embed-markdown/index.js

const { execSync } = require("child_process");
const fs = require("fs");

const core = require("@actions/core");
const github = require("@actions/github");

const apiUrl =
  core.getInput("api_url") || "http://agents-api.servc-agents:3000";
const apiToken = core.getInput("api_token");
const route = core.getInput("route") || "agent-lake";
const method = core.getInput("method") || "embeddings_insert";
const debug = (core.getInput("debug") || "false").toLowerCase() === "true";

const repoFull = process.env.GITHUB_REPOSITORY;
const [owner, repo] = repoFull.split("/");
const segment_id = ["docs", owner].join(".");

const serverUrl = (
  process.env.GITHUB_SERVER_URL ||
  github.context.serverUrl ||
  "https://git.yusufali.ca"
).replace(/\/$/, "");

const markdownFiles = execSync("git ls-files '*.md'", {
  encoding: "utf8",
})
  .trim()
  .split("\n")
  .filter(Boolean);

const headers = {
  "Content-Type": "application/json",
};

if (apiToken) {
  headers.Apitoken = apiToken;
}

async function post(requestPayload, retries=0) {
  try{
    const response = await fetch(apiUrl, {
      method: "POST",
      headers,
      body: JSON.stringify(requestPayload),
    });

    const responseText = await response.text();
    if (!response.ok) {
      core.setFailed(
        `Agent API request failed (${response.status}): ${responseText}`,
      );
    } else {
      core.info(`Agent response: ${responseText}`);
    }
  } catch(e){
    if(retries < 5){
      return post(requestPayload, retries-1)
    }
    core.setFailed(`Error sending task to agent: ${error}`)
  }
}

for (const file of markdownFiles) {
  const content = fs.readFileSync(file, "utf8").trim();
  const lines = content.split(/\r?\n/);
  const h1Line = lines.find((line) => /^#\s+/.test(line)) || "";

  const chunks = [];
  let current = [];

  for (const line of lines) {
    if (/^##\s+/.test(line)) {
      if (current.length) {
        chunks.push(current.join("\n").trim());
      }
      current = [line];
    } else {
      current.push(line);
    }
  }

  if (current.length) {
    chunks.push(current.join("\n").trim());
  }

  const normalizedChunks =
    chunks.length > 0
      ? chunks.map((chunk) => {
          let chunkLines = chunk.split(/\r?\n/);
          if (h1Line && chunkLines[0] === h1Line) {
            chunkLines = chunkLines.slice(1);
          }
          const body = chunkLines.join("\n").trim();
          return [h1Line, body].filter(Boolean).join("\n");
        })
      : [content];

  normalizedChunks.forEach((chunk, index) => {
    const requestPayload = {
      type: "input",
      route,
      argumentId: "plain",
      force: true,
      instanceId: null,
      inputs: {
        method,
        inputs: {
          segment_id,
          document_id: [repo, file.replace(".", ""), `part${index + 1}`].join(
            ".",
          ),
          embed_text: chunk,
          store_text: chunk,
        },
      },
    };

    if (debug) {
      core.info(`API URL: ${apiUrl}`);
      core.info(`Route: ${route}`);
      core.info(`Server URL: ${serverUrl}`);
      core.info(`Using auth: ${Boolean(apiToken)}`);
      core.info(`Request payload: ${JSON.stringify(requestPayload)}`);
    }

    post(requestPayload);
  });
}