Notes/notes-service/vendor/loco-rs/website/scripts/generate-llms-txt.mjs

import { readFileSync, readdirSync, mkdirSync, writeFileSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const __dirname = path.dirname(fileURLToPath(import.meta.url));

// Content root mirrors the site's URL space 1:1: `src/content/docs/docs/**`
// serves `/docs/**` (see astro.config.mjs `trailingSlash: 'always'`).
const CONTENT_ROOT = path.resolve(__dirname, '../src/content/docs/docs');
const OUT_DIR = path.resolve(__dirname, '../public');
const SITE = 'https://loco.rs';

// Same six Diátaxis groups + labels as the sidebar in astro.config.mjs.
const SECTION_LABELS = {
  tutorials: 'Tutorials',
  'how-to': 'How-to guides',
  reference: 'Reference',
  explanation: 'Explanation',
  extras: 'Extras',
  resources: 'Resources',
};
const SECTION_ORDER = ['tutorials', 'how-to', 'reference', 'explanation', 'extras', 'resources'];

function walk(dir) {
  const entries = readdirSync(dir, { withFileTypes: true });
  const files = [];
  for (const entry of entries) {
    const full = path.join(dir, entry.name);
    if (entry.isDirectory()) files.push(...walk(full));
    else if (entry.name.endsWith('.md')) files.push(full);
  }
  return files;
}

function unquote(value) {
  const trimmed = value.trim();
  if (
    (trimmed.startsWith('"') && trimmed.endsWith('"')) ||
    (trimmed.startsWith("'") && trimmed.endsWith("'"))
  ) {
    try {
      return JSON.parse(trimmed.replace(/^'|'$/g, '"'));
    } catch {
      return trimmed.slice(1, -1);
    }
  }
  return trimmed;
}

/**
 * Minimal parser for the flat YAML frontmatter this site's docs actually
 * produce (see scripts/convert-frontmatter.mjs): a `---` delimited block of
 * `title:`, `description:`, and `sidebar:\n  order: N`.
 *
 * @param {string} raw
 * @returns {{ title: string, description: string, order: number, body: string }}
 */
export function parseDoc(raw) {
  const match = raw.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/);
  if (!match) return { title: '', description: '', order: 999, body: raw };

  const fm = match[1];
  const body = raw.slice(match[0].length).trim();

  const title = unquote(fm.match(/^title:\s*(.+)$/m)?.[1] ?? '');
  const description = unquote(fm.match(/^description:\s*(.*)$/m)?.[1] ?? '');
  const order = Number(fm.match(/^\s+order:\s*(\d+)/m)?.[1] ?? 999);

  return { title, description, order, body };
}

export function urlFor(relPath) {
  // relPath is relative to CONTENT_ROOT, e.g. 'index.md', 'how-to/index.md',
  // 'how-to/add-model.md'.
  const dir = path.dirname(relPath);
  const base = path.basename(relPath, '.md');
  if (base === 'index') {
    return dir === '.' ? '/docs/' : `/docs/${dir}/`;
  }
  return `/docs/${dir}/${base}/`;
}

function loadPages() {
  return walk(CONTENT_ROOT)
    .map((file) => {
      const relPath = path.relative(CONTENT_ROOT, file);
      const raw = readFileSync(file, 'utf8');
      const doc = parseDoc(raw);
      const section = path.dirname(relPath).split(path.sep)[0]; // '.' for docs/index.md
      return { ...doc, url: urlFor(relPath), section, relPath };
    })
    .sort((a, b) => a.order - b.order || a.url.localeCompare(b.url));
}

/**
 * Builds `llms.txt` per the llmstxt.org convention: an H1 title, a summary
 * blockquote, then one H2 per Diátaxis section listing every page as a
 * `[title](url): description` bullet.
 */
function buildLlmsTxt(pages) {
  const root = pages.find((p) => p.section === '.');
  const lines = ['# Loco', ''];
  lines.push(`> ${root?.description || 'Loco is a Rust web framework for full-stack productivity, batteries included.'}`);
  lines.push('');
  if (root) {
    lines.push(`- [${root.title}](${SITE}${root.url})${root.description ? `: ${root.description}` : ''}`);
    lines.push('');
  }

  for (const section of SECTION_ORDER) {
    const sectionPages = pages.filter((p) => p.section === section);
    if (sectionPages.length === 0) continue;
    lines.push(`## ${SECTION_LABELS[section]}`);
    for (const page of sectionPages) {
      const desc = page.description ? `: ${page.description}` : '';
      lines.push(`- [${page.title}](${SITE}${page.url})${desc}`);
    }
    lines.push('');
  }

  return lines.join('\n').trimEnd() + '\n';
}

/**
 * Builds `llms-full.txt`: the full rendered-markdown body of every doc page,
 * concatenated in the same section/order as `llms.txt`, each preceded by
 * its title and canonical URL so an LLM can attribute/cite a passage.
 */
function buildLlmsFullTxt(pages) {
  const ordered = [
    ...pages.filter((p) => p.section === '.'),
    ...SECTION_ORDER.flatMap((section) => pages.filter((p) => p.section === section)),
  ];

  return (
    ordered
      .map((page) => `# ${page.title}\n\nSource: ${SITE}${page.url}\n\n${page.body}`)
      .join('\n\n---\n\n') + '\n'
  );
}

function main() {
  const pages = loadPages();
  mkdirSync(OUT_DIR, { recursive: true });

  const llmsTxt = buildLlmsTxt(pages);
  const llmsFullTxt = buildLlmsFullTxt(pages);

  writeFileSync(path.join(OUT_DIR, 'llms.txt'), llmsTxt, 'utf8');
  writeFileSync(path.join(OUT_DIR, 'llms-full.txt'), llmsFullTxt, 'utf8');

  console.log(`llms.txt: ${llmsTxt.length} bytes, ${pages.length} pages linked`);
  console.log(`llms-full.txt: ${llmsFullTxt.length} bytes, ${pages.length} pages inlined`);
}

if (import.meta.url === `file://${process.argv[1]}`) {
  main();
}