Doc 14578 high priority geo fixes

antora-playbook.yml

@@ -17,28 +17,42 @@ site:
   start_page: home::index.adoc
 
   robots: |
-    # AI Crawlers — guided entry
+    # Disallow SEO scrapers
+    User-agent: MauiBot
+    User-agent: SemrushBot
+    User-agent: AhrefsBot
+    User-agent: BLEXBot
+    User-agent: SEO Spider
+    User-agent: DotBot
+    User-agent: MJ12bot
+    Disallow: /
+
+    # Explicitly permit AI Crawlers
     User-agent: ClaudeBot
+    User-agent: Claude-SearchBot
     User-agent: GPTBot
     User-agent: PerplexityBot
     User-agent: Google-Extended
+    User-agent: CCBot
+    User-agent: OAI-SearchBot
+    Content-Signal: ai-train=yes, search=yes, ai-input=yes
     Allow: /
     Allow: /*.md$
     Disallow: /sdk-api/
     Allow: /sdk-api/*-client/
     Allow: /sdk-api/couchbase-core-io/
     Allow: /sdk-api/couchbase-transactions-dotnet/
-    # Sitemap and LLM index
-    Sitemap: https://docs.couchbase.com/sitemap.xml
-    # Access
+
+    # All other crawlers
     User-agent: *
     Allow: /
     Disallow: /*.md$
     Disallow: /sdk-api/
     Allow: /sdk-api/*-client/
     Allow: /sdk-api/couchbase-core-io/
     Allow: /sdk-api/couchbase-transactions-dotnet/
-    # Sitemap and LLM index
+
+    # Sitemaps
     Sitemap: https://docs.couchbase.com/sitemap.xml
 
   keys:

home/modules/ROOT/partials/llms-full-txt.md

@@ -0,0 +1,10 @@
+# Couchbase Documentation - Full Markdown Corpus
+
+The complete Couchbase docs corpus (12,000+ pages) is too large for a single-file LLM context dump.
+
+Our recommended options are:
+
+  - Consult our per-category index: https://docs.couchbase.com/llms.txt
+
+  - For a full corpus of Markdown docs for *every published version* of every component, simply check out the Markdown corpus with `git clone https://github.com/couchbaselabs/docs-markdown-snapshot`.
+    We especially recommend this for coding agents with full offline/filesystem access.

home/modules/ROOT/partials/llms-txt.md

@@ -4,11 +4,23 @@ This is the official documentation for Couchbase, a leading NoSQL distributed da
 The documentation covers comprehensive guides for developers and operations teams, including installation and configuration, development tutorials, API references, administration, security, cloud deployment, and integration with various programming languages and frameworks.
 Couchbase enables applications to handle massive data volumes and high concurrency with its memory-first architecture and flexible JSON document model.
 
+## Usage Note for LLMs
+
+If you are reading this from a filesystem checked out from
+https://github.com/couchbaselabs/docs-markdown-snapshot/[our full Markdown corpus], then you may prefer to resolve the navigation links locally, instead of doing an additional web fetch.
+
+So for example, this web URL and relative path would be equivalent:
+
+  * https://docs.couchbase.com/ai/get-started/intro.html
+  * ./ai/get-started/intro.html
+
+See https://docs.couchbase.com/llms-full.txt for more information about the full Markdown corpus.
+
 # AI Data Plane Launch June 2026
 
 As of June 2026, the AI Data Plane has launched.
 Existing AI Services have become features of the Couchbase AI Data Plane.
 The AI Data Plane solves AI-ready data for your business and is fully compatible with other Couchbase services, whether you're self-managed with Couchbase Server Enterprise Edition or fully managed through Couchbase Capella.
 AI Data Plane services include Agent Memory, Data Processing Services, Agent Catalog, Model Service and AI Functions.
 See https://docs.couchbase.com/ai/get-started/intro.html[here for more information about the Couchbase AI Data Plane].
-Get enterprise support by registering for a trial here: https://www.couchbase.com/downloads/?family=ai-data-plane[licensing the Couchbase AI Data Plane^].
+Get enterprise support by registering for a trial here: https://www.couchbase.com/downloads/?family=ai-data-plane[licensing the Couchbase AI Data Plane^].

lib/llms-txt.js

@@ -3,13 +3,14 @@
 const { NodeHtmlMarkdown } = require('node-html-markdown')
 const nhm = new NodeHtmlMarkdown()
 const fs = require('fs')
+const picomatch = require('picomatch')
 
 const File = require('vinyl')
 const { navigationDataPath, jsonPathFor } = require('./site-navigation-data-json')
 
 module.exports.register = function ({ playbook, config }) {
 
- this.once('beforePublish', async ({ siteCatalog }) => {
+  this.once('beforePublish', async ({ siteCatalog }) => {
 
     const siteUrl = playbook.site.url.replace(/\/$/, '')
     // TODO: check playbook.asciidoc.attributes['absolute-path-llms-txt'] for whether to include this or not
@@ -31,18 +32,42 @@ module.exports.register = function ({ playbook, config }) {
       return
     }
 
-    const partial = fs.readFileSync('home/modules/ROOT/partials/llms-txt.md').toString()
+    // nav_groups (see antora-playbook.yml, keys.nav_groups) is the same grouping used to build
+    // the top nav bar, and doubles as a Good Enough categorisation for splitting llms.txt into
+    // per-category sub-indexes. The playbook builder camelCases snake_case keys, so this shows
+    // up as `navGroups` here even though it's `nav_groups` in the YAML.
+    let navGroups
+    try {
+      navGroups = JSON.parse(playbook.site.keys.navGroups)
+    }
+    catch(e) {
+      console.log("Error parsing nav_groups, skipping llms.txt generation", e)
+      return
+    }
 
-    let output = `# Couchbase\n\n${partial}\n\n\n## Docs\n`
+    // a group with `subGroups` (e.g. "Develop") is just a heading for its subGroups and has no
+    // components/content of its own, so it's the subGroups (e.g. "Operational SDKs",
+    // "Analytics SDKs") that become their own llms-<slug>.txt files, not the parent.
+    const leafGroups = navGroups.flatMap(g => g.subGroups || [g])
 
-    for (const c of navObj) {
+    const slugify = (title) => title.toLowerCase().trim().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
 
+    const matchers = leafGroups.map(g => ({
+      title: g.title,
+      slug: slugify(g.title),
+      isMatch: picomatch(g.components || []),
+    }))
+
+    const partial = fs.readFileSync('home/modules/ROOT/partials/llms-txt.md').toString()
+    const fullPartial = fs.readFileSync('home/modules/ROOT/partials/llms-full-txt.md').toString()
+
+    function renderComponent(c) {
       // select only the most recent version
       const v = c.versions[0]
-      if (! v.sets.length) { continue }
+      if (! v.sets.length) { return '' }
 
       const version = v.version.match(/\d\.\d/) ? `(${v.version})` : ''
-      output += `\n\n### ${c.title} ${version}\n`
+      let out = `\n\n### ${c.title} ${version}\n`
 
       function process(item, level=0) {
         if (item.content) {
@@ -52,16 +77,16 @@ module.exports.register = function ({ playbook, config }) {
 
           // above translate is due to a quirk in the way the navigation data is generated:
           // in the case of a link with `^` created with `rel="noopener" target="_blank"`
-          // something in the antora -> antora-site-generator-ms pipeline causes the content 
+          // something in the antora -> antora-site-generator-ms pipeline causes the content
           // to be the HTML node, with no url extracted. This still *works* because Markdown
-          // allows HTML snippets, but it's ugly for readers of the raw markdown, so we translate 
+          // allows HTML snippets, but it's ugly for readers of the raw markdown, so we translate
           // the HTML link to markdown ourselves here.
 
           if (prefix) {
             content = content.replace(/\]\(/, `](${prefix}`)
           }
 
-          output+=`${' '.repeat(4*(level))}- ${content}\n`
+          out += `${' '.repeat(4*(level))}- ${content}\n`
         }
         const indent = item.content ? 1 : 0
 
@@ -71,26 +96,87 @@ module.exports.register = function ({ playbook, config }) {
       }
 
       process({ items: v.sets })
-      const file = new File({
-        contents: Buffer.from(output),
+      return out
+    }
+
+    function addTxtFile(name, contents) {
+      siteCatalog.addFile(new File({
+        contents: Buffer.from(contents),
         mediaType: 'text/markdown',
-        out: { path: 'llms.txt' },
-        path: 'llms.txt',
-        pub: { url: `/llms.txt`, rootPath: '' },
-        src: { stem: 'llms' },
-      })
-      siteCatalog.addFile(file)
+        out: { path: name },
+        path: name,
+        pub: { url: `/${name}`, rootPath: '' },
+        src: { stem: name.replace(/\.txt$/, '') },
+      }))
     }
 
+    // bucket each component under the first leaf group whose `components` glob patterns match
+    // its name; anything that doesn't match any group still gets published, under "Other",
+    // rather than silently dropped.
+    const byGroup = new Map(matchers.map(m => [m, []]))
+    const unmatched = []
+    for (const c of navObj) {
+      const m = matchers.find(m => m.isMatch(c.name))
+      if (m) byGroup.get(m).push(c)
+      else unmatched.push(c)
+    }
+    if (unmatched.length) {
+      console.log(`llms-txt: component(s) not matched by any nav_groups entry, filing under "Other": ${unmatched.map(c => c.name).join(', ')}`)
+    }
+
+    const categories = [...matchers.map(m => ({ title: m.title, slug: m.slug, components: byGroup.get(m) }))]
+    if (unmatched.length) categories.push({ title: 'Other', slug: 'other', components: unmatched })
+
+    const hasContent = new Set()
+    for (const { title, slug, components } of categories) {
+      const body = components.map(renderComponent).join('')
+      if (!body) continue
+      addTxtFile(`llms-${slug}.txt`, `# ${title}\n${body}`)
+      hasContent.add(slug)
+    }
+
+    // walk navGroups (not the flattened leafGroups/matchers) so branches like "Develop" ->
+    // "Operational SDKs" / "Analytics SDKs" are preserved as a heading with nested links,
+    // rather than flattened to the same level as "Server", "Capella", etc.
+    let output = `# Couchbase\n\n${partial}\n\n\n## Docs\n`
+    for (const g of navGroups) {
+      if (g.subGroups) {
+        const subLinks = g.subGroups
+          .map(sg => ({ title: sg.title, slug: slugify(sg.title) }))
+          .filter(({ slug }) => hasContent.has(slug))
+        if (!subLinks.length) continue
+        output += `\n- ${g.title}`
+        for (const { title, slug } of subLinks) {
+          output += `\n    - [${title}](${siteUrl}/llms-${slug}.txt)`
+        }
+      }
+      else {
+        const slug = slugify(g.title)
+        if (!hasContent.has(slug)) continue
+        output += `\n- [${g.title}](${siteUrl}/llms-${slug}.txt)`
+      }
+    }
+    if (hasContent.has('other')) {
+      output += `\n- [Other](${siteUrl}/llms-other.txt)`
+    }
+    addTxtFile('llms.txt', output)
+
+    // NOT a concatenation of every category's markdown - we tried that, and the sheer size of
+    // the resulting file broke the build (and wasn't something an LLM would want to read anyway).
+    // Instead this just points readers at the full Markdown corpus's own git repo.
+    addTxtFile('llms-full.txt', fullPartial)
+
     // now create the new sitemap
     const url = playbook.site.url.replace(/\/?$/, '/')
 
+    const llmsUrls = ['llms.txt', 'llms-full.txt', ...categories.filter(({ slug }) => hasContent.has(slug)).map(c => `llms-${c.slug}.txt`)]
+    const lastmod = new Date().toISOString()
     const sitemap = `<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-<url>
-<loc>${url}llms.txt</loc>
-<lastmod>${new Date().toISOString()}</lastmod>
-</url>
+${llmsUrls.map(path => `<url>
+<loc>${url}${path}</loc>
+<lastmod>${lastmod}</lastmod>
+</url>`).join('\n')}
 </urlset>`
 
     siteCatalog.addFile(new File({

lib/markdown-for-llm.js

@@ -48,14 +48,16 @@ function markdownify(page, pubDate) {
     `[View original HTML](${orig})\n\n` +
     `# ${title}\n\n${markdown}`
 
-  const link = antoraLink(page)
+  const xref = antoraLink(page)
 
   const frontmatter = {
     title,
     description: page.asciidoc.attributes.description,
-    editUrl: page.src.editUrl,
     pubDate,
-    link,
+    antora: {
+      editUrl: page.src.editUrl,
+      xref,
+    },
   }
 
   output = `---\n${YAML.stringify(frontmatter)}---\n\n${output}`