From c4fb5fbfe0d2309f896dae0d6f0a7f7175dc2e8f Mon Sep 17 00:00:00 2001
From: Stephen DeLorme <stephen@d.elor.me>
Date: Wed, 17 Jun 2026 14:18:01 -0400
Subject: [PATCH] Add agent-readiness surfaces (robots.txt, sitemap, llms.txt,
 markdown)

Make the site discoverable and consumable by AI agents and crawlers:

- public/robots.txt: crawl rules, explicit AI crawler groups (GPTBot,
  ClaudeBot, Google-Extended, etc.), a Content-Signal line, and a
  Sitemap: reference.
- app/sitemap.ts: /sitemap.xml generated from content.
- app/llms.txt/route.ts: /llms.txt index linking every event/post/page
  to its plain-markdown mirror.
- app/md/[...slug] + middleware.ts: serve a plain-markdown version of
  any content page, reached via /<section>/<slug>.md URLs or an
  Accept: text/markdown request. HTML stays the default for browsers.
- next.config.js headers(): RFC 8288 Link headers pointing agents at
  llms.txt/sitemap and each page's .md alternate; metadata adds a
  <link rel="alternate" type="text/markdown">.
- Footer with a subtle /llms.txt link.

Scoped to a static content site: OAuth/MCP/WebMCP/DNS-AID/api-catalog
items from the audit don't apply and were skipped.
---
 AGENTS.md                         | 11 ++++
 app/[contentType]/[slug]/page.tsx |  1 +
 app/layout.tsx                    |  2 +
 app/llms.txt/route.ts             | 65 +++++++++++++++++++++
 app/md/[...slug]/route.ts         | 48 ++++++++++++++++
 app/page/[slug]/page.tsx          |  1 +
 app/sitemap.ts                    | 24 ++++++++
 components/Footer.tsx             | 20 +++++++
 lib/agent-content.ts              | 96 +++++++++++++++++++++++++++++++
 lib/site-metadata.ts              |  6 ++
 middleware.ts                     | 41 +++++++++++++
 next.config.js                    | 34 ++++++++++-
 public/robots.txt                 | 44 ++++++++++++++
 13 files changed, 392 insertions(+), 1 deletion(-)
 create mode 100644 app/llms.txt/route.ts
 create mode 100644 app/md/[...slug]/route.ts
 create mode 100644 app/sitemap.ts
 create mode 100644 components/Footer.tsx
 create mode 100644 lib/agent-content.ts
 create mode 100644 middleware.ts
 create mode 100644 public/robots.txt

diff --git a/AGENTS.md b/AGENTS.md
index 500dd13..adffe1e 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -39,6 +39,17 @@ Use `yarn` here, not `pnpm`: the repo is pinned to Yarn 1 and includes `yarn.loc
 - `meetup.ts`: shared meetup/site metadata used on the homepage.
 - `public/`: static assets.
 
+## Agent & crawler discovery
+The site exposes machine-readable surfaces for AI agents and crawlers. These are generated from content, so they stay current automatically — no manual updates when adding events/posts.
+- `public/robots.txt`: crawl rules, explicit AI crawler (GPTBot, ClaudeBot, Google-Extended, …) groups, a `Content-Signal` line (contentsignals.org), and a `Sitemap:` reference.
+- `app/sitemap.ts`: generates `/sitemap.xml` from `allDocs`.
+- `app/llms.txt/route.ts`: serves `/llms.txt` (llmstxt.org) — a plain-text index linking every event/post/page to its `.md` mirror.
+- `app/md/[...slug]/route.ts`: serves the plain-markdown mirror of any content page. Reached via `middleware.ts`, which rewrites `/<section>/<slug>.md` URLs and `Accept: text/markdown` requests onto it. HTML stays the default for browsers.
+- `lib/agent-content.ts`: shared helpers (read from Contentlayer `allDocs`, not the filesystem) backing all of the above.
+- `next.config.js` `headers()`: adds RFC 8288 `Link` headers (homepage → llms.txt/sitemap; content pages → their `.md` alternate).
+- `buildPageMetadata` adds a `<link rel="alternate" type="text/markdown">` to content pages via the `markdownPath` option.
+- Footer (`components/Footer.tsx`) carries a subtle `/llms.txt` link.
+
 ## Maintenance guidelines
 - Prefer small content and styling changes over broad refactors.
 - Do not rename or move large content folders unless you also update the content loading logic.
diff --git a/app/[contentType]/[slug]/page.tsx b/app/[contentType]/[slug]/page.tsx
index d14f776..611eb05 100644
--- a/app/[contentType]/[slug]/page.tsx
+++ b/app/[contentType]/[slug]/page.tsx
@@ -61,6 +61,7 @@ export async function generateMetadata({ params }: PageProps): Promise<Metadata>
     pathname,
     image,
     type: 'article',
+    markdownPath: `${pathname}.md`,
   })
 }
 
diff --git a/app/layout.tsx b/app/layout.tsx
index 83f8fed..618568e 100644
--- a/app/layout.tsx
+++ b/app/layout.tsx
@@ -3,6 +3,7 @@ import './globals.css'
 import { IBM_Plex_Serif, Inter } from 'next/font/google'
 
 import Header from '@/components/Header/Header'
+import Footer from '@/components/Footer'
 import type { Metadata } from 'next'
 import { ThemeProvider } from './theme-provider'
 import { meetup } from '@/meetup'
@@ -59,6 +60,7 @@ export default function RootLayout({
         <ThemeProvider attribute="class" defaultTheme="system" enableSystem>
           <Header />
           {children}
+          <Footer />
         </ThemeProvider>
       </body>
     </html>
diff --git a/app/llms.txt/route.ts b/app/llms.txt/route.ts
new file mode 100644
index 0000000..056ea75
--- /dev/null
+++ b/app/llms.txt/route.ts
@@ -0,0 +1,65 @@
+import { getAgentDocs } from '@/lib/agent-content'
+import { getSiteName, getSiteUrl } from '@/lib/site-metadata'
+import { meetup } from '@/meetup'
+
+/*
+ * /llms.txt — a machine-readable index of the site for AI agents.
+ * https://llmstxt.org/
+ *
+ * Each linked event/post/page also has a plain-markdown mirror at the same
+ * URL with a `.md` suffix (e.g. /events/<slug>.md).
+ */
+
+export const dynamic = 'force-static'
+
+function section(title: string, docs: ReturnType<typeof getAgentDocs>) {
+  if (docs.length === 0) {
+    return ''
+  }
+
+  const siteUrl = getSiteUrl()
+  const lines = docs.map((doc) => {
+    const suffix = doc.date ? `: ${doc.date}` : ''
+    return `- [${doc.title}](${siteUrl}${doc.mdPath})${suffix}`
+  })
+
+  return [`## ${title}`, '', ...lines, ''].join('\n')
+}
+
+export function GET() {
+  const siteUrl = getSiteUrl()
+  const events = getAgentDocs('events')
+  const posts = getAgentDocs('posts')
+  const pages = getAgentDocs('pages')
+
+  const body = [
+    `# ${getSiteName()}`,
+    '',
+    `> ${meetup.description}`,
+    '',
+    'This site lists Atlanta BitDevs meetups, Socratic Seminars, and posts on',
+    'Bitcoin and related protocol research. Every event, post, and page below',
+    'is also available as plain markdown by appending `.md` to its URL.',
+    '',
+    '## Key pages',
+    '',
+    `- [Home](${siteUrl}/): Community overview and recent activity`,
+    `- [All events](${siteUrl}/events): Upcoming and past meetups`,
+    `- [All posts](${siteUrl}/posts): Blog posts and write-ups`,
+    `- [Meetup group](https://www.meetup.com/atlantabitdevs/): RSVP and event logistics`,
+    '',
+    section('Pages', pages),
+    section('Events', events),
+    section('Posts', posts),
+  ]
+    .join('\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trimEnd()
+
+  return new Response(`${body}\n`, {
+    headers: {
+      'content-type': 'text/plain; charset=utf-8',
+      'cache-control': 'public, max-age=0, must-revalidate',
+    },
+  })
+}
diff --git a/app/md/[...slug]/route.ts b/app/md/[...slug]/route.ts
new file mode 100644
index 0000000..143d4d2
--- /dev/null
+++ b/app/md/[...slug]/route.ts
@@ -0,0 +1,48 @@
+import {
+  findAgentDoc,
+  getAllAgentDocs,
+  renderDocMarkdown,
+} from '@/lib/agent-content'
+
+/*
+ * Plain-markdown mirror of content pages.
+ *
+ * Reached via middleware, which rewrites a public `.md` URL — or a request
+ * with `Accept: text/markdown` — onto this route. For example:
+ *   /events/<slug>.md            -> /md/events/<slug>
+ *   /page/about (Accept: md)     -> /md/page/about
+ */
+
+export const dynamic = 'force-static'
+
+export function generateStaticParams() {
+  return getAllAgentDocs().map((doc) => ({
+    // doc.webPath is like "/events/<slug>"; drop the leading slash and split.
+    slug: doc.webPath.slice(1).split('/'),
+  }))
+}
+
+interface RouteContext {
+  params: Promise<{ slug: string[] }>
+}
+
+export async function GET(_request: Request, { params }: RouteContext) {
+  const { slug } = await params
+  const [segment, ...rest] = slug
+
+  const doc = findAgentDoc(segment, rest.join('/'))
+
+  if (!doc) {
+    return new Response('Not found\n', {
+      status: 404,
+      headers: { 'content-type': 'text/plain; charset=utf-8' },
+    })
+  }
+
+  return new Response(renderDocMarkdown(doc), {
+    headers: {
+      'content-type': 'text/markdown; charset=utf-8',
+      'cache-control': 'public, max-age=0, must-revalidate',
+    },
+  })
+}
diff --git a/app/page/[slug]/page.tsx b/app/page/[slug]/page.tsx
index 288c8d6..762fc0e 100644
--- a/app/page/[slug]/page.tsx
+++ b/app/page/[slug]/page.tsx
@@ -45,6 +45,7 @@ export async function generateMetadata({ params }: PageProps): Promise<Metadata>
     ),
     pathname: `/page/${resolvedParams.slug}`,
     image: defaultMetadataImage,
+    markdownPath: `/page/${resolvedParams.slug}.md`,
   })
 }
 
diff --git a/app/sitemap.ts b/app/sitemap.ts
new file mode 100644
index 0000000..18176b4
--- /dev/null
+++ b/app/sitemap.ts
@@ -0,0 +1,24 @@
+import type { MetadataRoute } from 'next'
+
+import { getAllAgentDocs } from '@/lib/agent-content'
+import { getSiteUrl } from '@/lib/site-metadata'
+
+export const dynamic = 'force-static'
+
+export default function sitemap(): MetadataRoute.Sitemap {
+  const siteUrl = getSiteUrl()
+
+  const staticRoutes: MetadataRoute.Sitemap = ['', '/events', '/posts'].map(
+    (path) => ({
+      url: `${siteUrl}${path}`,
+      changeFrequency: 'weekly',
+    }),
+  )
+
+  const docRoutes: MetadataRoute.Sitemap = getAllAgentDocs().map((doc) => ({
+    url: `${siteUrl}${doc.webPath}`,
+    lastModified: doc.date ? new Date(doc.date) : undefined,
+  }))
+
+  return [...staticRoutes, ...docRoutes]
+}
diff --git a/components/Footer.tsx b/components/Footer.tsx
new file mode 100644
index 0000000..ab07d64
--- /dev/null
+++ b/components/Footer.tsx
@@ -0,0 +1,20 @@
+const Footer = () => {
+  const year = new Date().getFullYear()
+
+  return (
+    <footer className="mt-16 border-t border-neutral-200 py-8 font-sans text-sm text-neutral-500 dark:border-neutral-800 dark:text-neutral-400">
+      <div className="container mx-auto flex max-w-5xl flex-col gap-2 px-4 sm:flex-row sm:items-center sm:justify-between">
+        <p>© {year} Atlanta BitDevs · #LearnBitcoinTogether</p>
+        <a
+          href="/llms.txt"
+          className="opacity-50 underline-offset-4 transition hover:underline hover:opacity-100"
+          title="Machine-readable site index for AI agents"
+        >
+          llms.txt
+        </a>
+      </div>
+    </footer>
+  )
+}
+
+export default Footer
diff --git a/lib/agent-content.ts b/lib/agent-content.ts
new file mode 100644
index 0000000..4c8c130
--- /dev/null
+++ b/lib/agent-content.ts
@@ -0,0 +1,96 @@
+import { allDocs } from 'contentlayer/generated'
+
+import { getSiteUrl } from './site-metadata'
+
+/*
+ * Helpers that expose site content to AI agents and crawlers.
+ *
+ * These power the machine-readable surfaces (`/llms.txt`, the per-page
+ * `.md` mirrors, and the sitemap). Everything reads from Contentlayer's
+ * generated `allDocs`, which is bundled as plain JS — no filesystem access
+ * at request time, so it works the same at build and on serverless runtimes.
+ */
+
+type Doc = (typeof allDocs)[number]
+
+// URL path segment (as used by the Next.js routes) -> content/ folder name.
+const SEGMENT_TO_FOLDER: Record<string, string> = {
+  events: 'events',
+  posts: 'posts',
+  page: 'pages',
+}
+
+const FOLDER_TO_SEGMENT: Record<string, string> = {
+  events: 'events',
+  posts: 'posts',
+  pages: 'page',
+}
+
+export interface AgentDoc {
+  title: string
+  date?: string
+  webPath: string // e.g. /events/2026-05-20-bitcoin-socratic-seminar-50
+  mdPath: string // e.g. /events/2026-05-20-bitcoin-socratic-seminar-50.md
+  raw: string
+}
+
+function toAgentDoc(doc: Doc): AgentDoc {
+  const [folder, ...rest] = doc._raw.flattenedPath.split('/')
+  const segment = FOLDER_TO_SEGMENT[folder] ?? folder
+  const webPath = `/${segment}/${rest.join('/')}`
+
+  return {
+    title: doc.title,
+    date: doc.date,
+    webPath,
+    mdPath: `${webPath}.md`,
+    raw: doc.body.raw,
+  }
+}
+
+function byDateDesc(a: AgentDoc, b: AgentDoc) {
+  if (a.date && b.date) {
+    return a.date < b.date ? 1 : -1
+  }
+  return 0
+}
+
+/** Published docs for a single content folder, newest first. */
+export function getAgentDocs(folder: 'events' | 'posts' | 'pages'): AgentDoc[] {
+  return allDocs
+    .filter((doc) => doc._raw.flattenedPath.startsWith(`${folder}/`))
+    .filter((doc) => doc.published !== false)
+    .map(toAgentDoc)
+    .sort(byDateDesc)
+}
+
+/** Every published doc, used for sitemap and static param generation. */
+export function getAllAgentDocs(): AgentDoc[] {
+  return allDocs.filter((doc) => doc.published !== false).map(toAgentDoc)
+}
+
+/** Resolve a URL segment + slug (e.g. "events", "foo") to a doc, or null. */
+export function findAgentDoc(segment: string, slug: string): AgentDoc | null {
+  const folder = SEGMENT_TO_FOLDER[segment]
+  if (!folder) {
+    return null
+  }
+
+  const flattenedPath = `${folder}/${slug}`
+  const doc = allDocs.find((d) => d._raw.flattenedPath === flattenedPath)
+
+  return doc && doc.published !== false ? toAgentDoc(doc) : null
+}
+
+/** Render a single doc as standalone, agent-friendly markdown. */
+export function renderDocMarkdown(doc: AgentDoc): string {
+  const lines = [`# ${doc.title}`, '']
+
+  if (doc.date) {
+    lines.push(`*${doc.date}*`, '')
+  }
+
+  lines.push(doc.raw.trim(), '', '---', `Source: ${getSiteUrl()}${doc.webPath}`, '')
+
+  return lines.join('\n')
+}
diff --git a/lib/site-metadata.ts b/lib/site-metadata.ts
index dc8b365..07f1711 100644
--- a/lib/site-metadata.ts
+++ b/lib/site-metadata.ts
@@ -105,6 +105,8 @@ type BuildPageMetadataArgs = {
   pathname: string
   image?: string | null
   type?: 'website' | 'article'
+  /** When set, advertises a plain-markdown mirror via <link rel="alternate">. */
+  markdownPath?: string
 }
 
 export function buildPageMetadata({
@@ -113,6 +115,7 @@ export function buildPageMetadata({
   pathname,
   image,
   type = 'website',
+  markdownPath,
 }: BuildPageMetadataArgs): Metadata {
   const resolvedImage = normalizeImagePath(image)
 
@@ -121,6 +124,9 @@ export function buildPageMetadata({
     description,
     alternates: {
       canonical: pathname,
+      ...(markdownPath
+        ? { types: { 'text/markdown': markdownPath } }
+        : {}),
     },
     openGraph: {
       type,
diff --git a/middleware.ts b/middleware.ts
new file mode 100644
index 0000000..11a8a97
--- /dev/null
+++ b/middleware.ts
@@ -0,0 +1,41 @@
+import { NextResponse } from 'next/server'
+import type { NextRequest } from 'next/server'
+
+/*
+ * Serves plain-markdown versions of content pages to agents in two ways:
+ *
+ *   1. Explicit URL:   /events/<slug>.md
+ *   2. Negotiation:    /events/<slug> with `Accept: text/markdown`
+ *
+ * Both are rewritten onto the /md/[...slug] route handler. HTML stays the
+ * default for browsers (which send `Accept: text/html`).
+ */
+
+// Matches a single content page, e.g. /events/foo, /posts/bar, /page/about.
+const CONTENT_PATH = /^\/(events|posts|page)\/[^/]+$/
+
+export function middleware(request: NextRequest) {
+  const { pathname } = request.nextUrl
+
+  const isExplicitMarkdown = pathname.endsWith('.md')
+  const basePath = isExplicitMarkdown ? pathname.slice(0, -3) : pathname
+
+  if (!CONTENT_PATH.test(basePath)) {
+    return NextResponse.next()
+  }
+
+  const accept = request.headers.get('accept') ?? ''
+  const wantsMarkdown = accept.includes('text/markdown')
+
+  if (!isExplicitMarkdown && !wantsMarkdown) {
+    return NextResponse.next()
+  }
+
+  const url = request.nextUrl.clone()
+  url.pathname = `/md${basePath}`
+  return NextResponse.rewrite(url)
+}
+
+export const config = {
+  matcher: ['/events/:slug*', '/posts/:slug*', '/page/:slug*'],
+}
diff --git a/next.config.js b/next.config.js
index 767719f..f49811d 100644
--- a/next.config.js
+++ b/next.config.js
@@ -1,4 +1,36 @@
 /** @type {import('next').NextConfig} */
-const nextConfig = {}
+const markdownAlternate = (segment) => ({
+  source: `/${segment}/:slug`,
+  headers: [
+    {
+      // Advertise the plain-markdown mirror of this page (RFC 8288).
+      key: 'Link',
+      value: `</${segment}/:slug.md>; rel="alternate"; type="text/markdown"`,
+    },
+    // Responses vary on Accept because agents can request markdown directly.
+    { key: 'Vary', value: 'Accept' },
+  ],
+})
+
+const nextConfig = {
+  async headers() {
+    return [
+      {
+        source: '/',
+        headers: [
+          {
+            // Point agents at the machine-readable site index and sitemap.
+            key: 'Link',
+            value:
+              '</llms.txt>; rel="alternate"; type="text/plain"; title="llms.txt", </sitemap.xml>; rel="sitemap"; type="application/xml"',
+          },
+        ],
+      },
+      markdownAlternate('events'),
+      markdownAlternate('posts'),
+      markdownAlternate('page'),
+    ]
+  },
+}
 
 module.exports = nextConfig
diff --git a/public/robots.txt b/public/robots.txt
new file mode 100644
index 0000000..9f9d2c7
--- /dev/null
+++ b/public/robots.txt
@@ -0,0 +1,44 @@
+# Atlanta BitDevs — https://atlbit.dev
+# Free and open Bitcoin education. Our content is meant to be shared, so we
+# welcome search engines and AI agents. A machine-readable index of the site
+# is available at https://atlbit.dev/llms.txt and every content page has a
+# plain-markdown mirror at the same URL with a ".md" suffix.
+
+User-agent: *
+Allow: /
+Disallow: /experiment
+# Content usage preferences — https://contentsignals.org/
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
+
+# AI / LLM crawlers — explicitly welcomed
+User-agent: GPTBot
+Allow: /
+
+User-agent: OAI-SearchBot
+Allow: /
+
+User-agent: ChatGPT-User
+Allow: /
+
+User-agent: Claude-Web
+Allow: /
+
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: anthropic-ai
+Allow: /
+
+User-agent: Google-Extended
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+User-agent: CCBot
+Allow: /
+
+User-agent: Applebot-Extended
+Allow: /
+
+Sitemap: https://atlbit.dev/sitemap.xml