Add agent-readiness surfaces (robots.txt, sitemap, llms.txt, markdown)

AGENTS.md

@@ -39,6 +39,17 @@ Use `yarn` here, not `pnpm`: the repo is pinned to Yarn 1 and includes `yarn.loc
 - `meetup.ts`: shared meetup/site metadata used on the homepage.
 - `public/`: static assets.
 
+## Agent & crawler discovery
+The site exposes machine-readable surfaces for AI agents and crawlers. These are generated from content, so they stay current automatically — no manual updates when adding events/posts.
+- `public/robots.txt`: crawl rules, explicit AI crawler (GPTBot, ClaudeBot, Google-Extended, …) groups, a `Content-Signal` line (contentsignals.org), and a `Sitemap:` reference.
+- `app/sitemap.ts`: generates `/sitemap.xml` from `allDocs`.
+- `app/llms.txt/route.ts`: serves `/llms.txt` (llmstxt.org) — a plain-text index linking every event/post/page to its `.md` mirror.
+- `app/md/[...slug]/route.ts`: serves the plain-markdown mirror of any content page. Reached via `middleware.ts`, which rewrites `/<section>/<slug>.md` URLs and `Accept: text/markdown` requests onto it. HTML stays the default for browsers.
+- `lib/agent-content.ts`: shared helpers (read from Contentlayer `allDocs`, not the filesystem) backing all of the above.
+- `next.config.js` `headers()`: adds RFC 8288 `Link` headers (homepage → llms.txt/sitemap; content pages → their `.md` alternate).
+- `buildPageMetadata` adds a `<link rel="alternate" type="text/markdown">` to content pages via the `markdownPath` option.
+- Footer (`components/Footer.tsx`) carries a subtle `/llms.txt` link.
+
 ## Maintenance guidelines
 - Prefer small content and styling changes over broad refactors.
 - Do not rename or move large content folders unless you also update the content loading logic.

app/[contentType]/[slug]/page.tsx

@@ -61,6 +61,7 @@ export async function generateMetadata({ params }: PageProps): Promise<Metadata>
     pathname,
     image,
     type: 'article',
+    markdownPath: `${pathname}.md`,
   })
 }
 

app/layout.tsx

@@ -3,6 +3,7 @@ import './globals.css'
 import { IBM_Plex_Serif, Inter } from 'next/font/google'
 
 import Header from '@/components/Header/Header'
+import Footer from '@/components/Footer'
 import type { Metadata } from 'next'
 import { ThemeProvider } from './theme-provider'
 import { meetup } from '@/meetup'
@@ -59,6 +60,7 @@ export default function RootLayout({
         <ThemeProvider attribute="class" defaultTheme="system" enableSystem>
           <Header />
           {children}
+          <Footer />
         </ThemeProvider>
       </body>
     </html>

app/llms.txt/route.ts

@@ -0,0 +1,65 @@
+import { getAgentDocs } from '@/lib/agent-content'
+import { getSiteName, getSiteUrl } from '@/lib/site-metadata'
+import { meetup } from '@/meetup'
+
+/*
+ * /llms.txt — a machine-readable index of the site for AI agents.
+ * https://llmstxt.org/
+ *
+ * Each linked event/post/page also has a plain-markdown mirror at the same
+ * URL with a `.md` suffix (e.g. /events/<slug>.md).
+ */
+
+export const dynamic = 'force-static'
+
+function section(title: string, docs: ReturnType<typeof getAgentDocs>) {
+  if (docs.length === 0) {
+    return ''
+  }
+
+  const siteUrl = getSiteUrl()
+  const lines = docs.map((doc) => {
+    const suffix = doc.date ? `: ${doc.date}` : ''
+    return `- [${doc.title}](${siteUrl}${doc.mdPath})${suffix}`
+  })
+
+  return [`## ${title}`, '', ...lines, ''].join('\n')
+}
+
+export function GET() {
+  const siteUrl = getSiteUrl()
+  const events = getAgentDocs('events')
+  const posts = getAgentDocs('posts')
+  const pages = getAgentDocs('pages')
+
+  const body = [
+    `# ${getSiteName()}`,
+    '',
+    `> ${meetup.description}`,
+    '',
+    'This site lists Atlanta BitDevs meetups, Socratic Seminars, and posts on',
+    'Bitcoin and related protocol research. Every event, post, and page below',
+    'is also available as plain markdown by appending `.md` to its URL.',
+    '',
+    '## Key pages',
+    '',
+    `- [Home](${siteUrl}/): Community overview and recent activity`,
+    `- [All events](${siteUrl}/events): Upcoming and past meetups`,
+    `- [All posts](${siteUrl}/posts): Blog posts and write-ups`,
+    `- [Meetup group](https://www.meetup.com/atlantabitdevs/): RSVP and event logistics`,
+    '',
+    section('Pages', pages),
+    section('Events', events),
+    section('Posts', posts),
+  ]
+    .join('\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trimEnd()
+
+  return new Response(`${body}\n`, {
+    headers: {
+      'content-type': 'text/plain; charset=utf-8',
+      'cache-control': 'public, max-age=0, must-revalidate',
+    },
+  })
+}

app/md/[...slug]/route.ts

@@ -0,0 +1,48 @@
+import {
+  findAgentDoc,
+  getAllAgentDocs,
+  renderDocMarkdown,
+} from '@/lib/agent-content'
+
+/*
+ * Plain-markdown mirror of content pages.
+ *
+ * Reached via middleware, which rewrites a public `.md` URL — or a request
+ * with `Accept: text/markdown` — onto this route. For example:
+ *   /events/<slug>.md            -> /md/events/<slug>
+ *   /page/about (Accept: md)     -> /md/page/about
+ */
+
+export const dynamic = 'force-static'
+
+export function generateStaticParams() {
+  return getAllAgentDocs().map((doc) => ({
+    // doc.webPath is like "/events/<slug>"; drop the leading slash and split.
+    slug: doc.webPath.slice(1).split('/'),
+  }))
+}
+
+interface RouteContext {
+  params: Promise<{ slug: string[] }>
+}
+
+export async function GET(_request: Request, { params }: RouteContext) {
+  const { slug } = await params
+  const [segment, ...rest] = slug
+
+  const doc = findAgentDoc(segment, rest.join('/'))
+
+  if (!doc) {
+    return new Response('Not found\n', {
+      status: 404,
+      headers: { 'content-type': 'text/plain; charset=utf-8' },
+    })
+  }
+
+  return new Response(renderDocMarkdown(doc), {
+    headers: {
+      'content-type': 'text/markdown; charset=utf-8',
+      'cache-control': 'public, max-age=0, must-revalidate',
+    },
+  })
+}

app/page/[slug]/page.tsx

@@ -45,6 +45,7 @@ export async function generateMetadata({ params }: PageProps): Promise<Metadata>
     ),
     pathname: `/page/${resolvedParams.slug}`,
     image: defaultMetadataImage,
+    markdownPath: `/page/${resolvedParams.slug}.md`,
   })
 }
 

app/sitemap.ts

@@ -0,0 +1,24 @@
+import type { MetadataRoute } from 'next'
+
+import { getAllAgentDocs } from '@/lib/agent-content'
+import { getSiteUrl } from '@/lib/site-metadata'
+
+export const dynamic = 'force-static'
+
+export default function sitemap(): MetadataRoute.Sitemap {
+  const siteUrl = getSiteUrl()
+
+  const staticRoutes: MetadataRoute.Sitemap = ['', '/events', '/posts'].map(
+    (path) => ({
+      url: `${siteUrl}${path}`,
+      changeFrequency: 'weekly',
+    }),
+  )
+
+  const docRoutes: MetadataRoute.Sitemap = getAllAgentDocs().map((doc) => ({
+    url: `${siteUrl}${doc.webPath}`,
+    lastModified: doc.date ? new Date(doc.date) : undefined,
+  }))
+
+  return [...staticRoutes, ...docRoutes]
+}

components/Footer.tsx

@@ -0,0 +1,20 @@
+const Footer = () => {
+  const year = new Date().getFullYear()
+
+  return (
+    <footer className="mt-16 border-t border-neutral-200 py-8 font-sans text-sm text-neutral-500 dark:border-neutral-800 dark:text-neutral-400">
+      <div className="container mx-auto flex max-w-5xl flex-col gap-2 px-4 sm:flex-row sm:items-center sm:justify-between">
+        <p>© {year} Atlanta BitDevs · #LearnBitcoinTogether</p>
+        <a
+          href="/llms.txt"
+          className="opacity-50 underline-offset-4 transition hover:underline hover:opacity-100"
+          title="Machine-readable site index for AI agents"
+        >
+          llms.txt
+        </a>
+      </div>
+    </footer>
+  )
+}
+
+export default Footer

lib/agent-content.ts

@@ -0,0 +1,96 @@
+import { allDocs } from 'contentlayer/generated'
+
+import { getSiteUrl } from './site-metadata'
+
+/*
+ * Helpers that expose site content to AI agents and crawlers.
+ *
+ * These power the machine-readable surfaces (`/llms.txt`, the per-page
+ * `.md` mirrors, and the sitemap). Everything reads from Contentlayer's
+ * generated `allDocs`, which is bundled as plain JS — no filesystem access
+ * at request time, so it works the same at build and on serverless runtimes.
+ */
+
+type Doc = (typeof allDocs)[number]
+
+// URL path segment (as used by the Next.js routes) -> content/ folder name.
+const SEGMENT_TO_FOLDER: Record<string, string> = {
+  events: 'events',
+  posts: 'posts',
+  page: 'pages',
+}
+
+const FOLDER_TO_SEGMENT: Record<string, string> = {
+  events: 'events',
+  posts: 'posts',
+  pages: 'page',
+}
+
+export interface AgentDoc {
+  title: string
+  date?: string
+  webPath: string // e.g. /events/2026-05-20-bitcoin-socratic-seminar-50
+  mdPath: string // e.g. /events/2026-05-20-bitcoin-socratic-seminar-50.md
+  raw: string
+}
+
+function toAgentDoc(doc: Doc): AgentDoc {
+  const [folder, ...rest] = doc._raw.flattenedPath.split('/')
+  const segment = FOLDER_TO_SEGMENT[folder] ?? folder
+  const webPath = `/${segment}/${rest.join('/')}`
+
+  return {
+    title: doc.title,
+    date: doc.date,
+    webPath,
+    mdPath: `${webPath}.md`,
+    raw: doc.body.raw,
+  }
+}
+
+function byDateDesc(a: AgentDoc, b: AgentDoc) {
+  if (a.date && b.date) {
+    return a.date < b.date ? 1 : -1
+  }
+  return 0
+}
+
+/** Published docs for a single content folder, newest first. */
+export function getAgentDocs(folder: 'events' | 'posts' | 'pages'): AgentDoc[] {
+  return allDocs
+    .filter((doc) => doc._raw.flattenedPath.startsWith(`${folder}/`))
+    .filter((doc) => doc.published !== false)
+    .map(toAgentDoc)
+    .sort(byDateDesc)
+}
+
+/** Every published doc, used for sitemap and static param generation. */
+export function getAllAgentDocs(): AgentDoc[] {
+  return allDocs.filter((doc) => doc.published !== false).map(toAgentDoc)
+}
+
+/** Resolve a URL segment + slug (e.g. "events", "foo") to a doc, or null. */
+export function findAgentDoc(segment: string, slug: string): AgentDoc | null {
+  const folder = SEGMENT_TO_FOLDER[segment]
+  if (!folder) {
+    return null
+  }
+
+  const flattenedPath = `${folder}/${slug}`
+  const doc = allDocs.find((d) => d._raw.flattenedPath === flattenedPath)
+
+  return doc && doc.published !== false ? toAgentDoc(doc) : null
+}
+
+/** Render a single doc as standalone, agent-friendly markdown. */
+export function renderDocMarkdown(doc: AgentDoc): string {
+  const lines = [`# ${doc.title}`, '']
+
+  if (doc.date) {
+    lines.push(`*${doc.date}*`, '')
+  }
+
+  lines.push(doc.raw.trim(), '', '---', `Source: ${getSiteUrl()}${doc.webPath}`, '')
+
+  return lines.join('\n')
+}

lib/site-metadata.ts

@@ -105,6 +105,8 @@ type BuildPageMetadataArgs = {
   pathname: string
   image?: string | null
   type?: 'website' | 'article'
+  /** When set, advertises a plain-markdown mirror via <link rel="alternate">. */
+  markdownPath?: string
 }
 
 export function buildPageMetadata({
@@ -113,6 +115,7 @@ export function buildPageMetadata({
   pathname,
   image,
   type = 'website',
+  markdownPath,
 }: BuildPageMetadataArgs): Metadata {
   const resolvedImage = normalizeImagePath(image)
 
@@ -121,6 +124,9 @@ export function buildPageMetadata({
     description,
     alternates: {
       canonical: pathname,
+      ...(markdownPath
+        ? { types: { 'text/markdown': markdownPath } }
+        : {}),
     },
     openGraph: {
       type,

middleware.ts

@@ -0,0 +1,41 @@
+import { NextResponse } from 'next/server'
+import type { NextRequest } from 'next/server'
+
+/*
+ * Serves plain-markdown versions of content pages to agents in two ways:
+ *
+ *   1. Explicit URL:   /events/<slug>.md
+ *   2. Negotiation:    /events/<slug> with `Accept: text/markdown`
+ *
+ * Both are rewritten onto the /md/[...slug] route handler. HTML stays the
+ * default for browsers (which send `Accept: text/html`).
+ */
+
+// Matches a single content page, e.g. /events/foo, /posts/bar, /page/about.
+const CONTENT_PATH = /^\/(events|posts|page)\/[^/]+$/
+
+export function middleware(request: NextRequest) {
+  const { pathname } = request.nextUrl
+
+  const isExplicitMarkdown = pathname.endsWith('.md')
+  const basePath = isExplicitMarkdown ? pathname.slice(0, -3) : pathname
+
+  if (!CONTENT_PATH.test(basePath)) {
+    return NextResponse.next()
+  }
+
+  const accept = request.headers.get('accept') ?? ''
+  const wantsMarkdown = accept.includes('text/markdown')
+
+  if (!isExplicitMarkdown && !wantsMarkdown) {
+    return NextResponse.next()
+  }
+
+  const url = request.nextUrl.clone()
+  url.pathname = `/md${basePath}`
+  return NextResponse.rewrite(url)
+}
+
+export const config = {
+  matcher: ['/events/:slug*', '/posts/:slug*', '/page/:slug*'],
+}