catalog.json

{
  "datasets": {
    "beam": {
      "description": "Long-context memory benchmark: 100 conversations (100K\u201310M tokens), 2 000 questions across 10 memory ability categories.",
      "task": "LLM-judged",
      "splits": [
        "100k",
        "500k",
        "1m",
        "10m"
      ]
    },
    "lifebench": {
      "description": "Long-horizon multi-source personalized memory benchmark across 10 users.",
      "task": "LLM-judged",
      "splits": [
        "en"
      ]
    },
    "locomo": {
      "description": "Multi-session long-term conversations with 1,986 QA pairs.",
      "task": "LLM-judged",
      "splits": [
        "locomo10"
      ]
    },
    "longmemeval": {
      "description": "Long-term memory evaluation in LLM-based chat assistants.",
      "task": "LLM-judged",
      "splits": [
        "s"
      ]
    },
    "personamem": {
      "description": "Long-horizon personal preference tracking across conversation sessions.",
      "task": "MCQ",
      "splits": [
        "32k",
        "128k",
        "1M"
      ]
    },
    "sdebench": {
      "description": "Does memory help a coding agent? Bug-fix tasks whose obvious fix fails a hidden test.",
      "task": "LLM-judged",
      "splits": [
        "boltons"
      ]
    }
  },
  "providers": {
    "vanilla": {
      "key": "none",
      "description": "No memory system (vanilla baseline).",
      "kind": "local",
      "link": null,
      "logo": null
    },
    "hindsight": {
      "link": "https://github.com/vectorize-io/hindsight",
      "logo": null,
      "variants": {
        "coding-plugin": {
          "key": "hscoding",
          "description": "Hindsight coding-agents plugin: deepen-engine ingestion, agent-side reflect+inject.",
          "kind": "local"
        },
        "local": {
          "key": "hindsight",
          "description": "Embedded Hindsight fact store using gemini-2.5-flash-lite as the extraction model. Recall uses all memory types (world + experience + observation) with no type filter applied.",
          "kind": "local"
        },
        "cloud": {
          "key": "hindsight-cloud",
          "description": "Hindsight hosted cloud API. Recall uses all memory types (world + experience + observation) with no type filter applied.",
          "kind": "cloud"
        },
        "http": {
          "key": "hindsight-http",
          "description": "Hindsight via a self-hosted HTTP endpoint. Recall uses all memory types (world + experience + observation) with no type filter applied.",
          "kind": "cloud"
        }
      }
    },
    "bm25": {
      "key": "bm25",
      "description": "Keyword search baseline. No embeddings \u2014 splits docs into chunks and uses BM25 ranking.",
      "kind": "local",
      "link": null,
      "logo": null
    },
    "cognee": {
      "key": "cognee",
      "description": "Graph-based knowledge extraction with FastEmbed (BAAI/bge-small-en-v1.5) + OpenAI LLM.",
      "kind": "local",
      "link": "https://cognee.ai",
      "logo": "https://www.google.com/s2/favicons?sz=32&domain=cognee.ai"
    },
    "mastra": {
      "key": "mastra",
      "description": "Mastra semantic recall with LibSQL store and FastEmbed embeddings. topK=10.",
      "kind": "local",
      "link": "https://mastra.ai",
      "logo": "https://www.google.com/s2/favicons?sz=32&domain=mastra.ai"
    },
    "mastra-om": {
      "key": "mastra-om",
      "description": "Mastra Observational Memory: observer/reflector pattern with Gemini 2.5 Flash + GPT-4o.",
      "kind": "local",
      "link": "https://mastra.ai",
      "logo": "https://www.google.com/s2/favicons?sz=32&domain=mastra.ai"
    },
    "mem0": {
      "link": "https://mem0.ai",
      "logo": "https://www.google.com/s2/favicons?sz=32&domain=mem0.ai",
      "variants": {
        "local": {
          "key": "mem0",
          "description": "Agentic memory with Gemini 2.5 Flash for reflective extraction + local Qdrant store.",
          "kind": "local"
        },
        "cloud": {
          "key": "mem0-cloud",
          "description": "Mem0 cloud API. Async indexing \u2014 waits for indexing to complete before eval. k=20.",
          "kind": "cloud"
        }
      }
    },
    "ogham": {
      "link": "https://ogham-mcp.dev",
      "logo": null,
      "variants": {
        "local": {
          "key": "ogham",
          "description": "Ogham MCP: hybrid vector + BM25 search with entity overlap boost. Local Postgres + pgvector. Stores verbatim conversations and retrieves via Reciprocal Rank Fusion with optional read-time fact extraction.",
          "kind": "local"
        }
      }
    },
    "hybrid-search": {
      "key": "qdrant",
      "description": "Hybrid dense + sparse vector search via Qdrant with RRF fusion. Dense: Qwen3-Embedding-0.6B (1024d, asymmetric query/doc encoding). Sparse: BM42 (bm42-all-minilm-l6-v2-attentions). Documents chunked into 512-token windows before indexing. Retrieves top-k=50 chunks (prefetch 100 per branch).",
      "kind": "local",
      "link": "https://qdrant.tech",
      "logo": "https://www.google.com/s2/favicons?sz=32&domain=qdrant.tech"
    },
    "supermemory": {
      "key": "supermemory",
      "description": "Supermemory cloud API with temporal metadata support.",
      "kind": "cloud",
      "link": "https://supermemory.ai",
      "logo": "https://www.google.com/s2/favicons?sz=32&domain=supermemory.ai"
    }
  },
  "modes": {
    "rag": {
      "description": "Default. Provider retrieves top-k documents; they are injected into an LLM prompt as context. Supports both MCQ and open-ended questions."
    },
    "agentic-rag": {
      "description": "The LLM acts as an agent with a recall tool and can make multiple retrieval calls with different queries before finalising its answer."
    },
    "agent": {
      "description": "Bypasses the benchmark retrieval pipeline entirely \u2014 calls the provider's own native direct_answer() for providers that have built-in agentic answering."
    },
    "coding": {
      "description": "Build the task repo, run a coding agent with test-feedback interventions, grade by pytest."
    }
  }
}