Merge feature/knowledge-base: Knowledge Base RAG pipeline (v0.3.3)

Add documentation-aware RAG pipeline for Elixir/OTP knowledge:
- knowledge_entries table with pgvector 768-dim HNSW index
- Knowledge context module with keyword-first hybrid search
- SkillAPI knowledge_read/knowledge_write permissions
- LLM resolve_api_key config fallback for Gemini/Anthropic
- HexDocs scraper dynamic skill (modules + guides via sidebar JSON)
- Chat RAG integration with context source selector
- 452 tests passing, 0 failures

Author: thatsme

README.md

@@ -24,6 +24,7 @@ AlexClaw monitors the world (RSS feeds, web sources, GitHub repositories, APIs),
 - **Telegram Gateway** — Bidirectional communication via long-polling. Command routing is deterministic pattern-matching — no LLM involved in dispatch.
 - **Runtime Configuration** — All settings (API keys, prompts, limits, personas) are stored in PostgreSQL, cached in ETS, and editable at runtime via the admin UI. No restart required for any config change.
 - **Persistent Memory with Semantic Search** — PostgreSQL + pgvector for knowledge storage. Deduplication by URL. Hybrid search combines vector cosine similarity and keyword matching — vector results are prioritized, keyword results fill gaps for exact matches. Embeddings are generated asynchronously via the LLM router (Gemini `gemini-embedding-001`, Ollama `nomic-embed-text`, or any OpenAI-compatible endpoint). 768-dimension vectors with HNSW index. All skills that store knowledge auto-embed in the background.
+- **Knowledge Base RAG** — Separate `knowledge_entries` table for documentation and reference material, isolated from news/conversation memory. Scraper skills fetch, chunk, and embed documentation from hexdocs.pm (API reference + official guides). Chat integrates both Knowledge and Memory search with a context source selector (Docs only / Memory only / Both / None). System prompt instructs the LLM to cite provided documentation over general knowledge. Currently covers 22 Elixir ecosystem packages including full Elixir stdlib and 53 official guides.
 - **Cron Scheduler** — Quantum-based. Jobs defined in config or DB.
 
 ### Skills
@@ -44,6 +45,7 @@ AlexClaw monitors the world (RSS feeds, web sources, GitHub repositories, APIs),
 | `google_calendar` | Fetch upcoming Google Calendar events |
 | `google_tasks` | Manage Google Tasks lists and items |
 | `web_automation` | Browser automation via headless Playwright sidecar (**experimental**) |
+| `hexdocs_scraper` | Scrape hexdocs.pm docs into knowledge base embeddings (dynamic) |
 
 ### Dynamic Skill Loading (**experimental**)
 

ROADMAP.md

@@ -62,6 +62,10 @@ Additional notification/command channels beyond Telegram:
 
 ## Someday
 
+### ~~Knowledge Base RAG~~ ✅ Completed (v0.3.3)
+
+Separate `knowledge_entries` table with pgvector HNSW index for documentation and reference material. HexDocs scraper skill discovers modules via sidebar JSON, chunks by section/function, and embeds via local nomic-embed-text or Gemini. Chat RAG integration with context source selector (Docs/Memory/Both/None). Keyword-first hybrid search for precise documentation retrieval. LLM API key resolution falls back from provider record to config settings. Currently 22 packages scraped (4200+ chunks), including full Elixir stdlib and 53 official guides.
+
 ### ~~Semantic Search (Memory)~~ ✅ Completed (v0.2.1)
 
 Hybrid search combining pgvector cosine similarity and keyword matching. Embeddings generated asynchronously via Gemini `text-embedding-004`, Ollama `nomic-embed-text`, or any OpenAI-compatible endpoint. 768-dimension vectors with HNSW index. All skills auto-embed stored knowledge in the background. Batch re-embed support for model switching.

lib/alex_claw/knowledge.ex

@@ -0,0 +1,205 @@
+defmodule AlexClaw.Knowledge do
+  @moduledoc """
+  Knowledge base store for documentation, guides, and reference material.
+  Separate from Memory (news/facts) to keep embeddings cleanly partitioned.
+  Supports hybrid search: pgvector cosine similarity + keyword fallback.
+  Embeddings are generated asynchronously under TaskSupervisor.
+  """
+  require Logger
+  import Ecto.Query
+  alias AlexClaw.Repo
+  alias AlexClaw.Knowledge.Entry
+
+  @type store_opts :: [source: String.t() | nil, metadata: map(), expires_at: DateTime.t() | nil]
+
+  @spec store(atom() | String.t(), String.t(), store_opts()) ::
+          {:ok, Entry.t()} | {:error, Ecto.Changeset.t()}
+  def store(kind, content, opts \\ []) do
+    source = Keyword.get(opts, :source)
+    metadata = Keyword.get(opts, :metadata, %{})
+    expires_at = Keyword.get(opts, :expires_at)
+
+    result =
+      %Entry{}
+      |> Entry.changeset(%{
+        kind: to_string(kind),
+        content: content,
+        source: source,
+        embedding: nil,
+        metadata: metadata,
+        expires_at: expires_at
+      })
+      |> Repo.insert()
+
+    case result do
+      {:ok, entry} ->
+        async_embed(entry)
+        {:ok, entry}
+
+      error ->
+        error
+    end
+  end
+
+  @spec search(String.t(), keyword()) :: [Entry.t()]
+  def search(query, opts \\ []) do
+    limit = Keyword.get(opts, :limit, 10)
+    kind = Keyword.get(opts, :kind)
+
+    keyword_results = keyword_search(query, kind, limit)
+
+    case AlexClaw.LLM.embed(query) do
+      {:ok, embedding} ->
+        vector_results = vector_search(embedding, kind, limit)
+        merge_results(keyword_results, vector_results, limit)
+
+      {:error, _} ->
+        keyword_results
+    end
+  end
+
+  @spec exists?(String.t()) :: boolean()
+  def exists?(source_url) do
+    Entry
+    |> where([e], e.source == ^source_url)
+    |> Repo.exists?()
+  end
+
+  @spec recent(keyword()) :: [Entry.t()]
+  def recent(opts \\ []) do
+    limit = Keyword.get(opts, :limit, 20)
+    kind = Keyword.get(opts, :kind)
+
+    Entry
+    |> maybe_filter_kind(kind)
+    |> order_by([e], desc: e.inserted_at)
+    |> limit(^limit)
+    |> Repo.all()
+  end
+
+  @spec count(atom() | String.t() | nil) :: non_neg_integer()
+  def count(kind \\ nil) do
+    Entry
+    |> maybe_filter_kind(kind)
+    |> Repo.aggregate(:count)
+  end
+
+  @spec reembed_all(keyword()) :: {:ok, non_neg_integer()}
+  def reembed_all(opts \\ []) do
+    batch_size = Keyword.get(opts, :batch_size, 20)
+    max_concurrency = Keyword.get(opts, :max_concurrency, 2)
+
+    entries =
+      Entry
+      |> where([e], is_nil(e.embedding))
+      |> Repo.all()
+
+    count = length(entries)
+
+    if count > 0 do
+      caller = self()
+
+      Task.Supervisor.start_child(AlexClaw.TaskSupervisor, fn ->
+        sandbox_allow(caller)
+        Logger.info("Re-embedding #{count} knowledge entries...")
+
+        entries
+        |> Enum.chunk_every(batch_size)
+        |> Enum.each(fn batch ->
+          batch
+          |> Task.async_stream(
+            fn entry -> embed_entry(entry) end,
+            max_concurrency: max_concurrency,
+            timeout: 30_000,
+            on_timeout: :kill_task
+          )
+          |> Stream.run()
+        end)
+
+        Logger.info("Re-embedding complete: processed #{count} knowledge entries")
+      end)
+    end
+
+    {:ok, count}
+  end
+
+  # --- Internal ---
+
+  defp async_embed(%Entry{} = entry) do
+    caller = self()
+
+    Task.Supervisor.start_child(AlexClaw.TaskSupervisor, fn ->
+      sandbox_allow(caller)
+      embed_entry(entry)
+    end)
+  end
+
+  defp sandbox_allow(caller) do
+    if Application.get_env(:alex_claw, AlexClaw.Repo)[:pool] == Ecto.Adapters.SQL.Sandbox do
+      Ecto.Adapters.SQL.Sandbox.allow(AlexClaw.Repo, caller, self())
+    end
+  end
+
+  defp embed_entry(%Entry{id: id, content: content}) do
+    case AlexClaw.LLM.embed(content) do
+      {:ok, vector} when is_list(vector) ->
+        case Repo.get(Entry, id) do
+          nil -> :ok
+          entry -> entry |> Entry.changeset(%{embedding: vector}) |> Repo.update()
+        end
+
+      {:error, reason} ->
+        Logger.warning("Embedding failed for knowledge entry #{id}: #{inspect(reason)}")
+        :ok
+    end
+  end
+
+  defp merge_results(keyword_results, vector_results, limit) do
+    # Keyword matches are more precise for documentation, so they go first.
+    # Then fill with vector results that weren't already found by keyword.
+    keyword_ids = MapSet.new(keyword_results, & &1.id)
+
+    new_vector =
+      vector_results
+      |> Enum.reject(fn e -> MapSet.member?(keyword_ids, e.id) end)
+
+    (keyword_results ++ new_vector)
+    |> Enum.take(limit)
+  end
+
+  defp vector_search(embedding, kind, limit) do
+    Entry
+    |> maybe_filter_kind(kind)
+    |> where([e], not is_nil(e.embedding))
+    |> order_by([e], fragment("embedding <=> ?", ^embedding))
+    |> limit(^limit)
+    |> Repo.all()
+  end
+
+  defp keyword_search(query, kind, limit) do
+    terms =
+      query
+      |> String.replace(~r/[?!.,;:()\[\]{}"']/, " ")
+      |> String.split(~r/\s+/, trim: true)
+      |> Enum.reject(fn t -> String.length(t) < 3 end)
+      |> Enum.reject(fn t -> String.downcase(t) in ~w(the and for how does what which with from that this are was were can) end)
+      |> Enum.take(5)
+
+    case terms do
+      [] ->
+        []
+
+      terms ->
+        Enum.reduce(terms, Entry |> maybe_filter_kind(kind), fn term, q ->
+          pattern = "%#{term}%"
+          where(q, [e], ilike(e.content, ^pattern))
+        end)
+        |> order_by([e], desc: e.inserted_at)
+        |> limit(^limit)
+        |> Repo.all()
+    end
+  end
+
+  defp maybe_filter_kind(queryable, nil), do: queryable
+  defp maybe_filter_kind(queryable, kind), do: where(queryable, [e], e.kind == ^to_string(kind))
+end

lib/alex_claw/knowledge/entry.ex

@@ -0,0 +1,36 @@
+defmodule AlexClaw.Knowledge.Entry do
+  @moduledoc "Ecto schema for knowledge base entries with vector embeddings."
+
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  schema "knowledge_entries" do
+    field :kind, :string
+    field :content, :string
+    field :source, :string
+    field :embedding, Pgvector.Ecto.Vector
+    field :metadata, :map, default: %{}
+    field :expires_at, :utc_datetime
+
+    timestamps(type: :utc_datetime)
+  end
+
+  @type t :: %__MODULE__{
+          id: integer() | nil,
+          kind: String.t(),
+          content: String.t(),
+          source: String.t() | nil,
+          embedding: list(float()) | nil,
+          metadata: map(),
+          expires_at: DateTime.t() | nil,
+          inserted_at: DateTime.t() | nil,
+          updated_at: DateTime.t() | nil
+        }
+
+  @spec changeset(t() | Ecto.Changeset.t(), map()) :: Ecto.Changeset.t()
+  def changeset(entry, attrs) do
+    entry
+    |> cast(attrs, [:kind, :content, :source, :embedding, :metadata, :expires_at])
+    |> validate_required([:kind, :content])
+  end
+end

lib/alex_claw/llm.ex

@@ -169,7 +169,7 @@ defmodule AlexClaw.LLM do
   end
 
   defp call_embedding(%Provider{type: "gemini"} = p, text, model) do
-    api_key = p.api_key || ""
+    api_key = resolve_api_key(p)
 
     if api_key == "",
       do: {:error, :api_key_not_set},
@@ -194,6 +194,22 @@ defmodule AlexClaw.LLM do
     {:error, :anthropic_no_embeddings}
   end
 
+  # --- API Key Resolution ---
+
+  @config_key_map %{
+    "gemini" => "llm.gemini_api_key",
+    "anthropic" => "llm.anthropic_api_key"
+  }
+
+  defp resolve_api_key(%Provider{api_key: key}) when is_binary(key) and key != "", do: key
+
+  defp resolve_api_key(%Provider{type: type}) do
+    case Map.get(@config_key_map, type) do
+      nil -> ""
+      config_key -> AlexClaw.Config.get(config_key) || ""
+    end
+  end
+
   # --- Gemini Embeddings ---
 
   defp call_embedding_gemini(api_key, text, model) do
@@ -335,15 +351,15 @@ defmodule AlexClaw.LLM do
   # --- Provider Calls ---
 
   defp call_provider(%Provider{type: "gemini"} = p, prompt, system) do
-    api_key = p.api_key || ""
+    api_key = resolve_api_key(p)
 
     if api_key == "",
       do: {:error, :api_key_not_set},
       else: call_gemini(p.model, api_key, prompt, system)
   end
 
   defp call_provider(%Provider{type: "anthropic"} = p, prompt, system) do
-    api_key = p.api_key || ""
+    api_key = resolve_api_key(p)
 
     if api_key == "",
       do: {:error, :api_key_not_set},

lib/alex_claw/skills/skill_api.ex

@@ -10,7 +10,7 @@ defmodule AlexClaw.Skills.SkillAPI do
   """
   require Logger
 
-  @known_permissions ~w(llm telegram_send memory_read memory_write web_read config_read resources_read skill_invoke)a
+  @known_permissions ~w(llm telegram_send memory_read memory_write knowledge_read knowledge_write web_read config_read resources_read skill_invoke)a
 
   def known_permissions, do: @known_permissions
 
@@ -78,6 +78,29 @@ defmodule AlexClaw.Skills.SkillAPI do
     end
   end
 
+  # --- Knowledge ---
+
+  @doc "Search knowledge base by semantic similarity. Opts: :limit, :kind"
+  def knowledge_search(skill_module, query, opts \\ []) do
+    with :ok <- check_permission(skill_module, :knowledge_read) do
+      {:ok, AlexClaw.Knowledge.search(query, opts)}
+    end
+  end
+
+  @doc "Check if a source URL already exists in knowledge base."
+  def knowledge_exists?(skill_module, source_url) do
+    with :ok <- check_permission(skill_module, :knowledge_read) do
+      {:ok, AlexClaw.Knowledge.exists?(source_url)}
+    end
+  end
+
+  @doc "Store a knowledge entry. Opts: :source, :metadata, :expires_at"
+  def knowledge_store(skill_module, kind, content, opts \\ []) do
+    with :ok <- check_permission(skill_module, :knowledge_write) do
+      AlexClaw.Knowledge.store(kind, content, opts)
+    end
+  end
+
   # --- HTTP ---
 
   @doc "HTTP GET. All Req options are passed through (headers, receive_timeout, params, etc)."