From 7a71cdbd556acf06939b81759b52e2b8668edb30 Mon Sep 17 00:00:00 2001 From: Oleksandr Ostrovskyi Date: Tue, 24 Mar 2026 21:39:52 +0200 Subject: [PATCH 1/8] feat: edges table with migration, CRUD, and cascade support Add edges table to SQLite for vault graph. Supports wikilink and mention edge types. Bidirectional cleanup on delete_edges_for_file. ON DELETE CASCADE handles file deletion. INSERT OR IGNORE for dupes. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/store.rs | 236 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) diff --git a/src/store.rs b/src/store.rs index 30d1eef..f66b487 100644 --- a/src/store.rs +++ b/src/store.rs @@ -133,6 +133,30 @@ impl Store { // Always ensure the index exists (safe for both fresh and migrated DBs). self.conn .execute_batch("CREATE INDEX IF NOT EXISTS idx_files_docid ON files(docid);")?; + + // Check if edges table exists. + let has_edges: bool = { + let mut stmt = self + .conn + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='edges'")?; + let mut rows = stmt.query_map([], |row| row.get::<_, String>(0))?; + rows.next().is_some() + }; + if !has_edges { + self.conn.execute_batch( + "CREATE TABLE IF NOT EXISTS edges ( + id INTEGER PRIMARY KEY, + from_file INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + to_file INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + edge_type TEXT NOT NULL, + UNIQUE(from_file, to_file, edge_type) + ); + CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_file); + CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_file); + CREATE INDEX IF NOT EXISTS idx_edges_type ON edges(edge_type);", + )?; + } + Ok(()) } @@ -380,6 +404,100 @@ impl Store { Ok(()) } + // ── Edges ────────────────────────────────────────────────── + + /// Insert an edge. Uses INSERT OR IGNORE for the UNIQUE constraint. + pub fn insert_edge(&self, from_file: i64, to_file: i64, edge_type: &str) -> Result<()> { + self.conn.execute( + "INSERT OR IGNORE INTO edges (from_file, to_file, edge_type) VALUES (?1, ?2, ?3)", + params![from_file, to_file, edge_type], + )?; + Ok(()) + } + + /// Delete all edges involving a file (both directions: from_file OR to_file). + pub fn delete_edges_for_file(&self, file_id: i64) -> Result<()> { + self.conn.execute( + "DELETE FROM edges WHERE from_file = ?1 OR to_file = ?1", + params![file_id], + )?; + Ok(()) + } + + /// Clear all edges (used during --rebuild). + pub fn clear_edges(&self) -> Result<()> { + self.conn.execute("DELETE FROM edges", [])?; + Ok(()) + } + + /// Get outgoing edges, optionally filtered by type. + pub fn get_outgoing( + &self, + file_id: i64, + edge_type: Option<&str>, + ) -> Result> { + let mut results = Vec::new(); + match edge_type { + Some(et) => { + let mut stmt = self.conn.prepare( + "SELECT to_file, edge_type FROM edges WHERE from_file = ?1 AND edge_type = ?2", + )?; + let rows = stmt.query_map(params![file_id, et], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + })?; + for row in rows { + results.push(row?); + } + } + None => { + let mut stmt = self + .conn + .prepare("SELECT to_file, edge_type FROM edges WHERE from_file = ?1")?; + let rows = stmt.query_map(params![file_id], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + })?; + for row in rows { + results.push(row?); + } + } + } + Ok(results) + } + + /// Get incoming edges, optionally filtered by type. + pub fn get_incoming( + &self, + file_id: i64, + edge_type: Option<&str>, + ) -> Result> { + let mut results = Vec::new(); + match edge_type { + Some(et) => { + let mut stmt = self.conn.prepare( + "SELECT from_file, edge_type FROM edges WHERE to_file = ?1 AND edge_type = ?2", + )?; + let rows = stmt.query_map(params![file_id, et], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + })?; + for row in rows { + results.push(row?); + } + } + None => { + let mut stmt = self + .conn + .prepare("SELECT from_file, edge_type FROM edges WHERE to_file = ?1")?; + let rows = stmt.query_map(params![file_id], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + })?; + for row in rows { + results.push(row?); + } + } + } + Ok(results) + } + // ── Stats ─────────────────────────────────────────────────── pub fn stats(&self) -> Result { @@ -773,4 +891,122 @@ mod tests { // Non-existent docid returns None. assert!(store.get_file_by_docid("ffffff").unwrap().is_none()); } + + // ── Edge tests ───────────────────────────────────────────── + + /// Helper: create two files and return their IDs. + fn setup_two_files(store: &Store) -> (i64, i64) { + let a = store + .insert_file("notes/a.md", "ha", 100, &[], &generate_docid("notes/a.md")) + .unwrap(); + let b = store + .insert_file("notes/b.md", "hb", 100, &[], &generate_docid("notes/b.md")) + .unwrap(); + (a, b) + } + + #[test] + fn test_insert_and_get_edges() { + let store = Store::open_memory().unwrap(); + let (a, b) = setup_two_files(&store); + + store.insert_edge(a, b, "wikilink").unwrap(); + + let out = store.get_outgoing(a, None).unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out[0], (b, "wikilink".to_string())); + + let inc = store.get_incoming(b, None).unwrap(); + assert_eq!(inc.len(), 1); + assert_eq!(inc[0], (a, "wikilink".to_string())); + + // No edges in the other direction. + assert!(store.get_outgoing(b, None).unwrap().is_empty()); + assert!(store.get_incoming(a, None).unwrap().is_empty()); + } + + #[test] + fn test_delete_edges_for_file_both_directions() { + let store = Store::open_memory().unwrap(); + let (a, b) = setup_two_files(&store); + let c = store + .insert_file("notes/c.md", "hc", 100, &[], &generate_docid("notes/c.md")) + .unwrap(); + + // a -> b, c -> a + store.insert_edge(a, b, "wikilink").unwrap(); + store.insert_edge(c, a, "mention").unwrap(); + + // Delete edges for file a — should remove both. + store.delete_edges_for_file(a).unwrap(); + + assert!(store.get_outgoing(a, None).unwrap().is_empty()); + assert!(store.get_incoming(a, None).unwrap().is_empty()); + assert!(store.get_incoming(b, None).unwrap().is_empty()); + assert!(store.get_outgoing(c, None).unwrap().is_empty()); + } + + #[test] + fn test_edge_cascade_on_file_delete() { + let store = Store::open_memory().unwrap(); + let (a, b) = setup_two_files(&store); + let c = store + .insert_file("notes/c.md", "hc", 100, &[], &generate_docid("notes/c.md")) + .unwrap(); + + // a -> b, b -> c + store.insert_edge(a, b, "wikilink").unwrap(); + store.insert_edge(b, c, "mention").unwrap(); + + // Delete file b — CASCADE should remove both edges. + store.delete_file(b).unwrap(); + + assert!(store.get_outgoing(a, None).unwrap().is_empty()); + assert!(store.get_incoming(c, None).unwrap().is_empty()); + } + + #[test] + fn test_duplicate_edge_ignored() { + let store = Store::open_memory().unwrap(); + let (a, b) = setup_two_files(&store); + + store.insert_edge(a, b, "wikilink").unwrap(); + store.insert_edge(a, b, "wikilink").unwrap(); // duplicate + + let out = store.get_outgoing(a, None).unwrap(); + assert_eq!(out.len(), 1); + + // Same pair with different type is NOT a duplicate. + store.insert_edge(a, b, "mention").unwrap(); + let out = store.get_outgoing(a, None).unwrap(); + assert_eq!(out.len(), 2); + } + + #[test] + fn test_get_outgoing_filtered_by_type() { + let store = Store::open_memory().unwrap(); + let (a, b) = setup_two_files(&store); + let c = store + .insert_file("notes/c.md", "hc", 100, &[], &generate_docid("notes/c.md")) + .unwrap(); + + store.insert_edge(a, b, "wikilink").unwrap(); + store.insert_edge(a, c, "mention").unwrap(); + + let wikilinks = store.get_outgoing(a, Some("wikilink")).unwrap(); + assert_eq!(wikilinks.len(), 1); + assert_eq!(wikilinks[0].0, b); + + let mentions = store.get_outgoing(a, Some("mention")).unwrap(); + assert_eq!(mentions.len(), 1); + assert_eq!(mentions[0].0, c); + + // Incoming filtered. + let inc = store.get_incoming(b, Some("wikilink")).unwrap(); + assert_eq!(inc.len(), 1); + assert_eq!(inc[0].0, a); + + let inc = store.get_incoming(b, Some("mention")).unwrap(); + assert!(inc.is_empty()); + } } From 6493347bf1c5e33be33a6c7267c68abf42fb035b Mon Sep 17 00:00:00 2001 From: Oleksandr Ostrovskyi Date: Tue, 24 Mar 2026 21:43:05 +0200 Subject: [PATCH 2/8] =?UTF-8?q?feat:=20graph=20helper=20methods=20?= =?UTF-8?q?=E2=80=94=20neighbors,=20shared=20tags,=20FTS=20term=20check,?= =?UTF-8?q?=20edge=20stats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BFS-based neighbor traversal (Rust loop, not recursive CTE). Shared tags via JSON query. FTS5 term presence check with escaping. Best chunk lookup for snippet resolution. Edge statistics. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/store.rs | 279 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 279 insertions(+) diff --git a/src/store.rs b/src/store.rs index f66b487..60ece26 100644 --- a/src/store.rs +++ b/src/store.rs @@ -35,6 +35,16 @@ pub struct FtsResult { pub snippet: String, } +/// Statistics about edges in the graph. +#[derive(Debug)] +pub struct EdgeStats { + pub total_edges: usize, + pub wikilink_count: usize, + pub mention_count: usize, + pub connected_file_count: usize, + pub isolated_file_count: usize, +} + /// Summary statistics for the store. #[derive(Debug)] pub struct StoreStats { @@ -660,6 +670,113 @@ impl Store { } Ok(ids) } + + // ── Graph helpers ──────────────────────────────────────────── + + /// Get neighbor file IDs within N hops via wikilinks. + /// Uses Rust-side BFS, not recursive SQL CTE. + pub fn get_neighbors(&self, file_id: i64, depth: usize) -> Result> { + use std::collections::VecDeque; + let mut visited = HashSet::new(); + visited.insert(file_id); + let mut queue = VecDeque::new(); + let mut results = Vec::new(); + queue.push_back((file_id, 0usize)); + while let Some((current, current_depth)) = queue.pop_front() { + if current_depth >= depth { + continue; + } + let outgoing = self.get_outgoing(current, Some("wikilink"))?; + for (neighbor_id, _) in outgoing { + if visited.insert(neighbor_id) { + let hop = current_depth + 1; + results.push((neighbor_id, hop)); + queue.push_back((neighbor_id, hop)); + } + } + } + Ok(results) + } + + /// Find files that share at least one tag with the given file. + pub fn get_shared_tags_files(&self, file_id: i64, limit: usize) -> Result> { + let mut stmt = self.conn.prepare( + "SELECT DISTINCT f2.id + FROM files f1 + JOIN files f2 ON f2.id != f1.id + WHERE f1.id = ?1 + AND EXISTS ( + SELECT 1 FROM json_each(f1.tags) t1 + JOIN json_each(f2.tags) t2 ON t1.value = t2.value + ) + LIMIT ?2", + )?; + let rows = stmt.query_map(params![file_id, limit as i64], |row| row.get::<_, i64>(0))?; + let mut results = Vec::new(); + for row in rows { + results.push(row?); + } + Ok(results) + } + + /// Check if a file's FTS5 content contains a term. Escapes for FTS5. + pub fn file_contains_term(&self, file_id: i64, term: &str) -> Result { + let escaped = term.replace('"', "\"\""); + let query = format!("\"{}\"", escaped); + let result: Result = self.conn.query_row( + "SELECT 1 FROM chunks_fts WHERE chunks_fts MATCH ?1 AND file_id = ?2 LIMIT 1", + params![query, file_id], + |row| row.get(0), + ); + Ok(result.is_ok()) + } + + /// Get the best (highest token_count) chunk for a file. + pub fn get_best_chunk_for_file(&self, file_id: i64) -> Result> { + let mut stmt = self.conn.prepare( + "SELECT heading, snippet FROM chunks WHERE file_id = ?1 ORDER BY token_count DESC LIMIT 1", + )?; + let mut rows = stmt.query_map(params![file_id], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })?; + match rows.next() { + Some(r) => Ok(Some(r?)), + None => Ok(None), + } + } + + /// Get statistics about edges in the graph. + pub fn get_edge_stats(&self) -> Result { + let total: i64 = self + .conn + .query_row("SELECT COUNT(*) FROM edges", [], |r| r.get(0))?; + let wikilinks: i64 = self.conn.query_row( + "SELECT COUNT(*) FROM edges WHERE edge_type = 'wikilink'", + [], + |r| r.get(0), + )?; + let mentions: i64 = self.conn.query_row( + "SELECT COUNT(*) FROM edges WHERE edge_type = 'mention'", + [], + |r| r.get(0), + )?; + let connected: i64 = self.conn.query_row( + "SELECT COUNT(DISTINCT id) FROM files WHERE id IN \ + (SELECT from_file FROM edges UNION SELECT to_file FROM edges)", + [], + |r| r.get(0), + )?; + let total_files: i64 = self + .conn + .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))?; + Ok(EdgeStats { + total_edges: total as usize, + wikilink_count: wikilinks as usize, + mention_count: mentions as usize, + connected_file_count: connected as usize, + isolated_file_count: (total_files - connected) as usize, + }) + } } fn parse_tags(json: &str) -> Vec { @@ -1009,4 +1126,166 @@ mod tests { let inc = store.get_incoming(b, Some("mention")).unwrap(); assert!(inc.is_empty()); } + + // ── Graph helper tests ───────────────────────────────────── + + #[test] + fn test_get_neighbors_depth_1() { + let store = Store::open_memory().unwrap(); + let f1 = store + .insert_file("n/f1.md", "h1", 100, &[], &generate_docid("n/f1.md")) + .unwrap(); + let f2 = store + .insert_file("n/f2.md", "h2", 100, &[], &generate_docid("n/f2.md")) + .unwrap(); + let f3 = store + .insert_file("n/f3.md", "h3", 100, &[], &generate_docid("n/f3.md")) + .unwrap(); + + store.insert_edge(f1, f2, "wikilink").unwrap(); + store.insert_edge(f1, f3, "wikilink").unwrap(); + + let neighbors = store.get_neighbors(f1, 1).unwrap(); + assert_eq!(neighbors.len(), 2); + + let ids: Vec = neighbors.iter().map(|(id, _)| *id).collect(); + assert!(ids.contains(&f2)); + assert!(ids.contains(&f3)); + + // All at depth 1. + for (_, d) in &neighbors { + assert_eq!(*d, 1); + } + } + + #[test] + fn test_get_neighbors_depth_2() { + let store = Store::open_memory().unwrap(); + let f1 = store + .insert_file("n/f1.md", "h1", 100, &[], &generate_docid("n/f1.md")) + .unwrap(); + let f2 = store + .insert_file("n/f2.md", "h2", 100, &[], &generate_docid("n/f2.md")) + .unwrap(); + let f3 = store + .insert_file("n/f3.md", "h3", 100, &[], &generate_docid("n/f3.md")) + .unwrap(); + let f4 = store + .insert_file("n/f4.md", "h4", 100, &[], &generate_docid("n/f4.md")) + .unwrap(); + + // f1 -> f2 -> f3 -> f4 + store.insert_edge(f1, f2, "wikilink").unwrap(); + store.insert_edge(f2, f3, "wikilink").unwrap(); + store.insert_edge(f3, f4, "wikilink").unwrap(); + + let neighbors = store.get_neighbors(f1, 2).unwrap(); + assert_eq!(neighbors.len(), 2); + + // f2 at depth 1, f3 at depth 2, f4 NOT included. + let map: std::collections::HashMap = neighbors.into_iter().collect(); + assert_eq!(map[&f2], 1); + assert_eq!(map[&f3], 2); + assert!(!map.contains_key(&f4)); + } + + #[test] + fn test_get_shared_tags_files() { + let store = Store::open_memory().unwrap(); + let f1 = store + .insert_file( + "n/f1.md", + "h1", + 100, + &["rust".to_string(), "cli".to_string()], + &generate_docid("n/f1.md"), + ) + .unwrap(); + let f2 = store + .insert_file( + "n/f2.md", + "h2", + 100, + &["rust".to_string(), "web".to_string()], + &generate_docid("n/f2.md"), + ) + .unwrap(); + let _f3 = store + .insert_file( + "n/f3.md", + "h3", + 100, + &["python".to_string()], + &generate_docid("n/f3.md"), + ) + .unwrap(); + + let shared = store.get_shared_tags_files(f1, 10).unwrap(); + assert_eq!(shared.len(), 1); + assert_eq!(shared[0], f2); + } + + #[test] + fn test_file_contains_term() { + let store = Store::open_memory().unwrap(); + let f1 = store + .insert_file("n/fts.md", "h1", 100, &[], &generate_docid("n/fts.md")) + .unwrap(); + + store + .insert_fts_chunk(f1, 0, "BRE-2579 delivery date extension") + .unwrap(); + + assert!(store.file_contains_term(f1, "delivery").unwrap()); + assert!(store.file_contains_term(f1, "extension").unwrap()); + assert!(!store.file_contains_term(f1, "checkout").unwrap()); + } + + #[test] + fn test_get_best_chunk_for_file() { + let store = Store::open_memory().unwrap(); + let f1 = store + .insert_file("n/best.md", "h1", 100, &[], &generate_docid("n/best.md")) + .unwrap(); + + store + .insert_chunk(f1, "Small heading", "small snippet", 1, 10) + .unwrap(); + store + .insert_chunk(f1, "Big heading", "big snippet", 2, 100) + .unwrap(); + + let best = store.get_best_chunk_for_file(f1).unwrap().unwrap(); + assert_eq!(best.0, "Big heading"); + assert_eq!(best.1, "big snippet"); + } + + #[test] + fn test_get_edge_stats() { + let store = Store::open_memory().unwrap(); + let a = store + .insert_file("n/a.md", "ha", 100, &[], &generate_docid("n/a.md")) + .unwrap(); + let b = store + .insert_file("n/b.md", "hb", 100, &[], &generate_docid("n/b.md")) + .unwrap(); + let c = store + .insert_file("n/c.md", "hc", 100, &[], &generate_docid("n/c.md")) + .unwrap(); + // d is isolated (no edges). + let _d = store + .insert_file("n/d.md", "hd", 100, &[], &generate_docid("n/d.md")) + .unwrap(); + + store.insert_edge(a, b, "wikilink").unwrap(); + store.insert_edge(a, c, "wikilink").unwrap(); + store.insert_edge(b, c, "mention").unwrap(); + + let stats = store.get_edge_stats().unwrap(); + assert_eq!(stats.total_edges, 3); + assert_eq!(stats.wikilink_count, 2); + assert_eq!(stats.mention_count, 1); + assert_eq!(stats.connected_file_count, 3); // a, b, c + assert_eq!(stats.isolated_file_count, 1); // d + } } From fe1a31a61bf04a1f6c05e011d9ddda8bdd34d579 Mon Sep 17 00:00:00 2001 From: Oleksandr Ostrovskyi Date: Tue, 24 Mar 2026 21:44:49 +0200 Subject: [PATCH 3/8] feat: graph module with wikilink extraction and query term helpers Byte-scanning wikilink extractor handles [[target]], [[target|display]], [[target#heading]], skips embeds (![[...]]). Deduplicates targets. Query term extraction for graph agent relevance filtering. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/graph.rs | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + 2 files changed, 102 insertions(+) create mode 100644 src/graph.rs diff --git a/src/graph.rs b/src/graph.rs new file mode 100644 index 0000000..2276a56 --- /dev/null +++ b/src/graph.rs @@ -0,0 +1,101 @@ +use std::collections::HashSet; + +/// Extract unique wikilink targets from text. +/// Handles [[Target]], [[Target|Display]], [[Target#Heading]]. +/// Skips embeds (![[...]]). +pub fn extract_wikilink_targets(text: &str) -> Vec { + let bytes = text.as_bytes(); + let mut targets = Vec::new(); + let mut seen = HashSet::new(); + let mut i = 0; + + while i + 1 < bytes.len() { + if bytes[i] == b'[' && bytes[i + 1] == b'[' { + // Check for embed prefix (! before [[) + let is_embed = i > 0 && bytes[i - 1] == b'!'; + if let Some(rest) = text.get(i + 2..) + && let Some(close) = rest.find("]]") + { + let inner = &rest[..close]; + if !is_embed && !inner.is_empty() && !inner.contains('\n') { + // Strip heading: [[Note#Section]] → "Note" + let target = inner.split('#').next().unwrap_or(inner); + // Strip display: [[Note|Display]] → "Note" + let target = target.split('|').next().unwrap_or(target); + let target = target.trim().to_string(); + if !target.is_empty() && seen.insert(target.clone()) { + targets.push(target); + } + } + i += 2 + close + 2; + continue; + } + } + i += 1; + } + targets +} + +/// Extract query terms for relevance filtering. +/// Splits on whitespace, lowercases, drops terms shorter than 3 chars. +pub fn extract_query_terms(query: &str) -> Vec { + query + .split_whitespace() + .map(|t| t.to_lowercase()) + .filter(|t| t.len() >= 3) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_wikilink_targets() { + let text = + "See [[Note One]] and [[Note Two|display]] for details. Also [[Note One]] again."; + let targets = extract_wikilink_targets(text); + assert!(targets.contains(&"Note One".to_string())); + assert!(targets.contains(&"Note Two".to_string())); + assert_eq!(targets.len(), 2); // deduplicated + } + + #[test] + fn test_extract_wikilinks_with_headings() { + let text = "Link to [[Note#Section]] here."; + let targets = extract_wikilink_targets(text); + assert_eq!(targets, vec!["Note"]); + } + + #[test] + fn test_extract_wikilinks_empty() { + assert!(extract_wikilink_targets("no links here").is_empty()); + assert!(extract_wikilink_targets("").is_empty()); + } + + #[test] + fn test_extract_wikilinks_skip_embeds() { + let text = "![[embedded image.png]] and [[real link]]"; + let targets = extract_wikilink_targets(text); + assert_eq!(targets, vec!["real link"]); + } + + #[test] + fn test_extract_wikilinks_heading_and_display() { + let text = "[[Note#Section|Custom Display]]"; + let targets = extract_wikilink_targets(text); + assert_eq!(targets, vec!["Note"]); // strip both heading and display + } + + #[test] + fn test_extract_query_terms() { + let terms = extract_query_terms("BRE-2579 delivery date"); + assert_eq!(terms, vec!["bre-2579", "delivery", "date"]); + } + + #[test] + fn test_extract_query_terms_short_words_dropped() { + let terms = extract_query_terms("a is the big query"); + assert_eq!(terms, vec!["the", "big", "query"]); + } +} diff --git a/src/lib.rs b/src/lib.rs index 172715e..870efbf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub mod docid; pub mod embedder; pub mod fts; pub mod fusion; +pub mod graph; pub mod hnsw; pub mod indexer; pub mod model; From db3a359ad21a0f11e47d026af5d96d7bca99ab67 Mon Sep 17 00:00:00 2001 From: Oleksandr Ostrovskyi Date: Tue, 24 Mar 2026 21:47:36 +0200 Subject: [PATCH 4/8] =?UTF-8?q?feat:=20graph=20agent=20=E2=80=94=20expand?= =?UTF-8?q?=20search=20results=20by=20following=20wikilinks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1-2 hop expansion with decay (0.8x for 1-hop, 0.5x for 2-hop). Relevance filter: must contain query term (FTS5) or share tags. Multi-parent merge takes highest score. Skips seed files. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/graph.rs | 313 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 312 insertions(+), 1 deletion(-) diff --git a/src/graph.rs b/src/graph.rs index 2276a56..bb01f05 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -1,4 +1,9 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet, hash_map::Entry}; + +use anyhow::Result; + +use crate::fusion::RankedResult; +use crate::store::Store; /// Extract unique wikilink targets from text. /// Handles [[Target]], [[Target|Display]], [[Target#Heading]]. @@ -46,9 +51,112 @@ pub fn extract_query_terms(query: &str) -> Vec { .collect() } +/// Expand search results by following graph connections. +/// Seeds are the top results from semantic + FTS lanes. +/// Returns expanded results suitable for RRF fusion. +pub fn graph_expand( + store: &Store, + seeds: &[RankedResult], + query: &str, + max_hops: usize, + max_expansions: usize, +) -> Result> { + let query_terms = extract_query_terms(query); + let seed_ids: HashSet = seeds.iter().map(|s| s.file_id).collect(); + + // Track best score per expanded file (multi-parent merge: take highest) + // (file_id) → (best_score, hop_depth, seed_file_path) + let mut expansions: HashMap = HashMap::new(); + + for seed in seeds { + let neighbors = store.get_neighbors(seed.file_id, max_hops)?; + + for (neighbor_id, hop) in neighbors { + if seed_ids.contains(&neighbor_id) { + continue; + } + + let decay = match hop { + 1 => 0.8, + 2 => 0.5, + _ => 0.3, + }; + let mut expansion_score = seed.score * decay; + + // Relevance filter: must match a query term via FTS or share tags + let term_match = query_terms + .iter() + .any(|t| store.file_contains_term(neighbor_id, t).unwrap_or(false)); + + if !term_match { + let shared = store + .get_shared_tags_files(neighbor_id, 100) + .unwrap_or_default(); + if shared.contains(&seed.file_id) { + expansion_score *= 0.7; + } else { + continue; // tangential — skip + } + } + + // Multi-parent merge: keep highest score + match expansions.entry(neighbor_id) { + Entry::Occupied(mut e) => { + if expansion_score > e.get().0 { + e.insert((expansion_score, hop, seed.file_path.clone())); + } + } + Entry::Vacant(e) => { + e.insert((expansion_score, hop, seed.file_path.clone())); + } + } + } + } + + // Sort by score descending, cap at max_expansions + let mut results: Vec<(i64, f64, usize, String)> = expansions + .into_iter() + .map(|(fid, (score, hop, seed))| (fid, score, hop, seed)) + .collect(); + results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + results.truncate(max_expansions); + + // Convert to RankedResult + let mut ranked = Vec::new(); + for (file_id, score, _hop, _seed) in results { + let file = store.get_file_by_id(file_id)?; + let (file_path, docid) = match file { + Some(f) => (f.path, f.docid), + None => continue, + }; + let (heading, snippet) = store + .get_best_chunk_for_file(file_id)? + .unwrap_or_else(|| (String::new(), String::new())); + let heading = if heading.is_empty() { + None + } else { + Some(heading) + }; + + ranked.push(RankedResult { + file_path, + file_id, + score, + heading, + snippet, + docid, + }); + } + + Ok(ranked) +} + #[cfg(test)] mod tests { use super::*; + use crate::docid::generate_docid; + use crate::fusion::RankedResult; + use crate::store::Store; #[test] fn test_extract_wikilink_targets() { @@ -98,4 +206,207 @@ mod tests { let terms = extract_query_terms("a is the big query"); assert_eq!(terms, vec!["the", "big", "query"]); } + + #[test] + fn test_graph_expand_basic() { + let store = Store::open_memory().unwrap(); + let f1 = store + .insert_file( + "seed.md", + "h1", + 100, + &["rust".into()], + &generate_docid("seed.md"), + ) + .unwrap(); + let f2 = store + .insert_file( + "linked.md", + "h2", + 100, + &["rust".into()], + &generate_docid("linked.md"), + ) + .unwrap(); + let _f3 = store + .insert_file( + "unlinked.md", + "h3", + 100, + &[], + &generate_docid("unlinked.md"), + ) + .unwrap(); + + store.insert_edge(f1, f2, "wikilink").unwrap(); + store + .insert_chunk(f2, "## Linked", "Linked content about delivery", 10, 20) + .unwrap(); + store + .insert_fts_chunk(f2, 0, "Linked content about delivery") + .unwrap(); + + let seeds = vec![RankedResult { + file_path: "seed.md".into(), + file_id: f1, + score: 0.85, + heading: None, + snippet: "Seed".into(), + docid: None, + }]; + + let expanded = graph_expand(&store, &seeds, "delivery", 2, 20).unwrap(); + assert_eq!(expanded.len(), 1); + assert_eq!(expanded[0].file_path, "linked.md"); + assert!(expanded[0].score > 0.0 && expanded[0].score < 0.85); + } + + #[test] + fn test_graph_expand_skips_seeds() { + let store = Store::open_memory().unwrap(); + let f1 = store + .insert_file("a.md", "h1", 100, &[], &generate_docid("a.md")) + .unwrap(); + let f2 = store + .insert_file("b.md", "h2", 100, &[], &generate_docid("b.md")) + .unwrap(); + + store.insert_edge(f1, f2, "wikilink").unwrap(); + store.insert_chunk(f2, "## B", "Content B", 10, 20).unwrap(); + store.insert_fts_chunk(f2, 0, "Content B").unwrap(); + + let seeds = vec![ + RankedResult { + file_path: "a.md".into(), + file_id: f1, + score: 0.9, + heading: None, + snippet: "A".into(), + docid: None, + }, + RankedResult { + file_path: "b.md".into(), + file_id: f2, + score: 0.8, + heading: None, + snippet: "B".into(), + docid: None, + }, + ]; + + let expanded = graph_expand(&store, &seeds, "content", 2, 20).unwrap(); + assert!(expanded.is_empty()); + } + + #[test] + fn test_graph_expand_multi_parent_takes_highest() { + let store = Store::open_memory().unwrap(); + let f1 = store + .insert_file("a.md", "h1", 100, &[], &generate_docid("a.md")) + .unwrap(); + let f2 = store + .insert_file("b.md", "h2", 100, &[], &generate_docid("b.md")) + .unwrap(); + let f3 = store + .insert_file("shared.md", "h3", 100, &[], &generate_docid("shared.md")) + .unwrap(); + + store.insert_edge(f1, f3, "wikilink").unwrap(); + store.insert_edge(f2, f3, "wikilink").unwrap(); + store + .insert_chunk(f3, "## Shared", "Shared topic content", 10, 20) + .unwrap(); + store + .insert_fts_chunk(f3, 0, "Shared topic content") + .unwrap(); + + let seeds = vec![ + RankedResult { + file_path: "a.md".into(), + file_id: f1, + score: 0.9, + heading: None, + snippet: "A".into(), + docid: None, + }, + RankedResult { + file_path: "b.md".into(), + file_id: f2, + score: 0.5, + heading: None, + snippet: "B".into(), + docid: None, + }, + ]; + + let expanded = graph_expand(&store, &seeds, "topic", 1, 20).unwrap(); + assert_eq!(expanded.len(), 1); + assert_eq!(expanded[0].file_path, "shared.md"); + // Should use highest parent: 0.9 * 0.8 = 0.72 + assert!((expanded[0].score - 0.72).abs() < 0.01); + } + + #[test] + fn test_graph_expand_empty_graph() { + let store = Store::open_memory().unwrap(); + let f1 = store.insert_file("a.md", "h1", 100, &[], "aaa111").unwrap(); + + let seeds = vec![RankedResult { + file_path: "a.md".into(), + file_id: f1, + score: 0.9, + heading: None, + snippet: "A".into(), + docid: None, + }]; + + let expanded = graph_expand(&store, &seeds, "query", 2, 20).unwrap(); + assert!(expanded.is_empty()); + } + + #[test] + fn test_graph_expand_tag_fallback() { + let store = Store::open_memory().unwrap(); + let f1 = store + .insert_file( + "seed.md", + "h1", + 100, + &["rust".into(), "cli".into()], + &generate_docid("seed.md"), + ) + .unwrap(); + let f2 = store + .insert_file( + "linked.md", + "h2", + 100, + &["rust".into()], + &generate_docid("linked.md"), + ) + .unwrap(); + + store.insert_edge(f1, f2, "wikilink").unwrap(); + store + .insert_chunk(f2, "## Linked", "Unrelated content", 10, 20) + .unwrap(); + store + .insert_fts_chunk(f2, 0, "Unrelated content here") + .unwrap(); + + let seeds = vec![RankedResult { + file_path: "seed.md".into(), + file_id: f1, + score: 0.85, + heading: None, + snippet: "Seed".into(), + docid: None, + }]; + + // Query doesn't match FTS, but shared tag "rust" should keep it (with 0.7x penalty) + let expanded = graph_expand(&store, &seeds, "nonexistent query term", 2, 20).unwrap(); + assert_eq!(expanded.len(), 1); + // Score: 0.85 * 0.8 * 0.7 = 0.476 + assert!((expanded[0].score - 0.476).abs() < 0.01); + } } From bcd50e6b2c6c6cde2b312a69c88c9a3347865737 Mon Sep 17 00:00:00 2001 From: Oleksandr Ostrovskyi Date: Tue, 24 Mar 2026 21:51:49 +0200 Subject: [PATCH 5/8] =?UTF-8?q?feat:=20edge=20building=20in=20indexer=20?= =?UTF-8?q?=E2=80=94=20wikilinks=20and=20people=20detection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wikilink extraction runs on raw file content during indexing. Targets resolved by exact path then basename match (case-insensitive). Bidirectional edges inserted. People detection scans for name mentions in non-People files, with alias extraction from frontmatter. Rebuild clears edges before building. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/indexer.rs | 249 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 246 insertions(+), 3 deletions(-) diff --git a/src/indexer.rs b/src/indexer.rs index c2f69ff..3d96649 100644 --- a/src/indexer.rs +++ b/src/indexer.rs @@ -12,6 +12,7 @@ use crate::chunker::{chunk_markdown, split_oversized_chunks}; use crate::config::Config; use crate::docid::generate_docid; use crate::embedder::Embedder; +use crate::graph::extract_wikilink_targets; use crate::hnsw::HnswIndex; use crate::store::{FileRecord, Store}; @@ -129,6 +130,138 @@ pub fn diff_vault( Ok((new_files, changed_files, deleted)) } +/// Resolve a wikilink target name to a file ID in the store. +fn resolve_link_target(store: &Store, target: &str) -> Result> { + let with_ext = if target.ends_with(".md") { + target.to_string() + } else { + format!("{}.md", target) + }; + + // Try exact path match + if let Some(f) = store.get_file(&with_ext)? { + return Ok(Some(f.id)); + } + + // Try basename match (case-insensitive) + let all_files = store.get_all_files()?; + let target_lower = with_ext.to_lowercase(); + let mut matches: Vec<&FileRecord> = all_files + .iter() + .filter(|f| { + let path_lower = f.path.to_lowercase(); + path_lower == target_lower || path_lower.ends_with(&format!("/{}", target_lower)) + }) + .collect(); + + matches.sort_by_key(|f| f.path.len()); + Ok(matches.first().map(|f| f.id)) +} + +/// Build wikilink edges for a single file. +pub fn build_edges_for_file(store: &Store, file_id: i64, content: &str) -> Result<()> { + let targets = extract_wikilink_targets(content); + for target in targets { + if let Some(target_id) = resolve_link_target(store, &target)? + && target_id != file_id + { + store.insert_edge(file_id, target_id, "wikilink")?; + store.insert_edge(target_id, file_id, "wikilink")?; + } + } + Ok(()) +} + +/// Load people entities from the People folder. +/// Returns (file_id, [name, aliases...]) for each person note. +pub fn load_people_entities( + store: &Store, + people_folder: &str, + content_by_path: &HashMap, +) -> Result)>> { + let all_files = store.get_all_files()?; + let mut people = Vec::new(); + for file in &all_files { + if file.path.contains(people_folder) { + let basename = file.path.rsplit('/').next().unwrap_or(&file.path); + let name = basename.trim_end_matches(".md").to_string(); + let mut names = vec![name]; + + // Extract aliases from frontmatter + if let Some(content) = content_by_path.get(&file.path) + && let Some(aliases) = extract_aliases_from_frontmatter(content) + { + names.extend(aliases); + } + + people.push((file.id, names)); + } + } + Ok(people) +} + +/// Extract aliases from YAML frontmatter. +fn extract_aliases_from_frontmatter(content: &str) -> Option> { + let trimmed = content.trim_start(); + if !trimmed.starts_with("---") { + return None; + } + let after = trimmed[3..].trim_start_matches('-').strip_prefix('\n')?; + let end = after.find("\n---")?; + let yaml = &after[..end]; + + let lines: Vec<&str> = yaml.lines().collect(); + for (i, line) in lines.iter().enumerate() { + let t = line.trim(); + if t.starts_with("aliases:") { + let after_colon = t.strip_prefix("aliases:")?.trim(); + let mut aliases = Vec::new(); + if after_colon.starts_with('[') { + let inner = after_colon.trim_start_matches('[').trim_end_matches(']'); + for a in inner.split(',') { + let a = a.trim().trim_matches('"').trim_matches('\'').to_string(); + if !a.is_empty() { + aliases.push(a); + } + } + } else if after_colon.is_empty() { + for sub in &lines[i + 1..] { + let st = sub.trim(); + if st.starts_with("- ") { + aliases.push(st.strip_prefix("- ").unwrap().trim().to_string()); + } else if !st.is_empty() { + break; + } + } + } + return Some(aliases); + } + } + None +} + +/// Detect people mentions and create edges. +pub fn build_people_edges( + store: &Store, + file_id: i64, + content: &str, + people: &[(i64, Vec)], +) -> Result<()> { + let content_lower = content.to_lowercase(); + for (person_id, names) in people { + if *person_id == file_id { + continue; + } + let mentioned = names + .iter() + .any(|name| content_lower.contains(&name.to_lowercase())); + if mentioned { + store.insert_edge(file_id, *person_id, "mention")?; + } + } + Ok(()) +} + /// Main indexing orchestrator. /// /// Walks the vault, diffs against the store, processes new/changed/deleted files, @@ -205,6 +338,15 @@ pub fn run_index(vault_path: &Path, config: &Config, rebuild: bool) -> Result = file_contents + .iter() + .map(|(path, content)| { + let rel = path.strip_prefix(vault_path).unwrap_or(path); + (rel.to_string_lossy().to_string(), content.clone()) + }) + .collect(); + // Parallel chunking (embedding is serial since Embedder is not Send+Sync). let chunked_files: Vec<_> = file_contents .par_iter() @@ -318,7 +460,41 @@ pub fn run_index(vault_path: &Path, config: &Config, rebuild: bool) -> Result Result Result Date: Tue, 24 Mar 2026 21:54:01 +0200 Subject: [PATCH 6/8] =?UTF-8?q?feat:=203-lane=20RRF=20=E2=80=94=20graph=20?= =?UTF-8?q?agent=20as=20third=20search=20lane?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Graph lane expands seed results from semantic + FTS by following wikilinks. Weight 0.8 (slightly below direct lanes). Detail field on LaneContribution shows hop info in --explain output. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/fusion.rs | 18 ++++++++++++++---- src/search.rs | 20 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/fusion.rs b/src/fusion.rs index f1c7508..39b86e4 100644 --- a/src/fusion.rs +++ b/src/fusion.rs @@ -6,6 +6,7 @@ /// rrf_score = sum( weight_i / (k + rank_i) ) /// /// A ranked result from a single search lane. +#[derive(Clone)] pub struct RankedResult { pub file_path: String, pub file_id: i64, @@ -32,6 +33,7 @@ pub struct LaneContribution { pub rank: usize, pub raw_score: f64, pub weighted_contribution: f64, + pub detail: Option, // e.g., "1-hop from BRE-2579" } use std::collections::HashMap; @@ -93,6 +95,7 @@ pub fn rrf_fuse(lanes: &[(&str, &[RankedResult], f64)], k: usize) -> Vec Vec String { let mut out = format!(" RRF: {:.4}\n", result.rrf_score); for lc in &result.lane_contributions { - out.push_str(&format!( - " {}: rank #{}, raw {:.2}, +{:.4}\n", - lc.lane_name, lc.rank, lc.raw_score, lc.weighted_contribution, - )); + let detail_str = lc + .detail + .as_deref() + .map(|d| format!(" ({})", d)) + .unwrap_or_default(); + out += &format!( + " {}: rank #{}, raw {:.2}{}, +{:.4}\n", + lc.lane_name, lc.rank, lc.raw_score, detail_str, lc.weighted_contribution + ); } out } @@ -234,12 +242,14 @@ mod tests { rank: 1, raw_score: 0.87, weighted_contribution: 0.0164, + detail: None, }, LaneContribution { lane_name: "fts".to_string(), rank: 3, raw_score: 5.23, weighted_contribution: 0.0159, + detail: None, }, ], }; diff --git a/src/search.rs b/src/search.rs index 40376a5..89ec9c8 100644 --- a/src/search.rs +++ b/src/search.rs @@ -6,6 +6,7 @@ use serde_json::json; use crate::embedder::Embedder; use crate::fusion::{self, RankedResult}; +use crate::graph; use crate::hnsw::HnswIndex; use crate::store::{Store, StoreStats}; @@ -127,12 +128,31 @@ pub fn run_search( .unwrap_or(std::cmp::Ordering::Equal) }); + // --- Graph lane --- + // Combine seeds from semantic + FTS (deduplicated by file_path, take higher score) + let combined_seeds: Vec = { + let mut by_file: HashMap = HashMap::new(); + for r in semantic_results.iter().chain(fts_results.iter()) { + match by_file.get(&r.file_path) { + Some(existing) if r.score <= existing.score => {} + _ => { + by_file.insert(r.file_path.clone(), r.clone()); + } + } + } + by_file.into_values().collect() + }; + + let graph_results = + graph::graph_expand(&store, &combined_seeds, query, 2, 20).unwrap_or_default(); + // --- RRF Fusion --- const RRF_K: usize = 60; let fused = fusion::rrf_fuse( &[ ("semantic", &semantic_results, 1.0), ("fts", &fts_results, 1.0), + ("graph", &graph_results, 0.8), ], RRF_K, ); From fe0106a019418b24821df98f2722a60d5b4f27bf Mon Sep 17 00:00:00 2001 From: Oleksandr Ostrovskyi Date: Tue, 24 Mar 2026 21:57:08 +0200 Subject: [PATCH 7/8] =?UTF-8?q?feat:=20engraph=20graph=20CLI=20=E2=80=94?= =?UTF-8?q?=20show=20connections=20and=20stats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'engraph graph show ' displays outgoing/incoming wikilinks and mentions. Supports file path, basename, or #docid resolution. 'engraph graph stats' shows vault graph overview. Status output now includes edge counts. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/main.rs | 144 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/search.rs | 38 ++++++++++--- src/store.rs | 14 +++++ 3 files changed, 188 insertions(+), 8 deletions(-) diff --git a/src/main.rs b/src/main.rs index dfd6300..17d18a8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -81,6 +81,23 @@ enum Command { #[command(subcommand)] action: ModelsAction, }, + + /// Inspect vault graph connections. + Graph { + #[command(subcommand)] + action: GraphAction, + }, +} + +#[derive(Subcommand, Debug)] +enum GraphAction { + /// Show connections for a note. + Show { + /// File path or #docid. + file: String, + }, + /// Show vault graph statistics. + Stats, } #[derive(Subcommand, Debug)] @@ -316,6 +333,133 @@ fn main() -> Result<()> { ); } + Command::Graph { action } => { + if !index_exists(&data_dir) { + eprintln!("No index found. Run 'engraph index ' first."); + std::process::exit(1); + } + let db_path = data_dir.join("engraph.db"); + let store = store::Store::open(&db_path)?; + + match action { + GraphAction::Show { file } => { + // Resolve: docid first, then exact path, then basename + let record = if file.starts_with('#') && file.len() == 7 { + store.get_file_by_docid(&file[1..])? + } else if let Some(f) = store.get_file(&file)? { + Some(f) + } else { + // Basename match (case-insensitive) + let target = if file.ends_with(".md") { + file.clone() + } else { + format!("{}.md", file) + }; + let all = store.get_all_files()?; + let target_lower = target.to_lowercase(); + all.into_iter().find(|f| { + let path_lower = f.path.to_lowercase(); + path_lower == target_lower + || path_lower.ends_with(&format!("/{}", target_lower)) + }) + }; + + let record = match record { + Some(r) => r, + None => { + eprintln!("File not found: {file}"); + std::process::exit(1); + } + }; + + let docid_str = record + .docid + .as_deref() + .map(|d| format!(" (#{d})")) + .unwrap_or_default(); + println!("{}{}\n", record.path, docid_str); + + let outgoing_wl = store.get_outgoing(record.id, Some("wikilink"))?; + println!("Outgoing wikilinks ({}):", outgoing_wl.len()); + for (fid, _) in &outgoing_wl { + if let Some(f) = store.get_file_by_id(*fid)? { + let did = f + .docid + .as_deref() + .map(|d| format!(" (#{d})")) + .unwrap_or_default(); + println!(" → {}{}", f.path, did); + } + } + + println!(); + let incoming_wl = store.get_incoming(record.id, Some("wikilink"))?; + println!("Incoming wikilinks ({}):", incoming_wl.len()); + for (fid, _) in &incoming_wl { + if let Some(f) = store.get_file_by_id(*fid)? { + let did = f + .docid + .as_deref() + .map(|d| format!(" (#{d})")) + .unwrap_or_default(); + println!(" ← {}{}", f.path, did); + } + } + + println!(); + let mentions_out = store.get_outgoing(record.id, Some("mention"))?; + let mentions_in = store.get_incoming(record.id, Some("mention"))?; + println!("Mentions out ({}):", mentions_out.len()); + for (fid, _) in &mentions_out { + if let Some(f) = store.get_file_by_id(*fid)? { + let did = f + .docid + .as_deref() + .map(|d| format!(" (#{d})")) + .unwrap_or_default(); + println!(" → {}{}", f.path, did); + } + } + if !mentions_in.is_empty() { + println!("Mentioned by ({}):", mentions_in.len()); + for (fid, _) in &mentions_in { + if let Some(f) = store.get_file_by_id(*fid)? { + let did = f + .docid + .as_deref() + .map(|d| format!(" (#{d})")) + .unwrap_or_default(); + println!(" ← {}{}", f.path, did); + } + } + } + } + + GraphAction::Stats => { + let stats = store.get_edge_stats()?; + println!("Vault Graph:"); + println!( + " Wikilink edges: {} ({} bidirectional pairs)", + stats.wikilink_count, + stats.wikilink_count / 2 + ); + println!(" Mention edges: {}", stats.mention_count); + println!(" Total edges: {}", stats.total_edges); + let total_files = stats.connected_file_count + stats.isolated_file_count; + let pct = if total_files > 0 { + stats.connected_file_count as f64 / total_files as f64 * 100.0 + } else { + 0.0 + }; + println!( + " Connected files: {} / {} ({:.1}%)", + stats.connected_file_count, total_files, pct + ); + println!(" Isolated files: {}", stats.isolated_file_count); + } + } + } + Command::Models { action } => { let registry = model::ModelRegistry::default(); match action { diff --git a/src/search.rs b/src/search.rs index 89ec9c8..a89d299 100644 --- a/src/search.rs +++ b/src/search.rs @@ -263,7 +263,7 @@ pub fn format_status(stats: &StoreStats, index_size: u64, model_name: &str, json let last_indexed = stats.last_indexed_at.as_deref().unwrap_or("never"); if json { - let obj = json!({ + let mut obj = json!({ "vault": vault, "files": stats.file_count, "chunks": stats.chunk_count, @@ -272,24 +272,40 @@ pub fn format_status(stats: &StoreStats, index_size: u64, model_name: &str, json "index_size": index_size, "model": model_name, }); + if let (Some(edges), Some(wl), Some(mn)) = + (stats.edge_count, stats.wikilink_count, stats.mention_count) + { + obj["edges"] = json!(edges); + obj["wikilink_edges"] = json!(wl); + obj["mention_edges"] = json!(mn); + } format!("{}\n", serde_json::to_string_pretty(&obj).unwrap()) } else { - format!( + let mut out = format!( "Vault: {}\n\ Files: {}\n\ - Chunks: {}\n\ - Tombstones: {} (pending cleanup)\n\ + Chunks: {}\n", + vault, stats.file_count, stats.chunk_count, + ); + if let (Some(edges), Some(wl), Some(mn)) = + (stats.edge_count, stats.wikilink_count, stats.mention_count) + { + out.push_str(&format!( + "Edges: {} ({} wikilinks, {} mentions)\n", + edges, wl, mn + )); + } + out.push_str(&format!( + "Tombstones: {} (pending cleanup)\n\ Last index: {}\n\ Index size: {}\n\ Model: {}\n", - vault, - stats.file_count, - stats.chunk_count, stats.tombstone_count, last_indexed, format_bytes(index_size), model_name, - ) + )); + out } } @@ -412,6 +428,9 @@ mod tests { tombstone_count: 3, last_indexed_at: Some("2026-03-19 14:30:00".to_string()), vault_path: Some("/path/to/vault".to_string()), + edge_count: None, + wikilink_count: None, + mention_count: None, }; let output = format_status(&stats, 2_516_582, "all-MiniLM-L6-v2", false); @@ -432,6 +451,9 @@ mod tests { tombstone_count: 3, last_indexed_at: Some("2026-03-19 14:30:00".to_string()), vault_path: Some("/path/to/vault".to_string()), + edge_count: None, + wikilink_count: None, + mention_count: None, }; let output = format_status(&stats, 2_516_582, "all-MiniLM-L6-v2", true); let parsed: serde_json::Value = serde_json::from_str(&output).unwrap(); diff --git a/src/store.rs b/src/store.rs index 60ece26..c318122 100644 --- a/src/store.rs +++ b/src/store.rs @@ -53,6 +53,9 @@ pub struct StoreStats { pub tombstone_count: usize, pub last_indexed_at: Option, pub vault_path: Option, + pub edge_count: Option, + pub wikilink_count: Option, + pub mention_count: Option, } const SCHEMA: &str = r#" @@ -520,12 +523,23 @@ impl Store { let tombstone_count = self.tombstone_count()?; let last_indexed_at = self.get_meta("last_indexed_at")?; let vault_path = self.get_meta("vault_path")?; + let (edge_count, wikilink_count, mention_count) = match self.get_edge_stats() { + Ok(es) => ( + Some(es.total_edges), + Some(es.wikilink_count), + Some(es.mention_count), + ), + Err(_) => (None, None, None), + }; Ok(StoreStats { file_count: file_count as usize, chunk_count: chunk_count as usize, tombstone_count, last_indexed_at, vault_path, + edge_count, + wikilink_count, + mention_count, }) } From 88bda9541ff34009d9bff25a9e557359cfd0ea11 Mon Sep 17 00:00:00 2001 From: Oleksandr Ostrovskyi Date: Tue, 24 Mar 2026 21:58:22 +0200 Subject: [PATCH 8/8] =?UTF-8?q?chore:=20bump=20to=20v0.3.0=20=E2=80=94=20v?= =?UTF-8?q?ault=20graph=20and=20graph=20agent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Version bump and docs update for engraph v0.3.0: - Vault graph with wikilink + mention edges - Graph agent as 3rd RRF search lane - engraph graph show/stats CLI - 119 tests, 12 modules Co-Authored-By: Claude Opus 4.6 (1M context) --- CLAUDE.md | 42 ++++++++++++++++++++++-------------------- Cargo.toml | 2 +- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 44697a5..b9d8d0d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,10 +1,10 @@ # engraph -Local semantic search CLI for Obsidian vaults. Rust, MIT licensed. +Local hybrid search CLI for Obsidian vaults. Rust, MIT licensed. ## Architecture -Single binary with 11 modules behind a lib crate: +Single binary with 12 modules behind a lib crate: - `config.rs` — loads `~/.engraph/config.toml` and `vault.toml`, merges CLI args, provides `data_dir()` - `chunker.rs` — smart chunking with break-point scoring algorithm. Finds optimal split points considering headings, code fences, blank lines, and thematic breaks. `split_oversized_chunks()` handles token-aware secondary splitting with overlap @@ -12,48 +12,51 @@ Single binary with 11 modules behind a lib crate: - `embedder.rs` — downloads and runs `all-MiniLM-L6-v2` ONNX model (384-dim). SHA256-verified on download. Uses `ort` for inference, `tokenizers` for tokenization. Implements `ModelBackend` trait - `model.rs` — pluggable `ModelBackend` trait, model registry, and `parse_model_spec()`. Enables future model swapping without changing consumer code - `fts.rs` — FTS5 full-text search support. Re-exports `FtsResult` from store. BM25-ranked keyword search -- `fusion.rs` — Reciprocal Rank Fusion (RRF) engine. Merges semantic + FTS5 results. Supports lane weighting and `--explain` output +- `fusion.rs` — Reciprocal Rank Fusion (RRF) engine. Merges semantic + FTS5 + graph results. Supports lane weighting, `--explain` output with per-lane detail +- `graph.rs` — vault graph agent. Extracts wikilink targets, expands search results by following graph connections 1-2 hops. Relevance filtering via FTS5 term check and shared tags - `profile.rs` — vault profile detection. Auto-detects PARA/Folders/Flat structure, vault type (Obsidian/Logseq/Plain), wikilinks, frontmatter, tags. Writes/loads `vault.toml` -- `store.rs` — SQLite persistence. Tables: `meta`, `files` (with docid), `chunks` (with vector BLOBs), `chunks_fts` (FTS5 virtual table), `tombstones`. Handles incremental diffing via content hashes +- `store.rs` — SQLite persistence. Tables: `meta`, `files` (with docid), `chunks` (with vector BLOBs), `chunks_fts` (FTS5), `edges` (vault graph), `tombstones`. Handles incremental diffing via content hashes - `hnsw.rs` — thin wrapper around `hnsw_rs`. **Important:** `hnsw_rs` does not support inserting after `load_hnsw()`. The index is rebuilt from vectors stored in SQLite on every index run -- `indexer.rs` — orchestrates vault walking (via `ignore` crate for `.gitignore` support), diffing, chunking, embedding (Rayon for parallel chunking, serial embedding since `Embedder` is not `Send`), and serial writes to store + HNSW + FTS5 +- `indexer.rs` — orchestrates vault walking (via `ignore` crate for `.gitignore` support), diffing, chunking, embedding (Rayon for parallel chunking, serial embedding since `Embedder` is not `Send`), serial writes to store + HNSW + FTS5, and vault graph edge building (wikilinks + people detection) -`main.rs` is a thin clap CLI that wires the modules together. Subcommands: `index`, `search` (with `--explain`), `status`, `clear`, `init`, `configure`, `models`. +`main.rs` is a thin clap CLI. Subcommands: `index`, `search` (with `--explain`), `status`, `clear`, `init`, `configure`, `models`, `graph` (show/stats). ## Key patterns -- **Hybrid search:** Queries run through two lanes — semantic (HNSW embeddings) and keyword (FTS5 BM25). Results are fused via Reciprocal Rank Fusion (RRF) with configurable lane weights -- **Smart chunking:** Break-point scoring algorithm assigns scores to potential split points (headings 50-100, code fences 80, thematic breaks 60, blank lines 20). Chunks split at the highest-scored break point near the token target. Code fence protection prevents splitting inside code blocks -- **Incremental indexing:** `diff_vault()` compares file content hashes in SQLite against disk. Changed files have their old chunks deleted (cascade), then are re-embedded as new. FTS5 entries are cleaned up alongside vector entries -- **HNSW rebuild on every run:** Vectors are stored as BLOBs in the `chunks` table. After SQLite is updated, the full HNSW index is rebuilt from `store.get_all_vectors()`. This is necessary because `hnsw_rs` doesn't support append-after-load -- **Docids:** Each file gets a deterministic 6-char hex ID (SHA-256 of relative path). Displayed in search results for quick reference +- **3-lane hybrid search:** Queries run through three lanes — semantic (HNSW embeddings), keyword (FTS5 BM25), and graph (wikilink expansion). Results are fused via Reciprocal Rank Fusion (RRF) with configurable lane weights (semantic 1.0, FTS 1.0, graph 0.8) +- **Vault graph:** `edges` table stores bidirectional wikilink edges and mention edges. Built during indexing after all files are written. People detection scans for person name/alias mentions using notes from the configured People folder +- **Graph agent:** Expands seed results by following wikilinks 1-2 hops. Decay: 0.8× for 1-hop, 0.5× for 2-hop. Relevance filter: must contain query term (FTS5) or share tags with seed. Multi-parent merge takes highest score +- **Smart chunking:** Break-point scoring algorithm assigns scores to potential split points (headings 50-100, code fences 80, thematic breaks 60, blank lines 20). Code fence protection prevents splitting inside code blocks +- **Incremental indexing:** `diff_vault()` compares file content hashes in SQLite against disk. Changed files have their old chunks and edges deleted, then are re-processed. FTS5 entries cleaned up alongside vector entries +- **HNSW rebuild on every run:** Vectors stored as BLOBs. Full HNSW index rebuilt from `store.get_all_vectors()` after SQLite update (hnsw_rs limitation) +- **Docids:** Each file gets a deterministic 6-char hex ID. Displayed in search results - **Vault profiles:** `engraph init` auto-detects vault structure and writes `vault.toml` -- **Pluggable models:** `ModelBackend` trait enables future model swapping. Current implementation uses ONNX all-MiniLM-L6-v2 +- **Pluggable models:** `ModelBackend` trait enables future model swapping ## Data directory -`~/.engraph/` — hardcoded via `Config::data_dir()` (uses `dirs::home_dir()`). Contains `engraph.db` (SQLite with FTS5), `hnsw/` (index files), `models/` (ONNX model + tokenizer), `vault.toml` (vault profile), `config.toml` (user config). +`~/.engraph/` — hardcoded via `Config::data_dir()`. Contains `engraph.db` (SQLite with FTS5 + edges), `hnsw/` (index files), `models/` (ONNX model + tokenizer), `vault.toml` (vault profile), `config.toml` (user config). Single vault only. Re-indexing a different vault path triggers a confirmation prompt. ## Dependencies to be aware of -- `ort` (2.0.0-rc.12) — ONNX Runtime Rust bindings. Pre-release API. `Session::builder()?.commit_from_file()` pattern. Does not provide prebuilt binaries for all targets (no x86_64-apple-darwin) -- `hnsw_rs` (0.3) — pure Rust HNSW. `Box::leak` used in `load()` to satisfy `'static` lifetime on the loaded index. Read-only after load +- `ort` (2.0.0-rc.12) — ONNX Runtime Rust bindings. Pre-release API. Does not provide prebuilt binaries for all targets +- `hnsw_rs` (0.3) — pure Rust HNSW. `Box::leak` in `load()`. Read-only after load - `tokenizers` (0.22) — HuggingFace tokenizer. Needs `fancy-regex` feature -- `ignore` (0.4) — vault walking with automatic `.gitignore` support +- `ignore` (0.4) — vault walking with `.gitignore` support - `rusqlite` (0.32) — bundled SQLite with FTS5 support ## Testing -- Unit tests in each module (`cargo test --lib`) — 91 tests, no network required +- Unit tests in each module (`cargo test --lib`) — 119 tests, no network required - 1 ignored smoke test (`test_embed_smoke`) — downloads ONNX model, verifies embedding -- Integration tests (`cargo test --test integration -- --ignored`) — 8 tests, require model download. Use `tempfile` for isolated data dirs +- Integration tests (`cargo test --test integration -- --ignored`) — 8 tests, require model download ## CI/CD - CI: `cargo fmt --check` + `cargo clippy -- -D warnings` + `cargo test --lib` on macOS + Ubuntu -- Release: native builds on macOS arm64 (macos-14) + Linux x86_64 (ubuntu-latest). Triggered by `v*` tags. No x86_64 macOS build (ort-sys limitation) +- Release: native builds on macOS arm64 (macos-14) + Linux x86_64 (ubuntu-latest). Triggered by `v*` tags - Homebrew: `devwhodevs/homebrew-tap` — formula builds from source tarball ## Common tasks @@ -70,5 +73,4 @@ cargo build --release # Release: tag and push git tag v0.x.y && git push origin v0.x.y -# Then update homebrew-tap formula with new SHA256 ``` diff --git a/Cargo.toml b/Cargo.toml index 63f03c1..09f68e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "engraph" -version = "0.2.0" +version = "0.3.0" edition = "2024" description = "Local semantic search for Obsidian vaults" license = "MIT"