Watch selected folders and reconcile bounded changes with media freshness guards

crates/searchi-service/src/lib.rs

@@ -272,7 +272,7 @@ impl HybridSearch {
             });
         }
         ensure!(
-            (6..=7).contains(&version),
+            (6..=8).contains(&version),
             "Unsupported semantic schema version {version}"
         );
         let (generation,fingerprint,revision,current):(Option<i64>,Option<String>,Option<i64>,i64)=connection.query_row(

crates/searchi-service/src/media.rs

@@ -141,6 +141,11 @@ impl HybridSearch {
         task: &SourceTask,
         work: &mut BuildWork,
     ) -> Result<()> {
+        let change_revision = self.lexical.media_change_revision(
+            &path.to_string_lossy(),
+            identity.bytes,
+            identity.modified_ns,
+        )?;
         if let Some(count) = self
             .store
             .reuse_media_source(generation, task, id, identity)?
@@ -152,7 +157,7 @@ impl HybridSearch {
             if kind != MediaKind::Image {
                 // Metadata remains part of music search after path remapping.
                 let metadata = probe(tools, path, limits)?;
-                self.refresh_media_metadata(path, identity, &metadata)?;
+                self.refresh_media_metadata(path, identity, &metadata, change_revision)?;
             }
             identity.ensure_unchanged()?;
             work.report.chunks += count;
@@ -169,6 +174,7 @@ impl HybridSearch {
             tools,
             limits,
             identity,
+            change_revision,
             &mut work.report,
         )?;
         self.store
@@ -182,6 +188,7 @@ impl HybridSearch {
         path: &Path,
         identity: &MediaIdentity,
         metadata: &MediaProbe,
+        change_revision: i64,
     ) -> Result<()> {
         let tags = metadata
             .tags
@@ -189,12 +196,13 @@ impl HybridSearch {
             .map(|(k, v)| format!("{k}: {v}"))
             .collect::<Vec<_>>()
             .join("\n");
-        self.lexical.set_media_metadata(
+        self.lexical.set_media_metadata_at_revision(
             &path.to_string_lossy(),
             &tags,
             metadata.duration_ms,
             identity.bytes,
             identity.modified_ns,
+            change_revision,
         )
     }
 
@@ -208,6 +216,7 @@ impl HybridSearch {
         tools: &FfmpegTools,
         limits: &MediaLimits,
         identity: &MediaIdentity,
+        change_revision: i64,
         report: &mut EmbeddingReport,
     ) -> Result<()> {
         let mut ordinal = 0;
@@ -228,7 +237,7 @@ impl HybridSearch {
             report.chunks += 1;
         } else {
             let metadata = probe(tools, path, limits)?;
-            self.refresh_media_metadata(path, identity, &metadata)?;
+            self.refresh_media_metadata(path, identity, &metadata, change_revision)?;
             if metadata.has_video && kind == MediaKind::Video {
                 let mut seen = std::collections::HashSet::new();
                 let mut scenes = 0;

crates/searchi-service/src/store.rs

@@ -111,7 +111,7 @@ impl Store {
             |row| row.get(0),
         )?;
         ensure!(
-            (1..=7).contains(&version),
+            (1..=8).contains(&version),
             "Unsupported semantic schema version {version}"
         );
         initialization.execute_batch("
@@ -273,7 +273,7 @@ impl Store {
             )
         };
         let bytes: Vec<u8> = vector.iter().flat_map(|v| v.to_le_bytes()).collect();
-        self.connection.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime) VALUES(?1,?2,?3,?4,?5,?6,?7,0,0,?8,?9,?10,?11,?12)",params![generation,-file_id,ordinal as i64,path,kind,timestamp,identity.sha256,snippet,bytes,source_type,identity.bytes as i64,identity.modified_ns])?;
+        self.connection.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,file_change_revision) VALUES(?1,?2,?3,?4,?5,?6,?7,0,0,?8,?9,?10,?11,?12,(SELECT change_revision FROM files WHERE path=?4))",params![generation,-file_id,ordinal as i64,path,kind,timestamp,identity.sha256,snippet,bytes,source_type,identity.bytes as i64,identity.modified_ns])?;
         Ok(())
     }
 
@@ -991,7 +991,7 @@ fn collect_eligibility(
     };
     let mut statement=connection.prepare(&format!("SELECT c.rowid,c.passage_id,c.content_hash,p.text,c.path,c.source_type FROM semantic_chunks c
         LEFT JOIN passages p ON c.source_type='text' AND p.rowid=c.passage_id AND p.file_path=c.path AND p.kind=c.kind AND p.number=c.number
-        JOIN files f ON f.path=c.path WHERE c.generation=?1 AND ((c.source_type='text' AND p.rowid IS NOT NULL) OR (c.source_type!='text' AND c.file_size=f.size AND c.file_mtime=f.mtime))
+        JOIN files f ON f.path=c.path WHERE c.generation=?1 AND ((c.source_type='text' AND p.rowid IS NOT NULL) OR (c.source_type!='text' AND c.file_size=f.size AND c.file_mtime=f.mtime AND c.file_change_revision=f.change_revision))
         AND (?2 IS NULL OR substr(f.path,1,length(?2))=?2) AND (?3 IS NULL OR f.extension=?3)
         AND (?4 IS NULL OR f.size>=?4) AND (?5 IS NULL OR f.size<=?5)
         AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7)

crates/searchi-service/src/store/incremental.rs

@@ -22,6 +22,11 @@ pub(super) fn migrate(connection: &Connection) -> Result<()> {
             "INTEGER NOT NULL DEFAULT 0",
         ),
         ("semantic_chunks", "task_key", "TEXT"),
+        (
+            "semantic_chunks",
+            "file_change_revision",
+            "INTEGER NOT NULL DEFAULT 0",
+        ),
     ] {
         let exists: bool = connection.query_row(
             "SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name=?2)",
@@ -43,7 +48,7 @@ pub(super) fn migrate(connection: &Connection) -> Result<()> {
         PRIMARY KEY(generation,task_key));
         CREATE INDEX IF NOT EXISTS semantic_task_reuse ON semantic_tasks(source_hash,title_hash,policy,source_type,state);
         CREATE INDEX IF NOT EXISTS semantic_chunk_task ON semantic_chunks(generation,task_key);
-        UPDATE semantic_meta SET schema_version=7 WHERE singleton=1;")?;
+        UPDATE semantic_meta SET schema_version=8 WHERE singleton=1;")?;
     Ok(())
 }
 
@@ -294,8 +299,8 @@ impl Store {
         for (key, donor) in mappings {
             tx.execute("INSERT INTO semantic_tasks(generation,task_key,path,source_type,source_hash,title_hash,policy,passage_id,state,attempts,expected_chunks,message)
                 SELECT ?1,d.task_key,d.path,d.source_type,d.source_hash,d.title_hash,d.policy,d.passage_id,'done',old.attempts,old.expected_chunks,NULL FROM semantic_discovery_tasks d JOIN semantic_prune_tasks old ON old.task_key=?3 WHERE d.task_key=?2",params![generation,key,donor])?;
-            tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key)
-                SELECT ?1,d.passage_id,c.ordinal,d.path,c.kind,c.number,c.content_hash,c.start_byte,c.end_byte,c.snippet,c.vector,c.source_type,c.file_size,c.file_mtime,d.task_key FROM semantic_prune_chunks c JOIN semantic_discovery_tasks d ON d.task_key=?2 WHERE c.task_key=?3",params![generation,key,donor])?;
+            tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key,file_change_revision)
+                SELECT ?1,d.passage_id,c.ordinal,d.path,c.kind,c.number,c.content_hash,c.start_byte,c.end_byte,c.snippet,c.vector,c.source_type,c.file_size,c.file_mtime,d.task_key,COALESCE((SELECT change_revision FROM files WHERE path=d.path),c.file_change_revision) FROM semantic_prune_chunks c JOIN semantic_discovery_tasks d ON d.task_key=?2 WHERE c.task_key=?3",params![generation,key,donor])?;
         }
         tx.execute(
             "DELETE FROM semantic_media_errors WHERE generation=?1",
@@ -530,8 +535,8 @@ impl Store {
             "DELETE FROM semantic_chunks WHERE generation=?1 AND task_key=?2",
             params![generation, task.key],
         )?;
-        tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key)
-            SELECT ?1,?2,ordinal,?3,COALESCE(?4,kind),COALESCE(?5,number),content_hash,start_byte,end_byte,snippet,vector,source_type,COALESCE(?6,file_size),COALESCE(?7,file_mtime),?8 FROM semantic_reuse_chunks",params![generation,task.passage_id,task.path,location.map(|s|s.kind.as_str()),location.map(|s|s.number),identity.map(|i|i.bytes as i64),identity.map(|i|i.modified_ns),task.key])?;
+        tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key,file_change_revision)
+            SELECT ?1,?2,ordinal,?3,COALESCE(?4,kind),COALESCE(?5,number),content_hash,start_byte,end_byte,snippet,vector,source_type,COALESCE(?6,file_size),COALESCE(?7,file_mtime),?8,COALESCE((SELECT change_revision FROM files WHERE path=?3),file_change_revision) FROM semantic_reuse_chunks",params![generation,task.passage_id,task.path,location.map(|s|s.kind.as_str()),location.map(|s|s.number),identity.map(|i|i.bytes as i64),identity.map(|i|i.modified_ns),task.key])?;
         let changed=tx.execute("UPDATE semantic_tasks SET state='done',expected_chunks=?3,path=?4,passage_id=?5,message=NULL WHERE generation=?1 AND task_key=?2 AND source_hash=?6 AND title_hash=?7 AND policy=?8 AND source_type=?9",params![generation,task.key,count as i64,task.path,task.passage_id,task.source_hash,task.title_hash,task.policy,task.source_type])?;
         ensure!(changed == 1, "Missing source task during reuse");
         if task.source_type == "media" {