diff --git a/apps/desktop/src-tauri/src/media_preview.rs b/apps/desktop/src-tauri/src/media_preview.rs index acf42ac..b3859e8 100644 --- a/apps/desktop/src-tauri/src/media_preview.rs +++ b/apps/desktop/src-tauri/src/media_preview.rs @@ -13,7 +13,6 @@ use std::{ atomic::{AtomicUsize, Ordering}, Arc, Mutex, }, - time::UNIX_EPOCH, }; use tauri::http::{header, Method, Request, Response, StatusCode}; @@ -38,8 +37,7 @@ impl Identity { let modified_ns = metadata .modified() .ok() - .and_then(|value| value.duration_since(UNIX_EPOCH).ok()) - .and_then(|value| i64::try_from(value.as_nanos()).ok()) + .and_then(|value| searchi_media::unix_time_ns(value).ok()) .ok_or("Preview source timestamp is unavailable.")?; Ok(Self { len: metadata.len(), @@ -503,6 +501,35 @@ mod tests { } } + #[test] + fn pre_epoch_media_is_indexed_and_previewed_with_current_identity() { + let mut fixture = Fixture::new(64); + File::options() + .write(true) + .open(&fixture.path) + .unwrap() + .set_times( + std::fs::FileTimes::new() + .set_modified(std::time::UNIX_EPOCH - std::time::Duration::from_secs(1)), + ) + .unwrap(); + assert_eq!( + fixture.request(Method::GET, None).status(), + StatusCode::GONE + ); + SearchIndex::open(&fixture.database) + .unwrap() + .index_folder(fixture.path.parent().unwrap()) + .unwrap(); + fixture.token = fixture + .registry + .grant(fixture.database.clone(), fixture.path.clone(), "audio") + .unwrap(); + let response = fixture.request(Method::GET, None); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.body().len(), 64); + } + #[test] fn large_media_never_allocates_the_full_file_and_head_is_metadata_only() { let fixture = Fixture::new(8 * MAX_CHUNK); diff --git a/crates/searchi-core/src/lib.rs b/crates/searchi-core/src/lib.rs index d6d3e35..cbf3e5d 100644 --- a/crates/searchi-core/src/lib.rs +++ b/crates/searchi-core/src/lib.rs @@ -21,7 +21,6 @@ use std::{ fs, path::{Path, PathBuf}, sync::atomic::{AtomicBool, Ordering}, - time::UNIX_EPOCH, }; use walkdir::WalkDir; @@ -657,12 +656,7 @@ impl SearchIndex { report.discovered += 1; let metadata = entry.metadata()?; let size = i64::try_from(metadata.len()).context("file exceeds supported size")?; - let modified = metadata - .modified()? - .duration_since(UNIX_EPOCH) - .context("file timestamp predates Unix epoch")?; - let mtime = i64::try_from(modified.as_nanos()) - .context("file timestamp exceeds supported range")?; + let mtime = searchi_media::unix_time_ns(metadata.modified()?)?; let extension = entry .path() .extension() @@ -1257,7 +1251,7 @@ impl SearchIndex { pub fn file_signature(&self, path: &str) -> Result> { self.connection .query_row( - "SELECT size,mtime FROM files WHERE path=?1 AND size>=0 AND mtime>=0", + "SELECT size,mtime FROM files WHERE path=?1 AND size>=0", [path], |row| Ok((row.get(0)?, row.get(1)?)), ) @@ -1294,9 +1288,8 @@ impl SearchIndex { Ok(value) if value.is_file() => value, _ => return Ok(None), }; - let mtime = metadata.modified()?.duration_since(UNIX_EPOCH)?; let size = i64::try_from(metadata.len())?; - let modified = i64::try_from(mtime.as_nanos())?; + let modified = searchi_media::unix_time_ns(metadata.modified()?)?; let Some(expected_hash) = self.extracted_source_hash(path)? else { return Ok(None); }; diff --git a/crates/searchi-core/tests/raw_source_hash.rs b/crates/searchi-core/tests/raw_source_hash.rs index 19b6894..2a8c8c7 100644 --- a/crates/searchi-core/tests/raw_source_hash.rs +++ b/crates/searchi-core/tests/raw_source_hash.rs @@ -90,3 +90,40 @@ fn oversized_text_and_unsupported_files_remain_filename_only() -> Result<()> { assert_eq!(index.search("large.txt", 10)?.len(), 1); Ok(()) } + +#[test] +fn pre_epoch_source_remains_searchable_and_verifiable() -> Result<()> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("archived-notes.txt"); + let content = "Historical supplier cancellation notice\n"; + fs::write(&path, content)?; + fs::File::options().write(true).open(&path)?.set_times( + FileTimes::new().set_modified(std::time::UNIX_EPOCH - std::time::Duration::from_secs(1)), + )?; + let mut index = SearchIndex::open(":memory:")?; + let report = index.index_folder(directory.path())?; + assert_eq!(report.indexed, 1); + assert_eq!(report.failures, 0); + assert_eq!(index.search("archived-notes.txt", 10)?.len(), 1); + assert_eq!(index.search("cancellation", 10)?.len(), 1); + let passage = index.passage_batch(0, 1)?.remove(0); + assert_eq!( + index.file_signature(&passage.path)?, + Some((content.len() as u64, -1_000_000_000)) + ); + assert_eq!( + index.extracted_source_hash(&passage.path)?, + Some(format!("{:x}", Sha256::digest(content.as_bytes()))) + ); + assert_eq!( + index + .source_passage(&passage.path, &passage.source, 8192)? + .unwrap() + .text, + content + ); + let rescan = index.index_folder(directory.path())?; + assert_eq!(rescan.unchanged, 1); + assert_eq!(rescan.indexed, 0); + Ok(()) +} diff --git a/crates/searchi-media/src/lib.rs b/crates/searchi-media/src/lib.rs index 81fc185..2b41f77 100644 --- a/crates/searchi-media/src/lib.rs +++ b/crates/searchi-media/src/lib.rs @@ -14,7 +14,7 @@ use std::{ path::{Path, PathBuf}, process::{Command, Stdio}, thread, - time::{Duration, Instant, UNIX_EPOCH}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] @@ -28,6 +28,26 @@ pub enum MediaKind { mod tests { use super::*; + #[test] + fn signed_unix_time_preserves_epoch_and_storage_boundaries() -> Result<()> { + // Windows SystemTime has 100 ns precision; storage conversion itself + // must still handle every signed nanosecond boundary exactly. + assert_eq!(unix_time_ns(UNIX_EPOCH)?, 0); + assert_eq!(unix_time_ns(UNIX_EPOCH + Duration::from_nanos(100))?, 100); + assert_eq!(unix_time_ns(UNIX_EPOCH - Duration::from_nanos(100))?, -100); + assert_eq!( + signed_duration_ns(Duration::from_nanos(i64::MAX as u64), false)?, + i64::MAX + ); + assert_eq!( + signed_duration_ns(Duration::from_nanos(i64::MAX as u64 + 1), true)?, + i64::MIN + ); + assert!(signed_duration_ns(Duration::from_nanos(i64::MAX as u64 + 1), false).is_err()); + assert!(signed_duration_ns(Duration::from_nanos(i64::MAX as u64 + 2), true).is_err()); + Ok(()) + } + fn fixture(duration_ms: u32) -> MediaProbe { MediaProbe { duration_ms, @@ -414,9 +434,25 @@ impl MediaIdentity { Ok(()) } } +/// Convert a filesystem timestamp to signed nanoseconds since the Unix epoch. +/// Times outside the signed 64-bit storage range are rejected explicitly. +pub fn unix_time_ns(time: SystemTime) -> Result { + match time.duration_since(UNIX_EPOCH) { + Ok(duration) => signed_duration_ns(duration, false), + Err(error) => signed_duration_ns(error.duration(), true), + } +} + +fn signed_duration_ns(duration: Duration, before_epoch: bool) -> Result { + // Duration's u64 seconds fit in i128 nanoseconds, including i64::MIN's + // magnitude, which cannot first be represented as positive i64. + let nanos = duration.as_nanos() as i128; + i64::try_from(if before_epoch { -nanos } else { nanos }) + .context("File timestamp exceeds supported nanosecond range") +} + fn modified_ns(metadata: &fs::Metadata) -> Result { - i64::try_from(metadata.modified()?.duration_since(UNIX_EPOCH)?.as_nanos()) - .context("Unsupported media timestamp") + unix_time_ns(metadata.modified()?) } #[derive(Debug, Clone, Serialize)] diff --git a/crates/searchi-service/src/store.rs b/crates/searchi-service/src/store.rs index 8ebeb82..fefc0e6 100644 --- a/crates/searchi-service/src/store.rs +++ b/crates/searchi-service/src/store.rs @@ -778,34 +778,30 @@ impl Store { )) })?; if let (Some(size), Some(mtime)) = (size, mtime) { - let valid = *filesystem_validity - .entry(result.path.clone()) - .or_insert_with(|| { - #[cfg(test)] - MEDIA_STAT_CALLS.with(|count| count.set(count.get() + 1)); - let validation_span = trace.begin( - "semantic.parent_filesystem_validation", - materialization_span, - ); - let valid = - std::fs::metadata(&result.path) - .ok() - .is_some_and(|metadata| { - metadata.is_file() - && i64::try_from(metadata.len()).ok() == Some(size) - && metadata - .modified() - .ok() - .and_then(|t| { - t.duration_since(std::time::UNIX_EPOCH).ok() - }) - .and_then(|d| i64::try_from(d.as_nanos()).ok()) - == Some(mtime) - }); - trace.counter(validation_span, "valid", u64::from(valid)); - trace.end(validation_span, Default::default()); - valid - }); + let valid = + *filesystem_validity + .entry(result.path.clone()) + .or_insert_with(|| { + #[cfg(test)] + MEDIA_STAT_CALLS.with(|count| count.set(count.get() + 1)); + let validation_span = trace.begin( + "semantic.parent_filesystem_validation", + materialization_span, + ); + let valid = + std::fs::metadata(&result.path) + .ok() + .is_some_and(|metadata| { + metadata.is_file() + && i64::try_from(metadata.len()).ok() == Some(size) + && metadata.modified().ok().and_then(|t| { + searchi_media::unix_time_ns(t).ok() + }) == Some(mtime) + }); + trace.counter(validation_span, "valid", u64::from(valid)); + trace.end(validation_span, Default::default()); + valid + }); if !valid { continue; } @@ -1141,6 +1137,62 @@ mod tests { Ok(()) } + #[test] + fn pre_epoch_media_parent_remains_eligible_for_semantic_results() -> Result<()> { + let directory = tempfile::tempdir()?; + let root = directory.path().join("media"); + std::fs::create_dir(&root)?; + let path = root.join("archived.mp4"); + std::fs::write(&path, b"bounded media identity fixture")?; + std::fs::File::options() + .write(true) + .open(&path)? + .set_times( + std::fs::FileTimes::new() + .set_modified(std::time::UNIX_EPOCH - std::time::Duration::from_secs(1)), + )?; + let db = directory.path().join("index.db"); + let mut lexical = SearchIndex::open(&db)?; + lexical.index_folder(&root)?; + let mut store = Store::open(&db, DIMENSIONS)?; + let generation = store.begin("pre-epoch-media-proof")?; + let mut query = vec![0.; DIMENSIONS]; + query[0] = 1.; + let (id, indexed_path, _, _) = store.media_batch(0)?.remove(0); + let identity = searchi_media::MediaIdentity::read(Path::new(&indexed_path), 1024)?; + assert_eq!(identity.modified_ns, -1_000_000_000); + store.insert_media(generation, id, 0, &identity, "video", 0, &query)?; + store.activate(generation)?; + let mut trace = TraceCollector::new(MetricsMode::Basic); + let results = store.search_origin_traced( + &query, + "pre-epoch-media-proof", + 10, + &SearchOptions::default(), + None, + &mut trace, + None, + )?; + assert_eq!(results.len(), 1); + assert_eq!(results[0].path, indexed_path); + // Timestamp changes still invalidate the prepared media source. + std::fs::File::options() + .write(true) + .open(&path)? + .set_times(std::fs::FileTimes::new().set_modified(std::time::UNIX_EPOCH))?; + let results = store.search_origin_traced( + &query, + "pre-epoch-media-proof", + 10, + &SearchOptions::default(), + None, + &mut trace, + None, + )?; + assert!(results.is_empty()); + Ok(()) + } + #[test] fn many_video_windows_do_not_crowd_out_other_parent_files() -> Result<()> { // Geometry verifies grouped retrieval, not model inference or codecs. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index e4c4769..af2798b 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,6 +1,6 @@ # Production readiness roadmap -Status: initial Windows v0.1.0 published and installed; broader production gates remain open. Windows VM tests use CPU only. +Status: Windows v0.1.3 and Linux previews published; initial Windows v0.1.0 installed. Broader production gates remain open. Windows VM tests use CPU only. Karl's release priority, 2026-10-13: publish a usable first Windows version and install it on DEV-WIN11 now. Extensive platform/GPU, clean-machine, signing and @@ -335,4 +335,15 @@ build was inferred from that lookup; macOS execution gates remain unchecked. - [x] Current native Linux desktop release compilation and local loader closure on Ubuntu WSL; [evidence](LINUX-DESKTOP.md). - [x] Linux desktop relocated normal resource discovery and CPU query-worker startup under an owned software-display test; no native UI interaction claim. -- [ ] Linux desktop graphical interaction and distribution packaging. +- [x] Publish Ubuntu 24.04 x86-64 Debian desktop preview; verify public bytes and extracted package CPU startup. See [evidence](LINUX-DESKTOP.md). +- [ ] Linux desktop graphical interaction and broader distribution compatibility. + +## Rust audit and goal review, 2026-10-13 + +- [x] Audit core/service and desktop/CLI Rust boundaries; avoid unnecessary naming/module churn. +- [x] Correct valid pre-1970 filesystem timestamps across indexing, source eligibility and previews. Changed crates passed 100 tests (six ignored), desktop 47; strict Clippy and formatting passed. See [audit](RUST-AUDIT.md). +- [ ] Include the timestamp correction in the next packaged update; published v0.1.3 artifacts precede it. + +Initial Windows delivery is complete. Native interaction/upgrade, actual GPU, +macOS execution and independent media-quality work remain real follow-up; the +full production goal remains active. Preserve the detailed unchecked gates above. diff --git a/docs/RUST-AUDIT.md b/docs/RUST-AUDIT.md new file mode 100644 index 0000000..2bebe99 --- /dev/null +++ b/docs/RUST-AUDIT.md @@ -0,0 +1,60 @@ +# Rust audit and completion review — 2026-10-13 + +Reviewed the shared indexing/media/semantic service boundaries and the desktop, +CLI/MCP, setup and optional AI orchestration with two independent Rust reviewers. +The review found no reason to add a broad naming or module-splitting project. +Existing adapters isolate native processes and desktop concerns from retrieval; +long but linear command/setup flows do not need additional indirection merely +because of their length. This is a targeted review, not proof of zero defects. + +## Corrected issue + +A valid file modification time before 1970 aborted folder scanning. Other source +eligibility and preview checks also assumed nonnegative Unix timestamps. +The shared `searchi_media::unix_time_ns` now performs checked signed conversion, +including the minimum signed storage boundary. Indexing, signatures, excerpts, +media identity, semantic parent validation and preview identity use that policy. +Negative sizes remain invalid. No schema migration or dependency was added. + +Regression coverage checks archived documents, unchanged rescans, excerpts, +media eligibility and stale-preview revocation followed by successful reindexing. +The semantic eligibility fixture uses synthetic geometry to test filtering; +it does not claim model-quality validation. + +Windows CPU-only results for the final changes: + +- Core, media and service: 100 tests passed, six explicit integration tests ignored. +- Desktop: all 47 tests passed, including the new pre-epoch preview regression. +- Strict all-target Clippy passed for the changed crates and desktop. +- Workspace and desktop formatting and Git whitespace checks passed. + +The first preview-test assertion expected 404; the protocol correctly returns +410 for an expired capability. The test expectation was corrected; protocol +behavior was preserved. Windows filesystem timestamp precision is 100 ns; +exact storage boundaries are tested through the conversion helper. + +## Deferred under YAGNI + +Desktop setup could resolve `resource_dir()` once instead of repeatedly. This +does not justify changing working orchestration during this audit. No generic +service framework, speculative cache or cosmetic function-renaming sweep was added. + +## Goal review + +The initial usable Windows release is delivered. Windows desktop 0.1.3, +Windows CLI/MCP and Ubuntu headless packages are published and byte-verified. +An Ubuntu 24.04 desktop Debian preview now also has verified packaged CPU startup. +The timestamp correction is in source; those 0.1.3 artifacts precede this fix. + +The full cross-platform production goal remains open. Concrete follow-up is: + +1. Include this fix in the next packaged update; refresh extraction provenance. +2. Check real Windows interaction, preview playback/seek and installed upgrades. +3. Validate physical GPU use and fallback, and execute a native macOS build. +4. Improve media relevance using independently reviewed real files and queries; + the current experimental quality evidence is not sufficient for broad claims. +5. Retain clean-machine, signing and large-library checks as later release gates. + +These follow-ups do not justify withholding the already published preview. +Gitea Actions remain disabled at Karl's request. See ROADMAP.md for detailed +evidence and unchecked gates; this audit does not silently close them.