Handle pre-epoch file timestamps consistently and record Rust audit
This commit is contained in:
1 parent
59b2fc39b2
commit
14adbbc616
7 files changed
+262
-46
No files matched your search
@@ -778,34 +778,30 @@ impl Store {
|
||||
))
|
||||
})?;
|
||||
if let (Some(size), Some(mtime)) = (size, mtime) {
|
||||
let valid = *filesystem_validity
|
||||
.entry(result.path.clone())
|
||||
.or_insert_with(|| {
|
||||
#[cfg(test)]
|
||||
MEDIA_STAT_CALLS.with(|count| count.set(count.get() + 1));
|
||||
let validation_span = trace.begin(
|
||||
"semantic.parent_filesystem_validation",
|
||||
materialization_span,
|
||||
);
|
||||
let valid =
|
||||
std::fs::metadata(&result.path)
|
||||
.ok()
|
||||
.is_some_and(|metadata| {
|
||||
metadata.is_file()
|
||||
&& i64::try_from(metadata.len()).ok() == Some(size)
|
||||
&& metadata
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|t| {
|
||||
t.duration_since(std::time::UNIX_EPOCH).ok()
|
||||
})
|
||||
.and_then(|d| i64::try_from(d.as_nanos()).ok())
|
||||
== Some(mtime)
|
||||
});
|
||||
trace.counter(validation_span, "valid", u64::from(valid));
|
||||
trace.end(validation_span, Default::default());
|
||||
valid
|
||||
});
|
||||
let valid =
|
||||
*filesystem_validity
|
||||
.entry(result.path.clone())
|
||||
.or_insert_with(|| {
|
||||
#[cfg(test)]
|
||||
MEDIA_STAT_CALLS.with(|count| count.set(count.get() + 1));
|
||||
let validation_span = trace.begin(
|
||||
"semantic.parent_filesystem_validation",
|
||||
materialization_span,
|
||||
);
|
||||
let valid =
|
||||
std::fs::metadata(&result.path)
|
||||
.ok()
|
||||
.is_some_and(|metadata| {
|
||||
metadata.is_file()
|
||||
&& i64::try_from(metadata.len()).ok() == Some(size)
|
||||
&& metadata.modified().ok().and_then(|t| {
|
||||
searchi_media::unix_time_ns(t).ok()
|
||||
}) == Some(mtime)
|
||||
});
|
||||
trace.counter(validation_span, "valid", u64::from(valid));
|
||||
trace.end(validation_span, Default::default());
|
||||
valid
|
||||
});
|
||||
if !valid {
|
||||
continue;
|
||||
}
|
||||
@@ -1141,6 +1137,62 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_epoch_media_parent_remains_eligible_for_semantic_results() -> Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let root = directory.path().join("media");
|
||||
std::fs::create_dir(&root)?;
|
||||
let path = root.join("archived.mp4");
|
||||
std::fs::write(&path, b"bounded media identity fixture")?;
|
||||
std::fs::File::options()
|
||||
.write(true)
|
||||
.open(&path)?
|
||||
.set_times(
|
||||
std::fs::FileTimes::new()
|
||||
.set_modified(std::time::UNIX_EPOCH - std::time::Duration::from_secs(1)),
|
||||
)?;
|
||||
let db = directory.path().join("index.db");
|
||||
let mut lexical = SearchIndex::open(&db)?;
|
||||
lexical.index_folder(&root)?;
|
||||
let mut store = Store::open(&db, DIMENSIONS)?;
|
||||
let generation = store.begin("pre-epoch-media-proof")?;
|
||||
let mut query = vec![0.; DIMENSIONS];
|
||||
query[0] = 1.;
|
||||
let (id, indexed_path, _, _) = store.media_batch(0)?.remove(0);
|
||||
let identity = searchi_media::MediaIdentity::read(Path::new(&indexed_path), 1024)?;
|
||||
assert_eq!(identity.modified_ns, -1_000_000_000);
|
||||
store.insert_media(generation, id, 0, &identity, "video", 0, &query)?;
|
||||
store.activate(generation)?;
|
||||
let mut trace = TraceCollector::new(MetricsMode::Basic);
|
||||
let results = store.search_origin_traced(
|
||||
&query,
|
||||
"pre-epoch-media-proof",
|
||||
10,
|
||||
&SearchOptions::default(),
|
||||
None,
|
||||
&mut trace,
|
||||
None,
|
||||
)?;
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(results[0].path, indexed_path);
|
||||
// Timestamp changes still invalidate the prepared media source.
|
||||
std::fs::File::options()
|
||||
.write(true)
|
||||
.open(&path)?
|
||||
.set_times(std::fs::FileTimes::new().set_modified(std::time::UNIX_EPOCH))?;
|
||||
let results = store.search_origin_traced(
|
||||
&query,
|
||||
"pre-epoch-media-proof",
|
||||
10,
|
||||
&SearchOptions::default(),
|
||||
None,
|
||||
&mut trace,
|
||||
None,
|
||||
)?;
|
||||
assert!(results.is_empty());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn many_video_windows_do_not_crowd_out_other_parent_files() -> Result<()> {
|
||||
// Geometry verifies grouped retrieval, not model inference or codecs.
|
||||
|
||||
Reference in new issue
Block a user