Verify genuine Google music constraints and separate metadata cache revisions
This commit is contained in:
karl committed
1 parent
b002399217
commit
ce642f1594
7 files changed
+339
-12
No files matched your search
@@ -44,6 +44,7 @@ unchanged-vector reuse and production platform checks remain release gates.
|
||||
|
||||
Exact artist/title constraints use the core's normalized structured probe keys
|
||||
before ANN limits and in the eligibility cache key. Metadata-only writes advance
|
||||
the corpus revision and dirty path so warmed readers cannot retain old eligibility.
|
||||
a separate durable metadata epoch so warmed readers cannot retain old eligibility.
|
||||
They do not change source-corpus revisions during the indexer's own probe publication.
|
||||
Empty metadata-only hybrid requests use core results without model inference.
|
||||
These explicit constraints do not establish abstention for free-form topic queries.
|
||||
@@ -33,6 +33,7 @@ struct FilterKey {
|
||||
code_only: bool,
|
||||
media_artist: Option<String>,
|
||||
media_title: Option<String>,
|
||||
metadata_revision: i64,
|
||||
}
|
||||
|
||||
struct EligibilityCache {
|
||||
@@ -186,16 +187,18 @@ impl Store {
|
||||
}
|
||||
}
|
||||
initialization.execute_batch("CREATE TABLE IF NOT EXISTS semantic_media_errors(generation INTEGER NOT NULL REFERENCES semantic_generations(id) ON DELETE CASCADE,path TEXT NOT NULL,message TEXT NOT NULL,PRIMARY KEY(generation,path))")?;
|
||||
// Metadata-only writes change strict eligibility even when file bytes do not change.
|
||||
for (name, event, row) in [
|
||||
("insert", "INSERT", "new"),
|
||||
("update", "UPDATE", "new"),
|
||||
("delete", "DELETE", "old"),
|
||||
// Probe publication changes filter eligibility, not source discovery identity.
|
||||
// Keep its epoch separate so indexing does not invalidate its own corpus snapshot.
|
||||
initialization.execute_batch("CREATE TABLE IF NOT EXISTS semantic_media_metadata_revision(singleton INTEGER PRIMARY KEY CHECK(singleton=1), revision INTEGER NOT NULL);
|
||||
INSERT OR IGNORE INTO semantic_media_metadata_revision(singleton,revision) VALUES(1,0);")?;
|
||||
for (name, event) in [
|
||||
("insert", "INSERT"),
|
||||
("update", "UPDATE"),
|
||||
("delete", "DELETE"),
|
||||
] {
|
||||
initialization.execute_batch(&format!("CREATE TRIGGER IF NOT EXISTS semantic_media_metadata_{name} AFTER {event} ON media_metadata BEGIN
|
||||
UPDATE semantic_meta SET corpus_revision=corpus_revision+1 WHERE singleton=1;
|
||||
INSERT INTO semantic_dirty_paths(path,revision) VALUES({row}.path,(SELECT corpus_revision FROM semantic_meta WHERE singleton=1))
|
||||
ON CONFLICT(path) DO UPDATE SET revision=excluded.revision; END;"))?;
|
||||
initialization.execute_batch(&format!("DROP TRIGGER IF EXISTS semantic_media_metadata_{name};
|
||||
CREATE TRIGGER semantic_media_metadata_{name} AFTER {event} ON media_metadata BEGIN
|
||||
UPDATE semantic_media_metadata_revision SET revision=revision+1 WHERE singleton=1; END;"))?;
|
||||
}
|
||||
incremental::migrate(&initialization)?;
|
||||
initialization.commit()?;
|
||||
@@ -646,6 +649,19 @@ impl Store {
|
||||
.map(|ext| ext.trim_start_matches('.').to_ascii_lowercase());
|
||||
let min_size = options.min_size.map(i64::try_from).transpose()?;
|
||||
let max_size = options.max_size.map(i64::try_from).transpose()?;
|
||||
let metadata_revision = if options.media_artist.is_some()
|
||||
|| options.media_title.is_some()
|
||||
|| options.min_duration_ms.is_some()
|
||||
|| options.max_duration_ms.is_some()
|
||||
{
|
||||
snapshot.query_row(
|
||||
"SELECT revision FROM semantic_media_metadata_revision WHERE singleton=1",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)?
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let filter = FilterKey {
|
||||
folder,
|
||||
extension,
|
||||
@@ -667,6 +683,7 @@ impl Store {
|
||||
.as_deref()
|
||||
.map(searchi_core::normalize_media_field)
|
||||
.transpose()?,
|
||||
metadata_revision,
|
||||
};
|
||||
trace.end(filter_span, Default::default());
|
||||
let eligibility_span = trace.begin("semantic.eligibility", parent);
|
||||
@@ -1403,6 +1420,11 @@ mod tests {
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
let corpus_before: i64 = store.connection.query_row(
|
||||
"SELECT corpus_revision FROM semantic_meta WHERE singleton=1",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
tags.insert("artist".into(), "Replacement".into());
|
||||
lexical.set_structured_media_metadata_at_revision(
|
||||
&path,
|
||||
@@ -1412,6 +1434,15 @@ mod tests {
|
||||
identity.modified_ns,
|
||||
revision,
|
||||
)?;
|
||||
let corpus_after: i64 = store.connection.query_row(
|
||||
"SELECT corpus_revision FROM semantic_meta WHERE singleton=1",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
assert_eq!(
|
||||
corpus_before, corpus_after,
|
||||
"Probe publication must not invalidate source discovery"
|
||||
);
|
||||
assert!(store
|
||||
.search(&vector, "metadata-filter-proof", 1, &options)?
|
||||
.is_empty());
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
//! Explicit identities constrain genuine Google media retrieval, without a cosine cutoff.
|
||||
use anyhow::{Context, Result};
|
||||
use searchi_core::{IndexOptions, SearchIndex};
|
||||
use searchi_service::{FfmpegTools, HybridSearch, MediaLimits, SearchOptions};
|
||||
use std::{fs, path::Path, process::Command, sync::atomic::AtomicBool};
|
||||
|
||||
fn tagged_tone(ffmpeg: &Path, path: &Path, artist: &str, title: &str) -> Result<()> {
|
||||
let output = Command::new(ffmpeg)
|
||||
.args([
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-threads",
|
||||
"1",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=440:sample_rate=16000:duration=2",
|
||||
"-c:a",
|
||||
"pcm_s16le",
|
||||
"-metadata",
|
||||
])
|
||||
.arg(format!("artist={artist}"))
|
||||
.arg("-metadata")
|
||||
.arg(format!("title={title}"))
|
||||
.arg(path)
|
||||
.output()?;
|
||||
anyhow::ensure!(
|
||||
output.status.success(),
|
||||
"Owned FFmpeg fixture creation failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "Requires verified Google assets, normal native worker and FFmpeg helpers; CPU only"]
|
||||
fn genuine_google_exact_music_constraints_survive_publication_and_hint_refresh() -> Result<()> {
|
||||
assert_eq!(
|
||||
std::env::var("SEARCHI_FORCE_CPU")?,
|
||||
"1",
|
||||
"VM inference must use CPU only"
|
||||
);
|
||||
let model =
|
||||
std::env::var_os("SEARCHI_GEMMA_MODEL_DIR").context("Set SEARCHI_GEMMA_MODEL_DIR")?;
|
||||
let worker = std::env::var_os("SEARCHI_GEMMA_WORKER").context("Set SEARCHI_GEMMA_WORKER")?;
|
||||
let tools = FfmpegTools {
|
||||
ffmpeg: std::env::var_os("SEARCHI_TEST_FFMPEG")
|
||||
.context("Set SEARCHI_TEST_FFMPEG")?
|
||||
.into(),
|
||||
ffprobe: std::env::var_os("SEARCHI_TEST_FFPROBE")
|
||||
.context("Set SEARCHI_TEST_FFPROBE")?
|
||||
.into(),
|
||||
};
|
||||
let temp = tempfile::tempdir()?;
|
||||
let root = temp.path().join("music");
|
||||
fs::create_dir(&root)?;
|
||||
let target = root.join("z-target.wav");
|
||||
tagged_tone(&tools.ffmpeg, &target, "Björk", "Jóga")?;
|
||||
// Identical audio is a deliberately strong distractor; identity must come from tags.
|
||||
tagged_tone(
|
||||
&tools.ffmpeg,
|
||||
&root.join("a-distractor.wav"),
|
||||
"Other Artist",
|
||||
"Björk",
|
||||
)?;
|
||||
let db = temp.path().join("index.db");
|
||||
let mut lexical = SearchIndex::open(&db)?;
|
||||
lexical.index_folder(&root)?;
|
||||
let mut indexer = HybridSearch::open(&db, &model, &worker, 1)?;
|
||||
assert_eq!(
|
||||
indexer.worker_info().context("Google worker info")?.device,
|
||||
"cpu"
|
||||
);
|
||||
let report = indexer.index_embeddings_with_media(&tools, &MediaLimits::default())?;
|
||||
assert!(report.failures.is_empty());
|
||||
assert_eq!(report.media_files, 2);
|
||||
assert_eq!(report.audio_inference_calls, 2);
|
||||
drop(indexer);
|
||||
let mut search = HybridSearch::open_query(&db, &model, &worker, 1)?;
|
||||
let options = SearchOptions {
|
||||
media_artist: Some("BJÖRK".into()),
|
||||
media_title: Some("JÓGA".into()),
|
||||
..Default::default()
|
||||
};
|
||||
let metadata = search.search_measured("", 1, &options, true)?;
|
||||
assert_eq!(metadata.results.len(), 1);
|
||||
assert_eq!(metadata.results[0].filename, "z-target.wav");
|
||||
assert!(!metadata
|
||||
.metrics
|
||||
.trace
|
||||
.context("debug trace")?
|
||||
.iter()
|
||||
.any(|span| span.name == "semantic.query_embedding"));
|
||||
let query = "the sound of a sine wave tone";
|
||||
let semantic = search.search_semantic(query, 1, &options)?;
|
||||
assert_eq!(semantic.len(), 1);
|
||||
assert_eq!(semantic[0].filename, "z-target.wav");
|
||||
assert_eq!(
|
||||
search.search(query, 1, &options)?[0].filename,
|
||||
"z-target.wav"
|
||||
);
|
||||
let absent = SearchOptions {
|
||||
media_artist: Some("Absent Artist".into()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(search.search(query, 1, &absent)?.is_empty());
|
||||
assert!(search.search("", 1, &absent)?.is_empty());
|
||||
let collision = SearchOptions {
|
||||
media_artist: Some("Jóga".into()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(search.search(query, 1, &collision)?.is_empty());
|
||||
// A native same-stat hint invalidates old tags and warmed ANN eligibility.
|
||||
let canonical = target.canonicalize()?;
|
||||
lexical.index_paths_cancellable(
|
||||
&root,
|
||||
std::slice::from_ref(&canonical),
|
||||
&IndexOptions::default(),
|
||||
&AtomicBool::new(false),
|
||||
)?;
|
||||
assert!(search.search("", 1, &options)?.is_empty());
|
||||
assert!(search.search_semantic(query, 1, &options)?.is_empty());
|
||||
let mut indexer = HybridSearch::open(&db, &model, &worker, 1)?;
|
||||
let refreshed = indexer.index_embeddings_with_media(&tools, &MediaLimits::default())?;
|
||||
assert!(refreshed.failures.is_empty());
|
||||
drop(indexer);
|
||||
assert_eq!(search.search("", 1, &options)?[0].filename, "z-target.wav");
|
||||
assert_eq!(
|
||||
search.search_semantic(query, 1, &options)?[0].filename,
|
||||
"z-target.wav"
|
||||
);
|
||||
// Deletion is suppressed without retaining an exact identity match.
|
||||
fs::remove_file(&target)?;
|
||||
lexical.index_folder(&root)?;
|
||||
assert!(search.search("", 1, &options)?.is_empty());
|
||||
assert!(search.search_semantic(query, 1, &options)?.is_empty());
|
||||
Ok(())
|
||||
}
|
||||
Reference in new issue
Block a user