Filter exact music identities before lexical and semantic candidate limits

This commit is contained in:
karl committed 2026-10-13 18:36:54 +02:00
1 parent 3ac1201909
commit 2eace15a33
7 files changed
+458 -19

No files matched your search

+12
View File
@@ -136,3 +136,15 @@ opens. The measured Windows synthetic fixture grew from about 295 MiB to 605 MiB
- OCR, media extraction, and real model inference belong to subsequent modules. - OCR, media extraction, and real model inference belong to subsequent modules.
- Linux/macOS measurements, realistic mixed-document corpora, cold startup, - Linux/macOS measurements, realistic mixed-document corpora, cold startup,
memory budgets, and indexing performance remain unverified. memory budgets, and indexing performance remain unverified.
### Exact music identities
`SearchOptions.media_artist` and `media_title` constrain every retrieval lane to
complete genuine probe fields, with Unicode simple lowercase and outer whitespace
normalization. Both fields combine with AND; substring inference and full Unicode
case folding are not claimed. Each request field accepts 1–1024 UTF-8 bytes.
An empty query with either constraint returns bounded filename/path-ordered matches.
Missing tags and legacy flattened-only records are ineligible, rather than admitting
semantic neighbours. A normal media probe refresh populates the structured keys.
Oversized genuine fields remain keyword-searchable but receive no truncated exact
identity key. Guarded publication and same-stat event hints preserve invalidation.
+275 -7
View File
@@ -17,14 +17,14 @@ use rusqlite::{params, params_from_iter, types::Value, Connection, OptionalExten
use serde::Serialize; use serde::Serialize;
use std::{ use std::{
cell::RefCell, cell::RefCell,
collections::{BTreeSet, HashMap}, collections::{BTreeMap, BTreeSet, HashMap},
fs, fs,
path::{Path, PathBuf}, path::{Path, PathBuf},
sync::atomic::{AtomicBool, Ordering}, sync::atomic::{AtomicBool, Ordering},
}; };
use walkdir::WalkDir; use walkdir::WalkDir;
const SCHEMA_VERSION: i64 = 13; const SCHEMA_VERSION: i64 = 14;
const MAX_RESULTS: usize = 1000; const MAX_RESULTS: usize = 1000;
type PreviousExtraction = (i64, i64, String, Option<String>, Option<String>); type PreviousExtraction = (i64, i64, String, Option<String>, Option<String>);
@@ -70,6 +70,19 @@ pub struct SearchOptions {
pub modified_before_unix_ms: Option<i64>, pub modified_before_unix_ms: Option<i64>,
pub min_duration_ms: Option<u32>, pub min_duration_ms: Option<u32>,
pub max_duration_ms: Option<u32>, pub max_duration_ms: Option<u32>,
/// Exact complete artist tag, Unicode lowercase comparison. Never inferred from query text.
pub media_artist: Option<String>,
/// Exact complete title tag; missing structured tags are ineligible.
pub media_title: Option<String>,
}
/// Shared bounded normalization for exact metadata constraints and genuine probe values.
pub fn normalize_media_field(value: &str) -> Result<String> {
let value = value.trim();
if value.is_empty() || value.len() > 1024 {
bail!("Exact media field must contain 1..=1024 UTF-8 bytes");
}
Ok(value.to_lowercase())
} }
#[derive(Debug, Serialize)] #[derive(Debug, Serialize)]
@@ -282,7 +295,19 @@ impl SearchIndex {
[], [],
)?; )?;
} }
initialization.execute_batch("PRAGMA user_version=13;")?; for field in ["artist_key", "title_key"] {
let exists: bool = initialization.query_row(
"SELECT EXISTS(SELECT 1 FROM pragma_table_info('media_metadata') WHERE name=?1)",
[field], |row| row.get(0),
)?;
if !exists {
initialization.execute(
&format!("ALTER TABLE media_metadata ADD COLUMN {field} TEXT"),
[],
)?;
}
}
initialization.execute_batch("PRAGMA user_version=14;")?;
initialization.commit()?; initialization.commit()?;
} }
Ok(Self { Ok(Self {
@@ -356,6 +381,78 @@ impl SearchIndex {
size: u64, size: u64,
mtime_ns: i64, mtime_ns: i64,
expected_revision: i64, expected_revision: i64,
) -> Result<()> {
self.write_media_metadata(
path,
tags,
None,
None,
duration_ms,
size,
mtime_ns,
expected_revision,
)
}
/// Preserve structured fields directly from a genuine probe, never from flattened text.
/// Empty/oversized fields retain legacy keyword tags but have no exact identity key.
#[allow(clippy::too_many_arguments)]
pub fn set_structured_media_metadata_at_revision(
&mut self,
path: &str,
tags: &BTreeMap<String, String>,
duration_ms: u32,
size: u64,
mtime_ns: i64,
expected_revision: i64,
) -> Result<()> {
let field = |name: &str| -> Result<Option<String>> {
tags.get(name)
.filter(|value| !value.trim().is_empty() && value.trim().len() <= 1024)
.map(|value| normalize_media_field(value))
.transpose()
};
let mut total = 0usize;
for (key, value) in tags {
total = total
.checked_add(key.len())
.and_then(|n| n.checked_add(value.len()))
.and_then(|n| n.checked_add(3))
.context("Media tags size overflow")?;
if total > 32768 {
bail!("Media tags exceed 32 KiB bound");
}
}
let artist = field("artist")?;
let title = field("title")?;
let flattened = tags
.iter()
.map(|(key, value)| format!("{key}: {value}"))
.collect::<Vec<_>>()
.join("\n");
self.write_media_metadata(
path,
&flattened,
artist.as_deref(),
title.as_deref(),
duration_ms,
size,
mtime_ns,
expected_revision,
)
}
#[allow(clippy::too_many_arguments)]
fn write_media_metadata(
&mut self,
path: &str,
tags: &str,
artist: Option<&str>,
title: Option<&str>,
duration_ms: u32,
size: u64,
mtime_ns: i64,
expected_revision: i64,
) -> Result<()> { ) -> Result<()> {
if tags.len() > 32768 { if tags.len() > 32768 {
bail!("Media tags exceed 32 KiB bound"); bail!("Media tags exceed 32 KiB bound");
@@ -369,7 +466,7 @@ impl SearchIndex {
) )
.optional()?; .optional()?;
let id = id.context("Media metadata file changed since indexing")?; let id = id.context("Media metadata file changed since indexing")?;
tx.execute("INSERT INTO media_metadata(path,file_id,tags,duration_ms) VALUES(?1,?2,?3,?4) ON CONFLICT(path) DO UPDATE SET tags=excluded.tags,duration_ms=excluded.duration_ms",params![path,id,tags,duration_ms])?; tx.execute("INSERT INTO media_metadata(path,file_id,tags,duration_ms,artist_key,title_key) VALUES(?1,?2,?3,?4,?5,?6) ON CONFLICT(path) DO UPDATE SET tags=excluded.tags,duration_ms=excluded.duration_ms,artist_key=excluded.artist_key,title_key=excluded.title_key",params![path,id,tags,duration_ms,artist,title])?;
tx.commit()?; tx.commit()?;
Ok(()) Ok(())
} }
@@ -916,8 +1013,19 @@ impl SearchIndex {
.map(i64::try_from) .map(i64::try_from)
.transpose() .transpose()
.context("maximum size exceeds supported range")?; .context("maximum size exceeds supported range")?;
let artist = options
.media_artist
.as_deref()
.map(normalize_media_field)
.transpose()?;
let title = options
.media_title
.as_deref()
.map(normalize_media_field)
.transpose()?;
let metadata_only = query.trim().is_empty() && (artist.is_some() || title.is_some());
let query = query.trim(); let query = query.trim();
if query.is_empty() || limit == 0 { if (query.is_empty() && !metadata_only) || limit == 0 {
return Ok(Vec::new()); return Ok(Vec::new());
} }
if query.len() > 4096 { if query.len() > 4096 {
@@ -937,7 +1045,9 @@ impl SearchIndex {
AND (?5 IS NULL OR f.size>=?5) AND (?6 IS NULL OR f.size<=?6) AND (?5 IS NULL OR f.size>=?5) AND (?6 IS NULL OR f.size<=?6)
AND (?7 IS NULL OR f.mtime/1000000>=?7) AND (?8 IS NULL OR f.mtime/1000000<=?8) AND (?7 IS NULL OR f.mtime/1000000>=?7) AND (?8 IS NULL OR f.mtime/1000000<=?8)
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms>=?10)) AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms>=?10))
AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms<=?11))"); AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms<=?11))
AND (?12 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.artist_key=?12))
AND (?13 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.title_key=?13))");
if options.code_only { if options.code_only {
filters.push_str(" AND "); filters.push_str(" AND ");
filters.push_str(code_file_predicate()); filters.push_str(code_file_predicate());
@@ -957,7 +1067,9 @@ impl SearchIndex {
let has_filters = filter_values.iter().any(|value| *value != Value::Null) let has_filters = filter_values.iter().any(|value| *value != Value::Null)
|| options.code_only || options.code_only
|| options.min_duration_ms.is_some() || options.min_duration_ms.is_some()
|| options.max_duration_ms.is_some(); || options.max_duration_ms.is_some()
|| artist.is_some()
|| title.is_some();
let mut results = Vec::new(); let mut results = Vec::new();
let mut selected = Vec::<i64>::new(); let mut selected = Vec::<i64>::new();
// Each lane filters its entire candidate set before LIMIT. There is no // Each lane filters its entire candidate set before LIMIT. There is no
@@ -1084,6 +1196,15 @@ impl SearchIndex {
fts.clone(), fts.clone(),
"rank,f.id", "rank,f.id",
)); ));
if metadata_only {
lanes = vec![(
"metadata_filter",
"files f".into(),
"(?1 IS NOT NULL AND ?2 IS NOT NULL)".into(),
String::new(),
"f.filename,f.path",
)];
}
trace.end(planning, Default::default()); trace.end(planning, Default::default());
for (kind, from, condition, expression, ordering) in lanes { for (kind, from, condition, expression, ordering) in lanes {
if results.len() == limit { if results.len() == limit {
@@ -1161,6 +1282,10 @@ impl SearchIndex {
.map_or(Value::Null, |v| Value::Integer(i64::from(v))), .map_or(Value::Null, |v| Value::Integer(i64::from(v))),
); );
} }
if sql.contains("?12") {
values.push(artist.clone().map_or(Value::Null, Value::Text));
values.push(title.clone().map_or(Value::Null, Value::Text));
}
let sql_span = trace.begin("lexical.filtered_sql", lane); let sql_span = trace.begin("lexical.filtered_sql", lane);
trace.counter(sql_span, "filters_active", u64::from(has_filters)); trace.counter(sql_span, "filters_active", u64::from(has_filters));
let mut statement = self.connection.prepare(&sql)?; let mut statement = self.connection.prepare(&sql)?;
@@ -1435,6 +1560,149 @@ fn validate_normalized_absolute(path: &Path) -> Result<()> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
#[test]
fn legacy_metadata_migration_preserves_keyword_search_without_inventing_fields(
) -> anyhow::Result<()> {
use super::*;
let dir = tempfile::tempdir()?;
let root = dir.path().join("media");
fs::create_dir(&root)?;
let file = root.join("track.ogg");
fs::write(&file, b"legacy tag fixture")?;
let db = dir.path().join("index.db");
let mut index = SearchIndex::open(&db)?;
index.index_folder(&root)?;
let file = file.canonicalize()?;
let identity = searchi_media::MediaIdentity::read(&file, 1024)?;
index.set_media_metadata(
&file.to_string_lossy(),
"artist: Björk",
1000,
identity.bytes,
identity.modified_ns,
)?;
drop(index);
let connection = Connection::open(&db)?;
connection.execute_batch("ALTER TABLE media_metadata DROP COLUMN artist_key; ALTER TABLE media_metadata DROP COLUMN title_key; PRAGMA user_version=13;")?;
drop(connection);
let mut index = SearchIndex::open(&db)?;
assert_eq!(index.search("Björk", 1)?.len(), 1);
let options = SearchOptions {
media_artist: Some("Björk".into()),
..Default::default()
};
assert!(index.search_with_options("", 1, &options)?.is_empty());
let revision = index.media_change_revision(
&file.to_string_lossy(),
identity.bytes,
identity.modified_ns,
)?;
let tags = BTreeMap::from([
("artist".into(), "Björk".into()),
("title".into(), "x".repeat(1025)),
]);
index.set_structured_media_metadata_at_revision(
&file.to_string_lossy(),
&tags,
1000,
identity.bytes,
identity.modified_ns,
revision,
)?;
assert_eq!(index.search_with_options("", 1, &options)?.len(), 1);
assert_eq!(index.search("x".repeat(1025).as_str(), 1)?.len(), 1);
let too_short = SearchOptions {
media_title: Some("x".repeat(1024)),
..Default::default()
};
assert!(index.search_with_options("", 1, &too_short)?.is_empty());
drop(index);
let index = SearchIndex::open(&db)?;
assert_eq!(index.search_with_options("", 1, &options)?.len(), 1);
Ok(())
}
#[test]
fn exact_structured_media_filters_preserve_fields_and_hint_invalidation() -> anyhow::Result<()>
{
use super::*;
let dir = tempfile::tempdir()?;
let path = dir.path().join("track.ogg");
fs::write(&path, b"tag storage fixture")?;
let mut index = SearchIndex::open(":memory:")?;
index.index_folder(dir.path())?;
let identity = searchi_media::MediaIdentity::read(&path, 1024)?;
let canonical = path.canonicalize()?;
let path_string = canonical.to_string_lossy();
let revision =
index.media_change_revision(&path_string, identity.bytes, identity.modified_ns)?;
let tags = BTreeMap::from([
("artist".into(), "Björk".into()),
("title".into(), "Jóga".into()),
("comment".into(), "artist: Absent Artist".into()),
]);
index.set_structured_media_metadata_at_revision(
&path_string,
&tags,
1000,
identity.bytes,
identity.modified_ns,
revision,
)?;
let mut options = SearchOptions {
media_artist: Some(" BJÖRK ".into()),
..Default::default()
};
assert_eq!(index.search_with_options("", 1, &options)?.len(), 1);
assert_eq!(index.search_with_options("track", 1, &options)?.len(), 1);
options.media_title = Some("JÓGA".into());
assert_eq!(index.search_with_options("", 1, &options)?.len(), 1);
options.media_artist = Some("Absent Artist".into());
assert!(index.search_with_options("", 1, &options)?.is_empty());
options.media_artist = Some("Björ".into());
assert!(index.search_with_options("track", 1, &options)?.is_empty());
options.media_artist = Some(" ".into());
assert!(index.search_with_options("", 1, &options).is_err());
options.media_artist = Some("Björk".into());
index.index_paths_cancellable(
dir.path(),
std::slice::from_ref(&canonical),
&IndexOptions::default(),
&AtomicBool::new(false),
)?;
assert!(index.search_with_options("", 1, &options)?.is_empty());
assert!(index
.set_structured_media_metadata_at_revision(
&path_string,
&tags,
1000,
identity.bytes,
identity.modified_ns,
revision
)
.is_err());
let revision =
index.media_change_revision(&path_string, identity.bytes, identity.modified_ns)?;
index.set_structured_media_metadata_at_revision(
&path_string,
&tags,
1000,
identity.bytes,
identity.modified_ns,
revision,
)?;
assert_eq!(index.search_with_options("", 1, &options)?.len(), 1);
index.set_media_metadata(
&path_string,
"artist: Björk",
1000,
identity.bytes,
identity.modified_ns,
)?;
assert!(index.search_with_options("", 1, &options)?.is_empty());
assert_eq!(index.search("Björk", 1)?.len(), 1);
Ok(())
}
#[test] #[test]
fn measured_search_preserves_results_and_contains_private_free_nested_spans( fn measured_search_preserves_results_and_contains_private_free_nested_spans(
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
@@ -447,7 +447,7 @@ fn schema_twelve_migrates_change_revision_without_reextracting_records() -> Resu
let connection = rusqlite::Connection::open(&db)?; let connection = rusqlite::Connection::open(&db)?;
assert_eq!( assert_eq!(
connection.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))?, connection.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))?,
13 14
); );
assert_eq!( assert_eq!(
connection.query_row("SELECT change_revision FROM files", [], |row| row connection.query_row("SELECT change_revision FROM files", [], |row| row
+6
View File
@@ -41,3 +41,9 @@ Audio timestamps represent segment starts, not exact word locations. Smaller
windows alone have not demonstrated accurate spoken navigation. No-answer windows alone have not demonstrated accurate spoken navigation. No-answer
calibration, music discrimination, exact spoken positions, process isolation, calibration, music discrimination, exact spoken positions, process isolation,
unchanged-vector reuse and production platform checks remain release gates. unchanged-vector reuse and production platform checks remain release gates.
Exact artist/title constraints use the core's normalized structured probe keys
before ANN limits and in the eligibility cache key. Metadata-only writes advance
the corpus revision and dirty path so warmed readers cannot retain old eligibility.
Empty metadata-only hybrid requests use core results without model inference.
These explicit constraints do not establish abstention for free-form topic queries.
+19 -1
View File
@@ -513,7 +513,7 @@ impl HybridSearch {
.lexical .lexical
.search_with_options_traced(query, candidates, options, trace, parent)?; .search_with_options_traced(query, candidates, options, trace, parent)?;
if query.trim().is_empty() || limit == 0 { if query.trim().is_empty() || limit == 0 {
return Ok(Vec::new()); return Ok(lexical.into_iter().take(limit.min(1000)).collect());
} }
trace.timed("hybrid.identifier_priority", parent, |_, _| { trace.timed("hybrid.identifier_priority", parent, |_, _| {
promote_identifiers(query.trim(), &mut lexical) promote_identifiers(query.trim(), &mut lexical)
@@ -776,6 +776,24 @@ mod tests {
let directory = tempfile::tempdir()?; let directory = tempfile::tempdir()?;
let db = directory.path().join("index.db"); let db = directory.path().join("index.db");
let mut service = HybridSearch::with_provider(&db, Box::new(FailingProvider))?; let mut service = HybridSearch::with_provider(&db, Box::new(FailingProvider))?;
// The failure-only provider proves metadata-only searches never require inference.
let metadata = service.search_measured(
"",
5,
&SearchOptions {
media_artist: Some("Absent Artist".into()),
..Default::default()
},
true,
)?;
assert!(metadata.results.is_empty());
assert!(!metadata
.metrics
.trace
.unwrap()
.iter()
.any(|span| span.name == "semantic.query_embedding"));
let mut trace = TraceCollector::new(MetricsMode::Debug); let mut trace = TraceCollector::new(MetricsMode::Debug);
assert!(service assert!(service
.search_traced( .search_traced(
+2 -8
View File
@@ -190,15 +190,9 @@ impl HybridSearch {
metadata: &MediaProbe, metadata: &MediaProbe,
change_revision: i64, change_revision: i64,
) -> Result<()> { ) -> Result<()> {
let tags = metadata self.lexical.set_structured_media_metadata_at_revision(
.tags
.iter()
.map(|(k, v)| format!("{k}: {v}"))
.collect::<Vec<_>>()
.join("\n");
self.lexical.set_media_metadata_at_revision(
&path.to_string_lossy(), &path.to_string_lossy(),
&tags, &metadata.tags,
metadata.duration_ms, metadata.duration_ms,
identity.bytes, identity.bytes,
identity.modified_ns, identity.modified_ns,
+143 -2
View File
@@ -31,6 +31,8 @@ struct FilterKey {
max_duration: Option<u32>, max_duration: Option<u32>,
source_type: Option<String>, source_type: Option<String>,
code_only: bool, code_only: bool,
media_artist: Option<String>,
media_title: Option<String>,
} }
struct EligibilityCache { struct EligibilityCache {
@@ -184,6 +186,17 @@ impl Store {
} }
} }
initialization.execute_batch("CREATE TABLE IF NOT EXISTS semantic_media_errors(generation INTEGER NOT NULL REFERENCES semantic_generations(id) ON DELETE CASCADE,path TEXT NOT NULL,message TEXT NOT NULL,PRIMARY KEY(generation,path))")?; initialization.execute_batch("CREATE TABLE IF NOT EXISTS semantic_media_errors(generation INTEGER NOT NULL REFERENCES semantic_generations(id) ON DELETE CASCADE,path TEXT NOT NULL,message TEXT NOT NULL,PRIMARY KEY(generation,path))")?;
// Metadata-only writes change strict eligibility even when file bytes do not change.
for (name, event, row) in [
("insert", "INSERT", "new"),
("update", "UPDATE", "new"),
("delete", "DELETE", "old"),
] {
initialization.execute_batch(&format!("CREATE TRIGGER IF NOT EXISTS semantic_media_metadata_{name} AFTER {event} ON media_metadata BEGIN
UPDATE semantic_meta SET corpus_revision=corpus_revision+1 WHERE singleton=1;
INSERT INTO semantic_dirty_paths(path,revision) VALUES({row}.path,(SELECT corpus_revision FROM semantic_meta WHERE singleton=1))
ON CONFLICT(path) DO UPDATE SET revision=excluded.revision; END;"))?;
}
incremental::migrate(&initialization)?; incremental::migrate(&initialization)?;
initialization.commit()?; initialization.commit()?;
let canonical = db.canonicalize()?; let canonical = db.canonicalize()?;
@@ -644,6 +657,16 @@ impl Store {
max_duration: options.max_duration_ms, max_duration: options.max_duration_ms,
source_type: source_type.map(str::to_owned), source_type: source_type.map(str::to_owned),
code_only: options.code_only, code_only: options.code_only,
media_artist: options
.media_artist
.as_deref()
.map(searchi_core::normalize_media_field)
.transpose()?,
media_title: options
.media_title
.as_deref()
.map(searchi_core::normalize_media_field)
.transpose()?,
}; };
trace.end(filter_span, Default::default()); trace.end(filter_span, Default::default());
let eligibility_span = trace.begin("semantic.eligibility", parent); let eligibility_span = trace.begin("semantic.eligibility", parent);
@@ -883,6 +906,8 @@ impl Store {
AND (?3 IS NULL OR f.extension=?3) AND (?3 IS NULL OR f.extension=?3)
AND (?4 IS NULL OR f.size>=?4) AND (?5 IS NULL OR f.size<=?5) AND (?4 IS NULL OR f.size>=?4) AND (?5 IS NULL OR f.size<=?5)
AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7) AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7)
AND (?9 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.artist_key=?9))
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.title_key=?10))
ORDER BY c.passage_id,c.ordinal LIMIT ?8")?; ORDER BY c.passage_id,c.ordinal LIMIT ?8")?;
// Fetch one extra to detect the baseline cap instead of silently hiding // Fetch one extra to detect the baseline cap instead of silently hiding
// candidates. Metadata filtering precedes this cap and all top-k limits. // candidates. Metadata filtering precedes this cap and all top-k limits.
@@ -894,7 +919,17 @@ impl Store {
max_size, max_size,
options.modified_after_unix_ms, options.modified_after_unix_ms,
options.modified_before_unix_ms, options.modified_before_unix_ms,
MAX_SCAN_CHUNKS as i64 + 1 MAX_SCAN_CHUNKS as i64 + 1,
options
.media_artist
.as_deref()
.map(searchi_core::normalize_media_field)
.transpose()?,
options
.media_title
.as_deref()
.map(searchi_core::normalize_media_field)
.transpose()?
])?; ])?;
let mut hashes = HashMap::<i64, String>::new(); let mut hashes = HashMap::<i64, String>::new();
let mut best = HashMap::<String, SearchResult>::new(); let mut best = HashMap::<String, SearchResult>::new();
@@ -1008,6 +1043,8 @@ fn collect_eligibility(
AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7) AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7)
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms>=?10)) AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms>=?10))
AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms<=?11)) AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms<=?11))
AND (?13 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.artist_key=?13))
AND (?14 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.title_key=?14))
AND (?12 IS NULL OR c.source_type=?12) AND (?12 IS NULL OR c.source_type=?12)
AND (?9 IS NULL OR c.path=?9) {code} ORDER BY c.passage_id,c.ordinal LIMIT ?8"))?; AND (?9 IS NULL OR c.path=?9) {code} ORDER BY c.passage_id,c.ordinal LIMIT ?8"))?;
let mut rows = statement.query(params![ let mut rows = statement.query(params![
@@ -1022,7 +1059,9 @@ fn collect_eligibility(
path, path,
filter.min_duration, filter.min_duration,
filter.max_duration, filter.max_duration,
filter.source_type filter.source_type,
filter.media_artist,
filter.media_title
])?; ])?;
let mut hashes = HashMap::<i64, String>::new(); let mut hashes = HashMap::<i64, String>::new();
let mut keys = HashSet::new(); let mut keys = HashSet::new();
@@ -1293,6 +1332,108 @@ mod tests {
Ok(()) Ok(())
} }
#[test]
fn exact_metadata_filters_apply_before_ann_and_invalidate_warmed_eligibility() -> Result<()> {
let dir = tempfile::tempdir()?;
let root = dir.path().join("media");
std::fs::create_dir(&root)?;
std::fs::write(root.join("first.ogg"), b"metadata ANN eligibility fixture")?;
std::fs::write(root.join("other.ogg"), b"metadata distractor fixture")?;
let db = dir.path().join("index.db");
let mut lexical = SearchIndex::open(&db)?;
lexical.index_folder(&root)?;
let mut store = Store::open(&db, DIMENSIONS)?;
let mut files = store.media_batch(0)?;
files.sort_by(|a, b| a.1.cmp(&b.1));
let (other_id, other_path, _, _) = files.pop().context("distractor fixture")?;
let other_identity = searchi_media::MediaIdentity::read(Path::new(&other_path), 1024)?;
let (id, path, _, _) = files.remove(0);
let identity = searchi_media::MediaIdentity::read(Path::new(&path), 1024)?;
let revision =
lexical.media_change_revision(&path, identity.bytes, identity.modified_ns)?;
let mut tags = std::collections::BTreeMap::from([
("artist".into(), "Björk".into()),
("title".into(), "Jóga".into()),
]);
lexical.set_structured_media_metadata_at_revision(
&path,
&tags,
1000,
identity.bytes,
identity.modified_ns,
revision,
)?;
// Deterministic geometry tests storage/filter behavior, not model quality.
let mut vector = vec![0.; DIMENSIONS];
vector[0] = 1.;
let generation = store.begin("metadata-filter-proof")?;
let mut weaker = vector.clone();
weaker[0] = 0.8;
weaker[1] = 0.6;
store.insert_media(generation, id, 0, &identity, "audio", 0, &weaker)?;
store.insert_media(
generation,
other_id,
0,
&other_identity,
"audio",
0,
&vector,
)?;
store.activate(generation)?;
let mut options = SearchOptions {
media_artist: Some("BJÖRK".into()),
media_title: Some("JÓGA".into()),
..Default::default()
};
assert_eq!(
store
.search(&vector, "metadata-filter-proof", 1, &options)?
.len(),
1
);
options.media_artist = Some("Absent".into());
assert!(store
.search(&vector, "metadata-filter-proof", 1, &options)?
.is_empty());
options.media_artist = Some("BJÖRK".into());
assert_eq!(
store
.search(&vector, "metadata-filter-proof", 1, &options)?
.len(),
1
);
tags.insert("artist".into(), "Replacement".into());
lexical.set_structured_media_metadata_at_revision(
&path,
&tags,
1000,
identity.bytes,
identity.modified_ns,
revision,
)?;
assert!(store
.search(&vector, "metadata-filter-proof", 1, &options)?
.is_empty());
options.media_artist = Some("Replacement".into());
assert_eq!(
store
.search(&vector, "metadata-filter-proof", 1, &options)?
.len(),
1
);
lexical.index_paths_cancellable(
&root,
&[std::path::PathBuf::from(&path)],
&searchi_core::IndexOptions::default(),
&std::sync::atomic::AtomicBool::new(false),
)?;
assert!(store
.search(&vector, "metadata-filter-proof", 1, &options)?
.is_empty());
Ok(())
}
#[test] #[test]
fn media_origins_persist_filters_and_changes_remove_stale_candidates() -> Result<()> { fn media_origins_persist_filters_and_changes_remove_stale_candidates() -> Result<()> {
let directory = tempfile::tempdir()?; let directory = tempfile::tempdir()?;