Filter exact music identities before lexical and semantic candidate limits
This commit is contained in:
1 parent
3ac1201909
commit
2eace15a33
7 files changed
+458
-19
No files matched your search
@@ -136,3 +136,15 @@ opens. The measured Windows synthetic fixture grew from about 295 MiB to 605 MiB
|
||||
- OCR, media extraction, and real model inference belong to subsequent modules.
|
||||
- Linux/macOS measurements, realistic mixed-document corpora, cold startup,
|
||||
memory budgets, and indexing performance remain unverified.
|
||||
|
||||
### Exact music identities
|
||||
|
||||
`SearchOptions.media_artist` and `media_title` constrain every retrieval lane to
|
||||
complete genuine probe fields, with Unicode simple lowercase and outer whitespace
|
||||
normalization. Both fields combine with AND; substring inference and full Unicode
|
||||
case folding are not claimed. Each request field accepts 1–1024 UTF-8 bytes.
|
||||
An empty query with either constraint returns bounded filename/path-ordered matches.
|
||||
Missing tags and legacy flattened-only records are ineligible, rather than admitting
|
||||
semantic neighbours. A normal media probe refresh populates the structured keys.
|
||||
Oversized genuine fields remain keyword-searchable but receive no truncated exact
|
||||
identity key. Guarded publication and same-stat event hints preserve invalidation.
|
||||
@@ -17,14 +17,14 @@ use rusqlite::{params, params_from_iter, types::Value, Connection, OptionalExten
|
||||
use serde::Serialize;
|
||||
use std::{
|
||||
cell::RefCell,
|
||||
collections::{BTreeSet, HashMap},
|
||||
collections::{BTreeMap, BTreeSet, HashMap},
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
sync::atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use walkdir::WalkDir;
|
||||
|
||||
const SCHEMA_VERSION: i64 = 13;
|
||||
const SCHEMA_VERSION: i64 = 14;
|
||||
const MAX_RESULTS: usize = 1000;
|
||||
|
||||
type PreviousExtraction = (i64, i64, String, Option<String>, Option<String>);
|
||||
@@ -70,6 +70,19 @@ pub struct SearchOptions {
|
||||
pub modified_before_unix_ms: Option<i64>,
|
||||
pub min_duration_ms: Option<u32>,
|
||||
pub max_duration_ms: Option<u32>,
|
||||
/// Exact complete artist tag, Unicode lowercase comparison. Never inferred from query text.
|
||||
pub media_artist: Option<String>,
|
||||
/// Exact complete title tag; missing structured tags are ineligible.
|
||||
pub media_title: Option<String>,
|
||||
}
|
||||
|
||||
/// Shared bounded normalization for exact metadata constraints and genuine probe values.
|
||||
pub fn normalize_media_field(value: &str) -> Result<String> {
|
||||
let value = value.trim();
|
||||
if value.is_empty() || value.len() > 1024 {
|
||||
bail!("Exact media field must contain 1..=1024 UTF-8 bytes");
|
||||
}
|
||||
Ok(value.to_lowercase())
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
@@ -282,7 +295,19 @@ impl SearchIndex {
|
||||
[],
|
||||
)?;
|
||||
}
|
||||
initialization.execute_batch("PRAGMA user_version=13;")?;
|
||||
for field in ["artist_key", "title_key"] {
|
||||
let exists: bool = initialization.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM pragma_table_info('media_metadata') WHERE name=?1)",
|
||||
[field], |row| row.get(0),
|
||||
)?;
|
||||
if !exists {
|
||||
initialization.execute(
|
||||
&format!("ALTER TABLE media_metadata ADD COLUMN {field} TEXT"),
|
||||
[],
|
||||
)?;
|
||||
}
|
||||
}
|
||||
initialization.execute_batch("PRAGMA user_version=14;")?;
|
||||
initialization.commit()?;
|
||||
}
|
||||
Ok(Self {
|
||||
@@ -356,6 +381,78 @@ impl SearchIndex {
|
||||
size: u64,
|
||||
mtime_ns: i64,
|
||||
expected_revision: i64,
|
||||
) -> Result<()> {
|
||||
self.write_media_metadata(
|
||||
path,
|
||||
tags,
|
||||
None,
|
||||
None,
|
||||
duration_ms,
|
||||
size,
|
||||
mtime_ns,
|
||||
expected_revision,
|
||||
)
|
||||
}
|
||||
|
||||
/// Preserve structured fields directly from a genuine probe, never from flattened text.
|
||||
/// Empty/oversized fields retain legacy keyword tags but have no exact identity key.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn set_structured_media_metadata_at_revision(
|
||||
&mut self,
|
||||
path: &str,
|
||||
tags: &BTreeMap<String, String>,
|
||||
duration_ms: u32,
|
||||
size: u64,
|
||||
mtime_ns: i64,
|
||||
expected_revision: i64,
|
||||
) -> Result<()> {
|
||||
let field = |name: &str| -> Result<Option<String>> {
|
||||
tags.get(name)
|
||||
.filter(|value| !value.trim().is_empty() && value.trim().len() <= 1024)
|
||||
.map(|value| normalize_media_field(value))
|
||||
.transpose()
|
||||
};
|
||||
let mut total = 0usize;
|
||||
for (key, value) in tags {
|
||||
total = total
|
||||
.checked_add(key.len())
|
||||
.and_then(|n| n.checked_add(value.len()))
|
||||
.and_then(|n| n.checked_add(3))
|
||||
.context("Media tags size overflow")?;
|
||||
if total > 32768 {
|
||||
bail!("Media tags exceed 32 KiB bound");
|
||||
}
|
||||
}
|
||||
let artist = field("artist")?;
|
||||
let title = field("title")?;
|
||||
let flattened = tags
|
||||
.iter()
|
||||
.map(|(key, value)| format!("{key}: {value}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
self.write_media_metadata(
|
||||
path,
|
||||
&flattened,
|
||||
artist.as_deref(),
|
||||
title.as_deref(),
|
||||
duration_ms,
|
||||
size,
|
||||
mtime_ns,
|
||||
expected_revision,
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn write_media_metadata(
|
||||
&mut self,
|
||||
path: &str,
|
||||
tags: &str,
|
||||
artist: Option<&str>,
|
||||
title: Option<&str>,
|
||||
duration_ms: u32,
|
||||
size: u64,
|
||||
mtime_ns: i64,
|
||||
expected_revision: i64,
|
||||
) -> Result<()> {
|
||||
if tags.len() > 32768 {
|
||||
bail!("Media tags exceed 32 KiB bound");
|
||||
@@ -369,7 +466,7 @@ impl SearchIndex {
|
||||
)
|
||||
.optional()?;
|
||||
let id = id.context("Media metadata file changed since indexing")?;
|
||||
tx.execute("INSERT INTO media_metadata(path,file_id,tags,duration_ms) VALUES(?1,?2,?3,?4) ON CONFLICT(path) DO UPDATE SET tags=excluded.tags,duration_ms=excluded.duration_ms",params![path,id,tags,duration_ms])?;
|
||||
tx.execute("INSERT INTO media_metadata(path,file_id,tags,duration_ms,artist_key,title_key) VALUES(?1,?2,?3,?4,?5,?6) ON CONFLICT(path) DO UPDATE SET tags=excluded.tags,duration_ms=excluded.duration_ms,artist_key=excluded.artist_key,title_key=excluded.title_key",params![path,id,tags,duration_ms,artist,title])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -916,8 +1013,19 @@ impl SearchIndex {
|
||||
.map(i64::try_from)
|
||||
.transpose()
|
||||
.context("maximum size exceeds supported range")?;
|
||||
let artist = options
|
||||
.media_artist
|
||||
.as_deref()
|
||||
.map(normalize_media_field)
|
||||
.transpose()?;
|
||||
let title = options
|
||||
.media_title
|
||||
.as_deref()
|
||||
.map(normalize_media_field)
|
||||
.transpose()?;
|
||||
let metadata_only = query.trim().is_empty() && (artist.is_some() || title.is_some());
|
||||
let query = query.trim();
|
||||
if query.is_empty() || limit == 0 {
|
||||
if (query.is_empty() && !metadata_only) || limit == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
if query.len() > 4096 {
|
||||
@@ -937,7 +1045,9 @@ impl SearchIndex {
|
||||
AND (?5 IS NULL OR f.size>=?5) AND (?6 IS NULL OR f.size<=?6)
|
||||
AND (?7 IS NULL OR f.mtime/1000000>=?7) AND (?8 IS NULL OR f.mtime/1000000<=?8)
|
||||
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms>=?10))
|
||||
AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms<=?11))");
|
||||
AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms<=?11))
|
||||
AND (?12 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.artist_key=?12))
|
||||
AND (?13 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.title_key=?13))");
|
||||
if options.code_only {
|
||||
filters.push_str(" AND ");
|
||||
filters.push_str(code_file_predicate());
|
||||
@@ -957,7 +1067,9 @@ impl SearchIndex {
|
||||
let has_filters = filter_values.iter().any(|value| *value != Value::Null)
|
||||
|| options.code_only
|
||||
|| options.min_duration_ms.is_some()
|
||||
|| options.max_duration_ms.is_some();
|
||||
|| options.max_duration_ms.is_some()
|
||||
|| artist.is_some()
|
||||
|| title.is_some();
|
||||
let mut results = Vec::new();
|
||||
let mut selected = Vec::<i64>::new();
|
||||
// Each lane filters its entire candidate set before LIMIT. There is no
|
||||
@@ -1084,6 +1196,15 @@ impl SearchIndex {
|
||||
fts.clone(),
|
||||
"rank,f.id",
|
||||
));
|
||||
if metadata_only {
|
||||
lanes = vec![(
|
||||
"metadata_filter",
|
||||
"files f".into(),
|
||||
"(?1 IS NOT NULL AND ?2 IS NOT NULL)".into(),
|
||||
String::new(),
|
||||
"f.filename,f.path",
|
||||
)];
|
||||
}
|
||||
trace.end(planning, Default::default());
|
||||
for (kind, from, condition, expression, ordering) in lanes {
|
||||
if results.len() == limit {
|
||||
@@ -1161,6 +1282,10 @@ impl SearchIndex {
|
||||
.map_or(Value::Null, |v| Value::Integer(i64::from(v))),
|
||||
);
|
||||
}
|
||||
if sql.contains("?12") {
|
||||
values.push(artist.clone().map_or(Value::Null, Value::Text));
|
||||
values.push(title.clone().map_or(Value::Null, Value::Text));
|
||||
}
|
||||
let sql_span = trace.begin("lexical.filtered_sql", lane);
|
||||
trace.counter(sql_span, "filters_active", u64::from(has_filters));
|
||||
let mut statement = self.connection.prepare(&sql)?;
|
||||
@@ -1435,6 +1560,149 @@ fn validate_normalized_absolute(path: &Path) -> Result<()> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn legacy_metadata_migration_preserves_keyword_search_without_inventing_fields(
|
||||
) -> anyhow::Result<()> {
|
||||
use super::*;
|
||||
let dir = tempfile::tempdir()?;
|
||||
let root = dir.path().join("media");
|
||||
fs::create_dir(&root)?;
|
||||
let file = root.join("track.ogg");
|
||||
fs::write(&file, b"legacy tag fixture")?;
|
||||
let db = dir.path().join("index.db");
|
||||
let mut index = SearchIndex::open(&db)?;
|
||||
index.index_folder(&root)?;
|
||||
let file = file.canonicalize()?;
|
||||
let identity = searchi_media::MediaIdentity::read(&file, 1024)?;
|
||||
index.set_media_metadata(
|
||||
&file.to_string_lossy(),
|
||||
"artist: Björk",
|
||||
1000,
|
||||
identity.bytes,
|
||||
identity.modified_ns,
|
||||
)?;
|
||||
drop(index);
|
||||
let connection = Connection::open(&db)?;
|
||||
connection.execute_batch("ALTER TABLE media_metadata DROP COLUMN artist_key; ALTER TABLE media_metadata DROP COLUMN title_key; PRAGMA user_version=13;")?;
|
||||
drop(connection);
|
||||
let mut index = SearchIndex::open(&db)?;
|
||||
assert_eq!(index.search("Björk", 1)?.len(), 1);
|
||||
let options = SearchOptions {
|
||||
media_artist: Some("Björk".into()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(index.search_with_options("", 1, &options)?.is_empty());
|
||||
let revision = index.media_change_revision(
|
||||
&file.to_string_lossy(),
|
||||
identity.bytes,
|
||||
identity.modified_ns,
|
||||
)?;
|
||||
let tags = BTreeMap::from([
|
||||
("artist".into(), "Björk".into()),
|
||||
("title".into(), "x".repeat(1025)),
|
||||
]);
|
||||
index.set_structured_media_metadata_at_revision(
|
||||
&file.to_string_lossy(),
|
||||
&tags,
|
||||
1000,
|
||||
identity.bytes,
|
||||
identity.modified_ns,
|
||||
revision,
|
||||
)?;
|
||||
assert_eq!(index.search_with_options("", 1, &options)?.len(), 1);
|
||||
assert_eq!(index.search("x".repeat(1025).as_str(), 1)?.len(), 1);
|
||||
let too_short = SearchOptions {
|
||||
media_title: Some("x".repeat(1024)),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(index.search_with_options("", 1, &too_short)?.is_empty());
|
||||
drop(index);
|
||||
let index = SearchIndex::open(&db)?;
|
||||
assert_eq!(index.search_with_options("", 1, &options)?.len(), 1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_structured_media_filters_preserve_fields_and_hint_invalidation() -> anyhow::Result<()>
|
||||
{
|
||||
use super::*;
|
||||
let dir = tempfile::tempdir()?;
|
||||
let path = dir.path().join("track.ogg");
|
||||
fs::write(&path, b"tag storage fixture")?;
|
||||
let mut index = SearchIndex::open(":memory:")?;
|
||||
index.index_folder(dir.path())?;
|
||||
let identity = searchi_media::MediaIdentity::read(&path, 1024)?;
|
||||
let canonical = path.canonicalize()?;
|
||||
let path_string = canonical.to_string_lossy();
|
||||
let revision =
|
||||
index.media_change_revision(&path_string, identity.bytes, identity.modified_ns)?;
|
||||
let tags = BTreeMap::from([
|
||||
("artist".into(), "Björk".into()),
|
||||
("title".into(), "Jóga".into()),
|
||||
("comment".into(), "artist: Absent Artist".into()),
|
||||
]);
|
||||
index.set_structured_media_metadata_at_revision(
|
||||
&path_string,
|
||||
&tags,
|
||||
1000,
|
||||
identity.bytes,
|
||||
identity.modified_ns,
|
||||
revision,
|
||||
)?;
|
||||
let mut options = SearchOptions {
|
||||
media_artist: Some(" BJÖRK ".into()),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(index.search_with_options("", 1, &options)?.len(), 1);
|
||||
assert_eq!(index.search_with_options("track", 1, &options)?.len(), 1);
|
||||
options.media_title = Some("JÓGA".into());
|
||||
assert_eq!(index.search_with_options("", 1, &options)?.len(), 1);
|
||||
options.media_artist = Some("Absent Artist".into());
|
||||
assert!(index.search_with_options("", 1, &options)?.is_empty());
|
||||
options.media_artist = Some("Björ".into());
|
||||
assert!(index.search_with_options("track", 1, &options)?.is_empty());
|
||||
options.media_artist = Some(" ".into());
|
||||
assert!(index.search_with_options("", 1, &options).is_err());
|
||||
options.media_artist = Some("Björk".into());
|
||||
index.index_paths_cancellable(
|
||||
dir.path(),
|
||||
std::slice::from_ref(&canonical),
|
||||
&IndexOptions::default(),
|
||||
&AtomicBool::new(false),
|
||||
)?;
|
||||
assert!(index.search_with_options("", 1, &options)?.is_empty());
|
||||
assert!(index
|
||||
.set_structured_media_metadata_at_revision(
|
||||
&path_string,
|
||||
&tags,
|
||||
1000,
|
||||
identity.bytes,
|
||||
identity.modified_ns,
|
||||
revision
|
||||
)
|
||||
.is_err());
|
||||
let revision =
|
||||
index.media_change_revision(&path_string, identity.bytes, identity.modified_ns)?;
|
||||
index.set_structured_media_metadata_at_revision(
|
||||
&path_string,
|
||||
&tags,
|
||||
1000,
|
||||
identity.bytes,
|
||||
identity.modified_ns,
|
||||
revision,
|
||||
)?;
|
||||
assert_eq!(index.search_with_options("", 1, &options)?.len(), 1);
|
||||
index.set_media_metadata(
|
||||
&path_string,
|
||||
"artist: Björk",
|
||||
1000,
|
||||
identity.bytes,
|
||||
identity.modified_ns,
|
||||
)?;
|
||||
assert!(index.search_with_options("", 1, &options)?.is_empty());
|
||||
assert_eq!(index.search("Björk", 1)?.len(), 1);
|
||||
Ok(())
|
||||
}
|
||||
#[test]
|
||||
fn measured_search_preserves_results_and_contains_private_free_nested_spans(
|
||||
) -> anyhow::Result<()> {
|
||||
|
||||
@@ -447,7 +447,7 @@ fn schema_twelve_migrates_change_revision_without_reextracting_records() -> Resu
|
||||
let connection = rusqlite::Connection::open(&db)?;
|
||||
assert_eq!(
|
||||
connection.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))?,
|
||||
13
|
||||
14
|
||||
);
|
||||
assert_eq!(
|
||||
connection.query_row("SELECT change_revision FROM files", [], |row| row
|
||||
|
||||
@@ -41,3 +41,9 @@ Audio timestamps represent segment starts, not exact word locations. Smaller
|
||||
windows alone have not demonstrated accurate spoken navigation. No-answer
|
||||
calibration, music discrimination, exact spoken positions, process isolation,
|
||||
unchanged-vector reuse and production platform checks remain release gates.
|
||||
|
||||
Exact artist/title constraints use the core's normalized structured probe keys
|
||||
before ANN limits and in the eligibility cache key. Metadata-only writes advance
|
||||
the corpus revision and dirty path so warmed readers cannot retain old eligibility.
|
||||
Empty metadata-only hybrid requests use core results without model inference.
|
||||
These explicit constraints do not establish abstention for free-form topic queries.
|
||||
@@ -513,7 +513,7 @@ impl HybridSearch {
|
||||
.lexical
|
||||
.search_with_options_traced(query, candidates, options, trace, parent)?;
|
||||
if query.trim().is_empty() || limit == 0 {
|
||||
return Ok(Vec::new());
|
||||
return Ok(lexical.into_iter().take(limit.min(1000)).collect());
|
||||
}
|
||||
trace.timed("hybrid.identifier_priority", parent, |_, _| {
|
||||
promote_identifiers(query.trim(), &mut lexical)
|
||||
@@ -776,6 +776,24 @@ mod tests {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let db = directory.path().join("index.db");
|
||||
let mut service = HybridSearch::with_provider(&db, Box::new(FailingProvider))?;
|
||||
// The failure-only provider proves metadata-only searches never require inference.
|
||||
let metadata = service.search_measured(
|
||||
"",
|
||||
5,
|
||||
&SearchOptions {
|
||||
media_artist: Some("Absent Artist".into()),
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
)?;
|
||||
assert!(metadata.results.is_empty());
|
||||
assert!(!metadata
|
||||
.metrics
|
||||
.trace
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|span| span.name == "semantic.query_embedding"));
|
||||
|
||||
let mut trace = TraceCollector::new(MetricsMode::Debug);
|
||||
assert!(service
|
||||
.search_traced(
|
||||
|
||||
@@ -190,15 +190,9 @@ impl HybridSearch {
|
||||
metadata: &MediaProbe,
|
||||
change_revision: i64,
|
||||
) -> Result<()> {
|
||||
let tags = metadata
|
||||
.tags
|
||||
.iter()
|
||||
.map(|(k, v)| format!("{k}: {v}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
self.lexical.set_media_metadata_at_revision(
|
||||
self.lexical.set_structured_media_metadata_at_revision(
|
||||
&path.to_string_lossy(),
|
||||
&tags,
|
||||
&metadata.tags,
|
||||
metadata.duration_ms,
|
||||
identity.bytes,
|
||||
identity.modified_ns,
|
||||
|
||||
@@ -31,6 +31,8 @@ struct FilterKey {
|
||||
max_duration: Option<u32>,
|
||||
source_type: Option<String>,
|
||||
code_only: bool,
|
||||
media_artist: Option<String>,
|
||||
media_title: Option<String>,
|
||||
}
|
||||
|
||||
struct EligibilityCache {
|
||||
@@ -184,6 +186,17 @@ impl Store {
|
||||
}
|
||||
}
|
||||
initialization.execute_batch("CREATE TABLE IF NOT EXISTS semantic_media_errors(generation INTEGER NOT NULL REFERENCES semantic_generations(id) ON DELETE CASCADE,path TEXT NOT NULL,message TEXT NOT NULL,PRIMARY KEY(generation,path))")?;
|
||||
// Metadata-only writes change strict eligibility even when file bytes do not change.
|
||||
for (name, event, row) in [
|
||||
("insert", "INSERT", "new"),
|
||||
("update", "UPDATE", "new"),
|
||||
("delete", "DELETE", "old"),
|
||||
] {
|
||||
initialization.execute_batch(&format!("CREATE TRIGGER IF NOT EXISTS semantic_media_metadata_{name} AFTER {event} ON media_metadata BEGIN
|
||||
UPDATE semantic_meta SET corpus_revision=corpus_revision+1 WHERE singleton=1;
|
||||
INSERT INTO semantic_dirty_paths(path,revision) VALUES({row}.path,(SELECT corpus_revision FROM semantic_meta WHERE singleton=1))
|
||||
ON CONFLICT(path) DO UPDATE SET revision=excluded.revision; END;"))?;
|
||||
}
|
||||
incremental::migrate(&initialization)?;
|
||||
initialization.commit()?;
|
||||
let canonical = db.canonicalize()?;
|
||||
@@ -644,6 +657,16 @@ impl Store {
|
||||
max_duration: options.max_duration_ms,
|
||||
source_type: source_type.map(str::to_owned),
|
||||
code_only: options.code_only,
|
||||
media_artist: options
|
||||
.media_artist
|
||||
.as_deref()
|
||||
.map(searchi_core::normalize_media_field)
|
||||
.transpose()?,
|
||||
media_title: options
|
||||
.media_title
|
||||
.as_deref()
|
||||
.map(searchi_core::normalize_media_field)
|
||||
.transpose()?,
|
||||
};
|
||||
trace.end(filter_span, Default::default());
|
||||
let eligibility_span = trace.begin("semantic.eligibility", parent);
|
||||
@@ -883,6 +906,8 @@ impl Store {
|
||||
AND (?3 IS NULL OR f.extension=?3)
|
||||
AND (?4 IS NULL OR f.size>=?4) AND (?5 IS NULL OR f.size<=?5)
|
||||
AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7)
|
||||
AND (?9 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.artist_key=?9))
|
||||
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.title_key=?10))
|
||||
ORDER BY c.passage_id,c.ordinal LIMIT ?8")?;
|
||||
// Fetch one extra to detect the baseline cap instead of silently hiding
|
||||
// candidates. Metadata filtering precedes this cap and all top-k limits.
|
||||
@@ -894,7 +919,17 @@ impl Store {
|
||||
max_size,
|
||||
options.modified_after_unix_ms,
|
||||
options.modified_before_unix_ms,
|
||||
MAX_SCAN_CHUNKS as i64 + 1
|
||||
MAX_SCAN_CHUNKS as i64 + 1,
|
||||
options
|
||||
.media_artist
|
||||
.as_deref()
|
||||
.map(searchi_core::normalize_media_field)
|
||||
.transpose()?,
|
||||
options
|
||||
.media_title
|
||||
.as_deref()
|
||||
.map(searchi_core::normalize_media_field)
|
||||
.transpose()?
|
||||
])?;
|
||||
let mut hashes = HashMap::<i64, String>::new();
|
||||
let mut best = HashMap::<String, SearchResult>::new();
|
||||
@@ -1008,6 +1043,8 @@ fn collect_eligibility(
|
||||
AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7)
|
||||
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms>=?10))
|
||||
AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms<=?11))
|
||||
AND (?13 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.artist_key=?13))
|
||||
AND (?14 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.title_key=?14))
|
||||
AND (?12 IS NULL OR c.source_type=?12)
|
||||
AND (?9 IS NULL OR c.path=?9) {code} ORDER BY c.passage_id,c.ordinal LIMIT ?8"))?;
|
||||
let mut rows = statement.query(params![
|
||||
@@ -1022,7 +1059,9 @@ fn collect_eligibility(
|
||||
path,
|
||||
filter.min_duration,
|
||||
filter.max_duration,
|
||||
filter.source_type
|
||||
filter.source_type,
|
||||
filter.media_artist,
|
||||
filter.media_title
|
||||
])?;
|
||||
let mut hashes = HashMap::<i64, String>::new();
|
||||
let mut keys = HashSet::new();
|
||||
@@ -1293,6 +1332,108 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_metadata_filters_apply_before_ann_and_invalidate_warmed_eligibility() -> Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let root = dir.path().join("media");
|
||||
std::fs::create_dir(&root)?;
|
||||
std::fs::write(root.join("first.ogg"), b"metadata ANN eligibility fixture")?;
|
||||
std::fs::write(root.join("other.ogg"), b"metadata distractor fixture")?;
|
||||
let db = dir.path().join("index.db");
|
||||
let mut lexical = SearchIndex::open(&db)?;
|
||||
lexical.index_folder(&root)?;
|
||||
let mut store = Store::open(&db, DIMENSIONS)?;
|
||||
let mut files = store.media_batch(0)?;
|
||||
files.sort_by(|a, b| a.1.cmp(&b.1));
|
||||
let (other_id, other_path, _, _) = files.pop().context("distractor fixture")?;
|
||||
let other_identity = searchi_media::MediaIdentity::read(Path::new(&other_path), 1024)?;
|
||||
let (id, path, _, _) = files.remove(0);
|
||||
let identity = searchi_media::MediaIdentity::read(Path::new(&path), 1024)?;
|
||||
let revision =
|
||||
lexical.media_change_revision(&path, identity.bytes, identity.modified_ns)?;
|
||||
let mut tags = std::collections::BTreeMap::from([
|
||||
("artist".into(), "Björk".into()),
|
||||
("title".into(), "Jóga".into()),
|
||||
]);
|
||||
lexical.set_structured_media_metadata_at_revision(
|
||||
&path,
|
||||
&tags,
|
||||
1000,
|
||||
identity.bytes,
|
||||
identity.modified_ns,
|
||||
revision,
|
||||
)?;
|
||||
// Deterministic geometry tests storage/filter behavior, not model quality.
|
||||
let mut vector = vec![0.; DIMENSIONS];
|
||||
vector[0] = 1.;
|
||||
let generation = store.begin("metadata-filter-proof")?;
|
||||
let mut weaker = vector.clone();
|
||||
weaker[0] = 0.8;
|
||||
weaker[1] = 0.6;
|
||||
store.insert_media(generation, id, 0, &identity, "audio", 0, &weaker)?;
|
||||
store.insert_media(
|
||||
generation,
|
||||
other_id,
|
||||
0,
|
||||
&other_identity,
|
||||
"audio",
|
||||
0,
|
||||
&vector,
|
||||
)?;
|
||||
store.activate(generation)?;
|
||||
let mut options = SearchOptions {
|
||||
media_artist: Some("BJÖRK".into()),
|
||||
media_title: Some("JÓGA".into()),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
store
|
||||
.search(&vector, "metadata-filter-proof", 1, &options)?
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
options.media_artist = Some("Absent".into());
|
||||
assert!(store
|
||||
.search(&vector, "metadata-filter-proof", 1, &options)?
|
||||
.is_empty());
|
||||
options.media_artist = Some("BJÖRK".into());
|
||||
assert_eq!(
|
||||
store
|
||||
.search(&vector, "metadata-filter-proof", 1, &options)?
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
tags.insert("artist".into(), "Replacement".into());
|
||||
lexical.set_structured_media_metadata_at_revision(
|
||||
&path,
|
||||
&tags,
|
||||
1000,
|
||||
identity.bytes,
|
||||
identity.modified_ns,
|
||||
revision,
|
||||
)?;
|
||||
assert!(store
|
||||
.search(&vector, "metadata-filter-proof", 1, &options)?
|
||||
.is_empty());
|
||||
options.media_artist = Some("Replacement".into());
|
||||
assert_eq!(
|
||||
store
|
||||
.search(&vector, "metadata-filter-proof", 1, &options)?
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
lexical.index_paths_cancellable(
|
||||
&root,
|
||||
&[std::path::PathBuf::from(&path)],
|
||||
&searchi_core::IndexOptions::default(),
|
||||
&std::sync::atomic::AtomicBool::new(false),
|
||||
)?;
|
||||
assert!(store
|
||||
.search(&vector, "metadata-filter-proof", 1, &options)?
|
||||
.is_empty());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn media_origins_persist_filters_and_changes_remove_stale_candidates() -> Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
|
||||
Reference in new issue
Block a user