Filter exact music identities before lexical and semantic candidate limits
This commit is contained in:
1 parent
3ac1201909
commit
2eace15a33
7 files changed
+458
-19
No files matched your search
@@ -136,3 +136,15 @@ opens. The measured Windows synthetic fixture grew from about 295 MiB to 605 MiB
|
|||||||
- OCR, media extraction, and real model inference belong to subsequent modules.
|
- OCR, media extraction, and real model inference belong to subsequent modules.
|
||||||
- Linux/macOS measurements, realistic mixed-document corpora, cold startup,
|
- Linux/macOS measurements, realistic mixed-document corpora, cold startup,
|
||||||
memory budgets, and indexing performance remain unverified.
|
memory budgets, and indexing performance remain unverified.
|
||||||
|
|
||||||
|
### Exact music identities
|
||||||
|
|
||||||
|
`SearchOptions.media_artist` and `media_title` constrain every retrieval lane to
|
||||||
|
complete genuine probe fields, with Unicode simple lowercase and outer whitespace
|
||||||
|
normalization. Both fields combine with AND; substring inference and full Unicode
|
||||||
|
case folding are not claimed. Each request field accepts 1–1024 UTF-8 bytes.
|
||||||
|
An empty query with either constraint returns bounded filename/path-ordered matches.
|
||||||
|
Missing tags and legacy flattened-only records are ineligible, rather than admitting
|
||||||
|
semantic neighbours. A normal media probe refresh populates the structured keys.
|
||||||
|
Oversized genuine fields remain keyword-searchable but receive no truncated exact
|
||||||
|
identity key. Guarded publication and same-stat event hints preserve invalidation.
|
||||||
@@ -17,14 +17,14 @@ use rusqlite::{params, params_from_iter, types::Value, Connection, OptionalExten
|
|||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use std::{
|
use std::{
|
||||||
cell::RefCell,
|
cell::RefCell,
|
||||||
collections::{BTreeSet, HashMap},
|
collections::{BTreeMap, BTreeSet, HashMap},
|
||||||
fs,
|
fs,
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
sync::atomic::{AtomicBool, Ordering},
|
sync::atomic::{AtomicBool, Ordering},
|
||||||
};
|
};
|
||||||
use walkdir::WalkDir;
|
use walkdir::WalkDir;
|
||||||
|
|
||||||
const SCHEMA_VERSION: i64 = 13;
|
const SCHEMA_VERSION: i64 = 14;
|
||||||
const MAX_RESULTS: usize = 1000;
|
const MAX_RESULTS: usize = 1000;
|
||||||
|
|
||||||
type PreviousExtraction = (i64, i64, String, Option<String>, Option<String>);
|
type PreviousExtraction = (i64, i64, String, Option<String>, Option<String>);
|
||||||
@@ -70,6 +70,19 @@ pub struct SearchOptions {
|
|||||||
pub modified_before_unix_ms: Option<i64>,
|
pub modified_before_unix_ms: Option<i64>,
|
||||||
pub min_duration_ms: Option<u32>,
|
pub min_duration_ms: Option<u32>,
|
||||||
pub max_duration_ms: Option<u32>,
|
pub max_duration_ms: Option<u32>,
|
||||||
|
/// Exact complete artist tag, Unicode lowercase comparison. Never inferred from query text.
|
||||||
|
pub media_artist: Option<String>,
|
||||||
|
/// Exact complete title tag; missing structured tags are ineligible.
|
||||||
|
pub media_title: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shared bounded normalization for exact metadata constraints and genuine probe values.
|
||||||
|
pub fn normalize_media_field(value: &str) -> Result<String> {
|
||||||
|
let value = value.trim();
|
||||||
|
if value.is_empty() || value.len() > 1024 {
|
||||||
|
bail!("Exact media field must contain 1..=1024 UTF-8 bytes");
|
||||||
|
}
|
||||||
|
Ok(value.to_lowercase())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
@@ -282,7 +295,19 @@ impl SearchIndex {
|
|||||||
[],
|
[],
|
||||||
)?;
|
)?;
|
||||||
}
|
}
|
||||||
initialization.execute_batch("PRAGMA user_version=13;")?;
|
for field in ["artist_key", "title_key"] {
|
||||||
|
let exists: bool = initialization.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM pragma_table_info('media_metadata') WHERE name=?1)",
|
||||||
|
[field], |row| row.get(0),
|
||||||
|
)?;
|
||||||
|
if !exists {
|
||||||
|
initialization.execute(
|
||||||
|
&format!("ALTER TABLE media_metadata ADD COLUMN {field} TEXT"),
|
||||||
|
[],
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
initialization.execute_batch("PRAGMA user_version=14;")?;
|
||||||
initialization.commit()?;
|
initialization.commit()?;
|
||||||
}
|
}
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
@@ -356,6 +381,78 @@ impl SearchIndex {
|
|||||||
size: u64,
|
size: u64,
|
||||||
mtime_ns: i64,
|
mtime_ns: i64,
|
||||||
expected_revision: i64,
|
expected_revision: i64,
|
||||||
|
) -> Result<()> {
|
||||||
|
self.write_media_metadata(
|
||||||
|
path,
|
||||||
|
tags,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
duration_ms,
|
||||||
|
size,
|
||||||
|
mtime_ns,
|
||||||
|
expected_revision,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Preserve structured fields directly from a genuine probe, never from flattened text.
|
||||||
|
/// Empty/oversized fields retain legacy keyword tags but have no exact identity key.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub fn set_structured_media_metadata_at_revision(
|
||||||
|
&mut self,
|
||||||
|
path: &str,
|
||||||
|
tags: &BTreeMap<String, String>,
|
||||||
|
duration_ms: u32,
|
||||||
|
size: u64,
|
||||||
|
mtime_ns: i64,
|
||||||
|
expected_revision: i64,
|
||||||
|
) -> Result<()> {
|
||||||
|
let field = |name: &str| -> Result<Option<String>> {
|
||||||
|
tags.get(name)
|
||||||
|
.filter(|value| !value.trim().is_empty() && value.trim().len() <= 1024)
|
||||||
|
.map(|value| normalize_media_field(value))
|
||||||
|
.transpose()
|
||||||
|
};
|
||||||
|
let mut total = 0usize;
|
||||||
|
for (key, value) in tags {
|
||||||
|
total = total
|
||||||
|
.checked_add(key.len())
|
||||||
|
.and_then(|n| n.checked_add(value.len()))
|
||||||
|
.and_then(|n| n.checked_add(3))
|
||||||
|
.context("Media tags size overflow")?;
|
||||||
|
if total > 32768 {
|
||||||
|
bail!("Media tags exceed 32 KiB bound");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let artist = field("artist")?;
|
||||||
|
let title = field("title")?;
|
||||||
|
let flattened = tags
|
||||||
|
.iter()
|
||||||
|
.map(|(key, value)| format!("{key}: {value}"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n");
|
||||||
|
self.write_media_metadata(
|
||||||
|
path,
|
||||||
|
&flattened,
|
||||||
|
artist.as_deref(),
|
||||||
|
title.as_deref(),
|
||||||
|
duration_ms,
|
||||||
|
size,
|
||||||
|
mtime_ns,
|
||||||
|
expected_revision,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
fn write_media_metadata(
|
||||||
|
&mut self,
|
||||||
|
path: &str,
|
||||||
|
tags: &str,
|
||||||
|
artist: Option<&str>,
|
||||||
|
title: Option<&str>,
|
||||||
|
duration_ms: u32,
|
||||||
|
size: u64,
|
||||||
|
mtime_ns: i64,
|
||||||
|
expected_revision: i64,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
if tags.len() > 32768 {
|
if tags.len() > 32768 {
|
||||||
bail!("Media tags exceed 32 KiB bound");
|
bail!("Media tags exceed 32 KiB bound");
|
||||||
@@ -369,7 +466,7 @@ impl SearchIndex {
|
|||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
let id = id.context("Media metadata file changed since indexing")?;
|
let id = id.context("Media metadata file changed since indexing")?;
|
||||||
tx.execute("INSERT INTO media_metadata(path,file_id,tags,duration_ms) VALUES(?1,?2,?3,?4) ON CONFLICT(path) DO UPDATE SET tags=excluded.tags,duration_ms=excluded.duration_ms",params![path,id,tags,duration_ms])?;
|
tx.execute("INSERT INTO media_metadata(path,file_id,tags,duration_ms,artist_key,title_key) VALUES(?1,?2,?3,?4,?5,?6) ON CONFLICT(path) DO UPDATE SET tags=excluded.tags,duration_ms=excluded.duration_ms,artist_key=excluded.artist_key,title_key=excluded.title_key",params![path,id,tags,duration_ms,artist,title])?;
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -916,8 +1013,19 @@ impl SearchIndex {
|
|||||||
.map(i64::try_from)
|
.map(i64::try_from)
|
||||||
.transpose()
|
.transpose()
|
||||||
.context("maximum size exceeds supported range")?;
|
.context("maximum size exceeds supported range")?;
|
||||||
|
let artist = options
|
||||||
|
.media_artist
|
||||||
|
.as_deref()
|
||||||
|
.map(normalize_media_field)
|
||||||
|
.transpose()?;
|
||||||
|
let title = options
|
||||||
|
.media_title
|
||||||
|
.as_deref()
|
||||||
|
.map(normalize_media_field)
|
||||||
|
.transpose()?;
|
||||||
|
let metadata_only = query.trim().is_empty() && (artist.is_some() || title.is_some());
|
||||||
let query = query.trim();
|
let query = query.trim();
|
||||||
if query.is_empty() || limit == 0 {
|
if (query.is_empty() && !metadata_only) || limit == 0 {
|
||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
if query.len() > 4096 {
|
if query.len() > 4096 {
|
||||||
@@ -937,7 +1045,9 @@ impl SearchIndex {
|
|||||||
AND (?5 IS NULL OR f.size>=?5) AND (?6 IS NULL OR f.size<=?6)
|
AND (?5 IS NULL OR f.size>=?5) AND (?6 IS NULL OR f.size<=?6)
|
||||||
AND (?7 IS NULL OR f.mtime/1000000>=?7) AND (?8 IS NULL OR f.mtime/1000000<=?8)
|
AND (?7 IS NULL OR f.mtime/1000000>=?7) AND (?8 IS NULL OR f.mtime/1000000<=?8)
|
||||||
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms>=?10))
|
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms>=?10))
|
||||||
AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms<=?11))");
|
AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.duration_ms<=?11))
|
||||||
|
AND (?12 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.artist_key=?12))
|
||||||
|
AND (?13 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.file_id=f.id AND m.title_key=?13))");
|
||||||
if options.code_only {
|
if options.code_only {
|
||||||
filters.push_str(" AND ");
|
filters.push_str(" AND ");
|
||||||
filters.push_str(code_file_predicate());
|
filters.push_str(code_file_predicate());
|
||||||
@@ -957,7 +1067,9 @@ impl SearchIndex {
|
|||||||
let has_filters = filter_values.iter().any(|value| *value != Value::Null)
|
let has_filters = filter_values.iter().any(|value| *value != Value::Null)
|
||||||
|| options.code_only
|
|| options.code_only
|
||||||
|| options.min_duration_ms.is_some()
|
|| options.min_duration_ms.is_some()
|
||||||
|| options.max_duration_ms.is_some();
|
|| options.max_duration_ms.is_some()
|
||||||
|
|| artist.is_some()
|
||||||
|
|| title.is_some();
|
||||||
let mut results = Vec::new();
|
let mut results = Vec::new();
|
||||||
let mut selected = Vec::<i64>::new();
|
let mut selected = Vec::<i64>::new();
|
||||||
// Each lane filters its entire candidate set before LIMIT. There is no
|
// Each lane filters its entire candidate set before LIMIT. There is no
|
||||||
@@ -1084,6 +1196,15 @@ impl SearchIndex {
|
|||||||
fts.clone(),
|
fts.clone(),
|
||||||
"rank,f.id",
|
"rank,f.id",
|
||||||
));
|
));
|
||||||
|
if metadata_only {
|
||||||
|
lanes = vec![(
|
||||||
|
"metadata_filter",
|
||||||
|
"files f".into(),
|
||||||
|
"(?1 IS NOT NULL AND ?2 IS NOT NULL)".into(),
|
||||||
|
String::new(),
|
||||||
|
"f.filename,f.path",
|
||||||
|
)];
|
||||||
|
}
|
||||||
trace.end(planning, Default::default());
|
trace.end(planning, Default::default());
|
||||||
for (kind, from, condition, expression, ordering) in lanes {
|
for (kind, from, condition, expression, ordering) in lanes {
|
||||||
if results.len() == limit {
|
if results.len() == limit {
|
||||||
@@ -1161,6 +1282,10 @@ impl SearchIndex {
|
|||||||
.map_or(Value::Null, |v| Value::Integer(i64::from(v))),
|
.map_or(Value::Null, |v| Value::Integer(i64::from(v))),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if sql.contains("?12") {
|
||||||
|
values.push(artist.clone().map_or(Value::Null, Value::Text));
|
||||||
|
values.push(title.clone().map_or(Value::Null, Value::Text));
|
||||||
|
}
|
||||||
let sql_span = trace.begin("lexical.filtered_sql", lane);
|
let sql_span = trace.begin("lexical.filtered_sql", lane);
|
||||||
trace.counter(sql_span, "filters_active", u64::from(has_filters));
|
trace.counter(sql_span, "filters_active", u64::from(has_filters));
|
||||||
let mut statement = self.connection.prepare(&sql)?;
|
let mut statement = self.connection.prepare(&sql)?;
|
||||||
@@ -1435,6 +1560,149 @@ fn validate_normalized_absolute(path: &Path) -> Result<()> {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
#[test]
|
||||||
|
fn legacy_metadata_migration_preserves_keyword_search_without_inventing_fields(
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
use super::*;
|
||||||
|
let dir = tempfile::tempdir()?;
|
||||||
|
let root = dir.path().join("media");
|
||||||
|
fs::create_dir(&root)?;
|
||||||
|
let file = root.join("track.ogg");
|
||||||
|
fs::write(&file, b"legacy tag fixture")?;
|
||||||
|
let db = dir.path().join("index.db");
|
||||||
|
let mut index = SearchIndex::open(&db)?;
|
||||||
|
index.index_folder(&root)?;
|
||||||
|
let file = file.canonicalize()?;
|
||||||
|
let identity = searchi_media::MediaIdentity::read(&file, 1024)?;
|
||||||
|
index.set_media_metadata(
|
||||||
|
&file.to_string_lossy(),
|
||||||
|
"artist: Björk",
|
||||||
|
1000,
|
||||||
|
identity.bytes,
|
||||||
|
identity.modified_ns,
|
||||||
|
)?;
|
||||||
|
drop(index);
|
||||||
|
let connection = Connection::open(&db)?;
|
||||||
|
connection.execute_batch("ALTER TABLE media_metadata DROP COLUMN artist_key; ALTER TABLE media_metadata DROP COLUMN title_key; PRAGMA user_version=13;")?;
|
||||||
|
drop(connection);
|
||||||
|
let mut index = SearchIndex::open(&db)?;
|
||||||
|
assert_eq!(index.search("Björk", 1)?.len(), 1);
|
||||||
|
let options = SearchOptions {
|
||||||
|
media_artist: Some("Björk".into()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(index.search_with_options("", 1, &options)?.is_empty());
|
||||||
|
let revision = index.media_change_revision(
|
||||||
|
&file.to_string_lossy(),
|
||||||
|
identity.bytes,
|
||||||
|
identity.modified_ns,
|
||||||
|
)?;
|
||||||
|
let tags = BTreeMap::from([
|
||||||
|
("artist".into(), "Björk".into()),
|
||||||
|
("title".into(), "x".repeat(1025)),
|
||||||
|
]);
|
||||||
|
index.set_structured_media_metadata_at_revision(
|
||||||
|
&file.to_string_lossy(),
|
||||||
|
&tags,
|
||||||
|
1000,
|
||||||
|
identity.bytes,
|
||||||
|
identity.modified_ns,
|
||||||
|
revision,
|
||||||
|
)?;
|
||||||
|
assert_eq!(index.search_with_options("", 1, &options)?.len(), 1);
|
||||||
|
assert_eq!(index.search("x".repeat(1025).as_str(), 1)?.len(), 1);
|
||||||
|
let too_short = SearchOptions {
|
||||||
|
media_title: Some("x".repeat(1024)),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(index.search_with_options("", 1, &too_short)?.is_empty());
|
||||||
|
drop(index);
|
||||||
|
let index = SearchIndex::open(&db)?;
|
||||||
|
assert_eq!(index.search_with_options("", 1, &options)?.len(), 1);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exact_structured_media_filters_preserve_fields_and_hint_invalidation() -> anyhow::Result<()>
|
||||||
|
{
|
||||||
|
use super::*;
|
||||||
|
let dir = tempfile::tempdir()?;
|
||||||
|
let path = dir.path().join("track.ogg");
|
||||||
|
fs::write(&path, b"tag storage fixture")?;
|
||||||
|
let mut index = SearchIndex::open(":memory:")?;
|
||||||
|
index.index_folder(dir.path())?;
|
||||||
|
let identity = searchi_media::MediaIdentity::read(&path, 1024)?;
|
||||||
|
let canonical = path.canonicalize()?;
|
||||||
|
let path_string = canonical.to_string_lossy();
|
||||||
|
let revision =
|
||||||
|
index.media_change_revision(&path_string, identity.bytes, identity.modified_ns)?;
|
||||||
|
let tags = BTreeMap::from([
|
||||||
|
("artist".into(), "Björk".into()),
|
||||||
|
("title".into(), "Jóga".into()),
|
||||||
|
("comment".into(), "artist: Absent Artist".into()),
|
||||||
|
]);
|
||||||
|
index.set_structured_media_metadata_at_revision(
|
||||||
|
&path_string,
|
||||||
|
&tags,
|
||||||
|
1000,
|
||||||
|
identity.bytes,
|
||||||
|
identity.modified_ns,
|
||||||
|
revision,
|
||||||
|
)?;
|
||||||
|
let mut options = SearchOptions {
|
||||||
|
media_artist: Some(" BJÖRK ".into()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert_eq!(index.search_with_options("", 1, &options)?.len(), 1);
|
||||||
|
assert_eq!(index.search_with_options("track", 1, &options)?.len(), 1);
|
||||||
|
options.media_title = Some("JÓGA".into());
|
||||||
|
assert_eq!(index.search_with_options("", 1, &options)?.len(), 1);
|
||||||
|
options.media_artist = Some("Absent Artist".into());
|
||||||
|
assert!(index.search_with_options("", 1, &options)?.is_empty());
|
||||||
|
options.media_artist = Some("Björ".into());
|
||||||
|
assert!(index.search_with_options("track", 1, &options)?.is_empty());
|
||||||
|
options.media_artist = Some(" ".into());
|
||||||
|
assert!(index.search_with_options("", 1, &options).is_err());
|
||||||
|
options.media_artist = Some("Björk".into());
|
||||||
|
index.index_paths_cancellable(
|
||||||
|
dir.path(),
|
||||||
|
std::slice::from_ref(&canonical),
|
||||||
|
&IndexOptions::default(),
|
||||||
|
&AtomicBool::new(false),
|
||||||
|
)?;
|
||||||
|
assert!(index.search_with_options("", 1, &options)?.is_empty());
|
||||||
|
assert!(index
|
||||||
|
.set_structured_media_metadata_at_revision(
|
||||||
|
&path_string,
|
||||||
|
&tags,
|
||||||
|
1000,
|
||||||
|
identity.bytes,
|
||||||
|
identity.modified_ns,
|
||||||
|
revision
|
||||||
|
)
|
||||||
|
.is_err());
|
||||||
|
let revision =
|
||||||
|
index.media_change_revision(&path_string, identity.bytes, identity.modified_ns)?;
|
||||||
|
index.set_structured_media_metadata_at_revision(
|
||||||
|
&path_string,
|
||||||
|
&tags,
|
||||||
|
1000,
|
||||||
|
identity.bytes,
|
||||||
|
identity.modified_ns,
|
||||||
|
revision,
|
||||||
|
)?;
|
||||||
|
assert_eq!(index.search_with_options("", 1, &options)?.len(), 1);
|
||||||
|
index.set_media_metadata(
|
||||||
|
&path_string,
|
||||||
|
"artist: Björk",
|
||||||
|
1000,
|
||||||
|
identity.bytes,
|
||||||
|
identity.modified_ns,
|
||||||
|
)?;
|
||||||
|
assert!(index.search_with_options("", 1, &options)?.is_empty());
|
||||||
|
assert_eq!(index.search("Björk", 1)?.len(), 1);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
#[test]
|
#[test]
|
||||||
fn measured_search_preserves_results_and_contains_private_free_nested_spans(
|
fn measured_search_preserves_results_and_contains_private_free_nested_spans(
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
|
|||||||
@@ -447,7 +447,7 @@ fn schema_twelve_migrates_change_revision_without_reextracting_records() -> Resu
|
|||||||
let connection = rusqlite::Connection::open(&db)?;
|
let connection = rusqlite::Connection::open(&db)?;
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
connection.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))?,
|
connection.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))?,
|
||||||
13
|
14
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
connection.query_row("SELECT change_revision FROM files", [], |row| row
|
connection.query_row("SELECT change_revision FROM files", [], |row| row
|
||||||
|
|||||||
@@ -41,3 +41,9 @@ Audio timestamps represent segment starts, not exact word locations. Smaller
|
|||||||
windows alone have not demonstrated accurate spoken navigation. No-answer
|
windows alone have not demonstrated accurate spoken navigation. No-answer
|
||||||
calibration, music discrimination, exact spoken positions, process isolation,
|
calibration, music discrimination, exact spoken positions, process isolation,
|
||||||
unchanged-vector reuse and production platform checks remain release gates.
|
unchanged-vector reuse and production platform checks remain release gates.
|
||||||
|
|
||||||
|
Exact artist/title constraints use the core's normalized structured probe keys
|
||||||
|
before ANN limits and in the eligibility cache key. Metadata-only writes advance
|
||||||
|
the corpus revision and dirty path so warmed readers cannot retain old eligibility.
|
||||||
|
Empty metadata-only hybrid requests use core results without model inference.
|
||||||
|
These explicit constraints do not establish abstention for free-form topic queries.
|
||||||
@@ -513,7 +513,7 @@ impl HybridSearch {
|
|||||||
.lexical
|
.lexical
|
||||||
.search_with_options_traced(query, candidates, options, trace, parent)?;
|
.search_with_options_traced(query, candidates, options, trace, parent)?;
|
||||||
if query.trim().is_empty() || limit == 0 {
|
if query.trim().is_empty() || limit == 0 {
|
||||||
return Ok(Vec::new());
|
return Ok(lexical.into_iter().take(limit.min(1000)).collect());
|
||||||
}
|
}
|
||||||
trace.timed("hybrid.identifier_priority", parent, |_, _| {
|
trace.timed("hybrid.identifier_priority", parent, |_, _| {
|
||||||
promote_identifiers(query.trim(), &mut lexical)
|
promote_identifiers(query.trim(), &mut lexical)
|
||||||
@@ -776,6 +776,24 @@ mod tests {
|
|||||||
let directory = tempfile::tempdir()?;
|
let directory = tempfile::tempdir()?;
|
||||||
let db = directory.path().join("index.db");
|
let db = directory.path().join("index.db");
|
||||||
let mut service = HybridSearch::with_provider(&db, Box::new(FailingProvider))?;
|
let mut service = HybridSearch::with_provider(&db, Box::new(FailingProvider))?;
|
||||||
|
// The failure-only provider proves metadata-only searches never require inference.
|
||||||
|
let metadata = service.search_measured(
|
||||||
|
"",
|
||||||
|
5,
|
||||||
|
&SearchOptions {
|
||||||
|
media_artist: Some("Absent Artist".into()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
)?;
|
||||||
|
assert!(metadata.results.is_empty());
|
||||||
|
assert!(!metadata
|
||||||
|
.metrics
|
||||||
|
.trace
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.any(|span| span.name == "semantic.query_embedding"));
|
||||||
|
|
||||||
let mut trace = TraceCollector::new(MetricsMode::Debug);
|
let mut trace = TraceCollector::new(MetricsMode::Debug);
|
||||||
assert!(service
|
assert!(service
|
||||||
.search_traced(
|
.search_traced(
|
||||||
|
|||||||
@@ -190,15 +190,9 @@ impl HybridSearch {
|
|||||||
metadata: &MediaProbe,
|
metadata: &MediaProbe,
|
||||||
change_revision: i64,
|
change_revision: i64,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let tags = metadata
|
self.lexical.set_structured_media_metadata_at_revision(
|
||||||
.tags
|
|
||||||
.iter()
|
|
||||||
.map(|(k, v)| format!("{k}: {v}"))
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.join("\n");
|
|
||||||
self.lexical.set_media_metadata_at_revision(
|
|
||||||
&path.to_string_lossy(),
|
&path.to_string_lossy(),
|
||||||
&tags,
|
&metadata.tags,
|
||||||
metadata.duration_ms,
|
metadata.duration_ms,
|
||||||
identity.bytes,
|
identity.bytes,
|
||||||
identity.modified_ns,
|
identity.modified_ns,
|
||||||
|
|||||||
@@ -31,6 +31,8 @@ struct FilterKey {
|
|||||||
max_duration: Option<u32>,
|
max_duration: Option<u32>,
|
||||||
source_type: Option<String>,
|
source_type: Option<String>,
|
||||||
code_only: bool,
|
code_only: bool,
|
||||||
|
media_artist: Option<String>,
|
||||||
|
media_title: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct EligibilityCache {
|
struct EligibilityCache {
|
||||||
@@ -184,6 +186,17 @@ impl Store {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
initialization.execute_batch("CREATE TABLE IF NOT EXISTS semantic_media_errors(generation INTEGER NOT NULL REFERENCES semantic_generations(id) ON DELETE CASCADE,path TEXT NOT NULL,message TEXT NOT NULL,PRIMARY KEY(generation,path))")?;
|
initialization.execute_batch("CREATE TABLE IF NOT EXISTS semantic_media_errors(generation INTEGER NOT NULL REFERENCES semantic_generations(id) ON DELETE CASCADE,path TEXT NOT NULL,message TEXT NOT NULL,PRIMARY KEY(generation,path))")?;
|
||||||
|
// Metadata-only writes change strict eligibility even when file bytes do not change.
|
||||||
|
for (name, event, row) in [
|
||||||
|
("insert", "INSERT", "new"),
|
||||||
|
("update", "UPDATE", "new"),
|
||||||
|
("delete", "DELETE", "old"),
|
||||||
|
] {
|
||||||
|
initialization.execute_batch(&format!("CREATE TRIGGER IF NOT EXISTS semantic_media_metadata_{name} AFTER {event} ON media_metadata BEGIN
|
||||||
|
UPDATE semantic_meta SET corpus_revision=corpus_revision+1 WHERE singleton=1;
|
||||||
|
INSERT INTO semantic_dirty_paths(path,revision) VALUES({row}.path,(SELECT corpus_revision FROM semantic_meta WHERE singleton=1))
|
||||||
|
ON CONFLICT(path) DO UPDATE SET revision=excluded.revision; END;"))?;
|
||||||
|
}
|
||||||
incremental::migrate(&initialization)?;
|
incremental::migrate(&initialization)?;
|
||||||
initialization.commit()?;
|
initialization.commit()?;
|
||||||
let canonical = db.canonicalize()?;
|
let canonical = db.canonicalize()?;
|
||||||
@@ -644,6 +657,16 @@ impl Store {
|
|||||||
max_duration: options.max_duration_ms,
|
max_duration: options.max_duration_ms,
|
||||||
source_type: source_type.map(str::to_owned),
|
source_type: source_type.map(str::to_owned),
|
||||||
code_only: options.code_only,
|
code_only: options.code_only,
|
||||||
|
media_artist: options
|
||||||
|
.media_artist
|
||||||
|
.as_deref()
|
||||||
|
.map(searchi_core::normalize_media_field)
|
||||||
|
.transpose()?,
|
||||||
|
media_title: options
|
||||||
|
.media_title
|
||||||
|
.as_deref()
|
||||||
|
.map(searchi_core::normalize_media_field)
|
||||||
|
.transpose()?,
|
||||||
};
|
};
|
||||||
trace.end(filter_span, Default::default());
|
trace.end(filter_span, Default::default());
|
||||||
let eligibility_span = trace.begin("semantic.eligibility", parent);
|
let eligibility_span = trace.begin("semantic.eligibility", parent);
|
||||||
@@ -883,6 +906,8 @@ impl Store {
|
|||||||
AND (?3 IS NULL OR f.extension=?3)
|
AND (?3 IS NULL OR f.extension=?3)
|
||||||
AND (?4 IS NULL OR f.size>=?4) AND (?5 IS NULL OR f.size<=?5)
|
AND (?4 IS NULL OR f.size>=?4) AND (?5 IS NULL OR f.size<=?5)
|
||||||
AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7)
|
AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7)
|
||||||
|
AND (?9 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.artist_key=?9))
|
||||||
|
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.title_key=?10))
|
||||||
ORDER BY c.passage_id,c.ordinal LIMIT ?8")?;
|
ORDER BY c.passage_id,c.ordinal LIMIT ?8")?;
|
||||||
// Fetch one extra to detect the baseline cap instead of silently hiding
|
// Fetch one extra to detect the baseline cap instead of silently hiding
|
||||||
// candidates. Metadata filtering precedes this cap and all top-k limits.
|
// candidates. Metadata filtering precedes this cap and all top-k limits.
|
||||||
@@ -894,7 +919,17 @@ impl Store {
|
|||||||
max_size,
|
max_size,
|
||||||
options.modified_after_unix_ms,
|
options.modified_after_unix_ms,
|
||||||
options.modified_before_unix_ms,
|
options.modified_before_unix_ms,
|
||||||
MAX_SCAN_CHUNKS as i64 + 1
|
MAX_SCAN_CHUNKS as i64 + 1,
|
||||||
|
options
|
||||||
|
.media_artist
|
||||||
|
.as_deref()
|
||||||
|
.map(searchi_core::normalize_media_field)
|
||||||
|
.transpose()?,
|
||||||
|
options
|
||||||
|
.media_title
|
||||||
|
.as_deref()
|
||||||
|
.map(searchi_core::normalize_media_field)
|
||||||
|
.transpose()?
|
||||||
])?;
|
])?;
|
||||||
let mut hashes = HashMap::<i64, String>::new();
|
let mut hashes = HashMap::<i64, String>::new();
|
||||||
let mut best = HashMap::<String, SearchResult>::new();
|
let mut best = HashMap::<String, SearchResult>::new();
|
||||||
@@ -1008,6 +1043,8 @@ fn collect_eligibility(
|
|||||||
AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7)
|
AND (?6 IS NULL OR f.mtime/1000000>=?6) AND (?7 IS NULL OR f.mtime/1000000<=?7)
|
||||||
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms>=?10))
|
AND (?10 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms>=?10))
|
||||||
AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms<=?11))
|
AND (?11 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.duration_ms<=?11))
|
||||||
|
AND (?13 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.artist_key=?13))
|
||||||
|
AND (?14 IS NULL OR EXISTS(SELECT 1 FROM media_metadata m WHERE m.path=f.path AND m.title_key=?14))
|
||||||
AND (?12 IS NULL OR c.source_type=?12)
|
AND (?12 IS NULL OR c.source_type=?12)
|
||||||
AND (?9 IS NULL OR c.path=?9) {code} ORDER BY c.passage_id,c.ordinal LIMIT ?8"))?;
|
AND (?9 IS NULL OR c.path=?9) {code} ORDER BY c.passage_id,c.ordinal LIMIT ?8"))?;
|
||||||
let mut rows = statement.query(params![
|
let mut rows = statement.query(params![
|
||||||
@@ -1022,7 +1059,9 @@ fn collect_eligibility(
|
|||||||
path,
|
path,
|
||||||
filter.min_duration,
|
filter.min_duration,
|
||||||
filter.max_duration,
|
filter.max_duration,
|
||||||
filter.source_type
|
filter.source_type,
|
||||||
|
filter.media_artist,
|
||||||
|
filter.media_title
|
||||||
])?;
|
])?;
|
||||||
let mut hashes = HashMap::<i64, String>::new();
|
let mut hashes = HashMap::<i64, String>::new();
|
||||||
let mut keys = HashSet::new();
|
let mut keys = HashSet::new();
|
||||||
@@ -1293,6 +1332,108 @@ mod tests {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exact_metadata_filters_apply_before_ann_and_invalidate_warmed_eligibility() -> Result<()> {
|
||||||
|
let dir = tempfile::tempdir()?;
|
||||||
|
let root = dir.path().join("media");
|
||||||
|
std::fs::create_dir(&root)?;
|
||||||
|
std::fs::write(root.join("first.ogg"), b"metadata ANN eligibility fixture")?;
|
||||||
|
std::fs::write(root.join("other.ogg"), b"metadata distractor fixture")?;
|
||||||
|
let db = dir.path().join("index.db");
|
||||||
|
let mut lexical = SearchIndex::open(&db)?;
|
||||||
|
lexical.index_folder(&root)?;
|
||||||
|
let mut store = Store::open(&db, DIMENSIONS)?;
|
||||||
|
let mut files = store.media_batch(0)?;
|
||||||
|
files.sort_by(|a, b| a.1.cmp(&b.1));
|
||||||
|
let (other_id, other_path, _, _) = files.pop().context("distractor fixture")?;
|
||||||
|
let other_identity = searchi_media::MediaIdentity::read(Path::new(&other_path), 1024)?;
|
||||||
|
let (id, path, _, _) = files.remove(0);
|
||||||
|
let identity = searchi_media::MediaIdentity::read(Path::new(&path), 1024)?;
|
||||||
|
let revision =
|
||||||
|
lexical.media_change_revision(&path, identity.bytes, identity.modified_ns)?;
|
||||||
|
let mut tags = std::collections::BTreeMap::from([
|
||||||
|
("artist".into(), "Björk".into()),
|
||||||
|
("title".into(), "Jóga".into()),
|
||||||
|
]);
|
||||||
|
lexical.set_structured_media_metadata_at_revision(
|
||||||
|
&path,
|
||||||
|
&tags,
|
||||||
|
1000,
|
||||||
|
identity.bytes,
|
||||||
|
identity.modified_ns,
|
||||||
|
revision,
|
||||||
|
)?;
|
||||||
|
// Deterministic geometry tests storage/filter behavior, not model quality.
|
||||||
|
let mut vector = vec![0.; DIMENSIONS];
|
||||||
|
vector[0] = 1.;
|
||||||
|
let generation = store.begin("metadata-filter-proof")?;
|
||||||
|
let mut weaker = vector.clone();
|
||||||
|
weaker[0] = 0.8;
|
||||||
|
weaker[1] = 0.6;
|
||||||
|
store.insert_media(generation, id, 0, &identity, "audio", 0, &weaker)?;
|
||||||
|
store.insert_media(
|
||||||
|
generation,
|
||||||
|
other_id,
|
||||||
|
0,
|
||||||
|
&other_identity,
|
||||||
|
"audio",
|
||||||
|
0,
|
||||||
|
&vector,
|
||||||
|
)?;
|
||||||
|
store.activate(generation)?;
|
||||||
|
let mut options = SearchOptions {
|
||||||
|
media_artist: Some("BJÖRK".into()),
|
||||||
|
media_title: Some("JÓGA".into()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.search(&vector, "metadata-filter-proof", 1, &options)?
|
||||||
|
.len(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
options.media_artist = Some("Absent".into());
|
||||||
|
assert!(store
|
||||||
|
.search(&vector, "metadata-filter-proof", 1, &options)?
|
||||||
|
.is_empty());
|
||||||
|
options.media_artist = Some("BJÖRK".into());
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.search(&vector, "metadata-filter-proof", 1, &options)?
|
||||||
|
.len(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
tags.insert("artist".into(), "Replacement".into());
|
||||||
|
lexical.set_structured_media_metadata_at_revision(
|
||||||
|
&path,
|
||||||
|
&tags,
|
||||||
|
1000,
|
||||||
|
identity.bytes,
|
||||||
|
identity.modified_ns,
|
||||||
|
revision,
|
||||||
|
)?;
|
||||||
|
assert!(store
|
||||||
|
.search(&vector, "metadata-filter-proof", 1, &options)?
|
||||||
|
.is_empty());
|
||||||
|
options.media_artist = Some("Replacement".into());
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.search(&vector, "metadata-filter-proof", 1, &options)?
|
||||||
|
.len(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
lexical.index_paths_cancellable(
|
||||||
|
&root,
|
||||||
|
&[std::path::PathBuf::from(&path)],
|
||||||
|
&searchi_core::IndexOptions::default(),
|
||||||
|
&std::sync::atomic::AtomicBool::new(false),
|
||||||
|
)?;
|
||||||
|
assert!(store
|
||||||
|
.search(&vector, "metadata-filter-proof", 1, &options)?
|
||||||
|
.is_empty());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn media_origins_persist_filters_and_changes_remove_stale_candidates() -> Result<()> {
|
fn media_origins_persist_filters_and_changes_remove_stale_candidates() -> Result<()> {
|
||||||
let directory = tempfile::tempdir()?;
|
let directory = tempfile::tempdir()?;
|
||||||
|
|||||||
Reference in new issue
Block a user