Add local spoken passages with Google retrieval and isolated runtime setup
This commit is contained in:
karl committed
1 parent
bc9459785d
commit
dab703413e
55 files changed
+3633
-70
No files matched your search
@@ -11,6 +11,7 @@ serde.workspace = true
|
||||
searchi-core = { path = "../searchi-core" }
|
||||
searchi-semantic = { path = "../searchi-semantic" }
|
||||
searchi-media = { path = "../searchi-media" }
|
||||
searchi-speech = { path = "../searchi-speech" }
|
||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
sha2 = "0.10"
|
||||
fs2 = "0.4"
|
||||
|
||||
@@ -48,3 +48,48 @@ a separate durable metadata epoch so warmed readers cannot retain old eligibilit
|
||||
They do not change source-corpus revisions during the indexer's own probe publication.
|
||||
Empty metadata-only hybrid requests use core results without model inference.
|
||||
These explicit constraints do not establish abstention for free-form topic queries.
|
||||
|
||||
## Optional speech evidence
|
||||
|
||||
Attach a verified `searchi_speech::SpeechWorker` through `with_speech_provider`.
|
||||
`BudgetedSpeechProvider` admits one background thread per contiguous window;
|
||||
its admission time remains separate from native VAD/ASR/load/transport metrics.
|
||||
Aborted acceleration and CPU retry cost is aggregated separately as `fallback_us`.
|
||||
Recognition consumes at most thirty seconds of mono 16-kHz PCM, with two-second
|
||||
window overlap and an explicit 1024-window source ceiling. Four-hour default
|
||||
media fits the contiguous policy. Recognition never receives reference captions.
|
||||
|
||||
Speech uses `decode_audio_aligned`: source-frame timestamps preserve positive
|
||||
track origins, internal gaps, seeks and tails. Its CPU FFmpeg regression compares
|
||||
actual samples against known fixture clocks. Raw Google audio preprocessing is
|
||||
unchanged. DTW first/last point spans may have equal start/end; no invented duration
|
||||
is added. These functional contracts do not prove general timestamp accuracy.
|
||||
|
||||
Raw media, speech and document passages have independent completed source tasks.
|
||||
Adding/upgrading speech or retrying its failures reuses compatible verified raw
|
||||
vectors. Speech task identity includes native recognition assets/policy, aligned
|
||||
decoder policy, input SHA-256, title, maximum duration and maximum file bytes;
|
||||
raw sampling changes do not force speech recognition. Failed speech tasks publish
|
||||
explicit partial coverage while retaining raw evidence, and remain retryable.
|
||||
A genuine Google inference failure still aborts semantic publication.
|
||||
|
||||
`SpeechProvider` returns bounded recognized passages and numeric metrics. Google
|
||||
EmbeddingGemma 2 alone embeds those passages. Spoken-intent requests prefer
|
||||
`semantic_speech`, falling back to raw audio; music and sound requests use audio.
|
||||
Source locations retain `timestamp_ms`; recognized snippets are identified as
|
||||
`local_speech_recognition`, not exact document bytes. `speech_excerpt_matching`
|
||||
binds an issued result's recognized text when timestamps repeat, verifies current
|
||||
media SHA-256/revision and serves only active indexed evidence.
|
||||
|
||||
`speech_policy_key` and `EmbeddingStatus.corpus_revision` provide the stable
|
||||
(policy, source revision) automatic-retry key. Generation IDs and metadata-only
|
||||
updates must not trigger an endless retry loop. Status includes active speech
|
||||
policy and bounded explicit failures. Native interaction, full speech quality,
|
||||
other-platform execution and GPU evidence remain separate release gates.
|
||||
|
||||
Actual ASR/Google integration is an explicit ignored test:
|
||||
`cargo test -p searchi-service --test speech_google_cpu -- --ignored --nocapture --test-threads=1`.
|
||||
Provide verified `SEARCHI_SPEECH_WORKER`, `SEARCHI_SPEECH_BASE`, `SEARCHI_SPEECH_VAD`,
|
||||
Google model/worker variables, CPU FFmpeg/FFprobe helpers and a real short WAV in
|
||||
`SEARCHI_TEST_SPEECH_AUDIO`; Windows VM execution requires `SEARCHI_FORCE_CPU=1`.
|
||||
This tests publication/reuse and evidence binding, not independent relevance.
|
||||
@@ -2,9 +2,12 @@
|
||||
//! filtered HNSW retrieval. Production-scale validation remains a release gate.
|
||||
mod coordination;
|
||||
mod media;
|
||||
mod speech;
|
||||
pub use coordination::{
|
||||
BudgetedProvider, InferenceBudget, InferencePermit, InferenceRole, PendingForeground,
|
||||
};
|
||||
pub use searchi_speech::{SpeechMetrics, SpeechPassage, SpeechTranscription};
|
||||
pub use speech::{speech_policy_key, BudgetedSpeechProvider, SpeechExcerpt, SpeechProvider};
|
||||
mod prepared;
|
||||
pub use prepared::PreparedGraphs;
|
||||
mod pipeline;
|
||||
@@ -39,7 +42,23 @@ fn generation_fingerprint(model: &str) -> String {
|
||||
)
|
||||
}
|
||||
|
||||
fn raw_media_policy(policy: &str) -> &str {
|
||||
if let Some((base, hash)) = policy.rsplit_once(":speech-v1-") {
|
||||
if hash.len() == 64 && hash.bytes().all(|byte| byte.is_ascii_hexdigit()) {
|
||||
return base;
|
||||
}
|
||||
}
|
||||
policy
|
||||
}
|
||||
|
||||
fn speech_component_policy(policy: &str) -> String {
|
||||
let raw = raw_media_policy(policy);
|
||||
format!("{}{}", policy_base(policy), &policy[raw.len()..])
|
||||
}
|
||||
|
||||
fn reusable_policy(expected: &str, active: &str) -> bool {
|
||||
let expected = raw_media_policy(expected);
|
||||
let active = raw_media_policy(active);
|
||||
if active == expected || active == format!("{expected}:text-only") {
|
||||
return true;
|
||||
}
|
||||
@@ -56,6 +75,7 @@ fn reusable_policy(expected: &str, active: &str) -> bool {
|
||||
/// Resolve compatible publication coverage without broadening model or window
|
||||
/// policy compatibility. A retained reader can start under either coverage.
|
||||
fn policy_base(policy: &str) -> &str {
|
||||
let policy = raw_media_policy(policy);
|
||||
if let Some(base) = policy.strip_suffix(":text-only") {
|
||||
return base;
|
||||
}
|
||||
@@ -213,12 +233,21 @@ pub struct EmbeddingReport {
|
||||
pub image_inference_calls: usize,
|
||||
pub audio_inference_calls: usize,
|
||||
pub resumed: bool,
|
||||
pub speech_requested: bool,
|
||||
pub speech_windows: usize,
|
||||
pub speech_chunks: usize,
|
||||
pub speech_reused_sources: usize,
|
||||
pub speech_failures: Vec<MediaFailure>,
|
||||
pub speech_metrics: std::collections::BTreeMap<String, u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct EmbeddingStatus {
|
||||
/// Coverage of a compatible, prepared active generation, if any.
|
||||
pub coverage: Option<SemanticCoverage>,
|
||||
pub speech_policy: Option<String>,
|
||||
pub speech_failures: Vec<MediaFailure>,
|
||||
pub corpus_revision: i64,
|
||||
pub requested_coverage: SemanticCoverage,
|
||||
/// Early setup must preserve an existing compatible full generation.
|
||||
pub text_indexing_allowed: bool,
|
||||
@@ -241,6 +270,7 @@ enum QueryTask {
|
||||
pub struct HybridSearch {
|
||||
lexical: SearchIndex,
|
||||
embedder: Box<dyn EmbeddingProvider>,
|
||||
speech: Option<Box<dyn SpeechProvider>>,
|
||||
store: Store,
|
||||
generation_fingerprint: String,
|
||||
// Real provider outputs only, scoped to this model session. Queries never
|
||||
@@ -300,6 +330,9 @@ impl HybridSearch {
|
||||
if !exists {
|
||||
return Ok(EmbeddingStatus {
|
||||
coverage: None,
|
||||
speech_policy: None,
|
||||
speech_failures: Vec::new(),
|
||||
corpus_revision: 0,
|
||||
requested_coverage,
|
||||
text_indexing_allowed: true,
|
||||
failures: Vec::new(),
|
||||
@@ -318,6 +351,9 @@ impl HybridSearch {
|
||||
if version < 6 {
|
||||
return Ok(EmbeddingStatus {
|
||||
coverage: None,
|
||||
speech_policy: None,
|
||||
speech_failures: Vec::new(),
|
||||
corpus_revision: 0,
|
||||
requested_coverage,
|
||||
text_indexing_allowed: true,
|
||||
failures: Vec::new(),
|
||||
@@ -329,7 +365,7 @@ impl HybridSearch {
|
||||
});
|
||||
}
|
||||
ensure!(
|
||||
(6..=8).contains(&version),
|
||||
(6..=9).contains(&version),
|
||||
"Unsupported semantic schema version {version}"
|
||||
);
|
||||
let (generation,fingerprint,revision,current,dimensions):(Option<i64>,Option<String>,Option<i64>,i64,Option<usize>)=snapshot.query_row(
|
||||
@@ -349,6 +385,25 @@ impl HybridSearch {
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
let speech_policy = fingerprint
|
||||
.as_deref()
|
||||
.filter(|policy| raw_media_policy(policy) != *policy)
|
||||
.and_then(|policy| policy.rsplit_once(":speech-v1-"))
|
||||
.map(|(_, hash)| format!("speech-v1-{hash}"));
|
||||
let speech_failures = if version >= 8 {
|
||||
let mut speech_query=snapshot.prepare("SELECT path,message FROM semantic_tasks WHERE generation=?1 AND source_type='speech' AND state='failed' ORDER BY path LIMIT 1000")?;
|
||||
let failures = speech_query
|
||||
.query_map([generation], |row| {
|
||||
Ok(MediaFailure {
|
||||
path: row.get(0)?,
|
||||
message: row.get(1)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
failures
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let expected = generation_fingerprint(GEMMA_FINGERPRINT);
|
||||
let coverage = if dimensions == Some(GEMMA_DIMENSIONS) {
|
||||
fingerprint
|
||||
@@ -361,11 +416,16 @@ impl HybridSearch {
|
||||
let policy_matches = match requested_coverage {
|
||||
SemanticCoverage::TextOnly => prepared_coverage.is_some(),
|
||||
// Keep the default full-media status's exact configured policy rule.
|
||||
SemanticCoverage::FullMedia => fingerprint.as_deref() == Some(expected.as_str()),
|
||||
SemanticCoverage::FullMedia => {
|
||||
fingerprint.as_deref().map(raw_media_policy) == Some(expected.as_str())
|
||||
}
|
||||
};
|
||||
let stale = revision != Some(current) || !policy_matches;
|
||||
Ok(EmbeddingStatus {
|
||||
coverage: prepared_coverage,
|
||||
speech_policy,
|
||||
speech_failures,
|
||||
corpus_revision: current,
|
||||
requested_coverage,
|
||||
text_indexing_allowed: coverage != Some(SemanticCoverage::FullMedia),
|
||||
failures,
|
||||
@@ -416,6 +476,7 @@ impl HybridSearch {
|
||||
query_cache: VecDeque::new(),
|
||||
lexical,
|
||||
embedder,
|
||||
speech: None,
|
||||
store,
|
||||
generation_fingerprint,
|
||||
})
|
||||
|
||||
@@ -4,7 +4,7 @@ use searchi_media::*;
|
||||
use store::SourceTask;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct InferenceFailure(anyhow::Error);
|
||||
pub(super) struct InferenceFailure(pub(super) anyhow::Error);
|
||||
impl std::fmt::Display for InferenceFailure {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "Media inference failed: {}", self.0)
|
||||
@@ -42,6 +42,10 @@ impl HybridSearch {
|
||||
limits.max_file_bytes
|
||||
));
|
||||
}
|
||||
if let Some(provider) = &self.speech {
|
||||
fingerprint.push(':');
|
||||
fingerprint.push_str(&speech_policy_key(provider.fingerprint(), limits)?);
|
||||
}
|
||||
self.rebuild_generation(fingerprint, Some(limits), |service, generation, work| {
|
||||
service.build(generation, work)?;
|
||||
let mut after = 0;
|
||||
@@ -80,11 +84,12 @@ impl HybridSearch {
|
||||
.map(|i| i.sha256.clone())
|
||||
.unwrap_or_default(),
|
||||
title_hash: String::new(),
|
||||
policy: service.generation_fingerprint.clone(),
|
||||
policy: raw_media_policy(&service.generation_fingerprint).into(),
|
||||
passage_id: -id,
|
||||
};
|
||||
work.track(&task)?;
|
||||
service.store.stage_task(generation, &task)?;
|
||||
let speech_identity = identity.as_ref().ok().cloned();
|
||||
let result = identity.and_then(|identity| {
|
||||
service.build_media_source(
|
||||
generation, id, path, kind, tools, limits, &identity, &task, work,
|
||||
@@ -93,7 +98,16 @@ impl HybridSearch {
|
||||
Ok(())
|
||||
});
|
||||
match result {
|
||||
Ok(()) => work.report.media_files += 1,
|
||||
Ok(()) => {
|
||||
work.report.media_files += 1;
|
||||
if service.speech.is_some() && kind != MediaKind::Image {
|
||||
if let Some(identity) = speech_identity {
|
||||
service.build_speech_source(
|
||||
generation, id, path, tools, limits, &identity, work,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
if let Err(cleanup) =
|
||||
service
|
||||
|
||||
@@ -101,6 +101,12 @@ impl HybridSearch {
|
||||
image_inference_calls: 0,
|
||||
audio_inference_calls: 0,
|
||||
resumed,
|
||||
speech_requested: self.speech.is_some() && media_limits.is_some(),
|
||||
speech_windows: 0,
|
||||
speech_chunks: 0,
|
||||
speech_reused_sources: 0,
|
||||
speech_failures: Vec::new(),
|
||||
speech_metrics: Default::default(),
|
||||
},
|
||||
keys: Vec::new(),
|
||||
media: Vec::new(),
|
||||
@@ -197,13 +203,28 @@ impl HybridSearch {
|
||||
);
|
||||
tasks.push(SourceTask {
|
||||
key: task_key(&["media", &path]),
|
||||
path,
|
||||
path: path.clone(),
|
||||
source_type: "media".into(),
|
||||
source_hash: identity.map(|identity| identity.sha256).unwrap_or_default(),
|
||||
source_hash: identity
|
||||
.as_ref()
|
||||
.map(|identity| identity.sha256.clone())
|
||||
.unwrap_or_default(),
|
||||
title_hash: String::new(),
|
||||
policy: self.generation_fingerprint.clone(),
|
||||
policy: raw_media_policy(&self.generation_fingerprint).into(),
|
||||
passage_id: -id,
|
||||
});
|
||||
if self.speech.is_some() && kind != searchi_media::MediaKind::Image {
|
||||
ensure!(
|
||||
tasks.len() < MAX_SCAN_CHUNKS,
|
||||
"Semantic source task budget exceeded"
|
||||
);
|
||||
tasks.push(super::speech::speech_task(
|
||||
&path,
|
||||
id,
|
||||
identity.as_ref().ok(),
|
||||
&self.generation_fingerprint,
|
||||
)?);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ pub enum SemanticOrigin {
|
||||
Image,
|
||||
Video,
|
||||
Audio,
|
||||
Speech,
|
||||
}
|
||||
impl SemanticOrigin {
|
||||
fn source_type(self) -> Option<&'static str> {
|
||||
@@ -17,6 +18,7 @@ impl SemanticOrigin {
|
||||
Self::Image => Some("image"),
|
||||
Self::Video => Some("video"),
|
||||
Self::Audio => Some("audio"),
|
||||
Self::Speech => Some("speech"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -59,32 +61,15 @@ fn preferred_origin(query: &str) -> Option<SemanticOrigin> {
|
||||
return None;
|
||||
}
|
||||
if has(&[
|
||||
"audio",
|
||||
"spoken",
|
||||
"speech",
|
||||
"recording",
|
||||
"conversation",
|
||||
"interview",
|
||||
"podcast",
|
||||
"sound",
|
||||
"music",
|
||||
"gesprochen",
|
||||
"gespräch",
|
||||
"gespraech",
|
||||
"geräusch",
|
||||
"geraeusch",
|
||||
"musik",
|
||||
"hörbar",
|
||||
"hörbarer",
|
||||
"hörbare",
|
||||
"hoerbar",
|
||||
"hoerbarer",
|
||||
"hoerbare",
|
||||
"tonaufnahme",
|
||||
]) {
|
||||
return Some(SemanticOrigin::Audio);
|
||||
}
|
||||
if (has(&["someone", "speaker"])
|
||||
]) || (has(&["someone", "speaker"])
|
||||
&& has(&[
|
||||
"mentions",
|
||||
"mentioned",
|
||||
@@ -96,6 +81,24 @@ fn preferred_origin(query: &str) -> Option<SemanticOrigin> {
|
||||
|| (has(&["jemand", "sprecher", "sprecherin"])
|
||||
&& has(&["erwähnt", "erwaehnt", "sagt", "gesagt", "spricht"]))
|
||||
{
|
||||
return Some(SemanticOrigin::Speech);
|
||||
}
|
||||
if has(&[
|
||||
"audio",
|
||||
"recording",
|
||||
"sound",
|
||||
"music",
|
||||
"geräusch",
|
||||
"geraeusch",
|
||||
"musik",
|
||||
"hörbar",
|
||||
"hörbarer",
|
||||
"hörbare",
|
||||
"hoerbar",
|
||||
"hoerbarer",
|
||||
"hoerbare",
|
||||
"tonaufnahme",
|
||||
]) {
|
||||
return Some(SemanticOrigin::Audio);
|
||||
}
|
||||
None
|
||||
@@ -140,6 +143,20 @@ impl HybridSearch {
|
||||
if !preferred.is_empty() {
|
||||
return Ok(preferred);
|
||||
}
|
||||
if origin == SemanticOrigin::Speech {
|
||||
let raw = self.store.search_origin_traced(
|
||||
vector,
|
||||
&self.generation_fingerprint,
|
||||
limit,
|
||||
options,
|
||||
Some("audio"),
|
||||
trace,
|
||||
parent,
|
||||
)?;
|
||||
if !raw.is_empty() {
|
||||
return Ok(raw);
|
||||
}
|
||||
}
|
||||
}
|
||||
trace.counter(parent, "general_lane", 1);
|
||||
self.store.search_origin_traced(
|
||||
@@ -191,17 +208,44 @@ mod tests {
|
||||
));
|
||||
for active in ["other-model:pinned-media:preprocessing-v1:custom-240-2000-120-10000-2000-1800-14400000-17179869184","pinned-model:pinned-media:preprocessing-v2:custom-240-2000-120-10000-2000-1800-14400000-17179869184","pinned-model:pinned-media:preprocessing-v1:custom-unknown"] {assert!(!reusable_policy(expected,active));}
|
||||
}
|
||||
#[test]
|
||||
fn speech_reader_policy_preserves_exact_raw_and_text_components() {
|
||||
let raw = "google:pinned-media:windows-v1";
|
||||
let speech = format!("{raw}:speech-v1-{}", "a".repeat(64));
|
||||
assert!(reusable_policy(raw, &speech));
|
||||
assert_eq!(raw_media_policy(&speech), raw);
|
||||
assert_eq!(policy_base(&speech), raw);
|
||||
assert!(!reusable_policy(
|
||||
"different-google:pinned-media:windows-v1",
|
||||
&speech
|
||||
));
|
||||
assert!(!reusable_policy(
|
||||
raw,
|
||||
&format!("{raw}:speech-v1-not-a-valid-hash")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_spoken_intent_is_conservative_and_bilingual() {
|
||||
for query in [
|
||||
"where someone mentions a contract",
|
||||
"a speaker says the date",
|
||||
"spoken discussion of ecology",
|
||||
"audio of a waterfall",
|
||||
"jemand erwähnt das Budget",
|
||||
"Wo spricht jemand über Kündigung?",
|
||||
"Gespräch über die neue Wohnung",
|
||||
] {
|
||||
assert_eq!(
|
||||
preferred_origin(query),
|
||||
Some(SemanticOrigin::Speech),
|
||||
"{query}"
|
||||
);
|
||||
}
|
||||
for query in [
|
||||
"audio of a waterfall",
|
||||
"hörbarer Regen",
|
||||
"music with violin",
|
||||
"sound of thunder",
|
||||
] {
|
||||
assert_eq!(
|
||||
preferred_origin(query),
|
||||
|
||||
@@ -0,0 +1,421 @@
|
||||
//! Optional local speech evidence. Google remains the only embedding provider.
|
||||
use super::*;
|
||||
use searchi_core::SourceLocation;
|
||||
use searchi_media::{decode_audio_aligned, probe, AudioWindow, MediaIdentity};
|
||||
use store::SourceTask;
|
||||
|
||||
const MAX_SPEECH_WINDOWS: usize = 1024;
|
||||
const MAX_SEGMENTS: usize = 64;
|
||||
const MAX_SEGMENT_BYTES: usize = 4096;
|
||||
const MAX_WINDOW_TEXT: usize = 64 * 1024;
|
||||
|
||||
/// Stable automatic-retry identity; excludes generation IDs and metadata epochs.
|
||||
pub fn speech_policy_key(provider_fingerprint: &str, limits: &MediaLimits) -> Result<String> {
|
||||
ensure!(
|
||||
!provider_fingerprint.is_empty() && provider_fingerprint.len() <= 4096,
|
||||
"Invalid speech provider fingerprint"
|
||||
);
|
||||
Ok(format!("speech-v1-{}",store::content_hash(&format!("{provider_fingerprint};speech-window=30000;overlap=2000;max-windows=1024;aligned-decoder-v1;max-duration={};max-bytes={}",limits.max_duration_ms,limits.max_file_bytes))))
|
||||
}
|
||||
|
||||
pub trait SpeechProvider: Send {
|
||||
/// Includes actual ASR/VAD/alignment assets and policy, not a display name alone.
|
||||
fn fingerprint(&self) -> &str;
|
||||
fn admission_us(&self) -> u64 {
|
||||
0
|
||||
}
|
||||
fn transcribe(&mut self, pcm: &[f32], sample_rate: u32) -> Result<SpeechTranscription>;
|
||||
}
|
||||
|
||||
impl SpeechProvider for searchi_speech::SpeechWorker {
|
||||
fn fingerprint(&self) -> &str {
|
||||
searchi_speech::SpeechWorker::fingerprint(self)
|
||||
}
|
||||
fn transcribe(&mut self, pcm: &[f32], sample_rate: u32) -> Result<SpeechTranscription> {
|
||||
searchi_speech::SpeechWorker::transcribe(self, pcm, sample_rate)
|
||||
}
|
||||
}
|
||||
|
||||
/// One background admission per bounded window; never claims native preemption.
|
||||
pub struct BudgetedSpeechProvider {
|
||||
inner: Box<dyn SpeechProvider>,
|
||||
budget: InferenceBudget,
|
||||
admission_us: u64,
|
||||
}
|
||||
impl BudgetedSpeechProvider {
|
||||
pub fn new(inner: Box<dyn SpeechProvider>, budget: InferenceBudget) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
budget,
|
||||
admission_us: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
impl SpeechProvider for BudgetedSpeechProvider {
|
||||
fn fingerprint(&self) -> &str {
|
||||
self.inner.fingerprint()
|
||||
}
|
||||
fn admission_us(&self) -> u64 {
|
||||
self.admission_us
|
||||
}
|
||||
fn transcribe(&mut self, pcm: &[f32], sample_rate: u32) -> Result<SpeechTranscription> {
|
||||
let start = std::time::Instant::now();
|
||||
let permit = self.budget.acquire(InferenceRole::Background, 1);
|
||||
self.admission_us = start.elapsed().as_micros().min(u64::MAX as u128) as u64;
|
||||
let _permit = permit?;
|
||||
self.inner.transcribe(pcm, sample_rate)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct SpeechExcerpt {
|
||||
pub path: String,
|
||||
pub filename: String,
|
||||
pub source: SourceLocation,
|
||||
pub end_ms: u32,
|
||||
pub text: String,
|
||||
pub provenance: &'static str,
|
||||
}
|
||||
|
||||
pub(super) fn speech_task(
|
||||
path: &str,
|
||||
id: i64,
|
||||
identity: Option<&MediaIdentity>,
|
||||
policy: &str,
|
||||
) -> Result<SourceTask> {
|
||||
let filename = Path::new(path)
|
||||
.file_name()
|
||||
.unwrap_or_default()
|
||||
.to_string_lossy();
|
||||
Ok(SourceTask {
|
||||
key: super::pipeline::task_key(&["speech", path]),
|
||||
path: path.into(),
|
||||
source_type: "speech".into(),
|
||||
source_hash: identity
|
||||
.map(|identity| identity.sha256.clone())
|
||||
.unwrap_or_default(),
|
||||
title_hash: store::content_hash(bounded_title(&filename)),
|
||||
policy: speech_component_policy(policy),
|
||||
passage_id: store::media_passage_id(id, "speech")?,
|
||||
})
|
||||
}
|
||||
|
||||
fn speech_windows(start_ms: u32, end_ms: u32) -> Result<Vec<AudioWindow>> {
|
||||
ensure!(start_ms <= end_ms, "Invalid speech audio range");
|
||||
let mut windows = Vec::new();
|
||||
let mut start = start_ms;
|
||||
while start < end_ms {
|
||||
ensure!(
|
||||
windows.len() < MAX_SPEECH_WINDOWS,
|
||||
"Speech exceeds 1024 contiguous windows; coverage is incomplete"
|
||||
);
|
||||
windows.push(AudioWindow {
|
||||
start_ms: start,
|
||||
duration_ms: (end_ms - start).min(30000),
|
||||
});
|
||||
if end_ms - start <= 30000 {
|
||||
break;
|
||||
}
|
||||
start = start
|
||||
.checked_add(28000)
|
||||
.ok_or_else(|| anyhow::anyhow!("Speech timestamp overflow"))?;
|
||||
}
|
||||
Ok(windows)
|
||||
}
|
||||
|
||||
fn validate_transcription(result: &SpeechTranscription, duration_ms: u32) -> Result<()> {
|
||||
ensure!(
|
||||
result.passages.len() <= MAX_SEGMENTS,
|
||||
"Speech window exceeds 64 passages"
|
||||
);
|
||||
let mut total = 0usize;
|
||||
for passage in &result.passages {
|
||||
ensure!(
|
||||
passage.start_ms <= passage.end_ms && passage.end_ms <= duration_ms,
|
||||
"Speech timestamp lies outside decoded window"
|
||||
);
|
||||
ensure!(
|
||||
!passage.text.trim().is_empty() && passage.text.len() <= MAX_SEGMENT_BYTES,
|
||||
"Speech passage text is empty or oversized"
|
||||
);
|
||||
total = total
|
||||
.checked_add(passage.text.len())
|
||||
.ok_or_else(|| anyhow::anyhow!("Speech text budget overflow"))?;
|
||||
ensure!(
|
||||
total <= MAX_WINDOW_TEXT,
|
||||
"Speech window exceeds 64 KiB text"
|
||||
);
|
||||
}
|
||||
for value in [
|
||||
result.metrics.vad_ms,
|
||||
result.metrics.asr_ms,
|
||||
result.metrics.load_ms,
|
||||
result.metrics.transport_ms,
|
||||
result.metrics.fallback_ms,
|
||||
] {
|
||||
ensure!(
|
||||
value.is_finite() && value >= 0. && value <= (u64::MAX / 1000) as f64,
|
||||
"Invalid speech timing metric"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl HybridSearch {
|
||||
pub fn with_speech_provider(mut self, provider: Box<dyn SpeechProvider>) -> Result<Self> {
|
||||
ensure!(
|
||||
!provider.fingerprint().is_empty() && provider.fingerprint().len() <= 4096,
|
||||
"Invalid speech provider fingerprint"
|
||||
);
|
||||
self.speech = Some(provider);
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
/// Only returns active, byte-verified indexed speech. Caller may further bind a source handle.
|
||||
pub fn speech_excerpt(
|
||||
&self,
|
||||
path: &str,
|
||||
start_ms: u32,
|
||||
max_chars: usize,
|
||||
) -> Result<Option<SpeechExcerpt>> {
|
||||
self.store.speech_excerpt(path, start_ms, max_chars)
|
||||
}
|
||||
|
||||
/// Bind issued search evidence when multiple recognized passages share a timestamp.
|
||||
pub fn speech_excerpt_matching(
|
||||
&self,
|
||||
path: &str,
|
||||
start_ms: u32,
|
||||
expected_text: &str,
|
||||
max_chars: usize,
|
||||
) -> Result<Option<SpeechExcerpt>> {
|
||||
self.store
|
||||
.speech_excerpt_matching(path, start_ms, Some(expected_text), max_chars)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) fn build_speech_source(
|
||||
&mut self,
|
||||
generation: i64,
|
||||
id: i64,
|
||||
path: &Path,
|
||||
tools: &FfmpegTools,
|
||||
limits: &MediaLimits,
|
||||
identity: &MediaIdentity,
|
||||
work: &mut super::pipeline::BuildWork,
|
||||
) -> Result<()> {
|
||||
let task = speech_task(
|
||||
&path.to_string_lossy(),
|
||||
id,
|
||||
Some(identity),
|
||||
&self.generation_fingerprint,
|
||||
)?;
|
||||
work.track(&task)?;
|
||||
self.store.stage_task(generation, &task)?;
|
||||
let before = work.report.chunks;
|
||||
let before_speech = work.report.speech_chunks;
|
||||
let result =
|
||||
self.encode_speech_source(generation, id, path, tools, limits, identity, &task, work);
|
||||
match result {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) => {
|
||||
self.store
|
||||
.fail_source(generation, &task, &format!("{error:#}"))?;
|
||||
work.report.chunks = before;
|
||||
work.report.speech_chunks = before_speech;
|
||||
if error
|
||||
.downcast_ref::<super::media::InferenceFailure>()
|
||||
.is_some()
|
||||
{
|
||||
return Err(error);
|
||||
}
|
||||
if work.report.speech_failures.len() < 1000 {
|
||||
work.report.speech_failures.push(MediaFailure {
|
||||
path: task.path,
|
||||
message: format!("{error:#}").chars().take(2048).collect(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn encode_speech_source(
|
||||
&mut self,
|
||||
generation: i64,
|
||||
id: i64,
|
||||
path: &Path,
|
||||
tools: &FfmpegTools,
|
||||
limits: &MediaLimits,
|
||||
identity: &MediaIdentity,
|
||||
task: &SourceTask,
|
||||
work: &mut super::pipeline::BuildWork,
|
||||
) -> Result<()> {
|
||||
let revision =
|
||||
self.lexical
|
||||
.media_change_revision(&task.path, identity.bytes, identity.modified_ns)?;
|
||||
if let Some(count) = self
|
||||
.store
|
||||
.reuse_media_source(generation, task, id, identity)?
|
||||
{
|
||||
ensure!(
|
||||
work.report.chunks + count <= MAX_SCAN_CHUNKS,
|
||||
"Speech exceeds generation chunk budget"
|
||||
);
|
||||
identity.ensure_unchanged()?;
|
||||
work.report.chunks += count;
|
||||
work.report.speech_chunks += count;
|
||||
work.report.reused_chunks += count;
|
||||
work.report.reused_sources += 1;
|
||||
work.report.speech_reused_sources += 1;
|
||||
return Ok(());
|
||||
}
|
||||
let metadata = probe(tools, path, limits)?;
|
||||
let filename = path.file_name().unwrap_or_default().to_string_lossy();
|
||||
let mut ordinal = 0usize;
|
||||
if metadata.has_audio {
|
||||
for window in speech_windows(metadata.audio_start_ms, metadata.audio_end_ms)? {
|
||||
let pcm = decode_audio_aligned(tools, path, window, limits)?;
|
||||
let provider = self
|
||||
.speech
|
||||
.as_mut()
|
||||
.ok_or_else(|| anyhow::anyhow!("Speech provider unavailable"))?;
|
||||
work.report.speech_windows += 1;
|
||||
let result = provider.transcribe(&pcm, 16000);
|
||||
let admission = work
|
||||
.report
|
||||
.speech_metrics
|
||||
.entry("admission_us".into())
|
||||
.or_default();
|
||||
*admission = admission.saturating_add(provider.admission_us());
|
||||
let transcription = result?;
|
||||
validate_transcription(&transcription, window.duration_ms)?;
|
||||
for (key, value) in [
|
||||
("vad_us", transcription.metrics.vad_ms),
|
||||
("asr_us", transcription.metrics.asr_ms),
|
||||
("load_us", transcription.metrics.load_ms),
|
||||
("transport_us", transcription.metrics.transport_ms),
|
||||
("fallback_us", transcription.metrics.fallback_ms),
|
||||
] {
|
||||
let total = work.report.speech_metrics.entry(key.into()).or_default();
|
||||
*total = total.saturating_add((value * 1000.) as u64);
|
||||
}
|
||||
for passage in transcription.passages {
|
||||
ensure!(
|
||||
work.report.chunks < MAX_SCAN_CHUNKS,
|
||||
"Speech exceeds generation chunk budget"
|
||||
);
|
||||
let start = window
|
||||
.start_ms
|
||||
.checked_add(passage.start_ms)
|
||||
.ok_or_else(|| anyhow::anyhow!("Speech timestamp overflow"))?;
|
||||
let end = window
|
||||
.start_ms
|
||||
.checked_add(passage.end_ms)
|
||||
.ok_or_else(|| anyhow::anyhow!("Speech timestamp overflow"))?;
|
||||
let windows = passage_windows(
|
||||
&mut *self.embedder,
|
||||
&passage.text,
|
||||
bounded_title(&filename),
|
||||
)?;
|
||||
for text_window in windows {
|
||||
ensure!(
|
||||
work.report.chunks < MAX_SCAN_CHUNKS,
|
||||
"Speech exceeds generation chunk budget"
|
||||
);
|
||||
let calls = work
|
||||
.report
|
||||
.speech_metrics
|
||||
.entry("google_embedding_calls".into())
|
||||
.or_default();
|
||||
*calls = calls.saturating_add(1);
|
||||
let vectors = self
|
||||
.embedder
|
||||
.embed_documents(&[&text_window.text], bounded_title(&filename))
|
||||
.map_err(|error| {
|
||||
anyhow::Error::new(super::media::InferenceFailure(error))
|
||||
})?;
|
||||
ensure!(vectors.len() == 1, "Speech embedding count mismatch");
|
||||
self.store.insert_speech(
|
||||
generation,
|
||||
id,
|
||||
ordinal,
|
||||
identity,
|
||||
revision,
|
||||
start,
|
||||
end,
|
||||
&text_window.text,
|
||||
&vectors[0],
|
||||
)?;
|
||||
ordinal += 1;
|
||||
work.report.chunks += 1;
|
||||
work.report.speech_chunks += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
identity.ensure_unchanged()?;
|
||||
ensure!(
|
||||
self.lexical
|
||||
.media_change_revision(&task.path, identity.bytes, identity.modified_ns)?
|
||||
== revision,
|
||||
"Speech source changed during recognition"
|
||||
);
|
||||
self.store.complete_source(generation, task, ordinal)?;
|
||||
work.report.encoded_sources += 1;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn point_spans_are_legal_and_invalid_native_evidence_is_rejected() -> Result<()> {
|
||||
let mut result = SpeechTranscription {
|
||||
passages: vec![SpeechPassage {
|
||||
start_ms: 0,
|
||||
end_ms: 0,
|
||||
text: "Recognized point".into(),
|
||||
}],
|
||||
metrics: SpeechMetrics::default(),
|
||||
};
|
||||
validate_transcription(&result, 30000)?;
|
||||
result.passages[0].end_ms = 30001;
|
||||
assert!(validate_transcription(&result, 30000).is_err());
|
||||
result.passages[0].end_ms = 0;
|
||||
result.passages[0].text = "x".repeat(4097);
|
||||
assert!(validate_transcription(&result, 30000).is_err());
|
||||
result.passages[0].text = "valid".into();
|
||||
result.metrics.asr_ms = f64::NAN;
|
||||
assert!(validate_transcription(&result, 30000).is_err());
|
||||
result.metrics.asr_ms = 0.;
|
||||
result.metrics.fallback_ms = f64::INFINITY;
|
||||
assert!(validate_transcription(&result, 30000).is_err());
|
||||
result.metrics.fallback_ms = 12.5;
|
||||
validate_transcription(&result, 30000)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn contiguous_windows_cover_full_four_hours_and_overflow_is_explicit() -> Result<()> {
|
||||
let windows = speech_windows(0, 14400000)?;
|
||||
assert_eq!(windows[0].start_ms, 0);
|
||||
assert_eq!(
|
||||
windows.last().unwrap().start_ms + windows.last().unwrap().duration_ms,
|
||||
14400000
|
||||
);
|
||||
assert!(windows
|
||||
.windows(2)
|
||||
.all(|pair| pair[1].start_ms == pair[0].start_ms + 28000));
|
||||
assert!(speech_windows(0, u32::MAX).is_err());
|
||||
let offset = speech_windows(2000, 34000)?;
|
||||
assert_eq!(offset[0].start_ms, 2000);
|
||||
assert_eq!(
|
||||
offset.last().unwrap().start_ms + offset.last().unwrap().duration_ms,
|
||||
34000
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,19 @@ use usearch::{Index, IndexOptions, MetricKind, ScalarKind};
|
||||
pub(crate) const MAX_SCAN_CHUNKS: usize = 100_000;
|
||||
pub(crate) const MIN_COSINE: f32 = 0.25;
|
||||
|
||||
// Disjoint negative namespaces preserve existing document/raw-media IDs without a table rebuild.
|
||||
pub(crate) fn media_passage_id(file_id: i64, source_type: &str) -> Result<i64> {
|
||||
ensure!(
|
||||
file_id > 0 && file_id <= i64::MAX / 2,
|
||||
"Media ID exceeds supported namespace"
|
||||
);
|
||||
Ok(if source_type == "speech" {
|
||||
i64::MIN + file_id
|
||||
} else {
|
||||
-file_id
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
struct FilterKey {
|
||||
folder: Option<String>,
|
||||
@@ -123,7 +136,7 @@ impl Store {
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
ensure!(
|
||||
(1..=8).contains(&version),
|
||||
(1..=9).contains(&version),
|
||||
"Unsupported semantic schema version {version}"
|
||||
);
|
||||
initialization.execute_batch("
|
||||
@@ -187,6 +200,13 @@ impl Store {
|
||||
}
|
||||
}
|
||||
initialization.execute_batch("CREATE TABLE IF NOT EXISTS semantic_media_errors(generation INTEGER NOT NULL REFERENCES semantic_generations(id) ON DELETE CASCADE,path TEXT NOT NULL,message TEXT NOT NULL,PRIMARY KEY(generation,path))")?;
|
||||
let has_speech_end:bool=initialization.query_row("SELECT EXISTS(SELECT 1 FROM pragma_table_info('semantic_chunks') WHERE name='speech_end_ms')",[],|row|row.get(0))?;
|
||||
if !has_speech_end {
|
||||
initialization.execute(
|
||||
"ALTER TABLE semantic_chunks ADD COLUMN speech_end_ms INTEGER",
|
||||
[],
|
||||
)?;
|
||||
}
|
||||
// Probe publication changes filter eligibility, not source discovery identity.
|
||||
// Keep its epoch separate so indexing does not invalidate its own corpus snapshot.
|
||||
initialization.execute_batch("CREATE TABLE IF NOT EXISTS semantic_media_metadata_revision(singleton INTEGER PRIMARY KEY CHECK(singleton=1), revision INTEGER NOT NULL);
|
||||
@@ -254,7 +274,7 @@ impl Store {
|
||||
}
|
||||
pub(crate) fn media_failure(&self, generation: i64, path: &str, message: &str) -> Result<()> {
|
||||
self.connection.execute(
|
||||
"DELETE FROM semantic_chunks WHERE generation=?1 AND path=?2 AND source_type!='text'",
|
||||
"DELETE FROM semantic_chunks WHERE generation=?1 AND path=?2 AND source_type IN ('image','video','audio')",
|
||||
params![generation, path],
|
||||
)?;
|
||||
self.connection.execute("INSERT OR REPLACE INTO semantic_media_errors(generation,path,message) VALUES(?1,?2,?3)",params![generation,path,message.chars().take(2048).collect::<String>()])?;
|
||||
@@ -298,10 +318,96 @@ impl Store {
|
||||
)
|
||||
};
|
||||
let bytes: Vec<u8> = vector.iter().flat_map(|v| v.to_le_bytes()).collect();
|
||||
self.connection.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,file_change_revision) VALUES(?1,?2,?3,?4,?5,?6,?7,0,0,?8,?9,?10,?11,?12,(SELECT change_revision FROM files WHERE path=?4))",params![generation,-file_id,ordinal as i64,path,kind,timestamp,identity.sha256,snippet,bytes,source_type,identity.bytes as i64,identity.modified_ns])?;
|
||||
self.connection.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,file_change_revision) VALUES(?1,?2,?3,?4,?5,?6,?7,0,0,?8,?9,?10,?11,?12,(SELECT change_revision FROM files WHERE path=?4))",params![generation,media_passage_id(file_id,source_type)?,ordinal as i64,path,kind,timestamp,identity.sha256,snippet,bytes,source_type,identity.bytes as i64,identity.modified_ns])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn insert_speech(
|
||||
&self,
|
||||
generation: i64,
|
||||
file_id: i64,
|
||||
ordinal: usize,
|
||||
identity: &searchi_media::MediaIdentity,
|
||||
revision: i64,
|
||||
start_ms: u32,
|
||||
end_ms: u32,
|
||||
text: &str,
|
||||
vector: &[f32],
|
||||
) -> Result<()> {
|
||||
ensure!(
|
||||
start_ms <= end_ms && !text.trim().is_empty() && text.len() <= 4096,
|
||||
"Invalid recognized passage"
|
||||
);
|
||||
let current:Option<i64>=self.connection.query_row("SELECT change_revision FROM files WHERE id=?1 AND path=?2 AND size=?3 AND mtime=?4",params![file_id,identity.path.to_string_lossy(),i64::try_from(identity.bytes)?,identity.modified_ns],|row|row.get(0)).optional()?;
|
||||
ensure!(
|
||||
current == Some(revision),
|
||||
"Speech source changed since recognition admission"
|
||||
);
|
||||
self.insert_media(
|
||||
generation, file_id, ordinal, identity, "speech", start_ms, vector,
|
||||
)?;
|
||||
self.connection.execute("UPDATE semantic_chunks SET snippet=?4,speech_end_ms=?5 WHERE generation=?1 AND passage_id=?2 AND ordinal=?3 AND source_type='speech'",params![generation,media_passage_id(file_id,"speech")?,ordinal as i64,text,end_ms])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn speech_excerpt(
|
||||
&self,
|
||||
path: &str,
|
||||
start_ms: u32,
|
||||
max_chars: usize,
|
||||
) -> Result<Option<super::SpeechExcerpt>> {
|
||||
self.speech_excerpt_matching(path, start_ms, None, max_chars)
|
||||
}
|
||||
|
||||
pub(crate) fn speech_excerpt_matching(
|
||||
&self,
|
||||
path: &str,
|
||||
start_ms: u32,
|
||||
expected_text: Option<&str>,
|
||||
max_chars: usize,
|
||||
) -> Result<Option<super::SpeechExcerpt>> {
|
||||
if let Some(text) = expected_text {
|
||||
ensure!(
|
||||
!text.is_empty() && text.len() <= 4096,
|
||||
"Invalid recognized source identity"
|
||||
);
|
||||
}
|
||||
ensure!(
|
||||
max_chars > 0 && max_chars <= 8192,
|
||||
"Speech excerpt limit must be 1..=8192 characters"
|
||||
);
|
||||
let row:Option<(String,String,u32,u64,i64,String)>=self.connection.query_row("SELECT f.filename,c.snippet,c.speech_end_ms,c.file_size,c.file_mtime,c.content_hash FROM semantic_chunks c
|
||||
JOIN semantic_meta m ON m.active_generation=c.generation JOIN files f ON f.path=c.path
|
||||
WHERE c.path=?1 AND c.source_type='speech' AND c.number=?2 AND (?3 IS NULL OR c.snippet=?3) AND c.file_size=f.size AND c.file_mtime=f.mtime AND c.file_change_revision=f.change_revision
|
||||
ORDER BY c.ordinal LIMIT 1",params![path,start_ms,expected_text],|row|Ok((row.get(0)?,row.get(1)?,row.get(2)?,row.get(3)?,row.get(4)?,row.get(5)?))).optional()?;
|
||||
let Some((filename, text, end_ms, size, mtime, hash)) = row else {
|
||||
return Ok(None);
|
||||
};
|
||||
ensure!(
|
||||
size <= searchi_media::MediaLimits::default().max_file_bytes,
|
||||
"Speech source exceeds supported byte bound"
|
||||
);
|
||||
let current = match searchi_media::MediaIdentity::read(Path::new(path), size) {
|
||||
Ok(identity) => identity,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
if current.bytes != size || current.modified_ns != mtime || current.sha256 != hash {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(super::SpeechExcerpt {
|
||||
path: path.into(),
|
||||
filename,
|
||||
source: SourceLocation {
|
||||
kind: "timestamp_ms".into(),
|
||||
number: start_ms,
|
||||
},
|
||||
end_ms,
|
||||
text: text.chars().take(max_chars).collect(),
|
||||
provenance: "local_speech_recognition",
|
||||
}))
|
||||
}
|
||||
|
||||
pub(crate) fn insert(
|
||||
&mut self,
|
||||
generation: i64,
|
||||
|
||||
@@ -48,7 +48,7 @@ pub(super) fn migrate(connection: &Connection) -> Result<()> {
|
||||
PRIMARY KEY(generation,task_key));
|
||||
CREATE INDEX IF NOT EXISTS semantic_task_reuse ON semantic_tasks(source_hash,title_hash,policy,source_type,state);
|
||||
CREATE INDEX IF NOT EXISTS semantic_chunk_task ON semantic_chunks(generation,task_key);
|
||||
UPDATE semantic_meta SET schema_version=8 WHERE singleton=1;")?;
|
||||
UPDATE semantic_meta SET schema_version=9 WHERE singleton=1;")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -146,7 +146,7 @@ impl Store {
|
||||
let tx = self
|
||||
.connection
|
||||
.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
|
||||
let count:i64=tx.query_row("SELECT count(*) FROM semantic_chunks WHERE generation=?1 AND path=?2 AND ((?3='text' AND source_type='text' AND passage_id=?4) OR (?3='media' AND source_type!='text'))",params![generation,task.path,task.source_type,task.passage_id],|r|r.get(0))?;
|
||||
let count:i64=tx.query_row("SELECT count(*) FROM semantic_chunks WHERE generation=?1 AND path=?2 AND ((?3='text' AND source_type='text' AND passage_id=?4) OR (?3='media' AND source_type IN ('image','video','audio')) OR (?3='speech' AND source_type='speech'))",params![generation,task.path,task.source_type,task.passage_id],|r|r.get(0))?;
|
||||
ensure!(
|
||||
usize::try_from(count)? == expected_chunks,
|
||||
"Incomplete source chunk membership"
|
||||
@@ -160,7 +160,7 @@ impl Store {
|
||||
total <= MAX_SCAN_CHUNKS,
|
||||
"Generation exceeds semantic chunk budget"
|
||||
);
|
||||
tx.execute("UPDATE semantic_chunks SET task_key=?5 WHERE generation=?1 AND path=?2 AND ((?3='text' AND source_type='text' AND passage_id=?4) OR (?3='media' AND source_type!='text'))",params![generation,task.path,task.source_type,task.passage_id,task.key])?;
|
||||
tx.execute("UPDATE semantic_chunks SET task_key=?5 WHERE generation=?1 AND path=?2 AND ((?3='text' AND source_type='text' AND passage_id=?4) OR (?3='media' AND source_type IN ('image','video','audio')) OR (?3='speech' AND source_type='speech'))",params![generation,task.path,task.source_type,task.passage_id,task.key])?;
|
||||
let changed=tx.execute("UPDATE semantic_tasks SET state='done',expected_chunks=?3,path=?4,passage_id=?5,message=NULL WHERE generation=?1 AND task_key=?2 AND source_hash=?6 AND title_hash=?7 AND policy=?8 AND source_type=?9",params![generation,task.key,count,task.path,task.passage_id,task.source_hash,task.title_hash,task.policy,task.source_type])?;
|
||||
ensure!(changed == 1, "Missing matching source task");
|
||||
if task.source_type == "media" {
|
||||
@@ -180,7 +180,7 @@ impl Store {
|
||||
message: &str,
|
||||
) -> Result<()> {
|
||||
let tx = self.connection.transaction()?;
|
||||
tx.execute("DELETE FROM semantic_chunks WHERE generation=?1 AND (task_key=?2 OR (path=?3 AND ((?4='text' AND source_type='text' AND passage_id=?5) OR (?4='media' AND source_type!='text'))))",params![generation,task.key,task.path,task.source_type,task.passage_id])?;
|
||||
tx.execute("DELETE FROM semantic_chunks WHERE generation=?1 AND (task_key=?2 OR (path=?3 AND ((?4='text' AND source_type='text' AND passage_id=?5) OR (?4='media' AND source_type IN ('image','video','audio')) OR (?4='speech' AND source_type='speech'))))",params![generation,task.key,task.path,task.source_type,task.passage_id])?;
|
||||
tx.execute("UPDATE semantic_tasks SET state='failed',expected_chunks=NULL,message=?3 WHERE generation=?1 AND task_key=?2",params![generation,task.key,message.chars().take(2048).collect::<String>()])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
@@ -299,8 +299,8 @@ impl Store {
|
||||
for (key, donor) in mappings {
|
||||
tx.execute("INSERT INTO semantic_tasks(generation,task_key,path,source_type,source_hash,title_hash,policy,passage_id,state,attempts,expected_chunks,message)
|
||||
SELECT ?1,d.task_key,d.path,d.source_type,d.source_hash,d.title_hash,d.policy,d.passage_id,'done',old.attempts,old.expected_chunks,NULL FROM semantic_discovery_tasks d JOIN semantic_prune_tasks old ON old.task_key=?3 WHERE d.task_key=?2",params![generation,key,donor])?;
|
||||
tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key,file_change_revision)
|
||||
SELECT ?1,d.passage_id,c.ordinal,d.path,c.kind,c.number,c.content_hash,c.start_byte,c.end_byte,c.snippet,c.vector,c.source_type,c.file_size,c.file_mtime,d.task_key,COALESCE((SELECT change_revision FROM files WHERE path=d.path),c.file_change_revision) FROM semantic_prune_chunks c JOIN semantic_discovery_tasks d ON d.task_key=?2 WHERE c.task_key=?3",params![generation,key,donor])?;
|
||||
tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key,file_change_revision,speech_end_ms)
|
||||
SELECT ?1,d.passage_id,c.ordinal,d.path,c.kind,c.number,c.content_hash,c.start_byte,c.end_byte,c.snippet,c.vector,c.source_type,c.file_size,c.file_mtime,d.task_key,COALESCE((SELECT change_revision FROM files WHERE path=d.path),c.file_change_revision),c.speech_end_ms FROM semantic_prune_chunks c JOIN semantic_discovery_tasks d ON d.task_key=?2 WHERE c.task_key=?3",params![generation,key,donor])?;
|
||||
}
|
||||
tx.execute(
|
||||
"DELETE FROM semantic_media_errors WHERE generation=?1",
|
||||
@@ -378,11 +378,11 @@ fn validate_task(task: &SourceTask) -> Result<()> {
|
||||
"Invalid source task identity"
|
||||
);
|
||||
ensure!(
|
||||
matches!(task.source_type.as_str(), "text" | "media"),
|
||||
matches!(task.source_type.as_str(), "text" | "media" | "speech"),
|
||||
"Invalid source task type"
|
||||
);
|
||||
ensure!(
|
||||
(!task.source_hash.is_empty() || task.source_type == "media")
|
||||
(!task.source_hash.is_empty() || task.source_type != "text")
|
||||
&& task.source_hash.len() <= 256
|
||||
&& task.title_hash.len() <= 256
|
||||
&& !task.policy.is_empty()
|
||||
@@ -418,6 +418,13 @@ impl Store {
|
||||
candidate?;
|
||||
let compatible = source_dimensions == destination_dimensions
|
||||
&& (source_policy == destination_policy
|
||||
|| (task.source_type == "media"
|
||||
&& super::super::raw_media_policy(&destination_policy) == task.policy
|
||||
&& super::super::raw_media_policy(&source_policy) == task.policy)
|
||||
|| (task.source_type == "speech"
|
||||
&& super::super::speech_component_policy(&destination_policy)
|
||||
== task.policy
|
||||
&& super::super::speech_component_policy(&source_policy) == task.policy)
|
||||
|| (task.source_type == "text"
|
||||
&& super::super::policy_base(&destination_policy) == task.policy
|
||||
&& super::super::reusable_policy(&task.policy, &destination_policy)
|
||||
@@ -437,7 +444,7 @@ impl Store {
|
||||
let second = admitted.get(1).copied().unwrap_or(first);
|
||||
Ok(self.connection.query_row("SELECT t.generation,t.task_key,t.expected_chunks FROM semantic_tasks t JOIN semantic_generations g ON g.id=t.generation JOIN semantic_generations destination ON destination.id=?1
|
||||
WHERE t.source_hash=?2 AND t.title_hash=?3 AND t.policy=?4 AND t.source_type=?5 AND t.state='done' AND t.expected_chunks IS NOT NULL
|
||||
AND (g.fingerprint=destination.fingerprint OR (?5='text' AND t.generation IN (?6,?7))) AND g.dimensions=destination.dimensions
|
||||
AND (g.fingerprint=destination.fingerprint OR t.generation IN (?6,?7)) AND g.dimensions=destination.dimensions
|
||||
AND t.expected_chunks=(SELECT count(*) FROM semantic_chunks c WHERE c.generation=t.generation AND c.task_key=t.task_key)
|
||||
ORDER BY (t.generation=?1) DESC,t.generation DESC LIMIT 1",params![generation,task.source_hash,task.title_hash,task.policy,task.source_type,first,second],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()?)
|
||||
}
|
||||
@@ -510,8 +517,8 @@ impl Store {
|
||||
identity: &searchi_media::MediaIdentity,
|
||||
) -> Result<Option<usize>> {
|
||||
ensure!(
|
||||
task.source_type == "media"
|
||||
&& task.passage_id == -file_id
|
||||
matches!(task.source_type.as_str(), "media" | "speech")
|
||||
&& task.passage_id == media_passage_id(file_id, &task.source_type)?
|
||||
&& task.path == identity.path.to_string_lossy()
|
||||
&& task.source_hash == identity.sha256,
|
||||
"Media task mapping differs from current source"
|
||||
@@ -524,18 +531,34 @@ impl Store {
|
||||
"Reusable source exceeds chunk budget"
|
||||
);
|
||||
{
|
||||
let mut stmt=self.connection.prepare("SELECT vector,source_type,ordinal,content_hash FROM semantic_chunks WHERE generation=?1 AND task_key=?2 ORDER BY ordinal")?;
|
||||
let mut stmt=self.connection.prepare("SELECT vector,source_type,ordinal,content_hash,speech_end_ms,number,snippet,kind FROM semantic_chunks WHERE generation=?1 AND task_key=?2 ORDER BY ordinal")?;
|
||||
let mut rows = stmt.query(params![source, key])?;
|
||||
let mut seen = 0usize;
|
||||
while let Some(row) = rows.next()? {
|
||||
ensure!(
|
||||
matches!(
|
||||
row.get::<_, String>(1)?.as_str(),
|
||||
"image" | "audio" | "video"
|
||||
) && row.get::<_, usize>(2)? == seen
|
||||
(if task.source_type == "speech" {
|
||||
row.get::<_, String>(1)? == "speech"
|
||||
} else {
|
||||
matches!(
|
||||
row.get::<_, String>(1)?.as_str(),
|
||||
"image" | "audio" | "video"
|
||||
)
|
||||
}) && row.get::<_, usize>(2)? == seen
|
||||
&& row.get::<_, String>(3)? == task.source_hash,
|
||||
"Corrupt completed media source membership"
|
||||
);
|
||||
if task.source_type == "speech" {
|
||||
let start: u32 = row.get(5)?;
|
||||
let end: Option<u32> = row.get(4)?;
|
||||
let text: String = row.get(6)?;
|
||||
ensure!(
|
||||
end.is_some_and(|end| end >= start)
|
||||
&& row.get::<_, String>(7)? == "timestamp_ms"
|
||||
&& !text.trim().is_empty()
|
||||
&& text.len() <= 4096,
|
||||
"Corrupt completed speech evidence"
|
||||
);
|
||||
}
|
||||
validate_vector(&row.get::<_, Vec<u8>>(0)?, self.dimensions)?;
|
||||
seen += 1;
|
||||
}
|
||||
@@ -572,8 +595,8 @@ impl Store {
|
||||
"DELETE FROM semantic_chunks WHERE generation=?1 AND task_key=?2",
|
||||
params![generation, task.key],
|
||||
)?;
|
||||
tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key,file_change_revision)
|
||||
SELECT ?1,?2,ordinal,?3,COALESCE(?4,kind),COALESCE(?5,number),content_hash,start_byte,end_byte,snippet,vector,source_type,COALESCE(?6,file_size),COALESCE(?7,file_mtime),?8,COALESCE((SELECT change_revision FROM files WHERE path=?3),file_change_revision) FROM semantic_reuse_chunks",params![generation,task.passage_id,task.path,location.map(|s|s.kind.as_str()),location.map(|s|s.number),identity.map(|i|i.bytes as i64),identity.map(|i|i.modified_ns),task.key])?;
|
||||
tx.execute("INSERT INTO semantic_chunks(generation,passage_id,ordinal,path,kind,number,content_hash,start_byte,end_byte,snippet,vector,source_type,file_size,file_mtime,task_key,file_change_revision,speech_end_ms)
|
||||
SELECT ?1,?2,ordinal,?3,COALESCE(?4,kind),COALESCE(?5,number),content_hash,start_byte,end_byte,snippet,vector,source_type,COALESCE(?6,file_size),COALESCE(?7,file_mtime),?8,COALESCE((SELECT change_revision FROM files WHERE path=?3),file_change_revision),speech_end_ms FROM semantic_reuse_chunks",params![generation,task.passage_id,task.path,location.map(|s|s.kind.as_str()),location.map(|s|s.number),identity.map(|i|i.bytes as i64),identity.map(|i|i.modified_ns),task.key])?;
|
||||
let changed=tx.execute("UPDATE semantic_tasks SET state='done',expected_chunks=?3,path=?4,passage_id=?5,message=NULL WHERE generation=?1 AND task_key=?2 AND source_hash=?6 AND title_hash=?7 AND policy=?8 AND source_type=?9",params![generation,task.key,count as i64,task.path,task.passage_id,task.source_hash,task.title_hash,task.policy,task.source_type])?;
|
||||
ensure!(changed == 1, "Missing source task during reuse");
|
||||
if task.source_type == "media" {
|
||||
@@ -605,6 +628,148 @@ fn validate_vector(bytes: &[u8], dimensions: usize) -> Result<()> {
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod speech_lifecycle_tests {
|
||||
use super::*;
|
||||
use searchi_core::SearchIndex;
|
||||
#[test]
|
||||
fn speech_upgrade_and_failure_reuse_raw_work_and_verified_excerpts_reject_same_stat_changes(
|
||||
) -> Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let root = dir.path().join("media");
|
||||
std::fs::create_dir(&root)?;
|
||||
let file = root.join("meeting.wav");
|
||||
std::fs::write(&file, b"real source bytes for storage tests")?;
|
||||
let db = dir.path().join("index.db");
|
||||
let mut lexical = SearchIndex::open(&db)?;
|
||||
lexical.index_folder(&root)?;
|
||||
let mut store = Store::open(&db, 2)?;
|
||||
let (id, path, _, _) = store.media_batch(0)?.remove(0);
|
||||
let identity = searchi_media::MediaIdentity::read(Path::new(&path), 1024)?;
|
||||
let revision =
|
||||
lexical.media_change_revision(&path, identity.bytes, identity.modified_ns)?;
|
||||
let base = "storage-google-base";
|
||||
let raw = SourceTask {
|
||||
key: "raw-key".into(),
|
||||
path: path.clone(),
|
||||
source_type: "media".into(),
|
||||
source_hash: identity.sha256.clone(),
|
||||
title_hash: String::new(),
|
||||
policy: base.into(),
|
||||
passage_id: -id,
|
||||
};
|
||||
let original = store.begin(base)?;
|
||||
store.stage_task(original, &raw)?;
|
||||
// Synthetic unit geometry tests membership, not production inference or relevance.
|
||||
store.insert_media(original, id, 0, &identity, "audio", 0, &[1., 0.])?;
|
||||
store.complete_source(original, &raw, 1)?;
|
||||
store.activate(original)?;
|
||||
let speech_policy = format!("{base}:speech-v1-{}", "a".repeat(64));
|
||||
let first = store.begin(&speech_policy)?;
|
||||
store.stage_task(first, &raw)?;
|
||||
assert_eq!(
|
||||
store.reuse_media_source(first, &raw, id, &identity)?,
|
||||
Some(1)
|
||||
);
|
||||
let speech = SourceTask {
|
||||
key: "speech-key".into(),
|
||||
source_type: "speech".into(),
|
||||
policy: speech_policy.clone(),
|
||||
passage_id: media_passage_id(id, "speech")?,
|
||||
..raw.clone()
|
||||
};
|
||||
store.stage_task(first, &speech)?;
|
||||
store.insert_speech(
|
||||
first,
|
||||
id,
|
||||
0,
|
||||
&identity,
|
||||
revision,
|
||||
100,
|
||||
100,
|
||||
"Recognized words",
|
||||
&[0., 1.],
|
||||
)?;
|
||||
store.insert_speech(
|
||||
first,
|
||||
id,
|
||||
1,
|
||||
&identity,
|
||||
revision,
|
||||
100,
|
||||
110,
|
||||
"Different recognized words",
|
||||
&[0., 1.],
|
||||
)?;
|
||||
store.complete_source(first, &speech, 2)?;
|
||||
validate_completion(&store.connection, first)?;
|
||||
store.activate(first)?;
|
||||
let excerpt = store
|
||||
.speech_excerpt(&path, 100, 4096)?
|
||||
.expect("verified excerpt");
|
||||
assert_eq!(excerpt.text, "Recognized words");
|
||||
assert_eq!(excerpt.end_ms, 100);
|
||||
assert_eq!(excerpt.provenance, "local_speech_recognition");
|
||||
let exact = store
|
||||
.speech_excerpt_matching(&path, 100, Some("Different recognized words"), 4096)?
|
||||
.expect("chosen duplicate-time passage");
|
||||
assert_eq!(exact.end_ms, 110);
|
||||
assert_eq!(exact.text, "Different recognized words");
|
||||
assert!(store
|
||||
.speech_excerpt_matching(&path, 100, Some("not recognized"), 4096)?
|
||||
.is_none());
|
||||
let same = store.begin(&speech_policy)?;
|
||||
store.stage_task(same, &raw)?;
|
||||
store.stage_task(same, &speech)?;
|
||||
assert_eq!(
|
||||
store.reuse_media_source(same, &raw, id, &identity)?,
|
||||
Some(1)
|
||||
);
|
||||
assert_eq!(
|
||||
store.reuse_media_source(same, &speech, id, &identity)?,
|
||||
Some(2)
|
||||
);
|
||||
store.activate(same)?;
|
||||
let changed_policy = format!("{base}:speech-v1-{}", "b".repeat(64));
|
||||
let changed = store.begin(&changed_policy)?;
|
||||
store.stage_task(changed, &raw)?;
|
||||
assert_eq!(
|
||||
store.reuse_media_source(changed, &raw, id, &identity)?,
|
||||
Some(1)
|
||||
);
|
||||
let updated_speech = SourceTask {
|
||||
policy: changed_policy,
|
||||
..speech.clone()
|
||||
};
|
||||
store.stage_task(changed, &updated_speech)?;
|
||||
assert_eq!(
|
||||
store.reuse_media_source(changed, &updated_speech, id, &identity)?,
|
||||
None
|
||||
);
|
||||
store.fail_source(changed, &updated_speech, "explicit recognition failure")?;
|
||||
let raw_count: i64 = store.connection.query_row(
|
||||
"SELECT count(*) FROM semantic_chunks WHERE generation=?1 AND source_type='audio'",
|
||||
[changed],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
assert_eq!(raw_count, 1);
|
||||
validate_completion(&store.connection, changed)?;
|
||||
// Same-stat byte mutations never qualify for agent evidence.
|
||||
let modified = std::fs::metadata(&path)?.modified()?;
|
||||
let replacement = vec![b'x'; identity.bytes as usize];
|
||||
std::fs::write(&path, replacement)?;
|
||||
std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.open(&path)?
|
||||
.set_times(std::fs::FileTimes::new().set_modified(modified))?;
|
||||
assert!(store.speech_excerpt(&path, 100, 4096)?.is_none());
|
||||
assert!(store
|
||||
.speech_excerpt_matching(&path, 100, Some("Different recognized words"), 4096)?
|
||||
.is_none());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
//! Functional speech publication uses real recognition and real Google embeddings.
|
||||
//! Retrieval queries derive from recognized evidence; this is not a relevance benchmark.
|
||||
use anyhow::{Context, Result};
|
||||
use searchi_core::SearchIndex;
|
||||
use searchi_service::{
|
||||
FfmpegTools, HybridSearch, MediaLimits, SearchOptions, SpeechProvider, SpeechTranscription,
|
||||
};
|
||||
use searchi_speech::{Device, SpeechWorker};
|
||||
use std::{
|
||||
fs,
|
||||
path::PathBuf,
|
||||
sync::{
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
Arc,
|
||||
},
|
||||
};
|
||||
|
||||
struct FailOnceNative {
|
||||
worker: SpeechWorker,
|
||||
fingerprint: String,
|
||||
fail_once: bool,
|
||||
calls: Arc<AtomicUsize>,
|
||||
}
|
||||
impl SpeechProvider for FailOnceNative {
|
||||
fn fingerprint(&self) -> &str {
|
||||
&self.fingerprint
|
||||
}
|
||||
fn transcribe(&mut self, pcm: &[f32], rate: u32) -> Result<SpeechTranscription> {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.fail_once {
|
||||
self.fail_once = false;
|
||||
anyhow::bail!("Injected optional recognition failure");
|
||||
}
|
||||
self.worker.transcribe(pcm, rate)
|
||||
}
|
||||
}
|
||||
fn required(name: &str) -> Result<PathBuf> {
|
||||
std::env::var_os(name)
|
||||
.map(PathBuf::from)
|
||||
.with_context(|| format!("Set {name}"))
|
||||
}
|
||||
fn native() -> Result<SpeechWorker> {
|
||||
SpeechWorker::load(
|
||||
&required("SEARCHI_SPEECH_WORKER")?,
|
||||
&required("SEARCHI_SPEECH_BASE")?,
|
||||
&required("SEARCHI_SPEECH_VAD")?,
|
||||
Device::Cpu,
|
||||
)
|
||||
}
|
||||
fn raw_vectors(db: &std::path::Path) -> Result<Vec<Vec<u8>>> {
|
||||
let connection = rusqlite::Connection::open(db)?;
|
||||
let mut statement=connection.prepare("SELECT vector FROM semantic_chunks WHERE generation=(SELECT active_generation FROM semantic_meta WHERE singleton=1) AND source_type='audio' ORDER BY ordinal")?;
|
||||
let rows = statement
|
||||
.query_map([], |row| row.get(0))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "Requires verified real speech assets/runtime, Google assets/runtime and short real speech WAV; CPU only"]
|
||||
fn genuine_speech_policy_upgrade_retries_only_speech_and_preserves_warmed_query_reader(
|
||||
) -> Result<()> {
|
||||
assert_eq!(std::env::var("SEARCHI_FORCE_CPU")?, "1");
|
||||
let model = required("SEARCHI_GEMMA_MODEL_DIR")?;
|
||||
let gemma = required("SEARCHI_GEMMA_WORKER")?;
|
||||
let source = required("SEARCHI_TEST_SPEECH_AUDIO")?;
|
||||
anyhow::ensure!(
|
||||
fs::metadata(&source)?.len() <= 64 * 1024 * 1024,
|
||||
"Test source exceeds byte budget"
|
||||
);
|
||||
let tools = FfmpegTools {
|
||||
ffmpeg: required("SEARCHI_TEST_FFMPEG")?,
|
||||
ffprobe: required("SEARCHI_TEST_FFPROBE")?,
|
||||
};
|
||||
let limits = MediaLimits::default();
|
||||
let metadata = searchi_media::probe(&tools, &source, &limits)?;
|
||||
anyhow::ensure!(
|
||||
metadata.has_audio && metadata.duration_ms <= 30000,
|
||||
"Functional fixture must contain real speech in at most thirty seconds"
|
||||
);
|
||||
let dir = tempfile::tempdir()?;
|
||||
let root = dir.path().join("media");
|
||||
fs::create_dir(&root)?;
|
||||
let path = root.join("meeting.wav");
|
||||
fs::copy(&source, &path)?;
|
||||
let db = dir.path().join("index.db");
|
||||
let mut lexical = SearchIndex::open(&db)?;
|
||||
lexical.index_folder(&root)?;
|
||||
let mut indexer = HybridSearch::open(&db, &model, &gemma, 1)?;
|
||||
let raw = indexer.index_embeddings_with_media(&tools, &limits)?;
|
||||
assert!(raw.failures.is_empty());
|
||||
assert_eq!(raw.audio_inference_calls, 1);
|
||||
let original_vectors = raw_vectors(&db)?;
|
||||
assert_eq!(original_vectors.len(), 1);
|
||||
indexer = indexer.with_speech_provider(Box::new(native()?))?;
|
||||
let speech = indexer.index_embeddings_with_media(&tools, &limits)?;
|
||||
assert!(speech.speech_failures.is_empty());
|
||||
assert_eq!(speech.audio_inference_calls, 0);
|
||||
assert_eq!(speech.speech_windows, 1);
|
||||
assert!(speech.speech_chunks > 0);
|
||||
assert_eq!(raw_vectors(&db)?, original_vectors);
|
||||
let connection = rusqlite::Connection::open(&db)?;
|
||||
let recognized:String=connection.query_row("SELECT snippet FROM semantic_chunks WHERE generation=(SELECT active_generation FROM semantic_meta WHERE singleton=1) AND source_type='speech' ORDER BY ordinal LIMIT 1",[],|row|row.get(0))?;
|
||||
let mut reader = HybridSearch::open_query(&db, &model, &gemma, 1)?;
|
||||
let query = format!("someone says {recognized}");
|
||||
let hits = reader.search(&query, 1, &SearchOptions::default())?;
|
||||
assert_eq!(hits.len(), 1);
|
||||
assert_eq!(hits[0].match_kind, "semantic_speech");
|
||||
let location = hits[0]
|
||||
.source
|
||||
.as_ref()
|
||||
.context("recognized source timestamp")?;
|
||||
assert_eq!(location.kind, "timestamp_ms");
|
||||
let excerpt = reader
|
||||
.speech_excerpt_matching(&hits[0].path, location.number, &hits[0].snippet, 4096)?
|
||||
.context("verified recognized evidence")?;
|
||||
assert_eq!(excerpt.provenance, "local_speech_recognition");
|
||||
assert_eq!(excerpt.text, hits[0].snippet);
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let worker = native()?;
|
||||
let fingerprint = format!("{};functional-fault-injection-v1", worker.fingerprint());
|
||||
indexer = indexer.with_speech_provider(Box::new(FailOnceNative {
|
||||
worker,
|
||||
fingerprint,
|
||||
fail_once: true,
|
||||
calls: calls.clone(),
|
||||
}))?;
|
||||
let partial = indexer.index_embeddings_with_media(&tools, &limits)?;
|
||||
assert_eq!(partial.speech_failures.len(), 1);
|
||||
assert_eq!(partial.audio_inference_calls, 0);
|
||||
assert_eq!(raw_vectors(&db)?, original_vectors);
|
||||
let status = HybridSearch::embedding_status(&db)?;
|
||||
assert_eq!(status.speech_failures.len(), 1);
|
||||
let retry = indexer.index_embeddings_with_media(&tools, &limits)?;
|
||||
assert!(retry.speech_failures.is_empty());
|
||||
assert_eq!(retry.audio_inference_calls, 0);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 2);
|
||||
assert!(retry.speech_chunks > 0);
|
||||
assert_eq!(raw_vectors(&db)?, original_vectors);
|
||||
let current = HybridSearch::embedding_status(&db)?;
|
||||
assert!(current.speech_failures.is_empty());
|
||||
assert_eq!(
|
||||
current.corpus_revision, status.corpus_revision,
|
||||
"Partial publication must not reset automatic retry identity"
|
||||
);
|
||||
assert_eq!(current.speech_policy, status.speech_policy);
|
||||
// The retained foreground Google query session adopts the compatible publication.
|
||||
let hits = reader.search(&query, 1, &SearchOptions::default())?;
|
||||
assert_eq!(hits[0].match_kind, "semantic_speech");
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
//! Timeline contract uses actual CPU FFmpeg, without speech/model inference.
|
||||
use anyhow::{Context, Result};
|
||||
use searchi_media::{decode_audio, decode_audio_aligned, probe, AudioWindow};
|
||||
use searchi_service::{FfmpegTools, MediaLimits};
|
||||
use std::{path::Path, process::Command};
|
||||
|
||||
fn fixture(ffmpeg: &Path, args: &[&str], output: &Path) -> Result<()> {
|
||||
let result = Command::new(ffmpeg)
|
||||
.args([
|
||||
"-nostdin",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-threads",
|
||||
"1",
|
||||
])
|
||||
.args(args)
|
||||
.arg(output)
|
||||
.output()?;
|
||||
anyhow::ensure!(
|
||||
result.status.success(),
|
||||
"Owned CPU media fixture failed: {}",
|
||||
String::from_utf8_lossy(&result.stderr)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "Requires verified CPU FFmpeg/FFprobe helpers; no model inference"]
|
||||
fn positioned_audio_decodes_at_its_actual_file_timeline_origin() -> Result<()> {
|
||||
assert_eq!(std::env::var("SEARCHI_FORCE_CPU")?, "1");
|
||||
let tools = FfmpegTools {
|
||||
ffmpeg: std::env::var_os("SEARCHI_TEST_FFMPEG")
|
||||
.context("Set SEARCHI_TEST_FFMPEG")?
|
||||
.into(),
|
||||
ffprobe: std::env::var_os("SEARCHI_TEST_FFPROBE")
|
||||
.context("Set SEARCHI_TEST_FFPROBE")?
|
||||
.into(),
|
||||
};
|
||||
let directory = tempfile::tempdir()?;
|
||||
let ordinary = directory.path().join("ordinary.wav");
|
||||
let positioned = directory.path().join("positioned.mkv");
|
||||
fixture(
|
||||
&tools.ffmpeg,
|
||||
&[
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=440:sample_rate=16000:duration=2",
|
||||
"-c:a",
|
||||
"pcm_s16le",
|
||||
],
|
||||
&ordinary,
|
||||
)?;
|
||||
fixture(
|
||||
&tools.ffmpeg,
|
||||
&[
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=size=16x16:rate=1:duration=4",
|
||||
"-itsoffset",
|
||||
"2",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=440:sample_rate=16000:duration=2",
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"1:a:0",
|
||||
"-c:v",
|
||||
"ffv1",
|
||||
"-c:a",
|
||||
"pcm_s16le",
|
||||
"-threads",
|
||||
"1",
|
||||
],
|
||||
&positioned,
|
||||
)?;
|
||||
let limits = MediaLimits::default();
|
||||
let metadata = probe(&tools, &positioned, &limits)?;
|
||||
assert_eq!(metadata.start_ms, 0);
|
||||
assert_eq!(metadata.audio_start_ms, 2000);
|
||||
assert_eq!(metadata.audio_end_ms, 4000);
|
||||
let baseline = decode_audio(
|
||||
&tools,
|
||||
&ordinary,
|
||||
AudioWindow {
|
||||
start_ms: 0,
|
||||
duration_ms: 2000,
|
||||
},
|
||||
&limits,
|
||||
)?;
|
||||
let actual = decode_audio(
|
||||
&tools,
|
||||
&positioned,
|
||||
AudioWindow {
|
||||
start_ms: metadata.audio_start_ms,
|
||||
duration_ms: metadata.audio_end_ms - metadata.audio_start_ms,
|
||||
},
|
||||
&limits,
|
||||
)?;
|
||||
assert_eq!(baseline.len(), 32000);
|
||||
let aligned = decode_audio_aligned(
|
||||
&tools,
|
||||
&positioned,
|
||||
AudioWindow {
|
||||
start_ms: 2000,
|
||||
duration_ms: 2000,
|
||||
},
|
||||
&limits,
|
||||
)?;
|
||||
assert_eq!(aligned, baseline);
|
||||
let whole = decode_audio_aligned(
|
||||
&tools,
|
||||
&positioned,
|
||||
AudioWindow {
|
||||
start_ms: 0,
|
||||
duration_ms: 4000,
|
||||
},
|
||||
&limits,
|
||||
)?;
|
||||
assert_eq!(whole.len(), 64000);
|
||||
assert!(whole[..32000].iter().all(|sample| *sample == 0.));
|
||||
assert_eq!(&whole[32000..], baseline.as_slice());
|
||||
assert_eq!(
|
||||
actual, baseline,
|
||||
"First decoded sample must represent the requested actual audio origin"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "Requires verified CPU FFmpeg/FFprobe helpers; no model inference"]
|
||||
fn aligned_decoder_preserves_internal_gap_seek_and_tail_source_clock() -> Result<()> {
|
||||
assert_eq!(std::env::var("SEARCHI_FORCE_CPU")?, "1");
|
||||
let tools = FfmpegTools {
|
||||
ffmpeg: std::env::var_os("SEARCHI_TEST_FFMPEG")
|
||||
.context("Set SEARCHI_TEST_FFMPEG")?
|
||||
.into(),
|
||||
ffprobe: std::env::var_os("SEARCHI_TEST_FFPROBE")
|
||||
.context("Set SEARCHI_TEST_FFPROBE")?
|
||||
.into(),
|
||||
};
|
||||
let directory = tempfile::tempdir()?;
|
||||
let ordinary = directory.path().join("ramp.wav");
|
||||
let gapped = directory.path().join("gapped.mka");
|
||||
fixture(
|
||||
&tools.ffmpeg,
|
||||
&[
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"aevalsrc=0.1+0.1*t:s=16000:d=3",
|
||||
"-c:a",
|
||||
"pcm_s16le",
|
||||
],
|
||||
&ordinary,
|
||||
)?;
|
||||
// Three exact one-second frames carry PTS 0,2,3; the source clock has a one-second gap.
|
||||
fixture(
|
||||
&tools.ffmpeg,
|
||||
&[
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"aevalsrc=0.1+0.1*t:s=16000:d=3",
|
||||
"-af",
|
||||
"asetnsamples=n=16000:p=0,asetpts=PTS+gte(T\\,1)/TB",
|
||||
"-c:a",
|
||||
"pcm_s16le",
|
||||
],
|
||||
&gapped,
|
||||
)?;
|
||||
let limits = MediaLimits::default();
|
||||
let metadata = probe(&tools, &gapped, &limits)?;
|
||||
assert_eq!(metadata.start_ms, 0);
|
||||
assert_eq!(metadata.audio_start_ms, 0);
|
||||
assert_eq!(metadata.audio_end_ms, 4000);
|
||||
let source = decode_audio(
|
||||
&tools,
|
||||
&ordinary,
|
||||
AudioWindow {
|
||||
start_ms: 0,
|
||||
duration_ms: 3000,
|
||||
},
|
||||
&limits,
|
||||
)?;
|
||||
assert_eq!(source.len(), 48000);
|
||||
let mut expected = source[..16000].to_vec();
|
||||
expected.extend(vec![0.; 16000]);
|
||||
expected.extend_from_slice(&source[16000..]);
|
||||
let whole = decode_audio_aligned(
|
||||
&tools,
|
||||
&gapped,
|
||||
AudioWindow {
|
||||
start_ms: 0,
|
||||
duration_ms: 4000,
|
||||
},
|
||||
&limits,
|
||||
)?;
|
||||
assert_eq!(whole, expected);
|
||||
let seek = decode_audio_aligned(
|
||||
&tools,
|
||||
&gapped,
|
||||
AudioWindow {
|
||||
start_ms: 1500,
|
||||
duration_ms: 1500,
|
||||
},
|
||||
&limits,
|
||||
)?;
|
||||
assert_eq!(seek, &expected[24000..48000]);
|
||||
let tail = decode_audio_aligned(
|
||||
&tools,
|
||||
&gapped,
|
||||
AudioWindow {
|
||||
start_ms: 3500,
|
||||
duration_ms: 500,
|
||||
},
|
||||
&limits,
|
||||
)?;
|
||||
assert_eq!(tail, &expected[56000..64000]);
|
||||
Ok(())
|
||||
}
|
||||
Reference in new issue
Block a user