Reuse completed Google embeddings and resume semantic staging
This commit is contained in:
karl committed
1 parent
0b56ab553c
commit
5c18e7b49c
11 files changed
+1677
-148
No files matched your search
@@ -1,5 +1,7 @@
|
||||
use super::pipeline::BuildWork;
|
||||
use super::*;
|
||||
use searchi_media::*;
|
||||
use store::SourceTask;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct InferenceFailure(anyhow::Error);
|
||||
@@ -40,8 +42,8 @@ impl HybridSearch {
|
||||
limits.max_file_bytes
|
||||
));
|
||||
}
|
||||
self.rebuild_generation(fingerprint, |service, generation| {
|
||||
let mut report = service.build(generation)?;
|
||||
self.rebuild_generation(fingerprint, |service, generation, work| {
|
||||
service.build(generation, work)?;
|
||||
let mut after = 0;
|
||||
let mut media_count = 0usize;
|
||||
loop {
|
||||
@@ -60,42 +62,142 @@ impl HybridSearch {
|
||||
media_count <= 10000,
|
||||
"Media generation exceeds 10000-file resource budget"
|
||||
);
|
||||
let before_chunks = report.chunks;
|
||||
match service.build_media_file(
|
||||
generation,
|
||||
id,
|
||||
let before_chunks = work.report.chunks;
|
||||
let identity = MediaIdentity::read(
|
||||
path,
|
||||
kind,
|
||||
tools,
|
||||
limits,
|
||||
&mut report,
|
||||
) {
|
||||
Ok(()) => report.media_files += 1,
|
||||
if kind == MediaKind::Image {
|
||||
64 * 1024 * 1024
|
||||
} else {
|
||||
limits.max_file_bytes
|
||||
},
|
||||
);
|
||||
let task = SourceTask {
|
||||
key: super::pipeline::task_key(&["media", &path.to_string_lossy()]),
|
||||
path: path.to_string_lossy().into_owned(),
|
||||
source_type: "media".into(),
|
||||
source_hash: identity
|
||||
.as_ref()
|
||||
.map(|i| i.sha256.clone())
|
||||
.unwrap_or_default(),
|
||||
title_hash: String::new(),
|
||||
policy: service.generation_fingerprint.clone(),
|
||||
passage_id: -id,
|
||||
};
|
||||
work.track(&task)?;
|
||||
service.store.stage_task(generation, &task)?;
|
||||
let result = identity.and_then(|identity| {
|
||||
service.build_media_source(
|
||||
generation, id, path, kind, tools, limits, &identity, &task, work,
|
||||
)?;
|
||||
work.media.push(identity);
|
||||
Ok(())
|
||||
});
|
||||
match result {
|
||||
Ok(()) => work.report.media_files += 1,
|
||||
Err(error) => {
|
||||
if let Err(cleanup) =
|
||||
service
|
||||
.store
|
||||
.fail_source(generation, &task, &format!("{error:#}"))
|
||||
{
|
||||
return Err(error.context(format!(
|
||||
"Failed to discard incomplete media source: {cleanup:#}"
|
||||
)));
|
||||
}
|
||||
if error.downcast_ref::<InferenceFailure>().is_some() {
|
||||
return Err(error);
|
||||
}
|
||||
service.store.media_failure(
|
||||
generation,
|
||||
&path.to_string_lossy(),
|
||||
&task.path,
|
||||
&format!("{error:#}"),
|
||||
)?;
|
||||
report.chunks = before_chunks;
|
||||
if report.failures.len() < 1000 {
|
||||
report.failures.push(MediaFailure {
|
||||
path: path.to_string_lossy().into_owned(),
|
||||
work.report.chunks = before_chunks;
|
||||
if work.report.failures.len() < 1000 {
|
||||
work.report.failures.push(MediaFailure {
|
||||
path: task.path,
|
||||
message: format!("{error:#}").chars().take(2048).collect(),
|
||||
});
|
||||
}
|
||||
report.media_failures += 1;
|
||||
work.report.media_failures += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(report)
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn build_media_source(
|
||||
&mut self,
|
||||
generation: i64,
|
||||
id: i64,
|
||||
path: &Path,
|
||||
kind: MediaKind,
|
||||
tools: &FfmpegTools,
|
||||
limits: &MediaLimits,
|
||||
identity: &MediaIdentity,
|
||||
task: &SourceTask,
|
||||
work: &mut BuildWork,
|
||||
) -> Result<()> {
|
||||
if let Some(count) = self
|
||||
.store
|
||||
.reuse_media_source(generation, task, id, identity)?
|
||||
{
|
||||
ensure!(
|
||||
work.report.chunks + count <= MAX_SCAN_CHUNKS,
|
||||
"Media exceeds generation chunk budget"
|
||||
);
|
||||
if kind != MediaKind::Image {
|
||||
// Metadata remains part of music search after path remapping.
|
||||
let metadata = probe(tools, path, limits)?;
|
||||
self.refresh_media_metadata(path, identity, &metadata)?;
|
||||
}
|
||||
identity.ensure_unchanged()?;
|
||||
work.report.chunks += count;
|
||||
work.report.reused_chunks += count;
|
||||
work.report.reused_sources += 1;
|
||||
return Ok(());
|
||||
}
|
||||
let before = work.report.chunks;
|
||||
self.build_media_file(
|
||||
generation,
|
||||
id,
|
||||
path,
|
||||
kind,
|
||||
tools,
|
||||
limits,
|
||||
identity,
|
||||
&mut work.report,
|
||||
)?;
|
||||
self.store
|
||||
.complete_source(generation, task, work.report.chunks - before)?;
|
||||
work.report.encoded_sources += 1;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn refresh_media_metadata(
|
||||
&mut self,
|
||||
path: &Path,
|
||||
identity: &MediaIdentity,
|
||||
metadata: &MediaProbe,
|
||||
) -> Result<()> {
|
||||
let tags = metadata
|
||||
.tags
|
||||
.iter()
|
||||
.map(|(k, v)| format!("{k}: {v}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
self.lexical.set_media_metadata(
|
||||
&path.to_string_lossy(),
|
||||
&tags,
|
||||
metadata.duration_ms,
|
||||
identity.bytes,
|
||||
identity.modified_ns,
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn build_media_file(
|
||||
&mut self,
|
||||
@@ -105,16 +207,9 @@ impl HybridSearch {
|
||||
kind: MediaKind,
|
||||
tools: &FfmpegTools,
|
||||
limits: &MediaLimits,
|
||||
identity: &MediaIdentity,
|
||||
report: &mut EmbeddingReport,
|
||||
) -> Result<()> {
|
||||
let identity = MediaIdentity::read(
|
||||
path,
|
||||
if kind == MediaKind::Image {
|
||||
64 * 1024 * 1024
|
||||
} else {
|
||||
limits.max_file_bytes
|
||||
},
|
||||
)?;
|
||||
let mut ordinal = 0;
|
||||
if kind == MediaKind::Image {
|
||||
validate_image(path)?;
|
||||
@@ -122,29 +217,18 @@ impl HybridSearch {
|
||||
report.chunks < MAX_SCAN_CHUNKS,
|
||||
"Media exceeds generation chunk budget"
|
||||
);
|
||||
report.image_inference_calls += 1;
|
||||
let vectors = self
|
||||
.embedder
|
||||
.embed_images(&[path], 280)
|
||||
.map_err(|e| anyhow::Error::new(InferenceFailure(e)))?;
|
||||
ensure!(vectors.len() == 1, "Image embedding count mismatch");
|
||||
self.store
|
||||
.insert_media(generation, id, ordinal, &identity, "image", 0, &vectors[0])?;
|
||||
.insert_media(generation, id, ordinal, identity, "image", 0, &vectors[0])?;
|
||||
report.chunks += 1;
|
||||
} else {
|
||||
let metadata = probe(tools, path, limits)?;
|
||||
let tags = metadata
|
||||
.tags
|
||||
.iter()
|
||||
.map(|(k, v)| format!("{k}: {v}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
self.lexical.set_media_metadata(
|
||||
&path.to_string_lossy(),
|
||||
&tags,
|
||||
metadata.duration_ms,
|
||||
identity.bytes,
|
||||
identity.modified_ns,
|
||||
)?;
|
||||
self.refresh_media_metadata(path, identity, &metadata)?;
|
||||
if metadata.has_video && kind == MediaKind::Video {
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
let mut scenes = 0;
|
||||
@@ -168,6 +252,7 @@ impl HybridSearch {
|
||||
report.chunks < MAX_SCAN_CHUNKS,
|
||||
"Media exceeds generation chunk budget"
|
||||
);
|
||||
report.image_inference_calls += 1;
|
||||
let vectors = self
|
||||
.embedder
|
||||
.embed_images(&[frame.path.as_path()], 70)
|
||||
@@ -177,7 +262,7 @@ impl HybridSearch {
|
||||
generation,
|
||||
id,
|
||||
ordinal,
|
||||
&identity,
|
||||
identity,
|
||||
"video",
|
||||
frame.timestamp_ms,
|
||||
&vectors[0],
|
||||
@@ -194,6 +279,7 @@ impl HybridSearch {
|
||||
"Audio exceeds generation chunk budget"
|
||||
);
|
||||
let pcm = decode_audio(tools, path, window, limits)?;
|
||||
report.audio_inference_calls += 1;
|
||||
let vector = self
|
||||
.embedder
|
||||
.embed_audio(&pcm)
|
||||
@@ -202,7 +288,7 @@ impl HybridSearch {
|
||||
generation,
|
||||
id,
|
||||
ordinal,
|
||||
&identity,
|
||||
identity,
|
||||
"audio",
|
||||
window.start_ms,
|
||||
&vector,
|
||||
|
||||
Reference in new issue
Block a user