906 lines
30 KiB
Rust
906 lines
30 KiB
Rust
//! Local bounded media preparation. No network, model, executable installer, or
|
|
//! fabricated caption/transcript lives here. FFmpeg paths come from the caller.
|
|
//!
|
|
//! Video coverage is distributed sampling, not a guarantee that every scene is
|
|
//! represented. Audio windows cover the complete primary stream or return an
|
|
//! explicit budget error. Decoder process memory is not sandbox-limited yet.
|
|
use anyhow::{bail, ensure, Context, Result};
|
|
use serde::{Deserialize, Serialize};
|
|
use sha2::{Digest, Sha256};
|
|
use std::{
|
|
collections::BTreeMap,
|
|
fs::{self, File},
|
|
io::Read,
|
|
path::{Path, PathBuf},
|
|
process::{Command, Stdio},
|
|
thread,
|
|
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
|
|
};
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
pub enum MediaKind {
|
|
Image,
|
|
Video,
|
|
Audio,
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn signed_unix_time_preserves_epoch_and_storage_boundaries() -> Result<()> {
|
|
// Windows SystemTime has 100 ns precision; storage conversion itself
|
|
// must still handle every signed nanosecond boundary exactly.
|
|
assert_eq!(unix_time_ns(UNIX_EPOCH)?, 0);
|
|
assert_eq!(unix_time_ns(UNIX_EPOCH + Duration::from_nanos(100))?, 100);
|
|
assert_eq!(unix_time_ns(UNIX_EPOCH - Duration::from_nanos(100))?, -100);
|
|
assert_eq!(
|
|
signed_duration_ns(Duration::from_nanos(i64::MAX as u64), false)?,
|
|
i64::MAX
|
|
);
|
|
assert_eq!(
|
|
signed_duration_ns(Duration::from_nanos(i64::MAX as u64 + 1), true)?,
|
|
i64::MIN
|
|
);
|
|
assert!(signed_duration_ns(Duration::from_nanos(i64::MAX as u64 + 1), false).is_err());
|
|
assert!(signed_duration_ns(Duration::from_nanos(i64::MAX as u64 + 2), true).is_err());
|
|
Ok(())
|
|
}
|
|
|
|
fn fixture(duration_ms: u32) -> MediaProbe {
|
|
MediaProbe {
|
|
duration_ms,
|
|
start_ms: 0,
|
|
video_stream_index: 0,
|
|
width: 64,
|
|
height: 64,
|
|
has_video: true,
|
|
has_audio: true,
|
|
audio_start_ms: 0,
|
|
audio_end_ms: duration_ms,
|
|
tags: BTreeMap::new(),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn audio_covers_tail_with_overlap_and_rejects_budget() -> Result<()> {
|
|
let probe = fixture(65_000);
|
|
let limits = MediaLimits::default();
|
|
let windows = audio_windows(&probe, &limits)?;
|
|
assert_eq!(
|
|
windows.iter().map(|w| w.start_ms).collect::<Vec<_>>(),
|
|
vec![0, 28_000, 56_000]
|
|
);
|
|
assert_eq!(windows.last().unwrap().duration_ms, 9_000);
|
|
let tight = MediaLimits {
|
|
max_audio_windows: 2,
|
|
..limits
|
|
};
|
|
assert!(audio_windows(&probe, &tight).is_err());
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn finer_audio_windows_cover_long_movie_or_explicitly_reject() -> Result<()> {
|
|
let probe = fixture(3 * 60 * 60 * 1000);
|
|
let limits = MediaLimits {
|
|
audio_window_ms: 10_000,
|
|
audio_overlap_ms: 2000,
|
|
max_audio_windows: 1800,
|
|
..Default::default()
|
|
};
|
|
let windows = audio_windows(&probe, &limits)?;
|
|
assert_eq!(windows.first().unwrap().start_ms, 0);
|
|
let last = windows.last().unwrap();
|
|
assert_eq!(last.start_ms + last.duration_ms, probe.duration_ms);
|
|
assert!(windows.len() >= 1300 && windows.len() <= 1440);
|
|
let too_small = MediaLimits {
|
|
max_audio_windows: 1000,
|
|
..limits
|
|
};
|
|
assert!(audio_windows(&probe, &too_small).is_err());
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn frame_budget_spans_entire_movie() -> Result<()> {
|
|
let probe = fixture(3_600_000);
|
|
let times = frame_times(&probe, &MediaLimits::default())?;
|
|
assert!(times.len() <= 240);
|
|
assert_eq!(times.first(), Some(&0));
|
|
assert_eq!(times.last(), Some(&3_599_500));
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn owned_frame_scratch_validates_parent_and_cleans_batch() -> Result<()> {
|
|
let root = tempfile::tempdir()?;
|
|
let parent = root.path().join("owned scratch");
|
|
let limits = MediaLimits {
|
|
scratch_directory: Some(parent.clone()),
|
|
..Default::default()
|
|
};
|
|
let batch = frame_scratch(&limits)?;
|
|
assert_eq!(batch.path().parent(), Some(parent.as_path()));
|
|
let batch_path = batch.path().to_owned();
|
|
fs::write(batch.path().join("frame.png"), b"owned temporary bytes")?;
|
|
drop(batch);
|
|
assert!(!batch_path.exists());
|
|
assert!(parent.is_dir());
|
|
let relative = MediaLimits {
|
|
scratch_directory: Some(PathBuf::from("relative")),
|
|
..Default::default()
|
|
};
|
|
assert!(frame_scratch(&relative).is_err());
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
fn identity_rejects_changes_and_oversized_input() -> Result<()> {
|
|
let temp = tempfile::tempdir()?;
|
|
let file = temp.path().join("audio.wav");
|
|
fs::write(&file, b"first")?;
|
|
assert!(MediaIdentity::read(&file, 4).is_err());
|
|
let identity = MediaIdentity::read(&file, 10)?;
|
|
fs::write(&file, b"changed size")?;
|
|
assert!(identity.ensure_unchanged().is_err());
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "Requires explicitly provisioned CPU FFmpeg/ffprobe executables"]
|
|
fn decoder_rejects_network_references_inside_local_media() -> Result<()> {
|
|
let tools = FfmpegTools::from_environment()?.context("Set CPU decoder paths")?;
|
|
let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
|
|
listener.set_nonblocking(true)?;
|
|
let temp = tempfile::tempdir()?;
|
|
let file = temp.path().join("local.m3u8");
|
|
fs::write(&file,format!("#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:2\n#EXTINF:2.0,\nhttp://{}/segment.ts\n#EXT-X-ENDLIST\n",listener.local_addr()?))?;
|
|
let limits = MediaLimits {
|
|
command_timeout: Duration::from_secs(3),
|
|
..Default::default()
|
|
};
|
|
let error = super::probe(&tools, &file, &limits).unwrap_err();
|
|
assert!(format!("{error:#}").contains("whitelist"), "{error:#}");
|
|
assert!(decode_frames(&tools, &file, &fixture(2000), 0, false, &limits).is_err());
|
|
assert!(decode_audio(
|
|
&tools,
|
|
&file,
|
|
AudioWindow {
|
|
start_ms: 0,
|
|
duration_ms: 2000
|
|
},
|
|
&limits
|
|
)
|
|
.is_err());
|
|
match listener.accept() {
|
|
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {}
|
|
other => panic!("Decoder attempted network connection: {other:?}"),
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "Requires explicitly provisioned CPU FFmpeg/ffprobe executables"]
|
|
fn real_decoder_preserves_frame_timestamps_and_audio() -> Result<()> {
|
|
let tools = FfmpegTools::from_environment()?.context("Set CPU decoder paths")?;
|
|
let temp = tempfile::tempdir()?;
|
|
let video = temp.path().join("fixture.mkv");
|
|
let mut command = Command::new(&tools.ffmpeg);
|
|
command
|
|
.args([
|
|
"-nostdin",
|
|
"-v",
|
|
"error",
|
|
"-threads",
|
|
"1",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"color=c=red:s=64x64:r=10:d=4",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"sine=frequency=440:sample_rate=16000:duration=4",
|
|
"-c:v",
|
|
"ffv1",
|
|
"-c:a",
|
|
"pcm_s16le",
|
|
"-threads",
|
|
"1",
|
|
"-y",
|
|
])
|
|
.arg(&video);
|
|
run(command, Duration::from_secs(30), 4096, 65536)?;
|
|
let scratch = temp.path().join("owned scratch");
|
|
let limits = MediaLimits {
|
|
scratch_directory: Some(scratch.clone()),
|
|
..Default::default()
|
|
};
|
|
let probe = probe(&tools, &video, &limits)?;
|
|
assert!(probe.has_video && probe.has_audio);
|
|
let frames = decode_frames(&tools, &video, &probe, 2000, false, &limits)?;
|
|
assert!(frames.directory().starts_with(&scratch));
|
|
assert!(frames
|
|
.frames
|
|
.iter()
|
|
.all(|frame| frame.path.starts_with(&scratch)));
|
|
assert_eq!(frames.frames.len(), 1);
|
|
assert!((1900..=2100).contains(&frames.frames[0].timestamp_ms));
|
|
let audio = decode_audio(
|
|
&tools,
|
|
&video,
|
|
AudioWindow {
|
|
start_ms: 1000,
|
|
duration_ms: 2000,
|
|
},
|
|
&limits,
|
|
)?;
|
|
assert!((31_000..=32_000).contains(&audio.len()));
|
|
assert!(audio.iter().any(|x| x.abs() > 0.01));
|
|
assert!(audio.iter().all(|x| x.is_finite() && x.abs() <= 1.0));
|
|
let ogg = temp.path().join("overshoot.ogg");
|
|
let mut command = Command::new(&tools.ffmpeg);
|
|
command
|
|
.args([
|
|
"-nostdin",
|
|
"-v",
|
|
"error",
|
|
"-threads",
|
|
"1",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
"sine=frequency=440:sample_rate=44100:duration=31.123",
|
|
"-c:a",
|
|
"libvorbis",
|
|
"-metadata",
|
|
"artist=CPU decoder fixture",
|
|
"-threads",
|
|
"1",
|
|
"-y",
|
|
])
|
|
.arg(&ogg);
|
|
run(command, Duration::from_secs(30), 4096, 65536)?;
|
|
let music_probe = super::probe(&tools, &ogg, &limits)?;
|
|
assert_eq!(
|
|
music_probe.tags.get("artist").map(String::as_str),
|
|
Some("CPU decoder fixture")
|
|
);
|
|
let pcm = decode_audio(
|
|
&tools,
|
|
&ogg,
|
|
AudioWindow {
|
|
start_ms: 0,
|
|
duration_ms: 30000,
|
|
},
|
|
&limits,
|
|
)?;
|
|
assert_eq!(
|
|
pcm.len(),
|
|
480000,
|
|
"Vorbis packets must not overshoot model limit"
|
|
);
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
pub fn classify(path: &Path) -> Option<MediaKind> {
|
|
match path.extension()?.to_str()?.to_ascii_lowercase().as_str() {
|
|
"png" | "jpg" | "jpeg" | "webp" | "bmp" => Some(MediaKind::Image),
|
|
"mp4" | "mkv" | "mov" | "avi" | "webm" | "m4v" | "mpeg" | "mpg" => Some(MediaKind::Video),
|
|
"mp3" | "flac" | "wav" | "m4a" | "ogg" | "opus" | "aac" | "aiff" | "aif" => {
|
|
Some(MediaKind::Audio)
|
|
}
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
/// Validate bounded raster decoding before treating model errors as fatal.
|
|
/// The provider still performs its own bounds and EXIF-aware preprocessing.
|
|
pub fn validate_image(path: &Path) -> Result<()> {
|
|
ensure!(
|
|
fs::metadata(path)?.len() <= 64 * 1024 * 1024,
|
|
"Image exceeds 64 MiB"
|
|
);
|
|
let reader = image::ImageReader::open(path)?.with_guessed_format()?;
|
|
let (width, height) = reader.into_dimensions()?;
|
|
ensure!(
|
|
width <= 16384 && height <= 16384 && u64::from(width) * u64::from(height) <= 16_000_000,
|
|
"Image exceeds 16M pixels"
|
|
);
|
|
let mut reader = image::ImageReader::open(path)?.with_guessed_format()?;
|
|
let mut limits = image::Limits::default();
|
|
limits.max_alloc = Some(256 * 1024 * 1024);
|
|
limits.max_image_width = Some(16384);
|
|
limits.max_image_height = Some(16384);
|
|
reader.limits(limits);
|
|
use image::ImageDecoder;
|
|
let mut decoder = reader.into_decoder()?;
|
|
decoder
|
|
.orientation()
|
|
.context("Invalid image EXIF orientation")?;
|
|
image::DynamicImage::from_decoder(decoder).context("Image decoder rejected corrupt content")?;
|
|
Ok(())
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct FfmpegTools {
|
|
pub ffmpeg: PathBuf,
|
|
pub ffprobe: PathBuf,
|
|
}
|
|
impl FfmpegTools {
|
|
pub fn new(ffmpeg: impl AsRef<Path>, ffprobe: impl AsRef<Path>) -> Result<Self> {
|
|
ensure!(
|
|
ffmpeg.as_ref().is_file() && ffprobe.as_ref().is_file(),
|
|
"FFmpeg/ffprobe executable is missing"
|
|
);
|
|
Ok(Self {
|
|
ffmpeg: fs::canonicalize(ffmpeg)?,
|
|
ffprobe: fs::canonicalize(ffprobe)?,
|
|
})
|
|
}
|
|
pub fn from_environment() -> Result<Option<Self>> {
|
|
match (
|
|
std::env::var_os("SEARCHI_FFMPEG_PATH"),
|
|
std::env::var_os("SEARCHI_FFPROBE_PATH"),
|
|
) {
|
|
(None, None) => Ok(None),
|
|
(Some(ffmpeg), Some(ffprobe)) => Self::new(ffmpeg, ffprobe).map(Some),
|
|
_ => bail!("Set both SEARCHI_FFMPEG_PATH and SEARCHI_FFPROBE_PATH"),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct MediaLimits {
|
|
pub max_file_bytes: u64,
|
|
pub max_duration_ms: u32,
|
|
pub max_frames: usize,
|
|
pub frame_interval_ms: u32,
|
|
pub max_scene_frames: usize,
|
|
pub audio_window_ms: u32,
|
|
pub audio_overlap_ms: u32,
|
|
pub max_audio_windows: usize,
|
|
pub command_timeout: Duration,
|
|
/// Owned decoded-frame scratch parent. Desktop callers place this below
|
|
/// excluded application data to avoid indexing their own temporary frames.
|
|
/// None preserves the normal operating-system temporary directory.
|
|
pub scratch_directory: Option<PathBuf>,
|
|
}
|
|
impl Default for MediaLimits {
|
|
fn default() -> Self {
|
|
Self {
|
|
max_file_bytes: 16 * 1024 * 1024 * 1024,
|
|
max_duration_ms: 4 * 60 * 60 * 1000,
|
|
max_frames: 240,
|
|
frame_interval_ms: 2000,
|
|
max_scene_frames: 120,
|
|
audio_window_ms: 30_000,
|
|
audio_overlap_ms: 2000,
|
|
max_audio_windows: 600,
|
|
command_timeout: Duration::from_secs(45),
|
|
scratch_directory: None,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub struct MediaIdentity {
|
|
pub path: PathBuf,
|
|
pub bytes: u64,
|
|
pub modified_ns: i64,
|
|
pub sha256: String,
|
|
}
|
|
impl MediaIdentity {
|
|
pub fn read(path: &Path, maximum: u64) -> Result<Self> {
|
|
let file = File::open(path)?;
|
|
let before = file.metadata()?;
|
|
ensure!(
|
|
before.len() <= maximum,
|
|
"Media file exceeds {maximum} bytes"
|
|
);
|
|
let modified_ns = modified_ns(&before)?;
|
|
let mut reader = file.take(maximum.saturating_add(1));
|
|
let mut buffer = [0u8; 65536];
|
|
let mut hash = Sha256::new();
|
|
let mut count = 0u64;
|
|
loop {
|
|
let read = reader.read(&mut buffer)?;
|
|
if read == 0 {
|
|
break;
|
|
}
|
|
count += read as u64;
|
|
ensure!(count <= maximum, "Media grew past file budget");
|
|
hash.update(&buffer[..read]);
|
|
}
|
|
let identity = Self {
|
|
path: fs::canonicalize(path)?,
|
|
bytes: count,
|
|
modified_ns,
|
|
sha256: format!("{:x}", hash.finalize()),
|
|
};
|
|
ensure!(count == before.len(), "Media changed while fingerprinting");
|
|
identity.ensure_unchanged()?;
|
|
Ok(identity)
|
|
}
|
|
pub fn ensure_unchanged(&self) -> Result<()> {
|
|
let metadata = fs::metadata(&self.path)?;
|
|
ensure!(
|
|
metadata.len() == self.bytes && modified_ns(&metadata)? == self.modified_ns,
|
|
"Media changed during indexing"
|
|
);
|
|
Ok(())
|
|
}
|
|
}
|
|
/// Convert a filesystem timestamp to signed nanoseconds since the Unix epoch.
|
|
/// Times outside the signed 64-bit storage range are rejected explicitly.
|
|
pub fn unix_time_ns(time: SystemTime) -> Result<i64> {
|
|
match time.duration_since(UNIX_EPOCH) {
|
|
Ok(duration) => signed_duration_ns(duration, false),
|
|
Err(error) => signed_duration_ns(error.duration(), true),
|
|
}
|
|
}
|
|
|
|
fn signed_duration_ns(duration: Duration, before_epoch: bool) -> Result<i64> {
|
|
// Duration's u64 seconds fit in i128 nanoseconds, including i64::MIN's
|
|
// magnitude, which cannot first be represented as positive i64.
|
|
let nanos = duration.as_nanos() as i128;
|
|
i64::try_from(if before_epoch { -nanos } else { nanos })
|
|
.context("File timestamp exceeds supported nanosecond range")
|
|
}
|
|
|
|
fn modified_ns(metadata: &fs::Metadata) -> Result<i64> {
|
|
unix_time_ns(metadata.modified()?)
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub struct MediaProbe {
|
|
pub duration_ms: u32,
|
|
pub start_ms: i64,
|
|
pub video_stream_index: u32,
|
|
pub width: u32,
|
|
pub height: u32,
|
|
pub has_video: bool,
|
|
pub has_audio: bool,
|
|
pub audio_start_ms: u32,
|
|
pub audio_end_ms: u32,
|
|
pub tags: BTreeMap<String, String>,
|
|
}
|
|
#[derive(Deserialize)]
|
|
struct ProbeJson {
|
|
#[serde(default)]
|
|
streams: Vec<ProbeStream>,
|
|
format: ProbeFormat,
|
|
}
|
|
#[derive(Deserialize)]
|
|
struct ProbeFormat {
|
|
duration: String,
|
|
#[serde(default)]
|
|
start_time: Option<String>,
|
|
#[serde(default)]
|
|
tags: BTreeMap<String, String>,
|
|
}
|
|
#[derive(Deserialize)]
|
|
struct ProbeStream {
|
|
index: u32,
|
|
codec_type: String,
|
|
#[serde(default)]
|
|
width: u32,
|
|
#[serde(default)]
|
|
height: u32,
|
|
#[serde(default)]
|
|
duration: Option<String>,
|
|
#[serde(default)]
|
|
start_time: Option<String>,
|
|
#[serde(default)]
|
|
tags: BTreeMap<String, String>,
|
|
#[serde(default)]
|
|
disposition: BTreeMap<String, i64>,
|
|
}
|
|
fn milliseconds(value: &str) -> Result<i64> {
|
|
let value: f64 = value.parse()?;
|
|
ensure!(
|
|
value.is_finite() && value.abs() < 100_000_000.0,
|
|
"Invalid media duration/timestamp"
|
|
);
|
|
Ok((value * 1000.0).round() as i64)
|
|
}
|
|
|
|
pub fn probe(tools: &FfmpegTools, path: &Path, limits: &MediaLimits) -> Result<MediaProbe> {
|
|
let mut command = Command::new(&tools.ffprobe);
|
|
command.args(["-protocol_whitelist", "file,pipe"]);
|
|
command
|
|
.args([
|
|
"-v",
|
|
"error",
|
|
"-threads",
|
|
"1",
|
|
"-probesize",
|
|
"5000000",
|
|
"-analyzeduration",
|
|
"5000000",
|
|
"-show_streams",
|
|
"-show_format",
|
|
"-of",
|
|
"json",
|
|
])
|
|
.arg(path);
|
|
let output = run(command, limits.command_timeout, 65536, 65536)?;
|
|
let parsed: ProbeJson =
|
|
serde_json::from_slice(&output.stdout).context("Invalid ffprobe JSON")?;
|
|
let duration = milliseconds(&parsed.format.duration)?;
|
|
ensure!(
|
|
duration > 0 && duration <= i64::from(limits.max_duration_ms),
|
|
"Media duration exceeds policy or is unknown"
|
|
);
|
|
let start = parsed
|
|
.format
|
|
.start_time
|
|
.as_deref()
|
|
.map(milliseconds)
|
|
.transpose()?
|
|
.unwrap_or(0);
|
|
let video = parsed.streams.iter().find(|stream| {
|
|
stream.codec_type == "video" && stream.disposition.get("attached_pic") != Some(&1)
|
|
});
|
|
if let Some(video) = video {
|
|
ensure!(
|
|
u64::from(video.width) * u64::from(video.height) <= 16_000_000,
|
|
"Video dimensions exceed 16M pixels"
|
|
);
|
|
}
|
|
let audio = parsed
|
|
.streams
|
|
.iter()
|
|
.find(|stream| stream.codec_type == "audio");
|
|
let audio_start = audio
|
|
.and_then(|stream| stream.start_time.as_deref())
|
|
.map(milliseconds)
|
|
.transpose()?
|
|
.unwrap_or(start)
|
|
- start;
|
|
let audio_length = audio
|
|
.and_then(|stream| stream.duration.as_deref())
|
|
.map(milliseconds)
|
|
.transpose()?
|
|
.unwrap_or(duration);
|
|
let mut tags = parsed.format.tags;
|
|
for stream in &parsed.streams {
|
|
for (key, value) in &stream.tags {
|
|
tags.entry(key.clone()).or_insert_with(|| value.clone());
|
|
}
|
|
}
|
|
tags = tags
|
|
.into_iter()
|
|
.map(|(key, value)| {
|
|
(
|
|
key.to_ascii_lowercase(),
|
|
value.chars().take(2048).collect::<String>(),
|
|
)
|
|
})
|
|
.filter(|(key, _)| {
|
|
matches!(
|
|
key.as_str(),
|
|
"title"
|
|
| "artist"
|
|
| "album"
|
|
| "album_artist"
|
|
| "genre"
|
|
| "comment"
|
|
| "date"
|
|
| "composer"
|
|
| "description"
|
|
| "track"
|
|
)
|
|
})
|
|
.take(16)
|
|
.collect();
|
|
Ok(MediaProbe {
|
|
duration_ms: duration as u32,
|
|
start_ms: start,
|
|
video_stream_index: video.map_or(0, |stream| stream.index),
|
|
width: video.map_or(0, |stream| stream.width),
|
|
height: video.map_or(0, |stream| stream.height),
|
|
has_video: video.is_some(),
|
|
has_audio: audio.is_some(),
|
|
audio_start_ms: audio_start.max(0) as u32,
|
|
audio_end_ms: (audio_start + audio_length).clamp(0, duration) as u32,
|
|
tags,
|
|
})
|
|
}
|
|
|
|
/// Uniform coverage is distributed over the entire movie; this is not a prefix
|
|
/// truncation. Scene probes may add extra local cuts around these anchors.
|
|
pub fn frame_times(probe: &MediaProbe, limits: &MediaLimits) -> Result<Vec<u32>> {
|
|
if !probe.has_video {
|
|
return Ok(Vec::new());
|
|
}
|
|
ensure!(
|
|
limits.max_frames >= 2 && limits.max_frames <= 2000,
|
|
"Frame budget must be 2..2000"
|
|
);
|
|
let last = probe.duration_ms.saturating_sub(500);
|
|
let interval = limits
|
|
.frame_interval_ms
|
|
.max(last.div_ceil((limits.max_frames - 1) as u32))
|
|
.max(1);
|
|
let mut times = Vec::new();
|
|
let mut time = 0;
|
|
while time < last {
|
|
times.push(time);
|
|
time = time.saturating_add(interval);
|
|
}
|
|
times.push(last);
|
|
ensure!(
|
|
times.len() <= limits.max_frames,
|
|
"Frame sampling budget exceeded"
|
|
);
|
|
Ok(times)
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, Serialize)]
|
|
pub struct AudioWindow {
|
|
pub start_ms: u32,
|
|
pub duration_ms: u32,
|
|
}
|
|
pub fn audio_windows(probe: &MediaProbe, limits: &MediaLimits) -> Result<Vec<AudioWindow>> {
|
|
if !probe.has_audio {
|
|
return Ok(Vec::new());
|
|
}
|
|
ensure!(
|
|
limits.audio_window_ms >= 5_000
|
|
&& limits.audio_window_ms <= 30_000
|
|
&& limits.audio_overlap_ms < limits.audio_window_ms,
|
|
"Invalid audio window policy"
|
|
);
|
|
let step = limits.audio_window_ms - limits.audio_overlap_ms;
|
|
let mut windows = Vec::new();
|
|
let mut start = probe.audio_start_ms;
|
|
while start < probe.audio_end_ms {
|
|
ensure!(
|
|
windows.len() < limits.max_audio_windows,
|
|
"Whole audio coverage exceeds window budget; no partial index produced"
|
|
);
|
|
windows.push(AudioWindow {
|
|
start_ms: start,
|
|
duration_ms: limits.audio_window_ms.min(probe.audio_end_ms - start),
|
|
});
|
|
if start + limits.audio_window_ms >= probe.audio_end_ms {
|
|
break;
|
|
}
|
|
start = start.saturating_add(step);
|
|
}
|
|
Ok(windows)
|
|
}
|
|
|
|
pub struct DecodedFrame {
|
|
pub path: PathBuf,
|
|
pub timestamp_ms: u32,
|
|
}
|
|
pub struct FrameBatch {
|
|
directory: tempfile::TempDir,
|
|
pub frames: Vec<DecodedFrame>,
|
|
}
|
|
impl FrameBatch {
|
|
pub fn directory(&self) -> &Path {
|
|
self.directory.path()
|
|
}
|
|
}
|
|
fn frame_scratch(limits: &MediaLimits) -> Result<tempfile::TempDir> {
|
|
match &limits.scratch_directory {
|
|
Some(parent) => {
|
|
ensure!(
|
|
parent.is_absolute(),
|
|
"Media scratch directory must be absolute"
|
|
);
|
|
fs::create_dir_all(parent).context("Create owned media scratch directory")?;
|
|
tempfile::Builder::new()
|
|
.prefix("searchi-frames-")
|
|
.tempdir_in(parent)
|
|
.context("Create owned frame batch")
|
|
}
|
|
None => tempfile::tempdir().context("Create frame batch"),
|
|
}
|
|
}
|
|
|
|
pub fn decode_frames(
|
|
tools: &FfmpegTools,
|
|
path: &Path,
|
|
probe: &MediaProbe,
|
|
start_ms: u32,
|
|
include_scene: bool,
|
|
limits: &MediaLimits,
|
|
) -> Result<FrameBatch> {
|
|
let directory = frame_scratch(limits)?;
|
|
let mut command = Command::new(&tools.ffmpeg);
|
|
command.args(["-protocol_whitelist", "file,pipe"]);
|
|
command.args(["-nostdin","-hide_banner","-loglevel","info","-threads","1","-hwaccel","none","-copyts","-ss"])
|
|
.arg(format!("{:.3}",f64::from(start_ms)/1000.0)).args(["-t",if include_scene {"1.5"} else {"0.5"},"-i"]).arg(path)
|
|
.arg("-map").arg(format!("0:{}",probe.video_stream_index)).args(["-an","-filter_threads","1","-vf"])
|
|
.arg(if include_scene {"select=eq(n\\,0)+gt(scene\\,0.32),showinfo,scale=960:960:force_original_aspect_ratio=decrease"}
|
|
else {"showinfo,scale=960:960:force_original_aspect_ratio=decrease"})
|
|
.args(["-frames:v",if include_scene {"2"} else {"1"},"-fps_mode","vfr","-threads","1","-y"])
|
|
.arg(directory.path().join("frame-%02d.png"));
|
|
let output = run(command, limits.command_timeout, 4096, 256 * 1024)?;
|
|
let mut timestamps = Vec::new();
|
|
for line in String::from_utf8_lossy(&output.stderr).lines() {
|
|
if line.contains("showinfo") {
|
|
if let Some((_, tail)) = line.split_once("pts_time:") {
|
|
let value = tail
|
|
.split_whitespace()
|
|
.next()
|
|
.context("Missing frame PTS")?;
|
|
let actual = (milliseconds(value)? - probe.start_ms).max(0);
|
|
ensure!(
|
|
actual <= i64::from(probe.duration_ms) + 1000,
|
|
"Frame PTS lies beyond media timeline"
|
|
);
|
|
timestamps.push(actual as u32);
|
|
}
|
|
}
|
|
}
|
|
let mut frames = Vec::new();
|
|
for ordinal in 1..=if include_scene { 2 } else { 1 } {
|
|
let file = directory.path().join(format!("frame-{ordinal:02}.png"));
|
|
if file.is_file() {
|
|
ensure!(
|
|
fs::metadata(&file)?.len() <= 8 * 1024 * 1024,
|
|
"Decoded frame exceeds 8MiB"
|
|
);
|
|
let timestamp_ms = *timestamps
|
|
.get(ordinal - 1)
|
|
.context("Decoder did not report genuine frame PTS")?;
|
|
frames.push(DecodedFrame {
|
|
path: file,
|
|
timestamp_ms,
|
|
});
|
|
}
|
|
}
|
|
ensure!(
|
|
!frames.is_empty(),
|
|
"Decoder produced no frame at requested timeline position"
|
|
);
|
|
Ok(FrameBatch { directory, frames })
|
|
}
|
|
|
|
pub fn decode_audio(
|
|
tools: &FfmpegTools,
|
|
path: &Path,
|
|
window: AudioWindow,
|
|
limits: &MediaLimits,
|
|
) -> Result<Vec<f32>> {
|
|
decode_audio_inner(tools, path, window, limits, false)
|
|
}
|
|
|
|
/// Preserve the source clock across seek offsets and discontinuities for speech citations.
|
|
/// Silence is derived from decoded frame timestamps, never semantic guesses.
|
|
pub fn decode_audio_aligned(
|
|
tools: &FfmpegTools,
|
|
path: &Path,
|
|
window: AudioWindow,
|
|
limits: &MediaLimits,
|
|
) -> Result<Vec<f32>> {
|
|
decode_audio_inner(tools, path, window, limits, true)
|
|
}
|
|
|
|
fn decode_audio_inner(
|
|
tools: &FfmpegTools,
|
|
path: &Path,
|
|
window: AudioWindow,
|
|
limits: &MediaLimits,
|
|
aligned: bool,
|
|
) -> Result<Vec<f32>> {
|
|
ensure!(
|
|
window.duration_ms > 0 && window.duration_ms <= 30_000,
|
|
"Audio decode window exceeds 30 seconds"
|
|
);
|
|
let mut command = Command::new(&tools.ffmpeg);
|
|
command.args(["-protocol_whitelist", "file,pipe"]);
|
|
command
|
|
.args([
|
|
"-nostdin",
|
|
"-hide_banner",
|
|
"-v",
|
|
"error",
|
|
"-threads",
|
|
"1",
|
|
"-ss",
|
|
])
|
|
.arg(format!("{:.3}", f64::from(window.start_ms) / 1000.0))
|
|
.args(["-t"])
|
|
.arg(format!("{:.3}", f64::from(window.duration_ms) / 1000.0))
|
|
.arg("-i")
|
|
.arg(path)
|
|
.arg("-af")
|
|
.arg(format!(
|
|
"{},atrim=end_sample={},asetpts=PTS-STARTPTS",
|
|
if aligned {
|
|
"aresample=16000:async=1:first_pts=0"
|
|
} else {
|
|
"aresample=16000"
|
|
},
|
|
u64::from(window.duration_ms) * 16
|
|
))
|
|
.args([
|
|
"-map", "0:a:0", "-vn", "-ac", "1", "-ar", "16000", "-threads", "1", "-f", "s16le",
|
|
"pipe:1",
|
|
]);
|
|
let output = run(command, limits.command_timeout, 960_000, 65536)?;
|
|
ensure!(
|
|
!output.stdout.is_empty() && output.stdout.len() % 2 == 0,
|
|
"Empty or invalid decoded PCM audio"
|
|
);
|
|
Ok(output
|
|
.stdout
|
|
.as_chunks::<2>()
|
|
.0
|
|
.iter()
|
|
.map(|bytes| f32::from(i16::from_le_bytes([bytes[0], bytes[1]])) / 32768.0)
|
|
.collect())
|
|
}
|
|
|
|
struct Output {
|
|
stdout: Vec<u8>,
|
|
stderr: Vec<u8>,
|
|
}
|
|
fn run(
|
|
mut command: Command,
|
|
timeout: Duration,
|
|
stdout_limit: usize,
|
|
stderr_limit: usize,
|
|
) -> Result<Output> {
|
|
#[cfg(windows)]
|
|
{
|
|
use std::os::windows::process::CommandExt;
|
|
command.creation_flags(0x08000000);
|
|
}
|
|
let mut child = command
|
|
.stdin(Stdio::null())
|
|
.stdout(Stdio::piped())
|
|
.stderr(Stdio::piped())
|
|
.spawn()
|
|
.context("Launch configured FFmpeg tool")?;
|
|
let stdout = child.stdout.take().context("Missing decoder stdout")?;
|
|
let stderr = child.stderr.take().context("Missing decoder stderr")?;
|
|
let read = |reader: Box<dyn Read + Send>, limit: usize| {
|
|
thread::spawn(move || -> Result<Vec<u8>> {
|
|
let mut bytes = Vec::new();
|
|
reader.take(limit as u64 + 1).read_to_end(&mut bytes)?;
|
|
ensure!(bytes.len() <= limit, "Decoder output exceeded byte budget");
|
|
Ok(bytes)
|
|
})
|
|
};
|
|
let out = read(Box::new(stdout), stdout_limit);
|
|
let err = read(Box::new(stderr), stderr_limit);
|
|
let start = Instant::now();
|
|
let status = loop {
|
|
if let Some(status) = child.try_wait()? {
|
|
break Some(status);
|
|
}
|
|
if start.elapsed() > timeout {
|
|
child.kill()?;
|
|
child.wait()?;
|
|
break None;
|
|
}
|
|
thread::sleep(Duration::from_millis(10));
|
|
};
|
|
let stdout = out
|
|
.join()
|
|
.map_err(|_| anyhow::anyhow!("Decoder stdout reader panicked"))??;
|
|
let stderr = err
|
|
.join()
|
|
.map_err(|_| anyhow::anyhow!("Decoder stderr reader panicked"))??;
|
|
let status = status.context("Decoder exceeded time budget")?;
|
|
ensure!(
|
|
status.success(),
|
|
"Decoder failed: {}",
|
|
String::from_utf8_lossy(&stderr)
|
|
.chars()
|
|
.take(2048)
|
|
.collect::<String>()
|
|
);
|
|
Ok(Output { stdout, stderr })
|
|
}
|