videoctx 0.1.0__py3-none-any.whl

videoctx/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

videoctx/captions.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import base64
+from pathlib import Path
+
+import anthropic
+
+from .pricing import CostMeter
+from .schema import VideoContext
+
+CAPTION_MODEL = "claude-haiku-4-5"
+
+CAPTION_PROMPT = (
+    "Describe this video frame in ONE dense line: subjects, actions, setting, "
+    "notable objects, and any readable on-screen text (quote it). No preamble."
+)
+
+
+def caption_shots(
+    index_dir: Path,
+    ctx: VideoContext,
+    model: str = CAPTION_MODEL,
+    client: anthropic.Anthropic | None = None,
+    max_captions: int = 150,
+    progress=lambda *_: None,
+) -> CostMeter:
+    """Caption shots' middle keyframes with a cheap VLM and write the captions
+    back into index.json. One-time cost, amortized over all queries.
+
+    Long fast-cut videos can have many hundreds of shots; captioning is capped
+    at max_captions, subsampled evenly, to bound both the one-time cost and
+    the text-view size."""
+    client = client or anthropic.Anthropic()
+    meter = CostMeter(model=model)
+
+    # max_captions is a per-INDEX total, not per-call: repeat calls must not
+    # keep adding captions past the cap (bloats the text view).
+    budget = max_captions - sum(1 for s in ctx.shots if s.caption)
+    if budget <= 0:
+        return meter
+    todo = [s for s in ctx.shots if not s.caption and s.keyframes]
+    if len(todo) > budget:
+        step = (len(todo) - 1) / (budget - 1) if budget > 1 else 0
+        todo = [todo[round(i * step)] for i in range(budget)]
+
+    for shot in todo:
+        kf = shot.keyframes[len(shot.keyframes) // 2]
+        data = base64.standard_b64encode((index_dir / kf.file).read_bytes()).decode()
+        response = client.messages.create(
+            model=model,
+            max_tokens=200,
+            messages=[{
+                "role": "user",
+                "content": [
+                    {"type": "image", "source": {
+                        "type": "base64", "media_type": "image/jpeg", "data": data}},
+                    {"type": "text", "text": CAPTION_PROMPT},
+                ],
+            }],
+        )
+        meter.add(response.usage)
+        shot.caption = "".join(b.text for b in response.content if b.type == "text").strip()
+        shot.caption_model = model
+        progress(f"  shot {shot.id}: {shot.caption[:70]}")
+
+    (index_dir / "index.json").write_text(ctx.model_dump_json(indent=2))
+    return meter

videoctx/cli.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import click
+
+
+@click.group()
+def main():
+    """videoctx: index videos once, query them cheaply with any LLM."""
+
+
+@main.command()
+@click.argument("video", type=click.Path(exists=True, path_type=Path))
+@click.option("--out", type=click.Path(path_type=Path), default=None, help="Index output dir")
+@click.option("--max-edge", default=768, show_default=True, help="Keyframe long-edge pixels")
+@click.option("--whisper-model", default="small", show_default=True)
+@click.option("--no-transcript", is_flag=True, help="Skip transcription")
+def index(video: Path, out: Path | None, max_edge: int, whisper_model: str, no_transcript: bool):
+    """Index VIDEO into a .vctx directory."""
+    from .indexer import build_index
+
+    build_index(
+        video, out_dir=out, max_edge=max_edge,
+        whisper_model=whisper_model, skip_transcript=no_transcript,
+        progress=click.echo,
+    )
+
+
+@main.command()
+@click.argument("index_dir", type=click.Path(exists=True, path_type=Path))
+def caption(index_dir: Path):
+    """Add visual captions to an index's shots (one cheap VLM call per shot)."""
+    from .captions import caption_shots
+    from .indexer import load_index
+
+    ctx = load_index(index_dir)
+    meter = caption_shots(index_dir, ctx, progress=click.echo)
+    click.echo(f"captioned; cost: {json.dumps(meter.summary())}")
+
+
+@main.command()
+@click.argument("index_dir", type=click.Path(exists=True, path_type=Path))
+def show(index_dir: Path):
+    """Print the text view of an index (what the model reads)."""
+    from .indexer import load_index
+    from .textview import text_view
+
+    click.echo(text_view(load_index(index_dir)))
+
+
+@main.command()
+@click.argument("index_dir", type=click.Path(exists=True, path_type=Path))
+@click.argument("question")
+@click.option("--model", default="claude-opus-4-8", show_default=True)
+def ask(index_dir: Path, question: str, model: str):
+    """Ask a question about an indexed video."""
+    from .indexer import load_index
+    from .query import ask as _ask
+
+    ctx = load_index(index_dir)
+    answer, meter, frames = _ask(index_dir, ctx, question, model=model)
+    click.echo(answer)
+    click.echo("---")
+    click.echo(f"cost: {json.dumps(meter.summary())} frames_fetched={frames}")
+
+
+if __name__ == "__main__":
+    main()

videoctx/frames.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+from .schema import Keyframe
+
+
+def extract_frame(
+    video_path: Path,
+    t_s: float,
+    out_path: Path,
+    max_edge: int = 768,
+    quality: int = 4,
+) -> Keyframe:
+    """Extract one frame at t_s, scaled so the long edge is <= max_edge."""
+    scale = (
+        f"scale='if(gt(iw,ih),min(iw,{max_edge}),-2)'"
+        f":'if(gt(iw,ih),-2,min(ih,{max_edge}))'"
+    )
+    subprocess.run(
+        [
+            "ffmpeg", "-v", "error", "-y",
+            "-ss", f"{t_s:.3f}", "-i", str(video_path),
+            "-frames:v", "1", "-vf", scale, "-q:v", str(quality),
+            "-strict", "unofficial",  # ffmpeg 9 mjpeg rejects non-full-range YUV otherwise
+            str(out_path),
+        ],
+        check=True,
+    )
+    width, height = _image_size(out_path)
+    return Keyframe(t_s=round(t_s, 3), file=f"frames/{out_path.name}", width=width, height=height)
+
+
+def _image_size(path: Path) -> tuple[int, int]:
+    out = subprocess.run(
+        [
+            "ffprobe", "-v", "error",
+            "-select_streams", "v:0",
+            "-show_entries", "stream=width,height",
+            "-of", "csv=s=x:p=0", str(path),
+        ],
+        capture_output=True, text=True, check=True,
+    )
+    w, h = out.stdout.strip().split("x")
+    return int(w), int(h)

videoctx/indexer.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+from . import __version__
+from .frames import extract_frame
+from .probe import probe
+from .schema import VideoContext
+from .shots import detect_shots, keyframe_times
+from .transcribe import transcribe
+
+
+def build_index(
+    video_path: Path,
+    out_dir: Path | None = None,
+    max_edge: int = 768,
+    whisper_model: str = "small",
+    skip_transcript: bool = False,
+    progress=print,
+) -> Path:
+    """Index a video into a .vctx directory. Returns the index path."""
+    video_path = video_path.resolve()
+    if out_dir is None:
+        out_dir = video_path.with_suffix("").with_name(video_path.stem + ".vctx")
+    frames_dir = out_dir / "frames"
+    frames_dir.mkdir(parents=True, exist_ok=True)
+
+    t0 = time.monotonic()
+    progress(f"probing {video_path.name}")
+    info = probe(video_path)
+
+    progress(f"detecting shots ({info.duration_s:.0f}s video)")
+    shots = detect_shots(video_path, info.duration_s)
+    progress(f"  {len(shots)} shots")
+
+    progress("extracting keyframes")
+    n_frames = 0
+    for shot in shots:
+        for t in keyframe_times(shot):
+            out_path = frames_dir / f"shot{shot.id:04d}_t{t:08.2f}.jpg"
+            shot.keyframes.append(extract_frame(video_path, t, out_path, max_edge=max_edge))
+            n_frames += 1
+    progress(f"  {n_frames} keyframes")
+
+    if skip_transcript or not info.has_audio:
+        from .schema import Transcript
+        transcript = Transcript()
+        progress("skipping transcript" if skip_transcript else "no audio stream")
+    else:
+        progress(f"transcribing (faster-whisper/{whisper_model})")
+        transcript = transcribe(video_path, model_size=whisper_model)
+        progress(f"  {len(transcript.segments)} segments, language={transcript.language}")
+
+    ctx = VideoContext(video=info, shots=shots, transcript=transcript)
+    (out_dir / "index.json").write_text(ctx.model_dump_json(indent=2))
+
+    manifest = {
+        "format": "vctx",
+        "format_version": "0.1",
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "producer": {"name": "videoctx", "version": __version__},
+        "params": {
+            "max_edge": max_edge,
+            "whisper_model": whisper_model,
+            "shot_detector": "scenedetect/AdaptiveDetector",
+        },
+        "source": {
+            "filename": video_path.name,
+            "path": str(video_path),  # hint only; consumers must tolerate it moving
+            "sha256": _sha256(video_path),
+            "size_bytes": video_path.stat().st_size,
+        },
+        "indexing_wall_s": round(time.monotonic() - t0, 1),
+    }
+    (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2))
+    progress(f"index written to {out_dir} ({manifest['indexing_wall_s']}s)")
+    return out_dir
+
+
+def load_index(index_dir: Path) -> VideoContext:
+    return VideoContext.model_validate_json((Path(index_dir) / "index.json").read_text())
+
+
+def _sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()

videoctx/mcp_server.py

@@ -0,0 +1,161 @@
+"""videoctx MCP server.
+
+Gives any MCP client (Claude Code, Cursor, ...) the ability to read videos the
+way it reads images: index once, then work from the cheap text layer and pull
+frames only when needed. The calling agent's own model does the reasoning -
+the core tools below require no API key and make no model calls.
+
+Install into Claude Code:
+    claude mcp add videoctx -- /path/to/videoctx/.venv/bin/videoctx-mcp
+
+Intended agent loop:
+    1. video_overview(path)   -> shot structure + captions + transcript (cheap)
+    2. survey_video(path)     -> one thumbnail per shot, when unsure where to look
+    3. get_frames(path, t0, t1) -> full-res frames from the exact moment
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from mcp.server.mcpserver import Image, MCPServer
+
+from .indexer import build_index, load_index
+from .schema import VideoContext
+from .textview import text_view
+
+server = MCPServer(
+    "videoctx",
+    instructions=(
+        "Tools for reading video files. Call video_overview first: its text "
+        "layer (shots, captions, transcript) answers most questions without "
+        "any frames. Fetch frames only for visual details the text does not "
+        "cover, on the narrowest time range possible. Evidence rule: answer "
+        "visual questions only from frames you have actually seen; if fetched "
+        "frames do not show the answer, fetch a different range or higher "
+        "density instead of guessing."
+    ),
+)
+
+VIDEO_SUFFIXES = {".mp4", ".mov", ".mkv", ".webm", ".avi", ".m4v"}
+
+
+def _resolve(path: str) -> tuple[Path, VideoContext]:
+    """Accept a video file or a .vctx index dir; auto-index videos on first use."""
+    p = Path(path).expanduser().resolve()
+    if p.is_dir() and (p / "index.json").exists():
+        return p, load_index(p)
+    if p.suffix.lower() in VIDEO_SUFFIXES and p.exists():
+        index_dir = p.parent / f"{p.stem}.vctx"
+        if not (index_dir / "index.json").exists():
+            build_index(p, out_dir=index_dir, progress=lambda *_: None)
+        return index_dir, load_index(index_dir)
+    raise ValueError(f"not a video file or .vctx index: {path}")
+
+
+def _source_video(index_dir: Path) -> Path | None:
+    import json
+
+    manifest = index_dir / "manifest.json"
+    if manifest.exists():
+        source = json.loads(manifest.read_text())["source"]
+        for candidate in (Path(source.get("path", "")), index_dir.parent / source["filename"]):
+            if candidate.name and candidate.exists():
+                return candidate
+    return None
+
+
+@server.tool()
+def index_video(path: str) -> str:
+    """Index a video file for querying (shots, keyframes, speech transcript).
+    Runs locally; takes roughly 10-20% of the video's duration. Idempotent -
+    returns immediately if the video is already indexed. The other tools
+    auto-index too, so calling this first is only needed to control when the
+    wait happens."""
+    index_dir, ctx = _resolve(path)
+    return (
+        f"indexed: {index_dir}\n"
+        f"{ctx.video.duration_s:.0f}s, {len(ctx.shots)} shots, "
+        f"{len(ctx.transcript.segments)} transcript segments "
+        f"(language={ctx.transcript.language})"
+    )
+
+
+@server.tool()
+def video_overview(path: str) -> str:
+    """The cheap text layer of a video: shot list with timestamps and visual
+    captions (when present), interleaved with the speech transcript. Read this
+    before fetching any frames: it answers most questions by itself."""
+    _, ctx = _resolve(path)
+    return text_view(ctx)
+
+
+@server.tool()
+def survey_video(path: str) -> list:
+    """One small thumbnail per shot across the whole video (subsampled to at
+    most 24), each labeled with its timestamp. Cheap visual sweep for locating
+    the relevant moment before zooming in with get_frames."""
+    from .query import MAX_SURVEY_THUMBS, _thumb
+
+    index_dir, ctx = _resolve(path)
+    shots = [s for s in ctx.shots if s.keyframes]
+    if len(shots) > MAX_SURVEY_THUMBS:
+        step = (len(shots) - 1) / (MAX_SURVEY_THUMBS - 1)
+        shots = [shots[round(i * step)] for i in range(MAX_SURVEY_THUMBS)]
+    out: list = []
+    for shot in shots:
+        kf = shot.keyframes[len(shot.keyframes) // 2]
+        out.append(f"shot {shot.id} @ {kf.t_s:.1f}s:")
+        out.append(Image(path=_thumb(index_dir, kf)))
+    return out or ["no keyframes available"]
+
+
+@server.tool()
+def get_frames(path: str, start_s: float, end_s: float, max_frames: int = 4,
+               detail: str = "standard") -> list:
+    """Frames evenly spaced across [start_s, end_s]. Frames are expensive: use
+    the narrowest range the question allows, and raise max_frames (up to 8)
+    only when a sparse sample missed the moment. Set detail='high' (2x
+    resolution, ~4x token cost) when small text, colors, patterns, or fine
+    object details matter."""
+    from .query import extract_on_demand, select_frames
+
+    index_dir, ctx = _resolve(path)
+    max_frames = max(1, min(int(max_frames), 8))
+    edge = 1536 if detail == "high" else 768
+    video = _source_video(index_dir)
+    if video is not None:
+        frames = extract_on_demand(index_dir, video, ctx, start_s, end_s, max_frames, max_edge=edge)
+    else:
+        frames = select_frames(ctx, start_s, end_s, max_frames)
+    out: list = []
+    for kf in frames:
+        out.append(f"frame @ {kf.t_s:.1f}s:")
+        out.append(Image(path=index_dir / kf.file))
+    return out or ["no frames available in that range"]
+
+
+@server.tool()
+def transcript_search(path: str, query: str, context_s: float = 15.0) -> str:
+    """Case-insensitive search over the transcript. Returns matching segments
+    with surrounding context and timestamps: use the timestamps to target
+    get_frames."""
+    _, ctx = _resolve(path)
+    q = query.lower()
+    hits = []
+    segs = ctx.transcript.segments
+    for i, seg in enumerate(segs):
+        if q in seg.text.lower():
+            around = [s for s in segs if abs(s.start_s - seg.start_s) <= context_s]
+            block = " ".join(s.text for s in around)
+            hits.append(f"[{seg.start_s:.1f}s] ...{block}...")
+    if not hits:
+        return f"no transcript matches for {query!r}"
+    return "\n".join(hits[:10])
+
+
+def main() -> None:
+    server.run()
+
+
+if __name__ == "__main__":
+    main()

videoctx/pricing.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+# USD per million tokens (input, output). Update as pricing changes.
+PRICES: dict[str, tuple[float, float]] = {
+    "claude-opus-4-8": (5.00, 25.00),
+    "claude-sonnet-5": (3.00, 15.00),
+    "claude-haiku-4-5": (1.00, 5.00),
+    "gemini-native": (0.0, 0.0),  # filled in by the eval baseline, provider-specific
+}
+
+
+@dataclass
+class CostMeter:
+    """Accumulates token usage across API calls and prices it."""
+
+    model: str
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cache_read_tokens: int = 0
+    cache_write_tokens: int = 0
+    calls: int = 0
+    notes: list[str] = field(default_factory=list)
+
+    cache_write_1h_tokens: int = 0  # 1h-TTL writes bill at 2x, not 1.25x
+
+    def add(self, usage) -> None:
+        self.calls += 1
+        self.input_tokens += usage.input_tokens
+        self.output_tokens += usage.output_tokens
+        self.cache_read_tokens += getattr(usage, "cache_read_input_tokens", 0) or 0
+        total_write = getattr(usage, "cache_creation_input_tokens", 0) or 0
+        breakdown = getattr(usage, "cache_creation", None)
+        h1 = (getattr(breakdown, "ephemeral_1h_input_tokens", 0) or 0) if breakdown else 0
+        self.cache_write_1h_tokens += h1
+        self.cache_write_tokens += total_write - h1
+
+    @property
+    def usd(self) -> float:
+        inp, out = PRICES.get(self.model, (0.0, 0.0))
+        cost = (self.input_tokens * inp + self.output_tokens * out) / 1_000_000
+        cost += self.cache_read_tokens * inp * 0.1 / 1_000_000
+        cost += self.cache_write_tokens * inp * 1.25 / 1_000_000   # 5m-TTL write premium
+        cost += self.cache_write_1h_tokens * inp * 2.0 / 1_000_000  # 1h-TTL write premium
+        return cost
+
+    def summary(self) -> dict:
+        return {
+            "model": self.model,
+            "calls": self.calls,
+            "input_tokens": self.input_tokens,
+            "output_tokens": self.output_tokens,
+            "cache_read_tokens": self.cache_read_tokens,
+            "cache_write_tokens": self.cache_write_tokens,
+            "cache_write_1h_tokens": self.cache_write_1h_tokens,
+            "usd": round(self.usd, 6),
+        }

videoctx/probe.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import json
+import subprocess
+from pathlib import Path
+
+from .schema import VideoInfo
+
+
+def probe(video_path: Path) -> VideoInfo:
+    """Read duration, dimensions, fps, and audio presence via ffprobe."""
+    out = subprocess.run(
+        [
+            "ffprobe", "-v", "error",
+            "-show_entries", "format=duration",
+            "-show_entries", "stream=codec_type,width,height,avg_frame_rate",
+            "-of", "json", str(video_path),
+        ],
+        capture_output=True, text=True, check=True,
+    )
+    data = json.loads(out.stdout)
+    duration = float(data["format"]["duration"])
+    video_stream = next(s for s in data["streams"] if s.get("codec_type") == "video")
+    has_audio = any(s.get("codec_type") == "audio" for s in data["streams"])
+
+    num, _, den = video_stream.get("avg_frame_rate", "0/1").partition("/")
+    fps = float(num) / float(den) if den and float(den) else 0.0
+
+    return VideoInfo(
+        duration_s=duration,
+        width=video_stream["width"],
+        height=video_stream["height"],
+        fps=round(fps, 3),
+        has_audio=has_audio,
+    )