videopython 0.50.0__tar.gz → 0.51.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {videopython-0.50.0 → videopython-0.51.0}/PKG-INFO +6 -4
- {videopython-0.50.0 → videopython-0.51.0}/README.md +5 -3
- {videopython-0.50.0 → videopython-0.51.0}/pyproject.toml +1 -1
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/__init__.py +7 -10
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/_ollama.py +16 -1
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/_optional.py +13 -15
- videopython-0.51.0/src/videopython/ai/_predictor.py +70 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/auto_edit/__init__.py +3 -4
- videopython-0.51.0/src/videopython/ai/auto_edit/backend.py +28 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/auto_edit/editor.py +13 -9
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/auto_edit/local.py +12 -5
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/dubbing/__init__.py +0 -2
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/dubbing/config.py +3 -6
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/dubbing/dubber.py +3 -2
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/dubbing/loudness.py +1 -1
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/dubbing/pipeline.py +5 -5
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/dubbing/remux.py +6 -71
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/dubbing/timing.py +0 -39
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/generation/_tts_backend.py +7 -8
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/generation/audio.py +5 -19
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/generation/image.py +4 -7
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/generation/translation.py +1 -48
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/generation/video.py +7 -13
- videopython-0.51.0/src/videopython/ai/ops.py +23 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/understanding/__init__.py +3 -3
- videopython-0.51.0/src/videopython/ai/understanding/_yolo.py +111 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/understanding/audio.py +8 -25
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/understanding/faces.py +21 -67
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/understanding/objects.py +14 -68
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/understanding/separation.py +2 -10
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/understanding/temporal.py +2 -7
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/video_analysis/models.py +10 -3
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/video_analysis/sampling.py +1 -1
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/mcp/server.py +58 -22
- videopython-0.50.0/src/videopython/ai/_predictor.py +0 -54
- videopython-0.50.0/src/videopython/ai/auto_edit/backend.py +0 -37
- {videopython-0.50.0 → videopython-0.51.0}/.gitignore +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/LICENSE +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/__init__.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/_device.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/_revisions.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/auto_edit/catalog.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/auto_edit/models.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/auto_edit/resolve.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/dubbing/expressiveness.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/dubbing/models.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/dubbing/quality.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/dubbing/voice_sample.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/effects.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/generation/__init__.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/transforms.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/understanding/image.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/video_analysis/__init__.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/video_analysis/analyzer.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/ai/video_analysis/stages.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/audio/__init__.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/audio/analysis.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/audio/audio.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/__init__.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/_dimensions.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/_ffmpeg.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/_video_io.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/description.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/draw_detections.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/exceptions.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/fonts/Anton-OFL.txt +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/fonts/Anton-Regular.ttf +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/fonts/BebasNeue-OFL.txt +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/fonts/BebasNeue-Regular.ttf +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/fonts/DejaVuSans.ttf +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/fonts/LICENSE_DEJAVU +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/fonts/Lato-Bold.ttf +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/fonts/Lato-OFL.txt +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/fonts/Poppins-Bold.ttf +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/fonts/Poppins-OFL.txt +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/fonts/__init__.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/transcription.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/base/video.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/editing/__init__.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/editing/_ass.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/editing/_easing.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/editing/_schema.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/editing/audio_ops.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/editing/effects.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/editing/operation.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/editing/streaming.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/editing/transcription_overlay.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/editing/transforms.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/editing/video_edit.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/mcp/__init__.py +0 -0
- {videopython-0.50.0 → videopython-0.51.0}/src/videopython/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: videopython
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.51.0
|
|
4
4
|
Summary: Minimal video generation and processing library.
|
|
5
5
|
Project-URL: Homepage, https://videopython.com
|
|
6
6
|
Project-URL: Repository, https://github.com/bartwojtowicz/videopython/
|
|
@@ -128,11 +128,13 @@ Putting an LLM in the loop works three ways:
|
|
|
128
128
|
2. **`AutoEditor`** — a local Ollama vision model is the planner (see [Automatic editing](#automatic-editing-local-llm) above).
|
|
129
129
|
3. **MCP server** — `videopython-mcp` exposes the pipeline as [Model Context Protocol](https://modelcontextprotocol.io) tools, so an agent like Claude drives editing with its own model. Install `[ai,mcp]`, run `videopython-mcp`, and point your MCP client at it. See the [MCP Server Guide](https://videopython.com/guides/mcp/).
|
|
130
130
|
|
|
131
|
-
|
|
131
|
+
**Mode 1** in brief: every operation is a Pydantic model whose fields *are* the JSON wire format, so `VideoEdit.json_schema()` hands your model a ready-made tool schema — a discriminated union over every LLM-exposed op (pass `strict=True` for provider grammar modes). Plans parse permissively and own their numeric bounds at validation, so a refine loop converges fast:
|
|
132
132
|
|
|
133
|
-
|
|
133
|
+
- **`edit.check(meta)`** — collect *every* structured error in one pass, not just the first
|
|
134
|
+
- **`edit.repair(meta)`** — auto-clamp mechanical violations (overruns, negatives) with a changelog
|
|
135
|
+
- **`edit.normalize_dimensions(meta, target)`** — make heterogeneous segments concat-compatible
|
|
134
136
|
|
|
135
|
-
See the [LLM Integration Guide](https://videopython.com/guides/llm-integration/) for end-to-end examples, the
|
|
137
|
+
See the [LLM Integration Guide](https://videopython.com/guides/llm-integration/) for end-to-end examples (Anthropic / OpenAI tool use), the refine loop, and operation discovery.
|
|
136
138
|
|
|
137
139
|
## Features
|
|
138
140
|
|
|
@@ -84,11 +84,13 @@ Putting an LLM in the loop works three ways:
|
|
|
84
84
|
2. **`AutoEditor`** — a local Ollama vision model is the planner (see [Automatic editing](#automatic-editing-local-llm) above).
|
|
85
85
|
3. **MCP server** — `videopython-mcp` exposes the pipeline as [Model Context Protocol](https://modelcontextprotocol.io) tools, so an agent like Claude drives editing with its own model. Install `[ai,mcp]`, run `videopython-mcp`, and point your MCP client at it. See the [MCP Server Guide](https://videopython.com/guides/mcp/).
|
|
86
86
|
|
|
87
|
-
|
|
87
|
+
**Mode 1** in brief: every operation is a Pydantic model whose fields *are* the JSON wire format, so `VideoEdit.json_schema()` hands your model a ready-made tool schema — a discriminated union over every LLM-exposed op (pass `strict=True` for provider grammar modes). Plans parse permissively and own their numeric bounds at validation, so a refine loop converges fast:
|
|
88
88
|
|
|
89
|
-
|
|
89
|
+
- **`edit.check(meta)`** — collect *every* structured error in one pass, not just the first
|
|
90
|
+
- **`edit.repair(meta)`** — auto-clamp mechanical violations (overruns, negatives) with a changelog
|
|
91
|
+
- **`edit.normalize_dimensions(meta, target)`** — make heterogeneous segments concat-compatible
|
|
90
92
|
|
|
91
|
-
See the [LLM Integration Guide](https://videopython.com/guides/llm-integration/) for end-to-end examples, the
|
|
93
|
+
See the [LLM Integration Guide](https://videopython.com/guides/llm-integration/) for end-to-end examples (Anthropic / OpenAI tool use), the refine loop, and operation discovery.
|
|
92
94
|
|
|
93
95
|
## Features
|
|
94
96
|
|
|
@@ -2,12 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
Every public symbol is re-exported lazily via PEP 562 ``__getattr__`` so that
|
|
4
4
|
``import videopython.ai`` (or ``from videopython.ai import X``) loads ONLY the
|
|
5
|
-
leaf module backing the requested symbol — not every sibling.
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
``[
|
|
10
|
-
|
|
5
|
+
leaf module backing the requested symbol — not every sibling. All AI capabilities
|
|
6
|
+
ship in the single ``[ai]`` extra; the laziness keeps ``import videopython`` (and
|
|
7
|
+
importing one leaf class) light by deferring the heavy ML imports (torch /
|
|
8
|
+
transformers / diffusers / ultralytics) until a symbol is actually used. When
|
|
9
|
+
``[ai]`` is not installed, touching a symbol raises a clear
|
|
10
|
+
``pip install 'videopython[ai]'``-pointing ``ImportError`` at attribute access
|
|
11
|
+
instead of failing the whole package import.
|
|
11
12
|
|
|
12
13
|
The ``TYPE_CHECKING`` block below keeps the eager imports visible to mypy and
|
|
13
14
|
IDEs (static autocompletion / re-export checking) without executing them at
|
|
@@ -28,11 +29,9 @@ if TYPE_CHECKING:
|
|
|
28
29
|
from .auto_edit import AutoEditor as AutoEditor
|
|
29
30
|
from .auto_edit import EditCatalog as EditCatalog
|
|
30
31
|
from .auto_edit import EditPlan as EditPlan
|
|
31
|
-
from .auto_edit import ImagePart as ImagePart
|
|
32
32
|
from .auto_edit import OllamaVisionLLM as OllamaVisionLLM
|
|
33
33
|
from .auto_edit import PlannerError as PlannerError
|
|
34
34
|
from .auto_edit import StructuredVisionLLM as StructuredVisionLLM
|
|
35
|
-
from .auto_edit import TextPart as TextPart
|
|
36
35
|
from .auto_edit import build_catalog as build_catalog
|
|
37
36
|
from .effects import ObjectDetectionOverlay as ObjectDetectionOverlay
|
|
38
37
|
from .generation import ImageToVideo as ImageToVideo
|
|
@@ -82,8 +81,6 @@ _exports: dict[str, str] = {
|
|
|
82
81
|
"OllamaVisionLLM": ".auto_edit",
|
|
83
82
|
"StructuredVisionLLM": ".auto_edit",
|
|
84
83
|
"PlannerError": ".auto_edit",
|
|
85
|
-
"TextPart": ".auto_edit",
|
|
86
|
-
"ImagePart": ".auto_edit",
|
|
87
84
|
"EditCatalog": ".auto_edit",
|
|
88
85
|
"EditPlan": ".auto_edit",
|
|
89
86
|
"build_catalog": ".auto_edit",
|
|
@@ -7,6 +7,7 @@ import io
|
|
|
7
7
|
import json
|
|
8
8
|
from typing import Any
|
|
9
9
|
|
|
10
|
+
import cv2
|
|
10
11
|
import numpy as np
|
|
11
12
|
from PIL import Image
|
|
12
13
|
|
|
@@ -34,7 +35,7 @@ class OllamaStructuredClient:
|
|
|
34
35
|
|
|
35
36
|
def _get_client(self) -> Any:
|
|
36
37
|
if self._client is None:
|
|
37
|
-
ollama = require("ollama",
|
|
38
|
+
ollama = require("ollama", feature="Ollama")
|
|
38
39
|
self._client = ollama.Client(host=self.host)
|
|
39
40
|
return self._client
|
|
40
41
|
|
|
@@ -65,6 +66,20 @@ class OllamaStructuredClient:
|
|
|
65
66
|
self._client = None
|
|
66
67
|
|
|
67
68
|
|
|
69
|
+
# Used only on the MCP keyframe path (videopython.mcp). SceneVLM captioning and the local
|
|
70
|
+
# planner deliberately encode full-resolution frames via _encode_png_b64.
|
|
71
|
+
KEYFRAME_MAX_DIM = 768 # bound a keyframe's longest side before PNG-encoding for the MCP payload
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _downscale(frame: np.ndarray, max_dim: int = KEYFRAME_MAX_DIM) -> np.ndarray:
|
|
75
|
+
"""Shrink an RGB frame so its longest side is at most ``max_dim`` (aspect preserved; never upscales)."""
|
|
76
|
+
h, w = frame.shape[:2]
|
|
77
|
+
scale = max_dim / max(h, w)
|
|
78
|
+
if scale >= 1.0:
|
|
79
|
+
return frame
|
|
80
|
+
return cv2.resize(frame, (max(1, round(w * scale)), max(1, round(h * scale))), interpolation=cv2.INTER_AREA)
|
|
81
|
+
|
|
82
|
+
|
|
68
83
|
def _encode_png_b64(frame: np.ndarray) -> str:
|
|
69
84
|
buffer = io.BytesIO()
|
|
70
85
|
Image.fromarray(frame).save(buffer, format="PNG")
|
|
@@ -6,12 +6,11 @@ method that needs it (there are no top-level ``import torch``/``transformers``/
|
|
|
6
6
|
importable on a slim install: importing ``ai.understanding.objects`` only needs
|
|
7
7
|
core deps until a detector is actually constructed.
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
``pip install 'videopython[<extra>]'``.
|
|
9
|
+
All those heavy deps ship in the single ``[ai]`` extra. When it isn't installed
|
|
10
|
+
and a code path reaches one of them, the bare ``import`` would raise a stock
|
|
11
|
+
``ModuleNotFoundError`` with no hint about how to fix it. :func:`require` wraps
|
|
12
|
+
the import and turns that into an actionable ``ImportError`` pointing at
|
|
13
|
+
``pip install 'videopython[ai]'``.
|
|
15
14
|
"""
|
|
16
15
|
|
|
17
16
|
from __future__ import annotations
|
|
@@ -21,15 +20,13 @@ from types import ModuleType
|
|
|
21
20
|
from typing import Callable
|
|
22
21
|
|
|
23
22
|
|
|
24
|
-
def require(module: str,
|
|
25
|
-
"""Import ``module`` or raise a clear,
|
|
23
|
+
def require(module: str, *, feature: str | None = None) -> ModuleType:
|
|
24
|
+
"""Import ``module`` or raise a clear, ``[ai]``-pointing ``ImportError``.
|
|
26
25
|
|
|
27
26
|
Args:
|
|
28
27
|
module: Dotted module name to import (e.g. ``"chatterbox.mtl_tts"``,
|
|
29
28
|
``"torch"``). The fully-imported module object is returned so the
|
|
30
29
|
call site can bind the symbols it needs.
|
|
31
|
-
extra: The videopython extra that ships ``module`` (e.g. ``"tts"``,
|
|
32
|
-
``"asr"``). Surfaced verbatim in the install hint.
|
|
33
30
|
feature: Optional human-readable feature name for the message head. When
|
|
34
31
|
omitted the top-level package of ``module`` is used.
|
|
35
32
|
|
|
@@ -38,14 +35,14 @@ def require(module: str, extra: str, *, feature: str | None = None) -> ModuleTyp
|
|
|
38
35
|
|
|
39
36
|
Raises:
|
|
40
37
|
ImportError: If ``module`` cannot be imported. The message always
|
|
41
|
-
contains
|
|
42
|
-
|
|
38
|
+
contains a ``pip install 'videopython[ai]'`` hint so the caller
|
|
39
|
+
knows exactly how to fix it.
|
|
43
40
|
"""
|
|
44
41
|
try:
|
|
45
42
|
return importlib.import_module(module)
|
|
46
43
|
except ImportError as exc:
|
|
47
44
|
label = feature or module.split(".")[0]
|
|
48
|
-
raise ImportError(f"{label} requires the '
|
|
45
|
+
raise ImportError(f"{label} requires the 'ai' extra: pip install 'videopython[ai]'") from exc
|
|
49
46
|
|
|
50
47
|
|
|
51
48
|
def lazy_exports(package: str, exports: dict[str, str]) -> tuple[Callable[[str], object], Callable[[], list[str]]]:
|
|
@@ -53,8 +50,9 @@ def lazy_exports(package: str, exports: dict[str, str]) -> tuple[Callable[[str],
|
|
|
53
50
|
|
|
54
51
|
Re-exports a set of public symbols lazily: the submodule backing a symbol is
|
|
55
52
|
imported only on first attribute access, so importing the package does not
|
|
56
|
-
pull in any sibling leaf module
|
|
57
|
-
|
|
53
|
+
pull in any sibling leaf module. This keeps ``import videopython`` (and
|
|
54
|
+
importing a single leaf class) light by deferring the heavy ML imports
|
|
55
|
+
(torch / transformers / diffusers / ultralytics) until a symbol is used.
|
|
58
56
|
|
|
59
57
|
Args:
|
|
60
58
|
package: The ``__name__`` of the package defining the re-exports. Used
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Context-manager base with a default VRAM-releasing ``unload()``.
|
|
2
|
+
|
|
3
|
+
Every predictor under ``videopython.ai`` holds a lazily-loaded model (and
|
|
4
|
+
sometimes a processor / pipeline / VAD field) on the selected device, and needs
|
|
5
|
+
to drop those references and free the allocator cache when done. That teardown
|
|
6
|
+
is mechanical and identical across predictors: set the model field(s) to
|
|
7
|
+
``None`` and call :func:`videopython.ai._device.release_device_memory`.
|
|
8
|
+
|
|
9
|
+
:class:`ManagedPredictor` provides that as a default ``unload()`` driven by two
|
|
10
|
+
class attributes -- ``_model_attrs`` (the fields holding model state) and
|
|
11
|
+
``_device_attr`` (the field holding the device) -- and turns any predictor into
|
|
12
|
+
a context manager::
|
|
13
|
+
|
|
14
|
+
with SceneVLM(...) as vlm:
|
|
15
|
+
... # use vlm
|
|
16
|
+
# vlm.unload() has fired here, releasing VRAM
|
|
17
|
+
|
|
18
|
+
Subclasses with non-default model fields just declare ``_model_attrs``; those
|
|
19
|
+
whose teardown isn't "null the fields + release" (e.g. delegating to a client's
|
|
20
|
+
own ``unload()``) override ``unload()`` instead.
|
|
21
|
+
|
|
22
|
+
The base imports no torch / transformers / ultralytics (``release_device_memory``
|
|
23
|
+
defers its torch import), so it stays safe to mix into any predictor regardless
|
|
24
|
+
of how it is constructed.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
from types import TracebackType
|
|
30
|
+
from typing import TYPE_CHECKING, Literal
|
|
31
|
+
|
|
32
|
+
from videopython.ai._device import release_device_memory
|
|
33
|
+
|
|
34
|
+
if TYPE_CHECKING:
|
|
35
|
+
from typing_extensions import Self
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class ManagedPredictor:
|
|
39
|
+
"""Adds a default ``unload()`` plus ``with``-statement support.
|
|
40
|
+
|
|
41
|
+
``unload()`` clears each attribute named in ``_model_attrs`` to ``None`` and
|
|
42
|
+
releases the cache for the device named by ``_device_attr``. It is idempotent
|
|
43
|
+
(safe before the model is loaded and on repeated calls). ``__exit__`` always
|
|
44
|
+
returns ``False`` so exceptions inside the ``with`` block propagate --
|
|
45
|
+
``unload()`` runs on both the success and failure paths.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
# Attributes holding model state, cleared to None on unload. Override per
|
|
49
|
+
# predictor (e.g. ("_model", "_processor")).
|
|
50
|
+
_model_attrs: tuple[str, ...] = ("_model",)
|
|
51
|
+
# Attribute holding the resolved device passed to release_device_memory.
|
|
52
|
+
_device_attr: str = "device"
|
|
53
|
+
|
|
54
|
+
def unload(self) -> None:
|
|
55
|
+
"""Drop the model reference(s) and release device memory. Idempotent."""
|
|
56
|
+
for attr in self._model_attrs:
|
|
57
|
+
setattr(self, attr, None)
|
|
58
|
+
release_device_memory(getattr(self, self._device_attr, None))
|
|
59
|
+
|
|
60
|
+
def __enter__(self) -> Self:
|
|
61
|
+
return self
|
|
62
|
+
|
|
63
|
+
def __exit__(
|
|
64
|
+
self,
|
|
65
|
+
exc_type: type[BaseException] | None,
|
|
66
|
+
exc: BaseException | None,
|
|
67
|
+
tb: TracebackType | None,
|
|
68
|
+
) -> Literal[False]:
|
|
69
|
+
self.unload()
|
|
70
|
+
return False
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
import videopython.ai.ops # noqa: F401 -- registers the AI ops (face_crop, object_detection_overlay)
|
|
6
|
+
|
|
7
|
+
from .backend import PlannerError, StructuredVisionLLM
|
|
6
8
|
from .catalog import build_catalog
|
|
7
9
|
from .editor import AutoEditError, AutoEditor
|
|
8
10
|
from .local import OllamaVisionLLM
|
|
@@ -16,13 +18,10 @@ __all__ = [
|
|
|
16
18
|
"CatalogScene",
|
|
17
19
|
"EditCatalog",
|
|
18
20
|
"EditPlan",
|
|
19
|
-
"ImagePart",
|
|
20
21
|
"OllamaVisionLLM",
|
|
21
|
-
"Part",
|
|
22
22
|
"PlanSegment",
|
|
23
23
|
"PlannerError",
|
|
24
24
|
"StructuredVisionLLM",
|
|
25
|
-
"TextPart",
|
|
26
25
|
"UnknownSceneIdsError",
|
|
27
26
|
"build_catalog",
|
|
28
27
|
"resolve_plan",
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""The SDK-free seam between the editor and a structured-vision model."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class PlannerError(RuntimeError):
|
|
12
|
+
"""A backend produced unusable output; the editor retries (infra errors should propagate instead)."""
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@runtime_checkable
|
|
16
|
+
class StructuredVisionLLM(Protocol):
|
|
17
|
+
"""Returns schema-shaped JSON from a system prompt + text + optional keyframes.
|
|
18
|
+
|
|
19
|
+
The signature mirrors
|
|
20
|
+
:meth:`videopython.ai._ollama.OllamaStructuredClient.generate_json`, so any
|
|
21
|
+
structured-generation client satisfies it structurally. Implementations
|
|
22
|
+
raise :class:`PlannerError` on unusable output (the editor retries those);
|
|
23
|
+
infra errors should propagate so they are not silently retried.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def generate_json(
|
|
27
|
+
self, *, system: str, text: str, images: list[np.ndarray] | None, schema: dict[str, Any]
|
|
28
|
+
) -> dict[str, Any]: ...
|
|
@@ -8,12 +8,14 @@ from typing import TYPE_CHECKING, Any, Literal
|
|
|
8
8
|
|
|
9
9
|
from pydantic import ValidationError
|
|
10
10
|
|
|
11
|
-
from .backend import
|
|
11
|
+
from .backend import PlannerError, StructuredVisionLLM
|
|
12
12
|
from .catalog import build_catalog
|
|
13
13
|
from .models import CatalogBundle, CatalogScene, EditPlan
|
|
14
14
|
from .resolve import UnknownSceneIdsError, resolve_plan
|
|
15
15
|
|
|
16
16
|
if TYPE_CHECKING:
|
|
17
|
+
import numpy as np
|
|
18
|
+
|
|
17
19
|
from videopython.ai.video_analysis import VideoAnalysis, VideoAnalyzer
|
|
18
20
|
from videopython.base.video import VideoMetadata
|
|
19
21
|
from videopython.editing import VideoEdit
|
|
@@ -61,13 +63,13 @@ class AutoEditor:
|
|
|
61
63
|
metadata = _metadata_by_source(analyses)
|
|
62
64
|
run_context = _merge_context(analyses, context)
|
|
63
65
|
schema = EditPlan.json_schema(strict=True)
|
|
64
|
-
|
|
66
|
+
base_text, images = _build_prompt(brief, bundle)
|
|
65
67
|
|
|
66
68
|
feedback: str | None = None
|
|
67
69
|
for _ in range(self.max_rounds):
|
|
68
|
-
|
|
70
|
+
text = base_text if feedback is None else f"{base_text}\n\n{feedback}"
|
|
69
71
|
try:
|
|
70
|
-
raw = self.planner.generate_json(system=_SYSTEM_PROMPT,
|
|
72
|
+
raw = self.planner.generate_json(system=_SYSTEM_PROMPT, text=text, images=images or None, schema=schema)
|
|
71
73
|
edit = resolve_plan(EditPlan.model_validate(raw), bundle.catalog)
|
|
72
74
|
except (PlannerError, ValidationError, UnknownSceneIdsError) as exc:
|
|
73
75
|
feedback = _shape_feedback(exc)
|
|
@@ -89,14 +91,16 @@ class AutoEditor:
|
|
|
89
91
|
return self._analyzer
|
|
90
92
|
|
|
91
93
|
|
|
92
|
-
def
|
|
93
|
-
|
|
94
|
+
def _build_prompt(brief: str, bundle: CatalogBundle) -> tuple[str, list[np.ndarray]]:
|
|
95
|
+
"""The planner prompt: scene-line text plus the keyframes, in catalog order."""
|
|
96
|
+
lines = [f"Brief: {brief}\n\nCandidate scenes:"]
|
|
97
|
+
images: list[np.ndarray] = []
|
|
94
98
|
for scene in bundle.catalog.scenes:
|
|
95
|
-
|
|
99
|
+
lines.append(_scene_line(scene))
|
|
96
100
|
frame = bundle.keyframes.get(scene.id)
|
|
97
101
|
if frame is not None:
|
|
98
|
-
|
|
99
|
-
return
|
|
102
|
+
images.append(frame)
|
|
103
|
+
return "\n\n".join(lines), images
|
|
100
104
|
|
|
101
105
|
|
|
102
106
|
def _scene_line(scene: CatalogScene) -> str:
|
|
@@ -2,11 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
from typing import Any
|
|
5
|
+
from typing import TYPE_CHECKING, Any
|
|
6
6
|
|
|
7
7
|
from videopython.ai._ollama import OllamaError, OllamaStructuredClient
|
|
8
8
|
|
|
9
|
-
from .backend import
|
|
9
|
+
from .backend import PlannerError
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
import numpy as np
|
|
10
13
|
|
|
11
14
|
DEFAULT_OLLAMA_MODEL = "gemma3:27b"
|
|
12
15
|
|
|
@@ -19,6 +22,10 @@ class OllamaVisionLLM:
|
|
|
19
22
|
every model supports schema conditioning -- ``gemma3:27b`` is verified working;
|
|
20
23
|
some builds (e.g. certain MLX ones) fail it. ``ollama pull <model>`` first;
|
|
21
24
|
``options`` are extra generation options merged over ``temperature=0``.
|
|
25
|
+
|
|
26
|
+
Thin wrapper over the shared :class:`OllamaStructuredClient`: its only job is
|
|
27
|
+
to translate :class:`OllamaError` into the :class:`PlannerError` the editor
|
|
28
|
+
retries on.
|
|
22
29
|
"""
|
|
23
30
|
|
|
24
31
|
def __init__(
|
|
@@ -30,9 +37,9 @@ class OllamaVisionLLM:
|
|
|
30
37
|
) -> None:
|
|
31
38
|
self._client = OllamaStructuredClient(model=model, host=host, options=options)
|
|
32
39
|
|
|
33
|
-
def generate_json(
|
|
34
|
-
|
|
35
|
-
|
|
40
|
+
def generate_json(
|
|
41
|
+
self, *, system: str, text: str, images: list[np.ndarray] | None, schema: dict[str, Any]
|
|
42
|
+
) -> dict[str, Any]:
|
|
36
43
|
try:
|
|
37
44
|
return self._client.generate_json(system=system, text=text, schema=schema, images=images or None)
|
|
38
45
|
except OllamaError as exc:
|
|
@@ -29,7 +29,6 @@ if TYPE_CHECKING:
|
|
|
29
29
|
from videopython.ai.dubbing.quality import TranscriptQuality as TranscriptQuality
|
|
30
30
|
from videopython.ai.dubbing.quality import assess_transcript as assess_transcript
|
|
31
31
|
from videopython.ai.dubbing.timing import TimingSynchronizer as TimingSynchronizer
|
|
32
|
-
from videopython.ai.generation.translation import UnsupportedLanguageError as UnsupportedLanguageError
|
|
33
32
|
|
|
34
33
|
# Public symbol -> fully-qualified module that defines it.
|
|
35
34
|
_exports: dict[str, str] = {
|
|
@@ -45,7 +44,6 @@ _exports: dict[str, str] = {
|
|
|
45
44
|
"GarbageTranscriptError": "videopython.ai.dubbing.quality",
|
|
46
45
|
"TranscriptQuality": "videopython.ai.dubbing.quality",
|
|
47
46
|
"assess_transcript": "videopython.ai.dubbing.quality",
|
|
48
|
-
"UnsupportedLanguageError": "videopython.ai.generation.translation",
|
|
49
47
|
}
|
|
50
48
|
|
|
51
49
|
__all__ = list(_exports)
|
|
@@ -6,7 +6,6 @@ from typing import Literal
|
|
|
6
6
|
|
|
7
7
|
from pydantic import BaseModel, ConfigDict
|
|
8
8
|
|
|
9
|
-
TranslatorChoice = Literal["auto", "ollama"]
|
|
10
9
|
WhisperModel = Literal["tiny", "base", "small", "medium", "large", "turbo"]
|
|
11
10
|
|
|
12
11
|
|
|
@@ -47,9 +46,8 @@ class DubbingConfig(BaseModel):
|
|
|
47
46
|
WARNING but processing continues. Either way the
|
|
48
47
|
:class:`TranscriptQuality` is exposed on ``DubbingResult`` for
|
|
49
48
|
inspection.
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
``translator_host`` the server URL.
|
|
49
|
+
translator_model: Ollama tag for the translation model (``None`` uses the
|
|
50
|
+
translator's default). ``translator_host`` sets the server URL.
|
|
53
51
|
"""
|
|
54
52
|
|
|
55
53
|
model_config = ConfigDict(frozen=True)
|
|
@@ -62,7 +60,6 @@ class DubbingConfig(BaseModel):
|
|
|
62
60
|
logprob_threshold: float | None = -1.0
|
|
63
61
|
vocabulary: list[str] | None = None
|
|
64
62
|
strict_quality: bool = False
|
|
65
|
-
translator: TranslatorChoice = "auto"
|
|
66
63
|
translator_model: str | None = None
|
|
67
64
|
translator_host: str | None = None
|
|
68
65
|
|
|
@@ -75,5 +72,5 @@ class DubbingConfig(BaseModel):
|
|
|
75
72
|
"device": self.device.lower() if isinstance(self.device, str) else "auto",
|
|
76
73
|
"low_memory": self.low_memory,
|
|
77
74
|
"whisper_model": self.whisper_model,
|
|
78
|
-
"
|
|
75
|
+
"translator_model": self.translator_model,
|
|
79
76
|
}
|
|
@@ -36,8 +36,9 @@ class VideoDubber:
|
|
|
36
36
|
raise TypeError("Pass either `config=` or knob kwargs, not both")
|
|
37
37
|
self.config = config or DubbingConfig(**kwargs)
|
|
38
38
|
# Optional injected speech backend. None -> the pipeline lazily builds
|
|
39
|
-
# the local chatterbox-backed TextToSpeech (
|
|
40
|
-
#
|
|
39
|
+
# the local chatterbox-backed TextToSpeech (from the [ai] extra). Inject
|
|
40
|
+
# a SpeechBackend to run synthesis out-of-process (e.g. a remote/Modal
|
|
41
|
+
# function) without loading chatterbox here.
|
|
41
42
|
self._tts_backend = tts_backend
|
|
42
43
|
self._local_pipeline: Any = None
|
|
43
44
|
logger.info(
|
|
@@ -64,7 +64,7 @@ def loudness_match(target: Audio, reference: Audio) -> Audio:
|
|
|
64
64
|
|
|
65
65
|
from videopython.ai._optional import require
|
|
66
66
|
|
|
67
|
-
pyloudnorm = require("pyloudnorm",
|
|
67
|
+
pyloudnorm = require("pyloudnorm", feature="loudness matching")
|
|
68
68
|
|
|
69
69
|
target_lufs = pyloudnorm.Meter(target.metadata.sample_rate).integrated_loudness(target.data)
|
|
70
70
|
reference_lufs = pyloudnorm.Meter(reference.metadata.sample_rate).integrated_loudness(reference.data)
|
|
@@ -48,9 +48,9 @@ class LocalDubbingPipeline:
|
|
|
48
48
|
self.config = config or DubbingConfig(**kwargs)
|
|
49
49
|
# Injected speech backend (a SpeechBackend, e.g. a remote/out-of-process
|
|
50
50
|
# synthesizer). When None, _init_tts lazily constructs the local
|
|
51
|
-
# chatterbox-backed TextToSpeech
|
|
52
|
-
#
|
|
53
|
-
# chatterbox
|
|
51
|
+
# chatterbox-backed TextToSpeech (from the [ai] extra). Supplying a
|
|
52
|
+
# backend lets dubbing run synthesis out-of-process without loading
|
|
53
|
+
# chatterbox here.
|
|
54
54
|
self._tts_backend = tts_backend
|
|
55
55
|
logger.info(
|
|
56
56
|
"LocalDubbingPipeline initialized with %s%s",
|
|
@@ -211,8 +211,8 @@ class LocalDubbingPipeline:
|
|
|
211
211
|
When a ``tts_backend`` was injected at construction it is used as-is
|
|
212
212
|
(it owns its own language handling). Otherwise the local
|
|
213
213
|
chatterbox-backed :class:`TextToSpeech` is constructed for ``language``
|
|
214
|
-
— importing it requires the ``[
|
|
215
|
-
|
|
214
|
+
— importing it requires the ``[ai]`` extra and raises a clear
|
|
215
|
+
``pip install 'videopython[ai]'``-pointing ``ImportError`` at this point.
|
|
216
216
|
"""
|
|
217
217
|
if self._tts_backend is not None:
|
|
218
218
|
self._tts = self._tts_backend
|
|
@@ -38,71 +38,6 @@ def _build_stream_maps(keep_original_audio: bool) -> list[str]:
|
|
|
38
38
|
return maps
|
|
39
39
|
|
|
40
40
|
|
|
41
|
-
def replace_audio_stream(
|
|
42
|
-
video_path: str | Path,
|
|
43
|
-
audio_path: str | Path,
|
|
44
|
-
output_path: str | Path,
|
|
45
|
-
audio_codec: str = "aac",
|
|
46
|
-
audio_bitrate: str = "192k",
|
|
47
|
-
keep_original_audio: bool = False,
|
|
48
|
-
) -> None:
|
|
49
|
-
"""Copy ``video_path``'s video stream and mux in ``audio_path`` as the audio track.
|
|
50
|
-
|
|
51
|
-
Uses ffmpeg stream-copy for video (no re-encode) and encodes audio to AAC.
|
|
52
|
-
Subtitle streams from ``video_path`` are carried through unchanged
|
|
53
|
-
(stream-copy). ``-shortest`` trims to the shorter of the two streams so
|
|
54
|
-
the output duration matches the source video when the dubbed audio is
|
|
55
|
-
slightly longer.
|
|
56
|
-
|
|
57
|
-
Args:
|
|
58
|
-
video_path: Source video file (video + subtitle streams are copied unchanged).
|
|
59
|
-
audio_path: Audio file to use as the new (default) audio track.
|
|
60
|
-
output_path: Destination file. Overwritten if it exists.
|
|
61
|
-
audio_codec: ffmpeg audio codec name. Defaults to ``aac`` (MP4-compatible).
|
|
62
|
-
audio_bitrate: Audio bitrate passed to ffmpeg (``-b:a``).
|
|
63
|
-
keep_original_audio: If True, retain the source audio as a secondary
|
|
64
|
-
track behind the dubbed one. Useful for editorial A/B.
|
|
65
|
-
|
|
66
|
-
Raises:
|
|
67
|
-
FileNotFoundError: If ``video_path`` or ``audio_path`` does not exist.
|
|
68
|
-
RemuxError: If ffmpeg returns a non-zero exit code.
|
|
69
|
-
"""
|
|
70
|
-
video_path = Path(video_path)
|
|
71
|
-
audio_path = Path(audio_path)
|
|
72
|
-
output_path = Path(output_path)
|
|
73
|
-
|
|
74
|
-
if not video_path.exists():
|
|
75
|
-
raise FileNotFoundError(f"Video file not found: {video_path}")
|
|
76
|
-
if not audio_path.exists():
|
|
77
|
-
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
|
78
|
-
|
|
79
|
-
cmd = [
|
|
80
|
-
"ffmpeg",
|
|
81
|
-
"-y",
|
|
82
|
-
"-i",
|
|
83
|
-
str(video_path),
|
|
84
|
-
"-i",
|
|
85
|
-
str(audio_path),
|
|
86
|
-
*_build_stream_maps(keep_original_audio),
|
|
87
|
-
"-c:v",
|
|
88
|
-
"copy",
|
|
89
|
-
"-c:a",
|
|
90
|
-
audio_codec,
|
|
91
|
-
"-b:a",
|
|
92
|
-
audio_bitrate,
|
|
93
|
-
"-c:s",
|
|
94
|
-
"copy",
|
|
95
|
-
"-shortest",
|
|
96
|
-
str(output_path),
|
|
97
|
-
]
|
|
98
|
-
|
|
99
|
-
logger.info("replace_audio_stream: %s + %s -> %s", video_path, audio_path, output_path)
|
|
100
|
-
try:
|
|
101
|
-
_ffmpeg.run(cmd)
|
|
102
|
-
except FFmpegRunError as e:
|
|
103
|
-
raise RemuxError(str(e)) from e
|
|
104
|
-
|
|
105
|
-
|
|
106
41
|
def replace_audio_stream_from_audio(
|
|
107
42
|
video_path: str | Path,
|
|
108
43
|
audio: Audio,
|
|
@@ -111,13 +46,13 @@ def replace_audio_stream_from_audio(
|
|
|
111
46
|
audio_bitrate: str = "192k",
|
|
112
47
|
keep_original_audio: bool = False,
|
|
113
48
|
) -> None:
|
|
114
|
-
"""
|
|
49
|
+
"""Copy ``video_path``'s video stream and mux in an in-memory ``Audio`` as the audio track.
|
|
115
50
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
51
|
+
Serializes the WAV in memory and feeds it to ffmpeg via stdin, avoiding an
|
|
52
|
+
``Audio.save -> read-from-disk -> ffmpeg`` round-trip. For long dubs this
|
|
53
|
+
saves a full WAV write+read of the output audio (~10 GB for a 2h source).
|
|
54
|
+
Video is stream-copied (no re-encode); subtitle streams from ``video_path``
|
|
55
|
+
are carried through unchanged. ``-shortest`` trims to the shorter stream.
|
|
121
56
|
|
|
122
57
|
Args:
|
|
123
58
|
video_path: Source video file (video + subtitle streams are copied unchanged).
|