videopython 0.51.0__tar.gz → 0.52.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {videopython-0.51.0 → videopython-0.52.1}/PKG-INFO +1 -1
- {videopython-0.51.0 → videopython-0.52.1}/pyproject.toml +1 -1
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/__init__.py +7 -2
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/_ollama.py +4 -26
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/_revisions.py +1 -1
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/auto_edit/__init__.py +7 -1
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/auto_edit/backend.py +3 -1
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/auto_edit/editor.py +3 -1
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/auto_edit/resolve.py +2 -1
- videopython-0.51.0/src/videopython/ai/dubbing/loudness.py → videopython-0.52.1/src/videopython/ai/dubbing/audio_ops.py +46 -1
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/dubbing/config.py +13 -1
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/dubbing/dubber.py +5 -7
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/dubbing/models.py +22 -3
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/dubbing/pipeline.py +51 -53
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/dubbing/quality.py +3 -1
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/dubbing/remux.py +2 -1
- {videopython-0.51.0/src/videopython/ai/understanding → videopython-0.52.1/src/videopython/ai/dubbing}/separation.py +3 -8
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/dubbing/timing.py +1 -23
- videopython-0.52.1/src/videopython/ai/errors.py +13 -0
- videopython-0.52.1/src/videopython/ai/keyframe.py +43 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/transforms.py +2 -2
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/understanding/__init__.py +6 -4
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/understanding/audio.py +0 -178
- videopython-0.52.1/src/videopython/ai/understanding/classification.py +192 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/understanding/faces.py +80 -48
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/video_analysis/__init__.py +7 -1
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/video_analysis/analyzer.py +41 -141
- videopython-0.51.0/src/videopython/ai/video_analysis/stages.py → videopython-0.52.1/src/videopython/ai/video_analysis/detectors.py +2 -2
- videopython-0.52.1/src/videopython/ai/video_analysis/source_metadata.py +133 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/mcp/server.py +2 -2
- videopython-0.51.0/src/videopython/ai/dubbing/expressiveness.py +0 -47
- {videopython-0.51.0 → videopython-0.52.1}/.gitignore +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/LICENSE +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/README.md +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/__init__.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/_device.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/_optional.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/_predictor.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/auto_edit/catalog.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/auto_edit/local.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/auto_edit/models.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/dubbing/__init__.py +0 -0
- {videopython-0.51.0/src/videopython/ai/generation → videopython-0.52.1/src/videopython/ai/dubbing}/_tts_backend.py +0 -0
- {videopython-0.51.0/src/videopython/ai/generation → videopython-0.52.1/src/videopython/ai/dubbing}/translation.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/dubbing/voice_sample.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/effects.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/generation/__init__.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/generation/audio.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/generation/image.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/generation/video.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/ops.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/understanding/_yolo.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/understanding/image.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/understanding/objects.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/understanding/temporal.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/video_analysis/models.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/ai/video_analysis/sampling.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/audio/__init__.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/audio/analysis.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/audio/audio.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/__init__.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/_dimensions.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/_ffmpeg.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/_video_io.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/description.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/draw_detections.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/exceptions.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/fonts/Anton-OFL.txt +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/fonts/Anton-Regular.ttf +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/fonts/BebasNeue-OFL.txt +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/fonts/BebasNeue-Regular.ttf +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/fonts/DejaVuSans.ttf +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/fonts/LICENSE_DEJAVU +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/fonts/Lato-Bold.ttf +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/fonts/Lato-OFL.txt +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/fonts/Poppins-Bold.ttf +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/fonts/Poppins-OFL.txt +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/fonts/__init__.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/transcription.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/base/video.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/editing/__init__.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/editing/_ass.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/editing/_easing.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/editing/_schema.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/editing/audio_ops.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/editing/effects.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/editing/operation.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/editing/streaming.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/editing/transcription_overlay.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/editing/transforms.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/editing/video_edit.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/mcp/__init__.py +0 -0
- {videopython-0.51.0 → videopython-0.52.1}/src/videopython/py.typed +0 -0
|
@@ -34,6 +34,7 @@ if TYPE_CHECKING:
|
|
|
34
34
|
from .auto_edit import StructuredVisionLLM as StructuredVisionLLM
|
|
35
35
|
from .auto_edit import build_catalog as build_catalog
|
|
36
36
|
from .effects import ObjectDetectionOverlay as ObjectDetectionOverlay
|
|
37
|
+
from .errors import AiError as AiError
|
|
37
38
|
from .generation import ImageToVideo as ImageToVideo
|
|
38
39
|
from .generation import TextToImage as TextToImage
|
|
39
40
|
from .generation import TextToMusic as TextToMusic
|
|
@@ -42,7 +43,8 @@ if TYPE_CHECKING:
|
|
|
42
43
|
from .transforms import FaceTrackingCrop as FaceTrackingCrop
|
|
43
44
|
from .understanding import AudioClassifier as AudioClassifier
|
|
44
45
|
from .understanding import AudioToText as AudioToText
|
|
45
|
-
from .understanding import
|
|
46
|
+
from .understanding import FaceShotTracker as FaceShotTracker
|
|
47
|
+
from .understanding import FaceSmoothingTracker as FaceSmoothingTracker
|
|
46
48
|
from .understanding import ObjectDetector as ObjectDetector
|
|
47
49
|
from .understanding import SceneVLM as SceneVLM
|
|
48
50
|
from .understanding import SemanticSceneDetector as SemanticSceneDetector
|
|
@@ -63,7 +65,8 @@ _exports: dict[str, str] = {
|
|
|
63
65
|
# Understanding
|
|
64
66
|
"AudioToText": ".understanding",
|
|
65
67
|
"AudioClassifier": ".understanding",
|
|
66
|
-
"
|
|
68
|
+
"FaceShotTracker": ".understanding",
|
|
69
|
+
"FaceSmoothingTracker": ".understanding",
|
|
67
70
|
"ObjectDetector": ".understanding",
|
|
68
71
|
"SceneVLM": ".understanding",
|
|
69
72
|
"SemanticSceneDetector": ".understanding",
|
|
@@ -75,6 +78,8 @@ _exports: dict[str, str] = {
|
|
|
75
78
|
"VideoAnalysis": ".video_analysis",
|
|
76
79
|
"VideoAnalysisConfig": ".video_analysis",
|
|
77
80
|
"VideoAnalyzer": ".video_analysis",
|
|
81
|
+
# Errors
|
|
82
|
+
"AiError": ".errors",
|
|
78
83
|
# Auto-editing (LLM-authored edits)
|
|
79
84
|
"AutoEditor": ".auto_edit",
|
|
80
85
|
"AutoEditError": ".auto_edit",
|
|
@@ -2,19 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
import base64
|
|
6
|
-
import io
|
|
7
5
|
import json
|
|
8
6
|
from typing import Any
|
|
9
7
|
|
|
10
|
-
import cv2
|
|
11
8
|
import numpy as np
|
|
12
|
-
from PIL import Image
|
|
13
9
|
|
|
14
10
|
from videopython.ai._optional import require
|
|
11
|
+
from videopython.ai.errors import AiError
|
|
12
|
+
from videopython.ai.keyframe import encode_png_b64
|
|
15
13
|
|
|
16
14
|
|
|
17
|
-
class OllamaError(RuntimeError):
|
|
15
|
+
class OllamaError(AiError, RuntimeError):
|
|
18
16
|
"""Ollama returned unusable output (non-JSON or an unexpected shape)."""
|
|
19
17
|
|
|
20
18
|
|
|
@@ -50,7 +48,7 @@ class OllamaStructuredClient:
|
|
|
50
48
|
"""Return the parsed JSON object Ollama generates under ``schema``."""
|
|
51
49
|
user: dict[str, Any] = {"role": "user", "content": text}
|
|
52
50
|
if images:
|
|
53
|
-
user["images"] = [
|
|
51
|
+
user["images"] = [encode_png_b64(image) for image in images]
|
|
54
52
|
messages = [{"role": "system", "content": system}, user]
|
|
55
53
|
response = self._get_client().chat(model=self.model, messages=messages, format=schema, options=self.options)
|
|
56
54
|
content = response.message.content
|
|
@@ -64,23 +62,3 @@ class OllamaStructuredClient:
|
|
|
64
62
|
|
|
65
63
|
def unload(self) -> None:
|
|
66
64
|
self._client = None
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
# Used only on the MCP keyframe path (videopython.mcp). SceneVLM captioning and the local
|
|
70
|
-
# planner deliberately encode full-resolution frames via _encode_png_b64.
|
|
71
|
-
KEYFRAME_MAX_DIM = 768 # bound a keyframe's longest side before PNG-encoding for the MCP payload
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
def _downscale(frame: np.ndarray, max_dim: int = KEYFRAME_MAX_DIM) -> np.ndarray:
|
|
75
|
-
"""Shrink an RGB frame so its longest side is at most ``max_dim`` (aspect preserved; never upscales)."""
|
|
76
|
-
h, w = frame.shape[:2]
|
|
77
|
-
scale = max_dim / max(h, w)
|
|
78
|
-
if scale >= 1.0:
|
|
79
|
-
return frame
|
|
80
|
-
return cv2.resize(frame, (max(1, round(w * scale)), max(1, round(h * scale))), interpolation=cv2.INTER_AREA)
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
def _encode_png_b64(frame: np.ndarray) -> str:
|
|
84
|
-
buffer = io.BytesIO()
|
|
85
|
-
Image.fromarray(frame).save(buffer, format="PNG")
|
|
86
|
-
return base64.b64encode(buffer.getvalue()).decode("ascii")
|
|
@@ -65,7 +65,7 @@ MODEL_REVISIONS: dict[str, str] = {
|
|
|
65
65
|
# Speaker diarization (ai/understanding/audio.py: PYANNOTE_DIARIZATION_MODEL).
|
|
66
66
|
# Gated repo (auto-approved); from_pretrained needs an accepted HF token.
|
|
67
67
|
"pyannote/speaker-diarization-community-1": "3533c8cf8e369892e6b79ff1bf80f7b0286a54ee",
|
|
68
|
-
# Audio event classifier (ai/understanding/
|
|
68
|
+
# Audio event classifier (ai/understanding/classification.py: AudioClassifier)
|
|
69
69
|
"MIT/ast-finetuned-audioset-10-10-0.4593": "f826b80d28226b62986cc218e5cec390b1096902",
|
|
70
70
|
# Face detection checkpoint (ai/understanding/faces.py: hf_hub_download)
|
|
71
71
|
"arnabdhar/YOLOv8-Face-Detection": "52fa54977207fa4f021de949b515fb19dcab4488",
|
|
@@ -1,4 +1,10 @@
|
|
|
1
|
-
"""LLM-authored editing: build a scene catalog and plan a VideoEdit from it.
|
|
1
|
+
"""LLM-authored editing: build a scene catalog and plan a VideoEdit from it.
|
|
2
|
+
|
|
3
|
+
Uses eager imports (unlike the leaf packages): it is a thin orchestration
|
|
4
|
+
package whose submodules load together, and it MUST eagerly import
|
|
5
|
+
``videopython.ai.ops`` so the AI editing ops are registered before any
|
|
6
|
+
``EditPlan.json_schema()`` call. Heavy ML deps it pulls in stay lazy.
|
|
7
|
+
"""
|
|
2
8
|
|
|
3
9
|
from __future__ import annotations
|
|
4
10
|
|
|
@@ -4,11 +4,13 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
|
|
6
6
|
|
|
7
|
+
from videopython.ai.errors import AiError
|
|
8
|
+
|
|
7
9
|
if TYPE_CHECKING:
|
|
8
10
|
import numpy as np
|
|
9
11
|
|
|
10
12
|
|
|
11
|
-
class PlannerError(RuntimeError):
|
|
13
|
+
class PlannerError(AiError, RuntimeError):
|
|
12
14
|
"""A backend produced unusable output; the editor retries (infra errors should propagate instead)."""
|
|
13
15
|
|
|
14
16
|
|
|
@@ -8,6 +8,8 @@ from typing import TYPE_CHECKING, Any, Literal
|
|
|
8
8
|
|
|
9
9
|
from pydantic import ValidationError
|
|
10
10
|
|
|
11
|
+
from videopython.ai.errors import AiError
|
|
12
|
+
|
|
11
13
|
from .backend import PlannerError, StructuredVisionLLM
|
|
12
14
|
from .catalog import build_catalog
|
|
13
15
|
from .models import CatalogBundle, CatalogScene, EditPlan
|
|
@@ -30,7 +32,7 @@ _SYSTEM_PROMPT = (
|
|
|
30
32
|
_NormalizeTarget = tuple[int, int] | Literal["first", "largest", "match"]
|
|
31
33
|
|
|
32
34
|
|
|
33
|
-
class AutoEditError(RuntimeError):
|
|
35
|
+
class AutoEditError(AiError, RuntimeError):
|
|
34
36
|
"""The planner could not produce a valid edit within the retry budget."""
|
|
35
37
|
|
|
36
38
|
|
|
@@ -2,12 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
from videopython.ai.errors import AiError
|
|
5
6
|
from videopython.editing import SegmentConfig, VideoEdit
|
|
6
7
|
|
|
7
8
|
from .models import EditCatalog, EditPlan
|
|
8
9
|
|
|
9
10
|
|
|
10
|
-
class UnknownSceneIdsError(ValueError):
|
|
11
|
+
class UnknownSceneIdsError(AiError, ValueError):
|
|
11
12
|
"""An EditPlan referenced scene ids absent from the catalog."""
|
|
12
13
|
|
|
13
14
|
def __init__(self, ids: list[str]) -> None:
|
|
@@ -1,4 +1,11 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Audio DSP helpers for the dubbing pipeline.
|
|
2
|
+
|
|
3
|
+
Merges the former ``expressiveness`` and ``loudness`` leaf modules: RMS,
|
|
4
|
+
source-prosody -> Chatterbox expressiveness mapping, and LUFS/peak loudness
|
|
5
|
+
matching. These are the small numpy/DSP helpers the pipeline composes; keeping
|
|
6
|
+
them in one file (rather than two ~50-line files) makes the audio-side toolkit
|
|
7
|
+
legible.
|
|
8
|
+
"""
|
|
2
9
|
|
|
3
10
|
from __future__ import annotations
|
|
4
11
|
|
|
@@ -6,10 +13,48 @@ from typing import TYPE_CHECKING
|
|
|
6
13
|
|
|
7
14
|
import numpy as np
|
|
8
15
|
|
|
16
|
+
from videopython.ai.dubbing.models import Expressiveness
|
|
17
|
+
|
|
9
18
|
if TYPE_CHECKING:
|
|
10
19
|
from videopython.audio import Audio
|
|
11
20
|
|
|
12
21
|
|
|
22
|
+
def rms(data: np.ndarray) -> float:
|
|
23
|
+
"""RMS over samples; ``0.0`` for empty input. float64 reduction so a
|
|
24
|
+
long slice can't overflow the squared accumulator."""
|
|
25
|
+
if data.size == 0:
|
|
26
|
+
return 0.0
|
|
27
|
+
return float(np.sqrt(np.mean(np.square(data, dtype=np.float64))))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# --- Source-prosody-driven expressiveness (Chatterbox TTS knobs) -----------
|
|
31
|
+
# Source-segment RMS / whole-vocals RMS below CALM lands in the calm bucket;
|
|
32
|
+
# above DRAMATIC in the dramatic bucket; in between gets Chatterbox's defaults.
|
|
33
|
+
# Knob values picked by-ear on cam1_1min.mp4 -- see RELEASE_NOTES 0.29.0.
|
|
34
|
+
CALM_RATIO_THRESHOLD = 0.7
|
|
35
|
+
DRAMATIC_RATIO_THRESHOLD = 1.3
|
|
36
|
+
_CALM = Expressiveness(exaggeration=0.3, cfg_weight=0.7)
|
|
37
|
+
_DRAMATIC = Expressiveness(exaggeration=0.85, cfg_weight=0.35)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def expressiveness_for(source_slice: Audio, baseline_rms: float) -> Expressiveness:
|
|
41
|
+
"""Map a source vocals slice to a Chatterbox expressiveness profile
|
|
42
|
+
by RMS ratio. Falls back to the no-knobs default for empty or silent
|
|
43
|
+
inputs."""
|
|
44
|
+
if baseline_rms <= 0.0:
|
|
45
|
+
return Expressiveness()
|
|
46
|
+
segment_rms = rms(source_slice.data)
|
|
47
|
+
if segment_rms <= 0.0:
|
|
48
|
+
return Expressiveness()
|
|
49
|
+
ratio = segment_rms / baseline_rms
|
|
50
|
+
if ratio < CALM_RATIO_THRESHOLD:
|
|
51
|
+
return _CALM
|
|
52
|
+
if ratio > DRAMATIC_RATIO_THRESHOLD:
|
|
53
|
+
return _DRAMATIC
|
|
54
|
+
return Expressiveness()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# --- LUFS / peak loudness matching -----------------------------------------
|
|
13
58
|
# BS.1770 integrated-loudness measurement requires at least 400 ms of audio
|
|
14
59
|
# (one gating block). Below this, fall back to peak match -- pyloudnorm
|
|
15
60
|
# returns -inf or warns, neither of which gives a usable gain.
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
from typing import Literal
|
|
5
|
+
from typing import Any, Literal
|
|
6
6
|
|
|
7
7
|
from pydantic import BaseModel, ConfigDict
|
|
8
8
|
|
|
@@ -63,6 +63,18 @@ class DubbingConfig(BaseModel):
|
|
|
63
63
|
translator_model: str | None = None
|
|
64
64
|
translator_host: str | None = None
|
|
65
65
|
|
|
66
|
+
@classmethod
|
|
67
|
+
def from_args(cls, config: DubbingConfig | None = None, /, **kwargs: Any) -> DubbingConfig:
|
|
68
|
+
"""Resolve either a ``config`` object or flat knob kwargs into a config.
|
|
69
|
+
|
|
70
|
+
The shared accept-one-or-the-other guard for ``VideoDubber`` and
|
|
71
|
+
``LocalDubbingPipeline``, which both take ``config=DubbingConfig(...)``
|
|
72
|
+
or the flat kwargs (not both).
|
|
73
|
+
"""
|
|
74
|
+
if config is not None and kwargs:
|
|
75
|
+
raise TypeError("Pass either `config=` or knob kwargs, not both")
|
|
76
|
+
return config or cls(**kwargs)
|
|
77
|
+
|
|
66
78
|
def init_log_fields(self) -> dict[str, object]:
|
|
67
79
|
"""Subset of fields surfaced in the init-log line.
|
|
68
80
|
|
|
@@ -10,7 +10,7 @@ from videopython.ai.dubbing.config import DubbingConfig
|
|
|
10
10
|
from videopython.ai.dubbing.models import DubbingResult, RevoiceResult
|
|
11
11
|
|
|
12
12
|
if TYPE_CHECKING:
|
|
13
|
-
from videopython.ai.
|
|
13
|
+
from videopython.ai.dubbing._tts_backend import SpeechBackend
|
|
14
14
|
from videopython.base.video import Video
|
|
15
15
|
|
|
16
16
|
logger = logging.getLogger(__name__)
|
|
@@ -20,8 +20,8 @@ class VideoDubber:
|
|
|
20
20
|
"""Dubs videos into different languages using the local pipeline.
|
|
21
21
|
|
|
22
22
|
Accepts either a :class:`DubbingConfig` or the same knobs as flat kwargs
|
|
23
|
-
(``device``, ``low_memory``, ``whisper_model``, ``
|
|
24
|
-
the flat path builds a ``DubbingConfig`` internally. See
|
|
23
|
+
(``device``, ``low_memory``, ``whisper_model``, ``translator_model``, etc.)
|
|
24
|
+
-- the flat path builds a ``DubbingConfig`` internally. See
|
|
25
25
|
:class:`DubbingConfig` for the full knob list and defaults.
|
|
26
26
|
"""
|
|
27
27
|
|
|
@@ -32,9 +32,7 @@ class VideoDubber:
|
|
|
32
32
|
tts_backend: SpeechBackend | None = None,
|
|
33
33
|
**kwargs: Any,
|
|
34
34
|
):
|
|
35
|
-
|
|
36
|
-
raise TypeError("Pass either `config=` or knob kwargs, not both")
|
|
37
|
-
self.config = config or DubbingConfig(**kwargs)
|
|
35
|
+
self.config = DubbingConfig.from_args(config, **kwargs)
|
|
38
36
|
# Optional injected speech backend. None -> the pipeline lazily builds
|
|
39
37
|
# the local chatterbox-backed TextToSpeech (from the [ai] extra). Inject
|
|
40
38
|
# a SpeechBackend to run synthesis out-of-process (e.g. a remote/Modal
|
|
@@ -248,6 +246,6 @@ class VideoDubber:
|
|
|
248
246
|
|
|
249
247
|
@staticmethod
|
|
250
248
|
def get_supported_languages() -> dict[str, str]:
|
|
251
|
-
from videopython.ai.
|
|
249
|
+
from videopython.ai.dubbing.translation import OllamaTranslator
|
|
252
250
|
|
|
253
251
|
return OllamaTranslator.get_supported_languages()
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
from
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Annotated, Any
|
|
6
7
|
|
|
7
8
|
from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, PlainSerializer, model_validator
|
|
8
9
|
|
|
@@ -10,8 +11,26 @@ from videopython.ai.dubbing.quality import TranscriptQuality
|
|
|
10
11
|
from videopython.audio import Audio
|
|
11
12
|
from videopython.base.transcription import Transcription, TranscriptionSegment
|
|
12
13
|
|
|
13
|
-
|
|
14
|
-
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class TimingAdjustment:
|
|
17
|
+
"""Information about a timing adjustment made to a segment.
|
|
18
|
+
|
|
19
|
+
Attributes:
|
|
20
|
+
segment_index: Index of the segment that was adjusted.
|
|
21
|
+
original_duration: Original duration of the segment in seconds.
|
|
22
|
+
target_duration: Target duration to fit into.
|
|
23
|
+
actual_duration: Actual duration after adjustment.
|
|
24
|
+
speed_factor: Speed factor applied (> 1 means sped up).
|
|
25
|
+
was_truncated: Whether the segment had to be truncated.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
segment_index: int
|
|
29
|
+
original_duration: float
|
|
30
|
+
target_duration: float
|
|
31
|
+
actual_duration: float
|
|
32
|
+
speed_factor: float
|
|
33
|
+
was_truncated: bool
|
|
15
34
|
|
|
16
35
|
|
|
17
36
|
# Speed factors within this band of 1.0 are treated as a "clean" timing
|
|
@@ -7,16 +7,16 @@ import tempfile
|
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
from typing import TYPE_CHECKING, Any, Callable
|
|
9
9
|
|
|
10
|
-
from videopython.ai.dubbing import
|
|
10
|
+
from videopython.ai.dubbing import audio_ops, voice_sample
|
|
11
11
|
from videopython.ai.dubbing.config import DubbingConfig
|
|
12
12
|
from videopython.ai.dubbing.models import DubbingResult, Expressiveness, RevoiceResult, SeparatedAudio, TimingSummary
|
|
13
13
|
from videopython.ai.dubbing.quality import GarbageTranscriptError, assess_transcript
|
|
14
14
|
from videopython.ai.dubbing.timing import TimingSynchronizer
|
|
15
|
-
from videopython.ai.
|
|
15
|
+
from videopython.ai.dubbing.translation import DEFAULT_TRANSLATION_MODEL, OllamaTranslator
|
|
16
16
|
|
|
17
17
|
if TYPE_CHECKING:
|
|
18
|
+
from videopython.ai.dubbing._tts_backend import SpeechBackend
|
|
18
19
|
from videopython.ai.dubbing.models import TranslatedSegment
|
|
19
|
-
from videopython.ai.generation._tts_backend import SpeechBackend
|
|
20
20
|
from videopython.audio import Audio
|
|
21
21
|
from videopython.base.transcription import Transcription
|
|
22
22
|
|
|
@@ -40,12 +40,9 @@ class LocalDubbingPipeline:
|
|
|
40
40
|
tts_backend: SpeechBackend | None = None,
|
|
41
41
|
**kwargs: Any,
|
|
42
42
|
):
|
|
43
|
-
# ``DubbingConfig`` consolidates the
|
|
44
|
-
#
|
|
45
|
-
|
|
46
|
-
if config is not None and kwargs:
|
|
47
|
-
raise TypeError("Pass either `config=` or knob kwargs, not both")
|
|
48
|
-
self.config = config or DubbingConfig(**kwargs)
|
|
43
|
+
# ``DubbingConfig`` consolidates the knobs that used to be constructor
|
|
44
|
+
# kwargs. Either ``config=`` or the flat kwargs are accepted (not both).
|
|
45
|
+
self.config = DubbingConfig.from_args(config, **kwargs)
|
|
49
46
|
# Injected speech backend (a SpeechBackend, e.g. a remote/out-of-process
|
|
50
47
|
# synthesizer). When None, _init_tts lazily constructs the local
|
|
51
48
|
# chatterbox-backed TextToSpeech (from the [ai] extra). Supplying a
|
|
@@ -224,7 +221,7 @@ class LocalDubbingPipeline:
|
|
|
224
221
|
|
|
225
222
|
def _init_separator(self) -> None:
|
|
226
223
|
"""Initialize the audio separator."""
|
|
227
|
-
from videopython.ai.
|
|
224
|
+
from videopython.ai.dubbing.separation import AudioSeparator
|
|
228
225
|
|
|
229
226
|
self._separator = AudioSeparator(device=self.config.device)
|
|
230
227
|
|
|
@@ -232,6 +229,43 @@ class LocalDubbingPipeline:
|
|
|
232
229
|
"""Initialize the timing synchronizer."""
|
|
233
230
|
self._synchronizer = TimingSynchronizer()
|
|
234
231
|
|
|
232
|
+
def _separate(
|
|
233
|
+
self,
|
|
234
|
+
source_audio: Audio,
|
|
235
|
+
transcription: Transcription,
|
|
236
|
+
progress_fraction: float,
|
|
237
|
+
report_progress: Callable[[str, float], None],
|
|
238
|
+
) -> tuple[SeparatedAudio | None, Audio, Audio | None]:
|
|
239
|
+
"""Demucs source separation over the speech regions; shared by process()/revoice().
|
|
240
|
+
|
|
241
|
+
Limits Demucs to the speech-bearing portion of the audio: the transcription
|
|
242
|
+
has already located every speech region, and separating outside those is pure
|
|
243
|
+
overhead (no vocals to isolate). On talk-heavy sources with silence/music gaps
|
|
244
|
+
this roughly halves separation time; when speech covers most of the track,
|
|
245
|
+
separate_regions falls back to a full-track separate().
|
|
246
|
+
|
|
247
|
+
Returns ``(separated_audio, vocal_audio, background_audio)``. In low_memory
|
|
248
|
+
mode the ``SeparatedAudio`` container is dropped (returned as ``None``) so
|
|
249
|
+
vocals and background release as soon as their last local reference goes.
|
|
250
|
+
"""
|
|
251
|
+
report_progress("Separating audio", progress_fraction)
|
|
252
|
+
if self._separator is None:
|
|
253
|
+
self._init_separator()
|
|
254
|
+
|
|
255
|
+
from videopython.ai.dubbing.separation import merge_regions
|
|
256
|
+
|
|
257
|
+
speech_regions = merge_regions(
|
|
258
|
+
[(s.start, s.end) for s in transcription.segments],
|
|
259
|
+
audio_duration=source_audio.metadata.duration_seconds,
|
|
260
|
+
)
|
|
261
|
+
separated_audio = self._separator.separate_regions(source_audio, speech_regions)
|
|
262
|
+
self._maybe_unload("_separator")
|
|
263
|
+
vocal_audio = separated_audio.vocals
|
|
264
|
+
background_audio = separated_audio.background
|
|
265
|
+
if self.config.low_memory:
|
|
266
|
+
separated_audio = None
|
|
267
|
+
return separated_audio, vocal_audio, background_audio
|
|
268
|
+
|
|
235
269
|
def _finalise_audio(
|
|
236
270
|
self,
|
|
237
271
|
dubbed_speech: Audio,
|
|
@@ -253,7 +287,7 @@ class LocalDubbingPipeline:
|
|
|
253
287
|
final_audio = background_audio.overlay(dubbed_speech, position=0.0)
|
|
254
288
|
else:
|
|
255
289
|
final_audio = dubbed_speech
|
|
256
|
-
return
|
|
290
|
+
return audio_ops.loudness_match(final_audio, source_audio)
|
|
257
291
|
|
|
258
292
|
def process(
|
|
259
293
|
self,
|
|
@@ -358,32 +392,9 @@ class LocalDubbingPipeline:
|
|
|
358
392
|
background_audio: Audio | None = None
|
|
359
393
|
|
|
360
394
|
if preserve_background:
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
self._init_separator()
|
|
364
|
-
|
|
365
|
-
# Limit Demucs to the speech-bearing portion of the audio. The
|
|
366
|
-
# transcription has already located every speech region; running
|
|
367
|
-
# source separation outside those is pure overhead (no vocals to
|
|
368
|
-
# isolate). On talk-heavy sources with silence/music gaps this
|
|
369
|
-
# roughly halves separation time. When speech covers most of the
|
|
370
|
-
# track separate_regions falls back to a full-track separate().
|
|
371
|
-
from videopython.ai.understanding.separation import _merge_regions
|
|
372
|
-
|
|
373
|
-
speech_regions = _merge_regions(
|
|
374
|
-
[(s.start, s.end) for s in transcription.segments],
|
|
375
|
-
audio_duration=source_audio.metadata.duration_seconds,
|
|
395
|
+
separated_audio, vocal_audio, background_audio = self._separate(
|
|
396
|
+
source_audio, transcription, 0.15, report_progress
|
|
376
397
|
)
|
|
377
|
-
separated_audio = self._separator.separate_regions(source_audio, speech_regions)
|
|
378
|
-
self._maybe_unload("_separator")
|
|
379
|
-
vocal_audio = separated_audio.vocals
|
|
380
|
-
background_audio = separated_audio.background
|
|
381
|
-
# In low_memory mode, drop the SeparatedAudio container so vocals
|
|
382
|
-
# and background can be released as soon as their last local
|
|
383
|
-
# reference goes (after voice-sample extraction and final overlay
|
|
384
|
-
# respectively). The result will report separated_audio=None.
|
|
385
|
-
if self.config.low_memory:
|
|
386
|
-
separated_audio = None
|
|
387
398
|
|
|
388
399
|
voice_samples: dict[str, Audio] = {}
|
|
389
400
|
if voice_clone:
|
|
@@ -401,10 +412,10 @@ class LocalDubbingPipeline:
|
|
|
401
412
|
# — transcription timestamps can drift past the buffer tail
|
|
402
413
|
# (especially on synthetic test audio) and Audio.slice rejects
|
|
403
414
|
# out-of-range ends past a 0.1s tolerance.
|
|
404
|
-
baseline_rms =
|
|
415
|
+
baseline_rms = audio_ops.rms(vocal_audio.data)
|
|
405
416
|
vocal_duration = vocal_audio.metadata.duration_seconds
|
|
406
417
|
expressiveness_per_segment = [
|
|
407
|
-
|
|
418
|
+
audio_ops.expressiveness_for(
|
|
408
419
|
vocal_audio.slice(min(s.start, vocal_duration), min(s.end, vocal_duration)),
|
|
409
420
|
baseline_rms,
|
|
410
421
|
)
|
|
@@ -530,22 +541,9 @@ class LocalDubbingPipeline:
|
|
|
530
541
|
background_audio: Audio | None = None
|
|
531
542
|
|
|
532
543
|
if preserve_background:
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
self._init_separator()
|
|
536
|
-
|
|
537
|
-
from videopython.ai.understanding.separation import _merge_regions
|
|
538
|
-
|
|
539
|
-
speech_regions = _merge_regions(
|
|
540
|
-
[(s.start, s.end) for s in transcription.segments],
|
|
541
|
-
audio_duration=source_audio.metadata.duration_seconds,
|
|
544
|
+
separated_audio, vocal_audio, background_audio = self._separate(
|
|
545
|
+
source_audio, transcription, 0.20, report_progress
|
|
542
546
|
)
|
|
543
|
-
separated_audio = self._separator.separate_regions(source_audio, speech_regions)
|
|
544
|
-
self._maybe_unload("_separator")
|
|
545
|
-
vocal_audio = separated_audio.vocals
|
|
546
|
-
background_audio = separated_audio.background
|
|
547
|
-
if self.config.low_memory:
|
|
548
|
-
separated_audio = None
|
|
549
547
|
|
|
550
548
|
report_progress("Extracting voice sample", 0.40)
|
|
551
549
|
chosen_sample: Audio | None = None
|
|
@@ -24,6 +24,8 @@ from typing import TYPE_CHECKING, Literal
|
|
|
24
24
|
|
|
25
25
|
from pydantic import BaseModel, Field
|
|
26
26
|
|
|
27
|
+
from videopython.ai.errors import AiError
|
|
28
|
+
|
|
27
29
|
if TYPE_CHECKING:
|
|
28
30
|
from videopython.base.transcription import Transcription
|
|
29
31
|
|
|
@@ -74,7 +76,7 @@ class TranscriptQuality(BaseModel):
|
|
|
74
76
|
flags: list[str] = Field(default_factory=list)
|
|
75
77
|
|
|
76
78
|
|
|
77
|
-
class GarbageTranscriptError(RuntimeError):
|
|
79
|
+
class GarbageTranscriptError(AiError, RuntimeError):
|
|
78
80
|
"""Raised by the dubbing pipeline when ``strict_quality=True`` and the
|
|
79
81
|
transcript heuristic returns ``recommendation="reject"``.
|
|
80
82
|
|
|
@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING
|
|
|
10
10
|
|
|
11
11
|
import numpy as np
|
|
12
12
|
|
|
13
|
+
from videopython.ai.errors import AiError
|
|
13
14
|
from videopython.base import _ffmpeg
|
|
14
15
|
from videopython.base.exceptions import FFmpegRunError
|
|
15
16
|
|
|
@@ -19,7 +20,7 @@ if TYPE_CHECKING:
|
|
|
19
20
|
logger = logging.getLogger(__name__)
|
|
20
21
|
|
|
21
22
|
|
|
22
|
-
class RemuxError(RuntimeError):
|
|
23
|
+
class RemuxError(AiError, RuntimeError):
|
|
23
24
|
"""ffmpeg failed while replacing an audio stream."""
|
|
24
25
|
|
|
25
26
|
|
|
@@ -13,7 +13,7 @@ from videopython.audio import Audio, AudioMetadata
|
|
|
13
13
|
logger = logging.getLogger(__name__)
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def
|
|
16
|
+
def merge_regions(
|
|
17
17
|
regions: list[tuple[float, float]],
|
|
18
18
|
audio_duration: float,
|
|
19
19
|
pad: float = 0.5,
|
|
@@ -59,14 +59,9 @@ def _merge_regions(
|
|
|
59
59
|
class AudioSeparator(ManagedPredictor):
|
|
60
60
|
"""Separates audio into vocals and background components using Demucs."""
|
|
61
61
|
|
|
62
|
-
SUPPORTED_MODELS: list[str] = ["htdemucs", "htdemucs_ft", "htdemucs_6s", "mdx_extra"]
|
|
63
62
|
STEM_NAMES = ["drums", "bass", "other", "vocals"]
|
|
64
|
-
STEM_NAMES_6S = ["drums", "bass", "other", "vocals", "guitar", "piano"]
|
|
65
63
|
|
|
66
64
|
def __init__(self, model_name: str = "htdemucs", device: str | None = None):
|
|
67
|
-
if model_name not in self.SUPPORTED_MODELS:
|
|
68
|
-
raise ValueError(f"Model '{model_name}' not supported. Supported: {self.SUPPORTED_MODELS}")
|
|
69
|
-
|
|
70
65
|
self.model_name = model_name
|
|
71
66
|
self.device = device
|
|
72
67
|
self._model: Any = None
|
|
@@ -129,7 +124,7 @@ class AudioSeparator(ManagedPredictor):
|
|
|
129
124
|
sources_np = sources[0].cpu().numpy()
|
|
130
125
|
del sources
|
|
131
126
|
|
|
132
|
-
stem_names = self.
|
|
127
|
+
stem_names = self.STEM_NAMES
|
|
133
128
|
vocals_idx = stem_names.index("vocals")
|
|
134
129
|
non_vocal_indices = [i for i in range(len(stem_names)) if i != vocals_idx]
|
|
135
130
|
|
|
@@ -185,7 +180,7 @@ class AudioSeparator(ManagedPredictor):
|
|
|
185
180
|
audio: Source audio (typically the full track).
|
|
186
181
|
regions: List of ``(start, end)`` second pairs marking
|
|
187
182
|
speech-bearing portions. Caller is responsible for
|
|
188
|
-
merging/padding (use ``
|
|
183
|
+
merging/padding (use ``merge_regions``).
|
|
189
184
|
full_separation_threshold: If the regions cover more than
|
|
190
185
|
this fraction of the audio, fall back to full-track
|
|
191
186
|
``separate()`` since per-region slicing+stitching
|
|
@@ -2,34 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
from dataclasses import dataclass
|
|
6
|
-
|
|
7
5
|
import numpy as np
|
|
8
6
|
|
|
7
|
+
from videopython.ai.dubbing.models import TimingAdjustment
|
|
9
8
|
from videopython.audio import Audio, AudioMetadata
|
|
10
9
|
|
|
11
10
|
|
|
12
|
-
@dataclass
|
|
13
|
-
class TimingAdjustment:
|
|
14
|
-
"""Information about a timing adjustment made to a segment.
|
|
15
|
-
|
|
16
|
-
Attributes:
|
|
17
|
-
segment_index: Index of the segment that was adjusted.
|
|
18
|
-
original_duration: Original duration of the segment in seconds.
|
|
19
|
-
target_duration: Target duration to fit into.
|
|
20
|
-
actual_duration: Actual duration after adjustment.
|
|
21
|
-
speed_factor: Speed factor applied (> 1 means sped up).
|
|
22
|
-
was_truncated: Whether the segment had to be truncated.
|
|
23
|
-
"""
|
|
24
|
-
|
|
25
|
-
segment_index: int
|
|
26
|
-
original_duration: float
|
|
27
|
-
target_duration: float
|
|
28
|
-
actual_duration: float
|
|
29
|
-
speed_factor: float
|
|
30
|
-
was_truncated: bool
|
|
31
|
-
|
|
32
|
-
|
|
33
11
|
class TimingSynchronizer:
|
|
34
12
|
"""Synchronizes dubbed audio segments to match original timing.
|
|
35
13
|
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Common base for ``videopython.ai`` error types.
|
|
2
|
+
|
|
3
|
+
Every error the AI layer raises subclasses :class:`AiError` (alongside the
|
|
4
|
+
builtin it semantically is — ``RuntimeError`` for operational failures,
|
|
5
|
+
``ValueError`` for bad inputs), so a caller can ``except AiError`` to catch any
|
|
6
|
+
of them without enumerating each submodule's error type.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class AiError(Exception):
|
|
13
|
+
"""Base class for all ``videopython.ai`` errors."""
|