videopython 0.43.0__tar.gz → 0.43.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {videopython-0.43.0 → videopython-0.43.1}/PKG-INFO +1 -1
- {videopython-0.43.0 → videopython-0.43.1}/pyproject.toml +1 -1
- videopython-0.43.1/src/videopython/ai/_predictor.py +54 -0
- videopython-0.43.1/src/videopython/ai/_revisions.py +109 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/generation/audio.py +8 -4
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/generation/image.py +4 -1
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/generation/qwen3.py +4 -2
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/generation/translation.py +7 -3
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/generation/video.py +8 -4
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/understanding/audio.py +12 -5
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/understanding/faces.py +2 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/understanding/image.py +7 -3
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/understanding/objects.py +3 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/understanding/separation.py +2 -1
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/understanding/temporal.py +2 -1
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/exceptions.py +65 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/video.py +10 -0
- {videopython-0.43.0 → videopython-0.43.1}/.gitignore +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/LICENSE +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/README.md +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/__init__.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/__init__.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/_device.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/_optional.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/dubbing/__init__.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/dubbing/config.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/dubbing/dubber.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/dubbing/expressiveness.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/dubbing/loudness.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/dubbing/models.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/dubbing/pipeline.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/dubbing/quality.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/dubbing/remux.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/dubbing/timing.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/dubbing/voice_sample.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/effects.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/generation/__init__.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/generation/_tts_backend.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/transforms.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/understanding/__init__.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/video_analysis/__init__.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/video_analysis/analyzer.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/video_analysis/models.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/video_analysis/sampling.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/ai/video_analysis/stages.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/audio/__init__.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/audio/analysis.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/audio/audio.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/__init__.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/_dimensions.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/_ffmpeg.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/_video_io.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/description.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/draw_detections.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/fonts/Anton-OFL.txt +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/fonts/Anton-Regular.ttf +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/fonts/BebasNeue-OFL.txt +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/fonts/BebasNeue-Regular.ttf +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/fonts/DejaVuSans.ttf +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/fonts/LICENSE_DEJAVU +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/fonts/Lato-Bold.ttf +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/fonts/Lato-OFL.txt +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/fonts/Poppins-Bold.ttf +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/fonts/Poppins-OFL.txt +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/fonts/__init__.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/base/transcription.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/editing/__init__.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/editing/_ass.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/editing/_easing.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/editing/audio_ops.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/editing/effects.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/editing/operation.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/editing/streaming.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/editing/transcription_overlay.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/editing/transforms.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/editing/video_edit.py +0 -0
- {videopython-0.43.0 → videopython-0.43.1}/src/videopython/py.typed +0 -0
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Context-manager mixin for VRAM-releasing predictors.
|
|
2
|
+
|
|
3
|
+
Every predictor under ``videopython.ai`` already exposes an ``unload()`` method
|
|
4
|
+
that drops its model reference (``self._model = None``, plus any processor /
|
|
5
|
+
pipeline fields) and calls :func:`videopython.ai._device.release_device_memory`
|
|
6
|
+
to free the allocator cache. Today that has to be called by hand -- the dubbing
|
|
7
|
+
pipeline, for example, unloads each stage manually once it is done.
|
|
8
|
+
|
|
9
|
+
:class:`ManagedPredictor` makes that bookkeeping automatic by turning any such
|
|
10
|
+
predictor into a context manager::
|
|
11
|
+
|
|
12
|
+
with SceneVLM(...) as vlm:
|
|
13
|
+
... # use vlm
|
|
14
|
+
# vlm.unload() has fired here, releasing VRAM
|
|
15
|
+
|
|
16
|
+
The mixin is deliberately tiny and dependency-free: it imports no torch /
|
|
17
|
+
transformers / ultralytics and holds no state, so it is safe to mix into any
|
|
18
|
+
predictor class regardless of how it is constructed.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
from types import TracebackType
|
|
24
|
+
from typing import TYPE_CHECKING, Literal
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from typing_extensions import Self
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ManagedPredictor:
|
|
31
|
+
"""Adds ``with``-statement support that calls ``unload()`` on exit.
|
|
32
|
+
|
|
33
|
+
Subclasses MUST define their own ``unload()`` method; this mixin does not
|
|
34
|
+
provide one. The expected contract is the one shared by every predictor in
|
|
35
|
+
``videopython.ai``: ``unload()`` clears the model reference(s) and releases
|
|
36
|
+
device memory, and is safe to call more than once (including before the
|
|
37
|
+
model was ever loaded).
|
|
38
|
+
|
|
39
|
+
``__exit__`` always returns ``False`` so exceptions raised inside the ``with``
|
|
40
|
+
block are never suppressed -- ``unload()`` runs on both the success and
|
|
41
|
+
failure paths.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def __enter__(self) -> Self:
|
|
45
|
+
return self
|
|
46
|
+
|
|
47
|
+
def __exit__(
|
|
48
|
+
self,
|
|
49
|
+
exc_type: type[BaseException] | None,
|
|
50
|
+
exc: BaseException | None,
|
|
51
|
+
tb: TracebackType | None,
|
|
52
|
+
) -> Literal[False]:
|
|
53
|
+
self.unload() # type: ignore[attr-defined]
|
|
54
|
+
return False
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Centralized HuggingFace model-revision pins.
|
|
2
|
+
|
|
3
|
+
The AI predictors load weights via ``from_pretrained(...)``,
|
|
4
|
+
``hf_hub_download(...)``, and ``YOLO(...)`` with no revision pinning by
|
|
5
|
+
default, so a silent upstream weight change can alter production analysis
|
|
6
|
+
without any code change on our side. This module maps each *fixed* model
|
|
7
|
+
identifier to the commit SHA we want to pin to, so callers can pass
|
|
8
|
+
``revision=pinned(model_id)`` to lock the download to a known-good commit.
|
|
9
|
+
|
|
10
|
+
Resolution: each SHA below is the ``sha`` field returned by the HuggingFace
|
|
11
|
+
model API (``https://huggingface.co/api/models/<repo_id>``), i.e. the latest
|
|
12
|
+
commit on the repo's ``main`` branch at the time it was captured.
|
|
13
|
+
|
|
14
|
+
Refreshing SHAs:
|
|
15
|
+
For each ``model_id`` in ``MODEL_REVISIONS`` (and the TODO list, if any),
|
|
16
|
+
fetch ``https://huggingface.co/api/models/<model_id>`` and copy the JSON
|
|
17
|
+
``sha`` field into the dict. Bump deliberately -- the whole point of a pin
|
|
18
|
+
is that it does not move on its own. After editing, re-run the AI suite
|
|
19
|
+
so the new revision is exercised before release.
|
|
20
|
+
|
|
21
|
+
Usage:
|
|
22
|
+
>>> from videopython.ai._revisions import pinned
|
|
23
|
+
>>> pinned("facebook/musicgen-small") # -> "<sha>" or None
|
|
24
|
+
Pass the result straight through:
|
|
25
|
+
``AutoProcessor.from_pretrained(model_id, revision=pinned(model_id))``.
|
|
26
|
+
``revision=None`` is the library default and means "current behavior"
|
|
27
|
+
(track ``main``), so an unpinned or dynamic model stays safe.
|
|
28
|
+
|
|
29
|
+
# ---------------------------------------------------------------------------
|
|
30
|
+
# Unpinnable-by-design (intentionally absent from MODEL_REVISIONS)
|
|
31
|
+
# ---------------------------------------------------------------------------
|
|
32
|
+
# These load paths cannot be pinned to a HuggingFace commit SHA, so they are
|
|
33
|
+
# documented here rather than registered above. ``pinned()`` returns None for
|
|
34
|
+
# them, which leaves the caller on its current (unpinned) behavior.
|
|
35
|
+
#
|
|
36
|
+
# * ultralytics YOLO asset -- ai/understanding/objects.py loads
|
|
37
|
+
# ``YOLO(self.model_name)`` (e.g. "yolov8n.pt"). Ultralytics resolves and
|
|
38
|
+
# downloads this from its own GitHub release assets, not from a HF repo,
|
|
39
|
+
# so there is no HF revision to pin. (The face detector in
|
|
40
|
+
# ai/understanding/faces.py is different: it pulls a YOLOv8 *checkpoint*
|
|
41
|
+
# from the HF repo ``arnabdhar/YOLOv8-Face-Detection`` via
|
|
42
|
+
# hf_hub_download, and that one IS pinned below.)
|
|
43
|
+
#
|
|
44
|
+
# * Dynamic Marian pairs -- ai/generation/translation.py builds the model
|
|
45
|
+
# name per language pair at runtime (e.g. "Helsinki-NLP/opus-mt-en-de").
|
|
46
|
+
# The set of repos is open-ended (one per direction), so pinning each pair
|
|
47
|
+
# to a SHA is impractical and out of scope; left unpinned by design.
|
|
48
|
+
#
|
|
49
|
+
# * Chatterbox internal load -- ai/generation/audio.py calls
|
|
50
|
+
# ``ChatterboxMultilingualTTS.from_pretrained(device=...)`` with no repo
|
|
51
|
+
# argument. The repo id + revision are resolved internally by the
|
|
52
|
+
# chatterbox package, so there is nothing for us to pass a ``revision`` to.
|
|
53
|
+
#
|
|
54
|
+
# * openai-whisper CDN models -- ai/understanding/audio.py loads transcription
|
|
55
|
+
# weights via ``whisper.load_model(name="turbo", ...)`` (tiny/base/small/
|
|
56
|
+
# medium/large/turbo). openai-whisper downloads these from OpenAI's own CDN
|
|
57
|
+
# by name, NOT through a HF ``from_pretrained`` repo, so there is no HF
|
|
58
|
+
# commit SHA to pin. (The faster-whisper backend, if/when used, WOULD map to
|
|
59
|
+
# a HF repo and could be pinned -- this code path does not use it.)
|
|
60
|
+
# ---------------------------------------------------------------------------
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
from __future__ import annotations
|
|
64
|
+
|
|
65
|
+
# Exact model-identifier string -> pinned commit SHA. The key must match the
|
|
66
|
+
# literal value passed to from_pretrained/hf_hub_download at the call site so
|
|
67
|
+
# ``pinned(model_id)`` resolves with a plain dict lookup. SHAs captured from
|
|
68
|
+
# the HuggingFace model API (see module docstring for refresh instructions).
|
|
69
|
+
MODEL_REVISIONS: dict[str, str] = {
|
|
70
|
+
# SceneVLM (ai/understanding/image.py: SCENE_VLM_MODEL_IDS)
|
|
71
|
+
"Qwen/Qwen3.5-4B": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
|
|
72
|
+
"Qwen/Qwen3.5-9B": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
|
|
73
|
+
"Qwen/Qwen3.5-27B": "fc05daec18b0a78c049392ed2e771dde82bdf654",
|
|
74
|
+
# Speaker diarization (ai/understanding/audio.py: PYANNOTE_DIARIZATION_MODEL).
|
|
75
|
+
# Gated repo (auto-approved); from_pretrained needs an accepted HF token.
|
|
76
|
+
"pyannote/speaker-diarization-community-1": "3533c8cf8e369892e6b79ff1bf80f7b0286a54ee",
|
|
77
|
+
# Audio event classifier (ai/understanding/audio.py: AudioClassifier)
|
|
78
|
+
"MIT/ast-finetuned-audioset-10-10-0.4593": "f826b80d28226b62986cc218e5cec390b1096902",
|
|
79
|
+
# Face detection checkpoint (ai/understanding/faces.py: hf_hub_download)
|
|
80
|
+
"arnabdhar/YOLOv8-Face-Detection": "52fa54977207fa4f021de949b515fb19dcab4488",
|
|
81
|
+
# Qwen3 translation GGUF (ai/generation/qwen3.py: DEFAULT_REPO_ID)
|
|
82
|
+
"unsloth/Qwen3-4B-Instruct-2507-GGUF": "a06e946bb6b655725eafa393f4a9745d460374c9",
|
|
83
|
+
# MusicGen (ai/generation/audio.py: TextToMusic)
|
|
84
|
+
"facebook/musicgen-small": "4c8334b02c6ec4e8664a91979669a501ec497792",
|
|
85
|
+
# SDXL (ai/generation/image.py: TextToImage)
|
|
86
|
+
"stabilityai/stable-diffusion-xl-base-1.0": "462165984030d82259a11f4367a4eed129e94a7b",
|
|
87
|
+
# CogVideoX (ai/generation/video.py: TextToVideo / ImageToVideo)
|
|
88
|
+
"THUDM/CogVideoX1.5-5B": "fdc5267c90b5c06492985b966e43aae984e189e0",
|
|
89
|
+
"THUDM/CogVideoX1.5-5B-I2V": "46c90528707aebbe69066390b4fe7e7d24c9c2a4",
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def pinned(model_id: str) -> str | None:
|
|
94
|
+
"""Return the pinned commit SHA for ``model_id``, or ``None`` if unpinned.
|
|
95
|
+
|
|
96
|
+
``None`` is a valid, safe value to forward as ``revision=`` to
|
|
97
|
+
``from_pretrained`` / ``hf_hub_download`` -- it is their default and means
|
|
98
|
+
"no pin" (track the repo's ``main`` branch, i.e. the current behavior).
|
|
99
|
+
Dynamic and unpinnable-by-design models (see the module docstring) fall
|
|
100
|
+
into this case.
|
|
101
|
+
|
|
102
|
+
Args:
|
|
103
|
+
model_id: The exact repo id / model identifier string passed to the
|
|
104
|
+
loader at the call site (e.g. ``"facebook/musicgen-small"``).
|
|
105
|
+
|
|
106
|
+
Returns:
|
|
107
|
+
The pinned SHA, or ``None`` when no pin is registered for ``model_id``.
|
|
108
|
+
"""
|
|
109
|
+
return MODEL_REVISIONS.get(model_id)
|
|
@@ -5,13 +5,15 @@ from __future__ import annotations
|
|
|
5
5
|
from typing import TYPE_CHECKING, Any
|
|
6
6
|
|
|
7
7
|
from videopython.ai._device import log_device_initialization, release_device_memory, select_device
|
|
8
|
+
from videopython.ai._predictor import ManagedPredictor
|
|
9
|
+
from videopython.ai._revisions import pinned
|
|
8
10
|
from videopython.audio import Audio, AudioMetadata
|
|
9
11
|
|
|
10
12
|
if TYPE_CHECKING:
|
|
11
13
|
from pathlib import Path
|
|
12
14
|
|
|
13
15
|
|
|
14
|
-
class TextToSpeech:
|
|
16
|
+
class TextToSpeech(ManagedPredictor):
|
|
15
17
|
"""Generates speech audio from text using Chatterbox Multilingual.
|
|
16
18
|
|
|
17
19
|
Backed by Chatterbox Multilingual (Resemble AI). When ``voice_sample`` is
|
|
@@ -42,6 +44,8 @@ class TextToSpeech:
|
|
|
42
44
|
requested_device = self.device
|
|
43
45
|
device = select_device(self.device, mps_allowed=False)
|
|
44
46
|
|
|
47
|
+
# No repo id to key a revision on: Chatterbox resolves its own repo +
|
|
48
|
+
# revision internally, so there is nothing to pass revision= to.
|
|
45
49
|
self._model = ChatterboxMultilingualTTS.from_pretrained(device=device)
|
|
46
50
|
self.device = device
|
|
47
51
|
log_device_initialization(
|
|
@@ -146,7 +150,7 @@ class TextToSpeech:
|
|
|
146
150
|
release_device_memory(self.device)
|
|
147
151
|
|
|
148
152
|
|
|
149
|
-
class TextToMusic:
|
|
153
|
+
class TextToMusic(ManagedPredictor):
|
|
150
154
|
"""Generates music from text descriptions using MusicGen."""
|
|
151
155
|
|
|
152
156
|
def __init__(self, device: str | None = None):
|
|
@@ -170,8 +174,8 @@ class TextToMusic:
|
|
|
170
174
|
device = select_device(self.device, mps_allowed=True)
|
|
171
175
|
|
|
172
176
|
model_name = "facebook/musicgen-small"
|
|
173
|
-
self._processor = AutoProcessor.from_pretrained(model_name)
|
|
174
|
-
self._model = MusicgenForConditionalGeneration.from_pretrained(model_name)
|
|
177
|
+
self._processor = AutoProcessor.from_pretrained(model_name, revision=pinned(model_name))
|
|
178
|
+
self._model = MusicgenForConditionalGeneration.from_pretrained(model_name, revision=pinned(model_name))
|
|
175
179
|
self._model.to(device)
|
|
176
180
|
self.device = device
|
|
177
181
|
log_device_initialization(
|
|
@@ -7,9 +7,11 @@ from typing import Any
|
|
|
7
7
|
from PIL import Image
|
|
8
8
|
|
|
9
9
|
from videopython.ai._device import log_device_initialization, release_device_memory, select_device
|
|
10
|
+
from videopython.ai._predictor import ManagedPredictor
|
|
11
|
+
from videopython.ai._revisions import pinned
|
|
10
12
|
|
|
11
13
|
|
|
12
|
-
class TextToImage:
|
|
14
|
+
class TextToImage(ManagedPredictor):
|
|
13
15
|
"""Generates images from text descriptions using local models."""
|
|
14
16
|
|
|
15
17
|
def __init__(self, device: str | None = None):
|
|
@@ -32,6 +34,7 @@ class TextToImage:
|
|
|
32
34
|
model_name = "stabilityai/stable-diffusion-xl-base-1.0"
|
|
33
35
|
self._pipeline = DiffusionPipeline.from_pretrained(
|
|
34
36
|
model_name,
|
|
37
|
+
revision=pinned(model_name),
|
|
35
38
|
torch_dtype=dtype,
|
|
36
39
|
variant=variant,
|
|
37
40
|
use_safetensors=True,
|
|
@@ -22,6 +22,8 @@ from pathlib import Path
|
|
|
22
22
|
from typing import TYPE_CHECKING, Any, Callable
|
|
23
23
|
|
|
24
24
|
from videopython.ai._device import release_device_memory, select_device
|
|
25
|
+
from videopython.ai._predictor import ManagedPredictor
|
|
26
|
+
from videopython.ai._revisions import pinned
|
|
25
27
|
from videopython.ai.generation.translation import (
|
|
26
28
|
LANGUAGE_NAMES,
|
|
27
29
|
MarianTranslator,
|
|
@@ -208,7 +210,7 @@ def _parse_jsonl_response(raw: str) -> dict[int, str]:
|
|
|
208
210
|
return parsed
|
|
209
211
|
|
|
210
212
|
|
|
211
|
-
class Qwen3Translator:
|
|
213
|
+
class Qwen3Translator(ManagedPredictor):
|
|
212
214
|
"""Qwen3-Instruct translation via llama-cpp-python (GGUF).
|
|
213
215
|
|
|
214
216
|
Args:
|
|
@@ -277,7 +279,7 @@ class Qwen3Translator:
|
|
|
277
279
|
)
|
|
278
280
|
|
|
279
281
|
logger.info("Qwen3Translator: loading %s", self.filename)
|
|
280
|
-
model_path = Path(hf_hub_download(repo_id=self.repo_id, filename=self.filename))
|
|
282
|
+
model_path = Path(hf_hub_download(repo_id=self.repo_id, filename=self.filename, revision=pinned(self.repo_id)))
|
|
281
283
|
|
|
282
284
|
# n_gpu_layers=-1 offloads everything to GPU when one is available;
|
|
283
285
|
# 0 forces CPU. llama-cpp-python's Metal/CUDA support detects and
|
|
@@ -17,6 +17,8 @@ from __future__ import annotations
|
|
|
17
17
|
from typing import TYPE_CHECKING, Any, Callable, Protocol, runtime_checkable
|
|
18
18
|
|
|
19
19
|
from videopython.ai._device import log_device_initialization, release_device_memory, select_device
|
|
20
|
+
from videopython.ai._predictor import ManagedPredictor
|
|
21
|
+
from videopython.ai._revisions import pinned
|
|
20
22
|
from videopython.base.transcription import TranscriptionSegment
|
|
21
23
|
|
|
22
24
|
# Imported under TYPE_CHECKING to avoid a circular dep through
|
|
@@ -123,7 +125,7 @@ LANGUAGE_NAMES = {
|
|
|
123
125
|
}
|
|
124
126
|
|
|
125
127
|
|
|
126
|
-
class MarianTranslator:
|
|
128
|
+
class MarianTranslator(ManagedPredictor):
|
|
127
129
|
"""Translates text between languages using local Helsinki-NLP MarianMT models."""
|
|
128
130
|
|
|
129
131
|
# Languages without a direct opus-mt-{src}-{tgt} model. Maps (source, target)
|
|
@@ -181,8 +183,10 @@ class MarianTranslator:
|
|
|
181
183
|
requested_device = self.device
|
|
182
184
|
device = select_device(self.device, mps_allowed=True)
|
|
183
185
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
+
# Marian model names are dynamic per language pair, so pinned() returns
|
|
187
|
+
# None for them by design (revision=None tracks main, the safe default).
|
|
188
|
+
self._tokenizer = MarianTokenizer.from_pretrained(model_name, revision=pinned(model_name))
|
|
189
|
+
self._model = MarianMTModel.from_pretrained(model_name, revision=pinned(model_name)).to(device)
|
|
186
190
|
self.device = device
|
|
187
191
|
log_device_initialization(
|
|
188
192
|
"MarianTranslator",
|
|
@@ -7,6 +7,8 @@ from typing import TYPE_CHECKING, Any
|
|
|
7
7
|
import numpy as np
|
|
8
8
|
|
|
9
9
|
from videopython.ai._device import log_device_initialization, release_device_memory, select_device
|
|
10
|
+
from videopython.ai._predictor import ManagedPredictor
|
|
11
|
+
from videopython.ai._revisions import pinned
|
|
10
12
|
from videopython.base.video import Video
|
|
11
13
|
|
|
12
14
|
if TYPE_CHECKING:
|
|
@@ -23,7 +25,7 @@ def _get_torch_device_and_dtype(device: str | None) -> tuple[str, Any]:
|
|
|
23
25
|
return selected_device, torch.float32
|
|
24
26
|
|
|
25
27
|
|
|
26
|
-
class TextToVideo:
|
|
28
|
+
class TextToVideo(ManagedPredictor):
|
|
27
29
|
"""Generates videos from text descriptions using local diffusion models."""
|
|
28
30
|
|
|
29
31
|
def __init__(self, device: str | None = None):
|
|
@@ -39,7 +41,7 @@ class TextToVideo:
|
|
|
39
41
|
device, dtype = _get_torch_device_and_dtype(self.device)
|
|
40
42
|
|
|
41
43
|
model_name = "THUDM/CogVideoX1.5-5B"
|
|
42
|
-
self._pipeline = CogVideoXPipeline.from_pretrained(model_name, torch_dtype=dtype)
|
|
44
|
+
self._pipeline = CogVideoXPipeline.from_pretrained(model_name, revision=pinned(model_name), torch_dtype=dtype)
|
|
43
45
|
self._pipeline.to(device)
|
|
44
46
|
self.device = device
|
|
45
47
|
log_device_initialization(
|
|
@@ -77,7 +79,7 @@ class TextToVideo:
|
|
|
77
79
|
release_device_memory(self.device)
|
|
78
80
|
|
|
79
81
|
|
|
80
|
-
class ImageToVideo:
|
|
82
|
+
class ImageToVideo(ManagedPredictor):
|
|
81
83
|
"""Generates videos from static images using local video diffusion."""
|
|
82
84
|
|
|
83
85
|
def __init__(self, device: str | None = None):
|
|
@@ -95,7 +97,9 @@ class ImageToVideo:
|
|
|
95
97
|
device, dtype = _get_torch_device_and_dtype(self.device)
|
|
96
98
|
|
|
97
99
|
model_name = "THUDM/CogVideoX1.5-5B-I2V"
|
|
98
|
-
self._pipeline = CogVideoXImageToVideoPipeline.from_pretrained(
|
|
100
|
+
self._pipeline = CogVideoXImageToVideoPipeline.from_pretrained(
|
|
101
|
+
model_name, revision=pinned(model_name), torch_dtype=dtype
|
|
102
|
+
)
|
|
99
103
|
self._pipeline.to(device)
|
|
100
104
|
self.device = device
|
|
101
105
|
log_device_initialization(
|
|
@@ -6,6 +6,8 @@ import logging
|
|
|
6
6
|
from typing import Any, Literal
|
|
7
7
|
|
|
8
8
|
from videopython.ai._device import log_device_initialization, release_device_memory, select_device
|
|
9
|
+
from videopython.ai._predictor import ManagedPredictor
|
|
10
|
+
from videopython.ai._revisions import pinned
|
|
9
11
|
from videopython.audio import Audio
|
|
10
12
|
from videopython.base.description import AudioClassification, AudioEvent
|
|
11
13
|
from videopython.base.transcription import Transcription, TranscriptionSegment, TranscriptionWord
|
|
@@ -105,7 +107,7 @@ def _attach_confidence_by_overlap(
|
|
|
105
107
|
tgt.compression_ratio = best_src.compression_ratio
|
|
106
108
|
|
|
107
109
|
|
|
108
|
-
class AudioToText:
|
|
110
|
+
class AudioToText(ManagedPredictor):
|
|
109
111
|
"""Transcription service for audio and video using local Whisper models.
|
|
110
112
|
|
|
111
113
|
Uses openai-whisper for transcription (with word-level timestamps) and
|
|
@@ -187,6 +189,9 @@ class AudioToText:
|
|
|
187
189
|
|
|
188
190
|
whisper = require("whisper", "asr", feature="AudioToText")
|
|
189
191
|
|
|
192
|
+
# No revision pin: openai-whisper downloads weights by name from OpenAI's
|
|
193
|
+
# own CDN, not via a HF from_pretrained repo, so there is no HF commit
|
|
194
|
+
# SHA to pin (see videopython.ai._revisions module docstring).
|
|
190
195
|
self._model = whisper.load_model(name=self.model_name, device=self.device)
|
|
191
196
|
|
|
192
197
|
def _init_diarization(self) -> None:
|
|
@@ -197,7 +202,9 @@ class AudioToText:
|
|
|
197
202
|
|
|
198
203
|
Pipeline = require("pyannote.audio", "asr", feature="AudioToText diarization").Pipeline
|
|
199
204
|
|
|
200
|
-
self._diarization_pipeline = Pipeline.from_pretrained(
|
|
205
|
+
self._diarization_pipeline = Pipeline.from_pretrained(
|
|
206
|
+
self.PYANNOTE_DIARIZATION_MODEL, revision=pinned(self.PYANNOTE_DIARIZATION_MODEL)
|
|
207
|
+
)
|
|
201
208
|
self._diarization_pipeline.to(torch.device(self.device))
|
|
202
209
|
|
|
203
210
|
def _init_vad(self) -> None:
|
|
@@ -486,7 +493,7 @@ class AudioToText:
|
|
|
486
493
|
return self._transcribe_local(audio, effective_vocab)
|
|
487
494
|
|
|
488
495
|
|
|
489
|
-
class AudioClassifier:
|
|
496
|
+
class AudioClassifier(ManagedPredictor):
|
|
490
497
|
"""Audio event and sound classification using AST."""
|
|
491
498
|
|
|
492
499
|
SUPPORTED_MODELS: list[str] = ["MIT/ast-finetuned-audioset-10-10-0.4593"]
|
|
@@ -526,8 +533,8 @@ class AudioClassifier:
|
|
|
526
533
|
ASTFeatureExtractor = _transformers.ASTFeatureExtractor
|
|
527
534
|
ASTForAudioClassification = _transformers.ASTForAudioClassification
|
|
528
535
|
|
|
529
|
-
self._processor = ASTFeatureExtractor.from_pretrained(self.model_name)
|
|
530
|
-
self._model = ASTForAudioClassification.from_pretrained(self.model_name)
|
|
536
|
+
self._processor = ASTFeatureExtractor.from_pretrained(self.model_name, revision=pinned(self.model_name))
|
|
537
|
+
self._model = ASTForAudioClassification.from_pretrained(self.model_name, revision=pinned(self.model_name))
|
|
531
538
|
self._model.to(self.device)
|
|
532
539
|
self._model.eval()
|
|
533
540
|
|
|
@@ -17,6 +17,7 @@ from typing import Any, Literal
|
|
|
17
17
|
import numpy as np
|
|
18
18
|
|
|
19
19
|
from videopython.ai._device import select_device
|
|
20
|
+
from videopython.ai._revisions import pinned
|
|
20
21
|
from videopython.base.description import BoundingBox, DetectedFace, FaceTrack
|
|
21
22
|
|
|
22
23
|
logger = logging.getLogger(__name__)
|
|
@@ -82,6 +83,7 @@ class _FaceDetector:
|
|
|
82
83
|
model_path = hf_hub_download(
|
|
83
84
|
repo_id="arnabdhar/YOLOv8-Face-Detection",
|
|
84
85
|
filename="model.pt",
|
|
86
|
+
revision=pinned("arnabdhar/YOLOv8-Face-Detection"),
|
|
85
87
|
)
|
|
86
88
|
self._yolo_model = YOLO(model_path)
|
|
87
89
|
|
|
@@ -12,6 +12,8 @@ import numpy as np
|
|
|
12
12
|
from PIL import Image
|
|
13
13
|
|
|
14
14
|
from videopython.ai._device import log_device_initialization, release_device_memory, select_device
|
|
15
|
+
from videopython.ai._predictor import ManagedPredictor
|
|
16
|
+
from videopython.ai._revisions import pinned
|
|
15
17
|
from videopython.base.description import SceneDescription
|
|
16
18
|
|
|
17
19
|
logger = logging.getLogger(__name__)
|
|
@@ -74,7 +76,7 @@ _RETRY_PROMPT = (
|
|
|
74
76
|
)
|
|
75
77
|
|
|
76
78
|
|
|
77
|
-
class SceneVLM:
|
|
79
|
+
class SceneVLM(ManagedPredictor):
|
|
78
80
|
"""Generates structured scene descriptions with local Qwen3.5.
|
|
79
81
|
|
|
80
82
|
``model_size`` maps to Qwen3.5 dense vision-capable variants:
|
|
@@ -162,13 +164,15 @@ class SceneVLM:
|
|
|
162
164
|
requested_device = self.device
|
|
163
165
|
resolved_device = select_device(self.device, mps_allowed=True)
|
|
164
166
|
|
|
165
|
-
self._processor = AutoProcessor.from_pretrained(self.model_name)
|
|
167
|
+
self._processor = AutoProcessor.from_pretrained(self.model_name, revision=pinned(self.model_name))
|
|
166
168
|
# Save and restore default dtype -- transformers torch_dtype="auto" can
|
|
167
169
|
# mutate torch.get_default_dtype(), which breaks concurrent models
|
|
168
170
|
# (e.g. Whisper) that expect float32.
|
|
169
171
|
saved_dtype = torch.get_default_dtype()
|
|
170
172
|
try:
|
|
171
|
-
self._model = AutoModelForImageTextToText.from_pretrained(
|
|
173
|
+
self._model = AutoModelForImageTextToText.from_pretrained(
|
|
174
|
+
self.model_name, torch_dtype="auto", revision=pinned(self.model_name)
|
|
175
|
+
)
|
|
172
176
|
finally:
|
|
173
177
|
torch.set_default_dtype(saved_dtype)
|
|
174
178
|
self._model.to(resolved_device)
|
|
@@ -87,6 +87,9 @@ class ObjectDetector:
|
|
|
87
87
|
|
|
88
88
|
YOLO = require("ultralytics", "vision", feature="ObjectDetector").YOLO
|
|
89
89
|
|
|
90
|
+
# No revision pin: ultralytics resolves/downloads this asset from its own
|
|
91
|
+
# GitHub release assets, not a HF repo, and YOLO() takes no revision arg
|
|
92
|
+
# (see videopython.ai._revisions module docstring).
|
|
90
93
|
self._yolo_model = YOLO(self.model_name)
|
|
91
94
|
self._class_names = dict(self._yolo_model.names)
|
|
92
95
|
|
|
@@ -6,6 +6,7 @@ import logging
|
|
|
6
6
|
from typing import Any
|
|
7
7
|
|
|
8
8
|
from videopython.ai._device import log_device_initialization, release_device_memory, select_device
|
|
9
|
+
from videopython.ai._predictor import ManagedPredictor
|
|
9
10
|
from videopython.ai.dubbing.models import SeparatedAudio
|
|
10
11
|
from videopython.audio import Audio, AudioMetadata
|
|
11
12
|
|
|
@@ -55,7 +56,7 @@ def _merge_regions(
|
|
|
55
56
|
return merged
|
|
56
57
|
|
|
57
58
|
|
|
58
|
-
class AudioSeparator:
|
|
59
|
+
class AudioSeparator(ManagedPredictor):
|
|
59
60
|
"""Separates audio into vocals and background components using Demucs."""
|
|
60
61
|
|
|
61
62
|
SUPPORTED_MODELS: list[str] = ["htdemucs", "htdemucs_ft", "htdemucs_6s", "mdx_extra"]
|
|
@@ -10,13 +10,14 @@ from pathlib import Path
|
|
|
10
10
|
from typing import TYPE_CHECKING, Any
|
|
11
11
|
|
|
12
12
|
from videopython.ai._device import log_device_initialization, release_device_memory, select_device
|
|
13
|
+
from videopython.ai._predictor import ManagedPredictor
|
|
13
14
|
from videopython.base.description import SceneBoundary
|
|
14
15
|
|
|
15
16
|
if TYPE_CHECKING:
|
|
16
17
|
from videopython.base.video import Video
|
|
17
18
|
|
|
18
19
|
|
|
19
|
-
class SemanticSceneDetector:
|
|
20
|
+
class SemanticSceneDetector(ManagedPredictor):
|
|
20
21
|
"""ML-based scene detection using TransNetV2.
|
|
21
22
|
|
|
22
23
|
TransNetV2 is a neural network specifically designed for shot boundary
|
|
@@ -119,6 +119,62 @@ class PlanError:
|
|
|
119
119
|
predicted_duration: float | None = None
|
|
120
120
|
detail: str | None = None
|
|
121
121
|
|
|
122
|
+
def to_prompt_line(self) -> str:
|
|
123
|
+
"""Render this error as one deterministic, actionable feedback line.
|
|
124
|
+
|
|
125
|
+
Composes a single line from the structured fields so a *standalone*
|
|
126
|
+
:class:`PlanError` can reproduce a human message without the ad-hoc
|
|
127
|
+
prose built at the validation call sites. Intended for an LLM
|
|
128
|
+
refine-loop: feed these lines back so the model can repair the plan.
|
|
129
|
+
|
|
130
|
+
Format (each clause is appended only when its field is populated;
|
|
131
|
+
``None`` fields are skipped)::
|
|
132
|
+
|
|
133
|
+
<CODE_NAME> [at <location>] [(op '<op>')]: [<field>=<value>]
|
|
134
|
+
[(limit <limit>)] [(predicted duration <predicted_duration>s)]
|
|
135
|
+
[-- <detail>]
|
|
136
|
+
|
|
137
|
+
Every :class:`PlanErrorCode` yields a non-empty line: the code name is
|
|
138
|
+
always present, so a code carrying no other fields still renders
|
|
139
|
+
(e.g. ``"SOURCE_UNREADABLE"``). Numeric fields are formatted with
|
|
140
|
+
:func:`_fmt_num`, which drops trailing zeros for readability.
|
|
141
|
+
"""
|
|
142
|
+
line = self.code.name
|
|
143
|
+
if self.location is not None:
|
|
144
|
+
line += f" at {self.location}"
|
|
145
|
+
if self.op is not None:
|
|
146
|
+
line += f" (op '{self.op}')"
|
|
147
|
+
|
|
148
|
+
clauses: list[str] = []
|
|
149
|
+
if self.field is not None:
|
|
150
|
+
if self.value is not None:
|
|
151
|
+
clauses.append(f"{self.field}={_fmt_num(self.value)}")
|
|
152
|
+
else:
|
|
153
|
+
clauses.append(self.field)
|
|
154
|
+
elif self.value is not None:
|
|
155
|
+
clauses.append(f"value={_fmt_num(self.value)}")
|
|
156
|
+
if self.limit is not None:
|
|
157
|
+
clauses.append(f"limit {_fmt_num(self.limit)}")
|
|
158
|
+
if self.predicted_duration is not None:
|
|
159
|
+
clauses.append(f"predicted duration {_fmt_num(self.predicted_duration)}s")
|
|
160
|
+
|
|
161
|
+
if clauses:
|
|
162
|
+
line += ": " + ", ".join(clauses)
|
|
163
|
+
if self.detail is not None:
|
|
164
|
+
line += f" -- {self.detail}"
|
|
165
|
+
return line
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _fmt_num(value: float) -> str:
|
|
169
|
+
"""Format a numeric field readably: integers lose the ``.0``, floats keep it.
|
|
170
|
+
|
|
171
|
+
Keeps :meth:`PlanError.to_prompt_line` deterministic and free of noisy
|
|
172
|
+
trailing zeros (``3`` not ``3.0``, but ``2.5`` stays ``2.5``).
|
|
173
|
+
"""
|
|
174
|
+
if isinstance(value, float) and value.is_integer():
|
|
175
|
+
return str(int(value))
|
|
176
|
+
return str(value)
|
|
177
|
+
|
|
122
178
|
|
|
123
179
|
@dataclass
|
|
124
180
|
class PlanRepair:
|
|
@@ -156,3 +212,12 @@ class PlanValidationError(ValueError):
|
|
|
156
212
|
def __init__(self, message: str, errors: list[PlanError]):
|
|
157
213
|
super().__init__(message)
|
|
158
214
|
self.errors = errors
|
|
215
|
+
|
|
216
|
+
def prompt_feedback(self) -> str:
|
|
217
|
+
"""Every carried :class:`PlanError` as newline-joined feedback lines.
|
|
218
|
+
|
|
219
|
+
Joins :meth:`PlanError.to_prompt_line` over :attr:`errors`, one per
|
|
220
|
+
line -- a ready-to-feed block for an LLM refine loop. Returns an empty
|
|
221
|
+
string when no structured errors are attached.
|
|
222
|
+
"""
|
|
223
|
+
return "\n".join(e.to_prompt_line() for e in self.errors)
|
|
@@ -569,6 +569,16 @@ class Video:
|
|
|
569
569
|
Returns:
|
|
570
570
|
New Video with the audio added
|
|
571
571
|
"""
|
|
572
|
+
# When the incoming audio will be mixed with an existing (non-silent)
|
|
573
|
+
# track, reconcile sample rates first. concat()/overlay() require
|
|
574
|
+
# matching sample rates, and mixing at a mismatched rate would otherwise
|
|
575
|
+
# produce silent A/V drift. The existing track's sample rate is the
|
|
576
|
+
# canonical target (it is what gets encoded into the video). For a pure
|
|
577
|
+
# attach/replace (no existing audio, or overlay=False) the incoming rate
|
|
578
|
+
# is kept as-is.
|
|
579
|
+
if overlay and not self.audio.is_silent:
|
|
580
|
+
audio = audio.resample(self.audio.metadata.sample_rate)
|
|
581
|
+
|
|
572
582
|
video_duration = self.total_seconds
|
|
573
583
|
audio_duration = audio.metadata.duration_seconds
|
|
574
584
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|