continuityguard-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- continuityguard/__init__.py +43 -0
- continuityguard/cli.py +152 -0
- continuityguard/ingest/__init__.py +1 -0
- continuityguard/ingest/ffmpeg.py +295 -0
- continuityguard/py.typed +0 -0
- continuityguard/report/__init__.py +4 -0
- continuityguard/report/json_report.py +52 -0
- continuityguard/report/terminal.py +85 -0
- continuityguard/report/types.py +64 -0
- continuityguard/scan.py +147 -0
- continuityguard/score/__init__.py +2 -0
- continuityguard/score/consistency.py +226 -0
- continuityguard/score/models/NOTICE.md +42 -0
- continuityguard/score/models/mobilenetv2-7.onnx +0 -0
- continuityguard/score/physics.py +170 -0
- continuityguard_cli-0.1.0.dist-info/METADATA +296 -0
- continuityguard_cli-0.1.0.dist-info/RECORD +20 -0
- continuityguard_cli-0.1.0.dist-info/WHEEL +4 -0
- continuityguard_cli-0.1.0.dist-info/entry_points.txt +2 -0
- continuityguard_cli-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""
|
|
2
|
+
CG04 -- shared report shape consumed by both the JSON writer
|
|
3
|
+
(continuityguard/report/json_report.py) and the terminal writer
|
|
4
|
+
(continuityguard/report/terminal.py). Every flagged shot carries a
|
|
5
|
+
timestamp/frame index, a numeric score, and a plain-language reason --
|
|
6
|
+
never a bare pass/fail -- so a human reviewer or a parsing agent knows
|
|
7
|
+
*why* a shot was flagged.
|
|
8
|
+
|
|
9
|
+
Ported field-for-field from src/report/types.ts. Field names use
|
|
10
|
+
snake_case (not the TS camelCase) since that's the Python-idiomatic
|
|
11
|
+
convention and this matches the JSON keys the TS side already serializes
|
|
12
|
+
(the TS `ScanReport` interface itself uses snake_case JSON field names,
|
|
13
|
+
e.g. `scanned_directory`, `character_consistency`) -- so the JSON report
|
|
14
|
+
produced by this Python port is structurally identical to the TS one.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
from typing import List
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class ConsistencyFlagReport:
|
|
24
|
+
clip: str
|
|
25
|
+
character: str
|
|
26
|
+
reference_clip: str
|
|
27
|
+
similarity_score: float
|
|
28
|
+
reason: str
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class PhysicsFlagReport:
|
|
33
|
+
clip: str
|
|
34
|
+
frame_index_a: int
|
|
35
|
+
frame_index_b: int
|
|
36
|
+
discontinuity_ratio: float
|
|
37
|
+
reason: str
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class CharacterConsistencyReport:
|
|
42
|
+
characters_tracked: int
|
|
43
|
+
similarity_threshold: float
|
|
44
|
+
flagged_shots: List[ConsistencyFlagReport] = field(default_factory=list)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class PhysicsPlausibilityReport:
|
|
49
|
+
discontinuity_multiplier: float
|
|
50
|
+
flagged_shots: List[PhysicsFlagReport] = field(default_factory=list)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class ScanReport:
|
|
55
|
+
scan_id: str
|
|
56
|
+
scanned_directory: str
|
|
57
|
+
clips_scanned: int
|
|
58
|
+
frames_extracted: int
|
|
59
|
+
character_consistency: CharacterConsistencyReport
|
|
60
|
+
physics_plausibility: PhysicsPlausibilityReport
|
|
61
|
+
generated_at: str
|
|
62
|
+
tool_version: str
|
|
63
|
+
scan_duration_seconds: float
|
|
64
|
+
network_calls_made: int = 0
|
continuityguard/scan.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Top-level library entry point: CG01 ingestion -> CG02 character-consistency
|
|
3
|
+
scoring -> CG03 physics-plausibility heuristic -> a structured ScanReport.
|
|
4
|
+
|
|
5
|
+
This is the agent-native path: `from continuityguard import scan` gives a
|
|
6
|
+
CI script or an agent an in-process `ScanReport` object with no CLI
|
|
7
|
+
subprocess involved. `continuityguard/cli.py` is a thin argparse wrapper
|
|
8
|
+
around this same function, so the CLI and the library never drift.
|
|
9
|
+
|
|
10
|
+
Ported from the orchestration logic in src/cli.ts's `runScan`, split out
|
|
11
|
+
into its own importable module since the TypeScript CLI does not expose a
|
|
12
|
+
separate library entry point the way this Python port does.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import time
|
|
17
|
+
from datetime import datetime, timezone
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import List, Optional
|
|
20
|
+
|
|
21
|
+
from .ingest.ffmpeg import (
|
|
22
|
+
ExtractedFrames,
|
|
23
|
+
cleanup_frames,
|
|
24
|
+
extract_frames_from_directory,
|
|
25
|
+
list_clips,
|
|
26
|
+
)
|
|
27
|
+
from .report.types import (
|
|
28
|
+
CharacterConsistencyReport,
|
|
29
|
+
ConsistencyFlagReport,
|
|
30
|
+
PhysicsFlagReport,
|
|
31
|
+
PhysicsPlausibilityReport,
|
|
32
|
+
ScanReport,
|
|
33
|
+
)
|
|
34
|
+
from .score.consistency import CONSISTENCY_SIMILARITY_THRESHOLD, ShotInput, score_consistency
|
|
35
|
+
from .score.physics import DISCONTINUITY_MULTIPLIER, PhysicsShotInput, score_physics
|
|
36
|
+
|
|
37
|
+
TOOL_VERSION = "0.1.0"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class NoClipsFoundError(Exception):
|
|
41
|
+
"""Raised when the target directory contains no supported video clips.
|
|
42
|
+
|
|
43
|
+
Deliberately NOT a subclass of FileNotFoundError -- the directory
|
|
44
|
+
itself does exist and was read successfully; it just contains no
|
|
45
|
+
supported clip files. Callers (see cli.py) need to tell this apart
|
|
46
|
+
from "the directory path itself is wrong," which does raise a plain
|
|
47
|
+
FileNotFoundError from list_clips().
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def scan(
|
|
52
|
+
directory: str,
|
|
53
|
+
fps: Optional[float] = None,
|
|
54
|
+
) -> ScanReport:
|
|
55
|
+
"""
|
|
56
|
+
Scans a directory of generated clips for character-consistency and
|
|
57
|
+
physics-plausibility flags, and returns a structured ScanReport.
|
|
58
|
+
|
|
59
|
+
Raises FileNotFoundError if the directory does not exist,
|
|
60
|
+
NotADirectoryError if the path is not a directory, NoClipsFoundError if
|
|
61
|
+
the directory contains no supported clips, and RuntimeError if ffmpeg
|
|
62
|
+
fails to decode a clip. Callers that want the CLI's clean
|
|
63
|
+
stderr-message-plus-exit-code behavior instead of raised exceptions
|
|
64
|
+
should use `continuityguard.cli.run_scan`.
|
|
65
|
+
"""
|
|
66
|
+
target_dir = str(Path(directory).resolve())
|
|
67
|
+
clips = list_clips(target_dir)
|
|
68
|
+
if not clips:
|
|
69
|
+
raise NoClipsFoundError(
|
|
70
|
+
f"No supported video clips found in {target_dir}. "
|
|
71
|
+
"Supported extensions: .mp4 .mov .mkv .webm .avi"
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
started_at = time.time()
|
|
75
|
+
extracted: List[ExtractedFrames] = []
|
|
76
|
+
try:
|
|
77
|
+
extracted = extract_frames_from_directory(target_dir, fps=fps)
|
|
78
|
+
frames_extracted = sum(len(e.frame_paths) for e in extracted)
|
|
79
|
+
|
|
80
|
+
consistency_result = score_consistency(
|
|
81
|
+
[ShotInput(clip=e.clip.name, frame_paths=e.frame_paths) for e in extracted]
|
|
82
|
+
)
|
|
83
|
+
physics_result = score_physics(
|
|
84
|
+
[PhysicsShotInput(clip=e.clip.name, frame_paths=e.frame_paths) for e in extracted]
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
duration_seconds = time.time() - started_at
|
|
88
|
+
started_dt = datetime.fromtimestamp(started_at, tz=timezone.utc)
|
|
89
|
+
# Matches the TS side's `cg-${isoString.replace(/[:.]/g, '-')}`.
|
|
90
|
+
started_iso = (
|
|
91
|
+
started_dt.strftime("%Y-%m-%dT%H:%M:%S.")
|
|
92
|
+
+ f"{int(started_dt.microsecond / 1000):03d}Z"
|
|
93
|
+
)
|
|
94
|
+
scan_id = "cg-" + started_iso.replace(":", "-").replace(".", "-")
|
|
95
|
+
|
|
96
|
+
report = ScanReport(
|
|
97
|
+
scan_id=scan_id,
|
|
98
|
+
scanned_directory=target_dir,
|
|
99
|
+
clips_scanned=len(clips),
|
|
100
|
+
frames_extracted=frames_extracted,
|
|
101
|
+
character_consistency=CharacterConsistencyReport(
|
|
102
|
+
characters_tracked=consistency_result.characters_tracked,
|
|
103
|
+
similarity_threshold=CONSISTENCY_SIMILARITY_THRESHOLD,
|
|
104
|
+
flagged_shots=[
|
|
105
|
+
ConsistencyFlagReport(
|
|
106
|
+
clip=f.clip,
|
|
107
|
+
character=f.character,
|
|
108
|
+
reference_clip=f.reference_clip,
|
|
109
|
+
similarity_score=f.similarity_score,
|
|
110
|
+
reason=f.reason,
|
|
111
|
+
)
|
|
112
|
+
for f in consistency_result.flagged_shots
|
|
113
|
+
],
|
|
114
|
+
),
|
|
115
|
+
physics_plausibility=PhysicsPlausibilityReport(
|
|
116
|
+
discontinuity_multiplier=DISCONTINUITY_MULTIPLIER,
|
|
117
|
+
flagged_shots=[
|
|
118
|
+
PhysicsFlagReport(
|
|
119
|
+
clip=f.clip,
|
|
120
|
+
frame_index_a=f.frame_index_a,
|
|
121
|
+
frame_index_b=f.frame_index_b,
|
|
122
|
+
discontinuity_ratio=f.discontinuity_ratio,
|
|
123
|
+
reason=f.reason,
|
|
124
|
+
)
|
|
125
|
+
for f in physics_result.flagged_shots
|
|
126
|
+
],
|
|
127
|
+
),
|
|
128
|
+
generated_at=datetime.now(tz=timezone.utc).isoformat().replace("+00:00", "Z"),
|
|
129
|
+
tool_version=TOOL_VERSION,
|
|
130
|
+
scan_duration_seconds=duration_seconds,
|
|
131
|
+
network_calls_made=0,
|
|
132
|
+
)
|
|
133
|
+
return report
|
|
134
|
+
finally:
|
|
135
|
+
cleanup_frames(extracted)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
async def scan_async(directory: str, fps: Optional[float] = None) -> ScanReport:
|
|
139
|
+
"""Async wrapper around `scan`, run in a thread so it does not block an
|
|
140
|
+
event loop -- useful for callers already inside `asyncio` (e.g. an
|
|
141
|
+
agent framework's async tool-call handler). ffmpeg decode and ONNX
|
|
142
|
+
inference are both CPU/subprocess-bound work, not natively async, so
|
|
143
|
+
this offloads to a worker thread rather than reimplementing an async
|
|
144
|
+
ffmpeg/onnxruntime pipeline."""
|
|
145
|
+
import asyncio
|
|
146
|
+
|
|
147
|
+
return await asyncio.to_thread(scan, directory, fps)
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""
|
|
2
|
+
CG02 -- character-consistency scoring.
|
|
3
|
+
|
|
4
|
+
Computes a per-shot visual embedding and flags cross-shot pairs (claiming
|
|
5
|
+
to be the same named character) whose cosine similarity falls below a
|
|
6
|
+
threshold derived from this repo's own fixture run (see
|
|
7
|
+
src/score/testdata/ and the root CHANGELOG.md's "CG02 fixture
|
|
8
|
+
calibration" entry -- never an invented number; the Python port reuses
|
|
9
|
+
the exact same threshold since it uses the exact same bundled model file
|
|
10
|
+
and the same fixture clips).
|
|
11
|
+
|
|
12
|
+
IMPORTANT, stated plainly and repeated in the CLI output, the JSON
|
|
13
|
+
`reason` field, and the README: this technique is best-validated on
|
|
14
|
+
photorealistic/live-action content. Its accuracy on stylized or
|
|
15
|
+
anime-adjacent character designs -- the dominant visual style in the
|
|
16
|
+
short-drama category this tool targets -- is UNVERIFIED. Treat a flag on
|
|
17
|
+
stylized footage as "worth a second look," never as a confirmed defect.
|
|
18
|
+
|
|
19
|
+
Runtime note: this port uses `onnxruntime` (the Python package), the same
|
|
20
|
+
ONNX Runtime project as the TS side's `onnxruntime-node`, loading the
|
|
21
|
+
identical bundled `mobilenetv2-7.onnx` file the npm package ships (see
|
|
22
|
+
models/NOTICE.md for the model's provenance and the honest limitations of
|
|
23
|
+
using a generic ImageNet feature extractor instead of a dedicated face/CLIP
|
|
24
|
+
embedding model). Same model file, same runtime family, same preprocessing
|
|
25
|
+
-> same embeddings and the same flagged shots as the TypeScript CLI on
|
|
26
|
+
identical input.
|
|
27
|
+
|
|
28
|
+
Ported from src/score/consistency.ts.
|
|
29
|
+
"""
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import re
|
|
33
|
+
from dataclasses import dataclass, field
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
from typing import Dict, List, Optional, Sequence
|
|
36
|
+
|
|
37
|
+
import numpy as np
|
|
38
|
+
import onnxruntime as ort
|
|
39
|
+
|
|
40
|
+
from ..ingest.ffmpeg import FRAME_SIZE, read_raw_frame
|
|
41
|
+
|
|
42
|
+
_MODEL_PATH = Path(__file__).resolve().parent / "models" / "mobilenetv2-7.onnx"
|
|
43
|
+
|
|
44
|
+
# ImageNet normalization constants used by the bundled MobileNetV2 model's
|
|
45
|
+
# documented preprocessing (see continuityguard/score/models/NOTICE.md).
|
|
46
|
+
_IMAGENET_MEAN = (0.485, 0.456, 0.406)
|
|
47
|
+
_IMAGENET_STD = (0.229, 0.224, 0.225)
|
|
48
|
+
|
|
49
|
+
# Cross-shot cosine-similarity threshold below which a shot is flagged as a
|
|
50
|
+
# likely character-consistency break. Derived from this repo's own fixture
|
|
51
|
+
# run -- see the root CHANGELOG.md "CG02 fixture calibration" entry for the
|
|
52
|
+
# exact command and raw numbers this value came from, not an illustrative
|
|
53
|
+
# placeholder. Identical to the TS side's CONSISTENCY_SIMILARITY_THRESHOLD.
|
|
54
|
+
CONSISTENCY_SIMILARITY_THRESHOLD = 0.88
|
|
55
|
+
|
|
56
|
+
_cached_session: Optional[ort.InferenceSession] = None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def get_embedding_session() -> ort.InferenceSession:
|
|
60
|
+
"""Loads (and caches) the bundled ONNX embedding session. Never fetches
|
|
61
|
+
anything over the network -- the model file ships inside this package."""
|
|
62
|
+
global _cached_session
|
|
63
|
+
if _cached_session is None:
|
|
64
|
+
_cached_session = ort.InferenceSession(
|
|
65
|
+
str(_MODEL_PATH), providers=["CPUExecutionProvider"]
|
|
66
|
+
)
|
|
67
|
+
return _cached_session
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def preprocess_frame(raw: bytes) -> np.ndarray:
|
|
71
|
+
"""Converts a raw HWC RGB24 frame buffer into the CHW, ImageNet-normalized
|
|
72
|
+
float32 array MobileNetV2 expects, shape (3, FRAME_SIZE, FRAME_SIZE)."""
|
|
73
|
+
size = FRAME_SIZE
|
|
74
|
+
pixels = np.frombuffer(raw, dtype=np.uint8).reshape(size, size, 3).astype(np.float32) / 255.0
|
|
75
|
+
mean = np.array(_IMAGENET_MEAN, dtype=np.float32)
|
|
76
|
+
std = np.array(_IMAGENET_STD, dtype=np.float32)
|
|
77
|
+
normalized = (pixels - mean) / std
|
|
78
|
+
# HWC -> CHW
|
|
79
|
+
return np.transpose(normalized, (2, 0, 1))
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def embed_frame(raw: bytes, session: ort.InferenceSession) -> np.ndarray:
|
|
83
|
+
"""Runs one frame through the embedding model and returns its raw
|
|
84
|
+
1000-d output vector (see models/NOTICE.md for why this is the raw
|
|
85
|
+
classifier-logit vector rather than a dedicated embedding-layer
|
|
86
|
+
output)."""
|
|
87
|
+
input_array = preprocess_frame(raw)
|
|
88
|
+
tensor = input_array.reshape(1, 3, FRAME_SIZE, FRAME_SIZE)
|
|
89
|
+
input_name = session.get_inputs()[0].name
|
|
90
|
+
output_name = session.get_outputs()[0].name
|
|
91
|
+
results = session.run([output_name], {input_name: tensor})
|
|
92
|
+
return np.asarray(results[0], dtype=np.float32).reshape(-1)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def embed_shot(frame_paths: Sequence[str], session: ort.InferenceSession) -> np.ndarray:
|
|
96
|
+
"""Computes one embedding per shot by averaging the embeddings of every
|
|
97
|
+
sampled frame in that shot -- smooths single-frame noise into a more
|
|
98
|
+
stable per-shot signal than picking any single frame would."""
|
|
99
|
+
if not frame_paths:
|
|
100
|
+
raise ValueError("Cannot embed a shot with zero extracted frames")
|
|
101
|
+
total: Optional[np.ndarray] = None
|
|
102
|
+
for frame_path in frame_paths:
|
|
103
|
+
raw = read_raw_frame(frame_path)
|
|
104
|
+
embedding = embed_frame(raw, session)
|
|
105
|
+
total = embedding.copy() if total is None else total + embedding
|
|
106
|
+
assert total is not None
|
|
107
|
+
return total / len(frame_paths)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
|
|
111
|
+
if a.shape != b.shape:
|
|
112
|
+
raise ValueError("Cannot compare embeddings of different dimensionality")
|
|
113
|
+
norm_a = float(np.linalg.norm(a))
|
|
114
|
+
norm_b = float(np.linalg.norm(b))
|
|
115
|
+
if norm_a == 0 or norm_b == 0:
|
|
116
|
+
return 0.0
|
|
117
|
+
return float(np.dot(a, b) / (norm_a * norm_b))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
_CHARACTER_NAME_RE = re.compile(r"^([a-zA-Z][a-zA-Z0-9]*)_")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def parse_character_name(clip_file_name: str) -> Optional[str]:
|
|
124
|
+
"""
|
|
125
|
+
Infers the named character a clip belongs to from its filename, using
|
|
126
|
+
a `<character>_<shot-id>.<ext>` convention (e.g. "mei_shot01.mp4" ->
|
|
127
|
+
"mei"). This is a deliberate v0.1 simplification: there is no
|
|
128
|
+
industry-standard character-tagging metadata format across AI
|
|
129
|
+
short-drama pipelines, so ContinuityGuard reads it from the filename
|
|
130
|
+
rather than requiring a separate manifest. Clips that don't match the
|
|
131
|
+
convention return None and are never compared to each other for
|
|
132
|
+
consistency (no named-character claim to check them against).
|
|
133
|
+
"""
|
|
134
|
+
without_ext = re.sub(r"\.[^.]+$", "", clip_file_name)
|
|
135
|
+
match = _CHARACTER_NAME_RE.match(without_ext)
|
|
136
|
+
return match.group(1).lower() if match else None
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@dataclass(frozen=True)
|
|
140
|
+
class ShotInput:
|
|
141
|
+
clip: str
|
|
142
|
+
frame_paths: List[str]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@dataclass(frozen=True)
|
|
146
|
+
class ConsistencyFlag:
|
|
147
|
+
clip: str
|
|
148
|
+
character: str
|
|
149
|
+
reference_clip: str
|
|
150
|
+
similarity_score: float
|
|
151
|
+
reason: str
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@dataclass(frozen=True)
|
|
155
|
+
class ConsistencyResult:
|
|
156
|
+
characters_tracked: int
|
|
157
|
+
flagged_shots: List[ConsistencyFlag] = field(default_factory=list)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def build_consistency_reason(similarity: float) -> str:
|
|
161
|
+
"""Builds the honest, unverified-content-labeled reason string for a
|
|
162
|
+
flagged shot -- kept as its own function so every call site (CLI, JSON
|
|
163
|
+
report) uses identical, never-drifting wording."""
|
|
164
|
+
return (
|
|
165
|
+
f"Cross-shot embedding similarity {similarity:.2f} is below the "
|
|
166
|
+
f"{CONSISTENCY_SIMILARITY_THRESHOLD:.2f} threshold. Best-validated on "
|
|
167
|
+
"photorealistic content; accuracy on stylized/anime-adjacent character designs "
|
|
168
|
+
"is unverified -- treat this as a prompt for human review, not a confirmed defect."
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def flag_inconsistent_shots(
|
|
173
|
+
shots: Sequence[Dict], threshold: float = CONSISTENCY_SIMILARITY_THRESHOLD
|
|
174
|
+
) -> ConsistencyResult:
|
|
175
|
+
"""
|
|
176
|
+
Pure grouping + flagging logic, independent of the embedding model --
|
|
177
|
+
given already-computed per-shot embeddings, groups by character and
|
|
178
|
+
flags any shot whose similarity to that character's first-seen
|
|
179
|
+
("reference") shot falls below threshold. Split out from
|
|
180
|
+
`score_consistency` so this logic has fast, deterministic unit-test
|
|
181
|
+
coverage without depending on real ONNX inference for every test case.
|
|
182
|
+
|
|
183
|
+
Each item in `shots` is a dict with keys "clip" (str) and "embedding"
|
|
184
|
+
(np.ndarray).
|
|
185
|
+
"""
|
|
186
|
+
by_character: "Dict[str, List[Dict]]" = {}
|
|
187
|
+
for shot in shots:
|
|
188
|
+
character = parse_character_name(shot["clip"])
|
|
189
|
+
if not character:
|
|
190
|
+
continue
|
|
191
|
+
by_character.setdefault(character, []).append(shot)
|
|
192
|
+
|
|
193
|
+
flagged_shots: List[ConsistencyFlag] = []
|
|
194
|
+
for character, bucket in by_character.items():
|
|
195
|
+
if len(bucket) < 2:
|
|
196
|
+
continue
|
|
197
|
+
reference = bucket[0]
|
|
198
|
+
for candidate in bucket[1:]:
|
|
199
|
+
similarity = cosine_similarity(reference["embedding"], candidate["embedding"])
|
|
200
|
+
if similarity < threshold:
|
|
201
|
+
flagged_shots.append(
|
|
202
|
+
ConsistencyFlag(
|
|
203
|
+
clip=candidate["clip"],
|
|
204
|
+
character=character,
|
|
205
|
+
reference_clip=reference["clip"],
|
|
206
|
+
similarity_score=round(similarity, 4),
|
|
207
|
+
reason=build_consistency_reason(similarity),
|
|
208
|
+
)
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
return ConsistencyResult(characters_tracked=len(by_character), flagged_shots=flagged_shots)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def score_consistency(
|
|
215
|
+
shots: Sequence[ShotInput],
|
|
216
|
+
session: Optional[ort.InferenceSession] = None,
|
|
217
|
+
threshold: float = CONSISTENCY_SIMILARITY_THRESHOLD,
|
|
218
|
+
) -> ConsistencyResult:
|
|
219
|
+
"""End-to-end CG02 entry point: computes embeddings for every shot (via
|
|
220
|
+
the real ONNX model) and flags cross-shot consistency breaks."""
|
|
221
|
+
active_session = session or get_embedding_session()
|
|
222
|
+
embedded: List[Dict] = []
|
|
223
|
+
for shot in shots:
|
|
224
|
+
embedding = embed_shot(shot.frame_paths, active_session)
|
|
225
|
+
embedded.append({"clip": shot.clip, "embedding": embedding})
|
|
226
|
+
return flag_inconsistent_shots(embedded, threshold)
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Bundled model: MobileNetV2 (ONNX)
|
|
2
|
+
|
|
3
|
+
`mobilenetv2-7.onnx` is the MobileNetV2-1.0 image classifier from the
|
|
4
|
+
[ONNX Model Zoo](https://github.com/onnx/models), `validated/vision/classification/mobilenet`,
|
|
5
|
+
opset 7. It is trained on ImageNet (ILSVRC2012) and licensed **Apache-2.0**
|
|
6
|
+
(see the model directory's own `SPDX-License-Identifier: Apache-2.0` header
|
|
7
|
+
in that repo), compatible with ContinuityGuard's own Apache-2.0 license.
|
|
8
|
+
|
|
9
|
+
## Why this model, and not a face/CLIP embedding model
|
|
10
|
+
|
|
11
|
+
Character-consistency scoring uses a pretrained ONNX embedding model via
|
|
12
|
+
`onnxruntime-node`, with CLIP ViT-B/32 considered as one illustrative
|
|
13
|
+
option before landing on MobileNetV2 below.
|
|
14
|
+
`onnxruntime-node` itself works fine on this build's target
|
|
15
|
+
platform (see the root README's "onnxruntime-node vs. WASM" note) -- the
|
|
16
|
+
substitution here is the *model*, not the runtime.
|
|
17
|
+
|
|
18
|
+
v0.1 ships MobileNetV2 instead of a dedicated face-recognition or CLIP
|
|
19
|
+
embedding model for one practical reason: a specialized face/character
|
|
20
|
+
embedding model that is both small enough to bundle directly in an npm
|
|
21
|
+
package and unambiguously permissively licensed was not something this
|
|
22
|
+
build could source, convert, and license-verify inside this session's
|
|
23
|
+
scope. MobileNetV2 is small (14MB), Apache-2.0 licensed with no ambiguity,
|
|
24
|
+
and works today.
|
|
25
|
+
|
|
26
|
+
**What this means in practice, stated honestly:** ContinuityGuard's
|
|
27
|
+
character-consistency score is a cosine-similarity comparison of
|
|
28
|
+
MobileNetV2's final 1000-way ImageNet-logit vector between two character
|
|
29
|
+
crops, used as a generic visual-similarity proxy -- not a specialized
|
|
30
|
+
face-identity or CLIP embedding tuned for "is this the same character."
|
|
31
|
+
It will respond to overall color, texture, and coarse-shape similarity
|
|
32
|
+
between crops, which is a real but weaker signal than a dedicated
|
|
33
|
+
face-recognition embedding would give you on photorealistic faces, and an
|
|
34
|
+
even weaker, unvalidated signal on stylized/anime-adjacent character
|
|
35
|
+
designs (see the main README's "Known limitations" section, which applies
|
|
36
|
+
regardless of which embedding model is used underneath).
|
|
37
|
+
|
|
38
|
+
This is disclosed here, in the CLI output, in the JSON report's `reason`
|
|
39
|
+
field, and in the README -- not discovered later by a disappointed user.
|
|
40
|
+
A future version can swap in a dedicated face-embedding ONNX model behind
|
|
41
|
+
the same `computeEmbedding` interface in `src/score/consistency.ts` without
|
|
42
|
+
changing the CLI or report shape.
|
|
Binary file
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
"""
|
|
2
|
+
CG03 -- physics-plausibility heuristic.
|
|
3
|
+
|
|
4
|
+
This is a frame-to-frame motion-discontinuity proxy, computed as the mean
|
|
5
|
+
absolute per-channel pixel difference between consecutive sampled frames
|
|
6
|
+
of a shot (a frame-diff proxy, not optical-flow motion-vector extraction
|
|
7
|
+
-- see the root README "Known limitations" for why this simpler proxy was
|
|
8
|
+
chosen for v0.1). A shot is flagged when the diff between two consecutive
|
|
9
|
+
frames exceeds a multiple of that shot's own local baseline (the median
|
|
10
|
+
frame-to-frame diff across the whole shot).
|
|
11
|
+
|
|
12
|
+
HARD RULE, enforced everywhere this feature is surfaced (CLI output, the
|
|
13
|
+
JSON report's `reason` field, README copy): this heuristic never "detects"
|
|
14
|
+
a physics violation. It only "flags a shot for human review." It has real
|
|
15
|
+
false positives (legitimate fast motion, intentional stylized jump-cuts)
|
|
16
|
+
and false negatives (subtly implausible motion that stays under the
|
|
17
|
+
threshold) -- it is a proxy, not a ground-truth validator.
|
|
18
|
+
|
|
19
|
+
Ported from src/score/physics.ts.
|
|
20
|
+
"""
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import statistics
|
|
24
|
+
from dataclasses import dataclass, field
|
|
25
|
+
from typing import List, Optional, Sequence
|
|
26
|
+
|
|
27
|
+
import numpy as np
|
|
28
|
+
|
|
29
|
+
from ..ingest.ffmpeg import FRAME_SIZE, read_raw_frame
|
|
30
|
+
|
|
31
|
+
# A consecutive-frame diff is flagged when it exceeds this multiple of the
|
|
32
|
+
# shot's own median frame-to-frame diff. Derived from this repo's own
|
|
33
|
+
# fixture run -- see the root CHANGELOG.md "CG03 fixture calibration"
|
|
34
|
+
# entry for the exact command and raw numbers this value came from, not an
|
|
35
|
+
# illustrative placeholder. Identical to the TS side's
|
|
36
|
+
# DISCONTINUITY_MULTIPLIER.
|
|
37
|
+
DISCONTINUITY_MULTIPLIER = 3
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def compute_frame_diff(a: bytes, b: bytes) -> float:
|
|
41
|
+
"""Mean absolute per-channel pixel difference between two same-sized
|
|
42
|
+
raw RGB24 frame buffers, normalized to [0, 1]. Uses numpy for speed
|
|
43
|
+
(numpy is already a transitive dependency via onnxruntime); the result
|
|
44
|
+
is numerically identical to a pure-Python sum of absolute differences."""
|
|
45
|
+
if len(a) != len(b):
|
|
46
|
+
raise ValueError("Cannot diff frames of different sizes")
|
|
47
|
+
arr_a = np.frombuffer(a, dtype=np.uint8).astype(np.int32)
|
|
48
|
+
arr_b = np.frombuffer(b, dtype=np.uint8).astype(np.int32)
|
|
49
|
+
total = int(np.abs(arr_a - arr_b).sum())
|
|
50
|
+
return total / len(a) / 255
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def compute_diff_sequence(frames: Sequence[bytes]) -> List[float]:
|
|
54
|
+
"""Consecutive-frame diffs for an ordered sequence of frames. Length is
|
|
55
|
+
len(frames) - 1 (empty if fewer than 2 frames)."""
|
|
56
|
+
return [compute_frame_diff(frames[i - 1], frames[i]) for i in range(1, len(frames))]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _median(values: Sequence[float]) -> float:
|
|
60
|
+
return statistics.median(values)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass(frozen=True)
|
|
64
|
+
class Discontinuity:
|
|
65
|
+
frame_index_a: int
|
|
66
|
+
frame_index_b: int
|
|
67
|
+
diff: float
|
|
68
|
+
baseline: float
|
|
69
|
+
ratio: float
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def detect_discontinuities(
|
|
73
|
+
diffs: Sequence[float], multiplier: float = DISCONTINUITY_MULTIPLIER
|
|
74
|
+
) -> List[Discontinuity]:
|
|
75
|
+
"""Pure detection logic over an already-computed diff sequence -- split
|
|
76
|
+
out from `score_physics` so it has fast, deterministic unit-test
|
|
77
|
+
coverage independent of ffmpeg/frame extraction."""
|
|
78
|
+
if not diffs:
|
|
79
|
+
return []
|
|
80
|
+
baseline = _median(diffs)
|
|
81
|
+
found: List[Discontinuity] = []
|
|
82
|
+
for index, diff in enumerate(diffs):
|
|
83
|
+
if baseline == 0:
|
|
84
|
+
# A perfectly static baseline (e.g. an all-static shot) makes
|
|
85
|
+
# any ratio computation divide-by-zero/undefined; only flag if
|
|
86
|
+
# there is any motion at all against a genuinely zero baseline.
|
|
87
|
+
if diff > 0:
|
|
88
|
+
found.append(
|
|
89
|
+
Discontinuity(
|
|
90
|
+
frame_index_a=index,
|
|
91
|
+
frame_index_b=index + 1,
|
|
92
|
+
diff=diff,
|
|
93
|
+
baseline=baseline,
|
|
94
|
+
ratio=float("inf"),
|
|
95
|
+
)
|
|
96
|
+
)
|
|
97
|
+
continue
|
|
98
|
+
ratio = diff / baseline
|
|
99
|
+
if ratio > multiplier:
|
|
100
|
+
found.append(
|
|
101
|
+
Discontinuity(
|
|
102
|
+
frame_index_a=index,
|
|
103
|
+
frame_index_b=index + 1,
|
|
104
|
+
diff=diff,
|
|
105
|
+
baseline=baseline,
|
|
106
|
+
ratio=ratio,
|
|
107
|
+
)
|
|
108
|
+
)
|
|
109
|
+
return found
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def build_physics_reason(ratio: float) -> str:
|
|
113
|
+
ratio_text = f"{ratio:.1f}x" if ratio != float("inf") else "far above"
|
|
114
|
+
return (
|
|
115
|
+
f"Frame-to-frame motion discontinuity {ratio_text} this shot's local baseline. "
|
|
116
|
+
"This is a heuristic proxy, not a physics simulator -- it flags the shot for "
|
|
117
|
+
"human review, it does not detect a physics violation. Expect both false "
|
|
118
|
+
"positives (legitimate fast motion, stylized jump-cuts) and false negatives."
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass(frozen=True)
|
|
123
|
+
class PhysicsShotInput:
|
|
124
|
+
clip: str
|
|
125
|
+
frame_paths: List[str]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@dataclass(frozen=True)
|
|
129
|
+
class PhysicsFlag:
|
|
130
|
+
clip: str
|
|
131
|
+
frame_index_a: int
|
|
132
|
+
frame_index_b: int
|
|
133
|
+
discontinuity_ratio: float
|
|
134
|
+
reason: str
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
@dataclass(frozen=True)
|
|
138
|
+
class PhysicsResult:
|
|
139
|
+
flagged_shots: List[PhysicsFlag] = field(default_factory=list)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def score_physics(
|
|
143
|
+
shots: Sequence[PhysicsShotInput], multiplier: Optional[float] = None
|
|
144
|
+
) -> PhysicsResult:
|
|
145
|
+
"""End-to-end CG03 entry point: reads every shot's sampled raw frames
|
|
146
|
+
from disk, computes the diff sequence, and flags discontinuities."""
|
|
147
|
+
flagged_shots: List[PhysicsFlag] = []
|
|
148
|
+
active_multiplier = DISCONTINUITY_MULTIPLIER if multiplier is None else multiplier
|
|
149
|
+
for shot in shots:
|
|
150
|
+
if len(shot.frame_paths) < 2:
|
|
151
|
+
continue
|
|
152
|
+
frames = [read_raw_frame(frame_path) for frame_path in shot.frame_paths]
|
|
153
|
+
diffs = compute_diff_sequence(frames)
|
|
154
|
+
discontinuities = detect_discontinuities(diffs, active_multiplier)
|
|
155
|
+
for d in discontinuities:
|
|
156
|
+
flagged_shots.append(
|
|
157
|
+
PhysicsFlag(
|
|
158
|
+
clip=shot.clip,
|
|
159
|
+
frame_index_a=d.frame_index_a,
|
|
160
|
+
frame_index_b=d.frame_index_b,
|
|
161
|
+
discontinuity_ratio=round(d.ratio, 2) if d.ratio != float("inf") else -1,
|
|
162
|
+
reason=build_physics_reason(d.ratio),
|
|
163
|
+
)
|
|
164
|
+
)
|
|
165
|
+
return PhysicsResult(flagged_shots=flagged_shots)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
# Re-exported so callers (CLI, tests) can reference the expected frame size
|
|
169
|
+
# without importing from continuityguard.ingest directly.
|
|
170
|
+
EXPECTED_FRAME_SIZE = FRAME_SIZE
|