media-optimization-engine 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- media_engine/__init__.py +7 -0
- media_engine/adapters/__init__.py +0 -0
- media_engine/adapters/flutter/README.md +10 -0
- media_engine/adapters/flutter/__init__.py +0 -0
- media_engine/adapters/flutter/models.dart +88 -0
- media_engine/adapters/marzipano/__init__.py +2 -0
- media_engine/adapters/marzipano/manifest.py +53 -0
- media_engine/adapters/pannellum/__init__.py +2 -0
- media_engine/adapters/pannellum/manifest.py +23 -0
- media_engine/adapters/threejs/__init__.py +2 -0
- media_engine/adapters/threejs/manifest.py +10 -0
- media_engine/admin.py +36 -0
- media_engine/apps.py +11 -0
- media_engine/auth.py +33 -0
- media_engine/autoregister.py +108 -0
- media_engine/control_plane.py +274 -0
- media_engine/doctor.py +177 -0
- media_engine/focal_detectors.py +28 -0
- media_engine/health.py +78 -0
- media_engine/image_ops.py +142 -0
- media_engine/integration.py +34 -0
- media_engine/locks.py +20 -0
- media_engine/management/__init__.py +0 -0
- media_engine/management/commands/__init__.py +0 -0
- media_engine/management/commands/audit_media_engine.py +54 -0
- media_engine/management/commands/audit_panorama_media.py +20 -0
- media_engine/management/commands/backfill_panorama_media.py +32 -0
- media_engine/management/commands/backfill_registered_media.py +21 -0
- media_engine/management/commands/backfill_responsive_images.py +27 -0
- media_engine/management/commands/cleanup_image_derivatives.py +30 -0
- media_engine/management/commands/media_engine_doctor.py +15 -0
- media_engine/management/commands/media_engine_node_report.py +24 -0
- media_engine/management/commands/rollback_media_engine_config.py +14 -0
- media_engine/management/commands/sync_media_engine_hub.py +20 -0
- media_engine/manifest.py +89 -0
- media_engine/metrics.py +34 -0
- media_engine/migrations/0001_initial.py +87 -0
- media_engine/migrations/0002_panorama_multires.py +44 -0
- media_engine/migrations/0003_panorama_cube_tiles.py +31 -0
- media_engine/migrations/__init__.py +0 -0
- media_engine/models.py +145 -0
- media_engine/node.py +29 -0
- media_engine/processors/__init__.py +0 -0
- media_engine/processors/base.py +21 -0
- media_engine/processors/image/__init__.py +0 -0
- media_engine/processors/image/standard.py +16 -0
- media_engine/processors/panorama/__init__.py +5 -0
- media_engine/processors/panorama/cube_tiles.py +121 -0
- media_engine/processors/panorama/cubemap.py +40 -0
- media_engine/processors/panorama/detect.py +23 -0
- media_engine/processors/panorama/equirectangular.py +149 -0
- media_engine/processors/panorama/manifest.py +105 -0
- media_engine/processors/panorama/tiles.py +88 -0
- media_engine/processors/registry.py +17 -0
- media_engine/profiles.py +77 -0
- media_engine/queueing.py +34 -0
- media_engine/registry.py +49 -0
- media_engine/runtime_config.py +37 -0
- media_engine/serializers.py +29 -0
- media_engine/services.py +187 -0
- media_engine/signals.py +5 -0
- media_engine/spec/openapi.yaml +191 -0
- media_engine/static/media_engine/adaptive-media.js +77 -0
- media_engine/static/media_engine/media-cache-sw.js +17 -0
- media_engine/storage_paths.py +13 -0
- media_engine/system_views.py +102 -0
- media_engine/tasks.py +44 -0
- media_engine/templatetags/__init__.py +0 -0
- media_engine/templatetags/responsive_media.py +262 -0
- media_engine/tests/__init__.py +0 -0
- media_engine/tests/test_distributed_node.py +72 -0
- media_engine/tests/test_image_ops.py +14 -0
- media_engine/tests/test_panorama.py +31 -0
- media_engine/tests/test_paths.py +9 -0
- media_engine/tests/test_profiles.py +8 -0
- media_engine/tests/test_responsive_media.py +153 -0
- media_engine/urls.py +14 -0
- media_engine/validators.py +42 -0
- media_engine/views.py +111 -0
- media_optimization_engine-1.4.0.dist-info/METADATA +253 -0
- media_optimization_engine-1.4.0.dist-info/RECORD +85 -0
- media_optimization_engine-1.4.0.dist-info/WHEEL +5 -0
- media_optimization_engine-1.4.0.dist-info/entry_points.txt +2 -0
- media_optimization_engine-1.4.0.dist-info/licenses/LICENSE +21 -0
- media_optimization_engine-1.4.0.dist-info/top_level.txt +1 -0
media_engine/doctor.py
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""Integration diagnostics for Media Optimization Engine.
|
|
2
|
+
|
|
3
|
+
Can be used either as a Django management command or directly with:
|
|
4
|
+
python -m media_engine.doctor
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import importlib.util
|
|
9
|
+
import os
|
|
10
|
+
import sys
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Iterable
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class Check:
|
|
18
|
+
name: str
|
|
19
|
+
ok: bool
|
|
20
|
+
detail: str
|
|
21
|
+
level: str = "ERROR"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _yesno(value: bool) -> str:
|
|
25
|
+
return "OK" if value else "FAIL"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _package_checks() -> list[Check]:
|
|
29
|
+
checks: list[Check] = []
|
|
30
|
+
try:
|
|
31
|
+
import media_engine
|
|
32
|
+
checks.append(Check("package", True, f"media_engine {getattr(media_engine, '__version__', 'unknown')}"))
|
|
33
|
+
except Exception as exc:
|
|
34
|
+
return [Check("package", False, str(exc))]
|
|
35
|
+
|
|
36
|
+
for module, label in [
|
|
37
|
+
("PIL", "Pillow"),
|
|
38
|
+
("PIL.AvifImagePlugin", "AVIF plugin"),
|
|
39
|
+
("rest_framework", "Django REST Framework"),
|
|
40
|
+
]:
|
|
41
|
+
found = importlib.util.find_spec(module) is not None
|
|
42
|
+
checks.append(Check(label, found, "installed" if found else "missing"))
|
|
43
|
+
return checks
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def run_diagnostics() -> list[Check]:
|
|
47
|
+
checks = _package_checks()
|
|
48
|
+
try:
|
|
49
|
+
from django.conf import settings
|
|
50
|
+
from django.apps import apps
|
|
51
|
+
from django.core.cache import cache
|
|
52
|
+
from django.core.files.base import ContentFile
|
|
53
|
+
from django.core.files.storage import default_storage
|
|
54
|
+
from django.db import connection
|
|
55
|
+
from django.db.migrations.executor import MigrationExecutor
|
|
56
|
+
except Exception as exc:
|
|
57
|
+
checks.append(Check("django runtime", False, f"Django unavailable: {exc}"))
|
|
58
|
+
return checks
|
|
59
|
+
|
|
60
|
+
if not settings.configured:
|
|
61
|
+
checks.append(Check(
|
|
62
|
+
"django settings",
|
|
63
|
+
False,
|
|
64
|
+
"settings are not configured; set DJANGO_SETTINGS_MODULE or run through manage.py",
|
|
65
|
+
))
|
|
66
|
+
return checks
|
|
67
|
+
|
|
68
|
+
installed = "media_engine" in settings.INSTALLED_APPS or any(
|
|
69
|
+
str(item).startswith("media_engine.") for item in settings.INSTALLED_APPS
|
|
70
|
+
)
|
|
71
|
+
checks.append(Check(
|
|
72
|
+
"INSTALLED_APPS",
|
|
73
|
+
installed,
|
|
74
|
+
"media_engine registered" if installed else 'add "media_engine" to INSTALLED_APPS',
|
|
75
|
+
))
|
|
76
|
+
if not installed:
|
|
77
|
+
return checks
|
|
78
|
+
|
|
79
|
+
checks.append(Check("app registry", apps.is_installed("media_engine"), "Django app loaded"))
|
|
80
|
+
|
|
81
|
+
try:
|
|
82
|
+
with connection.cursor() as cursor:
|
|
83
|
+
cursor.execute("SELECT 1")
|
|
84
|
+
cursor.fetchone()
|
|
85
|
+
checks.append(Check("database", True, connection.vendor))
|
|
86
|
+
except Exception as exc:
|
|
87
|
+
checks.append(Check("database", False, str(exc)))
|
|
88
|
+
|
|
89
|
+
try:
|
|
90
|
+
executor = MigrationExecutor(connection)
|
|
91
|
+
targets = executor.loader.graph.leaf_nodes("media_engine")
|
|
92
|
+
plan = executor.migration_plan(targets)
|
|
93
|
+
pending = [f"{m.app_label}.{m.name}" for m, backwards in plan if not backwards and m.app_label == "media_engine"]
|
|
94
|
+
checks.append(Check(
|
|
95
|
+
"migrations",
|
|
96
|
+
not pending,
|
|
97
|
+
"all applied" if not pending else "pending: " + ", ".join(pending),
|
|
98
|
+
))
|
|
99
|
+
except Exception as exc:
|
|
100
|
+
checks.append(Check("migrations", False, str(exc)))
|
|
101
|
+
|
|
102
|
+
try:
|
|
103
|
+
key = "media_engine:doctor"
|
|
104
|
+
cache.set(key, "ok", timeout=10)
|
|
105
|
+
value = cache.get(key)
|
|
106
|
+
checks.append(Check("cache", value == "ok", settings.CACHES.get("default", {}).get("BACKEND", "unknown")))
|
|
107
|
+
cache.delete(key)
|
|
108
|
+
except Exception as exc:
|
|
109
|
+
checks.append(Check("cache", False, str(exc), level="WARN"))
|
|
110
|
+
|
|
111
|
+
try:
|
|
112
|
+
test_name = "media_engine/doctor/.write-test"
|
|
113
|
+
saved = default_storage.save(test_name, ContentFile(b"media-engine-doctor"))
|
|
114
|
+
exists = default_storage.exists(saved)
|
|
115
|
+
if exists:
|
|
116
|
+
default_storage.delete(saved)
|
|
117
|
+
checks.append(Check("storage", exists, f"writable via {default_storage.__class__.__name__}"))
|
|
118
|
+
except Exception as exc:
|
|
119
|
+
checks.append(Check("storage", False, str(exc)))
|
|
120
|
+
|
|
121
|
+
from .queueing import resolved_task_mode
|
|
122
|
+
configured_mode = getattr(settings, "MEDIA_ENGINE_TASK_MODE", "auto")
|
|
123
|
+
effective_mode = resolved_task_mode()
|
|
124
|
+
checks.append(Check("task mode", True, f"configured={configured_mode}; effective={effective_mode}"))
|
|
125
|
+
|
|
126
|
+
auto_fields = getattr(settings, "MEDIA_ENGINE_AUTO_FIELDS", {}) or {}
|
|
127
|
+
auto_discover = bool(getattr(settings, "MEDIA_ENGINE_AUTO_DISCOVER_IMAGE_FIELDS", False))
|
|
128
|
+
checks.append(Check("auto integration", True, f"configured_models={len(auto_fields)}; discover={auto_discover}"))
|
|
129
|
+
|
|
130
|
+
try:
|
|
131
|
+
from .processors.panorama.equirectangular import Panorama360Processor
|
|
132
|
+
checks.append(Check("panorama processor", True, Panorama360Processor.__name__))
|
|
133
|
+
except Exception as exc:
|
|
134
|
+
checks.append(Check("panorama processor", False, str(exc)))
|
|
135
|
+
|
|
136
|
+
try:
|
|
137
|
+
from PIL import features
|
|
138
|
+
checks.append(Check("WebP codec", bool(features.check("webp")), "Pillow WebP support"))
|
|
139
|
+
except Exception as exc:
|
|
140
|
+
checks.append(Check("WebP codec", False, str(exc), level="WARN"))
|
|
141
|
+
|
|
142
|
+
return checks
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def format_report(checks: Iterable[Check]) -> str:
|
|
146
|
+
rows = ["Media Optimization Engine doctor", "=" * 32]
|
|
147
|
+
failures = 0
|
|
148
|
+
warnings = 0
|
|
149
|
+
for c in checks:
|
|
150
|
+
status = _yesno(c.ok)
|
|
151
|
+
if not c.ok and c.level == "WARN":
|
|
152
|
+
status = "WARN"
|
|
153
|
+
warnings += 1
|
|
154
|
+
elif not c.ok:
|
|
155
|
+
failures += 1
|
|
156
|
+
rows.append(f"[{status:4}] {c.name}: {c.detail}")
|
|
157
|
+
rows.append("-" * 32)
|
|
158
|
+
rows.append(f"Result: {failures} error(s), {warnings} warning(s)")
|
|
159
|
+
return "\n".join(rows)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def main() -> int:
|
|
163
|
+
settings_module = os.getenv("DJANGO_SETTINGS_MODULE")
|
|
164
|
+
if settings_module:
|
|
165
|
+
try:
|
|
166
|
+
import django
|
|
167
|
+
django.setup()
|
|
168
|
+
except Exception as exc:
|
|
169
|
+
print(f"Unable to initialize Django: {exc}", file=sys.stderr)
|
|
170
|
+
return 2
|
|
171
|
+
checks = run_diagnostics()
|
|
172
|
+
print(format_report(checks))
|
|
173
|
+
return 0 if all(c.ok or c.level == "WARN" for c in checks) else 1
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
if __name__ == "__main__":
|
|
177
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
def detect_focal_point(image):
|
|
2
|
+
"""Optional face-aware focal detector.
|
|
3
|
+
|
|
4
|
+
Install requirements-ai.txt to enable OpenCV face detection. If OpenCV is
|
|
5
|
+
not installed or no face is found, return (None, None) and the normal
|
|
6
|
+
center crop remains in effect.
|
|
7
|
+
"""
|
|
8
|
+
try:
|
|
9
|
+
import cv2
|
|
10
|
+
import numpy as np
|
|
11
|
+
except Exception:
|
|
12
|
+
return None, None
|
|
13
|
+
|
|
14
|
+
rgb = image.convert('RGB')
|
|
15
|
+
arr = np.asarray(rgb)
|
|
16
|
+
gray = cv2.cvtColor(arr, cv2.COLOR_RGB2GRAY)
|
|
17
|
+
cascade = cv2.CascadeClassifier(
|
|
18
|
+
cv2.data.haarcascades + 'haarcascade_frontalface_default.xml'
|
|
19
|
+
)
|
|
20
|
+
faces = cascade.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=5, minSize=(32, 32))
|
|
21
|
+
if len(faces) == 0:
|
|
22
|
+
return None, None
|
|
23
|
+
|
|
24
|
+
x1 = min(int(x) for x, y, w, h in faces)
|
|
25
|
+
y1 = min(int(y) for x, y, w, h in faces)
|
|
26
|
+
x2 = max(int(x + w) for x, y, w, h in faces)
|
|
27
|
+
y2 = max(int(y + h) for x, y, w, h in faces)
|
|
28
|
+
return ((x1 + x2) / 2 / image.width, (y1 + y2) / 2 / image.height)
|
media_engine/health.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from django.conf import settings
|
|
3
|
+
from django.core.cache import cache
|
|
4
|
+
from django.core.files.base import ContentFile
|
|
5
|
+
from django.core.files.storage import default_storage
|
|
6
|
+
from django.db import connection
|
|
7
|
+
from .control_plane import control_plane_status
|
|
8
|
+
from .node import get_node_identity
|
|
9
|
+
|
|
10
|
+
HEARTBEAT_KEY = 'media-engine:worker-heartbeat'
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _check_database():
|
|
14
|
+
try:
|
|
15
|
+
with connection.cursor() as cursor:
|
|
16
|
+
cursor.execute('SELECT 1')
|
|
17
|
+
cursor.fetchone()
|
|
18
|
+
return {'state': 'healthy'}
|
|
19
|
+
except Exception as exc:
|
|
20
|
+
return {'state': 'unhealthy', 'error': str(exc)[:300]}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _check_cache():
|
|
24
|
+
try:
|
|
25
|
+
key = 'media-engine:health:cache'
|
|
26
|
+
cache.set(key, 'ok', timeout=10)
|
|
27
|
+
ok = cache.get(key) == 'ok'
|
|
28
|
+
return {'state': 'healthy' if ok else 'unhealthy'}
|
|
29
|
+
except Exception as exc:
|
|
30
|
+
return {'state': 'unhealthy', 'error': str(exc)[:300]}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _check_storage():
|
|
34
|
+
identity = get_node_identity()
|
|
35
|
+
name = f'media_engine/health/{identity.node_id}.txt'
|
|
36
|
+
try:
|
|
37
|
+
stored = default_storage.save(name, ContentFile(b'ok'))
|
|
38
|
+
ok = default_storage.exists(stored)
|
|
39
|
+
default_storage.delete(stored)
|
|
40
|
+
return {'state': 'healthy' if ok else 'unhealthy'}
|
|
41
|
+
except Exception as exc:
|
|
42
|
+
return {'state': 'unhealthy', 'error': str(exc)[:300]}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _check_worker():
|
|
46
|
+
try:
|
|
47
|
+
stamp = cache.get(HEARTBEAT_KEY)
|
|
48
|
+
if not stamp:
|
|
49
|
+
return {'state': 'degraded', 'reason': 'no_heartbeat'}
|
|
50
|
+
age = max(0, int(time.time() - float(stamp)))
|
|
51
|
+
max_age = int(getattr(settings, 'MEDIA_ENGINE_WORKER_HEARTBEAT_MAX_AGE', 120))
|
|
52
|
+
return {
|
|
53
|
+
'state': 'healthy' if age <= max_age else 'degraded',
|
|
54
|
+
'age_seconds': age,
|
|
55
|
+
}
|
|
56
|
+
except Exception as exc:
|
|
57
|
+
return {'state': 'degraded', 'error': str(exc)[:300]}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def build_health_report(*, deep=False):
|
|
61
|
+
db = _check_database()
|
|
62
|
+
cache_status = _check_cache()
|
|
63
|
+
storage = _check_storage() if deep else {'state': 'healthy', 'check': 'shallow', 'backend': default_storage.__class__.__name__}
|
|
64
|
+
worker = _check_worker()
|
|
65
|
+
control = control_plane_status()
|
|
66
|
+
core_healthy = all(part.get('state') == 'healthy' for part in (db, cache_status, storage))
|
|
67
|
+
state = 'healthy' if core_healthy and worker.get('state') == 'healthy' else ('degraded' if core_healthy else 'unhealthy')
|
|
68
|
+
return {
|
|
69
|
+
'state': state,
|
|
70
|
+
'node': get_node_identity().as_dict(),
|
|
71
|
+
'database': db,
|
|
72
|
+
'cache': cache_status,
|
|
73
|
+
'storage': storage,
|
|
74
|
+
'worker': worker,
|
|
75
|
+
'control_plane': control,
|
|
76
|
+
'control_plane_required_for_runtime': False,
|
|
77
|
+
'deep_storage_check': bool(deep),
|
|
78
|
+
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import io
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from PIL import Image, ImageOps
|
|
5
|
+
import numpy as np
|
|
6
|
+
from blurhash import encode as blurhash_encode
|
|
7
|
+
try:
|
|
8
|
+
import pillow_avif # noqa: F401
|
|
9
|
+
except Exception:
|
|
10
|
+
pillow_avif = None
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class EncodedImage:
|
|
15
|
+
content: bytes
|
|
16
|
+
width: int
|
|
17
|
+
height: int
|
|
18
|
+
quality: int
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def normalized_image(source):
|
|
22
|
+
image = Image.open(source)
|
|
23
|
+
image.load()
|
|
24
|
+
image = ImageOps.exif_transpose(image)
|
|
25
|
+
if image.mode not in ('RGB', 'RGBA'):
|
|
26
|
+
image = image.convert('RGBA' if 'A' in image.getbands() else 'RGB')
|
|
27
|
+
return image
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def dominant_color(image):
|
|
31
|
+
thumb = image.convert('RGB').resize((1, 1), Image.Resampling.LANCZOS)
|
|
32
|
+
r, g, b = thumb.getpixel((0, 0))
|
|
33
|
+
return f'#{r:02x}{g:02x}{b:02x}'
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def make_blurhash(image):
|
|
37
|
+
small = image.convert('RGB').resize((32, 32), Image.Resampling.LANCZOS)
|
|
38
|
+
pixels = np.asarray(small)
|
|
39
|
+
return blurhash_encode(pixels, components_x=4, components_y=3)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def placeholder_data_url(image):
|
|
43
|
+
small = image.convert('RGB')
|
|
44
|
+
small.thumbnail((32, 32), Image.Resampling.LANCZOS)
|
|
45
|
+
out = io.BytesIO()
|
|
46
|
+
small.save(out, format='WEBP', quality=25, method=4)
|
|
47
|
+
payload = base64.b64encode(out.getvalue()).decode('ascii')
|
|
48
|
+
return f'data:image/webp;base64,{payload}'
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def crop_to_ratio(image, ratio, focal_x=None, focal_y=None):
|
|
52
|
+
target_ratio = ratio[0] / ratio[1]
|
|
53
|
+
current_ratio = image.width / image.height
|
|
54
|
+
if abs(target_ratio - current_ratio) < 0.001:
|
|
55
|
+
return image
|
|
56
|
+
|
|
57
|
+
fx = 0.5 if focal_x is None else min(1.0, max(0.0, focal_x))
|
|
58
|
+
fy = 0.5 if focal_y is None else min(1.0, max(0.0, focal_y))
|
|
59
|
+
|
|
60
|
+
if current_ratio > target_ratio:
|
|
61
|
+
new_width = int(image.height * target_ratio)
|
|
62
|
+
max_left = image.width - new_width
|
|
63
|
+
left = int(max_left * fx)
|
|
64
|
+
left = min(max(0, left), max_left)
|
|
65
|
+
box = (left, 0, left + new_width, image.height)
|
|
66
|
+
else:
|
|
67
|
+
new_height = int(image.width / target_ratio)
|
|
68
|
+
max_top = image.height - new_height
|
|
69
|
+
top = int(max_top * fy)
|
|
70
|
+
top = min(max(0, top), max_top)
|
|
71
|
+
box = (0, top, image.width, top + new_height)
|
|
72
|
+
return image.crop(box)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def resized(image, width, fit='contain', aspect_ratio=None, focal_x=None, focal_y=None):
|
|
76
|
+
working = image
|
|
77
|
+
if fit == 'cover' and aspect_ratio:
|
|
78
|
+
working = crop_to_ratio(working, aspect_ratio, focal_x=focal_x, focal_y=focal_y)
|
|
79
|
+
width = min(width, working.width)
|
|
80
|
+
if width == working.width:
|
|
81
|
+
return working.copy()
|
|
82
|
+
height = round(working.height * width / working.width)
|
|
83
|
+
return working.resize((width, height), Image.Resampling.LANCZOS)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _save_kwargs(fmt, quality):
|
|
87
|
+
fmt = fmt.lower()
|
|
88
|
+
if fmt == 'avif':
|
|
89
|
+
return {'format': 'AVIF', 'quality': quality, 'speed': 6}
|
|
90
|
+
if fmt == 'webp':
|
|
91
|
+
return {'format': 'WEBP', 'quality': quality, 'method': 6}
|
|
92
|
+
if fmt in ('jpg', 'jpeg'):
|
|
93
|
+
return {'format': 'JPEG', 'quality': quality, 'optimize': True, 'progressive': True}
|
|
94
|
+
if fmt == 'png':
|
|
95
|
+
return {'format': 'PNG', 'optimize': True}
|
|
96
|
+
raise ValueError(f'Unsupported output format: {fmt}')
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _prepare_for_format(image, fmt):
|
|
100
|
+
if fmt.lower() in ('jpeg', 'jpg') and image.mode == 'RGBA':
|
|
101
|
+
canvas = Image.new('RGB', image.size, 'white')
|
|
102
|
+
canvas.paste(image, mask=image.getchannel('A'))
|
|
103
|
+
return canvas
|
|
104
|
+
if fmt.lower() in ('jpeg', 'jpg') and image.mode != 'RGB':
|
|
105
|
+
return image.convert('RGB')
|
|
106
|
+
return image
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def encode_with_budget(image, fmt, initial_quality, target_bytes=None, min_quality=35):
|
|
110
|
+
image = _prepare_for_format(image, fmt)
|
|
111
|
+
|
|
112
|
+
def encode(q):
|
|
113
|
+
out = io.BytesIO()
|
|
114
|
+
image.save(out, **_save_kwargs(fmt, q))
|
|
115
|
+
return out.getvalue()
|
|
116
|
+
|
|
117
|
+
quality = int(initial_quality)
|
|
118
|
+
content = encode(quality)
|
|
119
|
+
if not target_bytes or len(content) <= target_bytes or fmt.lower() == 'png':
|
|
120
|
+
return EncodedImage(content, image.width, image.height, quality)
|
|
121
|
+
|
|
122
|
+
low, high = min_quality, quality
|
|
123
|
+
best = None
|
|
124
|
+
best_q = min_quality
|
|
125
|
+
smallest = content
|
|
126
|
+
smallest_q = quality
|
|
127
|
+
while low <= high:
|
|
128
|
+
mid = (low + high) // 2
|
|
129
|
+
candidate = encode(mid)
|
|
130
|
+
if len(candidate) < len(smallest):
|
|
131
|
+
smallest = candidate
|
|
132
|
+
smallest_q = mid
|
|
133
|
+
if len(candidate) <= target_bytes:
|
|
134
|
+
best = candidate
|
|
135
|
+
best_q = mid
|
|
136
|
+
low = mid + 1
|
|
137
|
+
else:
|
|
138
|
+
high = mid - 1
|
|
139
|
+
if best is None:
|
|
140
|
+
best = smallest
|
|
141
|
+
best_q = smallest_q
|
|
142
|
+
return EncodedImage(best, image.width, image.height, best_q)
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
from django.contrib.contenttypes.models import ContentType
|
|
2
|
+
from django.db import transaction
|
|
3
|
+
from django.core.files.base import File
|
|
4
|
+
from .models import MediaBinding
|
|
5
|
+
from .services import ingest_uploaded_file
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def ingest_model_field(instance, *, field_name, profile='default', role='content'):
|
|
9
|
+
field_file = getattr(instance, field_name, None)
|
|
10
|
+
if not field_file or not getattr(field_file, 'name', None):
|
|
11
|
+
return None
|
|
12
|
+
with field_file.storage.open(field_file.name, 'rb') as source:
|
|
13
|
+
wrapped = File(source, name=field_file.name.rsplit('/', 1)[-1])
|
|
14
|
+
asset, created = ingest_uploaded_file(
|
|
15
|
+
wrapped,
|
|
16
|
+
owner_ref=f'{instance._meta.label}:{instance.pk}:{field_name}',
|
|
17
|
+
profile=profile,
|
|
18
|
+
enqueue=False,
|
|
19
|
+
)
|
|
20
|
+
ct = ContentType.objects.get_for_model(instance, for_concrete_model=False)
|
|
21
|
+
previous = MediaBinding.objects.filter(
|
|
22
|
+
content_type=ct, object_id=str(instance.pk), field_name=field_name
|
|
23
|
+
).select_related('asset').first()
|
|
24
|
+
unchanged = previous is not None and previous.asset_id == asset.id and previous.profile == profile
|
|
25
|
+
binding, _ = MediaBinding.objects.update_or_create(
|
|
26
|
+
content_type=ct,
|
|
27
|
+
object_id=str(instance.pk),
|
|
28
|
+
field_name=field_name,
|
|
29
|
+
defaults={'profile': profile, 'role': role, 'asset': asset},
|
|
30
|
+
)
|
|
31
|
+
if created or not unchanged:
|
|
32
|
+
from .queueing import enqueue_asset
|
|
33
|
+
transaction.on_commit(lambda: enqueue_asset(asset.id, profile=profile))
|
|
34
|
+
return binding
|
media_engine/locks.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
from django.core.cache import cache
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@contextmanager
|
|
7
|
+
def cache_lock(key, timeout=120, wait_timeout=8):
|
|
8
|
+
token = f'lock:{key}'
|
|
9
|
+
acquired = False
|
|
10
|
+
deadline = time.monotonic() + wait_timeout
|
|
11
|
+
while time.monotonic() < deadline:
|
|
12
|
+
if cache.add(token, '1', timeout=timeout):
|
|
13
|
+
acquired = True
|
|
14
|
+
break
|
|
15
|
+
time.sleep(0.1)
|
|
16
|
+
try:
|
|
17
|
+
yield acquired
|
|
18
|
+
finally:
|
|
19
|
+
if acquired:
|
|
20
|
+
cache.delete(token)
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
from django.conf import settings
|
|
2
|
+
from django.core.management.base import BaseCommand
|
|
3
|
+
from django.db.models import Sum
|
|
4
|
+
from media_engine.models import MediaAsset, MediaVariant
|
|
5
|
+
from media_engine.profiles import get_profile, requested_widths, all_formats
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Command(BaseCommand):
|
|
9
|
+
help = 'Audit media assets and responsive variants.'
|
|
10
|
+
|
|
11
|
+
def add_arguments(self, parser):
|
|
12
|
+
parser.add_argument('--profile', default='default')
|
|
13
|
+
parser.add_argument('--fail-on-incomplete', action='store_true')
|
|
14
|
+
|
|
15
|
+
def handle(self, *args, **options):
|
|
16
|
+
profile_name = options['profile']
|
|
17
|
+
profile = get_profile(profile_name)
|
|
18
|
+
version = getattr(settings, 'MEDIA_ENGINE_PIPELINE_VERSION', 1)
|
|
19
|
+
total = MediaAsset.objects.count()
|
|
20
|
+
ready_assets = 0
|
|
21
|
+
incomplete = 0
|
|
22
|
+
expected_total = 0
|
|
23
|
+
for asset in MediaAsset.objects.all().iterator():
|
|
24
|
+
expected = len(requested_widths(profile, asset.width)) * len(all_formats(profile))
|
|
25
|
+
expected_total += expected
|
|
26
|
+
ready = MediaVariant.objects.filter(
|
|
27
|
+
asset=asset,
|
|
28
|
+
profile=profile_name,
|
|
29
|
+
processor_version=version,
|
|
30
|
+
status=MediaVariant.Status.READY,
|
|
31
|
+
).count()
|
|
32
|
+
if ready >= expected:
|
|
33
|
+
ready_assets += 1
|
|
34
|
+
else:
|
|
35
|
+
incomplete += 1
|
|
36
|
+
|
|
37
|
+
original_bytes = MediaAsset.objects.aggregate(v=Sum('original_size'))['v'] or 0
|
|
38
|
+
derivative_bytes = MediaVariant.objects.filter(status=MediaVariant.Status.READY).aggregate(v=Sum('file_size'))['v'] or 0
|
|
39
|
+
avif = MediaVariant.objects.filter(format='avif', status=MediaVariant.Status.READY).count()
|
|
40
|
+
webp = MediaVariant.objects.filter(format='webp', status=MediaVariant.Status.READY).count()
|
|
41
|
+
failed = MediaVariant.objects.filter(status=MediaVariant.Status.FAILED).count()
|
|
42
|
+
|
|
43
|
+
self.stdout.write(f'Pipeline version: {version}')
|
|
44
|
+
self.stdout.write(f'Original images: {total}')
|
|
45
|
+
self.stdout.write(f'Ready responsive images: {ready_assets}')
|
|
46
|
+
self.stdout.write(f'Missing/incomplete: {incomplete}')
|
|
47
|
+
self.stdout.write(f'Expected variants: {expected_total}')
|
|
48
|
+
self.stdout.write(f'AVIF variants: {avif}')
|
|
49
|
+
self.stdout.write(f'WebP variants: {webp}')
|
|
50
|
+
self.stdout.write(f'Failed variants: {failed}')
|
|
51
|
+
self.stdout.write(f'Original storage: {original_bytes / (1024 * 1024):.2f} MB')
|
|
52
|
+
self.stdout.write(f'Responsive derivatives: {derivative_bytes / (1024 * 1024):.2f} MB')
|
|
53
|
+
if options['fail_on_incomplete'] and incomplete:
|
|
54
|
+
raise SystemExit(2)
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from django.core.management.base import BaseCommand
|
|
2
|
+
from media_engine.models import MediaAsset, PanoramaTile
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class Command(BaseCommand):
|
|
6
|
+
help = 'Audit 360 panorama assets and multiresolution tile completeness.'
|
|
7
|
+
|
|
8
|
+
def handle(self, *args, **options):
|
|
9
|
+
assets = MediaAsset.objects.filter(media_kind='panorama')
|
|
10
|
+
total = assets.count()
|
|
11
|
+
ready = assets.filter(status='READY').count()
|
|
12
|
+
failed = assets.filter(status='FAILED').count()
|
|
13
|
+
tiles = PanoramaTile.objects.count()
|
|
14
|
+
ready_tiles = PanoramaTile.objects.filter(status='READY').count()
|
|
15
|
+
bytes_total = sum(PanoramaTile.objects.filter(status='READY').values_list('file_size', flat=True))
|
|
16
|
+
self.stdout.write(f'Panorama assets: {total}')
|
|
17
|
+
self.stdout.write(f'Ready panoramas: {ready}')
|
|
18
|
+
self.stdout.write(f'Failed panoramas: {failed}')
|
|
19
|
+
self.stdout.write(f'Tiles: {ready_tiles}/{tiles}')
|
|
20
|
+
self.stdout.write(f'Tile storage: {bytes_total / (1024 * 1024):.2f} MB')
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from django.core.management.base import BaseCommand
|
|
2
|
+
from media_engine.models import MediaAsset
|
|
3
|
+
from media_engine.queueing import enqueue_asset
|
|
4
|
+
from media_engine.processors.panorama.detect import detect_equirectangular
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Command(BaseCommand):
|
|
8
|
+
help = 'Detect existing 2:1 equirectangular assets and queue panorama multires processing.'
|
|
9
|
+
|
|
10
|
+
def add_arguments(self, parser):
|
|
11
|
+
parser.add_argument('--dry-run', action='store_true')
|
|
12
|
+
parser.add_argument('--limit', type=int, default=0)
|
|
13
|
+
|
|
14
|
+
def handle(self, *args, **options):
|
|
15
|
+
changed = 0
|
|
16
|
+
candidates = MediaAsset.objects.exclude(media_kind='panorama').order_by('created_at')
|
|
17
|
+
if options['limit']:
|
|
18
|
+
candidates = candidates[:options['limit']]
|
|
19
|
+
for asset in candidates:
|
|
20
|
+
detection = detect_equirectangular(asset.width, asset.height, asset.metadata_json)
|
|
21
|
+
if not detection.is_panorama:
|
|
22
|
+
continue
|
|
23
|
+
self.stdout.write(f'{asset.id} {asset.width}x{asset.height}: {detection.reason}')
|
|
24
|
+
if options['dry_run']:
|
|
25
|
+
continue
|
|
26
|
+
asset.media_kind = 'panorama'
|
|
27
|
+
asset.projection = detection.projection
|
|
28
|
+
asset.processor = 'panorama_360'
|
|
29
|
+
asset.save(update_fields=['media_kind', 'projection', 'processor', 'updated_at'])
|
|
30
|
+
enqueue_asset(asset.id, profile='panorama.multires', force=False)
|
|
31
|
+
changed += 1
|
|
32
|
+
self.stdout.write(self.style.SUCCESS(f'Queued panorama assets: {changed}'))
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from django.core.management.base import BaseCommand
|
|
2
|
+
from media_engine.integration import ingest_model_field
|
|
3
|
+
from media_engine.registry import entries
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Command(BaseCommand):
|
|
7
|
+
help = 'Ingest image fields declared through register_model_image().'
|
|
8
|
+
|
|
9
|
+
def add_arguments(self, parser):
|
|
10
|
+
parser.add_argument('--limit', type=int)
|
|
11
|
+
|
|
12
|
+
def handle(self, *args, **options):
|
|
13
|
+
processed = 0
|
|
14
|
+
for entry in entries():
|
|
15
|
+
qs = entry.model._default_manager.all()
|
|
16
|
+
if options['limit']:
|
|
17
|
+
qs = qs[:options['limit']]
|
|
18
|
+
for obj in qs.iterator():
|
|
19
|
+
ingest_model_field(obj, field_name=entry.field_name, profile=entry.profile, role=entry.role)
|
|
20
|
+
processed += 1
|
|
21
|
+
self.stdout.write(self.style.SUCCESS(f'Processed {processed} registered model images.'))
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from django.core.management.base import BaseCommand
|
|
2
|
+
from media_engine.models import MediaAsset
|
|
3
|
+
from media_engine.tasks import generate_asset_variants
|
|
4
|
+
from media_engine.queueing import enqueue_asset
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Command(BaseCommand):
|
|
8
|
+
help = 'Generate missing responsive variants for existing MediaAsset rows.'
|
|
9
|
+
|
|
10
|
+
def add_arguments(self, parser):
|
|
11
|
+
parser.add_argument('--profile', default='default')
|
|
12
|
+
parser.add_argument('--force', action='store_true')
|
|
13
|
+
parser.add_argument('--sync', action='store_true')
|
|
14
|
+
parser.add_argument('--limit', type=int)
|
|
15
|
+
|
|
16
|
+
def handle(self, *args, **options):
|
|
17
|
+
qs = MediaAsset.objects.order_by('created_at')
|
|
18
|
+
if options['limit']:
|
|
19
|
+
qs = qs[:options['limit']]
|
|
20
|
+
count = 0
|
|
21
|
+
for asset in qs.iterator():
|
|
22
|
+
if options['sync']:
|
|
23
|
+
generate_asset_variants.apply(args=[str(asset.id), options['profile'], options['force']]).get()
|
|
24
|
+
else:
|
|
25
|
+
enqueue_asset(asset.id, profile=options['profile'], force=options['force'], backfill=True)
|
|
26
|
+
count += 1
|
|
27
|
+
self.stdout.write(self.style.SUCCESS(f'Queued {count} assets.'))
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from datetime import timedelta
|
|
2
|
+
from django.conf import settings
|
|
3
|
+
from django.core.files.storage import default_storage
|
|
4
|
+
from django.core.management.base import BaseCommand
|
|
5
|
+
from django.utils import timezone
|
|
6
|
+
from media_engine.models import MediaVariant
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Command(BaseCommand):
|
|
10
|
+
help = 'Delete old derivative versions after a retention period.'
|
|
11
|
+
|
|
12
|
+
def add_arguments(self, parser):
|
|
13
|
+
parser.add_argument('--days', type=int, default=30)
|
|
14
|
+
parser.add_argument('--confirm', action='store_true')
|
|
15
|
+
|
|
16
|
+
def handle(self, *args, **options):
|
|
17
|
+
version = getattr(settings, 'MEDIA_ENGINE_PIPELINE_VERSION', 1)
|
|
18
|
+
threshold = timezone.now() - timedelta(days=options['days'])
|
|
19
|
+
qs = MediaVariant.objects.filter(processor_version__lt=version, updated_at__lt=threshold)
|
|
20
|
+
self.stdout.write(f'Candidates: {qs.count()}')
|
|
21
|
+
if not options['confirm']:
|
|
22
|
+
self.stdout.write('Dry run. Re-run with --confirm to delete.')
|
|
23
|
+
return
|
|
24
|
+
removed = 0
|
|
25
|
+
for variant in qs.iterator():
|
|
26
|
+
if variant.file and default_storage.exists(variant.file.name):
|
|
27
|
+
default_storage.delete(variant.file.name)
|
|
28
|
+
variant.delete()
|
|
29
|
+
removed += 1
|
|
30
|
+
self.stdout.write(self.style.SUCCESS(f'Deleted {removed} old variants.'))
|