pixelcue 0.8.7__py3-none-any.whl

pixelcue/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.8.7"

pixelcue/analysis_store.py

@@ -0,0 +1,232 @@
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+import threading
+import time
+from pathlib import Path
+from typing import Any
+
+from platformdirs import user_data_dir
+
+from .logging_utils import log_event, log_exception
+
+
+SCHEMA_VERSION = 1
+
+
+def default_analysis_db_path() -> Path:
+    override = os.environ.get("PIXELCUE_ANALYSIS_DB", "").strip()
+    if override:
+        return Path(override).expanduser()
+    return Path(user_data_dir("PixelCue", "DanceFlow")) / "analysis-cache.sqlite3"
+
+
+def _jsonable(value: Any) -> Any:
+    if value is None or isinstance(value, (str, int, float, bool)):
+        return value
+    if isinstance(value, Path):
+        return str(value)
+    if isinstance(value, dict):
+        return {str(k): _jsonable(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple, set)):
+        return [_jsonable(v) for v in value]
+    item = getattr(value, "item", None)
+    if callable(item):
+        try:
+            return _jsonable(item())
+        except Exception:
+            pass
+    return str(value)
+
+
+def normalized_path(path: str | Path) -> str:
+    return os.path.normcase(os.path.abspath(os.fspath(path)))
+
+
+def file_signature(path: str | Path) -> tuple[str, int, int]:
+    p = Path(path)
+    st = p.stat()
+    return normalized_path(p), int(st.st_size), int(st.st_mtime_ns)
+
+
+class AnalysisStore:
+    """Permanent stage-oriented analysis cache backed by SQLite.
+
+    Results are immutable by file signature: if a path changes size or mtime, a
+    new row is written while the old result remains as historical analysis.
+    """
+
+    def __init__(self, db_path: str | Path | None = None) -> None:
+        self.db_path = Path(db_path or default_analysis_db_path()).expanduser()
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.RLock()
+        self._conn = sqlite3.connect(
+            self.db_path,
+            timeout=30.0,
+            check_same_thread=False,
+        )
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._conn.execute("PRAGMA synchronous=NORMAL")
+        self._conn.execute("PRAGMA busy_timeout=30000")
+        self._create_schema()
+        log_event(
+            "analysis-store",
+            "ready",
+            "persistent analysis cache opened",
+            db_path=str(self.db_path),
+        )
+
+    def _create_schema(self) -> None:
+        with self._lock, self._conn:
+            self._conn.executescript(
+                """
+                CREATE TABLE IF NOT EXISTS analysis_results (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    path TEXT NOT NULL,
+                    size_bytes INTEGER NOT NULL,
+                    mtime_ns INTEGER NOT NULL,
+                    kind TEXT NOT NULL,
+                    engine TEXT NOT NULL,
+                    payload_json TEXT NOT NULL,
+                    created_at REAL NOT NULL,
+                    updated_at REAL NOT NULL,
+                    UNIQUE(path, size_bytes, mtime_ns, kind, engine)
+                );
+                CREATE INDEX IF NOT EXISTS idx_analysis_lookup
+                    ON analysis_results(path, kind, engine, size_bytes, mtime_ns);
+                CREATE TABLE IF NOT EXISTS metadata (
+                    key TEXT PRIMARY KEY,
+                    value TEXT NOT NULL
+                );
+                """
+            )
+            self._conn.execute(
+                "INSERT OR REPLACE INTO metadata(key, value) VALUES('schema_version', ?)",
+                (str(SCHEMA_VERSION),),
+            )
+
+    def load(
+        self,
+        path: str | Path,
+        *,
+        kind: str,
+        engine: str,
+    ) -> dict[str, Any] | None:
+        try:
+            normalized, size_bytes, mtime_ns = file_signature(path)
+            with self._lock:
+                row = self._conn.execute(
+                    """
+                    SELECT payload_json
+                    FROM analysis_results
+                    WHERE path=? AND size_bytes=? AND mtime_ns=?
+                      AND kind=? AND engine=?
+                    ORDER BY updated_at DESC
+                    LIMIT 1
+                    """,
+                    (normalized, size_bytes, mtime_ns, kind, engine),
+                ).fetchone()
+            if row is None:
+                return None
+            payload = json.loads(row[0])
+            log_event(
+                "analysis-store",
+                "cache-hit",
+                "restored persisted analysis",
+                path=normalized,
+                kind=kind,
+                engine=engine,
+            )
+            return payload
+        except FileNotFoundError:
+            return None
+        except Exception as exc:
+            log_exception(
+                "analysis-store",
+                "load-failure",
+                exc,
+                path=str(path),
+                kind=kind,
+                engine=engine,
+            )
+            return None
+
+    def save(
+        self,
+        path: str | Path,
+        *,
+        kind: str,
+        engine: str,
+        payload: dict[str, Any],
+    ) -> bool:
+        try:
+            normalized, size_bytes, mtime_ns = file_signature(path)
+            encoded = json.dumps(
+                _jsonable(payload),
+                ensure_ascii=False,
+                separators=(",", ":"),
+                sort_keys=True,
+            )
+            now = time.time()
+            with self._lock, self._conn:
+                self._conn.execute(
+                    """
+                    INSERT INTO analysis_results(
+                        path, size_bytes, mtime_ns, kind, engine,
+                        payload_json, created_at, updated_at
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                    ON CONFLICT(path, size_bytes, mtime_ns, kind, engine)
+                    DO UPDATE SET payload_json=excluded.payload_json,
+                                  updated_at=excluded.updated_at
+                    """,
+                    (
+                        normalized,
+                        size_bytes,
+                        mtime_ns,
+                        kind,
+                        engine,
+                        encoded,
+                        now,
+                        now,
+                    ),
+                )
+            log_event(
+                "analysis-store",
+                "cache-save",
+                "persisted analysis result",
+                path=normalized,
+                kind=kind,
+                engine=engine,
+            )
+            return True
+        except FileNotFoundError:
+            return False
+        except Exception as exc:
+            log_exception(
+                "analysis-store",
+                "save-failure",
+                exc,
+                path=str(path),
+                kind=kind,
+                engine=engine,
+            )
+            return False
+
+    def count(self) -> int:
+        with self._lock:
+            row = self._conn.execute("SELECT COUNT(*) FROM analysis_results").fetchone()
+        return int(row[0] if row else 0)
+
+    def close(self) -> None:
+        with self._lock:
+            try:
+                self._conn.close()
+            finally:
+                log_event(
+                    "analysis-store",
+                    "finish",
+                    "persistent analysis cache closed",
+                    db_path=str(self.db_path),
+                )

pixelcue/archives.py

@@ -0,0 +1,319 @@
+from __future__ import annotations
+
+import random
+import shutil
+import tarfile
+import tempfile
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import BinaryIO, Callable
+
+from .media import is_image, is_video
+from .metadata import MagicDetector
+
+ARCHIVE_SUFFIXES = (
+    ".zip", ".7z", ".rar", ".tar", ".tgz", ".tbz2", ".txz",
+    ".tar.gz", ".tar.bz2", ".tar.xz",
+)
+
+ARCHIVE_MIMES = {
+    "application/zip",
+    "application/x-7z-compressed",
+    "application/vnd.rar",
+    "application/x-rar",
+    "application/x-rar-compressed",
+    "application/x-tar",
+    "application/gzip",
+    "application/x-gzip",
+    "application/x-bzip2",
+    "application/x-xz",
+}
+
+PROBE_BYTES = 128 * 1024
+
+
+def is_archive(path: Path, mime: str = "") -> bool:
+    """Content MIME wins completely whenever it is available."""
+    mime = (mime or "").strip().lower()
+    if mime:
+        return mime in ARCHIVE_MIMES
+
+    # Suffix fallback exists only for callers that genuinely have no MIME result.
+    name = path.name.lower()
+    return any(name.endswith(s) for s in ARCHIVE_SUFFIXES)
+
+
+def _archive_kind(path: Path, mime: str = "") -> str:
+    mime = (mime or "").strip().lower()
+    if mime == "application/zip":
+        return "zip"
+    if mime == "application/x-7z-compressed":
+        return "7z"
+    if mime in {
+        "application/vnd.rar",
+        "application/x-rar",
+        "application/x-rar-compressed",
+    }:
+        return "rar"
+    if mime in {
+        "application/x-tar",
+        "application/gzip",
+        "application/x-gzip",
+        "application/x-bzip2",
+        "application/x-xz",
+    }:
+        return "tar"
+
+    if mime:
+        return "unknown"
+
+    # Only callers with no content MIME may fall back to the suffix.
+    lower = path.name.lower()
+    if lower.endswith(".zip"):
+        return "zip"
+    if lower.endswith(".7z"):
+        return "7z"
+    if lower.endswith(".rar"):
+        return "rar"
+    if lower.endswith((
+        ".tar", ".tgz", ".tbz2", ".txz",
+        ".tar.gz", ".tar.bz2", ".tar.xz",
+    )):
+        return "tar"
+    return "unknown"
+
+
+def _safe_member_name(name: str) -> bool:
+    pp = PurePosixPath(name.replace("\\", "/"))
+    return not pp.is_absolute() and ".." not in pp.parts
+
+
+def _is_media_bytes(detector: MagicDetector, data: bytes) -> tuple[bool, str]:
+    """Return whether member bytes are image/video according to libmagic."""
+    mime, _desc = detector.identify_buffer(data)
+    # No filename is passed here: extensions have zero influence.
+    return is_image(Path("member"), mime) or is_video(Path("member"), mime), mime
+
+
+@dataclass
+class SampledArchive:
+    tempdir: tempfile.TemporaryDirectory
+    paths: list[Path]
+    member_names: list[str]
+    total_media_members: int
+    total_members: int
+
+    def cleanup(self) -> None:
+        self.tempdir.cleanup()
+
+
+def _safe_output(td: Path, idx: int, member_name: str) -> Path:
+    # Keep the original suffix only so downstream tools have a convenient temp
+    # filename. It has already passed content-based MIME classification.
+    suffix = Path(member_name).suffix.lower()
+    return td / f"member-{idx:02d}{suffix}"
+
+
+def _choose(items: list, sample_size: int) -> list:
+    if len(items) <= sample_size:
+        return list(items)
+    return random.SystemRandom().sample(items, sample_size)
+
+
+def sample_archive(
+    path: Path,
+    sample_size: int = 5,
+    mime: str = "",
+    detector: MagicDetector | None = None,
+) -> SampledArchive:
+    """Sample actual media members from an archive using content MIME.
+
+    Archive type is selected from libmagic MIME when supplied. Member filenames
+    are never used to decide whether an entry is image/video.
+    """
+    detector = detector or MagicDetector()
+    archive_kind = _archive_kind(path, mime)
+
+    if archive_kind == "unknown":
+        raise ValueError(
+            f"Unsupported archive content type: {mime or 'unknown'} ({path.name})"
+        )
+
+    td_obj = tempfile.TemporaryDirectory(prefix="pixelcue-archive-")
+    td = Path(td_obj.name)
+
+    extracted: list[Path] = []
+    sampled_names: list[str] = []
+    total_members = 0
+    total_media = 0
+
+    try:
+        if archive_kind == "zip":
+            with zipfile.ZipFile(path) as zf:
+                infos = [
+                    i for i in zf.infolist()
+                    if not i.is_dir() and _safe_member_name(i.filename)
+                ]
+                total_members = len(infos)
+
+                media_infos = []
+                for info in infos:
+                    try:
+                        with zf.open(info) as src:
+                            header = src.read(PROBE_BYTES)
+                        ok, _mime = _is_media_bytes(detector, header)
+                        if ok:
+                            media_infos.append(info)
+                    except Exception:
+                        continue
+
+                total_media = len(media_infos)
+                for idx, info in enumerate(_choose(media_infos, sample_size)):
+                    out = _safe_output(td, idx, info.filename)
+                    with zf.open(info) as src, out.open("wb") as dst:
+                        shutil.copyfileobj(src, dst)
+                    extracted.append(out)
+                    sampled_names.append(info.filename)
+
+        elif archive_kind == "tar":
+            with tarfile.open(path, mode="r:*") as tf:
+                infos = [
+                    i for i in tf.getmembers()
+                    if i.isfile() and _safe_member_name(i.name)
+                ]
+                total_members = len(infos)
+
+                media_infos = []
+                for info in infos:
+                    try:
+                        src = tf.extractfile(info)
+                        if src is None:
+                            continue
+                        with src:
+                            header = src.read(PROBE_BYTES)
+                        ok, _mime = _is_media_bytes(detector, header)
+                        if ok:
+                            media_infos.append(info)
+                    except Exception:
+                        continue
+
+                total_media = len(media_infos)
+                for idx, info in enumerate(_choose(media_infos, sample_size)):
+                    src = tf.extractfile(info)
+                    if src is None:
+                        continue
+                    out = _safe_output(td, idx, info.name)
+                    with src, out.open("wb") as dst:
+                        shutil.copyfileobj(src, dst)
+                    extracted.append(out)
+                    sampled_names.append(info.name)
+
+        elif archive_kind == "rar":
+            import rarfile
+
+            with rarfile.RarFile(path) as rf:
+                infos = [
+                    i for i in rf.infolist()
+                    if not i.isdir() and _safe_member_name(i.filename)
+                ]
+                total_members = len(infos)
+
+                media_infos = []
+                for info in infos:
+                    try:
+                        with rf.open(info) as src:
+                            header = src.read(PROBE_BYTES)
+                        ok, _mime = _is_media_bytes(detector, header)
+                        if ok:
+                            media_infos.append(info)
+                    except Exception:
+                        continue
+
+                total_media = len(media_infos)
+                for idx, info in enumerate(_choose(media_infos, sample_size)):
+                    out = _safe_output(td, idx, info.filename)
+                    with rf.open(info) as src, out.open("wb") as dst:
+                        shutil.copyfileobj(src, dst)
+                    extracted.append(out)
+                    sampled_names.append(info.filename)
+
+        elif archive_kind == "7z":
+            # py7zr does not expose a stable streaming member API across the
+            # supported versions, so probe members one at a time into a private
+            # temp directory. Crucially, the decision is still based on libmagic
+            # bytes, never the member suffix.
+            import py7zr
+
+            with py7zr.SevenZipFile(path, mode="r") as zf:
+                names = [
+                    n for n in zf.getnames()
+                    if _safe_member_name(n) and not n.endswith("/")
+                ]
+            total_members = len(names)
+
+            media_names: list[str] = []
+            probe_root = td / "_probe"
+            probe_root.mkdir()
+
+            for probe_idx, name in enumerate(names):
+                one = probe_root / f"p{probe_idx:08d}"
+                one.mkdir()
+                try:
+                    # Reopen for each target: some py7zr versions cannot perform
+                    # repeated extract() calls on one exhausted archive object.
+                    with py7zr.SevenZipFile(path, mode="r") as zf:
+                        zf.extract(path=one, targets=[name])
+
+                    candidate = (one / name).resolve()
+                    if (
+                        one.resolve() not in candidate.parents
+                        or not candidate.is_file()
+                    ):
+                        continue
+
+                    with candidate.open("rb") as src:
+                        header = src.read(PROBE_BYTES)
+                    ok, _mime = _is_media_bytes(detector, header)
+                    if ok:
+                        media_names.append(name)
+                except Exception:
+                    continue
+                finally:
+                    shutil.rmtree(one, ignore_errors=True)
+
+            total_media = len(media_names)
+            chosen = _choose(media_names, sample_size)
+            for idx, name in enumerate(chosen):
+                one = probe_root / f"chosen-{idx:04d}"
+                one.mkdir()
+                try:
+                    with py7zr.SevenZipFile(path, mode="r") as zf:
+                        zf.extract(path=one, targets=[name])
+                    candidate = (one / name).resolve()
+                    if (
+                        one.resolve() not in candidate.parents
+                        or not candidate.is_file()
+                    ):
+                        continue
+
+                    out = _safe_output(td, idx, name)
+                    shutil.copy2(candidate, out)
+                    extracted.append(out)
+                    sampled_names.append(name)
+                finally:
+                    shutil.rmtree(one, ignore_errors=True)
+
+            shutil.rmtree(probe_root, ignore_errors=True)
+
+        return SampledArchive(
+            td_obj,
+            extracted,
+            sampled_names,
+            total_media,
+            total_members,
+        )
+
+    except Exception:
+        td_obj.cleanup()
+        raise

pixelcue/cli.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from .model import TAGGER_PROFILES, DEFAULT_TAGGER_PROFILE
+
+
+def model_profile_from_argv(argv: list[str]) -> str:
+    """Read --model ID / --model=ID, falling back to configured default."""
+    for index, arg in enumerate(argv[1:], start=1):
+        if arg.startswith("--model="):
+            candidate = arg.split("=", 1)[1].strip()
+            return candidate if candidate in TAGGER_PROFILES else DEFAULT_TAGGER_PROFILE
+        if arg == "--model" and index + 1 < len(argv):
+            candidate = argv[index + 1].strip()
+            return candidate if candidate in TAGGER_PROFILES else DEFAULT_TAGGER_PROFILE
+    return DEFAULT_TAGGER_PROFILE
+
+
+def startup_path_from_argv(argv: list[str]) -> str | None:
+    """Return the first non-option command-line argument that is a valid path."""
+    skip_next = False
+    for index, arg in enumerate(argv[1:], start=1):
+        if skip_next:
+            skip_next = False
+            continue
+        if arg == "--model":
+            skip_next = True
+            continue
+        if arg.startswith("--model=") or arg.startswith("-"):
+            continue
+
+        candidate = Path(os.path.expandvars(os.path.expanduser(arg)))
+        if os.path.lexists(candidate):
+            try:
+                return str(candidate.absolute())
+            except OSError:
+                return str(candidate)
+    return None

pixelcue/cluster_process.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+from typing import Any
+
+from .face import (
+    FaceResult,
+    cluster_embeddings,
+    CLUSTER_COSINE_DISTANCE,
+    CLUSTER_MERGE_MARGIN,
+)
+from .logging_utils import log_event, log_exception
+
+
+def face_cluster_process_main(request_queue, result_queue) -> None:
+    """Persistent clustering subprocess; state is incrementally updated over IPC."""
+    worker = "face-cluster-process"
+    embeddings: dict[str, list[float]] = {}
+    log_event(worker, "start", "clustering subprocess started")
+    finish_reason = "normal stop"
+    success = True
+    try:
+        while True:
+            message = request_queue.get()
+            command = message.get("command")
+            if command == "stop":
+                finish_reason = str(message.get("reason") or "stop requested")
+                break
+            if command != "cluster":
+                continue
+
+            job_id = int(message.get("job_id", 0))
+            items = list(message.get("items") or [])
+            threshold = float(message.get("threshold", CLUSTER_COSINE_DISTANCE))
+            log_event(
+                worker,
+                "job-start",
+                "clustering embeddings",
+                job_id=job_id,
+                new_embeddings=len(items),
+                known_embeddings=len(embeddings),
+                assignment_threshold=threshold,
+                merge_threshold=min(
+                    1.0,
+                    threshold + max(0.0, CLUSTER_MERGE_MARGIN),
+                ),
+            )
+            try:
+                for item in items:
+                    path = str(item["path"])
+                    embeddings[path] = [float(x) for x in item["embedding"]]
+
+                results = [
+                    FaceResult(path=path, embedding=embedding, attributes={})
+                    for path, embedding in embeddings.items()
+                ]
+                clusters = cluster_embeddings(results, threshold=threshold)
+                payload = [
+                    {
+                        "cluster_id": int(cluster["cluster_id"]),
+                        "centroid": [float(x) for x in cluster["centroid"]],
+                        "representative_path": str(cluster["representative_path"]),
+                        "paths": list(cluster["paths"]),
+                        "size": len(cluster["paths"]),
+                    }
+                    for cluster in clusters
+                ]
+                result_queue.put({"job_id": job_id, "ok": True, "clusters": payload})
+                log_event(
+                    worker,
+                    "job-finish",
+                    "clustering succeeded",
+                    job_id=job_id,
+                    clusters=len(payload),
+                    embeddings=len(embeddings),
+                )
+            except BaseException as exc:
+                log_exception(worker, "job-failure", exc, job_id=job_id)
+                result_queue.put(
+                    {
+                        "job_id": job_id,
+                        "ok": False,
+                        "error": f"{type(exc).__name__}: {exc}",
+                    }
+                )
+    except BaseException as exc:
+        success = False
+        finish_reason = f"{type(exc).__name__}: {exc}"
+        log_exception(worker, "failure", exc)
+    finally:
+        log_event(
+            worker,
+            "finish",
+            finish_reason,
+            level="INFO" if success else "ERROR",
+            success=success,
+        )