classifyre-cli 0.6.11__tar.gz → 0.6.12__tar.gz

classifyre_cli-0.6.12/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 324 packages in 342ms
+Checked 51 packages in 4ms

{classifyre_cli-0.6.11 → classifyre_cli-0.6.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.6.11
+Version: 0.6.12
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.6.11 → classifyre_cli-0.6.12}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.6.11",
+  "version": "0.6.12",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.6.11 → classifyre_cli-0.6.12}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.6.11"
+version = "0.6.12"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 # Upper bound is load-bearing: onnxruntime publishes no cp315 wheels (notably

{classifyre_cli-0.6.11 → classifyre_cli-0.6.12}/src/main.py

@@ -553,6 +553,24 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                         await augmentation.startup()
 
                     scan_cache = ScanCache(recipe, source, payload_windows=payload_windows)
+
+                    # What the workspace embeds besides text follows its
+                    # embedding model, so the API says and the run asks once.
+                    # The pipeline prepares that media; the scan cache uses the
+                    # same plan to know which unchanged assets still owe theirs.
+                    if hasattr(sink, "fetch_embedding_plan"):
+                        from .pipeline.media_artifact import EmbeddingPlan
+
+                        embedding_plan = EmbeddingPlan.from_payload(
+                            await sink.fetch_embedding_plan()
+                        )
+                        if embedding_plan.collects_media:
+                            logger.info(
+                                "Preparing media for embedding: %s",
+                                ", ".join(sorted(embedding_plan.media)),
+                            )
+                        pipeline.set_embedding_plan(embedding_plan)
+                        scan_cache.set_embedding_plan(embedding_plan)
                     if scan_cache.enabled:
                         logger.info(
                             "Scan cache active (verify=%s): unchanged assets and"
@@ -675,7 +693,13 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                                     source.evict_asset_cache(asset_hash)
                                     return
 
-                                if plan.mode == "partial":
+                                if plan.media_only:
+                                    logger.info(
+                                        "Cached %s, but %s",
+                                        asset.name,
+                                        plan.reason,
+                                    )
+                                elif plan.mode == "partial":
                                     logger.info(
                                         "Partially cached %s: running %s",
                                         asset.name,
@@ -761,6 +785,13 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                                 f_total, f_by_sev, f_by_det = _compute_findings_counts(
                                     result.findings or []
                                 )
+                                if plan.media_only and plan.prior is not None:
+                                    # Read again only to send its picture or
+                                    # recording: no detector ran, so the counts
+                                    # of its last real scan are still its counts.
+                                    f_total = plan.prior.findings_total or 0
+                                    f_by_sev = plan.prior.findings_by_severity or {}
+                                    f_by_det = plan.prior.findings_by_detector or {}
                                 payload = _asset_to_payload(result)
 
                                 content_hash = plan.content_hash
@@ -785,6 +816,7 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                                     findings_total=f_total,
                                     findings_by_severity=f_by_sev,
                                     findings_by_detector=f_by_det,
+                                    media_state=scan_cache.media_state_for(asset),
                                 )
                                 if state is not None:
                                     payload["scan_cache"] = state

{classifyre_cli-0.6.11 → classifyre_cli-0.6.12}/src/outputs/rest.py

@@ -347,13 +347,29 @@ class RestOutputSink:
             )
 
     async def emit_text_chunks(self, asset_hash: str, artifact: TextArtifact) -> None:
-        """Send extracted text provenance; the API embeds it asynchronously."""
+        """Send extracted text provenance; the API embeds it asynchronously.
+
+        Prepared pictures and clips ride in the same request, because the API
+        replaces everything it holds for the asset in one step.
+        """
         source_id = self._require_source_id()
-        self._request_json(
-            "POST",
-            f"/sources/{source_id}/embeddings/chunks",
-            {"assetHash": asset_hash, "chunks": artifact.chunks},
-        )
+        body: dict[str, Any] = {"assetHash": asset_hash, "chunks": artifact.chunks}
+        if artifact.media:
+            body["media"] = artifact.media
+        self._request_json("POST", f"/sources/{source_id}/embeddings/chunks", body)
+
+    async def fetch_embedding_plan(self) -> dict[str, Any]:
+        """What this workspace wants prepared for embedding besides text.
+
+        Asked once per run. An API that predates the endpoint answers 404, and
+        any failure here means the same thing: prepare no media, and embed the
+        text as every scan before this did.
+        """
+        try:
+            return self._request_json("GET", "/embeddings/collection")
+        except Exception as exc:
+            logger.info("No embedding collection plan (%s); embedding text only", exc)
+            return {}
 
     def _split_by_size(self, assets: list[dict[str, Any]]) -> list[list[dict[str, Any]]]:
         """Split assets into chunks that each stay under _MAX_BATCH_BYTES."""

{classifyre_cli-0.6.11 → classifyre_cli-0.6.12}/src/pipeline/detector_pipeline.py

@@ -29,6 +29,8 @@ from ..utils.file_parser import TextExtractionCoverageError, resolve_mime_type
 from ..utils.redaction import Redactor, redact_generic, short_id_for_log
 from .content_provider import ContentProvider
 from .detector_breaker import DetectorBreaker
+from .finding_context import attach_context
+from .media_artifact import EmbeddingPlan, prepare_media
 from .text_artifact import TextArtifact
 from .worker_pool import DetectorWorkerPool, is_io_bound_detector
 
@@ -151,12 +153,60 @@ class DetectorPipeline:
         # disabled here is disabled for this run and nothing longer.
         self._breaker = DetectorBreaker()
         self._text_artifacts: dict[str, TextArtifact] = {}
+        # What the workspace wants embedded besides text. Empty until the run
+        # asks the API, and empty for good on a text-only embedding model.
+        self._embedding_plan = EmbeddingPlan()
         self._text_extraction_errors: dict[str, list[str]] = {}
         self._text_extraction_statuses: dict[str, TextExtractionStatus] = {}
 
     def take_text_artifact(self, asset_hash: str) -> TextArtifact | None:
         return self._text_artifacts.pop(asset_hash, None)
 
+    def set_embedding_plan(self, plan: EmbeddingPlan) -> None:
+        """Say which media, if any, this run prepares for embedding."""
+        self._embedding_plan = plan
+
+    async def _collect_media(self, asset: SingleAssetScanResults, artifact: TextArtifact) -> None:
+        """Attach the asset's own picture or recording, prepared for embedding.
+
+        Only for an asset that IS a picture or a recording, and only when the
+        workspace's model embeds that kind of content. Best-effort throughout:
+        media adds a way to find the asset, and failing to prepare it leaves
+        the asset exactly as findable as it was before this existed.
+        """
+        spec = self._embedding_plan.spec_for(asset.asset_type)
+        if spec is None:
+            return
+        for candidate in (asset.external_url, asset.hash):
+            asset_id = str(candidate or "").strip()
+            if not asset_id:
+                continue
+            try:
+                fetched = await self.content_provider.fetch_bytes(asset_id)
+            except Exception as exc:
+                logger.warning(
+                    "Could not read %s to prepare its %s for embedding: %s",
+                    asset.name,
+                    spec.modality,
+                    redact_generic(self._redactor.redact(str(exc))),
+                )
+                return
+            if not fetched or not fetched[0]:
+                continue
+            # Decoding and re-encoding is CPU work; keep it off the event loop
+            # the rest of the scan is running on.
+            items = await asyncio.to_thread(prepare_media, fetched[0], spec)
+            for mime_type, data in items:
+                artifact.add_media(spec.modality, mime_type, data)
+            if items:
+                logger.info(
+                    "Prepared %d %s item(s) of %s for embedding",
+                    len(items),
+                    spec.modality,
+                    asset.name,
+                )
+            return
+
     def _register_detector_info(self, detector: BaseDetector, info: _DetectorInfo) -> None:
         self._detector_info[id(detector)] = info
 
@@ -401,6 +451,9 @@ class DetectorPipeline:
         scan_warnings.extend(code_warnings)
         detector_types_run = self._merge_detector_types(detector_types_run, code_types_run)
 
+        if not records_only:
+            await self._collect_media(asset, artifact)
+
         scan_duration = int((datetime.now(UTC) - scan_started).total_seconds() * 1000)
 
         asset.findings = findings
@@ -1011,6 +1064,8 @@ class DetectorPipeline:
                             }
                         )
                         detector_findings.append(finding_with_meta)
+            if isinstance(content, str):
+                detector_findings = attach_context(detector_findings, content)
 
             tripped = self._breaker.record(
                 breaker_key,

classifyre_cli-0.6.12/src/pipeline/finding_context.py

@@ -0,0 +1,99 @@
+"""The text around a match.
+
+A finding is a value and where it was found. What the value means is in the
+words either side of it: "IBAN DE89..." in a payment reminder and the same
+number in a list of test fixtures are two different pieces of evidence, and
+nothing downstream can tell them apart from the value alone. The API ranks a
+finding by its surrounding text, embeds the match together with it, and shows
+it to the reviewer next to the match.
+
+No detector returns that text: each answers with the value and, at most, a
+position. So it is cut here, once, from the content the detectors were given,
+for every finding that does not already carry it.
+"""
+
+from __future__ import annotations
+
+import re
+
+from ..models.generated_single_asset_scan_results import DetectionResult
+
+# Characters kept on each side. The ranking treats 320 characters of evidence
+# as a full context, so a match of ordinary length with both sides reaches it.
+CONTEXT_CHARS = 160
+
+# A match longer than this is already a passage (an LLM detector quoting a
+# paragraph); text around it would be the next paragraph, not its context.
+MAX_MATCH_CHARS = 600
+
+_CHAR_RANGE = re.compile(r"character range (\d+)-(\d+)")
+
+
+def _locate(content: str, matched: str, finding: DetectionResult, start_at: int) -> int:
+    """Offset of this finding's match in `content`, or -1."""
+    description = getattr(finding.location, "description", None) if finding.location else None
+    stated = _CHAR_RANGE.search(description) if isinstance(description, str) else None
+    if stated:
+        start, end = int(stated.group(1)), int(stated.group(2))
+        if content[start:end] == matched:
+            return start
+    found = content.find(matched, start_at)
+    return found if found >= 0 else content.find(matched)
+
+
+def _before(content: str, start: int) -> str | None:
+    cut = max(0, start - CONTEXT_CHARS)
+    text = content[cut:start]
+    # A window that opens in the middle of a word starts with half of it.
+    if cut > 0 and not content[cut - 1].isspace() and text[:1].strip():
+        parts = text.split(None, 1)
+        text = parts[1] if len(parts) > 1 else ""
+    return " ".join(text.split()) or None
+
+
+def _after(content: str, end: int) -> str | None:
+    cut = min(len(content), end + CONTEXT_CHARS)
+    text = content[end:cut]
+    if cut < len(content) and not content[cut].isspace() and text[-1:].strip():
+        parts = text.rsplit(None, 1)
+        text = parts[0] if len(parts) > 1 else ""
+    return " ".join(text.split()) or None
+
+
+def attach_context(findings: list[DetectionResult], content: str) -> list[DetectionResult]:
+    """`findings` with the text around each match filled in where it was missing.
+
+    A value that occurs several times is matched to its occurrences in order,
+    so two findings of one value do not both get the first one's surroundings.
+    A match that cannot be found in the content (a detector that reports a
+    normalised value, or a verdict rather than a quotation) is left as it is.
+    """
+    if not content:
+        return findings
+    cursor: dict[str, int] = {}
+    out: list[DetectionResult] = []
+    for finding in findings:
+        matched = finding.matched_content
+        if (
+            finding.context_before
+            or finding.context_after
+            or not matched
+            or len(matched) > MAX_MATCH_CHARS
+        ):
+            out.append(finding)
+            continue
+        start = _locate(content, matched, finding, cursor.get(matched, 0))
+        if start < 0:
+            out.append(finding)
+            continue
+        end = start + len(matched)
+        cursor[matched] = end
+        out.append(
+            finding.model_copy(
+                update={
+                    "context_before": _before(content, start),
+                    "context_after": _after(content, end),
+                }
+            )
+        )
+    return out

classifyre_cli-0.6.12/src/pipeline/media_artifact.py

@@ -0,0 +1,262 @@
+"""Pictures and sound prepared for embedding as what they are.
+
+Text read out of an asset (OCR, transcription) is embedded as text. A model
+that also embeds pictures or sound can find an asset by what it shows or how it
+sounds, which is the only way to find a photograph with no writing in it. This
+module prepares that media the way the API asks for it: one small JPEG per
+picture, mono 16 kHz WAV clips per recording. The API stores the prepared copy,
+because a later rebuild re-embeds from it and the scan will not send it again.
+
+What to prepare is the API's decision (``GET /embeddings/collection``), since
+it follows the workspace's embedding model. With nothing asked for, nothing
+here runs.
+"""
+
+from __future__ import annotations
+
+import io
+import logging
+import wave
+from dataclasses import dataclass, field
+from typing import Any
+
+from ..detectors.dependencies import MissingDependencyError, require_module
+
+logger = logging.getLogger(__name__)
+
+# uv groups that carry Pillow, and PyAV (which arrives with faster-whisper). A
+# picture or a recording has normally been through OCR or transcription by the
+# time it gets here, so these are already installed; a scan that somehow is
+# not installs the first one on demand.
+_IMAGE_UV_GROUPS = ["file-processing", "content", "llm", "custom"]
+_AUDIO_UV_GROUPS = ["transcription"]
+
+# Scan-cache key under which an asset's state records that its media was
+# collected. Lives beside the detector fingerprints because it answers the same
+# question for the same reason: was this work done for this asset under the
+# rules in force now. The API names the key with the plan (it is the one that
+# takes the record back when the media is deleted); this is the same name, for
+# an API that does not send one.
+MEDIA_CACHE_KEY_PREFIX = "EMBED_MEDIA::"
+
+# Shortest clip worth sending: of a whole recording, and of what is left over
+# after the full clips of a longer one.
+_MIN_CLIP_SECONDS = 1
+_MIN_TAIL_SECONDS = 5
+
+# Asset types whose own bytes are a picture or a recording.
+_MODALITY_BY_ASSET_TYPE = {"IMAGE": "image", "AUDIO": "audio"}
+
+
+@dataclass(frozen=True)
+class MediaSpec:
+    """How one kind of media is prepared."""
+
+    modality: str
+    mime_type: str
+    max_items: int = 1
+    max_edge: int = 768
+    quality: int = 85
+    sample_rate: int = 16_000
+    clip_seconds: int = 30
+    cache_key: str = ""
+    # Changes when the way this kind of media is prepared changes.
+    fingerprint: str = ""
+
+    @classmethod
+    def from_payload(cls, raw: Any) -> MediaSpec | None:
+        if not isinstance(raw, dict):
+            return None
+        modality = str(raw.get("modality") or "").strip().lower()
+        if modality not in {"image", "audio"}:
+            return None
+
+        def number(key: str, default: int, minimum: int) -> int:
+            value = raw.get(key)
+            return max(minimum, int(value)) if isinstance(value, int | float) else default
+
+        return cls(
+            modality=modality,
+            mime_type=str(
+                raw.get("mimeType") or ("image/jpeg" if modality == "image" else "audio/wav")
+            ),
+            max_items=number("maxItems", 1, 1),
+            max_edge=number("maxEdge", 768, 32),
+            quality=min(100, number("quality", 85, 1)),
+            sample_rate=number("sampleRate", 16_000, 8_000),
+            clip_seconds=number("clipSeconds", 30, 1),
+            cache_key=str(raw.get("cacheKey") or f"{MEDIA_CACHE_KEY_PREFIX}{modality}"),
+            fingerprint=str(raw.get("fingerprint") or ""),
+        )
+
+
+@dataclass(frozen=True)
+class EmbeddingPlan:
+    """What this workspace wants sent for embedding."""
+
+    media: dict[str, MediaSpec] = field(default_factory=dict)
+
+    @classmethod
+    def from_payload(cls, raw: Any) -> EmbeddingPlan:
+        if not isinstance(raw, dict):
+            return cls()
+        specs: dict[str, MediaSpec] = {}
+        entries = raw.get("media")
+        for entry in entries if isinstance(entries, list) else []:
+            spec = MediaSpec.from_payload(entry)
+            if spec is not None:
+                specs[spec.modality] = spec
+        return cls(media=specs)
+
+    @property
+    def collects_media(self) -> bool:
+        return bool(self.media)
+
+    def spec_for(self, asset_type: Any) -> MediaSpec | None:
+        """The spec for an asset whose own bytes are media, if it is collected."""
+        name = str(getattr(asset_type, "value", asset_type) or "").upper()
+        modality = _MODALITY_BY_ASSET_TYPE.get(name)
+        return self.media.get(modality) if modality else None
+
+    def cache_entry_for(self, asset_type: Any) -> tuple[str, str] | None:
+        """Scan-cache (key, fingerprint) proving this asset's media was sent.
+
+        Per kind of media, so that starting to embed sound does not make every
+        picture read as not yet sent.
+        """
+        spec = self.spec_for(asset_type)
+        if spec is None:
+            return None
+        key = spec.cache_key or f"{MEDIA_CACHE_KEY_PREFIX}{spec.modality}"
+        return key, spec.fingerprint or "1"
+
+
+def prepare_image(raw: bytes, spec: MediaSpec) -> bytes | None:
+    """Re-encode a picture as an RGB JPEG no larger than ``spec.max_edge``.
+
+    Vision encoders work at a few hundred pixels a side, so a larger copy is
+    only storage. Returns None when the bytes are not a picture Pillow can
+    read, or Pillow is not available; the asset is then embedded by its text
+    alone, as it would have been anyway.
+    """
+    if not raw:
+        return None
+    try:
+        pil = require_module("PIL.Image", "media embedding", _IMAGE_UV_GROUPS)
+        ops = require_module("PIL.ImageOps", "media embedding", _IMAGE_UV_GROUPS)
+    except MissingDependencyError as exc:
+        logger.warning("Cannot prepare a picture for embedding: %s", exc)
+        return None
+    try:
+        # HEIC (every recent phone photograph) is read through a plug-in that
+        # has to be registered; without it Pillow does not know the format.
+        try:
+            import pillow_heif  # type: ignore[import-not-found]
+
+            pillow_heif.register_heif_opener()
+        except Exception:
+            pass
+        with pil.open(io.BytesIO(raw)) as image:
+            # A phone photograph is stored sideways with a tag saying which way
+            # is up; the encoder only sees pixels.
+            upright = ops.exif_transpose(image) or image
+            # Flatten transparency onto white: JPEG has no alpha, and dropping
+            # the channel turns a transparent background black.
+            if upright.mode in {"RGBA", "LA"} or "transparency" in upright.info:
+                rgba = upright.convert("RGBA")
+                canvas = pil.new("RGB", rgba.size, (255, 255, 255))
+                canvas.paste(rgba, mask=rgba.getchannel("A"))
+                rgb = canvas
+            else:
+                rgb = upright.convert("RGB")
+            rgb.thumbnail((spec.max_edge, spec.max_edge), pil.Resampling.LANCZOS)
+            buffer = io.BytesIO()
+            rgb.save(buffer, format="JPEG", quality=spec.quality, optimize=True)
+            return buffer.getvalue()
+    except Exception as exc:
+        logger.warning("Could not prepare a picture for embedding: %s", exc)
+        return None
+
+
+def _pcm_to_wav(pcm: bytes, sample_rate: int) -> bytes:
+    buffer = io.BytesIO()
+    with wave.open(buffer, "wb") as out:
+        out.setnchannels(1)
+        out.setsampwidth(2)
+        out.setframerate(sample_rate)
+        out.writeframes(pcm)
+    return buffer.getvalue()
+
+
+def prepare_audio_clips(raw: bytes, spec: MediaSpec) -> list[bytes]:
+    """Decode a recording to mono 16-bit WAV clips at ``spec.sample_rate``.
+
+    At most ``spec.max_items`` clips of ``spec.clip_seconds`` each, from the
+    start: the opening minutes say what a recording is, and every clip is a
+    megabyte kept. Decoding stops once enough has been read, so a long file is
+    not decoded to its end. Returns [] when the bytes hold no audio PyAV can
+    read, or PyAV is not available.
+    """
+    if not raw:
+        return []
+    try:
+        pyav = require_module("av", "media embedding", _AUDIO_UV_GROUPS)
+    except MissingDependencyError as exc:
+        logger.warning("Cannot prepare a recording for embedding: %s", exc)
+        return []
+
+    clip_bytes = spec.sample_rate * spec.clip_seconds * 2
+    wanted = clip_bytes * spec.max_items
+    pcm = bytearray()
+    container = None
+    try:
+        container = pyav.open(io.BytesIO(raw), metadata_errors="ignore")
+        streams = [stream for stream in container.streams if stream.type == "audio"]
+        if not streams:
+            return []
+        resampler = pyav.audio.resampler.AudioResampler(
+            format="s16", layout="mono", rate=spec.sample_rate
+        )
+
+        def drain(frames: Any) -> None:
+            for frame in frames if isinstance(frames, list) else [frames]:
+                if frame is not None:
+                    pcm.extend(bytes(frame.planes[0])[: frame.samples * 2])
+
+        for frame in container.decode(streams[0]):
+            drain(resampler.resample(frame))
+            if len(pcm) >= wanted:
+                break
+        else:
+            # Only a file read to its end has a resampler tail worth flushing.
+            try:
+                drain(resampler.resample(None))
+            except Exception:
+                pass
+    except Exception as exc:
+        logger.warning("Could not prepare a recording for embedding: %s", exc)
+        return []
+    finally:
+        if container is not None:
+            container.close()
+
+    clips: list[bytes] = []
+    for start in range(0, min(len(pcm), wanted), clip_bytes):
+        chunk = bytes(pcm[start : start + clip_bytes])
+        # A short recording is sent whole: a second of sound can be all there
+        # is. The tail of a long one is different, since the clips before it
+        # already say what the recording is and a few seconds more add a vector
+        # of their own for almost nothing.
+        leftover = bool(clips) and len(chunk) < clip_bytes
+        shortest = _MIN_TAIL_SECONDS if leftover else _MIN_CLIP_SECONDS
+        if len(chunk) >= spec.sample_rate * 2 * shortest:
+            clips.append(_pcm_to_wav(chunk, spec.sample_rate))
+    return clips
+
+
+def prepare_media(raw: bytes, spec: MediaSpec) -> list[tuple[str, bytes]]:
+    """Prepared (mime type, bytes) items for one asset under ``spec``."""
+    if spec.modality == "image":
+        image = prepare_image(raw, spec)
+        return [(spec.mime_type, image)] if image else []
+    return [(spec.mime_type, clip) for clip in prepare_audio_clips(raw, spec)]