classifyre-cli 0.4.52__tar.gz → 0.4.54__tar.gz

classifyre_cli-0.4.54/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 292 packages in 16ms
+Checked 51 packages in 4ms

{classifyre_cli-0.4.52 → classifyre_cli-0.4.54}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.52
+Version: 0.4.54
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.52 → classifyre_cli-0.4.54}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.52",
+  "version": "0.4.54",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.52 → classifyre_cli-0.4.54}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.52"
+version = "0.4.54"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -128,6 +128,7 @@ detectors = [
     { include-group = "regex" },
     { include-group = "llm" },
     { include-group = "transcription" },
+    { include-group = "video" },
 ]
 file-processing = [
     "filetype>=1.2.0",
@@ -145,9 +146,16 @@ transcription = [
     # PyAV (no system ffmpeg required) and runs ctranslate2 for inference.
     "faster-whisper>=1.1.0",
 ]
+video = [
+    # Use the server-safe OpenCV 5 wheel. RapidOCR's desktop OpenCV dependency
+    # is excluded below because both distributions install the same cv2 module.
+    "opencv-python-headless>=5.0.0.93,<6.0.0",
+    "rapidocr-onnxruntime>=1.4.0",
+]
 ocr = [
     { include-group = "file-processing" },
     "docling>=2.94.0",
+    "opencv-python-headless>=5.0.0.93,<6.0.0",
     "rapidocr-onnxruntime>=1.4.0",
     # torchvision is a transitive dep of docling-ibm-models, but [tool.uv.sources]
     # index pins only apply to dependencies declared directly in this project.
@@ -407,6 +415,10 @@ markers = [
 
 [tool.uv]
 package = true
+# RapidOCR is the sole requester of opencv-python and works with the headless
+# distribution declared by our video and OCR groups. Installing both would
+# overwrite the same cv2 namespace and add GUI libraries to CPU-only containers.
+exclude-dependencies = ["opencv-python"]
 # Every platform we ship (desktop bundles + CI) must resolve to versions with
 # usable wheels. Without this, uv happily locks versions that are missing a
 # platform — e.g. deltalake 1.6.1 published no win_amd64 wheel, which broke the

{classifyre_cli-0.4.52 → classifyre_cli-0.4.54}/src/detectors/custom/runners/_base.py

@@ -47,10 +47,10 @@ _IMAGE_INPUT_CONTENT_TYPES = [*_IMAGE_CONTENT_TYPES, "application/pdf"]
 
 logger = logging.getLogger(__name__)
 
-# Cap per-label entity spans embedded in each finding's metadata. Every finding
-# carries a copy of the pipeline result (the API builds extraction rows from it),
-# so an uncapped regex with thousands of matches would blow up the scan payload
-# quadratically.
+# Cap per-label entity spans carried through the dedicated extracted_data field.
+# The API moves this into CustomDetectorExtraction and does not persist it in
+# Finding.metadata. It is still repeated in the wire payload until extraction
+# becomes page-scoped, so the cap prevents quadratic request growth.
 _MAX_EMBEDDED_SPANS_PER_LABEL = 25
 
 
@@ -139,6 +139,7 @@ class BaseRunner(ABC):
         matched_content: str,
         location: Location | None,
         metadata: dict[str, Any],
+        extracted_data: dict[str, Any] | None = None,
     ) -> DetectionResult:
         return DetectionResult(
             detector_type=DetectorType.CUSTOM,
@@ -152,14 +153,19 @@ class BaseRunner(ABC):
             custom_detector_name=self._detector_name,
             detected_at=datetime.now(UTC),
             metadata=metadata,
+            extracted_data=extracted_data,
         )
 
     def _result_to_findings(self, text: str, result: PipelineResult) -> list[DetectionResult]:
         findings: list[DetectionResult] = []
-        runner_type = result.metadata.get("runner", "GLINER2")
+        # Every field on PipelineResult is optional, and runners return a bare
+        # PipelineResult() when a model fails to load — so none of these may be
+        # dereferenced directly. Doing so turned a handled model-load failure
+        # into an AttributeError from deep inside finding construction.
+        runner_type = (result.metadata or {}).get("runner", "GLINER2")
         pipeline_result_dump = _slim_pipeline_result(result)
 
-        for label, spans in result.entities.items():
+        for label, spans in (result.entities or {}).items():
             for span in spans:
                 confidence = float(span.get("confidence", 0.0))
                 value = str(span.get("value", ""))
@@ -181,7 +187,6 @@ class BaseRunner(ABC):
                 meta: dict[str, Any] = {
                     "runner": runner_type,
                     "entity_label": label,
-                    "pipeline_result": pipeline_result_dump,
                 }
                 if "groups" in span:
                     meta["capture_groups"] = span["groups"]
@@ -195,10 +200,11 @@ class BaseRunner(ABC):
                         matched_content=value,
                         location=loc,
                         metadata=meta,
+                        extracted_data=pipeline_result_dump,
                     )
                 )
 
-        for task, outcome in result.classification.items():
+        for task, outcome in (result.classification or {}).items():
             label = str(outcome.get("label", ""))
             confidence = float(outcome.get("confidence", 0.0))
             if not label:
@@ -216,8 +222,8 @@ class BaseRunner(ABC):
                         "runner": runner_type,
                         "task": task,
                         "label": label,
-                        "pipeline_result": pipeline_result_dump,
                     },
+                    extracted_data=pipeline_result_dump,
                 )
             )
 

{classifyre_cli-0.4.52 → classifyre_cli-0.4.54}/src/detectors/custom/runners/_gliner2.py

@@ -80,23 +80,41 @@ class GLiNER2Runner(BaseRunner):
         raw_entities: dict[str, list[dict[str, object]]] = {}
         raw_classification: dict[str, dict[str, object]] = {}
 
-        try:
-            if entity_schema:
+        # Entity extraction and each classification task are isolated. They used
+        # to share one try/except, so a classification failure discarded the
+        # entities already extracted in the same run and returned an empty
+        # result — the whole detector went silent because of one broken task.
+        if entity_schema:
+            try:
                 raw = self._extract_entities(model, text, entity_schema)
                 raw_entities = _normalise_entity_output(raw, text)
+            except Exception as exc:  # pragma: no cover - runtime specific
+                logger.error(
+                    "GLiNER2 entity extraction failed for detector '%s': %s",
+                    self._detector_key,
+                    exc,
+                )
 
-            for task_name, labels in classification_tasks.items():
+        for task_name, labels in classification_tasks.items():
+            try:
                 setfit = self._get_setfit_model(task_name)
                 if setfit is not None:
                     raw_cls: Any = self._run_setfit(setfit, task_name, text)
                 else:
-                    raw_cls = self._classify_chunked(model, text, labels)
+                    raw_cls = self._classify_chunked(model, text, task_name, labels)
                 raw_classification[task_name] = _normalise_classification_output(raw_cls)
+            except Exception as exc:  # pragma: no cover - runtime specific
+                logger.error(
+                    "GLiNER2 classification task '%s' failed for detector '%s': %s",
+                    task_name,
+                    self._detector_key,
+                    exc,
+                )
 
-        except Exception as exc:  # pragma: no cover - runtime specific
-            logger.error("GLiNER2 pipeline failed for detector '%s': %s", self._detector_key, exc)
-            return PipelineResult()
-
+        # No early return for an empty result: "ran and found nothing" is a
+        # legitimate outcome and must still produce a well-formed result with
+        # metadata. Failures are reported by the logs above and by the runner's
+        # detector outcome, not by an anonymous empty return.
         validation = self._schema.validation or PipelineValidationConfig()
         filtered_entities = _apply_entity_validation(raw_entities, validation, self._schema)
         filtered_classification = _apply_classification_validation(raw_classification, validation)
@@ -138,15 +156,46 @@ class GLiNER2Runner(BaseRunner):
                 )
         return model.extract_entities(text, entity_schema, **kwargs)
 
-    def _classify_chunked(self, model: Any, text: str, labels: list[str]) -> dict[str, object]:
+    def _classify_once(self, model: Any, text: str, task_name: str, labels: list[str]) -> Any:
+        """Run one classification task against whichever API the runtime exposes.
+
+        gliner2>=1.3 exposes `classify_text(text, tasks: dict, ...)`, where tasks
+        maps a task name to its labels and the result is keyed by that name.
+        Mirrors the defensive probing in _extract_entities rather than assuming
+        a method exists.
+        """
+        classify_text = getattr(model, "classify_text", None)
+        if classify_text is not None:
+            raw = classify_text(
+                text,
+                {task_name: list(labels)},
+                threshold=0.0,
+                include_confidence=True,
+            )
+            # classify_text returns {task_name: <result>}; unwrap to the result.
+            if isinstance(raw, dict) and task_name in raw:
+                return raw[task_name]
+            return raw
+
+        legacy_classify = getattr(model, "classify", None)
+        if legacy_classify is None:
+            raise AttributeError(
+                f"GLiNER2 model exposes neither 'classify_text' nor 'classify'; "
+                f"cannot run classification task '{task_name}'"
+            )
+        return legacy_classify(text, labels, threshold=0.0)
+
+    def _classify_chunked(
+        self, model: Any, text: str, task_name: str, labels: list[str]
+    ) -> dict[str, object]:
         """Classify text, chunking long inputs and keeping the max-confidence label."""
         chunks = _chunk_text(text, _CLS_CHUNK_SIZE, _CLS_CHUNK_OVERLAP)[:_MAX_CLS_CHUNKS]
-        if len(chunks) == 1:
-            return _normalise_classification_output(model.classify(text, labels, threshold=0.0))
 
         best: dict[str, object] = {}
         for chunk in chunks:
-            outcome = _normalise_classification_output(model.classify(chunk, labels, threshold=0.0))
+            outcome = _normalise_classification_output(
+                self._classify_once(model, chunk, task_name, labels)
+            )
             if outcome and float(outcome.get("confidence", 0.0)) > float(
                 best.get("confidence", -1.0)
             ):
@@ -312,14 +361,31 @@ def _normalise_span(span: Any, text: str) -> dict[str, object] | None:
     return {"value": value, "confidence": round(confidence, 4), "start": start, "end": end}
 
 
+def _confidence_of(item: Any) -> float:
+    """Read a confidence from either key the runtime may use.
+
+    gliner2's formatter emits {"label", "confidence"} for multi-label results.
+    Reading only "score" silently yields 0.0 for every label, which the
+    validation threshold then filters away — classification would appear to run
+    cleanly and produce nothing.
+    """
+    if not isinstance(item, dict):
+        return 0.0
+    value = item.get("confidence", item.get("score", 0.0))
+    try:
+        return float(value)  # type: ignore[arg-type]
+    except (TypeError, ValueError):
+        return 0.0
+
+
 def _normalise_classification_output(raw: Any) -> dict[str, object]:
     if isinstance(raw, dict):
         label = raw.get("label", "")
-        confidence = float(raw.get("confidence", raw.get("score", 0.0)))
+        confidence = _confidence_of(raw)
     elif isinstance(raw, (list, tuple)) and raw:
-        best = max(raw, key=lambda x: x.get("score", 0.0) if isinstance(x, dict) else 0.0)
+        best = max(raw, key=_confidence_of)
         label = best.get("label", "") if isinstance(best, dict) else str(best)
-        confidence = float(best.get("score", 0.0)) if isinstance(best, dict) else 1.0
+        confidence = _confidence_of(best) if isinstance(best, dict) else 1.0
     else:
         return {}
 

{classifyre_cli-0.4.52 → classifyre_cli-0.4.54}/src/detectors/pii/detector.py

@@ -20,6 +20,19 @@ logger = logging.getLogger(__name__)
 _PRESIDIO_LOG_FILTER_INSTALLED = False
 
 
+def _unwrap_int(value: Any) -> int | None:
+    """Unwrap a generated RootModel[int] config field to a plain int.
+
+    Numeric fields with constraints (chunk_size, chunk_overlap, max_length) are
+    generated as RootModel[int] wrappers. A wrapper instance is truthy and is
+    not an int, so `if not value` never short-circuits and any arithmetic or
+    comparison against it raises TypeError. Both failure modes are silent: the
+    detector fails per page while the run still reports success.
+    """
+    unwrapped = getattr(value, "root", value)
+    return unwrapped if isinstance(unwrapped, int) else None
+
+
 class _PresidioNoiseFilter(logging.Filter):
     """Suppresses noisy but harmless Presidio initialization warnings."""
 
@@ -342,7 +355,7 @@ class PIIDetector(BaseDetector):
             logger.warning("spaCy model '%s' not found; using default NLP engine", cfg_model)
             return None
 
-        spacy_max_length: int | None = getattr(self._cfg, "max_length", None)
+        spacy_max_length = _unwrap_int(getattr(self._cfg, "max_length", None))
         if spacy_max_length is not None:
             nlp.max_length = spacy_max_length
             logger.debug("Set spaCy nlp.max_length = %d", spacy_max_length)
@@ -757,6 +770,22 @@ class PIIDetector(BaseDetector):
     def _get_severity_for_entity(self, entity_type: str) -> Severity:
         e = entity_type.upper()
 
+        # An explicit override always wins: a recognizer's severity depends on
+        # the corpus, not on its label. A CREDIT_CARD hit is critical in a
+        # payments export and noise in an OCR'd court transcript.
+        overrides = getattr(self._cfg, "severity_overrides", None) or {}
+        if isinstance(overrides, dict):
+            override = overrides.get(e) or overrides.get(entity_type)
+            if override is not None:
+                try:
+                    return Severity(str(override).lower())
+                except ValueError:
+                    logger.warning(
+                        "Ignoring invalid severity override '%s' for entity %s",
+                        override,
+                        e,
+                    )
+
         # Critical — government IDs, financial account numbers, biometric IDs
         if e in {
             "CREDIT_CARD",
@@ -807,7 +836,12 @@ class PIIDetector(BaseDetector):
         if e in {"PERSON", "LOCATION", "DATE_TIME", "NRP", "URL"}:
             return Severity.medium
 
-        return Severity.high
+        # Unknown entity types — including every custom recognizer and anything
+        # Presidio adds upstream — default to medium rather than high. Severity
+        # is a claim about the label, not about evidence quality, and defaulting
+        # unrecognised labels to HIGH inflated the corpus's severity histogram
+        # without any review having happened.
+        return Severity.medium
 
     # ------------------------------------------------------------------
     # Public API
@@ -827,10 +861,10 @@ class PIIDetector(BaseDetector):
 
     def _chunk_text(self, text: str) -> list[tuple[str, int]]:
         """Return (chunk, offset) pairs. When chunk_size is null returns the full text at offset 0."""
-        chunk_size: int | None = getattr(self._cfg, "chunk_size", None)
+        chunk_size = _unwrap_int(getattr(self._cfg, "chunk_size", None))
         if not chunk_size:
             return [(text, 0)]
-        overlap: int = getattr(self._cfg, "chunk_overlap", None) or 0
+        overlap = _unwrap_int(getattr(self._cfg, "chunk_overlap", None)) or 0
         step = max(1, chunk_size - overlap)
         return [(text[i : i + chunk_size], i) for i in range(0, len(text), step)]
 

{classifyre_cli-0.4.52 → classifyre_cli-0.4.54}/src/models/generated_detectors.py

@@ -460,6 +460,10 @@ class PIIDetectorConfig(DetectorConfig):
         None,
         description='Presidio entity types to detect. When null, all supported entities are enabled. Use PIIEnabledPattern values (e.g. EMAIL_ADDRESS, US_SSN, CREDIT_CARD).',
     )
+    severity_overrides: dict[str, Severity] | None = Field(
+        None,
+        description='Per-entity severity overrides, keyed by Presidio entity type (e.g. {"CREDIT_CARD": "medium"}). Severity describes the label, not evidence quality: a recognizer that fires on OCR artifacts in one corpus may be genuinely critical in another. Overrides win over the built-in defaults.',
+    )
     language: str | None = Field(
         'en', description='BCP-47 language code for NER models (e.g. en, de, es)'
     )

{classifyre_cli-0.4.52 → classifyre_cli-0.4.54}/src/models/generated_input.py

@@ -112,14 +112,6 @@ class SamplingConfig(BaseModel):
         extra='forbid',
     )
     strategy: SamplingStrategy
-    enable_ocr: bool | None = Field(
-        False,
-        description='When true, enable OCR/text extraction for supported binary documents and images before routing text-capable detectors.',
-    )
-    enable_transcription: bool | None = Field(
-        False,
-        description='When true, transcribe audio and video files to text (via faster-whisper) before routing text-capable detectors. Slower and requires the transcription dependency.',
-    )
     order_by_column: str | None = Field(
         None,
         description='Column to use for LATEST sampling mode in tabular sources (usually created_at/updated_at). Auto-detected when not set.',
@@ -407,10 +399,6 @@ class YouTubeOptionalTranscript(BaseModel):
         None,
         description='Preferred caption language codes in priority order (e.g. ["en"]). Empty means accept any available language.',
     )
-    skip_transcript: bool | None = Field(
-        False,
-        description='When true, skip transcript fetching entirely (metadata-only assets, no detector content).',
-    )
 
 
 class YouTubeOptionalConnection(BaseModel):

{classifyre_cli-0.4.52 → classifyre_cli-0.4.54}/src/models/generated_single_asset_scan_results.py

@@ -87,31 +87,6 @@ class Location(BaseModel):
     end: int | None = Field(None, description='End offset (0-indexed)')
 
 
-class ScanStats(BaseModel):
-    """
-    Statistics about detector scan for an asset
-    """
-
-    scanned_at: AwareDatetime = Field(
-        ..., description='Timestamp when the scan started'
-    )
-    duration_ms: int = Field(..., description='Duration of the scan in milliseconds')
-    detectors_run: list[DetectorType] = Field(
-        ..., description='List of detector types that were run'
-    )
-    content_size_bytes: int | None = Field(
-        None, description='Size of the content that was scanned'
-    )
-    findings_count: int | None = Field(
-        None, description='Total number of findings detected'
-    )
-    warnings: list[str] | None = Field(
-        None,
-        description='Non-fatal issues during scan (e.g. content truncation, empty content)',
-    )
-    errors: list[str] | None = Field(None, description='Detector errors during scan')
-
-
 class DetectionResult(BaseModel):
     """
     Result from detector scan
@@ -191,6 +166,71 @@ class DetectionResult(BaseModel):
     )
 
 
+class Status(StrEnum):
+    """
+    OK means the detector completed; it may still have found nothing. ERROR means it raised and its result is unknown.
+    """
+
+    OK = 'OK'
+    ERROR = 'ERROR'
+
+
+class DetectorOutcome(BaseModel):
+    """
+    Outcome of a single detector on a single asset. A detector that raised is reported with status ERROR: it produced no findings, but that absence is a failure rather than a clean result. Consumers must not treat an ERROR detector's silence as evidence that its previous findings are gone.
+    """
+
+    detector_type: DetectorType = Field(
+        ..., description='Detector family that was attempted'
+    )
+    custom_detector_key: str | None = Field(
+        None,
+        description='Stable key of the custom detector when detector_type is CUSTOM. Required to tell two custom detectors apart, since both report detector_type CUSTOM.',
+        title='Custom Detector Key',
+    )
+    status: Status = Field(
+        ...,
+        description='OK means the detector completed; it may still have found nothing. ERROR means it raised and its result is unknown.',
+        title='Status',
+    )
+    error: str | None = Field(
+        None, description='Failure detail when status is ERROR', title='Error'
+    )
+
+
+class ScanStats(BaseModel):
+    """
+    Statistics about detector scan for an asset
+    """
+
+    scanned_at: AwareDatetime = Field(
+        ..., description='Timestamp when the scan started'
+    )
+    duration_ms: int = Field(..., description='Duration of the scan in milliseconds')
+    detectors_run: list[DetectorType] = Field(
+        ..., description='List of detector types that were run'
+    )
+    detector_outcomes: list[DetectorOutcome] | None = Field(
+        None,
+        description='Per-detector outcome for this asset. detectors_run lists what was attempted and cannot distinguish a detector that completed cleanly from one that crashed; this does.',
+    )
+    content_size_bytes: int | None = Field(
+        None, description='Size of the content that was scanned'
+    )
+    empty_text: bool | None = Field(
+        None,
+        description="True when an asset that should carry text yielded none. Most significant for assets whose text is derived rather than native (OCR'd images and scanned PDFs, transcribed audio/video): empty derived text means the content may have been missed entirely, not that the document is blank. Distinct from an error — nothing failed — so it is reported as coverage, not as a run failure.",
+    )
+    findings_count: int | None = Field(
+        None, description='Total number of findings detected'
+    )
+    warnings: list[str] | None = Field(
+        None,
+        description='Non-fatal issues during scan (e.g. content truncation, empty content)',
+    )
+    errors: list[str] | None = Field(None, description='Detector errors during scan')
+
+
 class SingleAssetScanResults(BaseModel):
     """
     Single asset scan results with detector findings