classifyre-cli 0.4.57__tar.gz → 0.4.58__tar.gz

classifyre_cli-0.4.58/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 300 packages in 202ms
+Checked 51 packages in 3ms

{classifyre_cli-0.4.57 → classifyre_cli-0.4.58}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.57
+Version: 0.4.58
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.57 → classifyre_cli-0.4.58}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.57",
+  "version": "0.4.58",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.57 → classifyre_cli-0.4.58}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.57"
+version = "0.4.58"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -60,7 +60,7 @@ privacy = [
     # phonenumbers 9.x introduced lazy region loading via __import__ which fails
     # mid-run in frozen/venv contexts. 8.x eagerly loads all data at import time,
     # avoiding ModuleNotFoundError during Presidio phone number analysis.
-    "phonenumbers>=8.13.0,<10.0.0",
+    "phonenumbers>=8.13.0,<9.0.0",
     "numpy>=1.26.0,<3.0.0",
 ]
 security = [
@@ -135,6 +135,18 @@ file-processing = [
     "openpyxl>=3.1.5",
     "chardet>=7.4.3",
     "pyarrow>=18.0.0",
+    # RTF text extraction (tiny, pure Python).
+    "striprtf>=0.0.32",
+    # Secure XML parsing (entity-expansion / XXE hardened stdlib drop-in).
+    "defusedxml>=0.7.1",
+    # Image decoding for GIF-frame extraction and HEIC/HEIF conversion before OCR.
+    "pillow>=12.2.0",
+    "pillow-heif>=1.4.0",
+    # Archive member extraction (zip/tar via stdlib). RAR additionally needs an
+    # external unrar/unar/bsdtar backend at runtime; rarfile degrades gracefully
+    # when none is installed.
+    "py7zr>=1.1.3",
+    "rarfile>=4.3",
     # Table layer: Iceberg metadata resolution + SQL over Parquet without Spark.
     "pyiceberg>=0.9.0",
     "duckdb>=1.1.0",
@@ -152,7 +164,8 @@ video = [
 ]
 ocr = [
     { include-group = "file-processing" },
-    "docling>=2.94.0",
+    # >=2.113 for the OpenDocument (odt/ods/odp) and email (eml/msg) backends.
+    "docling>=2.113.0",
     "opencv-python-headless>=5.0.0.93,<6.0.0",
     "rapidocr-onnxruntime>=1.4.0",
     # torchvision is a transitive dep of docling-ibm-models, but [tool.uv.sources]

{classifyre_cli-0.4.57 → classifyre_cli-0.4.58}/src/file_evaluation/runner.py

@@ -20,6 +20,10 @@ class FileEvaluationRunner:
 
     def __init__(self, detectors_config: list[dict[str, Any]]) -> None:
         self._config = detectors_config
+        # Detector construction failures, surfaced in the evaluate-file JSON
+        # output so callers can distinguish "misconfigured detector" from
+        # "no findings".
+        self.detector_errors: list[str] = []
 
     def _build_detectors(self) -> list[Any]:
         from ..detectors import get_detector
@@ -44,6 +48,7 @@ class FileEvaluationRunner:
                 logger.info(f"Initialized file-evaluation detector: {detector_name}")
             except Exception as e:
                 logger.error(f"Failed to initialize detector {detector_type}: {e}")
+                self.detector_errors.append(f"{detector_type}: {e}")
 
         return detectors
 

{classifyre_cli-0.4.57 → classifyre_cli-0.4.58}/src/main.py

@@ -1,4 +1,5 @@
 import argparse
+import asyncio
 import json
 import logging
 import os
@@ -17,6 +18,7 @@ _SURROGATE_RE = re.compile(r"[\ud800-\udfff]")
 
 _TIMEOUT_PHRASES = ("timed out", "timeout", "connection reset", "errno 110", "connection refused")
 _TIMEOUT_MYSQL_CODES = {2003, 2006, 2013}
+_TEXT_CHUNK_EMIT_ATTEMPTS = 3
 
 
 def _is_timeout_error(exc: BaseException) -> bool:
@@ -43,6 +45,34 @@ def _sanitize_for_json(value: Any) -> Any:
     return value
 
 
+async def _emit_text_chunks_with_retry(
+    sink: Any,
+    asset_hash: str,
+    artifact: Any,
+    *,
+    attempts: int = _TEXT_CHUNK_EMIT_ATTEMPTS,
+) -> None:
+    """Persist chunk provenance, retrying transient API failures.
+
+    The caller keeps the already-persisted asset successful, but propagates a
+    final failure at run level so missing chunks/embeddings cannot look healthy.
+    """
+    for attempt in range(1, attempts + 1):
+        try:
+            await sink.emit_text_chunks(asset_hash, artifact)
+            return
+        except Exception:
+            if attempt == attempts:
+                raise
+            logger.warning(
+                "Text-chunk emission failed for asset %s (attempt %d/%d); retrying",
+                asset_hash,
+                attempt,
+                attempts,
+            )
+            await asyncio.sleep(0.25 * (2 ** (attempt - 1)))
+
+
 def setup_logging() -> None:
     """Configure the logging facility."""
     logging.basicConfig(
@@ -250,13 +280,18 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                         stub_batch = [_asset_to_payload(asset) for asset in raw_batch]
                         for stub in stub_batch:
                             stub["findings"] = []
-                        await sink.emit_batch(stub_batch, skip_findings=True)
-                        output_batch_count += 1
 
+                        # Register BEFORE ingesting: the API stamps per-run
+                        # change_type onto runner_assets rows during ingest, so
+                        # the row must exist or the CREATED stamp is lost and
+                        # first-run counters report every asset as unchanged.
                         hashes = [s["hash"] for s in stub_batch if s.get("hash")]
                         if hasattr(sink, "register_discovered_assets") and hashes:
                             await sink.register_discovered_assets(hashes)
 
+                        await sink.emit_batch(stub_batch, skip_findings=True)
+                        output_batch_count += 1
+
                         all_stubs.extend(raw_batch)
                         logger.info(
                             "Discovered %s assets (total: %s)",
@@ -269,13 +304,11 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
 
                     # --- Phase 2: Processing ---
                     if all_stubs:
-                        import asyncio as _asyncio
-
                         processed_count = 0
                         _pw = worker_pool.max_workers if worker_pool else 4
                         max_concurrent = args.max_concurrent_assets or (_pw * 2)
                         max_concurrent = max(1, max_concurrent)
-                        _asset_semaphore = _asyncio.Semaphore(max_concurrent)
+                        _asset_semaphore = asyncio.Semaphore(max_concurrent)
                         logger.info(
                             "Phase 2 starting: %d assets, pool_workers=%s, max_concurrent_assets=%d",
                             len(all_stubs),
@@ -283,6 +316,7 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                             max_concurrent,
                         )
                         error_count = 0
+                        chunk_errors: list[str] = []
 
                         async def _process_one(asset: Any) -> None:
                             async with _asset_semaphore:
@@ -326,7 +360,24 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                                 if hasattr(sink, "emit_text_chunks"):
                                     artifact = pipeline.take_text_artifact(asset_hash)
                                     if artifact is not None:
-                                        await sink.emit_text_chunks(asset_hash, artifact)
+                                        try:
+                                            await _emit_text_chunks_with_retry(
+                                                sink, asset_hash, artifact
+                                            )
+                                        except Exception as chunk_exc:
+                                            # Findings and the asset remain persisted,
+                                            # preserving B-2's graceful degradation.
+                                            # The run fails after phase 2 so operators
+                                            # cannot mistake missing embeddings for a
+                                            # healthy completed scan.
+                                            detail = f"{asset_hash}: {chunk_exc}"
+                                            chunk_errors.append(detail)
+                                            logger.error(
+                                                "Text-chunk emission exhausted retries"
+                                                " for asset %s: %s",
+                                                asset_hash,
+                                                chunk_exc,
+                                            )
 
                                 if hasattr(sink, "update_asset_status"):
                                     f_total, f_by_sev, f_by_det = _compute_findings_counts(
@@ -354,13 +405,22 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                                     except Exception:
                                         pass
 
-                        tasks = [_asyncio.create_task(_process_one(a)) for a in all_stubs]
-                        await _asyncio.gather(*tasks, return_exceptions=True)
+                        tasks = [asyncio.create_task(_process_one(a)) for a in all_stubs]
+                        await asyncio.gather(*tasks, return_exceptions=True)
                         logger.info(
                             "Phase 2 complete: %d processed, %d errors",
                             processed_count,
                             error_count,
                         )
+                        if chunk_errors:
+                            preview = "; ".join(chunk_errors[:3])
+                            remaining = len(chunk_errors) - 3
+                            if remaining > 0:
+                                preview += f"; and {remaining} more"
+                            raise RuntimeError(
+                                f"Text-chunk emission failed for {len(chunk_errors)}"
+                                f" asset(s) after retries: {preview}"
+                            )
 
                     # Persist the advanced AUTOMATIC sampling cursor (no-op for
                     # other strategies, which return None). Only on the normal
@@ -506,6 +566,8 @@ def run_evaluate_file_command(args: argparse.Namespace) -> None:
         }
         if parsed.parse_error:
             output["parse_error"] = parsed.parse_error
+        if runner.detector_errors:
+            output["detector_errors"] = runner.detector_errors
         print(json.dumps(_sanitize_for_json(output), ensure_ascii=False))
     except Exception as e:
         logger.error("File evaluation failed: %s", e, exc_info=True)

{classifyre_cli-0.4.57 → classifyre_cli-0.4.58}/src/sources/object_storage/base.py

@@ -1,14 +1,16 @@
 from __future__ import annotations
 
 import asyncio
+import hashlib
 import itertools
 import logging
 import random
+import tempfile
 from abc import ABC, abstractmethod
 from collections.abc import AsyncGenerator, Iterator
 from dataclasses import dataclass
 from datetime import UTC, datetime
-from pathlib import PurePosixPath
+from pathlib import Path, PurePosixPath
 from typing import Any
 
 from ...models.generated_input import SamplingStrategy
@@ -20,6 +22,7 @@ from ...models.generated_single_asset_scan_results import (
     Location,
     SingleAssetScanResults,
 )
+from ...utils.archive_extraction import ArchiveMember, is_archive_mime, iter_archive_members
 from ...utils.embedded_images import EmbeddedImage, has_embedded_images, iter_embedded_images
 from ...utils.file_metadata import extract_file_metadata
 from ...utils.file_parser import infer_mime_type_from_file_name, resolve_mime_type
@@ -65,6 +68,8 @@ _FILE_EXTENSION_HINTS: dict[str, OutputAssetType] = {
     ".wav": OutputAssetType.AUDIO,
     ".aac": OutputAssetType.AUDIO,
     ".ogg": OutputAssetType.AUDIO,
+    ".heic": OutputAssetType.IMAGE,
+    ".heif": OutputAssetType.IMAGE,
     ".pdf": OutputAssetType.BINARY,
     ".doc": OutputAssetType.BINARY,
     ".docx": OutputAssetType.BINARY,
@@ -72,11 +77,18 @@ _FILE_EXTENSION_HINTS: dict[str, OutputAssetType] = {
     ".xlsx": OutputAssetType.TABLE,
     ".ppt": OutputAssetType.BINARY,
     ".pptx": OutputAssetType.BINARY,
+    ".odt": OutputAssetType.BINARY,
+    ".ods": OutputAssetType.TABLE,
+    ".odp": OutputAssetType.BINARY,
+    ".rtf": OutputAssetType.BINARY,
+    ".eml": OutputAssetType.BINARY,
+    ".msg": OutputAssetType.BINARY,
     ".zip": OutputAssetType.BINARY,
     ".rar": OutputAssetType.BINARY,
     ".7z": OutputAssetType.BINARY,
     ".tar": OutputAssetType.BINARY,
     ".gz": OutputAssetType.BINARY,
+    ".tgz": OutputAssetType.BINARY,
     ".parquet": OutputAssetType.TABLE,
     ".json": OutputAssetType.TXT,
     ".xml": OutputAssetType.TXT,
@@ -134,6 +146,11 @@ class ObjectStorageSourceBase(BaseSource, ABC):
         self._file_processing_deps_checked = False
         # Keyed by both asset_hash and external_url for O(1) lookup from either.
         self._bytes_cache: dict[str, bytes] = {}
+        # Archive members can expand to 100 MB per container. Keep their
+        # discovery-to-processing lifetime on disk so many archives cannot
+        # multiply that budget into unbounded resident memory.
+        self._archive_bytes_cache: dict[str, Path] = {}
+        self._archive_cache_dir: tempfile.TemporaryDirectory[str] | None = None
         self._mime_cache: dict[str, str] = {}
         # asset_ids for which fetch_content_pages ran the full bytes path
         # (even if it produced no text, e.g. all-silence audio).  Checked by
@@ -339,6 +356,13 @@ class ObjectStorageSourceBase(BaseSource, ABC):
         }
         return mapping.get(asset_type, "file")
 
+    @classmethod
+    def _asset_kind_for_mime(cls, mime_type: str | None, asset_type: OutputAssetType) -> str:
+        """Catalog kind for an asset: archives get their own kind for display."""
+        if mime_type and is_archive_mime(mime_type):
+            return "archive"
+        return cls._asset_kind_for_asset_type(asset_type)
+
     def _ensure_file_processing_dependencies(self) -> None:
         if self._file_processing_deps_checked:
             return
@@ -399,10 +423,9 @@ class ObjectStorageSourceBase(BaseSource, ABC):
 
         # Images and audio/video are extractable automatically. Opaque binaries
         # remain available to binary detectors but have no text page stream.
-        is_non_extractable = normalized_mime in (
-            "application/octet-stream",
-            "application/zip",
-        )
+        # Archives keep their bytes so member files can be expanded into child
+        # assets (the container itself still yields no text pages).
+        is_non_extractable = normalized_mime == "application/octet-stream"
 
         return ContentSnapshot(
             mime_type=mime_type,
@@ -479,7 +502,9 @@ class ObjectStorageSourceBase(BaseSource, ABC):
             created_at=ref.last_modified,
             updated_at=ref.last_modified,
             runner_id=self.runner_id,
-            **self.metadata_fields(self._asset_kind_for_asset_type(asset_type), asset_metadata),
+            **self.metadata_fields(
+                self._asset_kind_for_mime(snapshot.mime_type, asset_type), asset_metadata
+            ),
         )
         self._hash_to_uri[asset_hash] = external_url
         self._object_ref_by_hash[asset_hash] = ref
@@ -497,6 +522,18 @@ class ObjectStorageSourceBase(BaseSource, ABC):
                 ref=ref,
             )
 
+        # Archive containers (zip/tar/gz/7z/rar) expand into one child asset per
+        # member file so each member gets its own MIME resolution, text
+        # extraction, and detector pass. Nested archives are kept as opaque
+        # child assets (single-level expansion) to bound memory and fan-out.
+        if snapshot.raw_bytes is not None and is_archive_mime(snapshot.mime_type):
+            self._queue_child_archive_assets(
+                parent=asset,
+                file_bytes=snapshot.raw_bytes,
+                mime_type=snapshot.mime_type,
+                ref=ref,
+            )
+
         return asset
 
     def _queue_child_image_assets(
@@ -555,6 +592,84 @@ class ObjectStorageSourceBase(BaseSource, ABC):
             **self.metadata_fields("image", metadata),
         )
 
+    def _queue_child_archive_assets(
+        self,
+        *,
+        parent: SingleAssetScanResults,
+        file_bytes: bytes,
+        mime_type: str,
+        ref: ObjectRef,
+    ) -> None:
+        """Expand archive members into child assets linked from the parent
+        (appended to ``parent.links``, never removing existing links)."""
+        try:
+            for member in iter_archive_members(
+                file_bytes,
+                mime_type,
+                file_name=self._object_file_name(ref),
+            ):
+                child = self._build_child_archive_member_asset(parent, member, ref)
+                self._pending_child_assets.append(child)
+                if child.hash not in parent.links:
+                    parent.links.append(child.hash)
+        except Exception as exc:
+            logger.warning("Failed to expand archive members from %s: %s", parent.external_url, exc)
+
+    def _build_child_archive_member_asset(
+        self,
+        parent: SingleAssetScanResults,
+        member: ArchiveMember,
+        ref: ObjectRef,
+    ) -> SingleAssetScanResults:
+        child_url = f"{parent.external_url}#{member.location}"
+        child_hash = self.generate_hash_id(child_url)
+        member_type = self._asset_type_from_mime_or_key(member.mime_type, member.location)
+        metadata: dict[str, Any] = {
+            "provider": self.provider_label,
+            "object_key": f"{ref.key}#{member.location}",
+            "source_hash": parent.hash,
+            "location": member.location,
+            "mime_type": member.mime_type,
+            "size_bytes": len(member.member_bytes),
+        }
+        file_meta = extract_file_metadata(
+            member.member_bytes,
+            member.mime_type,
+            file_name=member.location,
+        )
+        metadata.update({k: v for k, v in file_meta.items() if v is not None})
+        # Spool member bytes to a per-scan temporary directory. Phase 1 must
+        # discover every asset before phase 2 starts (B-4), so retaining these
+        # bytes in memory would make the per-archive extraction cap unbounded
+        # across the whole source.
+        if self._archive_cache_dir is None:
+            self._archive_cache_dir = tempfile.TemporaryDirectory(
+                prefix="classifyre-archive-members-"
+            )
+        cache_name = hashlib.sha256(child_hash.encode()).hexdigest()
+        member_path = Path(self._archive_cache_dir.name) / cache_name
+        member_path.write_bytes(member.member_bytes)
+        self._archive_bytes_cache[child_hash] = member_path
+        self._archive_bytes_cache[child_url] = member_path
+        self._mime_cache[child_hash] = member.mime_type
+        self._mime_cache[child_url] = member.mime_type
+        self._hash_to_uri[child_hash] = child_url
+        return SingleAssetScanResults(
+            hash=child_hash,
+            checksum=self.calculate_checksum(metadata),
+            name=f"{parent.name}#{member.location}",
+            external_url=child_url,
+            links=[],
+            asset_type=member_type,
+            source_id=self.source_id,
+            created_at=ref.last_modified,
+            updated_at=ref.last_modified,
+            runner_id=self.runner_id,
+            **self.metadata_fields(
+                self._asset_kind_for_mime(member.mime_type, member_type), metadata
+            ),
+        )
+
     def test_connection(self) -> dict[str, Any]:
         result = {
             "timestamp": datetime.now(UTC).isoformat(),
@@ -581,6 +696,7 @@ class ObjectStorageSourceBase(BaseSource, ABC):
         self._hash_to_uri = {}
         self._object_ref_by_hash = {}
         self._bytes_cache = {}
+        self._reset_archive_cache()
         self._mime_cache = {}
         self._content_pages_processed = set()
         self._pending_child_assets = []
@@ -617,6 +733,10 @@ class ObjectStorageSourceBase(BaseSource, ABC):
 
     async def fetch_content_bytes(self, asset_id: str) -> tuple[bytes, str] | None:
         raw_bytes = self._bytes_cache.get(asset_id)
+        if raw_bytes is None:
+            member_path = self._archive_bytes_cache.get(asset_id)
+            if member_path is not None:
+                raw_bytes = member_path.read_bytes()
         mime = self._mime_cache.get(asset_id, "")
         if raw_bytes is not None and mime:
             return raw_bytes, mime
@@ -660,6 +780,10 @@ class ObjectStorageSourceBase(BaseSource, ABC):
 
     async def fetch_content_pages(self, asset_id: str) -> AsyncGenerator[tuple[str, str], None]:
         raw_bytes = self._bytes_cache.get(asset_id)
+        if raw_bytes is None:
+            member_path = self._archive_bytes_cache.get(asset_id)
+            if member_path is not None:
+                raw_bytes = member_path.read_bytes()
         mime = self._mime_cache.get(asset_id, "")
 
         logger.info(
@@ -807,20 +931,31 @@ class ObjectStorageSourceBase(BaseSource, ABC):
 
     def evict_asset_cache(self, asset_hash: str) -> None:
         external_url = self._hash_to_uri.get(asset_hash)
+        archive_path = self._archive_bytes_cache.pop(asset_hash, None)
         self._content_cache.pop(asset_hash, None)
         self._bytes_cache.pop(asset_hash, None)
         self._mime_cache.pop(asset_hash, None)
         self._object_ref_by_hash.pop(asset_hash, None)
         if external_url:
+            archive_path = self._archive_bytes_cache.pop(external_url, archive_path)
             self._content_cache.pop(external_url, None)
             self._bytes_cache.pop(external_url, None)
             self._mime_cache.pop(external_url, None)
+        if archive_path is not None:
+            archive_path.unlink(missing_ok=True)
+
+    def _reset_archive_cache(self) -> None:
+        self._archive_bytes_cache = {}
+        if self._archive_cache_dir is not None:
+            self._archive_cache_dir.cleanup()
+            self._archive_cache_dir = None
 
     def abort(self) -> None:
         logger.info("Aborting object storage extraction...")
         super().abort()
 
     def cleanup(self) -> None:
+        self._reset_archive_cache()
         client = self._cached_client
         if client is None:
             return