classifyre-cli 0.4.23__tar.gz → 0.4.24__tar.gz

{classifyre_cli-0.4.23 → classifyre_cli-0.4.24}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 262 packages in 370ms
+Resolved 262 packages in 149ms
 Checked 50 packages in 1ms

{classifyre_cli-0.4.23 → classifyre_cli-0.4.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.23
+Version: 0.4.24
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.23 → classifyre_cli-0.4.24}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.23",
+  "version": "0.4.24",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.23 → classifyre_cli-0.4.24}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.23"
+version = "0.4.24"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

{classifyre_cli-0.4.23 → classifyre_cli-0.4.24}/src/pipeline/detector_pipeline.py

@@ -297,13 +297,23 @@ class DetectorPipeline:
                 page_num=page_num,
             )
             elapsed = int((time.monotonic() - t0) * 1000)
+            snippet = page_content[:120].replace("\n", "\\n") if page_content else ""
             logger.info(
-                "  %s page %d done: %d findings (%dms)",
+                "  %s page %d: %d findings in %dms — snippet: %s",
                 asset.name,
                 page_num,
                 len(page_findings),
                 elapsed,
+                snippet,
             )
+            if page_findings:
+                for f in page_findings[:5]:
+                    logger.info(
+                        "    finding: type=%s detector=%s matched=%.100s",
+                        f.finding_type,
+                        f.detector_type,
+                        f.matched_content[:100].replace("\n", " "),
+                    )
             return page_findings, page_types, page_errors, page_content, page_num
 
         def _collect_done() -> None:
@@ -404,13 +414,23 @@ class DetectorPipeline:
                 page_num=page_num,
             )
             elapsed = int((time.monotonic() - t0) * 1000)
+            snippet = page_content[:120].replace("\n", "\\n") if page_content else ""
             logger.info(
-                "  %s page %d done: %d findings (%dms)",
+                "  %s page %d: %d findings in %dms — snippet: %s",
                 asset.name,
                 page_num,
                 len(page_findings),
                 elapsed,
+                snippet,
             )
+            if page_findings:
+                for f in page_findings[:5]:
+                    logger.info(
+                        "    finding: type=%s detector=%s matched=%.100s",
+                        f.finding_type,
+                        f.detector_type,
+                        f.matched_content[:100].replace("\n", " "),
+                    )
             return page_findings, page_types, page_errors, page_content, page_num
 
         async def _collect_done_and_flush(min_findings: int = 1) -> None:
@@ -491,6 +511,12 @@ class DetectorPipeline:
                 continue
             candidate_ids.append(value)
 
+        logger.info(
+            "_iter_text_content_pages(%s): trying candidates %s",
+            asset.name,
+            candidate_ids,
+        )
+
         for candidate_id in candidate_ids:
             saw_candidate_content = False
             async for text_content in self.content_provider.fetch_text_pages(candidate_id):
@@ -502,6 +528,16 @@ class DetectorPipeline:
             if saw_candidate_content:
                 return
 
+            # If fetch_content_pages ran the full bytes-path extraction (even
+            # yielding 0 text, e.g. silent audio), the source already did the
+            # expensive work.  Don't re-process with another candidate ID for
+            # the same asset.
+            source = getattr(self.content_provider, "_source", None)
+            if source is not None:
+                processed: set[str] = getattr(source, "_content_pages_processed", set())
+                if candidate_id in processed:
+                    return
+
     async def _run_binary_detectors_for_asset(
         self,
         *,

{classifyre_cli-0.4.23 → classifyre_cli-0.4.24}/src/pipeline/parsed_content_provider.py

@@ -3,11 +3,14 @@
 from __future__ import annotations
 
 import asyncio
+import logging
 from collections.abc import AsyncGenerator
 
 from ..models.generated_single_asset_scan_results import DetectionResult, SingleAssetScanResults
 from ..sources.base import BaseSource
 
+logger = logging.getLogger(__name__)
+
 
 class ParsedContentProvider:
     """
@@ -32,11 +35,30 @@ class ParsedContentProvider:
         if saw_text:
             return
 
+        # If fetch_content_pages already ran the full extraction pipeline for
+        # this asset (tracked via _content_pages_processed), skip the fallback
+        # iter_asset_pages call.  Without this, an all-silence audio file would
+        # trigger a redundant second transcription pass.
+        pages_processed: set[str] | None = getattr(self._source, "_content_pages_processed", None)
+        if isinstance(pages_processed, set) and asset_id in pages_processed:
+            logger.info(
+                "fetch_text_pages(%s): source already processed, skipping fallback",
+                asset_id,
+            )
+            return
+
         result = await self._source.fetch_content_bytes(asset_id)
         if result is None:
+            logger.info("fetch_text_pages(%s): fetch_content_bytes returned None", asset_id)
             return
 
         raw_bytes, mime = result
+        logger.info(
+            "fetch_text_pages(%s): fallback iter_asset_pages path (%s, %d bytes)",
+            asset_id,
+            mime,
+            len(raw_bytes),
+        )
         pages: list[str] = await asyncio.to_thread(
             list,
             self._source.iter_asset_pages(raw_bytes, mime),

{classifyre_cli-0.4.23 → classifyre_cli-0.4.24}/src/pipeline/worker_pool.py

@@ -21,6 +21,9 @@ from concurrent.futures import ProcessPoolExecutor
 from typing import Any
 
 from ..models.generated_single_asset_scan_results import DetectionResult
+from ..utils.resources import get_effective_cpu_count, get_effective_memory_mb
+
+__all__ = ["get_effective_cpu_count", "get_effective_memory_mb"]
 
 logger = logging.getLogger(__name__)
 
@@ -130,63 +133,6 @@ def is_io_bound_detector(detector_name: str) -> bool:
     return detector_name in _IO_BOUND_DETECTORS
 
 
-def get_effective_cpu_count() -> int:
-    """Return the number of usable CPUs, respecting cgroup limits (K8s/Docker).
-
-    ``os.cpu_count()`` returns the *host* CPU count, which can be much larger
-    than what the container is allowed to use.  This function reads the cgroup
-    v2 ``cpu.max`` (or v1 ``cpu.cfs_quota_us``/``cpu.cfs_period_us``) to
-    determine the actual allocation.
-    """
-    try:
-        data = open("/sys/fs/cgroup/cpu.max").read().strip()
-        quota_str, period_str = data.split()
-        if quota_str != "max":
-            cpus = int(quota_str) / int(period_str)
-            if cpus >= 0.5:
-                return max(1, int(cpus))
-    except (FileNotFoundError, OSError, ValueError):
-        pass
-
-    try:
-        quota = int(open("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read().strip())
-        period = int(open("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read().strip())
-        if quota > 0 and period > 0:
-            cpus = quota / period
-            if cpus >= 0.5:
-                return max(1, int(cpus))
-    except (FileNotFoundError, OSError, ValueError):
-        pass
-
-    return os.cpu_count() or 4
-
-
-def get_effective_memory_mb() -> int:
-    """Return usable memory in MB, respecting cgroup limits."""
-    try:
-        mem_bytes = int(open("/sys/fs/cgroup/memory.max").read().strip())
-        if mem_bytes < 2**50:
-            return max(256, mem_bytes // (1024 * 1024))
-    except (FileNotFoundError, OSError, ValueError):
-        pass
-
-    try:
-        mem_bytes = int(open("/sys/fs/cgroup/memory/memory.limit_in_bytes").read().strip())
-        if mem_bytes < 2**50:
-            return max(256, mem_bytes // (1024 * 1024))
-    except (FileNotFoundError, OSError, ValueError):
-        pass
-
-    try:
-        for line in open("/proc/meminfo"):
-            if line.startswith("MemTotal:"):
-                return max(256, int(line.split()[1]) // 1024)
-    except (FileNotFoundError, OSError, ValueError):
-        pass
-
-    return 4096
-
-
 def compute_pool_workers(override: int | None = None) -> int:
     """Compute optimal pool size from actual resource limits.
 

{classifyre_cli-0.4.23 → classifyre_cli-0.4.24}/src/sources/object_storage/base.py

@@ -135,6 +135,11 @@ class ObjectStorageSourceBase(BaseSource, ABC):
         # Keyed by both asset_hash and external_url for O(1) lookup from either.
         self._bytes_cache: dict[str, bytes] = {}
         self._mime_cache: dict[str, str] = {}
+        # asset_ids for which fetch_content_pages ran the full bytes path
+        # (even if it produced no text, e.g. all-silence audio).  Checked by
+        # ParsedContentProvider to skip its fallback iter_asset_pages path,
+        # which would otherwise re-run an expensive transcription a second time.
+        self._content_pages_processed: set[str] = set()
         # Child IMAGE assets queued while transforming the current object.
         self._pending_child_assets: list[SingleAssetScanResults] = []
 
@@ -302,6 +307,15 @@ class ObjectStorageSourceBase(BaseSource, ABC):
 
         return OutputAssetType.OTHER
 
+    @staticmethod
+    def _asset_kind_for_asset_type(asset_type: OutputAssetType) -> str:
+        mapping: dict[OutputAssetType, str] = {
+            OutputAssetType.IMAGE: "image",
+            OutputAssetType.AUDIO: "audio",
+            OutputAssetType.VIDEO: "video",
+        }
+        return mapping.get(asset_type, "file")
+
     def _ensure_file_processing_dependencies(self) -> None:
         if self._file_processing_deps_checked:
             return
@@ -446,7 +460,7 @@ class ObjectStorageSourceBase(BaseSource, ABC):
             created_at=ref.last_modified,
             updated_at=ref.last_modified,
             runner_id=self.runner_id,
-            **self.metadata_fields("file", asset_metadata),
+            **self.metadata_fields(self._asset_kind_for_asset_type(asset_type), asset_metadata),
         )
         self._hash_to_uri[asset_hash] = external_url
         self._object_ref_by_hash[asset_hash] = ref
@@ -549,6 +563,7 @@ class ObjectStorageSourceBase(BaseSource, ABC):
         self._object_ref_by_hash = {}
         self._bytes_cache = {}
         self._mime_cache = {}
+        self._content_pages_processed = set()
         self._pending_child_assets = []
 
         refs = self._list_objects()
@@ -628,26 +643,69 @@ class ObjectStorageSourceBase(BaseSource, ABC):
         raw_bytes = self._bytes_cache.get(asset_id)
         mime = self._mime_cache.get(asset_id, "")
 
+        logger.info(
+            "fetch_content_pages(%s): raw_bytes=%s mime=%s processed=%s",
+            asset_id,
+            f"{len(raw_bytes)} bytes" if raw_bytes is not None else "MISS",
+            mime or "MISS",
+            asset_id in self._content_pages_processed,
+        )
+
         if raw_bytes is not None:
             sampling = self.config.sampling
             batch_size = int(sampling.rows_per_page or 100)
             include_col_names = bool(
                 sampling.include_column_names if sampling.include_column_names is not None else True
             )
-            # Run the (potentially blocking) file parsing in a thread so pyarrow /
-            # pdfplumber can't freeze the event loop during large file iteration.
-            pages: list[str] = await asyncio.to_thread(
-                list,
-                self.iter_asset_pages(
-                    raw_bytes,
-                    mime,
-                    batch_size,
-                    include_col_names,
-                    file_name=self._file_name_for_asset_id(asset_id),
-                ),
+            file_name = self._file_name_for_asset_id(asset_id)
+
+            # Stream pages from a thread instead of materializing via list().
+            # For transcription this lets detectors start working on the first
+            # chunk while later chunks are still being transcribed.
+            loop = asyncio.get_running_loop()
+            queue: asyncio.Queue[str | None] = asyncio.Queue()
+
+            exc_info: list[BaseException | None] = [None]
+
+            page_count: int = 0
+
+            def _produce() -> None:
+                nonlocal page_count
+                try:
+                    for page in self.iter_asset_pages(
+                        raw_bytes,
+                        mime,
+                        batch_size,
+                        include_col_names,
+                        file_name=file_name,
+                    ):
+                        loop.call_soon_threadsafe(queue.put_nowait, page)
+                        page_count += 1
+                except BaseException as exc:
+                    exc_info[0] = exc
+                finally:
+                    loop.call_soon_threadsafe(queue.put_nowait, None)
+
+            task = loop.run_in_executor(None, _produce)
+
+            while True:
+                page = await queue.get()
+                if page is None:
+                    break
+                yield "", page
+
+            await task
+            if exc_info[0] is not None:
+                raise exc_info[0]  # type: ignore[misc]
+
+            logger.info(
+                "fetch_content_pages(%s): streamed %d page(s) from %s",
+                asset_id,
+                page_count,
+                file_name,
             )
-            for batch_text in pages:
-                yield "", batch_text
+
+            self._content_pages_processed.add(asset_id)
             return
 
         result = await self.fetch_content(asset_id)

classifyre_cli-0.4.24/src/utils/resources.py

@@ -0,0 +1,65 @@
+"""Cgroup-aware CPU and memory introspection.
+
+Shared by the detector worker pool (to size the process pool) and the
+transcription pipeline (to select the right Whisper model at runtime).
+"""
+
+from __future__ import annotations
+
+import os
+
+
+def get_effective_cpu_count() -> int:
+    """Return usable CPUs, respecting cgroup limits (K8s / Docker).
+
+    ``os.cpu_count()`` returns the *host* count, which is usually much larger
+    than the container's CPU quota.  This reads cgroup v2 / v1 to get the
+    actual allocation.
+    """
+    try:
+        data = open("/sys/fs/cgroup/cpu.max").read().strip()
+        quota_str, period_str = data.split()
+        if quota_str != "max":
+            cpus = int(quota_str) / int(period_str)
+            if cpus >= 0.5:
+                return max(1, int(cpus))
+    except (FileNotFoundError, OSError, ValueError):
+        pass
+
+    try:
+        quota = int(open("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read().strip())
+        period = int(open("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read().strip())
+        if quota > 0 and period > 0:
+            cpus = quota / period
+            if cpus >= 0.5:
+                return max(1, int(cpus))
+    except (FileNotFoundError, OSError, ValueError):
+        pass
+
+    return os.cpu_count() or 4
+
+
+def get_effective_memory_mb() -> int:
+    """Return usable memory in MB, respecting cgroup limits (K8s / Docker)."""
+    try:
+        mem_bytes = int(open("/sys/fs/cgroup/memory.max").read().strip())
+        if mem_bytes < 2**50:
+            return max(256, mem_bytes // (1024 * 1024))
+    except (FileNotFoundError, OSError, ValueError):
+        pass
+
+    try:
+        mem_bytes = int(open("/sys/fs/cgroup/memory/memory.limit_in_bytes").read().strip())
+        if mem_bytes < 2**50:
+            return max(256, mem_bytes // (1024 * 1024))
+    except (FileNotFoundError, OSError, ValueError):
+        pass
+
+    try:
+        for line in open("/proc/meminfo"):
+            if line.startswith("MemTotal:"):
+                return max(256, int(line.split()[1]) // 1024)
+    except (FileNotFoundError, OSError, ValueError):
+        pass
+
+    return 4096

{classifyre_cli-0.4.23 → classifyre_cli-0.4.24}/src/utils/transcription.py

@@ -89,14 +89,27 @@ def _get_whisper_model() -> tuple[object, str | None]:
                 detail="Transcription requires the faster-whisper optional dependency.",
             )
             cfg = get_whisper_config()
+
+            import os
+
+            explicit_model = os.environ.get("CLASSIFYRE_WHISPER_MODEL", "").strip()
+            if explicit_model:
+                model_name = cfg.model
+                model_source = "env"
+            else:
+                available_mb = _whisper_available_mb()
+                model_name = select_whisper_model(available_mb)
+                model_source = f"auto ({available_mb} MB available)"
+
             _whisper_state.model = whisper_module.WhisperModel(
-                cfg.model,
+                model_name,
                 device=cfg.device,
                 compute_type=cfg.compute_type,
             )
             logger.info(
-                "Loaded faster-whisper model %s (device=%s, compute_type=%s)",
-                cfg.model,
+                "Loaded faster-whisper model %s [%s] (device=%s, compute_type=%s)",
+                model_name,
+                model_source,
                 cfg.device,
                 cfg.compute_type,
             )
@@ -152,6 +165,51 @@ def _pcm_to_wav(pcm_bytes: bytes, sample_rate: int) -> bytes:
 _AUDIO_CHUNK_SECONDS = 600  # 10-minute chunks → ~38 MB decoded audio per chunk
 _TARGET_SAMPLE_RATE = 16_000
 
+# ---------------------------------------------------------------------------
+# Memory-aware model selection
+# ---------------------------------------------------------------------------
+
+# Budget to reserve for the detector worker process (presidio + spaCy) and
+# OS / Python overhead, before the remainder is available to the Whisper model.
+_DETECTOR_WORKER_RESERVE_MB = 1536  # ~1 GB presidio/spaCy + 512 MB headroom
+_OS_RESERVE_MB = 256
+
+# (model_name, min_available_mb) ordered largest → smallest.
+# min_available_mb accounts for beam-search working memory on top of the model
+# weights: tiny≈75 MB, base≈150 MB, small≈500 MB, medium≈1.5 GB.
+_WHISPER_MODEL_TIERS: tuple[tuple[str, int], ...] = (
+    ("medium", 2560),  # needs ≥ 2.5 GB available
+    ("small", 1000),  # needs ≥ 1 GB available
+    ("base", 512),  # needs ≥ 512 MB available
+    ("tiny", 0),  # always fits
+)
+
+
+def select_whisper_model(available_mb: int) -> str:
+    """Return the largest Whisper model that fits in *available_mb* of RAM.
+
+    Mirrors the logic of ``compute_pool_workers`` in the detector worker pool:
+    query the cgroup-aware memory budget, subtract reserved headroom, then pick
+    the best model tier that fits.  Callers pass *available_mb* so the function
+    is unit-testable without mocking cgroup files.
+
+    Example: a 4 GB pod with one detector worker has
+    ``4096 - 1536 - 256 = 2304 MB`` available → auto-selects "small" instead
+    of the default "medium", keeping peak RSS safely under the pod limit.
+    """
+    for model, threshold in _WHISPER_MODEL_TIERS:
+        if available_mb >= threshold:
+            return model
+    return "tiny"
+
+
+def _whisper_available_mb() -> int:
+    """Return MB available to the Whisper model after reserving headroom."""
+    from .resources import get_effective_memory_mb
+
+    total = get_effective_memory_mb()
+    return max(0, total - _DETECTOR_WORKER_RESERVE_MB - _OS_RESERVE_MB)
+
 
 def _split_audio_chunks(
     file_bytes: bytes,