classifyre-cli 0.4.78__tar.gz → 0.4.80__tar.gz

classifyre_cli-0.4.80/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 307 packages in 10ms
+Checked 51 packages in 7ms

{classifyre_cli-0.4.78 → classifyre_cli-0.4.80}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.78
+Version: 0.4.80
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.78 → classifyre_cli-0.4.80}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.78",
+  "version": "0.4.80",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.78 → classifyre_cli-0.4.80}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.78"
+version = "0.4.80"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -246,6 +246,16 @@ youtube = [
     "yt-dlp>=2025.1.0",
     "youtube-transcript-api>=1.0.0",
 ]
+reddit = [
+    # Official Reddit API wrapper. Handles OAuth (all three PRAW flows), token
+    # refresh and Reddit's own ratelimit headers.
+    "praw>=7.8.1,<8.0.0",
+    # Reddit posts routinely embed YouTube/Vimeo links; those are analysed with
+    # the same yt-dlp + captions + Whisper + frame-OCR pipeline the YouTube
+    # source uses (src/utils/external_video.py).
+    "yt-dlp>=2025.1.0",
+    "youtube-transcript-api>=1.0.0",
+]
 delta-lake = [
     # Pure-Python/Rust Delta reader (delta-rs) — no Spark, no JVM. Table metadata
     # and data-file lists come from deltalake; rows are sampled through DuckDB

{classifyre_cli-0.4.78 → classifyre_cli-0.4.80}/src/file_evaluation/runner.py

@@ -9,7 +9,7 @@ from pathlib import Path
 from typing import Any
 
 from ..models.generated_single_asset_scan_results import DetectionResult, Location
-from ..utils.embedded_images import has_embedded_images, iter_embedded_images
+from ..utils.embedded_files import has_embedded_files, iter_embedded_files
 from ..utils.file_parser import ParsedFile, parse_file
 
 logger = logging.getLogger(__name__)
@@ -148,11 +148,11 @@ class FileEvaluationRunner:
                     file_mime,
                 )
 
-        # Files that embed images (parquet image datasets, office docs) get each
-        # embedded image run through the image/binary detectors directly, with
-        # findings tagged by the embedded-image location so the UI can group them.
-        embedded_images = has_embedded_images(mime_type)
-        if not tasks and not embedded_images:
+        # Files that embed whole other files (parquet file columns, office media)
+        # get each embedded file run through the binary detectors directly, with
+        # findings tagged by the embedded location so the UI can group them.
+        embedded_files = has_embedded_files(mime_type)
+        if not tasks and not embedded_files:
             if not text.strip() and not file_mime:
                 logger.warning(
                     "No text content extracted from %s file; skipping text detectors.",
@@ -186,9 +186,9 @@ class FileEvaluationRunner:
                         )
                     )
 
-        if embedded_images:
+        if embedded_files:
             all_findings.extend(
-                await self._run_embedded_image_detectors(
+                await self._run_embedded_file_detectors(
                     detectors=detectors,
                     file_bytes=_get_raw_bytes(),
                     mime_type=mime_type,
@@ -198,7 +198,7 @@ class FileEvaluationRunner:
 
         return parsed, all_findings
 
-    async def _run_embedded_image_detectors(
+    async def _run_embedded_file_detectors(
         self,
         *,
         detectors: list[Any],
@@ -206,60 +206,65 @@ class FileEvaluationRunner:
         mime_type: str,
         detected_at: datetime,
     ) -> list[DetectionResult]:
-        """Run image/binary detectors over each image embedded in a container file.
+        """Run binary detectors over each file embedded in a container file.
 
-        Sequential by design (one image, one detector at a time) so the job/scan
+        Unwindowed on purpose: file evaluation judges one file the user handed over,
+        so there is no run-to-run cursor to advance. A scan of the same container
+        from a source expands its embedded files into child assets instead, one
+        sampling window at a time.
+
+        Sequential by design (one file, one detector at a time) so the job/scan
         logs read in order and each detector's outcome is attributable.
         """
         binary_detectors = [d for d in detectors if self._is_binary_detector(d)]
         if not binary_detectors:
             logger.info(
-                "File evaluation: %s embeds images but no image-capable detectors are configured; "
-                "skipping embedded-image scan",
+                "File evaluation: %s embeds files but no binary-capable detectors are configured; "
+                "skipping embedded-file scan",
                 mime_type,
             )
             return []
 
         logger.info(
-            "File evaluation: scanning images embedded in %s with image detectors: [%s]",
+            "File evaluation: scanning files embedded in %s with binary detectors: [%s]",
             mime_type,
             ", ".join(self._detector_label(d) for d in binary_detectors),
         )
 
         findings: list[DetectionResult] = []
-        image_count = 0
-        for image in iter_embedded_images(file_bytes, mime_type):
-            image_count += 1
-            size_kb = len(image.image_bytes) / 1024
+        embedded_count = 0
+        for embedded in iter_embedded_files(file_bytes, mime_type):
+            embedded_count += 1
+            size_kb = len(embedded.file_bytes) / 1024
             compatible = [
                 d
                 for d in binary_detectors
-                if self._supports_mime(d.get_supported_content_types(), image.mime_type)
+                if self._supports_mime(d.get_supported_content_types(), embedded.mime_type)
             ]
             if not compatible:
                 logger.info(
-                    "File evaluation: embedded image %s (%s, %.0f KB) — no compatible detector",
-                    image.location,
-                    image.mime_type,
+                    "File evaluation: embedded file %s (%s, %.0f KB) — no compatible detector",
+                    embedded.location,
+                    embedded.mime_type,
                     size_kb,
                 )
                 continue
             logger.info(
-                "File evaluation: embedded image %s (%s, %.0f KB) → dispatching [%s]",
-                image.location,
-                image.mime_type,
+                "File evaluation: embedded file %s (%s, %.0f KB) → dispatching [%s]",
+                embedded.location,
+                embedded.mime_type,
                 size_kb,
                 ", ".join(self._detector_label(d) for d in compatible),
             )
             for detector in compatible:
                 label = self._detector_label(detector)
                 try:
-                    result = await detector.detect(image.image_bytes, image.mime_type)
+                    result = await detector.detect(embedded.file_bytes, embedded.mime_type)
                 except Exception as exc:
                     logger.error(
-                        "File evaluation: '%s' failed on embedded image %s: %s",
+                        "File evaluation: '%s' failed on embedded file %s: %s",
                         label,
-                        image.location,
+                        embedded.location,
                         exc,
                     )
                     continue
@@ -269,17 +274,17 @@ class FileEvaluationRunner:
                     else []
                 )
                 logger.info(
-                    "File evaluation: '%s' on embedded image %s → %d finding(s)",
+                    "File evaluation: '%s' on embedded file %s → %d finding(s)",
                     label,
-                    image.location,
+                    embedded.location,
                     len(detector_findings),
                 )
                 for finding in detector_findings:
-                    findings.append(self._tag_embedded(finding, image.location, detected_at))
+                    findings.append(self._tag_embedded(finding, embedded.location, detected_at))
 
         logger.info(
-            "File evaluation: embedded-image scan complete — %d image(s) scanned, %d finding(s)",
-            image_count,
+            "File evaluation: embedded-file scan complete — %d file(s) scanned, %d finding(s)",
+            embedded_count,
             len(findings),
         )
         return findings
@@ -299,7 +304,7 @@ class FileEvaluationRunner:
     def _tag_embedded(
         finding: DetectionResult, location_label: str, detected_at: datetime
     ) -> DetectionResult:
-        """Attach the embedded-image location to a finding for parent-file grouping."""
+        """Attach the embedded-file location to a finding for parent-file grouping."""
         metadata = dict(finding.metadata or {})
         metadata["embedded_location"] = location_label
         location = finding.location or Location()

{classifyre_cli-0.4.78 → classifyre_cli-0.4.80}/src/main.py

@@ -265,9 +265,22 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                             worker_pool=worker_pool,
                         )
 
+                    from .pipeline.payload_window import PayloadWindowStore
                     from .pipeline.scan_cache import ScanCache
 
-                    scan_cache = ScanCache(recipe, source)
+                    # Bounds how much of each asset a run reads, not just how many
+                    # assets it reads. The scan cache consults it before skipping,
+                    # so a part-scanned payload is never mistaken for a finished one.
+                    payload_windows = PayloadWindowStore.from_recipe(recipe)
+                    if payload_windows.enabled:
+                        logger.info(
+                            "Payload sampling active (%s): tabular assets are read %d rows per run",
+                            payload_windows.strategy,
+                            payload_windows.rows_per_page,
+                        )
+                    source.attach_payload_windows(payload_windows)
+
+                    scan_cache = ScanCache(recipe, source, payload_windows=payload_windows)
                     if scan_cache.enabled:
                         logger.info(
                             "Scan cache active (verify=%s): unchanged assets and"
@@ -292,6 +305,13 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                         for stub in stub_batch:
                             stub["findings"] = []
 
+                        # Teach the payload windows each asset's ids and checksum
+                        # before any window is built: a stored row offset only
+                        # resumes when it still matches the file it was measured
+                        # against.
+                        for asset in raw_batch:
+                            payload_windows.bind(asset)
+
                         # Register BEFORE ingesting: the API stamps per-run
                         # change_type onto runner_assets rows during ingest, so
                         # the row must exist or the CREATED stamp is lost and
@@ -303,9 +323,15 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                         # or not at all.
                         hashes = [s["hash"] for s in stub_batch if s.get("hash")]
                         if hasattr(sink, "register_discovered_assets") and hashes:
+                            want_cursors = payload_windows.persists_cursor
                             prior = await sink.register_discovered_assets(
-                                hashes, include_scan_cache=scan_cache.enabled
+                                hashes,
+                                include_scan_cache=scan_cache.enabled,
+                                include_payload_cursor=want_cursors,
                             )
+                            if want_cursors and isinstance(prior, tuple):
+                                prior, payload_cursors = prior
+                                payload_windows.record_prior(payload_cursors)
                             scan_cache.record_prior(prior)
 
                         await sink.emit_batch(stub_batch, skip_findings=True)
@@ -477,6 +503,10 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                                 if state is not None:
                                     payload["scan_cache"] = state
 
+                                cursor = payload_windows.cursor_payload_for(asset)
+                                if cursor is not None:
+                                    payload["payload_cursor"] = cursor
+
                                 await sink.emit_batch([payload], skip_findings=False)
 
                                 if hasattr(sink, "update_asset_status"):

{classifyre_cli-0.4.78 → classifyre_cli-0.4.80}/src/models/generated_detectors.py

@@ -75,6 +75,7 @@ class SourceAssetType(StrEnum):
     ORACLE = 'ORACLE'
     HIVE = 'HIVE'
     POWERBI = 'POWERBI'
+    REDDIT = 'REDDIT'
     SANDBOX = 'SANDBOX'