classifyre-cli 0.4.2__tar.gz → 0.4.4__tar.gz

classifyre_cli-0.4.4/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 256 packages in 170ms
+Checked 49 packages in 1ms

{classifyre_cli-0.4.2 → classifyre_cli-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.2
+Version: 0.4.4
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.2 → classifyre_cli-0.4.4}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.2",
+  "version": "0.4.4",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.2 → classifyre_cli-0.4.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.2"
+version = "0.4.4"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

{classifyre_cli-0.4.2 → classifyre_cli-0.4.4}/src/detectors/custom/detector.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import logging
 
 from ...models.generated_detectors import (
@@ -34,6 +35,11 @@ class CustomDetector(BaseDetector):
 
     async def detect(
         self, content: str | bytes, content_type: str = "text/plain"
+    ) -> list[DetectionResult]:
+        return await asyncio.to_thread(self._detect_sync, content, content_type)
+
+    def _detect_sync(
+        self, content: str | bytes, content_type: str = "text/plain"
     ) -> list[DetectionResult]:
         findings = self._runner.detect(content, content_type)
         max_findings = self.custom_config.max_findings

{classifyre_cli-0.4.2 → classifyre_cli-0.4.4}/src/detectors/secrets/detector.py

@@ -5,6 +5,7 @@ plugin's ``analyze_line`` directly.  No temp files, no global Settings state,
 and no ``SecretsCollection`` needed.
 """
 
+import asyncio
 import importlib
 import logging
 import pkgutil
@@ -304,7 +305,9 @@ class SecretsDetector(BaseDetector):
                     len(content),
                 )
                 return []
+        return await asyncio.to_thread(self._detect_sync, content)
 
+    def _detect_sync(self, content: str) -> list[DetectionResult]:
         plugins = self._build_plugins()
         if not plugins:
             return []

{classifyre_cli-0.4.2 → classifyre_cli-0.4.4}/src/detectors/threat/code_security_detector.py

@@ -1,5 +1,6 @@
 """Code security detector using Bandit static analysis."""
 
+import asyncio
 import json
 import logging
 import subprocess
@@ -137,7 +138,9 @@ class CodeSecurityDetector(BaseDetector):
             return []
         if not content.strip():
             return []
+        return await asyncio.to_thread(self._detect_sync, content)
 
+    def _detect_sync(self, content: str) -> list[DetectionResult]:
         threshold = self._cfg.confidence_threshold or 0.7
         max_findings = self._cfg.max_findings or 25
         findings: list[DetectionResult] = []

{classifyre_cli-0.4.2 → classifyre_cli-0.4.4}/src/detectors/threat/yara_detector.py

@@ -1,5 +1,6 @@
 """YARA-based threat detector — compiles structured rule objects into a live ruleset."""
 
+import asyncio
 import logging
 import re
 
@@ -89,6 +90,13 @@ class YaraDetector(BaseDetector):
 
     async def detect(
         self, content: str | bytes, content_type: str = "text/plain"
+    ) -> list[DetectionResult]:
+        if self._rules is None:
+            return []
+        return await asyncio.to_thread(self._detect_sync, content, content_type)
+
+    def _detect_sync(
+        self, content: str | bytes, content_type: str = "text/plain"
     ) -> list[DetectionResult]:
         if self._rules is None:
             return []

{classifyre_cli-0.4.2 → classifyre_cli-0.4.4}/src/main.py

@@ -179,15 +179,32 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                     sink_started = True
 
                     from .pipeline.detector_pipeline import DetectorPipeline
+                    from .pipeline.worker_pool import (
+                        DetectorWorkerPool,
+                        compute_pool_workers,
+                    )
+
+                    pool_workers = compute_pool_workers(
+                        override=args.max_pool_workers,
+                    )
+                    worker_pool: DetectorWorkerPool | None = None
 
                     pipeline = DetectorPipeline.from_recipe(
                         recipe,
                         source,
                         runner_id,
-                        max_concurrent_assets=args.detector_max_concurrent,
                     )
                     has_detectors = bool(pipeline.detectors)
 
+                    if has_detectors:
+                        worker_pool = DetectorWorkerPool(max_workers=pool_workers)
+                        pipeline = DetectorPipeline.from_recipe(
+                            recipe,
+                            source,
+                            runner_id,
+                            worker_pool=worker_pool,
+                        )
+
                     # --- Phase 1: Discovery ---
                     source.set_discovery_only(True)
                     all_stubs: list[Any] = []
@@ -225,74 +242,84 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                     if has_detectors and all_stubs:
                         import asyncio as _asyncio
 
-                        workers = args.processing_workers
-                        semaphore = _asyncio.Semaphore(workers)
                         processed_count = 0
+                        _pw = worker_pool.max_workers if worker_pool else 4
+                        max_concurrent = args.max_concurrent_assets or (_pw * 2)
+                        max_concurrent = max(1, max_concurrent)
+                        _asset_semaphore = _asyncio.Semaphore(max_concurrent)
+                        logger.info(
+                            "Phase 2 starting: %d assets, pool_workers=%s, max_concurrent_assets=%d",
+                            len(all_stubs),
+                            worker_pool.max_workers if worker_pool else "none",
+                            max_concurrent,
+                        )
                         error_count = 0
 
                         async def _process_one(asset: Any) -> None:
-                            nonlocal processed_count, error_count
-                            async with semaphore:
-                                asset_hash = getattr(asset, "hash", None) or ""
-                                try:
-                                    if hasattr(sink, "update_asset_status"):
-                                        await sink.update_asset_status(asset_hash, "PROCESSING")
-
-                                    async def _on_findings_flushed(partial: list[Any]) -> None:
-                                        # partial is the full accumulated findings list from the pipeline
-                                        stub_payload = _asset_to_payload(asset)
-                                        stub_payload["findings"] = [
-                                            f.model_dump(mode="json", exclude_none=True)
-                                            if hasattr(f, "model_dump")
-                                            else f
-                                            for f in partial
-                                        ]
-                                        await sink.emit_batch([stub_payload], skip_findings=False)
-                                        if hasattr(sink, "update_asset_status"):
-                                            f_total, f_by_sev, f_by_det = _compute_findings_counts(
-                                                partial
-                                            )
-                                            await sink.update_asset_status(
-                                                asset_hash,
-                                                "PROCESSING",
-                                                findings_total=f_total,
-                                                findings_by_severity=f_by_sev,
-                                                findings_by_detector=f_by_det,
-                                            )
-
-                                    result = await pipeline.process_single_asset(
-                                        asset,
-                                        on_findings_flushed=_on_findings_flushed,
-                                        findings_flush_size=args.detector_flush_batch_size,
-                                    )
-                                    payload = _asset_to_payload(result)
-                                    await sink.emit_batch([payload], skip_findings=False)
+                            async with _asset_semaphore:
+                                await _process_one_inner(asset)
 
+                        async def _process_one_inner(asset: Any) -> None:
+                            nonlocal processed_count, error_count
+                            asset_hash = getattr(asset, "hash", None) or ""
+                            try:
+                                if hasattr(sink, "update_asset_status"):
+                                    await sink.update_asset_status(asset_hash, "PROCESSING")
+
+                                async def _on_findings_flushed(partial: list[Any]) -> None:
+                                    stub_payload = _asset_to_payload(asset)
+                                    stub_payload["findings"] = [
+                                        f.model_dump(mode="json", exclude_none=True)
+                                        if hasattr(f, "model_dump")
+                                        else f
+                                        for f in partial
+                                    ]
+                                    await sink.emit_batch([stub_payload], skip_findings=False)
                                     if hasattr(sink, "update_asset_status"):
                                         f_total, f_by_sev, f_by_det = _compute_findings_counts(
-                                            result.findings or []
+                                            partial
                                         )
                                         await sink.update_asset_status(
                                             asset_hash,
-                                            "PROCESSED",
+                                            "PROCESSING",
                                             findings_total=f_total,
                                             findings_by_severity=f_by_sev,
                                             findings_by_detector=f_by_det,
                                         )
 
-                                    source.evict_asset_cache(asset_hash)
-                                    processed_count += 1
-                                except Exception as exc:
-                                    error_count += 1
-                                    logger.error("Asset %s failed: %s", asset_hash, exc)
-                                    if hasattr(sink, "update_asset_status"):
-                                        try:
-                                            error_msg = str(exc) or type(exc).__name__
-                                            await sink.update_asset_status(
-                                                asset_hash, "ERROR", error_msg
-                                            )
-                                        except Exception:
-                                            pass
+                                result = await pipeline.process_single_asset(
+                                    asset,
+                                    on_findings_flushed=_on_findings_flushed,
+                                    findings_flush_size=args.detector_flush_batch_size,
+                                )
+                                payload = _asset_to_payload(result)
+                                await sink.emit_batch([payload], skip_findings=False)
+
+                                if hasattr(sink, "update_asset_status"):
+                                    f_total, f_by_sev, f_by_det = _compute_findings_counts(
+                                        result.findings or []
+                                    )
+                                    await sink.update_asset_status(
+                                        asset_hash,
+                                        "PROCESSED",
+                                        findings_total=f_total,
+                                        findings_by_severity=f_by_sev,
+                                        findings_by_detector=f_by_det,
+                                    )
+
+                                source.evict_asset_cache(asset_hash)
+                                processed_count += 1
+                            except Exception as exc:
+                                error_count += 1
+                                logger.error("Asset %s failed: %s", asset_hash, exc)
+                                if hasattr(sink, "update_asset_status"):
+                                    try:
+                                        error_msg = str(exc) or type(exc).__name__
+                                        await sink.update_asset_status(
+                                            asset_hash, "ERROR", error_msg
+                                        )
+                                    except Exception:
+                                        pass
 
                         tasks = [_asyncio.create_task(_process_one(a)) for a in all_stubs]
                         await _asyncio.gather(*tasks, return_exceptions=True)
@@ -340,6 +367,9 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                                 "Failed to mark sink failure: %s", sink_error, exc_info=True
                             )
                     raise
+                finally:
+                    if worker_pool is not None:
+                        worker_pool.shutdown(wait=True)
 
         except Exception as e:
             logger.debug("Traceback for %s failure:", args.command, exc_info=True)
@@ -527,16 +557,16 @@ def main() -> None:
         help="How many detector-processed assets to accumulate before pushing findings to the API (default: 5, env: CLASSIFYRE_DETECTOR_FLUSH_BATCH_SIZE)",
     )
     parser.add_argument(
-        "--detector-max-concurrent",
+        "--max-pool-workers",
         type=int,
         default=None,
-        help="Max assets processed in parallel by the detector pipeline (default: 10, env: CLASSIFYRE_DETECTOR_MAX_CONCURRENT)",
+        help="Max OS processes in the detector pool. Auto-sized from CPU/memory when omitted (env: CLASSIFYRE_MAX_POOL_WORKERS)",
     )
     parser.add_argument(
-        "--processing-workers",
+        "--max-concurrent-assets",
         type=int,
         default=None,
-        help="Number of parallel asset-processing workers in Phase 2 (default: 2, env: CLASSIFYRE_PROCESSING_WORKERS)",
+        help="Max assets processed concurrently in Phase 2. Controls DB connection usage. Defaults to pool_workers*2 (env: CLASSIFYRE_MAX_CONCURRENT_ASSETS)",
     )
 
     args = parser.parse_args()
@@ -549,21 +579,19 @@ def main() -> None:
             args.detector_flush_batch_size = 5
     args.detector_flush_batch_size = max(args.detector_flush_batch_size, 1)
 
-    if args.detector_max_concurrent is None:
-        env_val = os.environ.get("CLASSIFYRE_DETECTOR_MAX_CONCURRENT")
+    if args.max_pool_workers is None:
+        env_val = os.environ.get("CLASSIFYRE_MAX_POOL_WORKERS")
         try:
-            args.detector_max_concurrent = int(env_val) if env_val else 10
+            args.max_pool_workers = int(env_val) if env_val else None
         except ValueError:
-            args.detector_max_concurrent = 10
-    args.detector_max_concurrent = max(args.detector_max_concurrent, 1)
+            args.max_pool_workers = None
 
-    if args.processing_workers is None:
-        env_val = os.environ.get("CLASSIFYRE_PROCESSING_WORKERS")
+    if args.max_concurrent_assets is None:
+        env_val = os.environ.get("CLASSIFYRE_MAX_CONCURRENT_ASSETS")
         try:
-            args.processing_workers = int(env_val) if env_val else 2
+            args.max_concurrent_assets = int(env_val) if env_val else None
         except ValueError:
-            args.processing_workers = 2
-    args.processing_workers = max(args.processing_workers, 1)
+            args.max_concurrent_assets = None
 
     if args.debug:
         logging.getLogger().setLevel(logging.DEBUG)
@@ -585,6 +613,15 @@ def main() -> None:
 
     recipe = load_recipe(args.recipe)
 
+    # Resolve resource overrides from recipe when CLI args / env vars are not set
+    recipe_resources = recipe.get("resources") or {}
+    if args.max_pool_workers is None and isinstance(recipe_resources.get("max_pool_workers"), int):
+        args.max_pool_workers = recipe_resources["max_pool_workers"]
+    if args.max_concurrent_assets is None and isinstance(
+        recipe_resources.get("max_concurrent_assets"), int
+    ):
+        args.max_concurrent_assets = recipe_resources["max_concurrent_assets"]
+
     source_type = recipe.get("type", "").lower()
     if not source_type:
         logger.error(

{classifyre_cli-0.4.2 → classifyre_cli-0.4.4}/src/models/generated_input.py

@@ -41,6 +41,7 @@ class AssetType(StrEnum):
     CONFLUENCE = 'CONFLUENCE'
     JIRA = 'JIRA'
     SERVICEDESK = 'SERVICEDESK'
+    SQLITE = 'SQLITE'
 
 
 class SourceCategory(StrEnum):
@@ -163,15 +164,15 @@ class ResourceOverrides(BaseModel):
         ge=60,
         le=86400,
     )
-    processing_workers: int | None = Field(
+    max_pool_workers: int | None = Field(
         None,
-        description='Number of parallel asset-processing workers in Phase 2 (default: 2)',
+        description='Max OS processes in the detector pool. Auto-sized from CPU/memory limits when omitted.',
         ge=1,
-        le=20,
+        le=16,
     )
-    detector_max_concurrent: int | None = Field(
+    max_concurrent_assets: int | None = Field(
         None,
-        description='Max concurrent detector invocations across all pages (default: 5)',
+        description='Max assets processed concurrently. Controls parallel DB connections. Defaults to pool_workers * 2 when omitted.',
         ge=1,
         le=50,
     )
@@ -1842,6 +1843,7 @@ class Type(StrEnum):
     CONFLUENCE = 'CONFLUENCE'
     JIRA = 'JIRA'
     SERVICEDESK = 'SERVICEDESK'
+    SQLITE = 'SQLITE'
 
 
 class SlackInput(CoreInput):
@@ -2628,6 +2630,7 @@ class Type17(StrEnum):
     CONFLUENCE = 'CONFLUENCE'
     JIRA = 'JIRA'
     SERVICEDESK = 'SERVICEDESK'
+    SQLITE = 'SQLITE'
 
 
 class ConfluenceInput(CoreInput):
@@ -2682,6 +2685,59 @@ class ServiceDeskInput(CoreInput):
     resources: ResourceOverrides | None = None
 
 
+class SQLiteRequired(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    database_path: str = Field(
+        ...,
+        description='Absolute or relative path to the SQLite database file (e.g. /data/app.db)',
+    )
+
+
+class SQLiteOptionalScope(BaseModel):
+    """
+    Table selection scope.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    include_tables: list[str] | None = Field(
+        None,
+        description='Optional table allowlist. Only tables in this list will be scanned.',
+    )
+    table_limit: int | None = Field(
+        None, description='Optional cap on number of table assets extracted', ge=1
+    )
+
+
+class SQLiteOptional(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    scope: SQLiteOptionalScope | None = None
+
+
+class SQLiteInput(CoreInput):
+    type: Literal['SQLITE'] = Field('SQLITE', description='Type of the asset or source')
+    required: SQLiteRequired
+    masked: dict[str, Any] | None = Field(
+        None,
+        description='SQLite has no credentials; this section is intentionally empty.',
+    )
+    optional: SQLiteOptional | None = None
+    detectors: list[Detector] | None = Field(
+        None, description='Detectors to run on ingested content'
+    )
+    custom_detectors: list[CustomDetectorSelection] | None = Field(
+        None,
+        description='Reusable custom detector IDs selected from the custom detector catalog.',
+    )
+    sampling: SamplingConfig
+    resources: ResourceOverrides | None = None
+
+
 class SourceInput(
     RootModel[
         SlackInput
@@ -2703,6 +2759,7 @@ class SourceInput(
         | ConfluenceInput
         | JiraInput
         | ServiceDeskInput
+        | SQLiteInput
     ]
 ):
     root: (
@@ -2725,6 +2782,7 @@ class SourceInput(
         | ConfluenceInput
         | JiraInput
         | ServiceDeskInput
+        | SQLiteInput
     ) = Field(
         ...,
         description='Merged configuration schema with all source types and common definitions',