classifyre-cli 0.4.3__tar.gz → 0.4.4__tar.gz

{classifyre_cli-0.4.3 → classifyre_cli-0.4.4}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 256 packages in 159ms
+Resolved 256 packages in 170ms
 Checked 49 packages in 1ms

{classifyre_cli-0.4.3 → classifyre_cli-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.3
+Version: 0.4.4
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.3 → classifyre_cli-0.4.4}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.3",
+  "version": "0.4.4",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.3 → classifyre_cli-0.4.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.3"
+version = "0.4.4"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

{classifyre_cli-0.4.3 → classifyre_cli-0.4.4}/src/main.py

@@ -243,14 +243,23 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                         import asyncio as _asyncio
 
                         processed_count = 0
+                        _pw = worker_pool.max_workers if worker_pool else 4
+                        max_concurrent = args.max_concurrent_assets or (_pw * 2)
+                        max_concurrent = max(1, max_concurrent)
+                        _asset_semaphore = _asyncio.Semaphore(max_concurrent)
                         logger.info(
-                            "Phase 2 starting: %d assets, pool_workers=%s",
+                            "Phase 2 starting: %d assets, pool_workers=%s, max_concurrent_assets=%d",
                             len(all_stubs),
                             worker_pool.max_workers if worker_pool else "none",
+                            max_concurrent,
                         )
                         error_count = 0
 
                         async def _process_one(asset: Any) -> None:
+                            async with _asset_semaphore:
+                                await _process_one_inner(asset)
+
+                        async def _process_one_inner(asset: Any) -> None:
                             nonlocal processed_count, error_count
                             asset_hash = getattr(asset, "hash", None) or ""
                             try:
@@ -553,6 +562,12 @@ def main() -> None:
         default=None,
         help="Max OS processes in the detector pool. Auto-sized from CPU/memory when omitted (env: CLASSIFYRE_MAX_POOL_WORKERS)",
     )
+    parser.add_argument(
+        "--max-concurrent-assets",
+        type=int,
+        default=None,
+        help="Max assets processed concurrently in Phase 2. Controls DB connection usage. Defaults to pool_workers*2 (env: CLASSIFYRE_MAX_CONCURRENT_ASSETS)",
+    )
 
     args = parser.parse_args()
 
@@ -571,6 +586,13 @@ def main() -> None:
         except ValueError:
             args.max_pool_workers = None
 
+    if args.max_concurrent_assets is None:
+        env_val = os.environ.get("CLASSIFYRE_MAX_CONCURRENT_ASSETS")
+        try:
+            args.max_concurrent_assets = int(env_val) if env_val else None
+        except ValueError:
+            args.max_concurrent_assets = None
+
     if args.debug:
         logging.getLogger().setLevel(logging.DEBUG)
 
@@ -591,6 +613,15 @@ def main() -> None:
 
     recipe = load_recipe(args.recipe)
 
+    # Resolve resource overrides from recipe when CLI args / env vars are not set
+    recipe_resources = recipe.get("resources") or {}
+    if args.max_pool_workers is None and isinstance(recipe_resources.get("max_pool_workers"), int):
+        args.max_pool_workers = recipe_resources["max_pool_workers"]
+    if args.max_concurrent_assets is None and isinstance(
+        recipe_resources.get("max_concurrent_assets"), int
+    ):
+        args.max_concurrent_assets = recipe_resources["max_concurrent_assets"]
+
     source_type = recipe.get("type", "").lower()
     if not source_type:
         logger.error(

{classifyre_cli-0.4.3 → classifyre_cli-0.4.4}/src/models/generated_input.py

@@ -41,6 +41,7 @@ class AssetType(StrEnum):
     CONFLUENCE = 'CONFLUENCE'
     JIRA = 'JIRA'
     SERVICEDESK = 'SERVICEDESK'
+    SQLITE = 'SQLITE'
 
 
 class SourceCategory(StrEnum):
@@ -169,6 +170,12 @@ class ResourceOverrides(BaseModel):
         ge=1,
         le=16,
     )
+    max_concurrent_assets: int | None = Field(
+        None,
+        description='Max assets processed concurrently. Controls parallel DB connections. Defaults to pool_workers * 2 when omitted.',
+        ge=1,
+        le=50,
+    )
 
 
 class Detector(BaseModel):
@@ -1836,6 +1843,7 @@ class Type(StrEnum):
     CONFLUENCE = 'CONFLUENCE'
     JIRA = 'JIRA'
     SERVICEDESK = 'SERVICEDESK'
+    SQLITE = 'SQLITE'
 
 
 class SlackInput(CoreInput):
@@ -2622,6 +2630,7 @@ class Type17(StrEnum):
     CONFLUENCE = 'CONFLUENCE'
     JIRA = 'JIRA'
     SERVICEDESK = 'SERVICEDESK'
+    SQLITE = 'SQLITE'
 
 
 class ConfluenceInput(CoreInput):
@@ -2676,6 +2685,59 @@ class ServiceDeskInput(CoreInput):
     resources: ResourceOverrides | None = None
 
 
+class SQLiteRequired(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    database_path: str = Field(
+        ...,
+        description='Absolute or relative path to the SQLite database file (e.g. /data/app.db)',
+    )
+
+
+class SQLiteOptionalScope(BaseModel):
+    """
+    Table selection scope.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    include_tables: list[str] | None = Field(
+        None,
+        description='Optional table allowlist. Only tables in this list will be scanned.',
+    )
+    table_limit: int | None = Field(
+        None, description='Optional cap on number of table assets extracted', ge=1
+    )
+
+
+class SQLiteOptional(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    scope: SQLiteOptionalScope | None = None
+
+
+class SQLiteInput(CoreInput):
+    type: Literal['SQLITE'] = Field('SQLITE', description='Type of the asset or source')
+    required: SQLiteRequired
+    masked: dict[str, Any] | None = Field(
+        None,
+        description='SQLite has no credentials; this section is intentionally empty.',
+    )
+    optional: SQLiteOptional | None = None
+    detectors: list[Detector] | None = Field(
+        None, description='Detectors to run on ingested content'
+    )
+    custom_detectors: list[CustomDetectorSelection] | None = Field(
+        None,
+        description='Reusable custom detector IDs selected from the custom detector catalog.',
+    )
+    sampling: SamplingConfig
+    resources: ResourceOverrides | None = None
+
+
 class SourceInput(
     RootModel[
         SlackInput
@@ -2697,6 +2759,7 @@ class SourceInput(
         | ConfluenceInput
         | JiraInput
         | ServiceDeskInput
+        | SQLiteInput
     ]
 ):
     root: (
@@ -2719,6 +2782,7 @@ class SourceInput(
         | ConfluenceInput
         | JiraInput
         | ServiceDeskInput
+        | SQLiteInput
     ) = Field(
         ...,
         description='Merged configuration schema with all source types and common definitions',

{classifyre_cli-0.4.3 → classifyre_cli-0.4.4}/src/pipeline/detector_pipeline.py

@@ -66,9 +66,7 @@ class DetectorPipeline:
             self.content_provider = ParsedContentProvider(source)
         self.init_warnings: list[str] = []
 
-    def _register_detector_info(
-        self, detector: BaseDetector, info: _DetectorInfo
-    ) -> None:
+    def _register_detector_info(self, detector: BaseDetector, info: _DetectorInfo) -> None:
         self._detector_info[id(detector)] = info
 
     def _get_detector_info(self, detector: BaseDetector) -> _DetectorInfo | None:
@@ -156,9 +154,7 @@ class DetectorPipeline:
         all_active = text_detectors + binary_detectors + link_detectors
         detector_names = [self._detector_log_label(d) for d in all_active]
         pool_tag = "[pool]" if self._worker_pool else "[in-process]"
-        logger.info(
-            "%s Scanning %s [%s]", pool_tag, asset.name, ", ".join(detector_names)
-        )
+        logger.info("%s Scanning %s [%s]", pool_tag, asset.name, ", ".join(detector_names))
 
         findings: list[DetectionResult] = []
         detector_types_run: list[DetectorType] = []
@@ -241,12 +237,13 @@ class DetectorPipeline:
         if findings:
             logger.info(
                 "%s Scanned %s: %d finding(s) in %dms",
-                pool_tag, asset.name, len(findings), scan_duration,
+                pool_tag,
+                asset.name,
+                len(findings),
+                scan_duration,
             )
         else:
-            logger.info(
-                "%s Scanned %s: no findings (%dms)", pool_tag, asset.name, scan_duration
-            )
+            logger.info("%s Scanned %s: no findings (%dms)", pool_tag, asset.name, scan_duration)
 
         return asset
 
@@ -299,7 +296,10 @@ class DetectorPipeline:
             elapsed = int((time.monotonic() - t0) * 1000)
             logger.info(
                 "  %s page %d done: %d findings (%dms)",
-                asset.name, page_num, len(page_findings), elapsed,
+                asset.name,
+                page_num,
+                len(page_findings),
+                elapsed,
             )
             return page_findings, page_types, page_errors, page_content, page_num
 
@@ -322,9 +322,7 @@ class DetectorPipeline:
                         page_content,
                     )
 
-        max_pending = max(
-            2, self._worker_pool.max_workers * 2 if self._worker_pool else 4
-        )
+        max_pending = max(2, self._worker_pool.max_workers * 2 if self._worker_pool else 4)
 
         async for text_content in self._iter_text_content_pages(asset):
             page_index += 1
@@ -335,18 +333,22 @@ class DetectorPipeline:
 
             while len(pending_tasks) >= max_pending:
                 done, pending_tasks = await asyncio.wait(
-                    pending_tasks, return_when=asyncio.FIRST_COMPLETED,
+                    pending_tasks,
+                    return_when=asyncio.FIRST_COMPLETED,
                 )
                 for task in done:
                     page_findings, page_types, page_errors, page_content, _pn = task.result()
                     findings.extend(page_findings)
                     errors.extend(page_errors)
                     detector_types_run = self._merge_detector_types(
-                        detector_types_run, page_types,
+                        detector_types_run,
+                        page_types,
                     )
                     for finding in page_findings:
                         self.content_provider.enrich_finding_location(
-                            finding, asset, page_content,
+                            finding,
+                            asset,
+                            page_content,
                         )
 
             task = asyncio.create_task(_detect_page(text_content, page_index))
@@ -401,7 +403,10 @@ class DetectorPipeline:
             elapsed = int((time.monotonic() - t0) * 1000)
             logger.info(
                 "  %s page %d done: %d findings (%dms)",
-                asset.name, page_num, len(page_findings), elapsed,
+                asset.name,
+                page_num,
+                len(page_findings),
+                elapsed,
             )
             return page_findings, page_types, page_errors, page_content, page_num
 
@@ -413,26 +418,29 @@ class DetectorPipeline:
                 page_findings, page_types, page_errors, page_content, _pn = task.result()
                 for finding in page_findings:
                     self.content_provider.enrich_finding_location(
-                        finding, asset, page_content,
+                        finding,
+                        asset,
+                        page_content,
                     )
                 findings.extend(page_findings)
                 errors.extend(page_errors)
                 detector_types_run = self._merge_detector_types(
-                    detector_types_run, page_types,
+                    detector_types_run,
+                    page_types,
                 )
                 unflushed_count += len(page_findings)
 
             if unflushed_count >= findings_flush_size and unflushed_count > 0:
                 logger.debug(
                     "  %s flushing %d findings (%d total)",
-                    asset.name, unflushed_count, len(findings),
+                    asset.name,
+                    unflushed_count,
+                    len(findings),
                 )
                 await on_findings_flushed(list(findings))
                 unflushed_count = 0
 
-        max_pending = max(
-            2, self._worker_pool.max_workers * 2 if self._worker_pool else 4
-        )
+        max_pending = max(2, self._worker_pool.max_workers * 2 if self._worker_pool else 4)
 
         async for text_content in self._iter_text_content_pages(asset):
             page_index += 1
@@ -443,25 +451,31 @@ class DetectorPipeline:
 
             while len(pending_tasks) >= max_pending:
                 done, pending_tasks_set = await asyncio.wait(
-                    pending_tasks, return_when=asyncio.FIRST_COMPLETED,
+                    pending_tasks,
+                    return_when=asyncio.FIRST_COMPLETED,
                 )
                 pending_tasks = pending_tasks_set
                 for task in done:
                     page_findings, page_types, page_errors, page_content, _pn = task.result()
                     for finding in page_findings:
                         self.content_provider.enrich_finding_location(
-                            finding, asset, page_content,
+                            finding,
+                            asset,
+                            page_content,
                         )
                     findings.extend(page_findings)
                     errors.extend(page_errors)
                     detector_types_run = self._merge_detector_types(
-                        detector_types_run, page_types,
+                        detector_types_run,
+                        page_types,
                     )
                     unflushed_count += len(page_findings)
                 if unflushed_count >= findings_flush_size and unflushed_count > 0:
                     logger.info(
                         "  %s flushing %d findings (%d total)",
-                        asset.name, unflushed_count, len(findings),
+                        asset.name,
+                        unflushed_count,
+                        len(findings),
                     )
                     await on_findings_flushed(list(findings))
                     unflushed_count = 0
@@ -637,9 +651,7 @@ class DetectorPipeline:
         errors: list[str] = []
         detected_at = datetime.now(UTC)
 
-        for i, (detector, result) in enumerate(
-            zip(runnable_detectors, results, strict=False)
-        ):
+        for i, (detector, result) in enumerate(zip(runnable_detectors, results, strict=False)):
             detector_name = detector.__class__.__name__
             via = task_via[i]
             loc = f"{asset_name}:{page_tag}" if page_tag else asset_name
@@ -648,7 +660,11 @@ class DetectorPipeline:
                 wall_ms = int((time.monotonic() - task_start_times[i]) * 1000)
                 logger.error(
                     "  [%s] %s on %s: FAILED in %dms — %s",
-                    via, detector_name, loc, wall_ms, result,
+                    via,
+                    detector_name,
+                    loc,
+                    wall_ms,
+                    result,
                 )
                 errors.append(f"{detector_name}: {result}")
                 continue
@@ -677,12 +693,19 @@ class DetectorPipeline:
             if detector_findings:
                 logger.info(
                     "  [%s] %s on %s: %d finding(s) in %dms",
-                    pid_tag, detector_name, loc, len(detector_findings), worker_elapsed,
+                    pid_tag,
+                    detector_name,
+                    loc,
+                    len(detector_findings),
+                    worker_elapsed,
                 )
             else:
                 logger.info(
                     "  [%s] %s on %s: clean (%dms)",
-                    pid_tag, detector_name, loc, worker_elapsed,
+                    pid_tag,
+                    detector_name,
+                    loc,
+                    worker_elapsed,
                 )
 
             all_findings.extend(detector_findings)
@@ -835,7 +858,9 @@ class DetectorPipeline:
 
         if not detector_configs:
             return cls(
-                detectors=[], source=source, runner_id=runner_id,
+                detectors=[],
+                source=source,
+                runner_id=runner_id,
                 worker_pool=worker_pool,
             )
 

{classifyre_cli-0.4.3 → classifyre_cli-0.4.4}/src/pipeline/worker_pool.py

@@ -37,9 +37,7 @@ class _WorkerResult:
 
     __slots__ = ("elapsed_ms", "findings", "worker_pid")
 
-    def __init__(
-        self, findings: list[dict[str, Any]], worker_pid: int, elapsed_ms: int
-    ) -> None:
+    def __init__(self, findings: list[dict[str, Any]], worker_pid: int, elapsed_ms: int) -> None:
         self.findings = findings
         self.worker_pid = worker_pid
         self.elapsed_ms = elapsed_ms
@@ -80,9 +78,7 @@ def _detect_in_worker(
             from ..detectors import get_detector
             from ..detectors.config import parse_detector_config
 
-            name, typed_config = parse_detector_config(
-                detector_type, json.loads(config_json)
-            )
+            name, typed_config = parse_detector_config(detector_type, json.loads(config_json))
             detector = get_detector(name, typed_config)
             _worker_detector_cache[cache_key] = detector
             logging.getLogger(__name__).info(
@@ -103,7 +99,9 @@ def _detect_in_worker(
         elif detector_name == "custom":
             results = detector._detect_sync(content, content_type)
         else:
-            text = content if isinstance(content, str) else content.decode("utf-8", errors="replace")
+            text = (
+                content if isinstance(content, str) else content.decode("utf-8", errors="replace")
+            )
             results = detector._detect_sync(text)
     else:
         results = asyncio.run(detector.detect(content, content_type))
@@ -111,7 +109,10 @@ def _detect_in_worker(
     elapsed_ms = int((time.monotonic() - t0) * 1000)
     logging.getLogger(__name__).info(
         "Worker %d ran %s: %d findings in %dms",
-        pid, detector_name, len(results), elapsed_ms,
+        pid,
+        detector_name,
+        len(results),
+        elapsed_ms,
     )
 
     findings = [
@@ -209,7 +210,11 @@ def compute_pool_workers(override: int | None = None) -> int:
 
     logger.info(
         "Pool sizing: cpu_budget=%d (cpus=%d), mem_budget=%d (%dMB), effective=%d",
-        cpu_budget, cpus, mem_budget, mem_mb, effective,
+        cpu_budget,
+        cpus,
+        mem_budget,
+        mem_mb,
+        effective,
     )
     return effective
 
@@ -238,7 +243,9 @@ class DetectorWorkerPool:
         self._shutdown = False
         logger.info(
             "Detector pool started: %d workers (method=%s, pid=%d)",
-            effective_workers, mp_start_method, os.getpid(),
+            effective_workers,
+            mp_start_method,
+            os.getpid(),
         )
 
     @property