classifyre-cli 0.4.3__tar.gz → 0.4.5__tar.gz

{classifyre_cli-0.4.3 → classifyre_cli-0.4.5}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 256 packages in 159ms
+Resolved 256 packages in 203ms
 Checked 49 packages in 1ms

{classifyre_cli-0.4.3 → classifyre_cli-0.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.3
+Version: 0.4.5
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.3 → classifyre_cli-0.4.5}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.3",
+  "version": "0.4.5",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.3 → classifyre_cli-0.4.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.3"
+version = "0.4.5"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

classifyre_cli-0.4.5/src/detectors/custom/extractor.py

@@ -0,0 +1,261 @@
+"""Custom detector extraction engine — REGEX, GLINER, and CLASSIFIER_GLINER strategies."""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from ...models.generated_detectors import (
+    CustomDetectorMethod,
+    CustomExtractorConfig,
+    CustomExtractorField,
+)
+from ..dependencies import MissingDependencyError, require_module
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_GLINER2_MODEL = "fastino/gliner2-base-v1"
+
+# Extraction method tags sent to the API via DetectionResult.extraction_method
+EXTRACTION_METHOD_REGEX = "REGEX"
+EXTRACTION_METHOD_GLINER = "GLINER"
+EXTRACTION_METHOD_CLASSIFIER_GLINER = "CLASSIFIER_GLINER"
+
+
+@dataclass
+class ExtractionResult:
+    """Typed output from one extraction run."""
+
+    extracted_data: dict[str, Any]
+    method: str
+    populated_fields: list[str] = field(default_factory=list)
+    field_count: int = 0
+
+    def __post_init__(self) -> None:
+        self.populated_fields = [
+            k for k, v in self.extracted_data.items() if v is not None and v not in ([], "")
+        ]
+        self.field_count = len(self.extracted_data)
+
+
+class CustomExtractor:
+    """
+    Runs after a custom detector fires to pull structured data from the content.
+
+    Strategy selection:
+      RULESET     → REGEX    (named capture groups in field.regex_pattern)
+      ENTITY      → GLINER   (group GLiNER2 entity spans by entity_label into fields)
+      CLASSIFIER  → CLASSIFIER_GLINER  (second GLiNER2 pass on wider content slice)
+    """
+
+    def __init__(
+        self,
+        config: CustomExtractorConfig,
+        detector_method: CustomDetectorMethod,
+    ) -> None:
+        self._config = config
+        self._method = detector_method
+        self._gliner_model: Any | None = None
+        self._compiled: dict[str, re.Pattern[str]] = {}  # pattern cache
+
+    # ── Public API ───────────────────────────────────────────────────────────
+
+    def extract(
+        self,
+        matched_content: str,
+        content_for_extraction: str,
+    ) -> ExtractionResult | None:
+        """
+        Run extraction and return structured result, or None if nothing extracted.
+
+        Args:
+            matched_content:       The content stored in the finding (may be truncated).
+            content_for_extraction: Wider slice of the original document for GLiNER/regex.
+        """
+        if not self._config.enabled:
+            return None
+
+        if self._method == CustomDetectorMethod.RULESET:
+            return self._extract_regex(content_for_extraction)
+        if self._method == CustomDetectorMethod.ENTITY:
+            return self._extract_gliner(content_for_extraction, EXTRACTION_METHOD_GLINER)
+        if self._method == CustomDetectorMethod.CLASSIFIER:
+            return self._extract_gliner(content_for_extraction, EXTRACTION_METHOD_CLASSIFIER_GLINER)
+        logger.warning("CustomExtractor: unknown detector method %s", self._method)
+        return None
+
+    # ── RULESET — regex named groups ─────────────────────────────────────────
+
+    def _extract_regex(self, content: str) -> ExtractionResult | None:
+        data: dict[str, Any] = {}
+
+        for f in self._config.fields:
+            if not f.regex_pattern:
+                logger.debug(
+                    "Extractor field '%s' has no regex_pattern — skipped for RULESET", f.name
+                )
+                continue
+            value = self._apply_regex_field(content, f)
+            if value is not None:
+                data[f.name] = value
+
+        return self._finalize(data, EXTRACTION_METHOD_REGEX)
+
+    def _apply_regex_field(self, content: str, f: CustomExtractorField) -> Any:
+        pattern = self._compile(f.regex_pattern or "", f.regex_flags or "i")
+        if pattern is None:
+            return None
+
+        named_groups = pattern.groupindex
+        group_name = next(iter(named_groups), None)
+
+        matches: list[str] = []
+        for m in pattern.finditer(content):
+            captured = m.group(group_name) if group_name else m.group(0)
+            if captured and captured.strip():
+                matches.append(captured.strip())
+
+        return self._aggregate(matches, f) if matches else None
+
+    def _compile(self, pattern: str, flags_str: str) -> re.Pattern[str] | None:
+        cache_key = f"{pattern}::{flags_str}"
+        if cache_key in self._compiled:
+            return self._compiled[cache_key]
+
+        flags = 0
+        for ch in flags_str:
+            if ch == "i":
+                flags |= re.IGNORECASE
+            elif ch == "m":
+                flags |= re.MULTILINE
+            elif ch == "s":
+                flags |= re.DOTALL
+
+        try:
+            compiled = re.compile(pattern, flags=flags)
+            self._compiled[cache_key] = compiled
+            return compiled
+        except re.error as exc:
+            logger.warning("CustomExtractor: invalid regex pattern '%s': %s", pattern, exc)
+            return None
+
+    # ── ENTITY / CLASSIFIER — GLiNER2 entity spans ───────────────────────────
+
+    def _extract_gliner(self, content: str, method_tag: str) -> ExtractionResult | None:
+        label_to_fields: dict[str, list[CustomExtractorField]] = {}
+        for f in self._config.fields:
+            if f.entity_label:
+                label_to_fields.setdefault(f.entity_label, []).append(f)
+
+        if not label_to_fields:
+            logger.debug("CustomExtractor: no fields with entity_label — skipping GLiNER2")
+            return None
+
+        model = self._load_gliner()
+        if model is None:
+            return None
+
+        entity_schema = {
+            label: next(
+                (
+                    field.description
+                    for field in fields
+                    if isinstance(field.description, str) and field.description.strip()
+                ),
+                "",
+            )
+            for label, fields in label_to_fields.items()
+        }
+        try:
+            result = model.extract_entities(
+                content,
+                entity_schema,
+                threshold=0.0,
+                include_confidence=True,
+            )
+        except Exception as exc:  # pragma: no cover
+            logger.warning("CustomExtractor: GLiNER2 extraction failed: %s", exc)
+            return None
+
+        entities = result.get("entities", {})
+        if not isinstance(entities, dict):
+            return None
+
+        data: dict[str, Any] = {}
+        for entity_label, fields in label_to_fields.items():
+            raw_spans = entities.get(entity_label, [])
+            if not isinstance(raw_spans, list):
+                raw_spans = [raw_spans]
+
+            for f in fields:
+                threshold = f.min_confidence if f.min_confidence is not None else 0.4
+                values = self._filter_gliner2_values(raw_spans, threshold)
+                value = self._aggregate(values, f) if values else None
+                if value is not None:
+                    data[f.name] = value
+
+        return self._finalize(data, method_tag)
+
+    def _filter_gliner2_values(self, raw_spans: list[Any], threshold: float) -> list[str]:
+        values: list[str] = []
+        for raw_span in raw_spans:
+            if isinstance(raw_span, dict):
+                score = float(raw_span.get("confidence", raw_span.get("score", 0.0)))
+                text = str(raw_span.get("text", "")).strip()
+            else:
+                score = 1.0
+                text = str(raw_span).strip()
+
+            if score >= threshold and text:
+                values.append(text)
+
+        return values
+
+    def _load_gliner(self) -> Any | None:
+        if self._gliner_model is not None:
+            return self._gliner_model
+        try:
+            gliner2_module = require_module("gliner2", "custom", ["classification", "detectors"])
+            model_name = self._config.gliner_model or _DEFAULT_GLINER2_MODEL
+            self._gliner_model = gliner2_module.GLiNER2.from_pretrained(model_name)
+            return self._gliner_model
+        except MissingDependencyError:
+            raise
+        except Exception as exc:  # pragma: no cover
+            logger.warning("CustomExtractor: failed to load GLiNER2: %s", exc)
+            return None
+
+    # ── Shared helpers ────────────────────────────────────────────────────────
+
+    def _aggregate(self, values: list[str], f: CustomExtractorField) -> Any:
+        if not values:
+            return None
+        aggregate = f.aggregate or "list"
+        if aggregate == "first":
+            return values[0]
+        if aggregate == "last":
+            return values[-1]
+        if aggregate == "list":
+            return values
+        if aggregate == "join":
+            sep = f.join_separator if f.join_separator is not None else ", "
+            return sep.join(values)
+        if aggregate == "count":
+            return len(values)
+        return values  # fallback
+
+    def _finalize(self, data: dict[str, Any], method: str) -> ExtractionResult | None:
+        # Required fields gate: if any required field is missing, discard the result
+        for f in self._config.fields:
+            if f.required and f.name not in data:
+                logger.debug(
+                    "CustomExtractor: required field '%s' not populated — discarding", f.name
+                )
+                return None
+
+        if not data:
+            return None
+
+        return ExtractionResult(extracted_data=data, method=method)

{classifyre_cli-0.4.3 → classifyre_cli-0.4.5}/src/main.py

@@ -243,14 +243,23 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                         import asyncio as _asyncio
 
                         processed_count = 0
+                        _pw = worker_pool.max_workers if worker_pool else 4
+                        max_concurrent = args.max_concurrent_assets or (_pw * 2)
+                        max_concurrent = max(1, max_concurrent)
+                        _asset_semaphore = _asyncio.Semaphore(max_concurrent)
                         logger.info(
-                            "Phase 2 starting: %d assets, pool_workers=%s",
+                            "Phase 2 starting: %d assets, pool_workers=%s, max_concurrent_assets=%d",
                             len(all_stubs),
                             worker_pool.max_workers if worker_pool else "none",
+                            max_concurrent,
                         )
                         error_count = 0
 
                         async def _process_one(asset: Any) -> None:
+                            async with _asset_semaphore:
+                                await _process_one_inner(asset)
+
+                        async def _process_one_inner(asset: Any) -> None:
                             nonlocal processed_count, error_count
                             asset_hash = getattr(asset, "hash", None) or ""
                             try:
@@ -553,6 +562,12 @@ def main() -> None:
         default=None,
         help="Max OS processes in the detector pool. Auto-sized from CPU/memory when omitted (env: CLASSIFYRE_MAX_POOL_WORKERS)",
     )
+    parser.add_argument(
+        "--max-concurrent-assets",
+        type=int,
+        default=None,
+        help="Max assets processed concurrently in Phase 2. Controls DB connection usage. Defaults to pool_workers*2 (env: CLASSIFYRE_MAX_CONCURRENT_ASSETS)",
+    )
 
     args = parser.parse_args()
 
@@ -571,6 +586,13 @@ def main() -> None:
         except ValueError:
             args.max_pool_workers = None
 
+    if args.max_concurrent_assets is None:
+        env_val = os.environ.get("CLASSIFYRE_MAX_CONCURRENT_ASSETS")
+        try:
+            args.max_concurrent_assets = int(env_val) if env_val else None
+        except ValueError:
+            args.max_concurrent_assets = None
+
     if args.debug:
         logging.getLogger().setLevel(logging.DEBUG)
 
@@ -591,6 +613,15 @@ def main() -> None:
 
     recipe = load_recipe(args.recipe)
 
+    # Resolve resource overrides from recipe when CLI args / env vars are not set
+    recipe_resources = recipe.get("resources") or {}
+    if args.max_pool_workers is None and isinstance(recipe_resources.get("max_pool_workers"), int):
+        args.max_pool_workers = recipe_resources["max_pool_workers"]
+    if args.max_concurrent_assets is None and isinstance(
+        recipe_resources.get("max_concurrent_assets"), int
+    ):
+        args.max_concurrent_assets = recipe_resources["max_concurrent_assets"]
+
     source_type = recipe.get("type", "").lower()
     if not source_type:
         logger.error(

{classifyre_cli-0.4.3 → classifyre_cli-0.4.5}/src/models/generated_input.py

@@ -41,6 +41,7 @@ class AssetType(StrEnum):
     CONFLUENCE = 'CONFLUENCE'
     JIRA = 'JIRA'
     SERVICEDESK = 'SERVICEDESK'
+    SQLITE = 'SQLITE'
 
 
 class SourceCategory(StrEnum):
@@ -169,6 +170,12 @@ class ResourceOverrides(BaseModel):
         ge=1,
         le=16,
     )
+    max_concurrent_assets: int | None = Field(
+        None,
+        description='Max assets processed concurrently. Controls parallel DB connections. Defaults to pool_workers * 2 when omitted.',
+        ge=1,
+        le=50,
+    )
 
 
 class Detector(BaseModel):
@@ -1836,6 +1843,7 @@ class Type(StrEnum):
     CONFLUENCE = 'CONFLUENCE'
     JIRA = 'JIRA'
     SERVICEDESK = 'SERVICEDESK'
+    SQLITE = 'SQLITE'
 
 
 class SlackInput(CoreInput):
@@ -2622,6 +2630,7 @@ class Type17(StrEnum):
     CONFLUENCE = 'CONFLUENCE'
     JIRA = 'JIRA'
     SERVICEDESK = 'SERVICEDESK'
+    SQLITE = 'SQLITE'
 
 
 class ConfluenceInput(CoreInput):
@@ -2676,6 +2685,59 @@ class ServiceDeskInput(CoreInput):
     resources: ResourceOverrides | None = None
 
 
+class SQLiteRequired(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    database_path: str = Field(
+        ...,
+        description='Absolute or relative path to the SQLite database file (e.g. /data/app.db)',
+    )
+
+
+class SQLiteOptionalScope(BaseModel):
+    """
+    Table selection scope.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    include_tables: list[str] | None = Field(
+        None,
+        description='Optional table allowlist. Only tables in this list will be scanned.',
+    )
+    table_limit: int | None = Field(
+        None, description='Optional cap on number of table assets extracted', ge=1
+    )
+
+
+class SQLiteOptional(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    scope: SQLiteOptionalScope | None = None
+
+
+class SQLiteInput(CoreInput):
+    type: Literal['SQLITE'] = Field('SQLITE', description='Type of the asset or source')
+    required: SQLiteRequired
+    masked: dict[str, Any] | None = Field(
+        None,
+        description='SQLite has no credentials; this section is intentionally empty.',
+    )
+    optional: SQLiteOptional | None = None
+    detectors: list[Detector] | None = Field(
+        None, description='Detectors to run on ingested content'
+    )
+    custom_detectors: list[CustomDetectorSelection] | None = Field(
+        None,
+        description='Reusable custom detector IDs selected from the custom detector catalog.',
+    )
+    sampling: SamplingConfig
+    resources: ResourceOverrides | None = None
+
+
 class SourceInput(
     RootModel[
         SlackInput
@@ -2697,6 +2759,7 @@ class SourceInput(
         | ConfluenceInput
         | JiraInput
         | ServiceDeskInput
+        | SQLiteInput
     ]
 ):
     root: (
@@ -2719,6 +2782,7 @@ class SourceInput(
         | ConfluenceInput
         | JiraInput
         | ServiceDeskInput
+        | SQLiteInput
     ) = Field(
         ...,
         description='Merged configuration schema with all source types and common definitions',