classifyre-cli 0.4.4__tar.gz → 0.4.6__tar.gz

classifyre_cli-0.4.6/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 256 packages in 322ms
+Checked 49 packages in 0.88ms

{classifyre_cli-0.4.4 → classifyre_cli-0.4.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.4
+Version: 0.4.6
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.4 → classifyre_cli-0.4.6}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.4",
+  "version": "0.4.6",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.4 → classifyre_cli-0.4.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.4"
+version = "0.4.6"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

classifyre_cli-0.4.6/src/detectors/custom/extractor.py

@@ -0,0 +1,261 @@
+"""Custom detector extraction engine — REGEX, GLINER, and CLASSIFIER_GLINER strategies."""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from ...models.generated_detectors import (
+    CustomDetectorMethod,
+    CustomExtractorConfig,
+    CustomExtractorField,
+)
+from ..dependencies import MissingDependencyError, require_module
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_GLINER2_MODEL = "fastino/gliner2-base-v1"
+
+# Extraction method tags sent to the API via DetectionResult.extraction_method
+EXTRACTION_METHOD_REGEX = "REGEX"
+EXTRACTION_METHOD_GLINER = "GLINER"
+EXTRACTION_METHOD_CLASSIFIER_GLINER = "CLASSIFIER_GLINER"
+
+
+@dataclass
+class ExtractionResult:
+    """Typed output from one extraction run."""
+
+    extracted_data: dict[str, Any]
+    method: str
+    populated_fields: list[str] = field(default_factory=list)
+    field_count: int = 0
+
+    def __post_init__(self) -> None:
+        self.populated_fields = [
+            k for k, v in self.extracted_data.items() if v is not None and v not in ([], "")
+        ]
+        self.field_count = len(self.extracted_data)
+
+
+class CustomExtractor:
+    """
+    Runs after a custom detector fires to pull structured data from the content.
+
+    Strategy selection:
+      RULESET     → REGEX    (named capture groups in field.regex_pattern)
+      ENTITY      → GLINER   (group GLiNER2 entity spans by entity_label into fields)
+      CLASSIFIER  → CLASSIFIER_GLINER  (second GLiNER2 pass on wider content slice)
+    """
+
+    def __init__(
+        self,
+        config: CustomExtractorConfig,
+        detector_method: CustomDetectorMethod,
+    ) -> None:
+        self._config = config
+        self._method = detector_method
+        self._gliner_model: Any | None = None
+        self._compiled: dict[str, re.Pattern[str]] = {}  # pattern cache
+
+    # ── Public API ───────────────────────────────────────────────────────────
+
+    def extract(
+        self,
+        matched_content: str,
+        content_for_extraction: str,
+    ) -> ExtractionResult | None:
+        """
+        Run extraction and return structured result, or None if nothing extracted.
+
+        Args:
+            matched_content:       The content stored in the finding (may be truncated).
+            content_for_extraction: Wider slice of the original document for GLiNER/regex.
+        """
+        if not self._config.enabled:
+            return None
+
+        if self._method == CustomDetectorMethod.RULESET:
+            return self._extract_regex(content_for_extraction)
+        if self._method == CustomDetectorMethod.ENTITY:
+            return self._extract_gliner(content_for_extraction, EXTRACTION_METHOD_GLINER)
+        if self._method == CustomDetectorMethod.CLASSIFIER:
+            return self._extract_gliner(content_for_extraction, EXTRACTION_METHOD_CLASSIFIER_GLINER)
+        logger.warning("CustomExtractor: unknown detector method %s", self._method)
+        return None
+
+    # ── RULESET — regex named groups ─────────────────────────────────────────
+
+    def _extract_regex(self, content: str) -> ExtractionResult | None:
+        data: dict[str, Any] = {}
+
+        for f in self._config.fields:
+            if not f.regex_pattern:
+                logger.debug(
+                    "Extractor field '%s' has no regex_pattern — skipped for RULESET", f.name
+                )
+                continue
+            value = self._apply_regex_field(content, f)
+            if value is not None:
+                data[f.name] = value
+
+        return self._finalize(data, EXTRACTION_METHOD_REGEX)
+
+    def _apply_regex_field(self, content: str, f: CustomExtractorField) -> Any:
+        pattern = self._compile(f.regex_pattern or "", f.regex_flags or "i")
+        if pattern is None:
+            return None
+
+        named_groups = pattern.groupindex
+        group_name = next(iter(named_groups), None)
+
+        matches: list[str] = []
+        for m in pattern.finditer(content):
+            captured = m.group(group_name) if group_name else m.group(0)
+            if captured and captured.strip():
+                matches.append(captured.strip())
+
+        return self._aggregate(matches, f) if matches else None
+
+    def _compile(self, pattern: str, flags_str: str) -> re.Pattern[str] | None:
+        cache_key = f"{pattern}::{flags_str}"
+        if cache_key in self._compiled:
+            return self._compiled[cache_key]
+
+        flags = 0
+        for ch in flags_str:
+            if ch == "i":
+                flags |= re.IGNORECASE
+            elif ch == "m":
+                flags |= re.MULTILINE
+            elif ch == "s":
+                flags |= re.DOTALL
+
+        try:
+            compiled = re.compile(pattern, flags=flags)
+            self._compiled[cache_key] = compiled
+            return compiled
+        except re.error as exc:
+            logger.warning("CustomExtractor: invalid regex pattern '%s': %s", pattern, exc)
+            return None
+
+    # ── ENTITY / CLASSIFIER — GLiNER2 entity spans ───────────────────────────
+
+    def _extract_gliner(self, content: str, method_tag: str) -> ExtractionResult | None:
+        label_to_fields: dict[str, list[CustomExtractorField]] = {}
+        for f in self._config.fields:
+            if f.entity_label:
+                label_to_fields.setdefault(f.entity_label, []).append(f)
+
+        if not label_to_fields:
+            logger.debug("CustomExtractor: no fields with entity_label — skipping GLiNER2")
+            return None
+
+        model = self._load_gliner()
+        if model is None:
+            return None
+
+        entity_schema = {
+            label: next(
+                (
+                    field.description
+                    for field in fields
+                    if isinstance(field.description, str) and field.description.strip()
+                ),
+                "",
+            )
+            for label, fields in label_to_fields.items()
+        }
+        try:
+            result = model.extract_entities(
+                content,
+                entity_schema,
+                threshold=0.0,
+                include_confidence=True,
+            )
+        except Exception as exc:  # pragma: no cover
+            logger.warning("CustomExtractor: GLiNER2 extraction failed: %s", exc)
+            return None
+
+        entities = result.get("entities", {})
+        if not isinstance(entities, dict):
+            return None
+
+        data: dict[str, Any] = {}
+        for entity_label, fields in label_to_fields.items():
+            raw_spans = entities.get(entity_label, [])
+            if not isinstance(raw_spans, list):
+                raw_spans = [raw_spans]
+
+            for f in fields:
+                threshold = f.min_confidence if f.min_confidence is not None else 0.4
+                values = self._filter_gliner2_values(raw_spans, threshold)
+                value = self._aggregate(values, f) if values else None
+                if value is not None:
+                    data[f.name] = value
+
+        return self._finalize(data, method_tag)
+
+    def _filter_gliner2_values(self, raw_spans: list[Any], threshold: float) -> list[str]:
+        values: list[str] = []
+        for raw_span in raw_spans:
+            if isinstance(raw_span, dict):
+                score = float(raw_span.get("confidence", raw_span.get("score", 0.0)))
+                text = str(raw_span.get("text", "")).strip()
+            else:
+                score = 1.0
+                text = str(raw_span).strip()
+
+            if score >= threshold and text:
+                values.append(text)
+
+        return values
+
+    def _load_gliner(self) -> Any | None:
+        if self._gliner_model is not None:
+            return self._gliner_model
+        try:
+            gliner2_module = require_module("gliner2", "custom", ["classification", "detectors"])
+            model_name = self._config.gliner_model or _DEFAULT_GLINER2_MODEL
+            self._gliner_model = gliner2_module.GLiNER2.from_pretrained(model_name)
+            return self._gliner_model
+        except MissingDependencyError:
+            raise
+        except Exception as exc:  # pragma: no cover
+            logger.warning("CustomExtractor: failed to load GLiNER2: %s", exc)
+            return None
+
+    # ── Shared helpers ────────────────────────────────────────────────────────
+
+    def _aggregate(self, values: list[str], f: CustomExtractorField) -> Any:
+        if not values:
+            return None
+        aggregate = f.aggregate or "list"
+        if aggregate == "first":
+            return values[0]
+        if aggregate == "last":
+            return values[-1]
+        if aggregate == "list":
+            return values
+        if aggregate == "join":
+            sep = f.join_separator if f.join_separator is not None else ", "
+            return sep.join(values)
+        if aggregate == "count":
+            return len(values)
+        return values  # fallback
+
+    def _finalize(self, data: dict[str, Any], method: str) -> ExtractionResult | None:
+        # Required fields gate: if any required field is missing, discard the result
+        for f in self._config.fields:
+            if f.required and f.name not in data:
+                logger.debug(
+                    "CustomExtractor: required field '%s' not populated — discarding", f.name
+                )
+                return None
+
+        if not data:
+            return None
+
+        return ExtractionResult(extracted_data=data, method=method)

classifyre_cli-0.4.6/tests/test_custom_extractor.py

@@ -0,0 +1,291 @@
+"""Tests for CustomExtractor — all three strategies."""
+
+from __future__ import annotations
+
+from src.detectors.custom.extractor import (
+    EXTRACTION_METHOD_CLASSIFIER_GLINER,
+    EXTRACTION_METHOD_GLINER,
+    EXTRACTION_METHOD_REGEX,
+    CustomExtractor,
+    ExtractionResult,
+)
+from src.models.generated_detectors import (
+    CustomDetectorMethod,
+    CustomExtractorConfig,
+    CustomExtractorField,
+)
+
+
+def make_config(*fields_kwargs: dict, **config_kwargs) -> CustomExtractorConfig:
+    fields = [CustomExtractorField(**kw) for kw in fields_kwargs]
+    return CustomExtractorConfig(fields=fields, **config_kwargs)
+
+
+# ── RULESET / REGEX ──────────────────────────────────────────────────────────
+
+
+class TestRegexExtraction:
+    def _extractor(self, *fields_kwargs: dict) -> CustomExtractor:
+        return CustomExtractor(make_config(*fields_kwargs), CustomDetectorMethod.RULESET)
+
+    def test_extracts_named_group(self):
+        ex = self._extractor(
+            {
+                "name": "amount",
+                "regex_pattern": r"(?P<value>\d+[.,]\d+)\s*EUR",
+                "aggregate": "first",
+            }
+        )
+        result = ex.extract("price is 29.99 EUR today", "price is 29.99 EUR today")
+        assert result is not None
+        assert result.extracted_data["amount"] == "29.99"
+        assert result.method == EXTRACTION_METHOD_REGEX
+        assert "amount" in result.populated_fields
+
+    def test_list_aggregate_collects_all(self):
+        ex = self._extractor(
+            {
+                "name": "emails",
+                "regex_pattern": r"(?P<value>[a-z]+@[a-z]+\.[a-z]+)",
+                "aggregate": "list",
+            }
+        )
+        result = ex.extract("a@b.com and c@d.com", "a@b.com and c@d.com")
+        assert result is not None
+        assert result.extracted_data["emails"] == ["a@b.com", "c@d.com"]
+
+    def test_join_aggregate(self):
+        ex = self._extractor(
+            {
+                "name": "tags",
+                "regex_pattern": r"#(?P<value>\w+)",
+                "aggregate": "join",
+                "join_separator": " | ",
+            }
+        )
+        result = ex.extract("found #food and #recipe here", "found #food and #recipe here")
+        assert result is not None
+        assert result.extracted_data["tags"] == "food | recipe"
+
+    def test_count_aggregate(self):
+        ex = self._extractor(
+            {
+                "name": "mention_count",
+                "regex_pattern": r"(?P<value>car rental)",
+                "aggregate": "count",
+                "regex_flags": "i",
+            }
+        )
+        result = ex.extract(
+            "car rental here and car rental there", "car rental here and car rental there"
+        )
+        assert result is not None
+        assert result.extracted_data["mention_count"] == 2
+
+    def test_no_match_returns_none(self):
+        ex = self._extractor(
+            {"name": "iban", "regex_pattern": r"(?P<value>DE\d{20})", "aggregate": "first"}
+        )
+        result = ex.extract("no iban here", "no iban here")
+        assert result is None
+
+    def test_required_field_gates_result(self):
+        ex = self._extractor(
+            {"name": "optional", "regex_pattern": r"(?P<value>foo)", "aggregate": "first"},
+            {
+                "name": "must_have",
+                "regex_pattern": r"(?P<value>REQUIRED)",
+                "aggregate": "first",
+                "required": True,
+            },
+        )
+        result = ex.extract("foo bar baz", "foo bar baz")
+        assert result is None  # must_have not populated
+
+    def test_required_field_allows_result_when_present(self):
+        ex = self._extractor(
+            {
+                "name": "must_have",
+                "regex_pattern": r"(?P<value>REQUIRED)",
+                "aggregate": "first",
+                "required": True,
+            },
+        )
+        result = ex.extract("text with REQUIRED word", "text with REQUIRED word")
+        assert result is not None
+        assert result.extracted_data["must_have"] == "REQUIRED"
+
+    def test_invalid_regex_skipped_gracefully(self):
+        ex = self._extractor(
+            {"name": "bad", "regex_pattern": r"[invalid", "aggregate": "first"},
+            {"name": "good", "regex_pattern": r"(?P<value>ok)", "aggregate": "first"},
+        )
+        result = ex.extract("ok", "ok")
+        assert result is not None
+        assert "good" in result.extracted_data
+        assert "bad" not in result.extracted_data
+
+    def test_case_insensitive_flag(self):
+        ex = self._extractor(
+            {
+                "name": "word",
+                "regex_pattern": r"(?P<value>hello)",
+                "aggregate": "first",
+                "regex_flags": "i",
+            }
+        )
+        result = ex.extract("HELLO world", "HELLO world")
+        assert result is not None
+        assert result.extracted_data["word"].lower() == "hello"
+
+    def test_disabled_extractor_returns_none(self):
+        config = make_config(
+            {"name": "f", "regex_pattern": r"(?P<value>\w+)", "aggregate": "first"},
+            enabled=False,
+        )
+        ex = CustomExtractor(config, CustomDetectorMethod.RULESET)
+        assert ex.extract("hello", "hello") is None
+
+    def test_extraction_result_populated_fields(self):
+        ex = self._extractor(
+            {"name": "a", "regex_pattern": r"(?P<value>yes)", "aggregate": "first"},
+            {"name": "b", "regex_pattern": r"(?P<value>no)", "aggregate": "first"},
+        )
+        result = ex.extract("yes only", "yes only")
+        assert result is not None
+        assert "a" in result.populated_fields
+        assert "b" not in result.populated_fields
+
+
+# ── ENTITY / GLINER ──────────────────────────────────────────────────────────
+
+
+class TestGlinerExtraction:
+    """Tests using a mocked GLiNER model to avoid downloading models."""
+
+    def _extractor_with_mock_gliner(
+        self, fields: list[dict], mock_entities: list[dict]
+    ) -> CustomExtractor:
+        config = make_config(*fields)
+        ex = CustomExtractor(config, CustomDetectorMethod.ENTITY)
+
+        class MockGliner:
+            def extract_entities(self, content: str, labels: dict[str, str], **_kwargs) -> dict:
+                entities = {
+                    label: [
+                        {"text": e["text"], "confidence": e["score"]}
+                        for e in mock_entities
+                        if e.get("label") == label
+                    ]
+                    for label in labels
+                }
+                return {"entities": entities}
+
+        ex._gliner_model = MockGliner()
+        return ex
+
+    def test_groups_entity_spans_by_label(self):
+        ex = self._extractor_with_mock_gliner(
+            [
+                {
+                    "name": "persons",
+                    "entity_label": "person",
+                    "type": "list[string]",
+                    "aggregate": "list",
+                },
+                {
+                    "name": "orgs",
+                    "entity_label": "organization",
+                    "type": "list[string]",
+                    "aggregate": "list",
+                },
+            ],
+            [
+                {"label": "person", "text": "Alice", "score": 0.9},
+                {"label": "person", "text": "Bob", "score": 0.8},
+                {"label": "organization", "text": "Acme Corp", "score": 0.85},
+            ],
+        )
+        result = ex.extract("text", "Alice and Bob work at Acme Corp")
+        assert result is not None
+        assert result.extracted_data["persons"] == ["Alice", "Bob"]
+        assert result.extracted_data["orgs"] == ["Acme Corp"]
+        assert result.method == EXTRACTION_METHOD_GLINER
+
+    def test_min_confidence_filters_low_score(self):
+        ex = self._extractor_with_mock_gliner(
+            [{"name": "items", "entity_label": "item", "aggregate": "list", "min_confidence": 0.8}],
+            [
+                {"label": "item", "text": "high conf", "score": 0.9},
+                {"label": "item", "text": "low conf", "score": 0.3},
+            ],
+        )
+        result = ex.extract("text", "text")
+        assert result is not None
+        assert result.extracted_data["items"] == ["high conf"]
+
+    def test_first_aggregate_takes_first(self):
+        ex = self._extractor_with_mock_gliner(
+            [{"name": "role", "entity_label": "job title", "aggregate": "first"}],
+            [
+                {"label": "job title", "text": "CEO", "score": 0.9},
+                {"label": "job title", "text": "CFO", "score": 0.85},
+            ],
+        )
+        result = ex.extract("text", "text")
+        assert result is not None
+        assert result.extracted_data["role"] == "CEO"
+
+    def test_no_entities_returns_none(self):
+        ex = self._extractor_with_mock_gliner(
+            [{"name": "dish", "entity_label": "food dish", "aggregate": "list"}],
+            [],
+        )
+        result = ex.extract("no food here", "no food here")
+        assert result is None
+
+    def test_classifier_method_uses_classifier_gliner_tag(self):
+        config = make_config({"name": "dish", "entity_label": "food dish", "aggregate": "list"})
+        ex = CustomExtractor(config, CustomDetectorMethod.CLASSIFIER)
+
+        class MockGliner:
+            def extract_entities(self, content: str, labels: dict[str, str], **_kwargs) -> dict:
+                return {
+                    "entities": {label: [{"text": "pizza", "confidence": 0.9}] for label in labels}
+                }
+
+        ex._gliner_model = MockGliner()
+        result = ex.extract("text", "I ate pizza")
+        assert result is not None
+        assert result.method == EXTRACTION_METHOD_CLASSIFIER_GLINER
+
+    def test_required_field_gates_gliner_result(self):
+        ex = self._extractor_with_mock_gliner(
+            [
+                {"name": "optional_field", "entity_label": "item", "aggregate": "list"},
+                {
+                    "name": "required_field",
+                    "entity_label": "must",
+                    "aggregate": "first",
+                    "required": True,
+                },
+            ],
+            [{"label": "item", "text": "something", "score": 0.9}],
+        )
+        result = ex.extract("text", "text")
+        assert result is None  # required_field (label "must") was not found
+
+
+# ── ExtractionResult ─────────────────────────────────────────────────────────
+
+
+class TestExtractionResult:
+    def test_populated_fields_excludes_empty_list(self):
+        r = ExtractionResult(extracted_data={"a": ["x"], "b": [], "c": "hello"}, method="REGEX")
+        assert "a" in r.populated_fields
+        assert "b" not in r.populated_fields
+        assert "c" in r.populated_fields
+
+    def test_field_count_matches_data_keys(self):
+        r = ExtractionResult(extracted_data={"x": 1, "y": 2, "z": 3}, method="GLINER")
+        assert r.field_count == 3

{classifyre_cli-0.4.4 → classifyre_cli-0.4.6}/uv.lock

@@ -531,7 +531,7 @@ wheels = [
 
 [[package]]
 name = "classifyre-cli"
-version = "0.4.4"
+version = "0.4.6"
 source = { editable = "." }
 dependencies = [
     { name = "beautifulsoup4" },
@@ -838,7 +838,7 @@ threat-ml = [
 
 [[package]]
 name = "classifyre-schemas"
-version = "0.4.4"
+version = "0.4.6"
 source = { editable = "../../packages/schemas" }
 dependencies = [
     { name = "fastjsonschema" },

classifyre_cli-0.4.4/.turbo/turbo-build.log

@@ -1,3 +0,0 @@
-$ uv sync
-Resolved 256 packages in 170ms
-Checked 49 packages in 1ms