classifyre-cli 0.4.2__py3-none-any.whl

src/detectors/broken_links/detector.py

@@ -0,0 +1,280 @@
+"""Broken links detector for URL reachability and empty responses."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+import requests
+
+from ...models.generated_detectors import BrokenLinksDetectorConfig, DetectorConfig, Severity
+from ...models.generated_single_asset_scan_results import DetectionResult, DetectorType
+from ...utils.hashing import normalize_http_url
+from ..base import BaseDetector
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class LinkScanResult:
+    url: str
+    line: int
+    start: int
+    end: int
+    finding_type: str
+    confidence: float
+    metadata: dict[str, object]
+
+
+class BrokenLinksDetector(BaseDetector):
+    """
+    Detector for broken links and empty link targets.
+
+    Input content is expected to be newline-delimited URLs
+    (one URL per line), passed with `application/x.asset-links`.
+    """
+
+    detector_type = "broken_links"
+    detector_name = "broken_links"
+
+    _REQUEST_TIMEOUT_SECONDS = 8
+    _MAX_CONCURRENCY = 12
+    _USER_AGENT = "classifyre-broken-links-detector/1.0"
+
+    def __init__(self, config: DetectorConfig | None = None) -> None:
+        super().__init__(config)
+        self._cfg: BrokenLinksDetectorConfig = (
+            config if isinstance(config, BrokenLinksDetectorConfig) else BrokenLinksDetectorConfig()
+        )
+        self._session = requests.Session()
+        self._session.headers.update({"User-Agent": self._USER_AGENT})
+
+    async def detect(
+        self, content: str | bytes, content_type: str = "application/x.asset-links"
+    ) -> list[DetectionResult]:
+        if isinstance(content, bytes):
+            return []
+        if content_type not in self.get_supported_content_types():
+            return []
+
+        links = self._extract_links(content)
+        if not links:
+            return []
+
+        semaphore = asyncio.Semaphore(self._MAX_CONCURRENCY)
+
+        async def check_link(url: str, line: int, start: int, end: int) -> LinkScanResult | None:
+            async with semaphore:
+                return await asyncio.to_thread(
+                    self._scan_link,
+                    url,
+                    line,
+                    start,
+                    end,
+                )
+
+        tasks = [
+            check_link(url=url, line=line, start=start, end=end) for url, line, start, end in links
+        ]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        findings: list[DetectionResult] = []
+        for result in results:
+            if isinstance(result, Exception):
+                logger.debug("Broken links detector task failed: %s", result)
+                continue
+            if result is None:
+                continue
+
+            findings.append(
+                DetectionResult(
+                    detector_type=DetectorType.BROKEN_LINKS,
+                    finding_type=result.finding_type,
+                    category="link_integrity",
+                    severity=Severity.low,
+                    confidence=result.confidence,
+                    matched_content=result.url,
+                    location=None,
+                    metadata=result.metadata,
+                )
+            )
+
+        if self._cfg.max_findings and len(findings) > self._cfg.max_findings:
+            findings = findings[: self._cfg.max_findings]
+
+        return findings
+
+    def get_supported_content_types(self) -> list[str]:
+        return ["application/x.asset-links"]
+
+    def _extract_links(self, content: str) -> list[tuple[str, int, int, int]]:
+        links: list[tuple[str, int, int, int]] = []
+        seen: set[str] = set()
+        offset = 0
+        for line_number, raw_line in enumerate(content.splitlines(), start=1):
+            line = raw_line.strip()
+            if not line:
+                offset += len(raw_line) + 1
+                continue
+
+            normalized = normalize_http_url(line)
+            if not normalized:
+                offset += len(raw_line) + 1
+                continue
+
+            if normalized in seen:
+                offset += len(raw_line) + 1
+                continue
+
+            seen.add(normalized)
+            stripped_index = raw_line.find(line)
+            start = offset + (stripped_index if stripped_index >= 0 else 0)
+            end = start + len(line)
+            links.append((normalized, line_number, start, end))
+            offset += len(raw_line) + 1
+
+        return links
+
+    def _scan_link(
+        self,
+        url: str,
+        line: int,
+        start: int,
+        end: int,
+    ) -> LinkScanResult | None:
+        head_response: requests.Response | None = None
+        try:
+            head_response = self._session.head(
+                url,
+                allow_redirects=True,
+                timeout=self._REQUEST_TIMEOUT_SECONDS,
+            )
+            status_code = head_response.status_code
+
+            if status_code in {405, 501}:
+                return self._scan_with_get(url, line, start, end, "head_not_supported")
+
+            if status_code >= 400:
+                return LinkScanResult(
+                    url=url,
+                    line=line,
+                    start=start,
+                    end=end,
+                    finding_type="unreachable",
+                    confidence=0.95,
+                    metadata={"status_code": status_code, "reason": "http_error"},
+                )
+
+            content_length = self._parse_content_length(head_response.headers)
+            if content_length == 0:
+                return LinkScanResult(
+                    url=url,
+                    line=line,
+                    start=start,
+                    end=end,
+                    finding_type="empty_content",
+                    confidence=0.9,
+                    metadata={"status_code": status_code, "reason": "empty_head_content_length"},
+                )
+
+            # Some servers omit Content-Length, so perform a lightweight GET check.
+            if content_length is None:
+                return self._scan_with_get(url, line, start, end, "missing_content_length")
+
+            return None
+        except requests.RequestException as exc:
+            return LinkScanResult(
+                url=url,
+                line=line,
+                start=start,
+                end=end,
+                finding_type="unreachable",
+                confidence=0.95,
+                metadata={"reason": "request_exception", "error": str(exc)},
+            )
+        finally:
+            if head_response is not None:
+                head_response.close()
+
+    def _scan_with_get(
+        self,
+        url: str,
+        line: int,
+        start: int,
+        end: int,
+        reason: str,
+    ) -> LinkScanResult | None:
+        get_response: requests.Response | None = None
+        try:
+            get_response = self._session.get(
+                url,
+                allow_redirects=True,
+                timeout=self._REQUEST_TIMEOUT_SECONDS,
+                stream=True,
+            )
+            status_code = get_response.status_code
+            if status_code >= 400:
+                return LinkScanResult(
+                    url=url,
+                    line=line,
+                    start=start,
+                    end=end,
+                    finding_type="unreachable",
+                    confidence=0.95,
+                    metadata={"status_code": status_code, "reason": reason},
+                )
+
+            content_length = self._parse_content_length(get_response.headers)
+            if content_length == 0:
+                return LinkScanResult(
+                    url=url,
+                    line=line,
+                    start=start,
+                    end=end,
+                    finding_type="empty_content",
+                    confidence=0.9,
+                    metadata={"status_code": status_code, "reason": reason},
+                )
+
+            has_payload = self._response_has_payload(get_response.iter_content(chunk_size=1))
+            if not has_payload:
+                return LinkScanResult(
+                    url=url,
+                    line=line,
+                    start=start,
+                    end=end,
+                    finding_type="empty_content",
+                    confidence=0.9,
+                    metadata={"status_code": status_code, "reason": "empty_body"},
+                )
+            return None
+        except requests.RequestException as exc:
+            return LinkScanResult(
+                url=url,
+                line=line,
+                start=start,
+                end=end,
+                finding_type="unreachable",
+                confidence=0.95,
+                metadata={"reason": reason, "error": str(exc)},
+            )
+        finally:
+            if get_response is not None:
+                get_response.close()
+
+    def _parse_content_length(self, headers: dict[str, object]) -> int | None:
+        raw_length = headers.get("Content-Length")
+        if raw_length is None:
+            return None
+        try:
+            return int(str(raw_length))
+        except (TypeError, ValueError):
+            return None
+
+    def _response_has_payload(self, chunks: Iterable[bytes]) -> bool:
+        for chunk in chunks:
+            if chunk:
+                return True
+        return False

src/detectors/config.py

@@ -0,0 +1,59 @@
+"""Shared detector config resolution and type mapping."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..models.generated_detectors import (
+    BrokenLinksDetectorConfig,
+    CustomDetectorConfig,
+    DetectorConfig,
+    PIIDetectorConfig,
+    SecretsDetectorConfig,
+    ThreatDetectorConfig,
+)
+
+type DetectorTypedConfig = (
+    DetectorConfig
+    | CustomDetectorConfig
+    | SecretsDetectorConfig
+    | PIIDetectorConfig
+    | ThreatDetectorConfig
+    | BrokenLinksDetectorConfig
+)
+
+_DETECTOR_NAME_BY_TYPE: dict[str, str] = {
+    "SECRETS": "secrets",
+    "PII": "pii",
+    "YARA": "yara",
+    "BROKEN_LINKS": "broken_links",
+    "CODE_SECURITY": "code_security",
+    "CUSTOM": "custom",
+}
+
+_DETECTOR_CONFIG_BY_TYPE: dict[str, type[DetectorConfig]] = {
+    "SECRETS": SecretsDetectorConfig,
+    "PII": PIIDetectorConfig,
+    "YARA": ThreatDetectorConfig,
+    "BROKEN_LINKS": BrokenLinksDetectorConfig,
+    "CUSTOM": CustomDetectorConfig,
+}
+
+
+def normalize_detector_type(detector_type: str) -> str:
+    return detector_type.strip().upper()
+
+
+def get_detector_name(detector_type: str) -> str:
+    normalized = normalize_detector_type(detector_type)
+    return _DETECTOR_NAME_BY_TYPE.get(normalized, normalized.lower())
+
+
+def parse_detector_config(detector_type: str, raw_config: Any) -> tuple[str, DetectorTypedConfig]:
+    normalized = normalize_detector_type(detector_type)
+    detector_name = get_detector_name(normalized)
+    config_cls = _DETECTOR_CONFIG_BY_TYPE.get(normalized, DetectorConfig)
+    if not isinstance(raw_config, dict):
+        raw_config = {}
+    typed_config = config_cls.model_validate(raw_config)
+    return detector_name, typed_config

src/detectors/custom/__init__.py

@@ -0,0 +1,13 @@
+"""Custom detector implementations."""
+
+from .detector import CustomDetector
+from .runners import BaseRunner, GLiNER2Runner, LLMRunner, RegexRunner, create_runner
+
+__all__ = [
+    "BaseRunner",
+    "CustomDetector",
+    "GLiNER2Runner",
+    "LLMRunner",
+    "RegexRunner",
+    "create_runner",
+]

src/detectors/custom/detector.py

@@ -0,0 +1,45 @@
+"""Custom detector — delegates to the appropriate runner via the runner factory."""
+
+from __future__ import annotations
+
+import logging
+
+from ...models.generated_detectors import (
+    CustomDetectorConfig,
+    DetectorConfig,
+)
+from ...models.generated_single_asset_scan_results import DetectionResult
+from ..base import BaseDetector
+from .runners import BaseRunner, create_runner
+
+logger = logging.getLogger(__name__)
+
+
+class CustomDetector(BaseDetector):
+    """Schema-driven detector backed by a pluggable runner (GLINER2 | REGEX | LLM | transformer)."""
+
+    detector_type = "custom"
+    detector_name = "custom"
+
+    def __init__(self, config: DetectorConfig | None = None):
+        super().__init__(config)
+        if not isinstance(self.config, CustomDetectorConfig):
+            raise ValueError("CustomDetector requires CustomDetectorConfig with pipeline_schema")
+        self.custom_config: CustomDetectorConfig = self.config
+        self._runner: BaseRunner = create_runner(
+            self.custom_config.pipeline_schema,
+            detector_key=self.custom_config.custom_detector_key,
+            detector_name=self.custom_config.name,
+        )
+
+    async def detect(
+        self, content: str | bytes, content_type: str = "text/plain"
+    ) -> list[DetectionResult]:
+        findings = self._runner.detect(content, content_type)
+        max_findings = self.custom_config.max_findings
+        if isinstance(max_findings, int) and max_findings > 0:
+            findings = findings[:max_findings]
+        return findings
+
+    def get_supported_content_types(self) -> list[str]:
+        return self._runner.get_supported_content_types()

src/detectors/custom/runners/__init__.py

@@ -0,0 +1,56 @@
+"""Custom detector runner package.
+
+Public surface re-exported here so that existing imports of the form
+    from .runners import BaseRunner, create_runner, GLiNER2Runner, ...
+continue to work unchanged.
+"""
+
+from ._base import (
+    _DEFAULT_GLINER2_MODEL,
+    _DEFAULT_IMAGE_CLASSIFICATION_MODEL,
+    _IMAGE_CONTENT_TYPES,
+    _TEXT_CONTENT_TYPES,
+    BaseRunner,
+    _resolve_pipeline_severity,
+)
+from ._factory import create_runner
+from ._feature_extraction import FeatureExtractionRunner, _chunk_text_with_offsets, _pool_hidden
+from ._gliner2 import (
+    GLiNER2Runner,
+    _apply_classification_validation,
+    _apply_entity_validation,
+    _normalise_classification_output,
+    _normalise_entity_output,
+    _normalise_span,
+)
+from ._image_classification import ImageClassificationRunner
+from ._llm import LLMRunner
+from ._object_detection import ObjectDetectionRunner
+from ._regex import RegexRunner, _load_regex_engine
+from ._text_classification import TextClassificationRunner, _chunk_text
+
+__all__ = [
+    "_DEFAULT_GLINER2_MODEL",
+    "_DEFAULT_IMAGE_CLASSIFICATION_MODEL",
+    "_IMAGE_CONTENT_TYPES",
+    "_TEXT_CONTENT_TYPES",
+    "BaseRunner",
+    "FeatureExtractionRunner",
+    "GLiNER2Runner",
+    "ImageClassificationRunner",
+    "LLMRunner",
+    "ObjectDetectionRunner",
+    "RegexRunner",
+    "TextClassificationRunner",
+    "_apply_classification_validation",
+    "_apply_entity_validation",
+    "_chunk_text",
+    "_chunk_text_with_offsets",
+    "_load_regex_engine",
+    "_normalise_classification_output",
+    "_normalise_entity_output",
+    "_normalise_span",
+    "_pool_hidden",
+    "_resolve_pipeline_severity",
+    "create_runner",
+]

src/detectors/custom/runners/_base.py

@@ -0,0 +1,177 @@
+"""Base runner interface and shared utilities for all pipeline execution strategies."""
+
+from __future__ import annotations
+
+import re
+from abc import ABC, abstractmethod
+from datetime import UTC, datetime
+from typing import Any
+
+from ....models.generated_detectors import (
+    PipelineResult,
+    PipelineSeverityRule,
+    Severity,
+)
+from ....models.generated_single_asset_scan_results import (
+    DetectionResult,
+    DetectorType,
+    Location,
+)
+
+_DEFAULT_GLINER2_MODEL = "fastino/gliner2-base-v1"
+_DEFAULT_IMAGE_CLASSIFICATION_MODEL = "google/vit-base-patch16-224"
+
+_TEXT_CONTENT_TYPES = [
+    "text/plain",
+    "text/html",
+    "text/markdown",
+    "application/json",
+    "application/xml",
+    "text/xml",
+]
+_IMAGE_CONTENT_TYPES = [
+    "image/jpeg",
+    "image/jpg",
+    "image/png",
+    "image/gif",
+    "image/webp",
+    "image/bmp",
+    "image/tiff",
+]
+
+
+def _resolve_pipeline_severity(
+    label: str,
+    severity_map: list[PipelineSeverityRule] | None,
+    default: Severity = Severity.info,
+) -> Severity:
+    """Return the first severity whose pattern matches label (case-insensitive)."""
+    if not severity_map:
+        return default
+    label_lower = label.lower()
+    for rule in severity_map:
+        try:
+            if re.search(rule.pattern, label_lower, re.IGNORECASE):
+                return rule.severity
+        except re.error:
+            if rule.pattern.lower() in label_lower:
+                return rule.severity
+    return default
+
+
+class BaseRunner(ABC):
+    """Common interface for all pipeline execution strategies."""
+
+    _detector_key: str = ""
+    _detector_name: str = ""
+
+    @abstractmethod
+    def run(self, text: str) -> PipelineResult:
+        """Execute the pipeline on *text* and return a normalised PipelineResult."""
+        ...
+
+    def detect(self, content: str | bytes, content_type: str) -> list[DetectionResult]:
+        """Run detection and return findings. Default: text-only path via run()."""
+        if isinstance(content, bytes):
+            return []
+        text = content.strip()
+        if not text:
+            return []
+        result = self.run(text)
+        return self._result_to_findings(text, result)
+
+    def get_supported_content_types(self) -> list[str]:
+        return list(_TEXT_CONTENT_TYPES)
+
+    def _make_result(
+        self,
+        *,
+        finding_type: str,
+        category: str,
+        severity: Severity,
+        confidence: float,
+        matched_content: str,
+        location: Location | None,
+        metadata: dict[str, Any],
+    ) -> DetectionResult:
+        return DetectionResult(
+            detector_type=DetectorType.CUSTOM,
+            finding_type=finding_type,
+            category=category,
+            severity=severity,
+            confidence=confidence,
+            matched_content=matched_content,
+            location=location,
+            custom_detector_key=self._detector_key,
+            custom_detector_name=self._detector_name,
+            detected_at=datetime.now(UTC),
+            metadata=metadata,
+        )
+
+    def _result_to_findings(self, text: str, result: PipelineResult) -> list[DetectionResult]:
+        findings: list[DetectionResult] = []
+        runner_type = result.metadata.get("runner", "GLINER2")
+
+        for label, spans in result.entities.items():
+            for span in spans:
+                confidence = float(span.get("confidence", 0.0))
+                value = str(span.get("value", ""))
+                start = span.get("start")
+                end = span.get("end")
+
+                loc: Location | None = None
+                if isinstance(start, int) and isinstance(end, int):
+                    loc = Location(start=start, end=end, path=f"{runner_type.lower()}-entity")
+
+                finding_type = f"regex:{label}" if runner_type == "REGEX" else f"entity:{label}"
+
+                span_severity = span.get("severity")
+                if isinstance(span_severity, str) and span_severity in Severity.__members__:
+                    sev = Severity(span_severity)
+                else:
+                    sev = Severity.medium if confidence < 0.9 else Severity.high
+
+                meta: dict[str, Any] = {
+                    "runner": runner_type,
+                    "entity_label": label,
+                    "pipeline_result": result.model_dump(),
+                }
+                if "groups" in span:
+                    meta["capture_groups"] = span["groups"]
+
+                findings.append(
+                    self._make_result(
+                        finding_type=finding_type,
+                        category="CLASSIFICATION",
+                        severity=sev,
+                        confidence=min(0.99, confidence),
+                        matched_content=value,
+                        location=loc,
+                        metadata=meta,
+                    )
+                )
+
+        for task, outcome in result.classification.items():
+            label = str(outcome.get("label", ""))
+            confidence = float(outcome.get("confidence", 0.0))
+            if not label:
+                continue
+
+            findings.append(
+                self._make_result(
+                    finding_type=f"classification:{task}:{label}",
+                    category="CLASSIFICATION",
+                    severity=Severity.medium if confidence < 0.9 else Severity.high,
+                    confidence=min(0.99, confidence),
+                    matched_content=text[:320],
+                    location=None,
+                    metadata={
+                        "runner": runner_type,
+                        "task": task,
+                        "label": label,
+                        "pipeline_result": result.model_dump(),
+                    },
+                )
+            )
+
+        return findings

src/detectors/custom/runners/_factory.py

@@ -0,0 +1,51 @@
+"""Factory function for creating runner instances from pipeline schemas."""
+
+from __future__ import annotations
+
+from ....models.generated_detectors import (
+    FeatureExtractionPipelineSchema,
+    GLiNER2PipelineSchema,
+    ImageClassificationPipelineSchema,
+    LLMPipelineSchema,
+    ObjectDetectionPipelineSchema,
+    RegexPipelineSchema,
+    TextClassificationPipelineSchema,
+)
+from ._base import BaseRunner
+from ._feature_extraction import FeatureExtractionRunner
+from ._gliner2 import GLiNER2Runner
+from ._image_classification import ImageClassificationRunner
+from ._llm import LLMRunner
+from ._object_detection import ObjectDetectionRunner
+from ._regex import RegexRunner
+from ._text_classification import TextClassificationRunner
+
+
+def create_runner(
+    schema: (
+        GLiNER2PipelineSchema
+        | RegexPipelineSchema
+        | LLMPipelineSchema
+        | TextClassificationPipelineSchema
+        | ImageClassificationPipelineSchema
+        | FeatureExtractionPipelineSchema
+        | ObjectDetectionPipelineSchema
+    ),
+    detector_key: str = "",
+    detector_name: str = "",
+) -> BaseRunner:
+    """Return the appropriate runner for *schema* based on its type discriminator."""
+    if isinstance(schema, TextClassificationPipelineSchema):
+        return TextClassificationRunner(schema, detector_key, detector_name)
+    if isinstance(schema, ImageClassificationPipelineSchema):
+        return ImageClassificationRunner(schema, detector_key, detector_name)
+    if isinstance(schema, FeatureExtractionPipelineSchema):
+        return FeatureExtractionRunner(schema, detector_key, detector_name)
+    if isinstance(schema, ObjectDetectionPipelineSchema):
+        return ObjectDetectionRunner(schema, detector_key, detector_name)
+    if isinstance(schema, RegexPipelineSchema):
+        return RegexRunner(schema, detector_key, detector_name)
+    if isinstance(schema, LLMPipelineSchema):
+        return LLMRunner(schema, detector_key, detector_name)
+    # GLiNER2PipelineSchema is the default / backward-compat path
+    return GLiNER2Runner(schema, detector_key, detector_name)