classifyre-cli 0.4.2__py3-none-any.whl

src/utils/hashing.py

@@ -0,0 +1,82 @@
+import base64
+import hashlib
+import json
+from typing import Any
+from urllib.parse import urljoin, urlsplit, urlunsplit
+
+
+def hash_id(source_type: str, raw_id: str) -> str:
+    """
+    Hash the raw ID into a base64 encoded string with a source type prefix.
+    Note: This is actually just base64 encoding for reversibility in debugging,
+    not a cryptographic hash.
+    """
+    final_raw_id = f"{source_type}_#_{raw_id}"
+    return base64.urlsafe_b64encode(final_raw_id.encode()).decode().rstrip("=")
+
+
+def unhash_id(hashed_id: str) -> str:
+    """
+    Unhash the base64 encoded ID back to its raw form.
+    """
+    # Add padding back if necessary
+    padding = len(hashed_id) % 4
+    if padding:
+        hashed_id += "=" * (4 - padding)
+    return base64.urlsafe_b64decode(hashed_id.encode()).decode()
+
+
+def calculate_checksum(data: dict[str, Any]) -> str:
+    """
+    Calculate a stable SHA-256 checksum for a dictionary.
+    Keys are sorted to ensure stability.
+    """
+    # Use sort_keys=True for stability
+    dump = json.dumps(data, sort_keys=True, default=str).encode("utf-8")
+    return hashlib.sha256(dump).hexdigest()
+
+
+def normalize_http_url(url: str, *, base_url: str | None = None) -> str | None:
+    """
+    Normalize an HTTP(S) URL for stable hashing and deduplication.
+
+    - Resolves relative URLs against `base_url` when provided
+    - Rejects non-HTTP(S) schemes (mailto:, tel:, javascript:, data:, etc.)
+    - Removes URL fragments
+    """
+    candidate = (url or "").strip()
+    if not candidate:
+        return None
+
+    lowered = candidate.lower()
+    if lowered.startswith(("#", "javascript:", "mailto:", "tel:", "data:")):
+        return None
+
+    if base_url:
+        candidate = urljoin(f"{base_url.rstrip('/')}/", candidate)
+
+    parsed = urlsplit(candidate)
+    if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc:
+        return None
+
+    path = parsed.path or "/"
+    return urlunsplit(
+        (
+            parsed.scheme.lower(),
+            parsed.netloc.lower(),
+            path,
+            parsed.query,
+            "",
+        )
+    )
+
+
+def hash_url(url: str, *, base_url: str | None = None) -> str:
+    """
+    URL hash strategy: opaque SHA-256 digest of normalized absolute URL.
+    """
+    normalized = normalize_http_url(url, base_url=base_url)
+    if not normalized:
+        raise ValueError(f"Invalid URL for hash: {url}")
+    digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
+    return f"url_sha256:{digest}"

src/utils/uv_sync.py

@@ -0,0 +1,79 @@
+"""Shared uv-sync state so that every `uv sync` call includes ALL accumulated groups.
+
+`uv sync --group X` removes packages that belong to other groups.  When sources
+and detectors each call `uv sync --group <their_group>` independently, the last
+call uninstalls packages from earlier groups.  This module keeps a global set of
+requested groups and always passes them all to `uv sync`.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import threading
+
+logger = logging.getLogger(__name__)
+
+_lock = threading.Lock()
+_synced_groups: set[str] = set()
+_failed_groups: dict[str, str] = {}
+
+
+def _auto_install_enabled() -> bool:
+    value = os.environ.get("CLASSIFYRE_CLI_AUTO_INSTALL_OPTIONAL_DEPS", "1").strip().lower()
+    return value not in {"0", "false", "no"}
+
+
+def _uv_command() -> list[str]:
+    uv_binary = shutil.which("uv")
+    if uv_binary:
+        return [uv_binary]
+    return [sys.executable, "-m", "uv"]
+
+
+def sync_group(group: str) -> tuple[bool, str | None]:
+    """Ensure *group* is installed, re-syncing with ALL previously requested groups."""
+    with _lock:
+        if group in _synced_groups:
+            return True, None
+        if group in _failed_groups:
+            return False, _failed_groups[group]
+
+        all_groups = _synced_groups | {group}
+        timeout = int(os.environ.get("CLASSIFYRE_UV_SYNC_TIMEOUT_SECONDS", "900"))
+        command = [*_uv_command(), "sync", "--frozen", "--no-dev"]
+        for g in sorted(all_groups):
+            command.extend(["--group", g])
+
+        logger.info("Installing optional dependency group '%s'...", group)
+        try:
+            result = subprocess.run(
+                command,
+                check=False,
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+            )
+        except Exception as exc:
+            detail = f"Failed to execute uv sync for group '{group}': {exc}"
+            _failed_groups[group] = detail
+            logger.error(detail)
+            return False, detail
+
+        if result.returncode == 0:
+            _synced_groups.update(all_groups)
+            logger.info("Installed dependency group '%s'", group)
+            return True, None
+
+        detail = result.stderr.strip() or result.stdout.strip() or "Unknown uv sync error"
+        message = f"uv sync failed for group '{group}': {detail}"
+        _failed_groups[group] = message
+        logger.error(message)
+        return False, message
+
+
+def auto_install_enabled() -> bool:
+    return _auto_install_enabled()

src/utils/validation.py

@@ -0,0 +1,56 @@
+import json
+from pathlib import Path
+from typing import Any
+
+from jsonschema import validators
+
+import schemas
+
+
+def _load_schema(schema_filename: str) -> dict[str, Any]:
+    schema_dir = Path(schemas.__file__).parent
+    schema_path = schema_dir / schema_filename
+
+    with open(schema_path) as f:
+        return json.load(f)
+
+
+def _validate_schema(instance: dict[str, Any], schema: dict[str, Any]) -> None:
+    validator_cls = validators.validator_for(schema)
+    validator_cls.check_schema(schema)
+    validator = validator_cls(schema)
+    validator.validate(instance)
+
+
+def validate_input(data: dict[str, Any], schema_name: str = "") -> None:  # noqa: ARG001
+    """
+    Validate input data against the unified all_input_sources schema.
+    The schema_name parameter is kept for compatibility but ignored.
+    """
+    schema = _load_schema("all_input_sources.json")
+    _validate_schema(data, schema)
+
+
+def validate_output(data: dict[str, Any], schema_name: str = "") -> None:  # noqa: ARG001
+    """
+    Validate output data against the unified single_asset_scan_results schema.
+    The schema_name parameter is kept for compatibility but ignored.
+    """
+    schema = _load_schema("single_asset_scan_results.json")
+    _validate_schema(data, schema)
+
+
+def validate_test_connection(data: dict[str, Any]) -> None:
+    """
+    Validate test connection output against the core test-connection schema.
+    """
+    schema: dict[str, Any] = {
+        "type": "object",
+        "required": ["status"],
+        "properties": {
+            "status": {"type": "string", "enum": ["SUCCESS", "FAILURE"]},
+            "message": {"type": "string"},
+        },
+    }
+
+    _validate_schema(data, schema)