devtorch-core 3.0.1__py3-none-any.whl

devtorch_core/audit/exporter.py

@@ -0,0 +1,297 @@
+"""Audit report export engine.
+
+Reads ``.GCC/events.log.jsonl``, applies date/project/workspace/user/event-type
+filters, applies privacy policy + scrubbing, and emits JSON or PDF.
+
+PDF generation uses ``reportlab`` (optional ``devtorch-core[audit]`` dependency).
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import datetime
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from .privacy import PrivacyConfig, apply_privacy_policy
+from .scrubber import RegexScrubber
+
+
+def _parse_iso(value: str | datetime.datetime | None) -> Optional[datetime.datetime]:
+    """Parse an ISO timestamp or pass through a datetime object."""
+    if value is None or isinstance(value, datetime.datetime):
+        return value
+    # Try common formats.
+    for fmt in ("%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d"):
+        try:
+            return datetime.datetime.strptime(value, fmt)
+        except ValueError:
+            continue
+    # Python 3.10+ isoformat with timezone.
+    try:
+        return datetime.datetime.fromisoformat(value)
+    except ValueError:
+        return None
+
+
+@dataclasses.dataclass
+class AuditFilters:
+    """Filters for audit export queries."""
+
+    since: Optional[datetime.datetime] = None
+    until: Optional[datetime.datetime] = None
+    project: Optional[str] = None
+    workspace: Optional[str] = None
+    user: Optional[str] = None
+    event_type: Optional[str] = None
+    actor: Optional[str] = None
+    limit: Optional[int] = None
+
+    def matches(self, event: Dict[str, Any]) -> bool:
+        ts = event.get("timestamp")
+        if ts:
+            parsed = _parse_iso(ts)
+            if parsed:
+                if self.since is not None and parsed < self.since:
+                    return False
+                if self.until is not None and parsed > self.until:
+                    return False
+
+        if self.event_type and event.get("event_type") != self.event_type:
+            return False
+        if self.actor and event.get("actor") != self.actor:
+            return False
+
+        payload = event.get("payload", {}) or {}
+        if isinstance(payload, dict):
+            if self.project and payload.get("project") != self.project:
+                return False
+            if self.workspace and payload.get("workspace") != self.workspace:
+                return False
+            if self.user and payload.get("user") != self.user:
+                return False
+
+        return True
+
+
+class AuditExporter:
+    """Export filtered, scrubbed audit events."""
+
+    def __init__(
+        self,
+        gcc_dir: Path,
+        privacy_config: PrivacyConfig | None = None,
+        scrubber: RegexScrubber | None = None,
+    ):
+        self.gcc_dir = gcc_dir
+        self.privacy_config = privacy_config or PrivacyConfig()
+        self.scrubber = scrubber or RegexScrubber()
+        self.event_log = gcc_dir / "events.log.jsonl"
+
+    def load_events(self, filters: AuditFilters | None = None) -> List[Dict[str, Any]]:
+        """Read and filter events from the event log."""
+        filters = filters or AuditFilters()
+        events: List[Dict[str, Any]] = []
+        if not self.event_log.exists():
+            return events
+
+        with self.event_log.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    event = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+
+                if not filters.matches(event):
+                    continue
+
+                event = apply_privacy_policy(event, self.privacy_config, self.scrubber)
+                events.append(event)
+
+                if filters.limit is not None and len(events) >= filters.limit:
+                    break
+
+        return events
+
+    def export_json(
+        self,
+        events: List[Dict[str, Any]],
+        output_path: Path,
+        filters: AuditFilters | None = None,
+        repo_name: str = "",
+    ) -> None:
+        """Write events to a JSON file."""
+        report = self._build_report(events, filters, repo_name)
+        output_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
+
+    def export_pdf(
+        self,
+        events: List[Dict[str, Any]],
+        output_path: Path,
+        filters: AuditFilters | None = None,
+        repo_name: str = "",
+    ) -> None:
+        """Write events to a PDF file using reportlab."""
+        try:
+            from reportlab.lib import colors
+            from reportlab.lib.pagesizes import letter
+            from reportlab.platypus import (
+                SimpleDocTemplate,
+                Paragraph,
+                Spacer,
+                Table,
+                TableStyle,
+            )
+            from reportlab.lib.styles import getSampleStyleSheet
+            from reportlab.lib.units import inch
+        except ImportError as exc:
+            raise RuntimeError("reportlab is required for PDF export; install devtorch-core[audit]") from exc
+
+        report = self._build_report(events, filters, repo_name)
+        doc = SimpleDocTemplate(str(output_path), pagesize=letter)
+        styles = getSampleStyleSheet()
+        story: List[Any] = []
+
+        story.append(Paragraph("DevTorch Audit Report", styles["Title"]))
+        story.append(Paragraph(f"Repository: {report['repo_name']}", styles["Normal"]))
+        story.append(Paragraph(f"Generated: {report['generated_at']}", styles["Normal"]))
+        story.append(Paragraph(f"Event count: {report['event_count']}", styles["Normal"]))
+        story.append(Spacer(1, 0.2 * inch))
+
+        filters = report["filters"]
+        filter_text = (
+            f"Filters — since: {filters.get('since')}, until: {filters.get('until')}, "
+            f"project: {filters.get('project')}, workspace: {filters.get('workspace')}, "
+            f"user: {filters.get('user')}, event_type: {filters.get('event_type')}, "
+            f"actor: {filters.get('actor')}, limit: {filters.get('limit')}"
+        )
+        story.append(Paragraph(filter_text, styles["Normal"]))
+        story.append(Spacer(1, 0.2 * inch))
+
+        if events:
+            headers = ["Timestamp", "Actor", "Type", "Hash", "Payload"]
+            data = [headers]
+            for event in events:
+                payload = event.get("payload", "")
+                if isinstance(payload, dict):
+                    payload_text = json.dumps(payload, indent=2)
+                else:
+                    payload_text = str(payload)
+                data.append(
+                    [
+                        Paragraph(str(event.get("timestamp", "")), styles["Normal"]),
+                        Paragraph(str(event.get("actor", "")), styles["Normal"]),
+                        Paragraph(str(event.get("event_type", "")), styles["Normal"]),
+                        Paragraph(str(event.get("hash", "")[:16]), styles["Normal"]),
+                        Paragraph(payload_text, styles["Normal"]),
+                    ]
+                )
+
+            table = Table(data, colWidths=[1.1 * inch, 0.9 * inch, 0.9 * inch, 1.0 * inch, 3.2 * inch])
+            table.setStyle(
+                TableStyle(
+                    [
+                        ("BACKGROUND", (0, 0), (-1, 0), colors.grey),
+                        ("TEXTCOLOR", (0, 0), (-1, 0), colors.whitesmoke),
+                        ("ALIGN", (0, 0), (-1, -1), "LEFT"),
+                        ("VALIGN", (0, 0), (-1, -1), "TOP"),
+                        ("FONTNAME", (0, 0), (-1, 0), "Helvetica-Bold"),
+                        ("FONTSIZE", (0, 0), (-1, 0), 10),
+                        ("BOTTOMPADDING", (0, 0), (-1, 0), 12),
+                        ("BACKGROUND", (0, 1), (-1, -1), colors.beige),
+                        ("GRID", (0, 0), (-1, -1), 1, colors.black),
+                    ]
+                )
+            )
+            story.append(table)
+        else:
+            story.append(Paragraph("No events matched the selected filters.", styles["Normal"]))
+
+        doc.build(story, onFirstPage=self._footer, onLaterPages=self._footer)
+
+    def _build_report(
+        self,
+        events: List[Dict[str, Any]],
+        filters: AuditFilters | None,
+        repo_name: str,
+    ) -> Dict[str, Any]:
+        return {
+            "repo_name": repo_name or str(self.gcc_dir.parent),
+            "generated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+            "event_count": len(events),
+            "filters": self._filters_dict(filters),
+            "events": events,
+        }
+
+    def _filters_dict(self, filters: AuditFilters | None) -> Dict[str, Any]:
+        if filters is None:
+            return {
+                "since": None,
+                "until": None,
+                "project": None,
+                "workspace": None,
+                "user": None,
+                "event_type": None,
+                "actor": None,
+                "limit": None,
+            }
+
+        def fmt(value):
+            if value is None:
+                return None
+            if isinstance(value, datetime.datetime):
+                return value.isoformat()
+            return str(value)
+
+        return {
+            "since": fmt(filters.since),
+            "until": fmt(filters.until),
+            "project": filters.project,
+            "workspace": filters.workspace,
+            "user": filters.user,
+            "event_type": filters.event_type,
+            "actor": filters.actor,
+            "limit": filters.limit,
+        }
+
+    def _footer(self, canvas, doc) -> None:
+        """Draw page number and timestamp in the footer."""
+        from reportlab.lib.units import inch
+        canvas.saveState()
+        canvas.setFont("Helvetica", 8)
+        footer_text = (
+            f"Page {doc.page} • DevTorch Audit Report • "
+            f"{datetime.datetime.now(datetime.timezone.utc).isoformat()}"
+        )
+        canvas.drawString(0.75 * inch, 0.5 * inch, footer_text)
+        canvas.restoreState()
+
+
+# Default convenience wrapper.
+
+def export_audit(
+    gcc_dir: Path,
+    output_path: Path,
+    fmt: str = "json",
+    filters: AuditFilters | None = None,
+    privacy_config: PrivacyConfig | None = None,
+    repo_name: str = "",
+) -> Dict[str, Any]:
+    """One-shot export helper used by the CLI."""
+    exporter = AuditExporter(gcc_dir, privacy_config=privacy_config)
+    events = exporter.load_events(filters)
+    if fmt == "json":
+        exporter.export_json(events, output_path, filters=filters, repo_name=repo_name)
+    elif fmt == "pdf":
+        exporter.export_pdf(events, output_path, filters=filters, repo_name=repo_name)
+    else:
+        raise ValueError(f"Unsupported export format: {fmt}")
+    return {
+        "format": fmt,
+        "output_path": str(output_path),
+        "event_count": len(events),
+    }

devtorch_core/audit/privacy.py

@@ -0,0 +1,101 @@
+"""Privacy policy configuration and enforcement.
+
+The configuration lives in ``.GCC/privacy.json`` and is versioned with the
+repository. It controls whether audit logs, prompts, and telemetry may leave the
+local environment, and whether redaction is applied before export.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+from pathlib import Path
+from typing import Any, Dict
+
+from .scrubber import RegexScrubber, Scrubber
+
+
+PRIVACY_FILE_NAME = "privacy.json"
+
+PROMPT_EVENT_TYPES = {"PROMPT", "THINKING", "REASONING", "CHAT", "LLM_REQUEST", "PROMPT_ARTIFACT"}
+
+
+@dataclasses.dataclass
+class PrivacyConfig:
+    """Privacy policy knobs for a DevTorch repository."""
+
+    telemetry_enabled: bool = True
+    prompt_telemetry_enabled: bool = True
+    redaction_enabled: bool = True
+    allow_remote_endpoints: bool = False
+    retention_days: int = 365
+
+    def to_dict(self) -> Dict[str, Any]:
+        return dataclasses.asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "PrivacyConfig":
+        return cls(
+            telemetry_enabled=bool(data.get("telemetry_enabled", True)),
+            prompt_telemetry_enabled=bool(data.get("prompt_telemetry_enabled", True)),
+            redaction_enabled=bool(data.get("redaction_enabled", True)),
+            allow_remote_endpoints=bool(data.get("allow_remote_endpoints", False)),
+            retention_days=int(data.get("retention_days", 365)),
+        )
+
+
+def load_privacy_config(gcc_dir: Path) -> PrivacyConfig:
+    """Load privacy config from ``.GCC/privacy.json`` or return defaults."""
+    path = gcc_dir / PRIVACY_FILE_NAME
+    if not path.exists():
+        return PrivacyConfig()
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return PrivacyConfig()
+    return PrivacyConfig.from_dict(data)
+
+
+def save_privacy_config(gcc_dir: Path, config: PrivacyConfig) -> None:
+    """Persist privacy config to ``.GCC/privacy.json``."""
+    path = gcc_dir / PRIVACY_FILE_NAME
+    path.write_text(json.dumps(config.to_dict(), indent=2), encoding="utf-8")
+
+
+def _is_prompt_event(event: Dict[str, Any]) -> bool:
+    """Heuristic: event likely contains prompt text."""
+    event_type = event.get("event_type", "")
+    if isinstance(event_type, str) and event_type.upper() in PROMPT_EVENT_TYPES:
+        return True
+    payload = event.get("payload", {})
+    if isinstance(payload, dict):
+        keys = {k.lower() for k in payload.keys()}
+        if keys & {"prompt", "system_prompt", "messages", "prompt_text"}:
+            return True
+    return False
+
+
+def apply_privacy_policy(
+    event: Dict[str, Any],
+    config: PrivacyConfig,
+    scrubber: Scrubber | None = None,
+) -> Dict[str, Any]:
+    """Apply privacy policy to a single event.
+
+    * Redaction is applied when ``redaction_enabled`` is true.
+    * Prompt telemetry is suppressed when ``prompt_telemetry_enabled`` is false.
+    * Non-prompt telemetry is suppressed when ``telemetry_enabled`` is false.
+    * Suppressed events are returned with a ``_telemetry_excluded`` flag and the
+      payload replaced by ``{ "excluded": true }``.
+    """
+    scrubber = scrubber or RegexScrubber()
+    if not config.telemetry_enabled:
+        return {**event, "_telemetry_excluded": True, "payload": {"excluded": True}}
+
+    if not config.prompt_telemetry_enabled and _is_prompt_event(event):
+        return {**event, "_telemetry_excluded": True, "payload": {"excluded": True}}
+
+    if config.redaction_enabled:
+        return scrubber.scrub(event)
+
+    return event

devtorch_core/audit/scrubber.py

@@ -0,0 +1,149 @@
+"""PII and secret scrubbing for audit exports.
+
+Regex-based scrubber is the default. ``presidio-analyzer`` is supported as an
+opt-in backend because pulling a full spaCy model into the OSS core is
+prohibitively heavy.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, List, Pattern, Tuple
+
+
+DEFAULT_REDACTION = "[REDACTED]"
+
+
+# Patterns are tuples of (label, compiled_regex, replacement_template).
+# The replacement template may reference groups if the regex contains groups.
+DEFAULT_PATTERNS: List[Tuple[str, Pattern[str], str]] = [
+    (
+        "email",
+        re.compile(r"(?i)\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b"),
+        "[REDACTED_EMAIL]",
+    ),
+    (
+        "phone",
+        re.compile(
+            r"\b(?:\+\d{1,2}\s?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b"
+        ),
+        "[REDACTED_PHONE]",
+    ),
+    (
+        "ssn",
+        re.compile(r"\b\d{3}-\d{2}-\d{4}\b"),
+        "[REDACTED_SSN]",
+    ),
+    (
+        "credit_card",
+        re.compile(r"\b(?:\d[ -]*?){13,16}\b"),
+        "[REDACTED_CC]",
+    ),
+    (
+        "openai_key",
+        re.compile(r"\b(sk-[a-zA-Z0-9]{20,})\b"),
+        "[REDACTED_API_KEY]",
+    ),
+    (
+        "aws_key",
+        re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
+        "[REDACTED_AWS_KEY]",
+    ),
+    (
+        "github_token",
+        re.compile(
+            r"\b(ghp_[a-zA-Z0-9]{36}|github_pat_[a-zA-Z0-9_]{22,}|gho_[a-zA-Z0-9]{36})\b"
+        ),
+        "[REDACTED_GITHUB_TOKEN]",
+    ),
+    (
+        "bearer_token",
+        re.compile(r"\bBearer\s+[a-zA-Z0-9_\-\.]+\b", flags=re.IGNORECASE),
+        "[REDACTED_BEARER_TOKEN]",
+    ),
+    (
+        "private_key",
+        re.compile(
+            r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----",
+            flags=re.DOTALL | re.IGNORECASE,
+        ),
+        "[REDACTED_PRIVATE_KEY]",
+    ),
+    (
+        "generic_secret",
+        re.compile(
+            r"(?i)(api[_-]?key|secret|password|token)\s*[:=]\s*['\"]?([a-zA-Z0-9_\-/+=]{8,})['\"]?"
+        ),
+        r"\1=[REDACTED_SECRET]",
+    ),
+]
+
+
+class Scrubber:
+    """Abstract scrubber interface."""
+
+    def scrub(self, obj: Any) -> Any:
+        raise NotImplementedError
+
+
+class RegexScrubber(Scrubber):
+    """Redact PII and secrets using regex patterns."""
+
+    def __init__(self, patterns: List[Tuple[str, Pattern[str], str]] | None = None):
+        self.patterns = patterns or DEFAULT_PATTERNS
+
+    def scrub(self, obj: Any) -> Any:
+        if isinstance(obj, str):
+            return self._scrub_str(obj)
+        if isinstance(obj, dict):
+            return {k: self.scrub(v) for k, v in obj.items()}
+        if isinstance(obj, list):
+            return [self.scrub(item) for item in obj]
+        return obj
+
+    def _scrub_str(self, text: str) -> str:
+        for _label, pattern, replacement in self.patterns:
+            text = pattern.sub(replacement, text)
+        return text
+
+
+class PresidioScrubber(Scrubber):
+    """Lightweight wrapper around Microsoft Presidio analyzer.
+
+    Falls back to ``RegexScrubber`` for any entity type that Presidio does not
+    support (e.g., API keys, bearer tokens).
+    """
+
+    def __init__(self, additional_patterns: List[Tuple[str, Pattern[str], str]] | None = None):
+        from presidio_analyzer import AnalyzerEngine
+        self._analyzer = AnalyzerEngine()
+        self._fallback = RegexScrubber(additional_patterns or DEFAULT_PATTERNS)
+
+    def scrub(self, obj: Any) -> Any:
+        if not isinstance(obj, str):
+            return self._fallback.scrub(obj)
+        return self._scrub_str(obj)
+
+    def _scrub_str(self, text: str) -> str:
+        results = self._analyzer.analyze(text=text, language="en")
+        # Redact from end to start so indices stay stable.
+        redacted = text
+        for result in sorted(results, key=lambda r: r.start, reverse=True):
+            redacted = redacted[: result.start] + "[REDACTED]" + redacted[result.end :]
+        return self._fallback._scrub_str(redacted)
+
+
+def make_scrubber(use_presidio: bool = False) -> Scrubber:
+    """Factory: returns a regex scrubber unless ``use_presidio=True``."""
+    if use_presidio:
+        try:
+            return PresidioScrubber()
+        except Exception:
+            return RegexScrubber()
+    return RegexScrubber()
+
+
+def scrub_dict(obj: Any, scrubber: Scrubber | None = None) -> Any:
+    """Convenience helper for one-off scrubbing."""
+    scrubber = scrubber or RegexScrubber()
+    return scrubber.scrub(obj)

devtorch_core/audit/service.py

@@ -0,0 +1,67 @@
+"""High-level audit service used by the CLI and MCP server.
+
+Keeps the orchestration logic in one place so both command-line users and
+programmatic callers get the same behaviour.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Dict, List
+
+from .exporter import AuditExporter, AuditFilters, export_audit
+from .privacy import PrivacyConfig, apply_privacy_policy, load_privacy_config
+from .scrubber import RegexScrubber
+from .signing import ExportSigner
+
+
+class AuditService:
+    """Facade for audit export, scrubbing, and signing operations."""
+
+    def __init__(self, gcc_dir: Path):
+        self.gcc_dir = gcc_dir
+        self.exporter = AuditExporter(gcc_dir)
+        self.signer = ExportSigner()
+
+    def get_privacy_config(self) -> PrivacyConfig:
+        return load_privacy_config(self.gcc_dir)
+
+    def query_events(self, filters: AuditFilters | None = None) -> List[Dict[str, Any]]:
+        """Return filtered, policy-applied events."""
+        privacy = self.get_privacy_config()
+        return self.exporter.load_events(filters)
+
+    def export(
+        self,
+        output_path: Path,
+        fmt: str = "json",
+        filters: AuditFilters | None = None,
+        repo_name: str = "",
+        privacy_config: PrivacyConfig | None = None,
+    ) -> Dict[str, Any]:
+        """Export events to JSON or PDF."""
+        return export_audit(
+            self.gcc_dir,
+            output_path,
+            fmt=fmt,
+            filters=filters,
+            repo_name=repo_name,
+            privacy_config=privacy_config,
+        )
+
+    def sign_export(self, input_path: Path, output_path: Path, private_key_path: Path) -> None:
+        """Sign an exported JSON file and write the signed envelope."""
+        self.signer.sign_file(input_path, output_path, private_key_path)
+
+    def verify_export(self, envelope_path: Path, public_key_path: Path) -> bool:
+        """Verify a signed export envelope."""
+        return self.signer.verify_file(envelope_path, public_key_path)
+
+    def scrub(self, obj: Any) -> Any:
+        """Run the default regex scrubber on an arbitrary object."""
+        return RegexScrubber().scrub(obj)
+
+    def apply_privacy(self, event: Dict[str, Any], config: PrivacyConfig | None = None) -> Dict[str, Any]:
+        """Apply the repository privacy policy (or a supplied config) to an event."""
+        config = config or self.get_privacy_config()
+        return apply_privacy_policy(event, config, self.exporter.scrubber)