traceforge-toolkit 0.1.0__py3-none-any.whl

traceforge/gates/pii_patterns.yaml

@@ -0,0 +1,228 @@
+# PII detection patterns — extracted from Microsoft Presidio recognizers.
+# Pure regex, no NLP dependency. Patterns tested against Presidio's own test suites.
+#
+# Structure:
+#   entity_type: The PII category name
+#   patterns: list of {regex, score, name}
+#     - regex: Python re pattern (IGNORECASE applied globally)
+#     - score: 0.0-1.0 confidence (higher = fewer false positives)
+#     - name: human label for the pattern variant
+#   context: list of nearby words that boost confidence (+0.15)
+#   validator: optional Python function name for checksum validation (→ score 1.0)
+#   critical: if true, presence triggers SUPPRESS instead of REDACT
+
+version: "pii-v1"
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Credit Cards — Luhn-validated
+# Source: presidio-analyzer/predefined_recognizers/credit_card_recognizer.py
+# ═══════════════════════════════════════════════════════════════════════════════
+entities:
+  - entity_type: CREDIT_CARD
+    critical: true
+    context: [credit, card, visa, mastercard, amex, discover, diners, jcb, maestro, payment]
+    validator: luhn
+    patterns:
+      - name: visa
+        regex: '\b4\d{3}[\s\-]?\d{4}[\s\-]?\d{4}[\s\-]?\d{4}\b'
+        score: 0.5
+      - name: mastercard
+        regex: '\b5[1-5]\d{2}[\s\-]?\d{4}[\s\-]?\d{4}[\s\-]?\d{4}\b'
+        score: 0.5
+      - name: amex
+        regex: '\b3[47]\d{2}[\s\-]?\d{6}[\s\-]?\d{5}\b'
+        score: 0.5
+      - name: discover
+        regex: '\b6(?:011|5\d{2})[\s\-]?\d{4}[\s\-]?\d{4}[\s\-]?\d{4}\b'
+        score: 0.5
+      - name: generic_16
+        regex: '\b\d{4}[\s\-]?\d{4}[\s\-]?\d{4}[\s\-]?\d{4}\b'
+        score: 0.3
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # US Social Security Numbers
+  # Source: presidio-analyzer/predefined_recognizers/us_ssn_recognizer.py
+  # ═══════════════════════════════════════════════════════════════════════════
+  - entity_type: US_SSN
+    critical: true
+    context: [social, security, ssn, ss#, "social security", taxpayer, tin]
+    patterns:
+      - name: ssn_dashes
+        regex: '\b(?!000|666|9\d{2})\d{3}-(?!00)\d{2}-(?!0000)\d{4}\b'
+        score: 0.85
+      - name: ssn_no_dashes
+        regex: '\b(?!000|666|9\d{2})\d{3}(?!00)\d{2}(?!0000)\d{4}\b'
+        score: 0.3
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # Email Addresses
+  # Source: presidio-analyzer/predefined_recognizers/email_recognizer.py
+  # ═══════════════════════════════════════════════════════════════════════════
+  - entity_type: EMAIL_ADDRESS
+    critical: false
+    context: [email, mail, e-mail, address, contact, send, to, from, cc, bcc]
+    patterns:
+      - name: email
+        regex: '\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b'
+        score: 0.85
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # Phone Numbers (international)
+  # Source: presidio-analyzer/predefined_recognizers/phone_recognizer.py
+  # ═══════════════════════════════════════════════════════════════════════════
+  - entity_type: PHONE_NUMBER
+    critical: false
+    context: [phone, call, tel, telephone, mobile, cell, fax, contact, dial, number]
+    patterns:
+      - name: us_phone
+        regex: '(?:\+?1[\s\-.]?)?\(?\d{3}\)?[\s\-.]?\d{3}[\s\-.]?\d{4}\b'
+        score: 0.4
+      - name: intl_phone
+        regex: '\+\d{1,3}[\s\-.]?\d{1,4}[\s\-.]?\d{2,4}[\s\-.]?\d{2,4}(?:[\s\-.]?\d{2,4})?'
+        score: 0.4
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # IP Addresses
+  # Source: presidio-analyzer/predefined_recognizers/ip_recognizer.py
+  # ═══════════════════════════════════════════════════════════════════════════
+  - entity_type: IP_ADDRESS
+    critical: false
+    context: [ip, address, host, server, network, subnet, dns, proxy]
+    patterns:
+      - name: ipv4
+        regex: '\b(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b'
+        score: 0.6
+      - name: ipv6
+        regex: '\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b'
+        score: 0.6
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # IBAN (International Bank Account Number)
+  # Source: presidio-analyzer/predefined_recognizers/iban_recognizer.py
+  # ═══════════════════════════════════════════════════════════════════════════
+  - entity_type: IBAN_CODE
+    critical: true
+    context: [iban, bank, account, transfer, swift, bic, wire, payment]
+    validator: iban
+    patterns:
+      - name: iban_generic
+        regex: '\b[A-Z]{2}\d{2}[\s]?[\dA-Z]{4}[\s]?(?:[\dA-Z]{4}[\s]?){1,7}[\dA-Z]{1,4}\b'
+        score: 0.5
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # US Passport
+  # Source: presidio-analyzer/predefined_recognizers/us_passport_recognizer.py
+  # ═══════════════════════════════════════════════════════════════════════════
+  - entity_type: US_PASSPORT
+    critical: true
+    context: [passport, travel, document, us passport, state department]
+    patterns:
+      - name: us_passport
+        regex: '\b[A-Z]?\d{8,9}\b'
+        score: 0.1  # Very weak without context — lots of 8-9 digit numbers
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # US Bank Account / Routing Numbers
+  # Source: presidio-analyzer/predefined_recognizers/us_bank_recognizer.py
+  # ═══════════════════════════════════════════════════════════════════════════
+  - entity_type: US_BANK_NUMBER
+    critical: true
+    context: [bank, account, routing, aba, checking, savings, wire, ach]
+    patterns:
+      - name: us_routing
+        regex: '\b\d{9}\b'
+        score: 0.05  # Very weak alone — 9-digit numbers are everywhere
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # Cryptocurrency Addresses
+  # Source: presidio-analyzer/predefined_recognizers/crypto_recognizer.py
+  # ═══════════════════════════════════════════════════════════════════════════
+  - entity_type: CRYPTO
+    critical: false
+    context: [bitcoin, btc, eth, ethereum, wallet, crypto, blockchain, address]
+    patterns:
+      - name: btc_p2pkh
+        regex: '\b[13][a-km-zA-HJ-NP-Z1-9]{25,34}\b'
+        score: 0.5
+      - name: btc_bech32
+        regex: '\bbc1[a-z0-9]{39,59}\b'
+        score: 0.7
+      - name: eth
+        regex: '\b0x[0-9a-fA-F]{40}\b'
+        score: 0.7
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # API Keys / Secrets (traceforge addition — not in Presidio)
+  # ═══════════════════════════════════════════════════════════════════════════
+  - entity_type: API_KEY
+    critical: true
+    context: [key, token, secret, api, bearer, authorization, auth, credential, password]
+    patterns:
+      - name: openai_key
+        regex: '\bsk-[A-Za-z0-9]{20,}\b'
+        score: 0.9
+      - name: github_pat
+        regex: '\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b'
+        score: 0.95
+      - name: aws_access_key
+        regex: '\bAKIA[0-9A-Z]{16}\b'
+        score: 0.95
+      - name: aws_secret_key
+        regex: '\b[A-Za-z0-9/+=]{40}\b'
+        score: 0.1  # Very weak — needs context
+      - name: azure_key
+        regex: '\b[A-Za-z0-9+/]{86}==\b'
+        score: 0.6
+      - name: generic_bearer
+        regex: '\b(?:Bearer|token)\s+[A-Za-z0-9\-._~+/]+=*\b'
+        score: 0.4
+      - name: slack_token
+        regex: '\bxox[baprs]-[0-9a-zA-Z\-]{10,}\b'
+        score: 0.9
+      - name: stripe_key
+        regex: '\b(?:sk|pk)_(?:live|test)_[A-Za-z0-9]{20,}\b'
+        score: 0.95
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # URLs (for detecting leaked internal URLs)
+  # Source: presidio-analyzer/predefined_recognizers/url_recognizer.py
+  # ═══════════════════════════════════════════════════════════════════════════
+  - entity_type: URL
+    critical: false
+    context: [url, link, endpoint, webhook, redirect, href, src]
+    patterns:
+      - name: url_http
+        regex: '\bhttps?://[^\s<>\"\'']{3,}\b'
+        score: 0.5
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Allow list — values that match PII patterns but should never be flagged
+# ═══════════════════════════════════════════════════════════════════════════════
+default_allow_list:
+  - "127.0.0.1"
+  - "0.0.0.0"
+  - "localhost"
+  - "::1"
+  - "255.255.255.255"
+  - "192.168.0.0"
+  - "10.0.0.0"
+  - "172.16.0.0"
+  - "example.com"
+  - "test@example.com"
+  - "noreply@github.com"
+  - "0000-0000-0000-0000"
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Context boost — how much nearby context words increase confidence
+# ═══════════════════════════════════════════════════════════════════════════════
+context_boost: 0.25
+context_window: 50  # characters to look around the match for context words
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Score thresholds for gating decisions
+# ═══════════════════════════════════════════════════════════════════════════════
+thresholds:
+  # Minimum score to consider a match a true finding
+  detection: 0.4
+  # Minimum score for redaction (below this, no action taken)
+  redaction: 0.5

traceforge/governance/__init__.py

@@ -0,0 +1,121 @@
+"""Governance extensions — enrichment pipeline, labeling, and persistence."""
+
+from traceforge.governance.types import (
+    CommandAnalysis,
+    EnrichmentContext,
+    PipeSegment,
+    SessionEvent,
+    ToolCallEvent,
+    ToolResultEvent,
+    compute_source_event_key,
+)
+from traceforge.governance.state import (
+    BudgetSnapshot,
+    SessionState,
+    SessionStateSnapshot,
+    TaintEntry,
+)
+from traceforge.governance.persistence import SystemStore
+from traceforge.governance.labeler import GovernanceLabeler, GovernanceResult
+from traceforge.governance.results import (
+    Evidence,
+    EvidencePointer,
+    EscalationContext,
+    Phase3Result,
+    RecommendedAction,
+    RecommendationResult,
+    RiskRecommendation,
+    SessionMeta,
+    TransformSuggestion,
+)
+from traceforge.governance.pipeline import GovernancePipeline
+from traceforge.governance.rules import (
+    Predicate,
+    Rule,
+    RuleMatch,
+    RecommendationTemplate,
+    evaluate_rules,
+    parse_rules,
+)
+from traceforge.governance.risk_wrapper import RiskModifiers, assess_governance_risk
+from traceforge.governance.canonical import compute_canonical_hash
+from traceforge.governance.pii import PIIScanner
+from traceforge.governance.ifc import IFCChecker, SCOPE_TO_LABEL, PATH_LABEL_RULES
+from traceforge.governance.integrity import IntegrityVerifier
+from traceforge.governance.mcp_drift import MCPIntegrityScanner, MCPIntegrityAlert, MCPToolProfile
+from traceforge.governance.drift import DriftDetector, DriftAssessment
+from traceforge.governance.budget import BudgetThresholds, BudgetTracker
+from traceforge.governance.envelope import ContextGapEvent, EnrichedEvent
+from traceforge.governance.emitter import EnrichedEmitter
+from traceforge.governance.observer import (
+    AgentContext,
+    GovernanceObserver,
+    TraceforgeObserver,
+    create_observer,
+)
+
+__all__ = [
+    # Types
+    "CommandAnalysis",
+    "EnrichmentContext",
+    "PipeSegment",
+    "SessionEvent",
+    "ToolCallEvent",
+    "ToolResultEvent",
+    "compute_source_event_key",
+    # State
+    "BudgetSnapshot",
+    "SessionState",
+    "SessionStateSnapshot",
+    "TaintEntry",
+    # Persistence
+    "SystemStore",
+    # Labeler
+    "GovernanceLabeler",
+    "GovernanceResult",
+    # Pipeline
+    "Evidence",
+    "EvidencePointer",
+    "EscalationContext",
+    "GovernancePipeline",
+    "Phase3Result",
+    "RecommendedAction",
+    "RecommendationResult",
+    "RiskRecommendation",
+    "SessionMeta",
+    "TransformSuggestion",
+    # Rules
+    "Predicate",
+    "Rule",
+    "RuleMatch",
+    "RecommendationTemplate",
+    "evaluate_rules",
+    "parse_rules",
+    # Risk
+    "RiskModifiers",
+    "assess_governance_risk",
+    # Canonical
+    "compute_canonical_hash",
+    # Scanners
+    "PIIScanner",
+    "IFCChecker",
+    "SCOPE_TO_LABEL",
+    "PATH_LABEL_RULES",
+    "IntegrityVerifier",
+    "MCPIntegrityScanner",
+    "MCPIntegrityAlert",
+    "MCPToolProfile",
+    "DriftDetector",
+    "DriftAssessment",
+    # Budget
+    "BudgetThresholds",
+    "BudgetTracker",
+    # Envelope & Observer
+    "ContextGapEvent",
+    "EnrichedEvent",
+    "EnrichedEmitter",
+    "TraceforgeObserver",
+    "GovernanceObserver",
+    "AgentContext",
+    "create_observer",
+]