privaci 0.1.0b4__py3-none-any.whl

privaci/__init__.py

@@ -0,0 +1,5 @@
+"""PrivaCI — in-VPC PostgreSQL masking and anonymization engine."""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"

privaci/__main__.py

@@ -0,0 +1,8 @@
+"""Allow `python -m privaci` to invoke the CLI."""
+
+from __future__ import annotations
+
+from privaci.cli.app import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

privaci/autodetect/__init__.py

@@ -0,0 +1,27 @@
+"""PII column auto-detection."""
+
+from __future__ import annotations
+
+from privaci.autodetect.models import (
+    DetectionConfidence,
+    DetectionFinding,
+    DetectionResult,
+)
+from privaci.autodetect.report import write_detection_report
+from privaci.autodetect.resolve import (
+    build_detection,
+    resolve_effective_table_config,
+    uncovered_strict_columns,
+)
+from privaci.autodetect.scanner import scan_catalog
+
+__all__ = [
+    "DetectionConfidence",
+    "DetectionFinding",
+    "DetectionResult",
+    "build_detection",
+    "resolve_effective_table_config",
+    "scan_catalog",
+    "uncovered_strict_columns",
+    "write_detection_report",
+]

privaci/autodetect/actions.py

@@ -0,0 +1,70 @@
+"""Map pattern rules to config column actions, respecting column types."""
+
+from __future__ import annotations
+
+from privaci.autodetect.patterns import PatternRule
+from privaci.catalog.models import ColumnInfo
+from privaci.config.actions import (
+    ColumnAction,
+    FakeAction,
+    HashAction,
+    NerMaskAction,
+    StaticAction,
+)
+
+_TEXTLIKE = frozenset(
+    {"text", "character varying", "character", "varchar", "char", "citext", "name"}
+)
+
+# Column base types each fake provider can produce a valid value for. Providers
+# absent from this map are treated as text-only.
+_PROVIDER_TYPES: dict[str, frozenset[str]] = {
+    "dob": frozenset({"date"}) | _TEXTLIKE,
+    "ip_address": frozenset({"inet", "cidr"}) | _TEXTLIKE,
+    "uuid": frozenset({"uuid"}) | _TEXTLIKE,
+}
+
+
+def _base_type(data_type: str) -> str:
+    head, _, _ = data_type.strip().lower().partition("(")
+    return head.strip()
+
+
+def action_for_column(rule: PatternRule, column: ColumnInfo) -> ColumnAction | None:
+    """Build a type-compatible action for ``column``, or ``None`` to skip.
+
+    Returns ``None`` when the matched rule's action cannot produce a value
+    valid for the column's type (e.g. ``hash`` on a ``uuid`` column), so the
+    scanner can leave the column as passthrough rather than emit a broken mask.
+    """
+    base = _base_type(column.data_type)
+    if rule.action == "fake":
+        return _fake_action(rule, base)
+    if rule.action == "hash":
+        return _hash_action(base)
+    if rule.action == "static":
+        if base not in _TEXTLIKE:
+            return None
+        value = rule.static_value or "privaci-test-pw"
+        return StaticAction(action="static", value=value)
+    if base not in _TEXTLIKE:
+        return None
+    return NerMaskAction(action="ner_mask")
+
+
+def _fake_action(rule: PatternRule, base: str) -> ColumnAction | None:
+    if rule.provider is None:
+        msg = f"fake rule {rule.rule_id} missing provider"
+        raise ValueError(msg)
+    allowed = _PROVIDER_TYPES.get(rule.provider, _TEXTLIKE)
+    if base not in allowed:
+        return None
+    return FakeAction(action="fake", provider=rule.provider)
+
+
+def _hash_action(base: str) -> ColumnAction | None:
+    if base == "uuid":
+        return FakeAction(action="fake", provider="uuid")
+    if base in _TEXTLIKE:
+        return HashAction(action="hash")
+    return None

privaci/autodetect/freeform.py

@@ -0,0 +1,68 @@
+"""Freeform-text shape and confidence scoring."""
+
+from __future__ import annotations
+
+import re
+
+from privaci.autodetect.models import DetectionConfidence
+from privaci.autodetect.table_context import table_sensitivity
+from privaci.catalog.models import ColumnInfo
+
+_FREEFORM_MIN_AVG_WIDTH = 200
+_FREEFORM_UNCERTAIN_MIN_AVG_WIDTH = 100
+_TEXT_TYPES = frozenset({"text"})
+_VARCHAR_RE = re.compile(r"^character varying\((\d+)\)$", re.IGNORECASE)
+
+
+def is_freeform_eligible_type(column: ColumnInfo) -> bool:
+    """Return whether the column type can hold narrative freeform text."""
+    base = _base_type(column.data_type)
+    if base in _TEXT_TYPES:
+        return True
+    match = _VARCHAR_RE.match(column.data_type.strip())
+    if match is None:
+        return False
+    return int(match.group(1)) >= 500
+
+
+def score_freeform_confidence(
+    *,
+    table_name: str,
+    column: ColumnInfo,
+) -> tuple[DetectionConfidence, tuple[str, ...]]:
+    """Score a freeform name-match using type, stats, and table context."""
+    reasons: list[str] = []
+    if not is_freeform_eligible_type(column):
+        reasons.append(f"type {column.data_type!r} is not freeform-eligible")
+        return "low", tuple(reasons)
+
+    reasons.append("column type is freeform-eligible")
+    sensitivity = table_sensitivity(table_name)
+    avg_width = column.avg_width
+
+    if avg_width is None:
+        reasons.append("pg_stats.avg_width unavailable")
+        if sensitivity == "sensitive":
+            reasons.append("sensitive table context")
+            return "medium", tuple(reasons)
+        return "low", tuple(reasons)
+
+    reasons.append(f"pg_stats.avg_width={avg_width:.0f}")
+    if avg_width >= _FREEFORM_MIN_AVG_WIDTH:
+        if sensitivity == "low":
+            reasons.append("low-sensitivity table context downgrades confidence")
+            return "medium", tuple(reasons)
+        if sensitivity == "sensitive":
+            reasons.append("sensitive table context")
+        return "high", tuple(reasons)
+
+    if avg_width >= _FREEFORM_UNCERTAIN_MIN_AVG_WIDTH and sensitivity == "sensitive":
+        reasons.append("borderline avg_width on sensitive table")
+        return "medium", tuple(reasons)
+
+    reasons.append("avg_width below freeform threshold")
+    return "low", tuple(reasons)
+
+
+def _base_type(data_type: str) -> str:
+    return re.sub(r"\(.*\)$", "", data_type.strip().lower())

privaci/autodetect/matcher.py

@@ -0,0 +1,59 @@
+"""Column-name pattern matching for auto-detect."""
+
+from __future__ import annotations
+
+import re
+from typing import get_args
+
+from privaci.autodetect.patterns import (
+    BUILTIN_PATTERNS,
+    PatternKind,
+    PatternRule,
+    compile_patterns,
+)
+
+
+def match_column_name(column_name: str) -> PatternRule | None:
+    """Return the first pattern rule matching ``column_name``, if any.
+
+    Rules are evaluated in library order; earlier rules win.
+    """
+    lowered = column_name.lower()
+    compiled = compile_patterns()
+    for rule in BUILTIN_PATTERNS:
+        if _rule_matches(lowered, rule, compiled):
+            return rule
+    return None
+
+
+def _delimiter_bounded_substring(lowered: str, needle: str) -> bool:
+    """Match ``needle`` only at underscore boundaries or as the whole name."""
+    if lowered == needle:
+        return True
+    padded = f"_{lowered}_"
+    return f"_{needle}_" in padded
+
+
+def _rule_matches(
+    lowered: str,
+    rule: PatternRule,
+    compiled: dict[str, re.Pattern[str]],
+) -> bool:
+    kind = rule.kind
+    if kind == "substring":
+        return rule.pattern in lowered
+    elif kind == "bounded_substring":
+        return _delimiter_bounded_substring(lowered, rule.pattern)
+    elif kind == "suffix":
+        return lowered.endswith(rule.pattern)
+    elif kind in ("prefix", "wildcard_prefix"):
+        return lowered.startswith(rule.pattern)
+    elif kind == "regex":
+        pattern = compiled.get(rule.rule_id)
+        return pattern is not None and pattern.fullmatch(lowered) is not None
+    # CodeQL's mixed-return rule wants an explicit non-returning terminal rather
+    # than an implicit fall-through after the exhaustive ``kind`` chain above.
+    # Derive the valid kinds from PatternKind so the message can never drift.
+    expected = ", ".join(get_args(PatternKind))
+    msg = f"Unknown auto-detect pattern kind: {kind!r}. Expected one of: {expected}."
+    raise ValueError(msg)

privaci/autodetect/models.py

@@ -0,0 +1,63 @@
+"""Detection finding models for PII auto-detect."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+from privaci.config.actions import ColumnAction
+
+DetectionConfidence = Literal["high", "medium", "low"]
+DetectionSource = Literal["autodetect", "config"]
+
+
+@dataclass(frozen=True, slots=True)
+class DetectionFinding:
+    """Outcome of inspecting one column for PII.
+
+    Attributes:
+        table_id: Schema-qualified table identifier.
+        column_name: Column under inspection.
+        confidence: ``high`` auto-masks; ``medium`` flags review; ``low`` passthrough.
+        reasons: Human-readable scoring explanation (never contains cell values).
+        action: Proposed masking action, or ``None`` for passthrough.
+        provider: Inferred fake provider name when applicable.
+        matched_pattern: Pattern rule id that fired, if any.
+        source: Whether YAML or auto-detect produced the effective action.
+    """
+
+    table_id: str
+    column_name: str
+    confidence: DetectionConfidence
+    reasons: tuple[str, ...]
+    action: ColumnAction | None = None
+    provider: str | None = None
+    matched_pattern: str | None = None
+    source: DetectionSource = "autodetect"
+
+    def __repr__(self) -> str:
+        return (
+            f"DetectionFinding({self.table_id}.{self.column_name!r}, "
+            f"confidence={self.confidence!r})"
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class DetectionResult:
+    """All findings from one catalog scan."""
+
+    findings: tuple[DetectionFinding, ...]
+
+    def by_table(self, table_id: str) -> tuple[DetectionFinding, ...]:
+        """Return findings for ``table_id`` in column order."""
+        return tuple(f for f in self.findings if f.table_id == table_id)
+
+    def finding_for(self, table_id: str, column_name: str) -> DetectionFinding | None:
+        """Return the finding for one column, if present."""
+        for finding in self.findings:
+            if finding.table_id == table_id and finding.column_name == column_name:
+                return finding
+        return None
+
+    def __repr__(self) -> str:
+        return f"DetectionResult(findings={len(self.findings)})"

privaci/autodetect/patterns.py

@@ -0,0 +1,119 @@
+"""Built-in PII column-name pattern library."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Literal
+
+PatternKind = Literal[
+    "substring",
+    "bounded_substring",
+    "suffix",
+    "prefix",
+    "regex",
+    "wildcard_prefix",
+]
+
+
+@dataclass(frozen=True, slots=True)
+class PatternRule:
+    """One auto-detect pattern entry."""
+
+    rule_id: str
+    kind: PatternKind
+    pattern: str
+    provider: str | None
+    action: Literal["fake", "hash", "static", "ner_mask"]
+    static_value: str | None = None
+
+
+_STATIC_PASSWORD = (
+    "privaci-test-pw"  # noqa: S105 — documented test placeholder, not a secret
+)
+
+# SECURITY: fixed test placeholder for password columns — never a real hash.
+BUILTIN_PATTERNS: tuple[PatternRule, ...] = (
+    PatternRule("email_suffix", "suffix", "_email", "email", "fake"),
+    PatternRule("email_prefix", "prefix", "email_", "email", "fake"),
+    PatternRule("email", "substring", "email", "email", "fake"),
+    PatternRule("e_mail", "substring", "e_mail", "email", "fake"),
+    PatternRule("phone", "substring", "phone", "phone", "fake"),
+    PatternRule("mobile", "substring", "mobile", "phone", "fake"),
+    PatternRule("tel", "bounded_substring", "tel", "phone", "fake"),
+    PatternRule("cell", "bounded_substring", "cell", "phone", "fake"),
+    PatternRule("phone_suffix", "suffix", "_phone", "phone", "fake"),
+    PatternRule("ssn", "substring", "ssn", "ssn", "fake"),
+    PatternRule("social_security", "substring", "social_security", "ssn", "fake"),
+    PatternRule("tax_id", "substring", "tax_id", "ssn", "fake"),
+    PatternRule("national_id", "substring", "national_id", "ssn", "fake"),
+    PatternRule("first_name", "substring", "first_name", "first_name", "fake"),
+    PatternRule("fname", "substring", "fname", "first_name", "fake"),
+    PatternRule("given_name", "substring", "given_name", "first_name", "fake"),
+    PatternRule("last_name", "substring", "last_name", "last_name", "fake"),
+    PatternRule("lname", "substring", "lname", "last_name", "fake"),
+    PatternRule("surname", "substring", "surname", "last_name", "fake"),
+    PatternRule("family_name", "substring", "family_name", "last_name", "fake"),
+    PatternRule("full_name", "substring", "full_name", "full_name", "fake"),
+    PatternRule("display_name", "substring", "display_name", "full_name", "fake"),
+    PatternRule("name_exact", "regex", r"^name$", "full_name", "fake"),
+    # IP rules precede the generic `address` substring so `ip_address` is not
+    # mis-matched as a street address. `ip` is word-anchored to avoid matching
+    # substrings like `descr(ip)tion` or `sh(ip)ping`.
+    PatternRule("ip_address", "substring", "ip_address", "ip_address", "fake"),
+    PatternRule("ip_suffix", "suffix", "_ip", "ip_address", "fake"),
+    PatternRule("ip_prefix", "prefix", "ip_", "ip_address", "fake"),
+    PatternRule("ip_exact", "regex", r"^ip$", "ip_address", "fake"),
+    PatternRule("address", "substring", "address", "address", "fake"),
+    PatternRule("street", "substring", "street", "street", "fake"),
+    PatternRule("city", "substring", "city", "city", "fake"),
+    PatternRule("postcode", "substring", "postcode", "postcode", "fake"),
+    PatternRule("zip", "substring", "zip", "postcode", "fake"),
+    PatternRule("country", "substring", "country", "country", "fake"),
+    PatternRule("dob", "substring", "dob", "dob", "fake"),
+    PatternRule("date_of_birth", "substring", "date_of_birth", "dob", "fake"),
+    PatternRule("birth_date", "substring", "birth_date", "dob", "fake"),
+    PatternRule("birthday", "substring", "birthday", "dob", "fake"),
+    PatternRule("credit_card", "substring", "credit_card", "credit_card", "fake"),
+    PatternRule("card_number", "substring", "card_number", "credit_card", "fake"),
+    PatternRule("cc_number", "substring", "cc_number", "credit_card", "fake"),
+    PatternRule("pan", "bounded_substring", "pan", "credit_card", "fake"),
+    PatternRule(
+        "password_hash",
+        "substring",
+        "password_hash",
+        "password",
+        "static",
+        _STATIC_PASSWORD,
+    ),
+    PatternRule(
+        "password", "substring", "password", "password", "static", _STATIC_PASSWORD
+    ),
+    PatternRule(
+        "passwd", "substring", "passwd", "password", "static", _STATIC_PASSWORD
+    ),
+    PatternRule("pwd", "substring", "pwd", "password", "static", _STATIC_PASSWORD),
+    PatternRule("token", "substring", "token", None, "hash"),
+    PatternRule("api_key", "substring", "api_key", None, "hash"),
+    PatternRule("secret", "substring", "secret", None, "hash"),
+    PatternRule("token_suffix", "suffix", "_token", None, "hash"),
+    PatternRule("auth_prefix", "prefix", "auth_", None, "hash"),
+    PatternRule("notes_suffix", "suffix", "_notes", None, "ner_mask"),
+    PatternRule("notes_substring", "substring", "notes", None, "ner_mask"),
+    PatternRule("notes", "wildcard_prefix", "note", None, "ner_mask"),
+    PatternRule("comments", "wildcard_prefix", "comment", None, "ner_mask"),
+    PatternRule("description", "substring", "description", None, "ner_mask"),
+    PatternRule("bio", "substring", "bio", None, "ner_mask"),
+    PatternRule("about", "substring", "about", None, "ner_mask"),
+)
+
+_COMPILED_REGEX: dict[str, re.Pattern[str]] = {}
+
+
+def compile_patterns() -> dict[str, re.Pattern[str]]:
+    """Return cached compiled regex rules keyed by rule id."""
+    if not _COMPILED_REGEX:
+        for rule in BUILTIN_PATTERNS:
+            if rule.kind == "regex":
+                _COMPILED_REGEX[rule.rule_id] = re.compile(rule.pattern, re.IGNORECASE)
+    return _COMPILED_REGEX

privaci/autodetect/report.py

@@ -0,0 +1,71 @@
+"""Markdown report writer for ``privaci dry-run --report``."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from privaci.autodetect.models import DetectionFinding, DetectionResult
+from privaci.catalog.models import CatalogResult
+from privaci.config.models import Config
+
+
+def write_detection_report(
+    path: Path,
+    *,
+    catalog: CatalogResult,
+    detection: DetectionResult,
+    config: Config,
+) -> None:
+    """Write a human-readable markdown summary of detection outcomes."""
+    lines = [
+        "# PrivaCI auto-detect report",
+        "",
+        f"Tables inspected: {len(catalog.tables)}",
+        f"Auto-detect: {'on' if config.auto_detect else 'off'}",
+        f"Strict mode: {'on' if config.strict_autodetect else 'off'}",
+        "",
+    ]
+    table_ids = sorted({f.table_id for f in detection.findings})
+    for table_id in table_ids:
+        lines.extend(_table_section(table_id, detection))
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _table_section(table_id: str, detection: DetectionResult) -> list[str]:
+    findings = detection.by_table(table_id)
+    masked = [f for f in findings if f.action is not None and f.source == "autodetect"]
+    passthrough = [f for f in findings if f.action is None and f.confidence != "medium"]
+    uncertain = [f for f in findings if f.confidence == "medium"]
+    explicit = [f for f in findings if f.source == "config"]
+
+    lines = [f"## {table_id}", ""]
+    lines.append("### Masked (auto-detect)")
+    lines.extend(_finding_lines(masked))
+    lines.append("")
+    lines.append("### Explicit config")
+    lines.extend(_finding_lines(explicit))
+    lines.append("")
+    lines.append("### Uncertain (manual review)")
+    lines.extend(_finding_lines(uncertain))
+    lines.append("")
+    lines.append("### Passthrough")
+    lines.extend(_finding_lines(passthrough))
+    lines.append("")
+    return lines
+
+
+def _finding_lines(findings: list[DetectionFinding]) -> list[str]:
+    if not findings:
+        return ["- _(none)_"]
+    lines: list[str] = []
+    for finding in findings:
+        action = finding.action.action if finding.action else "passthrough"
+        provider = f", provider={finding.provider}" if finding.provider else ""
+        pattern = (
+            f", pattern={finding.matched_pattern}" if finding.matched_pattern else ""
+        )
+        lines.append(
+            f"- `{finding.column_name}`: {action}{provider}{pattern} "
+            f"({finding.confidence})"
+        )
+    return lines

privaci/autodetect/resolve.py

@@ -0,0 +1,73 @@
+"""Merge YAML config with auto-detect findings."""
+
+from __future__ import annotations
+
+from privaci.autodetect.models import DetectionResult
+from privaci.autodetect.scanner import scan_catalog
+from privaci.catalog.models import CatalogResult, TableInfo
+from privaci.config.actions import ColumnAction
+from privaci.config.models import Config, TableConfig
+
+
+def resolve_effective_table_config(
+    table: TableInfo,
+    config: Config,
+    detection: DetectionResult,
+) -> TableConfig:
+    """Return the masking config for one table after auto-detect merge."""
+    yaml_cfg = config.tables.get(table.identifier)
+    if yaml_cfg is None:
+        yaml_cfg = TableConfig()
+    if not config.auto_detect:
+        return yaml_cfg
+
+    merged_columns = dict(yaml_cfg.columns)
+    for column in table.columns:
+        if column.name in merged_columns:
+            continue
+        finding = detection.finding_for(table.identifier, column.name)
+        if finding is None or finding.action is None:
+            continue
+        merged_columns[column.name] = finding.action
+    return TableConfig(
+        strategy=yaml_cfg.strategy,
+        columns=merged_columns,
+        batch_size=yaml_cfg.batch_size,
+        null_orphan_fks=yaml_cfg.null_orphan_fks,
+    )
+
+
+def uncovered_strict_columns(
+    config: Config,
+    detection: DetectionResult,
+) -> tuple[str, ...]:
+    """Return schema-qualified columns that fail strict auto-detect."""
+    if not config.strict_autodetect:
+        return ()
+    uncovered: list[str] = []
+    for finding in detection.findings:
+        if finding.confidence not in {"high", "medium"}:
+            continue
+        if finding.matched_pattern is None:
+            continue
+        table_cfg = config.tables.get(finding.table_id)
+        if table_cfg is not None and table_cfg.strategy == "exclude":
+            continue
+        if table_cfg is not None and finding.column_name in table_cfg.columns:
+            continue
+        uncovered.append(f"{finding.table_id}.{finding.column_name}")
+    return tuple(sorted(uncovered))
+
+
+def build_detection(config: Config, catalog: CatalogResult) -> DetectionResult:
+    """Scan the catalog unless auto-detect is disabled."""
+    return scan_catalog(catalog, config)
+
+
+def effective_columns_for_table(
+    table: TableInfo,
+    config: Config,
+    detection: DetectionResult,
+) -> dict[str, ColumnAction]:
+    """Return the merged per-column action map for one table."""
+    return resolve_effective_table_config(table, config, detection).columns