policystrata 0.1.0__py3-none-any.whl

policystrata/__init__.py

@@ -0,0 +1,3 @@
+"""PolicyStrata research artifact."""
+
+__version__ = "0.1.0"

policystrata/__main__.py

@@ -0,0 +1,4 @@
+from policystrata.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

policystrata/artifact_report.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable
+from pathlib import Path
+from statistics import median
+from typing import Any
+
+try:
+    from importlib.resources.abc import Traversable
+except ImportError:  # Python 3.10 exposes Traversable from importlib.abc.
+    from importlib.abc import Traversable
+
+from policystrata.domain import BUILTIN_DOMAINS, domain_root
+from policystrata.evidence import load_run_metadata, markdown_table, run_artifact_path
+from policystrata.models import Trace, WitnessClass
+from policystrata.summary import load_traces
+
+
+def build_artifact_report(run_dir: Path, domain_path: Path | None = None) -> dict[str, Any]:
+    traces = load_traces(run_dir)
+    metadata = load_run_metadata(run_dir)
+    domain = str(metadata.get("domain", "support_saas"))
+    witness_sizes = witness_byte_sizes(run_dir, traces)
+    artifact_stats = local_tree_stats(run_dir)
+    fixture_stats = domain_fixture_stats(domain, domain_path)
+
+    return {
+        "run_dir": str(run_dir),
+        "domain": domain,
+        "suite": metadata.get("suite"),
+        "evidence_level": metadata.get("evidence_level", "deterministic_fixture"),
+        "suite_provenance": metadata.get("suite_provenance", "hand_authored"),
+        "detector_frozen": bool(metadata.get("detector_frozen", False)),
+        "traces": len(traces),
+        "non_clean_traces": sum(1 for trace in traces if trace.witness_class != WitnessClass.CLEAN),
+        "minimized_witnesses": len(witness_sizes),
+        "median_witness_bytes": int(median(witness_sizes)) if witness_sizes else 0,
+        "run_artifact_files": artifact_stats["files"],
+        "run_artifact_bytes": artifact_stats["bytes"],
+        "domain_fixture_files": fixture_stats["files"],
+        "domain_fixture_bytes": fixture_stats["bytes"],
+        "domain_fixture_lines": fixture_stats["lines"],
+        "avg_latency_ms": round(average_latency(traces), 4),
+        "p95_latency_ms": round(percentile_latency(traces, 0.95), 4),
+        "estimated_cost": sum(int(trace.cost.get("estimated", 0)) for trace in traces),
+        "requires_llm_api_key": False,
+    }
+
+
+def render_artifact_report(run_dir: Path, domain_path: Path | None = None) -> str:
+    report = build_artifact_report(run_dir, domain_path)
+    rows = [
+        ["Run", report["run_dir"]],
+        ["Domain", report["domain"]],
+        ["Suite", str(report["suite"])],
+        ["Evidence level", report["evidence_level"]],
+        ["Suite provenance", report["suite_provenance"]],
+        ["Detector frozen", "yes" if report["detector_frozen"] else "no"],
+        ["Traces", str(report["traces"])],
+        ["Non-clean traces", str(report["non_clean_traces"])],
+        ["Minimized witnesses", str(report["minimized_witnesses"])],
+        ["Median witness bytes", str(report["median_witness_bytes"])],
+        ["Average trace latency ms", str(report["avg_latency_ms"])],
+        ["P95 trace latency ms", str(report["p95_latency_ms"])],
+        ["Estimated policy cost", str(report["estimated_cost"])],
+        ["Run artifact files", str(report["run_artifact_files"])],
+        ["Run artifact bytes", str(report["run_artifact_bytes"])],
+        ["Domain fixture files", str(report["domain_fixture_files"])],
+        ["Domain fixture bytes", str(report["domain_fixture_bytes"])],
+        ["Domain fixture lines", str(report["domain_fixture_lines"])],
+        ["Requires LLM API key", "no"],
+    ]
+    return "# PolicyStrata Artifact Report\n\n" + markdown_table(["Metric", "Value"], rows) + "\n"
+
+
+def witness_byte_sizes(run_dir: Path, traces: Iterable[Trace]) -> list[int]:
+    sizes: list[int] = []
+    for trace in traces:
+        if trace.witness_path:
+            sizes.append(len(run_artifact_path(run_dir, trace.witness_path).read_bytes()))
+    return sizes
+
+
+def local_tree_stats(root: Path) -> dict[str, int]:
+    files = [path for path in root.rglob("*") if path.is_file()]
+    return {
+        "files": len(files),
+        "bytes": sum(path.stat().st_size for path in files),
+        "lines": sum(count_lines(path.read_text(encoding="utf-8")) for path in text_files(files)),
+    }
+
+
+def domain_fixture_stats(domain: str, domain_path: Path | None) -> dict[str, int]:
+    if domain_path is not None:
+        return local_tree_stats(domain_path)
+    if domain in BUILTIN_DOMAINS:
+        return traversable_tree_stats(domain_root(domain))
+    return {"files": 0, "bytes": 0, "lines": 0}
+
+
+def traversable_tree_stats(root: Traversable) -> dict[str, int]:
+    files = list(traversable_files(root))
+    return {
+        "files": len(files),
+        "bytes": sum(len(path.read_bytes()) for path in files),
+        "lines": sum(count_lines(path.read_text(encoding="utf-8")) for path in text_traversables(files)),
+    }
+
+
+def traversable_files(root: Traversable) -> Iterable[Traversable]:
+    for child in root.iterdir():
+        if child.is_dir():
+            yield from traversable_files(child)
+        elif child.is_file():
+            yield child
+
+
+def text_files(paths: Iterable[Path]) -> Iterable[Path]:
+    for path in paths:
+        if path.suffix in {".json", ".jsonl", ".md", ".sql", ".yaml", ".yml"}:
+            yield path
+
+
+def text_traversables(paths: Iterable[Traversable]) -> Iterable[Traversable]:
+    for path in paths:
+        if Path(path.name).suffix in {".sql", ".yaml", ".yml"}:
+            yield path
+
+
+def count_lines(text: str) -> int:
+    return sum(1 for line in text.splitlines() if line.strip())
+
+
+def average_latency(traces: list[Trace]) -> float:
+    if not traces:
+        return 0.0
+    return sum(trace.latency_ms for trace in traces) / len(traces)
+
+
+def percentile_latency(traces: list[Trace], percentile: float) -> float:
+    if not traces:
+        return 0.0
+    latencies = sorted(trace.latency_ms for trace in traces)
+    index = min(len(latencies) - 1, max(0, int(round(percentile * (len(latencies) - 1)))))
+    return latencies[index]
+
+
+def artifact_report_json(run_dir: Path, domain_path: Path | None = None) -> str:
+    return json.dumps(build_artifact_report(run_dir, domain_path), indent=2, sort_keys=True) + "\n"

policystrata/baselines.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from pathlib import Path
+
+from policystrata.models import Trace, WitnessClass
+from policystrata.summary import load_traces
+
+BaselinePredicate = Callable[[Trace], bool]
+
+
+def final_answer_only(trace: Trace) -> bool:
+    return trace.semantic_difference and trace.release_decision.allowed
+
+
+def sql_snapshot(trace: Trace) -> bool:
+    if trace.localized_surface != "compiler":
+        return False
+    return trace.mutation not in {"cost_estimate_ignores_expansion"}
+
+
+def validator_only(trace: Trace) -> bool:
+    return not trace.canonical_decision.allowed and trace.localized_surface != "validator"
+
+
+def db_rls_only(trace: Trace) -> bool:
+    return trace.containment_layer == "database" or (
+        trace.localized_surface == "database" and bool(trace.db_result.get("blocked_by_database"))
+    )
+
+
+def random_data_generation(trace: Trace) -> bool:
+    return trace.semantic_difference
+
+
+def naive_surface_equality(trace: Trace) -> bool:
+    canonical = trace.canonical_decision.allowed
+    return any(decision.allowed != canonical for decision in trace.surface_decisions.values())
+
+
+def defense_in_depth_stack(trace: Trace) -> bool:
+    return (
+        validator_only(trace)
+        or sql_snapshot(trace)
+        or db_rls_only(trace)
+        or final_answer_only(trace)
+    )
+
+
+BASELINES: dict[str, BaselinePredicate] = {
+    "final_answer_only": final_answer_only,
+    "sql_snapshot": sql_snapshot,
+    "validator_only": validator_only,
+    "db_rls_only": db_rls_only,
+    "random_data_generation": random_data_generation,
+    "naive_surface_equality": naive_surface_equality,
+    "defense_in_depth_stack": defense_in_depth_stack,
+}
+
+
+def evaluate_baselines(traces: list[Trace]) -> dict[str, dict[str, int | float]]:
+    total_failures = sum(1 for trace in traces if trace.witness_class != WitnessClass.CLEAN)
+    results: dict[str, dict[str, int | float]] = {}
+    for name, predicate in BASELINES.items():
+        caught = sum(1 for trace in traces if trace.witness_class != WitnessClass.CLEAN and predicate(trace))
+        results[name] = {
+            "caught": caught,
+            "total_failures": total_failures,
+            "missed": total_failures - caught,
+            "catch_rate": caught / total_failures if total_failures else 0.0,
+        }
+    return results
+
+
+def evaluate_baseline_run(run_dir: Path) -> dict[str, dict[str, int | float]]:
+    return evaluate_baselines(load_traces(run_dir))

policystrata/cli.py

@@ -0,0 +1,229 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from pydantic import ValidationError
+
+from policystrata.artifact_report import artifact_report_json, render_artifact_report
+from policystrata.baselines import evaluate_baseline_run
+from policystrata.demo import run_demo
+from policystrata.domain import BUILTIN_DOMAIN, BUILTIN_DOMAINS, copy_domain
+from policystrata.evidence import parse_run_args, render_evidence_tables
+from policystrata.exports import export_run
+from policystrata.generator import MAX_GENERATED_COUNT
+from policystrata.integrations.dbt_semantic import compare_dbt_semantic_model
+from policystrata.minimize import minimize_witness_file
+from policystrata.runner import run_suite
+from policystrata.scan_models import GateOutcome
+from policystrata.scanner import run_scan
+from policystrata.summary import summarize_run
+
+
+def generated_count_arg(value: str) -> int:
+    try:
+        count = int(value)
+    except ValueError as exc:
+        raise argparse.ArgumentTypeError("count must be an integer") from exc
+    if count < 1 or count > MAX_GENERATED_COUNT:
+        raise argparse.ArgumentTypeError(f"count must be between 1 and {MAX_GENERATED_COUNT}")
+    return count
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="policystrata",
+        description="Cross-layer policy regression testing.",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    init_parser = subparsers.add_parser(
+        "init-domain",
+        help="Copy a built-in domain fixture into the current tree.",
+    )
+    init_parser.add_argument("domain", choices=BUILTIN_DOMAINS)
+    init_parser.add_argument("--out", type=Path, default=Path("."))
+
+    demo_parser = subparsers.add_parser("demo", help="Run a 30-second built-in policy drift demo.")
+    demo_parser.add_argument("--out", type=Path, default=Path("runs/demo"))
+
+    run_parser = subparsers.add_parser("run", help="Run a deterministic benchmark suite.")
+    run_parser.add_argument("--domain", default=BUILTIN_DOMAIN)
+    run_parser.add_argument("--suite", default="seeded")
+    run_parser.add_argument("--out", type=Path, required=True)
+    run_parser.add_argument("--domain-path", type=Path, default=None)
+    run_parser.add_argument(
+        "--count",
+        type=generated_count_arg,
+        default=None,
+        help="Task count for generated suites.",
+    )
+    run_parser.add_argument(
+        "--seed",
+        type=int,
+        default=None,
+        help="Seed for generated suites.",
+    )
+
+    minimize_parser = subparsers.add_parser("minimize", help="Minimize a trace or witness JSON file.")
+    minimize_parser.add_argument("--witness", type=Path, required=True)
+
+    summarize_parser = subparsers.add_parser("summarize", help="Summarize a run directory.")
+    summarize_parser.add_argument("run_dir", type=Path)
+
+    baselines_parser = subparsers.add_parser("baselines", help="Evaluate baseline strategies for a run.")
+    baselines_parser.add_argument("run_dir", type=Path)
+
+    export_parser = subparsers.add_parser("export", help="Export a run through an external eval adapter.")
+    export_parser.add_argument("run_dir", type=Path)
+    export_parser.add_argument("--format", choices=["inspect", "benchflow"], required=True)
+    export_parser.add_argument("--out", type=Path, required=True)
+
+    evidence_parser = subparsers.add_parser("evidence", help="Render Markdown evidence tables.")
+    evidence_parser.add_argument("runs", nargs="+", help="Run directories, optionally named as suite=path.")
+    evidence_parser.add_argument("--out", type=Path, default=None)
+
+    artifact_parser = subparsers.add_parser(
+        "artifact-report",
+        help="Render reviewer-facing reproducibility and usability metrics for a run.",
+    )
+    artifact_parser.add_argument("run_dir", type=Path)
+    artifact_parser.add_argument("--domain-path", type=Path, default=None)
+    artifact_parser.add_argument("--format", choices=["markdown", "json"], default="markdown")
+    artifact_parser.add_argument("--out", type=Path, default=None)
+
+    integration_parser = subparsers.add_parser(
+        "check-integration",
+        help="Check a small external semantic-layer fixture against a PolicyStrata domain.",
+    )
+    integration_parser.add_argument("kind", choices=["dbt-semantic"])
+    integration_parser.add_argument("--domain", default=BUILTIN_DOMAIN, choices=BUILTIN_DOMAINS)
+    integration_parser.add_argument("--path", type=Path, required=True)
+    integration_parser.add_argument("--domain-path", type=Path, default=None)
+
+    scan_parser = subparsers.add_parser(
+        "scan",
+        help="Run a production policy-drift scan over configured adapters and traces.",
+    )
+    scan_parser.add_argument("--config", type=Path, default=Path("policystrata.yaml"))
+    scan_parser.add_argument("--out", type=Path, default=None)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    try:
+        return run_command(args)
+    except ValidationError as exc:
+        parser.error(format_validation_error(exc))
+    except (FileNotFoundError, ValueError) as exc:
+        parser.error(str(exc))
+    except TypeError as exc:
+        if not is_user_type_error(exc):
+            raise
+        parser.error(str(exc))
+    return 2
+
+
+def run_command(args: argparse.Namespace) -> int:
+    if args.command == "init-domain":
+        target = copy_domain(args.domain, args.out)
+        print(target)
+        return 0
+
+    if args.command == "demo":
+        print(run_demo(args.out), end="")
+        return 0
+
+    if args.command == "run":
+        traces = run_suite(args.domain, args.suite, args.out, args.domain_path, args.count, args.seed)
+        print(json.dumps({"traces": len(traces), "out": str(args.out)}, sort_keys=True))
+        return 0
+
+    if args.command == "minimize":
+        print(json.dumps(minimize_witness_file(args.witness), indent=2, sort_keys=True))
+        return 0
+
+    if args.command == "summarize":
+        print(summarize_run(args.run_dir).model_dump_json(indent=2))
+        return 0
+
+    if args.command == "baselines":
+        print(json.dumps(evaluate_baseline_run(args.run_dir), indent=2, sort_keys=True))
+        return 0
+
+    if args.command == "export":
+        print(json.dumps(export_run(args.run_dir, args.format, args.out), sort_keys=True))
+        return 0
+
+    if args.command == "evidence":
+        markdown = render_evidence_tables(parse_run_args(args.runs))
+        if args.out is not None:
+            args.out.write_text(markdown, encoding="utf-8")
+            print(json.dumps({"out": str(args.out)}, sort_keys=True))
+        else:
+            print(markdown, end="")
+        return 0
+
+    if args.command == "artifact-report":
+        output = (
+            artifact_report_json(args.run_dir, args.domain_path)
+            if args.format == "json"
+            else render_artifact_report(args.run_dir, args.domain_path)
+        )
+        if args.out is not None:
+            args.out.write_text(output, encoding="utf-8")
+            print(json.dumps({"out": str(args.out)}, sort_keys=True))
+        else:
+            print(output, end="")
+        return 0
+
+    if args.command == "check-integration" and args.kind == "dbt-semantic":
+        print(
+            json.dumps(
+                compare_dbt_semantic_model(args.domain, args.path, args.domain_path),
+                indent=2,
+                sort_keys=True,
+            )
+        )
+        return 0
+
+    if args.command == "scan":
+        result = run_scan(args.config, args.out)
+        print(
+            json.dumps(
+                {
+                    "gate": result.gate.outcome.value,
+                    "findings": result.summary.total_findings,
+                    "out": result.output_dir,
+                },
+                sort_keys=True,
+            )
+        )
+        return 1 if result.gate.outcome == GateOutcome.FAIL else 0
+
+    raise ValueError(f"unknown command: {args.command}")
+
+
+def format_validation_error(exc: ValidationError) -> str:
+    errors = exc.errors()
+    if not errors:
+        return str(exc)
+    first = errors[0]
+    loc = ".".join(str(part) for part in first.get("loc", ())) or "input"
+    return f"{loc}: {first.get('msg', 'validation failed')}"
+
+
+def is_user_type_error(exc: TypeError) -> bool:
+    message = str(exc)
+    return message.startswith(
+        (
+            "generated count must",
+            "matrix count must",
+            "suite name must",
+        )
+    )

policystrata/compiler.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+from policystrata.models import CompileResult, Policy, Principal, SemanticQuery
+from policystrata.policy import PolicyOracle
+
+DIMENSION_SQL = {
+    "region": "accounts.region",
+    "plan": "subscriptions.plan",
+    "month": "date_trunc('month', invoices.invoice_date)",
+    "severity": "support_tickets.severity",
+    "customer_email": "accounts.customer_email",
+    "tenant_id": "accounts.tenant_id",
+}
+
+TENANT_SCOPE_MUTATIONS = {
+    "compiler_drops_tenant_predicate",
+    "compiler_uses_old_tenant_key",
+    "compiler_swaps_tenant_account_id",
+}
+
+
+def compile_query(
+    policy: Policy,
+    principal: Principal,
+    query: SemanticQuery,
+    mutation: str = "none",
+    domain: str = "support_saas",
+) -> CompileResult:
+    oracle = PolicyOracle(policy)
+    metric_name = oracle.resolve_metric(query.metric)
+    metric = policy.metrics.get(metric_name)
+    metric_expression = metric.expression if metric is not None else "count(*)"
+
+    metric_expression = metric_expression_for_mutation(metric_expression, mutation, domain)
+
+    dimensions = [dimension_sql(policy, dim) for dim in query.dimensions]
+    select_parts = [f"{metric_expression} as value"]
+    select_parts.extend(
+        f"{dim_sql} as {dim}" for dim, dim_sql in zip(query.dimensions, dimensions, strict=True)
+    )
+
+    joins = join_path_for_mutation(domain, mutation)
+
+    includes_tenant_predicate = mutation not in TENANT_SCOPE_MUTATIONS
+    where = tenant_predicates(domain, principal, mutation)
+
+    date_column = time_column(domain, metric.table if metric is not None else "")
+    where.extend(time_predicates(query.time_range, date_column, mutation))
+
+    where_sql = " where " + " and ".join(where) if where else ""
+    group_sql = " group by " + ", ".join(dimensions) if dimensions else ""
+    limit_sql = f" limit {query.limit}"
+    sql = f"select {', '.join(select_parts)} {' '.join(joins)}{where_sql}{group_sql}{limit_sql}"
+
+    return CompileResult(
+        sql=sql,
+        estimated_cost=estimated_cost_for_query(oracle, query, mutation),
+        includes_tenant_predicate=includes_tenant_predicate,
+        metric_expression=metric_expression,
+        join_grain=join_grain_for_mutation(mutation),
+        time_semantics=time_semantics_for_query(query, mutation),
+    )
+
+
+def estimated_cost_for_query(oracle: PolicyOracle, query: SemanticQuery, mutation: str) -> int:
+    if mutation == "cost_estimate_ignores_expansion":
+        return 1
+    return oracle.estimate_cost(query)
+
+
+def join_grain_for_mutation(mutation: str) -> str:
+    if mutation == "fanout_join_drift":
+        return "ticket_event"
+    if mutation == "compiler_inner_join_drops_rows":
+        return "inner_join_required"
+    return "account"
+
+
+def time_semantics_for_query(query: SemanticQuery, mutation: str) -> str:
+    if query.time_range == "last_fiscal_month" and mutation != "fiscal_calendar_mismatch":
+        return "fiscal"
+    return "calendar"
+
+
+def metric_expression_for_mutation(metric_expression: str, mutation: str, domain: str) -> str:
+    if mutation == "gross_net_metric_drift":
+        return drift_metric_expression(domain)
+    if mutation == "fanout_join_drift":
+        return f"({metric_expression}) * 2"
+    if mutation == "compiler_removes_distinct":
+        return metric_expression.replace("count(distinct ", "count(").replace("distinct ", "")
+    return metric_expression
+
+
+def dimension_sql(policy: Policy, dimension: str) -> str:
+    configured = policy.dimensions.get(dimension)
+    if configured is not None:
+        return configured.column
+    return DIMENSION_SQL.get(dimension, dimension)
+
+
+def drift_metric_expression(domain: str) -> str:
+    if domain == "finance_saas":
+        return "sum(transactions.gross_amount_cents)"
+    return "sum(invoices.gross_amount_cents)"
+
+
+def join_path_for_mutation(domain: str, mutation: str) -> list[str]:
+    joins = join_path(domain)
+    if mutation == "compiler_inner_join_drops_rows":
+        return [join.replace("left join", "join") for join in joins]
+    return joins
+
+
+def join_path(domain: str) -> list[str]:
+    if domain == "finance_saas":
+        return [
+            "from households",
+            "left join advisors on advisors.id = households.advisor_id",
+            "left join accounts on accounts.household_id = households.id",
+            "left join transactions on transactions.account_id = accounts.id",
+            "left join balances on balances.account_id = accounts.id",
+        ]
+    return [
+        "from accounts",
+        "left join subscriptions on subscriptions.account_id = accounts.id",
+        "left join invoices on invoices.subscription_id = subscriptions.id",
+        "left join support_tickets on support_tickets.account_id = accounts.id",
+    ]
+
+
+def tenant_predicates(domain: str, principal: Principal, mutation: str) -> list[str]:
+    column = tenant_filter_column(domain, mutation)
+    if column is None:
+        return []
+    tenant_list = ", ".join(f"'{tenant}'" for tenant in principal.tenant_ids)
+    return [f"{column} in ({tenant_list})"]
+
+
+def tenant_filter_column(domain: str, mutation: str) -> str | None:
+    if mutation == "compiler_drops_tenant_predicate":
+        return None
+    if mutation == "compiler_uses_old_tenant_key":
+        return legacy_tenant_column(domain)
+    if mutation == "compiler_swaps_tenant_account_id":
+        return "accounts.id"
+    return tenant_column(domain)
+
+
+def time_predicates(time_range: str, date_column: str, mutation: str) -> list[str]:
+    if time_range == "last_month" or (
+        time_range == "last_fiscal_month" and mutation == "fiscal_calendar_mismatch"
+    ):
+        return [f"{date_column} >= date '2026-05-01'", f"{date_column} < date '2026-06-01'"]
+    if time_range == "last_fiscal_month":
+        return [f"{date_column} >= date '2026-04-27'", f"{date_column} < date '2026-05-25'"]
+    if time_range == "quarter_to_date":
+        return [f"{date_column} >= date '2026-04-01'", f"{date_column} < date '2026-06-24'"]
+    return []
+
+
+def tenant_column(domain: str) -> str:
+    if domain == "finance_saas":
+        return "households.firm_id"
+    return "accounts.tenant_id"
+
+
+def legacy_tenant_column(domain: str) -> str:
+    if domain == "finance_saas":
+        return "households.legacy_firm_id"
+    return "accounts.legacy_tenant_id"
+
+
+def time_column(domain: str, table: str) -> str:
+    if domain == "finance_saas":
+        if table == "balances":
+            return "balances.balance_date"
+        return "transactions.transaction_date"
+    return "invoices.invoice_date"