profilefoundry 1.0.0__py3-none-any.whl

profilefoundry/__init__.py

@@ -0,0 +1,15 @@
+"""ProfileFoundry — a generator + SDK for structured, internally-consistent,
+linked, temporally coherent synthetic Person Objects.
+
+The public surface area is intentionally tiny. See the project README for
+worked examples.
+"""
+
+__version__ = "1.0.0"
+
+from profilefoundry.schema import (  # noqa: E402,F401
+    CURRENT_SCHEMA_VERSION,
+    Person,
+)
+
+__all__ = ["__version__", "CURRENT_SCHEMA_VERSION", "Person"]

profilefoundry/cli.py

@@ -0,0 +1,361 @@
+"""Tiny CLI for hand-driving the generator.
+
+Examples::
+
+    profilefoundry person --locale US --seed 4321
+    profilefoundry scale --n 100 --locale US --seed 4321 --out /tmp/sample.jsonl
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from datetime import date
+from pathlib import Path
+from typing import Optional
+
+import click
+
+# Auto-load .env at the repo root so secrets (HIBP_API_KEY, etc.) flow into
+# any audit that needs them. Silent no-op if python-dotenv isn't installed.
+try:
+    from dotenv import load_dotenv as _load_dotenv
+    _load_dotenv(Path(__file__).resolve().parents[2] / ".env")
+except ImportError:
+    pass
+
+from profilefoundry import __version__
+from profilefoundry.generate.factory import make_person
+from profilefoundry.schema import Locale
+
+VALID_LOCALES = ("US", "UK", "CA", "AU", "NZ", "IE", "IN", "PH")
+
+
+@click.group()
+@click.version_option(__version__)
+def main() -> None:
+    """ProfileFoundry — synthetic Person Object generator."""
+
+
+@main.command("person")
+@click.option("--locale", type=click.Choice(VALID_LOCALES), default="US", show_default=True)
+@click.option("--seed", "global_seed", type=int, default=4321, show_default=True)
+@click.option("--profile-seq", type=int, default=1, show_default=True)
+def person_cmd(locale: Locale, global_seed: int, profile_seq: int) -> None:
+    """Generate one Person and print it as JSON to stdout."""
+    p = make_person(locale=locale, profile_seq=profile_seq, global_seed=global_seed)
+    click.echo(p.model_dump_json(indent=2))
+
+
+@main.command("scale")
+@click.option("--n", "count", type=int, required=True, help="Number of profiles to generate.")
+@click.option("--locale", type=click.Choice(VALID_LOCALES), default="US", show_default=True)
+@click.option("--seed", "global_seed", type=int, default=4321, show_default=True)
+@click.option(
+    "--out",
+    type=click.Path(dir_okay=False, writable=True, path_type=Path),
+    default=None,
+    help="Write JSONL output to this file (one Person per line). Default: stdout.",
+)
+def scale_cmd(count: int, locale: Locale, global_seed: int, out: Optional[Path]) -> None:
+    """Generate ``--n`` profiles into a JSONL file or stdout."""
+    sink = out.open("w", encoding="utf-8") if out else sys.stdout
+    try:
+        for seq in range(1, count + 1):
+            p = make_person(locale=locale, profile_seq=seq, global_seed=global_seed)
+            sink.write(p.model_dump_json() + "\n")
+    finally:
+        if out:
+            sink.close()
+    if out:
+        click.echo(f"wrote {count} profiles to {out}", err=True)
+
+
+@main.command("household")
+@click.option("--locale", type=click.Choice(VALID_LOCALES), default="US", show_default=True)
+@click.option("--seed", "global_seed", type=int, default=4321, show_default=True)
+@click.option("--seq", type=int, default=1, show_default=True, help="Household sequence number.")
+def household_cmd(locale: Locale, global_seed: int, seq: int) -> None:
+    """Generate one household (multiple linked Persons) and print as JSON."""
+    from profilefoundry.linkage.orchestrator import build_household
+
+    bundle = build_household(locale=locale, household_seq=seq, global_seed=global_seed)
+    payload = {
+        "household_id": bundle.spec.household_id,
+        "composition_type": bundle.spec.composition_type,
+        "members": [json.loads(p.model_dump_json()) for p in bundle.persons],
+    }
+    click.echo(json.dumps(payload, indent=2))
+
+
+@main.command("scale-households")
+@click.option("--n", "count", type=int, required=True, help="Number of households to generate.")
+@click.option("--locale", type=click.Choice(VALID_LOCALES), default="US", show_default=True)
+@click.option("--seed", "global_seed", type=int, default=4321, show_default=True)
+@click.option("--out", type=click.Path(dir_okay=False, writable=True, path_type=Path), default=None,
+              help="Write JSONL output (one Person per line). Default: stdout.")
+def scale_households_cmd(count: int, locale: Locale, global_seed: int, out: Optional[Path]) -> None:
+    """Generate ``--n`` households (linked profiles) as JSONL."""
+    from profilefoundry.linkage.orchestrator import iter_households
+
+    sink = out.open("w", encoding="utf-8") if out else sys.stdout
+    total_persons = 0
+    try:
+        for bundle in iter_households(locale, count, global_seed):
+            for p in bundle.persons:
+                sink.write(p.model_dump_json() + "\n")
+                total_persons += 1
+    finally:
+        if out:
+            sink.close()
+    if out:
+        click.echo(f"wrote {total_persons} persons in {count} households to {out}", err=True)
+
+
+@main.command("validate")
+@click.option("--n", "count", type=int, default=500, show_default=True)
+@click.option("--seed", "global_seed", type=int, default=4321, show_default=True)
+@click.option("--locales", default="US,UK,IN,CA,AU", show_default=True,
+              help="Comma-separated locales to validate.")
+@click.option(
+    "--hibp/--skip-hibp",
+    "use_hibp",
+    default=False,
+    show_default=True,
+    help="Run live HIBP breached-account checks. Default validate mode is syntax-only.",
+)
+@click.option(
+    "--hibp-max-emails",
+    type=click.IntRange(min=1),
+    default=100,
+    show_default=True,
+    help="Maximum unique emails to send to HIBP when --hibp is enabled.",
+)
+def validate_cmd(count: int, global_seed: int, locales: str, use_hibp: bool, hibp_max_emails: int) -> None:
+    """Run Phase 5 validation suite (KS gaps, leakage, consistency)."""
+    from profilefoundry.validate.consistency import aggregate_consistency
+    from profilefoundry.validate.distributional import validate_locale
+    from profilefoundry.validate.leakage import (
+        email_kanonymity_check,
+        self_collision_audit,
+        wikidata_leakage_scan,
+    )
+
+    today = date.today()
+    locale_list = [s.strip() for s in locales.split(",") if s.strip()]
+
+    click.echo(f"Validating n={count}/locale across {locale_list}\n")
+    for loc in locale_list:
+        click.echo(f"--- {loc} ---")
+        persons = [make_person(locale=loc, profile_seq=i, global_seed=global_seed) for i in range(1, count + 1)]
+        ks = validate_locale(persons, loc, today=today)  # type: ignore[arg-type]
+        click.echo(f"  age_ks    male={ks.age_ks_male:.3f}  female={ks.age_ks_female:.3f}"
+                   if ks.age_ks_male is not None else "  age_ks    n/a")
+        click.echo(f"  edu_ks    {ks.education_ks:.3f}" if ks.education_ks is not None else "  edu_ks    n/a")
+        click.echo(f"  marital_ks {ks.marital_ks:.3f}" if ks.marital_ks is not None else "  marital_ks n/a")
+
+        cr = aggregate_consistency(persons, today=today)
+        click.echo(f"  consistency  {cr.n_pass}/{cr.n_total} ({100*cr.rate:.2f}%)")
+        if cr.failures:
+            top = sorted(cr.failures.items(), key=lambda kv: -kv[1])[:3]
+            click.echo(f"    top failures: {top}")
+
+        scol = self_collision_audit(persons)
+        click.echo(f"  self-collisions  {scol.n_collisions}")
+        wd = wikidata_leakage_scan(persons)
+        click.echo(f"  wikidata leaks   {wd.n_leaks}")
+        em = email_kanonymity_check(
+            persons,
+            use_hibp=use_hibp,
+            max_emails=hibp_max_emails if use_hibp else None,
+        )
+        if use_hibp:
+            click.echo(
+                "  email leaks    "
+                f"{em.n_leaks} (mode={em.metadata.get('mode')}, "
+                f"checked={em.metadata.get('n_checked_emails', 0)})"
+            )
+        else:
+            click.echo(f"  email-syntax bad {em.n_leaks}")
+        click.echo("")
+
+
+@main.command("export")
+@click.option("--out", type=click.Path(file_okay=False, writable=True, path_type=Path), required=True,
+              help="Output directory.")
+@click.option("--n-per-locale", "n_per_locale", type=int, default=1000, show_default=True)
+@click.option("--seed", "global_seed", type=int, default=4321, show_default=True)
+@click.option("--locales", default="US,UK,IN,CA,AU", show_default=True)
+@click.option("--generation-date", type=click.DateTime(formats=["%Y-%m-%d"]), default=None,
+              help="Pin generated profile dates as YYYY-MM-DD.")
+@click.option("--exported-at", default=None,
+              help="Pin release metadata timestamp as YYYY-MM-DD or ISO-8601 for byte-stable artifacts.")
+@click.option("--skip-hibp", is_flag=True,
+              help="Skip live HIBP email checks even when HIBP_API_KEY is configured.")
+@click.option("--household-mode/--profile-mode", default=True, show_default=True,
+              help="--household-mode generates linked households (Phase 3+).")
+def export_cmd(
+    out: Path,
+    n_per_locale: int,
+    global_seed: int,
+    locales: str,
+    generation_date,
+    exported_at: str | None,
+    skip_hibp: bool,
+    household_mode: bool,
+) -> None:
+    """Export a multi-locale dataset (JSONL + Parquet + dataset card)."""
+    import time
+
+    from profilefoundry.io.hf_export import (
+        ExportStats,
+        normalize_exported_at,
+        write_addresses_parquet,
+        write_dataset_card,
+        write_education_parquet,
+        write_employers_parquet,
+        write_employment_parquet,
+        write_events_parquet,
+        write_flat_parquet,
+        write_health_allergies_parquet,
+        write_households_parquet,
+        write_jsonl,
+        write_manifest,
+        write_person_objects_parquet,
+        write_relationships_parquet,
+        write_social_handles_parquet,
+    )
+    from profilefoundry.linkage.orchestrator import iter_households_for_profile_target
+    from profilefoundry.validate.consistency import aggregate_consistency
+    from profilefoundry.validate.distributional import validate_locale
+    from profilefoundry.validate.leakage import (
+        email_kanonymity_check,
+        self_collision_audit,
+        wikidata_leakage_scan,
+    )
+
+    out.mkdir(parents=True, exist_ok=True)
+    loc_list = [s.strip() for s in locales.split(",") if s.strip()]
+    today = generation_date.date() if generation_date else date.today()
+    stats = ExportStats(locales={})
+    t0 = time.time()
+
+    all_persons = []
+    for loc in loc_list:
+        click.echo(f"Generating {loc} (~{n_per_locale} profiles)...", err=True)
+        loc_persons = []
+        if household_mode:
+            for b in iter_households_for_profile_target(loc, n_per_locale, global_seed, today=today):  # type: ignore[arg-type]
+                loc_persons.extend(b.persons)
+        else:
+            for seq in range(1, n_per_locale + 1):
+                loc_persons.append(make_person(locale=loc, profile_seq=seq, global_seed=global_seed, today=today))  # type: ignore[arg-type]
+        all_persons.extend(loc_persons)
+        stats.locales[loc] = len(loc_persons)
+
+    stats.n_profiles = len(all_persons)
+    stats.n_events = sum(len(p.events) for p in all_persons)
+    stats.n_addresses = sum(len(p.addresses) for p in all_persons)
+    stats.n_employment_records = sum(len(p.employment) for p in all_persons)
+    stats.n_education_records = sum(len(p.education) for p in all_persons)
+    stats.n_social_handles = sum(len(p.contact.social_handles) for p in all_persons)
+    stats.n_health_allergies = sum(len(p.health.allergies) for p in all_persons)
+    stats.n_households = len({p.household_id for p in all_persons})
+    stats.n_employers = len({e.employer_id for p in all_persons for e in p.employment})
+    stats.duration_seconds = time.time() - t0
+    stats.generation_date = today.isoformat()
+    stats.exported_at = normalize_exported_at(exported_at)
+
+    click.echo(f"Writing {stats.n_profiles} profiles → {out}", err=True)
+    row_counts: dict[str, int] = {}
+    row_counts["profiles.jsonl"] = write_jsonl(all_persons, out / "profiles.jsonl")
+    row_counts["person_objects.parquet"] = write_person_objects_parquet(all_persons, out / "person_objects.parquet")
+    row_counts["profiles.parquet"] = write_flat_parquet(all_persons, out / "profiles.parquet")
+    row_counts["addresses.parquet"] = write_addresses_parquet(all_persons, out / "addresses.parquet")
+    row_counts["employment.parquet"] = write_employment_parquet(all_persons, out / "employment.parquet")
+    row_counts["education.parquet"] = write_education_parquet(all_persons, out / "education.parquet")
+    row_counts["social_handles.parquet"] = write_social_handles_parquet(all_persons, out / "social_handles.parquet")
+    row_counts["health_allergies.parquet"] = write_health_allergies_parquet(all_persons, out / "health_allergies.parquet")
+    row_counts["events.parquet"] = write_events_parquet(all_persons, out / "events.parquet")
+    row_counts["households.parquet"] = write_households_parquet(all_persons, out / "households.parquet")
+    row_counts["employers.parquet"] = write_employers_parquet(all_persons, out / "employers.parquet")
+    row_counts["relationships.parquet"] = write_relationships_parquet(all_persons, out / "relationships.parquet")
+    stats.n_addresses = row_counts["addresses.parquet"]
+    stats.n_social_handles = row_counts["social_handles.parquet"]
+    stats.n_health_allergies = row_counts["health_allergies.parquet"]
+    stats.n_relationships = row_counts["relationships.parquet"]
+
+    # Per-locale validation snapshot for the dataset card.
+    validation: dict = {}
+    for loc in loc_list:
+        loc_ps = [p for p in all_persons if p.locale == loc]
+        if not loc_ps:
+            continue
+        ks = validate_locale(loc_ps, loc, today=today)  # type: ignore[arg-type]
+        cr = aggregate_consistency(loc_ps, today=today)
+        validation[loc] = {
+            "age_ks_male": ks.age_ks_male,
+            "age_ks_female": ks.age_ks_female,
+            "education_ks": ks.education_ks,
+            "marital_ks": ks.marital_ks,
+            "consistency_rate": cr.rate,
+        }
+
+    # Leakage summary for the dataset card.
+    leakage: dict = {}
+    sc = self_collision_audit(all_persons)
+    leakage["self_collision"] = {
+        "n_profiles": sc.n_profiles,
+        "n_collisions": sc.n_collisions,
+        "mode": "exact",
+    }
+    wk = wikidata_leakage_scan(all_persons)
+    leakage["wikidata"] = {
+        "n_profiles": wk.n_profiles,
+        "n_leaks": wk.n_leaks,
+        "mode": wk.metadata.get("mode", "demo"),
+    }
+    em = email_kanonymity_check(all_persons, use_hibp=False if skip_hibp else None)
+    leakage["email"] = {
+        "n_profiles": em.n_profiles,
+        "n_leaks": em.n_leaks,
+        "mode": em.metadata.get("mode", "syntax_only"),
+    }
+
+    write_dataset_card(out / "dataset_card.md", stats, validation, leakage)
+    write_manifest(out, stats, row_counts)
+    click.echo(f"Done ({stats.duration_seconds:.1f}s). Files written to {out}/", err=True)
+
+
+@main.command("scale-smoke")
+@click.option("--locale", type=click.Choice(VALID_LOCALES), default="US", show_default=True)
+@click.option("--sizes", default="1000,10000", show_default=True,
+              help="Comma-separated profile counts to time.")
+@click.option("--seed", "global_seed", type=int, default=4321, show_default=True)
+def scale_smoke_cmd(locale: Locale, sizes: str, global_seed: int) -> None:
+    """Time the generator at several scale points."""
+    import time
+    size_list = [int(s.strip()) for s in sizes.split(",")]
+    for n in size_list:
+        t0 = time.time()
+        for seq in range(1, n + 1):
+            make_person(locale=locale, profile_seq=seq, global_seed=global_seed)
+        dt = time.time() - t0
+        rate = n / dt if dt > 0 else 0
+        click.echo(f"  n={n:>8d}  {dt:6.2f}s  ({rate:.0f} profiles/s)")
+
+
+@main.command("verify")
+def verify_cmd() -> None:
+    """Run a quick self-check: generate one profile per locale."""
+    for loc in VALID_LOCALES:
+        try:
+            p = make_person(locale=loc, profile_seq=1, global_seed=4321)
+            click.echo(f"OK  {loc:>3}  {p.profile_id}  {p.identity.given_name} {p.identity.family_name}")
+        except Exception as exc:  # noqa: BLE001
+            click.echo(f"FAIL {loc:>3}  {exc}", err=True)
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

profilefoundry/data/loader.py

@@ -0,0 +1,80 @@
+"""Reference-data loader.
+
+Resolves a (locale, table-name) pair into a parsed Python object. Prefers
+``data/reference/derived/<locale>/<table>.parquet`` (live-ingested) over
+``data/reference/bootstrap/<locale>/<table>.json`` (committed minimum).
+
+The loader is intentionally cache-aware: a single Python process never reads
+the same file twice.
+"""
+
+from __future__ import annotations
+
+import functools
+import json
+from pathlib import Path
+from typing import Any
+
+from profilefoundry.data.paths import reference_root
+
+REF_ROOT = reference_root()
+BOOTSTRAP_ROOT = REF_ROOT / "bootstrap"
+DERIVED_ROOT = REF_ROOT / "derived"
+
+LOCALE_CODES = ("US", "UK", "CA", "AU", "NZ", "IE", "IN", "PH")
+
+
+class ReferenceDataMissing(FileNotFoundError):
+    """Raised when neither bootstrap nor derived data is available."""
+
+
+def locale_dir(locale: str, *, tier: str) -> Path:
+    if locale not in LOCALE_CODES:
+        raise ValueError(f"Unknown locale: {locale!r}; expected one of {LOCALE_CODES}")
+    root = BOOTSTRAP_ROOT if tier == "bootstrap" else DERIVED_ROOT
+    return root / locale.lower()
+
+
+@functools.lru_cache(maxsize=128)
+def load_table(locale: str, table: str) -> dict[str, Any]:
+    """Load a reference table for a locale.
+
+    Resolution order:
+        1. ``derived/<locale>/<table>.parquet`` (not implemented yet; will
+           wrap pandas.read_parquet when richer ingestion lands).
+        2. ``bootstrap/<locale>/<table>.json``.
+
+    Returns the parsed ``data`` payload of the bootstrap envelope. Source
+    metadata can be retrieved via :func:`load_source_metadata`.
+    """
+
+    boot_path = locale_dir(locale, tier="bootstrap") / f"{table}.json"
+    if boot_path.exists():
+        envelope = json.loads(boot_path.read_text(encoding="utf-8"))
+        return envelope.get("data", {})
+
+    raise ReferenceDataMissing(
+        f"No reference data for locale={locale!r}, table={table!r}. "
+        f"Looked in {boot_path}. Run scripts/ingest_{locale.lower()}.py "
+        f"to populate live data, or add a bootstrap file."
+    )
+
+
+@functools.lru_cache(maxsize=128)
+def load_source_metadata(locale: str, table: str) -> dict[str, Any]:
+    boot_path = locale_dir(locale, tier="bootstrap") / f"{table}.json"
+    if not boot_path.exists():
+        raise ReferenceDataMissing(f"No bootstrap file at {boot_path}")
+    envelope = json.loads(boot_path.read_text(encoding="utf-8"))
+    return envelope.get("source", {})
+
+
+def available_tables(locale: str) -> list[str]:
+    boot_dir = locale_dir(locale, tier="bootstrap")
+    if not boot_dir.is_dir():
+        return []
+    return sorted(p.stem for p in boot_dir.glob("*.json"))
+
+
+def all_locales_with_data() -> dict[str, list[str]]:
+    return {loc: available_tables(loc) for loc in LOCALE_CODES}

profilefoundry/data/manifest_hash.py

@@ -0,0 +1,54 @@
+"""Reference data manifest hash (PF-053).
+
+Computes a deterministic hash of all bootstrap reference files used by the
+generator. Stamped into ``Person.generation.reference_manifest_hash`` so a
+re-run with the same global_seed but a different reference-data version is
+detectable.
+"""
+
+from __future__ import annotations
+
+import functools
+import hashlib
+from pathlib import Path
+
+from profilefoundry.data.paths import reference_root
+
+REF_ROOT = reference_root()
+
+
+# Subtrees that influence generation output. Audit-only artifacts (leakage
+# bloom, ingestion cache) live elsewhere under data/reference/ and are
+# deliberately excluded so the hash stays stable across leakage ingests.
+_HASHED_SUBTREES = ("bootstrap", "names", "geo")
+
+
+@functools.lru_cache(maxsize=1)
+def reference_manifest_hash() -> str:
+    """blake2b digest of every committed reference-data file the generator reads.
+
+    Returns a 16-char hex digest derived from the sorted list of
+    (relative_path, file_bytes) tuples under ``data/reference/{bootstrap,
+    names, geo}/``. Audit-only data (``leakage/``, ``cache/``, ``derived/``)
+    is excluded so re-running the leakage ingest does not change the hash.
+    Cached for the lifetime of the process.
+    """
+    h = hashlib.blake2b(digest_size=8)
+    if not REF_ROOT.exists():
+        return h.hexdigest()
+    paths: list[Path] = []
+    for sub in _HASHED_SUBTREES:
+        root = REF_ROOT / sub
+        if not root.exists():
+            continue
+        paths.extend(p for p in root.rglob("*") if p.is_file())
+    for p in sorted(paths):
+        rel = p.relative_to(REF_ROOT).as_posix()
+        h.update(rel.encode("utf-8"))
+        h.update(b"\x00")
+        try:
+            h.update(p.read_bytes())
+        except OSError:
+            h.update(b"<unreadable>")
+        h.update(b"\x01")
+    return h.hexdigest()

profilefoundry/data/paths.py

@@ -0,0 +1,33 @@
+"""Runtime paths for ProfileFoundry reference data."""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+REFERENCE_ROOT_ENV = "PROFILEFOUNDRY_REFERENCE_ROOT"
+
+
+def reference_root() -> Path:
+    """Return the generation reference-data root for source or installed runs.
+
+    Source checkouts keep reference data at ``<repo>/data/reference`` while
+    wheels install the same committed files as ``<install-root>/data/reference``.
+    ``PROFILEFOUNDRY_REFERENCE_ROOT`` can point audits at an explicit copy.
+    """
+
+    env_root = os.environ.get(REFERENCE_ROOT_ENV)
+    if env_root:
+        return Path(env_root).expanduser().resolve()
+
+    module_path = Path(__file__).resolve()
+    candidates = (
+        Path(sys.prefix) / "data" / "reference",  # installed wheel
+        module_path.parents[2] / "data" / "reference",  # pip --target install
+        module_path.parents[3] / "data" / "reference",  # source checkout
+    )
+    for candidate in candidates:
+        if candidate.exists():
+            return candidate
+    return candidates[-1]