creforge 0.1.0__py3-none-any.whl

creforge/__init__.py

@@ -0,0 +1,21 @@
+"""creforge: PII-safe synthetic credit bureau data from explicit behavioural rules."""
+
+__version__ = "0.1.0"
+
+from .config import Config, Profile, list_profiles, load_profile  # noqa: E402
+from .dataset import Dataset, DiskDataset, generate, write_dataset  # noqa: E402
+from .validate import Report, validate  # noqa: E402
+
+__all__ = [
+    "Config",
+    "Dataset",
+    "DiskDataset",
+    "Profile",
+    "Report",
+    "__version__",
+    "generate",
+    "list_profiles",
+    "load_profile",
+    "validate",
+    "write_dataset",
+]

creforge/cli.py

@@ -0,0 +1,86 @@
+"""Command-line interface: ``creforge generate | validate | profiles``."""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+from pathlib import Path
+
+import click
+import yaml
+
+from . import __version__
+from .config import Config, list_profiles, load_profile
+from .dataset import write_dataset
+from .validate import validate
+
+
+@click.group()
+@click.version_option(__version__, prog_name="creforge")
+def main() -> None:
+    """PII-safe synthetic credit bureau data from explicit behavioural rules."""
+
+
+@main.command()
+@click.option("--profile", "-p", default="baseline", show_default=True,
+              help="Built-in profile name or path to a YAML profile.")
+@click.option("--subjects", "-n", type=click.IntRange(min=1), default=10_000, show_default=True)
+@click.option("--months", "-m", type=click.IntRange(12, 120), default=36, show_default=True)
+@click.option("--seed", "-s", type=click.IntRange(min=0), default=0, show_default=True)
+@click.option("--start-month", default="2023-01", show_default=True, help="YYYY-MM")
+@click.option("--out", "-o", type=click.Path(file_okay=False, path_type=Path), required=True)
+@click.option("--format", "fmt", type=click.Choice(["parquet", "csv"]), default="parquet",
+              show_default=True)
+@click.option("--workers", "-w", type=click.IntRange(min=1), default=1, show_default=True)
+@click.option("--chunk-size", type=click.IntRange(min=1), default=50_000, show_default=True)
+def generate(profile, subjects, months, seed, start_month, out, fmt, workers, chunk_size) -> None:
+    """Generate a dataset into OUT (one part file per chunk, plus manifest.json)."""
+    cfg = Config.from_profile(profile, subjects=subjects, months=months, seed=seed,
+                              start_month=start_month, chunk_size=chunk_size)
+    t0 = time.perf_counter()
+    manifest = write_dataset(cfg, out, format=fmt, workers=workers)
+    secs = time.perf_counter() - t0
+    click.echo(f"Wrote {out} in {secs:.1f}s (profile={cfg.profile.name}, seed={seed})")
+    for table, n in manifest["row_counts"].items():
+        click.echo(f"  {table:<14} {n:>14,}")
+
+
+@main.command(name="validate")
+@click.argument("path", type=click.Path(exists=True, file_okay=False, path_type=Path))
+@click.option("--json", "as_json", is_flag=True, help="Emit the report as JSON.")
+@click.option("--report", type=click.Path(dir_okay=False, path_type=Path),
+              help="Also write the Markdown report to this file.")
+@click.option("--strict", is_flag=True, help="Exit non-zero on calibration failures too.")
+def validate_cmd(path, as_json, report, strict) -> None:
+    """Check integrity and calibration of a dataset written by `generate`."""
+    rep = validate(path)
+    md = rep.to_markdown()
+    if report:
+        report.write_text(md, encoding="utf-8")
+    click.echo(json.dumps(rep.to_dict(), indent=2, default=str) if as_json else md)
+    if not rep.integrity_ok or (strict and not rep.calibration_ok):
+        sys.exit(1)
+
+
+@main.group()
+def profiles() -> None:
+    """Inspect built-in profiles."""
+
+
+@profiles.command(name="list")
+def profiles_list() -> None:
+    for name in list_profiles():
+        desc = " ".join(load_profile(name).description.split())
+        click.echo(f"{name:<12} {desc}")
+
+
+@profiles.command(name="show")
+@click.argument("name")
+def profiles_show(name) -> None:
+    """Print the fully resolved profile (inheritance applied) as YAML."""
+    click.echo(yaml.safe_dump(load_profile(name).model_dump(mode="json"), sort_keys=False))
+
+
+if __name__ == "__main__":
+    main()

creforge/config.py

@@ -0,0 +1,287 @@
+"""Profile and run configuration (validated with pydantic)."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from importlib import resources
+from pathlib import Path
+from typing import Annotated, Any, Literal
+
+import numpy as np
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+Prob = Annotated[float, Field(ge=0.0, le=1.0)]
+NonNeg = Annotated[float, Field(ge=0.0)]
+Positive = Annotated[float, Field(gt=0.0)]
+
+_SUM_TOL = 1e-6
+_NAME = re.compile(r"^[a-z][a-z0-9_]*$")
+
+
+class _Model(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+
+def _check_shares(items: dict[str, Any], what: str) -> None:
+    total = sum(v.share for v in items.values())
+    if abs(total - 1.0) > _SUM_TOL:
+        raise ValueError(f"{what} shares must sum to 1.0, got {total:.6f}")
+
+
+class LogNormal(_Model):
+    median: Positive
+    sigma: NonNeg
+
+
+class Behaviour(_Model):
+    """Monthly transition parameters for one product (before grade/seasoning/macro).
+
+    ``roll[i]``: bucket i -> i+1 for C, D1..D4. ``cure[i]``: D1..D5 -> current.
+    ``back[i]``: D2..D5 -> one bucket lower. ``restructure``: D2+ -> RS.
+    """
+
+    roll: list[Prob] = Field(min_length=5, max_length=5)
+    cure: list[Prob] = Field(min_length=5, max_length=5)
+    back: list[Prob] = Field(min_length=4, max_length=4)
+    restructure: Prob = 0.0
+    rs_cure: Prob = 0.0
+    rs_redefault: Prob = 0.0
+    close: Prob = 0.0
+
+    @model_validator(mode="after")
+    def _rows_are_substochastic(self) -> Behaviour:
+        rows = {
+            "C": self.roll[0] + self.close,
+            "D1": self.roll[1] + self.cure[0],
+            "D2": self.roll[2] + self.cure[1] + self.back[0] + self.restructure,
+            "D3": self.roll[3] + self.cure[2] + self.back[1] + self.restructure,
+            "D4": self.roll[4] + self.cure[3] + self.back[2] + self.restructure,
+            "D5": self.cure[4] + self.back[3] + self.restructure,
+            "RS": self.rs_cure + self.rs_redefault,
+        }
+        bad = {k: round(v, 6) for k, v in rows.items() if v > 1.0 + _SUM_TOL}
+        if bad:
+            raise ValueError(f"outflow probabilities exceed 1.0 for states {bad}")
+        return self
+
+
+class Product(_Model):
+    kind: Literal["installment", "revolving"]
+    holding_rate: NonNeg = Field(description="Expected open accounts per subject at window start")
+    inquiry_share: NonNeg = Field(description="Share of new inquiries for this product")
+    amount: LogNormal = Field(description="Principal (installment) or limit (revolving)")
+    round_to: Positive = 100.0
+    tenor_months: list[Annotated[int, Field(ge=1)]] | None = None
+    max_age_months: Annotated[int, Field(ge=0)] = 240
+    interest_rate: Annotated[float, Field(ge=0.0, le=1.0)]
+    rate_grade_sensitivity: NonNeg = 1.0
+    secured: bool = False
+    min_payment_pct: Prob | None = None
+    behaviour: Behaviour
+
+    @model_validator(mode="after")
+    def _kind_consistency(self) -> Product:
+        if self.kind == "installment":
+            if not self.tenor_months:
+                raise ValueError("installment products need tenor_months")
+        else:
+            if self.min_payment_pct is None or self.min_payment_pct <= 0:
+                raise ValueError("revolving products need min_payment_pct > 0")
+            if self.behaviour.restructure > 0:
+                raise ValueError("restructure is only supported for installment products")
+        return self
+
+
+class Grade(_Model):
+    share: Prob
+    roll_mult: Positive
+    cure_mult: Positive
+    inquiry_rate: NonNeg = Field(description="Expected inquiries per subject per month")
+    approval: Annotated[float, Field(gt=0.0, lt=1.0)]
+    utilization: Prob
+    transactor_share: Prob
+    rate_spread: NonNeg
+
+
+class IncomeBand(_Model):
+    share: Prob
+    amount_mult: Positive
+    holding_mult: NonNeg
+
+
+class Population(_Model):
+    age_years: tuple[int, int] = (21, 70)
+    regions: Annotated[int, Field(ge=1, le=99)] = 13
+    income_bands: dict[str, IncomeBand]
+
+    @model_validator(mode="after")
+    def _valid(self) -> Population:
+        lo, hi = self.age_years
+        if not 18 <= lo <= hi <= 100:
+            raise ValueError("age_years must satisfy 18 <= min <= max <= 100")
+        _check_shares(self.income_bands, "income band")
+        return self
+
+
+class Seasoning(_Model):
+    """Roll-rate multiplier by months on book: floor at 0, peak at peak_age, tail after."""
+
+    floor: Positive = 0.5
+    peak: Positive = 1.35
+    peak_age: Annotated[int, Field(ge=1)] = 15
+    tail: Positive = 0.85
+
+    def curve(self, age: np.ndarray) -> np.ndarray:
+        a = np.maximum(np.asarray(age, dtype=np.float64), 0.0) / self.peak_age
+        hump = a * np.exp(1.0 - a)
+        base = np.where(a <= 1.0, self.floor, self.tail)
+        return base + (self.peak - base) * hump
+
+
+class Macro(_Model):
+    """Piecewise-linear roll multiplier by window month; flat beyond the given points."""
+
+    points: list[tuple[Annotated[int, Field(ge=0)], Positive]] = [(0, 1.0)]
+
+    @field_validator("points")
+    @classmethod
+    def _sorted(cls, v: list[tuple[int, float]]) -> list[tuple[int, float]]:
+        if not v:
+            raise ValueError("macro.points must not be empty")
+        months = [m for m, _ in v]
+        if months != sorted(set(months)):
+            raise ValueError("macro.points months must be strictly increasing")
+        return v
+
+    def path(self, months: int) -> np.ndarray:
+        xs = np.array([m for m, _ in self.points], dtype=np.float64)
+        ys = np.array([y for _, y in self.points], dtype=np.float64)
+        return np.interp(np.arange(months, dtype=np.float64), xs, ys)
+
+    @property
+    def is_flat(self) -> bool:
+        return len({y for _, y in self.points}) == 1
+
+
+class Inquiries(_Model):
+    recent_window_months: Annotated[int, Field(ge=1)] = 6
+    recent_penalty: NonNeg = Field(0.35, description="Logit penalty per recent inquiry")
+    withdrawn_share: Prob = 0.05
+
+
+class Target(_Model):
+    dpd30_share: tuple[Prob, Prob]
+    annual_writeoff_rate: tuple[Prob, Prob]
+
+
+class Profile(_Model):
+    name: str
+    description: str = ""
+    sources: list[str] = []
+    population: Population
+    grades: dict[str, Grade]
+    products: dict[str, Product]
+    seasoning: Seasoning = Seasoning()
+    macro: Macro = Macro()
+    inquiries: Inquiries = Inquiries()
+    lenders: Annotated[int, Field(ge=1, le=9999)] = 40
+    writeoff_after_months: Annotated[int, Field(ge=1, le=36)] = Field(
+        6, description="Months spent in the 120+ bucket before write-off"
+    )
+    targets: dict[str, Target] = {}
+
+    @model_validator(mode="after")
+    def _valid(self) -> Profile:
+        _check_shares(self.grades, "grade")
+        if len(self.grades) < 2:
+            raise ValueError("at least two grades are required")
+        if not self.products:
+            raise ValueError("at least one product is required")
+        for name in self.products:
+            if not _NAME.match(name):
+                raise ValueError(f"product name {name!r} must be snake_case")
+        inq = sum(p.inquiry_share for p in self.products.values())
+        if abs(inq - 1.0) > _SUM_TOL:
+            raise ValueError(f"product inquiry_share must sum to 1.0, got {inq:.6f}")
+        unknown = set(self.targets) - set(self.products)
+        if unknown:
+            raise ValueError(f"targets for unknown products: {sorted(unknown)}")
+        return self
+
+
+class Config(_Model):
+    profile: Profile
+    subjects: Annotated[int, Field(ge=1)]
+    months: Annotated[int, Field(ge=12, le=120)] = 36
+    seed: Annotated[int, Field(ge=0)] = 0
+    start_month: str = "2023-01"
+    chunk_size: Annotated[int, Field(ge=1)] = 50_000
+
+    @field_validator("start_month")
+    @classmethod
+    def _month(cls, v: str) -> str:
+        if not re.fullmatch(r"\d{4}-(0[1-9]|1[0-2])", v):
+            raise ValueError("start_month must look like YYYY-MM")
+        return v
+
+    @classmethod
+    def from_profile(cls, profile: str | Path = "baseline", **kwargs: Any) -> Config:
+        return cls(profile=load_profile(profile), **kwargs)
+
+    @property
+    def n_chunks(self) -> int:
+        return -(-self.subjects // self.chunk_size)
+
+    def chunk_bounds(self, chunk: int) -> tuple[int, int]:
+        start = chunk * self.chunk_size
+        return start, min(start + self.chunk_size, self.subjects)
+
+    def sha256(self) -> str:
+        payload = self.model_dump(mode="json", exclude={"chunk_size"})
+        blob = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+        return hashlib.sha256(blob.encode()).hexdigest()
+
+
+# --- profile loading -----------------------------------------------------------------
+
+
+def _deep_merge(base: dict[str, Any], over: dict[str, Any]) -> dict[str, Any]:
+    out = dict(base)
+    for k, v in over.items():
+        out[k] = _deep_merge(out[k], v) if isinstance(v, dict) and isinstance(out.get(k), dict) else v
+    return out
+
+
+def _read_profile_dict(ref: str | Path, _seen: tuple[str, ...] = ()) -> dict[str, Any]:
+    path = Path(ref)
+    if path.suffix in {".yaml", ".yml"} and path.exists():
+        text, key, here = path.read_text(encoding="utf-8"), str(path.resolve()), path.parent
+    else:
+        name = str(ref)
+        if name not in list_profiles():
+            raise FileNotFoundError(f"unknown profile {name!r}; built-ins: {list_profiles()}")
+        text = resources.files("creforge.profiles").joinpath(f"{name}.yaml").read_text("utf-8")
+        key, here = f"builtin:{name}", None
+    if key in _seen:
+        raise ValueError(f"circular profile inheritance: {' -> '.join(_seen + (key,))}")
+    data = yaml.safe_load(text) or {}
+    parent = data.pop("extends", None)
+    if parent is None:
+        return data
+    parent_ref: str | Path = parent
+    if here is not None and (here / parent).exists():
+        parent_ref = here / parent
+    return _deep_merge(_read_profile_dict(parent_ref, _seen + (key,)), data)
+
+
+def load_profile(ref: str | Path) -> Profile:
+    """Load a built-in profile by name, or a YAML file by path (``extends:`` supported)."""
+    return Profile.model_validate(_read_profile_dict(ref))
+
+
+def list_profiles() -> list[str]:
+    root = resources.files("creforge.profiles")
+    return sorted(p.name[:-5] for p in root.iterdir() if p.name.endswith(".yaml"))

creforge/dataset.py

@@ -0,0 +1,170 @@
+"""Public generation API: in-memory ``Dataset`` and streaming ``write_dataset``."""
+
+from __future__ import annotations
+
+import json
+import multiprocessing
+from collections.abc import Iterator
+from concurrent.futures import ProcessPoolExecutor
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+import polars as pl
+
+from . import __version__
+from .config import Config
+from .generator import TABLES, generate_chunk, make_context
+
+Format = Literal["parquet", "csv"]
+MANIFEST = "manifest.json"
+PRIVACY_STATEMENT = (
+    "Generated entirely from explicit behavioural rules and random draws. No real, "
+    "row-level or personal data was read or used; identifiers are keyed permutations "
+    "of row counters. Any resemblance to real persons or accounts is coincidental."
+)
+
+
+@dataclass
+class Dataset:
+    """Generated tables, kept per chunk so validation can stream over them."""
+
+    config: Config
+    chunks: list[dict[str, pl.DataFrame]]
+
+    def table(self, name: str) -> pl.DataFrame:
+        return pl.concat([c[name] for c in self.chunks])
+
+    @property
+    def subject(self) -> pl.DataFrame:
+        return self.table("subject")
+
+    @property
+    def inquiry(self) -> pl.DataFrame:
+        return self.table("inquiry")
+
+    @property
+    def account(self) -> pl.DataFrame:
+        return self.table("account")
+
+    @property
+    def account_month(self) -> pl.DataFrame:
+        return self.table("account_month")
+
+    def iter_chunks(self) -> Iterator[dict[str, pl.DataFrame]]:
+        yield from self.chunks
+
+    def write(self, out: str | Path, format: Format = "parquet") -> dict:
+        out = Path(out)
+        counts = [_write_chunk(out, i, c, format) for i, c in enumerate(self.chunks)]
+        return _write_manifest(out, self.config, format, counts)
+
+
+def generate(config: Config) -> Dataset:
+    """Generate all chunks in memory. Use :func:`write_dataset` for large runs."""
+    ctx = make_context(config)
+    return Dataset(config, [generate_chunk(config, i, ctx) for i in range(config.n_chunks)])
+
+
+def write_dataset(
+    config: Config, out: str | Path, format: Format = "parquet", workers: int = 1
+) -> dict:
+    """Generate chunk by chunk straight to disk; memory stays bounded by one chunk per worker.
+
+    Output is byte-identical for a given config regardless of ``workers``.
+    """
+    out = Path(out)
+    out.mkdir(parents=True, exist_ok=True)
+    jobs = range(config.n_chunks)
+    if workers <= 1 or config.n_chunks == 1:
+        ctx = make_context(config)
+        counts = [_write_chunk(out, i, generate_chunk(config, i, ctx), format) for i in jobs]
+    else:
+        # "spawn": forking after Polars' thread pool has started can deadlock.
+        spawn = multiprocessing.get_context("spawn")
+        with ProcessPoolExecutor(max_workers=workers, mp_context=spawn) as pool:
+            counts = list(pool.map(_chunk_job, [(config, i, str(out), format) for i in jobs]))
+    return _write_manifest(out, config, format, counts)
+
+
+def _chunk_job(args: tuple[Config, int, str, Format]) -> dict[str, int]:
+    config, chunk, out, format = args
+    return _write_chunk(Path(out), chunk, generate_chunk(config, chunk), format)
+
+
+def part_path(out: Path, table: str, chunk: int, format: Format) -> Path:
+    return out / table / f"part-{chunk:05d}.{format}"
+
+
+def _write_chunk(out: Path, chunk: int, frames: dict[str, pl.DataFrame], format: Format) -> dict:
+    for name in TABLES:
+        path = part_path(out, name, chunk, format)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        df = frames[name]
+        if format == "parquet":
+            df.write_parquet(path, compression="zstd", statistics=True)
+        else:
+            df.write_csv(path)
+    return {name: frames[name].height for name in TABLES}
+
+
+def _write_manifest(out: Path, config: Config, format: Format, counts: list[dict]) -> dict:
+    out.mkdir(parents=True, exist_ok=True)
+    manifest = {
+        "creforge_version": __version__,
+        "format": format,
+        "chunks": config.n_chunks,
+        "config_sha256": config.sha256(),
+        "row_counts": {t: sum(c[t] for c in counts) for t in TABLES},
+        "privacy": PRIVACY_STATEMENT,
+        "config": config.model_dump(mode="json"),
+    }
+    (out / MANIFEST).write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
+    return manifest
+
+
+@dataclass
+class DiskDataset:
+    """A dataset previously written by :func:`write_dataset`, read one chunk at a time."""
+
+    path: Path
+    manifest: dict
+
+    @classmethod
+    def open(cls, path: str | Path) -> DiskDataset:
+        path = Path(path)
+        return cls(path, json.loads((path / MANIFEST).read_text(encoding="utf-8")))
+
+    @property
+    def config(self) -> Config:
+        return Config.model_validate(self.manifest["config"])
+
+    def iter_chunks(self) -> Iterator[dict[str, pl.DataFrame]]:
+        fmt = self.manifest["format"]
+        for i in range(self.manifest["chunks"]):
+            yield {t: _read(part_path(self.path, t, i, fmt), fmt) for t in TABLES}
+
+
+# CSV loses types; restore the non-string columns explicitly instead of guessing.
+_CSV_TYPES: dict[str, pl.DataType] = {
+    **dict.fromkeys(("created_month", "inquiry_date", "open_date", "close_date", "as_of_month"),
+                    pl.Date()),
+    **dict.fromkeys(("requested_amount", "credit_limit", "principal", "interest_rate", "balance",
+                     "amount_due", "amount_paid"), pl.Float64()),
+    **dict.fromkeys(("birth_year", "tenor_months", "months_in_arrears"), pl.Int16()),
+    "secured": pl.Boolean(),
+}
+
+
+def _read(path: Path, fmt: str) -> pl.DataFrame:
+    if fmt == "parquet":
+        return pl.read_parquet(path)
+    df = pl.read_csv(path, infer_schema=False)
+    def restore(c: str, t: pl.DataType) -> pl.Expr:
+        if t == pl.Date():
+            return pl.col(c).str.to_date()
+        if t == pl.Boolean():
+            return pl.col(c) == "true"
+        return pl.col(c).cast(t)
+
+    return df.with_columns(restore(c, t) for c, t in _CSV_TYPES.items() if c in df.columns)