scelo 0.1.0__py3-none-any.whl

scelo/__init__.py

@@ -0,0 +1,47 @@
+"""scelo: soft data → tools → hard data, for actuaries who write code.
+
+    import scelo as sc
+    df  = sc.load("claims.csv")          # soft: typed the way Scelo IDE types it
+    df  = sc.clean(df)                   # the IDE's safe cleaning ops, audited
+    res = sc.reserve(df)                 # tools: chain ladder, Mack, BF, bootstrap
+    sc.report(res, to="pack.html")       # hard: numbers that travel with their basis
+
+Everything is one ``sc.`` away and every table-shaped result is a
+:class:`Table`, a pandas DataFrame that carries its title, basis, notes and
+provenance. ``sc.cheatsheet()`` prints the one-screen map.
+"""
+
+from __future__ import annotations
+
+from ._version import __version__
+from ._table import Table, as_table, notes
+from ._alias import COLUMN_ALIASES, find_column, infer
+from ._audit import audit, clear_audit, enable_audit, content_hash
+from .io import *  # noqa: F401,F403
+from .profile import *  # noqa: F401,F403
+from .clean import *  # noqa: F401,F403
+from .combine import *  # noqa: F401,F403
+from .life import *  # noqa: F401,F403
+from .reserving import *  # noqa: F401,F403
+from .finance import *  # noqa: F401,F403
+from .risk import *  # noqa: F401,F403
+from .pricing import *  # noqa: F401,F403
+from .fairness import *  # noqa: F401,F403
+from .climate import *  # noqa: F401,F403
+from .wmtr import *  # noqa: F401,F403
+from .swarm import *  # noqa: F401,F403
+from .workspace import *  # noqa: F401,F403
+from .lifelib import *  # noqa: F401,F403
+from .hard import *  # noqa: F401,F403
+from .pipelines import *  # noqa: F401,F403
+from .viz import *  # noqa: F401,F403
+import importlib as _importlib  # noqa: E402
+
+_importlib.import_module("scelo.accessor")  # registers df.sc
+
+_MODULES = ("io", "profile", "clean", "combine", "life", "reserving", "finance", "risk", "pricing", "fairness", "climate", "wmtr", "swarm",
+            "workspace", "lifelib", "hard", "pipelines", "viz")
+__all__ = sorted(
+    {n for m in _MODULES for n in getattr(_importlib.import_module(f"scelo.{m}"), "__all__", [])}
+    | {"Table", "as_table", "notes", "COLUMN_ALIASES", "find_column", "infer", "audit", "clear_audit", "enable_audit", "content_hash", "__version__"}
+)

scelo/_alias.py

@@ -0,0 +1,121 @@
+"""Column inference: the reason ``sc.triangle(df)`` needs no arguments.
+
+Every tools function accepts explicit column names, but when you leave them
+out it looks the columns up here: case- and punctuation-insensitive matches
+against the alias lists Scelo IDE uses for its own table suggestions, so
+``accident_year`` / ``AY`` / ``Origin Year`` all resolve to the origin column.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Dict, Iterable, List, Optional, Sequence
+
+import pandas as pd
+
+_NON_ALNUM = re.compile(r"[^a-z0-9]")
+
+
+def _lc(s: str) -> str:
+    return _NON_ALNUM.sub("", str(s).lower())
+
+
+# Mirrors packages/scelo-core/src/actuarialTables.ts COLUMN_ALIASES, with the
+# extra spellings that show up on real extracts. Order matters: the first alias
+# that matches wins, so the canonical name leads each list.
+COLUMN_ALIASES: Dict[str, List[str]] = {
+    "age": [
+        "age", "age_at_entry", "ageatentry", "issue_age", "attained_age", "x", "age_x",
+        "age_band", "ageband", "age_last", "age_nearest", "entry_age", "age_years",
+    ],
+    "qx": ["qx", "q_x", "mortality", "mortality_rate", "death_rate", "q", "prob_death", "rate", "qx_ult"],
+    "mx": ["mx", "m_x", "central_rate", "hazard", "mu", "mu_x", "force_of_mortality"],
+    "lx": ["lx", "l_x", "lives", "survivors", "l"],
+    "deaths": ["deaths", "death", "d", "dx", "actual_deaths", "claims_count", "n_deaths", "died", "events", "actual"],
+    "exposure": [
+        "exposure", "exposures", "exposed", "exposed_to_risk", "etr", "lives_exposed", "person_years",
+        "policy_years", "central_exposure", "initial_exposure", "expo", "e", "ex", "time_at_risk",
+    ],
+    "expected": ["expected", "expected_deaths", "exp_deaths", "e_deaths", "expected_claims"],
+    "origin": [
+        "origin", "origin_year", "accident_year", "accidentyear", "ay", "uw_year", "underwriting_year",
+        "occurrence_year", "loss_year", "year_of_origin", "cohort", "origin_period", "accident_period",
+        "acc_year", "accyear", "uwy", "policy_year",
+    ],
+    "development": [
+        "development", "dev", "development_period", "dev_period", "development_year", "dev_year", "lag",
+        "delay", "age_months", "development_lag", "dev_lag", "period", "devyear", "development_months",
+    ],
+    "payment": [
+        "payment_year", "calendar_year", "paid_year", "settlement_year", "transaction_year", "report_year",
+        "valuation_year", "cal_year", "calendar_period", "payment_period", "cy",
+    ],
+    "value": [
+        "paid", "incurred", "paid_amount", "incurred_amount", "claims", "claim_amount", "amount", "loss",
+        "losses", "payments", "value", "cumulative", "reported", "paid_claims", "incurred_claims",
+        "claim", "cost", "severity",
+    ],
+    "premium": [
+        "premium_pp", "premium", "annual_premium", "monthly_premium", "prem", "earned_premium",
+        "written_premium", "gwp", "gep", "premiums", "ep",
+    ],
+    "tenor": ["tenor", "maturity", "term", "years", "year", "t", "maturity_years", "tenor_years"],
+    "rate": [
+        "rate", "zero_rate", "spot", "spot_rate", "yield", "zero", "swap_rate", "par_rate",
+        "interest_rate", "zero_coupon", "spot_yield", "r",
+    ],
+    "sex": ["sex", "gender", "male_female", "m_f"],
+    "policy_term": ["policy_term", "policyterm", "term", "term_years", "policy_term_years", "duration_years"],
+    "sum_assured": ["sum_assured", "sumassured", "sa", "face_amount", "face", "benefit", "sum_insured", "si", "coverage"],
+    "count": ["count", "policy_count", "policycount", "n", "number", "claim_count", "frequency", "num_claims", "nclaims", "claims_count", "policies"],
+    "year": ["year", "calendar_year", "cal_year", "period", "yr"],
+    "date": ["date", "as_at", "valuation_date", "effective_date", "start_date", "issue_date"],
+    "duration": ["duration", "time", "t", "survival_time", "policy_duration", "tenure"],
+    "event": ["event", "status", "died", "death", "claimed", "lapsed", "censored"],
+    "group": ["group", "segment", "class", "cohort", "risk_class", "region", "band"],
+    "actual": ["actual", "observed", "actual_claims", "actual_deaths", "deaths", "claims"],
+}
+
+
+def find_column(columns: Iterable[str], aliases: Sequence[str]) -> Optional[str]:
+    """First column whose normalised name matches one of the aliases, else None."""
+    cols = list(columns)
+    table = {}
+    for c in cols:
+        table.setdefault(_lc(c), c)
+    for a in aliases:
+        hit = table.get(_lc(a))
+        if hit is not None:
+            return hit
+    return None
+
+
+def infer(df: pd.DataFrame, role: str, explicit: Optional[str] = None, *, required: bool = True,
+          exclude: Iterable[str] = ()) -> Optional[str]:
+    """Resolve a column for ``role`` ("age", "origin", …).
+
+    ``explicit`` wins when given (and is validated). Otherwise the alias list
+    for the role is searched, skipping anything in ``exclude`` (so the same
+    column is not picked for two roles). Raises a ``KeyError`` naming the
+    role, the aliases tried and the columns available when ``required``.
+    """
+    if explicit is not None:
+        if explicit not in df.columns:
+            raise KeyError(f'column "{explicit}" is not in the data (have: {", ".join(map(str, df.columns))})')
+        return explicit
+    aliases = COLUMN_ALIASES.get(role)
+    if aliases is None:
+        raise KeyError(f"unknown column role {role!r}")
+    ex = set(exclude)
+    hit = find_column([c for c in df.columns if c not in ex], aliases)
+    if hit is None and required:
+        raise KeyError(
+            f"could not infer the {role} column: pass {role}=<name>. "
+            f"Tried {', '.join(aliases[:6])}…; columns: {', '.join(map(str, df.columns))}"
+        )
+    return hit
+
+
+def numeric_columns(df: pd.DataFrame) -> List[str]:
+    """Names of the numeric (non-boolean) columns."""
+    return [c for c in df.columns if pd.api.types.is_numeric_dtype(df[c]) and not pd.api.types.is_bool_dtype(df[c])]

scelo/_audit.py

@@ -0,0 +1,137 @@
+"""The audit trail: what the tools layer did, in order.
+
+Scelo's pipeline rule is that hard data never travels without the trail that
+produced it. Every tools function records one entry here (function, the
+arguments that matter, input/output shapes and content hashes, wall time).
+``scelo.audit()`` returns the trail as a DataFrame; ``scelo.hard()`` copies
+the relevant entries onto the table it stamps.
+"""
+
+from __future__ import annotations
+
+import functools
+import hashlib
+import inspect
+import time
+from datetime import datetime, timezone
+from typing import Any, Callable, Dict, List, Optional
+
+import numpy as np
+import pandas as pd
+
+from ._version import __version__
+
+_TRAIL: List[Dict[str, Any]] = []
+_ENABLED = True
+
+
+def content_hash(obj: Any) -> str:
+    """sha256 of a DataFrame / Series / array / scalar's content (not its identity)."""
+    h = hashlib.sha256()
+    if isinstance(obj, pd.DataFrame):
+        h.update(",".join(map(str, obj.columns)).encode())
+        try:
+            h.update(pd.util.hash_pandas_object(obj, index=False).values.tobytes())
+        except TypeError:  # unhashable cells (lists, dicts)
+            h.update(obj.to_csv(index=False).encode())
+    elif isinstance(obj, pd.Series):
+        try:
+            h.update(pd.util.hash_pandas_object(obj, index=False).values.tobytes())
+        except TypeError:
+            h.update(obj.to_csv(index=False).encode())
+    elif isinstance(obj, np.ndarray):
+        h.update(np.ascontiguousarray(obj).tobytes())
+    elif obj is None:
+        h.update(b"None")
+    else:
+        h.update(repr(obj).encode())
+    return h.hexdigest()
+
+
+def _shape(obj: Any) -> Optional[str]:
+    if isinstance(obj, pd.DataFrame):
+        return f"{len(obj)}×{obj.shape[1]}"
+    if isinstance(obj, (pd.Series, np.ndarray, list, tuple)):
+        return f"{len(obj)}"
+    return None
+
+
+def _summarise_arg(v: Any) -> Any:
+    if isinstance(v, (pd.DataFrame, pd.Series, np.ndarray)):
+        return f"<{type(v).__name__} {_shape(v)} {content_hash(v)[:8]}>"
+    if isinstance(v, (int, float, str, bool)) or v is None:
+        return v
+    if isinstance(v, (list, tuple)) and len(v) <= 12:
+        return [_summarise_arg(x) for x in v]
+    if isinstance(v, dict) and len(v) <= 12:
+        return {k: _summarise_arg(x) for k, x in v.items()}
+    return f"<{type(v).__name__}>"
+
+
+def record(fn: str, args: Dict[str, Any], inputs: Any, output: Any, ms: float, note: str = "") -> Dict[str, Any]:
+    entry = {
+        "at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        "fn": fn,
+        "args": {k: _summarise_arg(v) for k, v in args.items()},
+        "in": content_hash(inputs)[:16] if inputs is not None else None,
+        "in_shape": _shape(inputs),
+        "out": content_hash(output)[:16] if output is not None else None,
+        "out_shape": _shape(output),
+        "ms": round(ms, 2),
+        "scelo": __version__,
+        "note": note,
+    }
+    if _ENABLED:
+        _TRAIL.append(entry)
+    return entry
+
+
+def tool(fn: Callable) -> Callable:
+    """Decorator: time the call, hash its first DataFrame-like input and its output, log it."""
+    sig = inspect.signature(fn)
+
+    @functools.wraps(fn)
+    def wrapper(*args: Any, **kwargs: Any):
+        t0 = time.perf_counter()
+        out = fn(*args, **kwargs)
+        ms = (time.perf_counter() - t0) * 1000
+        if _ENABLED:
+            try:
+                bound = sig.bind_partial(*args, **kwargs)
+                argmap = dict(bound.arguments)
+            except TypeError:
+                argmap = {"args": args, "kwargs": kwargs}
+            first = next((v for v in argmap.values() if isinstance(v, (pd.DataFrame, pd.Series, np.ndarray))), None)
+            out_for_hash = out
+            if isinstance(out, tuple) and out and isinstance(out[0], (pd.DataFrame, pd.Series)):
+                out_for_hash = out[0]
+            elif not isinstance(out, (pd.DataFrame, pd.Series, np.ndarray)):
+                out_for_hash = getattr(out, "table", None)
+            record(fn.__name__, argmap, first, out_for_hash, ms)
+        return out
+
+    wrapper.__scelo_tool__ = True  # type: ignore[attr-defined]
+    return wrapper
+
+
+def audit(last: Optional[int] = None) -> pd.DataFrame:
+    """The audit trail as a DataFrame (most recent last). ``last=n`` for the tail."""
+    rows = _TRAIL[-last:] if last else list(_TRAIL)
+    if not rows:
+        return pd.DataFrame(columns=["at", "fn", "args", "in", "in_shape", "out", "out_shape", "ms", "scelo", "note"])
+    return pd.DataFrame(rows)
+
+
+def clear_audit() -> None:
+    """Forget the trail (a new session / a new deliverable)."""
+    _TRAIL.clear()
+
+
+def enable_audit(on: bool = True) -> None:
+    """Switch recording on/off (it is on by default; off saves a little time in tight loops)."""
+    global _ENABLED
+    _ENABLED = bool(on)
+
+
+def entries() -> List[Dict[str, Any]]:
+    return list(_TRAIL)

scelo/_table.py

@@ -0,0 +1,148 @@
+"""Table: a pandas DataFrame that carries its own caveats.
+
+Every Scelo output that is table-shaped is a :class:`Table`: a real
+``pandas.DataFrame`` (slice it, merge it, plot it, ``.to_csv`` it) with four
+extra attributes that survive most pandas operations:
+
+``title``       one line naming the table,
+``notes``       the things an actuary should know before trusting it,
+``basis``       a one-line provenance label ("Gompertz–Makeham (illustrative) · i = 4 %"),
+``provenance``  a dict stamped by :func:`scelo.hard`: content hash, time,
+                scelo version, the audit entries that produced it.
+
+Printing a Table prints the frame and then its notes, so the caveat is on
+screen at the moment the number is, the one-way pipeline rule in practice:
+a number never travels without its basis.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+import pandas as pd
+
+_NOTE_PREFIX = "  · "
+
+
+class Table(pd.DataFrame):
+    """A DataFrame with ``title``, ``notes``, ``basis`` and ``provenance``."""
+
+    _metadata = ["title", "notes", "basis", "provenance", "stage"]
+
+    def __init__(
+        self,
+        data=None,
+        *args: Any,
+        title: Optional[str] = None,
+        notes: Optional[List[str]] = None,
+        basis: Optional[str] = None,
+        provenance: Optional[Dict[str, Any]] = None,
+        stage: Optional[str] = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(data, *args, **kwargs)
+        # Preserve metadata when constructed from another Table.
+        src = data if isinstance(data, Table) else None
+        has = src is not None
+        object.__setattr__(self, "title", title if title is not None else (src.title if has else None))
+        object.__setattr__(self, "notes", list(notes) if notes is not None else (list(src.notes) if has else []))
+        object.__setattr__(self, "basis", basis if basis is not None else (src.basis if has else None))
+        object.__setattr__(
+            self, "provenance", dict(provenance) if provenance is not None else (dict(src.provenance) if has else {})
+        )
+        object.__setattr__(self, "stage", stage if stage is not None else (src.stage if has else "soft"))
+
+    # pandas subclassing contract ------------------------------------------------
+    @property
+    def _constructor(self):
+        return Table
+
+    @property
+    def _constructor_sliced(self):
+        return pd.Series
+
+    # presentation ----------------------------------------------------------------
+    def _footer(self) -> str:
+        lines: List[str] = []
+        if self.title:
+            lines.append(f"— {self.title}")
+        if self.basis:
+            lines.append(f"  basis: {self.basis}")
+        for n in self.notes or []:
+            lines.append(_NOTE_PREFIX + n)
+        if self.provenance:
+            h = self.provenance.get("sha256", "")
+            if h:
+                lines.append(f"  hard · {h[:12]} · scelo {self.provenance.get('scelo', '?')} · {self.provenance.get('at', '')}")
+        return "\n".join(lines)
+
+    def __repr__(self) -> str:  # pragma: no cover - presentation
+        body = super().__repr__()
+        foot = self._footer()
+        return body + ("\n" + foot if foot else "")
+
+    def _repr_html_(self) -> Optional[str]:  # pragma: no cover - presentation
+        body = super()._repr_html_()
+        foot = self._footer()
+        if not foot or body is None:
+            return body
+        esc = foot.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+        return body + f"<pre style='font-size:0.85em;opacity:0.8'>{esc}</pre>"
+
+    # convenience -------------------------------------------------------------------
+    def note(self, text: str) -> "Table":
+        """Append a note (returns self, for chaining)."""
+        self.notes.append(text)
+        return self
+
+    @property
+    def df(self) -> pd.DataFrame:
+        """A plain ``pandas.DataFrame`` copy (drops the Scelo metadata)."""
+        return pd.DataFrame(self)
+
+    def to_markdown_report(self) -> str:
+        """Title, basis, the table (markdown) and the notes, one block."""
+        out: List[str] = []
+        if self.title:
+            out.append(f"### {self.title}")
+        if self.basis:
+            out.append(f"*Basis:* {self.basis}")
+        out.append("")
+        try:
+            out.append(pd.DataFrame(self).to_markdown(index=False))
+        except ImportError:  # tabulate not installed
+            out.append("```\n" + pd.DataFrame(self).to_string(index=False) + "\n```")
+        if self.notes:
+            out.append("")
+            out.extend(f"- {n}" for n in self.notes)
+        if self.provenance:
+            out.append("")
+            out.append(
+                f"<sub>hard · sha256 {self.provenance.get('sha256', '')[:16]} · "
+                f"scelo {self.provenance.get('scelo', '')} · {self.provenance.get('at', '')}</sub>"
+            )
+        return "\n".join(out)
+
+
+def as_table(
+    df: pd.DataFrame,
+    title: Optional[str] = None,
+    notes: Optional[List[str]] = None,
+    basis: Optional[str] = None,
+    stage: str = "hard",
+) -> Table:
+    """Wrap a DataFrame as a :class:`Table` (copying metadata if it already is one)."""
+    t = Table(df)
+    if title is not None:
+        object.__setattr__(t, "title", title)
+    if notes is not None:
+        object.__setattr__(t, "notes", list(notes))
+    if basis is not None:
+        object.__setattr__(t, "basis", basis)
+    object.__setattr__(t, "stage", stage)
+    return t
+
+
+def notes(x: Any) -> List[str]:
+    """The notes attached to a Scelo result (empty list for plain objects)."""
+    return list(getattr(x, "notes", []) or [])

scelo/_version.py

@@ -0,0 +1,7 @@
+"""Version of the scelo Python package.
+
+Kept in its own module so `scelo.__version__`, the audit trail and the
+provenance stamp on every hard table all read one value.
+"""
+
+__version__ = "0.1.0"

scelo/accessor.py

@@ -0,0 +1,59 @@
+"""``df.sc``: the pandas accessor, for people who chain.
+
+    df.sc.profile()         df.sc.clean("all")       df.sc.triangle().sc.mack()
+
+Every Scelo function that takes a frame first is available as a method;
+the frame is passed as the first argument.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+
+import importlib
+
+_clean = importlib.import_module("scelo.clean")
+_combine = importlib.import_module("scelo.combine")
+_fairness = importlib.import_module("scelo.fairness")
+_life = importlib.import_module("scelo.life")
+_pricing = importlib.import_module("scelo.pricing")
+_profile = importlib.import_module("scelo.profile")
+_reserving = importlib.import_module("scelo.reserving")
+_workspace = importlib.import_module("scelo.workspace")
+_hard = importlib.import_module("scelo.hard").hard
+
+_METHODS = {
+    "profile": _profile.profile, "describe": _profile.describe, "types": _profile.types, "missing": _profile.missing, "tab": _profile.tab,
+    "corr": _profile.corr, "outliers": _profile.outliers, "inliers": _profile.inliers,
+    "suggest": _clean.suggest, "clean": _clean.clean, "dedupe": _clean.dedupe, "snake_names": _clean.snake_names, "impute": _clean.impute,
+    "cap_outliers": _clean.cap_outliers, "parse_dates": _clean.parse_dates, "parse_numbers": _clean.parse_numbers,
+    "triangle": _reserving.triangle, "chain_ladder": _reserving.chain_ladder, "mack": _reserving.mack, "bf": _reserving.bf,
+    "bootstrap": _reserving.bootstrap, "reserve": _reserving.reserve, "ldf": _reserving.ldf, "cdf": _reserving.cdf, "ata": _reserving.ata,
+    "life_table": lambda df, **kw: _life.life_table(None, df, **kw), "commutation": lambda df, **kw: _life.commutation(None, df, **kw),
+    "factors": lambda df, **kw: _life.factors(None, df, **kw), "ae": _life.ae, "model_points": _life.model_points, "graduate": _life.graduate,
+    "lee_carter": _life.lee_carter, "kaplan_meier": _life.kaplan_meier, "basicterm": _life.basicterm,
+    "glm": _pricing.glm, "freq_sev": _pricing.freq_sev, "loss_ratio": _pricing.loss_ratio,
+    "fairness": _fairness.fairness, "fairness_audit": _fairness.fairness_audit,
+    "join": _combine.join, "append": _combine.append, "combine": _combine.combine, "diff": _combine.diff,
+    "bottleneck": _workspace.bottleneck, "active_subspace": _workspace.active_subspace,
+    "hard": _hard,
+}
+
+
+@pd.api.extensions.register_dataframe_accessor("sc")
+class SceloAccessor:
+    """``df.sc.<function>(...)`` for every frame-first Scelo function."""
+
+    def __init__(self, obj: pd.DataFrame) -> None:
+        self._obj = obj
+
+    def __getattr__(self, name: str) -> Any:
+        fn = _METHODS.get(name)
+        if fn is None:
+            raise AttributeError(f"df.sc has no method {name!r}; available: {', '.join(sorted(_METHODS))}")
+        return lambda *a, **kw: fn(self._obj, *a, **kw)
+
+    def __dir__(self):  # pragma: no cover
+        return sorted(_METHODS)