tabint 0.1.1__py3-none-any.whl

tabint/__init__.py

@@ -0,0 +1,16 @@
+"""tabint — a deterministic, reproducible intelligence layer for single-table data.
+
+Public API is intentionally small. See docs/architecture.md for the design and
+docs/roadmap.md for what's implemented vs. planned.
+"""
+from tabint.session import Session
+from tabint.results import Result
+from tabint.workspace import Workspace, Table
+from tabint.relationships import RelationshipGraph, Relationship
+from tabint import persistence
+
+__all__ = [
+    "Session", "Result", "Workspace", "Table",
+    "RelationshipGraph", "Relationship", "persistence",
+]
+__version__ = "0.1.0"

tabint/_ducktable.py

@@ -0,0 +1,264 @@
+"""Low-level DuckDB table mechanics shared by Store and Workspace/Table.
+
+Both the single-table Store and the multi-table Workspace need the same three
+primitives: load a CSV into a table stamped with a stable row id, read it back in
+that order, and write a computed column back by position. Keeping that logic here
+means there is exactly one implementation of the ``_ti_row`` machinery.
+
+Every table is stored as:
+  - an internal table ``<internal>`` carrying a 0-based ``_ti_row`` id column
+  - a view ``<view>`` = the internal table minus ``_ti_row`` (what callers query)
+
+All identifiers (table, view, column names) are double-quoted before being spliced
+into SQL, so names containing spaces, punctuation, or reserved words are handled
+safely; the CSV path is single-quote-escaped for the same reason.
+
+All functions take an ibis DuckDB backend; ``backend.con`` is the underlying
+duckdb connection used for the low-level writes ibis doesn't cover.
+"""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+
+def quote_ident(name: str) -> str:
+    """Quote a SQL identifier, escaping embedded double quotes."""
+    return '"' + str(name).replace('"', '""') + '"'
+
+
+def _quote_literal(value: str) -> str:
+    """Escape a value for use inside a single-quoted SQL string literal."""
+    return value.replace("'", "''")
+
+
+def load_csv(backend: Any, csv_path: Path, internal: str, view: str) -> None:
+    """Create the internal table (with _ti_row) and its public view from a CSV."""
+    path_literal = _quote_literal(str(csv_path))
+    create_from_query(backend, f"SELECT * FROM read_csv_auto('{path_literal}')", internal, view)
+
+
+def create_from_query(backend: Any, select_sql: str, internal: str, view: str) -> None:
+    """Materialize a SELECT as a new internal table (with _ti_row) plus its view.
+
+    Used for CSV loads and for derived tables such as joins — anything that
+    produces rows and needs the same stable-row-id + hidden-view treatment.
+    """
+    qi, qv = quote_ident(internal), quote_ident(view)
+    con = backend.con
+    con.execute(
+        f"CREATE TABLE {qi} AS "
+        f"SELECT row_number() OVER () - 1 AS _ti_row, * FROM ({select_sql})"
+    )
+    con.execute(f"CREATE VIEW {qv} AS SELECT * EXCLUDE (_ti_row) FROM {qi}")
+
+
+def create_empty(backend: Any, columns: list[tuple[str, str]], internal: str, view: str) -> None:
+    """Create an empty structured table (with auto-assigned _ti_row) plus its view.
+
+    Unlike create_from_query, this table starts with no rows: it defines a schema
+    the caller then fills with insert_select. A DuckDB sequence supplies _ti_row
+    so ordinary INSERTs that omit it get a monotonically increasing id for free,
+    keeping the table compatible with the frame_in_order / write_back machinery.
+
+    Args:
+        columns: (name, sql_type) pairs. Types must already be validated.
+    """
+    qi, qv = quote_ident(internal), quote_ident(view)
+    seq = quote_ident(f"_seq_{internal}")
+    seq_literal = _quote_literal(f"_seq_{internal}")
+    con = backend.con
+    coldefs = ", ".join(f"{quote_ident(name)} {sql_type}" for name, sql_type in columns)
+    con.execute(f"CREATE SEQUENCE {seq}")
+    con.execute(
+        f"CREATE TABLE {qi} (_ti_row BIGINT DEFAULT nextval('{seq_literal}'), {coldefs})"
+    )
+    con.execute(f"CREATE VIEW {qv} AS SELECT * EXCLUDE (_ti_row) FROM {qi}")
+
+
+def insert_select(backend: Any, internal: str, source_sql: str) -> int:
+    """Append rows into an existing table from a SELECT/VALUES query; return the count.
+
+    The column list (every column except the auto-assigned _ti_row) is spliced in
+    explicitly so source_sql maps positionally to the table's own columns and the
+    sequence default fills _ti_row. Returns the number of rows inserted.
+    """
+    qi = quote_ident(internal)
+    con = backend.con
+    cols = [row[0] for row in con.execute(f"DESCRIBE {qi}").fetchall() if row[0] != "_ti_row"]
+    col_list = ", ".join(quote_ident(c) for c in cols)
+    before = con.execute(f"SELECT COUNT(*) FROM {qi}").fetchone()[0]
+    con.execute(f"INSERT INTO {qi} ({col_list}) {source_sql}")
+    after = con.execute(f"SELECT COUNT(*) FROM {qi}").fetchone()[0]
+    return int(after - before)
+
+
+# Persistent registry of derived (computed) columns — cluster labels, outlier
+# flags, predictions, etc. — that must be excluded from feature matrices. It
+# lives in the same DuckDB file as the data so it survives reopen/_reattach; keyed
+# by (table_key, column_name) where table_key is the public table/view name.
+_DERIVED_REGISTRY = "_ti_derived_columns"
+
+
+def _ensure_derived_registry(con: Any) -> None:
+    con.execute(
+        f"CREATE TABLE IF NOT EXISTS {quote_ident(_DERIVED_REGISTRY)} "
+        f"(table_key VARCHAR, column_name VARCHAR)"
+    )
+
+
+def register_derived(backend: Any, table_key: str, column_name: str) -> None:
+    """Mark a column as derived (a non-feature annotation) for the given table."""
+    con = backend.con
+    q = quote_ident(_DERIVED_REGISTRY)
+    _ensure_derived_registry(con)
+    con.execute(
+        f"DELETE FROM {q} WHERE table_key = ? AND column_name = ?", [table_key, column_name]
+    )
+    con.execute(f"INSERT INTO {q} VALUES (?, ?)", [table_key, column_name])
+
+
+def unregister_derived(backend: Any, table_key: str, column_name: str) -> None:
+    """Clear any derived mark on a column (e.g. when it is (re)written as a feature)."""
+    con = backend.con
+    _ensure_derived_registry(con)
+    con.execute(
+        f"DELETE FROM {quote_ident(_DERIVED_REGISTRY)} "
+        f"WHERE table_key = ? AND column_name = ?",
+        [table_key, column_name],
+    )
+
+
+def derived_columns(backend: Any, table_key: str) -> set[str]:
+    """Return the set of columns marked derived for the given table."""
+    con = backend.con
+    _ensure_derived_registry(con)
+    rows = con.execute(
+        f"SELECT column_name FROM {quote_ident(_DERIVED_REGISTRY)} WHERE table_key = ?",
+        [table_key],
+    ).fetchall()
+    return {r[0] for r in rows}
+
+
+# Tokens that must never appear in a feature expression: statement/DDL keywords,
+# subquery starters, and functions that reach the filesystem or catalog. Matched
+# as whole words, case-insensitively — the guard for agent-authored SQL that runs
+# server-side over massive tables (see add_computed_column).
+_SQL_EXPR_BLOCKLIST = frozenset({
+    "select", "insert", "update", "delete", "drop", "alter", "create", "attach",
+    "detach", "copy", "install", "load", "pragma", "call", "export", "import",
+    "set", "reset", "read_csv", "read_csv_auto", "read_parquet", "read_json",
+    "read_json_auto", "read_text", "read_blob", "glob", "system", "getvariable",
+})
+_WORD_RE = re.compile(r"[A-Za-z_][A-Za-z_0-9]*")
+
+
+def _validate_sql_expression(backend: Any, view: str, expression: str) -> None:
+    """Reject anything beyond a single scalar expression over the table's columns.
+
+    Layered guard: no statement chaining (``;``), no blocklisted keywords/functions
+    (subqueries, DDL/DML, file & catalog readers), then a zero-row bind check so
+    the expression must parse as a scalar over the real columns. Raises ValueError.
+    """
+    if not expression or not expression.strip():
+        raise ValueError("Feature expression is empty.")
+    if ";" in expression:
+        raise ValueError("Feature expression must be a single expression (no ';').")
+    lowered_words = {w.lower() for w in _WORD_RE.findall(expression)}
+    hit = lowered_words & _SQL_EXPR_BLOCKLIST
+    if hit:
+        raise ValueError(
+            f"Feature expression may not use {sorted(hit)} — only a scalar computation "
+            f"over the table's columns is allowed (no subqueries, DDL, or file access)."
+        )
+    # Bind-only check: WHERE false computes nothing but still parses/binds the
+    # expression and its column references. Cheap even on a huge table.
+    qv = quote_ident(view)
+    try:
+        backend.con.execute(f"SELECT ({expression}) FROM {qv} WHERE false")
+    except Exception as exc:
+        raise ValueError(f"Invalid feature expression: {exc}") from exc
+
+
+def add_computed_column(backend: Any, internal: str, view: str, name: str, expression: str) -> int:
+    """Add (or replace) a column computed by a SQL scalar expression, in-database.
+
+    Rebuilds the internal table with the expression as a new column and refreshes
+    the public view, preserving ``_ti_row``. Runs entirely inside DuckDB — no rows
+    are materialized in the app. Returns the count of non-null values produced.
+    """
+    _validate_sql_expression(backend, view, expression)
+    con = backend.con
+    qi, qv, qn = quote_ident(internal), quote_ident(view), quote_ident(name)
+    existing = {row[0] for row in con.execute(f"DESCRIBE {qi}").fetchall()}
+    keep = f"* EXCLUDE ({qn})" if name in existing else "*"
+    con.execute(f"CREATE OR REPLACE TABLE {qi} AS SELECT {keep}, ({expression}) AS {qn} FROM {qi}")
+    con.execute(f"CREATE OR REPLACE VIEW {qv} AS SELECT * EXCLUDE (_ti_row) FROM {qi}")
+    return int(con.execute(f"SELECT COUNT({qn}) FROM {qi}").fetchone()[0])
+
+
+def count_rows(backend: Any, view: str) -> int:
+    """Return the table's row count via an in-database COUNT(*) (no materialization)."""
+    return int(backend.con.execute(f"SELECT COUNT(*) FROM {quote_ident(view)}").fetchone()[0])
+
+
+def count_non_null(backend: Any, view: str, column: str) -> int:
+    """Return the count of non-NULL values in a column, computed inside DuckDB.
+
+    Raises KeyError if the column does not exist (COUNT(col) counts non-NULLs).
+    """
+    con = backend.con
+    qv = quote_ident(view)
+    existing = {row[0] for row in con.execute(f"DESCRIBE {qv}").fetchall()}
+    if column not in existing:
+        raise KeyError(f"No column {column!r}. Known: {sorted(existing)}")
+    return int(con.execute(f"SELECT COUNT({quote_ident(column)}) FROM {qv}").fetchone()[0])
+
+
+def frame_in_order(backend: Any, internal: str) -> Any:
+    """Return the table as a pandas DataFrame in stable _ti_row order (id excluded)."""
+    return backend.sql(
+        f"SELECT * EXCLUDE (_ti_row) FROM {quote_ident(internal)} ORDER BY _ti_row"
+    ).execute()
+
+
+def write_back(backend: Any, internal: str, view: str, name: str, values: Any) -> None:
+    """Add or replace a column by position (join on _ti_row), then refresh the view.
+
+    ``values[i]`` is written to the row whose ``_ti_row`` is ``i`` — the same
+    order frame_in_order returns, so per-row arrays align without realignment.
+
+    Raises ValueError if len(values) != the table's row count, since a mismatch
+    would silently drop rows through the positional inner join.
+    """
+    # DuckDB parameter binding can't consume numpy generics — coerce to Python.
+    col_list = [v.item() if isinstance(v, np.generic) else v for v in values]
+    n = len(col_list)
+    con = backend.con
+    qi, qv, qn = quote_ident(internal), quote_ident(view), quote_ident(name)
+    tmp = quote_ident(f"_wb_{internal}")
+
+    current = con.execute(f"SELECT COUNT(*) FROM {qi}").fetchone()[0]
+    if n != current:
+        raise ValueError(
+            f"write_back_column expected {current} values (one per row) but got {n}."
+        )
+
+    con.execute(
+        f"CREATE OR REPLACE TEMP TABLE {tmp} AS "
+        f"SELECT unnest(range({n})) AS _ti_row, unnest(?) AS {qn}",
+        [col_list],
+    )
+    existing = {row[0] for row in con.execute(f"DESCRIBE {qi}").fetchall()}
+    src_cols = f"{qi}.* EXCLUDE ({qn})" if name in existing else f"{qi}.*"
+
+    con.execute(
+        f"CREATE OR REPLACE TABLE {qi} AS "
+        f"SELECT {src_cols}, {tmp}.{qn} "
+        f"FROM {qi} JOIN {tmp} ON {qi}._ti_row = {tmp}._ti_row"
+    )
+    con.execute(f"CREATE OR REPLACE VIEW {qv} AS SELECT * EXCLUDE (_ti_row) FROM {qi}")
+    con.execute(f"DROP TABLE IF EXISTS {tmp}")

tabint/_serialize.py

@@ -0,0 +1,59 @@
+"""Shared JSON serialization for the CLI and MCP surfaces.
+
+Both front-ends must turn Result objects (and arbitrary values/metadata dicts that
+may contain numpy scalars or NaN) into strict-JSON-safe structures. Keeping that in
+one place means the two surfaces never diverge on how a result is rendered.
+"""
+from __future__ import annotations
+
+import datetime as _dt
+import math
+from decimal import Decimal
+from typing import Any
+
+import numpy as np
+
+
+def jsonable(obj: Any) -> Any:
+    """Recursively coerce numpy scalars, NaN, and ±Infinity into JSON-safe values.
+
+    NaN and infinities are not valid JSON (RFC 8259) and are rejected by strict
+    parsers (Node, jq, Go, Rust), so they become null — an agent must never get a
+    ``0``-exit success that it then fails to parse.
+    """
+    if isinstance(obj, dict):
+        return {k: jsonable(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [jsonable(v) for v in obj]
+    if isinstance(obj, np.generic):
+        obj = obj.item()
+    # DuckDB DECIMAL columns come back as Python Decimal, and DATE/TIMESTAMP as
+    # datetime objects — none are JSON-serializable, so coerce them here.
+    if isinstance(obj, Decimal):
+        obj = float(obj)
+    elif isinstance(obj, (_dt.date, _dt.datetime, _dt.time)):
+        return obj.isoformat()
+    if isinstance(obj, float) and (math.isnan(obj) or math.isinf(obj)):
+        return None
+    return obj
+
+
+def result_dict(res: Any) -> dict:
+    """Render a Result as a plain JSON dict (dropping any non-serializable artifact).
+
+    Always emits the honesty envelope — a ``trust`` block and a top-level
+    ``declined`` flag — so every tool's output carries confidence uniformly. An
+    analytic that hasn't set trust reports ``unassessed`` (never a fake ``high``).
+    """
+    from . import honesty
+
+    trust = getattr(res, "trust", None) or honesty.unassessed()
+    trust_json = trust.model_dump(mode="json")
+    return {
+        "method": res.method,
+        "summary": res.summary,
+        "trust": trust_json,
+        "declined": bool(trust_json.get("declined", False)),
+        "values": jsonable(res.values),
+        "metadata": jsonable(res.metadata),
+    }

tabint/analytics/__init__.py

@@ -0,0 +1,4 @@
+"""Analytics families: descriptive, association, clustering, supervised, interpretation,
+dimensionality reduction, time series, feature-computation (generic feature
+engineering), and the insight-extraction primitives (insights/key-drivers,
+basket/market-basket, causal, cohort/RFM+retention, compare)."""

tabint/analytics/_prep.py

@@ -0,0 +1,141 @@
+"""Shared feature-preparation helpers for the model-based families.
+
+Clustering, dimensionality reduction, and supervised learning all need to turn
+the stored table into a numeric, model-ready matrix. Doing that consistently —
+same column selection, same encoding, same imputation — is what keeps results
+comparable across families, so it lives here rather than being re-derived in each
+module.
+
+Column *classification* is never re-decided here; it is delegated to
+validation.dtypes (the single source of truth). This module only decides how a
+classified column is fed to scikit-learn.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+from sklearn.compose import ColumnTransformer
+from sklearn.impute import SimpleImputer
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import OneHotEncoder, StandardScaler
+
+from ..validation.dtypes import classify_column
+
+# Column types (from validation.dtypes) that carry modelling signal and how they
+# should be treated. Identifiers and datetimes are excluded from feature matrices.
+_NUMERIC_TYPES = {"continuous"}
+_CATEGORICAL_TYPES = {"categorical_nominal", "categorical_ordinal"}
+
+
+def get_frame(store: Any) -> pd.DataFrame:
+    """Return the full table in stable (_ti_row) order as a pandas DataFrame."""
+    return store.get_frame()
+
+
+def feature_columns(
+    store: Any,
+    exclude: tuple[str, ...] = (),
+) -> tuple[list[str], list[str]]:
+    """Split the table's columns into (numeric, categorical) feature lists.
+
+    Identifier and datetime columns are dropped — they are not features. Any
+    names in ``exclude`` (e.g. the supervised target, or a cluster-label column)
+    are dropped too.
+
+    Args:
+        store: The Store instance.
+        exclude: Column names to omit from both lists.
+
+    Returns:
+        (numeric_columns, categorical_columns).
+    """
+    # Derived annotations (outlier flags, cluster labels, predictions) are never
+    # features — drop them alongside the caller's explicit excludes so they can't
+    # leak into a model. reduce_dimensions marks its components feature=True, so
+    # those stay eligible.
+    get_derived = getattr(store, "derived_columns", None)
+    excluded = set(exclude) | (get_derived() if get_derived else set())
+
+    numeric: list[str] = []
+    categorical: list[str] = []
+    for name in store._table.schema():
+        if name in excluded:
+            continue
+        kind = classify_column(name, store)
+        if kind in _NUMERIC_TYPES:
+            numeric.append(name)
+        elif kind in _CATEGORICAL_TYPES:
+            categorical.append(name)
+        # identifier / datetime → intentionally skipped
+    return numeric, categorical
+
+
+def build_preprocessor(
+    numeric: list[str],
+    categorical: list[str],
+    *,
+    scale: bool,
+) -> ColumnTransformer:
+    """Build a ColumnTransformer that imputes, (optionally) scales, and one-hot encodes.
+
+    Args:
+        numeric: Numeric (continuous) feature column names.
+        categorical: Categorical feature column names.
+        scale: Whether to standard-scale numeric columns. Distance-based methods
+            (k-means, PCA) need this; tree ensembles do not.
+
+    Returns:
+        An unfitted ColumnTransformer.
+    """
+    numeric_steps: list[tuple[str, Any]] = [("impute", SimpleImputer(strategy="median"))]
+    if scale:
+        numeric_steps.append(("scale", StandardScaler()))
+    numeric_pipe = Pipeline(numeric_steps)
+
+    categorical_pipe = Pipeline([
+        ("impute", SimpleImputer(strategy="most_frequent")),
+        # Dense output: HistGradientBoosting and the manifold methods reject sparse.
+        ("onehot", OneHotEncoder(handle_unknown="ignore", sparse_output=False)),
+    ])
+
+    transformers = []
+    if numeric:
+        transformers.append(("numeric", numeric_pipe, numeric))
+    if categorical:
+        transformers.append(("categorical", categorical_pipe, categorical))
+    return ColumnTransformer(transformers, remainder="drop")
+
+
+def numeric_matrix(
+    store: Any,
+    exclude: tuple[str, ...] = (),
+    *,
+    scale: bool = True,
+) -> tuple[Any, pd.DataFrame, list[str]]:
+    """Materialize a fully numeric feature matrix for distance-based methods.
+
+    Used by clustering and dimensionality reduction. Numeric columns are scaled
+    (by default) and categorical columns one-hot encoded.
+
+    Args:
+        store: The Store instance.
+        exclude: Column names to omit (e.g. an existing cluster-label column).
+        scale: Whether to standard-scale numeric columns.
+
+    Returns:
+        (X, frame, feature_names) where X is the transformed 2-D array in stable
+        row order, frame is the source DataFrame, and feature_names are the
+        original feature column names (pre-encoding).
+    """
+    frame = get_frame(store)
+    numeric, categorical = feature_columns(store, exclude=exclude)
+    if not numeric and not categorical:
+        raise ValueError("No usable feature columns (all identifier/datetime).")
+    pre = build_preprocessor(numeric, categorical, scale=scale)
+    X = pre.fit_transform(frame)
+    # Densify sparse one-hot output so downstream estimators that dislike sparse
+    # input (e.g. some sklearn manifold methods) work uniformly.
+    if hasattr(X, "toarray"):
+        X = X.toarray()
+    return X, frame, numeric + categorical