sqldim 0.1.0__py3-none-any.whl

sqldim/__init__.py

@@ -0,0 +1,86 @@
+from sqldim.core.fields import Field
+from sqldim.core.models import DimensionModel, FactModel, BridgeModel
+from sqldim.core.mixins import SCD2Mixin, SCD3Mixin, DatelistMixin, CumulativeMixin
+from sqldim.core.facts import TransactionFact, PeriodicSnapshotFact, AccumulatingFact, CumulativeFact, ActivityFact
+from sqldim.scd.backfill import backfill_scd2
+from sqldim.core.graph import SchemaGraph
+from sqldim.models.graph import VertexModel, EdgeModel, Vertex
+from sqldim.graph import GraphModel, TraversalEngine
+from sqldim.graph.schema_graph import SchemaGraph as GraphSchemaGraph, GraphSchema
+from sqldim.narwhals.adapter import NarwhalsAdapter
+from sqldim.narwhals.transforms import col, TransformPipeline
+from sqldim.narwhals.backfill import backfill_scd2_narwhals
+from sqldim.narwhals.scd_engine import NarwhalsHashStrategy, NarwhalsSCDProcessor
+from sqldim.loaders.cumulative import CumulativeLoader
+from sqldim.loaders.bitmask import BitmaskerLoader
+from sqldim.loaders.array_metric import ArrayMetricLoader
+from sqldim.loaders.edge_projection import EdgeProjectionLoader
+from sqldim.exceptions import TransformTypeError
+from sqldim.scd.handler import SCDHandler, SCDResult
+from sqldim.loaders.dimensional import DimensionalLoader, SKResolver
+from sqldim.session import AsyncDimensionalSession
+from sqldim.config import SqldimConfig
+from sqldim.exceptions import (
+    SqldimError,
+    SchemaError, GrainViolationError, NaturalKeyError,
+    SCDError, DestructiveOperationError,
+    LoadError, SKResolutionError, IdempotencyError,
+    MigrationError, DestructiveMigrationError,
+    SemanticError, InvalidJoinError,
+)
+
+__all__ = [
+    # Core schema
+    "Field",
+    "DimensionModel",
+    "FactModel",
+    "BridgeModel",
+    "TransactionFact",
+    "PeriodicSnapshotFact",
+    "AccumulatingFact",
+    "CumulativeFact",
+    "ActivityFact",
+    "DatelistMixin",
+    "CumulativeMixin",
+    "backfill_scd2",
+    "SCD2Mixin",
+    "SCD3Mixin",
+    "SchemaGraph",
+    # Graph extension (Phase 6)
+    "VertexModel",
+    "EdgeModel",
+    "Vertex",
+    "GraphModel",
+    "TraversalEngine",
+    "GraphSchemaGraph",
+    "GraphSchema",
+    # Narwhals integration (Phase 7)
+    "NarwhalsAdapter",
+    "col",
+    "TransformPipeline",
+    "backfill_scd2_narwhals",
+    "NarwhalsHashStrategy",
+    "NarwhalsSCDProcessor",
+    "CumulativeLoader",
+    "BitmaskerLoader",
+    "ArrayMetricLoader",
+    "EdgeProjectionLoader",
+    "TransformTypeError",
+    # SCD
+    "SCDHandler",
+    "SCDResult",
+    # Loaders
+    "DimensionalLoader",
+    "SKResolver",
+    # Session
+    "AsyncDimensionalSession",
+    # Config
+    "SqldimConfig",
+    # Exceptions
+    "SqldimError",
+    "SchemaError", "GrainViolationError", "NaturalKeyError",
+    "SCDError", "DestructiveOperationError",
+    "LoadError", "SKResolutionError", "IdempotencyError",
+    "MigrationError", "DestructiveMigrationError",
+    "SemanticError", "InvalidJoinError",
+]

sqldim/cli.py

@@ -0,0 +1,82 @@
+"""
+sqldim CLI — dimensional migration management.
+
+Usage:
+    sqldim migrations generate "message"
+    sqldim migrations show
+"""
+import argparse
+import json
+import sys
+from typing import List
+
+
+def cmd_migrations_generate(args: argparse.Namespace) -> None:
+    """Generate a migration from a schema diff."""
+    print(f"[sqldim] Generating migration: '{args.message}'")
+    print("[sqldim] Tip: call generate_migration(models, current_state, message) in your project.")
+
+def cmd_migrations_show(args: argparse.Namespace) -> None:
+    """Show pending migration info."""
+    print("[sqldim] No pending migrations detected (run generate first).")
+
+def cmd_migrations_init(args: argparse.Namespace) -> None:
+    """Initialize sqldim migration environment."""
+    import os
+    migration_dir = getattr(args, "dir", "migrations")
+    os.makedirs(migration_dir, exist_ok=True)
+    init_file = os.path.join(migration_dir, "__init__.py")
+    if not os.path.exists(init_file):
+        open(init_file, "w").close()
+    print(f"[sqldim] Migration directory initialized at '{migration_dir}/'")
+
+def cmd_schema_graph(args: argparse.Namespace) -> None:
+    """Print schema graph as JSON or Mermaid."""
+    print("[sqldim] Pass your SchemaGraph instance to .to_dict() or .to_mermaid()")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="sqldim",
+        description="sqldim — Dimensional Modeling toolkit for Python",
+    )
+    subparsers = parser.add_subparsers(dest="command")
+
+    # migrations subcommand
+    mig = subparsers.add_parser("migrations", help="Migration management")
+    mig_sub = mig.add_subparsers(dest="subcommand")
+
+    gen = mig_sub.add_parser("generate", help="Generate a migration from model diff")
+    gen.add_argument("message", help="Migration message")
+    gen.set_defaults(func=cmd_migrations_generate)
+
+    show = mig_sub.add_parser("show", help="Show pending migrations")
+    show.set_defaults(func=cmd_migrations_show)
+
+    init = mig_sub.add_parser("init", help="Initialize migration directory")
+    init.add_argument("--dir", default="migrations", help="Migration directory path")
+    init.set_defaults(func=cmd_migrations_init)
+
+    # schema subcommand
+    schema = subparsers.add_parser("schema", help="Schema introspection")
+    schema_sub = schema.add_subparsers(dest="subcommand")
+    graph = schema_sub.add_parser("graph", help="Print schema graph")
+    graph.set_defaults(func=cmd_schema_graph)
+
+    return parser
+
+
+def main(argv: List[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    if not hasattr(args, "func"):
+        parser.print_help()
+        return 1
+
+    args.func(args)
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

sqldim/config.py

@@ -0,0 +1,22 @@
+from datetime import datetime
+from typing import Literal
+
+from pydantic_settings import BaseSettings
+
+class SqldimConfig(BaseSettings):
+    # SCD defaults
+    scd_default_type: int = 2
+    scd_epoch: datetime = datetime(1970, 1, 1)
+    scd_infinity: datetime = datetime(9999, 12, 31)
+    checksum_algorithm: Literal["md5", "sha1", "sha256"] = "md5"
+
+    # Load defaults
+    default_batch_size: int = 10_000
+    sk_lookup_cache: bool = True
+    sk_lookup_cache_ttl: int = 3600  # seconds
+
+    # Migration defaults
+    allow_destructive: bool = False
+    migration_dir: str = "migrations"
+
+    model_config = {"env_prefix": "SQLDIM_"}

sqldim/core/facts.py

@@ -0,0 +1,26 @@
+from enum import Enum
+from sqlmodel import SQLModel
+from typing import ClassVar, List, Optional
+from sqldim.core.models import FactModel
+
+class FactType(str, Enum):
+    TRANSACTION = "transaction"
+    PERIODIC_SNAPSHOT = "periodic_snapshot"
+    ACCUMULATING_SNAPSHOT = "accumulating_snapshot"
+    CUMULATIVE = "cumulative"
+    ACTIVITY = "activity"
+
+class TransactionFact(FactModel):
+    __fact_type__ = FactType.TRANSACTION
+
+class PeriodicSnapshotFact(FactModel):
+    __fact_type__ = FactType.PERIODIC_SNAPSHOT
+
+class AccumulatingFact(FactModel):
+    __fact_type__ = FactType.ACCUMULATING_SNAPSHOT
+
+class CumulativeFact(FactModel):
+    __fact_type__ = FactType.CUMULATIVE
+
+class ActivityFact(FactModel):
+    __fact_type__ = FactType.ACTIVITY

sqldim/core/fields.py

@@ -0,0 +1,54 @@
+from typing import Any, Optional, Type
+from sqlmodel import Field as SQLModelField
+
+def Field(
+    default: Any = ...,
+    *,
+    default_factory: Optional[Any] = None,
+    primary_key: bool = False,
+    foreign_key: Optional[str] = None,
+    index: bool = False,
+    nullable: Optional[bool] = None,
+    # Dimensional metadata
+    surrogate_key: bool = False,
+    natural_key: bool = False,
+    measure: bool = False,
+    additive: bool = True,
+    dimension: Optional[Type] = None,
+    role: Optional[str] = None,
+    scd: Optional[int] = None,
+    previous_column: Optional[str] = None,
+    **kwargs: Any,
+) -> Any:
+    """
+    Extends SQLModel's Field with dimensional metadata stored in sa_column_kwargs
+    for compatibility across SQLModel/SQLAlchemy versions.
+    """
+    # Collect dimensional metadata
+    dim_meta = {
+        "surrogate_key": surrogate_key,
+        "natural_key": natural_key,
+        "measure": measure,
+        "additive": additive,
+        "dimension": dimension,
+        "role": role,
+        "scd": scd,
+        "previous_column": previous_column,
+    }
+    
+    # Store in sa_column_kwargs so it persists to the SQLAlchemy Column info
+    sa_column_kwargs = kwargs.get("sa_column_kwargs", {})
+    if "info" not in sa_column_kwargs:
+        sa_column_kwargs["info"] = {}
+    sa_column_kwargs["info"].update(dim_meta)
+    kwargs["sa_column_kwargs"] = sa_column_kwargs
+
+    return SQLModelField(
+        default,
+        default_factory=default_factory,
+        primary_key=primary_key,
+        foreign_key=foreign_key,
+        index=index,
+        nullable=nullable,
+        **kwargs,
+    )

sqldim/core/graph.py

@@ -0,0 +1,135 @@
+from typing import List, Type, Dict, Any, Optional
+from sqldim.core.models import DimensionModel, FactModel
+
+
+class RolePlayingRef:
+    """Represents a dimension playing a named role in a fact table."""
+    def __init__(self, dimension: Type[DimensionModel], role: str, fk_column: str):
+        self.dimension = dimension
+        self.role = role
+        self.fk_column = fk_column
+
+    def __repr__(self) -> str:
+        return f"RolePlayingRef(dim={self.dimension.__name__}, role={self.role!r}, fk={self.fk_column!r})"
+
+
+class SchemaGraph:
+    def __init__(self, models: List[Type[Any]]):
+        self.models = models
+        self.dimensions = [m for m in models if issubclass(m, DimensionModel)]
+        self.facts = [m for m in models if issubclass(m, FactModel)]
+
+    @classmethod
+    def from_models(cls, models: List[Type[Any]]) -> "SchemaGraph":
+        return cls(models)
+
+    def get_star_schema(self, fact_model: Type[FactModel]) -> Dict[str, Any]:
+        relationships = {}
+        for name, field in fact_model.model_fields.items():
+            dim = None
+
+            # Check for dimension in field attributes (pydantic side)
+            if hasattr(field, "json_schema_extra") and field.json_schema_extra:
+                dim = field.json_schema_extra.get("dimension")
+
+            # Check for dimension in sa_column (SQLAlchemy side)
+            if not dim and hasattr(fact_model, "__table__"):
+                column = fact_model.__table__.columns.get(name)
+                if column is not None and column.info:
+                    dim = column.info.get("dimension")
+
+            if dim:
+                relationships[name] = dim
+        return {
+            "fact": fact_model,
+            "dimensions": relationships
+        }
+
+    def get_role_playing_dimensions(self, fact_model: Type[FactModel]) -> List[RolePlayingRef]:
+        """
+        Returns all role-playing dimension references in a fact table.
+        A role-playing dimension is when the same physical dimension is joined
+        multiple times under different logical roles (e.g. departure_date, arrival_date).
+        """
+        refs = []
+        if not hasattr(fact_model, "__table__"):
+            return refs
+        for col in fact_model.__table__.columns:
+            if col.info:
+                role = col.info.get("role")
+                dim = col.info.get("dimension")
+                if role and dim:
+                    refs.append(RolePlayingRef(
+                        dimension=dim,
+                        role=role,
+                        fk_column=col.name,
+                    ))
+        return refs
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return a JSON-serializable representation of the full schema graph."""
+        result: Dict[str, Any] = {"facts": [], "dimensions": []}
+
+        for dim in self.dimensions:
+            result["dimensions"].append({
+                "name": dim.__name__,
+                "natural_key": getattr(dim, "__natural_key__", []),
+                "scd_type": getattr(dim, "__scd_type__", 1),
+                "columns": list(dim.model_fields.keys()),
+            })
+
+        for fact in self.facts:
+            star = self.get_star_schema(fact)
+            roles = self.get_role_playing_dimensions(fact)
+            result["facts"].append({
+                "name": fact.__name__,
+                "grain": getattr(fact, "__grain__", None),
+                "dimensions": {
+                    fk: dim.__name__ for fk, dim in star["dimensions"].items()
+                },
+                "role_playing": [
+                    {"fk": r.fk_column, "dimension": r.dimension.__name__, "role": r.role}
+                    for r in roles
+                ],
+                "columns": list(fact.model_fields.keys()),
+            })
+
+        return result
+
+    def to_mermaid(self) -> str:
+        """Render the schema as a Mermaid ER diagram string."""
+        lines = ["erDiagram"]
+
+        for dim in self.dimensions:
+            lines.append(f"    {dim.__name__} {{")
+            for col_name, field in dim.model_fields.items():
+                col_type = "string"
+                annotation = dim.__annotations__.get(col_name)
+                if annotation in (int,):
+                    col_type = "int"
+                elif annotation in (float,):
+                    col_type = "float"
+                elif annotation in (bool,):
+                    col_type = "bool"
+                lines.append(f"        {col_type} {col_name}")
+            lines.append("    }")
+
+        for fact in self.facts:
+            lines.append(f"    {fact.__name__} {{")
+            for col_name, field in fact.model_fields.items():
+                col_type = "string"
+                annotation = fact.__annotations__.get(col_name)
+                if annotation in (int,):
+                    col_type = "int"
+                elif annotation in (float,):
+                    col_type = "float"
+                lines.append(f"        {col_type} {col_name}")
+            lines.append("    }")
+
+        # Relationships
+        for fact in self.facts:
+            star = self.get_star_schema(fact)
+            for fk_col, dim in star["dimensions"].items():
+                lines.append(f"    {fact.__name__} }}o--||  {dim.__name__} : \"{fk_col}\"")
+
+        return "\n".join(lines)

sqldim/core/mixins.py

@@ -0,0 +1,73 @@
+from datetime import date, datetime, timezone
+from typing import Optional, Any
+from sqlmodel import SQLModel
+from sqldim.core.fields import Field
+
+class SCD2Mixin(SQLModel):
+    """
+    Base mixin for SCD Type 2 tracking.
+    Default period columns are datetimes. For non-datetime periods (like integer seasons),
+    override valid_from/valid_to in the subclass.
+    """
+    valid_from: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    valid_to: Optional[datetime] = Field(default=None, index=True, nullable=True)
+    is_current: bool = Field(default=True, index=True)
+    checksum: Optional[str] = Field(default=None, index=True, nullable=True)
+
+class SCD3Mixin(SQLModel):
+    """Supports SCD Type 3 (Current + Previous value columns)."""
+    # Note: Users define the specific 'previous_X' columns in their model
+    # and we handle the rotation logic in SCDHandler.
+    pass
+
+class CumulativeMixin(SQLModel):
+    """
+    Mixin for Cumulative Dimensions/Facts (array-of-structs pattern).
+    Provides helpers for managing history arrays.
+    """
+    def current_value(self, column: str) -> Any:
+        """Returns the most recent element in the cumulative array."""
+        arr = getattr(self, column, [])
+        return arr[-1] if arr else None
+
+    def first_value(self, column: str) -> Any:
+        """Returns the first element in the cumulative array."""
+        arr = getattr(self, column, [])
+        return arr[0] if arr else None
+
+class DatelistMixin(SQLModel):
+    """
+    Mixin for tracking activity via dates lists.
+    Enables bitmask-based analysis (L7, L28).
+    """
+    def to_bitmask(self, reference_date: date, window: int = 32) -> int:
+        """Converts a list of active dates into an integer bitmask."""
+        # Implementation based on the provided theoretical requirement
+        mask = 0
+        dates = getattr(self, "dates_active", [])
+        if not dates:
+            return mask
+            
+        for active_date in dates:
+            # Handle both date objects and ISO strings (common in JSON serialization)
+            if isinstance(active_date, str):
+                active_date = date.fromisoformat(active_date)
+            diff = (reference_date - active_date).days
+            if 0 <= diff < window:
+                mask |= (1 << diff)
+        return mask
+
+    def activity_in_window(self, days: int, reference_date: date) -> bool:
+        """Returns True if any activity occurred within the last N days."""
+        mask = self.to_bitmask(reference_date, window=days)
+        return mask > 0
+
+    def l7(self, reference_date: date) -> int:
+        """Count of active days in the last 7 days."""
+        mask = self.to_bitmask(reference_date, window=7)
+        return bin(mask).count("1")
+
+    def l28(self, reference_date: date) -> int:
+        """Count of active days in the last 28 days."""
+        mask = self.to_bitmask(reference_date, window=28)
+        return bin(mask).count("1")

sqldim/core/models.py

@@ -0,0 +1,18 @@
+from typing import Optional, List, ClassVar
+from sqlmodel import SQLModel
+from sqldim.core.fields import Field
+
+class DimensionModel(SQLModel):
+    __natural_key__: ClassVar[List[str]] = []
+    __scd_type__: ClassVar[int] = 2
+
+class FactModel(SQLModel):
+    __grain__: ClassVar[Optional[str]] = None
+    __strategy__: ClassVar[Optional[str]] = None  # e.g. "bulk", "upsert", "merge", "accumulating"
+
+class BridgeModel(SQLModel):
+    """
+    Base class for Kimball Bridge Tables handling many-to-many relationships.
+    Includes weighting factors for multi-valued dimensions to prevent double-counting.
+    """
+    weight: float = Field(default=1.0, description="Allocation factor (0.0 to 1.0)")

sqldim/dimensions/date.py

@@ -0,0 +1,55 @@
+from datetime import date, timedelta
+from typing import Optional, List
+from sqlmodel import Session
+from sqldim import DimensionModel, Field
+
+class DateDimension(DimensionModel, table=True):
+    """Pre-built Date Dimension — generate a full date spine for any range."""
+    __natural_key__ = ["date_value"]
+
+    id: int = Field(primary_key=True, surrogate_key=True)
+    date_value: str = Field(natural_key=True, index=True, unique=True)  # YYYY-MM-DD
+    year: int
+    quarter: int
+    month: int
+    month_name: str
+    day_of_month: int
+    day_of_week: int        # 0=Monday, 6=Sunday
+    day_name: str
+    week_of_year: int
+    is_weekend: bool
+    is_leap_year: bool
+
+    @classmethod
+    def _from_date(cls, d: date) -> "DateDimension":
+        return cls(
+            date_value=d.isoformat(),
+            year=d.year,
+            quarter=(d.month - 1) // 3 + 1,
+            month=d.month,
+            month_name=d.strftime("%B"),
+            day_of_month=d.day,
+            day_of_week=d.weekday(),
+            day_name=d.strftime("%A"),
+            week_of_year=int(d.strftime("%W")),
+            is_weekend=d.weekday() >= 5,
+            is_leap_year=d.year % 4 == 0 and (d.year % 100 != 0 or d.year % 400 == 0),
+        )
+
+    @classmethod
+    def generate(cls, start: str, end: str, session: Session) -> List["DateDimension"]:
+        """
+        Generate and persist all DateDimension rows for [start, end] inclusive.
+        Skips dates already present (idempotent).
+        """
+        start_date = date.fromisoformat(start)
+        end_date = date.fromisoformat(end)
+        rows = []
+        current = start_date
+        while current <= end_date:
+            row = cls._from_date(current)
+            session.add(row)
+            rows.append(row)
+            current += timedelta(days=1)
+        session.commit()
+        return rows

sqldim/dimensions/junk.py

@@ -0,0 +1,60 @@
+from typing import Any, Dict, List, Optional, Tuple, Type
+from itertools import product
+from sqlmodel import Session, select
+from sqldim import DimensionModel, Field
+
+
+def make_junk_dimension(
+    name: str,
+    flags: Dict[str, List[Any]],
+) -> Type[DimensionModel]:
+    """
+    Dynamically creates a JunkDimension model class for a given set of
+    low-cardinality flag columns and their possible values.
+
+    Usage:
+        SalesFlags = make_junk_dimension("SalesFlags", {
+            "is_promo": [True, False],
+            "is_return": [True, False],
+            "channel": ["web", "store", "phone"],
+        })
+
+    The resulting model has one row per unique combination of flag values.
+    """
+    annotations: Dict[str, Any] = {
+        "id": int,
+        **{col: Optional[Any] for col in flags},
+    }
+    fields: Dict[str, Any] = {
+        "id": Field(primary_key=True, surrogate_key=True),
+        **{col: Field(default=None, nullable=True) for col in flags},
+        "__annotations__": annotations,
+        "__natural_key__": list(flags.keys()),
+    }
+    model_cls = type(name, (DimensionModel,), {**fields, "__tablename__": name.lower()})
+    return model_cls  # type: ignore
+
+
+def populate_junk_dimension(
+    model: Type[DimensionModel],
+    flags: Dict[str, List[Any]],
+    session: Session,
+) -> List[Any]:
+    """
+    Populate a junk dimension with all combinations of flag values (idempotent).
+    Returns list of inserted rows.
+    """
+    cols = list(flags.keys())
+    combos = list(product(*[flags[c] for c in cols]))
+    rows = []
+    for combo in combos:
+        row_data = dict(zip(cols, combo))
+        # Check if already exists
+        conditions = [model.__table__.c[col] == val for col, val in row_data.items()]
+        existing = session.exec(select(model).where(*conditions)).first()
+        if not existing:
+            row = model(**row_data)
+            session.add(row)
+            rows.append(row)
+    session.commit()
+    return rows

sqldim/dimensions/time.py

@@ -0,0 +1,53 @@
+from datetime import time
+from typing import List
+from sqlmodel import Session
+from sqldim import DimensionModel, Field
+
+class TimeDimension(DimensionModel, table=True):
+    """
+    Pre-built Time-of-Day Dimension for intra-day analysis.
+    Granularity: one row per minute (1440 rows total).
+    """
+    __natural_key__ = ["time_value"]
+
+    id: int = Field(primary_key=True, surrogate_key=True)
+    time_value: str = Field(natural_key=True, index=True, unique=True)  # HH:MM
+    hour: int
+    minute: int
+    period: str   # AM | PM
+    hour_12: int
+    time_of_day: str  # Morning | Afternoon | Evening | Night
+
+    @classmethod
+    def _from_time(cls, h: int, m: int) -> "TimeDimension":
+        t = time(h, m)
+        period = "AM" if h < 12 else "PM"
+        hour_12 = h % 12 or 12
+        if 5 <= h < 12:
+            tod = "Morning"
+        elif 12 <= h < 17:
+            tod = "Afternoon"
+        elif 17 <= h < 21:
+            tod = "Evening"
+        else:
+            tod = "Night"
+        return cls(
+            time_value=t.strftime("%H:%M"),
+            hour=h,
+            minute=m,
+            period=period,
+            hour_12=hour_12,
+            time_of_day=tod,
+        )
+
+    @classmethod
+    def generate(cls, session: Session) -> List["TimeDimension"]:
+        """Generate all 1440 minute-level time rows (idempotent)."""
+        rows = []
+        for h in range(24):
+            for m in range(60):
+                row = cls._from_time(h, m)
+                session.add(row)
+                rows.append(row)
+        session.commit()
+        return rows