sqldim 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqldim-0.1.0/PKG-INFO +60 -0
- sqldim-0.1.0/README.md +49 -0
- sqldim-0.1.0/pyproject.toml +24 -0
- sqldim-0.1.0/setup.cfg +4 -0
- sqldim-0.1.0/sqldim/__init__.py +86 -0
- sqldim-0.1.0/sqldim/cli.py +82 -0
- sqldim-0.1.0/sqldim/config.py +22 -0
- sqldim-0.1.0/sqldim/core/__init__.py +0 -0
- sqldim-0.1.0/sqldim/core/facts.py +26 -0
- sqldim-0.1.0/sqldim/core/fields.py +54 -0
- sqldim-0.1.0/sqldim/core/graph.py +135 -0
- sqldim-0.1.0/sqldim/core/mixins.py +73 -0
- sqldim-0.1.0/sqldim/core/models.py +18 -0
- sqldim-0.1.0/sqldim/dimensions/__init__.py +0 -0
- sqldim-0.1.0/sqldim/dimensions/date.py +55 -0
- sqldim-0.1.0/sqldim/dimensions/junk.py +60 -0
- sqldim-0.1.0/sqldim/dimensions/time.py +53 -0
- sqldim-0.1.0/sqldim/exceptions.py +61 -0
- sqldim-0.1.0/sqldim/graph/__init__.py +4 -0
- sqldim-0.1.0/sqldim/graph/registry.py +326 -0
- sqldim-0.1.0/sqldim/graph/schema_graph.py +252 -0
- sqldim-0.1.0/sqldim/graph/traversal.py +168 -0
- sqldim-0.1.0/sqldim/loaders/__init__.py +0 -0
- sqldim-0.1.0/sqldim/loaders/accumulating.py +78 -0
- sqldim-0.1.0/sqldim/loaders/array_metric.py +53 -0
- sqldim-0.1.0/sqldim/loaders/bitmask.py +53 -0
- sqldim-0.1.0/sqldim/loaders/cumulative.py +102 -0
- sqldim-0.1.0/sqldim/loaders/dimensional.py +110 -0
- sqldim-0.1.0/sqldim/loaders/edge_projection.py +47 -0
- sqldim-0.1.0/sqldim/loaders/resolution.py +81 -0
- sqldim-0.1.0/sqldim/loaders/snapshot.py +49 -0
- sqldim-0.1.0/sqldim/loaders/strategies.py +73 -0
- sqldim-0.1.0/sqldim/migrations/__init__.py +0 -0
- sqldim-0.1.0/sqldim/migrations/context.py +103 -0
- sqldim-0.1.0/sqldim/migrations/generator.py +98 -0
- sqldim-0.1.0/sqldim/migrations/guards.py +37 -0
- sqldim-0.1.0/sqldim/migrations/ops.py +37 -0
- sqldim-0.1.0/sqldim/models/__init__.py +3 -0
- sqldim-0.1.0/sqldim/models/graph.py +119 -0
- sqldim-0.1.0/sqldim/narwhals/__init__.py +16 -0
- sqldim-0.1.0/sqldim/narwhals/adapter.py +122 -0
- sqldim-0.1.0/sqldim/narwhals/backfill.py +100 -0
- sqldim-0.1.0/sqldim/narwhals/scd_engine.py +266 -0
- sqldim-0.1.0/sqldim/narwhals/sk_resolver.py +165 -0
- sqldim-0.1.0/sqldim/narwhals/transforms.py +224 -0
- sqldim-0.1.0/sqldim/query/__init__.py +0 -0
- sqldim-0.1.0/sqldim/query/builder.py +121 -0
- sqldim-0.1.0/sqldim/scd/__init__.py +0 -0
- sqldim-0.1.0/sqldim/scd/audit.py +54 -0
- sqldim-0.1.0/sqldim/scd/backfill.py +87 -0
- sqldim-0.1.0/sqldim/scd/detection.py +53 -0
- sqldim-0.1.0/sqldim/scd/handler.py +150 -0
- sqldim-0.1.0/sqldim/session.py +32 -0
- sqldim-0.1.0/sqldim.egg-info/PKG-INFO +60 -0
- sqldim-0.1.0/sqldim.egg-info/SOURCES.txt +80 -0
- sqldim-0.1.0/sqldim.egg-info/dependency_links.txt +1 -0
- sqldim-0.1.0/sqldim.egg-info/requires.txt +4 -0
- sqldim-0.1.0/sqldim.egg-info/top_level.txt +1 -0
- sqldim-0.1.0/tests/test_accumulating.py +72 -0
- sqldim-0.1.0/tests/test_cli.py +49 -0
- sqldim-0.1.0/tests/test_config.py +23 -0
- sqldim-0.1.0/tests/test_core.py +48 -0
- sqldim-0.1.0/tests/test_coverage_final.py +135 -0
- sqldim-0.1.0/tests/test_coverage_gap.py +169 -0
- sqldim-0.1.0/tests/test_coverage_gap_v2.py +116 -0
- sqldim-0.1.0/tests/test_dimensions.py +56 -0
- sqldim-0.1.0/tests/test_exceptions.py +32 -0
- sqldim-0.1.0/tests/test_graph_extended.py +79 -0
- sqldim-0.1.0/tests/test_hybrid_scd.py +93 -0
- sqldim-0.1.0/tests/test_junk_dimension.py +49 -0
- sqldim-0.1.0/tests/test_kimball_extended.py +79 -0
- sqldim-0.1.0/tests/test_loaders.py +126 -0
- sqldim-0.1.0/tests/test_migrations.py +205 -0
- sqldim-0.1.0/tests/test_nba_pipeline.py +72 -0
- sqldim-0.1.0/tests/test_query.py +116 -0
- sqldim-0.1.0/tests/test_resolution.py +75 -0
- sqldim-0.1.0/tests/test_scd.py +44 -0
- sqldim-0.1.0/tests/test_scd_detection.py +102 -0
- sqldim-0.1.0/tests/test_session.py +29 -0
- sqldim-0.1.0/tests/test_snapshot.py +77 -0
- sqldim-0.1.0/tests/test_strategies.py +62 -0
- sqldim-0.1.0/tests/test_time_dimension.py +50 -0
sqldim-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sqldim
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A Python toolkit for Dimensional Modeling (Kimball methodology) using SQLModel.
|
|
5
|
+
Requires-Python: >=3.13
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: sqlmodel>=0.0.34
|
|
8
|
+
Requires-Dist: pydantic-settings>=2.2.1
|
|
9
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0.29
|
|
10
|
+
Requires-Dist: alembic>=1.13.1
|
|
11
|
+
|
|
12
|
+
# sqldim
|
|
13
|
+
|
|
14
|
+
`sqldim` is a high-performance Python toolkit for **Unified Dimensional & Graph Modeling**. Built on top of `SQLModel`, it allows you to define complex Kimball-native structures that power both BI star schemas and network relationship traversals in a single, consistent model.
|
|
15
|
+
|
|
16
|
+
## Core Pillars
|
|
17
|
+
|
|
18
|
+
- **🚀 Vectorized ETL**: Powered by **Narwhals**. Handles millions of rows with single-pass joins using Polars, Pandas, or DuckDB backends.
|
|
19
|
+
- **🔄 Automated SCD**: Built-in support for Type 1, 2, 3, and 6. Vectorized change detection eliminates row-by-row Python loops.
|
|
20
|
+
- **🕸️ Dual-Paradigm Graph**: Project your dimensions as Vertices and facts as Edges. Traverse networks using Recursive CTEs without leaving your SQL database.
|
|
21
|
+
- **⏳ Semantic Time-Travel**: Fluent query builder with automatic `AS OF` join resolution for point-in-time analysis.
|
|
22
|
+
- **🧊 Open Table Ready**: Natively compatible with Apache Iceberg and Delta Lake via PyArrow/Narwhals integration.
|
|
23
|
+
|
|
24
|
+
## Key Features
|
|
25
|
+
|
|
26
|
+
- **Hybrid SCD**: Combine typed columns with versioned JSONB property bags for flexible, heterogeneous metadata.
|
|
27
|
+
- **Cumulative History**: Replicate the "Array-of-Structs" pattern for dense longitudinal data (e.g., player seasons).
|
|
28
|
+
- **Bitmask Activity**: Ultra-low footprint user retention tracking (L7/L28 metrics) encoded in 32-bit integers.
|
|
29
|
+
- **Topological Loading**: Automatic dependency resolution ensures dimensions load before facts.
|
|
30
|
+
|
|
31
|
+
## Quick Start
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
from sqldim import DimensionModel, SCD2Mixin, Field
|
|
35
|
+
|
|
36
|
+
class User(DimensionModel, SCD2Mixin, table=True):
|
|
37
|
+
__natural_key__ = ["email"]
|
|
38
|
+
email: str
|
|
39
|
+
plan_tier: str
|
|
40
|
+
|
|
41
|
+
# Loading is vectorized automatically if you pass a DataFrame
|
|
42
|
+
loader.register(User, source=polars_df)
|
|
43
|
+
await loader.run()
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Documentation
|
|
47
|
+
See the `/docs` folder for detailed guides on:
|
|
48
|
+
- [Vectorized ETL](./docs/guides/vectorized_etl.md)
|
|
49
|
+
- [Dual-Paradigm Graphing](./docs/guides/graph_extension.md)
|
|
50
|
+
- [Hybrid SCD Pattern](./docs/patterns/hybrid_scd.md)
|
|
51
|
+
- [Bitmask Retention](./docs/index.md)
|
|
52
|
+
|
|
53
|
+
## Examples
|
|
54
|
+
Check out our real-world showcases:
|
|
55
|
+
- `examples/nba_analytics/`: Longitudinal career analysis.
|
|
56
|
+
- `examples/user_activity/`: High-velocity retention tracking.
|
|
57
|
+
- `examples/saas_growth/`: Viral referral networks.
|
|
58
|
+
|
|
59
|
+
---
|
|
60
|
+
🌸 *Built with love by SylphAI*
|
sqldim-0.1.0/README.md
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# sqldim
|
|
2
|
+
|
|
3
|
+
`sqldim` is a high-performance Python toolkit for **Unified Dimensional & Graph Modeling**. Built on top of `SQLModel`, it allows you to define complex Kimball-native structures that power both BI star schemas and network relationship traversals in a single, consistent model.
|
|
4
|
+
|
|
5
|
+
## Core Pillars
|
|
6
|
+
|
|
7
|
+
- **🚀 Vectorized ETL**: Powered by **Narwhals**. Handles millions of rows with single-pass joins using Polars, Pandas, or DuckDB backends.
|
|
8
|
+
- **🔄 Automated SCD**: Built-in support for Type 1, 2, 3, and 6. Vectorized change detection eliminates row-by-row Python loops.
|
|
9
|
+
- **🕸️ Dual-Paradigm Graph**: Project your dimensions as Vertices and facts as Edges. Traverse networks using Recursive CTEs without leaving your SQL database.
|
|
10
|
+
- **⏳ Semantic Time-Travel**: Fluent query builder with automatic `AS OF` join resolution for point-in-time analysis.
|
|
11
|
+
- **🧊 Open Table Ready**: Natively compatible with Apache Iceberg and Delta Lake via PyArrow/Narwhals integration.
|
|
12
|
+
|
|
13
|
+
## Key Features
|
|
14
|
+
|
|
15
|
+
- **Hybrid SCD**: Combine typed columns with versioned JSONB property bags for flexible, heterogeneous metadata.
|
|
16
|
+
- **Cumulative History**: Replicate the "Array-of-Structs" pattern for dense longitudinal data (e.g., player seasons).
|
|
17
|
+
- **Bitmask Activity**: Ultra-low footprint user retention tracking (L7/L28 metrics) encoded in 32-bit integers.
|
|
18
|
+
- **Topological Loading**: Automatic dependency resolution ensures dimensions load before facts.
|
|
19
|
+
|
|
20
|
+
## Quick Start
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
from sqldim import DimensionModel, SCD2Mixin, Field
|
|
24
|
+
|
|
25
|
+
class User(DimensionModel, SCD2Mixin, table=True):
|
|
26
|
+
__natural_key__ = ["email"]
|
|
27
|
+
email: str
|
|
28
|
+
plan_tier: str
|
|
29
|
+
|
|
30
|
+
# Loading is vectorized automatically if you pass a DataFrame
|
|
31
|
+
loader.register(User, source=polars_df)
|
|
32
|
+
await loader.run()
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Documentation
|
|
36
|
+
See the `/docs` folder for detailed guides on:
|
|
37
|
+
- [Vectorized ETL](./docs/guides/vectorized_etl.md)
|
|
38
|
+
- [Dual-Paradigm Graphing](./docs/guides/graph_extension.md)
|
|
39
|
+
- [Hybrid SCD Pattern](./docs/patterns/hybrid_scd.md)
|
|
40
|
+
- [Bitmask Retention](./docs/index.md)
|
|
41
|
+
|
|
42
|
+
## Examples
|
|
43
|
+
Check out our real-world showcases:
|
|
44
|
+
- `examples/nba_analytics/`: Longitudinal career analysis.
|
|
45
|
+
- `examples/user_activity/`: High-velocity retention tracking.
|
|
46
|
+
- `examples/saas_growth/`: Viral referral networks.
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
🌸 *Built with love by SylphAI*
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "sqldim"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "A Python toolkit for Dimensional Modeling (Kimball methodology) using SQLModel."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.13"
|
|
7
|
+
dependencies = [
|
|
8
|
+
"sqlmodel>=0.0.34",
|
|
9
|
+
"pydantic-settings>=2.2.1",
|
|
10
|
+
"sqlalchemy[asyncio]>=2.0.29",
|
|
11
|
+
"alembic>=1.13.1",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
[tool.pytest.ini_options]
|
|
15
|
+
asyncio_mode = "auto"
|
|
16
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
17
|
+
|
|
18
|
+
[dependency-groups]
|
|
19
|
+
dev = [
|
|
20
|
+
"pytest>=9.0.2",
|
|
21
|
+
"pytest-asyncio>=1.3.0",
|
|
22
|
+
"pytest-cov>=7.0.0",
|
|
23
|
+
]
|
|
24
|
+
|
sqldim-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
from sqldim.core.fields import Field
|
|
2
|
+
from sqldim.core.models import DimensionModel, FactModel, BridgeModel
|
|
3
|
+
from sqldim.core.mixins import SCD2Mixin, SCD3Mixin, DatelistMixin, CumulativeMixin
|
|
4
|
+
from sqldim.core.facts import TransactionFact, PeriodicSnapshotFact, AccumulatingFact, CumulativeFact, ActivityFact
|
|
5
|
+
from sqldim.scd.backfill import backfill_scd2
|
|
6
|
+
from sqldim.core.graph import SchemaGraph
|
|
7
|
+
from sqldim.models.graph import VertexModel, EdgeModel, Vertex
|
|
8
|
+
from sqldim.graph import GraphModel, TraversalEngine
|
|
9
|
+
from sqldim.graph.schema_graph import SchemaGraph as GraphSchemaGraph, GraphSchema
|
|
10
|
+
from sqldim.narwhals.adapter import NarwhalsAdapter
|
|
11
|
+
from sqldim.narwhals.transforms import col, TransformPipeline
|
|
12
|
+
from sqldim.narwhals.backfill import backfill_scd2_narwhals
|
|
13
|
+
from sqldim.narwhals.scd_engine import NarwhalsHashStrategy, NarwhalsSCDProcessor
|
|
14
|
+
from sqldim.loaders.cumulative import CumulativeLoader
|
|
15
|
+
from sqldim.loaders.bitmask import BitmaskerLoader
|
|
16
|
+
from sqldim.loaders.array_metric import ArrayMetricLoader
|
|
17
|
+
from sqldim.loaders.edge_projection import EdgeProjectionLoader
|
|
18
|
+
from sqldim.exceptions import TransformTypeError
|
|
19
|
+
from sqldim.scd.handler import SCDHandler, SCDResult
|
|
20
|
+
from sqldim.loaders.dimensional import DimensionalLoader, SKResolver
|
|
21
|
+
from sqldim.session import AsyncDimensionalSession
|
|
22
|
+
from sqldim.config import SqldimConfig
|
|
23
|
+
from sqldim.exceptions import (
|
|
24
|
+
SqldimError,
|
|
25
|
+
SchemaError, GrainViolationError, NaturalKeyError,
|
|
26
|
+
SCDError, DestructiveOperationError,
|
|
27
|
+
LoadError, SKResolutionError, IdempotencyError,
|
|
28
|
+
MigrationError, DestructiveMigrationError,
|
|
29
|
+
SemanticError, InvalidJoinError,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
# Core schema
|
|
34
|
+
"Field",
|
|
35
|
+
"DimensionModel",
|
|
36
|
+
"FactModel",
|
|
37
|
+
"BridgeModel",
|
|
38
|
+
"TransactionFact",
|
|
39
|
+
"PeriodicSnapshotFact",
|
|
40
|
+
"AccumulatingFact",
|
|
41
|
+
"CumulativeFact",
|
|
42
|
+
"ActivityFact",
|
|
43
|
+
"DatelistMixin",
|
|
44
|
+
"CumulativeMixin",
|
|
45
|
+
"backfill_scd2",
|
|
46
|
+
"SCD2Mixin",
|
|
47
|
+
"SCD3Mixin",
|
|
48
|
+
"SchemaGraph",
|
|
49
|
+
# Graph extension (Phase 6)
|
|
50
|
+
"VertexModel",
|
|
51
|
+
"EdgeModel",
|
|
52
|
+
"Vertex",
|
|
53
|
+
"GraphModel",
|
|
54
|
+
"TraversalEngine",
|
|
55
|
+
"GraphSchemaGraph",
|
|
56
|
+
"GraphSchema",
|
|
57
|
+
# Narwhals integration (Phase 7)
|
|
58
|
+
"NarwhalsAdapter",
|
|
59
|
+
"col",
|
|
60
|
+
"TransformPipeline",
|
|
61
|
+
"backfill_scd2_narwhals",
|
|
62
|
+
"NarwhalsHashStrategy",
|
|
63
|
+
"NarwhalsSCDProcessor",
|
|
64
|
+
"CumulativeLoader",
|
|
65
|
+
"BitmaskerLoader",
|
|
66
|
+
"ArrayMetricLoader",
|
|
67
|
+
"EdgeProjectionLoader",
|
|
68
|
+
"TransformTypeError",
|
|
69
|
+
# SCD
|
|
70
|
+
"SCDHandler",
|
|
71
|
+
"SCDResult",
|
|
72
|
+
# Loaders
|
|
73
|
+
"DimensionalLoader",
|
|
74
|
+
"SKResolver",
|
|
75
|
+
# Session
|
|
76
|
+
"AsyncDimensionalSession",
|
|
77
|
+
# Config
|
|
78
|
+
"SqldimConfig",
|
|
79
|
+
# Exceptions
|
|
80
|
+
"SqldimError",
|
|
81
|
+
"SchemaError", "GrainViolationError", "NaturalKeyError",
|
|
82
|
+
"SCDError", "DestructiveOperationError",
|
|
83
|
+
"LoadError", "SKResolutionError", "IdempotencyError",
|
|
84
|
+
"MigrationError", "DestructiveMigrationError",
|
|
85
|
+
"SemanticError", "InvalidJoinError",
|
|
86
|
+
]
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""
|
|
2
|
+
sqldim CLI — dimensional migration management.
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
sqldim migrations generate "message"
|
|
6
|
+
sqldim migrations show
|
|
7
|
+
"""
|
|
8
|
+
import argparse
|
|
9
|
+
import json
|
|
10
|
+
import sys
|
|
11
|
+
from typing import List
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def cmd_migrations_generate(args: argparse.Namespace) -> None:
|
|
15
|
+
"""Generate a migration from a schema diff."""
|
|
16
|
+
print(f"[sqldim] Generating migration: '{args.message}'")
|
|
17
|
+
print("[sqldim] Tip: call generate_migration(models, current_state, message) in your project.")
|
|
18
|
+
|
|
19
|
+
def cmd_migrations_show(args: argparse.Namespace) -> None:
|
|
20
|
+
"""Show pending migration info."""
|
|
21
|
+
print("[sqldim] No pending migrations detected (run generate first).")
|
|
22
|
+
|
|
23
|
+
def cmd_migrations_init(args: argparse.Namespace) -> None:
|
|
24
|
+
"""Initialize sqldim migration environment."""
|
|
25
|
+
import os
|
|
26
|
+
migration_dir = getattr(args, "dir", "migrations")
|
|
27
|
+
os.makedirs(migration_dir, exist_ok=True)
|
|
28
|
+
init_file = os.path.join(migration_dir, "__init__.py")
|
|
29
|
+
if not os.path.exists(init_file):
|
|
30
|
+
open(init_file, "w").close()
|
|
31
|
+
print(f"[sqldim] Migration directory initialized at '{migration_dir}/'")
|
|
32
|
+
|
|
33
|
+
def cmd_schema_graph(args: argparse.Namespace) -> None:
|
|
34
|
+
"""Print schema graph as JSON or Mermaid."""
|
|
35
|
+
print("[sqldim] Pass your SchemaGraph instance to .to_dict() or .to_mermaid()")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
39
|
+
parser = argparse.ArgumentParser(
|
|
40
|
+
prog="sqldim",
|
|
41
|
+
description="sqldim — Dimensional Modeling toolkit for Python",
|
|
42
|
+
)
|
|
43
|
+
subparsers = parser.add_subparsers(dest="command")
|
|
44
|
+
|
|
45
|
+
# migrations subcommand
|
|
46
|
+
mig = subparsers.add_parser("migrations", help="Migration management")
|
|
47
|
+
mig_sub = mig.add_subparsers(dest="subcommand")
|
|
48
|
+
|
|
49
|
+
gen = mig_sub.add_parser("generate", help="Generate a migration from model diff")
|
|
50
|
+
gen.add_argument("message", help="Migration message")
|
|
51
|
+
gen.set_defaults(func=cmd_migrations_generate)
|
|
52
|
+
|
|
53
|
+
show = mig_sub.add_parser("show", help="Show pending migrations")
|
|
54
|
+
show.set_defaults(func=cmd_migrations_show)
|
|
55
|
+
|
|
56
|
+
init = mig_sub.add_parser("init", help="Initialize migration directory")
|
|
57
|
+
init.add_argument("--dir", default="migrations", help="Migration directory path")
|
|
58
|
+
init.set_defaults(func=cmd_migrations_init)
|
|
59
|
+
|
|
60
|
+
# schema subcommand
|
|
61
|
+
schema = subparsers.add_parser("schema", help="Schema introspection")
|
|
62
|
+
schema_sub = schema.add_subparsers(dest="subcommand")
|
|
63
|
+
graph = schema_sub.add_parser("graph", help="Print schema graph")
|
|
64
|
+
graph.set_defaults(func=cmd_schema_graph)
|
|
65
|
+
|
|
66
|
+
return parser
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def main(argv: List[str] | None = None) -> int:
|
|
70
|
+
parser = build_parser()
|
|
71
|
+
args = parser.parse_args(argv)
|
|
72
|
+
|
|
73
|
+
if not hasattr(args, "func"):
|
|
74
|
+
parser.print_help()
|
|
75
|
+
return 1
|
|
76
|
+
|
|
77
|
+
args.func(args)
|
|
78
|
+
return 0
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
if __name__ == "__main__": # pragma: no cover
|
|
82
|
+
sys.exit(main())
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
from typing import Literal
|
|
3
|
+
|
|
4
|
+
from pydantic_settings import BaseSettings
|
|
5
|
+
|
|
6
|
+
class SqldimConfig(BaseSettings):
|
|
7
|
+
# SCD defaults
|
|
8
|
+
scd_default_type: int = 2
|
|
9
|
+
scd_epoch: datetime = datetime(1970, 1, 1)
|
|
10
|
+
scd_infinity: datetime = datetime(9999, 12, 31)
|
|
11
|
+
checksum_algorithm: Literal["md5", "sha1", "sha256"] = "md5"
|
|
12
|
+
|
|
13
|
+
# Load defaults
|
|
14
|
+
default_batch_size: int = 10_000
|
|
15
|
+
sk_lookup_cache: bool = True
|
|
16
|
+
sk_lookup_cache_ttl: int = 3600 # seconds
|
|
17
|
+
|
|
18
|
+
# Migration defaults
|
|
19
|
+
allow_destructive: bool = False
|
|
20
|
+
migration_dir: str = "migrations"
|
|
21
|
+
|
|
22
|
+
model_config = {"env_prefix": "SQLDIM_"}
|
|
File without changes
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
from sqlmodel import SQLModel
|
|
3
|
+
from typing import ClassVar, List, Optional
|
|
4
|
+
from sqldim.core.models import FactModel
|
|
5
|
+
|
|
6
|
+
class FactType(str, Enum):
|
|
7
|
+
TRANSACTION = "transaction"
|
|
8
|
+
PERIODIC_SNAPSHOT = "periodic_snapshot"
|
|
9
|
+
ACCUMULATING_SNAPSHOT = "accumulating_snapshot"
|
|
10
|
+
CUMULATIVE = "cumulative"
|
|
11
|
+
ACTIVITY = "activity"
|
|
12
|
+
|
|
13
|
+
class TransactionFact(FactModel):
|
|
14
|
+
__fact_type__ = FactType.TRANSACTION
|
|
15
|
+
|
|
16
|
+
class PeriodicSnapshotFact(FactModel):
|
|
17
|
+
__fact_type__ = FactType.PERIODIC_SNAPSHOT
|
|
18
|
+
|
|
19
|
+
class AccumulatingFact(FactModel):
|
|
20
|
+
__fact_type__ = FactType.ACCUMULATING_SNAPSHOT
|
|
21
|
+
|
|
22
|
+
class CumulativeFact(FactModel):
|
|
23
|
+
__fact_type__ = FactType.CUMULATIVE
|
|
24
|
+
|
|
25
|
+
class ActivityFact(FactModel):
|
|
26
|
+
__fact_type__ = FactType.ACTIVITY
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
from typing import Any, Optional, Type
|
|
2
|
+
from sqlmodel import Field as SQLModelField
|
|
3
|
+
|
|
4
|
+
def Field(
|
|
5
|
+
default: Any = ...,
|
|
6
|
+
*,
|
|
7
|
+
default_factory: Optional[Any] = None,
|
|
8
|
+
primary_key: bool = False,
|
|
9
|
+
foreign_key: Optional[str] = None,
|
|
10
|
+
index: bool = False,
|
|
11
|
+
nullable: Optional[bool] = None,
|
|
12
|
+
# Dimensional metadata
|
|
13
|
+
surrogate_key: bool = False,
|
|
14
|
+
natural_key: bool = False,
|
|
15
|
+
measure: bool = False,
|
|
16
|
+
additive: bool = True,
|
|
17
|
+
dimension: Optional[Type] = None,
|
|
18
|
+
role: Optional[str] = None,
|
|
19
|
+
scd: Optional[int] = None,
|
|
20
|
+
previous_column: Optional[str] = None,
|
|
21
|
+
**kwargs: Any,
|
|
22
|
+
) -> Any:
|
|
23
|
+
"""
|
|
24
|
+
Extends SQLModel's Field with dimensional metadata stored in sa_column_kwargs
|
|
25
|
+
for compatibility across SQLModel/SQLAlchemy versions.
|
|
26
|
+
"""
|
|
27
|
+
# Collect dimensional metadata
|
|
28
|
+
dim_meta = {
|
|
29
|
+
"surrogate_key": surrogate_key,
|
|
30
|
+
"natural_key": natural_key,
|
|
31
|
+
"measure": measure,
|
|
32
|
+
"additive": additive,
|
|
33
|
+
"dimension": dimension,
|
|
34
|
+
"role": role,
|
|
35
|
+
"scd": scd,
|
|
36
|
+
"previous_column": previous_column,
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
# Store in sa_column_kwargs so it persists to the SQLAlchemy Column info
|
|
40
|
+
sa_column_kwargs = kwargs.get("sa_column_kwargs", {})
|
|
41
|
+
if "info" not in sa_column_kwargs:
|
|
42
|
+
sa_column_kwargs["info"] = {}
|
|
43
|
+
sa_column_kwargs["info"].update(dim_meta)
|
|
44
|
+
kwargs["sa_column_kwargs"] = sa_column_kwargs
|
|
45
|
+
|
|
46
|
+
return SQLModelField(
|
|
47
|
+
default,
|
|
48
|
+
default_factory=default_factory,
|
|
49
|
+
primary_key=primary_key,
|
|
50
|
+
foreign_key=foreign_key,
|
|
51
|
+
index=index,
|
|
52
|
+
nullable=nullable,
|
|
53
|
+
**kwargs,
|
|
54
|
+
)
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
from typing import List, Type, Dict, Any, Optional
|
|
2
|
+
from sqldim.core.models import DimensionModel, FactModel
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class RolePlayingRef:
|
|
6
|
+
"""Represents a dimension playing a named role in a fact table."""
|
|
7
|
+
def __init__(self, dimension: Type[DimensionModel], role: str, fk_column: str):
|
|
8
|
+
self.dimension = dimension
|
|
9
|
+
self.role = role
|
|
10
|
+
self.fk_column = fk_column
|
|
11
|
+
|
|
12
|
+
def __repr__(self) -> str:
|
|
13
|
+
return f"RolePlayingRef(dim={self.dimension.__name__}, role={self.role!r}, fk={self.fk_column!r})"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class SchemaGraph:
|
|
17
|
+
def __init__(self, models: List[Type[Any]]):
|
|
18
|
+
self.models = models
|
|
19
|
+
self.dimensions = [m for m in models if issubclass(m, DimensionModel)]
|
|
20
|
+
self.facts = [m for m in models if issubclass(m, FactModel)]
|
|
21
|
+
|
|
22
|
+
@classmethod
|
|
23
|
+
def from_models(cls, models: List[Type[Any]]) -> "SchemaGraph":
|
|
24
|
+
return cls(models)
|
|
25
|
+
|
|
26
|
+
def get_star_schema(self, fact_model: Type[FactModel]) -> Dict[str, Any]:
|
|
27
|
+
relationships = {}
|
|
28
|
+
for name, field in fact_model.model_fields.items():
|
|
29
|
+
dim = None
|
|
30
|
+
|
|
31
|
+
# Check for dimension in field attributes (pydantic side)
|
|
32
|
+
if hasattr(field, "json_schema_extra") and field.json_schema_extra:
|
|
33
|
+
dim = field.json_schema_extra.get("dimension")
|
|
34
|
+
|
|
35
|
+
# Check for dimension in sa_column (SQLAlchemy side)
|
|
36
|
+
if not dim and hasattr(fact_model, "__table__"):
|
|
37
|
+
column = fact_model.__table__.columns.get(name)
|
|
38
|
+
if column is not None and column.info:
|
|
39
|
+
dim = column.info.get("dimension")
|
|
40
|
+
|
|
41
|
+
if dim:
|
|
42
|
+
relationships[name] = dim
|
|
43
|
+
return {
|
|
44
|
+
"fact": fact_model,
|
|
45
|
+
"dimensions": relationships
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
def get_role_playing_dimensions(self, fact_model: Type[FactModel]) -> List[RolePlayingRef]:
|
|
49
|
+
"""
|
|
50
|
+
Returns all role-playing dimension references in a fact table.
|
|
51
|
+
A role-playing dimension is when the same physical dimension is joined
|
|
52
|
+
multiple times under different logical roles (e.g. departure_date, arrival_date).
|
|
53
|
+
"""
|
|
54
|
+
refs = []
|
|
55
|
+
if not hasattr(fact_model, "__table__"):
|
|
56
|
+
return refs
|
|
57
|
+
for col in fact_model.__table__.columns:
|
|
58
|
+
if col.info:
|
|
59
|
+
role = col.info.get("role")
|
|
60
|
+
dim = col.info.get("dimension")
|
|
61
|
+
if role and dim:
|
|
62
|
+
refs.append(RolePlayingRef(
|
|
63
|
+
dimension=dim,
|
|
64
|
+
role=role,
|
|
65
|
+
fk_column=col.name,
|
|
66
|
+
))
|
|
67
|
+
return refs
|
|
68
|
+
|
|
69
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
70
|
+
"""Return a JSON-serializable representation of the full schema graph."""
|
|
71
|
+
result: Dict[str, Any] = {"facts": [], "dimensions": []}
|
|
72
|
+
|
|
73
|
+
for dim in self.dimensions:
|
|
74
|
+
result["dimensions"].append({
|
|
75
|
+
"name": dim.__name__,
|
|
76
|
+
"natural_key": getattr(dim, "__natural_key__", []),
|
|
77
|
+
"scd_type": getattr(dim, "__scd_type__", 1),
|
|
78
|
+
"columns": list(dim.model_fields.keys()),
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
for fact in self.facts:
|
|
82
|
+
star = self.get_star_schema(fact)
|
|
83
|
+
roles = self.get_role_playing_dimensions(fact)
|
|
84
|
+
result["facts"].append({
|
|
85
|
+
"name": fact.__name__,
|
|
86
|
+
"grain": getattr(fact, "__grain__", None),
|
|
87
|
+
"dimensions": {
|
|
88
|
+
fk: dim.__name__ for fk, dim in star["dimensions"].items()
|
|
89
|
+
},
|
|
90
|
+
"role_playing": [
|
|
91
|
+
{"fk": r.fk_column, "dimension": r.dimension.__name__, "role": r.role}
|
|
92
|
+
for r in roles
|
|
93
|
+
],
|
|
94
|
+
"columns": list(fact.model_fields.keys()),
|
|
95
|
+
})
|
|
96
|
+
|
|
97
|
+
return result
|
|
98
|
+
|
|
99
|
+
def to_mermaid(self) -> str:
|
|
100
|
+
"""Render the schema as a Mermaid ER diagram string."""
|
|
101
|
+
lines = ["erDiagram"]
|
|
102
|
+
|
|
103
|
+
for dim in self.dimensions:
|
|
104
|
+
lines.append(f" {dim.__name__} {{")
|
|
105
|
+
for col_name, field in dim.model_fields.items():
|
|
106
|
+
col_type = "string"
|
|
107
|
+
annotation = dim.__annotations__.get(col_name)
|
|
108
|
+
if annotation in (int,):
|
|
109
|
+
col_type = "int"
|
|
110
|
+
elif annotation in (float,):
|
|
111
|
+
col_type = "float"
|
|
112
|
+
elif annotation in (bool,):
|
|
113
|
+
col_type = "bool"
|
|
114
|
+
lines.append(f" {col_type} {col_name}")
|
|
115
|
+
lines.append(" }")
|
|
116
|
+
|
|
117
|
+
for fact in self.facts:
|
|
118
|
+
lines.append(f" {fact.__name__} {{")
|
|
119
|
+
for col_name, field in fact.model_fields.items():
|
|
120
|
+
col_type = "string"
|
|
121
|
+
annotation = fact.__annotations__.get(col_name)
|
|
122
|
+
if annotation in (int,):
|
|
123
|
+
col_type = "int"
|
|
124
|
+
elif annotation in (float,):
|
|
125
|
+
col_type = "float"
|
|
126
|
+
lines.append(f" {col_type} {col_name}")
|
|
127
|
+
lines.append(" }")
|
|
128
|
+
|
|
129
|
+
# Relationships
|
|
130
|
+
for fact in self.facts:
|
|
131
|
+
star = self.get_star_schema(fact)
|
|
132
|
+
for fk_col, dim in star["dimensions"].items():
|
|
133
|
+
lines.append(f" {fact.__name__} }}o--|| {dim.__name__} : \"{fk_col}\"")
|
|
134
|
+
|
|
135
|
+
return "\n".join(lines)
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
from datetime import date, datetime, timezone
|
|
2
|
+
from typing import Optional, Any
|
|
3
|
+
from sqlmodel import SQLModel
|
|
4
|
+
from sqldim.core.fields import Field
|
|
5
|
+
|
|
6
|
+
class SCD2Mixin(SQLModel):
|
|
7
|
+
"""
|
|
8
|
+
Base mixin for SCD Type 2 tracking.
|
|
9
|
+
Default period columns are datetimes. For non-datetime periods (like integer seasons),
|
|
10
|
+
override valid_from/valid_to in the subclass.
|
|
11
|
+
"""
|
|
12
|
+
valid_from: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
|
13
|
+
valid_to: Optional[datetime] = Field(default=None, index=True, nullable=True)
|
|
14
|
+
is_current: bool = Field(default=True, index=True)
|
|
15
|
+
checksum: Optional[str] = Field(default=None, index=True, nullable=True)
|
|
16
|
+
|
|
17
|
+
class SCD3Mixin(SQLModel):
|
|
18
|
+
"""Supports SCD Type 3 (Current + Previous value columns)."""
|
|
19
|
+
# Note: Users define the specific 'previous_X' columns in their model
|
|
20
|
+
# and we handle the rotation logic in SCDHandler.
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
class CumulativeMixin(SQLModel):
|
|
24
|
+
"""
|
|
25
|
+
Mixin for Cumulative Dimensions/Facts (array-of-structs pattern).
|
|
26
|
+
Provides helpers for managing history arrays.
|
|
27
|
+
"""
|
|
28
|
+
def current_value(self, column: str) -> Any:
|
|
29
|
+
"""Returns the most recent element in the cumulative array."""
|
|
30
|
+
arr = getattr(self, column, [])
|
|
31
|
+
return arr[-1] if arr else None
|
|
32
|
+
|
|
33
|
+
def first_value(self, column: str) -> Any:
|
|
34
|
+
"""Returns the first element in the cumulative array."""
|
|
35
|
+
arr = getattr(self, column, [])
|
|
36
|
+
return arr[0] if arr else None
|
|
37
|
+
|
|
38
|
+
class DatelistMixin(SQLModel):
|
|
39
|
+
"""
|
|
40
|
+
Mixin for tracking activity via dates lists.
|
|
41
|
+
Enables bitmask-based analysis (L7, L28).
|
|
42
|
+
"""
|
|
43
|
+
def to_bitmask(self, reference_date: date, window: int = 32) -> int:
|
|
44
|
+
"""Converts a list of active dates into an integer bitmask."""
|
|
45
|
+
# Implementation based on the provided theoretical requirement
|
|
46
|
+
mask = 0
|
|
47
|
+
dates = getattr(self, "dates_active", [])
|
|
48
|
+
if not dates:
|
|
49
|
+
return mask
|
|
50
|
+
|
|
51
|
+
for active_date in dates:
|
|
52
|
+
# Handle both date objects and ISO strings (common in JSON serialization)
|
|
53
|
+
if isinstance(active_date, str):
|
|
54
|
+
active_date = date.fromisoformat(active_date)
|
|
55
|
+
diff = (reference_date - active_date).days
|
|
56
|
+
if 0 <= diff < window:
|
|
57
|
+
mask |= (1 << diff)
|
|
58
|
+
return mask
|
|
59
|
+
|
|
60
|
+
def activity_in_window(self, days: int, reference_date: date) -> bool:
|
|
61
|
+
"""Returns True if any activity occurred within the last N days."""
|
|
62
|
+
mask = self.to_bitmask(reference_date, window=days)
|
|
63
|
+
return mask > 0
|
|
64
|
+
|
|
65
|
+
def l7(self, reference_date: date) -> int:
|
|
66
|
+
"""Count of active days in the last 7 days."""
|
|
67
|
+
mask = self.to_bitmask(reference_date, window=7)
|
|
68
|
+
return bin(mask).count("1")
|
|
69
|
+
|
|
70
|
+
def l28(self, reference_date: date) -> int:
|
|
71
|
+
"""Count of active days in the last 28 days."""
|
|
72
|
+
mask = self.to_bitmask(reference_date, window=28)
|
|
73
|
+
return bin(mask).count("1")
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from typing import Optional, List, ClassVar
|
|
2
|
+
from sqlmodel import SQLModel
|
|
3
|
+
from sqldim.core.fields import Field
|
|
4
|
+
|
|
5
|
+
class DimensionModel(SQLModel):
|
|
6
|
+
__natural_key__: ClassVar[List[str]] = []
|
|
7
|
+
__scd_type__: ClassVar[int] = 2
|
|
8
|
+
|
|
9
|
+
class FactModel(SQLModel):
|
|
10
|
+
__grain__: ClassVar[Optional[str]] = None
|
|
11
|
+
__strategy__: ClassVar[Optional[str]] = None # e.g. "bulk", "upsert", "merge", "accumulating"
|
|
12
|
+
|
|
13
|
+
class BridgeModel(SQLModel):
|
|
14
|
+
"""
|
|
15
|
+
Base class for Kimball Bridge Tables handling many-to-many relationships.
|
|
16
|
+
Includes weighting factors for multi-valued dimensions to prevent double-counting.
|
|
17
|
+
"""
|
|
18
|
+
weight: float = Field(default=1.0, description="Allocation factor (0.0 to 1.0)")
|
|
File without changes
|