model2data 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {model2data-0.2.1 → model2data-0.2.2}/PKG-INFO +1 -1
- model2data-0.2.2/model2data/dbt/__init__.py +1 -0
- model2data-0.2.2/model2data/dbt/project.py +74 -0
- model2data-0.2.2/model2data/dbt/tests.py +92 -0
- model2data-0.2.2/model2data/generate/__init__.py +1 -0
- model2data-0.2.2/model2data/generate/core.py +166 -0
- model2data-0.2.2/model2data/generate/faker.py +122 -0
- model2data-0.2.2/model2data/generate/relationships.py +52 -0
- model2data-0.2.2/model2data/parse/__init__.py +1 -0
- model2data-0.2.2/model2data/parse/dbml.py +218 -0
- {model2data-0.2.1 → model2data-0.2.2}/model2data.egg-info/PKG-INFO +1 -1
- {model2data-0.2.1 → model2data-0.2.2}/model2data.egg-info/SOURCES.txt +9 -0
- {model2data-0.2.1 → model2data-0.2.2}/pyproject.toml +3 -2
- {model2data-0.2.1 → model2data-0.2.2}/LICENSE +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/README.md +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/model2data/__init__.py +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/model2data/cli.py +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/model2data/utils.py +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/model2data.egg-info/dependency_links.txt +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/model2data.egg-info/entry_points.txt +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/model2data.egg-info/requires.txt +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/model2data.egg-info/top_level.txt +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/setup.cfg +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/tests/test_cli.py +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/tests/test_coverage_gaps.py +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/tests/test_dbml_parser.py +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/tests/test_dbt_naming.py +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/tests/test_dbt_project.py +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/tests/test_dbt_tests.py +0 -0
- {model2data-0.2.1 → model2data-0.2.2}/tests/test_generation.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""dbt project generation utilities."""
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import jinja2
|
|
4
|
+
|
|
5
|
+
TEMPLATES_DIR = Path(__file__).parent / "templates"
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def create_project_scaffold(dest: Path, project_name: str, profile_name: str) -> None:
|
|
9
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
10
|
+
|
|
11
|
+
# dbt folders
|
|
12
|
+
(dest / "models" / "staging").mkdir(parents=True, exist_ok=True)
|
|
13
|
+
(dest / "seeds" / "raw").mkdir(parents=True, exist_ok=True)
|
|
14
|
+
(dest / "analysis").mkdir(exist_ok=True)
|
|
15
|
+
(dest / "macros").mkdir(exist_ok=True)
|
|
16
|
+
(dest / "tests").mkdir(exist_ok=True)
|
|
17
|
+
(dest / "snapshots").mkdir(exist_ok=True)
|
|
18
|
+
|
|
19
|
+
# dbt_project.yml
|
|
20
|
+
_render_template(
|
|
21
|
+
template_name="dbt_project.yml.jinja",
|
|
22
|
+
output_path=dest / "dbt_project.yml",
|
|
23
|
+
context={"project_name": project_name, "profile_name": profile_name},
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
# Copy over any macros from templates
|
|
27
|
+
template_macros_dir = Path("model2data/dbt/templates/macros")
|
|
28
|
+
if template_macros_dir.exists():
|
|
29
|
+
for macro_file in template_macros_dir.glob("*.sql"):
|
|
30
|
+
target_file = dest / "macros" / macro_file.name
|
|
31
|
+
if not target_file.exists():
|
|
32
|
+
target_file.write_text(macro_file.read_text())
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def create_staging_models(dest: Path, project_name: str) -> None:
|
|
36
|
+
"""
|
|
37
|
+
Creates staging models in models/staging/ folder that reference raw seed tables as sources.
|
|
38
|
+
"""
|
|
39
|
+
seeds_path = dest / "seeds" / "raw"
|
|
40
|
+
models_path = dest / "models" / "staging"
|
|
41
|
+
models_path.mkdir(parents=True, exist_ok=True)
|
|
42
|
+
|
|
43
|
+
for csv_file in seeds_path.glob("*.csv"):
|
|
44
|
+
table_name = csv_file.stem # keep full seed name, e.g., raw_stories
|
|
45
|
+
model_file = models_path / f"stg_{table_name}.sql"
|
|
46
|
+
|
|
47
|
+
if not model_file.exists():
|
|
48
|
+
sql_content = f"""\
|
|
49
|
+
-- Auto-generated staging model for {table_name}
|
|
50
|
+
select *
|
|
51
|
+
from {{{{ source('raw', '{table_name}') }}}}
|
|
52
|
+
"""
|
|
53
|
+
model_file.write_text(sql_content)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def create_profiles_yml(dest: Path, profile_name: str) -> None:
|
|
57
|
+
profiles_file = dest / "profiles.yml"
|
|
58
|
+
if profiles_file.exists():
|
|
59
|
+
content = profiles_file.read_text()
|
|
60
|
+
if profile_name in content:
|
|
61
|
+
return
|
|
62
|
+
_render_template(
|
|
63
|
+
template_name="profiles.yml.jinja",
|
|
64
|
+
output_path=profiles_file,
|
|
65
|
+
context={"profile_name": profile_name},
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _render_template(template_name: str, output_path: Path, context: dict) -> None:
|
|
70
|
+
template_path = TEMPLATES_DIR / template_name
|
|
71
|
+
if not template_path.exists():
|
|
72
|
+
raise FileNotFoundError(f"Template not found: {template_path}")
|
|
73
|
+
template = jinja2.Template(template_path.read_text())
|
|
74
|
+
output_path.write_text(template.render(**context))
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
from collections import defaultdict
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Any, Union
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def generate_dbt_yml(dest: Path, tables: dict, refs: list[dict], source_name: str = "hackernews"):
|
|
7
|
+
"""
|
|
8
|
+
Generate:
|
|
9
|
+
1) __sources.yml with all raw_* seeds (no tests)
|
|
10
|
+
2) One .yml per staging model (stg_*) with tests
|
|
11
|
+
Table and column names are used exactly as in DBML.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
staging_path = dest / "models" / "staging"
|
|
15
|
+
staging_path.mkdir(parents=True, exist_ok=True)
|
|
16
|
+
|
|
17
|
+
# -------------------------
|
|
18
|
+
# Build foreign key map
|
|
19
|
+
# -------------------------
|
|
20
|
+
fk_map = defaultdict(list)
|
|
21
|
+
for ref in refs:
|
|
22
|
+
fk_map[(ref["source_table"], ref["source_column"])].append(ref)
|
|
23
|
+
|
|
24
|
+
# -------------------------
|
|
25
|
+
# Generate __sources.yml
|
|
26
|
+
# -------------------------
|
|
27
|
+
sources_lines = ["version: 2", "", "sources:"]
|
|
28
|
+
sources_lines.append(" - name: raw")
|
|
29
|
+
sources_lines.append(" schema: raw")
|
|
30
|
+
sources_lines.append(f" description: {source_name.capitalize()} raw seed data")
|
|
31
|
+
sources_lines.append(" tables:")
|
|
32
|
+
|
|
33
|
+
for table in tables.values():
|
|
34
|
+
seed_name = table.name # keep exact name
|
|
35
|
+
table_desc = getattr(table, "description", None) or f"Table {seed_name}"
|
|
36
|
+
sources_lines.append(f" - name: {seed_name}")
|
|
37
|
+
sources_lines.append(f" description: {table_desc}")
|
|
38
|
+
|
|
39
|
+
sources_file = staging_path / "__sources.yml"
|
|
40
|
+
sources_file.write_text("\n".join(sources_lines))
|
|
41
|
+
|
|
42
|
+
# -------------------------
|
|
43
|
+
# Generate individual staging model YAMLs
|
|
44
|
+
# -------------------------
|
|
45
|
+
for table in tables.values():
|
|
46
|
+
stg_name = f"stg_{table.name}" # staging model names are prefixed, columns unchanged
|
|
47
|
+
model_columns = []
|
|
48
|
+
|
|
49
|
+
for col in table.columns:
|
|
50
|
+
tests: list[Union[str, dict[str, dict[str, Any]]]] = []
|
|
51
|
+
settings = col.settings or set()
|
|
52
|
+
|
|
53
|
+
if "not null" in settings or "pk" in settings:
|
|
54
|
+
tests.append("not_null")
|
|
55
|
+
if "unique" in settings or "pk" in settings:
|
|
56
|
+
tests.append("unique")
|
|
57
|
+
|
|
58
|
+
fk_refs = fk_map.get((table.name, col.name), [])
|
|
59
|
+
for fk in fk_refs:
|
|
60
|
+
tests.append(
|
|
61
|
+
{
|
|
62
|
+
"relationships": {
|
|
63
|
+
"to": f"ref('stg_{fk['target_table']}')",
|
|
64
|
+
"field": fk["target_column"],
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
model_columns.append({"name": col.name, "tests": tests if tests else None})
|
|
70
|
+
|
|
71
|
+
# Render model YAML
|
|
72
|
+
lines = ["version: 2", "", "models:"]
|
|
73
|
+
lines.append(f" - name: {stg_name}")
|
|
74
|
+
lines.append(" columns:")
|
|
75
|
+
for col in model_columns:
|
|
76
|
+
lines.append(f" - name: {col['name']}")
|
|
77
|
+
if col["tests"]:
|
|
78
|
+
lines.append(" tests:")
|
|
79
|
+
for test in col["tests"]:
|
|
80
|
+
if isinstance(test, str):
|
|
81
|
+
lines.append(f" - {test}")
|
|
82
|
+
else:
|
|
83
|
+
# relationships test with arguments
|
|
84
|
+
for k, v in test.items():
|
|
85
|
+
lines.append(f" - {k}:")
|
|
86
|
+
lines.append(" arguments:")
|
|
87
|
+
for fk_key, fk_val in v.items():
|
|
88
|
+
lines.append(f" {fk_key}: {fk_val}")
|
|
89
|
+
|
|
90
|
+
# Write YAML to same folder as SQL model
|
|
91
|
+
yml_file = staging_path / f"{stg_name}.yml"
|
|
92
|
+
yml_file.write_text("\n".join(lines))
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Data generation utilities."""
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import random
|
|
4
|
+
from collections import defaultdict, deque
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import pandas as pd
|
|
8
|
+
from faker import Faker
|
|
9
|
+
|
|
10
|
+
from model2data.generate.faker import generate_column_values
|
|
11
|
+
from model2data.generate.relationships import (
|
|
12
|
+
build_fk_lookup,
|
|
13
|
+
classify_refs,
|
|
14
|
+
)
|
|
15
|
+
from model2data.parse.dbml import TableDef
|
|
16
|
+
|
|
17
|
+
fake = Faker()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# ---------------------------------------------------------
|
|
21
|
+
# Public API
|
|
22
|
+
# ---------------------------------------------------------
|
|
23
|
+
def generate_data_from_dbml(
|
|
24
|
+
tables: dict[str, TableDef],
|
|
25
|
+
refs: list[dict],
|
|
26
|
+
base_rows: int = 100,
|
|
27
|
+
seed: Optional[int] = None,
|
|
28
|
+
) -> dict[str, pd.DataFrame]:
|
|
29
|
+
"""
|
|
30
|
+
Generate synthetic datasets from parsed DBML definitions.
|
|
31
|
+
|
|
32
|
+
This function is deterministic if a seed is provided.
|
|
33
|
+
It performs no filesystem I/O and returns pandas DataFrames.
|
|
34
|
+
"""
|
|
35
|
+
if seed is not None:
|
|
36
|
+
random.seed(seed)
|
|
37
|
+
Faker.seed(seed)
|
|
38
|
+
|
|
39
|
+
# ---------------------------------------------------------
|
|
40
|
+
# Classify references
|
|
41
|
+
# ---------------------------------------------------------
|
|
42
|
+
fk_refs, attribute_refs = classify_refs(tables, refs)
|
|
43
|
+
fk_lookup = build_fk_lookup(fk_refs)
|
|
44
|
+
|
|
45
|
+
# ---------------------------------------------------------
|
|
46
|
+
# Generate tables in dependency order
|
|
47
|
+
# ---------------------------------------------------------
|
|
48
|
+
ordered_tables = _topological_table_order(tables, fk_refs)
|
|
49
|
+
generated: dict[str, pd.DataFrame] = {}
|
|
50
|
+
|
|
51
|
+
for table_name in ordered_tables:
|
|
52
|
+
table_def = tables[table_name]
|
|
53
|
+
row_count = _determine_row_count(table_def.name, base_rows)
|
|
54
|
+
|
|
55
|
+
data: dict[str, list] = {}
|
|
56
|
+
|
|
57
|
+
# -----------------------
|
|
58
|
+
# First pass: columns + FKs
|
|
59
|
+
# -----------------------
|
|
60
|
+
for column in table_def.columns:
|
|
61
|
+
fk_series = None
|
|
62
|
+
fk_target = fk_lookup.get((table_name, column.name))
|
|
63
|
+
|
|
64
|
+
if fk_target:
|
|
65
|
+
parent_table, parent_column = fk_target
|
|
66
|
+
parent_df = generated.get(parent_table)
|
|
67
|
+
if parent_df is not None and parent_column in parent_df.columns:
|
|
68
|
+
fk_series = parent_df[parent_column]
|
|
69
|
+
|
|
70
|
+
ensure_unique = "pk" in column.settings
|
|
71
|
+
data[column.name] = generate_column_values(
|
|
72
|
+
column=column,
|
|
73
|
+
row_count=row_count,
|
|
74
|
+
fk_series=fk_series,
|
|
75
|
+
ensure_unique=ensure_unique,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
df = pd.DataFrame(data)
|
|
79
|
+
|
|
80
|
+
# -----------------------------------------------------
|
|
81
|
+
# Second pass: attribute mirroring (non-FK refs)
|
|
82
|
+
# -----------------------------------------------------
|
|
83
|
+
for ref in attribute_refs:
|
|
84
|
+
if ref["source_table"] != table_name:
|
|
85
|
+
continue
|
|
86
|
+
|
|
87
|
+
parent_table = ref["target_table"]
|
|
88
|
+
parent_column = ref["target_column"]
|
|
89
|
+
child_column = ref["source_column"]
|
|
90
|
+
|
|
91
|
+
parent_df = generated.get(parent_table)
|
|
92
|
+
if parent_df is None:
|
|
93
|
+
continue
|
|
94
|
+
|
|
95
|
+
# find FK linking child → parent
|
|
96
|
+
fk_column = next(
|
|
97
|
+
(
|
|
98
|
+
r["source_column"]
|
|
99
|
+
for r in fk_refs
|
|
100
|
+
if r["source_table"] == table_name and r["target_table"] == parent_table
|
|
101
|
+
),
|
|
102
|
+
None,
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
if not fk_column or fk_column not in df.columns:
|
|
106
|
+
continue
|
|
107
|
+
|
|
108
|
+
lookup = parent_df.groupby("id")[parent_column].first().to_dict()
|
|
109
|
+
|
|
110
|
+
df[child_column] = df[fk_column].map(lookup)
|
|
111
|
+
|
|
112
|
+
generated[table_name] = df
|
|
113
|
+
|
|
114
|
+
return generated
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# ---------------------------------------------------------
|
|
118
|
+
# Internal helpers
|
|
119
|
+
# ---------------------------------------------------------
|
|
120
|
+
def _determine_row_count(table_name: str, base_rows: int) -> int:
|
|
121
|
+
"""
|
|
122
|
+
Return the base number of rows for all tables.
|
|
123
|
+
"""
|
|
124
|
+
return base_rows
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _topological_table_order(
|
|
128
|
+
tables: dict[str, TableDef],
|
|
129
|
+
fk_refs: list[dict],
|
|
130
|
+
) -> list[str]:
|
|
131
|
+
"""
|
|
132
|
+
Order tables so parent tables are generated before children.
|
|
133
|
+
"""
|
|
134
|
+
graph: dict[str, set[str]] = defaultdict(set)
|
|
135
|
+
indegree: dict[str, int] = dict.fromkeys(tables.keys(), 0)
|
|
136
|
+
|
|
137
|
+
for ref in fk_refs:
|
|
138
|
+
parent = ref["target_table"]
|
|
139
|
+
child = ref["source_table"]
|
|
140
|
+
|
|
141
|
+
if parent == child:
|
|
142
|
+
continue
|
|
143
|
+
if parent not in tables or child not in tables:
|
|
144
|
+
continue
|
|
145
|
+
|
|
146
|
+
if child not in graph[parent]:
|
|
147
|
+
graph[parent].add(child)
|
|
148
|
+
indegree[child] += 1
|
|
149
|
+
|
|
150
|
+
queue = deque(sorted(name for name, deg in indegree.items() if deg == 0))
|
|
151
|
+
order: list[str] = []
|
|
152
|
+
|
|
153
|
+
while queue:
|
|
154
|
+
node = queue.popleft()
|
|
155
|
+
order.append(node)
|
|
156
|
+
for neighbor in sorted(graph.get(node, [])):
|
|
157
|
+
indegree[neighbor] -= 1
|
|
158
|
+
if indegree[neighbor] == 0:
|
|
159
|
+
queue.append(neighbor)
|
|
160
|
+
|
|
161
|
+
# Safety net for disconnected tables
|
|
162
|
+
for name in tables.keys():
|
|
163
|
+
if name not in order:
|
|
164
|
+
order.append(name)
|
|
165
|
+
|
|
166
|
+
return order
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import random
|
|
4
|
+
import uuid
|
|
5
|
+
from datetime import datetime, timedelta
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
import pandas as pd
|
|
9
|
+
from faker import Faker
|
|
10
|
+
|
|
11
|
+
from model2data.parse.dbml import ColumnDef
|
|
12
|
+
|
|
13
|
+
fake = Faker()
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
# ---------------------------------------------------------
|
|
17
|
+
# Public API
|
|
18
|
+
# ---------------------------------------------------------
|
|
19
|
+
def generate_column_values(
|
|
20
|
+
column: ColumnDef,
|
|
21
|
+
row_count: int,
|
|
22
|
+
fk_series: Optional[pd.Series] = None,
|
|
23
|
+
ensure_unique: bool = False,
|
|
24
|
+
) -> list:
|
|
25
|
+
"""
|
|
26
|
+
Generate synthetic values for a single column.
|
|
27
|
+
Respects FKs, uniqueness, and optional min/max hints in column notes.
|
|
28
|
+
"""
|
|
29
|
+
if fk_series is not None and not fk_series.empty:
|
|
30
|
+
fk_values = fk_series.tolist()
|
|
31
|
+
return [random.choice(fk_values) for _ in range(row_count)]
|
|
32
|
+
|
|
33
|
+
dtype = column.data_type.lower()
|
|
34
|
+
base_type = dtype.split("(")[0].strip()
|
|
35
|
+
values: list = []
|
|
36
|
+
|
|
37
|
+
# Extract min/max from note if present
|
|
38
|
+
min_val = None
|
|
39
|
+
max_val = None
|
|
40
|
+
if column.note:
|
|
41
|
+
min_val = column.note.get("min")
|
|
42
|
+
max_val = column.note.get("max")
|
|
43
|
+
|
|
44
|
+
# -----------------------------------------------------
|
|
45
|
+
# UUIDs / hashes
|
|
46
|
+
# -----------------------------------------------------
|
|
47
|
+
if "uuid" in base_type or "hash" in base_type:
|
|
48
|
+
values = [str(uuid.uuid4()) for _ in range(row_count)]
|
|
49
|
+
|
|
50
|
+
# -----------------------------------------------------
|
|
51
|
+
# Integers
|
|
52
|
+
# -----------------------------------------------------
|
|
53
|
+
elif any(key in base_type for key in ["int", "integer", "bigint", "smallint"]):
|
|
54
|
+
# Use note values if present, otherwise defaults
|
|
55
|
+
if min_val is None:
|
|
56
|
+
min_val = 0
|
|
57
|
+
if max_val is None:
|
|
58
|
+
max_val = 100
|
|
59
|
+
values = [random.randint(min_val, max_val) for _ in range(row_count)]
|
|
60
|
+
|
|
61
|
+
# -----------------------------------------------------
|
|
62
|
+
# Floats / decimals
|
|
63
|
+
# -----------------------------------------------------
|
|
64
|
+
elif any(key in base_type for key in ["decimal", "numeric", "float", "double"]):
|
|
65
|
+
if min_val is None:
|
|
66
|
+
min_val = 0
|
|
67
|
+
if max_val is None:
|
|
68
|
+
max_val = 10_000
|
|
69
|
+
values = [round(random.uniform(min_val, max_val), 2) for _ in range(row_count)]
|
|
70
|
+
|
|
71
|
+
# -----------------------------------------------------
|
|
72
|
+
# Booleans
|
|
73
|
+
# -----------------------------------------------------
|
|
74
|
+
elif "boolean" in base_type or "bool" in base_type:
|
|
75
|
+
values = [random.choice([True, False]) for _ in range(row_count)]
|
|
76
|
+
|
|
77
|
+
# -----------------------------------------------------
|
|
78
|
+
# Dates
|
|
79
|
+
# -----------------------------------------------------
|
|
80
|
+
elif "date" in base_type and "time" not in base_type:
|
|
81
|
+
values = [fake.date_between(start_date="-2y", end_date="today") for _ in range(row_count)]
|
|
82
|
+
|
|
83
|
+
elif "time" in base_type and "stamp" not in base_type:
|
|
84
|
+
values = [fake.time() for _ in range(row_count)]
|
|
85
|
+
|
|
86
|
+
elif any(key in base_type for key in ["timestamp", "datetime"]):
|
|
87
|
+
values = [_random_datetime().isoformat(sep=" ") for _ in range(row_count)]
|
|
88
|
+
|
|
89
|
+
# -----------------------------------------------------
|
|
90
|
+
# Fallback to Faker providers
|
|
91
|
+
# -----------------------------------------------------
|
|
92
|
+
else:
|
|
93
|
+
try:
|
|
94
|
+
values = [fake.format(base_type) for _ in range(row_count)]
|
|
95
|
+
except (AttributeError, TypeError):
|
|
96
|
+
if column.name.lower().endswith("_id") or ensure_unique:
|
|
97
|
+
values = [str(uuid.uuid4()) for _ in range(row_count)]
|
|
98
|
+
else:
|
|
99
|
+
values = [fake.sentence(nb_words=3) for _ in range(row_count)]
|
|
100
|
+
|
|
101
|
+
# -----------------------------------------------------
|
|
102
|
+
# Nullability
|
|
103
|
+
# -----------------------------------------------------
|
|
104
|
+
if "not null" not in column.settings:
|
|
105
|
+
null_fraction = max(0, min(0.2, 1 - (row_count / (row_count + 50))))
|
|
106
|
+
sample_size = int(row_count * null_fraction)
|
|
107
|
+
if sample_size:
|
|
108
|
+
for idx in random.sample(range(row_count), k=sample_size):
|
|
109
|
+
values[idx] = None
|
|
110
|
+
|
|
111
|
+
return values
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# ---------------------------------------------------------
|
|
115
|
+
# Internal helpers
|
|
116
|
+
# ---------------------------------------------------------
|
|
117
|
+
def _random_datetime(start_days: int = -365, end_days: int = 0) -> datetime:
|
|
118
|
+
start = datetime.now() + timedelta(days=start_days)
|
|
119
|
+
end = datetime.now() + timedelta(days=end_days)
|
|
120
|
+
delta = end - start
|
|
121
|
+
random_second = random.randint(0, int(delta.total_seconds()))
|
|
122
|
+
return start + timedelta(seconds=random_second)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
from typing import Dict, List, Tuple
|
|
2
|
+
|
|
3
|
+
from model2data.parse.dbml import TableDef
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
# ---------------------------------------------------------
|
|
7
|
+
# Public API
|
|
8
|
+
# ---------------------------------------------------------
|
|
9
|
+
def classify_refs(
|
|
10
|
+
tables: Dict[str, TableDef],
|
|
11
|
+
refs: List[Dict],
|
|
12
|
+
) -> Tuple[List[Dict], List[Dict]]:
|
|
13
|
+
"""
|
|
14
|
+
Classify references into:
|
|
15
|
+
- fk_refs: Foreign keys (target column looks like a PK)
|
|
16
|
+
- attribute_refs: Non-FK dependencies (mirroring parent attributes)
|
|
17
|
+
"""
|
|
18
|
+
fk_refs = []
|
|
19
|
+
attribute_refs = []
|
|
20
|
+
|
|
21
|
+
for ref in refs:
|
|
22
|
+
target_table = tables.get(ref["target_table"])
|
|
23
|
+
target_col = None
|
|
24
|
+
if target_table:
|
|
25
|
+
target_col = next(
|
|
26
|
+
(c for c in target_table.columns if c.name == ref["target_column"]), None
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
# FK if target column is a primary key or named "id"
|
|
30
|
+
if target_col and ("pk" in target_col.settings or target_col.name.lower() == "id"):
|
|
31
|
+
fk_refs.append(ref)
|
|
32
|
+
else:
|
|
33
|
+
attribute_refs.append(ref)
|
|
34
|
+
|
|
35
|
+
return fk_refs, attribute_refs
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def build_fk_lookup(fk_refs: List[Dict]) -> Dict[Tuple[str, str], Tuple[str, str]]:
|
|
39
|
+
"""
|
|
40
|
+
Build a lookup dictionary for FK relationships.
|
|
41
|
+
|
|
42
|
+
Returns:
|
|
43
|
+
{
|
|
44
|
+
(child_table, child_column): (parent_table, parent_column)
|
|
45
|
+
}
|
|
46
|
+
"""
|
|
47
|
+
lookup: Dict[Tuple[str, str], Tuple[str, str]] = {}
|
|
48
|
+
for ref in fk_refs:
|
|
49
|
+
key = (ref["source_table"], ref["source_column"])
|
|
50
|
+
value = (ref["target_table"], ref["target_column"])
|
|
51
|
+
lookup[key] = value
|
|
52
|
+
return lookup
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""DBML parsing utilities."""
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Optional
|
|
8
|
+
|
|
9
|
+
# -------------------------------
|
|
10
|
+
# Dataclasses
|
|
11
|
+
# -------------------------------
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class ColumnDef:
|
|
16
|
+
name: str
|
|
17
|
+
data_type: str
|
|
18
|
+
settings: set[str] = field(default_factory=set)
|
|
19
|
+
note: Optional[dict] = None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class TableDef:
|
|
24
|
+
name: str
|
|
25
|
+
columns: list[ColumnDef] = field(default_factory=list)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# -------------------------------
|
|
29
|
+
# Helpers
|
|
30
|
+
# -------------------------------
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _strip_quotes(value: str) -> str:
|
|
34
|
+
return value.strip().strip('"').strip("'")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _parse_column_settings(raw: Optional[str]) -> tuple[set[str], Optional[dict]]:
|
|
38
|
+
"""Parse column settings and extract note if present."""
|
|
39
|
+
if not raw:
|
|
40
|
+
return set(), None
|
|
41
|
+
|
|
42
|
+
settings = set()
|
|
43
|
+
note_dict = None
|
|
44
|
+
|
|
45
|
+
# Split by comma, but be careful with nested structures
|
|
46
|
+
parts = []
|
|
47
|
+
current = []
|
|
48
|
+
depth = 0
|
|
49
|
+
|
|
50
|
+
for char in raw:
|
|
51
|
+
if char in "{[":
|
|
52
|
+
depth += 1
|
|
53
|
+
elif char in "}]":
|
|
54
|
+
depth -= 1
|
|
55
|
+
elif char == "," and depth == 0:
|
|
56
|
+
parts.append("".join(current).strip())
|
|
57
|
+
current = []
|
|
58
|
+
continue
|
|
59
|
+
current.append(char)
|
|
60
|
+
|
|
61
|
+
if current:
|
|
62
|
+
parts.append("".join(current).strip())
|
|
63
|
+
|
|
64
|
+
for part in parts:
|
|
65
|
+
part = part.strip()
|
|
66
|
+
if not part:
|
|
67
|
+
continue
|
|
68
|
+
|
|
69
|
+
# Check if this is a note
|
|
70
|
+
if part.lower().startswith("note:"):
|
|
71
|
+
note_str = part[5:].strip()
|
|
72
|
+
# Remove surrounding quotes if present
|
|
73
|
+
note_str = _strip_quotes(note_str)
|
|
74
|
+
try:
|
|
75
|
+
# Try to parse as JSON
|
|
76
|
+
note_dict = json.loads(note_str)
|
|
77
|
+
except json.JSONDecodeError:
|
|
78
|
+
# If JSON parsing fails, ignore the note
|
|
79
|
+
pass
|
|
80
|
+
else:
|
|
81
|
+
# Regular setting (pk, not null, unique, etc.)
|
|
82
|
+
settings.add(part.strip("'").strip('"').lower())
|
|
83
|
+
|
|
84
|
+
return settings, note_dict
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def normalize_identifier(value: str) -> str:
|
|
88
|
+
cleaned = re.sub(r"[^0-9A-Za-z]+", "_", value).strip("_").lower()
|
|
89
|
+
if not cleaned:
|
|
90
|
+
cleaned = "table"
|
|
91
|
+
if cleaned[0].isdigit():
|
|
92
|
+
cleaned = f"t_{cleaned}"
|
|
93
|
+
return cleaned
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def parse_dbml(dbml_path: Path) -> tuple[dict[str, TableDef], list[dict]]:
|
|
97
|
+
text = dbml_path.read_text(encoding="utf-8")
|
|
98
|
+
lines = text.splitlines()
|
|
99
|
+
tables: dict[str, TableDef] = {}
|
|
100
|
+
refs: list[dict] = []
|
|
101
|
+
|
|
102
|
+
current_table: Optional[TableDef] = None
|
|
103
|
+
in_indexes_block = False
|
|
104
|
+
note_block_depth = 0
|
|
105
|
+
in_ref_block = False
|
|
106
|
+
|
|
107
|
+
for raw_line in lines:
|
|
108
|
+
line = raw_line.strip()
|
|
109
|
+
if not line or line.startswith("//"):
|
|
110
|
+
continue
|
|
111
|
+
|
|
112
|
+
cleaned = line.split("//", 1)[0].strip()
|
|
113
|
+
if not cleaned:
|
|
114
|
+
continue
|
|
115
|
+
|
|
116
|
+
triple_quote_count = cleaned.count("'''")
|
|
117
|
+
if triple_quote_count:
|
|
118
|
+
note_block_depth = (note_block_depth + triple_quote_count) % 2
|
|
119
|
+
if cleaned.startswith("Note:"):
|
|
120
|
+
continue
|
|
121
|
+
if note_block_depth:
|
|
122
|
+
continue
|
|
123
|
+
|
|
124
|
+
# ----------------------
|
|
125
|
+
# TABLE PARSING
|
|
126
|
+
# ----------------------
|
|
127
|
+
if cleaned.lower().startswith("table "):
|
|
128
|
+
table_name_section = cleaned[6:].split("{", 1)[0].strip()
|
|
129
|
+
if "[" in table_name_section:
|
|
130
|
+
table_name_section = table_name_section.split("[", 1)[0].strip()
|
|
131
|
+
table_name = _strip_quotes(table_name_section)
|
|
132
|
+
current_table = TableDef(name=table_name)
|
|
133
|
+
continue
|
|
134
|
+
|
|
135
|
+
if current_table:
|
|
136
|
+
if cleaned.startswith("indexes"):
|
|
137
|
+
in_indexes_block = True
|
|
138
|
+
continue
|
|
139
|
+
if in_indexes_block:
|
|
140
|
+
if cleaned.endswith("}"):
|
|
141
|
+
in_indexes_block = False
|
|
142
|
+
continue
|
|
143
|
+
if cleaned.startswith("}"):
|
|
144
|
+
tables[current_table.name] = current_table
|
|
145
|
+
current_table = None
|
|
146
|
+
continue
|
|
147
|
+
if cleaned.startswith("Note:"):
|
|
148
|
+
continue
|
|
149
|
+
|
|
150
|
+
col_match = re.match(
|
|
151
|
+
r'^(".*?"|`.*?`|[A-Za-z_][\w]*)\s+(.+?)(?:\s+\[(.+)\])?$',
|
|
152
|
+
cleaned,
|
|
153
|
+
)
|
|
154
|
+
if not col_match:
|
|
155
|
+
continue
|
|
156
|
+
|
|
157
|
+
col_name = _strip_quotes(col_match.group(1))
|
|
158
|
+
col_type = col_match.group(2).strip()
|
|
159
|
+
|
|
160
|
+
# 🚨 Reject invalid / sentence-like column definitions
|
|
161
|
+
if len(col_type.split()) > 3:
|
|
162
|
+
continue
|
|
163
|
+
|
|
164
|
+
settings, note_dict = _parse_column_settings(col_match.group(3))
|
|
165
|
+
|
|
166
|
+
current_table.columns.append(
|
|
167
|
+
ColumnDef(
|
|
168
|
+
name=col_name,
|
|
169
|
+
data_type=col_type,
|
|
170
|
+
settings=settings,
|
|
171
|
+
note=note_dict,
|
|
172
|
+
)
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
continue
|
|
176
|
+
|
|
177
|
+
# ----------------------
|
|
178
|
+
# REF BLOCK START
|
|
179
|
+
# ----------------------
|
|
180
|
+
if cleaned.startswith("Ref"):
|
|
181
|
+
in_ref_block = True
|
|
182
|
+
continue
|
|
183
|
+
|
|
184
|
+
if in_ref_block:
|
|
185
|
+
if cleaned.startswith("}"):
|
|
186
|
+
in_ref_block = False
|
|
187
|
+
continue
|
|
188
|
+
|
|
189
|
+
# Match: "table"."column" > "table"."column"
|
|
190
|
+
ref_match = re.match(
|
|
191
|
+
r'(".*?"|`.*?`|[\w]+)\.(".*?"|`.*?`|[\w]+)\s*([<>])\s*'
|
|
192
|
+
r'(".*?"|`.*?`|[\w]+)\.(".*?"|`.*?`|[\w]+)',
|
|
193
|
+
cleaned,
|
|
194
|
+
)
|
|
195
|
+
if not ref_match:
|
|
196
|
+
continue
|
|
197
|
+
|
|
198
|
+
left_table, left_column, operator, right_table, right_column = ref_match.groups()
|
|
199
|
+
|
|
200
|
+
# Ignore <> and other non-FK relations
|
|
201
|
+
if operator not in (">", "<"):
|
|
202
|
+
continue
|
|
203
|
+
|
|
204
|
+
if operator == "<":
|
|
205
|
+
left_table, right_table = right_table, left_table
|
|
206
|
+
left_column, right_column = right_column, left_column
|
|
207
|
+
|
|
208
|
+
refs.append(
|
|
209
|
+
{
|
|
210
|
+
"source_table": _strip_quotes(left_table),
|
|
211
|
+
"source_column": _strip_quotes(left_column),
|
|
212
|
+
"target_table": _strip_quotes(right_table),
|
|
213
|
+
"target_column": _strip_quotes(right_column),
|
|
214
|
+
}
|
|
215
|
+
)
|
|
216
|
+
continue
|
|
217
|
+
|
|
218
|
+
return tables, refs
|
|
@@ -10,6 +10,15 @@ model2data.egg-info/dependency_links.txt
|
|
|
10
10
|
model2data.egg-info/entry_points.txt
|
|
11
11
|
model2data.egg-info/requires.txt
|
|
12
12
|
model2data.egg-info/top_level.txt
|
|
13
|
+
model2data/dbt/__init__.py
|
|
14
|
+
model2data/dbt/project.py
|
|
15
|
+
model2data/dbt/tests.py
|
|
16
|
+
model2data/generate/__init__.py
|
|
17
|
+
model2data/generate/core.py
|
|
18
|
+
model2data/generate/faker.py
|
|
19
|
+
model2data/generate/relationships.py
|
|
20
|
+
model2data/parse/__init__.py
|
|
21
|
+
model2data/parse/dbml.py
|
|
13
22
|
tests/test_cli.py
|
|
14
23
|
tests/test_coverage_gaps.py
|
|
15
24
|
tests/test_dbml_parser.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "model2data"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
description = "Generate analytics-ready datasets from DBML models"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -33,7 +33,8 @@ dev = [
|
|
|
33
33
|
model2data = "model2data.cli:app"
|
|
34
34
|
|
|
35
35
|
[tool.setuptools.packages.find]
|
|
36
|
-
|
|
36
|
+
where = ["."]
|
|
37
|
+
include = ["model2data*"]
|
|
37
38
|
|
|
38
39
|
# Ruff configuration (fast Python linter)
|
|
39
40
|
[tool.ruff]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|