model2data 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {model2data-0.2.1 → model2data-0.2.2}/PKG-INFO +1 -1
  2. model2data-0.2.2/model2data/dbt/__init__.py +1 -0
  3. model2data-0.2.2/model2data/dbt/project.py +74 -0
  4. model2data-0.2.2/model2data/dbt/tests.py +92 -0
  5. model2data-0.2.2/model2data/generate/__init__.py +1 -0
  6. model2data-0.2.2/model2data/generate/core.py +166 -0
  7. model2data-0.2.2/model2data/generate/faker.py +122 -0
  8. model2data-0.2.2/model2data/generate/relationships.py +52 -0
  9. model2data-0.2.2/model2data/parse/__init__.py +1 -0
  10. model2data-0.2.2/model2data/parse/dbml.py +218 -0
  11. {model2data-0.2.1 → model2data-0.2.2}/model2data.egg-info/PKG-INFO +1 -1
  12. {model2data-0.2.1 → model2data-0.2.2}/model2data.egg-info/SOURCES.txt +9 -0
  13. {model2data-0.2.1 → model2data-0.2.2}/pyproject.toml +3 -2
  14. {model2data-0.2.1 → model2data-0.2.2}/LICENSE +0 -0
  15. {model2data-0.2.1 → model2data-0.2.2}/README.md +0 -0
  16. {model2data-0.2.1 → model2data-0.2.2}/model2data/__init__.py +0 -0
  17. {model2data-0.2.1 → model2data-0.2.2}/model2data/cli.py +0 -0
  18. {model2data-0.2.1 → model2data-0.2.2}/model2data/utils.py +0 -0
  19. {model2data-0.2.1 → model2data-0.2.2}/model2data.egg-info/dependency_links.txt +0 -0
  20. {model2data-0.2.1 → model2data-0.2.2}/model2data.egg-info/entry_points.txt +0 -0
  21. {model2data-0.2.1 → model2data-0.2.2}/model2data.egg-info/requires.txt +0 -0
  22. {model2data-0.2.1 → model2data-0.2.2}/model2data.egg-info/top_level.txt +0 -0
  23. {model2data-0.2.1 → model2data-0.2.2}/setup.cfg +0 -0
  24. {model2data-0.2.1 → model2data-0.2.2}/tests/test_cli.py +0 -0
  25. {model2data-0.2.1 → model2data-0.2.2}/tests/test_coverage_gaps.py +0 -0
  26. {model2data-0.2.1 → model2data-0.2.2}/tests/test_dbml_parser.py +0 -0
  27. {model2data-0.2.1 → model2data-0.2.2}/tests/test_dbt_naming.py +0 -0
  28. {model2data-0.2.1 → model2data-0.2.2}/tests/test_dbt_project.py +0 -0
  29. {model2data-0.2.1 → model2data-0.2.2}/tests/test_dbt_tests.py +0 -0
  30. {model2data-0.2.1 → model2data-0.2.2}/tests/test_generation.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: model2data
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Generate analytics-ready datasets from DBML models
5
5
  Requires-Python: >=3.10
6
6
  Description-Content-Type: text/markdown
@@ -0,0 +1 @@
1
+ """dbt project generation utilities."""
@@ -0,0 +1,74 @@
1
+ from pathlib import Path
2
+
3
+ import jinja2
4
+
5
+ TEMPLATES_DIR = Path(__file__).parent / "templates"
6
+
7
+
8
+ def create_project_scaffold(dest: Path, project_name: str, profile_name: str) -> None:
9
+ dest.mkdir(parents=True, exist_ok=True)
10
+
11
+ # dbt folders
12
+ (dest / "models" / "staging").mkdir(parents=True, exist_ok=True)
13
+ (dest / "seeds" / "raw").mkdir(parents=True, exist_ok=True)
14
+ (dest / "analysis").mkdir(exist_ok=True)
15
+ (dest / "macros").mkdir(exist_ok=True)
16
+ (dest / "tests").mkdir(exist_ok=True)
17
+ (dest / "snapshots").mkdir(exist_ok=True)
18
+
19
+ # dbt_project.yml
20
+ _render_template(
21
+ template_name="dbt_project.yml.jinja",
22
+ output_path=dest / "dbt_project.yml",
23
+ context={"project_name": project_name, "profile_name": profile_name},
24
+ )
25
+
26
+ # Copy over any macros from templates
27
+ template_macros_dir = Path("model2data/dbt/templates/macros")
28
+ if template_macros_dir.exists():
29
+ for macro_file in template_macros_dir.glob("*.sql"):
30
+ target_file = dest / "macros" / macro_file.name
31
+ if not target_file.exists():
32
+ target_file.write_text(macro_file.read_text())
33
+
34
+
35
+ def create_staging_models(dest: Path, project_name: str) -> None:
36
+ """
37
+ Creates staging models in models/staging/ folder that reference raw seed tables as sources.
38
+ """
39
+ seeds_path = dest / "seeds" / "raw"
40
+ models_path = dest / "models" / "staging"
41
+ models_path.mkdir(parents=True, exist_ok=True)
42
+
43
+ for csv_file in seeds_path.glob("*.csv"):
44
+ table_name = csv_file.stem # keep full seed name, e.g., raw_stories
45
+ model_file = models_path / f"stg_{table_name}.sql"
46
+
47
+ if not model_file.exists():
48
+ sql_content = f"""\
49
+ -- Auto-generated staging model for {table_name}
50
+ select *
51
+ from {{{{ source('raw', '{table_name}') }}}}
52
+ """
53
+ model_file.write_text(sql_content)
54
+
55
+
56
+ def create_profiles_yml(dest: Path, profile_name: str) -> None:
57
+ profiles_file = dest / "profiles.yml"
58
+ if profiles_file.exists():
59
+ content = profiles_file.read_text()
60
+ if profile_name in content:
61
+ return
62
+ _render_template(
63
+ template_name="profiles.yml.jinja",
64
+ output_path=profiles_file,
65
+ context={"profile_name": profile_name},
66
+ )
67
+
68
+
69
+ def _render_template(template_name: str, output_path: Path, context: dict) -> None:
70
+ template_path = TEMPLATES_DIR / template_name
71
+ if not template_path.exists():
72
+ raise FileNotFoundError(f"Template not found: {template_path}")
73
+ template = jinja2.Template(template_path.read_text())
74
+ output_path.write_text(template.render(**context))
@@ -0,0 +1,92 @@
1
+ from collections import defaultdict
2
+ from pathlib import Path
3
+ from typing import Any, Union
4
+
5
+
6
+ def generate_dbt_yml(dest: Path, tables: dict, refs: list[dict], source_name: str = "hackernews"):
7
+ """
8
+ Generate:
9
+ 1) __sources.yml with all raw_* seeds (no tests)
10
+ 2) One .yml per staging model (stg_*) with tests
11
+ Table and column names are used exactly as in DBML.
12
+ """
13
+
14
+ staging_path = dest / "models" / "staging"
15
+ staging_path.mkdir(parents=True, exist_ok=True)
16
+
17
+ # -------------------------
18
+ # Build foreign key map
19
+ # -------------------------
20
+ fk_map = defaultdict(list)
21
+ for ref in refs:
22
+ fk_map[(ref["source_table"], ref["source_column"])].append(ref)
23
+
24
+ # -------------------------
25
+ # Generate __sources.yml
26
+ # -------------------------
27
+ sources_lines = ["version: 2", "", "sources:"]
28
+ sources_lines.append(" - name: raw")
29
+ sources_lines.append(" schema: raw")
30
+ sources_lines.append(f" description: {source_name.capitalize()} raw seed data")
31
+ sources_lines.append(" tables:")
32
+
33
+ for table in tables.values():
34
+ seed_name = table.name # keep exact name
35
+ table_desc = getattr(table, "description", None) or f"Table {seed_name}"
36
+ sources_lines.append(f" - name: {seed_name}")
37
+ sources_lines.append(f" description: {table_desc}")
38
+
39
+ sources_file = staging_path / "__sources.yml"
40
+ sources_file.write_text("\n".join(sources_lines))
41
+
42
+ # -------------------------
43
+ # Generate individual staging model YAMLs
44
+ # -------------------------
45
+ for table in tables.values():
46
+ stg_name = f"stg_{table.name}" # staging model names are prefixed, columns unchanged
47
+ model_columns = []
48
+
49
+ for col in table.columns:
50
+ tests: list[Union[str, dict[str, dict[str, Any]]]] = []
51
+ settings = col.settings or set()
52
+
53
+ if "not null" in settings or "pk" in settings:
54
+ tests.append("not_null")
55
+ if "unique" in settings or "pk" in settings:
56
+ tests.append("unique")
57
+
58
+ fk_refs = fk_map.get((table.name, col.name), [])
59
+ for fk in fk_refs:
60
+ tests.append(
61
+ {
62
+ "relationships": {
63
+ "to": f"ref('stg_{fk['target_table']}')",
64
+ "field": fk["target_column"],
65
+ }
66
+ }
67
+ )
68
+
69
+ model_columns.append({"name": col.name, "tests": tests if tests else None})
70
+
71
+ # Render model YAML
72
+ lines = ["version: 2", "", "models:"]
73
+ lines.append(f" - name: {stg_name}")
74
+ lines.append(" columns:")
75
+ for col in model_columns:
76
+ lines.append(f" - name: {col['name']}")
77
+ if col["tests"]:
78
+ lines.append(" tests:")
79
+ for test in col["tests"]:
80
+ if isinstance(test, str):
81
+ lines.append(f" - {test}")
82
+ else:
83
+ # relationships test with arguments
84
+ for k, v in test.items():
85
+ lines.append(f" - {k}:")
86
+ lines.append(" arguments:")
87
+ for fk_key, fk_val in v.items():
88
+ lines.append(f" {fk_key}: {fk_val}")
89
+
90
+ # Write YAML to same folder as SQL model
91
+ yml_file = staging_path / f"{stg_name}.yml"
92
+ yml_file.write_text("\n".join(lines))
@@ -0,0 +1 @@
1
+ """Data generation utilities."""
@@ -0,0 +1,166 @@
1
+ from __future__ import annotations
2
+
3
+ import random
4
+ from collections import defaultdict, deque
5
+ from typing import Optional
6
+
7
+ import pandas as pd
8
+ from faker import Faker
9
+
10
+ from model2data.generate.faker import generate_column_values
11
+ from model2data.generate.relationships import (
12
+ build_fk_lookup,
13
+ classify_refs,
14
+ )
15
+ from model2data.parse.dbml import TableDef
16
+
17
+ fake = Faker()
18
+
19
+
20
+ # ---------------------------------------------------------
21
+ # Public API
22
+ # ---------------------------------------------------------
23
+ def generate_data_from_dbml(
24
+ tables: dict[str, TableDef],
25
+ refs: list[dict],
26
+ base_rows: int = 100,
27
+ seed: Optional[int] = None,
28
+ ) -> dict[str, pd.DataFrame]:
29
+ """
30
+ Generate synthetic datasets from parsed DBML definitions.
31
+
32
+ This function is deterministic if a seed is provided.
33
+ It performs no filesystem I/O and returns pandas DataFrames.
34
+ """
35
+ if seed is not None:
36
+ random.seed(seed)
37
+ Faker.seed(seed)
38
+
39
+ # ---------------------------------------------------------
40
+ # Classify references
41
+ # ---------------------------------------------------------
42
+ fk_refs, attribute_refs = classify_refs(tables, refs)
43
+ fk_lookup = build_fk_lookup(fk_refs)
44
+
45
+ # ---------------------------------------------------------
46
+ # Generate tables in dependency order
47
+ # ---------------------------------------------------------
48
+ ordered_tables = _topological_table_order(tables, fk_refs)
49
+ generated: dict[str, pd.DataFrame] = {}
50
+
51
+ for table_name in ordered_tables:
52
+ table_def = tables[table_name]
53
+ row_count = _determine_row_count(table_def.name, base_rows)
54
+
55
+ data: dict[str, list] = {}
56
+
57
+ # -----------------------
58
+ # First pass: columns + FKs
59
+ # -----------------------
60
+ for column in table_def.columns:
61
+ fk_series = None
62
+ fk_target = fk_lookup.get((table_name, column.name))
63
+
64
+ if fk_target:
65
+ parent_table, parent_column = fk_target
66
+ parent_df = generated.get(parent_table)
67
+ if parent_df is not None and parent_column in parent_df.columns:
68
+ fk_series = parent_df[parent_column]
69
+
70
+ ensure_unique = "pk" in column.settings
71
+ data[column.name] = generate_column_values(
72
+ column=column,
73
+ row_count=row_count,
74
+ fk_series=fk_series,
75
+ ensure_unique=ensure_unique,
76
+ )
77
+
78
+ df = pd.DataFrame(data)
79
+
80
+ # -----------------------------------------------------
81
+ # Second pass: attribute mirroring (non-FK refs)
82
+ # -----------------------------------------------------
83
+ for ref in attribute_refs:
84
+ if ref["source_table"] != table_name:
85
+ continue
86
+
87
+ parent_table = ref["target_table"]
88
+ parent_column = ref["target_column"]
89
+ child_column = ref["source_column"]
90
+
91
+ parent_df = generated.get(parent_table)
92
+ if parent_df is None:
93
+ continue
94
+
95
+ # find FK linking child → parent
96
+ fk_column = next(
97
+ (
98
+ r["source_column"]
99
+ for r in fk_refs
100
+ if r["source_table"] == table_name and r["target_table"] == parent_table
101
+ ),
102
+ None,
103
+ )
104
+
105
+ if not fk_column or fk_column not in df.columns:
106
+ continue
107
+
108
+ lookup = parent_df.groupby("id")[parent_column].first().to_dict()
109
+
110
+ df[child_column] = df[fk_column].map(lookup)
111
+
112
+ generated[table_name] = df
113
+
114
+ return generated
115
+
116
+
117
+ # ---------------------------------------------------------
118
+ # Internal helpers
119
+ # ---------------------------------------------------------
120
+ def _determine_row_count(table_name: str, base_rows: int) -> int:
121
+ """
122
+ Return the base number of rows for all tables.
123
+ """
124
+ return base_rows
125
+
126
+
127
+ def _topological_table_order(
128
+ tables: dict[str, TableDef],
129
+ fk_refs: list[dict],
130
+ ) -> list[str]:
131
+ """
132
+ Order tables so parent tables are generated before children.
133
+ """
134
+ graph: dict[str, set[str]] = defaultdict(set)
135
+ indegree: dict[str, int] = dict.fromkeys(tables.keys(), 0)
136
+
137
+ for ref in fk_refs:
138
+ parent = ref["target_table"]
139
+ child = ref["source_table"]
140
+
141
+ if parent == child:
142
+ continue
143
+ if parent not in tables or child not in tables:
144
+ continue
145
+
146
+ if child not in graph[parent]:
147
+ graph[parent].add(child)
148
+ indegree[child] += 1
149
+
150
+ queue = deque(sorted(name for name, deg in indegree.items() if deg == 0))
151
+ order: list[str] = []
152
+
153
+ while queue:
154
+ node = queue.popleft()
155
+ order.append(node)
156
+ for neighbor in sorted(graph.get(node, [])):
157
+ indegree[neighbor] -= 1
158
+ if indegree[neighbor] == 0:
159
+ queue.append(neighbor)
160
+
161
+ # Safety net for disconnected tables
162
+ for name in tables.keys():
163
+ if name not in order:
164
+ order.append(name)
165
+
166
+ return order
@@ -0,0 +1,122 @@
1
+ from __future__ import annotations
2
+
3
+ import random
4
+ import uuid
5
+ from datetime import datetime, timedelta
6
+ from typing import Optional
7
+
8
+ import pandas as pd
9
+ from faker import Faker
10
+
11
+ from model2data.parse.dbml import ColumnDef
12
+
13
+ fake = Faker()
14
+
15
+
16
+ # ---------------------------------------------------------
17
+ # Public API
18
+ # ---------------------------------------------------------
19
+ def generate_column_values(
20
+ column: ColumnDef,
21
+ row_count: int,
22
+ fk_series: Optional[pd.Series] = None,
23
+ ensure_unique: bool = False,
24
+ ) -> list:
25
+ """
26
+ Generate synthetic values for a single column.
27
+ Respects FKs, uniqueness, and optional min/max hints in column notes.
28
+ """
29
+ if fk_series is not None and not fk_series.empty:
30
+ fk_values = fk_series.tolist()
31
+ return [random.choice(fk_values) for _ in range(row_count)]
32
+
33
+ dtype = column.data_type.lower()
34
+ base_type = dtype.split("(")[0].strip()
35
+ values: list = []
36
+
37
+ # Extract min/max from note if present
38
+ min_val = None
39
+ max_val = None
40
+ if column.note:
41
+ min_val = column.note.get("min")
42
+ max_val = column.note.get("max")
43
+
44
+ # -----------------------------------------------------
45
+ # UUIDs / hashes
46
+ # -----------------------------------------------------
47
+ if "uuid" in base_type or "hash" in base_type:
48
+ values = [str(uuid.uuid4()) for _ in range(row_count)]
49
+
50
+ # -----------------------------------------------------
51
+ # Integers
52
+ # -----------------------------------------------------
53
+ elif any(key in base_type for key in ["int", "integer", "bigint", "smallint"]):
54
+ # Use note values if present, otherwise defaults
55
+ if min_val is None:
56
+ min_val = 0
57
+ if max_val is None:
58
+ max_val = 100
59
+ values = [random.randint(min_val, max_val) for _ in range(row_count)]
60
+
61
+ # -----------------------------------------------------
62
+ # Floats / decimals
63
+ # -----------------------------------------------------
64
+ elif any(key in base_type for key in ["decimal", "numeric", "float", "double"]):
65
+ if min_val is None:
66
+ min_val = 0
67
+ if max_val is None:
68
+ max_val = 10_000
69
+ values = [round(random.uniform(min_val, max_val), 2) for _ in range(row_count)]
70
+
71
+ # -----------------------------------------------------
72
+ # Booleans
73
+ # -----------------------------------------------------
74
+ elif "boolean" in base_type or "bool" in base_type:
75
+ values = [random.choice([True, False]) for _ in range(row_count)]
76
+
77
+ # -----------------------------------------------------
78
+ # Dates
79
+ # -----------------------------------------------------
80
+ elif "date" in base_type and "time" not in base_type:
81
+ values = [fake.date_between(start_date="-2y", end_date="today") for _ in range(row_count)]
82
+
83
+ elif "time" in base_type and "stamp" not in base_type:
84
+ values = [fake.time() for _ in range(row_count)]
85
+
86
+ elif any(key in base_type for key in ["timestamp", "datetime"]):
87
+ values = [_random_datetime().isoformat(sep=" ") for _ in range(row_count)]
88
+
89
+ # -----------------------------------------------------
90
+ # Fallback to Faker providers
91
+ # -----------------------------------------------------
92
+ else:
93
+ try:
94
+ values = [fake.format(base_type) for _ in range(row_count)]
95
+ except (AttributeError, TypeError):
96
+ if column.name.lower().endswith("_id") or ensure_unique:
97
+ values = [str(uuid.uuid4()) for _ in range(row_count)]
98
+ else:
99
+ values = [fake.sentence(nb_words=3) for _ in range(row_count)]
100
+
101
+ # -----------------------------------------------------
102
+ # Nullability
103
+ # -----------------------------------------------------
104
+ if "not null" not in column.settings:
105
+ null_fraction = max(0, min(0.2, 1 - (row_count / (row_count + 50))))
106
+ sample_size = int(row_count * null_fraction)
107
+ if sample_size:
108
+ for idx in random.sample(range(row_count), k=sample_size):
109
+ values[idx] = None
110
+
111
+ return values
112
+
113
+
114
+ # ---------------------------------------------------------
115
+ # Internal helpers
116
+ # ---------------------------------------------------------
117
+ def _random_datetime(start_days: int = -365, end_days: int = 0) -> datetime:
118
+ start = datetime.now() + timedelta(days=start_days)
119
+ end = datetime.now() + timedelta(days=end_days)
120
+ delta = end - start
121
+ random_second = random.randint(0, int(delta.total_seconds()))
122
+ return start + timedelta(seconds=random_second)
@@ -0,0 +1,52 @@
1
+ from typing import Dict, List, Tuple
2
+
3
+ from model2data.parse.dbml import TableDef
4
+
5
+
6
+ # ---------------------------------------------------------
7
+ # Public API
8
+ # ---------------------------------------------------------
9
+ def classify_refs(
10
+ tables: Dict[str, TableDef],
11
+ refs: List[Dict],
12
+ ) -> Tuple[List[Dict], List[Dict]]:
13
+ """
14
+ Classify references into:
15
+ - fk_refs: Foreign keys (target column looks like a PK)
16
+ - attribute_refs: Non-FK dependencies (mirroring parent attributes)
17
+ """
18
+ fk_refs = []
19
+ attribute_refs = []
20
+
21
+ for ref in refs:
22
+ target_table = tables.get(ref["target_table"])
23
+ target_col = None
24
+ if target_table:
25
+ target_col = next(
26
+ (c for c in target_table.columns if c.name == ref["target_column"]), None
27
+ )
28
+
29
+ # FK if target column is a primary key or named "id"
30
+ if target_col and ("pk" in target_col.settings or target_col.name.lower() == "id"):
31
+ fk_refs.append(ref)
32
+ else:
33
+ attribute_refs.append(ref)
34
+
35
+ return fk_refs, attribute_refs
36
+
37
+
38
+ def build_fk_lookup(fk_refs: List[Dict]) -> Dict[Tuple[str, str], Tuple[str, str]]:
39
+ """
40
+ Build a lookup dictionary for FK relationships.
41
+
42
+ Returns:
43
+ {
44
+ (child_table, child_column): (parent_table, parent_column)
45
+ }
46
+ """
47
+ lookup: Dict[Tuple[str, str], Tuple[str, str]] = {}
48
+ for ref in fk_refs:
49
+ key = (ref["source_table"], ref["source_column"])
50
+ value = (ref["target_table"], ref["target_column"])
51
+ lookup[key] = value
52
+ return lookup
@@ -0,0 +1 @@
1
+ """DBML parsing utilities."""
@@ -0,0 +1,218 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import re
5
+ from dataclasses import dataclass, field
6
+ from pathlib import Path
7
+ from typing import Optional
8
+
9
+ # -------------------------------
10
+ # Dataclasses
11
+ # -------------------------------
12
+
13
+
14
+ @dataclass
15
+ class ColumnDef:
16
+ name: str
17
+ data_type: str
18
+ settings: set[str] = field(default_factory=set)
19
+ note: Optional[dict] = None
20
+
21
+
22
+ @dataclass
23
+ class TableDef:
24
+ name: str
25
+ columns: list[ColumnDef] = field(default_factory=list)
26
+
27
+
28
+ # -------------------------------
29
+ # Helpers
30
+ # -------------------------------
31
+
32
+
33
+ def _strip_quotes(value: str) -> str:
34
+ return value.strip().strip('"').strip("'")
35
+
36
+
37
+ def _parse_column_settings(raw: Optional[str]) -> tuple[set[str], Optional[dict]]:
38
+ """Parse column settings and extract note if present."""
39
+ if not raw:
40
+ return set(), None
41
+
42
+ settings = set()
43
+ note_dict = None
44
+
45
+ # Split by comma, but be careful with nested structures
46
+ parts = []
47
+ current = []
48
+ depth = 0
49
+
50
+ for char in raw:
51
+ if char in "{[":
52
+ depth += 1
53
+ elif char in "}]":
54
+ depth -= 1
55
+ elif char == "," and depth == 0:
56
+ parts.append("".join(current).strip())
57
+ current = []
58
+ continue
59
+ current.append(char)
60
+
61
+ if current:
62
+ parts.append("".join(current).strip())
63
+
64
+ for part in parts:
65
+ part = part.strip()
66
+ if not part:
67
+ continue
68
+
69
+ # Check if this is a note
70
+ if part.lower().startswith("note:"):
71
+ note_str = part[5:].strip()
72
+ # Remove surrounding quotes if present
73
+ note_str = _strip_quotes(note_str)
74
+ try:
75
+ # Try to parse as JSON
76
+ note_dict = json.loads(note_str)
77
+ except json.JSONDecodeError:
78
+ # If JSON parsing fails, ignore the note
79
+ pass
80
+ else:
81
+ # Regular setting (pk, not null, unique, etc.)
82
+ settings.add(part.strip("'").strip('"').lower())
83
+
84
+ return settings, note_dict
85
+
86
+
87
+ def normalize_identifier(value: str) -> str:
88
+ cleaned = re.sub(r"[^0-9A-Za-z]+", "_", value).strip("_").lower()
89
+ if not cleaned:
90
+ cleaned = "table"
91
+ if cleaned[0].isdigit():
92
+ cleaned = f"t_{cleaned}"
93
+ return cleaned
94
+
95
+
96
+ def parse_dbml(dbml_path: Path) -> tuple[dict[str, TableDef], list[dict]]:
97
+ text = dbml_path.read_text(encoding="utf-8")
98
+ lines = text.splitlines()
99
+ tables: dict[str, TableDef] = {}
100
+ refs: list[dict] = []
101
+
102
+ current_table: Optional[TableDef] = None
103
+ in_indexes_block = False
104
+ note_block_depth = 0
105
+ in_ref_block = False
106
+
107
+ for raw_line in lines:
108
+ line = raw_line.strip()
109
+ if not line or line.startswith("//"):
110
+ continue
111
+
112
+ cleaned = line.split("//", 1)[0].strip()
113
+ if not cleaned:
114
+ continue
115
+
116
+ triple_quote_count = cleaned.count("'''")
117
+ if triple_quote_count:
118
+ note_block_depth = (note_block_depth + triple_quote_count) % 2
119
+ if cleaned.startswith("Note:"):
120
+ continue
121
+ if note_block_depth:
122
+ continue
123
+
124
+ # ----------------------
125
+ # TABLE PARSING
126
+ # ----------------------
127
+ if cleaned.lower().startswith("table "):
128
+ table_name_section = cleaned[6:].split("{", 1)[0].strip()
129
+ if "[" in table_name_section:
130
+ table_name_section = table_name_section.split("[", 1)[0].strip()
131
+ table_name = _strip_quotes(table_name_section)
132
+ current_table = TableDef(name=table_name)
133
+ continue
134
+
135
+ if current_table:
136
+ if cleaned.startswith("indexes"):
137
+ in_indexes_block = True
138
+ continue
139
+ if in_indexes_block:
140
+ if cleaned.endswith("}"):
141
+ in_indexes_block = False
142
+ continue
143
+ if cleaned.startswith("}"):
144
+ tables[current_table.name] = current_table
145
+ current_table = None
146
+ continue
147
+ if cleaned.startswith("Note:"):
148
+ continue
149
+
150
+ col_match = re.match(
151
+ r'^(".*?"|`.*?`|[A-Za-z_][\w]*)\s+(.+?)(?:\s+\[(.+)\])?$',
152
+ cleaned,
153
+ )
154
+ if not col_match:
155
+ continue
156
+
157
+ col_name = _strip_quotes(col_match.group(1))
158
+ col_type = col_match.group(2).strip()
159
+
160
+ # 🚨 Reject invalid / sentence-like column definitions
161
+ if len(col_type.split()) > 3:
162
+ continue
163
+
164
+ settings, note_dict = _parse_column_settings(col_match.group(3))
165
+
166
+ current_table.columns.append(
167
+ ColumnDef(
168
+ name=col_name,
169
+ data_type=col_type,
170
+ settings=settings,
171
+ note=note_dict,
172
+ )
173
+ )
174
+
175
+ continue
176
+
177
+ # ----------------------
178
+ # REF BLOCK START
179
+ # ----------------------
180
+ if cleaned.startswith("Ref"):
181
+ in_ref_block = True
182
+ continue
183
+
184
+ if in_ref_block:
185
+ if cleaned.startswith("}"):
186
+ in_ref_block = False
187
+ continue
188
+
189
+ # Match: "table"."column" > "table"."column"
190
+ ref_match = re.match(
191
+ r'(".*?"|`.*?`|[\w]+)\.(".*?"|`.*?`|[\w]+)\s*([<>])\s*'
192
+ r'(".*?"|`.*?`|[\w]+)\.(".*?"|`.*?`|[\w]+)',
193
+ cleaned,
194
+ )
195
+ if not ref_match:
196
+ continue
197
+
198
+ left_table, left_column, operator, right_table, right_column = ref_match.groups()
199
+
200
+ # Ignore <> and other non-FK relations
201
+ if operator not in (">", "<"):
202
+ continue
203
+
204
+ if operator == "<":
205
+ left_table, right_table = right_table, left_table
206
+ left_column, right_column = right_column, left_column
207
+
208
+ refs.append(
209
+ {
210
+ "source_table": _strip_quotes(left_table),
211
+ "source_column": _strip_quotes(left_column),
212
+ "target_table": _strip_quotes(right_table),
213
+ "target_column": _strip_quotes(right_column),
214
+ }
215
+ )
216
+ continue
217
+
218
+ return tables, refs
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: model2data
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Generate analytics-ready datasets from DBML models
5
5
  Requires-Python: >=3.10
6
6
  Description-Content-Type: text/markdown
@@ -10,6 +10,15 @@ model2data.egg-info/dependency_links.txt
10
10
  model2data.egg-info/entry_points.txt
11
11
  model2data.egg-info/requires.txt
12
12
  model2data.egg-info/top_level.txt
13
+ model2data/dbt/__init__.py
14
+ model2data/dbt/project.py
15
+ model2data/dbt/tests.py
16
+ model2data/generate/__init__.py
17
+ model2data/generate/core.py
18
+ model2data/generate/faker.py
19
+ model2data/generate/relationships.py
20
+ model2data/parse/__init__.py
21
+ model2data/parse/dbml.py
13
22
  tests/test_cli.py
14
23
  tests/test_coverage_gaps.py
15
24
  tests/test_dbml_parser.py
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "model2data"
7
- version = "0.2.1"
7
+ version = "0.2.2"
8
8
  description = "Generate analytics-ready datasets from DBML models"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -33,7 +33,8 @@ dev = [
33
33
  model2data = "model2data.cli:app"
34
34
 
35
35
  [tool.setuptools.packages.find]
36
- include = ["model2data"]
36
+ where = ["."]
37
+ include = ["model2data*"]
37
38
 
38
39
  # Ruff configuration (fast Python linter)
39
40
  [tool.ruff]
File without changes
File without changes
File without changes
File without changes
File without changes