model2data 0.1.1__tar.gz → 0.2.2__tar.gz

{model2data-0.1.1 → model2data-0.2.2}/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: model2data
-Version: 0.1.1
+Version: 0.2.2
 Summary: Generate analytics-ready datasets from DBML models
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: dbt-core>=1.5.0
@@ -14,6 +14,12 @@ Requires-Dist: typer>=0.20.0
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ty>=0.0.4; extra == "dev"
+Requires-Dist: types-pyyaml; extra == "dev"
+Requires-Dist: poethepoet>=0.38.0; extra == "dev"
 Dynamic: license-file
 
 # model2data
@@ -23,7 +29,7 @@ Dynamic: license-file
 [![codecov](https://codecov.io/gh/JB-Analytica/model2data/branch/main/graph/badge.svg)](https://codecov.io/gh/JB-Analytica/model2data)
 [![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
 
-`model2data` turns **data models into analytics-ready datasets** in seconds.  
+`model2data` turns **data models into analytics-ready datasets** in seconds.
 
 Given a **DBML file**, it generates synthetic but realistic data, a complete dbt project scaffold, and everything you need to start analyzing or testing data pipelines.
 

{model2data-0.1.1 → model2data-0.2.2}/README.md

@@ -5,7 +5,7 @@
 [![codecov](https://codecov.io/gh/JB-Analytica/model2data/branch/main/graph/badge.svg)](https://codecov.io/gh/JB-Analytica/model2data)
 [![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
 
-`model2data` turns **data models into analytics-ready datasets** in seconds.  
+`model2data` turns **data models into analytics-ready datasets** in seconds.
 
 Given a **DBML file**, it generates synthetic but realistic data, a complete dbt project scaffold, and everything you need to start analyzing or testing data pipelines.
 
@@ -134,4 +134,4 @@ Please read our [Code of Conduct](CODE_OF_CONDUCT.md) to understand our communit
 
 ## License
 
-MIT License. See LICENSE for details.
+MIT License. See LICENSE for details.

model2data-0.2.2/model2data/__init__.py

@@ -0,0 +1,3 @@
+"""Model2Data: Generate analytics-ready datasets from DBML models."""
+
+__version__ = "0.1.1"

{model2data-0.1.1 → model2data-0.2.2}/model2data/cli.py

@@ -1,19 +1,19 @@
-from pathlib import Path
-from typing import Optional
 import random
 import shutil
+from pathlib import Path
+from typing import Optional
 
 import typer
 from faker import Faker
 
-from model2data.parse.dbml import parse_dbml
-from model2data.generate.core import generate_data_from_dbml
 from model2data.dbt.project import (
-    create_project_scaffold,
     create_profiles_yml,
+    create_project_scaffold,
     create_staging_models,
 )
 from model2data.dbt.tests import generate_dbt_yml
+from model2data.generate.core import generate_data_from_dbml
+from model2data.parse.dbml import parse_dbml
 from model2data.utils import normalize_identifier
 
 app = typer.Typer(
@@ -30,7 +30,7 @@ app = typer.Typer(
 
 @app.command(help="Generate synthetic data and a dbt project from a DBML model.")
 def main(
-    file: Path = typer.Option(
+    file: Path = typer.Option(  # noqa: B008
         ...,
         "--file",
         "-f",
@@ -88,24 +88,13 @@ def main(
         typer.echo("❌ No tables found in the provided DBML file.")
         raise typer.Exit(1)
 
-    # -------------------------
-    # DBML → dbt name mapping
-    # -------------------------
-    dbt_name_map = {
-        table_name: normalize_identifier(table_name)
-        for table_name in tables.keys()
-    }
-
     project_name = normalize_identifier(name or file.stem)
     dest = Path.cwd() / f"dbt_{project_name}"
     profile_name = f"{project_name}_profile"
 
     if dest.exists():
         if not force:
-            typer.echo(
-                f"❌ Destination {dest} already exists.\n"
-                "Use --force to overwrite."
-            )
+            typer.echo(f"❌ Destination {dest} already exists.\nUse --force to overwrite.")
             raise typer.Exit(1)
         shutil.rmtree(dest)
 

model2data-0.2.2/model2data/dbt/__init__.py

@@ -0,0 +1 @@
+"""dbt project generation utilities."""

model2data-0.2.2/model2data/dbt/project.py

@@ -0,0 +1,74 @@
+from pathlib import Path
+
+import jinja2
+
+TEMPLATES_DIR = Path(__file__).parent / "templates"
+
+
+def create_project_scaffold(dest: Path, project_name: str, profile_name: str) -> None:
+    dest.mkdir(parents=True, exist_ok=True)
+
+    # dbt folders
+    (dest / "models" / "staging").mkdir(parents=True, exist_ok=True)
+    (dest / "seeds" / "raw").mkdir(parents=True, exist_ok=True)
+    (dest / "analysis").mkdir(exist_ok=True)
+    (dest / "macros").mkdir(exist_ok=True)
+    (dest / "tests").mkdir(exist_ok=True)
+    (dest / "snapshots").mkdir(exist_ok=True)
+
+    # dbt_project.yml
+    _render_template(
+        template_name="dbt_project.yml.jinja",
+        output_path=dest / "dbt_project.yml",
+        context={"project_name": project_name, "profile_name": profile_name},
+    )
+
+    # Copy over any macros from templates
+    template_macros_dir = Path("model2data/dbt/templates/macros")
+    if template_macros_dir.exists():
+        for macro_file in template_macros_dir.glob("*.sql"):
+            target_file = dest / "macros" / macro_file.name
+            if not target_file.exists():
+                target_file.write_text(macro_file.read_text())
+
+
+def create_staging_models(dest: Path, project_name: str) -> None:
+    """
+    Creates staging models in models/staging/ folder that reference raw seed tables as sources.
+    """
+    seeds_path = dest / "seeds" / "raw"
+    models_path = dest / "models" / "staging"
+    models_path.mkdir(parents=True, exist_ok=True)
+
+    for csv_file in seeds_path.glob("*.csv"):
+        table_name = csv_file.stem  # keep full seed name, e.g., raw_stories
+        model_file = models_path / f"stg_{table_name}.sql"
+
+        if not model_file.exists():
+            sql_content = f"""\
+-- Auto-generated staging model for {table_name}
+select *
+from {{{{ source('raw', '{table_name}') }}}}
+                """
+            model_file.write_text(sql_content)
+
+
+def create_profiles_yml(dest: Path, profile_name: str) -> None:
+    profiles_file = dest / "profiles.yml"
+    if profiles_file.exists():
+        content = profiles_file.read_text()
+        if profile_name in content:
+            return
+    _render_template(
+        template_name="profiles.yml.jinja",
+        output_path=profiles_file,
+        context={"profile_name": profile_name},
+    )
+
+
+def _render_template(template_name: str, output_path: Path, context: dict) -> None:
+    template_path = TEMPLATES_DIR / template_name
+    if not template_path.exists():
+        raise FileNotFoundError(f"Template not found: {template_path}")
+    template = jinja2.Template(template_path.read_text())
+    output_path.write_text(template.render(**context))

model2data-0.2.2/model2data/dbt/tests.py

@@ -0,0 +1,92 @@
+from collections import defaultdict
+from pathlib import Path
+from typing import Any, Union
+
+
+def generate_dbt_yml(dest: Path, tables: dict, refs: list[dict], source_name: str = "hackernews"):
+    """
+    Generate:
+      1) __sources.yml with all raw_* seeds (no tests)
+      2) One .yml per staging model (stg_*) with tests
+    Table and column names are used exactly as in DBML.
+    """
+
+    staging_path = dest / "models" / "staging"
+    staging_path.mkdir(parents=True, exist_ok=True)
+
+    # -------------------------
+    # Build foreign key map
+    # -------------------------
+    fk_map = defaultdict(list)
+    for ref in refs:
+        fk_map[(ref["source_table"], ref["source_column"])].append(ref)
+
+    # -------------------------
+    # Generate __sources.yml
+    # -------------------------
+    sources_lines = ["version: 2", "", "sources:"]
+    sources_lines.append("  - name: raw")
+    sources_lines.append("    schema: raw")
+    sources_lines.append(f"    description: {source_name.capitalize()} raw seed data")
+    sources_lines.append("    tables:")
+
+    for table in tables.values():
+        seed_name = table.name  # keep exact name
+        table_desc = getattr(table, "description", None) or f"Table {seed_name}"
+        sources_lines.append(f"      - name: {seed_name}")
+        sources_lines.append(f"        description: {table_desc}")
+
+    sources_file = staging_path / "__sources.yml"
+    sources_file.write_text("\n".join(sources_lines))
+
+    # -------------------------
+    # Generate individual staging model YAMLs
+    # -------------------------
+    for table in tables.values():
+        stg_name = f"stg_{table.name}"  # staging model names are prefixed, columns unchanged
+        model_columns = []
+
+        for col in table.columns:
+            tests: list[Union[str, dict[str, dict[str, Any]]]] = []
+            settings = col.settings or set()
+
+            if "not null" in settings or "pk" in settings:
+                tests.append("not_null")
+            if "unique" in settings or "pk" in settings:
+                tests.append("unique")
+
+            fk_refs = fk_map.get((table.name, col.name), [])
+            for fk in fk_refs:
+                tests.append(
+                    {
+                        "relationships": {
+                            "to": f"ref('stg_{fk['target_table']}')",
+                            "field": fk["target_column"],
+                        }
+                    }
+                )
+
+            model_columns.append({"name": col.name, "tests": tests if tests else None})
+
+        # Render model YAML
+        lines = ["version: 2", "", "models:"]
+        lines.append(f"  - name: {stg_name}")
+        lines.append("    columns:")
+        for col in model_columns:
+            lines.append(f"      - name: {col['name']}")
+            if col["tests"]:
+                lines.append("        tests:")
+                for test in col["tests"]:
+                    if isinstance(test, str):
+                        lines.append(f"          - {test}")
+                    else:
+                        # relationships test with arguments
+                        for k, v in test.items():
+                            lines.append(f"          - {k}:")
+                            lines.append("              arguments:")
+                            for fk_key, fk_val in v.items():
+                                lines.append(f"                {fk_key}: {fk_val}")
+
+        # Write YAML to same folder as SQL model
+        yml_file = staging_path / f"{stg_name}.yml"
+        yml_file.write_text("\n".join(lines))

model2data-0.2.2/model2data/generate/__init__.py

@@ -0,0 +1 @@
+"""Data generation utilities."""

model2data-0.2.2/model2data/generate/core.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import random
+from collections import defaultdict, deque
+from typing import Optional
+
+import pandas as pd
+from faker import Faker
+
+from model2data.generate.faker import generate_column_values
+from model2data.generate.relationships import (
+    build_fk_lookup,
+    classify_refs,
+)
+from model2data.parse.dbml import TableDef
+
+fake = Faker()
+
+
+# ---------------------------------------------------------
+# Public API
+# ---------------------------------------------------------
+def generate_data_from_dbml(
+    tables: dict[str, TableDef],
+    refs: list[dict],
+    base_rows: int = 100,
+    seed: Optional[int] = None,
+) -> dict[str, pd.DataFrame]:
+    """
+    Generate synthetic datasets from parsed DBML definitions.
+
+    This function is deterministic if a seed is provided.
+    It performs no filesystem I/O and returns pandas DataFrames.
+    """
+    if seed is not None:
+        random.seed(seed)
+        Faker.seed(seed)
+
+    # ---------------------------------------------------------
+    # Classify references
+    # ---------------------------------------------------------
+    fk_refs, attribute_refs = classify_refs(tables, refs)
+    fk_lookup = build_fk_lookup(fk_refs)
+
+    # ---------------------------------------------------------
+    # Generate tables in dependency order
+    # ---------------------------------------------------------
+    ordered_tables = _topological_table_order(tables, fk_refs)
+    generated: dict[str, pd.DataFrame] = {}
+
+    for table_name in ordered_tables:
+        table_def = tables[table_name]
+        row_count = _determine_row_count(table_def.name, base_rows)
+
+        data: dict[str, list] = {}
+
+        # -----------------------
+        # First pass: columns + FKs
+        # -----------------------
+        for column in table_def.columns:
+            fk_series = None
+            fk_target = fk_lookup.get((table_name, column.name))
+
+            if fk_target:
+                parent_table, parent_column = fk_target
+                parent_df = generated.get(parent_table)
+                if parent_df is not None and parent_column in parent_df.columns:
+                    fk_series = parent_df[parent_column]
+
+            ensure_unique = "pk" in column.settings
+            data[column.name] = generate_column_values(
+                column=column,
+                row_count=row_count,
+                fk_series=fk_series,
+                ensure_unique=ensure_unique,
+            )
+
+        df = pd.DataFrame(data)
+
+        # -----------------------------------------------------
+        # Second pass: attribute mirroring (non-FK refs)
+        # -----------------------------------------------------
+        for ref in attribute_refs:
+            if ref["source_table"] != table_name:
+                continue
+
+            parent_table = ref["target_table"]
+            parent_column = ref["target_column"]
+            child_column = ref["source_column"]
+
+            parent_df = generated.get(parent_table)
+            if parent_df is None:
+                continue
+
+            # find FK linking child → parent
+            fk_column = next(
+                (
+                    r["source_column"]
+                    for r in fk_refs
+                    if r["source_table"] == table_name and r["target_table"] == parent_table
+                ),
+                None,
+            )
+
+            if not fk_column or fk_column not in df.columns:
+                continue
+
+            lookup = parent_df.groupby("id")[parent_column].first().to_dict()
+
+            df[child_column] = df[fk_column].map(lookup)
+
+        generated[table_name] = df
+
+    return generated
+
+
+# ---------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------
+def _determine_row_count(table_name: str, base_rows: int) -> int:
+    """
+    Return the base number of rows for all tables.
+    """
+    return base_rows
+
+
+def _topological_table_order(
+    tables: dict[str, TableDef],
+    fk_refs: list[dict],
+) -> list[str]:
+    """
+    Order tables so parent tables are generated before children.
+    """
+    graph: dict[str, set[str]] = defaultdict(set)
+    indegree: dict[str, int] = dict.fromkeys(tables.keys(), 0)
+
+    for ref in fk_refs:
+        parent = ref["target_table"]
+        child = ref["source_table"]
+
+        if parent == child:
+            continue
+        if parent not in tables or child not in tables:
+            continue
+
+        if child not in graph[parent]:
+            graph[parent].add(child)
+            indegree[child] += 1
+
+    queue = deque(sorted(name for name, deg in indegree.items() if deg == 0))
+    order: list[str] = []
+
+    while queue:
+        node = queue.popleft()
+        order.append(node)
+        for neighbor in sorted(graph.get(node, [])):
+            indegree[neighbor] -= 1
+            if indegree[neighbor] == 0:
+                queue.append(neighbor)
+
+    # Safety net for disconnected tables
+    for name in tables.keys():
+        if name not in order:
+            order.append(name)
+
+    return order

model2data-0.2.2/model2data/generate/faker.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import random
+import uuid
+from datetime import datetime, timedelta
+from typing import Optional
+
+import pandas as pd
+from faker import Faker
+
+from model2data.parse.dbml import ColumnDef
+
+fake = Faker()
+
+
+# ---------------------------------------------------------
+# Public API
+# ---------------------------------------------------------
+def generate_column_values(
+    column: ColumnDef,
+    row_count: int,
+    fk_series: Optional[pd.Series] = None,
+    ensure_unique: bool = False,
+) -> list:
+    """
+    Generate synthetic values for a single column.
+    Respects FKs, uniqueness, and optional min/max hints in column notes.
+    """
+    if fk_series is not None and not fk_series.empty:
+        fk_values = fk_series.tolist()
+        return [random.choice(fk_values) for _ in range(row_count)]
+
+    dtype = column.data_type.lower()
+    base_type = dtype.split("(")[0].strip()
+    values: list = []
+
+    # Extract min/max from note if present
+    min_val = None
+    max_val = None
+    if column.note:
+        min_val = column.note.get("min")
+        max_val = column.note.get("max")
+
+    # -----------------------------------------------------
+    # UUIDs / hashes
+    # -----------------------------------------------------
+    if "uuid" in base_type or "hash" in base_type:
+        values = [str(uuid.uuid4()) for _ in range(row_count)]
+
+    # -----------------------------------------------------
+    # Integers
+    # -----------------------------------------------------
+    elif any(key in base_type for key in ["int", "integer", "bigint", "smallint"]):
+        # Use note values if present, otherwise defaults
+        if min_val is None:
+            min_val = 0
+        if max_val is None:
+            max_val = 100
+        values = [random.randint(min_val, max_val) for _ in range(row_count)]
+
+    # -----------------------------------------------------
+    # Floats / decimals
+    # -----------------------------------------------------
+    elif any(key in base_type for key in ["decimal", "numeric", "float", "double"]):
+        if min_val is None:
+            min_val = 0
+        if max_val is None:
+            max_val = 10_000
+        values = [round(random.uniform(min_val, max_val), 2) for _ in range(row_count)]
+
+    # -----------------------------------------------------
+    # Booleans
+    # -----------------------------------------------------
+    elif "boolean" in base_type or "bool" in base_type:
+        values = [random.choice([True, False]) for _ in range(row_count)]
+
+    # -----------------------------------------------------
+    # Dates
+    # -----------------------------------------------------
+    elif "date" in base_type and "time" not in base_type:
+        values = [fake.date_between(start_date="-2y", end_date="today") for _ in range(row_count)]
+
+    elif "time" in base_type and "stamp" not in base_type:
+        values = [fake.time() for _ in range(row_count)]
+
+    elif any(key in base_type for key in ["timestamp", "datetime"]):
+        values = [_random_datetime().isoformat(sep=" ") for _ in range(row_count)]
+
+    # -----------------------------------------------------
+    # Fallback to Faker providers
+    # -----------------------------------------------------
+    else:
+        try:
+            values = [fake.format(base_type) for _ in range(row_count)]
+        except (AttributeError, TypeError):
+            if column.name.lower().endswith("_id") or ensure_unique:
+                values = [str(uuid.uuid4()) for _ in range(row_count)]
+            else:
+                values = [fake.sentence(nb_words=3) for _ in range(row_count)]
+
+    # -----------------------------------------------------
+    # Nullability
+    # -----------------------------------------------------
+    if "not null" not in column.settings:
+        null_fraction = max(0, min(0.2, 1 - (row_count / (row_count + 50))))
+        sample_size = int(row_count * null_fraction)
+        if sample_size:
+            for idx in random.sample(range(row_count), k=sample_size):
+                values[idx] = None
+
+    return values
+
+
+# ---------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------
+def _random_datetime(start_days: int = -365, end_days: int = 0) -> datetime:
+    start = datetime.now() + timedelta(days=start_days)
+    end = datetime.now() + timedelta(days=end_days)
+    delta = end - start
+    random_second = random.randint(0, int(delta.total_seconds()))
+    return start + timedelta(seconds=random_second)

model2data-0.2.2/model2data/generate/relationships.py

@@ -0,0 +1,52 @@
+from typing import Dict, List, Tuple
+
+from model2data.parse.dbml import TableDef
+
+
+# ---------------------------------------------------------
+# Public API
+# ---------------------------------------------------------
+def classify_refs(
+    tables: Dict[str, TableDef],
+    refs: List[Dict],
+) -> Tuple[List[Dict], List[Dict]]:
+    """
+    Classify references into:
+        - fk_refs: Foreign keys (target column looks like a PK)
+        - attribute_refs: Non-FK dependencies (mirroring parent attributes)
+    """
+    fk_refs = []
+    attribute_refs = []
+
+    for ref in refs:
+        target_table = tables.get(ref["target_table"])
+        target_col = None
+        if target_table:
+            target_col = next(
+                (c for c in target_table.columns if c.name == ref["target_column"]), None
+            )
+
+        # FK if target column is a primary key or named "id"
+        if target_col and ("pk" in target_col.settings or target_col.name.lower() == "id"):
+            fk_refs.append(ref)
+        else:
+            attribute_refs.append(ref)
+
+    return fk_refs, attribute_refs
+
+
+def build_fk_lookup(fk_refs: List[Dict]) -> Dict[Tuple[str, str], Tuple[str, str]]:
+    """
+    Build a lookup dictionary for FK relationships.
+
+    Returns:
+        {
+            (child_table, child_column): (parent_table, parent_column)
+        }
+    """
+    lookup: Dict[Tuple[str, str], Tuple[str, str]] = {}
+    for ref in fk_refs:
+        key = (ref["source_table"], ref["source_column"])
+        value = (ref["target_table"], ref["target_column"])
+        lookup[key] = value
+    return lookup

model2data-0.2.2/model2data/parse/__init__.py

@@ -0,0 +1 @@
+"""DBML parsing utilities."""