model2data 0.1.0__tar.gz → 0.1.1__tar.gz

{model2data-0.1.0 → model2data-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: model2data
-Version: 0.1.0
+Version: 0.1.1
 Summary: Generate analytics-ready datasets from DBML models
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
@@ -13,6 +13,7 @@ Requires-Dist: pyyaml>=6.0.3
 Requires-Dist: typer>=0.20.0
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
 Dynamic: license-file
 
 # model2data

{model2data-0.1.0 → model2data-0.1.1}/model2data.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: model2data
-Version: 0.1.0
+Version: 0.1.1
 Summary: Generate analytics-ready datasets from DBML models
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
@@ -13,6 +13,7 @@ Requires-Dist: pyyaml>=6.0.3
 Requires-Dist: typer>=0.20.0
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
 Dynamic: license-file
 
 # model2data

{model2data-0.1.0 → model2data-0.1.1}/model2data.egg-info/SOURCES.txt

@@ -9,11 +9,6 @@ model2data.egg-info/dependency_links.txt
 model2data.egg-info/entry_points.txt
 model2data.egg-info/requires.txt
 model2data.egg-info/top_level.txt
-model2data/dbt/project.py
-model2data/generate/core.py
-model2data/generate/faker.py
-model2data/generate/relationships.py
-model2data/parse/dbml.py
 tests/test_cli_smoke.py
 tests/test_dbml_parser.py
 tests/test_dbt_naming.py

{model2data-0.1.0 → model2data-0.1.1}/model2data.egg-info/requires.txt

@@ -7,3 +7,4 @@ typer>=0.20.0
 
 [dev]
 pytest
+pytest-cov

{model2data-0.1.0 → model2data-0.1.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "model2data"
-version = "0.1.0"
+version = "0.1.1"
 description = "Generate analytics-ready datasets from DBML models"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -20,6 +20,7 @@ dependencies = [
 [project.optional-dependencies]
 dev = [
   "pytest",
+  "pytest-cov",
 ]
 
 [project.scripts]

model2data-0.1.0/model2data/dbt/project.py

@@ -1,74 +0,0 @@
-from pathlib import Path
-import shutil
-import jinja2
-
-TEMPLATES_DIR = Path(__file__).parent / "templates"
-
-
-def create_project_scaffold(dest: Path, project_name: str, profile_name: str) -> None:
-    dest.mkdir(parents=True, exist_ok=True)
-
-    # dbt folders
-    (dest / "models" / "staging").mkdir(parents=True, exist_ok=True)
-    (dest / "seeds" / "raw").mkdir(parents=True, exist_ok=True)
-    (dest / "analysis").mkdir(exist_ok=True)
-    (dest / "macros").mkdir(exist_ok=True)
-    (dest / "tests").mkdir(exist_ok=True)
-    (dest / "snapshots").mkdir(exist_ok=True)
-
-    # dbt_project.yml
-    _render_template(
-        template_name="dbt_project.yml.jinja",
-        output_path=dest / "dbt_project.yml",
-        context={"project_name": project_name, "profile_name": profile_name},
-    )
-
-    # Copy over any macros from templates
-    template_macros_dir = Path("model2data/dbt/templates/macros")
-    if template_macros_dir.exists():
-        for macro_file in template_macros_dir.glob("*.sql"):
-            target_file = dest / "macros" / macro_file.name
-            if not target_file.exists():
-                target_file.write_text(macro_file.read_text())
-
-
-def create_staging_models(dest: Path, project_name: str) -> None:
-    """
-    Creates staging models in models/staging/ folder that reference raw seed tables as sources.
-    """
-    seeds_path = dest / "seeds" / "raw"
-    models_path = dest / "models" / "staging"
-    models_path.mkdir(parents=True, exist_ok=True)
-
-    for csv_file in seeds_path.glob("*.csv"):
-        table_name = csv_file.stem  # keep full seed name, e.g., raw_stories
-        model_file = models_path / f"stg_{table_name}.sql"
-
-        if not model_file.exists():
-            sql_content = f"""\
--- Auto-generated staging model for {table_name}
-select *
-from {{{{ source('raw', '{table_name}') }}}}
-                """
-            model_file.write_text(sql_content)
-
-
-def create_profiles_yml(dest: Path, profile_name: str) -> None:
-    profiles_file = dest / "profiles.yml"
-    if profiles_file.exists():
-        content = profiles_file.read_text()
-        if profile_name in content:
-            return
-    _render_template(
-        template_name="profiles.yml.jinja",
-        output_path=profiles_file,
-        context={"profile_name": profile_name},
-    )
-
-
-def _render_template(template_name: str, output_path: Path, context: dict) -> None:
-    template_path = TEMPLATES_DIR / template_name
-    if not template_path.exists():
-        raise FileNotFoundError(f"Template not found: {template_path}")
-    template = jinja2.Template(template_path.read_text())
-    output_path.write_text(template.render(**context))

model2data-0.1.0/model2data/generate/core.py

@@ -1,171 +0,0 @@
-from __future__ import annotations
-
-import random
-from collections import defaultdict, deque
-from typing import Optional
-
-import pandas as pd
-from faker import Faker
-
-from model2data.generate.faker import generate_column_values
-from model2data.generate.relationships import (
-    classify_refs,
-    build_fk_lookup,
-)
-from model2data.parse.dbml import TableDef
-
-fake = Faker()
-
-
-# ---------------------------------------------------------
-# Public API
-# ---------------------------------------------------------
-def generate_data_from_dbml(
-    tables: dict[str, TableDef],
-    refs: list[dict],
-    base_rows: int = 100,
-    seed: Optional[int] = None,
-) -> dict[str, pd.DataFrame]:
-    """
-    Generate synthetic datasets from parsed DBML definitions.
-
-    This function is deterministic if a seed is provided.
-    It performs no filesystem I/O and returns pandas DataFrames.
-    """
-    if seed is not None:
-        random.seed(seed)
-        Faker.seed(seed)
-
-    # ---------------------------------------------------------
-    # Classify references
-    # ---------------------------------------------------------
-    fk_refs, attribute_refs = classify_refs(tables, refs)
-    fk_lookup = build_fk_lookup(fk_refs)
-
-    # ---------------------------------------------------------
-    # Generate tables in dependency order
-    # ---------------------------------------------------------
-    ordered_tables = _topological_table_order(tables, fk_refs)
-    generated: dict[str, pd.DataFrame] = {}
-
-    for table_name in ordered_tables:
-        table_def = tables[table_name]
-        row_count = _determine_row_count(table_def.name, base_rows)
-
-        data: dict[str, list] = {}
-
-        # -----------------------
-        # First pass: columns + FKs
-        # -----------------------
-        for column in table_def.columns:
-            fk_series = None
-            fk_target = fk_lookup.get((table_name, column.name))
-
-            if fk_target:
-                parent_table, parent_column = fk_target
-                parent_df = generated.get(parent_table)
-                if parent_df is not None and parent_column in parent_df.columns:
-                    fk_series = parent_df[parent_column]
-
-            ensure_unique = "pk" in column.settings
-            data[column.name] = generate_column_values(
-                column=column,
-                row_count=row_count,
-                fk_series=fk_series,
-                ensure_unique=ensure_unique,
-            )
-
-        df = pd.DataFrame(data)
-
-        # -----------------------------------------------------
-        # Second pass: attribute mirroring (non-FK refs)
-        # -----------------------------------------------------
-        for ref in attribute_refs:
-            if ref["source_table"] != table_name:
-                continue
-
-            parent_table = ref["target_table"]
-            parent_column = ref["target_column"]
-            child_column = ref["source_column"]
-
-            parent_df = generated.get(parent_table)
-            if parent_df is None:
-                continue
-
-            # find FK linking child → parent
-            fk_column = next(
-                (
-                    r["source_column"]
-                    for r in fk_refs
-                    if r["source_table"] == table_name
-                    and r["target_table"] == parent_table
-                ),
-                None,
-            )
-
-            if not fk_column or fk_column not in df.columns:
-                continue
-
-            lookup = (
-                parent_df.groupby("id")[parent_column]
-                .first()
-                .to_dict()
-            )
-
-            df[child_column] = df[fk_column].map(lookup)
-
-        generated[table_name] = df
-
-    return generated
-
-
-# ---------------------------------------------------------
-# Internal helpers
-# ---------------------------------------------------------
-def _determine_row_count(table_name: str, base_rows: int) -> int:
-    """
-    Return the base number of rows for all tables.
-    """
-    return base_rows
-
-
-def _topological_table_order(
-    tables: dict[str, TableDef],
-    fk_refs: list[dict],
-) -> list[str]:
-    """
-    Order tables so parent tables are generated before children.
-    """
-    graph: dict[str, set[str]] = defaultdict(set)
-    indegree: dict[str, int] = {name: 0 for name in tables.keys()}
-
-    for ref in fk_refs:
-        parent = ref["target_table"]
-        child = ref["source_table"]
-
-        if parent == child:
-            continue
-        if parent not in tables or child not in tables:
-            continue
-
-        if child not in graph[parent]:
-            graph[parent].add(child)
-            indegree[child] += 1
-
-    queue = deque(sorted(name for name, deg in indegree.items() if deg == 0))
-    order: list[str] = []
-
-    while queue:
-        node = queue.popleft()
-        order.append(node)
-        for neighbor in sorted(graph.get(node, [])):
-            indegree[neighbor] -= 1
-            if indegree[neighbor] == 0:
-                queue.append(neighbor)
-
-    # Safety net for disconnected tables
-    for name in tables.keys():
-        if name not in order:
-            order.append(name)
-
-    return order

model2data-0.1.0/model2data/generate/faker.py

@@ -1,113 +0,0 @@
-from __future__ import annotations
-
-import random
-import uuid
-from datetime import datetime, timedelta
-from typing import Optional
-
-import pandas as pd
-from faker import Faker
-
-from model2data.parse.dbml import ColumnDef
-
-fake = Faker()
-
-
-# ---------------------------------------------------------
-# Public API
-# ---------------------------------------------------------
-def generate_column_values(
-    column: ColumnDef,
-    row_count: int,
-    fk_series: Optional[pd.Series] = None,
-    ensure_unique: bool = False,
-) -> list:
-    """
-    Generate synthetic values for a single column.
-    Respects FKs, uniqueness, and optional min/max hints in column notes.
-    """
-    if fk_series is not None and not fk_series.empty:
-        fk_values = fk_series.tolist()
-        return [random.choice(fk_values) for _ in range(row_count)]
-
-    dtype = column.data_type.lower()
-    base_type = dtype.split("(")[0].strip()
-    values: list = []
-
-    # -----------------------------------------------------
-    # UUIDs / hashes
-    # -----------------------------------------------------
-    if "uuid" in base_type or "hash" in base_type:
-        values = [str(uuid.uuid4()) for _ in range(row_count)]
-
-    # -----------------------------------------------------
-    # Integers
-    # -----------------------------------------------------
-    elif any(key in base_type for key in ["int", "integer", "bigint", "smallint"]):
-        min_val = 0
-        max_val = 100
-        if column.note:
-            if "min" in column.note:
-                min_val = column.note["min"]
-            if "max" in column.note:
-                max_val = column.note["max"]
-        values = [random.randint(min_val, max_val) for _ in range(row_count)]
-
-    # -----------------------------------------------------
-    # Floats / decimals
-    # -----------------------------------------------------
-    elif any(key in base_type for key in ["decimal", "numeric", "float", "double"]):
-        values = [round(random.uniform(0, 10_000), 2) for _ in range(row_count)]
-
-    # -----------------------------------------------------
-    # Booleans
-    # -----------------------------------------------------
-    elif "boolean" in base_type or "bool" in base_type:
-        values = [random.choice([True, False]) for _ in range(row_count)]
-
-    # -----------------------------------------------------
-    # Dates
-    # -----------------------------------------------------
-    elif "date" in base_type and "time" not in base_type:
-        values = [fake.date_between(start_date="-2y", end_date="today") for _ in range(row_count)]
-
-    elif "time" in base_type and "stamp" not in base_type:
-        values = [fake.time() for _ in range(row_count)]
-
-    elif any(key in base_type for key in ["timestamp", "datetime"]):
-        values = [_random_datetime().isoformat(sep=" ") for _ in range(row_count)]
-
-    # -----------------------------------------------------
-    # Fallback to Faker providers
-    # -----------------------------------------------------
-    else:
-        try:
-            values = [fake.format(base_type) for _ in range(row_count)]
-        except:
-            if column.name.lower().endswith("_id") or ensure_unique:
-                values = [str(uuid.uuid4()) for _ in range(row_count)]
-            else:
-                values = [fake.sentence(nb_words=3) for _ in range(row_count)]
-
-    # -----------------------------------------------------
-    # Nullability
-    # -----------------------------------------------------
-    if "not null" not in column.settings:
-        null_fraction = max(0, min(0.2, 1 - (row_count / (row_count + 50))))
-        sample_size = int(row_count * null_fraction)
-        if sample_size:
-            for idx in random.sample(range(row_count), k=sample_size):
-                values[idx] = None
-
-    return values
-
-
-# ---------------------------------------------------------
-# Internal helpers
-# ---------------------------------------------------------
-def _random_datetime(start_days: int = -365, end_days: int = 0) -> datetime:
-    start = datetime.now() + timedelta(days=start_days)
-    end = datetime.now() + timedelta(days=end_days)
-    delta = end - start
-    random_second = random.randint(0, int(delta.total_seconds()))
-    return start + timedelta(seconds=random_second)

model2data-0.1.0/model2data/generate/relationships.py

@@ -1,49 +0,0 @@
-from typing import Tuple, List, Dict
-from model2data.parse.dbml import TableDef
-
-
-# ---------------------------------------------------------
-# Public API
-# ---------------------------------------------------------
-def classify_refs(
-    tables: Dict[str, TableDef],
-    refs: List[Dict],
-) -> Tuple[List[Dict], List[Dict]]:
-    """
-    Classify references into:
-        - fk_refs: Foreign keys (target column looks like a PK)
-        - attribute_refs: Non-FK dependencies (mirroring parent attributes)
-    """
-    fk_refs = []
-    attribute_refs = []
-
-    for ref in refs:
-        target_table = tables.get(ref["target_table"])
-        target_col = None
-        if target_table:
-            target_col = next((c for c in target_table.columns if c.name == ref["target_column"]), None)
-
-        # FK if target column is a primary key or named "id"
-        if target_col and ("pk" in target_col.settings or target_col.name.lower() == "id"):
-            fk_refs.append(ref)
-        else:
-            attribute_refs.append(ref)
-
-    return fk_refs, attribute_refs
-
-
-def build_fk_lookup(fk_refs: List[Dict]) -> Dict[Tuple[str, str], Tuple[str, str]]:
-    """
-    Build a lookup dictionary for FK relationships.
-
-    Returns:
-        {
-            (child_table, child_column): (parent_table, parent_column)
-        }
-    """
-    lookup: Dict[Tuple[str, str], Tuple[str, str]] = {}
-    for ref in fk_refs:
-        key = (ref["source_table"], ref["source_column"])
-        value = (ref["target_table"], ref["target_column"])
-        lookup[key] = value
-    return lookup

model2data-0.1.0/model2data/parse/dbml.py

@@ -1,162 +0,0 @@
-from __future__ import annotations
-from pathlib import Path
-from typing import Optional
-from dataclasses import dataclass, field
-import re
-import json
-import uuid
-
-# -------------------------------
-# Dataclasses
-# -------------------------------
-
-@dataclass
-class ColumnDef:
-    name: str
-    data_type: str
-    settings: set[str] = field(default_factory=set)
-    note: Optional[dict] = None
-
-@dataclass
-class TableDef:
-    name: str
-    columns: list[ColumnDef] = field(default_factory=list)
-
-# -------------------------------
-# Helpers
-# -------------------------------
-
-def _strip_quotes(value: str) -> str:
-    return value.strip().strip('"').strip("'")
-
-def _parse_column_settings(raw: Optional[str]) -> set[str]:
-    if not raw:
-        return set()
-    parts = [part.strip() for part in raw.split(",")]
-    return {part.strip("'").strip('"').lower() for part in parts if part}
-
-def normalize_identifier(value: str) -> str:
-    cleaned = re.sub(r"[^0-9A-Za-z]+", "_", value).strip("_").lower()
-    if not cleaned:
-        cleaned = "table"
-    if cleaned[0].isdigit():
-        cleaned = f"t_{cleaned}"
-    return cleaned
-
-def parse_dbml(dbml_path: Path) -> tuple[dict[str, TableDef], list[dict]]:
-    text = dbml_path.read_text(encoding="utf-8")
-    lines = text.splitlines()
-    tables: dict[str, TableDef] = {}
-    refs: list[dict] = []
-
-    current_table: Optional[TableDef] = None
-    in_indexes_block = False
-    note_block_depth = 0
-    in_ref_block = False  # NEW
-
-    for raw_line in lines:
-        line = raw_line.strip()
-        if not line or line.startswith("//"):
-            continue
-
-        cleaned = line.split("//", 1)[0].strip()
-        if not cleaned:
-            continue
-
-        triple_quote_count = cleaned.count("'''")
-        if triple_quote_count:
-            note_block_depth = (note_block_depth + triple_quote_count) % 2
-            if cleaned.startswith("Note:"):
-                continue
-        if note_block_depth:
-            continue
-
-        # ----------------------
-        # TABLE PARSING
-        # ----------------------
-        if cleaned.lower().startswith("table "):
-            table_name_section = cleaned[6:].split("{", 1)[0].strip()
-            if "[" in table_name_section:
-                table_name_section = table_name_section.split("[", 1)[0].strip()
-            table_name = _strip_quotes(table_name_section)
-            current_table = TableDef(name=table_name)
-            continue
-
-        if current_table:
-            if cleaned.startswith("indexes"):
-                in_indexes_block = True
-                continue
-            if in_indexes_block:
-                if cleaned.endswith("}"):
-                    in_indexes_block = False
-                continue
-            if cleaned.startswith("}"):
-                tables[current_table.name] = current_table
-                current_table = None
-                continue
-            if cleaned.startswith("Note:"):
-                continue
-
-            col_match = re.match(
-                r'(".*?"|`.*?`|[\w]+)\s+([^\[]+?)(?:\s+\[(.+)\])?$',
-                cleaned,
-            )
-            if not col_match:
-                continue
-
-            col_name = _strip_quotes(col_match.group(1))
-            col_type = col_match.group(2).strip()
-            settings = _parse_column_settings(col_match.group(3))
-
-            current_table.columns.append(
-                ColumnDef(
-                    name=col_name,
-                    data_type=col_type,
-                    settings=settings,
-                    note=None,
-                )
-            )
-            continue
-
-        # ----------------------
-        # REF BLOCK START
-        # ----------------------
-        if cleaned.startswith("Ref"):
-            in_ref_block = True
-            continue
-
-        if in_ref_block:
-            if cleaned.startswith("}"):
-                in_ref_block = False
-                continue
-
-            # Match: "table"."column" > "table"."column"
-            ref_match = re.match(
-                r'(".*?"|`.*?`|[\w]+)\.(".*?"|`.*?`|[\w]+)\s*([<>])\s*'
-                r'(".*?"|`.*?`|[\w]+)\.(".*?"|`.*?`|[\w]+)',
-                cleaned,
-            )
-            if not ref_match:
-                continue
-
-            left_table, left_column, operator, right_table, right_column = ref_match.groups()
-
-            # Ignore <> and other non-FK relations
-            if operator not in (">", "<"):
-                continue
-
-            if operator == "<":
-                left_table, right_table = right_table, left_table
-                left_column, right_column = right_column, left_column
-
-            refs.append(
-                {
-                    "source_table": _strip_quotes(left_table),
-                    "source_column": _strip_quotes(left_column),
-                    "target_table": _strip_quotes(right_table),
-                    "target_column": _strip_quotes(right_column),
-                }
-            )
-            continue
-
-    return tables, refs