model2data 0.4.3__tar.gz → 0.5.0__tar.gz

{model2data-0.4.3 → model2data-0.5.0}/PKG-INFO

@@ -1,18 +1,18 @@
 Metadata-Version: 2.4
 Name: model2data
-Version: 0.4.3
+Version: 0.5.0
 Summary: Generate analytics-ready datasets from DBML models
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: dbt-core>=1.5.0
-Requires-Dist: dbt-duckdb>=1.5.0
+Requires-Dist: dbt-core>=1.8.5
+Requires-Dist: dbt-duckdb>=1.8.4
 Requires-Dist: faker>=37.12.0
 Requires-Dist: pandas>=2.3.3
 Requires-Dist: pyyaml>=6.0.3
 Requires-Dist: typer>=0.20.0
 Provides-Extra: postgres
-Requires-Dist: dbt-postgres>=1.5.0; extra == "postgres"
+Requires-Dist: dbt-postgres>=1.8.0; extra == "postgres"
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
@@ -177,7 +177,13 @@ Pass `--unit-tests` to also generate deterministic dbt unit test fixtures (`mode
 model2data --file examples/hackernews.dbml --rows 200 --seed 42 --unit-tests
 ```
 
-This targets dbt-core's native unit testing feature, which requires **dbt-core >= 1.8**. Since this project only requires `dbt-core>=1.5.0` by default, unit test generation is opt-in — leave the flag off if your dbt-core version is older.
+This targets dbt-core's native unit testing feature, which requires dbt-core >= 1.8 — already
+covered by this project's `dbt-core>=1.8.5` floor, so `--unit-tests` works with the base install.
+Note: on dbt-core versions in the 1.8.x line specifically, a DBML column named after a SQL
+reserved word (e.g. `by`, as in `examples/hackernews.dbml`) can fail unit test execution with a
+`syntax error` — a dbt-core-internal identifier-quoting limitation in its unit test fixture
+rendering for that release line, not something under model2data's control. It's fixed in later
+dbt-core versions; every other `--unit-tests` path works fine on 1.8.x.
 
 ---
 

{model2data-0.4.3 → model2data-0.5.0}/README.md

@@ -152,7 +152,13 @@ Pass `--unit-tests` to also generate deterministic dbt unit test fixtures (`mode
 model2data --file examples/hackernews.dbml --rows 200 --seed 42 --unit-tests
 ```
 
-This targets dbt-core's native unit testing feature, which requires **dbt-core >= 1.8**. Since this project only requires `dbt-core>=1.5.0` by default, unit test generation is opt-in — leave the flag off if your dbt-core version is older.
+This targets dbt-core's native unit testing feature, which requires dbt-core >= 1.8 — already
+covered by this project's `dbt-core>=1.8.5` floor, so `--unit-tests` works with the base install.
+Note: on dbt-core versions in the 1.8.x line specifically, a DBML column named after a SQL
+reserved word (e.g. `by`, as in `examples/hackernews.dbml`) can fail unit test execution with a
+`syntax error` — a dbt-core-internal identifier-quoting limitation in its unit test fixture
+rendering for that release line, not something under model2data's control. It's fixed in later
+dbt-core versions; every other `--unit-tests` path works fine on 1.8.x.
 
 ---
 

{model2data-0.4.3 → model2data-0.5.0}/model2data/cli.py

@@ -12,9 +12,9 @@ from model2data.dbt.project import (
     create_staging_models,
 )
 from model2data.dbt.tests import generate_dbt_yml, generate_unit_tests
-from model2data.generate.core import generate_data_from_dbml
+from model2data.generate.core import generate_data_from_dbml, get_cyclic_tables
 from model2data.generate.faker import get_unmapped_columns, reset_stats
-from model2data.parse.dbml import parse_dbml
+from model2data.parse.dbml import get_parse_warnings, parse_dbml
 from model2data.utils import normalize_identifier
 
 SUPPORTED_ADAPTERS = ("duckdb", "postgres")
@@ -111,6 +111,7 @@ def main(
     # Parse DBML (names untouched)
     # -------------------------
     tables, refs = parse_dbml(file)
+    parse_warnings = get_parse_warnings()
     if not tables:
         typer.echo("❌ No tables found in the provided DBML file.")
         raise typer.Exit(1)
@@ -175,6 +176,7 @@ def main(
     # -------------------------
     total_rows = sum(len(df) for df in generated_tables.values())
     unmapped = get_unmapped_columns()
+    cyclic_tables = get_cyclic_tables()
 
     typer.echo("\n📊 Summary")
     typer.echo(f"  Tables generated:        {len(generated_tables)}")
@@ -186,6 +188,15 @@ def main(
             typer.echo(f"    - {col_name} ({data_type})")
     else:
         typer.echo("  Columns using generic fallback text: 0")
+    if cyclic_tables:
+        typer.echo(
+            "  ⚠️  Tables in an unresolved FK cycle (data may not respect "
+            f"all relationships): {', '.join(cyclic_tables)}"
+        )
+    if parse_warnings:
+        typer.echo(f"  ⚠️  DBML lines model2data could not fully parse: {len(parse_warnings)}")
+        for warning in parse_warnings:
+            typer.echo(f"    - {warning}")
 
     # -------------------------
     # Done

{model2data-0.4.3 → model2data-0.5.0}/model2data/dbt/project.py

@@ -45,10 +45,14 @@ def create_staging_models(dest: Path, project_name: str) -> None:
         model_file = models_path / f"stg_{table_name}.sql"
 
         if not model_file.exists():
+            # table_name is spliced into a single-quoted Jinja string literal;
+            # escape any embedded single quote so a DBML identifier containing
+            # one can't break the source() call.
+            escaped_name = table_name.replace("'", "\\'")
             sql_content = f"""\
 -- Auto-generated staging model for {table_name}
 select *
-from {{{{ source('raw', '{table_name}') }}}}
+from {{{{ source('raw', '{escaped_name}') }}}}
                 """
             model_file.write_text(sql_content)
 

{model2data-0.4.3 → model2data-0.5.0}/model2data/dbt/tests.py

@@ -10,10 +10,16 @@ from model2data.generate.faker import is_free_text_type
 from model2data.generate.relationships import classify_refs
 
 
-def _yaml_field_lines(key: str, value: str, indent: str) -> list[str]:
-    """Render `key: value` as safely-escaped YAML lines at the given indent."""
-    dumped = yaml.safe_dump({key: value}, default_flow_style=False, sort_keys=False)
-    return [f"{indent}{line}" for line in dumped.strip("\n").splitlines()]
+def _dump_yaml(data: dict) -> str:
+    """Dump a plain Python structure to YAML, safely escaping every value.
+
+    Every table/column/description string that ends up in generated YAML
+    goes through this single choke point instead of being hand-interpolated
+    into f-string lines, so an arbitrary (but valid) DBML identifier --
+    containing a space, colon, quote, etc. -- can never produce invalid or
+    silently-misparsed YAML.
+    """
+    return yaml.safe_dump(data, default_flow_style=False, sort_keys=False)
 
 
 def generate_dbt_yml(dest: Path, tables: dict, refs: list[dict], source_name: str = "hackernews"):
@@ -53,20 +59,25 @@ def generate_dbt_yml(dest: Path, tables: dict, refs: list[dict], source_name: st
     # -------------------------
     # Generate __sources.yml
     # -------------------------
-    sources_lines = ["version: 2", "", "sources:"]
-    sources_lines.append("  - name: raw")
-    sources_lines.append("    schema: raw")
-    sources_lines.append(f"    description: {source_name.capitalize()} raw seed data")
-    sources_lines.append("    tables:")
-
+    source_tables = []
     for table in tables.values():
         seed_name = table.name  # keep exact name
         table_desc = getattr(table, "description", None) or f"Table {seed_name}"
-        sources_lines.append(f"      - name: {seed_name}")
-        sources_lines.extend(_yaml_field_lines("description", table_desc, "        "))
-
+        source_tables.append({"name": seed_name, "description": table_desc})
+
+    sources_doc = {
+        "version": 2,
+        "sources": [
+            {
+                "name": "raw",
+                "schema": "raw",
+                "description": f"{source_name.capitalize()} raw seed data",
+                "tables": source_tables,
+            }
+        ],
+    }
     sources_file = staging_path / "__sources.yml"
-    sources_file.write_text("\n".join(sources_lines))
+    sources_file.write_text(_dump_yaml(sources_doc))
 
     # -------------------------
     # Generate individual staging model YAMLs
@@ -97,44 +108,22 @@ def generate_dbt_yml(dest: Path, tables: dict, refs: list[dict], source_name: st
                     }
                 )
 
-            model_columns.append(
-                {
-                    "name": col.name,
-                    "description": getattr(col, "description", None),
-                    "tests": tests if tests else None,
-                }
-            )
-
-        # Render model YAML
-        lines = ["version: 2", "", "models:"]
-        lines.append(f"  - name: {stg_name}")
-        lines.append("    columns:")
-        for col in model_columns:
-            lines.append(f"      - name: {col['name']}")
-            if col["description"]:
-                lines.extend(_yaml_field_lines("description", col["description"], "        "))
-            if col["tests"]:
-                lines.append("        tests:")
-                for test in col["tests"]:
-                    if isinstance(test, str):
-                        lines.append(f"          - {test}")
-                    elif "accepted_values" in test:
-                        lines.append("          - accepted_values:")
-                        lines.append("              arguments:")
-                        lines.append("                values:")
-                        for value in test["accepted_values"]["values"]:
-                            lines.append(f"                  - {value!r}")
-                    else:
-                        # relationships test with arguments
-                        for k, v in test.items():
-                            lines.append(f"          - {k}:")
-                            lines.append("              arguments:")
-                            for fk_key, fk_val in v.items():
-                                lines.append(f"                {fk_key}: {fk_val}")
+            col_doc: dict[str, Any] = {"name": col.name}
+            description = getattr(col, "description", None)
+            if description:
+                col_doc["description"] = description
+            if tests:
+                col_doc["tests"] = tests
+            model_columns.append(col_doc)
+
+        model_doc = {
+            "version": 2,
+            "models": [{"name": stg_name, "columns": model_columns}],
+        }
 
         # Write YAML to same folder as SQL model
         yml_file = staging_path / f"{stg_name}.yml"
-        yml_file.write_text("\n".join(lines))
+        yml_file.write_text(_dump_yaml(model_doc))
 
     # -------------------------
     # Composite key singular tests
@@ -155,8 +144,7 @@ def _generate_seed_config(dest: Path, tables: dict) -> None:
     zero-padded postcodes, ...) and would otherwise be silently loaded as
     an integer, overflowing or dropping leading zeros.
     """
-    lines = ["version: 2", "", "seeds:"]
-    any_column_types = False
+    seed_entries = []
 
     for table in tables.values():
         column_types = {
@@ -165,19 +153,28 @@ def _generate_seed_config(dest: Path, tables: dict) -> None:
         if not column_types:
             continue
 
-        any_column_types = True
-        lines.append(f"  - name: {table.name}")
-        lines.append("    config:")
-        lines.append("      column_types:")
-        for col_name, col_type in column_types.items():
-            lines.append(f"        {col_name}: {col_type}")
+        seed_entries.append({"name": table.name, "config": {"column_types": column_types}})
 
-    if not any_column_types:
+    if not seed_entries:
         return
 
+    seeds_doc = {"version": 2, "seeds": seed_entries}
+
     seed_raw_path = dest / "seeds" / "raw"
     seed_raw_path.mkdir(parents=True, exist_ok=True)
-    (seed_raw_path / "__seed_config.yml").write_text("\n".join(lines))
+    (seed_raw_path / "__seed_config.yml").write_text(_dump_yaml(seeds_doc))
+
+
+def _quote_sql_identifier(name: str) -> str:
+    """ANSI double-quote a raw column identifier for use in generated SQL.
+
+    Both supported adapters (DuckDB and Postgres) accept ANSI double-quoting,
+    which is required once a DBML identifier contains a space, colon, or
+    other character that would otherwise break an unquoted `select`/`group
+    by` clause. A literal `"` inside the identifier is escaped by doubling,
+    the standard ANSI SQL convention.
+    """
+    return '"' + name.replace('"', '""') + '"'
 
 
 def _generate_composite_key_tests(dest: Path, tables: dict) -> None:
@@ -193,7 +190,8 @@ def _generate_composite_key_tests(dest: Path, tables: dict) -> None:
             if len(columns) < 2:
                 continue
 
-            columns_csv = ", ".join(columns)
+            quoted_columns = [_quote_sql_identifier(c) for c in columns]
+            columns_csv = ", ".join(quoted_columns)
             test_name = "unique_combination_" + "_".join([stg_name, *columns])
             sql = (
                 f"select {columns_csv}, count(*) as n\n"
@@ -230,6 +228,11 @@ def generate_unit_tests(
 
         sample_rows = _rows_as_native_dicts(df.head(sample_size))
         stg_name = f"stg_{table.name}"
+        # table.name is spliced into a single-quoted Jinja string literal (the
+        # source() call is evaluated by dbt as an expression, not treated as
+        # a literal YAML string); escape any embedded single quote so an
+        # unusual DBML identifier can't break that call.
+        escaped_name = table.name.replace("'", "\\'")
 
         unit_test = {
             "unit_tests": [
@@ -238,7 +241,7 @@ def generate_unit_tests(
                     "model": stg_name,
                     "given": [
                         {
-                            "input": f"source('raw', '{table.name}')",
+                            "input": f"source('raw', '{escaped_name}')",
                             "rows": sample_rows,
                         }
                     ],

{model2data-0.4.3 → model2data-0.5.0}/model2data/generate/core.py

@@ -16,6 +16,23 @@ from model2data.parse.dbml import TableDef
 
 fake = Faker()
 
+# Tables the most recent generate_data_from_dbml() call found stuck in an
+# unresolved FK cycle (never reached indegree 0 during the topological
+# sort). Exposed out-of-band, mirroring generate.faker's
+# reset_stats()/get_unmapped_columns() pattern, so the CLI can surface a
+# warning without changing this module's existing return signature.
+_cycle_state: dict[str, list[str]] = {"cyclic_tables": []}
+
+
+def reset_cycle_state() -> None:
+    """Clear the record of tables found in an unresolved FK cycle."""
+    _cycle_state["cyclic_tables"] = []
+
+
+def get_cyclic_tables() -> list[str]:
+    """Return table names stuck in an unresolved FK cycle by the last run."""
+    return list(_cycle_state["cyclic_tables"])
+
 
 # ---------------------------------------------------------
 # Public API
@@ -36,6 +53,8 @@ def generate_data_from_dbml(
         random.seed(seed)
         Faker.seed(seed)
 
+    reset_cycle_state()
+
     # ---------------------------------------------------------
     # Classify references
     # ---------------------------------------------------------
@@ -259,9 +278,17 @@ def _topological_table_order(
             if indegree[neighbor] == 0:
                 queue.append(neighbor)
 
-    # Safety net for disconnected tables
-    for name in tables.keys():
-        if name not in order:
-            order.append(name)
+    # Any table not reached by the Kahn's-algorithm pass above never had its
+    # indegree reduced to 0, which (unlike a genuinely disconnected table,
+    # which starts at indegree 0 and is processed by the loop above) can only
+    # happen if it sits inside -- or depends on -- an unresolved multi-table
+    # FK cycle. Append it to the order anyway (still generate *something*
+    # rather than crash on an unusual-but-not-invalid schema), but record it
+    # so the CLI can warn the user their generated FK data may not respect
+    # every relationship.
+    leftover = sorted(name for name in tables if name not in order)
+    if leftover:
+        _cycle_state["cyclic_tables"] = leftover
+    order.extend(leftover)
 
     return order