dbt-4ps-generator 0.1.1__tar.gz

dbt_4ps_generator-0.1.1/.gitignore

@@ -0,0 +1,10 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

dbt_4ps_generator-0.1.1/.python-version

@@ -0,0 +1 @@
+3.13

dbt_4ps_generator-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Misja Pronk
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dbt_4ps_generator-0.1.1/PKG-INFO

@@ -0,0 +1,87 @@
+Metadata-Version: 2.4
+Name: dbt-4ps-generator
+Version: 0.1.1
+Summary: Generate dbt staging models for 4PS Construct (Business Central) from bc2adls CDM manifests
+Project-URL: Repository, https://github.com/misja-pronk/dbt-4ps-staging
+Project-URL: Documentation, https://misja-pronk.github.io/dbt-4ps-staging/
+Project-URL: Changelog, https://github.com/misja-pronk/dbt-4ps-staging/blob/main/CHANGELOG.md
+Author: Misja Pronk
+License-Expression: MIT
+License-File: LICENSE
+Keywords: 4ps,bc2adls,business-central,cdm,databricks,dbt
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Database
+Requires-Python: >=3.13
+Requires-Dist: databricks-sdk>=0.50.0
+Requires-Dist: humps>=0.2.2
+Requires-Dist: pydantic>=2.11.3
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: rich>=13.0.0
+Requires-Dist: shandy-sqlfmt[jinjafmt]>=0.26.0
+Requires-Dist: sqlglot>=26.15.0
+Requires-Dist: typer>=0.15.2
+Description-Content-Type: text/markdown
+
+# dbt-4ps-generator
+
+[![PyPI](https://img.shields.io/pypi/v/dbt-4ps-generator.svg)](https://pypi.org/project/dbt-4ps-generator/)
+
+CLI that generates dbt staging models from a [bc2adls](https://github.com/Bertverbeek4PS/bc2adls) CDM export. For every entity in the manifest it writes:
+
+- `stg_4ps__<table>.sql` — a `SELECT ... FROM STREAM READ_FILES(...)` model with the column types from the CDM metadata and columns renamed to snake_case (`CreditLimitLCY-20` → `credit_limit_lcy`)
+- `_4ps__models.yaml` — model + column descriptions, and a `dbt_utils.unique_combination_of_columns` test on each table's primary key
+
+Full documentation: <https://misja-pronk.github.io/dbt-4ps-staging/>
+
+## Install
+
+```bash
+uvx dbt-4ps-generator --help          # run once, ephemerally
+uv tool install dbt-4ps-generator     # install the command on PATH
+```
+
+Or from a checkout: `uv run dbt-4ps-generator` in this directory.
+
+## Usage
+
+Download the CDM metadata from the Unity Catalog volume bc2adls exports to (authenticates via `DATABRICKS_HOST`/`DATABRICKS_TOKEN` or a `~/.databrickscfg` profile):
+
+```bash
+dbt-4ps-generator download \
+  --volume-path /Volumes/<catalog>/<schema>/<volume> \
+  --output-directory ../_cdm
+```
+
+Generate the models:
+
+```bash
+dbt-4ps-generator generate \
+  --manifest ../_cdm/deltas.manifest.cdm.json \
+  --output-directory ../dbt_4ps_staging_package/models/staging/4ps
+```
+
+Options (see `--help`): `--model-prefix` (default `stg_4ps__`), `--schema-prefix` (default `_4ps__`), and `--no-file-metadata-column` to omit the `_metadata` source-file column.
+
+## Development
+
+```bash
+uv sync
+uv run pytest
+uv run ruff check . && uv run ruff format --check .
+uv run ty check
+```
+
+## Layout
+
+| File (`src/dbt_4ps_generator/`) | Purpose |
+|---|---|
+| `cli.py` | Typer CLI (`generate`, `download`) |
+| `cdm.py` | Pydantic models for the CDM manifest and entity files |
+| `converters.py` | CDM → dbt/SQL conversions (types, names, READ_FILES options) |
+| `dbt.py` | Builders that write the .sql models and schema yaml |
+| `sql.py` | SQL primitives (columns, options, streaming-table DDL builder) |
+| `databricks_helper.py` | Volume download via the Databricks SDK |

dbt_4ps_generator-0.1.1/README.md

@@ -0,0 +1,59 @@
+# dbt-4ps-generator
+
+[![PyPI](https://img.shields.io/pypi/v/dbt-4ps-generator.svg)](https://pypi.org/project/dbt-4ps-generator/)
+
+CLI that generates dbt staging models from a [bc2adls](https://github.com/Bertverbeek4PS/bc2adls) CDM export. For every entity in the manifest it writes:
+
+- `stg_4ps__<table>.sql` — a `SELECT ... FROM STREAM READ_FILES(...)` model with the column types from the CDM metadata and columns renamed to snake_case (`CreditLimitLCY-20` → `credit_limit_lcy`)
+- `_4ps__models.yaml` — model + column descriptions, and a `dbt_utils.unique_combination_of_columns` test on each table's primary key
+
+Full documentation: <https://misja-pronk.github.io/dbt-4ps-staging/>
+
+## Install
+
+```bash
+uvx dbt-4ps-generator --help          # run once, ephemerally
+uv tool install dbt-4ps-generator     # install the command on PATH
+```
+
+Or from a checkout: `uv run dbt-4ps-generator` in this directory.
+
+## Usage
+
+Download the CDM metadata from the Unity Catalog volume bc2adls exports to (authenticates via `DATABRICKS_HOST`/`DATABRICKS_TOKEN` or a `~/.databrickscfg` profile):
+
+```bash
+dbt-4ps-generator download \
+  --volume-path /Volumes/<catalog>/<schema>/<volume> \
+  --output-directory ../_cdm
+```
+
+Generate the models:
+
+```bash
+dbt-4ps-generator generate \
+  --manifest ../_cdm/deltas.manifest.cdm.json \
+  --output-directory ../dbt_4ps_staging_package/models/staging/4ps
+```
+
+Options (see `--help`): `--model-prefix` (default `stg_4ps__`), `--schema-prefix` (default `_4ps__`), and `--no-file-metadata-column` to omit the `_metadata` source-file column.
+
+## Development
+
+```bash
+uv sync
+uv run pytest
+uv run ruff check . && uv run ruff format --check .
+uv run ty check
+```
+
+## Layout
+
+| File (`src/dbt_4ps_generator/`) | Purpose |
+|---|---|
+| `cli.py` | Typer CLI (`generate`, `download`) |
+| `cdm.py` | Pydantic models for the CDM manifest and entity files |
+| `converters.py` | CDM → dbt/SQL conversions (types, names, READ_FILES options) |
+| `dbt.py` | Builders that write the .sql models and schema yaml |
+| `sql.py` | SQL primitives (columns, options, streaming-table DDL builder) |
+| `databricks_helper.py` | Volume download via the Databricks SDK |

dbt_4ps_generator-0.1.1/pyproject.toml

@@ -0,0 +1,74 @@
+[project]
+name = "dbt-4ps-generator"
+version = "0.1.1"
+description = "Generate dbt staging models for 4PS Construct (Business Central) from bc2adls CDM manifests"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Misja Pronk" }]
+requires-python = ">=3.13"
+keywords = ["dbt", "databricks", "bc2adls", "business-central", "4ps", "cdm"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Database",
+]
+dependencies = [
+    "databricks-sdk>=0.50.0",
+    "humps>=0.2.2",
+    "pydantic>=2.11.3",
+    "pyyaml>=6.0.2",
+    "rich>=13.0.0",
+    "shandy-sqlfmt[jinjafmt]>=0.26.0",
+    "sqlglot>=26.15.0",
+    "typer>=0.15.2",
+]
+
+[project.urls]
+Repository = "https://github.com/misja-pronk/dbt-4ps-staging"
+Documentation = "https://misja-pronk.github.io/dbt-4ps-staging/"
+Changelog = "https://github.com/misja-pronk/dbt-4ps-staging/blob/main/CHANGELOG.md"
+
+[project.scripts]
+dbt-4ps-generator = "dbt_4ps_generator.cli:app"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dbt_4ps_generator"]
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0.0",
+    "ruff>=0.6",
+    "ty>=0.0.1a1",
+]
+docs = [
+    "mkdocs-material>=9.7.5",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-q"
+
+[tool.ruff]
+line-length = 100
+target-version = "py313"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM"]
+
+[tool.ruff.lint.per-file-ignores]
+# typer's idiom puts typer.Option(...) in argument defaults
+"src/dbt_4ps_generator/cli.py" = ["B008"]
+
+[tool.ty.environment]
+python-version = "3.13"
+root = ["src"]
+extra-paths = ["tests"]

dbt_4ps_generator-0.1.1/src/dbt_4ps_generator/__init__.py

@@ -0,0 +1 @@
+"""Generate dbt staging models for 4PS Construct from bc2adls CDM manifests."""

dbt_4ps_generator-0.1.1/src/dbt_4ps_generator/cdm.py

@@ -0,0 +1,137 @@
+import json
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+
+class Argument(BaseModel):
+    name: str
+    value: str
+
+
+class PatternTraitReference(BaseModel):
+    trait_reference: str = Field(alias="traitReference")
+    arguments: list[Argument]
+
+
+class DataPartitionPattern(BaseModel):
+    name: str
+    root_location: str = Field(alias="rootLocation")
+    glob_pattern: str = Field(alias="globPattern")
+    exhibits_traits: list[PatternTraitReference] = Field(alias="exhibitsTraits")
+
+
+class ManifestEntity(BaseModel):
+    type: str
+    entity_name: str = Field(alias="entityName")
+    entity_path: str = Field(alias="entityPath")
+    data_partition_patterns: list[DataPartitionPattern] = Field(
+        default=[], alias="dataPartitionPatterns"
+    )
+
+    def get_entity_name(self) -> str:
+        return self.entity_name
+
+    def get_entity_path(self) -> str:
+        return self.entity_path
+
+    def get_root_location(self) -> str:
+        root_location = self.data_partition_patterns[0].root_location
+        return root_location
+
+    def get_glob_pattern(self) -> str:
+        glob_pattern = self.data_partition_patterns[0].glob_pattern
+        return glob_pattern
+
+    def get_trait_reference(self) -> str:
+        trait_reference = self.data_partition_patterns[0].exhibits_traits[0].trait_reference
+        return trait_reference
+
+    def get_arguments(self) -> list[Argument]:
+        arguments = self.data_partition_patterns[0].exhibits_traits[0].arguments
+        return arguments
+
+
+class Manifest(BaseModel):
+    json_schema_semantic_version: str = Field(alias="jsonSchemaSemanticVersion")
+    imports: list[str]
+    manifest_name: str = Field(alias="manifestName")
+    explanation: str
+    entities: list[ManifestEntity] = Field(alias="entities")
+    relationships: list[str]
+
+
+class AppliedTrait(BaseModel):
+    trait_reference: str = Field(alias="traitReference")
+    arguments: list[dict]
+
+
+class Attribute(BaseModel):
+    name: str
+    data_format: str = Field(alias="dataFormat")
+    applied_traits: list[AppliedTrait] = Field(default=[], alias="appliedTraits")
+    display_name: str = Field(alias="displayName")
+    maximum_length: int | None = Field(alias="maximumLength")
+    is_primary_key: bool = Field(default=False, alias="isPrimaryKey")
+
+
+class Definition(BaseModel):
+    entity_name: str = Field(alias="entityName")
+    exhibits_traits: list[str] = Field(default=[], alias="exhibitsTraits")
+    display_name: str = Field(alias="displayName")
+    description: str
+    has_attributes: list[Attribute] = Field(alias="hasAttributes")
+
+
+class ModelEntity(BaseModel):
+    json_schema_semantic_version: str = Field(alias="jsonSchemaSemanticVersion")
+    imports: list[dict]
+    definitions: list[Definition]
+
+    def get_description(self) -> str:
+        description = self.definitions[0].description
+        return description
+
+    def get_attributes(self) -> list[Attribute]:
+        attributes = self.definitions[0].has_attributes
+        return attributes
+
+    def get_display_name(self) -> str:
+        display_name = self.definitions[0].display_name
+        return display_name
+
+
+class Entity(BaseModel):
+    root_location: str
+    glob_pattern: str
+    description: str
+    attributes: list[Attribute]
+    entity_name: str
+    entity_path: str
+    trait_reference: str
+    arguments: list[Argument]
+    display_name: str
+
+
+def build_entities(manifest_path: Path) -> list[Entity]:
+    entities = []
+    manifest = Manifest(**json.loads(manifest_path.read_text()))
+    for manifest_entity in manifest.entities:
+        model_entity_path = manifest_path.parent / Path(manifest_entity.entity_path.split("/")[0])
+        model_entity = ModelEntity(**json.loads(model_entity_path.read_text()))
+
+        entity = Entity(
+            trait_reference=manifest_entity.get_trait_reference(),
+            root_location=manifest_entity.get_root_location(),
+            description=model_entity.get_description(),
+            glob_pattern=manifest_entity.get_glob_pattern(),
+            entity_name=manifest_entity.get_entity_name(),
+            display_name=model_entity.get_display_name(),
+            entity_path=manifest_entity.get_entity_path(),
+            arguments=manifest_entity.get_arguments(),
+            attributes=model_entity.get_attributes(),
+        )
+
+        entities.append(entity)
+
+    return entities

dbt_4ps_generator-0.1.1/src/dbt_4ps_generator/cli.py

@@ -0,0 +1,109 @@
+from importlib.metadata import version as pkg_version
+from pathlib import Path
+
+import typer
+
+from dbt_4ps_generator.cdm import build_entities
+from dbt_4ps_generator.converters import (
+    argument_to_option,
+    attribute_to_column,
+    construct_path,
+    to_snake_case_with_column_number_removed,
+    trait_reference_to_format,
+)
+from dbt_4ps_generator.databricks_helper import download_cdm_files_from_databricks_volume
+from dbt_4ps_generator.dbt import ModelBuilder, ModelsSchemaBuilder
+
+app = typer.Typer(help="Generate dbt staging models from bc2adls CDM manifests.")
+
+
+def _version_callback(value: bool):
+    if value:
+        typer.echo(pkg_version("dbt-4ps-generator"))
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    version: bool = typer.Option(
+        False, "--version", callback=_version_callback, is_eager=True, help="Show the version."
+    ),
+):
+    pass
+
+
+@app.command()
+def generate(
+    manifest: Path = typer.Option(
+        ...,
+        help="Path to the CDM manifest, e.g. _cdm/deltas.manifest.cdm.json. "
+        "Entity .cdm.json files are resolved relative to this file.",
+        exists=True,
+        dir_okay=False,
+    ),
+    output_directory: Path = typer.Option(
+        ...,
+        help="Directory the .sql models and schema yaml are written to, "
+        "e.g. dbt_4ps_staging_package/models/staging/4ps.",
+        file_okay=False,
+    ),
+    model_prefix: str = typer.Option("stg_4ps__", help="Prefix for generated model names."),
+    schema_name: str = typer.Option("models", help="Base name of the generated schema yaml."),
+    schema_prefix: str = typer.Option("_4ps__", help="Prefix for the generated schema yaml."),
+    file_metadata_column: bool = typer.Option(
+        True, help="Add the _metadata column (source file info) to every model."
+    ),
+):
+    """Generate one staging model per entity in the manifest, plus a schema yaml."""
+    output_directory.mkdir(parents=True, exist_ok=True)
+
+    entities = build_entities(manifest)
+    configuration_builder = (
+        ModelsSchemaBuilder()
+        .name(name=schema_name, prefix=schema_prefix)
+        .output_directory(output_directory)
+    )
+
+    for entity in entities:
+        model_builder = (
+            ModelBuilder()
+            .name(name=entity.entity_name, prefix=model_prefix)
+            .description(entity.description)
+            .add_rename_function(to_snake_case_with_column_number_removed)
+            .add_options([argument_to_option(a) for a in entity.arguments])
+            .format(trait_reference_to_format(entity.trait_reference))
+            .add_columns([attribute_to_column(a) for a in entity.attributes])
+            .path(
+                construct_path(root_location=entity.root_location, glob_pattern=entity.glob_pattern)
+            )
+            .add_file_metadata_column(file_metadata_column)
+            .output_directory(output_directory)
+            .build()
+        )
+        configuration_builder.add_model_builder(model_builder)
+
+    configuration_builder.build()
+    typer.echo(f"Generated {len(entities)} models in {output_directory}")
+
+
+@app.command()
+def download(
+    volume_path: str = typer.Option(
+        ...,
+        help="Unity Catalog volume path holding the bc2adls export, "
+        "e.g. /Volumes/<catalog>/<schema>/<volume>.",
+    ),
+    output_directory: Path = typer.Option(
+        ..., help="Local directory to download the CDM json files into.", file_okay=False
+    ),
+):
+    """Download CDM manifest and entity files from a Databricks volume.
+
+    Authenticates via the Databricks SDK's default chain: set DATABRICKS_HOST and
+    DATABRICKS_TOKEN, or use a configured profile (DATABRICKS_CONFIG_PROFILE).
+    """
+    download_cdm_files_from_databricks_volume(volume_path, output_directory)
+
+
+if __name__ == "__main__":
+    app()

dbt_4ps_generator-0.1.1/src/dbt_4ps_generator/converters.py

@@ -0,0 +1,110 @@
+import re
+
+from dbt_4ps_generator.cdm import Argument, Attribute
+from dbt_4ps_generator.sql import Column, Option
+
+
+def to_snake_case_with_column_number_removed(input_string):
+    # Remove trailing hyphen + digits first
+    input_string = re.sub(r"[-_]\d+$", "", input_string)
+
+    # Find all words correctly
+    words = re.findall(
+        r"(?:[A-Z]+(?=[A-Z][a-z]))|(?:[A-Z]?[a-z]+)|(?:[A-Z]+)|(?:\d+)", input_string
+    )
+
+    # Join with underscores and lowercase
+    to_snake_case = "_".join(map(str.lower, words))
+    return to_snake_case
+
+
+def trait_reference_to_format(trait_reference: str):
+    return trait_reference.split(".")[-1].lower()
+
+
+def argument_to_option(argument: Argument) -> Option:
+    match argument.name:
+        case "columnHeaders":
+            return Option(key="header", value=argument.value)
+
+        case "delimiter":
+            return Option(key="sep", value=argument.value)
+
+        case "encoding":
+            return Option(key="encoding", value=argument.value.upper())
+
+        case "escape":
+            # double backslashes so the value survives SQL string-literal parsing
+            return Option(key="escape", value=argument.value.replace("\\", "\\\\"))
+        case _:
+            return Option(key=argument.name, value=argument.value)
+
+
+def attribute_to_column(attribute: Attribute) -> Column:
+    match attribute.data_format.lower():
+        case "string":
+            type = "string"
+        case "int32":
+            type = "int"
+        case "decimal":
+            precision = None
+            scale = None
+
+            if attribute.maximum_length:
+                precision = attribute.maximum_length
+
+            for applied_trait in attribute.applied_traits:
+                if applied_trait.trait_reference == "is.dataFormat.numeric.shaped":
+                    for argument in applied_trait.arguments:
+                        if argument["name"] == "scale":
+                            scale = argument["value"]
+                        break
+
+            if precision and scale:
+                type = f"decimal({precision},{scale})"
+            elif precision:
+                type = f"decimal({precision})"
+            else:
+                type = "string"
+
+        case "double":
+            type = "double"
+        case "option":
+            type = "string"
+        case "int64":
+            type = "bigint"
+        case "boolean":
+            type = "boolean"
+        case "integer" | "interger":  # tolerate the misspelled variant seen in some exports
+            type = "int"
+        case "time":
+            type = "string"
+        case "code":
+            type = "string"
+        case "date":
+            type = "date"
+        case "datetime":
+            type = "timestamp"
+        case "datetimeoffset":
+            type = "timestamp"
+        case "duration":
+            type = "string"
+        case "guid":
+            type = "string"
+        case _:
+            type = "string"
+
+    return Column(
+        name=attribute.name,
+        type=type,
+        display_name=attribute.display_name,
+        is_primary_key=attribute.is_primary_key,
+    )
+
+
+def construct_path(root_location: str, glob_pattern: str) -> str:
+    volume = (
+        r'/Volumes/{{ var("source")["database"] }}'
+        r'/{{ var("source")["schema"] }}/{{ var("source")["volume"] }}'
+    )
+    return f"{volume}/{root_location}{glob_pattern}"

dbt_4ps_generator-0.1.1/src/dbt_4ps_generator/databricks_helper.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+
+from databricks.sdk import WorkspaceClient
+from rich import print
+
+
+def download_cdm_files_from_databricks_volume(volume_path: str, download_folder: Path) -> None:
+    """Download all .json files from a Unity Catalog volume to a local folder.
+
+    Uses the Databricks SDK default authentication chain (DATABRICKS_HOST /
+    DATABRICKS_TOKEN environment variables, or a profile from ~/.databrickscfg).
+    """
+    w = WorkspaceClient()
+    download_folder.mkdir(parents=True, exist_ok=True)
+
+    for directory_entry in w.files.list_directory_contents(volume_path):
+        name, path = directory_entry.name, directory_entry.path
+        if directory_entry.is_directory or not name or not path or not name.endswith(".json"):
+            continue
+        print(path)
+        resp = w.files.download(path)
+        if resp.contents is None:
+            raise RuntimeError(f"Empty response downloading {path}")
+        download_file = download_folder / Path(name)
+        download_file.write_text(data=str(resp.contents.read(), encoding="utf-8"))