dbt-py-wrap 0.0.2__tar.gz

dbt_py_wrap-0.0.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Bilbottom
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dbt_py_wrap-0.0.2/PKG-INFO

@@ -0,0 +1,144 @@
+Metadata-Version: 2.1
+Name: dbt-py-wrap
+Version: 0.0.2
+Summary: Python wrapper for dbt-core to extend dbt with custom Python.
+Home-page: https://github.com/Bilbottom/dbt-py
+Author: Bilbottom
+Requires-Python: >=3.11,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Project-URL: Repository, https://github.com/Bilbottom/dbt-py
+Description-Content-Type: text/markdown
+
+<div align="center">
+
+[![Python 3.11](https://img.shields.io/badge/python-3.11-blue.svg)](https://www.python.org/downloads/release/python-3110/)
+[![Python 3.11](https://img.shields.io/badge/dbt-1.5+-blue.svg)](https://github.com/dbt-labs/dbt-core/)
+[![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
+![GitHub last commit](https://img.shields.io/github/last-commit/Bilbottom/dbt-py)
+
+[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/Bilbottom/dbt-py/main.svg)](https://results.pre-commit.ci/latest/github/Bilbottom/dbt-py/main)
+[![application-tests](https://github.com/Bilbottom/db-query-profiler/actions/workflows/application-tests.yaml/badge.svg)](https://github.com/Bilbottom/db-query-profiler/actions/workflows/application-tests.yaml)
+
+</div>
+
+---
+
+# dbt-π 🧬
+
+Python wrapper for [dbt-core](https://github.com/dbt-labs/dbt-core) to extend dbt with custom Python.
+
+## Shimmy shimmy shim 🕺🕺🕺
+
+This package is a [shim](<https://en.wikipedia.org/wiki/Shim_(computing)>) for [dbt-core](https://github.com/dbt-labs/dbt-core), inspired by (_cough_ stolen from _cough_) my old boss, [@darkdreamingdan](https://github.com/darkdreamingdan):
+
+- https://gist.github.com/darkdreamingdan/c5ded709a90fc3c5b420cee5f644f499
+
+Before using this package, it's recommended to get up to speed with the Python modules that are already available in dbt:
+
+- https://docs.getdbt.com/reference/dbt-jinja-functions/modules
+
+The existing Python modules are available in the dbt Jinja context under the `modules` object, for example:
+
+```jinja
+{{ modules.datetime.datetime.now() }}
+```
+
+## Installation ⬇️
+
+While in preview, this package is only available from GitHub:
+
+```
+pip install git+https://github.com/Bilbottom/dbt-py@v0.0.2
+```
+
+This will be made available on PyPI once it's ready for general use.
+
+## Usage 📖
+
+This package adds a new executable, `dbt-py`, which injects your custom Python into dbt and then runs dbt. Either a custom module or a custom package can be injected. A custom module is the simplest to get started with.
+
+The default module/package name is `custom` which would make custom Python available in the dbt Jinja context under the `modules.custom` object. This can be configured (see the [Configuration](#configuration-) section below).
+
+### Custom Module 🐍
+
+Create a module called `custom.py` in the root of your dbt project. This module can contain any Python code you like, for example:
+
+```python
+def salutation(name: str) -> str:
+    return f"Hello, {name}!"
+```
+
+Reference this module and function in the dbt Jinja context of a dbt model:
+
+```jinja
+{{ modules.custom.salutation("World") }}
+```
+
+Rather than run dbt with the `dbt` command, instead run it with `dbt-py`:
+
+```
+dbt-py clean
+dbt-py build
+```
+
+Note that `dbt-py` is a wrapper around `dbt` so all the usual dbt commands are available -- all the arguments passed to `dbt-py` are passed through to `dbt`, too.
+
+```
+dbt-py --help
+dbt-py run --select my_model
+dbt-py test --select tag:unit-test
+```
+
+### Custom Package 📦
+
+Using a custom package is similar to using a custom module: create a package called `custom` in the root of your dbt project.
+
+The submodules of this package will be available in the dbt Jinja context too. For example, suppose you have a package called `custom` with a submodule called `greetings`:
+
+```
+custom/
+    __init__.py
+    greetings.py
+```
+
+If the `greetings.py` submodule contains the same `salutation` function as above, then it can be referenced in the dbt Jinja context as follows:
+
+```jinja
+{{ modules.custom.greetings.salutation("World") }}
+```
+
+Alternatively, you can expose the `salutation` function via the `__init__.py` file and then reference it directly via `custom`:
+
+```jinja
+{{ modules.custom.salutation("World") }}
+```
+
+### Configuration 🛠️
+
+The default module/package and Jinja context name is `custom` but both can be configured with the following environment variables:
+
+- `DBT_PY_PACKAGE_ROOT`: The Python-style ref to the custom module/package, e.g. `package.module.submodule`
+- `DBT_PY_PACKAGE_NAME`: The name to give the custom module/package in the dbt Jinja context, e.g. `custom_py`. Defaults to the value of `DBT_PY_PACKAGE_ROOT`
+
+In particular, you can use the `DBT_PY_PACKAGE_ROOT` environment variable to reference a custom module/package that is not at the root of your dbt project.
+
+#### Warning ⚠️
+
+If you set the `DBT_PY_PACKAGE_ROOT` environment variable to a name that already exists, this package will use the existing module/package rather than your custom one. Make sure that your custom module/package name does not clash with any existing modules/packages.
+
+This is likely to change in a future release, but for now you may choose to exploit this behaviour to use an existing module/package in your dbt Jinja context. For example, you could set `DBT_PY_PACKAGE_ROOT` to `math` and then reference the `math` standard library in your dbt Jinja context:
+
+```jinja
+{{ modules.math.pi }}
+```
+
+## Future Work 🚧
+
+This is still in preview, and there are a few things to be added before it's ready for general use:
+
+- Support for importing any number of packages (currently only one package is supported)
+- Configuration via config files and CLI arguments (currently only environment variables are supported)
+- More robust testing
+

dbt_py_wrap-0.0.2/README.md

@@ -0,0 +1,130 @@
+<div align="center">
+
+[![Python 3.11](https://img.shields.io/badge/python-3.11-blue.svg)](https://www.python.org/downloads/release/python-3110/)
+[![Python 3.11](https://img.shields.io/badge/dbt-1.5+-blue.svg)](https://github.com/dbt-labs/dbt-core/)
+[![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
+![GitHub last commit](https://img.shields.io/github/last-commit/Bilbottom/dbt-py)
+
+[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/Bilbottom/dbt-py/main.svg)](https://results.pre-commit.ci/latest/github/Bilbottom/dbt-py/main)
+[![application-tests](https://github.com/Bilbottom/db-query-profiler/actions/workflows/application-tests.yaml/badge.svg)](https://github.com/Bilbottom/db-query-profiler/actions/workflows/application-tests.yaml)
+
+</div>
+
+---
+
+# dbt-π 🧬
+
+Python wrapper for [dbt-core](https://github.com/dbt-labs/dbt-core) to extend dbt with custom Python.
+
+## Shimmy shimmy shim 🕺🕺🕺
+
+This package is a [shim](<https://en.wikipedia.org/wiki/Shim_(computing)>) for [dbt-core](https://github.com/dbt-labs/dbt-core), inspired by (_cough_ stolen from _cough_) my old boss, [@darkdreamingdan](https://github.com/darkdreamingdan):
+
+- https://gist.github.com/darkdreamingdan/c5ded709a90fc3c5b420cee5f644f499
+
+Before using this package, it's recommended to get up to speed with the Python modules that are already available in dbt:
+
+- https://docs.getdbt.com/reference/dbt-jinja-functions/modules
+
+The existing Python modules are available in the dbt Jinja context under the `modules` object, for example:
+
+```jinja
+{{ modules.datetime.datetime.now() }}
+```
+
+## Installation ⬇️
+
+While in preview, this package is only available from GitHub:
+
+```
+pip install git+https://github.com/Bilbottom/dbt-py@v0.0.2
+```
+
+This will be made available on PyPI once it's ready for general use.
+
+## Usage 📖
+
+This package adds a new executable, `dbt-py`, which injects your custom Python into dbt and then runs dbt. Either a custom module or a custom package can be injected. A custom module is the simplest to get started with.
+
+The default module/package name is `custom` which would make custom Python available in the dbt Jinja context under the `modules.custom` object. This can be configured (see the [Configuration](#configuration-) section below).
+
+### Custom Module 🐍
+
+Create a module called `custom.py` in the root of your dbt project. This module can contain any Python code you like, for example:
+
+```python
+def salutation(name: str) -> str:
+    return f"Hello, {name}!"
+```
+
+Reference this module and function in the dbt Jinja context of a dbt model:
+
+```jinja
+{{ modules.custom.salutation("World") }}
+```
+
+Rather than run dbt with the `dbt` command, instead run it with `dbt-py`:
+
+```
+dbt-py clean
+dbt-py build
+```
+
+Note that `dbt-py` is a wrapper around `dbt` so all the usual dbt commands are available -- all the arguments passed to `dbt-py` are passed through to `dbt`, too.
+
+```
+dbt-py --help
+dbt-py run --select my_model
+dbt-py test --select tag:unit-test
+```
+
+### Custom Package 📦
+
+Using a custom package is similar to using a custom module: create a package called `custom` in the root of your dbt project.
+
+The submodules of this package will be available in the dbt Jinja context too. For example, suppose you have a package called `custom` with a submodule called `greetings`:
+
+```
+custom/
+    __init__.py
+    greetings.py
+```
+
+If the `greetings.py` submodule contains the same `salutation` function as above, then it can be referenced in the dbt Jinja context as follows:
+
+```jinja
+{{ modules.custom.greetings.salutation("World") }}
+```
+
+Alternatively, you can expose the `salutation` function via the `__init__.py` file and then reference it directly via `custom`:
+
+```jinja
+{{ modules.custom.salutation("World") }}
+```
+
+### Configuration 🛠️
+
+The default module/package and Jinja context name is `custom` but both can be configured with the following environment variables:
+
+- `DBT_PY_PACKAGE_ROOT`: The Python-style ref to the custom module/package, e.g. `package.module.submodule`
+- `DBT_PY_PACKAGE_NAME`: The name to give the custom module/package in the dbt Jinja context, e.g. `custom_py`. Defaults to the value of `DBT_PY_PACKAGE_ROOT`
+
+In particular, you can use the `DBT_PY_PACKAGE_ROOT` environment variable to reference a custom module/package that is not at the root of your dbt project.
+
+#### Warning ⚠️
+
+If you set the `DBT_PY_PACKAGE_ROOT` environment variable to a name that already exists, this package will use the existing module/package rather than your custom one. Make sure that your custom module/package name does not clash with any existing modules/packages.
+
+This is likely to change in a future release, but for now you may choose to exploit this behaviour to use an existing module/package in your dbt Jinja context. For example, you could set `DBT_PY_PACKAGE_ROOT` to `math` and then reference the `math` standard library in your dbt Jinja context:
+
+```jinja
+{{ modules.math.pi }}
+```
+
+## Future Work 🚧
+
+This is still in preview, and there are a few things to be added before it's ready for general use:
+
+- Support for importing any number of packages (currently only one package is supported)
+- Configuration via config files and CLI arguments (currently only environment variables are supported)
+- More robust testing

dbt_py_wrap-0.0.2/dbt_py/__init__.py

@@ -0,0 +1,9 @@
+"""
+Python wrapper for dbt-core to extend dbt with custom Python.
+"""
+from dbt_py.main import PROJECT_ROOT, main
+
+__all__ = [
+    "PROJECT_ROOT",
+    "main",
+]

dbt_py_wrap-0.0.2/dbt_py/main.py

@@ -0,0 +1,66 @@
+"""
+Shim the dbt CLI to include our custom modules.
+"""
+import importlib
+import os
+import pathlib
+import pkgutil
+import sys
+from types import ModuleType
+from typing import Any
+
+import dbt.cli.main
+import dbt.context.base
+from dbt.context.base import get_context_modules as _get_context_modules
+
+PROJECT_ROOT = pathlib.Path(__file__).parent.parent
+# Python-style ref, e.g. `package.module.submodule`
+PACKAGE_ROOT: str = os.environ.get("DBT_PY_PACKAGE_ROOT")
+
+
+def import_submodules(
+    package_name: str,
+    recursive: bool = True,
+) -> dict[str, ModuleType]:
+    """
+    Import all submodules of a module, recursively, including subpackages.
+
+    - https://stackoverflow.com/a/25562415/10730311
+    """
+    package = importlib.import_module(package_name)
+    if not hasattr(package, "__path__"):
+        # `package` is a module, don't recurse any further
+        return {}
+
+    results = {}
+    for loader, name, is_pkg in pkgutil.walk_packages(package.__path__):
+        full_name = f"{package.__name__}.{name}"
+        results[full_name] = importlib.import_module(full_name)
+        if recursive and is_pkg:
+            results |= import_submodules(full_name)
+
+    return results
+
+
+def new_get_context_modules() -> dict[str, dict[str, Any]]:
+    """
+    Append the custom modules into the whitelisted dbt modules.
+    """
+    package_root: str = PACKAGE_ROOT or "custom"
+    package_name: str = os.environ.get("DBT_PY_PACKAGE_NAME") or package_root
+
+    import_submodules(package_root)
+    modules = _get_context_modules()
+    modules[package_name] = importlib.import_module(package_root)  # type: ignore
+
+    return modules
+
+
+def main() -> None:
+    """
+    Shim the dbt CLI to include our custom modules.
+
+    - https://docs.getdbt.com/reference/programmatic-invocations
+    """
+    dbt.context.base.get_context_modules = new_get_context_modules
+    dbt.cli.main.dbtRunner().invoke(sys.argv[1:])

dbt_py_wrap-0.0.2/pyproject.toml

@@ -0,0 +1,47 @@
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+
+[tool.poetry]
+name = "dbt-py-wrap"  # Not `dbt-py` to avoid confusion with the DbtPy package (PyPI requirement)
+version = "0.0.2"
+description = "Python wrapper for dbt-core to extend dbt with custom Python."
+readme = "README.md"
+authors = ["Bilbottom"]
+repository = "https://github.com/Bilbottom/dbt-py"
+packages = [
+    { include = "dbt_py" },
+]
+
+[tool.poetry.scripts]
+dbt-py = "dbt_py:main"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+
+[tool.poetry.group]
+dev.optional = true
+test.optional = true
+ide.optional = true
+
+[tool.poetry.group.dev.dependencies]
+pre-commit = "^3.3.3"
+
+[tool.poetry.group.test.dependencies]
+dbt-core = "^1.5.0"
+dbt-duckdb = "^1.5.0"
+pytest = "^7.4.0"
+pytest-cov = "^4.1.0"
+
+# Packages just for IDE integration
+[tool.poetry.group.ide.dependencies]
+black = "*"
+
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+
+[tool.isort]
+profile = "black"