netbek-dw-lib 1.32.0__tar.gz

netbek_dw_lib-1.32.0/PKG-INFO

@@ -0,0 +1,141 @@
+Metadata-Version: 2.3
+Name: netbek-dw-lib
+Version: 1.32.0
+Summary: Tools for working with Postgres, ClickHouse, and DuckDB.
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: lazy-loader>=0.5
+Requires-Dist: packaging>=26.0
+Requires-Dist: psutil>=7.1.3
+Requires-Dist: pydantic>=2.12.4
+Requires-Dist: pydash>=8.0.5
+Requires-Dist: requests>=2.32.5
+Requires-Dist: ruamel-yaml>=0.19.1
+Requires-Dist: sqlalchemy>=2.0.44
+Requires-Dist: sqlglot>=28.0.0,<28.1.0
+Requires-Dist: sqlmodel>=0.0.27
+Requires-Dist: sqlparse>=0.5.3
+Requires-Dist: clickhouse-connect>=0.10.0 ; extra == 'clickhouse'
+Requires-Dist: clickhouse-sqlalchemy ; extra == 'clickhouse'
+Requires-Dist: sqlglot>=28.0.0,<28.1.0 ; extra == 'clickhouse'
+Requires-Dist: sqlmodel>=0.0.27 ; extra == 'clickhouse'
+Requires-Dist: dagster>=1.12.19 ; extra == 'dagster'
+Requires-Dist: dagster-dbt>=0.28.19 ; extra == 'dagster'
+Requires-Dist: clickhouse-connect>=0.10.0 ; extra == 'dbt'
+Requires-Dist: clickhouse-sqlalchemy ; extra == 'dbt'
+Requires-Dist: dbt-clickhouse ; extra == 'dbt'
+Requires-Dist: dbt-core>=1.10.0,<1.11 ; extra == 'dbt'
+Requires-Dist: livereload>=2.7.1 ; extra == 'dbt'
+Requires-Dist: opentelemetry-sdk>=1.39.1 ; extra == 'dbt'
+Requires-Dist: polyfactory>=3.0.0 ; extra == 'dbt'
+Requires-Dist: pydantic>=2.12.4 ; extra == 'dbt'
+Requires-Dist: pydash>=8.0.5 ; extra == 'dbt'
+Requires-Dist: ruamel-yaml>=0.19.1 ; extra == 'dbt'
+Requires-Dist: sqlglot>=28.0.0,<28.1.0 ; extra == 'dbt'
+Requires-Dist: sqlmodel>=0.0.27 ; extra == 'dbt'
+Requires-Dist: duckdb>=1.4.2 ; extra == 'duckdb'
+Requires-Dist: duckdb-engine>=0.17.0 ; extra == 'duckdb'
+Requires-Dist: sqlglot>=28.0.0,<28.1.0 ; extra == 'duckdb'
+Requires-Dist: sqlmodel>=0.0.27 ; extra == 'duckdb'
+Requires-Dist: boto3>=1.41.1 ; extra == 'loader'
+Requires-Dist: chdb>=3.7.2 ; extra == 'loader'
+Requires-Dist: jinja2>=3.1.6 ; extra == 'loader'
+Requires-Dist: pydantic>=2.12.4 ; extra == 'loader'
+Requires-Dist: rich>=14.2.0 ; extra == 'loader'
+Requires-Dist: clickhouse-connect>=0.10.0 ; extra == 'peerdb'
+Requires-Dist: clickhouse-sqlalchemy ; extra == 'peerdb'
+Requires-Dist: pydantic>=2.12.4 ; extra == 'peerdb'
+Requires-Dist: pydash>=8.0.5 ; extra == 'peerdb'
+Requires-Dist: requests>=2.32.5 ; extra == 'peerdb'
+Requires-Dist: rich>=14.2.0 ; extra == 'peerdb'
+Requires-Dist: ruamel-yaml>=0.19.1 ; extra == 'peerdb'
+Requires-Dist: sqlalchemy>=2.0.44 ; extra == 'peerdb'
+Requires-Dist: sqlglot>=28.0.0 ; extra == 'peerdb'
+Requires-Dist: sqlmodel>=0.0.27 ; extra == 'peerdb'
+Requires-Dist: psycopg[binary]>=3.3.3 ; extra == 'psycopg'
+Requires-Dist: sqlglot>=28.0.0,<28.1.0 ; extra == 'psycopg'
+Requires-Dist: sqlmodel>=0.0.27 ; extra == 'psycopg'
+Requires-Dist: psycopg2-binary>=2.9.11 ; extra == 'psycopg2'
+Requires-Dist: sqlglot>=28.0.0,<28.1.0 ; extra == 'psycopg2'
+Requires-Dist: sqlmodel>=0.0.27 ; extra == 'psycopg2'
+Requires-Python: ==3.13.*
+Provides-Extra: clickhouse
+Provides-Extra: dagster
+Provides-Extra: dbt
+Provides-Extra: duckdb
+Provides-Extra: loader
+Provides-Extra: peerdb
+Provides-Extra: psycopg
+Provides-Extra: psycopg2
+Description-Content-Type: text/markdown
+
+# dw-lib
+
+Tools for working with Postgres, ClickHouse, and DuckDB.
+
+## Development: Installation
+
+1. Clone the repo:
+
+    ```shell
+    git clone git@github.com:netbek/dw-lib.git
+    ```
+
+2. Install [Docker Engine v23 or higher](https://docs.docker.com/engine/install/) and [Docker Compose v2 or higher](https://docs.docker.com/compose/install/). Follow the links for instructions or run this script:
+
+    ```shell
+    ./scripts/install.sh docker
+    ```
+
+3. Install Nix:
+
+    ```shell
+    sh <(curl -L https://nixos.org/nix/install) --daemon
+    ```
+
+4. Configure Nix. Edit `/etc/nix/nix.conf` (for a multi-user installation) or `~/.config/nix/nix.conf` (for a single-user installation) to include the following lines:
+
+    ```shell
+    experimental-features = nix-command flakes
+    trusted-users = root <USER>
+    ```
+
+    Replace `<USER>` with your username on your computer.
+
+5. Install direnv:
+
+    ```shell
+    sudo apt install direnv
+    ```
+
+6. Enable direnv in your shell by adding a line to your shell configuration file.
+
+    For Bash, edit `~/.bashrc`:
+
+    ```shell
+    eval "$(direnv hook bash)"
+    ```
+
+7. Allow `.envrc`:
+
+    ```shell
+    direnv allow
+    ```
+
+8. Enter a [PyPI API token](https://pypi.org/manage/account/#api-tokens) as the password in `.pypirc`.
+
+## Development: Usage
+
+Build and publish the Python distribution package:
+
+```shell
+make bump-version [major|minor|patch]
+git push
+make build
+git push
+make create-release
+make publish
+```
+
+## License
+
+Copyright (c) 2025 Hein Bekker. Licensed under the GNU Affero General Public License, version 3.

netbek_dw_lib-1.32.0/README.md

@@ -0,0 +1,71 @@
+# dw-lib
+
+Tools for working with Postgres, ClickHouse, and DuckDB.
+
+## Development: Installation
+
+1. Clone the repo:
+
+    ```shell
+    git clone git@github.com:netbek/dw-lib.git
+    ```
+
+2. Install [Docker Engine v23 or higher](https://docs.docker.com/engine/install/) and [Docker Compose v2 or higher](https://docs.docker.com/compose/install/). Follow the links for instructions or run this script:
+
+    ```shell
+    ./scripts/install.sh docker
+    ```
+
+3. Install Nix:
+
+    ```shell
+    sh <(curl -L https://nixos.org/nix/install) --daemon
+    ```
+
+4. Configure Nix. Edit `/etc/nix/nix.conf` (for a multi-user installation) or `~/.config/nix/nix.conf` (for a single-user installation) to include the following lines:
+
+    ```shell
+    experimental-features = nix-command flakes
+    trusted-users = root <USER>
+    ```
+
+    Replace `<USER>` with your username on your computer.
+
+5. Install direnv:
+
+    ```shell
+    sudo apt install direnv
+    ```
+
+6. Enable direnv in your shell by adding a line to your shell configuration file.
+
+    For Bash, edit `~/.bashrc`:
+
+    ```shell
+    eval "$(direnv hook bash)"
+    ```
+
+7. Allow `.envrc`:
+
+    ```shell
+    direnv allow
+    ```
+
+8. Enter a [PyPI API token](https://pypi.org/manage/account/#api-tokens) as the password in `.pypirc`.
+
+## Development: Usage
+
+Build and publish the Python distribution package:
+
+```shell
+make bump-version [major|minor|patch]
+git push
+make build
+git push
+make create-release
+make publish
+```
+
+## License
+
+Copyright (c) 2025 Hein Bekker. Licensed under the GNU Affero General Public License, version 3.

netbek_dw_lib-1.32.0/pyproject.toml

@@ -0,0 +1,148 @@
+[project]
+name = "netbek-dw-lib"
+version = "1.32.0"
+description = "Tools for working with Postgres, ClickHouse, and DuckDB."
+readme = "README.md"
+requires-python = "==3.13.*"
+
+# sqlglot pinned for compatibility with dagster-dbt https://github.com/dagster-io/dagster/blob/1.12.19/python_modules/libraries/dagster-dbt/pyproject.toml#L33
+dependencies = [
+    "jinja2>=3.1.6",
+    "lazy-loader>=0.5",
+    "packaging>=26.0",
+    "psutil>=7.1.3",
+    "pydantic>=2.12.4",
+    "pydash>=8.0.5",
+    "requests>=2.32.5",
+    "ruamel-yaml>=0.19.1",
+    "sqlalchemy>=2.0.44",
+    "sqlglot>=28.0.0,<28.1.0",
+    "sqlmodel>=0.0.27",
+    "sqlparse>=0.5.3",
+]
+
+[project.optional-dependencies]
+clickhouse = [
+    "clickhouse-connect>=0.10.0",
+    "clickhouse-sqlalchemy",
+    "sqlglot>=28.0.0,<28.1.0",
+    "sqlmodel>=0.0.27",
+]
+
+duckdb = [
+    "duckdb>=1.4.2",
+    "duckdb-engine>=0.17.0",
+    "sqlglot>=28.0.0,<28.1.0",
+    "sqlmodel>=0.0.27",
+]
+
+psycopg = [
+    "psycopg[binary]>=3.3.3",
+    "sqlglot>=28.0.0,<28.1.0",
+    "sqlmodel>=0.0.27",
+]
+
+psycopg2 = [
+    "psycopg2-binary>=2.9.11",
+    "sqlglot>=28.0.0,<28.1.0",
+    "sqlmodel>=0.0.27",
+]
+
+dagster = [
+    "dagster>=1.12.19",
+    "dagster-dbt>=0.28.19",
+]
+
+# dbt-core pinned for compatibility with dbt-clickhouse https://github.com/netbek/dbt-clickhouse/blob/v1.10.0-batch/pyproject.toml#L26
+dbt = [
+    "clickhouse-connect>=0.10.0",
+    "clickhouse-sqlalchemy",
+    "dbt-clickhouse",
+    "dbt-core>=1.10.0,<1.11",
+    "livereload>=2.7.1",
+    "opentelemetry-sdk>=1.39.1",
+    "polyfactory>=3.0.0",
+    "pydantic>=2.12.4",
+    "pydash>=8.0.5",
+    "ruamel-yaml>=0.19.1",
+    "sqlglot>=28.0.0,<28.1.0",
+    "sqlmodel>=0.0.27",
+]
+
+loader = [
+    "boto3>=1.41.1",
+    "chdb>=3.7.2",
+    "jinja2>=3.1.6",
+    "pydantic>=2.12.4",
+    "rich>=14.2.0",
+]
+
+peerdb = [
+    "clickhouse-connect>=0.10.0",
+    "clickhouse-sqlalchemy",
+    "pydantic>=2.12.4",
+    "pydash>=8.0.5",
+    "requests>=2.32.5",
+    "rich>=14.2.0",
+    "ruamel-yaml>=0.19.1",
+    "sqlalchemy>=2.0.44",
+    "sqlglot>=28.0.0",
+    "sqlmodel>=0.0.27",
+]
+
+[dependency-groups]
+dev = [
+    "docker>=7.1.0",
+    "pre-commit>=4.5.1",
+    "pytest>=9.0.2",
+    "pytest-docker>=3.2.5",
+    "ruff>=0.15.0",
+    "twine>=6.2.0",
+    "ty>=0.0.23",
+]
+
+[build-system]
+requires = ["uv_build>=0.9.17,<0.12.0"]
+build-backend = "uv_build"
+
+[tool.isort]
+force_alphabetical_sort = true
+line_length = 100
+profile = "black"
+py_version = 313
+
+[tool.ruff]
+line-length = 100
+target-version = "py313"
+extend-exclude = ["./vendor"]
+
+[tool.ruff.lint]
+ignore = ["D100", "D101", "D102", "D103", "D104", "D107", "D205"]
+
+# [tool.pyright]
+# include = ["src", "tests"]
+# pythonVersion = "3.13"
+# typeCheckingMode = "basic"
+# reportPrivateImportUsage = "none"
+
+[tool.ty.rules]
+invalid-method-override = "ignore" # https://github.com/astral-sh/ty/issues/2154
+
+[tool.ty.src]
+include = ["src", "tests"]
+
+[tool.pytest]
+addopts = ["-x", "--no-header", "--container-scope=module"]
+filterwarnings = [
+    "ignore:The 'u' type code is deprecated:DeprecationWarning", # Caused by clickhouse_connect
+    "ignore::DeprecationWarning:dbt_common.invocation", # https://github.com/dbt-labs/dbt-core/issues/9791
+    "ignore::DeprecationWarning:dbt.cli.options"        # https://github.com/dbt-labs/dbt-core/issues/12038
+]
+markers = ["docker_compose_file", "docker_skip_wait_until_responsive"]
+
+[tool.uv.build-backend]
+module-name = "dw_lib"
+
+[tool.uv.sources]
+clickhouse-sqlalchemy = { git = "https://github.com/netbek/clickhouse-sqlalchemy.git", rev = "datetime-uuid" }
+dbt-clickhouse = { git = "https://github.com/netbek/dbt-clickhouse.git", rev = "v1.10.0-batch" }

netbek_dw_lib-1.32.0/src/dw_lib/cloud/__init__.py

@@ -0,0 +1,9 @@
+from .adapters.s3 import S3Adapter
+from .types import S3Settings
+from .utils import s3_to_endpoint_uri
+
+__all__ = [
+    "s3_to_endpoint_uri",
+    "S3Adapter",
+    "S3Settings",
+]

netbek_dw_lib-1.32.0/src/dw_lib/cloud/adapters/s3.py

@@ -0,0 +1,49 @@
+from ..types import S3Settings
+from botocore.client import Config
+from typing import Any
+
+import boto3
+
+
+class S3Adapter:
+    def __init__(self, settings: S3Settings) -> None:
+        self.settings = settings
+
+    @property
+    def url(self) -> str:
+        return f"s3://{self.settings.bucket}"
+
+    def create_client(self):
+        if self.settings.use_ssl:
+            scheme = "https"
+        else:
+            scheme = "http"
+
+        endpoint_url = f"{scheme}://{self.settings.endpoint}"
+
+        client = boto3.client(
+            "s3",
+            endpoint_url=endpoint_url,
+            aws_access_key_id=self.settings.key_id,
+            aws_secret_access_key=self.settings.secret,
+            config=Config(signature_version="s3v4"),
+            region_name=self.settings.region,
+        )
+
+        return client
+
+    def can_connect(self) -> bool:
+        client = self.create_client()
+
+        try:
+            client.head_bucket(Bucket=self.settings.bucket)
+        except Exception:
+            return False
+
+        return True
+
+    def list_objects(self, prefix: str | None = None) -> list[dict[str, Any]]:
+        client = self.create_client()
+        response = client.list_objects_v2(Bucket=self.settings.bucket, Prefix=prefix)
+
+        return response.get("Contents")

netbek_dw_lib-1.32.0/src/dw_lib/cloud/types.py

@@ -0,0 +1,12 @@
+from pydantic import BaseModel
+
+
+class S3Settings(BaseModel):
+    key_id: str
+    secret: str
+    region: str
+    endpoint: str
+    use_ssl: bool
+    url_style: str = "path"
+    bucket: str
+    prefix: str | None = None

netbek_dw_lib-1.32.0/src/dw_lib/cloud/utils.py

@@ -0,0 +1,17 @@
+from urllib.parse import urlparse
+
+
+def s3_to_endpoint_uri(s3_uri: str, endpoint: str, use_ssl: bool = False) -> str:
+    parsed = urlparse(s3_uri)
+    if parsed.scheme != "s3":
+        raise ValueError(f"Invalid S3 URI: {s3_uri}")
+
+    if use_ssl:
+        scheme = "https"
+    else:
+        scheme = "http"
+
+    bucket = parsed.netloc
+    path = parsed.path.lstrip("/")
+
+    return f"{scheme}://{endpoint}/{bucket}/{path}"

netbek_dw_lib-1.32.0/src/dw_lib/constants.py

@@ -0,0 +1,37 @@
+PYTHON_RESERVED_WORDS = [
+    "false",
+    "none",
+    "true",
+    "and",
+    "as",
+    "assert",
+    "async",
+    "await",
+    "break",
+    "class",
+    "continue",
+    "def",
+    "del",
+    "elif",
+    "else",
+    "except",
+    "finally",
+    "for",
+    "from",
+    "global",
+    "if",
+    "import",
+    "in",
+    "is",
+    "lambda",
+    "nonlocal",
+    "not",
+    "or",
+    "pass",
+    "raise",
+    "return",
+    "try",
+    "while",
+    "with",
+    "yield",
+]

netbek_dw_lib-1.32.0/src/dw_lib/dagster/asset_utils.py

@@ -0,0 +1,118 @@
+from collections.abc import Mapping, Sequence
+from dagster import (
+    AssetsDefinition,
+    DagsterRunStatus,
+    DefaultScheduleStatus,
+    define_asset_job,
+    RunConfig,
+    RunRequest,
+    RunsFilter,
+    schedule,
+    ScheduleEvaluationContext,
+    SkipReason,
+)
+from dagster._core.definitions.target import ExecutableDefinition
+from dagster_dbt.asset_utils import (
+    build_dbt_asset_selection,
+    DBT_DEFAULT_EXCLUDE,
+    DBT_DEFAULT_SELECT,
+    DBT_DEFAULT_SELECTOR,
+)
+
+# https://github.com/dagster-io/dagster/blob/1.12.19/python_modules/dagster/dagster/_core/storage/dagster_run.py#L110
+NOT_FINISHED_STATUSES = [
+    DagsterRunStatus.QUEUED,
+    DagsterRunStatus.NOT_STARTED,
+    DagsterRunStatus.STARTING,
+    DagsterRunStatus.STARTED,
+]
+
+
+def build_singleton_schedule(
+    job: ExecutableDefinition,
+    cron_schedule: str,
+    schedule_name: str | None = None,
+    tags: Mapping[str, str] | None = None,
+    config: RunConfig | None = None,
+    execution_timezone: str | None = None,
+    default_status: DefaultScheduleStatus = DefaultScheduleStatus.STOPPED,
+):
+    """
+    Returns a schedule that only triggers if no other instance of the job is running.
+
+    Reference: https://docs.dagster.io/guides/operate/managing-concurrency/advanced
+    """
+    schedule_name = schedule_name or f"{job.name}_schedule"
+
+    @schedule(
+        name=schedule_name,
+        job=job,
+        cron_schedule=cron_schedule,
+        execution_timezone=execution_timezone,
+        default_status=default_status,
+    )
+    def _schedule(context: ScheduleEvaluationContext):
+        # Find an unfinished run of the job
+        runs = context.instance.get_runs(
+            filters=RunsFilter(job_name=job.name, statuses=NOT_FINISHED_STATUSES), limit=1
+        )
+
+        if runs:
+            return SkipReason(f"Skipping {job.name} because a run is already in progress.")
+
+        return RunRequest(run_config=config, tags=tags)
+
+    return _schedule
+
+
+def build_singleton_schedule_from_dbt_selection(
+    dbt_assets: Sequence[AssetsDefinition],
+    job_name: str,
+    cron_schedule: str,
+    dbt_select: str = DBT_DEFAULT_SELECT,
+    dbt_exclude: str | None = DBT_DEFAULT_EXCLUDE,
+    dbt_selector: str = DBT_DEFAULT_SELECTOR,
+    schedule_name: str | None = None,
+    tags: Mapping[str, str] | None = None,
+    config: RunConfig | None = None,
+    execution_timezone: str | None = None,
+    default_status: DefaultScheduleStatus = DefaultScheduleStatus.STOPPED,
+):
+    """
+    Returns a schedule for dbt assets that only triggers if no other instance of the job is running.
+
+    Based on https://github.com/dagster-io/dagster/blob/1.12.19/python_modules/libraries/dagster-dbt/dagster_dbt/asset_utils.py#L353
+    """
+    selection = build_dbt_asset_selection(
+        dbt_assets,
+        dbt_select=dbt_select,
+        dbt_exclude=dbt_exclude or DBT_DEFAULT_EXCLUDE,
+        dbt_selector=dbt_selector,
+    )
+    job = define_asset_job(
+        name=job_name,
+        selection=selection,
+        config=config,
+        tags=tags,
+    )
+    schedule_name = schedule_name or f"{job_name}_schedule"
+
+    @schedule(
+        name=schedule_name,
+        job=job,
+        cron_schedule=cron_schedule,
+        execution_timezone=execution_timezone,
+        default_status=default_status,
+    )
+    def _schedule(context: ScheduleEvaluationContext):
+        # Find an unfinished run of the job
+        runs = context.instance.get_runs(
+            filters=RunsFilter(job_name=job.name, statuses=NOT_FINISHED_STATUSES), limit=1
+        )
+
+        if runs:
+            return SkipReason(f"Skipping {job.name} because a run is already in progress.")
+
+        return RunRequest(run_config=config, tags=tags)
+
+    return _schedule

netbek_dw_lib-1.32.0/src/dw_lib/database/__init__.py

@@ -0,0 +1,40 @@
+from .types import (
+    ClickHouseRelation,
+    ClickHouseSettings,
+    DuckDBRelation,
+    DuckDBSettings,
+    PostgresRelation,
+    PostgresSettings,
+)
+from .utils import render_statement
+from typing import TYPE_CHECKING
+
+import lazy_loader as lazy
+
+if TYPE_CHECKING:
+    from .adapters.clickhouse import ClickHouseAdapter
+    from .adapters.duckdb import DuckDBAdapter
+    from .adapters.postgres import PostgresAdapter
+
+# TODO Replace with lazy keyword in Python 3.15+ https://docs.python.org/3.15/whatsnew/3.15.html#whatsnew315-lazy-imports
+__getattr__, __dir__, _ = lazy.attach(
+    __name__,
+    submod_attrs={
+        "adapters.clickhouse": ["ClickHouseAdapter"],
+        "adapters.duckdb": ["DuckDBAdapter"],
+        "adapters.postgres": ["PostgresAdapter"],
+    },
+)
+
+__all__ = [
+    "ClickHouseAdapter",
+    "ClickHouseRelation",
+    "ClickHouseSettings",
+    "DuckDBAdapter",
+    "DuckDBRelation",
+    "DuckDBSettings",
+    "PostgresAdapter",
+    "PostgresRelation",
+    "PostgresSettings",
+    "render_statement",
+]