dpone 0.1.0__tar.gz

dpone-0.1.0/PKG-INFO

@@ -0,0 +1,290 @@
+Metadata-Version: 2.4
+Name: dpone
+Version: 0.1.0
+Summary: Declarative ETL framework for YAML-driven data pipelines
+Keywords: etl,data-platform,data-engineering,yaml,dag,bigquery,postgres,clickhouse
+Author: PaulKov
+License-Expression: Apache-2.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Distributed Computing
+Classifier: Typing :: Typed
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: jinja2>=3.1.0
+Requires-Dist: sqlglot==27.13.2
+Requires-Dist: pendulum==3.1.0
+Requires-Dist: prettytable==3.17.0
+Requires-Dist: pytz>=2023.3
+Requires-Dist: requests>=2.32.5
+Requires-Dist: ijson>=3.3.0
+Requires-Dist: clickhouse-connect==0.6.22 ; extra == 'clickhouse'
+Requires-Dist: clickhouse-driver==0.2.9 ; extra == 'clickhouse'
+Requires-Dist: clickhouse-cityhash==1.0.2.5 ; extra == 'clickhouse'
+Requires-Dist: psycopg[binary]==3.2.9 ; extra == 'full'
+Requires-Dist: clickhouse-connect==0.6.22 ; extra == 'full'
+Requires-Dist: clickhouse-driver==0.2.9 ; extra == 'full'
+Requires-Dist: clickhouse-cityhash==1.0.2.5 ; extra == 'full'
+Requires-Dist: google-cloud-bigquery==3.38.0 ; extra == 'full'
+Requires-Dist: google-cloud-storage==3.7.0 ; extra == 'full'
+Requires-Dist: pandas==2.1.4 ; extra == 'full'
+Requires-Dist: numpy==1.26.4 ; extra == 'full'
+Requires-Dist: vault-kv-client>=0.1.0,<0.2.0 ; extra == 'full'
+Requires-Dist: google-ads>=28.0.0,<29.0.0 ; extra == 'full'
+Requires-Dist: google-cloud-bigquery==3.38.0 ; extra == 'gcp'
+Requires-Dist: google-cloud-storage==3.7.0 ; extra == 'gcp'
+Requires-Dist: google-ads>=28.0.0,<29.0.0 ; extra == 'google-ads'
+Requires-Dist: pandas==2.1.4 ; extra == 'pandas'
+Requires-Dist: numpy==1.26.4 ; extra == 'pandas'
+Requires-Dist: psycopg[binary]==3.2.9 ; extra == 'postgres'
+Requires-Dist: vault-kv-client>=0.1.0,<0.2.0 ; extra == 'vault'
+Maintainer: PaulKov
+Requires-Python: >=3.11
+Project-URL: Changelog, https://github.com/PaulKov/dpone/blob/main/CHANGELOG.md
+Project-URL: Documentation, https://github.com/PaulKov/dpone#readme
+Project-URL: Homepage, https://github.com/PaulKov/dpone
+Project-URL: Issues, https://github.com/PaulKov/dpone/issues
+Project-URL: Repository, https://github.com/PaulKov/dpone
+Provides-Extra: clickhouse
+Provides-Extra: full
+Provides-Extra: gcp
+Provides-Extra: google-ads
+Provides-Extra: pandas
+Provides-Extra: postgres
+Provides-Extra: vault
+Description-Content-Type: text/markdown
+
+# dpone
+
+`dpone` is a Python ETL framework for declarative, YAML-driven data pipelines. It helps data teams describe sources, sinks, load strategies, dependencies, conventions, and operational checks as reusable configuration instead of one-off scripts.
+
+The public package name, import name, GitHub repository name, and CLI name are all intentionally short: `dpone`.
+
+Repository: https://github.com/PaulKov/dpone
+
+## What dpone gives you
+
+- YAML manifests for single-process and batch ETL definitions.
+- Built-in DAG/dependency inspection for pipeline debugging.
+- Runtime abstractions for sources, sinks, connectors, state, reconciliation, and safe SQL logging.
+- Optional integrations for PostgreSQL, ClickHouse, BigQuery/GCS, pandas, Google Ads, and HashiCorp Vault.
+- A CLI designed for self-service validation, rendering, explainability, and documentation checks.
+- Compatibility shims for older import paths while the canonical package layout continues to stabilize.
+
+## Installation
+
+Install the core package from PyPI:
+
+```bash
+pip install dpone
+```
+
+Install common extras for local ETL development:
+
+```bash
+pip install "dpone[postgres,clickhouse,gcp,pandas,vault]"
+```
+
+Install everything currently published by the project:
+
+```bash
+pip install "dpone[full]"
+```
+
+With `uv`:
+
+```bash
+uv add dpone
+uv add "dpone[full]"
+```
+
+## Optional extras
+
+| Extra | Purpose |
+| --- | --- |
+| `postgres` | PostgreSQL connectivity via `psycopg` |
+| `clickhouse` | ClickHouse connectivity |
+| `gcp` | Google BigQuery and Google Cloud Storage support |
+| `pandas` | DataFrame-based extract/load helpers |
+| `vault` | HashiCorp Vault integration via public `vault-kv-client` |
+| `google_ads` | Google Ads API support |
+| `full` | All public extras above |
+
+Vault support uses [`vault-kv-client`](https://github.com/PaulKov/vault-kv-client), published on PyPI as `vault-kv-client`. New code should use `vault_kv_client`; the historical `vault_client` import path remains supported by that package as a compatibility layer.
+
+## Quick start
+
+Create a batch manifest, for example `examples/batch/landing_postgres_to_bq.batch.yaml`:
+
+```yaml
+# yaml-language-server: $schema=../../src/dpone/schema/etl-batch-manifest.schema.json
+kind: dpone.batch.v1
+convention: landing_raw_v1
+registry: ../registry/sources.yaml
+
+vars:
+  src_system: demo_source
+  src_database: demo_db
+  owner_team: data-platform
+  owner_contact: data-platform@example.com
+  sla: daily
+
+defaults:
+  source:
+    type: postgres
+    connection_type: vault
+    connection_id: postgres-demo
+    vault_path: postgres/demo-source
+    options:
+      batch_size: 100000
+      export_format: csv
+
+  sink:
+    type: bigquery
+    connection_type: vault
+    connection_id: bigquery-demo
+    vault_path: gcp/demo-project-prod/bq/service-account
+    staging:
+      schema: stg
+    strategy:
+      mode: full_refresh
+      overwrite_type: exchange
+
+schemas:
+  public:
+    tables:
+      - core_city
+```
+
+Validate and render it:
+
+```bash
+dpone manifest validate examples/batch/landing_postgres_to_bq.batch.yaml \
+  --profile landing_raw_v1 \
+  --registry examples/registry/sources.yaml
+
+dpone manifest render examples/batch/landing_postgres_to_bq.batch.yaml \
+  --selector public.core_city \
+  --registry examples/registry/sources.yaml
+```
+
+Inspect pipeline dependencies:
+
+```bash
+dpone dag report examples/batch/landing_postgres_to_bq.batch.yaml \
+  --base-path . \
+  --format json \
+  --preset ci \
+  --registry examples/registry/sources.yaml
+```
+
+## CLI overview
+
+```bash
+dpone --help
+dpone manifest --help
+dpone dag --help
+dpone docs --help
+```
+
+Common commands:
+
+```bash
+dpone manifest list examples/batch/landing_postgres_to_bq.batch.yaml
+dpone manifest validate examples/batch/landing_postgres_to_bq.batch.yaml --recursive
+dpone manifest render examples/batch/landing_postgres_to_bq.batch.yaml --selector public.core_city
+dpone manifest explain examples/batch/landing_postgres_to_bq.batch.yaml --selector public.core_city --why sink.table.schema
+dpone dag list-edges examples/batch/landing_postgres_to_bq.batch.yaml --with-groups --with-refs
+dpone dag explain-node examples/batch/landing_postgres_to_bq.batch.yaml --task public.core_city
+dpone dag report examples/batch/landing_postgres_to_bq.batch.yaml --preset ci --format md
+```
+
+## Repository layout
+
+```text
+src/dpone/      Python package source code
+docs/           User and developer documentation
+examples/       Public example manifests and registries
+tests/          Unit and integration tests
+tools/          Local smoke and release helper scripts
+```
+
+Canonical imports live under:
+
+- `dpone.manifest.*`
+- `dpone.dag.*`
+- `dpone.runtime.*`
+- `dpone.contracts.*`
+- `dpone.ports.*`
+- `dpone.adapters.*`
+
+Legacy paths such as `dpone.core.*`, `dpone.lib.*`, `dpone.source.*`, and `dpone.sink.*` are compatibility shims. Prefer canonical imports for new code.
+
+## Local development
+
+```bash
+uv sync --all-extras
+uv run ruff check .
+uv run ruff format --check .
+uv run mypy --config-file mypy.ini
+uv run pytest -m "not integration_live"
+```
+
+Build package artifacts:
+
+```bash
+uv build
+```
+
+Run the package smoke script from an installed environment:
+
+```bash
+python tools/package_smoke.py --project-root . --dpone-cmd dpone
+```
+
+## CI and releases
+
+The OSS repository uses GitHub Actions as the primary automation path:
+
+- `.github/workflows/ci.yml` runs linting, formatting checks, type checks, tests, and package build.
+- `.github/workflows/release.yml` builds and publishes tagged releases to PyPI.
+
+Release tags use the format `vX.Y.Z`, for example:
+
+```bash
+git tag -a v0.1.0 -m "Release v0.1.0"
+git push origin v0.1.0
+```
+
+Prefer PyPI Trusted Publishing for releases. Token-based publishing should only be used as a fallback with short-lived, scoped tokens.
+
+## Security
+
+Never commit API tokens, PyPI tokens, GitHub tokens, Vault credentials, service-account JSON, or live vendor credentials. If a secret is ever pasted into an issue, chat, commit, or CI log, revoke it before publishing or pushing public history.
+
+See [SECURITY.md](SECURITY.md) for the vulnerability reporting process.
+
+## Documentation
+
+Start with [docs/README.md](docs/README.md). Key pages:
+
+- [Architecture](docs/ARCHITECTURE.md)
+- [CLI reference](docs/CLI_REFERENCE.md)
+- [CLI examples](docs/CLI_EXAMPLES.md)
+- [Manifest conventions](docs/CONVENTIONS.md)
+- [DAG debugging](docs/DAG_DEBUGGING.md)
+- [Testing](docs/TESTING.md)
+- [Release](docs/RELEASE.md)
+
+## License
+
+`dpone` is licensed under the Apache License 2.0. See [LICENSE](LICENSE).

dpone-0.1.0/README.md

@@ -0,0 +1,225 @@
+# dpone
+
+`dpone` is a Python ETL framework for declarative, YAML-driven data pipelines. It helps data teams describe sources, sinks, load strategies, dependencies, conventions, and operational checks as reusable configuration instead of one-off scripts.
+
+The public package name, import name, GitHub repository name, and CLI name are all intentionally short: `dpone`.
+
+Repository: https://github.com/PaulKov/dpone
+
+## What dpone gives you
+
+- YAML manifests for single-process and batch ETL definitions.
+- Built-in DAG/dependency inspection for pipeline debugging.
+- Runtime abstractions for sources, sinks, connectors, state, reconciliation, and safe SQL logging.
+- Optional integrations for PostgreSQL, ClickHouse, BigQuery/GCS, pandas, Google Ads, and HashiCorp Vault.
+- A CLI designed for self-service validation, rendering, explainability, and documentation checks.
+- Compatibility shims for older import paths while the canonical package layout continues to stabilize.
+
+## Installation
+
+Install the core package from PyPI:
+
+```bash
+pip install dpone
+```
+
+Install common extras for local ETL development:
+
+```bash
+pip install "dpone[postgres,clickhouse,gcp,pandas,vault]"
+```
+
+Install everything currently published by the project:
+
+```bash
+pip install "dpone[full]"
+```
+
+With `uv`:
+
+```bash
+uv add dpone
+uv add "dpone[full]"
+```
+
+## Optional extras
+
+| Extra | Purpose |
+| --- | --- |
+| `postgres` | PostgreSQL connectivity via `psycopg` |
+| `clickhouse` | ClickHouse connectivity |
+| `gcp` | Google BigQuery and Google Cloud Storage support |
+| `pandas` | DataFrame-based extract/load helpers |
+| `vault` | HashiCorp Vault integration via public `vault-kv-client` |
+| `google_ads` | Google Ads API support |
+| `full` | All public extras above |
+
+Vault support uses [`vault-kv-client`](https://github.com/PaulKov/vault-kv-client), published on PyPI as `vault-kv-client`. New code should use `vault_kv_client`; the historical `vault_client` import path remains supported by that package as a compatibility layer.
+
+## Quick start
+
+Create a batch manifest, for example `examples/batch/landing_postgres_to_bq.batch.yaml`:
+
+```yaml
+# yaml-language-server: $schema=../../src/dpone/schema/etl-batch-manifest.schema.json
+kind: dpone.batch.v1
+convention: landing_raw_v1
+registry: ../registry/sources.yaml
+
+vars:
+  src_system: demo_source
+  src_database: demo_db
+  owner_team: data-platform
+  owner_contact: data-platform@example.com
+  sla: daily
+
+defaults:
+  source:
+    type: postgres
+    connection_type: vault
+    connection_id: postgres-demo
+    vault_path: postgres/demo-source
+    options:
+      batch_size: 100000
+      export_format: csv
+
+  sink:
+    type: bigquery
+    connection_type: vault
+    connection_id: bigquery-demo
+    vault_path: gcp/demo-project-prod/bq/service-account
+    staging:
+      schema: stg
+    strategy:
+      mode: full_refresh
+      overwrite_type: exchange
+
+schemas:
+  public:
+    tables:
+      - core_city
+```
+
+Validate and render it:
+
+```bash
+dpone manifest validate examples/batch/landing_postgres_to_bq.batch.yaml \
+  --profile landing_raw_v1 \
+  --registry examples/registry/sources.yaml
+
+dpone manifest render examples/batch/landing_postgres_to_bq.batch.yaml \
+  --selector public.core_city \
+  --registry examples/registry/sources.yaml
+```
+
+Inspect pipeline dependencies:
+
+```bash
+dpone dag report examples/batch/landing_postgres_to_bq.batch.yaml \
+  --base-path . \
+  --format json \
+  --preset ci \
+  --registry examples/registry/sources.yaml
+```
+
+## CLI overview
+
+```bash
+dpone --help
+dpone manifest --help
+dpone dag --help
+dpone docs --help
+```
+
+Common commands:
+
+```bash
+dpone manifest list examples/batch/landing_postgres_to_bq.batch.yaml
+dpone manifest validate examples/batch/landing_postgres_to_bq.batch.yaml --recursive
+dpone manifest render examples/batch/landing_postgres_to_bq.batch.yaml --selector public.core_city
+dpone manifest explain examples/batch/landing_postgres_to_bq.batch.yaml --selector public.core_city --why sink.table.schema
+dpone dag list-edges examples/batch/landing_postgres_to_bq.batch.yaml --with-groups --with-refs
+dpone dag explain-node examples/batch/landing_postgres_to_bq.batch.yaml --task public.core_city
+dpone dag report examples/batch/landing_postgres_to_bq.batch.yaml --preset ci --format md
+```
+
+## Repository layout
+
+```text
+src/dpone/      Python package source code
+docs/           User and developer documentation
+examples/       Public example manifests and registries
+tests/          Unit and integration tests
+tools/          Local smoke and release helper scripts
+```
+
+Canonical imports live under:
+
+- `dpone.manifest.*`
+- `dpone.dag.*`
+- `dpone.runtime.*`
+- `dpone.contracts.*`
+- `dpone.ports.*`
+- `dpone.adapters.*`
+
+Legacy paths such as `dpone.core.*`, `dpone.lib.*`, `dpone.source.*`, and `dpone.sink.*` are compatibility shims. Prefer canonical imports for new code.
+
+## Local development
+
+```bash
+uv sync --all-extras
+uv run ruff check .
+uv run ruff format --check .
+uv run mypy --config-file mypy.ini
+uv run pytest -m "not integration_live"
+```
+
+Build package artifacts:
+
+```bash
+uv build
+```
+
+Run the package smoke script from an installed environment:
+
+```bash
+python tools/package_smoke.py --project-root . --dpone-cmd dpone
+```
+
+## CI and releases
+
+The OSS repository uses GitHub Actions as the primary automation path:
+
+- `.github/workflows/ci.yml` runs linting, formatting checks, type checks, tests, and package build.
+- `.github/workflows/release.yml` builds and publishes tagged releases to PyPI.
+
+Release tags use the format `vX.Y.Z`, for example:
+
+```bash
+git tag -a v0.1.0 -m "Release v0.1.0"
+git push origin v0.1.0
+```
+
+Prefer PyPI Trusted Publishing for releases. Token-based publishing should only be used as a fallback with short-lived, scoped tokens.
+
+## Security
+
+Never commit API tokens, PyPI tokens, GitHub tokens, Vault credentials, service-account JSON, or live vendor credentials. If a secret is ever pasted into an issue, chat, commit, or CI log, revoke it before publishing or pushing public history.
+
+See [SECURITY.md](SECURITY.md) for the vulnerability reporting process.
+
+## Documentation
+
+Start with [docs/README.md](docs/README.md). Key pages:
+
+- [Architecture](docs/ARCHITECTURE.md)
+- [CLI reference](docs/CLI_REFERENCE.md)
+- [CLI examples](docs/CLI_EXAMPLES.md)
+- [Manifest conventions](docs/CONVENTIONS.md)
+- [DAG debugging](docs/DAG_DEBUGGING.md)
+- [Testing](docs/TESTING.md)
+- [Release](docs/RELEASE.md)
+
+## License
+
+`dpone` is licensed under the Apache License 2.0. See [LICENSE](LICENSE).

dpone-0.1.0/pyproject.toml

@@ -0,0 +1,156 @@
+[build-system]
+requires = ["uv_build>=0.9.18,<0.10.0"]
+build-backend = "uv_build"
+
+[project]
+name = "dpone"
+version = "0.1.0"
+description = "Declarative ETL framework for YAML-driven data pipelines"
+readme = "README.md"
+requires-python = ">=3.11"
+license = "Apache-2.0"
+authors = [{ name = "PaulKov" }]
+maintainers = [{ name = "PaulKov" }]
+keywords = [
+  "etl",
+  "data-platform",
+  "data-engineering",
+  "yaml",
+  "dag",
+  "bigquery",
+  "postgres",
+  "clickhouse",
+]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "Intended Audience :: Information Technology",
+  "License :: OSI Approved :: Apache Software License",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+  "Topic :: Database",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+  "Topic :: System :: Distributed Computing",
+  "Typing :: Typed",
+]
+dependencies = [
+  "pyyaml>=6.0.1",
+  "jinja2>=3.1.0",
+  "sqlglot==27.13.2",
+  "pendulum==3.1.0",
+  "prettytable==3.17.0",
+  "pytz>=2023.3",
+  "requests>=2.32.5",
+  "ijson>=3.3.0"
+]
+
+[project.scripts]
+dpone = "dpone.cli.main:main"
+dpone-runtime-install = "dpone.runtime.dev_install:main_install"
+dpone-runtime-exec = "dpone.runtime.dev_install:main_exec"
+
+[project.urls]
+Homepage = "https://github.com/PaulKov/dpone"
+Repository = "https://github.com/PaulKov/dpone"
+Issues = "https://github.com/PaulKov/dpone/issues"
+Documentation = "https://github.com/PaulKov/dpone#readme"
+Changelog = "https://github.com/PaulKov/dpone/blob/main/CHANGELOG.md"
+
+[project.optional-dependencies]
+postgres = ["psycopg[binary]==3.2.9"]
+clickhouse = ["clickhouse-connect==0.6.22", "clickhouse-driver==0.2.9", "clickhouse-cityhash==1.0.2.5"]
+gcp = ["google-cloud-bigquery==3.38.0", "google-cloud-storage==3.7.0"]
+pandas = ["pandas==2.1.4", "numpy==1.26.4"]
+vault = ["vault-kv-client>=0.1.0,<0.2.0"]
+google_ads = ["google-ads>=28.0.0,<29.0.0"]
+full = [
+  "psycopg[binary]==3.2.9",
+  "clickhouse-connect==0.6.22",
+  "clickhouse-driver==0.2.9",
+  "clickhouse-cityhash==1.0.2.5",
+  "google-cloud-bigquery==3.38.0",
+  "google-cloud-storage==3.7.0",
+  "pandas==2.1.4",
+  "numpy==1.26.4",
+  "vault-kv-client>=0.1.0,<0.2.0",
+  "google-ads>=28.0.0,<29.0.0",
+]
+
+[dependency-groups]
+dev = [
+  "pytest>=8.0.0",
+  "pytest-cov>=5.0.0",
+  "ruff>=0.6.9",
+  "mypy>=1.11.2",
+  "pre-commit>=3.8.0",
+  "types-PyYAML>=6.0.12.20240917",
+  "types-requests>=2.32.0.20240914",
+  "minio>=7.2.9",
+]
+
+[tool.uv]
+default-groups = ["dev"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-q"
+markers = [
+  "integration: tests that require external services (PostgreSQL/ClickHouse)",
+  "integration_extended: extended/nightly integration tests (MinIO/GCS-export and installed-package scenarios)",
+  "integration_live: manual/nightly live integration tests against real vendor APIs",
+  "nightly: tests intended for scheduled or manual nightly-like pipelines",
+  "packaging: tests that validate installed package artifacts",
+]
+
+[tool.ruff]
+target-version = "py311"
+line-length = 120
+src = ["src", "tests", "tools"]
+extend-exclude = [".cache"]
+
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F", "I", "UP"]
+ignore = ["E501"]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+line-ending = "lf"
+
+[tool.coverage.run]
+source = ["src/dpone"]
+omit = [
+  # Legacy compatibility import shims are covered through their canonical runtime modules.
+  "src/dpone/credentials/*",
+  "src/dpone/etl/*",
+  "src/dpone/etl_logging/*",
+  "src/dpone/reconciliation/**",
+  "src/dpone/sink/**",
+  "src/dpone/source/**",
+  "src/dpone/sql_helpers/**",
+  "src/dpone/state/**",
+  "src/dpone/xmin/**",
+  "src/dpone/yaml_config_handler/**",
+
+  # Live adapters require external services or vendor credentials and are
+  # exercised by explicit integration profiles, not by the public OSS CI gate.
+  "src/dpone/runtime/connectors/api/*",
+  "src/dpone/runtime/connectors/bigquery.py",
+  "src/dpone/runtime/connectors/postgres.py",
+  "src/dpone/runtime/connectors/clickhouse*.py",
+  "src/dpone/runtime/sources/strategies/api/**",
+  "src/dpone/runtime/sources/strategies/clickhouse/**",
+  "src/dpone/runtime/sources/strategies/postgres/**",
+  "src/dpone/runtime/sinks/bigquery*.py",
+  "src/dpone/runtime/sinks/postgres*.py",
+  "src/dpone/runtime/sinks/strategies/**",
+]
+
+[tool.coverage.report]
+fail_under = 70
+show_missing = true
+skip_covered = true