pydatarheo 0.1.0__tar.gz

pydatarheo-0.1.0/.github/workflows/docs.yml

@@ -0,0 +1,70 @@
+name: Publish docs
+
+# The API reference is generated from the source by pdoc, so it is rebuilt on
+# every release and on every push to main rather than being checked in.
+on:
+  push:
+    branches: [main]
+    tags: ["v*"]
+  workflow_dispatch:
+
+# Allow one docs deployment at a time. A queued run waits rather than cancelling,
+# so a release build is never dropped in favour of a later push to main.
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    outputs:
+      pages_ready: ${{ steps.configure_pages.outcome == 'success' }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # uv-dynamic-versioning needs full history and tags.
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Generate API reference
+        run: uv run poe docs-generate
+
+      - name: Fail if the generated docs are empty
+        run: test -s docs/generated/datarheo.html
+
+      # GitHub's workflow token can deploy to an existing Pages site, but it may not
+      # be allowed to create one for the repository. Keep the docs build green and
+      # only deploy when Pages is already enabled.
+      - id: configure_pages
+        continue-on-error: true
+        uses: actions/configure-pages@v5
+        with:
+          enablement: true
+      - name: Skip deploy when Pages is unavailable
+        if: steps.configure_pages.outcome != 'success'
+        run: echo "::warning::Skipping GitHub Pages deployment because this workflow cannot enable Pages for the repository."
+      - uses: actions/upload-pages-artifact@v3
+        if: steps.configure_pages.outcome == 'success'
+        with:
+          path: docs/generated
+
+  deploy:
+    if: needs.build.outputs.pages_ready == 'true'
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - id: deployment
+        uses: actions/deploy-pages@v4

pydatarheo-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,74 @@
+name: Publish to PyPI
+
+# Publishing is triggered by pushing a version tag, so the released version is
+# always the tag itself. `uv-dynamic-versioning` derives the package version
+# from that tag, which is why no version is written in pyproject.toml.
+on:
+  push:
+    tags:
+      - "v*"
+  workflow_dispatch:
+    inputs:
+      target:
+        description: "Where to publish"
+        required: true
+        default: testpypi
+        type: choice
+        options: [testpypi, pypi]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # uv-dynamic-versioning needs full history and tags.
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Build sdist and wheel
+        run: uv build
+
+      - name: Verify the wheel imports from a clean environment
+        run: |
+          uv venv -p 3.11 /tmp/verify
+          VIRTUAL_ENV=/tmp/verify uv pip install dist/*.whl
+          /tmp/verify/bin/python -c "import datarheo; print(datarheo.__name__, 'ok')"
+          /tmp/verify/bin/pydr --help > /dev/null
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish-testpypi:
+    needs: build
+    if: github.event_name == 'workflow_dispatch' && inputs.target == 'testpypi'
+    runs-on: ubuntu-latest
+    environment: testpypi
+    permissions:
+      id-token: write  # Required for Trusted Publishing; no API token is stored.
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+  publish-pypi:
+    needs: build
+    if: github.event_name == 'push' || inputs.target == 'pypi'
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # Required for Trusted Publishing; no API token is stored.
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

pydatarheo-0.1.0/.gitignore

@@ -0,0 +1,29 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.Python
+
+# Build artifacts
+/build/
+/dist/
+/htmlcov/
+.coverage
+coverage.xml
+
+# Environments
+.venv/
+/venv/
+.env
+
+# Tooling caches
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.pyre/
+.viztracer/
+
+# Local secrets
+/secrets/
+*.env
+!*.env.example

pydatarheo-0.1.0/AGENTS.md

@@ -0,0 +1,28 @@
+# Agents
+
+Notes for AI agents working in this repository.
+
+## Naming
+
+This project is **PyDataRheo**. The import package is `datarheo`, the PyPI distribution is
+`pydatarheo`, and the CLI entry points are `pydatarheo` / `pydr` and `datarheo-mcp`.
+
+References to **Airbyte** are deliberate and should not be renamed. They name external systems
+that PyDataRheo integrates with: the Airbyte protocol and its message classes, the `airbyte-cdk`
+/ `airbyte-api` / `airbyte-protocol-models-pdv2` packages, the public connector registry, and the
+Airbyte Cloud API. Renaming those would either break imports or make the documentation wrong.
+
+Everything that names *this* project uses the DataRheo name: exception classes (`DataRheoError`
+and friends), environment variables (`DATARHEO_*`), cache metadata columns (`_datarheo_raw_id`,
+`_datarheo_extracted_at`, `_datarheo_meta`), and the default cache schema (`datarheo_raw`).
+
+## MCP UI Development
+
+When adding or changing MCP tools that return UI elements:
+
+- Name UI-first tools with a `show_` prefix.
+- Return bounded agent-readable text plus structured UI content.
+- Make any capped agent preview explicit, because the agent cannot see the user-facing UI.
+- Verify the server-side payload contract.
+- Capture human-reviewable evidence with MCPJam or Goose Desktop when retesting is requested,
+  including the rendered widget and any important UI interaction.

pydatarheo-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,17 @@
+# Contributing
+
+Thank you for your interest in contributing to PyDataRheo!
+
+## Docstring Style
+
+Use **Markdown** formatting in all docstrings — not reStructuredText (reST).
+
+- Use single backticks for inline code: `` `MyClass` ``, not double backticks (` ``MyClass`` `).
+- Reference methods as `` `get_column_info()` ``, not `:meth:\`get_column_info\``.
+- Use standard Markdown for emphasis, lists, and links.
+
+## 🚀 Releasing
+
+This project uses [`semantic-pr-release-drafter`](https://github.com/aaronsteers/semantic-pr-release-drafter) for automated release management. To release, simply click "`Edit`" on the latest release draft from the [releases page](https://github.com/Vrahad-Analytics/pydatarheo/releases), and then click "`Publish release`". This publish operation will trigger all necessary downstream publish operations.
+
+ℹ️ For more detailed instructions, please see the [Releasing Guide](https://github.com/aaronsteers/semantic-pr-release-drafter/blob/main/docs/releasing.md).

pydatarheo-0.1.0/Dockerfile.mcp

@@ -0,0 +1,25 @@
+FROM python:3.12-slim
+
+COPY --from=ghcr.io/astral-sh/uv:0.8.17 /uv /uvx /bin/
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y git \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY pyproject.toml uv.lock README.md /app/
+COPY datarheo/ /app/datarheo/
+
+RUN uv sync --frozen --no-dev
+
+RUN useradd -m -u 10001 appuser \
+    && chown -R appuser:appuser /app
+
+ENV PATH="/app/.venv/bin:$PATH"
+
+EXPOSE 8080
+
+USER appuser
+
+CMD ["datarheo-mcp-http"]

pydatarheo-0.1.0/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026 Vrahad Analytics LLP
+Copyright (c) 2023-2025 Airbyte, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pydatarheo-0.1.0/NOTICE

@@ -0,0 +1,26 @@
+PyDataRheo
+Copyright (c) 2026 Vrahad Analytics LLP
+
+This product is a derivative work of PyAirbyte
+(https://github.com/airbytehq/PyAirbyte), Copyright (c) 2023-2025 Airbyte, Inc.,
+distributed under the MIT License. PyDataRheo is a rebranded and modified fork:
+the package, public API names, CLI commands, environment variables, and
+documentation have been renamed, and usage reporting has been changed to
+opt-in with no default endpoint.
+
+The MIT License requires that the original copyright notice be retained. Both
+copyright lines appear in the LICENSE file at the root of this repository.
+
+PyDataRheo executes connectors that implement the Airbyte protocol and can talk
+to the Airbyte Cloud API. "Airbyte" is a trademark of Airbyte, Inc. References
+to Airbyte in this project identify those external systems and do not imply
+endorsement by, or affiliation with, Airbyte, Inc.
+
+This product depends on the following third-party packages, which retain their
+own licenses and copyright:
+
+  - airbyte-cdk                   (Airbyte, Inc.)
+  - airbyte-api                   (Airbyte, Inc.)
+  - airbyte-protocol-models-pdv2  (Airbyte, Inc.)
+
+See the distribution metadata of each dependency for its full license text.

pydatarheo-0.1.0/PKG-INFO

@@ -0,0 +1,238 @@
+Metadata-Version: 2.5
+Name: pydatarheo
+Version: 0.1.0
+Summary: PyDataRheo: run data-integration connectors from Python.
+Project-URL: Homepage, https://github.com/Vrahad-Analytics/pydatarheo
+Project-URL: Documentation, https://vrahad-analytics.github.io/pydatarheo
+Project-URL: Repository, https://github.com/Vrahad-Analytics/pydatarheo
+Project-URL: Issues, https://github.com/Vrahad-Analytics/pydatarheo/issues
+Author-email: Vrahad Analytics LLP <ardb40@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+License-File: NOTICE
+Keywords: connectors,data-integration,datarheo,elt,etl,mcp
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Database
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: <3.13,>=3.10
+Requires-Dist: airbyte-api<1.0,>=0.53.0
+Requires-Dist: airbyte-cdk<8.0.0,>=7.28.3
+Requires-Dist: airbyte-protocol-models-pdv2<1.0,>=0.13.0
+Requires-Dist: cryptography!=45.0.0,!=45.0.1,<47.0.0,>=44.0.0
+Requires-Dist: cyclopts<5.0,>=4.0
+Requires-Dist: duckdb-engine==0.17.0
+Requires-Dist: duckdb==1.4.3
+Requires-Dist: fastmcp-extensions<1.0.0,>=0.22.0
+Requires-Dist: fastmcp<4.0,>=3.0
+Requires-Dist: google-auth<3.0,>=2.27.0
+Requires-Dist: google-cloud-bigquery-storage<3.0,>=2.25.0
+Requires-Dist: google-cloud-bigquery<4.0,>=3.12.0
+Requires-Dist: google-cloud-secret-manager<3.0,>=2.17.0
+Requires-Dist: jsonschema<5.0,>=3.2.0
+Requires-Dist: orjson<4.0,>=3.10
+Requires-Dist: overrides<8.0,>=7.4.0
+Requires-Dist: pandas<3.0,>=1.5.3
+Requires-Dist: prefab-ui<0.21,>=0.20.1
+Requires-Dist: psycopg2-binary<3.0,>=2.9.9
+Requires-Dist: psycopg[binary,pool]<4.0,>=3.1.19
+Requires-Dist: pyarrow<22.0,>=16.1
+Requires-Dist: pydantic-core
+Requires-Dist: pydantic<3.0,>=2.0
+Requires-Dist: python-dotenv<2.0,>=1.0.1
+Requires-Dist: python-ulid<4.0,>=3.0.0
+Requires-Dist: pyyaml<7.0,>=6.0.2
+Requires-Dist: requests!=3.32.0
+Requires-Dist: rich<14.0,>=13.7.0
+Requires-Dist: snowflake-connector-python<4.0,>=3.12.2
+Requires-Dist: snowflake-sqlalchemy<2.0,>=1.6.1
+Requires-Dist: sqlalchemy-bigquery==1.12.0
+Requires-Dist: sqlalchemy==2.0.43
+Requires-Dist: starlette
+Requires-Dist: structlog<25.0,>=24.4.0
+Requires-Dist: typing-extensions
+Requires-Dist: uuid7<1.0,>=0.1.0
+Requires-Dist: uv<0.9.0,>=0.5.0
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/Vrahad-Analytics/pydatarheo/main/docs/brand/pydatarheo-logo.svg" alt="PyDataRheo" width="260">
+</p>
+
+# PyDataRheo
+
+**PyDataRheo moves data between hundreds of systems, from plain Python.**
+
+Pick a source, read from it, and land the records in DuckDB, Postgres, Snowflake, BigQuery,
+MotherDuck, or any destination connector. No orchestrator, no cluster, no YAML pipeline
+definition. It is a library, so it runs wherever your Python runs: a script, a notebook, a
+Lambda, an Airflow or Dagster task.
+
+PyDataRheo runs [Airbyte-protocol connectors](https://docs.airbyte.com/integrations/), which is
+what gives it several hundred sources and destinations on day one.
+
+[![PyPI version](https://badge.fury.io/py/pydatarheo.svg)](https://badge.fury.io/py/pydatarheo)
+[![PyPI - Downloads](https://img.shields.io/pypi/dm/pydatarheo)](https://pypi.org/project/pydatarheo/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pydatarheo)](https://pypi.org/project/pydatarheo/)
+[![Star on GitHub](https://img.shields.io/github/stars/Vrahad-Analytics/pydatarheo.svg?style=social&label=★%20on%20GitHub)](https://github.com/Vrahad-Analytics/pydatarheo)
+
+## Install
+
+```bash
+pip install pydatarheo
+```
+
+The import package is `datarheo`; the distribution on PyPI is `pydatarheo`.
+
+## Quick start
+
+```py
+import datarheo as dr
+
+source = dr.get_source(
+    "source-faker",
+    config={"count": 5_000},
+    install_if_missing=True,
+)
+source.check()
+source.select_all_streams()
+
+result = source.read()  # Lands in a local DuckDB cache by default.
+
+for record in result["users"].records:
+    print(record)
+```
+
+Read into a dataframe:
+
+```py
+df = result["users"].to_pandas()
+```
+
+Write to a real warehouse instead of the default local cache:
+
+```py
+from datarheo.caches import SnowflakeCache
+
+cache = SnowflakeCache(
+    account="my-account",
+    username="my-user",
+    password=dr.get_secret("SNOWFLAKE_PASSWORD"),
+    database="my-db",
+    warehouse="my-wh",
+    role="my-role",
+)
+source.read(cache=cache)
+```
+
+## Command line
+
+These console scripts are installed:
+
+| Command | What it does |
+| --- | --- |
+| `pydatarheo` / `pydr` | Benchmark, validate, and smoke-test connectors |
+| `datarheo-mcp` | Run the DataRheo MCP server over stdio |
+| `datarheo-mcp-http` | Run the DataRheo MCP server over HTTP |
+
+```bash
+pydr --help
+pydr validate --connector=source-faker
+```
+
+## Connector installation
+
+### Declarative (YAML) sources
+
+Declarative sources are downloaded as a single YAML manifest and executed directly. These have
+the fastest install times, because there is nothing to build.
+
+### Python sources
+
+When a connector publishes a Python package, PyDataRheo installs it automatically. Installation
+uses [`uv`](https://docs.astral.sh/uv) rather than `pip`, which is both much faster and able to
+install a connector under a different Python version than the one PyDataRheo itself is running
+on. Set `DATARHEO_NO_UV=true` to fall back to `pip`.
+
+To pin a connector to a specific Python version:
+
+```py
+source = dr.get_source("source-faker", use_python="3.10.17")
+```
+
+### Docker
+
+Pass `docker_image=True` to `get_source()` or `get_destination()` to run the connector from its
+published image instead, or `docker_image="my-org/my-image:tag"` to pick an exact image. Docker
+is the most reproducible option, because every dependency is locked inside the image, and it is
+required for Java-based destinations.
+
+## Configuration
+
+Every environment variable PyDataRheo reads is prefixed `DATARHEO_`. The most useful ones:
+
+| Variable | Effect |
+| --- | --- |
+| `DATARHEO_CACHE_ROOT` | Where cache files are written (default `./.cache`) |
+| `DATARHEO_PROJECT_DIR` | Parent directory for cache and connector installs |
+| `DATARHEO_NO_UV` | Set to `1` to install connectors with `pip` instead of `uv` |
+| `DATARHEO_OFFLINE_MODE` | Tolerate an unreachable connector registry; also disables telemetry |
+| `DATARHEO_TEMP_DIR` | Directory for temporary files |
+| `DATARHEO_CLOUD_CLIENT_ID` / `DATARHEO_CLOUD_CLIENT_SECRET` | Credentials for the hosted Cloud API |
+
+An environment still carrying the older `AIRBYTE_`-prefixed names keeps working: each
+`AIRBYTE_FOO` is read as `DATARHEO_FOO` unless the `DATARHEO_` name is also set, in which case
+the explicit `DATARHEO_` value wins.
+
+## Telemetry
+
+**Off by default.** PyDataRheo sends nothing anywhere unless you set `DATARHEO_TRACKING_KEY` to
+a Segment write key that you own. Setting `DO_NOT_TRACK`, or enabling offline mode, disables
+reporting outright.
+
+## Documentation
+
+The full API reference is generated from the source with `poe docs-generate` and published at
+[vrahad-analytics.github.io/pydatarheo](https://vrahad-analytics.github.io/pydatarheo).
+
+## Contributing
+
+See the [Contributors Guide](https://github.com/Vrahad-Analytics/pydatarheo/blob/main/docs/CONTRIBUTING.md).
+
+## Frequently asked questions
+
+**Is PyDataRheo a replacement for a data platform?**
+No. It is a library. It has no orchestration, scheduling, alerting, or pipeline monitoring. Run
+it inside whatever scheduler you already use.
+
+**What is the cache? Is it a destination?**
+Effectively yes: it is a built-in destination implementation backed by a SQL engine. We call it
+a cache to keep it distinct from the destination connectors, which are separate executables.
+
+**Does it work with Airflow, Dagster, Prefect, or Snowpark?**
+Yes. It is an ordinary Python dependency with no background services.
+
+**Can I build a normal ETL pipeline with it?**
+Yes. Choose the cache type that matches where the data should land, such as `SnowflakeCache` for
+Snowflake.
+
+**Can it run a connector from a local directory?**
+Yes. Any connector exposing a CLI works, and connectors already on `PATH` are found by name.
+
+## License and provenance
+
+PyDataRheo is released under the MIT license and is a rebranded derivative of
+[PyAirbyte](https://github.com/airbytehq/PyAirbyte), which is also MIT licensed. See
+[LICENSE](https://github.com/Vrahad-Analytics/pydatarheo/blob/main/LICENSE) and [NOTICE](https://github.com/Vrahad-Analytics/pydatarheo/blob/main/NOTICE).
+
+## Changelog
+
+See the [GitHub Releases](https://github.com/Vrahad-Analytics/pydatarheo/releases) page.