sunstone-data-platform 1.0.0__tar.gz

sunstone_data_platform-1.0.0/.gitignore

@@ -0,0 +1,22 @@
+__marimo__
+__pycache__
+*.egg-info/
+.env
+
+# Generated by hatch-vcs at build time
+src/sunstone_data/_version.py
+
+# Agent/planning/private config (not for public tree)
+.agents/
+.planning/
+old-stuff/
+
+# Security prompts
+security-prompts/
+.DS_Store
+.claude/*.local.*
+.worktrees
+/TODO.md
+/research-stack
+/target/
+CLAUDE.local.md

sunstone_data_platform-1.0.0/PKG-INFO

@@ -0,0 +1,302 @@
+Metadata-Version: 2.5
+Name: sunstone-data-platform
+Version: 1.0.0
+Summary: Sunstone Data Platform - Data lakehouse with semantic metadata
+Author-email: Sunstone Institute <stig@sunstone.institute>
+License-Expression: MIT
+Requires-Python: >=3.12
+Requires-Dist: charset-normalizer>=3.4.0
+Requires-Dist: duckdb>=1.5.3
+Requires-Dist: fastexcel>=0.20.2
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: pyiceberg[duckdb,gcsfs,s3fs]>=0.10.0
+Requires-Dist: python-dateutil>=2.9.0.post0
+Requires-Dist: python-dotenv>=1.2.2
+Requires-Dist: python-slugify>=8.0.0
+Requires-Dist: rich>=14.0.0
+Requires-Dist: sunstone-py>=1.15.0
+Requires-Dist: typer>=0.27.1
+Provides-Extra: pipelines
+Requires-Dist: dagster-docker>=0.29.0; extra == 'pipelines'
+Requires-Dist: dagster-postgres>=0.29.0; extra == 'pipelines'
+Requires-Dist: dagster-webserver>=1.13.0; extra == 'pipelines'
+Requires-Dist: dagster>=1.13.9; extra == 'pipelines'
+Description-Content-Type: text/markdown
+
+# Sunstone Data Platform
+
+A data lakehouse for importing, versioning, and querying external datasets with full provenance tracking. Built on Apache Iceberg, iceberg-catalog, RustFS, and DuckDB.
+
+Data scientists can import CSV/Excel files from local paths or URLs, query them via DuckDB with git-like version references, and track lineage through notebook sessions. All provenance is captured using W3C PROV-DM so metadata can later materialize as a knowledge graph.
+
+## Architecture
+
+```
+                        ┌──────────────┐
+                        │  sunstone    │
+                        │  CLI (Typer) │
+                        └──────┬───────┘
+                               │
+              ┌────────────────┼────────────────┐
+              │                │                │
+     ┌────────▼───────┐ ┌─────▼──────┐ ┌───────▼────────┐
+     │ Import Pipeline │ │   Query    │ │   Versioning   │
+     │ fetch → parse → │ │  (DuckDB)  │ │ branches/tags/ │
+     │ store → metadata│ │            │ │ history/diff   │
+     └────────┬────────┘ └─────┬──────┘ └───────┬────────┘
+              │                │                │
+              └────────────────┼────────────────┘
+                               │
+                    ┌──────────▼──────────┐
+                    │  iceberg-catalog    │
+                    │  (Iceberg REST API) │
+                    └──────────┬──────────┘
+                               │
+                    ┌──────────▼──────────┐
+                    │   RustFS (S3)       │
+                    │   Iceberg tables +  │
+                    │   original files    │
+                    └─────────────────────┘
+```
+
+**Stack**: Python 3.11+ | PyIceberg | DuckDB | iceberg-catalog | RustFS | PostgreSQL 16
+
+## Endpoints
+
+- `/catalog` — DCAT JSON-LD view of the lakehouse, served by graph-server views
+  (see [the views design](docs/specs/2026-08-02-graph-server-views-design.md))
+- `/ui/` — the graph explorer, a 3D view of the data graph, served by
+  graph-server (see [docs/graph-explorer.md](docs/graph-explorer.md))
+
+## Local Development Setup
+
+### Prerequisites
+
+- Python 3.11+
+- [uv](https://docs.astral.sh/uv/) package manager
+- Docker and Docker Compose
+
+### 1. Clone and install
+
+```bash
+git clone <repo-url>
+cd data-platform
+uv sync
+```
+
+### 2. Configure environment
+
+```bash
+cp .env.example .env
+# Edit .env if you want to change default credentials
+```
+
+### 3. Start the infrastructure
+
+```bash
+docker compose --profile s3 up -d
+```
+
+This starts the graph stack (Postgres, `graph-migrate`, `oxigraph`,
+`graph-server`, `materializer` — see
+[docs/architecture.md](docs/architecture.md#infrastructure)) plus:
+- **iceberg-catalog** on port 8181 (Iceberg REST catalog, S3-backed)
+- **object-store** (RustFS) on port 9000 (S3-compatible storage, console on 9001)
+
+For GCS testing, use `docker compose --profile gcs up -d` instead, which
+starts `fake-gcs-server` — note there's no GCS-backed iceberg-catalog wired
+up yet, so this profile provides the emulator but not a working GCS
+catalog.
+
+### Cloudflare R2 backend
+
+The `r2` profile hasn't moved to the graph stack yet — it still runs Nessie
+directly, against a real Cloudflare R2 bucket (no local object store):
+
+1. Create an R2 bucket in the Cloudflare dashboard.
+2. Create an R2 API token (R2 > Manage R2 API Tokens) with **Object Read & Write** scoped to that bucket. Copy the Access Key ID and Secret Access Key.
+3. Copy `.env.r2.example` values into your `.env` and fill in `R2_ACCOUNT_ID`, `R2_BUCKET`, `R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`.
+4. If your Postgres volume already exists from an earlier `s3`/`gcs` run, create the R2 version-store database once (the init script in `docker/init-databases.sql` only runs on a *fresh* `iceberg-db` volume):
+
+   ```bash
+   docker compose up -d postgres
+   docker compose exec postgres \
+     psql -U iceberg -d iceberg \
+     -c "CREATE DATABASE nessie_r2; GRANT ALL PRIVILEGES ON DATABASE nessie_r2 TO iceberg;"
+   ```
+
+   (A fresh stack — or `docker compose down -v && docker compose --profile r2 up -d` — creates it automatically. The reset destroys all local data.)
+5. Start the R2 profile:
+
+   ```bash
+   docker compose --profile r2 up -d
+   ```
+
+   Nessie listens on port 19122. Point your client at `http://localhost:19122`.
+
+R2-specific Nessie settings the profile sets for you:
+
+- `region=auto` (R2 ignores region but the SDK requires a value).
+- `endpoint=https://<R2_ACCOUNT_ID>.r2.cloudflarestorage.com` (account-style host).
+- `path-style-access=true` so requests are `<endpoint>/<bucket>/<key>` — R2 accepts both styles but path-style avoids virtual-host DNS issues with some older clients.
+
+The local PyIceberg/DuckDB client reads Iceberg data files straight from R2, so it needs the same bucket and credentials. It derives those from the same four `R2_*` vars automatically — warehouse (`s3://<R2_BUCKET>/`), endpoint, access keys, `region=auto`, and a default `CATALOG_URI` of `http://localhost:19122`. No separate `S3_*`/`ICEBERG_WAREHOUSE` settings are required; if you set them explicitly they take precedence.
+
+### 4. Verify the stack
+
+```bash
+uv run scripts/smoke_test.py
+```
+
+### 5. Set up your environment
+
+The `sunstone` CLI comes from sunstone-py. data-platform registers as a plugin and adds the `data` subcommand group:
+
+```bash
+# Add your local Docker stack as an environment
+sunstone env add local --catalog-url http://localhost:19120 --s3-endpoint http://localhost:9000
+sunstone env use local
+```
+
+### 6. Try it out
+
+```bash
+# Import a local CSV
+sunstone data import path/to/data.csv
+
+# Import from a URL
+sunstone data import https://example.com/dataset.csv
+
+# List datasets
+sunstone data info
+
+# View dataset metadata
+sunstone data info namespace.table_name
+
+# Query history
+sunstone data history namespace.table_name
+
+# Compare versions
+sunstone data diff namespace.table_name @v1 @v2
+```
+
+## Running Tests
+
+```bash
+# Unit tests (fast, no Docker required)
+uv run pytest tests/unit/
+
+# Integration tests (requires Docker stack running)
+uv run pytest tests/integration/
+
+# All tests
+uv run pytest
+
+# Specific test file with verbose output
+uv run pytest tests/unit/test_query.py -v
+```
+
+## CLI Commands
+
+| Command | Description |
+|---------|-------------|
+| `sunstone data import <source>` | Import CSV/Excel from path or URL |
+| `sunstone data info [table]` | View metadata for one or all datasets |
+| `sunstone data history <table>` | Show snapshot history (git-log style) |
+| `sunstone data tag <table> <tag>` | Tag a snapshot version |
+| `sunstone data untag <table> <tag>` | Remove a tag |
+| `sunstone data branches` | List branches and tags |
+| `sunstone data diff <table> <from> [to]` | Compare two versions |
+| `sunstone data lineage <table>` | Show provenance and lineage |
+| `sunstone data original <table>` | Download the original imported source file |
+
+All commands support `--branch` for working on version branches and `--json` for machine-readable output. Run `sunstone data <command> --help` for full options.
+
+## Key Concepts
+
+- **Table references** use git-like syntax: `namespace.table@branch`, `@v1.0`, `@2026-01-25`, `@snapshot_id`
+- **Deduplication**: Imports are checksum-based (SHA256). Re-importing the same file is a no-op.
+- **Schema evolution**: Safe type promotions (e.g. int32 to int64) are allowed automatically. Unsafe changes require `--force-schema`.
+- **Provenance**: Every import captures source metadata, schema info, and W3C PROV-DM provenance as Iceberg table properties.
+- **Original files**: Source files are stored content-addressable in S3 alongside the Iceberg data.
+
+## sunstone: URLs
+
+Datasets in the lakehouse can be referenced as URLs anywhere sunstone-py accepts a data source:
+
+```python
+import sunstone
+
+df = sunstone.read("sunstone:namespace/table@v1.0")
+```
+
+Supports the full ref syntax: `@branch`, `@tag`, `@2026-01-25`, `@snapshot_id`.
+
+## Project Structure
+
+```
+src/sunstone_data/
+├── plugin.py          # sunstone-py plugin (CLIProvider + URLHandler)
+├── url.py             # sunstone: URL parser
+├── config.py          # Settings from DataEnvironment or .env
+├── catalog.py         # PyIceberg RestCatalog connection to the Iceberg REST catalog
+├── query.py           # DuckDB QuerySession with time-travel support
+├── lineage.py         # Lineage in Iceberg table properties
+├── cli/               # Typer CLI commands + Rich formatters
+├── import_/           # Pipeline: fetch → parse → store → metadata
+├── metadata/          # Frozen dataclasses, MetadataBuilder, W3C PROV-DM
+├── versioning/        # Version branches/tags, refs, schema evolution, diff
+└── utils/             # Checksums, encoding detection, naming
+```
+
+## Documentation
+
+- [Getting Started](docs/getting-started.md) - Detailed setup instructions
+- [Architecture](docs/architecture.md) - System design and data flow
+- [CLI Reference](docs/cli-reference.md) - Complete command documentation
+- [Import Pipeline](docs/import-pipeline.md) - How imports work
+- [Development Guide](docs/development.md) - Contributing and testing
+
+## Resetting the Stack
+
+To wipe all data and start fresh:
+
+```bash
+docker compose --profile s3 --profile gcs down -v
+docker compose --profile s3 up -d
+```
+
+## License
+
+MIT
+
+## Graph server authentication
+
+`graph-server` requires a Keycloak bearer token on every route except
+`/healthz`/`/readyz`. Access is role-based: the client roles in your token
+determine which environment you can use. See
+`docs/specs/2026-07-05-graph-server-auth-design.md`.
+
+No hostnames or client ids ship in this package. Each environment's
+`graph_url` comes from config: Sunstone-managed machines get
+`/etc/sunstone/data_platform.toml` from the `fleet-config` repo, and other
+machines set `GRAPH_URL` or add an environment with `sunstone env add`.
+graph-server publishes the OIDC issuer and public client id at
+`$GRAPH_URL/ui/config.json`; the CLI reads them from there.
+
+### Humans (CLI, local agents)
+
+```bash
+sunstone env init          # cache discovery for configured environments
+sunstone data login        # browser sign-in (authorization code + PKCE)
+```
+
+A browser window opens for Keycloak SSO; the refresh token renews silently
+until the SSO session expires.
+
+### Services (CI, automation)
+
+Set `SUNSTONE_CLIENT_ID` and `SUNSTONE_CLIENT_SECRET` (or `client_id` and
+`client_secret` in the environment's `data-platform` section). The client then
+uses the OAuth2 client-credentials flow against the discovered issuer instead
+of a cached human login.

sunstone_data_platform-1.0.0/README.md

@@ -0,0 +1,277 @@
+# Sunstone Data Platform
+
+A data lakehouse for importing, versioning, and querying external datasets with full provenance tracking. Built on Apache Iceberg, iceberg-catalog, RustFS, and DuckDB.
+
+Data scientists can import CSV/Excel files from local paths or URLs, query them via DuckDB with git-like version references, and track lineage through notebook sessions. All provenance is captured using W3C PROV-DM so metadata can later materialize as a knowledge graph.
+
+## Architecture
+
+```
+                        ┌──────────────┐
+                        │  sunstone    │
+                        │  CLI (Typer) │
+                        └──────┬───────┘
+                               │
+              ┌────────────────┼────────────────┐
+              │                │                │
+     ┌────────▼───────┐ ┌─────▼──────┐ ┌───────▼────────┐
+     │ Import Pipeline │ │   Query    │ │   Versioning   │
+     │ fetch → parse → │ │  (DuckDB)  │ │ branches/tags/ │
+     │ store → metadata│ │            │ │ history/diff   │
+     └────────┬────────┘ └─────┬──────┘ └───────┬────────┘
+              │                │                │
+              └────────────────┼────────────────┘
+                               │
+                    ┌──────────▼──────────┐
+                    │  iceberg-catalog    │
+                    │  (Iceberg REST API) │
+                    └──────────┬──────────┘
+                               │
+                    ┌──────────▼──────────┐
+                    │   RustFS (S3)       │
+                    │   Iceberg tables +  │
+                    │   original files    │
+                    └─────────────────────┘
+```
+
+**Stack**: Python 3.11+ | PyIceberg | DuckDB | iceberg-catalog | RustFS | PostgreSQL 16
+
+## Endpoints
+
+- `/catalog` — DCAT JSON-LD view of the lakehouse, served by graph-server views
+  (see [the views design](docs/specs/2026-08-02-graph-server-views-design.md))
+- `/ui/` — the graph explorer, a 3D view of the data graph, served by
+  graph-server (see [docs/graph-explorer.md](docs/graph-explorer.md))
+
+## Local Development Setup
+
+### Prerequisites
+
+- Python 3.11+
+- [uv](https://docs.astral.sh/uv/) package manager
+- Docker and Docker Compose
+
+### 1. Clone and install
+
+```bash
+git clone <repo-url>
+cd data-platform
+uv sync
+```
+
+### 2. Configure environment
+
+```bash
+cp .env.example .env
+# Edit .env if you want to change default credentials
+```
+
+### 3. Start the infrastructure
+
+```bash
+docker compose --profile s3 up -d
+```
+
+This starts the graph stack (Postgres, `graph-migrate`, `oxigraph`,
+`graph-server`, `materializer` — see
+[docs/architecture.md](docs/architecture.md#infrastructure)) plus:
+- **iceberg-catalog** on port 8181 (Iceberg REST catalog, S3-backed)
+- **object-store** (RustFS) on port 9000 (S3-compatible storage, console on 9001)
+
+For GCS testing, use `docker compose --profile gcs up -d` instead, which
+starts `fake-gcs-server` — note there's no GCS-backed iceberg-catalog wired
+up yet, so this profile provides the emulator but not a working GCS
+catalog.
+
+### Cloudflare R2 backend
+
+The `r2` profile hasn't moved to the graph stack yet — it still runs Nessie
+directly, against a real Cloudflare R2 bucket (no local object store):
+
+1. Create an R2 bucket in the Cloudflare dashboard.
+2. Create an R2 API token (R2 > Manage R2 API Tokens) with **Object Read & Write** scoped to that bucket. Copy the Access Key ID and Secret Access Key.
+3. Copy `.env.r2.example` values into your `.env` and fill in `R2_ACCOUNT_ID`, `R2_BUCKET`, `R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`.
+4. If your Postgres volume already exists from an earlier `s3`/`gcs` run, create the R2 version-store database once (the init script in `docker/init-databases.sql` only runs on a *fresh* `iceberg-db` volume):
+
+   ```bash
+   docker compose up -d postgres
+   docker compose exec postgres \
+     psql -U iceberg -d iceberg \
+     -c "CREATE DATABASE nessie_r2; GRANT ALL PRIVILEGES ON DATABASE nessie_r2 TO iceberg;"
+   ```
+
+   (A fresh stack — or `docker compose down -v && docker compose --profile r2 up -d` — creates it automatically. The reset destroys all local data.)
+5. Start the R2 profile:
+
+   ```bash
+   docker compose --profile r2 up -d
+   ```
+
+   Nessie listens on port 19122. Point your client at `http://localhost:19122`.
+
+R2-specific Nessie settings the profile sets for you:
+
+- `region=auto` (R2 ignores region but the SDK requires a value).
+- `endpoint=https://<R2_ACCOUNT_ID>.r2.cloudflarestorage.com` (account-style host).
+- `path-style-access=true` so requests are `<endpoint>/<bucket>/<key>` — R2 accepts both styles but path-style avoids virtual-host DNS issues with some older clients.
+
+The local PyIceberg/DuckDB client reads Iceberg data files straight from R2, so it needs the same bucket and credentials. It derives those from the same four `R2_*` vars automatically — warehouse (`s3://<R2_BUCKET>/`), endpoint, access keys, `region=auto`, and a default `CATALOG_URI` of `http://localhost:19122`. No separate `S3_*`/`ICEBERG_WAREHOUSE` settings are required; if you set them explicitly they take precedence.
+
+### 4. Verify the stack
+
+```bash
+uv run scripts/smoke_test.py
+```
+
+### 5. Set up your environment
+
+The `sunstone` CLI comes from sunstone-py. data-platform registers as a plugin and adds the `data` subcommand group:
+
+```bash
+# Add your local Docker stack as an environment
+sunstone env add local --catalog-url http://localhost:19120 --s3-endpoint http://localhost:9000
+sunstone env use local
+```
+
+### 6. Try it out
+
+```bash
+# Import a local CSV
+sunstone data import path/to/data.csv
+
+# Import from a URL
+sunstone data import https://example.com/dataset.csv
+
+# List datasets
+sunstone data info
+
+# View dataset metadata
+sunstone data info namespace.table_name
+
+# Query history
+sunstone data history namespace.table_name
+
+# Compare versions
+sunstone data diff namespace.table_name @v1 @v2
+```
+
+## Running Tests
+
+```bash
+# Unit tests (fast, no Docker required)
+uv run pytest tests/unit/
+
+# Integration tests (requires Docker stack running)
+uv run pytest tests/integration/
+
+# All tests
+uv run pytest
+
+# Specific test file with verbose output
+uv run pytest tests/unit/test_query.py -v
+```
+
+## CLI Commands
+
+| Command | Description |
+|---------|-------------|
+| `sunstone data import <source>` | Import CSV/Excel from path or URL |
+| `sunstone data info [table]` | View metadata for one or all datasets |
+| `sunstone data history <table>` | Show snapshot history (git-log style) |
+| `sunstone data tag <table> <tag>` | Tag a snapshot version |
+| `sunstone data untag <table> <tag>` | Remove a tag |
+| `sunstone data branches` | List branches and tags |
+| `sunstone data diff <table> <from> [to]` | Compare two versions |
+| `sunstone data lineage <table>` | Show provenance and lineage |
+| `sunstone data original <table>` | Download the original imported source file |
+
+All commands support `--branch` for working on version branches and `--json` for machine-readable output. Run `sunstone data <command> --help` for full options.
+
+## Key Concepts
+
+- **Table references** use git-like syntax: `namespace.table@branch`, `@v1.0`, `@2026-01-25`, `@snapshot_id`
+- **Deduplication**: Imports are checksum-based (SHA256). Re-importing the same file is a no-op.
+- **Schema evolution**: Safe type promotions (e.g. int32 to int64) are allowed automatically. Unsafe changes require `--force-schema`.
+- **Provenance**: Every import captures source metadata, schema info, and W3C PROV-DM provenance as Iceberg table properties.
+- **Original files**: Source files are stored content-addressable in S3 alongside the Iceberg data.
+
+## sunstone: URLs
+
+Datasets in the lakehouse can be referenced as URLs anywhere sunstone-py accepts a data source:
+
+```python
+import sunstone
+
+df = sunstone.read("sunstone:namespace/table@v1.0")
+```
+
+Supports the full ref syntax: `@branch`, `@tag`, `@2026-01-25`, `@snapshot_id`.
+
+## Project Structure
+
+```
+src/sunstone_data/
+├── plugin.py          # sunstone-py plugin (CLIProvider + URLHandler)
+├── url.py             # sunstone: URL parser
+├── config.py          # Settings from DataEnvironment or .env
+├── catalog.py         # PyIceberg RestCatalog connection to the Iceberg REST catalog
+├── query.py           # DuckDB QuerySession with time-travel support
+├── lineage.py         # Lineage in Iceberg table properties
+├── cli/               # Typer CLI commands + Rich formatters
+├── import_/           # Pipeline: fetch → parse → store → metadata
+├── metadata/          # Frozen dataclasses, MetadataBuilder, W3C PROV-DM
+├── versioning/        # Version branches/tags, refs, schema evolution, diff
+└── utils/             # Checksums, encoding detection, naming
+```
+
+## Documentation
+
+- [Getting Started](docs/getting-started.md) - Detailed setup instructions
+- [Architecture](docs/architecture.md) - System design and data flow
+- [CLI Reference](docs/cli-reference.md) - Complete command documentation
+- [Import Pipeline](docs/import-pipeline.md) - How imports work
+- [Development Guide](docs/development.md) - Contributing and testing
+
+## Resetting the Stack
+
+To wipe all data and start fresh:
+
+```bash
+docker compose --profile s3 --profile gcs down -v
+docker compose --profile s3 up -d
+```
+
+## License
+
+MIT
+
+## Graph server authentication
+
+`graph-server` requires a Keycloak bearer token on every route except
+`/healthz`/`/readyz`. Access is role-based: the client roles in your token
+determine which environment you can use. See
+`docs/specs/2026-07-05-graph-server-auth-design.md`.
+
+No hostnames or client ids ship in this package. Each environment's
+`graph_url` comes from config: Sunstone-managed machines get
+`/etc/sunstone/data_platform.toml` from the `fleet-config` repo, and other
+machines set `GRAPH_URL` or add an environment with `sunstone env add`.
+graph-server publishes the OIDC issuer and public client id at
+`$GRAPH_URL/ui/config.json`; the CLI reads them from there.
+
+### Humans (CLI, local agents)
+
+```bash
+sunstone env init          # cache discovery for configured environments
+sunstone data login        # browser sign-in (authorization code + PKCE)
+```
+
+A browser window opens for Keycloak SSO; the refresh token renews silently
+until the SSO session expires.
+
+### Services (CI, automation)
+
+Set `SUNSTONE_CLIENT_ID` and `SUNSTONE_CLIENT_SECRET` (or `client_id` and
+`client_secret` in the environment's `data-platform` section). The client then
+uses the OAuth2 client-credentials flow against the discovered issuer instead
+of a cached human login.

sunstone_data_platform-1.0.0/pyproject.toml

@@ -0,0 +1,89 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "sunstone-data-platform"
+dynamic = ["version"]
+description = "Sunstone Data Platform - Data lakehouse with semantic metadata"
+readme = "README.md"
+requires-python = ">=3.12"
+license = "MIT"
+authors = [
+    { name = "Sunstone Institute", email = "stig@sunstone.institute" }
+]
+dependencies = [
+    "sunstone-py>=1.15.0",
+    "pyiceberg[s3fs,gcsfs,duckdb]>=0.10.0",
+    "duckdb>=1.5.3",
+    "python-dotenv>=1.2.2",
+    "typer>=0.27.1",
+    "rich>=14.0.0",
+    "charset-normalizer>=3.4.0",
+    "httpx>=0.27.0",
+    "fastexcel>=0.20.2",
+    "python-slugify>=8.0.0",
+    "python-dateutil>=2.9.0.post0",
+]
+
+[project.entry-points."sunstone.plugins"]
+data_platform = "sunstone_data.plugin:SunstoneDataPlugin"
+
+[project.optional-dependencies]
+pipelines = [
+    "dagster>=1.13.9",
+    "dagster-webserver>=1.13.0",
+    "dagster-docker>=0.29.0",
+    "dagster-postgres>=0.29.0",
+]
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.build.hooks.vcs]
+version-file = "src/sunstone_data/_version.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/sunstone_data", "src/sunstone_pipeline"]
+
+# The sdist is published to PyPI. Limit it to what the wheel is built from,
+# so deploy manifests, workflows and runbooks stay out of it.
+[tool.hatch.build.targets.sdist]
+include = ["/src/sunstone_data", "/src/sunstone_pipeline", "/README.md"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+    "integration: marks tests as integration tests (require Docker stack)",
+]
+filterwarnings = [
+    "ignore::pyparsing.exceptions.PyparsingDeprecationWarning",
+    "ignore::pydantic.warnings.PydanticDeprecatedSince20",
+    "ignore::pydantic.warnings.PydanticDeprecatedSince212",
+    # gcsfs imports moved google.cloud.storage._experimental modules — fixed upstream,
+    # check if this filter can be removed after upgrading gcsfs past 2025.3.2
+    "ignore:google\\.cloud\\.storage\\._experimental:DeprecationWarning:gcsfs",
+]
+
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.lint]
+select = ["E", "F", "I"]
+ignore = ["E501"]
+
+[dependency-groups]
+dev = [
+    "ruff>=0.15.17",
+    "frictionless>=5.19.0",
+    "gitpython>=3.1.58",
+    "openpyxl>=3.1.5",
+    "pandas>=3.0.5",
+    "pre-commit>=4.6.0",
+    "pytest>=9.0.3",
+    "pytest-asyncio>=0.24.0",
+    "pytest-cov>=4.0.0",
+    "pyyaml>=6.0.3",
+    "ruamel-yaml>=0.19.1",
+    "tomlkit>=0.15.1",
+]