steve-cli 0.4.1__tar.gz → 0.5.2__tar.gz

steve_cli-0.5.2/PKG-INFO

@@ -0,0 +1,283 @@
+Metadata-Version: 2.4
+Name: steve-cli
+Version: 0.5.2
+Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
+Author: Frank
+License: MIT
+Project-URL: Homepage, https://github.com/7frank/ds-steve-cli
+Project-URL: Repository, https://github.com/7frank/ds-steve-cli
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: boto3>=1.37.38
+Requires-Dist: click>=8.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: questionary>=2.0.0
+Requires-Dist: rdflib>=6.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: black>=22.0; extra == "dev"
+Requires-Dist: isort>=5.0; extra == "dev"
+Requires-Dist: polars>=1.8.2; extra == "dev"
+Requires-Dist: pyarrow>=17.0.0; extra == "dev"
+Requires-Dist: validoopsie>=0.1.0; extra == "dev"
+Requires-Dist: openlineage-python>=1.0.0; extra == "dev"
+Provides-Extra: polars
+Requires-Dist: polars>=1.8.2; extra == "polars"
+Requires-Dist: pyarrow>=17.0.0; extra == "polars"
+Provides-Extra: pandas
+Requires-Dist: pandas>=2.0.3; extra == "pandas"
+Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
+Provides-Extra: lineage
+Requires-Dist: openlineage-python>=1.0.0; extra == "lineage"
+Provides-Extra: validoopsie
+Requires-Dist: validoopsie>=0.1.0; extra == "validoopsie"
+Requires-Dist: polars>=1.8.2; extra == "validoopsie"
+Provides-Extra: great-expectations
+Requires-Dist: great-expectations>=0.18.0; extra == "great-expectations"
+Requires-Dist: pandas>=2.0.3; extra == "great-expectations"
+Provides-Extra: excel
+Requires-Dist: openpyxl>=3.1.0; extra == "excel"
+Provides-Extra: trino
+Requires-Dist: trino>=0.330; extra == "trino"
+Requires-Dist: pyarrow>=17.0.0; extra == "trino"
+Requires-Dist: requests>=2.31.0; extra == "trino"
+Requires-Dist: pyiceberg>=0.7.0; extra == "trino"
+Requires-Dist: s3fs>=2024.1.0; extra == "trino"
+Provides-Extra: visidata
+Requires-Dist: visidata>=3.0; extra == "visidata"
+Provides-Extra: all
+Requires-Dist: openlineage-python>=1.0.0; extra == "all"
+Requires-Dist: pyarrow>=17.0.0; extra == "all"
+Requires-Dist: polars>=1.8.2; extra == "all"
+Requires-Dist: validoopsie>=0.1.0; extra == "all"
+Requires-Dist: great-expectations>=0.18.0; extra == "all"
+Requires-Dist: pandas>=2.0.3; extra == "all"
+Requires-Dist: openpyxl>=3.1.0; extra == "all"
+Requires-Dist: visidata>=3.0; extra == "all"
+
+# Steve CLI
+
+A CLI tool for the data platform. Commands are organized by concern: general workspace automation, data lakehouse operations, semantic knowledge graphs, and governance.
+
+## Installation
+
+```bash
+# Add to your workspace
+uv add steve-cli
+
+# Or pin to a local editable copy (in your workspace pyproject.toml)
+[tool.uv.sources]
+steve-cli = { path = "../../packages/steve-cli", editable = true }
+```
+
+Upgrade to the latest version:
+
+```bash
+steve upgrade
+```
+
+---
+
+## Platform
+
+General-purpose commands for running jobs, managing local development processes, and working with the core infrastructure (object storage, secrets).
+
+### `steve jobs` — run automation jobs
+
+Runs tasks defined in `jobs.yaml`. Each job declares a command and optional env vars. Designed for data pipeline steps, seeding, bootstrapping, etc.
+
+```bash
+steve jobs              # interactive picker
+steve jobs ls           # list all jobs with commands and env vars
+steve jobs run <name>   # run a job by name
+steve jobs run <name> -f path/to/jobs.yaml
+```
+
+`jobs.yaml` example:
+
+```yaml
+jobs:
+  - name: seed-data
+    command: ["uv", "run", "src/02_seed_data.py"]
+    env:
+      BATCH_SIZE: "500"
+```
+
+### `steve apps` — manage long-running apps
+
+Starts, stops, and monitors local apps defined in `apps.yaml`. Apps are registered with the auth-proxy so they get a public URL inside the platform.
+
+```bash
+steve apps              # interactive picker
+steve apps ls           # list apps with status and URLs
+steve apps status       # show running/stopped status for all apps
+steve apps start <name>
+steve apps stop <name>
+steve apps restart <name>
+steve apps logs <name> [-n 100]
+```
+
+### `steve setup env` — decrypt secrets
+
+Decrypts SOPS-encrypted `*.enc.env` files in the current directory and writes plaintext `.env` files.
+
+```bash
+steve setup env
+```
+
+### `steve buckets` — browse S3 storage
+
+Interactive browser for S3/MinIO buckets. Detects available buckets from env vars (`BRONZE_ACCESS_KEY`, `SILVER_ACCESS_KEY`, `GOLD_ACCESS_KEY`, or `{WORKSPACE}_ACCESS_KEY`) and shows their contents as a file tree. Files can be opened in visidata.
+
+```bash
+steve buckets
+```
+
+---
+
+## Data Lakehouse
+
+Commands for working with the lakehouse layer: Iceberg tables accessed via Trino.
+
+### `steve tables` — browse Iceberg tables via Trino
+
+Interactive browser for Iceberg tables. Lists schemas and tables available via Trino, and opens them in visidata for inspection.
+
+```bash
+steve tables
+```
+
+Requires `TRINO_ENDPOINT` to be set. Install visidata for table preview:
+
+```bash
+uv pip install 'steve-cli[visidata]'
+```
+
+---
+
+## Semantics & Knowledge Graphs
+
+Commands for authoring and operating Virtual Knowledge Graphs (VKGs) — semantic layers that expose relational Iceberg data as RDF/SPARQL via Ontop.
+
+There are two sides:
+- **`ontology`** — the *author/publish* side: define concepts, register them in the Data Product Registry, compile VKG artifacts
+- **`vkg`** — the *runtime/ops* side: manage named Ontop instances on the VKG control plane, run SPARQL queries
+
+### `steve ontology` — author and publish ontologies
+
+```bash
+steve ontology ls                          # list registered packages in the registry
+steve ontology ls --show-bindings          # include binding packages
+steve ontology register <file.yaml>        # register a concept or binding YAML
+steve ontology add <uri>                   # add a dependency URI to ontology.yaml
+steve ontology build                       # resolve deps + compile → write vkg/ folder locally
+steve ontology compile --root <uri> --binding <uri>   # compile and print artifact info
+steve ontology push                        # compile + upload to VKG control plane (default slot)
+steve ontology push --name <slot>          # push to a named VKG slot
+steve ontology push -f ontology-consumer.yaml  # use a different manifest file
+steve ontology push --frozen               # fail if lock file is absent or stale (like npm ci)
+```
+
+Manifest format (`ontology.yaml`):
+
+```yaml
+# Option A: workspace that owns ontology files
+packages:
+  - path: ontologies/order_core.yaml
+bindings:
+  - path: ontologies/binding_order_iceberg.yaml
+compile:
+  root: dp://o/acme/concepts/order/core
+  binding: dp://o/acme/bindings/order/iceberg
+
+# Option B: consumer workspace (resolves deps from registry, no local files)
+dependencies:
+  - dp://o/acme/concepts/party/core
+  - dp://o/acme/concepts/order/core
+compile:
+  root: dp://o/acme/concepts/order/core
+  binding: dp://o/acme/bindings/order/iceberg
+```
+
+After `push`, a lock file (`ontology.lock.yaml`) is written with resolved package versions and Iceberg snapshot IDs for audit and future time-travel SQL rewriting.
+
+### `steve vkg` — operate VKG instances
+
+```bash
+steve vkg ls                              # list all slots: status, concept counts, SPARQL URL
+steve vkg status <name>                   # detailed status for one slot
+steve vkg start <name>                    # start a stopped slot
+steve vkg stop <name>                     # stop a running slot (persists desired=stopped)
+steve vkg sparql <name> "<SPARQL query>"  # run a SPARQL query and print results as a table
+steve vkg sparql <name> "<query>" --format json
+steve vkg logs <name> [-n 50]            # tail the Ontop process log for a slot
+```
+
+Desired state is persisted per slot — slots with `desired=running` are automatically restored after a pod restart.
+
+---
+
+## Governance
+
+### `steve policies apply` — apply access policies
+
+Applies role-based access policies from a YAML file to the Policy Control Plane.
+
+```bash
+steve policies apply                      # apply from ./policies/access.yaml
+steve policies apply -f custom/path.yaml
+```
+
+---
+
+## Environment variables
+
+| Variable | Used by | Description |
+|---|---|---|
+| `REGISTRY_URL` | `ontology` | Data Product Registry URL (default: `http://localhost:8765`) |
+| `REGISTRY_TOKEN` | `ontology` | Pre-issued JWT for the registry |
+| `REGISTRY_JWT_SECRET` | `ontology` | Secret to mint a JWT (if no token) |
+| `REGISTRY_ORG` | `ontology` | Org claim for minted JWT |
+| `VKG_CONTROL_PLANE_URL` | `ontology push`, `vkg` | Control plane URL (default: `http://localhost:18081`) |
+| `TRINO_ENDPOINT` | `ontology push`, `tables` | Trino HTTP URL for snapshot capture and table browsing |
+
+---
+
+## File Metadata Extraction
+
+Steve automatically extracts metadata from files read or written via `MetadataRegistry`. The extractor is chosen by file extension.
+
+| Extension | Extractor | Requires |
+|---|---|---|
+| `.parquet`, `.pq` | `ParquetExtractor` | `pip install steve-cli[polars]` |
+| `.csv`, `.tsv`, `.txt` | `CsvExtractor` | stdlib only |
+| `.json`, `.jsonl`, `.ndjson` | `JsonExtractor` | stdlib only |
+| `.xlsx`, `.xls`, `.xlsm` | `ExcelExtractor` | `pip install steve-cli[excel]` |
+| anything else | `GenericExtractor` | stdlib only |
+
+Custom extractors can be registered at startup:
+
+```python
+from steve_cli.storage.metadata.port import MetadataExtractorPort, FileMetadata, ColumnMetadata
+from steve_cli.storage.metadata.registry import MetadataRegistry
+
+class AvroExtractor(MetadataExtractorPort):
+    extensions = (".avro",)
+
+    def extract(self, data: bytes, path: str) -> FileMetadata:
+        import fastavro, io
+        reader = fastavro.reader(io.BytesIO(data))
+        schema = reader.writer_schema
+        columns = [ColumnMetadata(name=f["name"], type=str(f["type"])) for f in schema.get("fields", [])]
+        return FileMetadata(format="avro", size_bytes=len(data), rows=len(list(reader)), columns=columns)
+
+MetadataRegistry.register("avro", AvroExtractor)
+```

steve_cli-0.5.2/README.md

@@ -0,0 +1,218 @@
+# Steve CLI
+
+A CLI tool for the data platform. Commands are organized by concern: general workspace automation, data lakehouse operations, semantic knowledge graphs, and governance.
+
+## Installation
+
+```bash
+# Add to your workspace
+uv add steve-cli
+
+# Or pin to a local editable copy (in your workspace pyproject.toml)
+[tool.uv.sources]
+steve-cli = { path = "../../packages/steve-cli", editable = true }
+```
+
+Upgrade to the latest version:
+
+```bash
+steve upgrade
+```
+
+---
+
+## Platform
+
+General-purpose commands for running jobs, managing local development processes, and working with the core infrastructure (object storage, secrets).
+
+### `steve jobs` — run automation jobs
+
+Runs tasks defined in `jobs.yaml`. Each job declares a command and optional env vars. Designed for data pipeline steps, seeding, bootstrapping, etc.
+
+```bash
+steve jobs              # interactive picker
+steve jobs ls           # list all jobs with commands and env vars
+steve jobs run <name>   # run a job by name
+steve jobs run <name> -f path/to/jobs.yaml
+```
+
+`jobs.yaml` example:
+
+```yaml
+jobs:
+  - name: seed-data
+    command: ["uv", "run", "src/02_seed_data.py"]
+    env:
+      BATCH_SIZE: "500"
+```
+
+### `steve apps` — manage long-running apps
+
+Starts, stops, and monitors local apps defined in `apps.yaml`. Apps are registered with the auth-proxy so they get a public URL inside the platform.
+
+```bash
+steve apps              # interactive picker
+steve apps ls           # list apps with status and URLs
+steve apps status       # show running/stopped status for all apps
+steve apps start <name>
+steve apps stop <name>
+steve apps restart <name>
+steve apps logs <name> [-n 100]
+```
+
+### `steve setup env` — decrypt secrets
+
+Decrypts SOPS-encrypted `*.enc.env` files in the current directory and writes plaintext `.env` files.
+
+```bash
+steve setup env
+```
+
+### `steve buckets` — browse S3 storage
+
+Interactive browser for S3/MinIO buckets. Detects available buckets from env vars (`BRONZE_ACCESS_KEY`, `SILVER_ACCESS_KEY`, `GOLD_ACCESS_KEY`, or `{WORKSPACE}_ACCESS_KEY`) and shows their contents as a file tree. Files can be opened in visidata.
+
+```bash
+steve buckets
+```
+
+---
+
+## Data Lakehouse
+
+Commands for working with the lakehouse layer: Iceberg tables accessed via Trino.
+
+### `steve tables` — browse Iceberg tables via Trino
+
+Interactive browser for Iceberg tables. Lists schemas and tables available via Trino, and opens them in visidata for inspection.
+
+```bash
+steve tables
+```
+
+Requires `TRINO_ENDPOINT` to be set. Install visidata for table preview:
+
+```bash
+uv pip install 'steve-cli[visidata]'
+```
+
+---
+
+## Semantics & Knowledge Graphs
+
+Commands for authoring and operating Virtual Knowledge Graphs (VKGs) — semantic layers that expose relational Iceberg data as RDF/SPARQL via Ontop.
+
+There are two sides:
+- **`ontology`** — the *author/publish* side: define concepts, register them in the Data Product Registry, compile VKG artifacts
+- **`vkg`** — the *runtime/ops* side: manage named Ontop instances on the VKG control plane, run SPARQL queries
+
+### `steve ontology` — author and publish ontologies
+
+```bash
+steve ontology ls                          # list registered packages in the registry
+steve ontology ls --show-bindings          # include binding packages
+steve ontology register <file.yaml>        # register a concept or binding YAML
+steve ontology add <uri>                   # add a dependency URI to ontology.yaml
+steve ontology build                       # resolve deps + compile → write vkg/ folder locally
+steve ontology compile --root <uri> --binding <uri>   # compile and print artifact info
+steve ontology push                        # compile + upload to VKG control plane (default slot)
+steve ontology push --name <slot>          # push to a named VKG slot
+steve ontology push -f ontology-consumer.yaml  # use a different manifest file
+steve ontology push --frozen               # fail if lock file is absent or stale (like npm ci)
+```
+
+Manifest format (`ontology.yaml`):
+
+```yaml
+# Option A: workspace that owns ontology files
+packages:
+  - path: ontologies/order_core.yaml
+bindings:
+  - path: ontologies/binding_order_iceberg.yaml
+compile:
+  root: dp://o/acme/concepts/order/core
+  binding: dp://o/acme/bindings/order/iceberg
+
+# Option B: consumer workspace (resolves deps from registry, no local files)
+dependencies:
+  - dp://o/acme/concepts/party/core
+  - dp://o/acme/concepts/order/core
+compile:
+  root: dp://o/acme/concepts/order/core
+  binding: dp://o/acme/bindings/order/iceberg
+```
+
+After `push`, a lock file (`ontology.lock.yaml`) is written with resolved package versions and Iceberg snapshot IDs for audit and future time-travel SQL rewriting.
+
+### `steve vkg` — operate VKG instances
+
+```bash
+steve vkg ls                              # list all slots: status, concept counts, SPARQL URL
+steve vkg status <name>                   # detailed status for one slot
+steve vkg start <name>                    # start a stopped slot
+steve vkg stop <name>                     # stop a running slot (persists desired=stopped)
+steve vkg sparql <name> "<SPARQL query>"  # run a SPARQL query and print results as a table
+steve vkg sparql <name> "<query>" --format json
+steve vkg logs <name> [-n 50]            # tail the Ontop process log for a slot
+```
+
+Desired state is persisted per slot — slots with `desired=running` are automatically restored after a pod restart.
+
+---
+
+## Governance
+
+### `steve policies apply` — apply access policies
+
+Applies role-based access policies from a YAML file to the Policy Control Plane.
+
+```bash
+steve policies apply                      # apply from ./policies/access.yaml
+steve policies apply -f custom/path.yaml
+```
+
+---
+
+## Environment variables
+
+| Variable | Used by | Description |
+|---|---|---|
+| `REGISTRY_URL` | `ontology` | Data Product Registry URL (default: `http://localhost:8765`) |
+| `REGISTRY_TOKEN` | `ontology` | Pre-issued JWT for the registry |
+| `REGISTRY_JWT_SECRET` | `ontology` | Secret to mint a JWT (if no token) |
+| `REGISTRY_ORG` | `ontology` | Org claim for minted JWT |
+| `VKG_CONTROL_PLANE_URL` | `ontology push`, `vkg` | Control plane URL (default: `http://localhost:18081`) |
+| `TRINO_ENDPOINT` | `ontology push`, `tables` | Trino HTTP URL for snapshot capture and table browsing |
+
+---
+
+## File Metadata Extraction
+
+Steve automatically extracts metadata from files read or written via `MetadataRegistry`. The extractor is chosen by file extension.
+
+| Extension | Extractor | Requires |
+|---|---|---|
+| `.parquet`, `.pq` | `ParquetExtractor` | `pip install steve-cli[polars]` |
+| `.csv`, `.tsv`, `.txt` | `CsvExtractor` | stdlib only |
+| `.json`, `.jsonl`, `.ndjson` | `JsonExtractor` | stdlib only |
+| `.xlsx`, `.xls`, `.xlsm` | `ExcelExtractor` | `pip install steve-cli[excel]` |
+| anything else | `GenericExtractor` | stdlib only |
+
+Custom extractors can be registered at startup:
+
+```python
+from steve_cli.storage.metadata.port import MetadataExtractorPort, FileMetadata, ColumnMetadata
+from steve_cli.storage.metadata.registry import MetadataRegistry
+
+class AvroExtractor(MetadataExtractorPort):
+    extensions = (".avro",)
+
+    def extract(self, data: bytes, path: str) -> FileMetadata:
+        import fastavro, io
+        reader = fastavro.reader(io.BytesIO(data))
+        schema = reader.writer_schema
+        columns = [ColumnMetadata(name=f["name"], type=str(f["type"])) for f in schema.get("fields", [])]
+        return FileMetadata(format="avro", size_bytes=len(data), rows=len(list(reader)), columns=columns)
+
+MetadataRegistry.register("avro", AvroExtractor)
+```

{steve_cli-0.4.1 → steve_cli-0.5.2}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "steve-cli"
-version = "0.4.1"
+version = "0.5.2"
 description = "A simple CLI tool to run jobs from jobs.yaml with proper environment setup"
 readme = "README.md"
 license = {text = "MIT"}
@@ -26,6 +26,7 @@ dependencies = [
     "python-dotenv>=1.0.0",
     "pyyaml>=6.0",
     "questionary>=2.0.0",
+    "rdflib>=6.0",
 ]
 
 [project.optional-dependencies]

{steve_cli-0.4.1 → steve_cli-0.5.2}/steve_cli/cli.py

@@ -913,5 +913,11 @@ def upgrade():
     sys.exit(result.returncode)
 
 
+from steve_cli.ontology import ontology
+from steve_cli.vkg import vkg
+main.add_command(ontology)
+main.add_command(vkg)
+
+
 if __name__ == '__main__':
     main()

steve_cli-0.5.2/steve_cli/lineage/adapters/composite.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from steve_cli.lineage.port import LineageEvent, LineagePort
+
+
+class CompositeLineageAdapter(LineagePort):
+    def __init__(self, adapters: list[LineagePort]):
+        self._adapters = adapters
+
+    def emit(self, event: LineageEvent) -> None:
+        for adapter in self._adapters:
+            adapter.emit(event)

steve_cli-0.5.2/steve_cli/lineage/adapters/dataproductregistry.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import logging
+import os
+
+import requests
+
+from steve_cli.lineage.port import DatasetRef, LineageEvent, LineagePort
+
+logger = logging.getLogger(__name__)
+
+
+class RegistryLineageAdapter(LineagePort):
+    def __init__(self):
+        self._url = os.getenv("REGISTRY_URL", "http://localhost:8765")
+        self._token = os.getenv("REGISTRY_TOKEN", "")
+
+    def emit(self, event: LineageEvent) -> None:
+        if event.state != "COMPLETE":
+            return
+        for ds in event.outputs:
+            self._register(ds)
+
+    def _register(self, ds: DatasetRef) -> None:
+        product = {
+            "uri": f"dp://d/{ds.namespace}/{ds.name}",
+            "name": ds.name,
+            "namespace": ds.namespace,
+            "description": "Auto-registered from lineage job",
+        }
+        headers = {}
+        if self._token:
+            headers["Authorization"] = f"Bearer {self._token}"
+        try:
+            requests.post(
+                f"{self._url}/api/v1/register",
+                json={"product": product},
+                headers=headers,
+                timeout=10,
+            )
+        except Exception as exc:
+            logger.warning("Failed to register dataset %s/%s in registry: %s", ds.namespace, ds.name, exc)

{steve_cli-0.4.1 → steve_cli-0.5.2}/steve_cli/lineage/registry.py

@@ -25,6 +25,11 @@ class LineageRegistry:
     @classmethod
     def create(cls, name: str | None = None, **kwargs: Any) -> LineagePort:
         provider = name or os.getenv("LINEAGE_PROVIDER", "openlineage")
+        if "+" in provider:
+            from .adapters.composite import CompositeLineageAdapter
+            parts = [p.strip() for p in provider.split("+")]
+            adapters = [cls.get(p)(**kwargs) for p in parts]
+            return CompositeLineageAdapter(adapters)
         return cls.get(provider)(**kwargs)
 
     @classmethod
@@ -46,5 +51,9 @@ def _register_defaults() -> None:
     except ImportError:
         LineageRegistry.register("openlineage", NullLineageAdapter)
 
+    from .adapters.dataproductregistry import RegistryLineageAdapter
+
+    LineageRegistry.register("dataproductregistry", RegistryLineageAdapter)
+
 
 _register_defaults()