steve-cli 0.3.20__tar.gz → 0.4.1__tar.gz

{steve_cli-0.3.20 → steve_cli-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: steve-cli
-Version: 0.3.20
+Version: 0.4.1
 Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
 Author: Frank
 License: MIT
@@ -44,6 +44,12 @@ Requires-Dist: great-expectations>=0.18.0; extra == "great-expectations"
 Requires-Dist: pandas>=2.0.3; extra == "great-expectations"
 Provides-Extra: excel
 Requires-Dist: openpyxl>=3.1.0; extra == "excel"
+Provides-Extra: trino
+Requires-Dist: trino>=0.330; extra == "trino"
+Requires-Dist: pyarrow>=17.0.0; extra == "trino"
+Requires-Dist: requests>=2.31.0; extra == "trino"
+Requires-Dist: pyiceberg>=0.7.0; extra == "trino"
+Requires-Dist: s3fs>=2024.1.0; extra == "trino"
 Provides-Extra: visidata
 Requires-Dist: visidata>=3.0; extra == "visidata"
 Provides-Extra: all
@@ -72,7 +78,7 @@ uv pip install -e .
 
 ### Install from GitHub
 
-`uv add steve-cli` 
+`uv add steve-cli`
 
 OR
 
@@ -80,7 +86,14 @@ OR
 uv add git+https://github.com/your-org/tilt-ts-4.git#subdirectory=apps/example-hehnke/packages/steve-cli
 ```
 
+### dev
 
+For local development add the following to your pyproj.toml which allows using the local copy instead of the one from the registry
+
+```toml
+[tool.uv.sources]
+steve-cli = { path = "../../packages/steve-cli", editable = true }
+```
 
 ## Usage
 
@@ -162,13 +175,13 @@ Running `steve extract-data` will:
 
 Steve automatically extracts metadata from files read or written via `MetadataRegistry`. The extractor is chosen by file extension — no configuration needed.
 
-| Extension | Extractor | Requires |
-|---|---|---|
-| `.parquet`, `.pq` | `ParquetExtractor` | `pip install steve-cli[polars]` |
-| `.csv`, `.tsv`, `.txt` | `CsvExtractor` | stdlib only |
-| `.json`, `.jsonl`, `.ndjson` | `JsonExtractor` | stdlib only |
-| `.xlsx`, `.xls`, `.xlsm` | `ExcelExtractor` | `pip install steve-cli[excel]` |
-| anything else | `GenericExtractor` | stdlib only |
+| Extension                    | Extractor          | Requires                        |
+| ---------------------------- | ------------------ | ------------------------------- |
+| `.parquet`, `.pq`            | `ParquetExtractor` | `pip install steve-cli[polars]` |
+| `.csv`, `.tsv`, `.txt`       | `CsvExtractor`     | stdlib only                     |
+| `.json`, `.jsonl`, `.ndjson` | `JsonExtractor`    | stdlib only                     |
+| `.xlsx`, `.xls`, `.xlsm`     | `ExcelExtractor`   | `pip install steve-cli[excel]`  |
+| anything else                | `GenericExtractor` | stdlib only                     |
 
 ### Adding a custom extractor
 
@@ -230,10 +243,9 @@ black steve_cli/
 isort steve_cli/
 ```
 
-
-
 # use cli
-source .venv/bin/activate 
+
+source .venv/bin/activate
 steve
 
 #

{steve_cli-0.3.20 → steve_cli-0.4.1}/README.md

@@ -14,7 +14,7 @@ uv pip install -e .
 
 ### Install from GitHub
 
-`uv add steve-cli` 
+`uv add steve-cli`
 
 OR
 
@@ -22,7 +22,14 @@ OR
 uv add git+https://github.com/your-org/tilt-ts-4.git#subdirectory=apps/example-hehnke/packages/steve-cli
 ```
 
+### dev
 
+For local development add the following to your pyproj.toml which allows using the local copy instead of the one from the registry
+
+```toml
+[tool.uv.sources]
+steve-cli = { path = "../../packages/steve-cli", editable = true }
+```
 
 ## Usage
 
@@ -104,13 +111,13 @@ Running `steve extract-data` will:
 
 Steve automatically extracts metadata from files read or written via `MetadataRegistry`. The extractor is chosen by file extension — no configuration needed.
 
-| Extension | Extractor | Requires |
-|---|---|---|
-| `.parquet`, `.pq` | `ParquetExtractor` | `pip install steve-cli[polars]` |
-| `.csv`, `.tsv`, `.txt` | `CsvExtractor` | stdlib only |
-| `.json`, `.jsonl`, `.ndjson` | `JsonExtractor` | stdlib only |
-| `.xlsx`, `.xls`, `.xlsm` | `ExcelExtractor` | `pip install steve-cli[excel]` |
-| anything else | `GenericExtractor` | stdlib only |
+| Extension                    | Extractor          | Requires                        |
+| ---------------------------- | ------------------ | ------------------------------- |
+| `.parquet`, `.pq`            | `ParquetExtractor` | `pip install steve-cli[polars]` |
+| `.csv`, `.tsv`, `.txt`       | `CsvExtractor`     | stdlib only                     |
+| `.json`, `.jsonl`, `.ndjson` | `JsonExtractor`    | stdlib only                     |
+| `.xlsx`, `.xls`, `.xlsm`     | `ExcelExtractor`   | `pip install steve-cli[excel]`  |
+| anything else                | `GenericExtractor` | stdlib only                     |
 
 ### Adding a custom extractor
 
@@ -172,10 +179,9 @@ black steve_cli/
 isort steve_cli/
 ```
 
-
-
 # use cli
-source .venv/bin/activate 
+
+source .venv/bin/activate
 steve
 
 #

{steve_cli-0.3.20 → steve_cli-0.4.1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "steve-cli"
-version = "0.3.20"
+version = "0.4.1"
 description = "A simple CLI tool to run jobs from jobs.yaml with proper environment setup"
 readme = "README.md"
 license = {text = "MIT"}
@@ -60,6 +60,13 @@ great-expectations = [
 excel = [
     "openpyxl>=3.1.0",
 ]
+trino = [
+    "trino>=0.330",
+    "pyarrow>=17.0.0",
+    "requests>=2.31.0",
+    "pyiceberg>=0.7.0",
+    "s3fs>=2024.1.0",
+]
 visidata = [
     "visidata>=3.0",
 ]

{steve_cli-0.3.20 → steve_cli-0.4.1}/steve_cli/cli.py

@@ -781,6 +781,126 @@ def buckets(env_file: tuple):
         click.secho(f"❌ Could not open file: {e}", fg="red", err=True)
 
 
+def _list_trino_tables(storage_kwargs: dict, label: str) -> tuple:
+    click.echo(f"  {click.style(label, fg='cyan')}")
+    try:
+        from steve_cli.storage.trino import TrinoStorage
+        storage = TrinoStorage(**storage_kwargs)
+        tables = storage.list()
+        if not tables:
+            click.echo("    (no tables)")
+        else:
+            for t in tables:
+                click.echo(f"    ├── {t}")
+        return storage, tables
+    except EnvironmentError as e:
+        click.echo(f"    ⚠️  {e}", err=True)
+    except Exception as e:
+        click.echo(f"    ❌ {e}", err=True)
+    return None, []
+
+
+@main.command("tables")
+@click.option('--env-file', '-e', type=click.Path(path_type=Path), multiple=True,
+              help='Path to .env file(s). Can be specified multiple times. Defaults to .env and .workspaces.env')
+def tables(env_file: tuple):
+    """List Iceberg tables via Trino and view their contents."""
+    if not os.getenv("TRINO_ENDPOINT"):
+        click.secho("TRINO_ENDPOINT is not set — Trino is not available.", fg="yellow")
+        return
+
+    cwd = Path.cwd()
+    env_files = [Path(f) for f in env_file] if env_file else [cwd / ".env", cwd / ".workspaces.env"]
+    for ef in env_files:
+        load_dotenv(ef)
+
+    tiers = ["bronze", "silver", "gold"]
+    options: List[Dict[str, Any]] = []
+
+    bare_tiers = [t for t in tiers if os.getenv(f"{t.upper()}_ACCESS_KEY")]
+    for tier in bare_tiers:
+        options.append({
+            "label": f"default / {tier}",
+            "kwargs": {"tier": tier},
+        })
+
+    for ws in _detect_workspaces():
+        for tier in tiers:
+            if not os.getenv(f"{ws}_ACCESS_KEY_{tier.upper()}") and not os.getenv(f"{ws}_ACCESS_KEY"):
+                continue
+            options.append({
+                "label": f"{ws} / {tier}",
+                "kwargs": {"tier": tier, "workspace": ws.lower().replace("_", "-")},
+            })
+
+    if not options:
+        click.echo("No Trino storage env variables found (expected: TRINO_ENDPOINT + BRONZE_ACCESS_KEY or {WORKSPACE}_ACCESS_KEY).")
+        return
+
+    choice = questionary.select(
+        "Select a schema to list:",
+        choices=[o["label"] for o in options],
+    ).ask()
+
+    if choice is None:
+        sys.exit(0)
+
+    selected = next(o for o in options if o["label"] == choice)
+    storage, table_names = _list_trino_tables(selected["kwargs"], selected["label"])
+
+    if not storage or not table_names:
+        return
+
+    table_choice = questionary.select(
+        "View a table (or press Esc to exit):",
+        choices=["(done)"] + table_names,
+    ).ask()
+
+    if not table_choice or table_choice == "(done)":
+        return
+
+    click.echo(f"\n📊 {click.style(table_choice, fg='cyan')}\n")
+    try:
+        import tempfile
+        data = storage.get_bytes(table_choice)
+        with tempfile.NamedTemporaryFile(suffix=".parquet", delete=False) as tmp:
+            tmp.write(data)
+            tmp_path = tmp.name
+        if not shutil.which("vd"):
+            click.secho("visidata not found. Install it with: uv pip install 'steve-cli[visidata]'", fg="yellow")
+            return
+        subprocess.call(["vd", tmp_path])
+    except Exception as e:
+        click.secho(f"❌ Could not open table: {e}", fg="red", err=True)
+
+
+@main.group()
+def policies():
+    """Manage and apply access policies via the Policy Control Plane."""
+    pass
+
+
+@policies.command("apply")
+@click.option('--file', '-f', type=click.Path(path_type=Path),
+              help='Path to policies YAML file (default: ./policies/access.yaml)')
+def policies_apply(file: Path | None):
+    """Apply access policies from a YAML file to the Policy Control Plane."""
+    from dotenv import load_dotenv
+    load_dotenv(".env", override=False)
+
+    from steve_cli.policies import PolicyClient
+    policies_file = Path(file) if file else None
+    try:
+        PolicyClient().apply_from_file(policies_file)
+        click.secho("Policies applied successfully.", fg="green")
+    except FileNotFoundError as e:
+        click.secho(str(e), fg="red", err=True)
+        sys.exit(1)
+    except Exception as e:
+        click.secho(f"ERROR: {e}", fg="red", bold=True, err=True)
+        sys.exit(1)
+
+
 @main.command("upgrade")
 def upgrade():
     """Upgrade steve-cli to the latest version."""

steve_cli-0.4.1/steve_cli/decorators/__init__.py

@@ -0,0 +1,10 @@
+from collections.abc import Callable
+
+from steve_cli.lineage.storage import LineageStorage
+
+from .lineage_job import lineage_job
+
+GetTables = Callable[[str, str | None], LineageStorage]
+GetStorage = Callable[[str, str | None], LineageStorage]
+
+__all__ = ["GetStorage", "GetTables", "lineage_job"]

{steve_cli-0.3.20 → steve_cli-0.4.1}/steve_cli/decorators/lineage_job.py

@@ -7,6 +7,7 @@ from typing import Any, Callable
 
 from steve_cli.lineage.collector import make_session
 from steve_cli.storage.s3 import S3Storage
+from steve_cli.storage.trino import TrinoStorage
 from steve_cli.lineage.storage import LineageStorage
 
 
@@ -48,14 +49,24 @@ def lineage_job(
                     session.namespace = ls._dataset_namespace
                 return ls
 
+            def get_tables(tier: str = "bronze", workspace: str | None = None) -> LineageStorage:
+                return LineageStorage(
+                    storage=lambda: TrinoStorage(tier=tier, workspace=workspace),
+                    session=session,
+                )
+
+            sig = inspect.signature(fn)
+            available = {"get_storage": get_storage, "get_tables": get_tables}
+            injectable = {k: v for k, v in available.items() if k in sig.parameters}
+
             try:
-                result = fn(*args, get_storage=get_storage, **kwargs)
+                result = fn(*args, **injectable, **kwargs)
             except Exception as exc:
                 session.fail(exc)
                 import sys
+                import click
                 from steve_cli.validation.port import DataQualityError
                 if isinstance(exc, (DataQualityError, EnvironmentError)):
-                    import click
                     click.secho(f"ERROR {exc}", fg="red", bold=True, err=True)
                     sys.exit(1)
                 raise

steve_cli-0.4.1/steve_cli/policies/__init__.py

@@ -0,0 +1,3 @@
+from .client import PolicyClient
+
+__all__ = ["PolicyClient"]

steve_cli-0.4.1/steve_cli/policies/client.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+_POLICY_VERSION = "v0"
+
+
+class PolicyClient:
+    def __init__(
+        self,
+        endpoint: str | None = None,
+        token: str | None = None,
+    ):
+        self.endpoint = (endpoint or os.environ.get("PCP_ENDPOINT", "http://policy-control-plane:3010")).rstrip("/")
+        self.token = token or os.environ.get("PCP_TOKEN", "")
+
+    def __trpc(self, method: str, path: str, payload: dict | None = None) -> dict:
+        url = f"{self.endpoint}/api/trpc/{path}"
+        headers = {"Authorization": f"Bearer {self.token}"} if self.token else {}
+        if method == "query":
+            resp = httpx.get(url, params={"input": json.dumps(payload or {})}, headers=headers, timeout=10)
+        else:
+            resp = httpx.post(url, json=payload or {}, headers=headers, timeout=10)
+        resp.raise_for_status()
+        data = resp.json()
+        if "error" in data:
+            raise RuntimeError(f"tRPC error on {path}: {data['error']}")
+        return data.get("result", {}).get("data", data.get("result", {}))
+
+    def _trpc(self, method: str, path: str, payload: dict | None = None) -> dict:
+        try:
+            return self.__trpc(method, path, payload)
+        except httpx.ConnectError as e:
+            raise EnvironmentError(
+                f"Cannot reach Policy Control Plane at {self.endpoint} — "
+                f"set PCP_ENDPOINT in your .env"
+            ) from e
+
+    def register_pdp(self, name: str, endpoint: str, pdp_type: str = "opa", is_default: bool = True) -> dict:
+        return self._trpc("mutation", "pdp.register", {
+            "name": name,
+            "type": pdp_type,
+            "endpoint": endpoint,
+            "isDefault": is_default,
+        })
+
+    def attach_pdp(self, workspace: str, pdp_id: str) -> None:
+        self._trpc("mutation", "workspace.attachPdp", {"workspace": workspace, "pdpId": pdp_id})
+
+    def apply_policies(self, workspace: str, pdp_name: str, rules: list[dict[str, Any]]) -> dict:
+        return self._trpc("mutation", "workspace.applyPolicies", {
+            "workspace": workspace,
+            "fragments": [
+                {
+                    "syntax": "dsl",
+                    "target": pdp_name,
+                    "content": {"rules": rules},
+                }
+            ],
+        })
+
+    def apply_from_file(self, policies_file: Path | None = None) -> None:
+        path = policies_file or Path.cwd() / "policies" / "access.yaml"
+        if not path.exists():
+            raise FileNotFoundError(f"Policies file not found: {path}")
+
+        import yaml
+        config: dict[str, Any] = yaml.safe_load(path.read_text())
+        workspace = config["workspace"]
+        pdp_cfg = config["pdp"]
+        rules = config.get("rules", [])
+
+        pdp = self.register_pdp(
+            name=pdp_cfg["name"],
+            endpoint=pdp_cfg["endpoint"],
+            pdp_type=pdp_cfg.get("type", "opa"),
+        )
+        self.attach_pdp(workspace, pdp["id"])
+        result = self.apply_policies(workspace, pdp_cfg["name"], rules)
+
+        errors = result.get("errors", [])
+        if errors:
+            reasons = "; ".join(e["reason"] for e in errors)
+            raise RuntimeError(f"Failed to apply {len(errors)} policy fragment(s): {reasons}")

{steve_cli-0.3.20 → steve_cli-0.4.1}/steve_cli/storage/__init__.py

@@ -1,11 +1,13 @@
 from .protocol import Storage
 from .s3 import S3Storage
+from .trino import TrinoStorage
 from .parquet import ParquetMetadata, extract_parquet_metadata
 from .metadata import FileMetadata, ColumnMetadata, MetadataExtractorPort, MetadataRegistry
 
 __all__ = [
     "Storage",
     "S3Storage",
+    "TrinoStorage",
     "ParquetMetadata",
     "extract_parquet_metadata",
     "FileMetadata",

steve_cli-0.4.1/steve_cli/storage/trino.py

@@ -0,0 +1,199 @@
+from __future__ import annotations
+
+import io
+import logging
+import os
+import time
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+def _derive_schema(tier: str, workspace: str | None) -> str | None:
+    if workspace:
+        prefix = workspace.lower().replace("-", "_")
+        return f"ws_{prefix}_{tier.lower()}"
+    return None
+
+
+class TrinoStorage:
+    def __init__(self, tier: str = "bronze", workspace: str | None = None):
+        trino_endpoint = os.getenv("TRINO_ENDPOINT")
+        if not trino_endpoint:
+            raise EnvironmentError("TRINO_ENDPOINT is not set — Trino is not available in this environment")
+        self.catalog = os.getenv("TRINO_CATALOG", "minio")
+        resolved_workspace = workspace or os.getenv("WORKSPACE_NAME")
+        self.schema = _derive_schema(tier, resolved_workspace) or os.environ.get("TRINO_SCHEMA", "")
+        if not self.schema:
+            raise EnvironmentError("TRINO_SCHEMA is not set and no WORKSPACE_NAME or workspace was provided")
+        self.user = os.getenv("TRINO_USER", "admin")
+        self._base = trino_endpoint.rstrip("/")
+        self._lakekeeper_endpoint = os.getenv("LAKEKEEPER_ENDPOINT")
+        default_warehouse = f"{resolved_workspace}-{tier}" if resolved_workspace else "minio"
+        self._lakekeeper_warehouse = os.getenv("LAKEKEEPER_WAREHOUSE", default_warehouse)
+        self.__iceberg_catalog = None
+
+    @property
+    def _iceberg_catalog(self):
+        if self.__iceberg_catalog is None:
+            from pyiceberg.catalog.rest import RestCatalog
+            from pyiceberg.io import load_file_io
+            from pyiceberg.table import Table
+
+            # FIXME: Interim workaround — pyiceberg cannot use Lakekeeper's remote signing
+            # endpoint when running outside the cluster (hostname 'lakekeeper' doesn't resolve
+            # locally). We subclass RestCatalog to inject our local signer URI after pyiceberg
+            # merges table config (which overwrites s3.signer.uri with the internal hostname).
+            # Remove once Increment 5 (STS credential vending) is wired.
+            tier = self.schema.rsplit("_", 1)[-1].upper()
+            lakekeeper_base = self._lakekeeper_endpoint or "http://lakekeeper:8181/catalog"
+            s3_overrides = {
+                "s3.endpoint": os.getenv("S3_ENDPOINT", "http://minio:9000"),
+                "s3.access-key-id": os.getenv(f"{tier}_ACCESS_KEY") or os.getenv("AWS_ACCESS_KEY_ID", ""),
+                "s3.secret-access-key": os.getenv(f"{tier}_SECRET_KEY") or os.getenv("AWS_SECRET_ACCESS_KEY", ""),
+                "s3.path-style-access": "true",
+                "s3.signer.uri": lakekeeper_base,
+            } if self._lakekeeper_endpoint else {}
+
+            class _PatchedRestCatalog(RestCatalog):
+                def _response_to_table(self_, identifier_tuple, table_response):
+                    merged = {**table_response.metadata.properties, **table_response.config, **s3_overrides, "uri": lakekeeper_base}
+                    return Table(
+                        identifier=identifier_tuple,
+                        metadata_location=table_response.metadata_location,
+                        metadata=table_response.metadata,
+                        io=load_file_io(merged, table_response.metadata_location),
+                        catalog=self_,
+                        config=table_response.config,
+                    )
+
+            catalog = _PatchedRestCatalog(
+                "lakekeeper",
+                uri=lakekeeper_base,
+                warehouse=self._lakekeeper_warehouse,
+                **s3_overrides,
+            )
+            if self._lakekeeper_endpoint:
+                catalog.uri = lakekeeper_base
+            self.__iceberg_catalog = catalog
+        return self.__iceberg_catalog
+
+    def _execute(self, sql: str) -> list[dict]:
+        headers = {
+            "X-Trino-User": self.user,
+            "X-Trino-Catalog": self.catalog,
+            "X-Trino-Schema": self.schema,
+        }
+        resp = requests.post(f"{self._base}/v1/statement", data=sql, headers=headers)
+        resp.raise_for_status()
+        data = resp.json()
+        rows: list[dict] = []
+        while True:
+            if "data" in data and "columns" in data:
+                col_names = [c["name"] for c in data["columns"]]
+                for row in data["data"]:
+                    rows.append(dict(zip(col_names, row)))
+            next_uri = data.get("nextUri")
+            if not next_uri:
+                break
+            time.sleep(0.1)
+            resp = requests.get(next_uri, headers=headers)
+            resp.raise_for_status()
+            data = resp.json()
+        return rows
+
+    @staticmethod
+    def _table_name(path: str) -> str:
+        return path.strip("/").replace("/", "_").replace(".", "_")
+
+    def list(self, prefix: str = "") -> list[str]:
+        rows = self._execute(f"SHOW TABLES FROM {self.catalog}.{self.schema}")
+        tables = [r["Table"] for r in rows]
+        if prefix:
+            clean = prefix.strip("/")
+            tables = [t for t in tables if t.startswith(clean)]
+        return tables
+
+    def list_all(self) -> list[str]:
+        return self.list()
+
+    def get_bytes(self, path: str) -> bytes:
+        rows = self._execute(f"SELECT * FROM {self.catalog}.{self.schema}.{self._table_name(path)}")
+        if not rows:
+            return b""
+        arrow_table = pa.Table.from_pylist(rows)
+        buf = io.BytesIO()
+        pq.write_table(arrow_table, buf)
+        return buf.getvalue()
+
+    def get_file(self, path: str, local_path: str) -> None:
+        Path(local_path).parent.mkdir(parents=True, exist_ok=True)
+        Path(local_path).write_bytes(self.get_bytes(path))
+
+    def _ensure_namespace(self) -> None:
+        from pyiceberg.exceptions import NamespaceAlreadyExistsError
+        try:
+            self._iceberg_catalog.create_namespace(self.schema)
+            logger.info("Created namespace %s", self.schema)
+        except NamespaceAlreadyExistsError:
+            pass
+
+    def put_bytes(self, data: bytes, path: str) -> None:
+        from pyiceberg.exceptions import NoSuchTableError
+
+        arrow_table = pq.read_table(io.BytesIO(data))
+        table_id = f"{self.schema}.{self._table_name(path)}"
+        catalog = self._iceberg_catalog
+
+        self._ensure_namespace()
+
+        try:
+            iceberg_table = catalog.load_table(table_id)
+            iceberg_table.append(arrow_table)
+        except NoSuchTableError:
+            from pyiceberg.schema import Schema
+            from pyiceberg.types import (
+                BinaryType,
+                BooleanType,
+                DateType,
+                DoubleType,
+                FloatType,
+                IntegerType,
+                LongType,
+                NestedField,
+                StringType,
+                TimestampType,
+            )
+
+            _PYARROW_TO_ICEBERG = {
+                pa.bool_(): BooleanType(),
+                pa.int32(): IntegerType(),
+                pa.int64(): LongType(),
+                pa.float32(): FloatType(),
+                pa.float64(): DoubleType(),
+                pa.large_utf8(): StringType(),
+                pa.utf8(): StringType(),
+                pa.large_binary(): BinaryType(),
+                pa.binary(): BinaryType(),
+                pa.date32(): DateType(),
+                pa.timestamp("us"): TimestampType(),
+                pa.timestamp("us", tz="UTC"): TimestampType(),
+            }
+
+            fields = []
+            for i, field in enumerate(arrow_table.schema):
+                iceberg_type = _PYARROW_TO_ICEBERG.get(field.type, StringType())
+                fields.append(NestedField(field_id=i + 1, name=field.name, field_type=iceberg_type, required=not field.nullable))
+
+            iceberg_schema = Schema(*fields)
+            iceberg_table = catalog.create_table(table_id, schema=iceberg_schema)
+            iceberg_table.append(arrow_table)
+
+        logger.info("Written %d rows to %s", len(arrow_table), table_id)
+
+    def put_file(self, local_path: str, path: str) -> None:
+        self.put_bytes(Path(local_path).read_bytes(), path)

{steve_cli-0.3.20 → steve_cli-0.4.1}/steve_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: steve-cli
-Version: 0.3.20
+Version: 0.4.1
 Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
 Author: Frank
 License: MIT
@@ -44,6 +44,12 @@ Requires-Dist: great-expectations>=0.18.0; extra == "great-expectations"
 Requires-Dist: pandas>=2.0.3; extra == "great-expectations"
 Provides-Extra: excel
 Requires-Dist: openpyxl>=3.1.0; extra == "excel"
+Provides-Extra: trino
+Requires-Dist: trino>=0.330; extra == "trino"
+Requires-Dist: pyarrow>=17.0.0; extra == "trino"
+Requires-Dist: requests>=2.31.0; extra == "trino"
+Requires-Dist: pyiceberg>=0.7.0; extra == "trino"
+Requires-Dist: s3fs>=2024.1.0; extra == "trino"
 Provides-Extra: visidata
 Requires-Dist: visidata>=3.0; extra == "visidata"
 Provides-Extra: all
@@ -72,7 +78,7 @@ uv pip install -e .
 
 ### Install from GitHub
 
-`uv add steve-cli` 
+`uv add steve-cli`
 
 OR
 
@@ -80,7 +86,14 @@ OR
 uv add git+https://github.com/your-org/tilt-ts-4.git#subdirectory=apps/example-hehnke/packages/steve-cli
 ```
 
+### dev
 
+For local development add the following to your pyproj.toml which allows using the local copy instead of the one from the registry
+
+```toml
+[tool.uv.sources]
+steve-cli = { path = "../../packages/steve-cli", editable = true }
+```
 
 ## Usage
 
@@ -162,13 +175,13 @@ Running `steve extract-data` will:
 
 Steve automatically extracts metadata from files read or written via `MetadataRegistry`. The extractor is chosen by file extension — no configuration needed.
 
-| Extension | Extractor | Requires |
-|---|---|---|
-| `.parquet`, `.pq` | `ParquetExtractor` | `pip install steve-cli[polars]` |
-| `.csv`, `.tsv`, `.txt` | `CsvExtractor` | stdlib only |
-| `.json`, `.jsonl`, `.ndjson` | `JsonExtractor` | stdlib only |
-| `.xlsx`, `.xls`, `.xlsm` | `ExcelExtractor` | `pip install steve-cli[excel]` |
-| anything else | `GenericExtractor` | stdlib only |
+| Extension                    | Extractor          | Requires                        |
+| ---------------------------- | ------------------ | ------------------------------- |
+| `.parquet`, `.pq`            | `ParquetExtractor` | `pip install steve-cli[polars]` |
+| `.csv`, `.tsv`, `.txt`       | `CsvExtractor`     | stdlib only                     |
+| `.json`, `.jsonl`, `.ndjson` | `JsonExtractor`    | stdlib only                     |
+| `.xlsx`, `.xls`, `.xlsm`     | `ExcelExtractor`   | `pip install steve-cli[excel]`  |
+| anything else                | `GenericExtractor` | stdlib only                     |
 
 ### Adding a custom extractor
 
@@ -230,10 +243,9 @@ black steve_cli/
 isort steve_cli/
 ```
 
-
-
 # use cli
-source .venv/bin/activate 
+
+source .venv/bin/activate
 steve
 
 #

{steve_cli-0.3.20 → steve_cli-0.4.1}/steve_cli.egg-info/SOURCES.txt

@@ -20,10 +20,13 @@ steve_cli/lineage/adapters/__init__.py
 steve_cli/lineage/adapters/logging.py
 steve_cli/lineage/adapters/null.py
 steve_cli/lineage/adapters/openlineage.py
+steve_cli/policies/__init__.py
+steve_cli/policies/client.py
 steve_cli/storage/__init__.py
 steve_cli/storage/parquet.py
 steve_cli/storage/protocol.py
 steve_cli/storage/s3.py
+steve_cli/storage/trino.py
 steve_cli/storage/metadata/__init__.py
 steve_cli/storage/metadata/port.py
 steve_cli/storage/metadata/registry.py

{steve_cli-0.3.20 → steve_cli-0.4.1}/steve_cli.egg-info/requires.txt

@@ -41,6 +41,13 @@ pyarrow>=17.0.0
 polars>=1.8.2
 pyarrow>=17.0.0
 
+[trino]
+trino>=0.330
+pyarrow>=17.0.0
+requests>=2.31.0
+pyiceberg>=0.7.0
+s3fs>=2024.1.0
+
 [validoopsie]
 validoopsie>=0.1.0
 polars>=1.8.2

steve_cli-0.3.20/steve_cli/decorators/__init__.py

@@ -1,3 +0,0 @@
-from .lineage_job import lineage_job
-
-__all__ = ["lineage_job"]