steve-cli 0.3.6__tar.gz → 0.3.8__tar.gz

{steve_cli-0.3.6/steve_cli.egg-info → steve_cli-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: steve-cli
-Version: 0.3.6
+Version: 0.3.8
 Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
 Author: Frank
 License: MIT
@@ -10,27 +10,48 @@ Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Python: >=3.8
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: boto3>=1.37.38
 Requires-Dist: click>=8.0.0
+Requires-Dist: python-dotenv>=1.0.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: questionary>=2.0.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: black>=22.0; extra == "dev"
 Requires-Dist: isort>=5.0; extra == "dev"
+Requires-Dist: polars>=1.8.2; extra == "dev"
+Requires-Dist: pyarrow>=17.0.0; extra == "dev"
+Requires-Dist: validoopsie>=0.1.0; extra == "dev"
+Requires-Dist: openlineage-python>=1.0.0; extra == "dev"
 Provides-Extra: polars
 Requires-Dist: polars>=1.8.2; extra == "polars"
 Requires-Dist: pyarrow>=17.0.0; extra == "polars"
 Provides-Extra: pandas
 Requires-Dist: pandas>=2.0.3; extra == "pandas"
 Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
+Provides-Extra: lineage
+Requires-Dist: openlineage-python>=1.0.0; extra == "lineage"
+Provides-Extra: validoopsie
+Requires-Dist: validoopsie>=0.1.0; extra == "validoopsie"
+Requires-Dist: polars>=1.8.2; extra == "validoopsie"
+Provides-Extra: great-expectations
+Requires-Dist: great-expectations>=0.18.0; extra == "great-expectations"
+Requires-Dist: pandas>=2.0.3; extra == "great-expectations"
+Provides-Extra: excel
+Requires-Dist: openpyxl>=3.1.0; extra == "excel"
+Provides-Extra: all
+Requires-Dist: openlineage-python>=1.0.0; extra == "all"
+Requires-Dist: pyarrow>=17.0.0; extra == "all"
+Requires-Dist: polars>=1.8.2; extra == "all"
+Requires-Dist: validoopsie>=0.1.0; extra == "all"
+Requires-Dist: great-expectations>=0.18.0; extra == "all"
+Requires-Dist: pandas>=2.0.3; extra == "all"
+Requires-Dist: openpyxl>=3.1.0; extra == "all"
 
 # Steve CLI
 
@@ -134,6 +155,54 @@ Running `steve extract-data` will:
 - ✅ **Colorful**: Nice colored output for better readability
 - ✅ **Error handling**: Clear error messages for missing jobs or files
 
+## File Metadata Extraction
+
+Steve automatically extracts metadata from files read or written via `MetadataRegistry`. The extractor is chosen by file extension — no configuration needed.
+
+| Extension | Extractor | Requires |
+|---|---|---|
+| `.parquet`, `.pq` | `ParquetExtractor` | `pip install steve-cli[polars]` |
+| `.csv`, `.tsv`, `.txt` | `CsvExtractor` | stdlib only |
+| `.json`, `.jsonl`, `.ndjson` | `JsonExtractor` | stdlib only |
+| `.xlsx`, `.xls`, `.xlsm` | `ExcelExtractor` | `pip install steve-cli[excel]` |
+| anything else | `GenericExtractor` | stdlib only |
+
+### Adding a custom extractor
+
+Implement `MetadataExtractorPort`, declare which extensions it handles, and register it once at startup:
+
+```python
+from steve_cli.storage.metadata.port import MetadataExtractorPort, FileMetadata, ColumnMetadata
+from steve_cli.storage.metadata.registry import MetadataRegistry
+
+class AvroExtractor(MetadataExtractorPort):
+    extensions = (".avro",)
+
+    def extract(self, data: bytes, path: str) -> FileMetadata:
+        import fastavro, io
+        reader = fastavro.reader(io.BytesIO(data))
+        schema = reader.writer_schema
+        columns = [
+            ColumnMetadata(name=f["name"], type=str(f["type"]))
+            for f in schema.get("fields", [])
+        ]
+        records = list(reader)
+        return FileMetadata(
+            format="avro",
+            size_bytes=len(data),
+            rows=len(records),
+            columns=columns,
+        )
+
+MetadataRegistry.register("avro", AvroExtractor)
+```
+
+After registration, `MetadataRegistry.extract(data, "output.avro")` picks `AvroExtractor` automatically. You can also force a specific extractor for any file via the env var:
+
+```bash
+METADATA_EXTRACTOR=avro steve jobs run my-job
+```
+
 ## Why Steve?
 
 Named after Steve Jobs - because it helps you run **jobs** locally! 😄

steve_cli-0.3.6/PKG-INFO → steve_cli-0.3.8/README.md

@@ -1,37 +1,3 @@
-Metadata-Version: 2.4
-Name: steve-cli
-Version: 0.3.6
-Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
-Author: Frank
-License: MIT
-Project-URL: Homepage, https://github.com/7frank/ds-steve-cli
-Project-URL: Repository, https://github.com/7frank/ds-steve-cli
-Classifier: Development Status :: 3 - Alpha
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-Requires-Dist: boto3>=1.37.38
-Requires-Dist: click>=8.0.0
-Requires-Dist: pyyaml>=6.0
-Requires-Dist: questionary>=2.0.0
-Provides-Extra: dev
-Requires-Dist: pytest>=7.0; extra == "dev"
-Requires-Dist: black>=22.0; extra == "dev"
-Requires-Dist: isort>=5.0; extra == "dev"
-Provides-Extra: polars
-Requires-Dist: polars>=1.8.2; extra == "polars"
-Requires-Dist: pyarrow>=17.0.0; extra == "polars"
-Provides-Extra: pandas
-Requires-Dist: pandas>=2.0.3; extra == "pandas"
-Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
-
 # Steve CLI
 
 A simple CLI tool to run jobs from `jobs.yaml` with proper environment setup. Perfect for local development and testing of automation kernel jobs.
@@ -134,6 +100,54 @@ Running `steve extract-data` will:
 - ✅ **Colorful**: Nice colored output for better readability
 - ✅ **Error handling**: Clear error messages for missing jobs or files
 
+## File Metadata Extraction
+
+Steve automatically extracts metadata from files read or written via `MetadataRegistry`. The extractor is chosen by file extension — no configuration needed.
+
+| Extension | Extractor | Requires |
+|---|---|---|
+| `.parquet`, `.pq` | `ParquetExtractor` | `pip install steve-cli[polars]` |
+| `.csv`, `.tsv`, `.txt` | `CsvExtractor` | stdlib only |
+| `.json`, `.jsonl`, `.ndjson` | `JsonExtractor` | stdlib only |
+| `.xlsx`, `.xls`, `.xlsm` | `ExcelExtractor` | `pip install steve-cli[excel]` |
+| anything else | `GenericExtractor` | stdlib only |
+
+### Adding a custom extractor
+
+Implement `MetadataExtractorPort`, declare which extensions it handles, and register it once at startup:
+
+```python
+from steve_cli.storage.metadata.port import MetadataExtractorPort, FileMetadata, ColumnMetadata
+from steve_cli.storage.metadata.registry import MetadataRegistry
+
+class AvroExtractor(MetadataExtractorPort):
+    extensions = (".avro",)
+
+    def extract(self, data: bytes, path: str) -> FileMetadata:
+        import fastavro, io
+        reader = fastavro.reader(io.BytesIO(data))
+        schema = reader.writer_schema
+        columns = [
+            ColumnMetadata(name=f["name"], type=str(f["type"]))
+            for f in schema.get("fields", [])
+        ]
+        records = list(reader)
+        return FileMetadata(
+            format="avro",
+            size_bytes=len(data),
+            rows=len(records),
+            columns=columns,
+        )
+
+MetadataRegistry.register("avro", AvroExtractor)
+```
+
+After registration, `MetadataRegistry.extract(data, "output.avro")` picks `AvroExtractor` automatically. You can also force a specific extractor for any file via the env var:
+
+```bash
+METADATA_EXTRACTOR=avro steve jobs run my-job
+```
+
 ## Why Steve?
 
 Named after Steve Jobs - because it helps you run **jobs** locally! 😄

{steve_cli-0.3.6 → steve_cli-0.3.8}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "steve-cli"
-version = "0.3.6"
+version = "0.3.8"
 description = "A simple CLI tool to run jobs from jobs.yaml with proper environment setup"
 readme = "README.md"
 license = {text = "MIT"}
@@ -15,16 +15,15 @@ classifiers = [
     "Intended Audience :: Developers",
     "License :: OSI Approved :: MIT License",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
 ]
-requires-python = ">=3.8"
+requires-python = ">=3.10"
 dependencies = [
     "boto3>=1.37.38",
     "click>=8.0.0",
+    "python-dotenv>=1.0.0",
     "pyyaml>=6.0",
     "questionary>=2.0.0",
 ]
@@ -34,6 +33,10 @@ dev = [
     "pytest>=7.0",
     "black>=22.0",
     "isort>=5.0",
+    "polars>=1.8.2",
+    "pyarrow>=17.0.0",
+    "validoopsie>=0.1.0",
+    "openlineage-python>=1.0.0",
 ]
 polars = [
     "polars>=1.8.2",
@@ -43,6 +46,29 @@ pandas = [
     "pandas>=2.0.3",
     "pyarrow>=17.0.0",
 ]
+lineage = [
+    "openlineage-python>=1.0.0",
+]
+validoopsie = [
+    "validoopsie>=0.1.0",
+    "polars>=1.8.2",
+]
+great-expectations = [
+    "great-expectations>=0.18.0",
+    "pandas>=2.0.3",
+]
+excel = [
+    "openpyxl>=3.1.0",
+]
+all = [
+    "openlineage-python>=1.0.0",
+    "pyarrow>=17.0.0",
+    "polars>=1.8.2",
+    "validoopsie>=0.1.0",
+    "great-expectations>=0.18.0",
+    "pandas>=2.0.3",
+    "openpyxl>=3.1.0",
+]
 
 [project.scripts]
 steve = "steve_cli.cli:main"
@@ -63,3 +89,8 @@ target-version = ['py38']
 
 [tool.isort]
 profile = "black"
+
+[dependency-groups]
+dev = [
+    "great-expectations>=1.18.2",
+]

{steve_cli-0.3.6 → steve_cli-0.3.8}/steve_cli/cli.py

@@ -5,7 +5,7 @@ Steve CLI - A simple tool to run jobs from jobs.yaml with proper environment set
 Usage:
     steve jobs ls              List all available jobs
     steve jobs run <job-name>  Run a job with its environment variables
-    steve setup all            Decrypt SOPS-encoded .env files
+    steve setup all            Decrypt SOPS-encrypted .env files
 """
 
 import os
@@ -18,6 +18,7 @@ from pathlib import Path
 from typing import Dict, List, Any, Optional
 
 import click
+from dotenv import load_dotenv
 import questionary
 import yaml
 
@@ -319,6 +320,10 @@ def apps_list(apps_file: Optional[Path]):
         if command:
             click.echo(f"    Command: {click.style(' '.join(command), fg='cyan')}")
         click.echo(f"    Port: {click.style(str(port), fg='magenta')}")
+        if pid:
+            uf = _url_file(name)
+            url = uf.read_text().strip() if uf.exists() else f'http://localhost:{port}'
+            click.echo(f"    URL: {click.style(url, fg='cyan')}")
         if env_vars:
             click.echo(f"    Environment: {click.style(f'{len(env_vars)} variables', fg='green')}")
         click.echo()
@@ -449,16 +454,16 @@ def setup():
 
 @setup.command("env")
 def setup_env():
-    """Decrypt SOPS-encoded .env files in current directory and write plaintext .env files."""
+    """Decrypt SOPS-encrypted .env files in current directory and write plaintext .env files."""
     cwd = Path.cwd()
     found: List[Path] = []
-    for pattern in ["*.enc.env", "*.encoded.env"]:
+    for pattern in ["*.enc.env", "*.encrypted.env"]:
         found.extend(sorted(cwd.glob(pattern)))
 
     found = [f for f in found if f.name not in SETUP_IGNORE]
 
     if not found:
-        click.echo("No *.enc.env or *.encoded.env files found.")
+        click.echo("No *.enc.env or *.encrypted.env files found.")
         return
 
     for enc_file in found:
@@ -488,7 +493,7 @@ def setup_env():
         if keys:
             click.echo(f"   🔑 Keys: {click.style(', '.join(keys), fg='green')}")
 
-        stem = enc_file.name.replace(".encoded.env", "").replace(".enc.env", "")
+        stem = enc_file.name.replace(".encrypted.env", "").replace(".enc.env", "")
         out_file = cwd / f"{stem}.env"
         out_file.write_text(result.stdout)
 
@@ -550,8 +555,14 @@ def _list_bucket(storage_kwargs: dict, label: str, bucket_name: str) -> None:
 
 
 @main.command("buckets")
-def buckets():
+@click.option('--env-file', '-e', type=click.Path(path_type=Path), multiple=True,
+              help='Path to .env file(s). Can be specified multiple times. Defaults to .env and .workspace.env')
+def buckets(env_file: tuple):
     """List all S3 buckets detected from env variables and show their files as a tree."""
+    cwd = Path.cwd()
+    env_files = [Path(f) for f in env_file] if env_file else [cwd / ".env", cwd / ".workspace.env"]
+    for ef in env_files:
+        load_dotenv(ef)
     tiers = ["bronze", "silver", "gold"]
 
     options: List[Dict[str, Any]] = []

steve_cli-0.3.8/steve_cli/decorators/__init__.py

@@ -0,0 +1,3 @@
+from .lineage_job import lineage_job
+
+__all__ = ["lineage_job"]

steve_cli-0.3.8/steve_cli/decorators/lineage_job.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import functools
+import inspect
+from pathlib import Path
+from typing import Any, Callable
+
+from steve_cli.lineage.collector import make_session
+from steve_cli.storage.s3 import S3Storage
+from steve_cli.lineage.storage import LineageStorage
+
+
+def lineage_job(
+    name: str | None = None,
+    namespace: str | None = None,
+    lineage_provider: str | None = None,
+    lineage_enabled: bool = True,
+) -> Callable:
+    def decorator(fn: Callable) -> Callable:
+        if name:
+            job_name = name
+        else:
+            caller_file = inspect.getfile(fn)
+            stem = Path(caller_file).stem
+            job_name = f"{stem}.{fn.__name__}"
+
+        @functools.wraps(fn)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            session = make_session(
+                namespace=namespace,
+                job_name=job_name,
+                provider=lineage_provider,
+                enabled=lineage_enabled,
+            )
+
+            session.start()
+
+            def get_storage(tier: str = "bronze", workspace: str | None = None) -> LineageStorage:
+                return LineageStorage(
+                    storage=lambda: S3Storage(tier=tier, workspace=workspace),
+                    session=session,
+                )
+
+            try:
+                result = fn(*args, get_storage=get_storage, **kwargs)
+            except Exception as exc:
+                session.fail(exc)
+                from steve_cli.validation.port import DataQualityError
+                if isinstance(exc, DataQualityError):
+                    import sys
+                    print(f"ERROR {exc}", file=sys.stderr)
+                    sys.exit(1)
+                raise
+
+            session.complete()
+            return result
+
+        if fn.__module__ == "__main__":
+            import logging
+            import os
+            logging.basicConfig(level=os.getenv("LOG_LEVEL", "WARNING"), format="%(levelname)s %(name)s: %(message)s")
+            wrapper()
+
+        return wrapper
+
+    return decorator

steve_cli-0.3.8/steve_cli/lineage/__init__.py

@@ -0,0 +1,14 @@
+from .port import LineagePort, LineageEvent, DatasetRef
+from .collector import LineageSession, make_session
+from .registry import LineageRegistry
+from .storage import LineageStorage
+
+__all__ = [
+    "LineagePort",
+    "LineageEvent",
+    "DatasetRef",
+    "LineageSession",
+    "make_session",
+    "LineageRegistry",
+    "LineageStorage",
+]

steve_cli-0.3.8/steve_cli/lineage/adapters/logging.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import json
+import logging
+
+from steve_cli.lineage.port import LineageEvent, LineagePort
+
+logger = logging.getLogger("steve_cli.lineage")
+
+
+class LoggingLineageAdapter(LineagePort):
+    def emit(self, event: LineageEvent) -> None:
+        logger.info(
+            "lineage event",
+            extra={
+                "lineage": {
+                    "state": event.state,
+                    "job": event.job_name,
+                    "namespace": event.namespace,
+                    "run_id": event.run_id,
+                    "inputs": [{"namespace": d.namespace, "name": d.name} for d in event.inputs],
+                    "outputs": [{"namespace": d.namespace, "name": d.name} for d in event.outputs],
+                }
+            },
+        )

steve_cli-0.3.8/steve_cli/lineage/adapters/null.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from steve_cli.lineage.port import LineageEvent, LineagePort
+
+
+class NullLineageAdapter(LineagePort):
+    def emit(self, event: LineageEvent) -> None:
+        pass

steve_cli-0.3.8/steve_cli/lineage/adapters/openlineage.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+import logging
+
+from steve_cli.lineage.port import DatasetRef, LineageEvent, LineagePort
+
+logger = logging.getLogger(__name__)
+
+
+class OpenLineageAdapter(LineagePort):
+    def __init__(self, url: str):
+        try:
+            from openlineage.client import OpenLineageClient
+            from openlineage.client.transport.http import HttpConfig, HttpTransport
+        except ImportError as exc:
+            raise ImportError(
+                "openlineage-python is required. Install it with: pip install steve-cli[lineage]"
+            ) from exc
+
+        # Strip any path suffix — OpenLineageClient expects just the base URL (scheme+host+port)
+        from urllib.parse import urlparse
+        parsed = urlparse(url)
+        base_url = f"{parsed.scheme}://{parsed.netloc}"
+
+        transport = HttpTransport(HttpConfig.from_dict({"url": base_url}))
+        self._client = OpenLineageClient(transport=transport)
+
+    def emit(self, event: LineageEvent) -> None:
+        from openlineage.client.event_v2 import Dataset, InputDataset, Job, Run, RunEvent, RunState
+        from openlineage.client.facet import (
+            Assertion,
+            ColumnMetric,
+            DataQualityAssertionsDatasetFacet,
+            DataQualityMetricsInputDatasetFacet,
+            ErrorMessageRunFacet,
+            SchemaDatasetFacet,
+            SchemaField,
+            StorageDatasetFacet,
+        )
+
+        run_facets = {}
+        if "errorMessage" in event.run_facets:
+            err = event.run_facets["errorMessage"]
+            stack_trace = None
+            if err.get("description"):
+                stack_trace = "\n".join(
+                    f"  [{a['column'] or a['assertion']}] {a['message']}"
+                    for a in err["description"]
+                )
+            run_facets["errorMessage"] = ErrorMessageRunFacet(
+                message=err.get("message", ""),
+                programmingLanguage=err.get("programmingLanguage", "python"),
+                stackTrace=stack_trace,
+            )
+
+        def _build_dataset_facets(raw: dict) -> dict:
+            facets: dict = {}
+            if "schema" in raw:
+                s = raw["schema"]
+                facets["schema"] = SchemaDatasetFacet(
+                    fields=[SchemaField(name=f["name"], type=f.get("type", "string"), description=f.get("description")) for f in s.get("fields", [])]
+                )
+            if "storage" in raw:
+                st = raw["storage"]
+                facets["storage"] = StorageDatasetFacet(
+                    storageLayer=st.get("storageLayer", "s3"),
+                    fileFormat=st.get("fileFormat", ""),
+                )
+            return facets
+
+        def _build_input_facets(raw: dict) -> dict:
+            input_facets: dict = {}
+            if "dataQualityAssertions" in raw:
+                assertions = [
+                    Assertion(assertion=a["assertion"], success=a.get("success", False), column=a.get("column"))
+                    for a in raw["dataQualityAssertions"]
+                ]
+                if assertions:
+                    input_facets["dataQualityAssertions"] = DataQualityAssertionsDatasetFacet(assertions=assertions)
+            if "dataQualityMetrics" in raw:
+                m = raw["dataQualityMetrics"]
+                col_metrics = {
+                    col: ColumnMetric(nullCount=metrics.get("nullCount"))
+                    for col, metrics in m.get("columnMetrics", {}).items()
+                }
+                input_facets["dataQualityMetrics"] = DataQualityMetricsInputDatasetFacet(
+                    rowCount=m.get("rowCount"),
+                    columnMetrics=col_metrics if col_metrics else None,
+                )
+            return input_facets
+
+        def _to_ol_input(ref: DatasetRef) -> InputDataset:
+            raw = ref.facets or {}
+            input_facets = _build_input_facets(raw)
+            return InputDataset(
+                namespace=ref.namespace,
+                name=ref.name,
+                facets=_build_dataset_facets(raw),
+                inputFacets=input_facets if input_facets else None,
+            )
+
+        def _to_ol_output(ref: DatasetRef) -> Dataset:
+            return Dataset(
+                namespace=ref.namespace,
+                name=ref.name,
+                facets=_build_dataset_facets(ref.facets) if ref.facets else {},
+            )
+
+        ol_event = RunEvent(
+            eventType=getattr(RunState, event.state),
+            eventTime=event.event_time,
+            run=Run(runId=event.run_id, facets=run_facets),
+            job=Job(namespace=event.namespace, name=event.job_name),
+            inputs=[_to_ol_input(d) for d in event.inputs],
+            outputs=[_to_ol_output(d) for d in event.outputs],
+        )
+
+        try:
+            self._client.emit(ol_event)
+        except Exception as exc:
+            logger.warning("Failed to emit lineage event to Marquez: %s", exc)