spadelib 0.1.0__tar.gz

spadelib-0.1.0/PKG-INFO

@@ -0,0 +1,156 @@
+Metadata-Version: 2.4
+Name: spadelib
+Version: 0.1.0
+Summary: Spade block authoring library for Python
+Keywords: spade,blocks,pipeline,data
+Author: krbundy
+Author-email: krbundy <kenneth.bundy@maine.edu>
+License-Expression: Apache-2.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Classifier: Typing :: Typed
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: pyyaml>=6.0
+Requires-Python: >=3.12
+Project-URL: Repository, https://github.com/krbundy/psae_monorepo
+Project-URL: Issues, https://github.com/krbundy/psae_monorepo/issues
+Description-Content-Type: text/markdown
+
+# Spade Python Library
+
+The Python library for authoring [Spade](../../spec/core.md) blocks. Define a handler function, call `run()`, and the library handles parameter loading, input discovery, and output writing.
+
+## Installation
+
+```bash
+uv add spadelib
+```
+
+
+```bash
+uv sync
+```
+
+## Quick Start
+
+```python
+from spadelib import run, RasterFile
+
+def handler(source: RasterFile, buffer_distance: int) -> RasterFile:
+    # Your processing logic here
+    result_path = process(source.path, buffer_distance)
+    return RasterFile(path=result_path)
+
+if __name__ == "__main__":
+    run(handler)
+```
+
+That's it. The `run()` function:
+
+1. Loads scalar parameters from `params.yaml`
+2. Scans `inputs/` for file-based arguments, matched by name to your function parameters
+3. Calls your handler with the combined arguments
+4. Writes the return value to `outputs/`
+
+## Types
+
+Use type hints on your handler to control how inputs are loaded and outputs are written.
+
+### File Types
+
+| Type | Description |
+|------|-------------|
+| `File` | Generic single file |
+| `RasterFile` | Raster data (e.g., GeoTIFF) |
+| `VectorFile` | Vector data (e.g., GeoJSON) |
+| `TabularFile` | Tabular data (e.g., CSV) |
+| `JsonFile` | JSON data |
+
+Each has a `path: str` field pointing to the file location.
+
+### Directory Type
+
+| Type | Description |
+|------|-------------|
+| `Directory` | Directory-based input (e.g., shapefiles) |
+
+Has a `path: str` field pointing to the directory.
+
+### Collection Types
+
+| Type | Description |
+|------|-------------|
+| `FileCollection` | Collection of files |
+| `RasterFileCollection` | Collection of raster files |
+| `VectorFileCollection` | Collection of vector files |
+| `TabularFileCollection` | Collection of tabular files |
+
+Each has a `paths: list[str]` field.
+
+## How Inputs Are Resolved
+
+The handler's parameter names are matched against two sources:
+
+- **Scalar parameters** (`str`, `int`, `float`, `bool`): loaded from `params.yaml`
+- **File-based inputs**: discovered from subdirectories in `inputs/`, where each subdirectory name matches a parameter name
+
+```
+params.yaml              # scalar args: {"resolution": 10, "method": "nearest"}
+inputs/
+  reference/
+    data.tif             # -> handler(reference=RasterFile(path="inputs/reference/data.tif"))
+  target/
+    data.tif             # -> handler(target=RasterFile(path="inputs/target/data.tif"))
+```
+
+Type hints determine how each input is constructed:
+
+- `File` subclass: expects a single file in the subdirectory
+- `Directory`: uses the subdirectory path itself
+- `FileCollection` subclass: collects all files in the subdirectory
+
+## Output Handling
+
+Return a typed value from your handler and `run()` writes it to `outputs/`:
+
+```python
+# Single output
+def handler(source: RasterFile) -> RasterFile:
+    return RasterFile(path="result.tif")
+
+# Multiple outputs
+def handler(source: RasterFile) -> dict:
+    return {
+        "raster": RasterFile(path="result.tif"),
+        "summary": JsonFile(path="stats.json"),
+    }
+```
+
+Output names are resolved from `block.yaml` (if available) or inferred from the return type.
+
+## The `build()` Function
+
+Generate a block manifest from a handler's signature:
+
+```python
+from spadelib import build, RasterFile, VectorFile
+
+def handler(raster: RasterFile, buffer: int) -> VectorFile:
+    """Converts raster boundaries to vector polygons."""
+    ...
+
+manifest = build(handler)
+# {'description': 'Converts raster boundaries to vector polygons.',
+#  'inputs': {'raster': {'type': 'file', 'format': 'GeoTIFF'},
+#             'buffer': {'type': 'number'}},
+#  'outputs': {'vector': {'type': 'file', 'format': 'GeoJSON'}}}
+```
+
+## Testing
+
+```bash
+uv run pytest
+```

spadelib-0.1.0/README.md

@@ -0,0 +1,135 @@
+# Spade Python Library
+
+The Python library for authoring [Spade](../../spec/core.md) blocks. Define a handler function, call `run()`, and the library handles parameter loading, input discovery, and output writing.
+
+## Installation
+
+```bash
+uv add spadelib
+```
+
+
+```bash
+uv sync
+```
+
+## Quick Start
+
+```python
+from spadelib import run, RasterFile
+
+def handler(source: RasterFile, buffer_distance: int) -> RasterFile:
+    # Your processing logic here
+    result_path = process(source.path, buffer_distance)
+    return RasterFile(path=result_path)
+
+if __name__ == "__main__":
+    run(handler)
+```
+
+That's it. The `run()` function:
+
+1. Loads scalar parameters from `params.yaml`
+2. Scans `inputs/` for file-based arguments, matched by name to your function parameters
+3. Calls your handler with the combined arguments
+4. Writes the return value to `outputs/`
+
+## Types
+
+Use type hints on your handler to control how inputs are loaded and outputs are written.
+
+### File Types
+
+| Type | Description |
+|------|-------------|
+| `File` | Generic single file |
+| `RasterFile` | Raster data (e.g., GeoTIFF) |
+| `VectorFile` | Vector data (e.g., GeoJSON) |
+| `TabularFile` | Tabular data (e.g., CSV) |
+| `JsonFile` | JSON data |
+
+Each has a `path: str` field pointing to the file location.
+
+### Directory Type
+
+| Type | Description |
+|------|-------------|
+| `Directory` | Directory-based input (e.g., shapefiles) |
+
+Has a `path: str` field pointing to the directory.
+
+### Collection Types
+
+| Type | Description |
+|------|-------------|
+| `FileCollection` | Collection of files |
+| `RasterFileCollection` | Collection of raster files |
+| `VectorFileCollection` | Collection of vector files |
+| `TabularFileCollection` | Collection of tabular files |
+
+Each has a `paths: list[str]` field.
+
+## How Inputs Are Resolved
+
+The handler's parameter names are matched against two sources:
+
+- **Scalar parameters** (`str`, `int`, `float`, `bool`): loaded from `params.yaml`
+- **File-based inputs**: discovered from subdirectories in `inputs/`, where each subdirectory name matches a parameter name
+
+```
+params.yaml              # scalar args: {"resolution": 10, "method": "nearest"}
+inputs/
+  reference/
+    data.tif             # -> handler(reference=RasterFile(path="inputs/reference/data.tif"))
+  target/
+    data.tif             # -> handler(target=RasterFile(path="inputs/target/data.tif"))
+```
+
+Type hints determine how each input is constructed:
+
+- `File` subclass: expects a single file in the subdirectory
+- `Directory`: uses the subdirectory path itself
+- `FileCollection` subclass: collects all files in the subdirectory
+
+## Output Handling
+
+Return a typed value from your handler and `run()` writes it to `outputs/`:
+
+```python
+# Single output
+def handler(source: RasterFile) -> RasterFile:
+    return RasterFile(path="result.tif")
+
+# Multiple outputs
+def handler(source: RasterFile) -> dict:
+    return {
+        "raster": RasterFile(path="result.tif"),
+        "summary": JsonFile(path="stats.json"),
+    }
+```
+
+Output names are resolved from `block.yaml` (if available) or inferred from the return type.
+
+## The `build()` Function
+
+Generate a block manifest from a handler's signature:
+
+```python
+from spadelib import build, RasterFile, VectorFile
+
+def handler(raster: RasterFile, buffer: int) -> VectorFile:
+    """Converts raster boundaries to vector polygons."""
+    ...
+
+manifest = build(handler)
+# {'description': 'Converts raster boundaries to vector polygons.',
+#  'inputs': {'raster': {'type': 'file', 'format': 'GeoTIFF'},
+#             'buffer': {'type': 'number'}},
+#  'outputs': {'vector': {'type': 'file', 'format': 'GeoJSON'}}}
+```
+
+## Testing
+
+```bash
+uv run pytest
+```

spadelib-0.1.0/pyproject.toml

@@ -0,0 +1,42 @@
+[project]
+# Distribution (PyPI) name. The import name stays "spadelib" via
+# [tool.uv.build-backend].module-name below, so block code keeps using
+# `import spadelib` regardless of this.
+name = "spadelib"
+version = "0.1.0"
+description = "Spade block authoring library for Python"
+readme = "README.md"
+requires-python = ">=3.12"
+license = "Apache-2.0"
+authors = [
+    { name = "krbundy", email = "kenneth.bundy@maine.edu" }
+]
+keywords = ["spade", "blocks", "pipeline", "data"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Operating System :: OS Independent",
+    "Typing :: Typed",
+]
+dependencies = [
+    "pydantic>=2.12.5",
+    "pyyaml>=6.0",
+]
+
+[project.urls]
+Repository = "https://github.com/krbundy/psae_monorepo"
+Issues = "https://github.com/krbundy/psae_monorepo/issues"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0",
+]
+
+[build-system]
+requires = ["uv_build>=0.10.0,<0.11.0"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+module-name = "spadelib"

spadelib-0.1.0/src/spadelib/__init__.py

@@ -0,0 +1,31 @@
+from spadelib.types import (
+    Directory,
+    File,
+    FileCollection,
+    JsonFile,
+    RasterFile,
+    RasterFileCollection,
+    TabularFile,
+    TabularFileCollection,
+    VectorFile,
+    VectorFileCollection,
+)
+from spadelib.run import run
+from spadelib.build import build
+from spadelib.secrets import get_secret
+
+__all__ = [
+    "File",
+    "Directory",
+    "RasterFile",
+    "VectorFile",
+    "TabularFile",
+    "JsonFile",
+    "FileCollection",
+    "RasterFileCollection",
+    "VectorFileCollection",
+    "TabularFileCollection",
+    "run",
+    "build",
+    "get_secret",
+]

spadelib-0.1.0/src/spadelib/_output.py

@@ -0,0 +1,101 @@
+import os
+import shutil
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from spadelib.types import (
+    Directory,
+    File,
+    FileCollection,
+    JsonFile,
+    RasterFile,
+    RasterFileCollection,
+    TabularFile,
+    TabularFileCollection,
+    VectorFile,
+    VectorFileCollection,
+)
+
+_TYPE_TO_DEFAULT_NAME: dict[type, str] = {
+    File: "file",
+    RasterFile: "raster",
+    VectorFile: "vector",
+    TabularFile: "tabular",
+    JsonFile: "json",
+    Directory: "directory",
+    FileCollection: "files",
+    RasterFileCollection: "rasters",
+    VectorFileCollection: "vectors",
+    TabularFileCollection: "tables",
+}
+
+
+def read_block_manifest() -> dict | None:
+    """Attempt to read the block manifest for output declarations.
+
+    Checks (in order):
+    1. SPADE_BLOCK_MANIFEST environment variable
+    2. block.yaml in the current working directory
+    """
+    manifest_path = os.environ.get("SPADE_BLOCK_MANIFEST")
+    if manifest_path and Path(manifest_path).exists():
+        with open(manifest_path, "r") as f:
+            manifest = yaml.safe_load(f)
+        return manifest.get("outputs") if manifest else None
+
+    block_yaml = Path("block.yaml")
+    if block_yaml.exists():
+        with open(block_yaml, "r") as f:
+            manifest = yaml.safe_load(f)
+        return manifest.get("outputs") if manifest else None
+
+    return None
+
+
+def _infer_output_name(value: Any) -> str:
+    """Infer output name from the type of the return value."""
+    for type_cls, name in _TYPE_TO_DEFAULT_NAME.items():
+        if type(value) is type_cls:
+            return name
+    return type(value).__name__.lower()
+
+
+def _write_single_output(name: str, value: Any) -> None:
+    """Write a single output value to outputs/<name>/."""
+    output_dir = Path("outputs") / name
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    if isinstance(value, FileCollection):
+        for file_path in value.paths:
+            src = Path(file_path)
+            shutil.copy2(src, output_dir / src.name)
+    elif isinstance(value, Directory):
+        src = Path(value.path)
+        for item in src.iterdir():
+            if item.is_file():
+                shutil.copy2(item, output_dir / item.name)
+            elif item.is_dir():
+                shutil.copytree(item, output_dir / item.name)
+    elif isinstance(value, File):
+        src = Path(value.path)
+        shutil.copy2(src, output_dir / src.name)
+
+
+def write_outputs(result: Any, manifest_outputs: dict | None = None) -> None:
+    """Write handler return value(s) to the outputs/ directory."""
+    if result is None:
+        return
+
+    Path("outputs").mkdir(exist_ok=True)
+
+    if isinstance(result, dict):
+        for name, value in result.items():
+            _write_single_output(name, value)
+    elif isinstance(result, (File, Directory, FileCollection)):
+        if manifest_outputs and len(manifest_outputs) == 1:
+            name = next(iter(manifest_outputs))
+        else:
+            name = _infer_output_name(result)
+        _write_single_output(name, result)

spadelib-0.1.0/src/spadelib/_scanning.py

@@ -0,0 +1,89 @@
+from pathlib import Path
+from typing import Any, Callable, get_type_hints
+
+import yaml
+
+from spadelib.types import (
+    Directory,
+    File,
+    FileCollection,
+)
+
+
+def load_params() -> dict:
+    """Load scalar parameters from params.yaml."""
+    params_path = Path("params.yaml")
+    if not params_path.exists():
+        return {}
+    with open(params_path, "r") as f:
+        params = yaml.safe_load(f)
+    return params if params is not None else {}
+
+
+def scan_inputs(type_hints: dict[str, type]) -> dict[str, Any]:
+    """Scan the inputs/ directory and build typed arguments.
+
+    Args:
+        type_hints: Mapping of parameter name -> expected type from the
+                    handler's annotations.
+
+    Returns:
+        Dict mapping parameter name -> typed value.
+    """
+    inputs_dir = Path("inputs")
+    if not inputs_dir.exists():
+        return {}
+
+    result = {}
+    for subdir in sorted(inputs_dir.iterdir()):
+        if not subdir.is_dir():
+            continue
+
+        name = subdir.name
+        expected_type = type_hints.get(name)
+
+        files = sorted([f for f in subdir.iterdir() if f.is_file()])
+
+        if expected_type is not None:
+            try:
+                if issubclass(expected_type, Directory):
+                    result[name] = expected_type(path=str(subdir))
+                    continue
+                elif issubclass(expected_type, FileCollection):
+                    result[name] = expected_type(paths=[str(f) for f in files])
+                    continue
+                elif issubclass(expected_type, File):
+                    if not files:
+                        raise ValueError(
+                            f"Input '{name}' expects a file but directory "
+                            f"'{subdir}' is empty"
+                        )
+                    result[name] = expected_type(path=str(files[0]))
+                    continue
+            except TypeError:
+                pass  # expected_type is not a class, fall through
+
+        # Default: infer from file count
+        if not files:
+            raise ValueError(f"Input directory '{subdir}' is empty")
+        if len(files) == 1:
+            result[name] = File(path=str(files[0]))
+        else:
+            result[name] = FileCollection(paths=[str(f) for f in files])
+
+    return result
+
+
+def build_function_args(fn: Callable) -> dict[str, Any]:
+    """Build the full arguments dict for the handler function.
+
+    Merges scalar parameters from params.yaml with file-based inputs
+    from the inputs/ directory. Inputs take precedence over params.
+    """
+    type_hints = get_type_hints(fn)
+    type_hints.pop("return", None)
+
+    params = load_params()
+    inputs = scan_inputs(type_hints)
+
+    return params | inputs

spadelib-0.1.0/src/spadelib/build.py

@@ -0,0 +1,88 @@
+import inspect
+from typing import Any, Callable, get_type_hints
+
+from spadelib.types import (
+    Directory,
+    File,
+    FileCollection,
+    JsonFile,
+    RasterFile,
+    RasterFileCollection,
+    TabularFile,
+    TabularFileCollection,
+    VectorFile,
+    VectorFileCollection,
+)
+
+_PYTHON_TYPE_TO_MANIFEST: dict[type, dict[str, str]] = {
+    File: {"type": "file"},
+    RasterFile: {"type": "file", "format": "GeoTIFF"},
+    VectorFile: {"type": "file", "format": "GeoJSON"},
+    TabularFile: {"type": "file", "format": "CSV"},
+    JsonFile: {"type": "json"},
+    Directory: {"type": "directory"},
+    FileCollection: {"type": "collection", "item_type": "file"},
+    RasterFileCollection: {"type": "collection", "item_type": "file", "format": "GeoTIFF"},
+    VectorFileCollection: {"type": "collection", "item_type": "file", "format": "GeoJSON"},
+    TabularFileCollection: {"type": "collection", "item_type": "file", "format": "CSV"},
+    str: {"type": "string"},
+    int: {"type": "number"},
+    float: {"type": "number"},
+    bool: {"type": "boolean"},
+}
+
+_TYPE_TO_OUTPUT_NAME: dict[type, str] = {
+    File: "file",
+    RasterFile: "raster",
+    VectorFile: "vector",
+    TabularFile: "tabular",
+    JsonFile: "json",
+    Directory: "directory",
+    FileCollection: "files",
+    RasterFileCollection: "rasters",
+    VectorFileCollection: "vectors",
+    TabularFileCollection: "tables",
+}
+
+
+def build(fn: Callable) -> dict:
+    """Generate a block manifest dict from a handler function's signature.
+
+    Inspects the function's type hints and docstring to produce a dict
+    that can be serialized to block.yaml.
+
+    Args:
+        fn: The handler function to inspect.
+
+    Returns:
+        A dict representing the block manifest (inputs, outputs, description).
+    """
+    hints = get_type_hints(fn)
+    sig = inspect.signature(fn)
+
+    inputs: dict[str, Any] = {}
+    for param_name in sig.parameters:
+        param_type = hints.get(param_name)
+        if param_type is None:
+            continue
+        manifest_entry = _PYTHON_TYPE_TO_MANIFEST.get(param_type)
+        if manifest_entry is not None:
+            inputs[param_name] = dict(manifest_entry)
+
+    outputs: dict[str, Any] = {}
+    return_type = hints.get("return")
+    if return_type is not None and return_type is not type(None):
+        manifest_entry = _PYTHON_TYPE_TO_MANIFEST.get(return_type)
+        if manifest_entry is not None:
+            output_name = _TYPE_TO_OUTPUT_NAME.get(return_type, "output")
+            outputs[output_name] = dict(manifest_entry)
+
+    manifest: dict[str, Any] = {}
+
+    if fn.__doc__:
+        manifest["description"] = fn.__doc__.strip()
+
+    manifest["inputs"] = inputs
+    manifest["outputs"] = outputs
+
+    return manifest

spadelib-0.1.0/src/spadelib/run.py

@@ -0,0 +1,44 @@
+import inspect
+from typing import Callable
+
+from spadelib._output import read_block_manifest, write_outputs
+from spadelib._scanning import build_function_args
+from spadelib.secrets import _load_secrets
+
+
+def run(fn: Callable) -> None:
+    """Execute a handler function as a Spade block.
+
+    1. Load scalar parameters from params.yaml
+    2. Scan inputs/ directory for file-based arguments
+    3. Build the function arguments dict
+    4. Call the handler function
+    5. Write return value(s) to outputs/ directory
+
+    Args:
+        fn: The handler function to execute. Its type hints determine
+            how inputs are loaded and outputs are written.
+    """
+    # Scrub SPADE_SECRETS from the environment early (before the handler runs),
+    # even if the block never calls get_secret. Idempotent.
+    _load_secrets()
+
+    args = build_function_args(fn)
+
+    sig = inspect.signature(fn)
+    param_names = set(sig.parameters.keys())
+
+    has_var_keyword = any(
+        p.kind == inspect.Parameter.VAR_KEYWORD
+        for p in sig.parameters.values()
+    )
+
+    if has_var_keyword:
+        filtered_args = args
+    else:
+        filtered_args = {k: v for k, v in args.items() if k in param_names}
+
+    result = fn(**filtered_args)
+
+    manifest_outputs = read_block_manifest()
+    write_outputs(result, manifest_outputs)

spadelib-0.1.0/src/spadelib/secrets.py

@@ -0,0 +1,50 @@
+"""Access to secrets injected into a block by the Spade runtime.
+
+Secrets are delivered as the ``SPADE_SECRETS`` environment variable — a JSON
+object mapping the block's logical secret names to their values — by the worker
+(cloud) or the CLI (local). This module parses that blob, serves values through
+:func:`get_secret`, and scrubs the variable from the environment so it is not
+inherited by any subprocess the block spawns. See ``spec/secrets.md`` §4.
+"""
+
+import json
+import os
+from typing import Dict, Optional
+
+_secrets: Optional[Dict[str, str]] = None
+
+
+def _parse_secrets(raw: Optional[str]) -> Dict[str, str]:
+    if not raw:
+        return {}
+    return json.loads(raw)
+
+
+def _load_secrets() -> Dict[str, str]:
+    """Parse and cache ``SPADE_SECRETS``, removing it from the environment.
+
+    The variable is popped on first read so it is not inherited by subprocesses
+    the block spawns. Idempotent: subsequent calls return the cached mapping.
+    """
+    global _secrets
+    if _secrets is None:
+        _secrets = _parse_secrets(os.environ.pop("SPADE_SECRETS", None))
+    return _secrets
+
+
+def get_secret(name: str) -> str:
+    """Return the secret bound to a logical ``name`` for this block.
+
+    The mapping from logical name to a stored secret is declared in the
+    pipeline (``spec/secrets.md`` §3.2); the value is injected by the worker
+    (cloud) or CLI (local). Raises :class:`KeyError` if the name was not
+    provided — a declared-but-unresolved secret is a real error, not empty.
+    """
+    secrets = _load_secrets()
+    try:
+        return secrets[name]
+    except KeyError:
+        raise KeyError(
+            f"secret {name!r} was not provided to this block; "
+            "declare it in the pipeline's 'secrets' mapping"
+        ) from None

spadelib-0.1.0/src/spadelib/types.py

@@ -0,0 +1,51 @@
+from pydantic import BaseModel
+
+
+class File(BaseModel):
+    """Base class for single-file inputs/outputs."""
+    path: str
+
+
+class Directory(BaseModel):
+    """Base class for directory-based inputs/outputs."""
+    path: str
+
+
+class RasterFile(File):
+    """Raster data file (e.g., GeoTIFF)."""
+    pass
+
+
+class VectorFile(File):
+    """Vector data file (e.g., GeoJSON, Shapefile)."""
+    pass
+
+
+class TabularFile(File):
+    """Tabular data file (e.g., CSV, Parquet)."""
+    pass
+
+
+class JsonFile(File):
+    """JSON data file."""
+    pass
+
+
+class FileCollection(BaseModel):
+    """Base class for a collection of files."""
+    paths: list[str]
+
+
+class RasterFileCollection(FileCollection):
+    """Collection of raster data files."""
+    pass
+
+
+class VectorFileCollection(FileCollection):
+    """Collection of vector data files."""
+    pass
+
+
+class TabularFileCollection(FileCollection):
+    """Collection of tabular data files."""
+    pass