disscube 0.4.0__py3-none-any.whl

disscube/__init__.py

@@ -0,0 +1,19 @@
+from disscube.client import CubeClient
+from disscube.models import (
+    Derivation,
+    DerivedVariable,
+    GridSpec,
+    SpatialDerivation,
+    SpatialSource,
+    Variable,
+)
+
+__all__ = [
+    "CubeClient",
+    "Derivation",
+    "DerivedVariable",
+    "GridSpec",
+    "SpatialDerivation",
+    "SpatialSource",
+    "Variable",
+]

disscube/catalog/__init__.py

@@ -0,0 +1,5 @@
+from .json_store import JsonCatalogStore
+from .protocol import CatalogStore
+from .sqlite_store import SqliteCatalogStore
+
+__all__ = ["CatalogStore", "JsonCatalogStore", "SqliteCatalogStore"]

disscube/catalog/json_store.py

@@ -0,0 +1,94 @@
+import json
+from pathlib import Path
+from typing import Any
+
+from disscube.models import DerivedVariable, GridSpec, SpatialRelation, SpatialSource
+
+
+class JsonCatalogStore:
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self._data: dict[str, Any] = {"grids": {}, "sources": {}, "derived": {}, "relations": []}
+        if self.path.exists():
+            with open(self.path, "r") as f:
+                self._data = json.load(f)
+            if "relations" not in self._data:
+                self._data["relations"] = []
+
+    def _save(self):
+        with open(self.path, "w") as f:
+            json.dump(self._data, f, indent=2)
+
+    def save_grid(self, grid: GridSpec) -> None:
+        self._data["grids"][grid.id] = grid.model_dump()
+        self._save()
+
+    def get_grid(self, grid_id: str) -> GridSpec | None:
+        data = self._data["grids"].get(grid_id)
+        return GridSpec(**data) if data else None
+
+    def list_grids(self) -> list[GridSpec]:
+        return [GridSpec(**g) for g in self._data["grids"].values()]
+
+    def save_spatial_source(self, source: SpatialSource) -> None:
+        self._data["sources"][source.id] = source.model_dump()
+        self._save()
+
+    def get_spatial_source(self, source_id: str) -> SpatialSource | None:
+        data = self._data["sources"].get(source_id)
+        return SpatialSource(**data) if data else None
+
+    def list_spatial_sources(self) -> list[SpatialSource]:
+        return [SpatialSource(**s) for s in self._data["sources"].values()]
+
+    def save_derived(self, derived: DerivedVariable) -> None:
+        self._data["derived"][derived.id] = derived.model_dump()
+        self._save()
+
+    def delete_derived(self, derived_id: str) -> None:
+        self._data["derived"].pop(derived_id, None)
+        self._save()
+
+    def search_derived_variables(self, grid_id: str | None = None, role: str | None = None, tile_id: str | None = None) -> list[DerivedVariable]:
+        results = []
+        for d in self._data["derived"].values():
+            if grid_id and d["grid_id"] != grid_id:
+                continue
+            if role and d["role"] != role:
+                continue
+            if tile_id and d.get("tile_id") != tile_id:
+                continue
+            results.append(DerivedVariable(**d))
+        return results
+
+    def get_derived_by_hash(self, spec_hash: str) -> DerivedVariable | None:
+        for d in self._data["derived"].values():
+            if d["spec_hash"] == spec_hash:
+                return DerivedVariable(**d)
+        return None
+
+    def save_relation(self, relation: SpatialRelation) -> None:
+        # Check if already exists to avoid duplicates
+        existing = self.get_relation(relation.source_grid_id, relation.target_grid_id)
+        if existing:
+            # Update
+            for i, r in enumerate(self._data["relations"]):
+                if r["source_grid_id"] == relation.source_grid_id and r["target_grid_id"] == relation.target_grid_id:
+                    self._data["relations"][i] = relation.model_dump()
+                    break
+        else:
+            self._data["relations"].append(relation.model_dump())
+        self._save()
+
+    def get_relations(self, grid_id: str) -> list[SpatialRelation]:
+        """Busca bidirecional: source_grid_id OU target_grid_id."""
+        return [
+            SpatialRelation(**r) for r in self._data["relations"]
+            if r["source_grid_id"] == grid_id or r["target_grid_id"] == grid_id
+        ]
+
+    def get_relation(self, source_id: str, target_id: str) -> SpatialRelation | None:
+        for r in self._data["relations"]:
+            if r["source_grid_id"] == source_id and r["target_grid_id"] == target_id:
+                return SpatialRelation(**r)
+        return None

disscube/catalog/protocol.py

@@ -0,0 +1,22 @@
+from typing import Protocol
+
+from disscube.models import DerivedVariable, GridSpec, SpatialRelation, SpatialSource
+
+
+class CatalogStore(Protocol):
+    def save_grid(self, grid: GridSpec) -> None: ...
+    def get_grid(self, grid_id: str) -> GridSpec | None: ...
+    def list_grids(self) -> list[GridSpec]: ...
+
+    def save_spatial_source(self, source: SpatialSource) -> None: ...
+    def get_spatial_source(self, source_id: str) -> SpatialSource | None: ...
+    def list_spatial_sources(self) -> list[SpatialSource]: ...
+    
+    def save_derived(self, derived: DerivedVariable) -> None: ...
+    def delete_derived(self, derived_id: str) -> None: ...
+    def search_derived_variables(self, grid_id: str | None = None, role: str | None = None, tile_id: str | None = None) -> list[DerivedVariable]: ...
+    def get_derived_by_hash(self, spec_hash: str) -> DerivedVariable | None: ...
+
+    def save_relation(self, relation: SpatialRelation) -> None: ...
+    def get_relations(self, grid_id: str) -> list[SpatialRelation]: ...
+    def get_relation(self, source_id: str, target_id: str) -> SpatialRelation | None: ...

disscube/catalog/sqlite_store.py

@@ -0,0 +1,143 @@
+import json
+import sqlite3
+from pathlib import Path
+
+from disscube.models import DerivedVariable, GridSpec, SpatialRelation, SpatialSource
+
+
+class SqliteCatalogStore:
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self._init_db()
+
+    def _get_connection(self):
+        conn = sqlite3.connect(self.path)
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    def _init_db(self):
+        with self._get_connection() as conn:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS grids (
+                    id TEXT PRIMARY KEY,
+                    data TEXT NOT NULL
+                )
+            """)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS sources (
+                    id TEXT PRIMARY KEY,
+                    data TEXT NOT NULL
+                )
+            """)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS derived (
+                    id TEXT PRIMARY KEY,
+                    grid_id TEXT,
+                    spec_hash TEXT,
+                    tile_id TEXT,
+                    role TEXT,
+                    data TEXT NOT NULL
+                )
+            """)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS relations (
+                    source_grid_id TEXT,
+                    target_grid_id TEXT,
+                    data TEXT NOT NULL,
+                    PRIMARY KEY (source_grid_id, target_grid_id)
+                )
+            """)
+            # Indexes for performance
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_derived_grid ON derived(grid_id)")
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_derived_hash ON derived(spec_hash)")
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_derived_tile ON derived(tile_id)")
+
+    def save_grid(self, grid: GridSpec) -> None:
+        with self._get_connection() as conn:
+            conn.execute(
+                "INSERT OR REPLACE INTO grids (id, data) VALUES (?, ?)",
+                (grid.id, grid.model_dump_json())
+            )
+
+    def get_grid(self, grid_id: str) -> GridSpec | None:
+        with self._get_connection() as conn:
+            row = conn.execute("SELECT data FROM grids WHERE id = ?", (grid_id,)).fetchone()
+            return GridSpec(**json.loads(row["data"])) if row else None
+
+    def list_grids(self) -> list[GridSpec]:
+        with self._get_connection() as conn:
+            rows = conn.execute("SELECT data FROM grids").fetchall()
+            return [GridSpec(**json.loads(r["data"])) for r in rows]
+
+    def save_spatial_source(self, source: SpatialSource) -> None:
+        with self._get_connection() as conn:
+            conn.execute(
+                "INSERT OR REPLACE INTO sources (id, data) VALUES (?, ?)",
+                (source.id, source.model_dump_json())
+            )
+
+    def get_spatial_source(self, source_id: str) -> SpatialSource | None:
+        with self._get_connection() as conn:
+            row = conn.execute("SELECT data FROM sources WHERE id = ?", (source_id,)).fetchone()
+            return SpatialSource(**json.loads(row["data"])) if row else None
+
+    def list_spatial_sources(self) -> list[SpatialSource]:
+        with self._get_connection() as conn:
+            rows = conn.execute("SELECT data FROM sources").fetchall()
+            return [SpatialSource(**json.loads(r["data"])) for r in rows]
+
+    def save_derived(self, derived: DerivedVariable) -> None:
+        with self._get_connection() as conn:
+            conn.execute(
+                "INSERT OR REPLACE INTO derived (id, grid_id, spec_hash, tile_id, role, data) VALUES (?, ?, ?, ?, ?, ?)",
+                (derived.id, derived.grid_id, derived.spec_hash, derived.tile_id, derived.role, derived.model_dump_json())
+            )
+
+    def delete_derived(self, derived_id: str) -> None:
+        with self._get_connection() as conn:
+            conn.execute("DELETE FROM derived WHERE id = ?", (derived_id,))
+
+    def search_derived_variables(self, grid_id: str | None = None, role: str | None = None, tile_id: str | None = None) -> list[DerivedVariable]:
+        query = "SELECT data FROM derived WHERE 1=1"
+        params = []
+        if grid_id:
+            query += " AND grid_id = ?"
+            params.append(grid_id)
+        if role:
+            query += " AND role = ?"
+            params.append(role)
+        if tile_id:
+            query += " AND tile_id = ?"
+            params.append(tile_id)
+        
+        with self._get_connection() as conn:
+            rows = conn.execute(query, params).fetchall()
+            return [DerivedVariable(**json.loads(r["data"])) for r in rows]
+
+    def get_derived_by_hash(self, spec_hash: str) -> DerivedVariable | None:
+        with self._get_connection() as conn:
+            row = conn.execute("SELECT data FROM derived WHERE spec_hash = ?", (spec_hash,)).fetchone()
+            return DerivedVariable(**json.loads(row["data"])) if row else None
+
+    def save_relation(self, relation: SpatialRelation) -> None:
+        with self._get_connection() as conn:
+            conn.execute(
+                "INSERT OR REPLACE INTO relations (source_grid_id, target_grid_id, data) VALUES (?, ?, ?)",
+                (relation.source_grid_id, relation.target_grid_id, relation.model_dump_json())
+            )
+
+    def get_relations(self, grid_id: str) -> list[SpatialRelation]:
+        with self._get_connection() as conn:
+            rows = conn.execute(
+                "SELECT data FROM relations WHERE source_grid_id = ? OR target_grid_id = ?",
+                (grid_id, grid_id)
+            )
+            return [SpatialRelation(**json.loads(r["data"])) for r in rows]
+
+    def get_relation(self, source_id: str, target_id: str) -> SpatialRelation | None:
+        with self._get_connection() as conn:
+            row = conn.execute(
+                "SELECT data FROM relations WHERE source_grid_id = ? AND target_grid_id = ?",
+                (source_id, target_id)
+            ).fetchone()
+            return SpatialRelation(**json.loads(row["data"])) if row else None

disscube/cli.py

@@ -0,0 +1,144 @@
+"""
+Command line: run DisSCube pipeline files.
+
+    disscube validate pipeline.toml [--json]
+    disscube fetch pipeline.toml
+    disscube run pipeline.toml [--workspace DIR] [--output OUT.tif] [--dry-run] [--json] [-v]
+    disscube export pipeline.toml --output OUT.tif [--workspace DIR] [--variables ...] [--json] [-v]
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="disscube", description="Run DisSCube pipeline files.")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_val = sub.add_parser("validate", help="check a pipeline file without fetching anything")
+    p_val.add_argument("file")
+    p_val.add_argument("--json", action="store_true", help="output validation result as JSON")
+
+    p_fetch = sub.add_parser("fetch", help="download and verify remote file sources declared in the pipeline")
+    p_fetch.add_argument("file")
+
+    p_run = sub.add_parser("run", help="fetch the sources and derive the variables")
+    p_run.add_argument("file")
+    p_run.add_argument("--workspace", help="output folder (default: the file's 'workspace', "
+                                           "else a folder named after the file)")
+    p_run.add_argument("--output", "-o", help="export derived variables (GeoTIFF; netCDF if the file ends in .nc)")
+    p_run.add_argument("--dry-run", action="store_true", help="simulate plan execution without downloading or computing")
+    p_run.add_argument("--json", action="store_true", help="output execution report as JSON")
+    p_run.add_argument("-v", "--verbose", action="store_true", help="log each step")
+
+    p_exp = sub.add_parser("export", help="export derived variables from an existing data cube to GeoTIFF or netCDF")
+    p_exp.add_argument("file", help="pipeline TOML file")
+    p_exp.add_argument("--output", "-o", required=True, help="output file: GeoTIFF, or netCDF if it ends in .nc (e.g. data/cellspace.tif)")
+    p_exp.add_argument("--workspace", help="workspace folder (default: data/cube or from pipeline)")
+    p_exp.add_argument("--variables", nargs="*", help="specific variables to export (default: all derived)")
+    p_exp.add_argument("--json", action="store_true", help="output export result as JSON")
+    p_exp.add_argument("-v", "--verbose", action="store_true", help="log each step")
+
+    args = parser.parse_args(argv)
+    from disscube.pipeline import PipelineError, plan, run
+
+    logging.basicConfig(level=logging.INFO if getattr(args, "verbose", False) else logging.WARNING,
+                        format="%(levelname)s %(name)s: %(message)s")
+    is_json = getattr(args, "json", False)
+    try:
+        p = plan(args.file)
+        if not is_json:
+            print(p.summary())
+        if args.command == "fetch":
+            from disscube.pipeline.runner import _fetch_file_source, _local_file, _raw_cache_dir, _resolve
+            from disscube.pipeline.schema import FileSource
+            raw = _raw_cache_dir()
+            raw.mkdir(parents=True, exist_ok=True)
+            fetched = 0
+            for s in p.sources:
+                if isinstance(s.config, FileSource) and getattr(s.config, 'url', None):
+                    target = _local_file(_resolve(raw, s.config.path))
+                    if target is not None:
+                        print(f"Fetching {s.id} -> {target.name}...")
+                        _fetch_file_source(s.config, target)
+                        fetched += 1
+            print(f"Fetch completed: {fetched} remote sources verified.")
+            return 0
+        if args.command == "validate":
+            if is_json:
+                import json
+                print(json.dumps({
+                    "status": "ok",
+                    "file": str(p.file.path),
+                    "name": p.file.config.name,
+                    "grid": p.grid.name if p.grid else None,
+                    "sources": [s.id for s in p.sources],
+                    "derives": [d.target for d in p.derives],
+                }, indent=2))
+            else:
+                print("OK")
+            return 0
+        if getattr(args, "dry_run", False):
+            if is_json:
+                import json
+                print(json.dumps({
+                    "status": "ok",
+                    "dry_run": True,
+                    "file": str(p.file.path),
+                    "grid": p.grid.name if p.grid else None,
+                    "sources": [{"id": s.id, "type": s.config.type} for s in p.sources],
+                    "derives": [{"target": d.target, "source": d.source, "operator": d.operator} for d in p.derives],
+                }, indent=2))
+            else:
+                print(f"[dry-run] Plan is valid. Would process {len(p.sources)} sources and derive {len(p.derives)} variables.")
+            return 0
+        if args.command == "export":
+            from disscube.pipeline import export_cube
+            exp = export_cube(p, output=args.output, workspace=args.workspace, variables=args.variables)
+            if is_json:
+                import json
+                print(json.dumps({
+                    "status": "ok",
+                    "workspace": str(exp.workspace),
+                    "grid_id": exp.grid_id,
+                    "variables": exp.variables,
+                    "output": str(exp.output),
+                }, indent=2))
+            else:
+                print(f"workspace : {exp.workspace}")
+                print(f"exported  : {len(exp.variables)} variables on {exp.grid_id}")
+                print(f"output    : {exp.output}")
+            return 0
+        report = run(p, workspace=args.workspace, export_geotiff=getattr(args, "output", None))
+        if is_json:
+            import json
+            print(json.dumps({
+                "status": "ok",
+                "workspace": str(report.workspace),
+                "grid_id": report.grid_id,
+                "sources": report.sources,
+                "derived": report.derived,
+                "exported": str(report.exported) if report.exported else None,
+                "record": str(report.record),
+            }, indent=2, default=str))
+            return 0
+    except PipelineError as exc:
+        if is_json:
+            import json
+            print(json.dumps({"status": "error", "error": str(exc)}, indent=2), file=sys.stderr)
+        else:
+            print(f"error: {exc}", file=sys.stderr)
+        return 2
+    print(f"workspace : {report.workspace}")
+    print(f"derived   : {len(report.derived)} products on {report.grid_id}")
+    if report.exported:
+        print(f"exported  : {report.exported}")
+    print(f"record    : {report.record}")
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())