spai 2026.7.7.dev2__tar.gz → 2026.9.8.dev0__tar.gz

{spai-2026.7.7.dev2 → spai-2026.9.8.dev0}/.gitignore

@@ -2,6 +2,7 @@ __pycache__
 .pytest_cache
 data
 outputs
+spai/tests/unit/storage/output
 .env
 .spai
 tmp

{spai-2026.7.7.dev2 → spai-2026.9.8.dev0}/PKG-INFO

@@ -1,9 +1,8 @@
 Metadata-Version: 2.4
 Name: spai
-Version: 2026.7.7.dev2
+Version: 2026.9.8.dev0
 Author-email: Jan Sensio <it@earthpulse.es>
 Requires-Python: >=3.12
-Requires-Dist: bson>=0.5.10
 Requires-Dist: pandas>=2.0.2
 Requires-Dist: pillow>=10.2.0
 Requires-Dist: pydantic>=2.0.3
@@ -22,7 +21,7 @@ Requires-Dist: odc-stac>=0.4.0; extra == 'all'
 Requires-Dist: planetary-computer>=1.0.0; extra == 'all'
 Requires-Dist: pyarrow>=19.0.0; extra == 'all'
 Requires-Dist: pystac-client>=0.9.0; extra == 'all'
-Requires-Dist: rasterio<=1.3.10; extra == 'all'
+Requires-Dist: rasterio>=1.3.11; extra == 'all'
 Requires-Dist: rioxarray>=0.18.2; extra == 'all'
 Requires-Dist: s3fs>=2025.9.0; extra == 'all'
 Requires-Dist: xarray>=2025.1.1; extra == 'all'
@@ -37,7 +36,7 @@ Requires-Dist: odc-geo==0.4.7; extra == 'data'
 Requires-Dist: odc-stac>=0.4.0; extra == 'data'
 Requires-Dist: planetary-computer>=1.0.0; extra == 'data'
 Requires-Dist: pystac-client>=0.9.0; extra == 'data'
-Requires-Dist: rasterio<=1.3.10; extra == 'data'
+Requires-Dist: rasterio>=1.3.11; extra == 'data'
 Requires-Dist: rioxarray>=0.18.2; extra == 'data'
 Requires-Dist: xarray>=2025.1.1; extra == 'data'
 Provides-Extra: storage

{spai-2026.7.7.dev2 → spai-2026.9.8.dev0}/makefile

@@ -13,3 +13,4 @@ build:
 
 publish:
 	uv publish --username "__token__" --password $(token)
+

{spai-2026.7.7.dev2 → spai-2026.9.8.dev0}/pyproject.toml

@@ -1,12 +1,11 @@
 [project]
 name = "spai"
-version = "2026.7.7.dev2"
+version = "2026.09.08.dev0"
 description = ""
 authors = [{name = "Jan Sensio", email = "it@earthpulse.es"}]
 readme = "README.md"
 requires-python = ">=3.12"
 dependencies = [
-    "bson>=0.5.10",
     "pandas>=2.0.2",
     "pillow>=10.2.0",
     "pydantic>=2.0.3",
@@ -21,7 +20,7 @@ dependencies = [
 
 [project.optional-dependencies]
 data = [
-    "rasterio<=1.3.10",
+    "rasterio>=1.3.11",
     "geopandas>=1.1.1",
     "xarray>=2025.1.1",
     "rioxarray>=0.18.2",
@@ -49,7 +48,7 @@ all = [
     "pystac-client>=0.9.0",
     "planetary-computer>=1.0.0",
     "pyarrow>=19.0.0",
-    "rasterio<=1.3.10",
+    "rasterio>=1.3.11",
     "s3fs>=2025.9.0",
     "rioxarray>=0.18.2",
     "xarray>=2025.1.1",
@@ -66,3 +65,8 @@ build-backend = "hatchling.build"
 [tool.hatch.build.targets.wheel]
 packages = ["spai"]
 
+[tool.pytest.ini_options]
+addopts = "--import-mode=importlib"
+# Needed with importlib mode so `import spai` resolves without an editable install (e.g. CI).
+pythonpath = ["."]
+

spai-2026.9.8.dev0/spai/__init__.py

@@ -0,0 +1 @@
+__version__ = "2026.09.08.dev0"

{spai-2026.7.7.dev2 → spai-2026.9.8.dev0}/spai/cli.py

@@ -1,6 +1,7 @@
 import typer
 from pathlib import Path
 from typing import List
+import traceback
 
 
 # Add the cli directory to the Python path
@@ -93,6 +94,7 @@ def run(
             install_requirements(path, typer, False)
         return run_local(path, variables, typer)
     except Exception as e:
+        traceback.print_exc()
         typer.echo(f"Error: {e}")
 
 
@@ -157,7 +159,6 @@ def clone(
         typer.echo(f"Error: {e}")
 
 
-
 @app.command(help="get owned project")
 def get_owned_project(
     project_name: str = typer.Argument(..., help="Project name"),

{spai-2026.7.7.dev2 → spai-2026.9.8.dev0}/spai/data/climate/copernicus/download_copernicus.py

@@ -188,7 +188,7 @@ def download_era5_copernicus(
                     end_date, aoi, end_month_days, variables
                 )
             else:
-                request = populate_reananalysis_request(end_date, aoi, end_month_days)
+                request = populate_reananalysis_request(end_date, aoi, end_month_days, variables)
             logger.debug(f"Request: {request}")
             zip_file = target_dir / "second_month.zip"
             client.retrieve(dataset, request).download(zip_file)

{spai-2026.7.7.dev2 → spai-2026.9.8.dev0}/spai/models/StorageConfig.py

@@ -34,13 +34,12 @@ class S3StorageCredentials(BaseModel):
     access_key: str
     secret_key: str
     bucket: str
-    region: Optional[str] = (
-        None  # en local no hace falta, además afecta a la conexión segura o no
-    )
+    region: str | None = None
 
 
 class S3StorageConfig(StorageConfig):
     credentials: Optional[S3StorageCredentials] = None
+    managed: bool = False  # whether the bucket is managed by SPAI or the user
 
     @field_validator("type")
     @classmethod

{spai-2026.7.7.dev2 → spai-2026.9.8.dev0}/spai/project/deploy_folder.py

@@ -65,7 +65,7 @@ def deploy_folder(user, path, variables, typer, verbose=False):
 
     if data:
         if (
-            os.getenv("SPAI_API_URL", "https://spai-test.earthpulse.ai/").strip("/")
+            os.getenv("SPAI_API_URL").strip("/")
             == "https://spai.staging.api.earthpulse.ai"
         ):
             typer.echo(
@@ -73,8 +73,7 @@ def deploy_folder(user, path, variables, typer, verbose=False):
                 f"Check status at: https://spai.staging.earthpulse.ai/projects/{data['project_id']}"
             )
         elif any(
-            host in (os.getenv("SPAI_API_URL", "https://spai-test.earthpulse.ai/"))
-            for host in ("localhost", "0.0.0.0")
+            host in (os.getenv("SPAI_API_URL")) for host in ("localhost", "0.0.0.0")
         ):
             typer.echo("Deployed to development")
         else:

{spai-2026.7.7.dev2 → spai-2026.9.8.dev0}/spai/project/run_local.py

@@ -108,6 +108,7 @@ def run_local(dir, variables, typer):
                 storage.credentials = create_or_retrieve_s3_bucket(
                     config.project, storage.name
                 )
+                storage.managed = True
         # set env variables
         prefix = "SPAI_STORAGE"
         names = []
@@ -126,8 +127,15 @@ def run_local(dir, variables, typer):
                 os.environ[f"{prefix}_S3_{name}_SECRET"] = (
                     storage.credentials.secret_key
                 )
-                os.environ[f"{prefix}_S3_{name}_REGION"] = storage.credentials.region
+                if storage.credentials.region:
+                    os.environ[f"{prefix}_S3_{name}_REGION"] = (
+                        storage.credentials.region
+                    )
                 os.environ[f"{prefix}_S3_{name}_BUCKET"] = storage.credentials.bucket
+                if storage.managed:
+                    os.environ[f"{prefix}_S3_{name}_MANAGED"] = "True"
+                else:
+                    os.environ[f"{prefix}_S3_{name}_MANAGED"] = "False"
         os.environ[f"{prefix}_NAMES"] = ",".join(names)
     if config.scripts:
         typer.echo("Deploying scripts...")

{spai-2026.7.7.dev2 → spai-2026.9.8.dev0}/spai/repos/APIRepo.py

@@ -233,3 +233,17 @@ class APIRepo:
             timeout=10,
         )
         return self.format_response(response)
+
+    def generate_presigned_url(
+        self, project_name, storage_name, object_name, workspace_name
+    ):
+        api_key = os.getenv("SPAI_API_KEY")
+        if not api_key:
+            raise Exception("SPAI_API_KEY is required to generate presigned URLs")
+        response = requests.get(
+            self.url
+            + f"data/{object_name}?project_name={project_name}&storage_name={storage_name}&workspace_name={workspace_name}",
+            headers={"X-API-Key": api_key},
+            timeout=10,
+        )
+        return self.format_response(response)

{spai-2026.7.7.dev2 → spai-2026.9.8.dev0}/spai/storage/LocalStorage.py

@@ -19,8 +19,9 @@ DRIVER_MAP = {"shp": "ESRI Shapefile", "gml": "GML"}
 
 
 class LocalStorage(BaseStorage):
-    def __init__(self, path="data"):
+    def __init__(self, name, path="data"):
         super().__init__()
+        self.name = name
         self.path = path
         if not os.path.exists(self.path):
             os.makedirs(self.path)
@@ -235,6 +236,7 @@ class LocalStorage(BaseStorage):
             gdf = gpd.GeoDataFrame(df, crs="EPSG:4326", geometry=geom_column)
 
             return gdf
+        return df
 
     @with_geopandas
     def read_from_geo_file(self, gpd, name, driver):

{spai-2026.7.7.dev2 → spai-2026.9.8.dev0}/spai/storage/S3Storage.py

@@ -13,26 +13,38 @@ from .decorators import (
     with_geopandas,
     with_rioxarray,
     with_pyarrow,
-    with_s3fs,
     with_xarray,
     with_zarr,
 )
 from pathlib import Path
+import requests
 
-
-# Ver funciones en CloudStorage si las de aquí no acaban de funcionar...
+from ..repos import APIRepo
+from spai.config import SPAIConfig
+from .presigned_object_response import PresignedObjectResponse
+from .s3_get_object_store import S3GetObjectStore
 
 
 class S3Storage(BaseStorage):
-    def __init__(self, url, access, secret, bucket, region=None):
+    def __init__(self, name, url, access, secret, bucket, region=None, managed=False):
         super().__init__()
+        self.name = name
         self.url = url
         self.access = access
         self.secret = secret
         self.region = region
         self.bucket = bucket
-
-        # print("holaaa", self.url, self.access, self.secret, self.region, self.bucket)
+        self.managed = managed if managed is "True" else False
+        # self.managed = False  # TODO: remove this once we have a way to manage S3 storages in the cloud
+
+        self.config = SPAIConfig()
+        self.workspace = self.config["workspace"]
+        self.api_repo = None
+        if managed:
+            print("This is a SPAI managed S3 storage")
+            self.api_repo = APIRepo()
+        else:
+            print("This is a user managed S3 storage")
 
         if self.access and self.secret:
             # Create a client
@@ -70,14 +82,32 @@ class S3Storage(BaseStorage):
             self.client.remove_object(self.bucket, name)
 
     def get_url(self, name):
-        return self.client.presigned_get_object(self.bucket, name)
+        if self.managed:
+            return self.api_repo.generate_presigned_url(
+                self.config["project"],
+                self.name,
+                name,
+                self.workspace if self.workspace else "default",
+            )
+        else:
+            return self.client.presigned_get_object(self.bucket, name)
+
+    def get_object(self, name):
+        if self.managed:
+            url = self.get_url(name)
+            if isinstance(url, tuple):
+                url, error = url
+                if error:
+                    raise RuntimeError(error)
+            response = requests.get(url, stream=True)
+            response.raise_for_status()
+            return PresignedObjectResponse(response)
+        return self.client.get_object(self.bucket, name)
 
     def exists(self, name):
         try:
             self.client.stat_object(self.bucket, name)
             return True
-        # except error.S3Error: # this is not working in soil moisture template
-        #     return False
         except Exception:
             return False
 
@@ -325,10 +355,9 @@ class S3Storage(BaseStorage):
         parquet_name = name.rsplit(".", 1)[0] + ".parquet"
         return self.create_from_parquet(data, parquet_name)
 
-    @with_s3fs
     @with_zarr
     @with_xarray
-    def create_from_zarr(self, xr, s3fs, data, name):
+    def create_from_zarr(self, xr, data, name):
         # Create a temporary directory to store the zarr
         with tempfile.TemporaryDirectory() as temp_dir:
             # First save the zarr locally
@@ -364,7 +393,14 @@ class S3Storage(BaseStorage):
         return self.get_url(name)
 
     def read_object(self, name):
-        return BytesIO(self.client.get_object(self.bucket, name).read())
+        response = None
+        try:
+            response = self.get_object(name)
+            return BytesIO(response.read())
+        finally:
+            if response:
+                response.close()
+                response.release_conn()
 
     def read_from_json(self, name):
         return pd.read_json(self.read_object(name))
@@ -376,7 +412,7 @@ class S3Storage(BaseStorage):
     def read_from_geojson(self, gpd, name):
         response = None
         try:
-            response = self.client.get_object(self.bucket, name)
+            response = self.get_object(name)
             data = json.load(response)
         finally:
             if response:
@@ -394,8 +430,14 @@ class S3Storage(BaseStorage):
     @with_geopandas
     @with_pyarrow
     def read_from_parquet(self, pa, gpd, name):
-        response = self.client.get_object(self.bucket, name)
-        data = response.read()
+        response = None
+        try:
+            response = self.get_object(name)
+            data = response.read()
+        finally:
+            if response:
+                response.close()
+                response.release_conn()
         df = pd.read_parquet(BytesIO(data))
         if "geometry" in df.columns or "geom" in df.columns:
             from shapely import wkb
@@ -410,34 +452,26 @@ class S3Storage(BaseStorage):
 
     @with_xarray
     def read_from_netcdf(self, xr, name):
-        return xr.open_dataset(self.read_object(name), decode_timedelta=True)
+        # netCDF backends generally need a real filesystem path, not a BytesIO
+        with tempfile.NamedTemporaryFile(suffix=".nc") as temp_file:
+            temp_file.write(self.read_object(name).read())
+            temp_file.flush()
+            with xr.open_dataset(temp_file.name, decode_timedelta=True) as ds:
+                return ds.load()
 
-    @with_s3fs
     @with_zarr
     @with_xarray
-    def read_from_zarr(self, xr, s3fs, name):
-        # Construye el path al archivo Zarr en el bucket S3
-        s3_path = f"s3://{self.bucket}/{name}"
-
-        # Lee los datos desde Zarr utilizando xarray
-        ds = xr.open_zarr(
-            s3_path,
-            storage_options={
-                "key": self.access,
-                "secret": self.secret,
-                "client_kwargs": {"endpoint_url": f"https://{self.url}"},
-            },
-            decode_timedelta=True,
-        )
-
-        return ds
+    def read_from_zarr(self, xr, name):
+        # Zarr is a multi-object store; each key is fetched via get_object
+        # so managed (presigned) and unmanaged (MinIO) both work.
+        return xr.open_zarr(S3GetObjectStore(self, name), decode_timedelta=True)
 
     def read_file(self, name):
         # Using BytesIO to be equivalent to LocalStorage
-        return io.BytesIO(self.client.get_object(self.bucket, name).read())
+        return self.read_object(name)
 
     async def data_stream(self, name):
-        with self.client.get_object(self.bucket, name) as stream:
+        with self.get_object(name) as stream:
             for chunk in stream.stream(1024 * 1024):  # Stream in chunks of 1MB
                 yield chunk
 

{spai-2026.7.7.dev2 → spai-2026.9.8.dev0}/spai/storage/Storage.py

@@ -27,13 +27,15 @@ class Storage(BaseStorage):
                 if key.startswith(f"SPAI_STORAGE_{storage_name.upper()}")
             }
             envs = {key.split("_")[-1].lower(): value for key, value in envs.items()}
-            if storage_name.split("_")[0] == "local":
+            storage_type = storage_name.split("_")[0]
+            storage_name = storage_name.split("_")[1]
+            if storage_type == "local":
                 self.initialize_local(storage_name, envs)
-            elif storage_name.split("_")[0] == "s3":
+            elif storage_type == "s3":
                 self.initialize_s3(storage_name, envs)
 
     def initialize_local(self, name, envs):
-        self.storages[f"{name.split('_')[1]}"] = LocalStorage(**envs)
+        self.storages[name] = LocalStorage(**envs, name=name)
 
     def initialize_s3(self, name, envs):
-        self.storages[f"{name.split('_')[1]}"] = S3Storage(**envs)
+        self.storages[name] = S3Storage(**envs, name=name)

spai-2026.9.8.dev0/spai/storage/presigned_object_response.py

@@ -0,0 +1,28 @@
+import requests
+
+
+class PresignedObjectResponse:
+    """MinIO-compatible wrapper for object data fetched via presigned URL."""
+
+    def __init__(self, http_response: requests.Response):
+        self._http_response = http_response
+        self._raw = http_response.raw
+
+    def read(self, amt=None):
+        return self._raw.read(amt)
+
+    def stream(self, amt=32 * 1024):
+        yield from self._raw.stream(amt)
+
+    def close(self):
+        self._raw.close()
+
+    def release_conn(self):
+        self._raw.release_conn()
+
+    def __enter__(self):
+        self._raw.__enter__()
+        return self
+
+    def __exit__(self, *args):
+        return self._raw.__exit__(*args)

spai-2026.9.8.dev0/spai/storage/s3_get_object_store.py

@@ -0,0 +1,44 @@
+from collections.abc import MutableMapping
+
+
+class S3GetObjectStore(MutableMapping):
+    """Read-only zarr store that fetches keys via S3Storage.get_object."""
+
+    def __init__(self, storage, root):
+        self.storage = storage
+        self.root = root.rstrip("/")
+
+    def _path(self, key):
+        key = str(key).lstrip("/")
+        return f"{self.root}/{key}" if key else self.root
+
+    def __getitem__(self, key):
+        response = None
+        try:
+            response = self.storage.get_object(self._path(key))
+            return response.read()
+        except Exception as exc:
+            raise KeyError(key) from exc
+        finally:
+            if response:
+                response.close()
+                response.release_conn()
+
+    def __setitem__(self, key, value):
+        raise PermissionError("S3GetObjectStore is read-only")
+
+    def __delitem__(self, key):
+        raise PermissionError("S3GetObjectStore is read-only")
+
+    def __contains__(self, key):
+        return self.storage.exists(self._path(key))
+
+    def __iter__(self):
+        prefix = f"{self.root}/"
+        for obj in self.storage.client.list_objects(
+            self.storage.bucket, prefix=prefix, recursive=True
+        ):
+            yield obj.object_name[len(prefix) :]
+
+    def __len__(self):
+        return sum(1 for _ in self)

{spai-2026.7.7.dev2 → spai-2026.9.8.dev0}/tests/config/test_vars.py

@@ -2,7 +2,7 @@ import json
 import pathlib
 import pytest
 
-from spai.spai.config.vars import SPAIVars
+from spai.config.vars import SPAIVars
 
 @pytest.fixture(autouse=False)
 def temp_vars_file(request):

{spai-2026.7.7.dev2/tests → spai-2026.9.8.dev0/tests/deprecated}/task_logging/test_task_logger.py

@@ -1,7 +1,7 @@
 from httpx import Response
 import respx
 
-from spai.spai.task_logging.task_logger import TaskLogger, TaskCreate
+from spai.task_logging.task_logger import TaskLogger, TaskCreate
 
 
 def test_task_logger_init():

{spai-2026.7.7.dev2/tests → spai-2026.9.8.dev0/tests/deprecated}/test_processing/test_autocategorize1D.py

@@ -1,7 +1,7 @@
 import numpy as np
 from unittest.mock import patch
 
-from spai.spai.processing import autocategorize1D
+from spai.processing import autocategorize1D
 
 
 @patch('numpy.argmin', side_effect=lambda arr, axis: np.zeros(arr.shape[0], dtype=int))

{spai-2026.7.7.dev2/tests → spai-2026.9.8.dev0/tests/deprecated}/test_processing/test_mask_raster.py

@@ -2,7 +2,7 @@ from unittest import mock
 from unittest.mock import patch
 import pytest
 
-from spai.spai.processing.mask_raster import mask_raster
+from spai.processing.mask_raster import mask_raster
 
 # Test mask_raster function
 @patch('rasterio.mask.mask')

{spai-2026.7.7.dev2/tests → spai-2026.9.8.dev0/tests/deprecated}/test_processing/test_normalised_difference.py

@@ -1,7 +1,7 @@
 import numpy as np
 import pytest
 
-from spai.spai.processing import normalised_difference
+from spai.processing import normalised_difference
 
 # Test cases
 

{spai-2026.7.7.dev2/tests → spai-2026.9.8.dev0/tests/deprecated}/test_processing/test_px_count.py

@@ -1,6 +1,6 @@
 import numpy as np
 
-from spai.spai.processing.px_count import px_count
+from spai.processing.px_count import px_count
 
 
 # Test px_count function

{spai-2026.7.7.dev2/tests → spai-2026.9.8.dev0/tests/deprecated}/test_processing/test_read_raster.py

@@ -1,7 +1,7 @@
 from unittest import mock
 import pytest
 
-from spai.spai.processing.read_raster import read_raster
+from spai.processing.read_raster import read_raster
 
 def test_read_raster_from_rasterio():
     storage = mock.Mock()

{spai-2026.7.7.dev2/tests → spai-2026.9.8.dev0/tests/deprecated}/test_processing/test_save_table.py

@@ -1,6 +1,6 @@
 from unittest import mock
 import pandas as pd
-from spai.spai.processing.save_table import save_table
+from spai.processing.save_table import save_table
 
 # Test save_table function
 def test_save_table():

spai-2026.9.8.dev0/tests/deprecated/test_pulses/spai

@@ -0,0 +1 @@
+../../spai