terrakio-core 0.3.6__tar.gz → 0.3.8__tar.gz

{terrakio_core-0.3.6 → terrakio_core-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: terrakio-core
-Version: 0.3.6
+Version: 0.3.8
 Summary: Core components for Terrakio API clients
 Author-email: Yupeng Chao <yupeng@haizea.com.au>
 Project-URL: Homepage, https://github.com/HaizeaAnalytics/terrakio-python-api
@@ -23,6 +23,12 @@ Requires-Dist: shapely>=2.0.0
 Requires-Dist: geopandas>=0.13.0
 Requires-Dist: google-cloud-storage>=2.0.0
 Requires-Dist: nest_asyncio
+Provides-Extra: ml
+Requires-Dist: torch>=2.7.1; extra == "ml"
+Requires-Dist: scikit-learn>=1.7.0; extra == "ml"
+Requires-Dist: skl2onnx>=1.19.1; extra == "ml"
+Requires-Dist: onnx>=1.18.0; extra == "ml"
+Requires-Dist: onnxruntime>=1.10.0; extra == "ml"
 
 # Terrakio Core
 

{terrakio_core-0.3.6 → terrakio_core-0.3.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "terrakio-core"
-version = "0.3.6"
+version = "0.3.8"
 authors = [
     {name = "Yupeng Chao", email = "yupeng@haizea.com.au"},
 ]
@@ -32,6 +32,15 @@ dependencies = [
     "nest_asyncio",
 ]
 
+[project.optional-dependencies]
+ml = [
+    "torch>=2.7.1",
+    "scikit-learn>=1.7.0",
+    "skl2onnx>=1.19.1",
+    "onnx>=1.18.0",
+    "onnxruntime>=1.10.0",
+]
+
 [project.urls]
 "Homepage" = "https://github.com/HaizeaAnalytics/terrakio-python-api"
 "Bug Tracker" = "https://github.com/HaizeaAnalytics/terrakio-python-api/issues" 

{terrakio_core-0.3.6 → terrakio_core-0.3.8}/terrakio_core/__init__.py

@@ -5,7 +5,7 @@ Terrakio Core
 Core components for Terrakio API clients.
 """
 
-__version__ = "0.3.4"
+__version__ = "0.3.8"
 
 from .async_client import AsyncClient
 from .sync_client import SyncClient

{terrakio_core-0.3.6 → terrakio_core-0.3.8}/terrakio_core/async_client.py

@@ -46,65 +46,65 @@ class AsyncClient(BaseClient):
         else:
             return await self._make_request_with_retry(self._session, method, endpoint, **kwargs)
 
-
     async def _make_request_with_retry(self, session: aiohttp.ClientSession, method: str, endpoint: str, **kwargs) -> Dict[Any, Any]:
         url = f"{self.url}/{endpoint.lstrip('/')}"
+        last_exception = None
+        
         for attempt in range(self.retry + 1):
-            try:                
+            try:
                 async with session.request(method, url, **kwargs) as response:
-                    response_text = await response.text()
+                    if not response.ok and self._should_retry(response.status, attempt):
+                        self.logger.info(f"Request failed (attempt {attempt+1}/{self.retry+1}): {response.status}. Retrying...")
+                        continue
                     if not response.ok:
-                        should_retry = False
-                        
-                        if response.status == 400:
-                            should_retry = False
-                        else:
-                            if response.status in [408, 502, 503, 504]:
-                                should_retry = True
-                            elif response.status == 500:
-                                try:
-                                    response_text = await response.text()
-                                    if "Internal server error" not in response_text:
-                                        should_retry = True
-                                except:
-                                    should_retry = True
-                        
-                        if should_retry and attempt < self.retry:
-                            self.logger.info(f"Request failed (attempt {attempt+1}/{self.retry+1}): {response.status} {response.reason}. Retrying...")
-                            continue
-                        else:
-                            error_msg = f"API request failed: {response.status} {response.reason}"
-                            try:
-                                error_data = await response.json()
-                                if "detail" in error_data:
-                                    error_msg += f" - {error_data['detail']}"
-                            except:
-                                pass
-                            raise APIError(error_msg, status_code=response.status)
-                    
-                    content_type = response.headers.get('content-type', '').lower()
-                    content = await response.read()
-                    if 'json' in content_type:
-                        return json.loads(content.decode('utf-8'))
-                    elif 'csv' in content_type:
-                        return pd.read_csv(BytesIO(content))
-                    elif 'image/' in content_type:
-                        return content
-                    elif 'text' in content_type:
-                        return content.decode('utf-8')
-                    else:
+                        error_msg = f"API request failed: {response.status} {response.reason}"
                         try:
-                            return xr.open_dataset(BytesIO(content))
+                            error_data = await response.json()
+                            if "detail" in error_data:
+                                error_msg += f" - {error_data['detail']}"
                         except:
-                            raise APIError(f"Unknown response format. Content-Type: {response.headers.get('content-type', 'unknown')}", status_code=response.status)
+                            pass
+                        raise APIError(error_msg, status_code=response.status)
+                    return await self._parse_response(response)
+                    
             except aiohttp.ClientError as e:
+                last_exception = e
                 if attempt < self.retry:
-                    self.logger.info(f"Request failed (attempt {attempt+1}/{self.retry+1}): {e}. Retrying...")
+                    self.logger.info(f"Networking error (attempt {attempt+1}/{self.retry+1}): {e}. Retrying...")
                     continue
                 else:
-                    raise APIError(f"Request failed after {self.retry+1} attempts: {e}", status_code=None)
-                
+                    break
+        
+        raise APIError(f"Networking error, request failed after {self.retry+1} attempts: {last_exception}", status_code=None)
+    
+    def _should_retry(self, status_code: int, attempt: int) -> bool:
+        """Determine if the request should be retried based on status code."""
+        if attempt >= self.retry:
+            return False
+        elif status_code in [408, 502, 503, 504]:
+            return True
+        else:
+            return False
+
+    async def _parse_response(self, response) -> Any:
+        """Parse response based on content type."""
+        content_type = response.headers.get('content-type', '').lower()
+        content = await response.read()
 
+        if 'json' in content_type:
+            return json.loads(content.decode('utf-8'))
+        elif 'csv' in content_type:
+            return pd.read_csv(BytesIO(content))
+        elif 'image/' in content_type:
+            return content
+        elif 'text' in content_type:
+            return content.decode('utf-8')
+        else:
+            try:
+                return xr.open_dataset(BytesIO(content))
+            except:
+                raise APIError(f"Unknown response format: {content_type}", status_code=response.status)
+            
     async def _regular_request(self, method: str, endpoint: str, **kwargs):
         url = endpoint.lstrip('/')
         if self._session is None:
@@ -134,6 +134,7 @@ class AsyncClient(BaseClient):
         output: str = "csv",
         resolution: int = -1,
         geom_fix: bool = False,
+        validated: bool = True,
         **kwargs
     ):
         """
@@ -147,6 +148,7 @@ class AsyncClient(BaseClient):
             output (str): Output format ('csv' or 'netcdf')
             resolution (int): Resolution parameter
             geom_fix (bool): Whether to fix the geometry (default False)
+            validated (bool): Whether to use validated data (default True)
             **kwargs: Additional parameters to pass to the WCS request
             
         Returns:
@@ -169,6 +171,7 @@ class AsyncClient(BaseClient):
             "resolution": resolution,
             "expr": expr,
             "buffer": geom_fix,
+            "validated": validated,
             **kwargs
         }
         return await self._terrakio_request("POST", "geoquery", json=payload)

{terrakio_core-0.3.6 → terrakio_core-0.3.8}/terrakio_core/endpoints/dataset_management.py

@@ -42,7 +42,7 @@ class DatasetManagement:
         return self._client._terrakio_request("GET", f"/datasets/{name}", params = params)
 
     @require_api_key
-    def create_dataset(
+    async def create_dataset(
         self, 
         name: str, 
         collection: str = "terrakio-datasets",
@@ -59,7 +59,8 @@ class DatasetManagement:
         proj4: Optional[str] = None,
         abstract: Optional[str] = None,
         geotransform: Optional[List[float]] = None,
-        padding: Optional[Any] = None
+        padding: Optional[Any] = None,
+        input: Optional[str] = None
     ) -> Dict[str, Any]:
         """
         Create a new dataset.
@@ -104,12 +105,13 @@ class DatasetManagement:
             "proj4": proj4,
             "abstract": abstract,
             "geotransform": geotransform,
-            "padding": padding
+            "padding": padding,
+            "input": input
         }
         for param, value in param_mapping.items():
             if value is not None:
                 payload[param] = value
-        return self._client._terrakio_request("POST", "/datasets", params = params, json = payload)
+        return await self._client._terrakio_request("POST", "/datasets", params = params, json = payload)
     
     @require_api_key
     def update_dataset(

{terrakio_core-0.3.6 → terrakio_core-0.3.8}/terrakio_core/endpoints/mass_stats.py

@@ -12,7 +12,7 @@ class MassStats:
         self._client = client
 
     @require_api_key
-    async def upload_request(
+    async def _upload_request(
         self,
         name: str,
         size: int,
@@ -220,7 +220,7 @@ class MassStats:
         return self._client._terrakio_request("GET", "mass_stats/download", params=params)
     
     @require_api_key
-    async def upload_file(self, file_path: str, url: str, use_gzip: bool = False):
+    async def _upload_file(self, file_path: str, url: str, use_gzip: bool = False):
         """
         Helper method to upload a JSON file to a signed URL.
         
@@ -427,7 +427,7 @@ class MassStats:
             return e
         except json.JSONDecodeError as e:
             return e
-        upload_result = await self.upload_request(name = name, size = size, region = region, output = output, config = config, location = location, force_loc = force_loc, overwrite = overwrite, server = server, skip_existing = skip_existing)
+        upload_result = await self._upload_request(name = name, size = size, region = region, output = output, config = config, location = location, force_loc = force_loc, overwrite = overwrite, server = server, skip_existing = skip_existing)
         requests_url = upload_result.get('requests_url')
         manifest_url = upload_result.get('manifest_url')
         if not requests_url:
@@ -436,7 +436,7 @@ class MassStats:
         try:
             # in this place we are uploading the request json file, we need to check whether the json is in the correct format or not
             self.validate_request(request_json)
-            requests_response = await self.upload_file(request_json, requests_url, use_gzip=True)
+            requests_response = await self._upload_file(request_json, requests_url, use_gzip=True)
             if requests_response.status not in [200, 201, 204]:
                 self._client.logger.error(f"Requests upload error: {requests_response.text()}")
                 raise Exception(f"Failed to upload request JSON: {requests_response.text()}")
@@ -447,7 +447,7 @@ class MassStats:
             raise ValueError("No manifest_url returned from server for manifest JSON upload")
         
         try:
-            manifest_response = await self.upload_file(manifest_json, manifest_url, use_gzip=False)
+            manifest_response = await self._upload_file(manifest_json, manifest_url, use_gzip=False)
             if manifest_response.status not in [200, 201, 204]:
                 self._client.logger.error(f"Manifest upload error: {manifest_response.text()}")
                 raise Exception(f"Failed to upload manifest JSON: {manifest_response.text()}")