terrakio-core 0.4.8__py3-none-any.whl → 0.4.94__py3-none-any.whl

terrakio_core/endpoints/mass_stats.py

@@ -2,6 +2,8 @@ from typing import Dict, Any, Optional
 import json
 import gzip        
 import os
+import weakref
+import weakref
 from pathlib import Path
 from urllib.parse import urlparse
 from ..helper.decorators import require_token, require_api_key, require_auth
@@ -12,7 +14,6 @@ import xarray as xr
 from io import BytesIO
 import geopandas as gpd
 from shapely.geometry import shape
-from ..convenience_functions.convenience_functions import expand_on_variables_and_time
 
 class MassStats:
     def __init__(self, client):
@@ -144,8 +145,9 @@ class MassStats:
         params = {"limit": limit}
         return self._client._terrakio_request("GET", "mass_stats/history", params=params)
     
-    @require_api_key
-    def start_post_processing(
+
+    @require_api_key     
+    async def start_post_processing(
         self,
         process_name: str,
         data_name: str,
@@ -153,30 +155,43 @@ class MassStats:
         consumer: str,
         overwrite: bool = False
     ) -> Dict[str, Any]:
-        """
-        Start post processing for a mass stats job.
-
-        Args:
-            process_name: The name of the post processing process
-            data_name: The name of the data to process
-            output: The output of the post processing
-            consumer: The consumer of the post processing
-            overwrite: Whether to overwrite the post processing
-
-        Returns:
-            API response as a dictionary
+                
+        with open(consumer, 'rb') as f:
+            script_bytes = f.read()
+        
+        data = aiohttp.FormData()
+        data.add_field('process_name', process_name)
+        data.add_field('data_name', data_name) 
+        data.add_field('output', output)
+        data.add_field('overwrite', str(overwrite).lower())
+        data.add_field('consumer', script_bytes, filename=os.path.basename(consumer), content_type='text/x-python')
+        
+        return await self._client._terrakio_request(
+            "POST", 
+            "mass_stats/post_process", 
+            data=data,
+        )
 
-        Raises:
-            APIError: If the API request fails
-        """
-        payload ={
-            "process_name": process_name,
-            "data_name": data_name,
-            "output": output,
-            "consumer": consumer,
-            "overwrite": overwrite
-        }
-        return self._client._terrakio_request("POST", "post_processing/start", json=payload)
+    @require_api_key     
+    async def zonal_stats_transform(
+        self,
+        data_name: str,
+        output: str,
+        consumer: bytes,
+        overwrite: bool = False
+    ) -> Dict[str, Any]:
+                
+        data = aiohttp.FormData()
+        data.add_field('data_name', data_name) 
+        data.add_field('output', output)
+        data.add_field('overwrite', str(overwrite).lower())
+        data.add_field('consumer', consumer, filename="consumer.py", content_type='text/x-python')
+
+        return await self._client._terrakio_request(
+            "POST", 
+            "mass_stats/transform", 
+            data=data,
+        )
     
     @require_api_key
     def download_results(
@@ -377,11 +392,22 @@ class MassStats:
         except Exception as e:
             raise Exception(f"Error in download process: {e}")
 
-    def validate_request(self, request_json_path: str):
-        with open(request_json_path, 'r') as file:
-            request_data = json.load(file)
+    def validate_request(self, request_json: Union[str, List[Dict]]):
+        # Handle both file path and direct JSON data
+        if isinstance(request_json, str):
+            # It's a file path
+            with open(request_json, 'r') as file:
+                request_data = json.load(file)
+        elif isinstance(request_json, list):
+            # It's already JSON data
+            request_data = request_json
+        else:
+            raise ValueError("request_json must be either a file path (str) or JSON data (list)")
+        
+        # Rest of validation logic stays exactly the same
         if not isinstance(request_data, list):
-            raise ValueError(f"Request JSON file {request_json_path} should contain a list of dictionaries")
+            raise ValueError("Request JSON should contain a list of dictionaries")
+        
         for i, request in enumerate(request_data):
             if not isinstance(request, dict):
                 raise ValueError(f"Request {i} should be a dictionary")
@@ -405,7 +431,7 @@ class MassStats:
         name: str, 
         output: str, 
         config: Dict[str, Any],
-        request_json: str,  # Path to request JSON file
+        request_json: Union[str, List[Dict]],  # ← Accept both file path OR data
         region: str = None, 
         overwrite: bool = False, 
         skip_existing: bool = False,  
@@ -453,19 +479,38 @@ class MassStats:
                     
             return groups
         
-        # Load and validate request JSON
+        # # Load and validate request JSON
+        # try:
+        #     with open(request_json, 'r') as file:
+        #         request_data = json.load(file)
+        #         if isinstance(request_data, list):
+        #             size = len(request_data)
+        #         else:
+        #             raise ValueError(f"Request JSON file {request_json} should contain a list of dictionaries")
+        # except FileNotFoundError as e:
+        #     return e
+        # except json.JSONDecodeError as e:
+        #     return e
         try:
-            with open(request_json, 'r') as file:
-                request_data = json.load(file)
-                if isinstance(request_data, list):
-                    size = len(request_data)
-                else:
-                    raise ValueError(f"Request JSON file {request_json} should contain a list of dictionaries")
+            if isinstance(request_json, str):
+                # It's a file path
+                with open(request_json, 'r') as file:
+                    request_data = json.load(file)
+            elif isinstance(request_json, list):
+                # It's already JSON data
+                request_data = request_json
+            else:
+                raise ValueError("request_json must be either a file path (str) or JSON data (list)")
+                
+            if isinstance(request_data, list):
+                size = len(request_data)
+            else:
+                raise ValueError("Request JSON should contain a list of dictionaries")
         except FileNotFoundError as e:
             return e
         except json.JSONDecodeError as e:
             return e
-        
+
         # Generate manifest from request data (kept in memory)
         try:
             manifest_groups = extract_manifest_from_request(request_data)
@@ -500,8 +545,16 @@ class MassStats:
         # Upload request JSON file
         try:
             self.validate_request(request_json)
-            requests_response = await self._upload_file(request_json, requests_url, use_gzip=True)
+            
+            if isinstance(request_json, str):
+                # File path - use existing _upload_file method
+                requests_response = await self._upload_file(request_json, requests_url, use_gzip=True)
+            else:
+                # JSON data - use _upload_json_data method
+                requests_response = await self._upload_json_data(request_json, requests_url, use_gzip=True)
+                
             if requests_response.status not in [200, 201, 204]:
+                # ... rest stays the same
                 self._client.logger.error(f"Requests upload error: {requests_response.text()}")
                 raise Exception(f"Failed to upload request JSON: {requests_response.text()}")
         except Exception as e:
@@ -656,125 +709,4 @@ class MassStats:
             'overwrite': str(overwrite).lower(),
             'max_file_size_mb': max_file_size_mb
         }
-        return await self._client._terrakio_request("POST", "mass_stats/combine_tiles", json=payload)
-
-    @require_api_key
-    async def load_zonal_stats(self, job_id: str, max_files: int = 5, poll_interval: int = 30):
-        """
-        Load zonal stats results from a completed mass stats job.
-        
-        Args:
-            job_id: The job ID returned from the mass stats execution
-            max_files: Maximum number of files to download (default: 5)
-            poll_interval: Seconds to wait between status checks (default: 30)
-            
-        Returns:
-            GeoDataFrame with geometry and dataset columns, or None if failed
-        """
-        try:
-            while True:
-                try:
-                    track_info = await self.track_job([job_id])
-                    job_info = track_info[job_id]
-                    status = job_info['status']
-                    
-                    self._client.logger.info(f"Job {job_id} status: {status}")
-                    
-                    if status == 'Completed':
-                        self._client.logger.info('Job completed successfully!')
-                        break
-                    elif status in ['Failed', 'Cancelled', 'Error']:
-                        raise RuntimeError(f"Job {job_id} failed with status: {status}")
-                    
-                    await asyncio.sleep(poll_interval)
-                    
-                except KeyboardInterrupt:
-                    self._client.logger.info(f"\nInterrupted! Job {job_id} is still running.")
-                    raise
-            
-            async with aiohttp.ClientSession() as session:
-                payload = {
-                    "job_name": job_info['name'],
-                    "file_type": "raw",
-                    "bucket": job_info['bucket']
-                }
-                
-                result = await self._client._terrakio_request("POST", "mass_stats/download_files", json=payload)
-                download_urls = result['download_urls'][:max_files]
-                
-                self._client.logger.info(f"Downloading {len(download_urls)} dataset files...")
-                
-                datasets = []
-                for i, url in enumerate(download_urls):
-                    try:
-                        self._client.logger.info(f"Downloading dataset {i+1}/{len(download_urls)}...")
-                        async with session.get(url) as response:
-                            if response.status == 200:
-                                content = await response.read()
-                                dataset = xr.open_dataset(BytesIO(content))
-                                datasets.append(dataset)
-                                self._client.logger.info(f"Successfully processed dataset {i+1}")
-                            else:
-                                self._client.logger.warning(f"Failed to download dataset {i+1}: HTTP {response.status}")
-                    except Exception as e:
-                        self._client.logger.error(f"Error downloading dataset {i+1}: {e}")
-                        continue
-                
-                if not datasets:
-                    self._client.logger.warning("No datasets were successfully downloaded")
-                    return gpd.GeoDataFrame({'geometry': [], 'dataset': []})
-                
-                try:
-                    json_response = await self._client._terrakio_request(
-                        "POST", "mass_stats/download_json", 
-                        params={"job_name": job_info['name']}
-                    )
-                    json_url = json_response["download_url"]
-                    
-                    async with session.get(json_url) as response:
-                        if response.status == 200:
-                            json_data = await response.json()
-                            self._client.logger.info("Successfully downloaded geometry data")
-                            
-                            geometries = []
-                            max_geometries = min(max_files, len(json_data), len(datasets))
-                            
-                            for i in range(max_geometries):
-                                try:
-                                    geom_dict = json_data[i]["request"]["feature"]["geometry"]
-                                    shapely_geom = shape(geom_dict)
-                                    geometries.append(shapely_geom)
-                                except (KeyError, ValueError) as e:
-                                    self._client.logger.warning(f"Error parsing geometry {i}: {e}")
-                                    continue
-                            
-                            min_length = min(len(datasets), len(geometries))
-                            if min_length == 0:
-                                self._client.logger.warning("No matching datasets and geometries found")
-                                return gpd.GeoDataFrame({'geometry': [], 'dataset': []})
-                            
-                            gdf = gpd.GeoDataFrame({
-                                'geometry': geometries[:min_length],
-                                'dataset': datasets[:min_length]
-                            })
-                            
-                            self._client.logger.info(f"Created GeoDataFrame with {len(gdf)} rows")
-                            
-                            try:
-                                expanded_gdf = expand_on_variables_and_time(gdf)
-                                return expanded_gdf
-                            except NameError:
-                                self._client.logger.warning("expand_on_variables_and_time function not found, returning raw GeoDataFrame")
-                                return gdf
-                            
-                        else:
-                            self._client.logger.warning(f"Failed to download geometry data: HTTP {response.status}")
-                            return gpd.GeoDataFrame({'geometry': [], 'dataset': []})
-                            
-                except Exception as e:
-                    self._client.logger.error(f"Error downloading geometry data: {e}")
-                    return gpd.GeoDataFrame({'geometry': [], 'dataset': []})
-        
-        except Exception as e:
-            self._client.logger.error(f"Failed to load zonal stats for job {job_id}: {e}")
-            return None
+        return await self._client._terrakio_request("POST", "mass_stats/combine_tiles", json=payload)

terrakio_core/sync_client.py

@@ -1,343 +1,3 @@
-# import asyncio
-# import concurrent.futures
-# import threading
-# import functools
-# import inspect
-# from typing import Optional, Dict, Any, Union
-# from geopandas import GeoDataFrame
-# from shapely.geometry.base import BaseGeometry as ShapelyGeometry
-# from .async_client import AsyncClient
-# from typing import TYPE_CHECKING
-
-# # # Add this after your other imports
-# # if TYPE_CHECKING:
-# #     from .endpoints.dataset_management import DatasetManagement
-# #     from .endpoints.user_management import UserManagement
-# #     from .endpoints.mass_stats import MassStats
-# #     from .endpoints.group_management import GroupManagement
-# #     from .endpoints.space_management import SpaceManagement
-# #     from .endpoints.model_management import ModelManagement
-# #     from .endpoints.auth import AuthClient
-
-
-# class SyncWrapper:
-#     """Generic synchronous wrapper with __dir__ support for runtime autocomplete."""
-    
-#     def __init__(self, async_obj, sync_client):
-#         self._async_obj = async_obj
-#         self._sync_client = sync_client
-    
-#     def __dir__(self):
-#         """Return list of attributes for autocomplete in interactive environments."""
-#         async_attrs = [attr for attr in dir(self._async_obj) if not attr.startswith('_')]
-#         wrapper_attrs = [attr for attr in object.__dir__(self) if not attr.startswith('_')]
-#         return list(set(async_attrs + wrapper_attrs))
-    
-#     def __getattr__(self, name):
-#         """Dynamically wrap any method call to convert async to sync."""
-#         attr = getattr(self._async_obj, name)
-        
-#         if callable(attr):
-#             @functools.wraps(attr)
-#             def sync_wrapper(*args, **kwargs):
-#                 result = attr(*args, **kwargs)
-#                 if hasattr(result, '__await__'):
-#                     return self._sync_client._run_async(result)
-#                 return result
-#             return sync_wrapper
-        
-#         return attr
-
-
-# class SyncClient:
-#     """
-#     Thread-safe synchronous wrapper for AsyncClient.
-#     Uses a persistent event loop in a dedicated thread to avoid event loop conflicts.
-#     """
-
-#     # datasets: 'DatasetManagement'
-#     # users: 'UserManagement' 
-#     # mass_stats: 'MassStats'
-#     # groups: 'GroupManagement'
-#     # space: 'SpaceManagement'
-#     # model: 'ModelManagement'
-#     # auth: 'AuthClient'
-    
-#     def __init__(self, url: Optional[str] = None, api_key: Optional[str] = None, verbose: bool = False):
-#         self._async_client = AsyncClient(url=url, api_key=api_key, verbose=verbose)
-#         self._context_entered = False
-#         self._closed = False
-        
-#         # Thread and event loop management
-#         self._loop = None
-#         self._thread = None
-#         self._loop_ready = None
-#         self._loop_exception = None
-        
-#         # Initialize endpoint managers
-#         print("we are here!!!!!!!!!!!!!!!!!")
-#         self.datasets = SyncWrapper(self._async_client.datasets, self)
-#         self.users = SyncWrapper(self._async_client.users, self)
-#         self.mass_stats = SyncWrapper(self._async_client.mass_stats, self)
-#         self.groups = SyncWrapper(self._async_client.groups, self)
-#         self.space = SyncWrapper(self._async_client.space, self)
-#         self.model = SyncWrapper(self._async_client.model, self)
-#         self.auth = SyncWrapper(self._async_client.auth, self)
-        
-#         # Register cleanup
-#         import atexit
-#         atexit.register(self._cleanup)
-    
-#     def _ensure_event_loop(self):
-#         """Ensure we have a persistent event loop in a dedicated thread."""
-#         if self._loop is None or self._loop.is_closed():
-#             self._loop_ready = threading.Event()
-#             self._loop_exception = None
-            
-#             def run_loop():
-#                 """Run the event loop in a dedicated thread."""
-#                 try:
-#                     # Create a new event loop for this thread
-#                     self._loop = asyncio.new_event_loop()
-#                     asyncio.set_event_loop(self._loop)
-                    
-#                     # Signal that the loop is ready
-#                     self._loop_ready.set()
-                    
-#                     # Run the loop forever (until stopped)
-#                     self._loop.run_forever()
-#                 except Exception as e:
-#                     self._loop_exception = e
-#                     self._loop_ready.set()
-#                 finally:
-#                     # Clean up when the loop stops
-#                     if self._loop and not self._loop.is_closed():
-#                         self._loop.close()
-            
-#             # Start the thread
-#             self._thread = threading.Thread(target=run_loop, daemon=True)
-#             self._thread.start()
-            
-#             # Wait for the loop to be ready
-#             self._loop_ready.wait(timeout=10)
-            
-#             if self._loop_exception:
-#                 raise self._loop_exception
-            
-#             if not self._loop_ready.is_set():
-#                 raise RuntimeError("Event loop failed to start within timeout")
-    
-#     def _run_async(self, coro):
-#         """
-#         Run async coroutine using persistent event loop.
-#         This is the core method that makes everything work.
-#         """
-#         # Ensure we have an event loop
-#         self._ensure_event_loop()
-        
-#         if self._loop.is_closed():
-#             raise RuntimeError("Event loop is closed")
-        
-#         # Create a future to get the result back from the event loop thread
-#         future = concurrent.futures.Future()
-        
-#         async def run_with_context():
-#             """Run the coroutine with proper context management."""
-#             try:
-#                 # Ensure the async client is properly initialized
-#                 await self._ensure_context()
-                
-#                 # Run the actual coroutine
-#                 result = await coro
-                
-#                 # Set the result on the future
-#                 future.set_result(result)
-#             except Exception as e:
-#                 # Set the exception on the future
-#                 future.set_exception(e)
-        
-#         # Schedule the coroutine on the persistent event loop
-#         self._loop.call_soon_threadsafe(
-#             lambda: asyncio.create_task(run_with_context())
-#         )
-        
-#         # Wait for the result (with timeout to avoid hanging)
-#         try:
-#             return future.result(timeout=300)  # 5 minute timeout
-#         except concurrent.futures.TimeoutError:
-#             raise RuntimeError("Async operation timed out after 5 minutes")
-    
-#     async def _ensure_context(self):
-#         """Ensure the async client context is entered."""
-#         if not self._context_entered and not self._closed:
-#             await self._async_client.__aenter__()
-#             self._context_entered = True
-    
-#     async def _exit_context(self):
-#         """Exit the async client context."""
-#         if self._context_entered and not self._closed:
-#             await self._async_client.__aexit__(None, None, None)
-#             self._context_entered = False
-    
-#     def close(self):
-#         """Close the underlying async client session and stop the event loop."""
-#         if not self._closed:
-#             if self._loop and not self._loop.is_closed():
-#                 # Schedule cleanup on the event loop
-#                 future = concurrent.futures.Future()
-                
-#                 async def cleanup():
-#                     """Clean up the async client."""
-#                     try:
-#                         await self._exit_context()
-#                         future.set_result(None)
-#                     except Exception as e:
-#                         future.set_exception(e)
-                
-#                 # Run cleanup
-#                 self._loop.call_soon_threadsafe(
-#                     lambda: asyncio.create_task(cleanup())
-#                 )
-                
-#                 # Wait for cleanup to complete
-#                 try:
-#                     future.result(timeout=10)
-#                 except:
-#                     pass  # Ignore cleanup errors
-                
-#                 # Stop the event loop
-#                 self._loop.call_soon_threadsafe(self._loop.stop)
-                
-#                 # Wait for thread to finish
-#                 if self._thread and self._thread.is_alive():
-#                     self._thread.join(timeout=5)
-            
-#             self._closed = True
-    
-#     def _cleanup(self):
-#         """Internal cleanup method called by atexit."""
-#         if not self._closed:
-#             try:
-#                 self.close()
-#             except Exception:
-#                 pass  # Ignore cleanup errors
-    
-#     def __dir__(self):
-#         """Return list of attributes for autocomplete in interactive environments."""
-#         default_attrs = [attr for attr in object.__dir__(self) if not attr.startswith('_')]
-#         async_client_attrs = [attr for attr in dir(self._async_client) if not attr.startswith('_')]
-#         endpoint_attrs = ['datasets', 'users', 'mass_stats', 'groups', 'space', 'model', 'auth']
-#         all_attrs = default_attrs + async_client_attrs + endpoint_attrs
-#         return list(set(all_attrs))
-    
-#     # Your existing methods (geoquery, zonal_stats, etc.)
-#     def geoquery(
-#         self,
-#         expr: str,
-#         feature: Union[Dict[str, Any], ShapelyGeometry],
-#         in_crs: str = "epsg:4326",
-#         out_crs: str = "epsg:4326",
-#         resolution: int = -1,
-#         geom_fix: bool = False,
-#         **kwargs
-#     ):
-#         """Compute WCS query for a single geometry (synchronous version)."""
-#         coro = self._async_client.geoquery(
-#             expr=expr,
-#             feature=feature,
-#             in_crs=in_crs,
-#             out_crs=out_crs,
-#             output="netcdf",
-#             resolution=resolution,
-#             geom_fix=geom_fix,
-#             **kwargs
-#         )
-#         return self._run_async(coro)
-
-#     def zonal_stats(
-#         self,
-#         gdf: GeoDataFrame,
-#         expr: str,
-#         conc: int = 20,
-#         inplace: bool = False,
-#         in_crs: str = "epsg:4326",
-#         out_crs: str = "epsg:4326",
-#         resolution: int = -1,
-#         geom_fix: bool = False,
-#         drop_nan: bool = False,
-#         spatial_reduction: str = None,
-#         temporal_reduction: str = None,
-#         max_memory_mb: int = 500,
-#         stream_to_disk: bool = False,
-#     ):
-#         """Compute zonal statistics for all geometries in a GeoDataFrame (synchronous version)."""
-#         coro = self._async_client.zonal_stats(
-#             gdf=gdf,
-#             expr=expr,
-#             conc=conc,
-#             inplace=inplace,
-#             in_crs=in_crs,
-#             out_crs=out_crs,
-#             resolution=resolution,
-#             geom_fix=geom_fix,
-#             drop_nan=drop_nan,
-#             spatial_reduction=spatial_reduction,
-#             temporal_reduction=temporal_reduction,
-#             max_memory_mb=max_memory_mb,
-#             stream_to_disk=stream_to_disk
-#         )
-#         return self._run_async(coro)
-    
-#     def create_dataset_file(
-#         self,
-#         aoi: str,
-#         expression: str,
-#         output: str,
-#         in_crs: str = "epsg:4326",
-#         res: float = 0.0001,
-#         region: str = "aus",
-#         to_crs: str = "epsg:4326",
-#         overwrite: bool = True,
-#         skip_existing: bool = False,
-#         non_interactive: bool = True,
-#         poll_interval: int = 30,
-#         download_path: str = "/home/user/Downloads",
-#     ) -> dict:
-#         """Create a dataset file using mass stats operations (synchronous version)."""
-#         coro = self._async_client.create_dataset_file(
-#             aoi=aoi,
-#             expression=expression,
-#             output=output,
-#             in_crs=in_crs,
-#             res=res,
-#             region=region,
-#             to_crs=to_crs,
-#             overwrite=overwrite,
-#             skip_existing=skip_existing,
-#             non_interactive=non_interactive,
-#             poll_interval=poll_interval,
-#             download_path=download_path,
-#         )
-#         return self._run_async(coro)
-    
-#     # Context manager support
-#     def __enter__(self):
-#         """Context manager entry."""
-#         return self
-    
-#     def __exit__(self, exc_type, exc_val, exc_tb):
-#         """Context manager exit."""
-#         self.close()
-    
-#     def __del__(self):
-#         """Destructor to ensure session is closed."""
-#         if not self._closed:
-#             try:
-#                 self._cleanup()
-#             except Exception:
-#                 pass
-
-
 import asyncio
 import concurrent.futures
 import threading