terrakio-core 0.3.0__tar.gz → 0.3.1__tar.gz

{terrakio_core-0.3.0 → terrakio_core-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: terrakio-core
-Version: 0.3.0
+Version: 0.3.1
 Summary: Core components for Terrakio API clients
 Author-email: Yupeng Chao <yupeng@haizea.com.au>
 Project-URL: Homepage, https://github.com/HaizeaAnalytics/terrakio-python-api

{terrakio_core-0.3.0 → terrakio_core-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "terrakio-core"
-version = "0.3.0"
+version = "0.3.1"
 authors = [
     {name = "Yupeng Chao", email = "yupeng@haizea.com.au"},
 ]

{terrakio_core-0.3.0 → terrakio_core-0.3.1}/terrakio_core/__init__.py

@@ -4,4 +4,4 @@ Terrakio Core
 Core components for Terrakio API clients.
 """
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"

{terrakio_core-0.3.0 → terrakio_core-0.3.1}/terrakio_core/client.py

@@ -13,6 +13,7 @@ from shapely.geometry import shape, mapping
 from shapely.geometry.base import BaseGeometry as ShapelyGeometry
 from google.cloud import storage
 from .exceptions import APIError, ConfigurationError
+from .decorators import admin_only_params
 import logging
 import textwrap
 
@@ -536,8 +537,8 @@ class BaseClient:
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.close()
 
-    # Mass Stats methods
-    def upload_mass_stats(self, name, size, bucket, output, location=None, **kwargs):
+    @admin_only_params('location', 'force_loc', 'server')
+    def execute_job(self, name, region, output, config, overwrite=False, skip_existing=False, request_json=None, manifest_json=None, location=None, force_loc=None, server="dev-au.terrak.io"):
         if not self.mass_stats:
             from terrakio_core.mass_stats import MassStats
             if not self.url or not self.key:
@@ -548,20 +549,8 @@ class BaseClient:
                 verify=self.verify,
                 timeout=self.timeout
             )
-        return self.mass_stats.upload_request(name, size, bucket, output, location, **kwargs)
+        return self.mass_stats.execute_job(name, region, output, config, overwrite, skip_existing, request_json, manifest_json, location, force_loc, server)
 
-    def start_mass_stats_job(self, task_id):
-        if not self.mass_stats:
-            from terrakio_core.mass_stats import MassStats
-            if not self.url or not self.key:
-                raise ConfigurationError("Mass Stats client not initialized. Make sure API URL and key are set.")
-            self.mass_stats = MassStats(
-                base_url=self.url,
-                api_key=self.key,
-                verify=self.verify,
-                timeout=self.timeout
-            )
-        return self.mass_stats.start_job(task_id)
 
     def get_mass_stats_task_id(self, name, stage, uid=None):
         if not self.mass_stats:
@@ -1123,22 +1112,66 @@ class BaseClient:
         return task_id
 
 
-    def train_model(self, model_name: str, training_data: dict) -> dict:
+    # def train_model(self, model_name: str, training_data: dict) -> dict:
+    #     """
+    #     Train a model using the external model training API.
+
+    #     Args:
+    #         model_name (str): The name of the model to train.
+    #         training_data (dict): Dictionary containing training data parameters.
+
+    #     Returns:
+    #         dict: The response from the model training API.
+    #     """
+    #     endpoint = "https://modeltraining-573248941006.australia-southeast1.run.app/train_model"
+    #     payload = {
+    #         "model_name": model_name,
+    #         "training_data": training_data
+    #     }
+    #     try:
+    #         response = self.session.post(endpoint, json=payload, timeout=self.timeout, verify=self.verify)
+    #         if not response.ok:
+    #             error_msg = f"Model training request failed: {response.status_code} {response.reason}"
+    #             try:
+    #                 error_data = response.json()
+    #                 if "detail" in error_data:
+    #                     error_msg += f" - {error_data['detail']}"
+    #             except Exception:
+    #                 if response.text:
+    #                     error_msg += f" - {response.text}"
+    #             raise APIError(error_msg)
+    #         return response.json()
+    #     except requests.RequestException as e:
+    #         raise APIError(f"Model training request failed: {str(e)}")
+
+
+    def train_model(self, model_name: str, training_dataset: str, task_type: str, model_category: str, architecture: str, region: str, hyperparameters: dict = None) -> dict:
         """
         Train a model using the external model training API.
-
+        
         Args:
             model_name (str): The name of the model to train.
-            training_data (dict): Dictionary containing training data parameters.
-
+            training_dataset (str): The training dataset identifier.
+            task_type (str): The type of ML task (e.g., regression, classification).
+            model_category (str): The category of model (e.g., random_forest).
+            architecture (str): The model architecture.
+            region (str): The region identifier.
+            hyperparameters (dict, optional): Additional hyperparameters for training.
+            
         Returns:
             dict: The response from the model training API.
         """
-        endpoint = "https://modeltraining-573248941006.australia-southeast1.run.app/train_model"
         payload = {
             "model_name": model_name,
-            "training_data": training_data
+            "training_dataset": training_dataset,
+            "task_type": task_type,
+            "model_category": model_category,
+            "architecture": architecture,
+            "region": region,
+            "hyperparameters": hyperparameters
         }
+        endpoint = f"{self.url.rstrip('/')}/train_model"
+        print("the payload is ", payload)
         try:
             response = self.session.post(endpoint, json=payload, timeout=self.timeout, verify=self.verify)
             if not response.ok:
@@ -1155,22 +1188,158 @@ class BaseClient:
         except requests.RequestException as e:
             raise APIError(f"Model training request failed: {str(e)}")
 
-    def deploy_model(self, dataset: str, product:str, model_name:str, input_expression: str, model_training_job_name: str, uid: str, dates_iso8601: list):
-        # we have the dataset and we have the product, and we have the model name, we need to create a new json file and add that to the dataset as our virtual dataset
-        # upload the script to the bucket, the script should be able to download the model and do the inferencing
-        # we need to upload the the json to the to the dataset as our virtual dataset
-        # then we do nothing and wait for the user to make the request call to the explorer
-        # we should have a uniform script for the random forest deployment
-        # create a script for each model
-        # upload script to google bucket, 
-        # 
+    # Mass Stats methods
+    def combine_tiles(self, 
+        data_name: str,
+        usezarr: bool,
+        overwrite: bool,
+        output : str) -> dict:
+
+        if not self.mass_stats:
+            from terrakio_core.mass_stats import MassStats
+            if not self.url or not self.key:
+                raise ConfigurationError("Mass Stats client not initialized. Make sure API URL and key are set.")
+            self.mass_stats = MassStats(
+                base_url=self.url,
+                api_key=self.key,
+                verify=self.verify,
+                timeout=self.timeout
+            )
+        return self.mass_stats.combine_tiles(data_name, usezarr, overwrite, output)
+    
+
+
+    def generate_combine_tiles(
+        self,
+        name: str,
+        aoi: str,
+        expression: str,
+        output: str,
+        tile_size: float = 128.0,
+        crs: str = "epsg:4326",
+        res: float = 0.0001,
+        region: str = "aus",
+        to_crs: str = "epsg:4326",
+        overwrite: bool = True,
+        skip_existing: bool = False,
+        non_interactive: bool = True,
+        usezarr: bool = False,
+        poll_interval: int = 30   # seconds between job status checks
+    ) -> dict:
+        
+        from terrakio_core.generation.tiles import tiles
+        import tempfile
+        import time
+
+        body, reqs, groups = tiles(
+            name = name, 
+            aoi = aoi, 
+            expression = expression,
+            output = output,
+            tile_size = tile_size,
+            crs = crs,
+            res = res,
+            region = region,
+            to_crs = to_crs,
+            fully_cover = True,
+            overwrite = overwrite,
+            skip_existing = skip_existing,
+            non_interactive = non_interactive
+        )
+
+        # Create temp json files before upload
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as tempreq:
+            tempreq.write(reqs)
+            tempreqname = tempreq.name
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as tempmanifest:
+            tempmanifest.write(groups)
+            tempmanifestname = tempmanifest.name
+        
+        if not self.mass_stats:
+            from terrakio_core.mass_stats import MassStats
+            if not self.url or not self.key:
+                raise ConfigurationError("Mass Stats client not initialized. Make sure API URL and key are set.")
+            self.mass_stats = MassStats(
+                base_url=self.url,
+                api_key=self.key,
+                verify=self.verify,
+                timeout=self.timeout
+            )
+
+        task_id = self.mass_stats.execute_job(
+            name=body["name"],
+            region=body["region"],
+            output=body["output"],
+            config = {},
+            overwrite=body["overwrite"],
+            skip_existing=body["skip_existing"],
+            request_json=tempreqname,
+            manifest_json=tempmanifestname,
+        )
+
+        ### Start combining tiles when generation-tiles job is done
+        start_time = time.time()
+        status = None
+        
+        while True:
+            try:
+                taskid = task_id['task_id']
+                trackinfo = self.mass_stats.track_job([taskid])
+                status = trackinfo[taskid]['status']
+                
+                # Check completion states
+                if status == 'Completed':
+                    print('Tiles generated successfully!')
+                    break
+                elif status in ['Failed', 'Cancelled', 'Error']:
+                    raise RuntimeError(f"Job {taskid} failed with status: {status}")
+                else:
+                    # Job is still running
+                    elapsed_time = time.time() - start_time
+                    print(f"Job status: {status} - Elapsed time: {elapsed_time:.1f}s", end='\r')
+                    
+                    # Sleep before next check
+                    time.sleep(poll_interval)
+                    
+                    
+            except KeyboardInterrupt:
+                print(f"\nInterrupted! Job {taskid} is still running in the background.")
+                raise
+            except Exception as e:
+                print(f"\nError tracking job: {e}")
+                raise
+
+        # Clean up temporary files
+        import os
+        os.unlink(tempreqname)
+        os.unlink(tempmanifestname)
+
 
+        # Start combining tiles
+        if not self.mass_stats:
+            from terrakio_core.mass_stats import MassStats
+            if not self.url or not self.key:
+                raise ConfigurationError("Mass Stats client not initialized. Make sure API URL and key are set.")
+            self.mass_stats = MassStats(
+                base_url=self.url,
+                api_key=self.key,
+                verify=self.verify,
+                timeout=self.timeout
+            )
+        
+        return self.mass_stats.combine_tiles(body["name"], usezarr, body["overwrite"], body["output"])
+
+
+
+
+
+
+
+
+
+    def deploy_model(self, dataset: str, product:str, model_name:str, input_expression: str, model_training_job_name: str, uid: str, dates_iso8601: list):
         script_content = self._generate_script(model_name, product, model_training_job_name, uid)
-        # self.create_dataset(collection = "terrakio-datasets", input = input, )
-        # we have the script, we need to upload it to the bucket
         script_name = f"{product}.py"
-        print("the script content is ", script_content)
-        print("the script name is ", script_name)
         self._upload_script_to_bucket(script_content, script_name, model_training_job_name, uid)
         # after uploading the script, we need to create a new virtual dataset
         self._create_dataset(name = dataset, collection = "terrakio-datasets", products = [product], path = f"gs://terrakio-mass-requests/{uid}/{model_training_job_name}/inference_scripts", input = input_expression, dates_iso8601 = dates_iso8601, padding = 0)
@@ -1238,7 +1407,6 @@ class BaseClient:
         client = storage.Client()
         bucket = client.get_bucket('terrakio-mass-requests')
         blob = bucket.blob(f'{uid}/{model_training_job_name}/inference_scripts/{script_name}')
-        # the first layer is the uid, the second layer is the model training job name
         blob.upload_from_string(script_content, content_type='text/plain')
         logging.info(f"Script uploaded successfully to {uid}/{model_training_job_name}/inference_scripts/{script_name}")
 

terrakio_core-0.3.1/terrakio_core/decorators.py

@@ -0,0 +1,18 @@
+# terrakio_core/decorators.py
+def admin_only_params(*restricted_params):
+    """
+    Decorator factory for restricting method parameters to admin users only.
+    """
+    def decorator(func):
+        def wrapper(self, *args, **kwargs):
+            if hasattr(self, '_is_admin') and self._is_admin:
+                return func(self, *args, **kwargs)
+            
+            admin_params_used = set(kwargs.keys()) & set(restricted_params)
+            if admin_params_used:
+                raise PermissionError(f"Parameters {admin_params_used} are only available to admin users")
+            
+            filtered_kwargs = {k: v for k, v in kwargs.items() if k not in restricted_params}
+            return func(self, *args, **filtered_kwargs)
+        return wrapper
+    return decorator

terrakio_core-0.3.1/terrakio_core/generation/tiles.py

@@ -0,0 +1,95 @@
+### implementing generation-tiles in python api
+### function should just generate the json file for mass_stats to pick up.
+
+import geopandas as gpd
+import shapely.geometry
+import json
+from rich import print
+
+def escape_newline(string):
+    if isinstance(string, list):
+        return [s.replace('\\n', '\n') for s in string]
+    else:
+        return string.replace('\\n', '\n')
+
+def get_bounds(aoi, crs, to_crs = None):
+    aoi : gpd.GeoDataFrame = gpd.read_file(aoi)
+    aoi = aoi.set_crs(crs, allow_override=True)
+    if to_crs:
+        aoi = aoi.to_crs(to_crs)
+    bounds = aoi.geometry[0].bounds
+    return *bounds, aoi
+
+def tile_generator(x_min, y_min, x_max, y_max, aoi, crs, res, tile_size, expression, output, fully_cover=True):
+    i_max = int((x_max-x_min)/(tile_size*res))
+    j_max = int((y_max-y_min)/(tile_size*res))
+    if fully_cover:
+        i_max += 1
+        j_max += 1
+    for j in range(0, int(j_max)):
+        for i in range(0, int(i_max)):
+            #print(f"Processing tile {i} {j}")
+            x = x_min + i*(tile_size*res)
+            y = y_max - j*(tile_size*res)
+            bbox = shapely.geometry.box(x, y-(tile_size*res), x + (tile_size*res), y)
+            if not aoi.geometry[0].intersects(bbox):
+                continue
+            feat  = {"type": "Feature", "geometry": bbox.__geo_interface__}
+            data = {
+                "feature": feat,
+                "in_crs": crs,
+                "out_crs": crs,
+                "resolution": res,
+                "expr" : expression,
+                "output" : output,
+            }
+            yield data, i , j
+
+
+def tiles(
+    name: str,
+    aoi : str, 
+    expression: str = "red=S2v2#(year,median).red@(year =2024) \n red",
+    output: str = "netcdf",
+    tile_size : float = 512,
+    crs : str = "epsg:3577",
+    res: float = 10,
+    region : str = "eu",
+    to_crs: str = None,
+    fully_cover: bool = True,
+    overwrite: bool = False,
+    skip_existing: bool = False,
+    non_interactive: bool = False,
+):
+    
+    # Create requests for each tile
+    reqs = []
+    x_min, y_min, x_max, y_max, aoi = get_bounds(aoi, crs, to_crs)
+    #print(f"Bounds: {x_min}, {y_min}, {x_max}, {y_max}")
+
+    if to_crs is None:
+        to_crs = crs
+    for tile_req, i, j in tile_generator(x_min, y_min, x_max, y_max, aoi, to_crs, res, tile_size, expression, output, fully_cover):
+        req_name = f"{name}_{i:02d}_{j:02d}"
+        reqs.append({"group": "tiles", "file": req_name, "request": tile_req})
+
+    #print(f"Generated {len(reqs)} tile requests.")
+
+
+    count = len(reqs)
+    groups = list(set(dic["group"] for dic in reqs))
+
+    body = {
+        "name" : name,
+        "output" : output,
+        "region" : region,
+        "size" : count,
+        "overwrite" : overwrite,
+        "non_interactive": non_interactive,
+        "skip_existing" : skip_existing,
+    }
+    request_json = json.dumps(reqs)
+    manifest_json = json.dumps(groups)
+
+    return body, request_json, manifest_json
+    

terrakio_core-0.3.1/terrakio_core/mass_stats.py

@@ -0,0 +1,496 @@
+import requests
+from typing import Optional, Dict, Any, List
+import json
+import json as json_lib
+import gzip
+
+class MassStats:
+    def __init__(self, base_url: str, api_key: str, verify: bool = True, timeout: int = 60):
+        self.base_url = base_url.rstrip('/')
+        self.api_key = api_key
+        self.verify = verify
+        self.timeout = timeout
+        self.session = requests.Session()
+        self.session.headers.update({
+            'x-api-key': self.api_key
+        })
+
+    def _upload_file(self, file_path: str, url: str, use_gzip: bool = False):
+        """
+        Helper method to upload a JSON file to a signed URL.
+        
+        Args:
+            file_path: Path to the JSON file
+            url: Signed URL to upload to
+            use_gzip: Whether to compress the file with gzip
+        """
+        try:
+            with open(file_path, 'r') as file:
+                json_data = json_lib.load(file)
+        except FileNotFoundError:
+            raise FileNotFoundError(f"JSON file not found: {file_path}")
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON in file {file_path}: {e}")
+        
+        # Check if using simplejson and support ignore_nan
+        if hasattr(json_lib, 'dumps') and 'ignore_nan' in json_lib.dumps.__code__.co_varnames:
+            dumps_kwargs = {'ignore_nan': True}
+        else:
+            dumps_kwargs = {}
+        
+        if use_gzip:
+            # Serialize and compress the JSON data
+            body = gzip.compress(json_lib.dumps(json_data, **dumps_kwargs).encode('utf-8'))
+            headers = {
+                'Content-Type': 'application/json',
+                'Content-Encoding': 'gzip'
+            }
+        else:
+            body = json_lib.dumps(json_data, **dumps_kwargs).encode('utf-8')
+            headers = {
+                'Content-Type': 'application/json'
+            }
+        
+        # Make the PUT request to the signed URL
+        response = requests.put(
+            url,
+            data=body,
+            headers=headers
+        )
+        
+        return response
+    
+
+    # def _download_file(self, url: str, output_path: str) -> str:
+    #     """
+    #     Helper method to download a file from a signed URL.
+        
+    #     Args:
+    #         url: Signed URL to download from
+    #         output_path: Path where the file should be saved
+            
+    #     Returns:
+    #         str: Path to the downloaded file
+    #     """
+
+    #     try:
+    #         response = requests.get(
+    #             url,
+    #             verify=self.verify,
+    #             timeout=self.timeout
+    #         )
+    #         response.raise_for_status()
+            
+    #         # Download and write the file
+    #         with open(output_path, 'wb') as file:
+    #             file.write(response.content)
+    #         print(f"File downloaded successfully to {output_path}")    
+    #         return output_path
+            
+    #     except requests.exceptions.RequestException as e:
+    #         raise Exception(f"Error downloading file from {url}: {e}")
+    #     except IOError as e:
+    #         raise Exception(f"Error writing file to {output_path}: {e}")
+    
+
+
+
+    def upload_request(
+        self,
+        name: str,
+        size: int,
+        region: List[str],
+        output: str,
+        config: Dict[str, Any],
+        location: Optional[str] = None,
+        force_loc: Optional[bool] = None,
+        overwrite: bool = False,
+        server: Optional[str] = None,
+        skip_existing: bool = False,
+    ) -> Dict[str, Any]:
+        """
+        Initiate a mass stats upload job.
+        
+        Args:
+            name: Name of the job
+            size: Size of the job
+            region: Region to run job [aus, eu, us]
+            output: Output type
+            config: Configuration dictionary
+            location: (Optional) Location for the upload
+            force_loc: Force location usage
+            overwrite: Overwrite existing data
+            server: Optional server
+            skip_existing: Skip existing files
+        """
+        
+
+        
+        # Step 2: Create the upload job and get signed URLs
+        url = f"{self.base_url}/mass_stats/upload"
+        
+        data = {
+            "name": name,
+            "size": size,
+            "region": region,
+            "output": output,
+            "config": config,
+            "overwrite": overwrite,
+            "skip_existing": skip_existing
+        }
+        
+        if location is not None:
+            data["location"] = location
+        if force_loc is not None:
+            data["force_loc"] = force_loc
+        if server is not None:
+            data["server"] = server
+        response = self.session.post(
+            url, 
+            json=data, 
+            verify=self.verify, 
+            timeout=self.timeout
+        )      
+        return response.json()
+    
+
+    # def construct_download_url(
+    #     self,
+    #     name: str,
+    #     output: str,
+    #     region: Optional[str] = None,
+    # ) -> Dict[str, Any]:
+    #     """
+    #     Request a signed download URL for a file.
+        
+    #     Args:
+    #         name: job name
+    #         file_type: Type of file to download (e.g., "output", "manifest", "log")
+    #         region: Region where the file is stored
+            
+    #     Returns:
+    #         Dict containing download_url and file metadata
+    #     """
+    #     url = f"{self.base_url}/mass_stats/download"
+        
+    #     data = {
+    #         "name": name,
+    #         "output": output
+    #     }
+        
+    #     if region is not None:
+    #         data["region"] = region
+        
+    #     response = self.session.post(
+    #         url,
+    #         json=data,
+    #         verify=self.verify,
+    #         timeout=self.timeout
+    #     )
+        
+    #     return response.json()
+
+
+    # def testdownload(
+    #     self,
+    #     name: str,
+    #     region: str,
+    #     output: str,
+    # ):
+    #     upload_result = self.construct_download_url(name, region, output)
+    #     return upload_result
+
+
+    
+    def execute_job(
+        self,
+        name: str,
+        region: str,
+        output: str,
+        config: Dict[str, Any],
+        overwrite: bool = False,
+        skip_existing: bool = False,
+        request_json: Optional[str] = None,
+        manifest_json: Optional[str] = None,
+        location: Optional[str] = None,
+        force_loc: Optional[bool] = None,
+        server: Optional[str] = None
+    ) -> Dict[str, Any]:
+        # Step 1: Calculate size from request JSON file if provided
+        size = 0
+        if request_json is not None:
+            try:
+                with open(request_json, 'r') as file:
+                    request_data = json_lib.load(file)
+                
+                if isinstance(request_data, list):
+                    size = len(request_data)
+                else:
+                    raise ValueError(f"Request JSON file {request_json} should contain a list of dictionaries")
+                    
+            except FileNotFoundError:
+                raise FileNotFoundError(f"Request JSON file not found: {request_json}")
+            except json.JSONDecodeError as e:
+                raise ValueError(f"Invalid JSON in request file {request_json}: {e}")
+            
+        upload_result = self.upload_request(name, size, region, output, config, location, force_loc, overwrite, server, skip_existing)
+        
+        # Step 3: Upload JSON files if provided
+        if request_json is not None or manifest_json is not None:
+            requests_url = upload_result.get('requests_url')
+            manifest_url = upload_result.get('manifest_url')
+            
+            if request_json is not None:
+                if not requests_url:
+                    raise ValueError("No requests_url returned from server for request JSON upload")
+                
+                try:
+                    requests_response = self._upload_file(request_json, requests_url, use_gzip=True)
+                    if requests_response.status_code not in [200, 201, 204]:
+                        print(f"Requests upload error: {requests_response.text}")
+                        raise Exception(f"Failed to upload request JSON: {requests_response.text}")
+                except Exception as e:
+                    raise Exception(f"Error uploading request JSON file {request_json}: {e}")
+            
+            if manifest_json is not None:
+                if not manifest_url:
+                    raise ValueError("No manifest_url returned from server for manifest JSON upload")
+                
+                try:
+                    manifest_response = self._upload_file(manifest_json, manifest_url, use_gzip=False)
+                    if manifest_response.status_code not in [200, 201, 204]:
+                        print(f"Manifest upload error: {manifest_response.text}")
+                        raise Exception(f"Failed to upload manifest JSON: {manifest_response.text}")
+                except Exception as e:
+                    raise Exception(f"Error uploading manifest JSON file {manifest_json}: {e}")
+            
+
+        start_job_task_id =self.start_job(upload_result.get("id"))
+        return start_job_task_id
+
+
+    def start_job(self, task_id: str) -> Dict[str, Any]:
+        """
+        Start a mass stats job by task ID.
+        """
+        url = f"{self.base_url}/mass_stats/start/{task_id}"
+        response = self.session.post(url, verify=self.verify, timeout=self.timeout)
+        response.raise_for_status()
+        return response.json()
+
+    def get_task_id(self, name: str, stage: str, uid: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Get the task ID for a mass stats job by name and stage (and optionally user ID).
+        """
+        url = f"{self.base_url}/mass_stats/job_id?name={name}&stage={stage}"
+        if uid is not None:
+            url += f"&uid={uid}"
+        response = self.session.get(url, verify=self.verify, timeout=self.timeout)
+        print("response text is ", response.text)
+        return response.json()
+
+    def track_job(self, ids: Optional[list] = None) -> Dict[str, Any]:
+        """
+        Track the status of one or more mass stats jobs.
+        If ids is None, gets progress for all of the user's jobs.
+        """
+        url = f"{self.base_url}/mass_stats/track"
+        data = {"ids": ids} if ids is not None else {}
+        response = self.session.post(url, json=data, verify=self.verify, timeout=self.timeout)
+        response.raise_for_status()
+        return response.json()
+
+    def get_history(self, limit: int = 100) -> Dict[str, Any]:
+        """
+        Get the history of mass stats jobs.
+        """
+        url = f"{self.base_url}/mass_stats/history"
+        params = {"limit": limit}
+        response = self.session.get(url, params=params, verify=self.verify, timeout=self.timeout)
+        response.raise_for_status()
+        return response.json()
+
+    def start_post_processing(
+        self,
+        process_name: str,
+        data_name: str,
+        output: str,
+        consumer_path: str,
+        overwrite: bool = False
+    ) -> Dict[str, Any]:
+        """
+        Start post processing for a mass stats job.
+        Args:
+            process_name: Folder to store output
+            data_name: Name of job used to create data
+            output: Output type
+            consumer_path: Path to the post processing script (Python file)
+            overwrite: Overwrite existing post processing output in same location
+        Returns:
+            Dict with task_id
+        """
+        url = f"{self.base_url}/mass_stats/post_process"
+        files = {
+            'consumer': (consumer_path, open(consumer_path, 'rb'), 'text/x-python')
+        }
+        data = {
+            'process_name': process_name,
+            'data_name': data_name,
+            'output': output,
+            'overwrite': str(overwrite).lower()
+        }
+        response = self.session.post(url, data=data, files=files, verify=self.verify, timeout=self.timeout)
+        print("the response is ", response.text)
+        # response.raise_for_status()
+        return response.json()
+
+    def download_results(
+        self,
+        id: Optional[str] = None,
+        force_loc: bool = False,
+        bucket: Optional[str] = None,
+        location: Optional[str] = None,
+        output: Optional[str] = None,
+        file_name: Optional[str] = None
+    ) -> bytes:
+        """
+        Download results from a mass stats job or arbitrary results if force_loc is True.
+        Returns the content of the .zip file.
+        """
+        url = f"{self.base_url}/mass_stats/download"
+        data = {}
+        if id is not None:
+            data["id"] = id
+        if force_loc:
+            data["force_loc"] = True
+            if bucket is not None:
+                data["bucket"] = bucket
+            if location is not None:
+                data["location"] = location
+            if output is not None:
+                data["output"] = output
+        if file_name is not None:
+            data["file_name"] = file_name
+        response = self.session.post(url, json=data, verify=self.verify, timeout=self.timeout)
+        print("the response is ", response.text)
+        # response.raise_for_status()
+        print("the response content is ", response.content)
+        return response.content
+
+    def cancel_job(self, id: str) -> Dict[str, Any]:
+        """
+        Cancel a mass stats job by ID.
+        """
+        url = f"{self.base_url}/mass_stats/cancel/{id}"
+        response = self.session.post(url, verify=self.verify, timeout=self.timeout)
+        response.raise_for_status()
+        return response.json()
+
+    def cancel_all_jobs(self) -> Dict[str, Any]:
+        """
+        Cancel all mass stats jobs for the user.
+        """
+        url = f"{self.base_url}/mass_stats/cancel"
+        response = self.session.post(url, verify=self.verify, timeout=self.timeout)
+        response.raise_for_status()
+        return response.json()
+
+    def create_pyramids(self, name: str, levels: int, config: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Create pyramids for a dataset.
+        Args:
+            name: Name for the pyramid job
+            levels: Number of zoom levels to compute
+            config: Dataset config (mapping)
+        Returns:
+            Dict with task_id
+        """
+        url = f"{self.base_url}/pyramids/create"
+        data = {
+            "name": name,
+            "levels": levels,
+            "config": config
+        }
+        response = self.session.post(url, json=data, verify=self.verify, timeout=self.timeout)
+        print("the url is   ", url)
+        print("the response is ", response.text)
+        print("the response status code is ", response.status_code)
+        # response.raise_for_status()
+        return response.json()
+
+    def random_sample(
+        self,
+        name: str,
+        config: dict,
+        aoi: dict,
+        samples: int,
+        year_range: list,
+        crs: str,
+        tile_size: int,
+        res: float,
+        output: str,
+        server: str,
+        region: str,
+        bucket: str,
+        overwrite: bool = False
+    ) -> Dict[str, Any]:
+        """
+        Submit a random sample job.
+        """
+        if year_range is None or len(year_range) != 2:
+            raise ValueError("year_range must be a list of two integers")
+        start_year, end_year = year_range
+        if start_year is None or end_year is None:
+            raise ValueError("Both start_year and end_year must be provided for year_range.")
+
+        url = f"{self.base_url}/random_sample"
+        data = {
+            "name": name,
+            "overwrite": overwrite,
+            "config": config,
+            "aoi": aoi,
+            "samples": samples,
+            "year_range": [start_year, end_year],
+            "crs": crs,
+            "tile_size": tile_size,
+            "res": res,
+            "output": output,
+            "server": server,
+            "region": region,
+            "bucket": bucket
+        }
+        print("the data is ", data)
+        print("the url is ", url)
+        response = self.session.post(url, json=data, verify=self.verify, timeout=self.timeout)
+        print("Status code:", response.status_code)
+        print("Response text:", response.text)
+        # response.raise_for_status()
+        return response.json() 
+
+
+    ### Adding the wrapper function to call endpoint /mass_stats/combine_tiles
+    def combine_tiles(
+        self, 
+        data_name: str,
+        usezarr: bool = False,
+        overwrite: bool = True,
+        output : str = "netcdf"
+    ) -> Dict[str, Any]:
+
+        url = f"{self.base_url}/mass_stats/combine_tiles"
+        request_body = {
+            'data_name': data_name,
+            'usezarr': str(usezarr).lower(),
+            'output': output,
+            'overwrite': str(overwrite).lower()
+        }
+        print(f"Request body: {json.dumps(request_body, indent=2)}")
+        response = self.session.post(url, json=request_body, verify=self.verify, timeout=self.timeout)
+        print(f"Response text: {response.text}")
+        return response.json()
+
+
+
+
+
+    

{terrakio_core-0.3.0 → terrakio_core-0.3.1}/terrakio_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: terrakio-core
-Version: 0.3.0
+Version: 0.3.1
 Summary: Core components for Terrakio API clients
 Author-email: Yupeng Chao <yupeng@haizea.com.au>
 Project-URL: Homepage, https://github.com/HaizeaAnalytics/terrakio-python-api

{terrakio_core-0.3.0 → terrakio_core-0.3.1}/terrakio_core.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ terrakio_core/auth.py
 terrakio_core/client.py
 terrakio_core/config.py
 terrakio_core/dataset_management.py
+terrakio_core/decorators.py
 terrakio_core/exceptions.py
 terrakio_core/group_access_management.py
 terrakio_core/mass_stats.py
@@ -14,4 +15,5 @@ terrakio_core.egg-info/PKG-INFO
 terrakio_core.egg-info/SOURCES.txt
 terrakio_core.egg-info/dependency_links.txt
 terrakio_core.egg-info/requires.txt
-terrakio_core.egg-info/top_level.txt
+terrakio_core.egg-info/top_level.txt
+terrakio_core/generation/tiles.py

terrakio_core-0.3.0/terrakio_core/mass_stats.py

@@ -1,262 +0,0 @@
-import requests
-from typing import Optional, Dict, Any
-
-class MassStats:
-    def __init__(self, base_url: str, api_key: str, verify: bool = True, timeout: int = 60):
-        self.base_url = base_url.rstrip('/')
-        self.api_key = api_key
-        self.verify = verify
-        self.timeout = timeout
-        self.session = requests.Session()
-        self.session.headers.update({
-            'x-api-key': self.api_key
-        })
-
-    def upload_request(
-        self,
-        name: str,
-        size: int,
-        bucket: str,
-        output: str,
-        location: Optional[str] = None,
-        force_loc: bool = False,
-        config: Optional[Dict[str, Any]] = None,
-        overwrite: bool = False,
-        server: Optional[str] = None,
-        skip_existing: bool = False
-    ) -> Dict[str, Any]:
-        """
-        Initiate a mass stats upload job.
-        
-        Args:
-            name: Name of the job
-            size: Size of the data
-            bucket: Storage bucket
-            output: Output path or identifier
-            location: (Optional) Location for the upload
-            force_loc: Force location usage
-            config: Optional configuration dictionary
-            overwrite: Overwrite existing data
-            server: Optional server
-            skip_existing: Skip existing files
-        """
-        url = f"{self.base_url}/mass_stats/upload"
-        data = {
-            "name": name,
-            "size": size,
-            "bucket": bucket,
-            "output": output,
-            "force_loc": force_loc,
-            "overwrite": overwrite,
-            "skip_existing": skip_existing
-        }
-        if location is not None:
-            data["location"] = location
-        if config is not None:
-            data["config"] = config
-        if server is not None:
-            data["server"] = server
-        response = self.session.post(url, json=data, verify=self.verify, timeout=self.timeout)
-        print("the response is ", response.text)
-        # response.raise_for_status()
-        return response.json()
-
-    def start_job(self, task_id: str) -> Dict[str, Any]:
-        """
-        Start a mass stats job by task ID.
-        """
-        url = f"{self.base_url}/mass_stats/start/{task_id}"
-        print("the self session header is ", self.session.headers)
-        response = self.session.post(url, verify=self.verify, timeout=self.timeout)
-        response.raise_for_status()
-        return response.json()
-
-    def get_task_id(self, name: str, stage: str, uid: Optional[str] = None) -> Dict[str, Any]:
-        """
-        Get the task ID for a mass stats job by name and stage (and optionally user ID).
-        """
-        url = f"{self.base_url}/mass_stats/job_id?name={name}&stage={stage}"
-        if uid is not None:
-            url += f"&uid={uid}"
-        response = self.session.get(url, verify=self.verify, timeout=self.timeout)
-        print("response text is ", response.text)
-        return response.json()
-
-    def track_job(self, ids: Optional[list] = None) -> Dict[str, Any]:
-        """
-        Track the status of one or more mass stats jobs.
-        If ids is None, gets progress for all of the user's jobs.
-        """
-        url = f"{self.base_url}/mass_stats/track"
-        data = {"ids": ids} if ids is not None else {}
-        response = self.session.post(url, json=data, verify=self.verify, timeout=self.timeout)
-        response.raise_for_status()
-        return response.json()
-
-    def get_history(self, limit: int = 100) -> Dict[str, Any]:
-        """
-        Get the history of mass stats jobs.
-        """
-        url = f"{self.base_url}/mass_stats/history"
-        params = {"limit": limit}
-        response = self.session.get(url, params=params, verify=self.verify, timeout=self.timeout)
-        response.raise_for_status()
-        return response.json()
-
-    def start_post_processing(
-        self,
-        process_name: str,
-        data_name: str,
-        output: str,
-        consumer_path: str,
-        overwrite: bool = False
-    ) -> Dict[str, Any]:
-        """
-        Start post processing for a mass stats job.
-        Args:
-            process_name: Folder to store output
-            data_name: Name of job used to create data
-            output: Output type
-            consumer_path: Path to the post processing script (Python file)
-            overwrite: Overwrite existing post processing output in same location
-        Returns:
-            Dict with task_id
-        """
-        url = f"{self.base_url}/mass_stats/post_process"
-        files = {
-            'consumer': (consumer_path, open(consumer_path, 'rb'), 'text/x-python')
-        }
-        data = {
-            'process_name': process_name,
-            'data_name': data_name,
-            'output': output,
-            'overwrite': str(overwrite).lower()
-        }
-        response = self.session.post(url, data=data, files=files, verify=self.verify, timeout=self.timeout)
-        print("the response is ", response.text)
-        # response.raise_for_status()
-        return response.json()
-
-    def download_results(
-        self,
-        id: Optional[str] = None,
-        force_loc: bool = False,
-        bucket: Optional[str] = None,
-        location: Optional[str] = None,
-        output: Optional[str] = None,
-        file_name: Optional[str] = None
-    ) -> bytes:
-        """
-        Download results from a mass stats job or arbitrary results if force_loc is True.
-        Returns the content of the .zip file.
-        """
-        url = f"{self.base_url}/mass_stats/download"
-        data = {}
-        if id is not None:
-            data["id"] = id
-        if force_loc:
-            data["force_loc"] = True
-            if bucket is not None:
-                data["bucket"] = bucket
-            if location is not None:
-                data["location"] = location
-            if output is not None:
-                data["output"] = output
-        if file_name is not None:
-            data["file_name"] = file_name
-        response = self.session.post(url, json=data, verify=self.verify, timeout=self.timeout)
-        print("the response is ", response.text)
-        # response.raise_for_status()
-        print("the response content is ", response.content)
-        return response.content
-
-    def cancel_job(self, id: str) -> Dict[str, Any]:
-        """
-        Cancel a mass stats job by ID.
-        """
-        url = f"{self.base_url}/mass_stats/cancel/{id}"
-        response = self.session.post(url, verify=self.verify, timeout=self.timeout)
-        response.raise_for_status()
-        return response.json()
-
-    def cancel_all_jobs(self) -> Dict[str, Any]:
-        """
-        Cancel all mass stats jobs for the user.
-        """
-        url = f"{self.base_url}/mass_stats/cancel"
-        response = self.session.post(url, verify=self.verify, timeout=self.timeout)
-        response.raise_for_status()
-        return response.json()
-
-    def create_pyramids(self, name: str, levels: int, config: Dict[str, Any]) -> Dict[str, Any]:
-        """
-        Create pyramids for a dataset.
-        Args:
-            name: Name for the pyramid job
-            levels: Number of zoom levels to compute
-            config: Dataset config (mapping)
-        Returns:
-            Dict with task_id
-        """
-        url = f"{self.base_url}/pyramids/create"
-        data = {
-            "name": name,
-            "levels": levels,
-            "config": config
-        }
-        response = self.session.post(url, json=data, verify=self.verify, timeout=self.timeout)
-        print("the url is   ", url)
-        print("the response is ", response.text)
-        print("the response status code is ", response.status_code)
-        # response.raise_for_status()
-        return response.json()
-
-    def random_sample(
-        self,
-        name: str,
-        config: dict,
-        aoi: dict,
-        samples: int,
-        year_range: list,
-        crs: str,
-        tile_size: int,
-        res: float,
-        output: str,
-        server: str,
-        region: str,
-        bucket: str,
-        overwrite: bool = False
-    ) -> Dict[str, Any]:
-        """
-        Submit a random sample job.
-        """
-        if year_range is None or len(year_range) != 2:
-            raise ValueError("year_range must be a list of two integers")
-        start_year, end_year = year_range
-        if start_year is None or end_year is None:
-            raise ValueError("Both start_year and end_year must be provided for year_range.")
-
-        url = f"{self.base_url}/random_sample"
-        data = {
-            "name": name,
-            "overwrite": overwrite,
-            "config": config,
-            "aoi": aoi,
-            "samples": samples,
-            "year_range": [start_year, end_year],
-            "crs": crs,
-            "tile_size": tile_size,
-            "res": res,
-            "output": output,
-            "server": server,
-            "region": region,
-            "bucket": bucket
-        }
-        print("the data is ", data)
-        print("the url is ", url)
-        response = self.session.post(url, json=data, verify=self.verify, timeout=self.timeout)
-        print("Status code:", response.status_code)
-        print("Response text:", response.text)
-        # response.raise_for_status()
-        return response.json() 
-