aimodelshare 0.1.32__py3-none-any.whl → 0.1.62__py3-none-any.whl

aimodelshare/moral_compass/config.py

@@ -0,0 +1,187 @@
+"""
+Configuration module for moral_compass API client.
+
+Provides API base URL discovery via:
+1. Environment variable MORAL_COMPASS_API_BASE_URL or AIMODELSHARE_API_BASE_URL
+2. Cached terraform outputs file (infra/terraform_outputs.json)
+3. Terraform command execution (fallback)
+
+Also provides AWS region discovery for region-aware table naming.
+"""
+
+import os
+import json
+import logging
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger("aimodelshare.moral_compass")
+
+
+def get_aws_region() -> Optional[str]:
+    """
+    Discover AWS region from multiple sources.
+    
+    Resolution order:
+    1. AWS_REGION environment variable
+    2. AWS_DEFAULT_REGION environment variable
+    3. Cached terraform outputs file
+    4. None (caller should handle default)
+    
+    Returns:
+        Optional[str]: AWS region name or None
+    """
+    # Strategy 1: Check environment variables
+    region = os.getenv("AWS_REGION") or os.getenv("AWS_DEFAULT_REGION")
+    if region:
+        logger.debug(f"Using AWS region from environment: {region}")
+        return region
+    
+    # Strategy 2: Try cached terraform outputs
+    cached_region = _get_region_from_cached_outputs()
+    if cached_region:
+        logger.debug(f"Using AWS region from cached terraform outputs: {cached_region}")
+        return cached_region
+    
+    # No region found - return None and let caller decide default
+    logger.debug("AWS region not found, caller should use default")
+    return None
+
+
+def get_api_base_url() -> str:
+    """
+    Discover API base URL using multiple strategies in order:
+    1. Environment variables (MORAL_COMPASS_API_BASE_URL or AIMODELSHARE_API_BASE_URL)
+    2. Cached terraform outputs file
+    3. Terraform command execution
+    
+    Returns:
+        str: The API base URL
+        
+    Raises:
+        RuntimeError: If API base URL cannot be determined
+    """
+    # Strategy 1: Check environment variables
+    env_url = os.getenv("MORAL_COMPASS_API_BASE_URL") or os.getenv("AIMODELSHARE_API_BASE_URL")
+    if env_url:
+        logger.debug(f"Using API base URL from environment: {env_url}")
+        return env_url.rstrip("/")
+    
+    # Strategy 2: Try cached terraform outputs
+    cached_url = _get_url_from_cached_outputs()
+    if cached_url:
+        logger.debug(f"Using API base URL from cached terraform outputs: {cached_url}")
+        return cached_url
+    
+    # Strategy 3: Try terraform command (last resort)
+    terraform_url = _get_url_from_terraform_command()
+    if terraform_url:
+        logger.debug(f"Using API base URL from terraform command: {terraform_url}")
+        return terraform_url
+    
+    raise RuntimeError(
+        "Could not determine API base URL. Please set MORAL_COMPASS_API_BASE_URL "
+        "environment variable or ensure terraform outputs are accessible."
+    )
+
+
+def _get_url_from_cached_outputs() -> Optional[str]:
+    """
+    Read API base URL from cached terraform outputs file.
+    
+    Returns:
+        Optional[str]: API base URL if found in cache, None otherwise
+    """
+    # Look for terraform_outputs.json in infra directory
+    repo_root = Path(__file__).parent.parent.parent.parent
+    outputs_file = repo_root / "infra" / "terraform_outputs.json"
+    
+    if not outputs_file.exists():
+        logger.debug(f"Cached terraform outputs not found at {outputs_file}")
+        return None
+    
+    try:
+        with open(outputs_file, "r") as f:
+            outputs = json.load(f)
+        
+        # Handle both formats: {"api_base_url": {"value": "..."}} or {"api_base_url": "..."}
+        api_base_url = outputs.get("api_base_url")
+        if isinstance(api_base_url, dict):
+            url = api_base_url.get("value")
+        else:
+            url = api_base_url
+        
+        if url and url != "null":
+            return url.rstrip("/")
+    except (json.JSONDecodeError, IOError) as e:
+        logger.warning(f"Error reading cached terraform outputs: {e}")
+    
+    return None
+
+
+def _get_region_from_cached_outputs() -> Optional[str]:
+    """
+    Read AWS region from cached terraform outputs file.
+    
+    Returns:
+        Optional[str]: AWS region if found in cache, None otherwise
+    """
+    # Look for terraform_outputs.json in infra directory
+    repo_root = Path(__file__).parent.parent.parent.parent
+    outputs_file = repo_root / "infra" / "terraform_outputs.json"
+    
+    if not outputs_file.exists():
+        logger.debug(f"Cached terraform outputs not found at {outputs_file}")
+        return None
+    
+    try:
+        with open(outputs_file, "r") as f:
+            outputs = json.load(f)
+        
+        # Handle both formats: {"region": {"value": "..."}} or {"region": "..."}
+        region = outputs.get("region") or outputs.get("aws_region")
+        if isinstance(region, dict):
+            region_value = region.get("value")
+        else:
+            region_value = region
+        
+        if region_value and region_value != "null":
+            return region_value
+    except (json.JSONDecodeError, IOError) as e:
+        logger.warning(f"Error reading cached terraform outputs: {e}")
+    
+    return None
+
+
+def _get_url_from_terraform_command() -> Optional[str]:
+    """
+    Execute terraform command to get API base URL.
+    
+    Returns:
+        Optional[str]: API base URL if terraform command succeeds, None otherwise
+    """
+    repo_root = Path(__file__).parent.parent.parent.parent
+    infra_dir = repo_root / "infra"
+    
+    if not infra_dir.exists():
+        logger.debug(f"Infra directory not found at {infra_dir}")
+        return None
+    
+    try:
+        result = subprocess.run(
+            ["terraform", "output", "-raw", "api_base_url"],
+            cwd=infra_dir,
+            capture_output=True,
+            text=True,
+            timeout=10
+        )
+        
+        if result.returncode == 0:
+            url = result.stdout.strip()
+            if url and url != "null":
+                return url.rstrip("/")
+    except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError) as e:
+        logger.debug(f"Terraform command failed: {e}")
+    
+    return None

aimodelshare/playground.py

@@ -1246,13 +1246,19 @@ class ModelPlayground:
             with HiddenPrints():
                 competition = Competition(self.playground_url)
 
-                version_comp, model_page = competition.submit_model(model=model,
-                                                                    prediction_submission=prediction_submission,
-                                                                    preprocessor=preprocessor,
-                                                                    reproducibility_env_filepath=reproducibility_env_filepath,
-                                                                    custom_metadata=custom_metadata,
-                                                                    input_dict=input_dict,
-                                                                    print_output=False)
+                comp_result = competition.submit_model(model=model,
+                                                       prediction_submission=prediction_submission,
+                                                       preprocessor=preprocessor,
+                                                       reproducibility_env_filepath=reproducibility_env_filepath,
+                                                       custom_metadata=custom_metadata,
+                                                       input_dict=input_dict,
+                                                       print_output=False)
+                
+                # Validate return structure before unpacking
+                if not isinstance(comp_result, tuple) or len(comp_result) != 2:
+                    raise RuntimeError(f"Invalid return from competition.submit_model: expected (version, url) tuple, got {type(comp_result)}")
+                
+                version_comp, model_page = comp_result
 
             print(f"Your model has been submitted to competition as model version {version_comp}.")
 
@@ -1260,13 +1266,19 @@ class ModelPlayground:
             with HiddenPrints():
                 experiment = Experiment(self.playground_url)
 
-                version_exp, model_page = experiment.submit_model(model=model,
-                                                                  prediction_submission=prediction_submission,
-                                                                  preprocessor=preprocessor,
-                                                                  reproducibility_env_filepath=reproducibility_env_filepath,
-                                                                  custom_metadata=custom_metadata,
-                                                                  input_dict=input_dict,
-                                                                  print_output=False)
+                exp_result = experiment.submit_model(model=model,
+                                                     prediction_submission=prediction_submission,
+                                                     preprocessor=preprocessor,
+                                                     reproducibility_env_filepath=reproducibility_env_filepath,
+                                                     custom_metadata=custom_metadata,
+                                                     input_dict=input_dict,
+                                                     print_output=False)
+                
+                # Validate return structure before unpacking
+                if not isinstance(exp_result, tuple) or len(exp_result) != 2:
+                    raise RuntimeError(f"Invalid return from experiment.submit_model: expected (version, url) tuple, got {type(exp_result)}")
+                
+                version_exp, model_page = exp_result
 
             print(f"Your model has been submitted to experiment as model version {version_exp}.")
 

aimodelshare/preprocessormodules.py

@@ -116,6 +116,26 @@ def import_preprocessor(filepath):
 
     return preprocessor
 
+def _test_object_serialization(obj, obj_name):
+    """
+    Test if an object can be serialized with pickle.
+    
+    Args:
+        obj: Object to test
+        obj_name: Name of the object for error reporting
+        
+    Returns:
+        tuple: (success: bool, error_msg: str or None)
+    """
+    import pickle
+    
+    try:
+        pickle.dumps(obj)
+        return True, None
+    except Exception as e:
+        return False, f"{type(e).__name__}: {str(e)}"
+
+
 def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
     """
     Exports preprocessor and related objects into zip file for model deployment
@@ -167,7 +187,7 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
         function_objects=list(inspect.getclosurevars(preprocessor_fxn).globals.keys())
         
         import sys
-        import imp
+        import importlib.util
         modulenames = ["sklearn","keras","tensorflow","cv2","resize","pytorch","librosa","pyspark"]
 
         # List all standard libraries not covered by sys.builtin_module_names
@@ -185,9 +205,12 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
                     modulenames.append(module_name)
                     continue
 
-                module_path = imp.find_module(module_name)[1]
-                if os.path.dirname(module_path) in stdlib:
-                    modulenames.append(module_name)
+                # Use importlib.util instead of deprecated imp
+                spec = importlib.util.find_spec(module_name)
+                if spec and spec.origin:
+                    module_path = spec.origin
+                    if os.path.dirname(module_path) in stdlib:
+                        modulenames.append(module_name)
             except Exception as e:
                 # print(e)
                 continue
@@ -232,12 +255,19 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
 
         export_methods = []
         savedpreprocessorobjectslist = []
+        failed_objects = []  # Track failed serializations for better diagnostics
+        
         for function_objects_nomodule in function_objects_nomodules:
             try:
                 savedpreprocessorobjectslist.append(savetopickle(function_objects_nomodule))
                 export_methods.append("pickle")
             except Exception as e:
-                # print(e)
+                # Track this failure for diagnostics
+                can_serialize, error_msg = _test_object_serialization(
+                    globals().get(function_objects_nomodule), 
+                    function_objects_nomodule
+                )
+                
                 try:
                     os.remove(os.path.join(temp_dir, function_objects_nomodule+".pkl"))
                 except:
@@ -246,7 +276,14 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
                 try:
                     savedpreprocessorobjectslist.append(save_to_zip(function_objects_nomodule))
                     export_methods.append("zip")
-                except Exception as e:
+                except Exception as zip_e:
+                    # Both pickle and zip failed - record this
+                    failed_objects.append({
+                        'name': function_objects_nomodule,
+                        'type': type(globals().get(function_objects_nomodule, None)).__name__,
+                        'pickle_error': str(e),
+                        'zip_error': str(zip_e)
+                    })
                     # print(e)
                     pass
         
@@ -265,6 +302,20 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
         # close the Zip File
         zipObj.close()
 
+        # If any critical objects failed to serialize, raise an error with details
+        if failed_objects:
+            failed_names = [obj['name'] for obj in failed_objects]
+            error_details = "\n".join([
+                f"  - {obj['name']} (type: {obj['type']}): {obj['pickle_error'][:100]}"
+                for obj in failed_objects
+            ])
+            raise RuntimeError(
+                f"Preprocessor export encountered serialization failures for {len(failed_objects)} closure variable(s): "
+                f"{', '.join(failed_names)}.\n\nDetails:\n{error_details}\n\n"
+                f"These objects are referenced by your preprocessor function but cannot be serialized. "
+                f"Common causes include open file handles, database connections, or thread locks."
+            )
+
         try:
             # clean up temp directory files for future runs
             os.remove(os.path.join(temp_dir,"preprocessor.py"))
@@ -279,6 +330,9 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
             pass
 
     except Exception as e:
+        # Re-raise RuntimeError with preserved message
+        if isinstance(e, RuntimeError):
+            raise
         print(e)
 
     return print("Your preprocessor is now saved to 'preprocessor.zip'")