aimodelshare 0.1.21__py3-none-any.whl → 0.1.62__py3-none-any.whl

aimodelshare/moral_compass/config.py

@@ -0,0 +1,187 @@
+"""
+Configuration module for moral_compass API client.
+
+Provides API base URL discovery via:
+1. Environment variable MORAL_COMPASS_API_BASE_URL or AIMODELSHARE_API_BASE_URL
+2. Cached terraform outputs file (infra/terraform_outputs.json)
+3. Terraform command execution (fallback)
+
+Also provides AWS region discovery for region-aware table naming.
+"""
+
+import os
+import json
+import logging
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger("aimodelshare.moral_compass")
+
+
+def get_aws_region() -> Optional[str]:
+    """
+    Discover AWS region from multiple sources.
+    
+    Resolution order:
+    1. AWS_REGION environment variable
+    2. AWS_DEFAULT_REGION environment variable
+    3. Cached terraform outputs file
+    4. None (caller should handle default)
+    
+    Returns:
+        Optional[str]: AWS region name or None
+    """
+    # Strategy 1: Check environment variables
+    region = os.getenv("AWS_REGION") or os.getenv("AWS_DEFAULT_REGION")
+    if region:
+        logger.debug(f"Using AWS region from environment: {region}")
+        return region
+    
+    # Strategy 2: Try cached terraform outputs
+    cached_region = _get_region_from_cached_outputs()
+    if cached_region:
+        logger.debug(f"Using AWS region from cached terraform outputs: {cached_region}")
+        return cached_region
+    
+    # No region found - return None and let caller decide default
+    logger.debug("AWS region not found, caller should use default")
+    return None
+
+
+def get_api_base_url() -> str:
+    """
+    Discover API base URL using multiple strategies in order:
+    1. Environment variables (MORAL_COMPASS_API_BASE_URL or AIMODELSHARE_API_BASE_URL)
+    2. Cached terraform outputs file
+    3. Terraform command execution
+    
+    Returns:
+        str: The API base URL
+        
+    Raises:
+        RuntimeError: If API base URL cannot be determined
+    """
+    # Strategy 1: Check environment variables
+    env_url = os.getenv("MORAL_COMPASS_API_BASE_URL") or os.getenv("AIMODELSHARE_API_BASE_URL")
+    if env_url:
+        logger.debug(f"Using API base URL from environment: {env_url}")
+        return env_url.rstrip("/")
+    
+    # Strategy 2: Try cached terraform outputs
+    cached_url = _get_url_from_cached_outputs()
+    if cached_url:
+        logger.debug(f"Using API base URL from cached terraform outputs: {cached_url}")
+        return cached_url
+    
+    # Strategy 3: Try terraform command (last resort)
+    terraform_url = _get_url_from_terraform_command()
+    if terraform_url:
+        logger.debug(f"Using API base URL from terraform command: {terraform_url}")
+        return terraform_url
+    
+    raise RuntimeError(
+        "Could not determine API base URL. Please set MORAL_COMPASS_API_BASE_URL "
+        "environment variable or ensure terraform outputs are accessible."
+    )
+
+
+def _get_url_from_cached_outputs() -> Optional[str]:
+    """
+    Read API base URL from cached terraform outputs file.
+    
+    Returns:
+        Optional[str]: API base URL if found in cache, None otherwise
+    """
+    # Look for terraform_outputs.json in infra directory
+    repo_root = Path(__file__).parent.parent.parent.parent
+    outputs_file = repo_root / "infra" / "terraform_outputs.json"
+    
+    if not outputs_file.exists():
+        logger.debug(f"Cached terraform outputs not found at {outputs_file}")
+        return None
+    
+    try:
+        with open(outputs_file, "r") as f:
+            outputs = json.load(f)
+        
+        # Handle both formats: {"api_base_url": {"value": "..."}} or {"api_base_url": "..."}
+        api_base_url = outputs.get("api_base_url")
+        if isinstance(api_base_url, dict):
+            url = api_base_url.get("value")
+        else:
+            url = api_base_url
+        
+        if url and url != "null":
+            return url.rstrip("/")
+    except (json.JSONDecodeError, IOError) as e:
+        logger.warning(f"Error reading cached terraform outputs: {e}")
+    
+    return None
+
+
+def _get_region_from_cached_outputs() -> Optional[str]:
+    """
+    Read AWS region from cached terraform outputs file.
+    
+    Returns:
+        Optional[str]: AWS region if found in cache, None otherwise
+    """
+    # Look for terraform_outputs.json in infra directory
+    repo_root = Path(__file__).parent.parent.parent.parent
+    outputs_file = repo_root / "infra" / "terraform_outputs.json"
+    
+    if not outputs_file.exists():
+        logger.debug(f"Cached terraform outputs not found at {outputs_file}")
+        return None
+    
+    try:
+        with open(outputs_file, "r") as f:
+            outputs = json.load(f)
+        
+        # Handle both formats: {"region": {"value": "..."}} or {"region": "..."}
+        region = outputs.get("region") or outputs.get("aws_region")
+        if isinstance(region, dict):
+            region_value = region.get("value")
+        else:
+            region_value = region
+        
+        if region_value and region_value != "null":
+            return region_value
+    except (json.JSONDecodeError, IOError) as e:
+        logger.warning(f"Error reading cached terraform outputs: {e}")
+    
+    return None
+
+
+def _get_url_from_terraform_command() -> Optional[str]:
+    """
+    Execute terraform command to get API base URL.
+    
+    Returns:
+        Optional[str]: API base URL if terraform command succeeds, None otherwise
+    """
+    repo_root = Path(__file__).parent.parent.parent.parent
+    infra_dir = repo_root / "infra"
+    
+    if not infra_dir.exists():
+        logger.debug(f"Infra directory not found at {infra_dir}")
+        return None
+    
+    try:
+        result = subprocess.run(
+            ["terraform", "output", "-raw", "api_base_url"],
+            cwd=infra_dir,
+            capture_output=True,
+            text=True,
+            timeout=10
+        )
+        
+        if result.returncode == 0:
+            url = result.stdout.strip()
+            if url and url != "null":
+                return url.rstrip("/")
+    except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError) as e:
+        logger.debug(f"Terraform command failed: {e}")
+    
+    return None

aimodelshare/playground.py

@@ -1246,13 +1246,19 @@ class ModelPlayground:
             with HiddenPrints():
                 competition = Competition(self.playground_url)
 
-                version_comp, model_page = competition.submit_model(model=model,
-                                                                    prediction_submission=prediction_submission,
-                                                                    preprocessor=preprocessor,
-                                                                    reproducibility_env_filepath=reproducibility_env_filepath,
-                                                                    custom_metadata=custom_metadata,
-                                                                    input_dict=input_dict,
-                                                                    print_output=False)
+                comp_result = competition.submit_model(model=model,
+                                                       prediction_submission=prediction_submission,
+                                                       preprocessor=preprocessor,
+                                                       reproducibility_env_filepath=reproducibility_env_filepath,
+                                                       custom_metadata=custom_metadata,
+                                                       input_dict=input_dict,
+                                                       print_output=False)
+                
+                # Validate return structure before unpacking
+                if not isinstance(comp_result, tuple) or len(comp_result) != 2:
+                    raise RuntimeError(f"Invalid return from competition.submit_model: expected (version, url) tuple, got {type(comp_result)}")
+                
+                version_comp, model_page = comp_result
 
             print(f"Your model has been submitted to competition as model version {version_comp}.")
 
@@ -1260,13 +1266,19 @@ class ModelPlayground:
             with HiddenPrints():
                 experiment = Experiment(self.playground_url)
 
-                version_exp, model_page = experiment.submit_model(model=model,
-                                                                  prediction_submission=prediction_submission,
-                                                                  preprocessor=preprocessor,
-                                                                  reproducibility_env_filepath=reproducibility_env_filepath,
-                                                                  custom_metadata=custom_metadata,
-                                                                  input_dict=input_dict,
-                                                                  print_output=False)
+                exp_result = experiment.submit_model(model=model,
+                                                     prediction_submission=prediction_submission,
+                                                     preprocessor=preprocessor,
+                                                     reproducibility_env_filepath=reproducibility_env_filepath,
+                                                     custom_metadata=custom_metadata,
+                                                     input_dict=input_dict,
+                                                     print_output=False)
+                
+                # Validate return structure before unpacking
+                if not isinstance(exp_result, tuple) or len(exp_result) != 2:
+                    raise RuntimeError(f"Invalid return from experiment.submit_model: expected (version, url) tuple, got {type(exp_result)}")
+                
+                version_exp, model_page = exp_result
 
             print(f"Your model has been submitted to experiment as model version {version_exp}.")
 

aimodelshare/preprocessormodules.py

@@ -116,6 +116,26 @@ def import_preprocessor(filepath):
 
     return preprocessor
 
+def _test_object_serialization(obj, obj_name):
+    """
+    Test if an object can be serialized with pickle.
+    
+    Args:
+        obj: Object to test
+        obj_name: Name of the object for error reporting
+        
+    Returns:
+        tuple: (success: bool, error_msg: str or None)
+    """
+    import pickle
+    
+    try:
+        pickle.dumps(obj)
+        return True, None
+    except Exception as e:
+        return False, f"{type(e).__name__}: {str(e)}"
+
+
 def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
     """
     Exports preprocessor and related objects into zip file for model deployment
@@ -167,7 +187,7 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
         function_objects=list(inspect.getclosurevars(preprocessor_fxn).globals.keys())
         
         import sys
-        import imp
+        import importlib.util
         modulenames = ["sklearn","keras","tensorflow","cv2","resize","pytorch","librosa","pyspark"]
 
         # List all standard libraries not covered by sys.builtin_module_names
@@ -185,9 +205,12 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
                     modulenames.append(module_name)
                     continue
 
-                module_path = imp.find_module(module_name)[1]
-                if os.path.dirname(module_path) in stdlib:
-                    modulenames.append(module_name)
+                # Use importlib.util instead of deprecated imp
+                spec = importlib.util.find_spec(module_name)
+                if spec and spec.origin:
+                    module_path = spec.origin
+                    if os.path.dirname(module_path) in stdlib:
+                        modulenames.append(module_name)
             except Exception as e:
                 # print(e)
                 continue
@@ -232,12 +255,19 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
 
         export_methods = []
         savedpreprocessorobjectslist = []
+        failed_objects = []  # Track failed serializations for better diagnostics
+        
         for function_objects_nomodule in function_objects_nomodules:
             try:
                 savedpreprocessorobjectslist.append(savetopickle(function_objects_nomodule))
                 export_methods.append("pickle")
             except Exception as e:
-                # print(e)
+                # Track this failure for diagnostics
+                can_serialize, error_msg = _test_object_serialization(
+                    globals().get(function_objects_nomodule), 
+                    function_objects_nomodule
+                )
+                
                 try:
                     os.remove(os.path.join(temp_dir, function_objects_nomodule+".pkl"))
                 except:
@@ -246,7 +276,14 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
                 try:
                     savedpreprocessorobjectslist.append(save_to_zip(function_objects_nomodule))
                     export_methods.append("zip")
-                except Exception as e:
+                except Exception as zip_e:
+                    # Both pickle and zip failed - record this
+                    failed_objects.append({
+                        'name': function_objects_nomodule,
+                        'type': type(globals().get(function_objects_nomodule, None)).__name__,
+                        'pickle_error': str(e),
+                        'zip_error': str(zip_e)
+                    })
                     # print(e)
                     pass
         
@@ -265,6 +302,20 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
         # close the Zip File
         zipObj.close()
 
+        # If any critical objects failed to serialize, raise an error with details
+        if failed_objects:
+            failed_names = [obj['name'] for obj in failed_objects]
+            error_details = "\n".join([
+                f"  - {obj['name']} (type: {obj['type']}): {obj['pickle_error'][:100]}"
+                for obj in failed_objects
+            ])
+            raise RuntimeError(
+                f"Preprocessor export encountered serialization failures for {len(failed_objects)} closure variable(s): "
+                f"{', '.join(failed_names)}.\n\nDetails:\n{error_details}\n\n"
+                f"These objects are referenced by your preprocessor function but cannot be serialized. "
+                f"Common causes include open file handles, database connections, or thread locks."
+            )
+
         try:
             # clean up temp directory files for future runs
             os.remove(os.path.join(temp_dir,"preprocessor.py"))
@@ -279,6 +330,9 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
             pass
 
     except Exception as e:
+        # Re-raise RuntimeError with preserved message
+        if isinstance(e, RuntimeError):
+            raise
         print(e)
 
     return print("Your preprocessor is now saved to 'preprocessor.zip'")

aimodelshare/reproducibility.py

@@ -3,11 +3,22 @@ import sys
 import json
 import random
 import tempfile
-import pkg_resources
 import requests
 
 import numpy as np
-import tensorflow as tf
+
+# TensorFlow is optional - only needed for reproducibility setup with TF models
+try:
+    import tensorflow as tf
+    _TF_AVAILABLE = True
+except ImportError:
+    _TF_AVAILABLE = False
+    tf = None
+
+try:
+    import importlib.metadata as md
+except ImportError:  # pragma: no cover
+    import importlib_metadata as md
 
 from aimodelshare.aws import get_s3_iam_client, run_function_on_lambda, get_aws_client
 
@@ -44,9 +55,13 @@ def export_reproducibility_env(seed, directory, mode="gpu"):
   else:
     raise Exception("Error: unknown 'mode' value, expected 'gpu' or 'cpu'")
 
-  installed_packages = pkg_resources.working_set
-  installed_packages_list = sorted(["%s==%s" % (i.key, i.version)
-    for i in installed_packages])
+  # Get installed packages using importlib.metadata
+  installed_packages_list = []
+  for dist in md.distributions():
+    name = dist.metadata.get("Name") or "unknown"
+    version = dist.version
+    installed_packages_list.append(f"{name}=={version}")
+  installed_packages_list = sorted(installed_packages_list)
 
   data["session_runtime_info"] = {
     "installed_packages": installed_packages_list,

aimodelshare/utils/__init__.py

@@ -0,0 +1,78 @@
+"""Utility modules for aimodelshare."""
+import os
+import sys
+import shutil
+import tempfile
+import functools
+import warnings
+from typing import Type
+
+from .optional_deps import check_optional
+
+
+def delete_files_from_temp_dir(temp_dir_file_deletion_list):
+    temp_dir = tempfile.gettempdir()
+    for file_name in temp_dir_file_deletion_list:
+        file_path = os.path.join(temp_dir, file_name)
+        if os.path.exists(file_path):
+            os.remove(file_path)
+
+
+def delete_folder(folder_path):
+    if os.path.exists(folder_path):
+        shutil.rmtree(folder_path)
+
+
+def make_folder(folder_path):
+    os.makedirs(folder_path, exist_ok=True)
+
+
+class HiddenPrints:
+    """Context manager that suppresses stdout and stderr (used for silencing noisy outputs)."""
+    def __enter__(self):
+        self._original_stdout = sys.stdout
+        self._original_stderr = sys.stderr
+        self._devnull_stdout = open(os.devnull, 'w')
+        self._devnull_stderr = open(os.devnull, 'w')
+        sys.stdout = self._devnull_stdout
+        sys.stderr = self._devnull_stderr
+        return self
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        sys.stdout = self._original_stdout
+        sys.stderr = self._original_stderr
+        self._devnull_stdout.close()
+        self._devnull_stderr.close()
+
+
+def ignore_warning(warning: Type[Warning]):
+    """
+    Ignore a given warning occurring during method execution.
+
+    Args:
+        warning (Warning): warning type to ignore.
+
+    Returns:
+        the inner function
+    """
+
+    def inner(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            with warnings.catch_warnings():
+                warnings.filterwarnings("ignore", category=warning)
+                return func(*args, **kwargs)
+
+        return wrapper
+
+    return inner
+
+
+__all__ = [
+    "check_optional",
+    "HiddenPrints",
+    "ignore_warning",
+    "delete_files_from_temp_dir",
+    "delete_folder",
+    "make_folder",
+]

aimodelshare/utils/optional_deps.py

@@ -0,0 +1,38 @@
+"""Optional dependency checking utilities."""
+import os
+import importlib.util
+import warnings
+
+_DEF_SUPPRESS_ENV = "AIMODELSHARE_SUPPRESS_OPTIONAL_WARNINGS"
+
+
+def check_optional(name: str, feature_label: str, suppress_env: str = _DEF_SUPPRESS_ENV) -> bool:
+    """Check if an optional dependency is available.
+    
+    Print a single warning (via warnings) if missing and suppression env var is not set.
+    Returns True if available, False otherwise.
+    
+    Parameters
+    ----------
+    name : str
+        The name of the module to check (e.g., 'xgboost', 'pyspark')
+    feature_label : str
+        A human-readable label for the feature that requires this dependency
+    suppress_env : str, optional
+        Environment variable name to check for suppression (default: AIMODELSHARE_SUPPRESS_OPTIONAL_WARNINGS)
+    
+    Returns
+    -------
+    bool
+        True if the module is available, False otherwise
+    """
+    spec = importlib.util.find_spec(name)
+    if spec is None:
+        if not os.environ.get(suppress_env):
+            warnings.warn(
+                f"{feature_label} support unavailable. Install `{name}` to enable.",
+                category=UserWarning,
+                stacklevel=2,
+            )
+        return False
+    return True