aimodelshare 0.1.54__py3-none-any.whl → 0.1.59__py3-none-any.whl

aimodelshare/preprocessormodules.py

@@ -116,6 +116,26 @@ def import_preprocessor(filepath):
 
     return preprocessor
 
+def _test_object_serialization(obj, obj_name):
+    """
+    Test if an object can be serialized with pickle.
+    
+    Args:
+        obj: Object to test
+        obj_name: Name of the object for error reporting
+        
+    Returns:
+        tuple: (success: bool, error_msg: str or None)
+    """
+    import pickle
+    
+    try:
+        pickle.dumps(obj)
+        return True, None
+    except Exception as e:
+        return False, f"{type(e).__name__}: {str(e)}"
+
+
 def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
     """
     Exports preprocessor and related objects into zip file for model deployment
@@ -167,7 +187,7 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
         function_objects=list(inspect.getclosurevars(preprocessor_fxn).globals.keys())
         
         import sys
-        import imp
+        import importlib.util
         modulenames = ["sklearn","keras","tensorflow","cv2","resize","pytorch","librosa","pyspark"]
 
         # List all standard libraries not covered by sys.builtin_module_names
@@ -185,9 +205,12 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
                     modulenames.append(module_name)
                     continue
 
-                module_path = imp.find_module(module_name)[1]
-                if os.path.dirname(module_path) in stdlib:
-                    modulenames.append(module_name)
+                # Use importlib.util instead of deprecated imp
+                spec = importlib.util.find_spec(module_name)
+                if spec and spec.origin:
+                    module_path = spec.origin
+                    if os.path.dirname(module_path) in stdlib:
+                        modulenames.append(module_name)
             except Exception as e:
                 # print(e)
                 continue
@@ -232,12 +255,19 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
 
         export_methods = []
         savedpreprocessorobjectslist = []
+        failed_objects = []  # Track failed serializations for better diagnostics
+        
         for function_objects_nomodule in function_objects_nomodules:
             try:
                 savedpreprocessorobjectslist.append(savetopickle(function_objects_nomodule))
                 export_methods.append("pickle")
             except Exception as e:
-                # print(e)
+                # Track this failure for diagnostics
+                can_serialize, error_msg = _test_object_serialization(
+                    globals().get(function_objects_nomodule), 
+                    function_objects_nomodule
+                )
+                
                 try:
                     os.remove(os.path.join(temp_dir, function_objects_nomodule+".pkl"))
                 except:
@@ -246,7 +276,14 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
                 try:
                     savedpreprocessorobjectslist.append(save_to_zip(function_objects_nomodule))
                     export_methods.append("zip")
-                except Exception as e:
+                except Exception as zip_e:
+                    # Both pickle and zip failed - record this
+                    failed_objects.append({
+                        'name': function_objects_nomodule,
+                        'type': type(globals().get(function_objects_nomodule, None)).__name__,
+                        'pickle_error': str(e),
+                        'zip_error': str(zip_e)
+                    })
                     # print(e)
                     pass
         
@@ -265,6 +302,20 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
         # close the Zip File
         zipObj.close()
 
+        # If any critical objects failed to serialize, raise an error with details
+        if failed_objects:
+            failed_names = [obj['name'] for obj in failed_objects]
+            error_details = "\n".join([
+                f"  - {obj['name']} (type: {obj['type']}): {obj['pickle_error'][:100]}"
+                for obj in failed_objects
+            ])
+            raise RuntimeError(
+                f"Preprocessor export encountered serialization failures for {len(failed_objects)} closure variable(s): "
+                f"{', '.join(failed_names)}.\n\nDetails:\n{error_details}\n\n"
+                f"These objects are referenced by your preprocessor function but cannot be serialized. "
+                f"Common causes include open file handles, database connections, or thread locks."
+            )
+
         try:
             # clean up temp directory files for future runs
             os.remove(os.path.join(temp_dir,"preprocessor.py"))
@@ -279,6 +330,9 @@ def export_preprocessor(preprocessor_fxn,directory, globs=globals()):
             pass
 
     except Exception as e:
+        # Re-raise RuntimeError with preserved message
+        if isinstance(e, RuntimeError):
+            raise
         print(e)
 
     return print("Your preprocessor is now saved to 'preprocessor.zip'")

aimodelshare/pyspark/authorization.txt

@@ -1,258 +1,258 @@
-from __future__ import print_function
-
-import re
-import json
-
-
-def handler(event, context):
-    print("Client token: " + event['authorizationToken'])
-    print("Method ARN: " + event['methodArn'])
-
-    '''
-    Validate the incoming token and produce the principal user identifier
-    associated with the token. This can be accomplished in a number of ways:
-
-    1. Call out to the OAuth provider
-    2. Decode a JWT token inline
-    3. Lookup in a self-managed DB
-    '''
-    idtoken=event['authorizationToken']
-    try:
-        tokenbodydata=json.loads(idtoken)
-        idtoken=tokenbodydata['token']
-        evaltrue=tokenbodydata['eval']
-    except:
-        evaltrue="FALSE"
-        
-    print(event)
-    principalId = idtoken
-    import jwt
-    decoded = jwt.decode(idtoken, options={"verify_signature": False})  # works in PyJWT < v2.0
-    
-    # Check token access with live rest api (with cognito auth activated) to ensure user isn't sending incorrect username
-    # 200 status indicates accept, 403 means unauthorized
-
-
-    #==============AIMODELSHARE Note: accept / deny languange located on line 67=======================================================
-
-    '''
-    You can send a 401 Unauthorized response to the client by failing like so:
-
-      raise Exception('Unauthorized')
-
-    If the token is valid, a policy must be generated which will allow or deny
-    access to the client. If access is denied, the client will receive a 403
-    Access Denied response. If access is allowed, API Gateway will proceed with
-    the backend integration configured on the method that was called.
-
-    This function must generate a policy that is associated with the recognized
-    principal user identifier. Depending on your use case, you might store
-    policies in a DB, or generate them on the fly.
-
-    Keep in mind, the policy is cached for 5 minutes by default (TTL is
-    configurable in the authorizer) and will apply to subsequent calls to any
-    method/resource in the RestApi made with the same token.
-
-    The example policy below denies access to all resources in the RestApi.
-    '''
-    tmp = event['methodArn'].split(':')
-    apiGatewayArnTmp = tmp[5].split('/')
-    awsAccountId = tmp[4]
-
-    policy = AuthPolicy(principalId, awsAccountId)
-    policy.restApiId = apiGatewayArnTmp[0]
-    policy.region = tmp[3]
-    policy.stage = apiGatewayArnTmp[1]
-    
-    
-    #====================Accept/Deny auth for AIMODELSHARE begins here===========================================
-    import requests
-    
-
-    headers_with_authentication = {'Content-Type': 'application/json', 'Authorization': idtoken}
-    
-    response=requests.post("https://1b861v14ne.execute-api.us-east-1.amazonaws.com/dev/redisusagetrackerembedded",
-                         json={"apiid": str(policy.restApiId),"eval":evaltrue}, headers=headers_with_authentication)
-    print(response.status_code)
-    uniquemodversion=str(response.text)
-
-    print(response.text)
-    if any([response.status_code!=200,response.text.find("Missing")>0,response.text.find("Monthly")>0,response.text.find("monthly")>0]):
-        policyresult="deny"
-    else:
-        policyresult="accept"
-
-    if policyresult=="deny":
-        policy.denyAllMethods()
-    else:
-        policy.allowAllMethods()
-
-    # Finally, build the policy
-    authResponse = policy.build()
-    authResponse['context']['uniquemodversion']=uniquemodversion
-
-    # new! -- add additional key-value pairs associated with the authenticated principal
-    # these are made available by APIGW like so: $context.authorizer.<key>
-    # additional context is cached
-    
-
-    
-    return authResponse
-
-
-class HttpVerb:
-    GET = 'GET'
-    POST = 'POST'
-    PUT = 'PUT'
-    PATCH = 'PATCH'
-    HEAD = 'HEAD'
-    DELETE = 'DELETE'
-    OPTIONS = 'OPTIONS'
-    ALL = '*'
-
-
-class AuthPolicy(object):
-    # The AWS account id the policy will be generated for. This is used to create the method ARNs.
-    awsAccountId = ''
-    # The principal used for the policy, this should be a unique identifier for the end user.
-    principalId = ''
-    # The policy version used for the evaluation. This should always be '2012-10-17'
-    version = '2012-10-17'
-    # The regular expression used to validate resource paths for the policy
-    pathRegex = '^[/.a-zA-Z0-9-\*]+$'
-
-    '''Internal lists of allowed and denied methods.
-
-    These are lists of objects and each object has 2 properties: A resource
-    ARN and a nullable conditions statement. The build method processes these
-    lists and generates the approriate statements for the final policy.
-    '''
-    allowMethods = []
-    denyMethods = []
-
-    # The API Gateway API id. By default this is set to '*'
-    restApiId = '*'
-    # The region where the API is deployed. By default this is set to '*'
-    region = '*'
-    # The name of the stage used in the policy. By default this is set to '*'
-    stage = '*'
-
-    def __init__(self, principal, awsAccountId):
-        self.awsAccountId = awsAccountId
-        self.principalId = principal
-        self.allowMethods = []
-        self.denyMethods = []
-
-    def _addMethod(self, effect, verb, resource, conditions):
-        '''Adds a method to the internal lists of allowed or denied methods. Each object in
-        the internal list contains a resource ARN and a condition statement. The condition
-        statement can be null.'''
-        if verb != '*' and not hasattr(HttpVerb, verb):
-            raise NameError('Invalid HTTP verb ' + verb + '. Allowed verbs in HttpVerb class')
-        resourcePattern = re.compile(self.pathRegex)
-        if not resourcePattern.match(resource):
-            raise NameError('Invalid resource path: ' + resource + '. Path should match ' + self.pathRegex)
-
-        if resource[:1] == '/':
-            resource = resource[1:]
-
-        resourceArn = 'arn:aws:execute-api:{}:{}:{}/{}/{}/{}'.format(self.region, self.awsAccountId, self.restApiId, self.stage, verb, resource)
-
-        if effect.lower() == 'allow':
-            self.allowMethods.append({
-                'resourceArn': resourceArn,
-                'conditions': conditions
-            })
-        elif effect.lower() == 'deny':
-            self.denyMethods.append({
-                'resourceArn': resourceArn,
-                'conditions': conditions
-            })
-
-    def _getEmptyStatement(self, effect):
-        '''Returns an empty statement object prepopulated with the correct action and the
-        desired effect.'''
-        statement = {
-            'Action': 'execute-api:Invoke',
-            'Effect': effect[:1].upper() + effect[1:].lower(),
-            'Resource': []
-        }
-
-        return statement
-
-    def _getStatementForEffect(self, effect, methods):
-        '''This function loops over an array of objects containing a resourceArn and
-        conditions statement and generates the array of statements for the policy.'''
-        statements = []
-
-        if len(methods) > 0:
-            statement = self._getEmptyStatement(effect)
-
-            for curMethod in methods:
-                if curMethod['conditions'] is None or len(curMethod['conditions']) == 0:
-                    statement['Resource'].append(curMethod['resourceArn'])
-                else:
-                    conditionalStatement = self._getEmptyStatement(effect)
-                    conditionalStatement['Resource'].append(curMethod['resourceArn'])
-                    conditionalStatement['Condition'] = curMethod['conditions']
-                    statements.append(conditionalStatement)
-
-            if statement['Resource']:
-                statements.append(statement)
-
-        return statements
-
-    def allowAllMethods(self):
-        '''Adds a '*' allow to the policy to authorize access to all methods of an API'''
-        self._addMethod('Allow', HttpVerb.ALL, '*', [])
-
-    def denyAllMethods(self):
-        '''Adds a '*' allow to the policy to deny access to all methods of an API'''
-        self._addMethod('Deny', HttpVerb.ALL, '*', [])
-
-    def allowMethod(self, verb, resource):
-        '''Adds an API Gateway method (Http verb + Resource path) to the list of allowed
-        methods for the policy'''
-        self._addMethod('Allow', verb, resource, [])
-
-    def denyMethod(self, verb, resource):
-        '''Adds an API Gateway method (Http verb + Resource path) to the list of denied
-        methods for the policy'''
-        self._addMethod('Deny', verb, resource, [])
-
-    def allowMethodWithConditions(self, verb, resource, conditions):
-        '''Adds an API Gateway method (Http verb + Resource path) to the list of allowed
-        methods and includes a condition for the policy statement. More on AWS policy
-        conditions here: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Condition'''
-        self._addMethod('Allow', verb, resource, conditions)
-
-    def denyMethodWithConditions(self, verb, resource, conditions):
-        '''Adds an API Gateway method (Http verb + Resource path) to the list of denied
-        methods and includes a condition for the policy statement. More on AWS policy
-        conditions here: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Condition'''
-        self._addMethod('Deny', verb, resource, conditions)
-
-    def build(self):
-        '''Generates the policy document based on the internal lists of allowed and denied
-        conditions. This will generate a policy with two main statements for the effect:
-        one statement for Allow and one statement for Deny.
-        Methods that includes conditions will have their own statement in the policy.'''
-        if ((self.allowMethods is None or len(self.allowMethods) == 0) and
-                (self.denyMethods is None or len(self.denyMethods) == 0)):
-            raise NameError('No statements defined for the policy')
-
-        policy = {
-            'principalId': self.principalId,
-            'policyDocument': {
-                'Version': self.version,
-                'Statement': []
-            },
-           "context": {
-                "uniquemodversion":"string"
-              }
-        }
-
-        policy['policyDocument']['Statement'].extend(self._getStatementForEffect('Allow', self.allowMethods))
-        policy['policyDocument']['Statement'].extend(self._getStatementForEffect('Deny', self.denyMethods))
-
-        return policy
+from __future__ import print_function
+
+import re
+import json
+
+
+def handler(event, context):
+    print("Client token: " + event['authorizationToken'])
+    print("Method ARN: " + event['methodArn'])
+
+    '''
+    Validate the incoming token and produce the principal user identifier
+    associated with the token. This can be accomplished in a number of ways:
+
+    1. Call out to the OAuth provider
+    2. Decode a JWT token inline
+    3. Lookup in a self-managed DB
+    '''
+    idtoken=event['authorizationToken']
+    try:
+        tokenbodydata=json.loads(idtoken)
+        idtoken=tokenbodydata['token']
+        evaltrue=tokenbodydata['eval']
+    except:
+        evaltrue="FALSE"
+        
+    print(event)
+    principalId = idtoken
+    import jwt
+    decoded = jwt.decode(idtoken, options={"verify_signature": False})  # works in PyJWT < v2.0
+    
+    # Check token access with live rest api (with cognito auth activated) to ensure user isn't sending incorrect username
+    # 200 status indicates accept, 403 means unauthorized
+
+
+    #==============AIMODELSHARE Note: accept / deny languange located on line 67=======================================================
+
+    '''
+    You can send a 401 Unauthorized response to the client by failing like so:
+
+      raise Exception('Unauthorized')
+
+    If the token is valid, a policy must be generated which will allow or deny
+    access to the client. If access is denied, the client will receive a 403
+    Access Denied response. If access is allowed, API Gateway will proceed with
+    the backend integration configured on the method that was called.
+
+    This function must generate a policy that is associated with the recognized
+    principal user identifier. Depending on your use case, you might store
+    policies in a DB, or generate them on the fly.
+
+    Keep in mind, the policy is cached for 5 minutes by default (TTL is
+    configurable in the authorizer) and will apply to subsequent calls to any
+    method/resource in the RestApi made with the same token.
+
+    The example policy below denies access to all resources in the RestApi.
+    '''
+    tmp = event['methodArn'].split(':')
+    apiGatewayArnTmp = tmp[5].split('/')
+    awsAccountId = tmp[4]
+
+    policy = AuthPolicy(principalId, awsAccountId)
+    policy.restApiId = apiGatewayArnTmp[0]
+    policy.region = tmp[3]
+    policy.stage = apiGatewayArnTmp[1]
+    
+    
+    #====================Accept/Deny auth for AIMODELSHARE begins here===========================================
+    import requests
+    
+
+    headers_with_authentication = {'Content-Type': 'application/json', 'Authorization': idtoken}
+    
+    response=requests.post("https://1b861v14ne.execute-api.us-east-1.amazonaws.com/dev/redisusagetrackerembedded",
+                         json={"apiid": str(policy.restApiId),"eval":evaltrue}, headers=headers_with_authentication)
+    print(response.status_code)
+    uniquemodversion=str(response.text)
+
+    print(response.text)
+    if any([response.status_code!=200,response.text.find("Missing")>0,response.text.find("Monthly")>0,response.text.find("monthly")>0]):
+        policyresult="deny"
+    else:
+        policyresult="accept"
+
+    if policyresult=="deny":
+        policy.denyAllMethods()
+    else:
+        policy.allowAllMethods()
+
+    # Finally, build the policy
+    authResponse = policy.build()
+    authResponse['context']['uniquemodversion']=uniquemodversion
+
+    # new! -- add additional key-value pairs associated with the authenticated principal
+    # these are made available by APIGW like so: $context.authorizer.<key>
+    # additional context is cached
+    
+
+    
+    return authResponse
+
+
+class HttpVerb:
+    GET = 'GET'
+    POST = 'POST'
+    PUT = 'PUT'
+    PATCH = 'PATCH'
+    HEAD = 'HEAD'
+    DELETE = 'DELETE'
+    OPTIONS = 'OPTIONS'
+    ALL = '*'
+
+
+class AuthPolicy(object):
+    # The AWS account id the policy will be generated for. This is used to create the method ARNs.
+    awsAccountId = ''
+    # The principal used for the policy, this should be a unique identifier for the end user.
+    principalId = ''
+    # The policy version used for the evaluation. This should always be '2012-10-17'
+    version = '2012-10-17'
+    # The regular expression used to validate resource paths for the policy
+    pathRegex = '^[/.a-zA-Z0-9-\*]+$'
+
+    '''Internal lists of allowed and denied methods.
+
+    These are lists of objects and each object has 2 properties: A resource
+    ARN and a nullable conditions statement. The build method processes these
+    lists and generates the approriate statements for the final policy.
+    '''
+    allowMethods = []
+    denyMethods = []
+
+    # The API Gateway API id. By default this is set to '*'
+    restApiId = '*'
+    # The region where the API is deployed. By default this is set to '*'
+    region = '*'
+    # The name of the stage used in the policy. By default this is set to '*'
+    stage = '*'
+
+    def __init__(self, principal, awsAccountId):
+        self.awsAccountId = awsAccountId
+        self.principalId = principal
+        self.allowMethods = []
+        self.denyMethods = []
+
+    def _addMethod(self, effect, verb, resource, conditions):
+        '''Adds a method to the internal lists of allowed or denied methods. Each object in
+        the internal list contains a resource ARN and a condition statement. The condition
+        statement can be null.'''
+        if verb != '*' and not hasattr(HttpVerb, verb):
+            raise NameError('Invalid HTTP verb ' + verb + '. Allowed verbs in HttpVerb class')
+        resourcePattern = re.compile(self.pathRegex)
+        if not resourcePattern.match(resource):
+            raise NameError('Invalid resource path: ' + resource + '. Path should match ' + self.pathRegex)
+
+        if resource[:1] == '/':
+            resource = resource[1:]
+
+        resourceArn = 'arn:aws:execute-api:{}:{}:{}/{}/{}/{}'.format(self.region, self.awsAccountId, self.restApiId, self.stage, verb, resource)
+
+        if effect.lower() == 'allow':
+            self.allowMethods.append({
+                'resourceArn': resourceArn,
+                'conditions': conditions
+            })
+        elif effect.lower() == 'deny':
+            self.denyMethods.append({
+                'resourceArn': resourceArn,
+                'conditions': conditions
+            })
+
+    def _getEmptyStatement(self, effect):
+        '''Returns an empty statement object prepopulated with the correct action and the
+        desired effect.'''
+        statement = {
+            'Action': 'execute-api:Invoke',
+            'Effect': effect[:1].upper() + effect[1:].lower(),
+            'Resource': []
+        }
+
+        return statement
+
+    def _getStatementForEffect(self, effect, methods):
+        '''This function loops over an array of objects containing a resourceArn and
+        conditions statement and generates the array of statements for the policy.'''
+        statements = []
+
+        if len(methods) > 0:
+            statement = self._getEmptyStatement(effect)
+
+            for curMethod in methods:
+                if curMethod['conditions'] is None or len(curMethod['conditions']) == 0:
+                    statement['Resource'].append(curMethod['resourceArn'])
+                else:
+                    conditionalStatement = self._getEmptyStatement(effect)
+                    conditionalStatement['Resource'].append(curMethod['resourceArn'])
+                    conditionalStatement['Condition'] = curMethod['conditions']
+                    statements.append(conditionalStatement)
+
+            if statement['Resource']:
+                statements.append(statement)
+
+        return statements
+
+    def allowAllMethods(self):
+        '''Adds a '*' allow to the policy to authorize access to all methods of an API'''
+        self._addMethod('Allow', HttpVerb.ALL, '*', [])
+
+    def denyAllMethods(self):
+        '''Adds a '*' allow to the policy to deny access to all methods of an API'''
+        self._addMethod('Deny', HttpVerb.ALL, '*', [])
+
+    def allowMethod(self, verb, resource):
+        '''Adds an API Gateway method (Http verb + Resource path) to the list of allowed
+        methods for the policy'''
+        self._addMethod('Allow', verb, resource, [])
+
+    def denyMethod(self, verb, resource):
+        '''Adds an API Gateway method (Http verb + Resource path) to the list of denied
+        methods for the policy'''
+        self._addMethod('Deny', verb, resource, [])
+
+    def allowMethodWithConditions(self, verb, resource, conditions):
+        '''Adds an API Gateway method (Http verb + Resource path) to the list of allowed
+        methods and includes a condition for the policy statement. More on AWS policy
+        conditions here: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Condition'''
+        self._addMethod('Allow', verb, resource, conditions)
+
+    def denyMethodWithConditions(self, verb, resource, conditions):
+        '''Adds an API Gateway method (Http verb + Resource path) to the list of denied
+        methods and includes a condition for the policy statement. More on AWS policy
+        conditions here: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Condition'''
+        self._addMethod('Deny', verb, resource, conditions)
+
+    def build(self):
+        '''Generates the policy document based on the internal lists of allowed and denied
+        conditions. This will generate a policy with two main statements for the effect:
+        one statement for Allow and one statement for Deny.
+        Methods that includes conditions will have their own statement in the policy.'''
+        if ((self.allowMethods is None or len(self.allowMethods) == 0) and
+                (self.denyMethods is None or len(self.denyMethods) == 0)):
+            raise NameError('No statements defined for the policy')
+
+        policy = {
+            'principalId': self.principalId,
+            'policyDocument': {
+                'Version': self.version,
+                'Statement': []
+            },
+           "context": {
+                "uniquemodversion":"string"
+              }
+        }
+
+        policy['policyDocument']['Statement'].extend(self._getStatementForEffect('Allow', self.allowMethods))
+        policy['policyDocument']['Statement'].extend(self._getStatementForEffect('Deny', self.denyMethods))
+
+        return policy

aimodelshare/pyspark/eval_lambda.txt

@@ -827,7 +827,7 @@ def get_leaderboard(task_type="$task_type", verbose=3, columns=None):
             newleaderboard = pd.read_csv("/tmp/"+"model_eval_data_mastertable_v"+str(i)+".csv", sep="\t")
             newleaderboard.drop(newleaderboard.filter(regex="Unname"),axis=1, inplace=True)
             
-            leaderboard=pd.concat([leaderboard, newleaderboard]).drop_duplicates()
+            leaderboard=leaderboard.append(newleaderboard).drop_duplicates()
             
         leaderboard.drop(leaderboard.filter(regex="Unname"),axis=1, inplace=True)
     #save new leaderboard here