aimodelshare 0.1.32__py3-none-any.whl → 0.1.62__py3-none-any.whl

aimodelshare/pyspark/authorization.txt

@@ -1,258 +1,258 @@
-from __future__ import print_function
-
-import re
-import json
-
-
-def handler(event, context):
-    print("Client token: " + event['authorizationToken'])
-    print("Method ARN: " + event['methodArn'])
-
-    '''
-    Validate the incoming token and produce the principal user identifier
-    associated with the token. This can be accomplished in a number of ways:
-
-    1. Call out to the OAuth provider
-    2. Decode a JWT token inline
-    3. Lookup in a self-managed DB
-    '''
-    idtoken=event['authorizationToken']
-    try:
-        tokenbodydata=json.loads(idtoken)
-        idtoken=tokenbodydata['token']
-        evaltrue=tokenbodydata['eval']
-    except:
-        evaltrue="FALSE"
-        
-    print(event)
-    principalId = idtoken
-    import jwt
-    decoded = jwt.decode(idtoken, options={"verify_signature": False})  # works in PyJWT < v2.0
-    
-    # Check token access with live rest api (with cognito auth activated) to ensure user isn't sending incorrect username
-    # 200 status indicates accept, 403 means unauthorized
-
-
-    #==============AIMODELSHARE Note: accept / deny languange located on line 67=======================================================
-
-    '''
-    You can send a 401 Unauthorized response to the client by failing like so:
-
-      raise Exception('Unauthorized')
-
-    If the token is valid, a policy must be generated which will allow or deny
-    access to the client. If access is denied, the client will receive a 403
-    Access Denied response. If access is allowed, API Gateway will proceed with
-    the backend integration configured on the method that was called.
-
-    This function must generate a policy that is associated with the recognized
-    principal user identifier. Depending on your use case, you might store
-    policies in a DB, or generate them on the fly.
-
-    Keep in mind, the policy is cached for 5 minutes by default (TTL is
-    configurable in the authorizer) and will apply to subsequent calls to any
-    method/resource in the RestApi made with the same token.
-
-    The example policy below denies access to all resources in the RestApi.
-    '''
-    tmp = event['methodArn'].split(':')
-    apiGatewayArnTmp = tmp[5].split('/')
-    awsAccountId = tmp[4]
-
-    policy = AuthPolicy(principalId, awsAccountId)
-    policy.restApiId = apiGatewayArnTmp[0]
-    policy.region = tmp[3]
-    policy.stage = apiGatewayArnTmp[1]
-    
-    
-    #====================Accept/Deny auth for AIMODELSHARE begins here===========================================
-    import requests
-    
-
-    headers_with_authentication = {'Content-Type': 'application/json', 'Authorization': idtoken}
-    
-    response=requests.post("https://1b861v14ne.execute-api.us-east-1.amazonaws.com/dev/redisusagetrackerembedded",
-                         json={"apiid": str(policy.restApiId),"eval":evaltrue}, headers=headers_with_authentication)
-    print(response.status_code)
-    uniquemodversion=str(response.text)
-
-    print(response.text)
-    if any([response.status_code!=200,response.text.find("Missing")>0,response.text.find("Monthly")>0,response.text.find("monthly")>0]):
-        policyresult="deny"
-    else:
-        policyresult="accept"
-
-    if policyresult=="deny":
-        policy.denyAllMethods()
-    else:
-        policy.allowAllMethods()
-
-    # Finally, build the policy
-    authResponse = policy.build()
-    authResponse['context']['uniquemodversion']=uniquemodversion
-
-    # new! -- add additional key-value pairs associated with the authenticated principal
-    # these are made available by APIGW like so: $context.authorizer.<key>
-    # additional context is cached
-    
-
-    
-    return authResponse
-
-
-class HttpVerb:
-    GET = 'GET'
-    POST = 'POST'
-    PUT = 'PUT'
-    PATCH = 'PATCH'
-    HEAD = 'HEAD'
-    DELETE = 'DELETE'
-    OPTIONS = 'OPTIONS'
-    ALL = '*'
-
-
-class AuthPolicy(object):
-    # The AWS account id the policy will be generated for. This is used to create the method ARNs.
-    awsAccountId = ''
-    # The principal used for the policy, this should be a unique identifier for the end user.
-    principalId = ''
-    # The policy version used for the evaluation. This should always be '2012-10-17'
-    version = '2012-10-17'
-    # The regular expression used to validate resource paths for the policy
-    pathRegex = '^[/.a-zA-Z0-9-\*]+$'
-
-    '''Internal lists of allowed and denied methods.
-
-    These are lists of objects and each object has 2 properties: A resource
-    ARN and a nullable conditions statement. The build method processes these
-    lists and generates the approriate statements for the final policy.
-    '''
-    allowMethods = []
-    denyMethods = []
-
-    # The API Gateway API id. By default this is set to '*'
-    restApiId = '*'
-    # The region where the API is deployed. By default this is set to '*'
-    region = '*'
-    # The name of the stage used in the policy. By default this is set to '*'
-    stage = '*'
-
-    def __init__(self, principal, awsAccountId):
-        self.awsAccountId = awsAccountId
-        self.principalId = principal
-        self.allowMethods = []
-        self.denyMethods = []
-
-    def _addMethod(self, effect, verb, resource, conditions):
-        '''Adds a method to the internal lists of allowed or denied methods. Each object in
-        the internal list contains a resource ARN and a condition statement. The condition
-        statement can be null.'''
-        if verb != '*' and not hasattr(HttpVerb, verb):
-            raise NameError('Invalid HTTP verb ' + verb + '. Allowed verbs in HttpVerb class')
-        resourcePattern = re.compile(self.pathRegex)
-        if not resourcePattern.match(resource):
-            raise NameError('Invalid resource path: ' + resource + '. Path should match ' + self.pathRegex)
-
-        if resource[:1] == '/':
-            resource = resource[1:]
-
-        resourceArn = 'arn:aws:execute-api:{}:{}:{}/{}/{}/{}'.format(self.region, self.awsAccountId, self.restApiId, self.stage, verb, resource)
-
-        if effect.lower() == 'allow':
-            self.allowMethods.append({
-                'resourceArn': resourceArn,
-                'conditions': conditions
-            })
-        elif effect.lower() == 'deny':
-            self.denyMethods.append({
-                'resourceArn': resourceArn,
-                'conditions': conditions
-            })
-
-    def _getEmptyStatement(self, effect):
-        '''Returns an empty statement object prepopulated with the correct action and the
-        desired effect.'''
-        statement = {
-            'Action': 'execute-api:Invoke',
-            'Effect': effect[:1].upper() + effect[1:].lower(),
-            'Resource': []
-        }
-
-        return statement
-
-    def _getStatementForEffect(self, effect, methods):
-        '''This function loops over an array of objects containing a resourceArn and
-        conditions statement and generates the array of statements for the policy.'''
-        statements = []
-
-        if len(methods) > 0:
-            statement = self._getEmptyStatement(effect)
-
-            for curMethod in methods:
-                if curMethod['conditions'] is None or len(curMethod['conditions']) == 0:
-                    statement['Resource'].append(curMethod['resourceArn'])
-                else:
-                    conditionalStatement = self._getEmptyStatement(effect)
-                    conditionalStatement['Resource'].append(curMethod['resourceArn'])
-                    conditionalStatement['Condition'] = curMethod['conditions']
-                    statements.append(conditionalStatement)
-
-            if statement['Resource']:
-                statements.append(statement)
-
-        return statements
-
-    def allowAllMethods(self):
-        '''Adds a '*' allow to the policy to authorize access to all methods of an API'''
-        self._addMethod('Allow', HttpVerb.ALL, '*', [])
-
-    def denyAllMethods(self):
-        '''Adds a '*' allow to the policy to deny access to all methods of an API'''
-        self._addMethod('Deny', HttpVerb.ALL, '*', [])
-
-    def allowMethod(self, verb, resource):
-        '''Adds an API Gateway method (Http verb + Resource path) to the list of allowed
-        methods for the policy'''
-        self._addMethod('Allow', verb, resource, [])
-
-    def denyMethod(self, verb, resource):
-        '''Adds an API Gateway method (Http verb + Resource path) to the list of denied
-        methods for the policy'''
-        self._addMethod('Deny', verb, resource, [])
-
-    def allowMethodWithConditions(self, verb, resource, conditions):
-        '''Adds an API Gateway method (Http verb + Resource path) to the list of allowed
-        methods and includes a condition for the policy statement. More on AWS policy
-        conditions here: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Condition'''
-        self._addMethod('Allow', verb, resource, conditions)
-
-    def denyMethodWithConditions(self, verb, resource, conditions):
-        '''Adds an API Gateway method (Http verb + Resource path) to the list of denied
-        methods and includes a condition for the policy statement. More on AWS policy
-        conditions here: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Condition'''
-        self._addMethod('Deny', verb, resource, conditions)
-
-    def build(self):
-        '''Generates the policy document based on the internal lists of allowed and denied
-        conditions. This will generate a policy with two main statements for the effect:
-        one statement for Allow and one statement for Deny.
-        Methods that includes conditions will have their own statement in the policy.'''
-        if ((self.allowMethods is None or len(self.allowMethods) == 0) and
-                (self.denyMethods is None or len(self.denyMethods) == 0)):
-            raise NameError('No statements defined for the policy')
-
-        policy = {
-            'principalId': self.principalId,
-            'policyDocument': {
-                'Version': self.version,
-                'Statement': []
-            },
-           "context": {
-                "uniquemodversion":"string"
-              }
-        }
-
-        policy['policyDocument']['Statement'].extend(self._getStatementForEffect('Allow', self.allowMethods))
-        policy['policyDocument']['Statement'].extend(self._getStatementForEffect('Deny', self.denyMethods))
-
-        return policy
+from __future__ import print_function
+
+import re
+import json
+
+
+def handler(event, context):
+    print("Client token: " + event['authorizationToken'])
+    print("Method ARN: " + event['methodArn'])
+
+    '''
+    Validate the incoming token and produce the principal user identifier
+    associated with the token. This can be accomplished in a number of ways:
+
+    1. Call out to the OAuth provider
+    2. Decode a JWT token inline
+    3. Lookup in a self-managed DB
+    '''
+    idtoken=event['authorizationToken']
+    try:
+        tokenbodydata=json.loads(idtoken)
+        idtoken=tokenbodydata['token']
+        evaltrue=tokenbodydata['eval']
+    except:
+        evaltrue="FALSE"
+        
+    print(event)
+    principalId = idtoken
+    import jwt
+    decoded = jwt.decode(idtoken, options={"verify_signature": False})  # works in PyJWT < v2.0
+    
+    # Check token access with live rest api (with cognito auth activated) to ensure user isn't sending incorrect username
+    # 200 status indicates accept, 403 means unauthorized
+
+
+    #==============AIMODELSHARE Note: accept / deny languange located on line 67=======================================================
+
+    '''
+    You can send a 401 Unauthorized response to the client by failing like so:
+
+      raise Exception('Unauthorized')
+
+    If the token is valid, a policy must be generated which will allow or deny
+    access to the client. If access is denied, the client will receive a 403
+    Access Denied response. If access is allowed, API Gateway will proceed with
+    the backend integration configured on the method that was called.
+
+    This function must generate a policy that is associated with the recognized
+    principal user identifier. Depending on your use case, you might store
+    policies in a DB, or generate them on the fly.
+
+    Keep in mind, the policy is cached for 5 minutes by default (TTL is
+    configurable in the authorizer) and will apply to subsequent calls to any
+    method/resource in the RestApi made with the same token.
+
+    The example policy below denies access to all resources in the RestApi.
+    '''
+    tmp = event['methodArn'].split(':')
+    apiGatewayArnTmp = tmp[5].split('/')
+    awsAccountId = tmp[4]
+
+    policy = AuthPolicy(principalId, awsAccountId)
+    policy.restApiId = apiGatewayArnTmp[0]
+    policy.region = tmp[3]
+    policy.stage = apiGatewayArnTmp[1]
+    
+    
+    #====================Accept/Deny auth for AIMODELSHARE begins here===========================================
+    import requests
+    
+
+    headers_with_authentication = {'Content-Type': 'application/json', 'Authorization': idtoken}
+    
+    response=requests.post("https://1b861v14ne.execute-api.us-east-1.amazonaws.com/dev/redisusagetrackerembedded",
+                         json={"apiid": str(policy.restApiId),"eval":evaltrue}, headers=headers_with_authentication)
+    print(response.status_code)
+    uniquemodversion=str(response.text)
+
+    print(response.text)
+    if any([response.status_code!=200,response.text.find("Missing")>0,response.text.find("Monthly")>0,response.text.find("monthly")>0]):
+        policyresult="deny"
+    else:
+        policyresult="accept"
+
+    if policyresult=="deny":
+        policy.denyAllMethods()
+    else:
+        policy.allowAllMethods()
+
+    # Finally, build the policy
+    authResponse = policy.build()
+    authResponse['context']['uniquemodversion']=uniquemodversion
+
+    # new! -- add additional key-value pairs associated with the authenticated principal
+    # these are made available by APIGW like so: $context.authorizer.<key>
+    # additional context is cached
+    
+
+    
+    return authResponse
+
+
+class HttpVerb:
+    GET = 'GET'
+    POST = 'POST'
+    PUT = 'PUT'
+    PATCH = 'PATCH'
+    HEAD = 'HEAD'
+    DELETE = 'DELETE'
+    OPTIONS = 'OPTIONS'
+    ALL = '*'
+
+
+class AuthPolicy(object):
+    # The AWS account id the policy will be generated for. This is used to create the method ARNs.
+    awsAccountId = ''
+    # The principal used for the policy, this should be a unique identifier for the end user.
+    principalId = ''
+    # The policy version used for the evaluation. This should always be '2012-10-17'
+    version = '2012-10-17'
+    # The regular expression used to validate resource paths for the policy
+    pathRegex = '^[/.a-zA-Z0-9-\*]+$'
+
+    '''Internal lists of allowed and denied methods.
+
+    These are lists of objects and each object has 2 properties: A resource
+    ARN and a nullable conditions statement. The build method processes these
+    lists and generates the approriate statements for the final policy.
+    '''
+    allowMethods = []
+    denyMethods = []
+
+    # The API Gateway API id. By default this is set to '*'
+    restApiId = '*'
+    # The region where the API is deployed. By default this is set to '*'
+    region = '*'
+    # The name of the stage used in the policy. By default this is set to '*'
+    stage = '*'
+
+    def __init__(self, principal, awsAccountId):
+        self.awsAccountId = awsAccountId
+        self.principalId = principal
+        self.allowMethods = []
+        self.denyMethods = []
+
+    def _addMethod(self, effect, verb, resource, conditions):
+        '''Adds a method to the internal lists of allowed or denied methods. Each object in
+        the internal list contains a resource ARN and a condition statement. The condition
+        statement can be null.'''
+        if verb != '*' and not hasattr(HttpVerb, verb):
+            raise NameError('Invalid HTTP verb ' + verb + '. Allowed verbs in HttpVerb class')
+        resourcePattern = re.compile(self.pathRegex)
+        if not resourcePattern.match(resource):
+            raise NameError('Invalid resource path: ' + resource + '. Path should match ' + self.pathRegex)
+
+        if resource[:1] == '/':
+            resource = resource[1:]
+
+        resourceArn = 'arn:aws:execute-api:{}:{}:{}/{}/{}/{}'.format(self.region, self.awsAccountId, self.restApiId, self.stage, verb, resource)
+
+        if effect.lower() == 'allow':
+            self.allowMethods.append({
+                'resourceArn': resourceArn,
+                'conditions': conditions
+            })
+        elif effect.lower() == 'deny':
+            self.denyMethods.append({
+                'resourceArn': resourceArn,
+                'conditions': conditions
+            })
+
+    def _getEmptyStatement(self, effect):
+        '''Returns an empty statement object prepopulated with the correct action and the
+        desired effect.'''
+        statement = {
+            'Action': 'execute-api:Invoke',
+            'Effect': effect[:1].upper() + effect[1:].lower(),
+            'Resource': []
+        }
+
+        return statement
+
+    def _getStatementForEffect(self, effect, methods):
+        '''This function loops over an array of objects containing a resourceArn and
+        conditions statement and generates the array of statements for the policy.'''
+        statements = []
+
+        if len(methods) > 0:
+            statement = self._getEmptyStatement(effect)
+
+            for curMethod in methods:
+                if curMethod['conditions'] is None or len(curMethod['conditions']) == 0:
+                    statement['Resource'].append(curMethod['resourceArn'])
+                else:
+                    conditionalStatement = self._getEmptyStatement(effect)
+                    conditionalStatement['Resource'].append(curMethod['resourceArn'])
+                    conditionalStatement['Condition'] = curMethod['conditions']
+                    statements.append(conditionalStatement)
+
+            if statement['Resource']:
+                statements.append(statement)
+
+        return statements
+
+    def allowAllMethods(self):
+        '''Adds a '*' allow to the policy to authorize access to all methods of an API'''
+        self._addMethod('Allow', HttpVerb.ALL, '*', [])
+
+    def denyAllMethods(self):
+        '''Adds a '*' allow to the policy to deny access to all methods of an API'''
+        self._addMethod('Deny', HttpVerb.ALL, '*', [])
+
+    def allowMethod(self, verb, resource):
+        '''Adds an API Gateway method (Http verb + Resource path) to the list of allowed
+        methods for the policy'''
+        self._addMethod('Allow', verb, resource, [])
+
+    def denyMethod(self, verb, resource):
+        '''Adds an API Gateway method (Http verb + Resource path) to the list of denied
+        methods for the policy'''
+        self._addMethod('Deny', verb, resource, [])
+
+    def allowMethodWithConditions(self, verb, resource, conditions):
+        '''Adds an API Gateway method (Http verb + Resource path) to the list of allowed
+        methods and includes a condition for the policy statement. More on AWS policy
+        conditions here: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Condition'''
+        self._addMethod('Allow', verb, resource, conditions)
+
+    def denyMethodWithConditions(self, verb, resource, conditions):
+        '''Adds an API Gateway method (Http verb + Resource path) to the list of denied
+        methods and includes a condition for the policy statement. More on AWS policy
+        conditions here: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Condition'''
+        self._addMethod('Deny', verb, resource, conditions)
+
+    def build(self):
+        '''Generates the policy document based on the internal lists of allowed and denied
+        conditions. This will generate a policy with two main statements for the effect:
+        one statement for Allow and one statement for Deny.
+        Methods that includes conditions will have their own statement in the policy.'''
+        if ((self.allowMethods is None or len(self.allowMethods) == 0) and
+                (self.denyMethods is None or len(self.denyMethods) == 0)):
+            raise NameError('No statements defined for the policy')
+
+        policy = {
+            'principalId': self.principalId,
+            'policyDocument': {
+                'Version': self.version,
+                'Statement': []
+            },
+           "context": {
+                "uniquemodversion":"string"
+              }
+        }
+
+        policy['policyDocument']['Statement'].extend(self._getStatementForEffect('Allow', self.allowMethods))
+        policy['policyDocument']['Statement'].extend(self._getStatementForEffect('Deny', self.denyMethods))
+
+        return policy

aimodelshare/pyspark/eval_lambda.txt

@@ -827,7 +827,7 @@ def get_leaderboard(task_type="$task_type", verbose=3, columns=None):
             newleaderboard = pd.read_csv("/tmp/"+"model_eval_data_mastertable_v"+str(i)+".csv", sep="\t")
             newleaderboard.drop(newleaderboard.filter(regex="Unname"),axis=1, inplace=True)
             
-            leaderboard=pd.concat([leaderboard, newleaderboard]).drop_duplicates()
+            leaderboard=leaderboard.append(newleaderboard).drop_duplicates()
             
         leaderboard.drop(leaderboard.filter(regex="Unname"),axis=1, inplace=True)
     #save new leaderboard here

aimodelshare/reproducibility.py

@@ -3,11 +3,22 @@ import sys
 import json
 import random
 import tempfile
-import pkg_resources
 import requests
 
 import numpy as np
-import tensorflow as tf
+
+# TensorFlow is optional - only needed for reproducibility setup with TF models
+try:
+    import tensorflow as tf
+    _TF_AVAILABLE = True
+except ImportError:
+    _TF_AVAILABLE = False
+    tf = None
+
+try:
+    import importlib.metadata as md
+except ImportError:  # pragma: no cover
+    import importlib_metadata as md
 
 from aimodelshare.aws import get_s3_iam_client, run_function_on_lambda, get_aws_client
 
@@ -44,9 +55,13 @@ def export_reproducibility_env(seed, directory, mode="gpu"):
   else:
     raise Exception("Error: unknown 'mode' value, expected 'gpu' or 'cpu'")
 
-  installed_packages = pkg_resources.working_set
-  installed_packages_list = sorted(["%s==%s" % (i.key, i.version)
-    for i in installed_packages])
+  # Get installed packages using importlib.metadata
+  installed_packages_list = []
+  for dist in md.distributions():
+    name = dist.metadata.get("Name") or "unknown"
+    version = dist.version
+    installed_packages_list.append(f"{name}=={version}")
+  installed_packages_list = sorted(installed_packages_list)
 
   data["session_runtime_info"] = {
     "installed_packages": installed_packages_list,

aimodelshare/utils/__init__.py

@@ -0,0 +1,78 @@
+"""Utility modules for aimodelshare."""
+import os
+import sys
+import shutil
+import tempfile
+import functools
+import warnings
+from typing import Type
+
+from .optional_deps import check_optional
+
+
+def delete_files_from_temp_dir(temp_dir_file_deletion_list):
+    temp_dir = tempfile.gettempdir()
+    for file_name in temp_dir_file_deletion_list:
+        file_path = os.path.join(temp_dir, file_name)
+        if os.path.exists(file_path):
+            os.remove(file_path)
+
+
+def delete_folder(folder_path):
+    if os.path.exists(folder_path):
+        shutil.rmtree(folder_path)
+
+
+def make_folder(folder_path):
+    os.makedirs(folder_path, exist_ok=True)
+
+
+class HiddenPrints:
+    """Context manager that suppresses stdout and stderr (used for silencing noisy outputs)."""
+    def __enter__(self):
+        self._original_stdout = sys.stdout
+        self._original_stderr = sys.stderr
+        self._devnull_stdout = open(os.devnull, 'w')
+        self._devnull_stderr = open(os.devnull, 'w')
+        sys.stdout = self._devnull_stdout
+        sys.stderr = self._devnull_stderr
+        return self
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        sys.stdout = self._original_stdout
+        sys.stderr = self._original_stderr
+        self._devnull_stdout.close()
+        self._devnull_stderr.close()
+
+
+def ignore_warning(warning: Type[Warning]):
+    """
+    Ignore a given warning occurring during method execution.
+
+    Args:
+        warning (Warning): warning type to ignore.
+
+    Returns:
+        the inner function
+    """
+
+    def inner(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            with warnings.catch_warnings():
+                warnings.filterwarnings("ignore", category=warning)
+                return func(*args, **kwargs)
+
+        return wrapper
+
+    return inner
+
+
+__all__ = [
+    "check_optional",
+    "HiddenPrints",
+    "ignore_warning",
+    "delete_files_from_temp_dir",
+    "delete_folder",
+    "make_folder",
+]