aimodelshare 0.3.7__py3-none-any.whl

aimodelshare/aws_client.py

@@ -0,0 +1,173 @@
+import time
+import json
+import boto3
+
+delay=2
+
+class AWSClient():
+
+    def __init__(self, user_session):
+
+        self.ecr_client = user_session.client('ecr')
+        self.s3_client = user_session.client("s3")
+        self.iam_client = user_session.client("iam")
+        self.sts_client = user_session.client("sts")
+        self.lambda_client = user_session.client("lambda")
+        self.codebuild_client = user_session.client("codebuild")
+        self.apigateway_client = user_session.client("apigateway")
+        self.account_id = self.sts_client.get_caller_identity()["Account"]
+
+    def get_repository_details(self, repository_name):
+        try:
+            response = self.ecr_client.describe_repositories(repositoryNames=[repository_name])
+            repository_details = response['repositories']
+        except:
+            repository_details = []
+        return repository_details
+
+    def get_image_details(self, repository_name, image_tag):
+        try:
+            response = self.ecr_client.describe_images(repositoryName=repository_name, imageIds=[{'imageTag': image_tag}])
+            image_details = response['imageDetails']
+        except:
+            image_details = []
+        return image_details
+
+    def get_role_details(self, role_name):
+        try:
+            response = self.iam_client.get_role(RoleName=role_name)
+            role_details = response['Role']
+        except:
+            role_details = {}
+        return role_details
+
+    def get_policy_details(self, policy_arn):
+        try:
+            response = self.iam_client.get_policy(PolicyArn=policy_arn)
+            policy_details = response['Policy']
+        except:
+            policy_details = {}
+        return policy_details
+
+    def detach_policies_from_role(self, role_name):
+        response = self.iam_client.list_attached_role_policies(RoleName=role_name)
+        policies = response['AttachedPolicies']
+        for policy in policies:
+            response = self.iam_client.detach_role_policy(RoleName=role_name, PolicyArn=policy['PolicyArn'])
+            time.sleep(delay)
+
+    def delete_iam_role(self, role_name):
+        if(len(self.get_role_details(role_name))):
+            response = self.detach_policies_from_role(role_name)
+            response = self.iam_client.delete_role(RoleName=role_name)
+            time.sleep(delay)
+        
+    def delete_iam_policy(self, policy_name):
+        policy_arn = "arn:aws:iam::" + self.account_id + ":policy/" + policy_name
+        if(len(self.get_policy_details(policy_arn))):
+            response = self.iam_client.delete_policy(PolicyArn=policy_arn)
+            time.sleep(delay)
+
+    def create_iam_role(self, role_name, trust_relationship):
+        response = self.iam_client.create_role(RoleName=role_name, AssumeRolePolicyDocument=json.dumps(trust_relationship))
+        time.sleep(delay)
+
+    def create_iam_policy(self, policy_name, policy):
+        policy_arn = "arn:aws:iam::" + self.account_id + ":policy/" + policy_name
+        response = self.iam_client.create_policy(PolicyName=policy_name, PolicyDocument=json.dumps(policy))
+        time.sleep(delay)
+
+    def attach_policy_to_role(self, role_name, policy_name):
+        policy_arn = "arn:aws:iam::" + self.account_id + ":policy/" + policy_name
+        response = self.iam_client.attach_role_policy(RoleName = role_name, PolicyArn = policy_arn)
+        time.sleep(delay)
+
+    def create_repository(self, repository_name):
+        response = self.ecr_client.create_repository(repositoryName=repository_name)
+        time.sleep(delay)
+
+    def delete_repository(self, repository_name):
+        pass
+        time.sleep(delay)
+
+    def upload_file_to_s3(self, local_file_path, bucket_name, bucket_file_path):
+        response = self.s3_client.upload_file(local_file_path, bucket_name, bucket_file_path)
+        time.sleep(delay)
+
+    def delete_file_from_s3(self, bucket_name, bucket_file_path):
+        response = self.s3_client.delete_object(Bucket=bucket_name, Key=bucket_file_path)
+        time.sleep(delay)
+
+    def create_s3_bucket(self, bucket_name, region):
+        try:
+            response=self.s3_client.head_bucket(Bucket=bucket_name)
+        except:
+            if(region=="us-east-1"):
+                response = self.s3_client.create_bucket(
+                    ACL="private",
+                    Bucket=bucket_name
+                )
+            else:
+                location = {'LocationConstraint': region}
+                response = self.s3_client.create_bucket(
+                    ACL="private",
+                    Bucket=bucket_name,
+                    CreateBucketConfiguration=location
+                )
+        return response
+
+    ####################################################################################################
+
+    def add_invoke_resource_policy_to_lambda(self, lambda_function, statement_id, source_arn):
+        response = self.lambda_client.add_permission(
+            FunctionName=lambda_function,
+            StatementId=statement_id,
+            Action='lambda:InvokeFunction',
+            Principal='apigateway.amazonaws.com',
+            SourceArn=source_arn
+        )
+        return response
+
+    def get_api_resources(self, api_id):
+        try:
+            response = self.apigateway_client.get_resources(restApiId=api_id)
+            resources = response['items']
+        except:
+            resources = []
+        return resources
+
+    def integration_setup(self, api_id, resource_id, uri_str, credentials, integration_response):
+        response_1 = self.apigateway_client.put_integration(
+            restApiId=api_id,
+            resourceId=resource_id,
+            httpMethod='POST',
+            type='AWS_PROXY',
+            integrationHttpMethod='POST',
+            uri=uri_str,
+            credentials=credentials
+        )
+        response_2 = self.apigateway_client.put_integration(
+            restApiId=api_id,
+            resourceId=resource_id,
+            httpMethod='OPTIONS',
+            type='MOCK',
+            requestTemplates={"application/json": '{"statusCode": 200}'},
+            integrationHttpMethod='OPTIONS',
+            uri=uri_str,
+            credentials=credentials
+        )
+        response_3 = self.apigateway_client.put_integration(
+            restApiId=api_id,
+            resourceId=resource_id,
+            httpMethod='OPTIONS',
+            type='MOCK',
+            requestTemplates={"application/json": '{"statusCode": 200}'}
+        )
+        response_4 = self.apigateway_client.put_integration_response(
+            restApiId=api_id,
+            resourceId=resource_id,
+            httpMethod='OPTIONS',
+            statusCode='200',
+            responseParameters=integration_response,
+            responseTemplates={"application/json": '{"statusCode": 200}'}
+        )

aimodelshare/base_image.py

@@ -0,0 +1,154 @@
+import json
+import boto3
+import os
+import shutil
+import time
+import tempfile
+import zipfile
+import importlib.resources as pkg_resources
+from string import Template
+
+def lambda_using_base_image(account_id, region, session, project_name, model_dir, requirements_file_path, apiid, memory_size='3000', timeout='90', python_version='3.7'):
+
+    codebuild_bucket_name=os.environ.get("BUCKET_NAME") # s3 bucket name to create  #TODO: use same bucket and subfolder we used previously to store this data
+                                                                                    #Why?  AWS limits users to 100 total buckets!  Our old code only creates one per user per acct.
+
+    repository=project_name+'-repository' # repository name to create
+    
+    template_folder=tempfile.gettempdir()+'/'+project_name # folder to create for sam
+
+    stack_name=project_name+'-stack' # stack name to be created in cloudformation
+
+    docker_tag='latest'
+    function_name=project_name
+    role_name=project_name+'-lambda-role'
+    policy_name=project_name+'-lambda-policy'
+    
+    codebuild_role_name=project_name+'-codebuild-role'
+    codebuild_policies_name=project_name+'-codebuild-policies'
+
+    codebuild_project_name=project_name+'-project'
+
+    s3_client = session.resource('s3', region_name=region)
+
+    s3_resource = session.resource('s3', region_name=region)
+
+    bucket_versioning = s3_resource.BucketVersioning(codebuild_bucket_name)
+    response = bucket_versioning.enable()
+
+    ecr = session.client('ecr')
+
+    response = ecr.create_repository(
+        repositoryName=repository
+    )
+
+    iam = session.client('iam')
+
+    import importlib_resources as pkg_resources
+    #from . import sam
+
+    os.mkdir(template_folder)
+
+    response = shutil.copytree(model_dir, '/'.join([template_folder, model_dir]))
+
+    def get_all_file_paths(directory):
+        file_paths = []
+        for root, directories, files in os.walk(directory):
+            for filename in files:
+                filepath = os.path.join(root, filename)
+                file_paths.append(filepath)
+        return file_paths
+
+    file_paths = get_all_file_paths(template_folder)
+
+    template_folder_len = len(template_folder)
+
+    with zipfile.ZipFile(''.join([template_folder, '.zip']),'w') as zip:
+        for file in file_paths:
+            zip.write(file, file[template_folder_len:])
+
+    s3_client = session.client('s3')
+    s3_client.upload_file(''.join([template_folder, '.zip']),
+                          codebuild_bucket_name,
+                          ''.join([apiid, '/', project_name, '.zip']))
+
+    iam_client = session.client('iam')
+    
+    response = iam_client.create_policy(
+        PolicyName=policy_name,
+        PolicyDocument=json.dumps({
+            "Version": "2012-10-17",
+            "Statement": [
+                {
+                    "Action": "logs:CreateLogStream",
+                    "Resource": "*",
+                    "Effect": "Allow"
+                },
+                {
+                    "Action": "logs:PutLogEvents",
+                    "Resource": "*",
+                    "Effect": "Allow"
+                },
+                {
+                    "Action": "s3:*",
+                    "Resource": "*",
+                    "Effect": "Allow"
+                }
+            ]
+        })
+    )
+    
+    time.sleep(5)
+
+    response = iam_client.create_role(
+        RoleName=role_name,
+        AssumeRolePolicyDocument=json.dumps({
+            "Version": "2012-10-17",
+            "Statement": [
+                {
+                    "Effect": "Allow",
+                    "Principal": {
+                        "Service": "lambda.amazonaws.com"
+                    },
+                    "Action": "sts:AssumeRole"
+                }
+            ]
+        })
+    )
+    
+    time.sleep(5)
+        
+    response = iam_client.attach_role_policy(
+        RoleName=role_name,
+        PolicyArn='arn:aws:iam::' + account_id + ':policy/' + policy_name
+    )
+    
+    time.sleep(5)
+    
+    lambda_client = session.client('lambda')
+
+    response = lambda_client.create_function(
+        FunctionName=project_name,
+        Role='arn:aws:iam::' + account_id + ':role/' + role_name,
+        Code={
+            'ImageUri': account_id + '.dkr.ecr.' + region + '.amazonaws.com/aimodelshare-base-image:latest'
+        },
+        PackageType="Image",
+        Timeout=int(timeout),
+        MemorySize=int(memory_size),
+        Environment={
+            'Variables': {
+                'bucket': codebuild_bucket_name,
+                'key': apiid + '/' + project_name + '.zip',
+                'final_location': '/tmp/' + model_dir + '.zip'
+            }
+        }
+    )
+    
+    s3_client.delete_object(Bucket=codebuild_bucket_name, Key=apiid)
+    
+    shutil.rmtree(template_folder)
+    
+__all__ = [
+    lambda_using_base_image
+]

aimodelshare/bucketpolicy.py

@@ -0,0 +1,106 @@
+def _custom_s3_policy(bucketname):
+  this_policy = {
+          "Version": "2012-10-17",
+          "Statement": [
+            {
+              "Effect": "Allow",
+              "Action": "s3:*",
+              "Resource": [
+                "arn:aws:s3:::"+bucketname,
+                "arn:aws:s3:::"+bucketname+"/*"
+                      ]
+                },{
+              "Effect": "Allow",
+              "Action": "apigateway:*",
+              "Resource":"*"
+                },{
+              "Effect": "Allow",
+              "Action": "lambda:*",
+              "Resource":"*"
+                },{
+              "Effect": "Allow",
+              "Action": "sts:GetCallerIdentity",
+              "Resource":"*"
+                },{
+              "Effect": "Allow",
+              "Action": ["iam:ListRoles","iam:PutRolePolicy","iam:CreateRole", "iam:ListRolePolicies", "iam:ListAttachedRolePolicies","iam:GetRole","iam:PassRole"],
+              "Resource":[ "arn:aws:lambda:*:*:function:*", "arn:aws:iam::*:role/*","arn:aws:apigateway:*:*","arn:aws:execute-api:*:*"]
+                }
+                 ]
+
+                }
+
+  return this_policy
+
+
+def _custom_upload_policy(bucket_name, unique_model_id):
+  managed_policy = {
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "AllowUserToSeeBucketListInTheConsole"+str(unique_model_id),
+            "Action": [
+                "s3:ListAllMyBuckets",
+                "s3:GetBucketLocation"
+            ],
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:s3:::*"
+            ]
+        },
+        {
+            "Sid": "AllowRootAndHomeListingOfCompanyBucket"+str(unique_model_id),
+            "Action": [
+                "s3:ListBucket"
+            ],
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:s3:::"+bucket_name
+            ],
+            "Condition": {
+                "StringEquals": {
+                    "s3:prefix": [
+                        "",
+                        unique_model_id+"/"
+                    ],
+                    "s3:delimiter": [
+                        "/"
+                    ]
+                }
+            }
+        },
+        {
+            "Sid": "AllowListingOfUserFolder"+str(unique_model_id),
+            "Action": [
+                "s3:ListBucket"
+            ],
+            "Effect": "Allow",
+            "Resource": [
+                "arn:aws:s3:::"+bucket_name
+            ],
+            "Condition": {
+                "StringLike": {
+                    "s3:prefix": [
+                        unique_model_id+"/*"
+                    ]
+                }
+            }
+        },
+        {
+            "Sid": "AllowAllS3ActionsInUserFolder"+str(unique_model_id),
+            "Effect": "Allow",
+            "Action": [
+                "s3:*"
+            ],
+            "Resource": [
+                "arn:aws:s3:::"+bucket_name+"/"+unique_model_id+"/*"
+            ]
+        }
+    ]
+    }
+  return managed_policy
+
+__all__ = [
+    _custom_s3_policy,
+    _custom_upload_policy,
+]

aimodelshare/color_mappings/color_mapping_keras.csv

@@ -0,0 +1,121 @@
+,keras,hex,
+0,AbstractRNNCell,#f1e2cc,
+1,Activation,,
+2,ActivityRegularization,#cbd5e8,
+3,Add,,
+4,AdditiveAttention,#f4cae4,
+5,AlphaDropout,#cbd5e8,
+6,Attention,#f4cae4,
+7,Average,,
+8,AveragePooling1D,#e6f5c9,
+9,AveragePooling2D,#e6f5c9,
+10,AveragePooling3D,#e6f5c9,
+11,AvgPool1D,#e6f5c9,
+12,AvgPool2D,#e6f5c9,
+13,AvgPool3D,#e6f5c9,
+14,BatchNormalization,#cbd5e8,
+15,Bidirectional,#f1e2cc,
+16,CategoryEncoding,,
+17,CenterCrop,,
+18,Concatenate,,
+19,Conv1D,#fdcdac,
+20,Conv1DTranspose,#fdcdac,
+21,Conv2D,#fdcdac,
+22,Conv2DTranspose,#fdcdac,
+23,Conv3D,#fdcdac,
+24,Conv3DTranspose,#fdcdac,
+25,ConvLSTM1D,#fdcdac,
+26,ConvLSTM2D,#fdcdac,
+27,ConvLSTM3D,#fdcdac,
+28,Convolution1D,#fdcdac,
+29,Convolution1DTranspose,#fdcdac,
+30,Convolution2D,#fdcdac,
+31,Convolution2DTranspose,#fdcdac,
+32,Convolution3D,#fdcdac,
+33,Convolution3DTranspose,#fdcdac,
+34,Cropping1D,,
+35,Cropping2D,,
+36,Cropping3D,,
+37,Dense,#fff2ae,
+38,DenseFeatures,,
+39,DepthwiseConv2D,#fdcdac,
+40,Discretization,,
+41,Dot,,
+42,Dropout,#cbd5e8,
+43,Embedding,#b3e2cd,
+44,Flatten,,
+45,GRU,#f1e2cc,
+46,GRUCell,#f1e2cc,
+47,GaussianDropout,#cbd5e8,
+48,GaussianNoise,#cbd5e8,
+49,GlobalAveragePooling1D,#e6f5c9,
+50,GlobalAveragePooling2D,#e6f5c9,
+51,GlobalAveragePooling3D,#e6f5c9,
+52,GlobalAvgPool1D,#e6f5c9,
+53,GlobalAvgPool2D,#e6f5c9,
+54,GlobalAvgPool3D,#e6f5c9,
+55,GlobalMaxPool1D,#e6f5c9,
+56,GlobalMaxPool2D,#e6f5c9,
+57,GlobalMaxPool3D,#e6f5c9,
+58,GlobalMaxPooling1D,#e6f5c9,
+59,GlobalMaxPooling2D,#e6f5c9,
+60,GlobalMaxPooling3D,#e6f5c9,
+61,Hashing,,
+62,Input,,
+63,InputLayer,,
+64,InputSpec,,
+65,IntegerLookup,,
+66,LSTM,#f1e2cc,
+67,LSTMCell,#f1e2cc,
+68,Lambda,,
+69,Layer,,
+70,LayerNormalization,#cbd5e8,
+71,LocallyConnected1D,#fdcdac,
+72,LocallyConnected2D,#fdcdac,
+73,Masking,,
+74,MaxPool1D,#e6f5c9,
+75,MaxPool2D,#e6f5c9,
+76,MaxPool3D,#e6f5c9,
+77,MaxPooling1D,#e6f5c9,
+78,MaxPooling2D,#e6f5c9,
+79,MaxPooling3D,#e6f5c9,
+80,Maximum,,
+81,Minimum,,
+82,MultiHeadAttention,#f4cae4,
+83,Multiply,,
+84,Normalization,,
+85,Permute,,
+86,RNN,#f1e2cc,
+87,RandomContrast,,
+88,RandomCrop,,
+89,RandomFlip,,
+90,RandomHeight,,
+91,RandomRotation,,
+92,RandomTranslation,,
+93,RandomWidth,,
+94,RandomZoom,,
+95,RepeatVector,,
+96,Rescaling,,
+97,Reshape,,
+98,Resizing,,
+99,SeparableConv1D,#fdcdac,
+100,SeparableConv2D,#fdcdac,
+101,SeparableConvolution1D,#fdcdac,
+102,SeparableConvolution2D,#fdcdac,
+103,SimpleRNN,#f1e2cc,
+104,SimpleRNNCell,#f1e2cc,
+105,SpatialDropout1D,#cbd5e8,
+106,SpatialDropout2D,#cbd5e8,
+107,SpatialDropout3D,#cbd5e8,
+108,StackedRNNCells,#f1e2cc,
+109,StringLookup,,
+110,Subtract,,
+111,TextVectorization,,
+112,TimeDistributed,,
+113,UpSampling1D,,
+114,UpSampling2D,,
+115,UpSampling3D,,
+116,Wrapper,,
+117,ZeroPadding1D,#cccccc,
+118,ZeroPadding2D,#cccccc,
+119,ZeroPadding3D,#cccccc,

aimodelshare/color_mappings/color_mapping_pytorch.csv

@@ -0,0 +1,117 @@
+,pytorch,hex
+0,AdaptiveAvgPool1d,#e6f5c9
+1,AdaptiveAvgPool2d,#e6f5c9
+2,AdaptiveAvgPool3d,#e6f5c9
+3,AdaptiveMaxPool1d,#e6f5c9
+4,AdaptiveMaxPool2d,#e6f5c9
+5,AdaptiveMaxPool3d,#e6f5c9
+6,AlphaDropout,#cbd5e8
+7,AvgPool1d,#e6f5c9
+8,AvgPool2d,#e6f5c9
+9,AvgPool3d,#e6f5c9
+10,BatchNorm1d,#cbd5e8
+11,BatchNorm2d,#cbd5e8
+12,BatchNorm3d,#cbd5e8
+13,Bilinear,
+14,ConstantPad1d,#cccccc
+15,ConstantPad2d,#cccccc
+16,ConstantPad3d,#cccccc
+17,Container,
+18,Conv1d,#fdcdac
+19,Conv2d,#fdcdac
+20,Conv3d,#fdcdac
+21,ConvTranspose1d,#fdcdac
+22,ConvTranspose2d,#fdcdac
+23,ConvTranspose3d,#fdcdac
+24,CosineSimilarity,
+25,CrossMapLRN2d,
+26,DataParallel,
+27,Dropout,#cbd5e8
+28,Dropout2d,#cbd5e8
+29,Dropout3d,#cbd5e8
+30,Embedding,#b3e2cd
+31,EmbeddingBag,#b3e2cd
+32,FeatureAlphaDropout,#cbd5e8
+33,Flatten,
+34,Fold,
+35,FractionalMaxPool2d,#e6f5c9
+36,FractionalMaxPool3d,#e6f5c9
+37,GRU,#f1e2cc
+38,GRUCell,#f1e2cc
+39,GroupNorm,
+40,Identity,
+41,InstanceNorm1d,#cbd5e8
+42,InstanceNorm2d,#cbd5e8
+43,InstanceNorm3d,#cbd5e8
+44,LPPool1d,#e6f5c9
+45,LPPool2d,#e6f5c9
+46,LSTM,#f1e2cc
+47,LSTMCell,#f1e2cc
+48,LayerNorm,
+49,Linear,#fff2ae
+50,LocalResponseNorm,
+51,MaxPool1d,#e6f5c9
+52,MaxPool2d,#e6f5c9
+53,MaxPool3d,#e6f5c9
+54,MaxUnpool1d,#e6f5c9
+55,MaxUnpool2d,#e6f5c9
+56,MaxUnpool3d,#e6f5c9
+57,Module,
+58,ModuleDict,
+59,ModuleList,
+60,PairwiseDistance,
+61,Parameter,
+62,ParameterDict,
+63,ParameterList,
+64,PixelShuffle,
+65,RNN,#f1e2cc
+66,RNNBase,#f1e2cc
+67,RNNCell,#f1e2cc
+68,RNNCellBase,#f1e2cc
+69,ReflectionPad1d,#cccccc
+70,ReflectionPad2d,#cccccc
+71,ReplicationPad1d,#cccccc
+72,ReplicationPad2d,#cccccc
+73,ReplicationPad3d,#cccccc
+74,Sequential,
+75,SyncBatchNorm,
+76,Transformer,#f4cae4
+77,TransformerDecoder,#f4cae4
+78,TransformerDecoderLayer,#f4cae4
+79,TransformerEncoder,#f4cae4
+80,TransformerEncoderLayer,#f4cae4
+81,Unfold,
+82,Upsample,
+83,UpsamplingBilinear2d,
+84,UpsamplingNearest2d,
+85,ZeroPad2d,#cccccc
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,
+,,