aimodelshare 0.3.7__py3-none-any.whl

aimodelshare/containerization_templates/Dockerfile.txt

@@ -0,0 +1,8 @@
+FROM public.ecr.aws/lambda/python:$python_version
+
+COPY lambda_function.py ./
+COPY requirements.txt ./
+
+RUN pip install -r ./requirements.txt
+
+CMD ["lambda_function.lambda_handler"]

aimodelshare/containerization_templates/Dockerfile_PySpark.txt

@@ -0,0 +1,23 @@
+FROM public.ecr.aws/lambda/python:$python_version
+
+RUN yum -y install java-1.8.0-openjdk
+
+COPY $requirements_file_path ./
+
+RUN pip install -r ./requirements.txt
+
+ENV SPARK_HOME="/var/lang/lib/python$python_version/site-packages/pyspark"
+ENV PATH=$PATH:$SPARK_HOME/bin
+ENV PATH=$PATH:$SPARK_HOME/sbin
+ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9-src.zip:$PYTHONPATH
+ENV PATH=$SPARK_HOME/python:$PATH
+ENV SPARK_MASTER_HOST="localhost"
+ENV SPARK_LOCAL_IP="127.0.0.1"
+
+ENV JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk*/jre"
+ENV PATH=$PATH:$JAVA_HOME/bin
+
+COPY spark-class $SPARK_HOME/bin/
+RUN chmod +x $SPARK_HOME/bin/spark-class
+
+CMD ["lambda_function.lambda_handler"]

aimodelshare/containerization_templates/buildspec.txt

@@ -0,0 +1,14 @@
+version: 0.2
+
+phases:
+  pre_build:
+    commands:
+      - aws ecr get-login-password --region $region | docker login --username AWS --password-stdin $account_id.dkr.ecr.$region.amazonaws.com
+  build:
+    commands:
+      - docker build --label $label --tag $repository:$image_tag .
+      - docker tag $repository:$image_tag $account_id.dkr.ecr.$region.amazonaws.com/$repository:$image_tag
+      - docker push $account_id.dkr.ecr.$region.amazonaws.com/$repository:$image_tag  
+  post_build:
+    commands:
+      - echo Build Successful.

aimodelshare/containerization_templates/lambda_function.txt

@@ -0,0 +1,40 @@
+# SET ENVIRONMENT VARIALBES FOR BUCKETS
+
+# "bucket": "aimodehshare",
+# "api_id": "354mdp3021",
+# "function_name": "modfunction161070"
+
+import sys
+import os
+import json
+import zipfile
+import tempfile
+import boto3
+
+temp_path = tempfile.gettempdir()
+
+bucket = os.getenv("bucket")
+api_id = os.getenv("api_id")
+function_name = os.getenv("function_name")
+
+key = api_id + "/" + function_name + ".zip"
+temp_path_directory = "/".join([temp_path, function_name])
+temp_path_directory_zip_file = temp_path_directory + ".zip"
+
+s3_client = boto3.client('s3')
+response = s3_client.download_file(
+  bucket,
+  key,
+  temp_path_directory_zip_file
+)
+
+with zipfile.ZipFile(temp_path_directory_zip_file, 'r') as zip_ref:
+    zip_ref.extractall(temp_path)
+
+sys.path.append(temp_path)
+from model import handler
+
+def lambda_handler(event, context):
+    print("lambda_handler from containerization_templates")
+    result = handler(event, context)
+    return result

aimodelshare/custom_approach/__init__.py

@@ -0,0 +1 @@
+

aimodelshare/custom_approach/lambda_function.py

@@ -0,0 +1,17 @@
+import json
+from model import handler
+import os
+
+def lambda_handler(event, context):
+
+    return {
+        "statusCode": 200,
+        "headers": {
+            "Access-Control-Allow-Origin" : "*",
+            "Access-Control-Allow-Credentials": True,
+            "Allow" : "GET, OPTIONS, POST",
+            "Access-Control-Allow-Methods" : "GET, OPTIONS, POST",
+            "Access-Control-Allow-Headers" : "*"
+        },
+        "body" : json.dumps(handler(event, context))
+    }

aimodelshare/custom_eval_metrics.py

@@ -0,0 +1,103 @@
+import os
+import zipfile
+import sys
+import pickle
+import tempfile
+import dill
+import importlib
+import inspect
+#from aimodelshare.python.my_preprocessor import *
+
+def export_eval_metric(eval_metric_fxn, directory, name='eval_metric', globs=globals()):
+    """
+    Exports evaluation metric and related objects into zip file for model deployment
+    Inputs: 2 
+    Output: zipfile named 'eval_metric.zip'
+    
+    Parameters:
+    -----------
+    `eval_metric_fxn`: name of eval metric function
+    `directory`: ``string`` folderpath to eval metric function
+        use "" to reference current working directory
+    `name`: name of the custom eval metric 
+    
+    Returns:
+    --------
+    file named 'name.zip' in the correct format for model deployment
+    """
+    #evaluation metric fxn should always be named "eval_metric" to work properly in aimodelshare process.
+
+    try:
+      import tempfile
+      from zipfile import ZipFile
+      import inspect
+      import os
+
+      globals().update(eval_metric_fxn.__globals__)
+
+      folderpath=directory
+
+      #create temporary folder
+      temp_dir=tempfile.gettempdir()
+      try:
+          os.remove(os.path.join(folderpath, name+".zip"))
+      except:
+          pass
+      #save function code within temporary folder
+      source = inspect.getsource(eval_metric_fxn)
+      with open(os.path.join(temp_dir,"{}.py".format(name)), "w") as f:
+          f.write(source)
+
+      # create a ZipFile object
+      zipObj = ZipFile(os.path.join(folderpath,"{}.zip".format(name)), 'w')
+      # Add preprocessor function to the zipfile
+      zipObj.write(os.path.join(temp_dir,"{}.py".format(name)),"{}.py".format(name))
+
+      #getting list of global variables used in function
+
+      import inspect
+      function_objects=list(inspect.getclosurevars(eval_metric_fxn).globals.keys())
+
+      import sys
+      modulenames = ["sklearn","keras","tensorflow","cv2","resize","pytorch", "librosa"]
+      function_objects_nomodules = [i for i in function_objects if i not in list(modulenames)]
+
+      def savetopickle(function_objects_listelement):
+        import pickle
+        pickle.dump(globals()[function_objects_listelement], open( os.path.join(temp_dir,function_objects_listelement+".pkl"), "wb" ) )
+        return function_objects_listelement
+
+      savedevalmetricobjectslist = list(map(savetopickle, function_objects_nomodules))
+
+      # take savedpreprocessorobjectslist pkl files saved to tempdir to zipfile
+      import pickle
+      import string
+
+      for i in savedevalmetricobjectslist: 
+          objectname=str(i)+".pkl"
+          zipObj.write(os.path.join(temp_dir,objectname),objectname)
+
+      # close the Zip File
+      zipObj.close()
+
+      try:
+          # clean up temp directory files for future runs
+          os.remove(os.path.join(temp_dir,"{}.py".format(name)))
+
+          for i in savedevalmetricobjectslist: 
+              objectname=str(i)+".pkl"
+              os.remove(os.path.join(temp_dir,objectname))
+      except:
+          pass
+
+    except Exception as e:
+        print(e)
+
+    return print("Your eval_metric is now saved to '{}.zip'".format(name))
+
+   
+
+__all__ = [
+    export_eval_metric    
+]
+

aimodelshare/data_sharing/data_sharing_templates/Dockerfile.txt

@@ -0,0 +1,3 @@
+FROM public.ecr.aws/lambda/python:$python_version
+
+COPY ./tmp_dataset_dir ./

aimodelshare/data_sharing/data_sharing_templates/__init__.py

@@ -0,0 +1 @@
+

aimodelshare/data_sharing/data_sharing_templates/buildspec.txt

@@ -0,0 +1,15 @@
+version: 0.2
+
+phases:
+  pre_build:
+    commands:
+      - aws configure set aws_access_key_id $aws_access_key_id
+      - aws configure set aws_secret_access_key $aws_secret_access_key
+      - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
+  build:
+    commands:
+      - docker build -t $repository:$dataset_tag .
+      - docker tag $repository:$dataset_tag $registry_uri/$repository:$dataset_tag      
+  post_build:
+    commands:
+      - docker push $registry_uri/$repository:$dataset_tag

aimodelshare/data_sharing/data_sharing_templates/codebuild_policies.txt

@@ -0,0 +1,129 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Resource": "*",
+            "Action": [
+                "logs:CreateLogGroup",
+                "logs:CreateLogStream",
+                "logs:PutLogEvents"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": "*",
+            "Action": [
+                "s3:PutObject",
+                "s3:GetObject",
+                "s3:GetObjectVersion",
+                "s3:GetBucketAcl",
+                "s3:GetBucketLocation"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "codebuild:CreateReportGroup",
+                "codebuild:CreateReport",
+                "codebuild:UpdateReport",
+                "codebuild:BatchPutTestCases",
+                "codebuild:BatchPutCodeCoverages"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ecr:*",
+                "cloudtrail:LookupEvents"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iam:CreateServiceLinkedRole"
+            ],
+            "Resource": "*",
+            "Condition": {
+                "StringEquals": {
+                    "iam:AWSServiceName": [
+                        "replication.ecr.amazonaws.com"
+                    ]
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "cloudformation:*"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "cloudformation:DescribeStacks",
+                "cloudformation:ListStackResources",
+                "cloudwatch:ListMetrics",
+                "cloudwatch:GetMetricData",
+                "ec2:DescribeSecurityGroups",
+                "ec2:DescribeSubnets",
+                "ec2:DescribeVpcs",
+                "kms:ListAliases",
+                "iam:GetPolicy",
+                "iam:GetPolicyVersion",
+                "iam:GetRole",
+                "iam:GetRolePolicy",
+                "iam:ListAttachedRolePolicies",
+                "iam:ListRolePolicies",
+                "iam:ListRoles",
+                "lambda:*",
+                "logs:DescribeLogGroups",
+                "states:DescribeStateMachine",
+                "states:ListStateMachines",
+                "tag:GetResources",
+                "xray:GetTraceSummaries",
+                "xray:BatchGetTraces"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": "iam:PassRole",
+            "Resource": "*",
+            "Condition": {
+                "StringEquals": {
+                    "iam:PassedToService": "lambda.amazonaws.com"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "logs:DescribeLogStreams",
+                "logs:GetLogEvents",
+                "logs:FilterLogEvents"
+            ],
+            "Resource": "arn:aws:logs:*:*:log-group:/aws/lambda/*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iam:*",
+                "organizations:DescribeAccount",
+                "organizations:DescribeOrganization",
+                "organizations:DescribeOrganizationalUnit",
+                "organizations:DescribePolicy",
+                "organizations:ListChildren",
+                "organizations:ListParents",
+                "organizations:ListPoliciesForTarget",
+                "organizations:ListRoots",
+                "organizations:ListPolicies",
+                "organizations:ListTargetsForPolicy"
+            ],
+            "Resource": "*"
+        }
+    ]
+}

aimodelshare/data_sharing/data_sharing_templates/codebuild_trust_relationship.txt

@@ -0,0 +1,12 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "Service": "codebuild.amazonaws.com"
+            },
+            "Action": "sts:AssumeRole"
+        }
+    ]
+}