aimodelshare 0.3.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. aimodelshare/README.md +26 -0
  2. aimodelshare/__init__.py +100 -0
  3. aimodelshare/aimsonnx.py +2381 -0
  4. aimodelshare/api.py +836 -0
  5. aimodelshare/auth.py +163 -0
  6. aimodelshare/aws.py +511 -0
  7. aimodelshare/aws_client.py +173 -0
  8. aimodelshare/base_image.py +154 -0
  9. aimodelshare/bucketpolicy.py +106 -0
  10. aimodelshare/color_mappings/color_mapping_keras.csv +121 -0
  11. aimodelshare/color_mappings/color_mapping_pytorch.csv +117 -0
  12. aimodelshare/containerisation.py +244 -0
  13. aimodelshare/containerization.py +712 -0
  14. aimodelshare/containerization_templates/Dockerfile.txt +8 -0
  15. aimodelshare/containerization_templates/Dockerfile_PySpark.txt +23 -0
  16. aimodelshare/containerization_templates/buildspec.txt +14 -0
  17. aimodelshare/containerization_templates/lambda_function.txt +40 -0
  18. aimodelshare/custom_approach/__init__.py +1 -0
  19. aimodelshare/custom_approach/lambda_function.py +17 -0
  20. aimodelshare/custom_eval_metrics.py +103 -0
  21. aimodelshare/data_sharing/__init__.py +0 -0
  22. aimodelshare/data_sharing/data_sharing_templates/Dockerfile.txt +3 -0
  23. aimodelshare/data_sharing/data_sharing_templates/__init__.py +1 -0
  24. aimodelshare/data_sharing/data_sharing_templates/buildspec.txt +15 -0
  25. aimodelshare/data_sharing/data_sharing_templates/codebuild_policies.txt +129 -0
  26. aimodelshare/data_sharing/data_sharing_templates/codebuild_trust_relationship.txt +12 -0
  27. aimodelshare/data_sharing/download_data.py +620 -0
  28. aimodelshare/data_sharing/share_data.py +373 -0
  29. aimodelshare/data_sharing/utils.py +8 -0
  30. aimodelshare/deploy_custom_lambda.py +246 -0
  31. aimodelshare/documentation/Makefile +20 -0
  32. aimodelshare/documentation/karma_sphinx_theme/__init__.py +28 -0
  33. aimodelshare/documentation/karma_sphinx_theme/_version.py +2 -0
  34. aimodelshare/documentation/karma_sphinx_theme/breadcrumbs.html +70 -0
  35. aimodelshare/documentation/karma_sphinx_theme/layout.html +172 -0
  36. aimodelshare/documentation/karma_sphinx_theme/search.html +50 -0
  37. aimodelshare/documentation/karma_sphinx_theme/searchbox.html +14 -0
  38. aimodelshare/documentation/karma_sphinx_theme/static/css/custom.css +2 -0
  39. aimodelshare/documentation/karma_sphinx_theme/static/css/custom.css.map +1 -0
  40. aimodelshare/documentation/karma_sphinx_theme/static/css/theme.css +2751 -0
  41. aimodelshare/documentation/karma_sphinx_theme/static/css/theme.css.map +1 -0
  42. aimodelshare/documentation/karma_sphinx_theme/static/css/theme.min.css +2 -0
  43. aimodelshare/documentation/karma_sphinx_theme/static/css/theme.min.css.map +1 -0
  44. aimodelshare/documentation/karma_sphinx_theme/static/font/fontello.eot +0 -0
  45. aimodelshare/documentation/karma_sphinx_theme/static/font/fontello.svg +32 -0
  46. aimodelshare/documentation/karma_sphinx_theme/static/font/fontello.ttf +0 -0
  47. aimodelshare/documentation/karma_sphinx_theme/static/font/fontello.woff +0 -0
  48. aimodelshare/documentation/karma_sphinx_theme/static/font/fontello.woff2 +0 -0
  49. aimodelshare/documentation/karma_sphinx_theme/static/js/theme.js +68 -0
  50. aimodelshare/documentation/karma_sphinx_theme/theme.conf +9 -0
  51. aimodelshare/documentation/make.bat +35 -0
  52. aimodelshare/documentation/requirements.txt +2 -0
  53. aimodelshare/documentation/source/about.rst +18 -0
  54. aimodelshare/documentation/source/advanced_features.rst +137 -0
  55. aimodelshare/documentation/source/competition.rst +218 -0
  56. aimodelshare/documentation/source/conf.py +58 -0
  57. aimodelshare/documentation/source/create_credentials.rst +86 -0
  58. aimodelshare/documentation/source/example_notebooks.rst +132 -0
  59. aimodelshare/documentation/source/functions.rst +151 -0
  60. aimodelshare/documentation/source/gettingstarted.rst +390 -0
  61. aimodelshare/documentation/source/images/creds1.png +0 -0
  62. aimodelshare/documentation/source/images/creds2.png +0 -0
  63. aimodelshare/documentation/source/images/creds3.png +0 -0
  64. aimodelshare/documentation/source/images/creds4.png +0 -0
  65. aimodelshare/documentation/source/images/creds5.png +0 -0
  66. aimodelshare/documentation/source/images/creds_file_example.png +0 -0
  67. aimodelshare/documentation/source/images/predict_tab.png +0 -0
  68. aimodelshare/documentation/source/index.rst +110 -0
  69. aimodelshare/documentation/source/modelplayground.rst +132 -0
  70. aimodelshare/exceptions.py +11 -0
  71. aimodelshare/generatemodelapi.py +1270 -0
  72. aimodelshare/iam/codebuild_policy.txt +129 -0
  73. aimodelshare/iam/codebuild_trust_relationship.txt +12 -0
  74. aimodelshare/iam/lambda_policy.txt +15 -0
  75. aimodelshare/iam/lambda_trust_relationship.txt +12 -0
  76. aimodelshare/json_templates/__init__.py +1 -0
  77. aimodelshare/json_templates/api_json.txt +155 -0
  78. aimodelshare/json_templates/auth/policy.txt +1 -0
  79. aimodelshare/json_templates/auth/role.txt +1 -0
  80. aimodelshare/json_templates/eval/policy.txt +1 -0
  81. aimodelshare/json_templates/eval/role.txt +1 -0
  82. aimodelshare/json_templates/function/policy.txt +1 -0
  83. aimodelshare/json_templates/function/role.txt +1 -0
  84. aimodelshare/json_templates/integration_response.txt +5 -0
  85. aimodelshare/json_templates/lambda_policy_1.txt +15 -0
  86. aimodelshare/json_templates/lambda_policy_2.txt +8 -0
  87. aimodelshare/json_templates/lambda_role_1.txt +12 -0
  88. aimodelshare/json_templates/lambda_role_2.txt +16 -0
  89. aimodelshare/leaderboard.py +174 -0
  90. aimodelshare/main/1.txt +132 -0
  91. aimodelshare/main/1B.txt +112 -0
  92. aimodelshare/main/2.txt +153 -0
  93. aimodelshare/main/3.txt +134 -0
  94. aimodelshare/main/4.txt +128 -0
  95. aimodelshare/main/5.txt +109 -0
  96. aimodelshare/main/6.txt +105 -0
  97. aimodelshare/main/7.txt +144 -0
  98. aimodelshare/main/8.txt +142 -0
  99. aimodelshare/main/__init__.py +1 -0
  100. aimodelshare/main/authorization.txt +275 -0
  101. aimodelshare/main/eval_classification.txt +79 -0
  102. aimodelshare/main/eval_lambda.txt +1709 -0
  103. aimodelshare/main/eval_regression.txt +80 -0
  104. aimodelshare/main/lambda_function.txt +8 -0
  105. aimodelshare/main/nst.txt +149 -0
  106. aimodelshare/model.py +1543 -0
  107. aimodelshare/modeluser.py +215 -0
  108. aimodelshare/moral_compass/README.md +408 -0
  109. aimodelshare/moral_compass/__init__.py +65 -0
  110. aimodelshare/moral_compass/_version.py +3 -0
  111. aimodelshare/moral_compass/api_client.py +601 -0
  112. aimodelshare/moral_compass/apps/__init__.py +69 -0
  113. aimodelshare/moral_compass/apps/ai_consequences.py +540 -0
  114. aimodelshare/moral_compass/apps/bias_detective.py +714 -0
  115. aimodelshare/moral_compass/apps/ethical_revelation.py +898 -0
  116. aimodelshare/moral_compass/apps/fairness_fixer.py +889 -0
  117. aimodelshare/moral_compass/apps/judge.py +888 -0
  118. aimodelshare/moral_compass/apps/justice_equity_upgrade.py +853 -0
  119. aimodelshare/moral_compass/apps/mc_integration_helpers.py +820 -0
  120. aimodelshare/moral_compass/apps/model_building_game.py +1104 -0
  121. aimodelshare/moral_compass/apps/model_building_game_beginner.py +687 -0
  122. aimodelshare/moral_compass/apps/moral_compass_challenge.py +858 -0
  123. aimodelshare/moral_compass/apps/session_auth.py +254 -0
  124. aimodelshare/moral_compass/apps/shared_activity_styles.css +349 -0
  125. aimodelshare/moral_compass/apps/tutorial.py +481 -0
  126. aimodelshare/moral_compass/apps/what_is_ai.py +853 -0
  127. aimodelshare/moral_compass/challenge.py +365 -0
  128. aimodelshare/moral_compass/config.py +187 -0
  129. aimodelshare/placeholders/model.onnx +0 -0
  130. aimodelshare/placeholders/preprocessor.zip +0 -0
  131. aimodelshare/playground.py +1968 -0
  132. aimodelshare/postprocessormodules.py +157 -0
  133. aimodelshare/preprocessormodules.py +373 -0
  134. aimodelshare/pyspark/1.txt +195 -0
  135. aimodelshare/pyspark/1B.txt +181 -0
  136. aimodelshare/pyspark/2.txt +220 -0
  137. aimodelshare/pyspark/3.txt +204 -0
  138. aimodelshare/pyspark/4.txt +187 -0
  139. aimodelshare/pyspark/5.txt +178 -0
  140. aimodelshare/pyspark/6.txt +174 -0
  141. aimodelshare/pyspark/7.txt +211 -0
  142. aimodelshare/pyspark/8.txt +206 -0
  143. aimodelshare/pyspark/__init__.py +1 -0
  144. aimodelshare/pyspark/authorization.txt +258 -0
  145. aimodelshare/pyspark/eval_classification.txt +79 -0
  146. aimodelshare/pyspark/eval_lambda.txt +1441 -0
  147. aimodelshare/pyspark/eval_regression.txt +80 -0
  148. aimodelshare/pyspark/lambda_function.txt +8 -0
  149. aimodelshare/pyspark/nst.txt +213 -0
  150. aimodelshare/python/my_preprocessor.py +58 -0
  151. aimodelshare/readme.md +26 -0
  152. aimodelshare/reproducibility.py +181 -0
  153. aimodelshare/sam/Dockerfile.txt +8 -0
  154. aimodelshare/sam/Dockerfile_PySpark.txt +24 -0
  155. aimodelshare/sam/__init__.py +1 -0
  156. aimodelshare/sam/buildspec.txt +11 -0
  157. aimodelshare/sam/codebuild_policies.txt +129 -0
  158. aimodelshare/sam/codebuild_trust_relationship.txt +12 -0
  159. aimodelshare/sam/codepipeline_policies.txt +173 -0
  160. aimodelshare/sam/codepipeline_trust_relationship.txt +12 -0
  161. aimodelshare/sam/spark-class.txt +2 -0
  162. aimodelshare/sam/template.txt +54 -0
  163. aimodelshare/tools.py +103 -0
  164. aimodelshare/utils/__init__.py +78 -0
  165. aimodelshare/utils/optional_deps.py +38 -0
  166. aimodelshare/utils.py +57 -0
  167. aimodelshare-0.3.7.dist-info/METADATA +298 -0
  168. aimodelshare-0.3.7.dist-info/RECORD +171 -0
  169. aimodelshare-0.3.7.dist-info/WHEEL +5 -0
  170. aimodelshare-0.3.7.dist-info/licenses/LICENSE +5 -0
  171. aimodelshare-0.3.7.dist-info/top_level.txt +1 -0
@@ -0,0 +1,712 @@
1
+ import json
2
+ import os
3
+ import shutil
4
+ import time
5
+ import tempfile
6
+ import zipfile
7
+ from string import Template
8
+ import sys
9
+ import boto3
10
+ import importlib_resources as pkg_resources
11
+
12
+ import uuid
13
+ import requests
14
+
15
+ time_delay=2
16
+
17
+ # abstraction to return list of strings of paths of all files present in a given directory
18
+ def get_all_file_paths_in_directory(directory):
19
+ file_paths = []
20
+ for root, directories, files in os.walk(directory):
21
+ for filename in files:
22
+ filepath = os.path.join(root, filename)
23
+ file_paths.append(filepath)
24
+ return file_paths
25
+
26
+ # abstraction to get the details of a repository
27
+ def get_repository_details(user_session, repo_name):
28
+ #print("Fetching details of repository \"" + repo_name + "\".")
29
+ ecr_client = user_session.client("ecr")
30
+ try:
31
+ repo_details = ecr_client.describe_repositories(repositoryNames=[repo_name])['repositories']
32
+ except:
33
+ repo_details = [] # no details
34
+ return repo_details
35
+
36
+ # abstraction to get the details of an image
37
+ def get_image_details(user_session, repo_name, image_tag):
38
+ #print("Fecthing details of image \"" + repo_name + ":" + image_tag + "\".")
39
+ ecr_client = user_session.client('ecr')
40
+ try:
41
+ image_details = ecr_client.describe_images(repositoryName=repo_name, imageIds=[{'imageTag': image_tag}])['imageDetails']
42
+ #print("Fetched details of image \"" + repo_name + ":" + image_tag + "\" successfully.")
43
+ except:
44
+ #print("No such image \"" + repo_name + ":" + image_tag + "\" exists.")
45
+ image_details = []
46
+ return image_details
47
+
48
+ # abstraction to create repository with name repo_name
49
+ def create_repository(user_session, repo_name):
50
+ #print("Creating repository \"" + repo_name + "\".")
51
+ ecr_client = user_session.client('ecr')
52
+ response = ecr_client.create_repository(
53
+ repositoryName=repo_name
54
+ )
55
+ #print("Created repository \"" + repo_name + "\" successfully.")
56
+
57
+ # abstraction to upload file to S3 bucket
58
+ def upload_file_to_s3(user_session, local_file_path, bucket_name, bucket_file_path):
59
+ #print("Uploading function files to \"" + bucket_file_path +"\".")
60
+ s3_client = user_session.client("s3")
61
+ s3_client.upload_file(
62
+ local_file_path, # path to file in the local environment
63
+ bucket_name, # S3 bucket name
64
+ bucket_file_path # path to file in the S3 bucket
65
+ )
66
+ #print("Uploaded function files to \"" + bucket_file_path +"\" successfully.")
67
+
68
+ # abstraction to delete file from S3 bucket
69
+ def delete_file_from_s3(user_session, bucket_name, bucket_file_path):
70
+ #print("Deleting file \"" + bucket_file_path +"\".")
71
+ s3_client = user_session.client("s3")
72
+ s3_client.delete_object(
73
+ Bucket=bucket_name, # S3 bucket name
74
+ Key=bucket_file_path # path to file in the S3 bucket
75
+ )
76
+ #print("Deleted file \"" + bucket_file_path +"\" successfully.")
77
+
78
+ # abstraction to delete IAM role
79
+ def delete_iam_role(user_session, role_name):
80
+ #print("Deleting IAM role \"" + role_name + "\".")
81
+ iam_client = user_session.client("iam")
82
+ # see if role exists
83
+ try:
84
+ response = iam_client.get_role(RoleName=role_name)
85
+ except:
86
+ return
87
+ # once role's existence is verified, fetch all attached policies
88
+ response = iam_client.list_attached_role_policies(
89
+ RoleName=role_name
90
+ )
91
+ # detach the policy from the role
92
+ policies = response['AttachedPolicies']
93
+ for policy in policies:
94
+ response = iam_client.detach_role_policy(
95
+ RoleName=role_name,
96
+ PolicyArn=policy['PolicyArn']
97
+ )
98
+ # delete role
99
+ response = iam_client.delete_role(
100
+ RoleName=role_name
101
+ )
102
+ # give time to reflect in IAM
103
+ time.sleep(time_delay)
104
+ #print("Deleted IAM role \"" + role_name + "\" successfully.")
105
+
106
+ # abstraction to delete IAM policy
107
+ def delete_iam_policy(user_session, policy_name):
108
+
109
+ sts_client = user_session.client("sts")
110
+ account_id = sts_client.get_caller_identity()["Account"]
111
+
112
+ #print("Deleting IAM policy \"" + policy_name + "\".")
113
+ iam_client = user_session.client("iam")
114
+ policy_arn = "arn:aws:iam::" + account_id + ":policy/" + policy_name
115
+ # see if policy exists
116
+ try:
117
+ response = iam_client.get_policy(PolicyArn=policy_arn)
118
+ except:
119
+ return
120
+ # once policy's existence is verified, delete policy
121
+ response = iam_client.delete_policy(
122
+ PolicyArn=policy_arn
123
+ )
124
+ # give time to reflect in IAM
125
+ time.sleep(time_delay)
126
+ #print("Deleted IAM policy \"" + policy_name + "\" successfully.")
127
+
128
+ # abstraction to create IAM role
129
+ def create_iam_role(user_session, role_name, trust_relationship):
130
+ #print("Creating IAM role \"" + role_name + "\".")
131
+ iam_client = user_session.client("iam")
132
+ response = iam_client.create_role(
133
+ RoleName=role_name,
134
+ AssumeRolePolicyDocument=json.dumps(trust_relationship) # convert JSON to string
135
+ )
136
+ # give time to reflect in IAM
137
+ time.sleep(time_delay)
138
+ #print("Created IAM role \"" + role_name + "\" successfully.")
139
+
140
+ # abstraction to create IAM policy
141
+ def create_iam_policy(user_session, policy_name, policy):
142
+
143
+ sts_client = user_session.client("sts")
144
+ account_id = sts_client.get_caller_identity()["Account"]
145
+
146
+ #print("Creating IAM policy \"" + policy_name + "\".")
147
+ iam_client = user_session.client("iam")
148
+ policy_arn = "arn:aws:iam::" + account_id + ":policy/" + policy_name
149
+ response = iam_client.create_policy(
150
+ PolicyName=policy_name,
151
+ PolicyDocument=json.dumps(policy) # convert JSON to string
152
+ )
153
+ # give time to reflect in IAM
154
+ time.sleep(time_delay)
155
+ #print("Created IAM policy \"" + policy_name + "\" successfully.")
156
+
157
+ # abstraction to attach IAM policy to IAM role
158
+ def attach_policy_to_role(user_session, role_name, policy_name):
159
+
160
+ sts_client = user_session.client("sts")
161
+ account_id = sts_client.get_caller_identity()["Account"]
162
+
163
+ #print("Attaching IAM policy \"" + policy_name +"\" to IAM role \"" + role_name + "\".")
164
+ iam_client = user_session.client("iam")
165
+ policy_arn = "arn:aws:iam::" + account_id + ":policy/" + policy_name
166
+ response = iam_client.attach_role_policy(
167
+ RoleName = role_name,
168
+ PolicyArn = policy_arn
169
+ )
170
+ # give time to reflect in IAM
171
+ time.sleep(time_delay)
172
+ #print("Attached IAM policy \"" + policy_name +"\" to IAM role \"" + role_name + "\" successfully.")
173
+
174
+ # build image using CodeBuild from files in zip file
175
+ def build_image(user_session, bucket_name, zip_file, image_name, image):
176
+
177
+ from . import iam
178
+
179
+ # upload zip file to S3 bucket
180
+ upload_file_to_s3(user_session, zip_file, bucket_name, image_name+'.zip')
181
+
182
+ # reading JSON of the trust relationship required to create role and authorize it to use CodeBuild to build Docker image
183
+ role_name = "codebuild_role"
184
+ trust_relationship = json.loads(pkg_resources.read_text(iam, "codebuild_trust_relationship.txt"))
185
+
186
+ # delete role for CodeBuild if role with same name exists
187
+ delete_iam_role(user_session, role_name)
188
+
189
+ # creating role for CodeBuild
190
+ create_iam_role(user_session, role_name, trust_relationship)
191
+
192
+ # reading JSON of all the policies that CodeBuild requires for accessing AWS services
193
+ policy_name = "codebuild_policy"
194
+ policy = json.loads(pkg_resources.read_text(iam, "codebuild_policy.txt"))
195
+
196
+ # delete policy for CodeBuild if policy with same name exists
197
+ delete_iam_policy(user_session, policy_name)
198
+
199
+ # creating policy for CodeBuild
200
+ create_iam_policy(user_session, policy_name, policy)
201
+
202
+ # attaching policies to role to execute CodeBuild to build Docker image
203
+ attach_policy_to_role(user_session, role_name, policy_name)
204
+
205
+ time.sleep(5)
206
+
207
+ # creating CodeBuild project
208
+ # specify which zip to be sourced from S3 that contains all the files to create the image
209
+ # and specify the Linux environment that will be used to build the image
210
+ codebuild_project_name = 'codebuild_' + image_name.replace("/", "_") + '_project'
211
+ codebuild_client = user_session.client("codebuild")
212
+ counter=1
213
+ while(counter<=3):
214
+ try:
215
+ print("Attempt " + str(counter) + " to create CodeBuild project.")
216
+ response = codebuild_client.create_project(
217
+ name = codebuild_project_name,
218
+ source = {
219
+ "type": "S3", # where to fetch the source files from
220
+ "location": bucket_name + "/" + image_name + ".zip" # exact location of the zip in S3 bucket
221
+ },
222
+ artifacts = {
223
+ "type": "S3", # where to store the artifacts
224
+ "location": bucket_name # which bucket to store artifacts in
225
+ },
226
+ environment = {
227
+ "type": "LINUX_CONTAINER", # using a Linux environment to build the Docker image
228
+ "image": "aws/codebuild/standard:5.0", # type of image to use to build Docker image
229
+ "computeType": "BUILD_GENERAL1_SMALL", # compute type to use based on the user's choice
230
+ "privilegedMode": True # so that a Docker image can be built inside the image
231
+ },
232
+ serviceRole=role_name # role that CodeBuild will use to build project
233
+ )
234
+ response = codebuild_client.start_build(
235
+ projectName = codebuild_project_name
236
+ )
237
+ break
238
+ except:
239
+ counter+=1
240
+ if(counter<=3):
241
+ print("CodeBuild project creation failed. Waiting for dependent resources to reflect. Retrying again in 10 seconds.")
242
+ time.sleep(time_delay)
243
+ else:
244
+ print("CodeBuild project creation failed.")
245
+ delete_file_from_s3(user_session, bucket_name, image_name+'.zip') # delete zip file from S3 bucket
246
+ return
247
+
248
+ # running through loop while build status shows termination/successful completion
249
+ counter=0
250
+ while(True):
251
+ build_response = codebuild_client.batch_get_builds(ids=[response['build']['id']])
252
+ build_status = build_response['builds'][0]['buildStatus']
253
+ if build_status == 'SUCCEEDED':
254
+ response = codebuild_client.delete_project( # delete CodeBuild project after process completes
255
+ name = codebuild_project_name
256
+ )
257
+ print("Image successfully built.")
258
+ print("CodeBuild finished with status " + build_status)
259
+ print("\nNew Base Image URI: " + image)
260
+ break
261
+ elif build_status == 'FAILED' or build_status == 'FAULT' or build_status == 'STOPPED' or build_status == 'TIMED_OUT':
262
+ response = codebuild_client.delete_project( # delete CodeBuild project after process completes
263
+ name = codebuild_project_name
264
+ )
265
+ print("Image not successfully built.")
266
+ print("CodeBuild finished with status " + build_status)
267
+ break
268
+ sys.stdout.write('\r')
269
+ sys.stdout.write("Waiting" + "."*counter)
270
+ sys.stdout.flush()
271
+ counter=(counter+1)%4
272
+ time.sleep(1)
273
+
274
+ # delete zip file from S3 bucket
275
+ delete_file_from_s3(user_session, bucket_name, image_name+'.zip')
276
+
277
+ # create a base image containing a particular set of libraries in repository with specific image tag
278
+ def build_new_base_image(libraries, repository, image_tag, python_version):
279
+
280
+ from . import containerization_templates
281
+
282
+ user_session = boto3.session.Session(aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID_AIMS"),
283
+ aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY_AIMS"),
284
+ region_name=os.environ.get("AWS_REGION_AIMS"))
285
+
286
+ bucket_name = os.environ.get("BUCKET_NAME")
287
+
288
+ s3_client = user_session.client("s3")
289
+
290
+ def create_bucket(s3_client, bucket_name, region):
291
+ try:
292
+ response=s3_client.head_bucket(Bucket=bucket_name)
293
+ except:
294
+ if(region=="us-east-1"):
295
+ response = s3_client.create_bucket(
296
+ ACL="private",
297
+ Bucket=bucket_name
298
+ )
299
+ else:
300
+ location={'LocationConstraint': region}
301
+ response=s3_client.create_bucket(
302
+ ACL="private",
303
+ Bucket=bucket_name,
304
+ CreateBucketConfiguration=location
305
+ )
306
+ return response
307
+
308
+ create_bucket(s3_client, bucket_name, os.environ.get("AWS_REGION_AIMS"))
309
+
310
+ print("S3 Bucket \"" + bucket_name + "\" used for all storage purposes.")
311
+
312
+ unique_name = repository + "_" + image_tag
313
+
314
+ try:
315
+ create_repository(user_session, repository)
316
+ except:
317
+ print("Repository already exists.")
318
+
319
+ sts_client = user_session.client("sts")
320
+ account_id = sts_client.get_caller_identity()["Account"]
321
+ region = user_session.region_name
322
+
323
+ print("Building new base image.")
324
+
325
+ folder_name = "unique_name"
326
+ ##################################################
327
+ #label = "libraries=" + ",".join(libraries) # label of image will be all string of all libraries
328
+ label = "libraries=test" # label of image will be all string of all libraries
329
+ ##################################################
330
+
331
+ # temporary folder path where we will create all files and folder
332
+ temp_dir = tempfile.gettempdir() + "/" + folder_name
333
+
334
+ if(os.path.isdir(temp_dir)):
335
+ shutil.rmtree(temp_dir)
336
+ os.mkdir(temp_dir)
337
+
338
+ # list of all Python libraries (with their versions if required) required to be downloaded from PyPI into Docker image
339
+ with open(os.path.join(temp_dir, "requirements.txt"), "a") as f:
340
+ for lib in libraries:
341
+ f.write('%s\n' % lib)
342
+
343
+ # Dockerfile.txt template being read and appropriate variables being assigned to generate Dockerfile
344
+ data = pkg_resources.read_text(containerization_templates, "Dockerfile.txt") # read template from containerization folder
345
+ template = Template(data)
346
+ newdata = template.substitute(
347
+ python_version=python_version) # AWS maintained images with speicific python versions
348
+ with open(os.path.join(temp_dir, "Dockerfile"), "w") as file:
349
+ file.write(newdata)
350
+
351
+ # buildspec.txt template being read and appropriate variables being assigned to generate buildspec.yml
352
+ data = pkg_resources.read_text(containerization_templates, "buildspec.txt") # read template from containerization folder
353
+ template = Template(data)
354
+ newdata = template.substitute(
355
+ account_id=account_id, # AWS account id
356
+ region=region, # region in which the repository is / should be created
357
+ repository=repository, # name of the repository
358
+ image_tag=image_tag, # version / tag to be given to the image
359
+ label=label) #label of the library
360
+ with open(os.path.join(temp_dir, "buildspec.yml"), "w") as file:
361
+ file.write(newdata)
362
+
363
+ # lambda_function.py being generated which has the handler that will be called when Docker image is invoked
364
+ data = pkg_resources.read_text(containerization_templates, "lambda_function.txt") # read template from containerization folder
365
+ with open(os.path.join(temp_dir, "lambda_function.py"), "w") as file:
366
+ file.write(data)
367
+
368
+ file_paths = get_all_file_paths_in_directory(temp_dir) # getting list of strings containing paths of all files
369
+
370
+ # zipping all files in the temporary folder to be uploaded to the S3 bucket
371
+ with zipfile.ZipFile(temp_dir + ".zip", "w") as zip:
372
+ for file in file_paths:
373
+ zip.write(file, file.replace(temp_dir, "")) # ignore temporary file path when copying to zip file
374
+
375
+ build_image(user_session, bucket_name, temp_dir + ".zip", unique_name + "_base_image", repository+":"+image_tag)
376
+
377
+ if(os.path.isdir(temp_dir)):
378
+ shutil.rmtree(temp_dir)
379
+
380
+ return
381
+
382
+ # create lambda function using a base image from a specific repository having a specific tag
383
+ def create_lambda_using_base_image(user_session, bucket_name, directory, lambda_name, api_id, repository, image_tag, memory_size, timeout):
384
+
385
+ from . import iam
386
+
387
+ cloudformation_client = user_session.client('cloudformation')
388
+
389
+ sts_client = user_session.client("sts")
390
+ account_id = sts_client.get_caller_identity()["Account"]
391
+ region = user_session.region_name
392
+
393
+ temp_dir = tempfile.gettempdir() + "/" + lambda_name
394
+
395
+ if(os.path.isdir(temp_dir)):
396
+ shutil.rmtree(temp_dir)
397
+
398
+ shutil.copytree(directory, temp_dir) # copying files from the local directory to tmp folder directory
399
+
400
+ temp_path_directory_file_paths = get_all_file_paths_in_directory(temp_dir) # getting list of strings containing paths of all files
401
+
402
+ # zipping all files in a temporary folder to be uploaded to the S3 bucket
403
+ with zipfile.ZipFile(temp_dir + ".zip", "w") as zip: # remove temporary path from directory if present
404
+ for file in temp_path_directory_file_paths:
405
+ zip.write(file, file.replace(temp_dir, "")) # ignore temporary path when copying to zip file
406
+
407
+ upload_file_to_s3(user_session, temp_dir + ".zip", bucket_name, api_id + "/" + lambda_name + ".zip") # upload zip file to S3 bucket
408
+
409
+ # reading JSON of the trust relationship required to create role and authorize it to use CodeBuild to build Docker image
410
+ role_name = "lambda_role_" + api_id
411
+ trust_relationship = json.loads(pkg_resources.read_text(iam, "lambda_trust_relationship.txt"))
412
+
413
+ # creating role for CodeBuild
414
+ create_iam_role(user_session, role_name, trust_relationship)
415
+
416
+ # reading JSON of all the policies that CodeBuild requires for accessing AWS services
417
+ policy_name = "lambda_policy_" + api_id
418
+ policy = json.loads(pkg_resources.read_text(iam, "lambda_policy.txt"))
419
+
420
+ # creating policy for CodeBuild
421
+ create_iam_policy(user_session, policy_name, policy)
422
+
423
+ # attaching policies to role to execute CodeBuild to build Docker image
424
+ attach_policy_to_role(user_session, role_name, policy_name)
425
+
426
+ #print("Creating Lambda function \"" + lambda_name + "\".")
427
+
428
+ role = 'arn:aws:iam::' + account_id + ':role/' + role_name
429
+ code = {
430
+ 'ImageUri': account_id + '.dkr.ecr.' + region + '.amazonaws.com/' + repository + ":" + image_tag
431
+ }
432
+ package_type = "Image"
433
+ timeout = int(timeout)
434
+ memory_size = int(memory_size)
435
+ environment = {
436
+ 'Variables': {
437
+ 'bucket': bucket_name, # bucket where zip file is located
438
+ 'api_id': api_id, # api_id in the bucket in which zip file is stored
439
+ 'function_name': lambda_name, # Lambda function name
440
+ 'NUMBA_CACHE_DIR': '/tmp'
441
+ }
442
+ }
443
+
444
+ lambda_client = user_session.client("lambda")
445
+ counter=1
446
+ while(counter<=3):
447
+ try:
448
+ #print("Attempt " + str(counter) + " to create Lambda function.")
449
+ response_lambda = lambda_client.create_function(
450
+ FunctionName=lambda_name,
451
+ Role = role,
452
+ Code = code,
453
+ PackageType = package_type,
454
+ Timeout = timeout,
455
+ MemorySize = memory_size,
456
+ Environment = environment
457
+ )
458
+ break
459
+ except:
460
+ counter+=1
461
+ if(counter<=3):
462
+ time.sleep(5)
463
+ else:
464
+ print("Lambda function creation failed.")
465
+
466
+ # running through loop until function reflects
467
+ counter=1
468
+ while(counter<=3):
469
+ try:
470
+ response = lambda_client.get_function(
471
+ FunctionName=lambda_name
472
+ )
473
+ #print("Created Lambda function " + lambda_name + "\" successfully.")
474
+ break
475
+ except:
476
+ counter+=1
477
+ if(counter<=3):
478
+ #print("Lambda function did not reflected. Waiting for Lambda function to reflect.")
479
+ time.sleep(5)
480
+ else:
481
+ None
482
+ #print("Lambda function did not reflected.")
483
+
484
+ # stack_name = 'stack-' + api_id
485
+ # role_name = "lambda_role_" + api_id
486
+ # trust_policy = json.loads(pkg_resources.read_text(iam, "lambda_trust_relationship.txt"))
487
+ # policy_name = "lambda_policy_" + api_id
488
+ # policy = json.loads(pkg_resources.read_text(iam, "lambda_policy.txt"))
489
+
490
+ # role = 'arn:aws:iam::' + account_id + ':role/' + role_name
491
+ # code = {
492
+ # 'ImageUri': account_id + '.dkr.ecr.' + region + '.amazonaws.com/' + repository + ":" + image_tag
493
+ # }
494
+ # package_type = "Image"
495
+ # timeout = int(timeout)
496
+ # memory_size = int(memory_size)
497
+ # environment = {
498
+ # 'Variables': {
499
+ # 'bucket': bucket_name, # bucket where zip file is located
500
+ # 'api_id': api_id, # api_id in the bucket in which zip file is stored
501
+ # 'function_name': lambda_name, # Lambda function name
502
+ # 'NUMBA_CACHE_DIR': '/tmp'
503
+ # }
504
+ # }
505
+ # arn_prefix = "arn:aws:execute-api:" + region + ":" + account_id + ":" + api_id
506
+ # template_body = get_cloudformation_template()
507
+ # template = Template(template_body)
508
+ # new_template = template.substitute(
509
+ # PolicyDocument = json.dumps(policy),
510
+ # PolicyName = policy_name,
511
+ # TrustPolicy = json.dumps(trust_policy),
512
+ # Policies = json.dumps([policy_name]),
513
+ # RoleName = role_name,
514
+ # Code = json.dumps(code),
515
+ # Environment = json.dumps(environment),
516
+ # FunctionName = lambda_name,
517
+ # MemorySize = memory_size,
518
+ # PackageType = package_type,
519
+ # Timeout = timeout,
520
+ # Action1 = 'lambda:InvokeFunction',
521
+ # Principal1 = 'apigateway.amazonaws.com',
522
+ # SourceArn1 = arn_prefix + "/*/POST/m"
523
+ # )
524
+ # template_body = new_template
525
+
526
+ # response = cloudformation_client.create_stack(
527
+ # StackName = stack_name,
528
+ # TemplateBody = template_body,
529
+ # DisableRollback=False,
530
+ # Capabilities=['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND']
531
+ # )
532
+
533
+ os.remove(temp_dir + ".zip") # delete the zip file created in tmp directory
534
+ if(os.path.isdir(temp_dir)): # delete the temporary folder created in tmp directory
535
+ shutil.rmtree(temp_dir)
536
+
537
+ return response
538
+
539
+ # check if the image exists in the specified repository with specified image tag
540
+ def check_if_image_exists(user_session, repo_name, image_tag):
541
+ #print("Checking if image \"" + repo_name + ":" + image_tag +"\" exists.")
542
+ import datetime
543
+ ecr_client = user_session.client('ecr')
544
+ try:
545
+ image_details = ecr_client.describe_images(
546
+ repositoryName=repo_name,
547
+ imageIds=[{'imageTag': image_tag}]
548
+ )
549
+ result = True
550
+ #adding date check of last image updates for any image tags.
551
+ #if user has old image then we update it using this approach
552
+ #result=image_details['imageDetails'][0]['imagePushedAt'].date()<=datetime.date(2021, 10, 6)
553
+
554
+ #print("The image " + "\"" + repo_name + ":" + image_tag + "\"" + " exists.")
555
+ # print details of image
556
+ #print(image_details)
557
+ return result
558
+ except:
559
+ #print("The image " + "\"" + repo_name + ":" + image_tag + "\"" + " does not exist.")
560
+ return False
561
+
562
+ # check if repo exists
563
+ def check_if_repo_exists(user_session, repo_name):
564
+ #print("Checking if repository \"" + repo_name + "\" exists.")
565
+ ecr_client = user_session.client("ecr")
566
+ try:
567
+ repo_details = ecr_client.describe_images(
568
+ repositoryName=repo_name
569
+ )
570
+ #print("The repository \"" + repo_name + "\" exists.")
571
+ # print details all images in repo
572
+ #print(repo_details)
573
+ return True
574
+ except:
575
+ #print("The repository \"" + repo_name + "\" does not exist.")
576
+ return False
577
+
578
+ # delete statement with specific Sid in ECR
579
+ def delete_registry_permission(user_session, statement_id):
580
+ ecr_client = user_session.client("ecr")
581
+ try:
582
+ registry_policy = ecr_client.get_registry_policy()
583
+ policy_json = json.loads(registry_policy['policyText'])
584
+ for statement in policy_json["Statement"]:
585
+ if(statement["Sid"]==statement_id):
586
+ policy_json["Statement"].remove(statement)
587
+ response = ecr_client.put_registry_policy(json.dumps(policy_json))
588
+ break
589
+ time.sleep(5)
590
+ except:
591
+ None
592
+
593
+ # add statement with specific Sid in ECR
594
+ def add_registry_permission(user_session, statement_id, source_account_id, account_id, region):
595
+ ecr_client = user_session.client("ecr")
596
+ policy=json.dumps({
597
+ "Version":"2012-10-17",
598
+ "Statement":[
599
+ {
600
+ "Sid": statement_id,
601
+ "Principal": {
602
+ "AWS": "arn:aws:iam::" + source_account_id + ":root"
603
+ },
604
+ "Action": "ecr:*",
605
+ "Resource": [ "arn:aws:ecr:"+region+":"+account_id+":repository/*"],
606
+ "Effect": "Allow"
607
+ }
608
+ ]
609
+ })
610
+ response = ecr_client.put_registry_policy(policyText=policy)
611
+ time.sleep(5)
612
+
613
+ # create base image having repository:image_tag identity from source account if image not present, update base image if update flag set to true
614
+ def clone_base_image(user_session, repository, image_tag, source_account_id, api_endpoint="", update=False):
615
+
616
+ if(check_if_image_exists(user_session, repository, image_tag) and update==False):
617
+ result = {"Status": 1, "Success" : "Base image \"" + repository + ":" + image_tag + "\" already exists on this user's account."}
618
+ elif(len(api_endpoint)>0):
619
+ sts_client = user_session.client("sts")
620
+
621
+ account_id = sts_client.get_caller_identity()["Account"]
622
+ region = user_session.region_name
623
+
624
+ ecr_client = user_session.client('ecr')
625
+
626
+ statement_id = source_account_id + "_image_clone_access"
627
+
628
+ delete_registry_permission(user_session, statement_id)
629
+ add_registry_permission(user_session, statement_id, source_account_id, account_id, region)
630
+
631
+ bodydata = {
632
+ "useraccountnumber": str(account_id), # change this to first and last name
633
+ "region": region,
634
+ "repository": repository,
635
+ "image_tag": image_tag
636
+ }
637
+
638
+ # datasets api
639
+ headers_with_authentication = {'Content-Type': 'application/json',
640
+ 'authorizationToken': os.environ.get("JWT_AUTHORIZATION_TOKEN"),
641
+ 'Access-Control-Allow-Headers': 'Content-Type,X-Amz-Date,authorizationToken,Access-Control-Allow-Origin,X-Api-Key,X-Amz-Security-Token,Authorization',
642
+ 'Access-Control-Allow-Origin': '*'}
643
+
644
+ # modeltoapi lambda function invoked through below url to return new prediction api in response
645
+ response = requests.post(api_endpoint, json=bodydata, headers=headers_with_authentication)
646
+ time.sleep(5)
647
+
648
+ # Delete registry policy
649
+ delete_registry_permission(user_session, statement_id)
650
+
651
+ #client.delete_registry_policy()
652
+ result = {"Status": 1, "Success" : "New base image loaded to user's account"}
653
+
654
+ #print(result)
655
+ else:
656
+ result = {"Status": 0, "Success" : "API endpoint not valid/not provided and image does not exist either."}
657
+ return result
658
+
659
+
660
+ def get_cloudformation_template():
661
+ return """
662
+ {
663
+ "AWSTemplateFormatVersion": "2010-09-09",
664
+ "Description": "Creation of Role, Policy, Lambda, Lambda Permission",
665
+ "Resources": {
666
+ "Role": {
667
+ "Type" : "AWS::IAM::Role",
668
+ "Properties" : {
669
+ "AssumeRolePolicyDocument" : $TrustPolicy,
670
+ "RoleName" : "$RoleName"
671
+ }
672
+ },
673
+ "Policy": {
674
+ "DependsOn": "Role",
675
+ "Type" : "AWS::IAM::Policy",
676
+ "Properties" : {
677
+ "PolicyDocument" : $PolicyDocument,
678
+ "PolicyName" : "$PolicyName",
679
+ "Roles": ["$RoleName"]
680
+ }
681
+ },
682
+ "Lambda": {
683
+ "DependsOn": "Policy",
684
+ "Type" : "AWS::Lambda::Function",
685
+ "Properties" : {
686
+ "Code" : $Code,
687
+ "Environment" : $Environment,
688
+ "FunctionName" : "$FunctionName",
689
+ "MemorySize" : $MemorySize,
690
+ "PackageType" : "$PackageType",
691
+ "Role" : {
692
+ "Fn::GetAtt": [
693
+ "Role",
694
+ "Arn"
695
+ ]
696
+ },
697
+ "Timeout" : $Timeout
698
+ }
699
+ },
700
+ "LambdaInvokePermission1": {
701
+ "DependsOn": "Lambda",
702
+ "Type" : "AWS::Lambda::Permission",
703
+ "Properties" : {
704
+ "Action" : "$Action1",
705
+ "FunctionName" : "$FunctionName",
706
+ "Principal" : "$Principal1",
707
+ "SourceArn" : "$SourceArn1"
708
+ }
709
+ }
710
+ }
711
+ }
712
+ """