aimodelshare 0.3.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. aimodelshare/README.md +26 -0
  2. aimodelshare/__init__.py +100 -0
  3. aimodelshare/aimsonnx.py +2381 -0
  4. aimodelshare/api.py +836 -0
  5. aimodelshare/auth.py +163 -0
  6. aimodelshare/aws.py +511 -0
  7. aimodelshare/aws_client.py +173 -0
  8. aimodelshare/base_image.py +154 -0
  9. aimodelshare/bucketpolicy.py +106 -0
  10. aimodelshare/color_mappings/color_mapping_keras.csv +121 -0
  11. aimodelshare/color_mappings/color_mapping_pytorch.csv +117 -0
  12. aimodelshare/containerisation.py +244 -0
  13. aimodelshare/containerization.py +712 -0
  14. aimodelshare/containerization_templates/Dockerfile.txt +8 -0
  15. aimodelshare/containerization_templates/Dockerfile_PySpark.txt +23 -0
  16. aimodelshare/containerization_templates/buildspec.txt +14 -0
  17. aimodelshare/containerization_templates/lambda_function.txt +40 -0
  18. aimodelshare/custom_approach/__init__.py +1 -0
  19. aimodelshare/custom_approach/lambda_function.py +17 -0
  20. aimodelshare/custom_eval_metrics.py +103 -0
  21. aimodelshare/data_sharing/__init__.py +0 -0
  22. aimodelshare/data_sharing/data_sharing_templates/Dockerfile.txt +3 -0
  23. aimodelshare/data_sharing/data_sharing_templates/__init__.py +1 -0
  24. aimodelshare/data_sharing/data_sharing_templates/buildspec.txt +15 -0
  25. aimodelshare/data_sharing/data_sharing_templates/codebuild_policies.txt +129 -0
  26. aimodelshare/data_sharing/data_sharing_templates/codebuild_trust_relationship.txt +12 -0
  27. aimodelshare/data_sharing/download_data.py +620 -0
  28. aimodelshare/data_sharing/share_data.py +373 -0
  29. aimodelshare/data_sharing/utils.py +8 -0
  30. aimodelshare/deploy_custom_lambda.py +246 -0
  31. aimodelshare/documentation/Makefile +20 -0
  32. aimodelshare/documentation/karma_sphinx_theme/__init__.py +28 -0
  33. aimodelshare/documentation/karma_sphinx_theme/_version.py +2 -0
  34. aimodelshare/documentation/karma_sphinx_theme/breadcrumbs.html +70 -0
  35. aimodelshare/documentation/karma_sphinx_theme/layout.html +172 -0
  36. aimodelshare/documentation/karma_sphinx_theme/search.html +50 -0
  37. aimodelshare/documentation/karma_sphinx_theme/searchbox.html +14 -0
  38. aimodelshare/documentation/karma_sphinx_theme/static/css/custom.css +2 -0
  39. aimodelshare/documentation/karma_sphinx_theme/static/css/custom.css.map +1 -0
  40. aimodelshare/documentation/karma_sphinx_theme/static/css/theme.css +2751 -0
  41. aimodelshare/documentation/karma_sphinx_theme/static/css/theme.css.map +1 -0
  42. aimodelshare/documentation/karma_sphinx_theme/static/css/theme.min.css +2 -0
  43. aimodelshare/documentation/karma_sphinx_theme/static/css/theme.min.css.map +1 -0
  44. aimodelshare/documentation/karma_sphinx_theme/static/font/fontello.eot +0 -0
  45. aimodelshare/documentation/karma_sphinx_theme/static/font/fontello.svg +32 -0
  46. aimodelshare/documentation/karma_sphinx_theme/static/font/fontello.ttf +0 -0
  47. aimodelshare/documentation/karma_sphinx_theme/static/font/fontello.woff +0 -0
  48. aimodelshare/documentation/karma_sphinx_theme/static/font/fontello.woff2 +0 -0
  49. aimodelshare/documentation/karma_sphinx_theme/static/js/theme.js +68 -0
  50. aimodelshare/documentation/karma_sphinx_theme/theme.conf +9 -0
  51. aimodelshare/documentation/make.bat +35 -0
  52. aimodelshare/documentation/requirements.txt +2 -0
  53. aimodelshare/documentation/source/about.rst +18 -0
  54. aimodelshare/documentation/source/advanced_features.rst +137 -0
  55. aimodelshare/documentation/source/competition.rst +218 -0
  56. aimodelshare/documentation/source/conf.py +58 -0
  57. aimodelshare/documentation/source/create_credentials.rst +86 -0
  58. aimodelshare/documentation/source/example_notebooks.rst +132 -0
  59. aimodelshare/documentation/source/functions.rst +151 -0
  60. aimodelshare/documentation/source/gettingstarted.rst +390 -0
  61. aimodelshare/documentation/source/images/creds1.png +0 -0
  62. aimodelshare/documentation/source/images/creds2.png +0 -0
  63. aimodelshare/documentation/source/images/creds3.png +0 -0
  64. aimodelshare/documentation/source/images/creds4.png +0 -0
  65. aimodelshare/documentation/source/images/creds5.png +0 -0
  66. aimodelshare/documentation/source/images/creds_file_example.png +0 -0
  67. aimodelshare/documentation/source/images/predict_tab.png +0 -0
  68. aimodelshare/documentation/source/index.rst +110 -0
  69. aimodelshare/documentation/source/modelplayground.rst +132 -0
  70. aimodelshare/exceptions.py +11 -0
  71. aimodelshare/generatemodelapi.py +1270 -0
  72. aimodelshare/iam/codebuild_policy.txt +129 -0
  73. aimodelshare/iam/codebuild_trust_relationship.txt +12 -0
  74. aimodelshare/iam/lambda_policy.txt +15 -0
  75. aimodelshare/iam/lambda_trust_relationship.txt +12 -0
  76. aimodelshare/json_templates/__init__.py +1 -0
  77. aimodelshare/json_templates/api_json.txt +155 -0
  78. aimodelshare/json_templates/auth/policy.txt +1 -0
  79. aimodelshare/json_templates/auth/role.txt +1 -0
  80. aimodelshare/json_templates/eval/policy.txt +1 -0
  81. aimodelshare/json_templates/eval/role.txt +1 -0
  82. aimodelshare/json_templates/function/policy.txt +1 -0
  83. aimodelshare/json_templates/function/role.txt +1 -0
  84. aimodelshare/json_templates/integration_response.txt +5 -0
  85. aimodelshare/json_templates/lambda_policy_1.txt +15 -0
  86. aimodelshare/json_templates/lambda_policy_2.txt +8 -0
  87. aimodelshare/json_templates/lambda_role_1.txt +12 -0
  88. aimodelshare/json_templates/lambda_role_2.txt +16 -0
  89. aimodelshare/leaderboard.py +174 -0
  90. aimodelshare/main/1.txt +132 -0
  91. aimodelshare/main/1B.txt +112 -0
  92. aimodelshare/main/2.txt +153 -0
  93. aimodelshare/main/3.txt +134 -0
  94. aimodelshare/main/4.txt +128 -0
  95. aimodelshare/main/5.txt +109 -0
  96. aimodelshare/main/6.txt +105 -0
  97. aimodelshare/main/7.txt +144 -0
  98. aimodelshare/main/8.txt +142 -0
  99. aimodelshare/main/__init__.py +1 -0
  100. aimodelshare/main/authorization.txt +275 -0
  101. aimodelshare/main/eval_classification.txt +79 -0
  102. aimodelshare/main/eval_lambda.txt +1709 -0
  103. aimodelshare/main/eval_regression.txt +80 -0
  104. aimodelshare/main/lambda_function.txt +8 -0
  105. aimodelshare/main/nst.txt +149 -0
  106. aimodelshare/model.py +1543 -0
  107. aimodelshare/modeluser.py +215 -0
  108. aimodelshare/moral_compass/README.md +408 -0
  109. aimodelshare/moral_compass/__init__.py +65 -0
  110. aimodelshare/moral_compass/_version.py +3 -0
  111. aimodelshare/moral_compass/api_client.py +601 -0
  112. aimodelshare/moral_compass/apps/__init__.py +69 -0
  113. aimodelshare/moral_compass/apps/ai_consequences.py +540 -0
  114. aimodelshare/moral_compass/apps/bias_detective.py +714 -0
  115. aimodelshare/moral_compass/apps/ethical_revelation.py +898 -0
  116. aimodelshare/moral_compass/apps/fairness_fixer.py +889 -0
  117. aimodelshare/moral_compass/apps/judge.py +888 -0
  118. aimodelshare/moral_compass/apps/justice_equity_upgrade.py +853 -0
  119. aimodelshare/moral_compass/apps/mc_integration_helpers.py +820 -0
  120. aimodelshare/moral_compass/apps/model_building_game.py +1104 -0
  121. aimodelshare/moral_compass/apps/model_building_game_beginner.py +687 -0
  122. aimodelshare/moral_compass/apps/moral_compass_challenge.py +858 -0
  123. aimodelshare/moral_compass/apps/session_auth.py +254 -0
  124. aimodelshare/moral_compass/apps/shared_activity_styles.css +349 -0
  125. aimodelshare/moral_compass/apps/tutorial.py +481 -0
  126. aimodelshare/moral_compass/apps/what_is_ai.py +853 -0
  127. aimodelshare/moral_compass/challenge.py +365 -0
  128. aimodelshare/moral_compass/config.py +187 -0
  129. aimodelshare/placeholders/model.onnx +0 -0
  130. aimodelshare/placeholders/preprocessor.zip +0 -0
  131. aimodelshare/playground.py +1968 -0
  132. aimodelshare/postprocessormodules.py +157 -0
  133. aimodelshare/preprocessormodules.py +373 -0
  134. aimodelshare/pyspark/1.txt +195 -0
  135. aimodelshare/pyspark/1B.txt +181 -0
  136. aimodelshare/pyspark/2.txt +220 -0
  137. aimodelshare/pyspark/3.txt +204 -0
  138. aimodelshare/pyspark/4.txt +187 -0
  139. aimodelshare/pyspark/5.txt +178 -0
  140. aimodelshare/pyspark/6.txt +174 -0
  141. aimodelshare/pyspark/7.txt +211 -0
  142. aimodelshare/pyspark/8.txt +206 -0
  143. aimodelshare/pyspark/__init__.py +1 -0
  144. aimodelshare/pyspark/authorization.txt +258 -0
  145. aimodelshare/pyspark/eval_classification.txt +79 -0
  146. aimodelshare/pyspark/eval_lambda.txt +1441 -0
  147. aimodelshare/pyspark/eval_regression.txt +80 -0
  148. aimodelshare/pyspark/lambda_function.txt +8 -0
  149. aimodelshare/pyspark/nst.txt +213 -0
  150. aimodelshare/python/my_preprocessor.py +58 -0
  151. aimodelshare/readme.md +26 -0
  152. aimodelshare/reproducibility.py +181 -0
  153. aimodelshare/sam/Dockerfile.txt +8 -0
  154. aimodelshare/sam/Dockerfile_PySpark.txt +24 -0
  155. aimodelshare/sam/__init__.py +1 -0
  156. aimodelshare/sam/buildspec.txt +11 -0
  157. aimodelshare/sam/codebuild_policies.txt +129 -0
  158. aimodelshare/sam/codebuild_trust_relationship.txt +12 -0
  159. aimodelshare/sam/codepipeline_policies.txt +173 -0
  160. aimodelshare/sam/codepipeline_trust_relationship.txt +12 -0
  161. aimodelshare/sam/spark-class.txt +2 -0
  162. aimodelshare/sam/template.txt +54 -0
  163. aimodelshare/tools.py +103 -0
  164. aimodelshare/utils/__init__.py +78 -0
  165. aimodelshare/utils/optional_deps.py +38 -0
  166. aimodelshare/utils.py +57 -0
  167. aimodelshare-0.3.7.dist-info/METADATA +298 -0
  168. aimodelshare-0.3.7.dist-info/RECORD +171 -0
  169. aimodelshare-0.3.7.dist-info/WHEEL +5 -0
  170. aimodelshare-0.3.7.dist-info/licenses/LICENSE +5 -0
  171. aimodelshare-0.3.7.dist-info/top_level.txt +1 -0
aimodelshare/api.py ADDED
@@ -0,0 +1,836 @@
1
+ import time
2
+ import sys
3
+ import os
4
+ import shutil
5
+ import random
6
+ import tempfile
7
+ import functools
8
+ import json
9
+ import requests
10
+ import math
11
+ from zipfile import ZipFile
12
+ from string import Template
13
+
14
+ import shortuuid
15
+ import boto3
16
+ import botocore
17
+
18
+ try:
19
+ import importlib.resources as pkg_resources
20
+ except ImportError:
21
+ import importlib_resources as pkg_resources
22
+
23
+ from . import main # relative-import the *package* containing the templates
24
+
25
+ from .utils import *
26
+
27
+ class create_prediction_api_class():
28
+
29
+ def __init__(self, model_filepath, unique_model_id, model_type, categorical, labels, apiid, custom_libraries, requirements, repo_name="", image_tag="", memory=None, timeout=90, pyspark_support=False):
30
+
31
+ #####
32
+ self.user_session = boto3.session.Session(
33
+ aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID_AIMS"),
34
+ aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY_AIMS"),
35
+ region_name=os.environ.get("AWS_REGION_AIMS")
36
+ )
37
+ self.sts_client = self.user_session.client("sts")
38
+ self.account_id = self.sts_client.get_caller_identity()["Account"]
39
+ #####
40
+
41
+ from .aws_client import AWSClient
42
+ self.aws_client = AWSClient(self.user_session)
43
+
44
+ #####
45
+ self.account_id = self.account_id
46
+ self.unique_model_id = unique_model_id
47
+ self.categorical = categorical
48
+ self.model_type = model_type
49
+ self.labels = labels
50
+ self.apiid = apiid
51
+ self.custom_libraries = custom_libraries
52
+ self.requirements = requirements
53
+ self.repo_name = repo_name
54
+ self.image_tag = image_tag
55
+ self.memory = memory
56
+ self.timeout = timeout
57
+ self.pyspark_support = pyspark_support
58
+ self.region = os.environ.get("AWS_REGION")
59
+ self.bucket_name = os.environ.get("BUCKET_NAME")
60
+ self.python_runtime = os.environ.get("PYTHON_RUNTIME", "python3.12")
61
+ #####
62
+
63
+ self.model_type = self.model_type.lower()
64
+
65
+ if(self.categorical == "TRUE"):
66
+ self.categorical = True
67
+ elif(self.categorical == "FALSE"):
68
+ self.categorical = False
69
+
70
+ if(self.custom_libraries == "TRUE"):
71
+ self.custom_libraries = True
72
+ elif(self.custom_libraries == "FALSE"):
73
+ self.custom_libraries = False
74
+
75
+ self.memory_model_mapping = {
76
+ "tabular": 1024,
77
+ "text": 1024,
78
+ "image": 1024,
79
+ "video": 1024,
80
+ "custom":1024
81
+ }
82
+
83
+ self.timeout_model_mapping = {
84
+ "tabular": 90,
85
+ "text": 90,
86
+ "image": 90,
87
+ "video": 90,
88
+ "custom": 90
89
+ }
90
+ # UPDATED: New eval layer ARNs (python3.12)
91
+ self.eval_layer_map = {
92
+ "us-east-1": "arn:aws:lambda:us-east-1:585666012274:layer:eval-layer-python3-12:4",
93
+ "us-east-2": "arn:aws:lambda:us-east-2:517169013426:layer:eval_layer_test:5",
94
+ "us-west-1": "arn:aws:lambda:us-west-1:517169013426:layer:eval_layer_test:1",
95
+ "us-west-2": "arn:aws:lambda:us-west-2:517169013426:layer:eval_layer_test:1",
96
+ "eu-west-1": "arn:aws:lambda:eu-west-1:585666012274:layer:eval-layer-python3-12:1",
97
+ "eu-west-2": "arn:aws:lambda:eu-west-2:517169013426:layer:eval_layer_test:1",
98
+ "eu-west-3": "arn:aws:lambda:eu-west-3:517169013426:layer:eval_layer_test:1"
99
+ }
100
+
101
+ # UPDATED: New auth layer ARNs (python3.12)
102
+ self.auth_layer_map = {
103
+ "us-east-1": "arn:aws:lambda:us-east-1:585666012274:layer:aimsauth-layer-python3-12:6",
104
+ "us-east-2": "arn:aws:lambda:us-east-2:517169013426:layer:aimsauth_layer:9",
105
+ "us-west-1": "arn:aws:lambda:us-west-1:517169013426:layer:aimsauth_layer:1",
106
+ "us-west-2": "arn:aws:lambda:us-west-2:517169013426:layer:aimsauth_layer:1",
107
+ "eu-west-1": "arn:aws:lambda:eu-west-1:585666012274:layer:aimsauth-layer-python3-12:6",
108
+ "eu-west-2": "arn:aws:lambda:eu-west-2:517169013426:layer:aimsauth_layer:1",
109
+ "eu-west-3": "arn:aws:lambda:eu-west-3:517169013426:layer:aimsauth_layer:1"
110
+ }
111
+
112
+ try:
113
+ onnx_size = math.ceil(os.path.getsize(model_filepath)/(1024*1024))
114
+ except:
115
+ onxx_size = 500
116
+
117
+ self.temp_dir_file_deletion_list = ['archive2.zip', 'archive3.zip', 'archive.zip', 'archivetest.zip', 'archiveeval.zip', 'archiveauth.zip', 'main.py', 'ytest.pkl']
118
+ self.memory = self.memory_model_mapping[self.model_type] if self.memory==None else memory
119
+ self.timeout = self.timeout_model_mapping[self.model_type] if self.timeout==None else timeout
120
+
121
+ self.eval_layer = self.eval_layer_map[self.region]
122
+ self.auth_layer = self.auth_layer_map[self.region]
123
+
124
+ self.temp_dir = tempfile.gettempdir()
125
+ self.file_objects_folder_path = os.path.join(self.temp_dir, 'file_objects')
126
+
127
+ self.memory_lambda = {
128
+ "main": 1024,
129
+ "eval": 2048,
130
+ "auth": 512
131
+ }
132
+
133
+ if self.categorical == True:
134
+ self.task_type="classification"
135
+ elif self.categorical == False:
136
+ self.task_type="regression"
137
+ else:
138
+ self.task_type="custom"
139
+
140
+ if(self.model_type=="custom"):
141
+ self.model_class="custom"
142
+ elif(self.model_type=="neural style transfer"):
143
+ self.model_class="generative"
144
+ elif(self.categorical==True):
145
+ self.model_class="classification"
146
+ elif(self.categorical==False):
147
+ self.model_class="regression"
148
+
149
+ self.model_template_mapping = {
150
+ "classification": {
151
+ "text": "1.txt",
152
+ "image": "2.txt",
153
+ "tabular": "4.txt",
154
+ "audio": "7.txt",
155
+ "video": "8.txt"
156
+ },
157
+ "regression": {
158
+ "text": "1B.txt",
159
+ "image": "3.txt",
160
+ "tabular": "5.txt",
161
+ "timeseries": "6.txt"
162
+ },
163
+ "generative": {
164
+ "neural style transfer": "nst.txt"
165
+ }
166
+ }
167
+
168
+ def create_prediction_api(self):
169
+ global main
170
+ if self.pyspark_support:
171
+ try:
172
+ from . import pyspark as main
173
+ except:
174
+ raise("Error: Please install pyspark to enable pyspark features")
175
+
176
+ delete_files_from_temp_dir(self.temp_dir_file_deletion_list)
177
+
178
+ if self.model_type != "custom":
179
+ delete_folder(self.file_objects_folder_path)
180
+ make_folder(self.file_objects_folder_path)
181
+
182
+ if self.model_type == 'custom':
183
+ with open("custom_lambda.py", 'r') as in_file:
184
+ newdata = in_file.read()
185
+ else:
186
+ data = pkg_resources.read_text(main, self.model_template_mapping[self.model_class][self.model_type])
187
+ t = Template(data)
188
+ if(self.model_class=="classification"):
189
+ newdata = t.substitute(
190
+ bucket_name=os.environ.get("BUCKET_NAME"),
191
+ unique_model_id=self.unique_model_id,
192
+ labels=self.labels
193
+ )
194
+ else:
195
+ newdata = t.substitute(
196
+ bucket_name=os.environ.get("BUCKET_NAME"),
197
+ unique_model_id=self.unique_model_id
198
+ )
199
+
200
+ with open(os.path.join(self.file_objects_folder_path, 'model.py'), 'w') as file:
201
+ file.write(newdata)
202
+
203
+ if(self.model_type == 'custom'):
204
+ from . import custom_approach
205
+ data = pkg_resources.read_text(custom_approach, 'lambda_function.py')
206
+ else:
207
+ data = pkg_resources.read_text(main, 'lambda_function.txt')
208
+
209
+ with open(os.path.join(self.file_objects_folder_path, 'lambda_function.py'), 'w') as file:
210
+ file.write(data)
211
+
212
+ ###
213
+ api_key = str(shortuuid.uuid())
214
+
215
+ t = Template(pkg_resources.read_text(main, 'eval_lambda.txt').replace("$apikey",api_key).replace("$task_type",self.task_type))
216
+
217
+ data = t.substitute(bucket_name = self.bucket_name, unique_model_id = self.unique_model_id, task_type = self.task_type)
218
+ with open(os.path.join(self.temp_dir, 'main.py'), 'w') as file:
219
+ file.write(data)
220
+ with ZipFile(os.path.join(self.temp_dir, 'archive2.zip'), 'a') as z:
221
+ z.write(os.path.join(self.temp_dir, 'main.py'), 'main.py')
222
+ self.aws_client.upload_file_to_s3(os.path.join(self.temp_dir, 'archive2.zip'), os.environ.get("BUCKET_NAME"), self.unique_model_id+"/"+'archiveeval.zip')
223
+
224
+ data2 = pkg_resources.read_text(main, 'authorization.txt').replace("$apikey",api_key)
225
+ with open(os.path.join(self.temp_dir, 'main.py'), 'w') as file:
226
+ file.write(data2)
227
+ with ZipFile(os.path.join(self.temp_dir, 'archive3.zip'), 'a') as z:
228
+ z.write(os.path.join(self.temp_dir, 'main.py'), 'main.py')
229
+ self.aws_client.upload_file_to_s3(os.path.join(self.temp_dir, 'archive3.zip'), os.environ.get("BUCKET_NAME"), self.unique_model_id+"/"+'archiveauth.zip')
230
+ ###
231
+
232
+ if self.model_type.lower() == 'custom':
233
+ self.aws_client.upload_file_to_s3(os.path.join(self.temp_dir, 'exampledata.json'), os.environ.get("BUCKET_NAME"), self.unique_model_id+"/"+"exampledata.json")
234
+
235
+ delete_files_from_temp_dir(self.temp_dir_file_deletion_list)
236
+
237
+ ####################
238
+
239
+ short_uuid = str(shortuuid.uuid())
240
+
241
+ lambdarolename = 'myService-dev-us-' + self.region + '-lambdaRole'+short_uuid
242
+ lambdapolicyname = 'myService-dev-' + self.region + '-lambdaPolicy'+short_uuid
243
+ lambdafxnname = 'modfunction'+short_uuid
244
+ lambdaauthfxnname = 'redisAccess'+short_uuid
245
+ lambdaevalfxnname = 'evalfunction'+short_uuid
246
+
247
+ from . import json_templates
248
+
249
+ lambdarole1 = json.loads(pkg_resources.read_text(json_templates, 'lambda_role_1.txt'))
250
+ lambdapolicy1 = json.loads(pkg_resources.read_text(json_templates, 'lambda_policy_1.txt'))
251
+
252
+ self.aws_client.delete_iam_role(lambdarolename) # delete role for CodeBuild if role with same name exists
253
+ self.aws_client.create_iam_role(lambdarolename, lambdarole1) # creating role for CodeBuild
254
+ self.aws_client.delete_iam_policy(lambdapolicyname) # delete policy for CodeBuild if policy with same name exists
255
+ self.aws_client.create_iam_policy(lambdapolicyname, lambdapolicy1) # creating policy for CodeBuild
256
+ self.aws_client.attach_policy_to_role(lambdarolename, lambdapolicyname)
257
+
258
+ sys.stdout.write('\r')
259
+ sys.stdout.write("[============ ] Progress: 40% - Creating custom containers... ")
260
+ sys.stdout.flush()
261
+
262
+ #########
263
+
264
+ if(self.custom_libraries == False):
265
+ from aimodelshare.containerization import create_lambda_using_base_image
266
+ response6 = create_lambda_using_base_image(self.user_session, os.getenv("BUCKET_NAME"), self.file_objects_folder_path, lambdafxnname, self.apiid, self.repo_name, self.image_tag, self.memory, self.timeout)
267
+ elif(self.custom_libraries == True):
268
+ requirements = self.requirements.split(",")
269
+ for i in range(len(requirements)):
270
+ requirements[i] = requirements[i].strip(" ")
271
+ with open(os.path.join(self.file_objects_folder_path, 'requirements.txt'), 'a') as f:
272
+ for lib in requirements:
273
+ f.write('%s\n' % lib)
274
+ requirements_file_path = os.path.join(self.file_objects_folder_path, 'requirements.txt')
275
+ from aimodelshare.containerisation import deploy_container
276
+ response6 = deploy_container(self.account_id, os.environ.get("AWS_REGION"), self.user_session, lambdafxnname, self.file_objects_folder_path,requirements_file_path,self.apiid, pyspark_support=self.pyspark_support)
277
+
278
+ ##########
279
+
280
+ lambdaclient = self.user_session.client('lambda')
281
+ role_arn = 'arn:aws:iam::' + self.account_id + ':role/' + lambdarolename
282
+ handler = 'main.handler'
283
+
284
+ def create_lambda_function(function_name, python_runtime, role_arn, handler, code_source, timeout, memory_size, layers):
285
+ response = self.aws_client.lambda_client.create_function(
286
+ FunctionName = function_name,
287
+ Runtime = python_runtime,
288
+ Role = role_arn,
289
+ Handler = handler,
290
+ Code = code_source,
291
+ Timeout = timeout,
292
+ MemorySize = memory_size,
293
+ Layers = layers
294
+ )
295
+
296
+ eval_code_source = {'S3Bucket': self.bucket_name, 'S3Key': self.unique_model_id + "/" + "archiveeval.zip"}
297
+ eval_layers = [self.eval_layer, self.auth_layer]
298
+ create_lambda_function(lambdaevalfxnname, self.python_runtime, role_arn, handler, eval_code_source, 90, 2048, eval_layers)
299
+
300
+ auth_code_source = {'S3Bucket': self.bucket_name, 'S3Key': self.unique_model_id + "/" + "archiveauth.zip"}
301
+ auth_layers = [self.auth_layer]
302
+ create_lambda_function(lambdaauthfxnname, self.python_runtime, role_arn, handler, auth_code_source, 90, 512, auth_layers)
303
+
304
+ sys.stdout.write('\r')
305
+ sys.stdout.write("[========================== ] Progress: 75% - Deploying prediction API... ")
306
+ sys.stdout.flush()
307
+ # }}}
308
+
309
+ ##############################
310
+
311
+ api_id = self.apiid
312
+ stmt_id = 'apigateway-prod-'+str(shortuuid.uuid())
313
+
314
+ resourceidlist = self.aws_client.get_api_resources(api_id)
315
+ api_path_id = {}
316
+ for i in resourceidlist:
317
+ api_path_id.update({i['path']: i['id']})
318
+
319
+ resource_id_parent = api_path_id['/']
320
+ resource_id_lambda = api_path_id['/m']
321
+ resource_id_eval = api_path_id['/eval']
322
+
323
+ fxn_list = self.aws_client.lambda_client.list_functions()
324
+
325
+ arn_prefix = "arn:aws:execute-api:" + self.region + ":" + self.account_id + ":" + api_id
326
+ self.aws_client.add_invoke_resource_policy_to_lambda(lambdaauthfxnname, stmt_id, arn_prefix + "/*/*")
327
+ #self.aws_client.add_invoke_resource_policy_to_lambda(lambdafxnname, 'apigateway-prod-2', arn_prefix + "/*/POST/m")
328
+ #self.aws_client.add_invoke_resource_policy_to_lambda(lambdafxnname, 'apigateway-test-2', arn_prefix + "/*/POST/m")
329
+ self.aws_client.add_invoke_resource_policy_to_lambda(lambdaevalfxnname, 'apigateway-prod-3', arn_prefix + "/*/POST/eval")
330
+ self.aws_client.add_invoke_resource_policy_to_lambda(lambdaevalfxnname, 'apigateway-test-3', arn_prefix + "/*/POST/eval")
331
+
332
+ integration_response = json.loads(pkg_resources.read_text(json_templates, 'integration_response.txt'))
333
+
334
+ lambdarole2 = json.loads(pkg_resources.read_text(json_templates, 'lambda_role_2.txt'))
335
+ lambdarolename2 = 'lambda_invoke_function_assume_apigw_role_2'
336
+ lambdapolicy2 = json.loads(pkg_resources.read_text(json_templates, 'lambda_policy_2.txt'))
337
+ lambdapolicyname2 = 'invokelambda'
338
+
339
+ self.aws_client.delete_iam_role(lambdarolename2) # delete role for CodeBuild if role with same name exists
340
+ self.aws_client.create_iam_role(lambdarolename2, lambdarole2) # creating role for CodeBuild
341
+ self.aws_client.delete_iam_policy(lambdapolicyname2) # delete policy for CodeBuild if policy with same name exists
342
+ self.aws_client.create_iam_policy(lambdapolicyname2, lambdapolicy2) # creating policy for CodeBuild
343
+ self.aws_client.attach_policy_to_role(lambdarolename2, lambdapolicyname2)
344
+
345
+ uri_str = "arn:aws:apigateway:" + self.region + ":lambda:path/2015-03-31/functions/arn:aws:lambda:" + self.region + ":" + self.account_id + ':function:' + lambdafxnname + '/invocations'
346
+ credentials = 'arn:aws:iam::'+self.account_id+':role/' + lambdarolename2
347
+ self.aws_client.integration_setup(api_id, resource_id_lambda, uri_str, credentials, integration_response)
348
+
349
+ uri_str_2 = "arn:aws:apigateway:" + self.region + ":lambda:path/2015-03-31/functions/arn:aws:lambda:" + self.region + ":" + self.account_id + ':function:' + lambdaevalfxnname + '/invocations'
350
+ credentials_2 = 'arn:aws:iam::'+self.account_id+':role/' + lambdarolename2
351
+ self.aws_client.integration_setup(api_id, resource_id_eval, uri_str_2, credentials_2, integration_response)
352
+
353
+ response = self.aws_client.apigateway_client.update_rest_api(
354
+ restApiId=api_id,
355
+ patchOperations=[
356
+ {
357
+ "op": "replace",
358
+ "path": "/policy",
359
+ "value": '{"Version": "2012-10-17","Statement": [{"Effect": "Allow","Principal": "*","Action": "execute-api:Invoke","Resource": "arn:aws:execute-api:'+os.environ.get("AWS_REGION")+':'+self.account_id+':'+api_id+'/prod/OPTIONS/*"}]}'
360
+ }
361
+ ]
362
+ )
363
+
364
+ auth_uri_str = "arn:aws:apigateway:"+ os.environ.get("AWS_REGION") +":lambda:path/2015-03-31/functions/arn:aws:lambda:"+os.environ.get("AWS_REGION")+":"+self.account_id+":function:"+lambdaauthfxnname+"/invocations"
365
+
366
+ responseauthfxnapigateway = self.aws_client.apigateway_client.create_authorizer(
367
+ restApiId=api_id,
368
+ name='aimscustomauthfxn',
369
+ type='TOKEN',
370
+ authorizerUri=auth_uri_str,
371
+ identitySource="method.request.header.authorizationToken",
372
+ authorizerResultTtlInSeconds=0
373
+ )
374
+
375
+ responseauthfxnapigateway = self.aws_client.apigateway_client.get_authorizers(
376
+ restApiId=api_id
377
+ )
378
+
379
+ authorizerid=responseauthfxnapigateway['items'][0]['id']
380
+
381
+ stmt_idauth = 'apigateway-prod-'+str(shortuuid.uuid())
382
+ response70 = self.user_session.client('lambda').add_permission(
383
+ FunctionName=lambdaauthfxnname,
384
+ StatementId=stmt_idauth,
385
+ Action='lambda:InvokeFunction',
386
+ Principal='apigateway.amazonaws.com',
387
+ SourceArn='arn:aws:execute-api:'+self.region+':' + self.account_id+':'+api_id + '/authorizers/' + authorizerid
388
+ )
389
+
390
+ response_modmthd_addauth = self.user_session.client('apigateway').update_method(
391
+ restApiId=api_id,
392
+ resourceId=resource_id_lambda,
393
+ httpMethod='POST',
394
+ patchOperations=[
395
+ {
396
+ 'op': 'replace',
397
+ 'path': '/authorizationType',
398
+ 'value': 'CUSTOM',
399
+ 'from': 'NONE'
400
+ },
401
+ {
402
+ 'op': 'replace',
403
+ 'path': '/authorizerId',
404
+ 'value': authorizerid
405
+ }
406
+ ]
407
+ )
408
+
409
+ response_evalmthd_addauth = self.user_session.client('apigateway').update_method(
410
+ restApiId=api_id,
411
+ resourceId=resource_id_eval,
412
+ httpMethod='POST',
413
+ patchOperations=[
414
+ {
415
+ 'op': 'replace',
416
+ 'path': '/authorizationType',
417
+ 'value': 'CUSTOM',
418
+ 'from': 'NONE'
419
+ },
420
+ {
421
+ 'op': 'replace',
422
+ 'path': '/authorizerId',
423
+ 'value': authorizerid
424
+ }])
425
+ response12 = self.user_session.client('apigateway').create_deployment(
426
+ restApiId=api_id,
427
+ stageName='prod'
428
+ )
429
+
430
+ result = 'https://'+ api_id + '.execute-api.' + os.environ.get("AWS_REGION") + '.amazonaws.com/prod/m'
431
+
432
+ if self.model_type=='custom':
433
+ ### Progress Update #6/6 {{{
434
+ sys.stdout.write('\r')
435
+ sys.stdout.write("[=====================================] Progress: 100% - API deployment completed! ")
436
+ sys.stdout.flush()
437
+ # }}}
438
+
439
+ return {"statusCode": 200,
440
+ "headers": {
441
+ "Access-Control-Allow-Origin": "*",
442
+ "Access-Control-Allow-Credentials": True
443
+ },
444
+ "body": json.dumps(result)}
445
+
446
+
447
+
448
+ def create_prediction_api(model_filepath, unique_model_id, model_type,
449
+ categorical, labels, apiid, custom_libraries,
450
+ requirements, repo_name="", image_tag="",
451
+ memory=None, timeout=None, pyspark_support=False):
452
+ api_class = create_prediction_api_class(model_filepath, unique_model_id,
453
+ model_type, categorical, labels, apiid,
454
+ custom_libraries, requirements, repo_name,
455
+ image_tag, memory, timeout, pyspark_support)
456
+ return api_class.create_prediction_api()
457
+
458
+ def get_api_json():
459
+ region = "us-east-2"
460
+ apijson = '''
461
+ {
462
+ "openapi": "3.0.1",
463
+ "info": {
464
+ "title": "modapi36146",
465
+ "description": "This is a copy of my first API okay",
466
+ "version": "2020-05-12T21:25:38Z"
467
+ },
468
+ "servers": [
469
+ {
470
+ "url": "https://8nee9nskdb.execute-api.%s.amazonaws.com/{basePath}",
471
+ "variables": {
472
+ "basePath": {
473
+ "default": "/prod"
474
+ }
475
+ }
476
+ }
477
+ ],
478
+ "paths": {
479
+ "/eval": {
480
+ "post": {
481
+ "responses": {
482
+ "200": {
483
+ "description": "200 response",
484
+ "headers": {
485
+ "Access-Control-Allow-Origin": {
486
+ "schema": {
487
+ "type": "string"
488
+ }
489
+ }
490
+ },
491
+ "content": {
492
+ "application/json": {
493
+ "schema": {
494
+ "$ref": "#/components/schemas/outputmodel"
495
+ }
496
+ }
497
+ }
498
+ }
499
+ }
500
+ },
501
+ "options": {
502
+ "responses": {
503
+ "200": {
504
+ "description": "200 response",
505
+ "headers": {
506
+ "Access-Control-Allow-Origin": {
507
+ "schema": {
508
+ "type": "string"
509
+ }
510
+ },
511
+ "Access-Control-Allow-Methods": {
512
+ "schema": {
513
+ "type": "string"
514
+ }
515
+ },
516
+ "Access-Control-Allow-Headers": {
517
+ "schema": {
518
+ "type": "string"
519
+ }
520
+ }
521
+ },
522
+ "content": {
523
+ "application/json": {
524
+ "schema": {
525
+ "$ref": "#/components/schemas/Empty"
526
+ }
527
+ }
528
+ }
529
+ }
530
+ }
531
+ }
532
+ },
533
+ "/m": {
534
+ "post": {
535
+ "responses": {
536
+ "200": {
537
+ "description": "200 response",
538
+ "headers": {
539
+ "Access-Control-Allow-Origin": {
540
+ "schema": {
541
+ "type": "string"
542
+ }
543
+ }
544
+ },
545
+ "content": {
546
+ "application/json": {
547
+ "schema": {
548
+ "$ref": "#/components/schemas/outputmodel"
549
+ }
550
+ }
551
+ }
552
+ }
553
+ }
554
+ },
555
+ "options": {
556
+ "responses": {
557
+ "200": {
558
+ "description": "200 response",
559
+ "headers": {
560
+ "Access-Control-Allow-Origin": {
561
+ "schema": {
562
+ "type": "string"
563
+ }
564
+ },
565
+ "Access-Control-Allow-Methods": {
566
+ "schema": {
567
+ "type": "string"
568
+ }
569
+ },
570
+ "Access-Control-Allow-Headers": {
571
+ "schema": {
572
+ "type": "string"
573
+ }
574
+ }
575
+ },
576
+ "content": {
577
+ "application/json": {
578
+ "schema": {
579
+ "$ref": "#/components/schemas/Empty"
580
+ }
581
+ }
582
+ }
583
+ }
584
+ }
585
+ }
586
+ }
587
+ },
588
+ "components": {
589
+ "schemas": {
590
+ "Empty": {
591
+ "title": "Empty Schema",
592
+ "type": "object"
593
+ },
594
+ "outputmodel": {
595
+ "title": "Output",
596
+ "type": "object",
597
+ "properties": {
598
+ "body": {
599
+ "type": "string"
600
+ }
601
+ }
602
+ }
603
+ }
604
+ },
605
+ "x-amazon-apigateway-policy": {
606
+ "Version": "2012-10-17",
607
+ "Statement": [
608
+ {
609
+ "Effect": "Allow",
610
+ "Principal": "*",
611
+ "Action": "execute-api:Invoke",
612
+ "Resource": "arn:aws:execute-api:%s:517169013426:8nee9nskdb/prod/OPTIONS/*"
613
+ }
614
+ ]
615
+ }
616
+ }
617
+ ''' % (region, region)
618
+ return apijson
619
+
620
+ def delete_deployment(apiurl, confirmation=True):
621
+ """
622
+ apiurl: string of API URL the user wishes to delete
623
+ WARNING: User must supply high-level credentials in order to delete an API.
624
+ """
625
+ from aimodelshare.aws import run_function_on_lambda
626
+
627
+ if confirmation==True:
628
+ # Provide Warning & Have user confirm deletion
629
+ print("Running this function will permanently delete all resources tied to this deployment, \n including the eval lambda and all models submitted to the model competition.\n")
630
+ confirmation = input(prompt="To confirm, type 'permanently delete':")
631
+ if confirmation.lower() == "permanently delete" or confirmation.lower() == "'permanently delete'":
632
+ pass
633
+ else:
634
+ return print("'Delete Deployment' unsuccessful: operation cancelled by user.")
635
+ else:
636
+ pass
637
+ import os
638
+ if os.environ.get("cloud_location") is not None:
639
+ cloudlocation=os.environ.get("cloud_location")
640
+ else:
641
+ cloudlocation="not set"
642
+ if "model_share"==cloudlocation:
643
+ def nonecheck(objinput=""):
644
+ if objinput==None:
645
+ objinput="None"
646
+ else:
647
+ objinput="'/tmp/"+objinput+"'"
648
+ return objinput
649
+
650
+ delplaygroundstring="delete_deployment('"+apiurl+"',"+",confirmation=False)"
651
+ import base64
652
+ import requests
653
+ import json
654
+
655
+ api_url = "https://z4kvag4sxdnv2mvs2b6c4thzj40bxnuw.lambda-url.us-east-2.on.aws/"
656
+
657
+ data = json.dumps({"code": """from aimodelshare.api import delete_deployment;"""+delplaygroundstring, "zipfilename": "","username":os.environ.get("username"), "password":os.environ.get("password"),"token":os.environ.get("JWT_AUTHORIZATION_TOKEN"),"s3keyid":"xrjpv1i7xe"})
658
+
659
+ headers = {"Content-Type": "application/json"}
660
+ response = requests.request("POST", api_url, headers = headers, data=data)
661
+ # Print response
662
+ result=json.loads(response.text)
663
+
664
+ for i in json.loads(result['body']):
665
+ print(i)
666
+
667
+ else:
668
+
669
+ # Confirm that creds are loaded, print warning if not
670
+ if all(["AWS_ACCESS_KEY_ID_AIMS" in os.environ,
671
+ "AWS_SECRET_ACCESS_KEY_AIMS" in os.environ,
672
+ "AWS_REGION_AIMS" in os.environ,
673
+ "username" in os.environ,
674
+ "password" in os.environ]):
675
+ pass
676
+ else:
677
+ return print("'Delete Deployment' unsuccessful. Please provide credentials with set_credentials().")
678
+
679
+ # get api_id from apiurl
680
+ api_url_trim = apiurl.split('https://')[1]
681
+ api_id = api_url_trim.split(".")[0]
682
+
683
+ # Create User Session
684
+ user_sess = boto3.session.Session(aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID_AIMS'),
685
+ aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY_AIMS'),
686
+ region_name=os.environ.get('AWS_REGION_AIMS'))
687
+
688
+ s3 = user_sess.resource('s3')
689
+
690
+ # Get bucket and model_id subfolder for user based on apiurl {{{
691
+ response, error = run_function_on_lambda(
692
+ apiurl, **{"delete": "FALSE", "versionupdateget": "TRUE"}
693
+ )
694
+ if error is not None:
695
+ raise error
696
+ import json
697
+ _, api_bucket, model_id = json.loads(response.content.decode("utf-8"))
698
+ # }}}
699
+
700
+ #Confirm username in bucket name
701
+ ## TODO: Update this check to more secure process
702
+ if os.environ.get("username").lower() in api_bucket:
703
+ pass
704
+ else:
705
+ print("Permission denied. Please provide credentials that allow administrator access to this api.")
706
+ return
707
+
708
+
709
+ # get api resources
710
+ api = user_sess.client('apigateway')
711
+ resources = api.get_resources(
712
+ restApiId=api_id
713
+ )
714
+
715
+ #get lambda arns
716
+ lambda_arns = list()
717
+ for i in range(len(resources['items'])):
718
+ if len(resources['items'][i]) > 2:
719
+ resource_id = resources['items'][i]['id']
720
+ integration = api.get_integration(
721
+ restApiId=api_id,
722
+ resourceId=resource_id,
723
+ httpMethod='POST'
724
+ )
725
+ uri=integration['uri']
726
+ ans1=uri.split('functions/')
727
+ lambda_arn = ans1[1].split('/invocations')[0]
728
+ lambda_arns.append(lambda_arn)
729
+ else:
730
+ pass
731
+
732
+ # get authorizer arn
733
+ authorizers = api.get_authorizers(
734
+ restApiId=api_id
735
+ )
736
+
737
+ authorizer_full_arn=authorizers['items'][0]['authorizerUri']
738
+ ans1=authorizer_full_arn.split('functions/')
739
+ auth_arn = ans1[1].split('/invocations')[0]
740
+ lambda_arns.append(auth_arn)
741
+
742
+ # delete lambdas & authorizer
743
+ client = boto3.client('lambda', region_name=os.environ.get("AWS_REGION"))
744
+ for arn in lambda_arns:
745
+ lambda_response = client.delete_function(
746
+ FunctionName = arn
747
+ )
748
+
749
+ # delete api
750
+ client = boto3.client('apigateway', region_name=os.environ.get("AWS_REGION"))
751
+ api_response = client.delete_rest_api(
752
+ restApiId=api_id
753
+ )
754
+ import requests
755
+
756
+ # delete api page on front end
757
+ bodydata = {'apiurl': apiurl,
758
+ 'delete': "TRUE"
759
+ }
760
+ headers_with_authentication = {'Content-Type': 'application/json', 'authorizationToken': os.environ.get("JWT_AUTHORIZATION_TOKEN"), 'Access-Control-Allow-Headers':
761
+ 'Content-Type,X-Amz-Date,authorizationToken,Access-Control-Allow-Origin,X-Api-Key,X-Amz-Security-Token,Authorization', 'Access-Control-Allow-Origin': '*'}
762
+
763
+ requests.post("https://bhrdesksak.execute-api.us-east-1.amazonaws.com/dev/modeldata",
764
+ json=bodydata, headers=headers_with_authentication)
765
+
766
+ # delete competition posting
767
+ bodydata = {"apiurl": apiurl,
768
+ "delete":"TRUE",
769
+ "experiment":"FALSE"
770
+ }
771
+
772
+ # Get the response
773
+ headers_with_authentication = {'Content-Type': 'application/json', 'authorizationToken': os.environ.get("AWS_TOKEN"), 'Access-Control-Allow-Headers':
774
+ 'Content-Type,X-Amz-Date,authorizationToken,Access-Control-Allow-Origin,X-Api-Key,X-Amz-Security-Token,Authorization', 'Access-Control-Allow-Origin': '*'}
775
+ # competitiondata lambda function invoked through below url to update model submissions and contributors
776
+ requests.post("https://o35jwfakca.execute-api.us-east-1.amazonaws.com/dev/modeldata",
777
+ json=bodydata, headers=headers_with_authentication)
778
+
779
+ # delete experiment posting
780
+ bodydata = {"apiurl": apiurl,
781
+ "delete":"TRUE",
782
+ "experiment":"TRUE"
783
+ }
784
+
785
+ # Get the response
786
+ headers_with_authentication = {'Content-Type': 'application/json', 'authorizationToken': os.environ.get("AWS_TOKEN"), 'Access-Control-Allow-Headers':
787
+ 'Content-Type,X-Amz-Date,authorizationToken,Access-Control-Allow-Origin,X-Api-Key,X-Amz-Security-Token,Authorization', 'Access-Control-Allow-Origin': '*'}
788
+ # competitiondata lambda function invoked through below url to update model submissions and contributors
789
+ requests.post("https://o35jwfakca.execute-api.us-east-1.amazonaws.com/dev/modeldata",
790
+ json=bodydata, headers=headers_with_authentication)
791
+
792
+ # Delete competition data container image
793
+ try:
794
+ content_object = s3.Object(bucket_name=api_bucket, key=model_id + "/competitionuserdata.json")
795
+ file_content = content_object.get()['Body'].read().decode('utf-8')
796
+ json_content = json.loads(file_content)
797
+ ecr_uri=json_content['datauri']
798
+
799
+ ecr_client = user_sess.client('ecr-public')
800
+
801
+ repository_image = ecr_uri.split('/')[2]
802
+
803
+ repository = repository_image.split(':')[0]
804
+ image = repository_image.split(':')[1]
805
+
806
+ response = ecr_client.batch_delete_image(
807
+ repositoryName=repository,
808
+ imageIds=[
809
+ {
810
+ 'imageTag': image
811
+ }
812
+ ]
813
+ )
814
+
815
+ image_details = ecr_client.describe_images(
816
+ repositoryName=repository
817
+ )
818
+
819
+ if len(image_details['imageDetails'])==0:
820
+ response = ecr_client.delete_repository(
821
+ repositoryName=repository
822
+ )
823
+ except:
824
+ pass
825
+ # delete s3 folder
826
+ bucket = s3.Bucket(api_bucket)
827
+ bucket.objects.filter(Prefix= model_id+'/').delete()
828
+
829
+ return "Deployment deleted successfully."
830
+
831
+
832
+ __all__ = [
833
+ get_api_json,
834
+ create_prediction_api,
835
+ delete_deployment
836
+ ]