aimodelshare 0.3.7__py3-none-any.whl

aimodelshare/pyspark/eval_regression.txt

@@ -0,0 +1,80 @@
+import boto3
+import pandas as pd
+import os
+import numpy as np
+import onnxruntime as rt
+import json
+import sklearn
+from sklearn.metrics import accuracy_score
+from sklearn.metrics import f1_score
+from sklearn.metrics import precision_score
+from sklearn.metrics import recall_score
+from sklearn.metrics import roc_auc_score
+from sklearn.metrics import mean_squared_error
+from sklearn.metrics import r2_score
+from sklearn.metrics import mean_absolute_error
+from math import sqrt
+import pickle
+import six
+
+def get_ytestdata(ytest_s3_filename="ytest.pkl"):
+  s3 = boto3.resource("s3")
+  bucket = s3.Bucket("$bucket_name")
+
+  with open("/tmp/ytest.pkl", "wb") as ytestfo:
+      bucket.download_fileobj("$unique_model_id/ytest.pkl",  ytestfo)
+  ytestdata = pickle.load(open("/tmp/ytest.pkl", "rb" ) )
+  return ytestdata
+
+def model_eval_metrics(y_true, y_pred,classification="TRUE"):
+     if classification=="TRUE":
+        accuracy_eval = accuracy_score(y_true, y_pred)
+        f1_score_eval = f1_score(y_true, y_pred,average="macro",zero_division=0)
+        precision_eval = precision_score(y_true, y_pred,average="macro",zero_division=0)
+        recall_eval = recall_score(y_true, y_pred,average="macro",zero_division=0)
+        mse_eval = 0
+        rmse_eval = 0
+        mae_eval = 0
+        r2_eval = 0
+        metricdata = {'accuracy': [accuracy_eval], 'f1_score': [f1_score_eval], 'precision': [precision_eval], 'recall': [recall_eval], 'mse': [mse_eval], 'rmse': [rmse_eval], 'mae': [mae_eval], 'r2': [r2_eval]}
+        finalmetricdata = pd.DataFrame.from_dict(metricdata)
+     else:
+        accuracy_eval = 0
+        f1_score_eval = 0
+        precision_eval = 0
+        recall_eval = 0
+        mse_eval = mean_squared_error(y_true, y_pred)
+        rmse_eval = sqrt(mean_squared_error(y_true, y_pred))
+        mae_eval = mean_absolute_error(y_true, y_pred)
+        r2_eval = r2_score(y_true, y_pred)
+        metricdata = {'accuracy': [accuracy_eval], 'f1_score': [f1_score_eval], 'precision': [precision_eval], 'recall': [recall_eval], 'mse': [mse_eval], 'rmse': [rmse_eval], 'mae': [mae_eval], 'r2': [r2_eval]}
+        finalmetricdata = pd.DataFrame.from_dict(metricdata)
+     return finalmetricdata.to_dict('records')[0]
+
+ytestdata=get_ytestdata(ytest_s3_filename="ytest.pkl")
+
+def evaluate_model(event,ytestdata):
+    body = event["body"]
+    print(body)
+    import six
+    if isinstance(event["body"], six.string_types):
+        prediction_list = json.loads(event["body"])
+        print(prediction_list)
+    else:
+        prediction_list = event["body"]
+        print(prediction_list)
+
+    result=model_eval_metrics(ytestdata,prediction_list,classification="FALSE")
+    return result
+
+def handler(event, context):
+        result = evaluate_model(event,ytestdata)
+        return {"statusCode": 200,
+        "headers": {
+        "Access-Control-Allow-Origin" : "*",
+        "Access-Control-Allow-Credentials": True,
+        "Allow" : "GET, OPTIONS, POST",
+        "Access-Control-Allow-Methods" : "GET, OPTIONS, POST",
+        "Access-Control-Allow-Headers" : "*"
+        },
+        "body": json.dumps(result)}

aimodelshare/pyspark/lambda_function.txt

@@ -0,0 +1,8 @@
+import json
+from model import handler
+
+def lambda_handler(event, context):
+
+    print("lambda_handler from main")
+
+    return handler(event, context)

aimodelshare/pyspark/nst.txt

@@ -0,0 +1,213 @@
+#Neural Style Transfer Prediction Runtime Code
+
+import boto3
+import cv2
+import os
+import numpy as np
+import json
+import onnxruntime as rt
+import base64
+import imghdr
+import six
+from functools import partial
+import os.path
+from os import path
+from io import BytesIO
+from PIL import Image
+import pyspark
+import pyspark.ml
+import pandas as pd
+
+def get_model_onnx(runtimemodel_s3_filename="runtime_model.onnx"):
+    s3 = boto3.resource('s3')
+    obj = s3.Object("$bucket_name", "$unique_model_id" +
+                    "/runtime_model.onnx")
+    model = rt.InferenceSession(obj.get()['Body'].read())
+    return model
+
+def _get_pyspark_modules():
+    import re
+    pyspark_modules = ['ml', 'ml.feature', 'ml.classification', 'ml.clustering', 'ml.regression']
+
+    models_modules_dict = {}
+
+    for i in pyspark_modules:
+        models_list = [j for j in dir(eval('pyspark.'+i)) if callable(getattr(eval('pyspark.'+i), j))]
+        models_list = [j for j in models_list if re.match('^[A-Z]', j)]
+
+        for k in models_list: 
+            models_modules_dict[k] = 'pyspark.'+i
+    
+    return models_modules_dict
+
+def pyspark_model_from_string(model_type):
+    import importlib
+
+    models_modules_dict = _get_pyspark_modules()
+    module = models_modules_dict[model_type]
+    model_class = getattr(importlib.import_module(module), model_type)
+    return model_class
+
+def get_preprocessor(preprocessor_s3_filename="runtime_preprocessor.zip"):
+    import os
+    import pickle
+    import tempfile
+    from io import BytesIO
+    from pathlib import Path
+    from zipfile import ZipFile
+    from pyspark.sql import SparkSession
+    
+
+    #create temporary folder
+    temp_dir = tempfile.gettempdir()
+
+    s3 = boto3.resource("s3")
+    bucket = s3.Bucket("$bucket_name")
+    
+    zip_obj = s3.Object(bucket_name="$bucket_name",
+                        key="$unique_model_id/runtime_preprocessor.zip")
+    buffer = BytesIO(zip_obj.get()["Body"].read())
+    z = ZipFile(buffer)
+    # Extract all the contents of zip file in temp directory
+    z.extractall(temp_dir)
+    
+    # Then import all pkl files you want from bucket (need to generate this list from
+    # function globals
+    pickle_file_list = []
+    zip_file_list = []
+    for file in os.listdir(temp_dir):
+        if file.endswith(".pkl"):
+            pickle_file_list.append(os.path.join(temp_dir, file))
+        if file.endswith(".zip"):
+            zip_file_list.append(os.path.join(temp_dir, file))
+    
+    for i in pickle_file_list: 
+        objectname = str(os.path.basename(i)).replace(".pkl", "")
+        objects = { objectname: "" }
+        globals()[objectname] = pickle.load(open(str(i), "rb"))
+    
+    # Need spark session and context to instantiate model object
+    # zip_file_list is only used by pyspark
+    if len(zip_file_list):
+        spark = SparkSession \
+            .builder \
+            .appName('Pyspark Model') \
+            .getOrCreate()
+    
+    for i in zip_file_list:
+        objectnames = str(os.path.basename(i)).replace(".zip", "").split("__")
+        dir_path = i.replace(".zip", "")
+        Path(dir_path).mkdir(parents=True, exist_ok=True)
+          
+        # Create a ZipFile Object and load module.zip in it
+        with ZipFile(i, 'r') as zipObj:
+            # Extract all the contents of zip file in current directory
+            zipObj.extractall(dir_path)
+
+        preprocessor_type = objectnames[0].split("_")[0]
+        objectname = objectnames[1]
+        preprocessor_class = pyspark_model_from_string(preprocessor_type)
+        if preprocessor_type == "PipelineModel":
+            print(preprocessor_class)
+            preprocessor_model = preprocessor_class(stages=None)
+        else:
+            preprocessor_model = preprocessor_class()
+
+        preprocessor_model = preprocessor_model.load(dir_path)
+        globals()[objectname] = preprocessor_model
+
+      # First import preprocessor function to session from preprocessor.py
+    exec(open(os.path.join(temp_dir, 'preprocessor.py')).read(),globals())
+    return preprocessor
+
+def get_runtimedata(runtimedata_s3_filename="runtime_data.json"):
+
+    s3 = boto3.resource('s3')
+    obj = s3.Object("$bucket_name", "$unique_model_id"+"/"+runtimedata_s3_filename)
+    runtime_data = json.load(obj.get()['Body'])
+
+    return runtime_data
+
+
+
+runtime_data=get_runtimedata(runtimedata_s3_filename="runtime_data.json")
+
+preprocessor_type=runtime_data["runtime_preprocessor"]
+
+runtime_model=runtime_data["runtime_model"]["name"]
+
+model=get_model_onnx(runtimemodel_s3_filename='runtime_model.onnx')
+
+# Load preprocessor
+
+preprocessor=get_preprocessor(preprocessor_s3_filename="runtime_preprocessor.zip")
+
+
+
+def predict(event,model,preprocessor):
+
+# Load base64 encoded image stored within "data" key of event dictionary
+    body = event["body"]
+
+    if isinstance(event["body"], six.string_types):
+        body = json.loads(event["body"])
+
+    content= body["content"]
+    style= body["style"]
+
+# Extract image file extension (e.g.-jpg, png, etc.)
+
+    sample_content = base64.decodebytes(bytearray(content, "utf-8"))
+    sample_style = base64.decodebytes(bytearray(style, "utf-8"))
+
+    content_file_type=None
+    for tf in imghdr.tests:
+        image_file_type = tf(sample_content, None)
+        if image_file_type:
+            break
+    content_file_type=image_file_type
+    if(content_file_type==None):
+        print("This file is not an image, please submit an image base64 encoded image file.")
+
+    style_file_type=None
+    for tf in imghdr.tests:
+        image_file_type = tf(sample_style, None)
+        if image_file_type:
+            break
+    style_file_type=image_file_type
+    if(style_file_type==None):
+        print("This file is not an image, please submit an image base64 encoded image file.")
+
+# Save image to local file, read into session, and preprocess image with preprocessor function
+    with open("/tmp/imagetopredict."+content_file_type, "wb") as fh:
+        fh.write(base64.b64decode(content))
+    content_image = preprocessor("/tmp/imagetopredict."+content_file_type)
+    with open("/tmp/imagetopredict."+style_file_type, "wb") as fh:
+        fh.write(base64.b64decode(style))
+    style_image = preprocessor("/tmp/imagetopredict."+style_file_type)
+
+# Generate prediction using preprocessed input data
+    print("The model expects input shape:", model.get_inputs()[0].shape)
+    input_name = model.get_inputs()[0].name
+    res = model.run(None, {
+        "placeholder": content_image,
+        "placeholder_1": style_image
+    })
+    
+    pil_img = Image.fromarray(np.uint8(res[0].squeeze(axis=0)*255))
+    buffered = BytesIO()
+    pil_img.save(buffered, format="JPEG")
+    img_str = base64.b64encode(buffered.getvalue())
+    return img_str.decode("utf-8")
+
+def handler(event, context):
+    result = predict(event,model,preprocessor)
+    return {"statusCode": 200,
+    "headers": {
+    "Access-Control-Allow-Origin" : "*",
+    "Access-Control-Allow-Credentials": True,
+    "Allow" : "GET, OPTIONS, POST",
+    "Access-Control-Allow-Methods" : "GET, OPTIONS, POST",
+    "Access-Control-Allow-Headers" : "*"
+    },
+    "body" : json.dumps(result)}

aimodelshare/python/my_preprocessor.py

@@ -0,0 +1,58 @@
+import pandas as pd
+import numpy as np
+
+from sklearn.compose import ColumnTransformer
+from sklearn.pipeline import Pipeline
+from sklearn.impute import SimpleImputer
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.linear_model import LogisticRegression
+from sklearn.model_selection import train_test_split, GridSearchCV
+
+np.random.seed(0)
+
+# Read data from Titanic dataset.
+titanic_url = ('https://raw.githubusercontent.com/amueller/'
+               'scipy-2017-sklearn/091d371/notebooks/datasets/titanic3.csv')
+data = pd.read_csv(titanic_url)
+
+# We will train our classifier with the following features:
+# Numeric Features:
+# - age: float.
+# - fare: float.
+# Categorical Features:
+# - embarked: categories encoded as strings {'C', 'S', 'Q'}.
+# - sex: categories encoded as strings {'female', 'male'}.
+# - pclass: ordinal integers {1, 2, 3}.
+
+# We create the preprocessing pipelines for both numeric and categorical data.
+numeric_features = ['age', 'fare']
+numeric_transformer = Pipeline(steps=[
+    ('imputer', SimpleImputer(strategy='median')),
+    ('scaler', StandardScaler())])
+
+categorical_features = ['embarked', 'sex', 'pclass']
+
+# Replacing missing values with Modal value and then one-hot encoding.
+categorical_transformer = Pipeline(steps=[
+    ('imputer', SimpleImputer(strategy='most_frequent')),
+    ('onehot', OneHotEncoder(handle_unknown='ignore'))])
+
+# Final preprocessor object set up with ColumnTransformer...
+
+preprocess = ColumnTransformer(
+    transformers=[
+        ('num', numeric_transformer, numeric_features),
+        ('cat', categorical_transformer, categorical_features)])
+
+
+X = data.drop('survived', axis=1)
+X = data.drop('name', axis=1)
+y = data['survived']
+
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+preprocess = preprocess.fit(X_train)
+def preprocessor(data):
+    import sklearn
+    preprocessed_data=preprocess.transform(data)
+    return preprocessed_data

aimodelshare/readme.md

@@ -0,0 +1,26 @@
+<p align="center"><img width="40%" src="https://github.com/AIModelShare/aimodelshare/blob/master/docs/aimodshare_banner.jpg" /></p>
+
+### The mission of the AI Model Share Platform is to provide a trusted non profit repository for machine learning model prediction APIs (python library + integrated website at modelshare.org).  A beta version of the platform is currently being used by Columbia University students, faculty, and staff to test and improve platform functionality.
+
+### In a matter of seconds, data scientists can launch a model into this infrastructure and end-users the world over will be able to engage their machine learning models.
+
+* ***Launch machine learning models into scalable production ready prediction REST APIs using a single Python function.*** 
+
+* ***Details about each model, how to use the model's API, and the model's author(s) are deployed simultaneously into a searchable website at modelshare.org.*** 
+
+* ***Deployed models receive an individual Model Playground listing information about all deployed models. Each of these pages includes a fully functional prediction dashboard that allows end-users to input text, tabular, or image data and receive live predictions.*** 
+
+* ***Moreover, users can build on model playgrounds by 1) creating ML model competitions, 2) uploading Jupyter notebooks to share code, 3) sharing model architectures and 4) sharing data... with all shared artifacts automatically creating a data science user portfolio.*** 
+
+# Use aimodelshare Python library to deploy your model, create a new ML competition, and more.
+* [Tutorials for deploying models](https://www.modelshare.org/search/deploy?search=ALL&problemdomain=ALL&gettingstartedguide=TRUE&pythonlibrariesused=ALL&tags=ALL&pageNum=1).
+
+# Find model playground web-dashboards to generate predictions now.
+* [View deployed models and generate predictions at modelshare.org](https://www.modelshare.org)
+
+# Installation
+
+You can then install aimodelshare from PyPi
+```
+pip install aimodelshare
+```

aimodelshare/reproducibility.py

@@ -0,0 +1,181 @@
+import os
+import sys
+import json
+import random
+import tempfile
+import requests
+
+import numpy as np
+
+# TensorFlow is optional - only needed for reproducibility setup with TF models
+try:
+    import tensorflow as tf
+    _TF_AVAILABLE = True
+except ImportError:
+    _TF_AVAILABLE = False
+    tf = None
+
+try:
+    import importlib.metadata as md
+except ImportError:  # pragma: no cover
+    import importlib_metadata as md
+
+from aimodelshare.aws import get_s3_iam_client, run_function_on_lambda, get_aws_client
+
+def export_reproducibility_env(seed, directory, mode="gpu"):
+  # Change the output into json.dumps
+  # Argument single seed for all inputs & mode
+  data = {
+    "global_seed_code": [
+      "os.environ['PYTHONHASHSEED'] = '{}'".format(seed),
+      "random.seed({})".format(seed),
+      "tf.random.set_seed({})".format(seed),
+      "np.random.seed({})".format(seed),
+    ]
+  }
+
+  # Ignore this part for now
+  # Local seed codes are tensorflow code that are
+  # not affected by the global seed and sometimes require us
+  # to define the seed in the function call
+  data["local_seed_code"] = [
+    "train_ds = tf.keras.preprocessing.image_dataset_from_directory(data_dir, validation_split=0.2, subset='training', seed={}, image_size=(img_height, img_width), batch_size=batch_size)".format(seed),
+    "val_ds = tf.keras.preprocessing.image_dataset_from_directory(data_dir, validation_split=0.2, subset='validation', seed={}, image_size=(img_height, img_width), batch_size=batch_size)".format(seed),
+  ]
+
+  if mode == "gpu":
+    data["gpu_cpu_parallelism_ops"] = [
+      "os.environ['TF_DETERMINISTIC_OPS'] = '1'",
+      "os.environ['TF_CUDNN_DETERMINISTIC'] = '1'"
+    ]
+  elif mode == "cpu":
+    data["gpu_cpu_parallelism_ops"] = [
+      "tf.config.threading.set_inter_op_parallelism_threads(1)"
+    ]
+  else:
+    raise Exception("Error: unknown 'mode' value, expected 'gpu' or 'cpu'")
+
+  # Get installed packages using importlib.metadata
+  installed_packages_list = []
+  for dist in md.distributions():
+    name = dist.metadata.get("Name") or "unknown"
+    version = dist.version
+    installed_packages_list.append(f"{name}=={version}")
+  installed_packages_list = sorted(installed_packages_list)
+
+  data["session_runtime_info"] = {
+    "installed_packages": installed_packages_list,
+    "python_version": sys.version,
+  }
+
+  with open(os.path.join(directory, "reproducibility.json"), "w") as fp:
+    json.dump(data, fp)
+
+  return print("Your reproducibility environment is now saved to 'reproducibility.json'")
+
+def set_reproducibility_env(reproducibility_env):
+  # Change the input into dict / json
+  for global_code in reproducibility_env["global_seed_code"]:
+    exec("%s" % (global_code))
+
+  for parallelism_ops in reproducibility_env["gpu_cpu_parallelism_ops"]:
+    exec("%s" % (parallelism_ops))
+
+def import_reproducibility_env(reproducibility_env_file):
+  with open(reproducibility_env_file) as json_file:
+    reproducibility_env = json.load(json_file)
+    set_reproducibility_env(reproducibility_env)
+
+  print("Your reproducibility environment is successfully setup")
+
+def import_reproducibility_env_from_competition_model(apiurl,version,submission_type): 
+  # Confirm that creds are loaded, print warning if not
+  if all(["username" in os.environ, 
+        "password" in os.environ]):
+    pass
+  else:
+    return print("Credentials not found. Please provide credentials with set_credentials().")
+
+  post_dict = {
+      "y_pred": [],
+      "return_eval": "False",
+      "return_y": "False",
+      "inspect_model": "False",
+      "version": "None", 
+      "compare_models": "False",
+      "version_list": "None",
+      "get_leaderboard": "False",
+      "instantiate_model": "False",
+      "reproduce": "True",
+      "trained": "False",
+      "model_version": version,
+      "submission_type": submission_type
+  }
+
+  headers = { 'Content-Type':'application/json', 'authorizationToken': os.environ.get("AWS_TOKEN"),} 
+
+  apiurl_eval=apiurl[:-1]+"eval"
+
+  resp = requests.post(apiurl_eval,headers=headers,data=json.dumps(post_dict)) 
+
+  # Check for appropriate response from Lambda. 
+  try :
+      resp.raise_for_status()
+  except requests.exceptions.HTTPError :
+      raise Exception(f"Error: Received {resp.status_code} from AWS, Please check if Model Version is correct.")
+
+  # Load Dictionary
+  resp_dict = json.loads(resp.text)
+
+  # Check if key,value pair exists
+  if resp_dict['reproducibility_env'] != None:
+      set_reproducibility_env(resp_dict['reproducibility_env'])
+      # Store version of reproducibility_environment_set 
+      os.environ['reproducibility_environment_version'] = str(version)
+      print("Your reproducibility environment is successfully setup. Please setup data and then call `replicate_model` ")
+  else:
+      print("Reproducibility environment is not found")
+
+
+def import_reproducibility_env_from_model(apiurl):
+    if all(["AWS_ACCESS_KEY_ID_AIMS" in os.environ, 
+            "AWS_SECRET_ACCESS_KEY_AIMS" in os.environ,
+            "AWS_REGION_AIMS" in os.environ, 
+            "username" in os.environ, 
+            "password" in os.environ]):
+        pass
+    else:
+        return print("'Instantiate Model' unsuccessful. Please provide credentials with set_credentials().")
+
+    aws_client = get_aws_client()
+    reproducibility_env_filename = "/runtime_reproducibility.json"
+
+    # Get bucket and model_id for user
+    response, error = run_function_on_lambda(
+        apiurl, **{"delete": "FALSE", "versionupdateget": "TRUE"}
+    )
+    if error is not None:
+        raise error
+
+    _, bucket, model_id = json.loads(response.content.decode("utf-8"))
+
+    try:
+        resp_string = aws_client["client"].get_object(
+            Bucket=bucket, Key=model_id + reproducibility_env_filename
+        )
+
+        reproducibility_env_string = resp_string['Body'].read()
+
+    except Exception as err:
+        print("This model was not deployed with reproducibility support")
+        raise err
+
+    # generate tempfile for onnx object 
+    temp_dir = tempfile.gettempdir()
+    temp_path = os.path.join(temp_dir, 'temp_file_name')
+
+    # save onnx to temporary path
+    with open(temp_path, "wb") as f:
+        f.write(reproducibility_env_string)
+
+    import_reproducibility_env(temp_path)

aimodelshare/sam/Dockerfile.txt

@@ -0,0 +1,8 @@
+FROM public.ecr.aws/lambda/python:$python_version
+
+COPY $directory/. ./
+COPY $requirements_file_path ./
+
+RUN pip install -r ./requirements.txt
+
+CMD ["lambda_function.lambda_handler"]

aimodelshare/sam/Dockerfile_PySpark.txt

@@ -0,0 +1,24 @@
+FROM public.ecr.aws/lambda/python:$python_version
+
+RUN yum -y install java-1.8.0-openjdk
+
+COPY $directory/. ./
+COPY $requirements_file_path ./
+
+RUN pip install -r ./requirements.txt
+
+ENV SPARK_HOME="/var/lang/lib/python$python_version/site-packages/pyspark"
+ENV PATH=$PATH:$SPARK_HOME/bin
+ENV PATH=$PATH:$SPARK_HOME/sbin
+ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9-src.zip:$PYTHONPATH
+ENV PATH=$SPARK_HOME/python:$PATH
+ENV SPARK_MASTER_HOST="localhost"
+ENV SPARK_LOCAL_IP="127.0.0.1"
+
+ENV JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk*/jre"
+ENV PATH=$PATH:$JAVA_HOME/bin
+
+COPY spark-class $SPARK_HOME/bin/
+RUN chmod +x $SPARK_HOME/bin/spark-class
+
+CMD ["lambda_function.lambda_handler"]

aimodelshare/sam/__init__.py

@@ -0,0 +1 @@
+

aimodelshare/sam/buildspec.txt

@@ -0,0 +1,11 @@
+version: 0.2
+
+phases:
+  pre_build:
+    commands:
+      - aws ecr get-login-password --region $region | docker login --username AWS --password-stdin $account_id.dkr.ecr.$region.amazonaws.com
+  build:
+    commands:
+      - sam build
+      - sam package --output-template-file packaged-template.yml --image-repository $account_id.dkr.ecr.$region.amazonaws.com/$repository_name
+      - sam deploy --template-file packaged-template.yml --stack-name $stack_name --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND --region $region --image-repository $account_id.dkr.ecr.$region.amazonaws.com/$repository_name