aimodelshare 0.1.29__tar.gz → 0.1.31__tar.gz

{aimodelshare-0.1.29/aimodelshare.egg-info → aimodelshare-0.1.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aimodelshare
-Version: 0.1.29
+Version: 0.1.31
 Summary: Deploy locally saved machine learning models to a live rest API and web-dashboard.  Share it with the world via modelshare.org
 Home-page: https://www.modelshare.org
 Author: Michael Parrott

{aimodelshare-0.1.29 → aimodelshare-0.1.31}/aimodelshare/aimsonnx.py

@@ -549,197 +549,146 @@ def _pyspark_to_onnx(model, initial_types, spark_session,
     return onx
 
 def _keras_to_onnx(model, transfer_learning=None,
-                  deep_learning=None, task_type=None, epochs=None):
-    '''Extracts metadata from keras model object.'''
+                   deep_learning=None, task_type=None, epochs=None):
+    '''Converts a Keras model to ONNX and extracts metadata.'''
 
-    # check whether this is a fitted keras model
-    # isinstance...
+    import tf2onnx
+    import tensorflow as tf
+    import numpy as np
+    import onnx
+    import pickle
+    import psutil
+    import warnings
+    from pympler import asizeof
+    import logging
+    import os
+    import sys
+    from contextlib import contextmanager
+
+    # -- Helper to suppress tf2onnx stderr (NumPy warnings etc.)
+    @contextmanager
+    def suppress_stderr():
+        with open(os.devnull, "w") as devnull:
+            old_stderr = sys.stderr
+            sys.stderr = devnull
+            try:
+                yield
+            finally:
+                sys.stderr = old_stderr
+
+    # Reduce logging output
+    tf2onnx_logger = logging.getLogger("tf2onnx")
+    tf2onnx_logger.setLevel(logging.CRITICAL)
+
+    # Unwrap scikeras, sklearn pipelines etc.
+    from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
+    from sklearn.pipeline import Pipeline
+    from scikeras.wrappers import KerasClassifier, KerasRegressor
 
-    # handle keras models in sklearn wrapper
     if isinstance(model, (GridSearchCV, RandomizedSearchCV)):
         model = model.best_estimator_
-
-    if isinstance(model, sklearn.pipeline.Pipeline):
+    if isinstance(model, Pipeline):
         model = model.steps[-1][1]
-
-    sklearn_wrappers = (KerasClassifier,KerasRegressor)
-
-    if isinstance(model, sklearn_wrappers):
+    if isinstance(model, (KerasClassifier, KerasRegressor)):
         model = model.model
-    
-    # convert to onnx
-    #onx = convert_keras(model)
-    # generate tempfile for onnx object 
-    temp_dir = tempfile.mkdtemp()
-
-
-
-    
-    tf.get_logger().setLevel('ERROR') # probably not good practice
-    output_path = os.path.join(temp_dir, 'temp.onnx')
-    
-    
-    model.save(temp_dir)
-
-    # # Convert the model
-    try:
-            modelstringtest="python -m tf2onnx.convert --saved-model  "+temp_dir+" --output "+output_path+" --opset 13"
-            resultonnx=os.system(modelstringtest)
-            resultonnx2=1
-            if resultonnx==0:
-              pass
-            else:
-              raise Exception('Model conversion to onnx unsuccessful.  Please try different model or submit predictions to leaderboard without submitting preprocessor or model files.')
-    except:
-            converter = tf.lite.TFLiteConverter.from_saved_model(temp_dir) # path to the SavedModel directory
-            converter.target_spec.supported_ops = [
-                tf.lite.OpsSet.TFLITE_BUILTINS, # enable TensorFlow Lite ops.
-                tf.lite.OpsSet.SELECT_TF_OPS # enable TensorFlow ops.
-              ]
-            tflite_model = converter.convert()
-
-            # Save the model.
-            with open(os.path.join(temp_dir,'tempmodel.tflite'), 'wb') as f:
-              f.write(tflite_model)
-
-            modelstringtest="python -m tf2onnx.convert --tflite "+os.path.join(temp_dir,'tempmodel.tflite')+" --output "+output_path+" --opset 13"
-            resultonnx2=os.system(modelstringtest)
-            pass
-
-    if any([resultonnx==0, resultonnx2==0]):
-      pass
-    else:
-      return print("Model conversion to onnx unsuccessful.  Please try different model or submit\npredictions to leaderboard without submitting preprocessor or model files.")
-
-    onx = onnx.load(output_path)
-
-
-    # generate metadata dict 
-    metadata = {}
-
-    # placeholders, need to be generated elsewhere
-    metadata['model_id'] = None
-    metadata['data_id'] = None
-    metadata['preprocessor_id'] = None
-
-    # infer ml framework from function call
-    metadata['ml_framework'] = 'keras'
 
-    # get model type from model object
-    metadata['model_type'] =  str(model.__class__.__name__)
-
-    # get transfer learning bool from user input
-    metadata['transfer_learning'] = transfer_learning
-
-    # get deep learning bool from user input
-    metadata['deep_learning'] = deep_learning
-
-    # get task type from user input
-    metadata['task_type'] = task_type
-
-    # placeholders, need to be inferred from data 
-    metadata['target_distribution'] = None
-    metadata['input_type'] = None
-    metadata['input_shape'] = None
-    metadata['input_dtypes'] = None       
-    metadata['input_distribution'] = None
+    # Input signature
+    input_shape = model.input_shape
+    if isinstance(input_shape, list):
+        input_shape = input_shape[0]
+    input_signature = [tf.TensorSpec(input_shape, tf.float32, name="input")]
+
+    # Wrap model in tf.function
+    @tf.function(input_signature=input_signature)
+    def model_fn(x):
+        return model(x)
+
+    concrete_func = model_fn
+
+    # Convert to ONNX
+    with suppress_stderr():
+        onx_model, _ = tf2onnx.convert.from_function(
+            concrete_func,
+            input_signature=input_signature,
+            opset=13,
+            output_path=None
+        )
 
-    # get model config dict from keras model object
-    metadata['model_config'] = str(model.get_config())
+    # Extract metadata
+    metadata = {
+        'model_id': None,
+        'data_id': None,
+        'preprocessor_id': None,
+        'ml_framework': 'keras',
+        'model_type': model.__class__.__name__,
+        'transfer_learning': transfer_learning,
+        'deep_learning': deep_learning,
+        'task_type': task_type,
+        'target_distribution': None,
+        'input_type': None,
+        'input_shape': input_shape,
+        'input_dtypes': None,
+        'input_distribution': None,
+        'model_config': str(model.get_config()),
+        'model_state': None,
+        'eval_metrics': None,
+        'model_graph': "",
+        'metadata_onnx': None,
+        'epochs': epochs
+    }
 
-    # get model weights from keras object 
     model_size = asizeof.asizeof(model.get_weights())
     mem = psutil.virtual_memory()
 
-    if model_size > mem.available: 
-
-        warnings.warn(f"Model size ({model_size/1e6} MB) exceeds available memory ({mem.available/1e6} MB). Skipping extraction of model weights.")
-
+    if model_size > mem.available:
+        warnings.warn(f"Model size ({model_size/1e6} MB) exceeds available memory.")
         metadata['model_weights'] = None
-
-    else: 
-        
+    else:
         metadata['model_weights'] = pickle.dumps(model.get_weights())
 
-    # get model state from pytorch model object
-    metadata['model_state'] = None
-    
-    # get list of current layer types 
-    layer_list, activation_list = _get_layer_names()
+    # Extract architecture
+    from aimodelshare.model import keras_unpack, model_summary_keras
 
-    # extract model architecture metadata 
+    keras_layers = keras_unpack(model)
     layers = []
     layers_n_params = []
     layers_shapes = []
     activations = []
 
-
-    keras_layers = keras_unpack(model)
-
-    for i in keras_layers: 
-        
-        # get layer names 
-        if i.__class__.__name__ in layer_list:
-            layers.append(i.__class__.__name__)
-            layers_n_params.append(i.count_params())
-            layers_shapes.append(i.output_shape)
-        
-        # get activation names
-        if i.__class__.__name__ in activation_list: 
-            activations.append(i.__class__.__name__.lower())
-        if hasattr(i, 'activation') and i.activation.__name__ in activation_list:
-            activations.append(i.activation.__name__)
-
-    if hasattr(model, 'loss'):
-        loss = model.loss.__class__.__name__
-    else:
-        loss = None
-        
-    if hasattr(model, 'optimizer'):
-        optimizer = model.optimizer.__class__.__name__
-    else:
-        optimizer = None 
-
-    model_summary_pd = model_summary_keras(model)
-    
-    # insert data into model architecture dict 
-    model_architecture = {'layers_number': len(layers),
-                          'layers_sequence': layers,
-                          'layers_summary': {i:layers.count(i) for i in set(layers)},
-                          'layers_n_params': layers_n_params,
-                          'layers_shapes': layers_shapes,
-                          'activations_sequence': activations,
-                          'activations_summary': {i:activations.count(i) for i in set(activations)},
-                          'loss':loss,
-                          'optimizer': optimizer
-                         }
+    for layer in keras_layers:
+        layers.append(layer.__class__.__name__)
+        layers_n_params.append(layer.count_params())
+        layers_shapes.append(getattr(layer, 'output_shape', None))
+        if hasattr(layer, 'activation'):
+            act = getattr(layer.activation, '__name__', None)
+            if act: activations.append(act)
+
+    optimizer = getattr(model.optimizer, '__class__', None)
+    loss = getattr(model.loss, '__class__', None)
+
+    model_architecture = {
+        'layers_number': len(layers),
+        'layers_sequence': layers,
+        'layers_summary': {i: layers.count(i) for i in set(layers)},
+        'layers_n_params': layers_n_params,
+        'layers_shapes': layers_shapes,
+        'activations_sequence': activations,
+        'activations_summary': {i: activations.count(i) for i in set(activations)},
+        'loss': loss.__name__ if loss else None,
+        'optimizer': optimizer.__name__ if optimizer else None
+    }
 
     metadata['model_architecture'] = str(model_architecture)
-
-
-    metadata['model_summary'] = model_summary_pd.to_json()
-
+    metadata['model_summary'] = model_summary_keras(model).to_json()
     metadata['memory_size'] = model_size
 
-    metadata['epochs'] = epochs
-
-    # model graph 
-    #G = model_graph_keras(model)
-    #metadata['model_graph'] = G.create_dot().decode('utf-8')
-    metadata['model_graph'] = ""
-    # placeholder, needs evaluation engine
-    metadata['eval_metrics'] = None
-
-    # add metadata from onnx object
-    # metadata['metadata_onnx'] = str(_extract_onnx_metadata(onx, framework='keras'))
-    metadata['metadata_onnx'] = None
-    # add metadata dict to onnx object
-    
-    meta = onx.metadata_props.add()
+    # Embed metadata in ONNX
+    meta = onx_model.metadata_props.add()
     meta.key = 'model_metadata'
     meta.value = str(metadata)
 
-    return onx
+    return onx_model
+
 
 
 def _pytorch_to_onnx(model, model_input, transfer_learning=None, 

{aimodelshare-0.1.29 → aimodelshare-0.1.31}/aimodelshare/data_sharing/download_data.py

@@ -76,110 +76,87 @@ def download_layer(layer, layer_count, tmp_img_dir, blobs_resp):
 	return layer_id, layer_dir
 
 def pull_image(image_uri):
-    import os
-    import requests
-    import tempfile
-    import json
-    import shutil
-    import tarfile
-    from aimodelshare.data_sharing.utils import redo_with_write
-
-    image_uri_parts = image_uri.split('/')
-
-    registry = image_uri_parts[0]    
-    image, tag = image_uri_parts[2].split(':')
-    repository = '/'.join([image_uri_parts[1], image])
-
-    auth_url = get_auth_url(registry)
-
-    # Request manifest with correct Accept header
-    auth_head = get_auth_head(auth_url, registry, repository)
-    manifest_url = f'https://{registry}/v2/{repository}/manifests/{tag}'
-    resp = requests.get(manifest_url, headers=auth_head, verify=False)
-
-    # --- PATCH: Handle manifest list (multi-platform images) ---
-    if resp.headers.get('Content-Type') == 'application/vnd.docker.distribution.manifest.list.v2+json':
-        manifest_list = resp.json()
-
-        # Find the first linux/amd64 image (or fallback to first available)
-        target_manifest = next(
-            (m for m in manifest_list['manifests']
-             if m['platform'].get('architecture') == 'amd64' and m['platform'].get('os') == 'linux'),
-            manifest_list['manifests'][0]
-        )
-        digest = target_manifest['digest']
-
-        # Get the actual image manifest now
-        resp = requests.get(
-            f'https://{registry}/v2/{repository}/manifests/{digest}',
-            headers=auth_head,
-            verify=False
-        )
-    # -----------------------------------------------------------
-
-    manifest = resp.json()
-
-    # Safely check and fail early if config key is still missing
-    if 'config' not in manifest:
-        raise ValueError("Manifest response missing 'config'. This image may not follow Docker V2 manifest schema.")
-
-    config = manifest['config']['digest']
-    config_resp = requests.get(f'https://{registry}/v2/{repository}/blobs/{config}', headers=auth_head, verify=False)
-
-    tmp_img_dir = os.path.join(tempfile.gettempdir(), f'tmp_{image}_{tag}')
-    os.mkdir(tmp_img_dir)
-
-    with open(f'{tmp_img_dir}/{config[7:]}.json', 'wb') as file:
-        file.write(config_resp.content)
-
-    content = [{
-        'Config': config[7:] + '.json',
-        'RepoTags': [image_uri],
-        'Layers': []
-    }]
-
-    # Skip first 6 layers? Keep original logic for compatibility
-    layers = manifest['layers'][6:]
-    layer_count = 0
-
-    for layer in layers:
-        layer_count += 1
-        auth_head = get_auth_head(auth_url, registry, repository)
-        blobs_resp = requests.get(
-            f'https://{registry}/v2/{repository}/blobs/{layer["digest"]}',
-            headers=auth_head,
-            stream=True,
-            verify=False
-        )
-
-        layer_id, layer_dir = download_layer(layer, layer_count, tmp_img_dir, blobs_resp)
-        content[0]['Layers'].append(layer_id + '/layer.tar')
-
-        json_path = os.path.join(layer_dir, 'json')
-        with open(json_path, 'w') as file:
-            if layers[-1]['digest'] == layer['digest']:
-                json_obj = json.loads(config_resp.content)
-                json_obj.pop('history', None)
-                json_obj.pop('rootfs', None)
-            else:
-                json_obj = {}
-            json_obj['id'] = layer_id
-            file.write(json.dumps(json_obj))
-
-    with open(os.path.join(tmp_img_dir, 'manifest.json'), 'w') as f:
-        f.write(json.dumps(content))
-
-    repo_dict = {'/'.join(image_uri_parts[:-1]) + '/' + image: {tag: layer_id}}
-    with open(os.path.join(tmp_img_dir, 'repositories'), 'w') as f:
-        f.write(json.dumps(repo_dict))
-
-    # Create tar archive from temp image directory
-    docker_tar = os.path.join(tempfile.gettempdir(), f'{repository.replace("/", "_")}_{tag}.tar')
-    with tarfile.open(docker_tar, "w") as tar:
-        tar.add(tmp_img_dir, arcname=os.path.sep)
-
-    shutil.rmtree(tmp_img_dir, onerror=redo_with_write)
-    return docker_tar
+
+	image_uri_parts = image_uri.split('/')
+
+	registry = image_uri_parts[0]	
+	image, tag = image_uri_parts[2].split(':')
+	repository = '/'.join([image_uri_parts[1], image])
+
+	auth_url = get_auth_url(registry)
+
+	auth_head = get_auth_head(auth_url, registry, repository)
+
+	resp = requests.get('https://{}/v2/{}/manifests/{}'.format(registry, repository, tag), headers=auth_head, verify=False)
+	
+	config = resp.json()['config']['digest']
+	config_resp = requests.get('https://{}/v2/{}/blobs/{}'.format(registry, repository, config), headers=auth_head, verify=False)
+
+	tmp_img_dir = tempfile.gettempdir() + '/' + 'tmp_{}_{}'.format(image, tag)
+	os.mkdir(tmp_img_dir)
+
+	file = open('{}/{}.json'.format(tmp_img_dir, config[7:]), 'wb')
+	file.write(config_resp.content)
+	file.close()
+
+	content = [{
+		'Config': config[7:] + '.json',
+		'RepoTags': [],
+		'Layers': []
+	}]
+	content[0]['RepoTags'].append(image_uri)
+
+	layer_count=0
+	layers = resp.json()['layers'][6:]
+
+	for layer in layers:
+
+		layer_count += 1
+
+		auth_head = get_auth_head(auth_url, registry, repository) # done to keep from expiring
+		blobs_resp = requests.get('https://{}/v2/{}/blobs/{}'.format(registry, repository, layer['digest']), headers=auth_head, stream=True, verify=False)
+
+		layer_id, layer_dir = download_layer(layer, layer_count, tmp_img_dir, blobs_resp)
+		content[0]['Layers'].append(layer_id + '/layer.tar')
+
+		# Creating json file
+		file = open(layer_dir + '/json', 'w')
+
+		# last layer = config manifest - history - rootfs
+		if layers[-1]['digest'] == layer['digest']:
+			json_obj = json.loads(config_resp.content)
+			del json_obj['history']
+			del json_obj['rootfs']
+		else: # other layers json are empty
+			json_obj = json.loads('{}')
+		
+		json_obj['id'] = layer_id
+		file.write(json.dumps(json_obj))
+		file.close()
+
+	file = open(tmp_img_dir + '/manifest.json', 'w')
+	file.write(json.dumps(content))
+	file.close()
+
+	content = {
+		'/'.join(image_uri_parts[:-1]) + '/' + image : { tag : layer_id }
+	}
+
+	file = open(tmp_img_dir + '/repositories', 'w')
+	file.write(json.dumps(content))
+	file.close()
+
+	# Create image tar and clean tmp folder
+	docker_tar = tempfile.gettempdir() + '/' + '_'.join([repository.replace('/', '_'), tag]) + '.tar'
+	sys.stdout.flush()
+
+	tar = tarfile.open(docker_tar, "w")
+	tar.add(tmp_img_dir, arcname=os.path.sep)
+	tar.close()
+
+	shutil.rmtree(tmp_img_dir, onerror=redo_with_write)
+
+	return docker_tar
 
 
 def extract_data_from_image(image_name, file_name, location):
@@ -219,10 +196,11 @@ def import_quickstart_data(tutorial, section="modelplayground"):
    
     #Download Quick Start materials
     if all([tutorial == "flowers", section == "modelplayground"]):
-        quickstart_repository = "public.ecr.aws/y2e2a1d6/quickstart_materials-repository:latest"   
+        quickstart_repository = "public.ecr.aws/z5w0c9e9/quickstart_materials-repository:latest"
         existing_folder = 'flower_competition_data'
+
     if all([tutorial == "flowers", section == "competition"]):
-        quickstart_repository = "public.ecr.aws/y2e2a1d6/quickstart_flowers_competition-repository:latest"
+        quickstart_repository = "public.ecr.aws/z5w0c9e9/quickstart_flowers_competition-repository:latest"
         existing_folder = 'flower_competition_data'
     
     if all([tutorial == "mnist", section == "modelplayground"]):

{aimodelshare-0.1.29 → aimodelshare-0.1.31/aimodelshare.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aimodelshare
-Version: 0.1.29
+Version: 0.1.31
 Summary: Deploy locally saved machine learning models to a live rest API and web-dashboard.  Share it with the world via modelshare.org
 Home-page: https://www.modelshare.org
 Author: Michael Parrott

{aimodelshare-0.1.29 → aimodelshare-0.1.31}/setup.py

@@ -6,7 +6,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name='aimodelshare', #TODO:update
-    version='0.1.29',        #TODO:update
+    version='0.1.31',        #TODO:update
     author="Michael Parrott",
     author_email="mikedparrott@modelshare.org",
     description="Deploy locally saved machine learning models to a live rest API and web-dashboard.  Share it with the world via modelshare.org",