aimodelshare 0.1.30__py3-none-any.whl → 0.1.32__py3-none-any.whl

aimodelshare/aimsonnx.py

@@ -549,197 +549,145 @@ def _pyspark_to_onnx(model, initial_types, spark_session,
     return onx
 
 def _keras_to_onnx(model, transfer_learning=None,
-                  deep_learning=None, task_type=None, epochs=None):
-    '''Extracts metadata from keras model object.'''
+                   deep_learning=None, task_type=None, epochs=None):
+    '''Converts a Keras model to ONNX and extracts metadata.'''
 
-    # check whether this is a fitted keras model
-    # isinstance...
+    import tf2onnx
+    import tensorflow as tf
+    import numpy as np
+    import onnx
+    import pickle
+    import psutil
+    import warnings
+    from pympler import asizeof
+    import logging
+    import os
+    import sys
+    from contextlib import contextmanager
+
+    # -- Helper to suppress tf2onnx stderr (NumPy warnings etc.)
+    @contextmanager
+    def suppress_stderr():
+        with open(os.devnull, "w") as devnull:
+            old_stderr = sys.stderr
+            sys.stderr = devnull
+            try:
+                yield
+            finally:
+                sys.stderr = old_stderr
+
+    # Reduce logging output
+    tf2onnx_logger = logging.getLogger("tf2onnx")
+    tf2onnx_logger.setLevel(logging.CRITICAL)
+
+    # Unwrap scikeras, sklearn pipelines etc.
+    from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
+    from sklearn.pipeline import Pipeline
+    from scikeras.wrappers import KerasClassifier, KerasRegressor
 
-    # handle keras models in sklearn wrapper
     if isinstance(model, (GridSearchCV, RandomizedSearchCV)):
         model = model.best_estimator_
-
-    if isinstance(model, sklearn.pipeline.Pipeline):
+    if isinstance(model, Pipeline):
         model = model.steps[-1][1]
-
-    sklearn_wrappers = (KerasClassifier,KerasRegressor)
-
-    if isinstance(model, sklearn_wrappers):
+    if isinstance(model, (KerasClassifier, KerasRegressor)):
         model = model.model
-    
-    # convert to onnx
-    #onx = convert_keras(model)
-    # generate tempfile for onnx object 
-    temp_dir = tempfile.mkdtemp()
-
-
-
-    
-    tf.get_logger().setLevel('ERROR') # probably not good practice
-    output_path = os.path.join(temp_dir, 'temp.onnx')
-    
-    
-    tf.saved_model.save(model, temp_dir)
 
-    # # Convert the model
-    try:
-            modelstringtest="python -m tf2onnx.convert --saved-model  "+temp_dir+" --output "+output_path+" --opset 13"
-            resultonnx=os.system(modelstringtest)
-            resultonnx2=1
-            if resultonnx==0:
-              pass
-            else:
-              raise Exception('Model conversion to onnx unsuccessful.  Please try different model or submit predictions to leaderboard without submitting preprocessor or model files.')
-    except:
-            converter = tf.lite.TFLiteConverter.from_saved_model(temp_dir) # path to the SavedModel directory
-            converter.target_spec.supported_ops = [
-                tf.lite.OpsSet.TFLITE_BUILTINS, # enable TensorFlow Lite ops.
-                tf.lite.OpsSet.SELECT_TF_OPS # enable TensorFlow ops.
-              ]
-            tflite_model = converter.convert()
-
-            # Save the model.
-            with open(os.path.join(temp_dir,'tempmodel.tflite'), 'wb') as f:
-              f.write(tflite_model)
-
-            modelstringtest="python -m tf2onnx.convert --tflite "+os.path.join(temp_dir,'tempmodel.tflite')+" --output "+output_path+" --opset 13"
-            resultonnx2=os.system(modelstringtest)
-            pass
-
-    if any([resultonnx==0, resultonnx2==0]):
-      pass
-    else:
-      return print("Model conversion to onnx unsuccessful.  Please try different model or submit\npredictions to leaderboard without submitting preprocessor or model files.")
-
-    onx = onnx.load(output_path)
-
-
-    # generate metadata dict 
-    metadata = {}
-
-    # placeholders, need to be generated elsewhere
-    metadata['model_id'] = None
-    metadata['data_id'] = None
-    metadata['preprocessor_id'] = None
-
-    # infer ml framework from function call
-    metadata['ml_framework'] = 'keras'
-
-    # get model type from model object
-    metadata['model_type'] =  str(model.__class__.__name__)
-
-    # get transfer learning bool from user input
-    metadata['transfer_learning'] = transfer_learning
-
-    # get deep learning bool from user input
-    metadata['deep_learning'] = deep_learning
-
-    # get task type from user input
-    metadata['task_type'] = task_type
-
-    # placeholders, need to be inferred from data 
-    metadata['target_distribution'] = None
-    metadata['input_type'] = None
-    metadata['input_shape'] = None
-    metadata['input_dtypes'] = None       
-    metadata['input_distribution'] = None
+    # Input signature
+    input_shape = model.input_shape
+    if isinstance(input_shape, list):
+        input_shape = input_shape[0]
+    input_signature = [tf.TensorSpec(input_shape, tf.float32, name="input")]
+
+    # Wrap model in tf.function
+    @tf.function(input_signature=input_signature)
+    def model_fn(x):
+        return model(x)
+
+    concrete_func = model_fn
+
+    # Convert to ONNX
+    with suppress_stderr():
+        onx_model, _ = tf2onnx.convert.from_function(
+            concrete_func,
+            input_signature=input_signature,
+            opset=13,
+            output_path=None
+        )
 
-    # get model config dict from keras model object
-    metadata['model_config'] = str(model.get_config())
+    # Extract metadata
+    metadata = {
+        'model_id': None,
+        'data_id': None,
+        'preprocessor_id': None,
+        'ml_framework': 'keras',
+        'model_type': model.__class__.__name__,
+        'transfer_learning': transfer_learning,
+        'deep_learning': deep_learning,
+        'task_type': task_type,
+        'target_distribution': None,
+        'input_type': None,
+        'input_shape': input_shape,
+        'input_dtypes': None,
+        'input_distribution': None,
+        'model_config': str(model.get_config()),
+        'model_state': None,
+        'eval_metrics': None,
+        'model_graph': "",
+        'metadata_onnx': None,
+        'epochs': epochs
+    }
 
-    # get model weights from keras object 
     model_size = asizeof.asizeof(model.get_weights())
     mem = psutil.virtual_memory()
 
-    if model_size > mem.available: 
-
-        warnings.warn(f"Model size ({model_size/1e6} MB) exceeds available memory ({mem.available/1e6} MB). Skipping extraction of model weights.")
-
+    if model_size > mem.available:
+        warnings.warn(f"Model size ({model_size/1e6} MB) exceeds available memory.")
         metadata['model_weights'] = None
-
-    else: 
-        
+    else:
         metadata['model_weights'] = pickle.dumps(model.get_weights())
 
-    # get model state from pytorch model object
-    metadata['model_state'] = None
-    
-    # get list of current layer types 
-    layer_list, activation_list = _get_layer_names()
+    # Extract architecture
 
-    # extract model architecture metadata 
+    keras_layers = keras_unpack(model)
     layers = []
     layers_n_params = []
     layers_shapes = []
     activations = []
 
-
-    keras_layers = keras_unpack(model)
-
-    for i in keras_layers: 
-        
-        # get layer names 
-        if i.__class__.__name__ in layer_list:
-            layers.append(i.__class__.__name__)
-            layers_n_params.append(i.count_params())
-            layers_shapes.append(i.output_shape)
-        
-        # get activation names
-        if i.__class__.__name__ in activation_list: 
-            activations.append(i.__class__.__name__.lower())
-        if hasattr(i, 'activation') and i.activation.__name__ in activation_list:
-            activations.append(i.activation.__name__)
-
-    if hasattr(model, 'loss'):
-        loss = model.loss.__class__.__name__
-    else:
-        loss = None
-        
-    if hasattr(model, 'optimizer'):
-        optimizer = model.optimizer.__class__.__name__
-    else:
-        optimizer = None 
-
-    model_summary_pd = model_summary_keras(model)
-    
-    # insert data into model architecture dict 
-    model_architecture = {'layers_number': len(layers),
-                          'layers_sequence': layers,
-                          'layers_summary': {i:layers.count(i) for i in set(layers)},
-                          'layers_n_params': layers_n_params,
-                          'layers_shapes': layers_shapes,
-                          'activations_sequence': activations,
-                          'activations_summary': {i:activations.count(i) for i in set(activations)},
-                          'loss':loss,
-                          'optimizer': optimizer
-                         }
+    for layer in keras_layers:
+        layers.append(layer.__class__.__name__)
+        layers_n_params.append(layer.count_params())
+        layers_shapes.append(getattr(layer, 'output_shape', None))
+        if hasattr(layer, 'activation'):
+            act = getattr(layer.activation, '__name__', None)
+            if act: activations.append(act)
+
+    optimizer = getattr(model.optimizer, '__class__', None)
+    loss = getattr(model.loss, '__class__', None)
+
+    model_architecture = {
+        'layers_number': len(layers),
+        'layers_sequence': layers,
+        'layers_summary': {i: layers.count(i) for i in set(layers)},
+        'layers_n_params': layers_n_params,
+        'layers_shapes': layers_shapes,
+        'activations_sequence': activations,
+        'activations_summary': {i: activations.count(i) for i in set(activations)},
+        'loss': loss.__name__ if loss else None,
+        'optimizer': optimizer.__name__ if optimizer else None
+    }
 
     metadata['model_architecture'] = str(model_architecture)
-
-
-    metadata['model_summary'] = model_summary_pd.to_json()
-
+    metadata['model_summary'] = model_summary_keras(model).to_json()
     metadata['memory_size'] = model_size
 
-    metadata['epochs'] = epochs
-
-    # model graph 
-    #G = model_graph_keras(model)
-    #metadata['model_graph'] = G.create_dot().decode('utf-8')
-    metadata['model_graph'] = ""
-    # placeholder, needs evaluation engine
-    metadata['eval_metrics'] = None
-
-    # add metadata from onnx object
-    # metadata['metadata_onnx'] = str(_extract_onnx_metadata(onx, framework='keras'))
-    metadata['metadata_onnx'] = None
-    # add metadata dict to onnx object
-    
-    meta = onx.metadata_props.add()
+    # Embed metadata in ONNX
+    meta = onx_model.metadata_props.add()
     meta.key = 'model_metadata'
     meta.value = str(metadata)
 
-    return onx
+    return onx_model
+
 
 
 def _pytorch_to_onnx(model, model_input, transfer_learning=None, 
@@ -1844,22 +1792,12 @@ def torch_unpack(model):
 
     
 def keras_unpack(model):
-    
     layers = []
-    
     for module in model.layers:
-                
         if isinstance(module, (tf.keras.Model, tf.keras.Sequential)):
-            
-            layers_out = keras_unpack(module)
-            
-            layers = layers + layers_out
-            
-            
+            layers += keras_unpack(module)
         else:
-            
             layers.append(module)
-            
     return layers
 
 

aimodelshare/data_sharing/download_data.py

@@ -89,7 +89,6 @@ def pull_image(image_uri):
 
 	resp = requests.get('https://{}/v2/{}/manifests/{}'.format(registry, repository, tag), headers=auth_head, verify=False)
 	
-	print(resp.json())
 	config = resp.json()['config']['digest']
 	config_resp = requests.get('https://{}/v2/{}/blobs/{}'.format(registry, repository, config), headers=auth_head, verify=False)
 

{aimodelshare-0.1.30.dist-info → aimodelshare-0.1.32.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aimodelshare
-Version: 0.1.30
+Version: 0.1.32
 Summary: Deploy locally saved machine learning models to a live rest API and web-dashboard.  Share it with the world via modelshare.org
 Home-page: https://www.modelshare.org
 Author: Michael Parrott

{aimodelshare-0.1.30.dist-info → aimodelshare-0.1.32.dist-info}/RECORD

@@ -1,6 +1,6 @@
 aimodelshare/README.md,sha256=_OMdUIeIYZnpFlKdafM1KNWaANO2nWdx0QpLE_ZC-Qs,2014
 aimodelshare/__init__.py,sha256=CS0iFxgAic21gBcQE6NSZ-D_ElHw80_A3OWrEYo9Dks,539
-aimodelshare/aimsonnx.py,sha256=fmOgHVc6SL1cH9zt2EHiPOVLXGunQuteo6XE4cc_LA0,69298
+aimodelshare/aimsonnx.py,sha256=rGhnNBonyrwpNgkuPTJ0hJFx_RGHQnNsCeAERZsPdcU,67129
 aimodelshare/api.py,sha256=jeCIMbpBllTYi-bPvCdNvI6yHaT3JYakq9fgaz10s_E,34920
 aimodelshare/aws.py,sha256=jn99R9-N77Qac-_eYm-LaCQUPd-RnE7oVULm9rh-3RY,15232
 aimodelshare/aws_client.py,sha256=Ce19iwf69BwpuyyJlVN8z1da3c5jf93svsTgx1OWhaA,6784
@@ -31,7 +31,7 @@ aimodelshare/containerization_templates/lambda_function.txt,sha256=nEFoPDXemNcQZ
 aimodelshare/custom_approach/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 aimodelshare/custom_approach/lambda_function.py,sha256=d1HZlgviHZq4mNBKx4q-RCunDK8P8i9DKZcfv6Nmgzc,479
 aimodelshare/data_sharing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-aimodelshare/data_sharing/download_data.py,sha256=Hd9QUKfC2GQqBuJhzjVIHNYz8rksN7Nl3B52U7WIqxA,22815
+aimodelshare/data_sharing/download_data.py,sha256=SshJCz3jJaL1gqZmWjFMOKzUVhijqsisnPGA6jYXbq8,22795
 aimodelshare/data_sharing/share_data.py,sha256=dMOP0-PTSpviOeHi3Nvj-uiq5PlIfk_SN5nN92j4PnI,13964
 aimodelshare/data_sharing/utils.py,sha256=865lN8-oGFi_U_zRaNnGB8Bd0sC8dN_iI5krZOSt_Ts,236
 aimodelshare/data_sharing/data_sharing_templates/Dockerfile.txt,sha256=27wmp7b0rXqJQsumhPxCvGHmUcDiiVgrC6i7DmY7KQA,77
@@ -140,11 +140,11 @@ aimodelshare/sam/codepipeline_policies.txt,sha256=267HMXMnbP7qRASkmFZYSx-2HmKf5o
 aimodelshare/sam/codepipeline_trust_relationship.txt,sha256=yfPYvZlN3fnaIHs7I3ENMMveigIE89mufV9pvR8EQH8,245
 aimodelshare/sam/spark-class.txt,sha256=chyJBxDzCzlUKXzVQYTzuJ2PXCTwg8_gd1yfnI-xbRw,217
 aimodelshare/sam/template.txt,sha256=JKSvEOZNaaLalHSx7r9psJg_6LLCb0XLAYi1-jYPu3M,1195
-aimodelshare-0.1.30.dist-info/licenses/LICENSE,sha256=JXBYLriXYgTloZs-9CJPZY76dqkuDT5df_HghMnljx8,1134
+aimodelshare-0.1.32.dist-info/licenses/LICENSE,sha256=JXBYLriXYgTloZs-9CJPZY76dqkuDT5df_HghMnljx8,1134
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/test_aimsonnx.py,sha256=-GOF1_qXGQaMxHyqK0GPg7dD1meE-S7CZea4pLmBDTk,3906
 tests/test_playground.py,sha256=vdFWPRrZNQ2poiBOoN3l7HsXB5yc3p3rrrclNYJHnaw,24574
-aimodelshare-0.1.30.dist-info/METADATA,sha256=VsmKA64cn84DXDDgHNTM92DE2chwVJU_mVyN1fm7QHw,3523
-aimodelshare-0.1.30.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
-aimodelshare-0.1.30.dist-info/top_level.txt,sha256=2KJgeHQ0BmZuilB75J203i7W4vri6CON2kdbwk9BNpU,19
-aimodelshare-0.1.30.dist-info/RECORD,,
+aimodelshare-0.1.32.dist-info/METADATA,sha256=4tr51rAZknBEIiALek1441tjpMArQd3xn-5zAZ5jElQ,3523
+aimodelshare-0.1.32.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+aimodelshare-0.1.32.dist-info/top_level.txt,sha256=2KJgeHQ0BmZuilB75J203i7W4vri6CON2kdbwk9BNpU,19
+aimodelshare-0.1.32.dist-info/RECORD,,