leap-model-parser 0.1.216__py3-none-any.whl → 0.1.232__py3-none-any.whl

leap_model_parser/contract/graph.py

@@ -47,6 +47,7 @@ class Node:
     outputs: Dict[str, ConnectionOutput] = field(default_factory=dict)
     pruning_plan_id: Optional[str] = None
     wrapper: Optional[WrapperData] = None
+    shape: Optional[List[str]] = None
 
     def __key(self):
         return (self.id, self.name)

leap_model_parser/keras_json_model_import.py

@@ -5,7 +5,6 @@ from typing import Set, Dict, Any, List, Type, Optional, Tuple
 
 import tensorflow as tf  # type: ignore
 from keras.engine.keras_tensor import KerasTensor  # type: ignore
-from keras.engine.node import Node  # type: ignore
 from keras.engine.node import Node as keras_node # type: ignore
 from keras.layers import Layer # type: ignore
 from keras.layers.convolutional.base_conv import Conv # type: ignore
@@ -206,7 +205,7 @@ class KerasJsonModelImport:
                            'serialized_call_kwargs': serialized_call_kwargs})
 
     def generate_regular_node(self, layer: Dict[str, Any], layer_metadata: Dict[str, Any], node_key: str,
-                              layer_name_to_inbound_nodes: Dict[str, List[Node]]):
+                              layer_name_to_inbound_nodes: Dict[str, List[keras_node]]):
         data = layer['config']
         if layer['class_name'] in ('TFOpLambda', 'SlicingOpLambda') or layer['class_name'] in self.custom_layers:
             call_args = layer_name_to_inbound_nodes[layer['config']
@@ -221,11 +220,17 @@ class KerasJsonModelImport:
 
         self.layer_data_adjustments(data, layer_metadata)
         node = Node(id=str(self.id), name=layer_metadata.get(
-            "class_name", layer["class_name"]), data=data, position=[0, 0])
+            "class_name", layer["class_name"]), data=data, position=[0, 0],
+                    shape=self._convert_layer_shape_to_string(layer['output_shape']))
         if 'wrapper' in layer:
             node.wrapper = layer['wrapper']
         self.nodes_cache[node_key] = node
 
+    @staticmethod
+    def _convert_layer_shape_to_string(s: List[Optional[int]]) -> List[str]:
+        return [str(dim) if dim is not None else 'dynamic_shape' for dim in s]
+
+
     @classmethod
     def handle_wrapper_layer(cls, layer):
         wrapped_layer = layer['config']['layer']
@@ -332,7 +337,8 @@ class KerasJsonModelImport:
         self.connected_inputs.append(input_info)
         input_copy = {
             'type': 'Input',
-            'output_name': input_name
+            'output_name': input_name,
+            'batch_input_shape_origin': shape
         }
         inputNode = Node(
             id=str(self.id), name='Input', data=input_copy, position=[0, 0])

leap_model_parser/leap_graph_editor.py

@@ -31,6 +31,8 @@ class LeapGraphEditor:
                 f"Prediction node with name {prediction_node.name} not found in model graph"
             prediction_node_id = prediction_mapping_node.id
             self.model_graph[prediction_node_id].data['prediction_type'] = prediction_labels_name
+        elif 'Input' in node_connection.node.type.value:
+            self._find_or_add_input_node(node_connection.node)
         elif node_connection.node.type == NodeMappingType.Visualizer:
             new_visualizer_node_id = self._add_visualizer_node(
                 node_connection.node.name, node_connection.node.sub_type,
@@ -47,6 +49,7 @@ class LeapGraphEditor:
                 self._add_connection_to_node(new_metric_node_id, input_name, input_node_id)
         elif node_connection.node.type in (NodeMappingType.Loss, NodeMappingType.CustomLoss):
             new_loss_node_id = self._add_loss_node(node_connection.node.name,
+                                                   node_connection.node.user_unique_name,
                                                    node_connection.node.type == NodeMappingType.CustomLoss,
                                                    node_connection.node.arg_names)
             for input_name, node in node_connection.node_inputs.items():
@@ -124,6 +127,22 @@ class LeapGraphEditor:
         self.model_graph[new_node_id] = ground_truth_node
         return new_node_id
 
+    def _add_input_encoder_not_connected_to_the_model_node(self, input_name: str) -> str:
+        new_node_id = self._generate_new_node_id()
+        ground_truth_node = Node(
+            new_node_id,
+            'Input',
+            position=[0, 0],
+            data={'name': input_name, 'output_name': input_name,
+                  'type': 'Input', "selected": input_name},
+            inputs={},
+            outputs={
+                f'{new_node_id}-{input_name}': ConnectionOutput([])
+            }
+        )
+        self.model_graph[new_node_id] = ground_truth_node
+        return new_node_id
+
     def _add_visualizer_node(self, visualizer_name: str, visualizer_type: str,
                              user_unique_name: str, arg_names: List[str]) -> str:
         new_node_id = self._generate_new_node_id()
@@ -157,7 +176,7 @@ class LeapGraphEditor:
         self.model_graph[new_node_id] = metric_node
         return new_node_id
 
-    def _add_loss_node(self, loss_name: str, is_custom_loss: bool, arg_names: Optional[List[str]]=None) -> str:
+    def _add_loss_node(self, loss_name: str, user_unique_name:str, is_custom_loss: bool, arg_names: Optional[List[str]]=None) -> str:
         new_node_id = self._generate_new_node_id()
 
         loss_type = 'CustomLoss' if is_custom_loss else 'Loss'
@@ -167,7 +186,7 @@ class LeapGraphEditor:
             new_node_id,
             loss_node_name,
             position=[0, 0],
-            data={'type': loss_type, 'selected': loss_name, 'name': loss_name},
+            data={'type': loss_type, 'selected': loss_name, 'name': loss_name, 'user_unique_name': user_unique_name},
             inputs={},
             outputs={
                 f'{new_node_id}-loss': ConnectionOutput([])
@@ -237,6 +256,8 @@ class LeapGraphEditor:
             if input_node_id is None:
                 if input_node.type == NodeMappingType.GroundTruth:
                     input_node_id = self._add_ground_truth_node(input_node.name)
+                elif input_node.type == NodeMappingType.Input:
+                    input_node_id = self._add_input_encoder_not_connected_to_the_model_node(input_node.name)
                 else:
                     raise Exception(f'Couldnt find input node name {input_node.name}')
         elif 'Input' in input_node.type.value:

leap_model_parser/model_parser.py

@@ -1,13 +1,13 @@
+# mypy: ignore-errors
+
 import glob
 import json
 import ntpath
-import pickle
 import tarfile
 import tempfile
 from importlib.util import find_spec
 from pathlib import Path
 
-
 import tensorflow as tf  # type: ignore
 from code_loader.contract.mapping import NodeConnection, NodeMapping  # type: ignore
 from keras import Model  # type: ignore
@@ -34,6 +34,7 @@ if spec is not None:
 
     onnx_imported = True
 
+
 class ModelParser:
     def __init__(self, should_transform_inputs_and_outputs=False,
                  custom_layers=None,
@@ -54,8 +55,14 @@ class ModelParser:
             ImportModelTypeEnum.PB_TF2.value: self.convert_pb_model,
         }
 
+    @staticmethod
+    def _add_output_node_shape_to_model_schema(model_schema: Dict, keras_model: Model):
+        for i, layer in enumerate(keras_model.layers):
+            model_schema['config']['layers'][i]['output_shape'] = list(layer.output_shape)
+
     def get_keras_model_and_model_graph(
-            self, model_path: Path, model_type: ImportModelTypeEnum) -> Tuple[Dict[str, Node], List[InputInfo], Optional[Model], Optional[str]]:
+            self, model_path: Path, model_type: ImportModelTypeEnum) -> Tuple[
+        Dict[str, Node], List[InputInfo], Optional[Model], Optional[str]]:
         model_to_keras_converter: Optional[Callable[[str], Tuple[Dict[str, Node], Model, Optional[str]]]] = \
             self._model_types_converter.get(model_type.value)
         if model_to_keras_converter is None:
@@ -64,6 +71,9 @@ class ModelParser:
 
         file_path = str(model_path)
         model_schema, keras_model_with_weights, error_info = model_to_keras_converter(file_path)
+
+        self._add_output_node_shape_to_model_schema(model_schema, keras_model_with_weights)
+
         model_generator = KerasJsonModelImport(self.custom_layers)
 
         keras_model = keras_model_with_weights
@@ -76,13 +86,19 @@ class ModelParser:
 
         graph, connected_inputs = model_generator.generate_graph(
             model_schema, layer_name_to_inbound_nodes)
-
+        # make sure input order is kept with original model
+        input_list = []
+        for inp in keras_model.inputs:
+            name = inp.name
+            name = name.replace(".", "_")
+            for inp_graph in connected_inputs:
+                if inp_graph.name == name:
+                    input_list.append(inp_graph)
         if self.mapping_connections is not None:
             leap_graph_editor = LeapGraphEditor(graph, keras_model_with_weights)
             leap_graph_editor.add_connections_to_graph(self.mapping_connections)
 
-        return graph, connected_inputs, keras_model_with_weights, error_info
-
+        return graph, input_list, keras_model_with_weights, error_info
 
     def _get_k_model_from_pb_path(self, file_path: str):
         tar_file = tarfile.open(file_path)
@@ -98,7 +114,8 @@ class ModelParser:
             k_model = self._load_keras_model_with_custom_layers(pb_folder_path)
         return k_model
 
-    def generate_model_graph(self, model_path: Path, model_type: ImportModelTypeEnum) -> Tuple[Dict[str, Node], List[InputInfo]]:
+    def generate_model_graph(self, model_path: Path, model_type: ImportModelTypeEnum) -> Tuple[
+        Dict[str, Node], List[InputInfo]]:
         model_graph, connected_inputs, _, error_info = self.get_keras_model_and_model_graph(
             model_path, model_type)
         return model_graph, connected_inputs
@@ -121,25 +138,58 @@ class ModelParser:
         input_all = [_input.name for _input in onnx_model.graph.input]
         input_initializer = [
             node.name for node in onnx_model.graph.initializer]
-        input_names = list(set(input_all) - set(input_initializer))
+        input_names = [name for name in input_all if name not in input_initializer]
         converted_response: ConvertedResponse = onnx_to_keras(onnx_model, input_names=input_names,
-                                name_policy='attach_weights_name', allow_partial_compilation=False)
+                                                              name_policy='attach_weights_name',
+                                                              allow_partial_compilation=False)
         return self.convert_to_keras_model(converted_response.converted_model, converted_response.error_info)
 
     def _load_keras_model_with_custom_layers(self, file_path: str):
         custom_objects = {}
         if self.custom_layers is not None:
             custom_objects = self.custom_layers
-        return load_model(file_path, custom_objects=custom_objects, compile=False)
+
+        try:
+            return load_model(file_path, custom_objects=custom_objects, compile=False)
+        except OSError as e:
+            if 'signature' in str(e):
+                raise Exception('Unable to open model file. The model might be corrupted or not a valid.')
+            else:
+                raise e
+
 
     def convert_h5_model(self, file_path: str) -> Tuple[Dict[str, Node], Model, Optional[str]]:
         imported_model = self._load_keras_model_with_custom_layers(file_path)
         return self.convert_to_keras_model(imported_model)
 
-    def convert_to_keras_model(self, k_model, error_info: Optional[str] = None) -> Tuple[Dict[str, Node], Model, Optional[str]]:
+    def convert_to_keras_model(self, k_model, error_info: Optional[str] = None) -> Tuple[
+        Dict[str, Node], Model, Optional[str]]:
         converted_k_model = convert_channels_first_to_last(
             k_model, self._should_transform_inputs_and_outputs, self.custom_layers)
 
+        from keras.saving.legacy.saved_model import json_utils  # type: ignore
+        import numpy as np
+
+        _orig_get_json_type = json_utils.get_json_type
+
+
+        def _patched_get_json_type(obj):  # type: ignore
+            # Handle numpy dtype explicitly
+            if isinstance(obj, np.dtype):
+                return obj.name  # e.g. "int64"
+            # Make sure common numpy scalars/containers are handled robustly
+            if isinstance(obj, np.integer):
+                return int(obj)
+            if isinstance(obj, np.floating):
+                return float(obj)
+            if isinstance(obj, np.bool_):
+                return bool(obj)
+            if isinstance(obj, np.ndarray):
+                return obj.tolist()
+            return _orig_get_json_type(obj)
+
+        json_utils.get_json_type = _patched_get_json_type
+
         model_schema = json.loads(converted_k_model.to_json())
 
         return model_schema, converted_k_model, error_info

{leap_model_parser-0.1.216.dist-info → leap_model_parser-0.1.232.dist-info}/METADATA

@@ -1,9 +1,9 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: leap-model-parser
-Version: 0.1.216
+Version: 0.1.232
 Summary: 
-Home-page: https://github.com/tensorleap/leap-model-parser
 License: MIT
+License-File: LICENSE
 Author: idan
 Author-email: idan.yogev@tensorleap.ai
 Requires-Python: >=3.8,<3.11
@@ -12,15 +12,16 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
-Requires-Dist: code-loader (>=1.0.115)
-Requires-Dist: keras-data-format-converter (==0.1.22)
+Requires-Dist: code-loader (>=1.0.127)
+Requires-Dist: keras-data-format-converter (==0.1.24)
 Requires-Dist: leap-model-rebuilder (==0.1.7)
 Requires-Dist: numpy (>=1.22.3,<2.0.0)
 Requires-Dist: onnx (==1.13.0)
-Requires-Dist: onnx2kerastl (==0.0.175)
+Requires-Dist: onnx2kerastl (==0.0.180)
 Requires-Dist: tensorflow (==2.12.0) ; platform_machine == "x86_64"
 Requires-Dist: tensorflow-io-gcs-filesystem (==0.34.0)
 Requires-Dist: tensorflow-macos (==2.12.0) ; platform_machine == "arm64"
+Project-URL: Homepage, https://github.com/tensorleap/leap-model-parser
 Project-URL: Repository, https://github.com/tensorleap/leap-model-parser
 Description-Content-Type: text/markdown
 

{leap_model_parser-0.1.216.dist-info → leap_model_parser-0.1.232.dist-info}/RECORD

@@ -1,13 +1,12 @@
-LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 leap_model_parser/__init__.py,sha256=OAU7rFHAVVWUM-cDtQ4Ohum567KN8M-YTkHZp5KiYbo,132
 leap_model_parser/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-leap_model_parser/contract/graph.py,sha256=c5rSlJJQ3JzLhyeniglzrP5GH0fp8UZDQKZw1fcrNe8,1132
+leap_model_parser/contract/graph.py,sha256=lbs_IUnlkvoVjiARBMJee-aNbqeNXNPsUeOEwQliKgU,1170
 leap_model_parser/contract/importmodelresponse.py,sha256=GlvnKS8xrebU2Sj0dxqtEhAOOo3DlOtT7AMJ2BlzH9E,145
 leap_model_parser/contract/nodedata.py,sha256=1_ML0nzp3QUZ0_9mGSLhfO4_hqjYMwi0DWLwymUnWEs,43326
 leap_model_parser/contract/ui_components.json,sha256=0lsxwOLElW1E-imCcdh3zKPWgzFuQ_bApG6aHvYfTvo,410591
-leap_model_parser/keras_json_model_import.py,sha256=x7HOH6iaASfzJgwMRHgF5SQS-iFOF5j9yCG0mDC9HEA,16794
-leap_model_parser/leap_graph_editor.py,sha256=fGbGo_28nG_KU7vFB3-d_zkXVvYWwtqOk_1ZkBeg6So,13289
-leap_model_parser/model_parser.py,sha256=Je37-FjrNgi8w17rBIt_b71xm-BoiopBAvgW30qfb7o,6614
+leap_model_parser/keras_json_model_import.py,sha256=h9-eb41W67goutRV5c3VkStxvntgb6EmPFdPeshhtsY,17058
+leap_model_parser/leap_graph_editor.py,sha256=s6iVO06xM6LJ8ZjSxkr1srYxsHhGl2KnQjQoc9g4_iE,14329
+leap_model_parser/model_parser.py,sha256=672YTDpQP38439juRr22UbrLG8YQng8HVmMmLVOy1mI,8510
 leap_model_parser/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 leap_model_parser/utils/layerpedia/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 leap_model_parser/utils/layerpedia/layerpedia.py,sha256=1syubfXBTB630TVkgcQ-Ge7Qe9Zbr6EtZRreuqCJnQ8,9292
@@ -18,8 +17,8 @@ leap_model_parser/utils/uicomponents/generatenodedata.py,sha256=LRaPlO5jJ9pUtkvL
 leap_model_parser/utils/uicomponents/tensorflowinscpection.py,sha256=ym613z9iQKPDBpr0RYD35bTABdm1L-Ez86G47BYT7qw,6775
 leap_model_parser/utils/uicomponents/ui_components.json,sha256=0lsxwOLElW1E-imCcdh3zKPWgzFuQ_bApG6aHvYfTvo,410591
 leap_model_parser/utils/uicomponents/ui_components_config.yaml,sha256=cRH8T-c3TAL0nfefRvt9pFsjbTWNEg38NRyHR7RpJsk,19534
-leap_model_parser-0.1.216.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-leap_model_parser-0.1.216.dist-info/METADATA,sha256=2wxebfyNwh5M_-qk4uyURhnXyOsiYngrwhxC3CN16Ps,1104
-leap_model_parser-0.1.216.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-leap_model_parser-0.1.216.dist-info/entry_points.txt,sha256=ZvV6EuQt1uAqwapNg5Lo2qjJM9ZG5g2wfzZoLh_Ztyk,77
-leap_model_parser-0.1.216.dist-info/RECORD,,
+leap_model_parser-0.1.232.dist-info/METADATA,sha256=3jnqLtGeBomuwwUqg-9mV4mVHvl-h3fOTuaoKTN5EOE,1138
+leap_model_parser-0.1.232.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+leap_model_parser-0.1.232.dist-info/entry_points.txt,sha256=ZvV6EuQt1uAqwapNg5Lo2qjJM9ZG5g2wfzZoLh_Ztyk,77
+leap_model_parser-0.1.232.dist-info/licenses/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+leap_model_parser-0.1.232.dist-info/RECORD,,

{leap_model_parser-0.1.216.dist-info → leap_model_parser-0.1.232.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 2.2.1
 Root-Is-Purelib: true
 Tag: py3-none-any

leap_model_parser-0.1.216.dist-info/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2021 TensorLeap
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.