bigdl-core-npu 2.5.0__cp310-cp310-win_amd64.whl

intel_npu_acceleration_library/external/openvino/tools/benchmark/utils/utils.py

@@ -0,0 +1,836 @@
+# Copyright (C) 2018-2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+from collections import defaultdict
+from datetime import timedelta
+import enum
+from openvino.runtime import Core, Model, PartialShape, Dimension, Layout, Type, serialize, properties, OVAny
+from openvino.preprocess import PrePostProcessor
+
+from .constants import DEVICE_DURATION_IN_SECS, UNKNOWN_DEVICE_TYPE, \
+    AUTO_DEVICE_NAME, MULTI_DEVICE_NAME, HETERO_DEVICE_NAME
+from .logging import logger
+
+import json
+import re
+import numpy as np
+
+def static_vars(**kwargs):
+    def decorate(func):
+        for k in kwargs:
+            setattr(func, k, kwargs[k])
+        return func
+
+    return decorate
+
+
+@static_vars(step_id=0)
+def next_step(additional_info='', step_id=0):
+    step_names = {
+        1: "Parsing and validating input arguments",
+        2: "Loading OpenVINO Runtime",
+        3: "Setting device configuration",
+        4: "Reading model files",
+        5: "Resizing model to match image sizes and given batch",
+        6: "Configuring input of the model",
+        7: "Loading the model to the device",
+        8: "Querying optimal runtime parameters",
+        9: "Creating infer requests and preparing input tensors",
+        10: "Measuring performance",
+        11: "Dumping statistics report",
+    }
+    if step_id != 0:
+        next_step.step_id = step_id
+    else:
+        next_step.step_id += 1
+
+    if next_step.step_id not in step_names.keys():
+        raise Exception(f'Step ID {next_step.step_id} is out of total steps number {str(len(step_names))}')
+
+    step_info_template = '[Step {}/{}] {}'
+    step_name = step_names[next_step.step_id] + (f' ({additional_info})' if additional_info else '')
+    step_info_template = step_info_template.format(next_step.step_id, len(step_names), step_name)
+    print(step_info_template)
+
+
+def get_element_type(precision):
+    format_map = {
+      'bool'    : Type.boolean,
+
+      'f16'  : Type.f16,
+      'f32'  : Type.f32,
+      'f64'  : Type.f64,
+
+      'i8'   : Type.i8,
+      'i16'  : Type.i16,
+      'i32'  : Type.i32,
+      'i64'  : Type.i64,
+
+      'u8'   : Type.u8,
+      'u16'  : Type.u16,
+      'u32'  : Type.u32,
+      'u64'  : Type.u64,
+    }
+    if precision in format_map.keys():
+        return format_map[precision]
+
+    raise Exception(f"Undefined precision: '{precision}' !")
+
+
+def fuse_mean_scale(preproc: PrePostProcessor, app_inputs_info):
+    # TODO: remove warning after 23.3 release
+    warned = False
+    warn_msg = 'Mean/scale values are fused into the model. This slows down performance compared to --imean and --iscale which existed before'
+    for input_info in app_inputs_info:
+        if input_info.mean.size:
+            if not warned:
+                logger.warning(warn_msg)
+                warned = True
+            preproc.input(input_info.name).preprocess().convert_element_type(Type.f32).mean(input_info.mean)
+        if input_info.scale.size:
+            if not warned:
+                logger.warning(warn_msg)
+                warned = True
+            preproc.input(input_info.name).preprocess().convert_element_type(Type.f32).scale(input_info.scale)
+
+
+def pre_post_processing(model: Model, app_inputs_info, input_precision: str, output_precision: str, input_output_precision: str):
+    pre_post_processor = PrePostProcessor(model)
+    if input_precision:
+        element_type = get_element_type(input_precision)
+        for i in range(len(model.inputs)):
+            pre_post_processor.input(i).tensor().set_element_type(element_type)
+            app_inputs_info[i].element_type = element_type
+    if output_precision:
+        element_type = get_element_type(output_precision)
+        for i in range(len(model.outputs)):
+            pre_post_processor.output(i).tensor().set_element_type(element_type)
+    user_precision_map = {}
+    if input_output_precision:
+        user_precision_map = parse_input_output_precision(input_output_precision)
+        input_names = get_input_output_names(model.inputs)
+        input_node_names = get_node_names(model.inputs)
+        output_names = get_input_output_names(model.outputs)
+        output_node_names = get_node_names(model.outputs)
+        for node_name, precision in user_precision_map.items():
+            user_precision_map[node_name] = get_element_type(precision)
+        for name, element_type in user_precision_map.items():
+            if name in input_names or name in input_node_names:
+                input_index = input_names.index(name) if name in input_names else input_node_names.index(name)
+                app_inputs_info[input_index].element_type = element_type
+                pre_post_processor.input(input_index).tensor().set_element_type(element_type)
+            elif name in output_names or name in output_node_names:
+                if name in output_names:
+                    pre_post_processor.output(name).tensor().set_element_type(element_type)
+                else:
+                    pre_post_processor.output(output_node_names.index(name)).tensor().set_element_type(element_type)
+            else:
+                raise Exception(f"Node '{name}' does not exist in model")
+
+    # update app_inputs_info
+    if not input_precision:
+        inputs = model.inputs
+        input_node_names = get_node_names(model.inputs)
+        for i in range(len(inputs)):
+            if app_inputs_info[i].name in user_precision_map:
+                app_inputs_info[i].element_type = user_precision_map[app_inputs_info[i].name]
+            elif input_node_names[i] in user_precision_map:
+                app_inputs_info[i].element_type = user_precision_map[input_node_names[i]]
+            elif app_inputs_info[i].is_image:
+                app_inputs_info[i].element_type = Type.u8
+                pre_post_processor.input(i).tensor().set_element_type(Type.u8)
+
+    fuse_mean_scale(pre_post_processor, app_inputs_info)
+
+    # set layout for model input
+    for info in app_inputs_info:
+        pre_post_processor.input(info.name).model().set_layout(info.layout)
+
+    model = pre_post_processor.build()
+
+
+def parse_input_output_precision(arg_map: str):
+    arg_map = arg_map.replace(" ", "")
+    pairs = [x.strip() for x in arg_map.split(',')]
+
+    parsed_map = {}
+    for pair in pairs:
+        key_value = [x.strip() for x in pair.split(':')]
+        name, precision = key_value[0], key_value[1]
+        # input's name can contain ':'
+        if len(key_value) == 3:
+            name = key_value[0] + ':' + key_value[1]
+            precision = key_value[2]
+        parsed_map.update({name:precision})
+
+    return parsed_map
+
+
+def print_inputs_and_outputs_info(model: Model):
+    inputs = model.inputs
+    logger.info("Model inputs:")
+    for input in inputs:
+        in_name = " , ".join(input.get_names())
+        node_name = input.node.get_friendly_name()
+
+        if in_name=="": in_name = "***NO_NAME***"
+        if node_name=="": node_name = "***NO_NAME***"
+
+        logger.info(f"    {in_name} (node: {node_name}) : {input.element_type.get_type_name()} / "
+                    f"{str(input.node.layout)} / {input.partial_shape}")
+
+    outputs = model.outputs
+    logger.info("Model outputs:")
+    for output in outputs:
+        out_name = " , ".join(output.get_names())
+        node_name = output.get_node().input(0).get_source_output().get_node().get_friendly_name()
+
+        if out_name=="": out_name = "***NO_NAME***"
+        if node_name=="": node_name = "***NO_NAME***"
+
+        logger.info(f"    {out_name} (node: {node_name}) : {output.element_type.get_type_name()} / "
+                    f"{str(output.node.layout)} / {output.partial_shape}")
+
+
+def get_number_iterations(number_iterations: int, nireq: int, num_shapes: int, api_type: str):
+    niter = number_iterations
+
+    if api_type == 'async' and niter:
+        if num_shapes > nireq:
+            niter = int(((niter + num_shapes -1) / num_shapes) * num_shapes)
+            if number_iterations != niter:
+                logger.warning('Number of iterations was aligned by number of input shapes '
+                            f'from {number_iterations} to {niter} using number of possible input shapes {num_shapes}')
+        else:
+            niter = int((niter + nireq - 1) / nireq) * nireq
+            if number_iterations != niter:
+                logger.warning('Number of iterations was aligned by request number '
+                            f'from {number_iterations} to {niter} using number of requests {nireq}')
+
+    return niter
+
+def get_duration_seconds(time, number_iterations, device):
+    if time:
+        # time limit
+        return time
+
+    if not number_iterations:
+        return get_duration_in_secs(device)
+    return 0
+
+
+class LatencyGroup:
+    def __init__(self, input_names, input_shapes):
+        self.input_names = input_names
+        self.input_shapes = input_shapes
+        self.times = list()
+        self.median = 0.
+        self.avg = 0.
+        self.min = 0.
+        self.max = 0.
+
+    def __str__(self):
+        return str().join(f" {name}: {str(shape)}" for name, shape in zip(self.input_names, self.input_shapes))
+
+
+def get_latency_groups(app_input_info):
+    num_groups = max(len(info.shapes) for info in app_input_info)
+    latency_groups = []
+    for i in range(num_groups):
+        names = list()
+        shapes = list()
+        for info in app_input_info:
+            names.append(info.name)
+            shapes.append(info.shapes[i % len(info.shapes)])
+        latency_groups.append(LatencyGroup(names, shapes))
+    return latency_groups
+
+
+def get_duration_in_milliseconds(duration):
+    return duration * 1000
+
+
+def get_duration_in_secs(target_device):
+    duration = 0
+    for device in DEVICE_DURATION_IN_SECS:
+        if device in target_device:
+            duration = max(duration, DEVICE_DURATION_IN_SECS[device])
+
+    if duration == 0:
+        duration = DEVICE_DURATION_IN_SECS[UNKNOWN_DEVICE_TYPE]
+        logger.warning(f'Default duration {duration} seconds is used for unknown device {target_device}')
+
+    return duration
+
+
+def check_for_static(app_input_info):
+    for info in app_input_info:
+        if info.is_dynamic:
+            return False
+    return True
+
+
+def can_measure_as_static(app_input_info):
+    for info in app_input_info:
+        if len(info.shapes) > 1:
+            return False
+    return True
+
+meta_plugins = [ MULTI_DEVICE_NAME, HETERO_DEVICE_NAME, AUTO_DEVICE_NAME ]
+def is_virtual_device(device_name) -> bool:
+    return device_name in meta_plugins
+
+def is_virtual_device_found(device_names) -> bool:
+    return any(is_virtual_device(device_name) for device_name in device_names)
+
+def parse_devices(device_string):
+    result = []
+    target_device = device_string.partition(":")[0]
+    result.append(target_device)
+    if device_string.find(":") != -1:
+        hw_devices_str = device_string.partition(":")[-1]
+        for hw_device in hw_devices_str.split(','):
+            if hw_device[0] == '-':
+                hw_device = hw_device[1:]
+            result.append(hw_device)
+    return result
+
+def parse_value_per_device(devices, values_string, value_type):
+    # Format: <device1>:<value1>,<device2>:<value2> or just <value>
+    result = {}
+    if not values_string:
+        return result
+    device_value_strings = values_string.split(',')
+    for device_value_string in device_value_strings:
+        device_value_vec = device_value_string.split(':')
+        if len(device_value_vec) == 2:
+            device_name = device_value_vec[0]
+            value = device_value_vec[1]
+            if device_name in devices:
+                result[device_name] = value
+            else:
+                devices_str = ""
+                for device in devices:
+                    devices_str += device + " "
+                devices_str = devices_str.strip()
+                raise Exception(f"Failed to set property to '{device_name}' " \
+                                f"which is not found in the target devices list '{devices_str}'!")
+        elif len(device_value_vec) == 1:
+            value = device_value_vec[0]
+            for device in devices:
+                result[device] = value
+        elif not device_value_vec:
+            raise Exception('Unknown string format: ' + values_string)
+    return result
+
+def parse_value_for_virtual_device(device, values_string):
+    isExist = device in values_string.keys()
+    if isExist and len(values_string) > 1:
+        if device == MULTI_DEVICE_NAME:
+            # Remove the element that the key is virtual device MULTI
+            # e.g. MULTI:xxx -nstreams 2 will set nstreams 2 to xxx.
+            values_string.pop(device)
+        elif device == AUTO_DEVICE_NAME or device == HETERO_DEVICE_NAME:
+            # Just keep the element that the key is virtual device AUTO/HETERO
+            # e.g. AUTO:xxx,xxx -nstreams 2 will trigger exception that AUTO plugin didn't support nstream property.
+            value = values_string.get(device)
+            values_string.clear()
+            values_string[device] = value
+    keys = values_string.keys()
+    for key in list(values_string):
+        if device not in list(values_string):
+            values_string[device] = '{'
+        else:
+            values_string[device] += ','
+        values_string[device] += key + ":" + values_string.get(key)
+        del values_string[key]
+    if device in values_string.keys():
+        values_string[device] = values_string[device].strip()
+        if values_string[device] != '':
+            values_string[device] += '}'
+    return
+
+def process_help_inference_string(benchmark_app, device_number_streams):
+    output_string = f'Start inference {benchmark_app.api_type}hronously'
+    if benchmark_app.api_type == 'async':
+        output_string += f', {benchmark_app.nireq} inference requests'
+
+        device_ss = ''
+        for device, streams in device_number_streams.items():
+            device_ss += ', ' if device_ss else ''
+            device_ss += f'{streams} streams for {device}'
+
+        if device_ss:
+            output_string += ' using ' + device_ss
+
+    output_string += ', limits: '
+
+    if benchmark_app.duration_seconds:
+        output_string += f'{get_duration_in_milliseconds(benchmark_app.duration_seconds)} ms duration'
+    if benchmark_app.niter:
+        if benchmark_app.duration_seconds > 0:
+            output_string += ', '
+        output_string += f'{benchmark_app.niter} iterations'
+
+    return output_string
+
+
+def dump_exec_graph(compiled_model, model_path):
+    serialize(compiled_model.get_runtime_model(), model_path)
+
+def print_perf_counters_sort(perf_counts_list,sort_flag="sort"):
+    """ Print opts time cost and can be sorted according by each opts time cost
+    """
+    for ni in range(len(perf_counts_list)):
+        perf_counts = perf_counts_list[ni]
+        total_time = timedelta()
+        total_time_cpu = timedelta()
+        logger.info(f"Performance counts sorted for {ni}-th infer request")
+        for pi in perf_counts:
+            total_time += pi.real_time
+            total_time_cpu += pi.cpu_time
+
+        total_time = total_time.microseconds
+        total_time_cpu = total_time_cpu.microseconds
+        total_real_time_proportion = 0
+        total_detail_data=[]
+        for pi in perf_counts:
+            node_name = pi.node_name
+            layerStatus = pi.status
+            layerType = pi.node_type
+            real_time = pi.real_time.microseconds
+            cpu_time = pi.cpu_time.microseconds
+            real_proportion = round(real_time/total_time,4)
+            execType = pi.exec_type
+            tmp_data=[node_name,layerStatus,layerType,real_time,cpu_time,real_proportion,execType]
+            total_detail_data.append(tmp_data)
+            total_real_time_proportion += real_proportion
+        total_detail_data = np.array(total_detail_data)
+        if sort_flag=="sort":
+            total_detail_data = sorted(total_detail_data,key=lambda tmp_data:tmp_data[-4],reverse=True)
+        elif sort_flag=="no_sort":
+            total_detail_data = total_detail_data
+        elif sort_flag=="simple_sort":
+            total_detail_data = sorted(total_detail_data,key=lambda tmp_data:tmp_data[-4],reverse=True)
+            total_detail_data = [tmp_data for tmp_data in total_detail_data if str(tmp_data[1])!="Status.NOT_RUN"]
+        print_detail_result(total_detail_data)
+        print(f'Total time:       {total_time / 1000:.3f} milliseconds')
+        print(f'Total CPU time:   {total_time_cpu / 1000:.3f} milliseconds')
+        print(f'Total proportion: {"%.2f"%(round(total_real_time_proportion)*100)} % \n')
+    return total_detail_data
+
+def print_detail_result(result_list):
+    """ Print_perf_counters_sort result
+    """
+    max_print_length = 20
+    for tmp_result in result_list:
+        node_name = tmp_result[0]
+        layerStatus = tmp_result[1]
+        layerType = tmp_result[2]
+        real_time = tmp_result[3]
+        cpu_time = tmp_result[4]
+        real_proportion = "%.2f" % (tmp_result[5] * 100)
+        if real_proportion == "0.00":
+            real_proportion = "N/A"
+        execType = tmp_result[6]
+        print(f"{node_name[:max_print_length - 4] + '...' if (len(node_name) >= max_print_length) else node_name:<20} "
+                f"{str(layerStatus):<20} "
+                f"layerType: {layerType[:max_print_length - 4] + '...' if (len(layerType) >= max_print_length) else layerType:<20} "
+                f"execType: {execType:<20} "
+                f"realTime (ms): {real_time / 1000:<10.3f} "
+                f"cpuTime (ms): {cpu_time / 1000:<10.3f}"
+                f"proportion: {str(real_proportion +'%'):<8}")
+
+def print_perf_counters(perf_counts_list):
+    max_print_length = 20
+    for ni in range(len(perf_counts_list)):
+        perf_counts = perf_counts_list[ni]
+        total_time = timedelta()
+        total_time_cpu = timedelta()
+        logger.info(f"Performance counts for {ni}-th infer request")
+        for pi in perf_counts:
+            print(f"{pi.node_name[:max_print_length - 4] + '...' if (len(pi.node_name) >= max_print_length) else pi.node_name:<20} "
+                f"{str(pi.status):<20} "
+                f"layerType: {pi.node_type[:max_print_length - 4] + '...' if (len(pi.node_type) >= max_print_length) else pi.node_type:<20} "
+                f"execType: {pi.exec_type:<20} "
+                f"realTime (ms): {pi.real_time / timedelta(milliseconds=1):<10.3f} "
+                f"cpuTime (ms): {pi.cpu_time / timedelta(milliseconds=1):<10.3f}")
+
+            total_time += pi.real_time
+            total_time_cpu += pi.cpu_time
+        print(f'Total time:     {total_time / timedelta(milliseconds=1)} milliseconds')
+        print(f'Total CPU time: {total_time_cpu / timedelta(milliseconds=1)} milliseconds\n')
+
+
+def get_command_line_arguments(argv):
+    parameters = []
+    arg_name = ''
+    arg_value = ''
+    for arg in argv[1:]:
+        if '=' in arg:
+            if arg_name != '':
+                parameters.append((arg_name, arg_value))
+            arg_name, arg_value = arg.split('=')
+            parameters.append((arg_name, arg_value))
+            arg_name = ''
+            arg_value = ''
+        else:
+          if arg[0] == '-':
+              if arg_name != '':
+                parameters.append((arg_name, arg_value))
+                arg_value = ''
+              arg_name = arg
+          else:
+              arg_value = arg
+    if arg_name != '':
+        parameters.append((arg_name, arg_value))
+    return parameters
+
+
+def get_input_output_names(ports):
+    return [port.any_name for port in ports]
+
+def get_node_names(ports):
+    return [port.node.friendly_name for port in ports]
+
+def get_data_shapes_map(data_shape_string, input_names):
+    # Parse parameter string like "input0[shape1][shape2],input1[shape1]" or "[shape1][shape2]" (applied to all inputs)
+    return_value = {}
+    if data_shape_string:
+        data_shape_string += ','
+        matches = re.findall(r'(.*?\[.*?\]),', data_shape_string)
+        if matches:
+            for match in matches:
+                input_name = match[:match.find('[')]
+                shapes = re.findall(r'\[(.*?)\]', match[len(input_name):])
+                if input_name:
+                    return_value[input_name] = list(PartialShape(shape_str) for shape_str in shapes)
+                else:
+                    data_shapes = list(PartialShape(shape_str) for shape_str in shapes)
+                    num_inputs, num_shapes = len(input_names), len(data_shapes)
+                    if num_shapes != 1 and num_shapes % num_inputs != 0:
+                        raise Exception(f"Number of provided data_shapes is not a multiple of the number of model inputs!")
+                    return_value = defaultdict(list)
+                    for i in range(max(num_shapes, num_inputs)):
+                        return_value[input_names[i % num_inputs]].append(data_shapes[i % num_shapes])
+                    return return_value
+        else:
+            raise Exception(f"Can't parse input parameter: {data_shape_string}")
+    return return_value
+
+
+
+def parse_input_parameters(parameter_string, input_names):
+    # Parse parameter string like "input0[value0],input1[value1]" or "[value]" (applied to all inputs)
+    return_value = {}
+    if parameter_string:
+        matches = re.findall(r'(.*?)\[(.*?)\],?', parameter_string)
+        if matches:
+            for match in matches:
+                input_name, value = match
+                if input_name != '':
+                    return_value[input_name] = value
+                else:
+                    return_value  = { k:value for k in input_names }
+                    break
+        else:
+            raise Exception(f"Can't parse input parameter: {parameter_string}")
+    return return_value
+
+
+def parse_scale_or_mean(parameter_string, input_info):
+    # Parse parameter string like "input0[value0],input1[value1]" or "[value]" (applied to all inputs)
+    return_value = {}
+    if parameter_string:
+        matches = re.findall(r'(.*?)\[(.*?)\],?', parameter_string)
+        if matches:
+            for match in matches:
+                input_name, value = match
+                f_value = np.array(value.split(",")).astype(float)
+                if input_name != '':
+                    return_value[input_name] = f_value
+                else:
+                    for input in input_info:
+                        if input.is_image:
+                            return_value[input.name] = f_value
+        else:
+            raise Exception(f"Can't parse input parameter: {parameter_string}")
+    return return_value
+
+class AppInputInfo:
+    def __init__(self):
+        self.element_type = None
+        self.layout = Layout()
+        self.partial_shape = None
+        self.data_shapes = []
+        self.scale = np.empty([0])
+        self.mean = np.empty([0])
+        self.name = None
+        self.node_name = None
+
+    @property
+    def is_image(self):
+        if str(self.layout) not in [ "[N,C,H,W]", "[N,H,W,C]", "[C,H,W]", "[H,W,C]" ]:
+            return False
+        return self.channels == 3
+
+    @property
+    def is_image_info(self):
+        if str(self.layout) != "[N,C]":
+            return False
+        return len(self.channels) >= 2 if self.channels.is_static else self.channels.relaxes(Dimension(2))
+
+    def getDimensionByLayout(self, character):
+        if self.layout.has_name(character):
+            return self.partial_shape[self.layout.get_index_by_name(character)]
+        else:
+            return Dimension(0)
+
+    def getDimensionsByLayout(self, character):
+        if self.layout.has_name(character):
+            d_index = self.layout.get_index_by_name(character)
+            dims = []
+            for shape in self.data_shapes:
+                dims.append(shape[d_index])
+            return dims
+        else:
+            return [0] * len(self.data_shapes)
+
+    @property
+    def shapes(self):
+        if self.is_static:
+            return [self.partial_shape.to_shape()]
+        else:
+            return self.data_shapes
+
+    @property
+    def width(self):
+        return len(self.getDimensionByLayout("W"))
+
+    @property
+    def widths(self):
+        return self.getDimensionsByLayout("W")
+
+    @property
+    def height(self):
+        return len(self.getDimensionByLayout("H"))
+
+    @property
+    def heights(self):
+        return self.getDimensionsByLayout("H")
+
+    @property
+    def channels(self):
+        return self.getDimensionByLayout("C")
+
+    @property
+    def is_static(self):
+        return self.partial_shape.is_static
+
+    @property
+    def is_dynamic(self):
+        return self.partial_shape.is_dynamic
+
+
+def get_inputs_info(shape_string, data_shape_string, layout_string, batch_size, scale_string, mean_string, inputs):
+    input_names = get_input_output_names(inputs)
+    input_node_names = get_node_names(inputs)
+    shape_map = parse_input_parameters(shape_string, input_names)
+    data_shape_map = get_data_shapes_map(data_shape_string, input_names)
+    layout_map = parse_input_parameters(layout_string, input_names)
+    batch_size = Dimension(batch_size)
+    reshape = False
+    batch_found = False
+    input_info = []
+    for i in range(len(inputs)):
+        info = AppInputInfo()
+        # Input name
+        info.name = input_names[i]
+        # Input node name
+        info.node_name = input_node_names[i]
+        # Input precision
+        info.element_type = inputs[i].element_type
+        # Shape
+        if info.name in shape_map:
+            info.partial_shape = PartialShape(shape_map[info.name])
+            reshape = True
+        elif info.node_name in shape_map:
+            info.partial_shape = PartialShape(shape_map[info.node_name])
+            reshape = True
+        else:
+            info.partial_shape = inputs[i].partial_shape
+
+        # Layout
+        if info.name in layout_map:
+            info.layout = Layout(layout_map[info.name])
+        elif info.node_name in layout_map:
+            info.layout = Layout(layout_map[info.node_name])
+        elif inputs[i].node.layout != Layout():
+            info.layout = inputs[i].node.layout
+        else:
+            image_colors_dim_max = 4
+            shape = info.partial_shape
+            num_dims = len(shape)
+            if num_dims == 4:
+                if shape[1].get_max_length() <= image_colors_dim_max and shape[3].get_max_length() > image_colors_dim_max:
+                    info.layout = Layout("NCHW")
+                elif shape[3].get_max_length() <= image_colors_dim_max and shape[1].get_max_length() > image_colors_dim_max:
+                    info.layout = Layout("NHWC")
+            elif num_dims == 3:
+                if shape[0].get_max_length() <= image_colors_dim_max and shape[2].get_max_length() > image_colors_dim_max:
+                    info.layout = Layout("CHW")
+                elif shape[2].get_max_length() <= image_colors_dim_max and shape[0].get_max_length() > image_colors_dim_max:
+                    info.layout = Layout("HWC")
+
+        # Update shape with batch if needed
+        if batch_size != 0:
+            if batch_size.is_static and data_shape_map:
+                 logger.warning(f"Batch size will be ignored. Provide batch deminsion in data_shape parameter.")
+            else:
+                batch_index = -1
+                if info.layout.has_name('N'):
+                    batch_index = info.layout.get_index_by_name('N')
+                elif info.layout == Layout():
+                    supposed_batch = info.partial_shape[0]
+                    if supposed_batch.is_dynamic or supposed_batch in [0, 1]:
+                        logger.warning(f"Batch dimension is not specified for input '{info.name}'. "
+                                        "The first dimension will be interpreted as batch size.")
+                        batch_index = 0
+                        info.layout = Layout("N...")
+                if batch_index != -1 and info.partial_shape[batch_index] != batch_size:
+                    info.partial_shape[batch_index] = batch_size
+                    reshape = True
+                    batch_found = True
+                elif batch_index == -1 and not batch_found and i == len(inputs) - 1:
+                    raise RuntimeError("-b option is provided in command line, but there's no inputs with batch(B) " \
+                            "dimension in input layout, so batch cannot be set. " \
+                            "You may specify layout explicitly using -layout option.")
+
+        # Data shape
+        if (info.name in data_shape_map or info.node_name in data_shape_map) and info.is_dynamic:
+            used_name = info.name if info.name in data_shape_map else info.node_name
+            for p_shape in data_shape_map[used_name]:
+                if p_shape.is_dynamic:
+                    raise Exception(f"Data shape always should be static, {str(p_shape)} is dynamic.")
+                elif info.partial_shape.compatible(p_shape):
+                    info.data_shapes.append(p_shape.to_shape())
+                else:
+                    raise Exception(f"Data shape '{str(p_shape)}' provided for input '{info.name}' "
+                                    f"is not compatible with partial shape '{str(info.partial_shape)}' for this input.")
+        elif info.name in data_shape_map or input_node_names[i] in data_shape_map:
+            logger.warning(f"Input '{info.name}' has static shape. Provided data shapes for this input will be ignored.")
+
+        input_info.append(info)
+
+    # Update scale and mean
+    scale_map = parse_scale_or_mean(scale_string, input_info)
+    mean_map = parse_scale_or_mean(mean_string, input_info)
+
+    for input in input_info:
+        if input.name in scale_map:
+            input.scale = scale_map[input.name]
+        elif input.node_name in scale_map:
+            input.scale = scale_map[input.node_name]
+        if input.name in mean_map:
+            input.mean = mean_map[input.name]
+        elif input.node_name in mean_map:
+            input.mean = mean_map[input.node_name]
+
+    return input_info, reshape
+
+
+def get_network_batch_size(inputs_info):
+    null_dimension = Dimension(0)
+    batch_size = null_dimension
+    for info in inputs_info:
+        batch_index = info.layout.get_index_by_name('N') if info.layout.has_name('N') else -1
+        if batch_index != -1:
+            if batch_size == null_dimension:
+                batch_size = info.partial_shape[batch_index]
+            elif batch_size != info.partial_shape[batch_index]:
+                raise Exception("Can't deterimine batch size: batch is different for different inputs!")
+    if batch_size == null_dimension:
+        batch_size = Dimension(1)
+    return batch_size
+
+
+def show_available_devices():
+    print("\nAvailable target devices:  ", ("  ".join(Core().available_devices)))
+
+
+def device_properties_to_string(config):
+    ret = "{"
+    for k, v in config.items():
+        if isinstance(v, dict):
+            sub_str = "{"
+            for sk, sv in v.items():
+                if isinstance(sv, bool):
+                    sv = "YES" if sv else "NO"
+                if isinstance(sv, properties.Affinity):
+                    sv = sv.name
+                sub_str += "{0}:{1},".format(sk, sv)
+            sub_str = sub_str[:-1]
+            sub_str += "}"
+            ret += "{0}:{1},".format(k, sub_str)
+        else:
+            ret += "{0}:{1},".format(k, v)
+    ret = ret[:-1]
+    ret += "}"
+    return ret
+
+
+def string_to_device_properties(device_properties_str):
+    ret = {}
+    if not device_properties_str:
+        return ret
+    if not device_properties_str.startswith("{") or not device_properties_str.endswith("}"):
+        raise Exception(
+            "Failed to parse device properties. Value of device properties should be started with '{' and ended with '}'."
+            "They are actually {} and {}".format(device_properties_str[0], device_properties_str[-1]))
+    pattern = r'(\w+):({.+?}|[^,}]+)'
+    pairs = re.findall(pattern, device_properties_str)
+    for key, value in pairs:
+        if value.startswith("{") and value.endswith("}"):
+            value = value[1:-1]
+            nested_pairs = re.findall(pattern, value)
+            nested_dict = {}
+            for nested_key, nested_value in nested_pairs:
+                nested_dict[nested_key] = nested_value
+            value = nested_dict
+        ret[key] = value
+    return ret
+
+
+def dump_config(filename, config):
+    json_config = {}
+    for device_name, device_config in config.items():
+        json_config[device_name] = {}
+        for key, value in device_config.items():
+            if isinstance(value, OVAny) and (isinstance(value.value, dict)):
+                value_string = device_properties_to_string(value.get())
+            elif isinstance(value, (properties.hint.PerformanceMode, properties.Affinity)):
+                value_string = value.name
+            elif isinstance(value, OVAny):
+                value_string = str(value.value)
+            else:
+                value_string = str(value)
+                if isinstance(value, bool):
+                    value_string = "YES" if value else "NO"
+            json_config[device_name][key] = value_string
+
+    with open(filename, 'w') as f:
+        json.dump(json_config, f, indent=4)
+
+
+def load_config(filename, config):
+    with open(filename) as f:
+        original_config = json.load(f)
+    for device in original_config:
+        config[device] = {}
+        for property_name in original_config[device]:
+            property_value = original_config[device][property_name]
+            if property_name == properties.device.properties():
+                property_value = string_to_device_properties(property_value)
+            elif property_value in ("YES", "NO"):
+                property_value = True if property_value == "YES" else False
+            config[device][property_name] = OVAny(property_value)