powertrain-build 1.13.1__py3-none-any.whl → 1.13.3.dev3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- powertrain_build/__init__.py +40 -40
- powertrain_build/__main__.py +6 -6
- powertrain_build/a2l.py +582 -582
- powertrain_build/a2l_merge.py +650 -650
- powertrain_build/a2l_templates.py +717 -717
- powertrain_build/build.py +985 -985
- powertrain_build/build_defs.py +309 -309
- powertrain_build/build_proj_config.py +690 -690
- powertrain_build/check_interface.py +575 -575
- powertrain_build/cli.py +141 -141
- powertrain_build/config.py +542 -542
- powertrain_build/core.py +395 -395
- powertrain_build/core_dummy.py +343 -343
- powertrain_build/create_conversion_table.py +73 -73
- powertrain_build/dids.py +916 -916
- powertrain_build/dummy.py +157 -157
- powertrain_build/dummy_spm.py +252 -252
- powertrain_build/environmentcheck.py +52 -52
- powertrain_build/ext_dbg.py +255 -255
- powertrain_build/ext_var.py +327 -327
- powertrain_build/feature_configs.py +301 -301
- powertrain_build/gen_allsysteminfo.py +227 -227
- powertrain_build/gen_label_split.py +449 -449
- powertrain_build/handcode_replacer.py +124 -124
- powertrain_build/html_report.py +133 -133
- powertrain_build/interface/__init__.py +4 -4
- powertrain_build/interface/application.py +511 -511
- powertrain_build/interface/base.py +500 -500
- powertrain_build/interface/csp_api.py +490 -490
- powertrain_build/interface/device_proxy.py +677 -677
- powertrain_build/interface/ems.py +67 -67
- powertrain_build/interface/export_global_vars.py +121 -121
- powertrain_build/interface/generate_adapters.py +132 -132
- powertrain_build/interface/generate_hi_interface.py +87 -87
- powertrain_build/interface/generate_service.py +69 -69
- powertrain_build/interface/generate_wrappers.py +147 -147
- powertrain_build/interface/generation_utils.py +142 -142
- powertrain_build/interface/hal.py +194 -194
- powertrain_build/interface/model_yaml_verification.py +348 -348
- powertrain_build/interface/service.py +296 -296
- powertrain_build/interface/simulink.py +249 -249
- powertrain_build/interface/update_call_sources.py +180 -180
- powertrain_build/interface/update_model_yaml.py +186 -186
- powertrain_build/interface/zone_controller.py +362 -362
- powertrain_build/lib/__init__.py +4 -4
- powertrain_build/lib/helper_functions.py +127 -127
- powertrain_build/lib/logger.py +55 -55
- powertrain_build/matlab_scripts/CodeGen/BuildAutomationPyBuild.m +78 -78
- powertrain_build/matlab_scripts/CodeGen/Generate_A2L.m +154 -154
- powertrain_build/matlab_scripts/CodeGen/generateTLUnit.m +239 -239
- powertrain_build/matlab_scripts/CodeGen/getAsilClassification.m +28 -28
- powertrain_build/matlab_scripts/CodeGen/modelConfiguredForTL.m +28 -28
- powertrain_build/matlab_scripts/CodeGen/moveDefOutports.m +88 -88
- powertrain_build/matlab_scripts/CodeGen/parseCalMeasData.m +410 -410
- powertrain_build/matlab_scripts/CodeGen/parseCoreIdentifiers.m +139 -139
- powertrain_build/matlab_scripts/CodeGen/parseDIDs.m +141 -141
- powertrain_build/matlab_scripts/CodeGen/parseInPorts.m +106 -106
- powertrain_build/matlab_scripts/CodeGen/parseIncludeConfigs.m +25 -25
- powertrain_build/matlab_scripts/CodeGen/parseModelInfo.m +38 -38
- powertrain_build/matlab_scripts/CodeGen/parseNVM.m +81 -81
- powertrain_build/matlab_scripts/CodeGen/parseOutPorts.m +120 -120
- powertrain_build/matlab_scripts/CodeGen/parsePreProcBlks.m +23 -23
- powertrain_build/matlab_scripts/CodeGen/struct2JSON.m +128 -128
- powertrain_build/matlab_scripts/CodeGen/updateCodeSwConfig.m +31 -31
- powertrain_build/matlab_scripts/Init_PyBuild.m +91 -91
- powertrain_build/matlab_scripts/__init__.py +2 -2
- powertrain_build/matlab_scripts/helperFunctions/Get_Full_Name.m +46 -46
- powertrain_build/matlab_scripts/helperFunctions/Get_SrcLines.m +12 -12
- powertrain_build/matlab_scripts/helperFunctions/Init_Models.m +78 -78
- powertrain_build/matlab_scripts/helperFunctions/Init_Projects.m +67 -67
- powertrain_build/matlab_scripts/helperFunctions/Read_Units.m +34 -34
- powertrain_build/matlab_scripts/helperFunctions/SetProjectTimeSamples.m +26 -26
- powertrain_build/matlab_scripts/helperFunctions/Strip_Suffix.m +16 -16
- powertrain_build/matlab_scripts/helperFunctions/followLink.m +118 -118
- powertrain_build/matlab_scripts/helperFunctions/getCodeSwitches.m +50 -50
- powertrain_build/matlab_scripts/helperFunctions/getConsumerBlocks.m +30 -30
- powertrain_build/matlab_scripts/helperFunctions/getDefBlock.m +39 -39
- powertrain_build/matlab_scripts/helperFunctions/getDefOutport.m +58 -58
- powertrain_build/matlab_scripts/helperFunctions/getDstBlocks.m +19 -19
- powertrain_build/matlab_scripts/helperFunctions/getDstLines.m +13 -13
- powertrain_build/matlab_scripts/helperFunctions/getInterfaceSignals.m +37 -37
- powertrain_build/matlab_scripts/helperFunctions/getName.m +37 -37
- powertrain_build/matlab_scripts/helperFunctions/getPath.m +6 -6
- powertrain_build/matlab_scripts/helperFunctions/getProperValue.m +21 -21
- powertrain_build/matlab_scripts/helperFunctions/getSrcBlocks.m +19 -19
- powertrain_build/matlab_scripts/helperFunctions/getSrcLines.m +13 -13
- powertrain_build/matlab_scripts/helperFunctions/loadLibraries.m +10 -10
- powertrain_build/matlab_scripts/helperFunctions/loadjson.m +6 -6
- powertrain_build/matlab_scripts/helperFunctions/modifyEnumStructField.m +21 -21
- powertrain_build/matlab_scripts/helperFunctions/removeConfigDuplicates.m +31 -31
- powertrain_build/matlab_scripts/helperFunctions/sortSystemByClass.m +26 -26
- powertrain_build/matlab_scripts/helperFunctions/tl_getfast.m +89 -89
- powertrain_build/matlab_scripts/helperFunctions/topLevelSystem.m +20 -20
- powertrain_build/matlab_scripts/helperFunctions/updateModels.m +131 -131
- powertrain_build/memory_section.py +224 -224
- powertrain_build/nvm_def.py +729 -729
- powertrain_build/problem_logger.py +86 -86
- powertrain_build/pt_matlab.py +430 -430
- powertrain_build/pt_win32.py +144 -144
- powertrain_build/replace_compu_tab_ref.py +105 -105
- powertrain_build/rte_dummy.py +254 -254
- powertrain_build/sched_funcs.py +209 -207
- powertrain_build/signal.py +7 -7
- powertrain_build/signal_if_html_rep.py +221 -221
- powertrain_build/signal_if_html_rep_all.py +302 -302
- powertrain_build/signal_incons_html_rep.py +180 -180
- powertrain_build/signal_incons_html_rep_all.py +366 -366
- powertrain_build/signal_incons_html_rep_base.py +168 -168
- powertrain_build/signal_inconsistency_check.py +641 -641
- powertrain_build/signal_interfaces.py +864 -864
- powertrain_build/templates/Index_SigCheck_All.html +22 -22
- powertrain_build/templates/Index_SigIf_All.html +19 -19
- powertrain_build/types.py +218 -218
- powertrain_build/unit_configs.py +419 -419
- powertrain_build/user_defined_types.py +660 -660
- powertrain_build/versioncheck.py +66 -66
- powertrain_build/wrapper.py +512 -512
- powertrain_build/xlrd_csv.py +87 -87
- powertrain_build/zone_controller/__init__.py +4 -4
- powertrain_build/zone_controller/calibration.py +176 -176
- powertrain_build/zone_controller/composition_yaml.py +880 -878
- {powertrain_build-1.13.1.dist-info → powertrain_build-1.13.3.dev3.dist-info}/METADATA +100 -100
- powertrain_build-1.13.3.dev3.dist-info/RECORD +130 -0
- {powertrain_build-1.13.1.dist-info → powertrain_build-1.13.3.dev3.dist-info}/WHEEL +1 -1
- {powertrain_build-1.13.1.dist-info → powertrain_build-1.13.3.dev3.dist-info}/licenses/LICENSE +202 -202
- powertrain_build-1.13.3.dev3.dist-info/pbr.json +1 -0
- powertrain_build-1.13.1.dist-info/RECORD +0 -130
- powertrain_build-1.13.1.dist-info/pbr.json +0 -1
- {powertrain_build-1.13.1.dist-info → powertrain_build-1.13.3.dev3.dist-info}/entry_points.txt +0 -0
- {powertrain_build-1.13.1.dist-info → powertrain_build-1.13.3.dev3.dist-info}/licenses/AUTHORS +0 -0
- {powertrain_build-1.13.1.dist-info → powertrain_build-1.13.3.dev3.dist-info}/licenses/NOTICE +0 -0
- {powertrain_build-1.13.1.dist-info → powertrain_build-1.13.3.dev3.dist-info}/top_level.txt +0 -0
powertrain_build/wrapper.py
CHANGED
|
@@ -1,512 +1,512 @@
|
|
|
1
|
-
# Copyright 2024 Volvo Car Corporation
|
|
2
|
-
# Licensed under Apache 2.0.
|
|
3
|
-
|
|
4
|
-
"""Performs compatibility upgrade of models to PyBuild using Matlab."""
|
|
5
|
-
|
|
6
|
-
import argparse
|
|
7
|
-
import hashlib
|
|
8
|
-
import json
|
|
9
|
-
import os
|
|
10
|
-
import sys
|
|
11
|
-
from pathlib import Path
|
|
12
|
-
from re import search
|
|
13
|
-
from typing import List, Optional
|
|
14
|
-
|
|
15
|
-
try:
|
|
16
|
-
from importlib.resources import files
|
|
17
|
-
except ImportError:
|
|
18
|
-
from importlib_resources import files
|
|
19
|
-
import shutil
|
|
20
|
-
|
|
21
|
-
import git
|
|
22
|
-
from powertrain_build import build, pt_matlab
|
|
23
|
-
from powertrain_build.lib import logger, helper_functions
|
|
24
|
-
|
|
25
|
-
LOGGER = logger.create_logger(__file__)
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
class PyBuildWrapper(pt_matlab.Matlab):
|
|
29
|
-
"""Performs upgrade of Matlab models to PyBuild system."""
|
|
30
|
-
|
|
31
|
-
HASH_FILE_NAME = "pybuild_file_hashes.json"
|
|
32
|
-
PARSER_HELP = "Run PyBuild, update and/or generate code for selected models and/or build."
|
|
33
|
-
|
|
34
|
-
def __init__(self, args):
|
|
35
|
-
"""Constructor, initializes paths for PyBuild upgrader.
|
|
36
|
-
|
|
37
|
-
Args:
|
|
38
|
-
args (argument parser): see add_args static method.
|
|
39
|
-
"""
|
|
40
|
-
super().__init__(dry_run=args.dry_run, matlab_bin=args.matlab_bin, nojvm=False)
|
|
41
|
-
|
|
42
|
-
self.root_path = helper_functions.get_repo_root()
|
|
43
|
-
self.repo = git.Repo(self.root_path)
|
|
44
|
-
|
|
45
|
-
self.target_link_settings_hash_file_path = Path("ConfigDocuments", "target_link_settings_file_hashes.json")
|
|
46
|
-
self.target_link_settings_folders = [
|
|
47
|
-
Path("ConfigDocuments", "targetlinkSettings"),
|
|
48
|
-
Path("ConfigDocuments", "TL4_3_settings"),
|
|
49
|
-
]
|
|
50
|
-
|
|
51
|
-
self.update = args.update
|
|
52
|
-
self.codegen = args.codegen
|
|
53
|
-
|
|
54
|
-
self.build = args.build
|
|
55
|
-
self.build_specific = getattr(args, "project_config", None) is not None
|
|
56
|
-
self.project_config = self._set_project_configuration(args)
|
|
57
|
-
self.generate_system_info = getattr(args, "generate_system_info", False)
|
|
58
|
-
self.core_dummy = getattr(args, "core_dummy", True)
|
|
59
|
-
self.rte_dummy = getattr(args, "rte_dummy", False)
|
|
60
|
-
self.debug = getattr(args, "debug", True)
|
|
61
|
-
self.no_abort = getattr(args, "no_abort", True)
|
|
62
|
-
self.no_nvm_a2l = getattr(args, "no_nvm_a2l", False)
|
|
63
|
-
self.complete_a2l = getattr(args, "complete_a2l", False)
|
|
64
|
-
self.silver_a2l = getattr(args, "silver_a2l", False)
|
|
65
|
-
self.generate_rte_checkpoint_calls = getattr(args, "generate_rte_checkpoint_calls", False)
|
|
66
|
-
self.interface = args.interface
|
|
67
|
-
self.matlab_include = args.include
|
|
68
|
-
# Always default to conversion table in pytools/config/conversion_table.json.
|
|
69
|
-
# Unless otherwise specified.
|
|
70
|
-
conv_tab_path = getattr(args, "generate_custom_conv_tab", None)
|
|
71
|
-
default_conv_tab_path = os.path.join("ConfigDocuments", "pytools_settings", "conversion_table.json")
|
|
72
|
-
self.conv_tab = conv_tab_path if conv_tab_path is not None else default_conv_tab_path
|
|
73
|
-
|
|
74
|
-
self.should_run, self.models = self._evaluate_run_and_models(args)
|
|
75
|
-
|
|
76
|
-
@staticmethod
|
|
77
|
-
def _set_project_configuration(args):
|
|
78
|
-
"""Evaluate path to project configuration file.
|
|
79
|
-
|
|
80
|
-
Args:
|
|
81
|
-
args (argument parser): see add_args static method.
|
|
82
|
-
Returns:
|
|
83
|
-
project_config: Path to project configuration file.
|
|
84
|
-
"""
|
|
85
|
-
if getattr(args, "project_config", None) is not None:
|
|
86
|
-
project_config = args.project_config.replace("/", os.sep)
|
|
87
|
-
elif args.build is not None:
|
|
88
|
-
if args.build.lower() == "custom":
|
|
89
|
-
# TODO: Change other scripts to accept custom config
|
|
90
|
-
# Change this to point to a config file instead
|
|
91
|
-
project_config = os.path.join(os.environ.get("PROJECT_DIR"), "ProjectCfg.json")
|
|
92
|
-
else:
|
|
93
|
-
project_config = os.path.join("Projects", args.build, "ProjectCfg.json")
|
|
94
|
-
else:
|
|
95
|
-
project_config = None
|
|
96
|
-
|
|
97
|
-
return project_config
|
|
98
|
-
|
|
99
|
-
@staticmethod
|
|
100
|
-
def convert_path_sep(paths):
|
|
101
|
-
"""Matlab requires forward-spaces for model paths. Convert backslashes.
|
|
102
|
-
|
|
103
|
-
Args:
|
|
104
|
-
paths (list): Model paths to fix path separators for.
|
|
105
|
-
Returns:
|
|
106
|
-
(list): Models paths separated with forward slashes.
|
|
107
|
-
"""
|
|
108
|
-
return [path.replace("\\", "/") for path in paths]
|
|
109
|
-
|
|
110
|
-
@staticmethod
|
|
111
|
-
def get_matlab_scripts_commit_sha():
|
|
112
|
-
"""Get current commit sha of matlab-scripts submodule, if available.
|
|
113
|
-
|
|
114
|
-
Returns:
|
|
115
|
-
matlab_scripts_commit_sha (str): Commit sha of matlab-scripts submodule.
|
|
116
|
-
None, if not available.
|
|
117
|
-
"""
|
|
118
|
-
repo = git.Repo()
|
|
119
|
-
try:
|
|
120
|
-
matlab_scripts = repo.submodule("matlab-scripts")
|
|
121
|
-
matlab_scripts_commit_sha = matlab_scripts.hexsha
|
|
122
|
-
except ValueError:
|
|
123
|
-
LOGGER.info("Submodule matlab-scripts not available, " "skipping adding its commit sha to hash file.")
|
|
124
|
-
matlab_scripts_commit_sha = None
|
|
125
|
-
return matlab_scripts_commit_sha
|
|
126
|
-
|
|
127
|
-
@staticmethod
|
|
128
|
-
def read_bytes(file_path):
|
|
129
|
-
"""Read file contents in byte mode.
|
|
130
|
-
|
|
131
|
-
Args:
|
|
132
|
-
file_path (Path): Path to file.
|
|
133
|
-
Returns:
|
|
134
|
-
file_bytes (bytes): Content of file read in binary mode.
|
|
135
|
-
"""
|
|
136
|
-
if file_path.suffix in [".slx", ".mdl", ".mexw64"]:
|
|
137
|
-
# Already treated as binary file
|
|
138
|
-
file_bytes = file_path.read_bytes()
|
|
139
|
-
else:
|
|
140
|
-
# Hack to make this script os independent
|
|
141
|
-
with file_path.open(encoding="iso-8859-1") as file_handle:
|
|
142
|
-
content = file_handle.read()
|
|
143
|
-
file_bytes = content.encode("iso-8859-1")
|
|
144
|
-
return file_bytes
|
|
145
|
-
|
|
146
|
-
@staticmethod
|
|
147
|
-
def get_files_to_hash(model_folder):
|
|
148
|
-
"""Get files to generate hashes for (PyBuild specific files).
|
|
149
|
-
|
|
150
|
-
Args:
|
|
151
|
-
model_folder (Path): Path to model folder.
|
|
152
|
-
Returns:
|
|
153
|
-
files_to_hash (list): List of, pybuild specific, files to generate hashes for.
|
|
154
|
-
"""
|
|
155
|
-
valid_file_endings = [
|
|
156
|
-
# source files
|
|
157
|
-
".a2l",
|
|
158
|
-
".c",
|
|
159
|
-
".h",
|
|
160
|
-
".mexw64",
|
|
161
|
-
".tlc",
|
|
162
|
-
# config files
|
|
163
|
-
".json",
|
|
164
|
-
# model files
|
|
165
|
-
".mdl",
|
|
166
|
-
".m",
|
|
167
|
-
]
|
|
168
|
-
valid_folders = [
|
|
169
|
-
model_folder,
|
|
170
|
-
Path(model_folder, "matlab_src"),
|
|
171
|
-
Path(model_folder, "pybuild_cfg"),
|
|
172
|
-
Path(model_folder, "pybuild_src"),
|
|
173
|
-
]
|
|
174
|
-
hash_file_path = Path(model_folder, PyBuildWrapper.HASH_FILE_NAME)
|
|
175
|
-
|
|
176
|
-
model_files = list(model_folder.rglob("*.*"))
|
|
177
|
-
if hash_file_path.exists():
|
|
178
|
-
model_files.remove(hash_file_path)
|
|
179
|
-
files_to_hash = [f for f in model_files if f.parent in valid_folders and f.suffix in valid_file_endings]
|
|
180
|
-
return files_to_hash
|
|
181
|
-
|
|
182
|
-
@staticmethod
|
|
183
|
-
def get_shared_function_files():
|
|
184
|
-
"""Get shared function files, files generated by VcSharedFunctions.mdl.
|
|
185
|
-
|
|
186
|
-
Returns:
|
|
187
|
-
shared_function_files (list): List of files generated by VcSharedFunctions.mdl.
|
|
188
|
-
"""
|
|
189
|
-
shared_function_files = []
|
|
190
|
-
common_source_folder = Path("Models/Common/pybuild_src")
|
|
191
|
-
for source_file in common_source_folder.glob("*.*"):
|
|
192
|
-
with source_file.open() as file_handle:
|
|
193
|
-
content = file_handle.read()
|
|
194
|
-
if search(r"Simulink model\s+: VcSharedFunctions", content) is not None:
|
|
195
|
-
shared_function_files.append(source_file)
|
|
196
|
-
return shared_function_files
|
|
197
|
-
|
|
198
|
-
@staticmethod
|
|
199
|
-
def get_file_hashes(model_paths, write_to_file=True):
|
|
200
|
-
"""Calculate SHA256 file hashes for files in given model folders.
|
|
201
|
-
|
|
202
|
-
Args:
|
|
203
|
-
model_paths (list): List of model paths.
|
|
204
|
-
write_to_file (bool): True/False whether calculated hashes should be written to file.
|
|
205
|
-
Returns:
|
|
206
|
-
model_to_files_hash_dict (dict): Dict mapping model name to files and its hashes.
|
|
207
|
-
"""
|
|
208
|
-
model_to_files_hash_dict = {}
|
|
209
|
-
model_folders = [(Path(m).parent, Path(m).stem) for m in model_paths]
|
|
210
|
-
for model_folder, model_name in model_folders:
|
|
211
|
-
file_to_hash_dict = {}
|
|
212
|
-
files_to_hash = PyBuildWrapper.get_files_to_hash(model_folder)
|
|
213
|
-
if "VcSharedFunctions" in model_folder.as_posix():
|
|
214
|
-
shared_function_files = PyBuildWrapper.get_shared_function_files()
|
|
215
|
-
files_to_hash.extend(shared_function_files)
|
|
216
|
-
for file_to_hash in files_to_hash:
|
|
217
|
-
file_bytes = PyBuildWrapper.read_bytes(file_to_hash)
|
|
218
|
-
file_to_hash_dict[file_to_hash.name] = hashlib.sha256(file_bytes).hexdigest()
|
|
219
|
-
commit_sha = PyBuildWrapper.get_matlab_scripts_commit_sha()
|
|
220
|
-
if commit_sha is not None:
|
|
221
|
-
file_to_hash_dict["matlab-scripts"] = commit_sha
|
|
222
|
-
if write_to_file:
|
|
223
|
-
with Path(model_folder, PyBuildWrapper.HASH_FILE_NAME).open("w", encoding="utf-8") as file_handle:
|
|
224
|
-
json.dump(file_to_hash_dict, file_handle, indent=4)
|
|
225
|
-
model_to_files_hash_dict[model_name] = file_to_hash_dict
|
|
226
|
-
return model_to_files_hash_dict
|
|
227
|
-
|
|
228
|
-
def _evaluate_run_and_models(self, args):
|
|
229
|
-
"""Evaluate if PyBuild Matlab related parts should run.
|
|
230
|
-
|
|
231
|
-
Additionally, it sets which models to update and/or generate code for, based on arguments.
|
|
232
|
-
NOTE: model_list=None indicates all models when run is True. Deprecated, add force flag?
|
|
233
|
-
|
|
234
|
-
Args:
|
|
235
|
-
args (argument parser): see add_args static method.
|
|
236
|
-
Returns:
|
|
237
|
-
should_run: True/False whether PyBuild should run or not.
|
|
238
|
-
model_list: List of models to update/generate code for.
|
|
239
|
-
"""
|
|
240
|
-
run_powertrain_build = args.build is not None or getattr(args, "project_config", None) is not None
|
|
241
|
-
should_run = args.update or args.codegen or run_powertrain_build
|
|
242
|
-
model_list = args.models if args.models else self.get_changed_models()
|
|
243
|
-
LOGGER.info("Affected models: %s", ", ".join(model_list))
|
|
244
|
-
if not model_list:
|
|
245
|
-
# PyBuild should not run if there were no model changes
|
|
246
|
-
should_run = run_powertrain_build
|
|
247
|
-
model_list = None
|
|
248
|
-
|
|
249
|
-
return should_run, model_list
|
|
250
|
-
|
|
251
|
-
def regenerate_target_link_settings_file_hashes(self):
|
|
252
|
-
"""Regenerate the file mapping TargetLink settings files to their hashes."""
|
|
253
|
-
target_link_settings_file_hashes = self.calculate_target_link_settings_file_hashes()
|
|
254
|
-
with self.target_link_settings_hash_file_path.open("w", encoding="utf-8") as file_handle:
|
|
255
|
-
json.dump(target_link_settings_file_hashes, file_handle, indent=4)
|
|
256
|
-
|
|
257
|
-
def calculate_target_link_settings_file_hashes(self):
|
|
258
|
-
"""Calculate SHA256 file hashes for files in TargetLink settings folders.
|
|
259
|
-
|
|
260
|
-
Returns:
|
|
261
|
-
file_to_hash_dict (dict): Dict mapping settings files and their hashes.
|
|
262
|
-
"""
|
|
263
|
-
file_to_hash_dict = {}
|
|
264
|
-
for settings_folder in self.target_link_settings_folders:
|
|
265
|
-
for settings_file in settings_folder.rglob("*.*"):
|
|
266
|
-
file_bytes = self.read_bytes(settings_file)
|
|
267
|
-
file_to_hash_dict[settings_file.name] = hashlib.sha256(file_bytes).hexdigest()
|
|
268
|
-
return file_to_hash_dict
|
|
269
|
-
|
|
270
|
-
def verify_target_link_settings(self):
|
|
271
|
-
"""Verify current TargetLink settings, comparing against commited file hashes file.
|
|
272
|
-
|
|
273
|
-
Returns:
|
|
274
|
-
(bool): True/False, depending on if the TargetLink settings have changed.
|
|
275
|
-
"""
|
|
276
|
-
if not self.target_link_settings_hash_file_path.exists():
|
|
277
|
-
message = (
|
|
278
|
-
"Could not read TargetLink settings file hashes file: "
|
|
279
|
-
f"{self.target_link_settings_hash_file_path.as_posix()}.\n"
|
|
280
|
-
"If your repo runs the jobb PyBuildDiff in hash mode, make sure to generate one."
|
|
281
|
-
"Ignoring settings verification."
|
|
282
|
-
)
|
|
283
|
-
LOGGER.warning(message)
|
|
284
|
-
return True
|
|
285
|
-
|
|
286
|
-
with self.target_link_settings_hash_file_path.open(encoding="utf-8") as file_handle:
|
|
287
|
-
current_file_hashes_dict = json.load(file_handle)
|
|
288
|
-
|
|
289
|
-
new_file_hashes_dict = self.calculate_target_link_settings_file_hashes()
|
|
290
|
-
|
|
291
|
-
return new_file_hashes_dict == current_file_hashes_dict
|
|
292
|
-
|
|
293
|
-
def get_changed_models(self):
|
|
294
|
-
"""Get changed models in current commit."""
|
|
295
|
-
changed_files_tmp = self.repo.git.diff("--diff-filter=d", "--name-only", "HEAD~1")
|
|
296
|
-
changed_files = changed_files_tmp.splitlines()
|
|
297
|
-
changed_models = [m for m in changed_files if m.endswith(".mdl") or m.endswith(".slx")]
|
|
298
|
-
return changed_models
|
|
299
|
-
|
|
300
|
-
def check_generate_shared_functions(self, matlab_command_list):
|
|
301
|
-
"""Check if shared function files should be generated.
|
|
302
|
-
|
|
303
|
-
Args:
|
|
304
|
-
matlab_command_list ([str]): list of matlab commands.
|
|
305
|
-
"""
|
|
306
|
-
if [model for model in self.models if model.endswith("VcSharedFunctions.mdl")]:
|
|
307
|
-
matlab_command_list.append(pt_matlab.cmd_callfunc("generateSharedFunctions", True))
|
|
308
|
-
|
|
309
|
-
def build_automation(self, mode):
|
|
310
|
-
"""Run Matlab with a specific task and specific models.
|
|
311
|
-
|
|
312
|
-
Args:
|
|
313
|
-
mode (str): Matlab run mode (update, codegen).
|
|
314
|
-
Returns:
|
|
315
|
-
exit_code (int): Exit code from Matlab.
|
|
316
|
-
"""
|
|
317
|
-
# Will be used in submodule matlab-scripts (if up to date), CodeGen/updateCodeSwConfig.m
|
|
318
|
-
calling_python = sys.version_info
|
|
319
|
-
calling_python_string = f"py -{calling_python.major}.{calling_python.minor}"
|
|
320
|
-
os.environ.setdefault("CALLING_PYTHON", calling_python_string)
|
|
321
|
-
|
|
322
|
-
# Specify a new script and log file name for each run
|
|
323
|
-
script_name = f"powertrain_build_matlab_{mode}.m"
|
|
324
|
-
self.log = f"powertrain_build_matlab_{mode}.log"
|
|
325
|
-
|
|
326
|
-
# Set up matlab path
|
|
327
|
-
matlab_scripts_path = os.path.join("powertrain_build_matlab_scripts")
|
|
328
|
-
|
|
329
|
-
# Copy matlab-scripts to the project root
|
|
330
|
-
matlab_script_folder = files("powertrain_build.matlab_scripts")
|
|
331
|
-
shutil.rmtree(matlab_scripts_path, ignore_errors=True)
|
|
332
|
-
shutil.copytree(matlab_script_folder, matlab_scripts_path)
|
|
333
|
-
|
|
334
|
-
cmds = []
|
|
335
|
-
cmds.append(f"cd '{self.root_path}'")
|
|
336
|
-
cmds.append(pt_matlab.cmd_path(matlab_scripts_path, True))
|
|
337
|
-
cmds.append(pt_matlab.cmd_path(self.matlab_include, True))
|
|
338
|
-
|
|
339
|
-
if mode == "codegen":
|
|
340
|
-
cmds.append(pt_matlab.cmd_callfunc("loadLibraries", self.matlab_include))
|
|
341
|
-
|
|
342
|
-
# Generate the command for calling the main build script:
|
|
343
|
-
args = [mode, True]
|
|
344
|
-
if self.models:
|
|
345
|
-
self.check_generate_shared_functions(cmds)
|
|
346
|
-
args.append(self.convert_path_sep(self.models))
|
|
347
|
-
|
|
348
|
-
cmds.append(pt_matlab.cmd_callfunc("BuildAutomationPyBuild", *args))
|
|
349
|
-
pt_matlab.write_m_script(script_name, pt_matlab.cmds_join(cmds), wrap_cmd=False)
|
|
350
|
-
|
|
351
|
-
# Reset the Matlab watcher before running a new m-script
|
|
352
|
-
self.matlab_watcher.reset_errors()
|
|
353
|
-
|
|
354
|
-
self.run_m_script(script_name, wrap_cmd=False, attempts=2)
|
|
355
|
-
|
|
356
|
-
if not self.matlab_watcher.task_success:
|
|
357
|
-
LOGGER.error("PyBuild %s error!", mode)
|
|
358
|
-
return 1
|
|
359
|
-
|
|
360
|
-
if not self.verify_target_link_settings():
|
|
361
|
-
LOGGER.error("Your TargetLink settings differ from the ones set by current branch.")
|
|
362
|
-
return 1
|
|
363
|
-
|
|
364
|
-
self.get_file_hashes(self.models)
|
|
365
|
-
return 0
|
|
366
|
-
|
|
367
|
-
def run_powertrain_build(self):
|
|
368
|
-
"""Execute powertrain-build.
|
|
369
|
-
|
|
370
|
-
Returns:
|
|
371
|
-
exit_code: Exit code from powertrain-build build step.
|
|
372
|
-
"""
|
|
373
|
-
try:
|
|
374
|
-
exit_code = build.build(
|
|
375
|
-
self.project_config,
|
|
376
|
-
interface=self.interface,
|
|
377
|
-
core_dummy=self.core_dummy,
|
|
378
|
-
rte_dummy=self.rte_dummy,
|
|
379
|
-
no_abort=self.no_abort,
|
|
380
|
-
no_nvm_a2l=self.no_nvm_a2l,
|
|
381
|
-
debug=self.debug,
|
|
382
|
-
generate_system_info=self.generate_system_info,
|
|
383
|
-
generate_custom_conversion_table=self.conv_tab,
|
|
384
|
-
complete_a2l=self.complete_a2l,
|
|
385
|
-
silver_a2l=self.silver_a2l,
|
|
386
|
-
generate_rte_checkpoint_calls=self.generate_rte_checkpoint_calls,
|
|
387
|
-
)
|
|
388
|
-
except (FileNotFoundError, PermissionError) as ex:
|
|
389
|
-
LOGGER.error(ex)
|
|
390
|
-
exit_code = 1
|
|
391
|
-
return exit_code
|
|
392
|
-
|
|
393
|
-
def run(self):
|
|
394
|
-
"""Run PyBuild, update and/or generate code for selected models and/or build.
|
|
395
|
-
|
|
396
|
-
Returns:
|
|
397
|
-
exit_code: Exit code from Matlab and build step.
|
|
398
|
-
"""
|
|
399
|
-
if not self.should_run:
|
|
400
|
-
return 0
|
|
401
|
-
|
|
402
|
-
LOGGER.info("Preparing workspace for PyBuild!")
|
|
403
|
-
|
|
404
|
-
exit_code = 0
|
|
405
|
-
if self.update:
|
|
406
|
-
LOGGER.info("Running PyBuild update!")
|
|
407
|
-
exit_code |= self.build_automation(mode="update")
|
|
408
|
-
|
|
409
|
-
if self.codegen:
|
|
410
|
-
LOGGER.info("Running PyBuild generate code!")
|
|
411
|
-
|
|
412
|
-
exit_code |= self.build_automation(mode="codegen")
|
|
413
|
-
|
|
414
|
-
if self.build:
|
|
415
|
-
LOGGER.info("Running PyBuild.build for %s!", self.build)
|
|
416
|
-
exit_code |= self.run_powertrain_build()
|
|
417
|
-
|
|
418
|
-
if self.build_specific:
|
|
419
|
-
LOGGER.info("Running PyBuild specific build for %s!", self.project_config)
|
|
420
|
-
exit_code |= self.run_powertrain_build()
|
|
421
|
-
|
|
422
|
-
return exit_code
|
|
423
|
-
|
|
424
|
-
@staticmethod
|
|
425
|
-
def add_args(parser):
|
|
426
|
-
"""Add expected arguments to the supplied parser.
|
|
427
|
-
|
|
428
|
-
Args:
|
|
429
|
-
parser (ArgumentParser): parser to add arguments to.
|
|
430
|
-
"""
|
|
431
|
-
parser.add_argument(
|
|
432
|
-
"--dry-run",
|
|
433
|
-
action="store_true",
|
|
434
|
-
help="Dry run: No changes, simulation only (default: False).",
|
|
435
|
-
)
|
|
436
|
-
parser.add_argument("--repo-root", help="Path to repository where work should be done")
|
|
437
|
-
parser.add_argument(
|
|
438
|
-
"--interface",
|
|
439
|
-
action="store_true",
|
|
440
|
-
help="Create interface consistency report (default: False)" "NOTE: This requires the --build flag.",
|
|
441
|
-
)
|
|
442
|
-
parser.add_argument(
|
|
443
|
-
"--models",
|
|
444
|
-
nargs="+",
|
|
445
|
-
default=None,
|
|
446
|
-
help="List of model files (full path, "
|
|
447
|
-
"e.g. Models/<SSP>/<MODEL>/<MDL-FILE>) to upgrade, separated with "
|
|
448
|
-
"spaces. Takes precedence over Git HEAD.",
|
|
449
|
-
)
|
|
450
|
-
parser.add_argument(
|
|
451
|
-
"--update",
|
|
452
|
-
action="store_true",
|
|
453
|
-
help="Run PyBuild update on models (default: False).",
|
|
454
|
-
)
|
|
455
|
-
parser.add_argument(
|
|
456
|
-
"--codegen",
|
|
457
|
-
action="store_true",
|
|
458
|
-
help="Run PyBuild code generation on models (default: False). "
|
|
459
|
-
"NOTE: This requires either the --update flag or already updated models.",
|
|
460
|
-
)
|
|
461
|
-
parser.add_argument(
|
|
462
|
-
"--build",
|
|
463
|
-
default=None,
|
|
464
|
-
help="Run PyBuild for project with standard settings, VCC SPM SW release " "(default: None).",
|
|
465
|
-
)
|
|
466
|
-
parser.add_argument(
|
|
467
|
-
"--regenerate-tl-settings-hashes",
|
|
468
|
-
action="store_true",
|
|
469
|
-
help="Regenerate the file mapping TargetLink settings files to their hashes.",
|
|
470
|
-
)
|
|
471
|
-
parser.add_argument(
|
|
472
|
-
"--include",
|
|
473
|
-
default="matlab-scripts",
|
|
474
|
-
help="Path to folder containing Matlab scripts and simulink libraries to include.",
|
|
475
|
-
)
|
|
476
|
-
parser.set_defaults(func=run_wrapper)
|
|
477
|
-
|
|
478
|
-
subparsers = parser.add_subparsers(help="PyBuild specific build.")
|
|
479
|
-
build_specific_parser = subparsers.add_parser(
|
|
480
|
-
"build-specific", help="Run PyBuild for project with specific settings."
|
|
481
|
-
)
|
|
482
|
-
build.add_args(build_specific_parser)
|
|
483
|
-
|
|
484
|
-
# Matlab arguments added by parent:
|
|
485
|
-
pt_matlab.Matlab.add_args(parser)
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
def run_wrapper(args: argparse.Namespace) -> int:
|
|
489
|
-
"""Run PyBuildWrapper."""
|
|
490
|
-
if args.build is not None and getattr(args, "project_config", None) is not None:
|
|
491
|
-
LOGGER.error("Cannot run both PyBuild quick build (--build <PROJECT>) " "and specific build (build-specific).")
|
|
492
|
-
return 1
|
|
493
|
-
|
|
494
|
-
wrapper = PyBuildWrapper(args)
|
|
495
|
-
|
|
496
|
-
if args.regenerate_tl_settings_hashes:
|
|
497
|
-
wrapper.regenerate_target_link_settings_file_hashes()
|
|
498
|
-
return 0
|
|
499
|
-
|
|
500
|
-
return wrapper.run()
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
def main(argv: Optional[List[str]] = None) -> int:
|
|
504
|
-
"""Run PyBuildWrapper"""
|
|
505
|
-
parser = argparse.ArgumentParser("PyBuild Wrapper")
|
|
506
|
-
PyBuildWrapper.add_args(parser)
|
|
507
|
-
args = parser.parse_args(argv)
|
|
508
|
-
return run_wrapper(args)
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
if __name__ == "__main__":
|
|
512
|
-
sys.exit(main(sys.argv[1:]))
|
|
1
|
+
# Copyright 2024 Volvo Car Corporation
|
|
2
|
+
# Licensed under Apache 2.0.
|
|
3
|
+
|
|
4
|
+
"""Performs compatibility upgrade of models to PyBuild using Matlab."""
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import sys
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from re import search
|
|
13
|
+
from typing import List, Optional
|
|
14
|
+
|
|
15
|
+
try:
|
|
16
|
+
from importlib.resources import files
|
|
17
|
+
except ImportError:
|
|
18
|
+
from importlib_resources import files
|
|
19
|
+
import shutil
|
|
20
|
+
|
|
21
|
+
import git
|
|
22
|
+
from powertrain_build import build, pt_matlab
|
|
23
|
+
from powertrain_build.lib import logger, helper_functions
|
|
24
|
+
|
|
25
|
+
LOGGER = logger.create_logger(__file__)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class PyBuildWrapper(pt_matlab.Matlab):
|
|
29
|
+
"""Performs upgrade of Matlab models to PyBuild system."""
|
|
30
|
+
|
|
31
|
+
HASH_FILE_NAME = "pybuild_file_hashes.json"
|
|
32
|
+
PARSER_HELP = "Run PyBuild, update and/or generate code for selected models and/or build."
|
|
33
|
+
|
|
34
|
+
def __init__(self, args):
|
|
35
|
+
"""Constructor, initializes paths for PyBuild upgrader.
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
args (argument parser): see add_args static method.
|
|
39
|
+
"""
|
|
40
|
+
super().__init__(dry_run=args.dry_run, matlab_bin=args.matlab_bin, nojvm=False)
|
|
41
|
+
|
|
42
|
+
self.root_path = helper_functions.get_repo_root()
|
|
43
|
+
self.repo = git.Repo(self.root_path)
|
|
44
|
+
|
|
45
|
+
self.target_link_settings_hash_file_path = Path("ConfigDocuments", "target_link_settings_file_hashes.json")
|
|
46
|
+
self.target_link_settings_folders = [
|
|
47
|
+
Path("ConfigDocuments", "targetlinkSettings"),
|
|
48
|
+
Path("ConfigDocuments", "TL4_3_settings"),
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
self.update = args.update
|
|
52
|
+
self.codegen = args.codegen
|
|
53
|
+
|
|
54
|
+
self.build = args.build
|
|
55
|
+
self.build_specific = getattr(args, "project_config", None) is not None
|
|
56
|
+
self.project_config = self._set_project_configuration(args)
|
|
57
|
+
self.generate_system_info = getattr(args, "generate_system_info", False)
|
|
58
|
+
self.core_dummy = getattr(args, "core_dummy", True)
|
|
59
|
+
self.rte_dummy = getattr(args, "rte_dummy", False)
|
|
60
|
+
self.debug = getattr(args, "debug", True)
|
|
61
|
+
self.no_abort = getattr(args, "no_abort", True)
|
|
62
|
+
self.no_nvm_a2l = getattr(args, "no_nvm_a2l", False)
|
|
63
|
+
self.complete_a2l = getattr(args, "complete_a2l", False)
|
|
64
|
+
self.silver_a2l = getattr(args, "silver_a2l", False)
|
|
65
|
+
self.generate_rte_checkpoint_calls = getattr(args, "generate_rte_checkpoint_calls", False)
|
|
66
|
+
self.interface = args.interface
|
|
67
|
+
self.matlab_include = args.include
|
|
68
|
+
# Always default to conversion table in pytools/config/conversion_table.json.
|
|
69
|
+
# Unless otherwise specified.
|
|
70
|
+
conv_tab_path = getattr(args, "generate_custom_conv_tab", None)
|
|
71
|
+
default_conv_tab_path = os.path.join("ConfigDocuments", "pytools_settings", "conversion_table.json")
|
|
72
|
+
self.conv_tab = conv_tab_path if conv_tab_path is not None else default_conv_tab_path
|
|
73
|
+
|
|
74
|
+
self.should_run, self.models = self._evaluate_run_and_models(args)
|
|
75
|
+
|
|
76
|
+
@staticmethod
|
|
77
|
+
def _set_project_configuration(args):
|
|
78
|
+
"""Evaluate path to project configuration file.
|
|
79
|
+
|
|
80
|
+
Args:
|
|
81
|
+
args (argument parser): see add_args static method.
|
|
82
|
+
Returns:
|
|
83
|
+
project_config: Path to project configuration file.
|
|
84
|
+
"""
|
|
85
|
+
if getattr(args, "project_config", None) is not None:
|
|
86
|
+
project_config = args.project_config.replace("/", os.sep)
|
|
87
|
+
elif args.build is not None:
|
|
88
|
+
if args.build.lower() == "custom":
|
|
89
|
+
# TODO: Change other scripts to accept custom config
|
|
90
|
+
# Change this to point to a config file instead
|
|
91
|
+
project_config = os.path.join(os.environ.get("PROJECT_DIR"), "ProjectCfg.json")
|
|
92
|
+
else:
|
|
93
|
+
project_config = os.path.join("Projects", args.build, "ProjectCfg.json")
|
|
94
|
+
else:
|
|
95
|
+
project_config = None
|
|
96
|
+
|
|
97
|
+
return project_config
|
|
98
|
+
|
|
99
|
+
@staticmethod
|
|
100
|
+
def convert_path_sep(paths):
|
|
101
|
+
"""Matlab requires forward-spaces for model paths. Convert backslashes.
|
|
102
|
+
|
|
103
|
+
Args:
|
|
104
|
+
paths (list): Model paths to fix path separators for.
|
|
105
|
+
Returns:
|
|
106
|
+
(list): Models paths separated with forward slashes.
|
|
107
|
+
"""
|
|
108
|
+
return [path.replace("\\", "/") for path in paths]
|
|
109
|
+
|
|
110
|
+
@staticmethod
|
|
111
|
+
def get_matlab_scripts_commit_sha():
|
|
112
|
+
"""Get current commit sha of matlab-scripts submodule, if available.
|
|
113
|
+
|
|
114
|
+
Returns:
|
|
115
|
+
matlab_scripts_commit_sha (str): Commit sha of matlab-scripts submodule.
|
|
116
|
+
None, if not available.
|
|
117
|
+
"""
|
|
118
|
+
repo = git.Repo()
|
|
119
|
+
try:
|
|
120
|
+
matlab_scripts = repo.submodule("matlab-scripts")
|
|
121
|
+
matlab_scripts_commit_sha = matlab_scripts.hexsha
|
|
122
|
+
except ValueError:
|
|
123
|
+
LOGGER.info("Submodule matlab-scripts not available, " "skipping adding its commit sha to hash file.")
|
|
124
|
+
matlab_scripts_commit_sha = None
|
|
125
|
+
return matlab_scripts_commit_sha
|
|
126
|
+
|
|
127
|
+
@staticmethod
|
|
128
|
+
def read_bytes(file_path):
|
|
129
|
+
"""Read file contents in byte mode.
|
|
130
|
+
|
|
131
|
+
Args:
|
|
132
|
+
file_path (Path): Path to file.
|
|
133
|
+
Returns:
|
|
134
|
+
file_bytes (bytes): Content of file read in binary mode.
|
|
135
|
+
"""
|
|
136
|
+
if file_path.suffix in [".slx", ".mdl", ".mexw64"]:
|
|
137
|
+
# Already treated as binary file
|
|
138
|
+
file_bytes = file_path.read_bytes()
|
|
139
|
+
else:
|
|
140
|
+
# Hack to make this script os independent
|
|
141
|
+
with file_path.open(encoding="iso-8859-1") as file_handle:
|
|
142
|
+
content = file_handle.read()
|
|
143
|
+
file_bytes = content.encode("iso-8859-1")
|
|
144
|
+
return file_bytes
|
|
145
|
+
|
|
146
|
+
@staticmethod
|
|
147
|
+
def get_files_to_hash(model_folder):
|
|
148
|
+
"""Get files to generate hashes for (PyBuild specific files).
|
|
149
|
+
|
|
150
|
+
Args:
|
|
151
|
+
model_folder (Path): Path to model folder.
|
|
152
|
+
Returns:
|
|
153
|
+
files_to_hash (list): List of, pybuild specific, files to generate hashes for.
|
|
154
|
+
"""
|
|
155
|
+
valid_file_endings = [
|
|
156
|
+
# source files
|
|
157
|
+
".a2l",
|
|
158
|
+
".c",
|
|
159
|
+
".h",
|
|
160
|
+
".mexw64",
|
|
161
|
+
".tlc",
|
|
162
|
+
# config files
|
|
163
|
+
".json",
|
|
164
|
+
# model files
|
|
165
|
+
".mdl",
|
|
166
|
+
".m",
|
|
167
|
+
]
|
|
168
|
+
valid_folders = [
|
|
169
|
+
model_folder,
|
|
170
|
+
Path(model_folder, "matlab_src"),
|
|
171
|
+
Path(model_folder, "pybuild_cfg"),
|
|
172
|
+
Path(model_folder, "pybuild_src"),
|
|
173
|
+
]
|
|
174
|
+
hash_file_path = Path(model_folder, PyBuildWrapper.HASH_FILE_NAME)
|
|
175
|
+
|
|
176
|
+
model_files = list(model_folder.rglob("*.*"))
|
|
177
|
+
if hash_file_path.exists():
|
|
178
|
+
model_files.remove(hash_file_path)
|
|
179
|
+
files_to_hash = [f for f in model_files if f.parent in valid_folders and f.suffix in valid_file_endings]
|
|
180
|
+
return files_to_hash
|
|
181
|
+
|
|
182
|
+
@staticmethod
|
|
183
|
+
def get_shared_function_files():
|
|
184
|
+
"""Get shared function files, files generated by VcSharedFunctions.mdl.
|
|
185
|
+
|
|
186
|
+
Returns:
|
|
187
|
+
shared_function_files (list): List of files generated by VcSharedFunctions.mdl.
|
|
188
|
+
"""
|
|
189
|
+
shared_function_files = []
|
|
190
|
+
common_source_folder = Path("Models/Common/pybuild_src")
|
|
191
|
+
for source_file in common_source_folder.glob("*.*"):
|
|
192
|
+
with source_file.open() as file_handle:
|
|
193
|
+
content = file_handle.read()
|
|
194
|
+
if search(r"Simulink model\s+: VcSharedFunctions", content) is not None:
|
|
195
|
+
shared_function_files.append(source_file)
|
|
196
|
+
return shared_function_files
|
|
197
|
+
|
|
198
|
+
@staticmethod
|
|
199
|
+
def get_file_hashes(model_paths, write_to_file=True):
|
|
200
|
+
"""Calculate SHA256 file hashes for files in given model folders.
|
|
201
|
+
|
|
202
|
+
Args:
|
|
203
|
+
model_paths (list): List of model paths.
|
|
204
|
+
write_to_file (bool): True/False whether calculated hashes should be written to file.
|
|
205
|
+
Returns:
|
|
206
|
+
model_to_files_hash_dict (dict): Dict mapping model name to files and its hashes.
|
|
207
|
+
"""
|
|
208
|
+
model_to_files_hash_dict = {}
|
|
209
|
+
model_folders = [(Path(m).parent, Path(m).stem) for m in model_paths]
|
|
210
|
+
for model_folder, model_name in model_folders:
|
|
211
|
+
file_to_hash_dict = {}
|
|
212
|
+
files_to_hash = PyBuildWrapper.get_files_to_hash(model_folder)
|
|
213
|
+
if "VcSharedFunctions" in model_folder.as_posix():
|
|
214
|
+
shared_function_files = PyBuildWrapper.get_shared_function_files()
|
|
215
|
+
files_to_hash.extend(shared_function_files)
|
|
216
|
+
for file_to_hash in files_to_hash:
|
|
217
|
+
file_bytes = PyBuildWrapper.read_bytes(file_to_hash)
|
|
218
|
+
file_to_hash_dict[file_to_hash.name] = hashlib.sha256(file_bytes).hexdigest()
|
|
219
|
+
commit_sha = PyBuildWrapper.get_matlab_scripts_commit_sha()
|
|
220
|
+
if commit_sha is not None:
|
|
221
|
+
file_to_hash_dict["matlab-scripts"] = commit_sha
|
|
222
|
+
if write_to_file:
|
|
223
|
+
with Path(model_folder, PyBuildWrapper.HASH_FILE_NAME).open("w", encoding="utf-8") as file_handle:
|
|
224
|
+
json.dump(file_to_hash_dict, file_handle, indent=4)
|
|
225
|
+
model_to_files_hash_dict[model_name] = file_to_hash_dict
|
|
226
|
+
return model_to_files_hash_dict
|
|
227
|
+
|
|
228
|
+
def _evaluate_run_and_models(self, args):
|
|
229
|
+
"""Evaluate if PyBuild Matlab related parts should run.
|
|
230
|
+
|
|
231
|
+
Additionally, it sets which models to update and/or generate code for, based on arguments.
|
|
232
|
+
NOTE: model_list=None indicates all models when run is True. Deprecated, add force flag?
|
|
233
|
+
|
|
234
|
+
Args:
|
|
235
|
+
args (argument parser): see add_args static method.
|
|
236
|
+
Returns:
|
|
237
|
+
should_run: True/False whether PyBuild should run or not.
|
|
238
|
+
model_list: List of models to update/generate code for.
|
|
239
|
+
"""
|
|
240
|
+
run_powertrain_build = args.build is not None or getattr(args, "project_config", None) is not None
|
|
241
|
+
should_run = args.update or args.codegen or run_powertrain_build
|
|
242
|
+
model_list = args.models if args.models else self.get_changed_models()
|
|
243
|
+
LOGGER.info("Affected models: %s", ", ".join(model_list))
|
|
244
|
+
if not model_list:
|
|
245
|
+
# PyBuild should not run if there were no model changes
|
|
246
|
+
should_run = run_powertrain_build
|
|
247
|
+
model_list = None
|
|
248
|
+
|
|
249
|
+
return should_run, model_list
|
|
250
|
+
|
|
251
|
+
def regenerate_target_link_settings_file_hashes(self):
|
|
252
|
+
"""Regenerate the file mapping TargetLink settings files to their hashes."""
|
|
253
|
+
target_link_settings_file_hashes = self.calculate_target_link_settings_file_hashes()
|
|
254
|
+
with self.target_link_settings_hash_file_path.open("w", encoding="utf-8") as file_handle:
|
|
255
|
+
json.dump(target_link_settings_file_hashes, file_handle, indent=4)
|
|
256
|
+
|
|
257
|
+
def calculate_target_link_settings_file_hashes(self):
|
|
258
|
+
"""Calculate SHA256 file hashes for files in TargetLink settings folders.
|
|
259
|
+
|
|
260
|
+
Returns:
|
|
261
|
+
file_to_hash_dict (dict): Dict mapping settings files and their hashes.
|
|
262
|
+
"""
|
|
263
|
+
file_to_hash_dict = {}
|
|
264
|
+
for settings_folder in self.target_link_settings_folders:
|
|
265
|
+
for settings_file in settings_folder.rglob("*.*"):
|
|
266
|
+
file_bytes = self.read_bytes(settings_file)
|
|
267
|
+
file_to_hash_dict[settings_file.name] = hashlib.sha256(file_bytes).hexdigest()
|
|
268
|
+
return file_to_hash_dict
|
|
269
|
+
|
|
270
|
+
def verify_target_link_settings(self):
|
|
271
|
+
"""Verify current TargetLink settings, comparing against commited file hashes file.
|
|
272
|
+
|
|
273
|
+
Returns:
|
|
274
|
+
(bool): True/False, depending on if the TargetLink settings have changed.
|
|
275
|
+
"""
|
|
276
|
+
if not self.target_link_settings_hash_file_path.exists():
|
|
277
|
+
message = (
|
|
278
|
+
"Could not read TargetLink settings file hashes file: "
|
|
279
|
+
f"{self.target_link_settings_hash_file_path.as_posix()}.\n"
|
|
280
|
+
"If your repo runs the jobb PyBuildDiff in hash mode, make sure to generate one."
|
|
281
|
+
"Ignoring settings verification."
|
|
282
|
+
)
|
|
283
|
+
LOGGER.warning(message)
|
|
284
|
+
return True
|
|
285
|
+
|
|
286
|
+
with self.target_link_settings_hash_file_path.open(encoding="utf-8") as file_handle:
|
|
287
|
+
current_file_hashes_dict = json.load(file_handle)
|
|
288
|
+
|
|
289
|
+
new_file_hashes_dict = self.calculate_target_link_settings_file_hashes()
|
|
290
|
+
|
|
291
|
+
return new_file_hashes_dict == current_file_hashes_dict
|
|
292
|
+
|
|
293
|
+
def get_changed_models(self):
|
|
294
|
+
"""Get changed models in current commit."""
|
|
295
|
+
changed_files_tmp = self.repo.git.diff("--diff-filter=d", "--name-only", "HEAD~1")
|
|
296
|
+
changed_files = changed_files_tmp.splitlines()
|
|
297
|
+
changed_models = [m for m in changed_files if m.endswith(".mdl") or m.endswith(".slx")]
|
|
298
|
+
return changed_models
|
|
299
|
+
|
|
300
|
+
def check_generate_shared_functions(self, matlab_command_list):
|
|
301
|
+
"""Check if shared function files should be generated.
|
|
302
|
+
|
|
303
|
+
Args:
|
|
304
|
+
matlab_command_list ([str]): list of matlab commands.
|
|
305
|
+
"""
|
|
306
|
+
if [model for model in self.models if model.endswith("VcSharedFunctions.mdl")]:
|
|
307
|
+
matlab_command_list.append(pt_matlab.cmd_callfunc("generateSharedFunctions", True))
|
|
308
|
+
|
|
309
|
+
def build_automation(self, mode):
|
|
310
|
+
"""Run Matlab with a specific task and specific models.
|
|
311
|
+
|
|
312
|
+
Args:
|
|
313
|
+
mode (str): Matlab run mode (update, codegen).
|
|
314
|
+
Returns:
|
|
315
|
+
exit_code (int): Exit code from Matlab.
|
|
316
|
+
"""
|
|
317
|
+
# Will be used in submodule matlab-scripts (if up to date), CodeGen/updateCodeSwConfig.m
|
|
318
|
+
calling_python = sys.version_info
|
|
319
|
+
calling_python_string = f"py -{calling_python.major}.{calling_python.minor}"
|
|
320
|
+
os.environ.setdefault("CALLING_PYTHON", calling_python_string)
|
|
321
|
+
|
|
322
|
+
# Specify a new script and log file name for each run
|
|
323
|
+
script_name = f"powertrain_build_matlab_{mode}.m"
|
|
324
|
+
self.log = f"powertrain_build_matlab_{mode}.log"
|
|
325
|
+
|
|
326
|
+
# Set up matlab path
|
|
327
|
+
matlab_scripts_path = os.path.join("powertrain_build_matlab_scripts")
|
|
328
|
+
|
|
329
|
+
# Copy matlab-scripts to the project root
|
|
330
|
+
matlab_script_folder = files("powertrain_build.matlab_scripts")
|
|
331
|
+
shutil.rmtree(matlab_scripts_path, ignore_errors=True)
|
|
332
|
+
shutil.copytree(matlab_script_folder, matlab_scripts_path)
|
|
333
|
+
|
|
334
|
+
cmds = []
|
|
335
|
+
cmds.append(f"cd '{self.root_path}'")
|
|
336
|
+
cmds.append(pt_matlab.cmd_path(matlab_scripts_path, True))
|
|
337
|
+
cmds.append(pt_matlab.cmd_path(self.matlab_include, True))
|
|
338
|
+
|
|
339
|
+
if mode == "codegen":
|
|
340
|
+
cmds.append(pt_matlab.cmd_callfunc("loadLibraries", self.matlab_include))
|
|
341
|
+
|
|
342
|
+
# Generate the command for calling the main build script:
|
|
343
|
+
args = [mode, True]
|
|
344
|
+
if self.models:
|
|
345
|
+
self.check_generate_shared_functions(cmds)
|
|
346
|
+
args.append(self.convert_path_sep(self.models))
|
|
347
|
+
|
|
348
|
+
cmds.append(pt_matlab.cmd_callfunc("BuildAutomationPyBuild", *args))
|
|
349
|
+
pt_matlab.write_m_script(script_name, pt_matlab.cmds_join(cmds), wrap_cmd=False)
|
|
350
|
+
|
|
351
|
+
# Reset the Matlab watcher before running a new m-script
|
|
352
|
+
self.matlab_watcher.reset_errors()
|
|
353
|
+
|
|
354
|
+
self.run_m_script(script_name, wrap_cmd=False, attempts=2)
|
|
355
|
+
|
|
356
|
+
if not self.matlab_watcher.task_success:
|
|
357
|
+
LOGGER.error("PyBuild %s error!", mode)
|
|
358
|
+
return 1
|
|
359
|
+
|
|
360
|
+
if not self.verify_target_link_settings():
|
|
361
|
+
LOGGER.error("Your TargetLink settings differ from the ones set by current branch.")
|
|
362
|
+
return 1
|
|
363
|
+
|
|
364
|
+
self.get_file_hashes(self.models)
|
|
365
|
+
return 0
|
|
366
|
+
|
|
367
|
+
def run_powertrain_build(self):
|
|
368
|
+
"""Execute powertrain-build.
|
|
369
|
+
|
|
370
|
+
Returns:
|
|
371
|
+
exit_code: Exit code from powertrain-build build step.
|
|
372
|
+
"""
|
|
373
|
+
try:
|
|
374
|
+
exit_code = build.build(
|
|
375
|
+
self.project_config,
|
|
376
|
+
interface=self.interface,
|
|
377
|
+
core_dummy=self.core_dummy,
|
|
378
|
+
rte_dummy=self.rte_dummy,
|
|
379
|
+
no_abort=self.no_abort,
|
|
380
|
+
no_nvm_a2l=self.no_nvm_a2l,
|
|
381
|
+
debug=self.debug,
|
|
382
|
+
generate_system_info=self.generate_system_info,
|
|
383
|
+
generate_custom_conversion_table=self.conv_tab,
|
|
384
|
+
complete_a2l=self.complete_a2l,
|
|
385
|
+
silver_a2l=self.silver_a2l,
|
|
386
|
+
generate_rte_checkpoint_calls=self.generate_rte_checkpoint_calls,
|
|
387
|
+
)
|
|
388
|
+
except (FileNotFoundError, PermissionError) as ex:
|
|
389
|
+
LOGGER.error(ex)
|
|
390
|
+
exit_code = 1
|
|
391
|
+
return exit_code
|
|
392
|
+
|
|
393
|
+
def run(self):
|
|
394
|
+
"""Run PyBuild, update and/or generate code for selected models and/or build.
|
|
395
|
+
|
|
396
|
+
Returns:
|
|
397
|
+
exit_code: Exit code from Matlab and build step.
|
|
398
|
+
"""
|
|
399
|
+
if not self.should_run:
|
|
400
|
+
return 0
|
|
401
|
+
|
|
402
|
+
LOGGER.info("Preparing workspace for PyBuild!")
|
|
403
|
+
|
|
404
|
+
exit_code = 0
|
|
405
|
+
if self.update:
|
|
406
|
+
LOGGER.info("Running PyBuild update!")
|
|
407
|
+
exit_code |= self.build_automation(mode="update")
|
|
408
|
+
|
|
409
|
+
if self.codegen:
|
|
410
|
+
LOGGER.info("Running PyBuild generate code!")
|
|
411
|
+
|
|
412
|
+
exit_code |= self.build_automation(mode="codegen")
|
|
413
|
+
|
|
414
|
+
if self.build:
|
|
415
|
+
LOGGER.info("Running PyBuild.build for %s!", self.build)
|
|
416
|
+
exit_code |= self.run_powertrain_build()
|
|
417
|
+
|
|
418
|
+
if self.build_specific:
|
|
419
|
+
LOGGER.info("Running PyBuild specific build for %s!", self.project_config)
|
|
420
|
+
exit_code |= self.run_powertrain_build()
|
|
421
|
+
|
|
422
|
+
return exit_code
|
|
423
|
+
|
|
424
|
+
@staticmethod
|
|
425
|
+
def add_args(parser):
|
|
426
|
+
"""Add expected arguments to the supplied parser.
|
|
427
|
+
|
|
428
|
+
Args:
|
|
429
|
+
parser (ArgumentParser): parser to add arguments to.
|
|
430
|
+
"""
|
|
431
|
+
parser.add_argument(
|
|
432
|
+
"--dry-run",
|
|
433
|
+
action="store_true",
|
|
434
|
+
help="Dry run: No changes, simulation only (default: False).",
|
|
435
|
+
)
|
|
436
|
+
parser.add_argument("--repo-root", help="Path to repository where work should be done")
|
|
437
|
+
parser.add_argument(
|
|
438
|
+
"--interface",
|
|
439
|
+
action="store_true",
|
|
440
|
+
help="Create interface consistency report (default: False)" "NOTE: This requires the --build flag.",
|
|
441
|
+
)
|
|
442
|
+
parser.add_argument(
|
|
443
|
+
"--models",
|
|
444
|
+
nargs="+",
|
|
445
|
+
default=None,
|
|
446
|
+
help="List of model files (full path, "
|
|
447
|
+
"e.g. Models/<SSP>/<MODEL>/<MDL-FILE>) to upgrade, separated with "
|
|
448
|
+
"spaces. Takes precedence over Git HEAD.",
|
|
449
|
+
)
|
|
450
|
+
parser.add_argument(
|
|
451
|
+
"--update",
|
|
452
|
+
action="store_true",
|
|
453
|
+
help="Run PyBuild update on models (default: False).",
|
|
454
|
+
)
|
|
455
|
+
parser.add_argument(
|
|
456
|
+
"--codegen",
|
|
457
|
+
action="store_true",
|
|
458
|
+
help="Run PyBuild code generation on models (default: False). "
|
|
459
|
+
"NOTE: This requires either the --update flag or already updated models.",
|
|
460
|
+
)
|
|
461
|
+
parser.add_argument(
|
|
462
|
+
"--build",
|
|
463
|
+
default=None,
|
|
464
|
+
help="Run PyBuild for project with standard settings, VCC SPM SW release " "(default: None).",
|
|
465
|
+
)
|
|
466
|
+
parser.add_argument(
|
|
467
|
+
"--regenerate-tl-settings-hashes",
|
|
468
|
+
action="store_true",
|
|
469
|
+
help="Regenerate the file mapping TargetLink settings files to their hashes.",
|
|
470
|
+
)
|
|
471
|
+
parser.add_argument(
|
|
472
|
+
"--include",
|
|
473
|
+
default="matlab-scripts",
|
|
474
|
+
help="Path to folder containing Matlab scripts and simulink libraries to include.",
|
|
475
|
+
)
|
|
476
|
+
parser.set_defaults(func=run_wrapper)
|
|
477
|
+
|
|
478
|
+
subparsers = parser.add_subparsers(help="PyBuild specific build.")
|
|
479
|
+
build_specific_parser = subparsers.add_parser(
|
|
480
|
+
"build-specific", help="Run PyBuild for project with specific settings."
|
|
481
|
+
)
|
|
482
|
+
build.add_args(build_specific_parser)
|
|
483
|
+
|
|
484
|
+
# Matlab arguments added by parent:
|
|
485
|
+
pt_matlab.Matlab.add_args(parser)
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def run_wrapper(args: argparse.Namespace) -> int:
|
|
489
|
+
"""Run PyBuildWrapper."""
|
|
490
|
+
if args.build is not None and getattr(args, "project_config", None) is not None:
|
|
491
|
+
LOGGER.error("Cannot run both PyBuild quick build (--build <PROJECT>) " "and specific build (build-specific).")
|
|
492
|
+
return 1
|
|
493
|
+
|
|
494
|
+
wrapper = PyBuildWrapper(args)
|
|
495
|
+
|
|
496
|
+
if args.regenerate_tl_settings_hashes:
|
|
497
|
+
wrapper.regenerate_target_link_settings_file_hashes()
|
|
498
|
+
return 0
|
|
499
|
+
|
|
500
|
+
return wrapper.run()
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
504
|
+
"""Run PyBuildWrapper"""
|
|
505
|
+
parser = argparse.ArgumentParser("PyBuild Wrapper")
|
|
506
|
+
PyBuildWrapper.add_args(parser)
|
|
507
|
+
args = parser.parse_args(argv)
|
|
508
|
+
return run_wrapper(args)
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
if __name__ == "__main__":
|
|
512
|
+
sys.exit(main(sys.argv[1:]))
|