shrinkwrap-tool 2026.2.1__py3-none-any.whl

shrinkwrap/__init__.py

@@ -0,0 +1 @@
+__version__ = "2026.3.0.dev0"

shrinkwrap/__main__.py

@@ -0,0 +1,4 @@
+from .shrinkwrap_entry import main
+
+if __name__ == "__main__":
+    main()

shrinkwrap/commands/build.py

@@ -0,0 +1,91 @@
+# Copyright (c) 2022, Arm Limited.
+# SPDX-License-Identifier: MIT
+
+import os
+import shrinkwrap.commands.buildall as buildall
+import shrinkwrap.utils.vars as vars
+
+
+cmd_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+def add_parser(parser, formatter):
+	"""
+	Part of the command interface expected by shrinkwrap.py. Adds the
+	subcommand to the parser, along with all options and documentation.
+	Returns the subcommand name.
+	"""
+	cmdp = parser.add_parser(cmd_name,
+		formatter_class=formatter,
+		help="""Builds a specified config and packages it ready
+		     to run.""",
+		epilog="""Custom config store(s) can be defined at at
+		     <SHRINKWRAP_CONFIG> as a colon-separated list of
+		     directories. Building is done at <SHRINKWRAP_BUILD> and
+		     output is saved to <SHRINKWRAP_PACKAGE>. The package
+		     includes all FW binaries, a manifest and a build.sh script
+		     containing all the commands that were executed per config.
+		     Any pre-existing config package directory is first deleted.
+		     Shrinkwrap will always search its default config store even
+		     if <SHRINKWRAP_CONFIG> is not defined. <SHRINKWRAP_BUILD>
+		     and <SHRINKWRAP_PACKAGE> default to '~/.shrinkwrap/build'
+		     and '~/.shrinkwrap/package'. The user can override them by
+		     setting the environment variables.""")
+
+	cmdp.add_argument('config',
+		metavar='config',
+		help="""Config to build. If the config exists relative to the
+		     current directory that config is used. Else if the config
+		     exists relative to the config store then it is used.""")
+
+	cmdp.add_argument('-b', '--btvar',
+		metavar='key=value', required=False, default=[],
+		action='append',
+		help="""Override value for a single build-time variable defined
+		     by the config. Specify option multiple times for multiple
+		     variables. Overrides for variables that have a default
+		     specified by the config are optional.""")
+
+	cmdp.add_argument('-s', '--no-sync',
+		metavar='component', required=False, default=[],
+		action='append',
+		help="""Optionally specify any components whose git repos should
+		     not be synced. For all other components, Shrinkwrap ensures
+		     that all repos are clean and checked out at the correct
+		     revision. Option can be specified multiple times.""")
+
+	cmdp.add_argument('--no-sync-all',
+		required=False, default=False, action='store_true',
+		help="""Do not sync repos for any component, as if --no-sync was
+		     specified for every component in the config.""")
+
+	cmdp.add_argument('--force-sync',
+		metavar='component', required=False, default=[], action='append',
+		help="""Synchronize the repo of the given component even if the local
+		     source directory contains unsaved changes. YOUR CHANGES WILL BE
+		     LOST! In addition, download updates from remote branches.""")
+
+	cmdp.add_argument('--force-sync-all',
+		required=False, default=False, action='store_true',
+		help="""Synchronize all components even if the local source directories
+		     contain unsaved changes. YOUR CHANGES WILL BE LOST! In addition,
+		     download all remote branch updates. Note that this will override
+		     any 'sync: false' config option.""")
+
+	buildall.add_common_args(cmdp)
+
+	return cmd_name
+
+
+def dispatch(args):
+	"""
+	Part of the command interface expected by shrinkwrap.py. Called to
+	execute the subcommand, with the arguments the user passed on the
+	command line. The arguments comply with those requested in add_parser().
+	"""
+	btvars = vars.parse(args.btvar, type='bt')
+	if args.no_sync_all:
+		args.no_sync = True
+	if args.force_sync_all:
+		args.force_sync = True
+	buildall.build([args.config], [btvars], args.no_sync, args.force_sync, args)

shrinkwrap/commands/buildall.py

@@ -0,0 +1,180 @@
+# Copyright (c) 2022, Arm Limited.
+# SPDX-License-Identifier: MIT
+
+import os
+import yaml
+import shrinkwrap.utils.config as config
+import shrinkwrap.utils.graph as ugraph
+import shrinkwrap.utils.runtime as runtime
+import shrinkwrap.utils.workspace as workspace
+
+
+cmd_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+def dflt_jobs():
+	return min(os.cpu_count() // 2, 32)
+
+
+def add_parser(parser, formatter):
+	"""
+	Part of the command interface expected by shrinkwrap.py. Adds the
+	subcommand to the parser, along with all options and documentation.
+	Returns the subcommand name.
+	"""
+	cmdp = parser.add_parser(cmd_name,
+		formatter_class=formatter,
+		help="""Builds either all concrete standard configs or an
+		     explicitly specified set of configs and packages them ready
+		     to run.""",
+		epilog="""Custom config store(s) can be defined at at
+		     <SHRINKWRAP_CONFIG> as a colon-separated list of
+		     directories. Building is done at <SHRINKWRAP_BUILD> and
+		     output is saved to <SHRINKWRAP_PACKAGE>. The package
+		     includes all FW binaries, a manifest and a build.sh script
+		     containing all the commands that were executed per config.
+		     Any pre-existing config package directory is first deleted.
+		     Shrinkwrap will always search its default config store even
+		     if <SHRINKWRAP_CONFIG> is not defined. <SHRINKWRAP_BUILD>
+		     and <SHRINKWRAP_PACKAGE> default to '~/.shrinkwrap/build'
+		     and '~/.shrinkwrap/package'. The user can override them by
+		     setting the environment variables.""")
+
+	cmdp.add_argument('configs',
+		metavar='yamlfile',
+		help="""A yaml file containing all the configs to be built. The
+		     top level dictionary contains a 'configs' key, whose value
+		     is a list of dictionaries, each with a 'config' key, whose
+		     value is a config filename.""")
+
+	add_common_args(cmdp)
+
+	return cmd_name
+
+
+def add_common_args(cmdp):
+	"""
+	Common args shared between build and buildmulti.
+	"""
+	cmdp.add_argument('-o', '--overlay',
+		metavar='cfgfile', required=False, default=[],
+		action='append',
+		help="""Optional config file overlay to override run-time and
+		     build-time settings. Only entries within the "build",
+		     "buildex" and "run" sections are used. Applied to all
+		     configs being built. Can be specified multiple times;
+		     left-most overlay is the first overlay applied.""")
+
+	cmdp.add_argument('-t', '--tasks',
+		required=False, default=dflt_jobs(), metavar='count', type=int,
+		help="""Maximum number of "high-level" tasks that will be
+		     performed in parallel by Shrinkwrap. Tasks include syncing
+		     git repositories, building components and copying
+		     artifacts. Default={}""".format(dflt_jobs()))
+
+	cmdp.add_argument('-j', '--jobs',
+		required=False, default=dflt_jobs(), metavar='count', type=int,
+		help="""Maximum number of low-level jobs that will be
+		     performed in parallel by each component build task.
+		     Default={}""".format(dflt_jobs()))
+
+	cmdp.add_argument('-v', '--verbose',
+		required=False, default=False, action='store_true',
+		help="""If specified, the output from all executed commands will
+		     be displayed. It is advisable to set tasks to 1 when
+		     this option is selected.""")
+
+	cmdp.add_argument('-n', '--dry-run',
+		required=False, default=False, action='store_true',
+		help="""If specified, <SHRINKWRAP_BUILD> and
+		     <SHRINKWRAP_PACKAGE> will not be touched and none of the
+		     build commands will be executed. Instead the set of
+		     commands that would have been executed are output to stdout
+		     as a bash script.""")
+
+	cmdp.add_argument('-c', '--no-color',
+		required=False, default=False, action='store_true',
+		help="""If specified, logs will not be colorized.""")
+
+
+def dispatch(args):
+	"""
+	Part of the command interface expected by shrinkwrap.py. Called to
+	execute the subcommand, with the arguments the user passed on the
+	command line. The arguments comply with those requested in add_parser().
+	"""
+	with open(args.configs) as file:
+		cfgs = yaml.safe_load(file)
+
+	configs = [c['config'] for c in cfgs['configs']]
+	btvarss = [c['btvars'] for c in cfgs['configs']]
+	build(configs, btvarss, [], [], args)
+
+
+def build(configs, btvarss, nosync, force_sync, args):
+	"""
+	Concurrently builds a list of configs. Intended to be called as a common
+	handler for the build and buildmulti commands.
+	"""
+	clivars = {'jobs': args.jobs}
+	configs = config.load_resolveb_all(configs, args.overlay, clivars, btvarss)
+	graph = config.build_graph(configs, args.verbose, nosync, force_sync)
+
+	if args.dry_run:
+		script = ugraph.make_script(graph)
+		print(script)
+	else:
+		if args.verbose:
+			workspace.dump()
+
+		# Run under a runtime environment, which may just run commands
+		# natively on the host or may execute commands in a container,
+		# depending on what the user specified.
+		with runtime.Runtime(name=args.runtime, image=config.get_image(configs, args),
+		       			ssh_agent_keys=args.ssh_agent_keys) as rt:
+			def add_volume(path, levels_up=0):
+				while levels_up:
+					path = os.path.dirname(path)
+					levels_up -= 1
+				os.makedirs(path, exist_ok=True)
+				rt.add_volume(path)
+
+			add_volume(workspace.build)
+			add_volume(workspace.package)
+			for c in workspace.configs():
+				add_volume(c)
+
+			for conf in configs:
+				for comp in conf['build'].values():
+					add_volume(comp['sourcedir'], 1)
+					add_volume(comp['builddir'])
+
+				for btvar in conf['buildex']['btvars'].values():
+					if btvar['type'] == 'path':
+						rt.add_volume(btvar['value'])
+
+			if workspace.project_cache:
+				add_volume(workspace.project_cache)
+
+			rt.start()
+
+			ugraph.execute(graph,
+				       args.tasks,
+				       args.verbose,
+				       not args.no_color)
+
+		for c in configs:
+			# Dump the config.
+			cfg_name = os.path.join(workspace.package,
+						f'{c["name"]}.yaml')
+			with open(cfg_name, 'w') as cfg:
+				config.dump(c, cfg)
+
+			# Dump the script to build the config.
+			graph = config.build_graph([c], args.verbose, nosync, force_sync)
+			script = ugraph.make_script(graph)
+			build_name = os.path.join(workspace.package,
+						  c['name'],
+						  'build.sh')
+			with open(build_name, 'w') as build:
+				build.write(script)

shrinkwrap/commands/clean.py

@@ -0,0 +1,161 @@
+# Copyright (c) 2022, Arm Limited.
+# SPDX-License-Identifier: MIT
+
+import os
+import shrinkwrap.utils.config as config
+import shrinkwrap.utils.graph as ugraph
+import shrinkwrap.utils.runtime as runtime
+import shrinkwrap.utils.workspace as workspace
+
+
+cmd_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+def dflt_jobs():
+	return min(os.cpu_count() // 2, 32)
+
+
+def add_parser(parser, formatter):
+	"""
+	Part of the command interface expected by shrinkwrap.py. Adds the
+	subcommand to the parser, along with all options and documentation.
+	Returns the subcommand name.
+	"""
+	cmdp = parser.add_parser(cmd_name,
+		formatter_class=formatter,
+		help="""Cleans either all concrete standard configs or an
+		     explicitly specified set of configs, ready to be rebuilt
+		     from scratch.""")
+
+	cmdp.add_argument('configs',
+		metavar='config', nargs='*',
+		help="""0 or more configs to clean. If a config exists relative
+		     to the current directory that config is used. Else if a
+		     config exists relative to the config store then it is used.
+		     If no configs are provided, all concrete configs in the
+		     config store are cleaned.""")
+
+	cmdp.add_argument('-o', '--overlay',
+		metavar='cfgfile', required=False, default=[],
+		action='append',
+		help="""Optional config file overlay to override run-time and
+		     build-time settings. Only entries within the "build",
+		     "buildex" and "run" sections are used. Applied to all
+		     configs being built. Can be specified multiple times;
+		     left-most overlay is the first overlay applied.""")
+
+	cmdp.add_argument('-t', '--tasks',
+		required=False, default=dflt_jobs(), metavar='count', type=int,
+		help="""Maximum number of "high-level" tasks that will be
+		     performed in parallel by Shrinkwrap. Tasks include cleaning
+		     components. Default={}""".format(dflt_jobs()))
+
+	cmdp.add_argument('-j', '--jobs',
+		required=False, default=dflt_jobs(), metavar='count', type=int,
+		help="""Maximum number of low-level jobs that will be
+		     performed in parallel by each component clean task.
+		     Default={}""".format(dflt_jobs()))
+
+	cmdp.add_argument('-v', '--verbose',
+		required=False, default=False, action='store_true',
+		help="""If specified, the output from all executed commands will
+		     be displayed. It is advisable to set tasks to 1 when
+		     this option is selected.""")
+
+	cmdp.add_argument('-n', '--dry-run',
+		required=False, default=False, action='store_true',
+		help="""If specified, none of the clean commands will be
+		     executed. Instead the set of commands that would have been
+		     executed are output to stdout as a bash script.""")
+
+	cmdp.add_argument('-c', '--no-color',
+		required=False, default=False, action='store_true',
+		help="""If specified, logs will not be colorized.""")
+
+	cmdp.add_argument('-f', '--filter',
+		metavar='[config.]component', required=False, default=[],
+		action='append',
+		help="""Optionally specify the components to clean. Option can
+		     be specified multiple times to clean multiple components.
+		     If not specified, all components are cleaned. If 'config.'
+		     is omitted, component is cleaned for all configs being
+		     cleaned (component must exist in all configs being
+		     cleaned).""")
+
+	return cmd_name
+
+
+def dispatch(args):
+	"""
+	Part of the command interface expected by shrinkwrap.py. Called to
+	execute the subcommand, with the arguments the user passed on the
+	command line. The arguments comply with those requested in add_parser().
+	"""
+	clivars = {'jobs': args.jobs}
+	configs = config.load_resolveb_all(args.configs, args.overlay, clivars)
+	if len(args.configs) == 0:
+		configs = [c for c in configs if c['concrete']]
+	for conf in configs:
+		conf['graph'] = _filter(conf['name'],
+					conf['graph'],
+					args.filter)
+
+	graph = config.clean_graph(configs, args.verbose)
+
+	if args.dry_run:
+		script = ugraph.make_script(graph)
+		print(script)
+	else:
+		if args.verbose:
+			workspace.dump()
+
+		# Run under a runtime environment, which may just run commands
+		# natively on the host or may execute commands in a container,
+		# depending on what the user specified.
+		with runtime.Runtime(name=args.runtime, image=config.get_image(configs, args),
+		       			ssh_agent_keys=args.ssh_agent_keys) as rt:
+			def add_volume(path, levels_up=0):
+				while levels_up:
+					path = os.path.dirname(path)
+					levels_up -= 1
+				os.makedirs(path, exist_ok=True)
+				rt.add_volume(path)
+
+			add_volume(workspace.build)
+			add_volume(workspace.package)
+
+			for conf in configs:
+				for comp in conf['build'].values():
+					add_volume(comp['sourcedir'], 1)
+					add_volume(comp['builddir'], 1)
+
+			rt.start()
+
+			ugraph.execute(graph,
+				       args.tasks,
+				       args.verbose,
+				       not args.no_color)
+
+
+def _filter(name, graph, args):
+	components = []
+	pruned = {}
+
+	for arg in args:
+		try:
+			conf, comp = arg.split('.', maxsplit=1)
+			if conf == name:
+				components.append(comp)
+		except ValueError:
+			components.append(arg)
+
+	if len(components) == 0:
+		return graph
+
+	for c in components:
+		if c not in graph:
+			raise Exception(f'Bad filter: {c} not a '
+					f'component of {name}')
+		pruned[c] = []
+
+	return pruned

shrinkwrap/commands/inspect.py

@@ -0,0 +1,235 @@
+# Copyright (c) 2022, Arm Limited.
+# SPDX-License-Identifier: MIT
+
+import json
+import io
+import os
+import re
+import textwrap
+import shrinkwrap.utils.config as config
+
+
+cmd_name = os.path.splitext(os.path.basename(__file__))[0]
+
+
+def add_parser(parser, formatter):
+	"""
+	Part of the command interface expected by shrinkwrap.py. Adds the
+	subcommand to the parser, along with all options and documentation.
+	Returns the subcommand name.
+	"""
+	cmdp = parser.add_parser(cmd_name,
+		formatter_class=formatter,
+		help="""Outputs to stdout info about either all concrete
+		     standard configs or an explicitly specified set of configs.
+		     Info includes name, description and runtime variables with
+		     their default values.""",
+		epilog="""Custom config store(s) can be defined at at
+		     <SHRINKWRAP_CONFIG> as a colon-separated list of
+		     directories. Shrinkwrap will always search its default
+		     config store even if <SHRINKWRAP_CONFIG> is not
+		     defined.""")
+
+	cmdp.add_argument('configs',
+		metavar='config', nargs='*',
+		help="""0 or more configs to inspect. If a config exists
+		     relative to the current directory that config is used. Else
+		     if a config exists relative to the config store then it is
+		     used. If no configs are provided, all concrete configs
+		     in the config store are built.""")
+
+	cmdp.add_argument('-a', '--all',
+		required=False, default=False, action='store_true',
+		help="""If specified, and no configs were explicitly provided,
+		     lists all standard configs rather than just the concrete
+		     ones.""")
+
+	cmdp.add_argument('-j', '--json',
+		required=False, default=False, action='store_true',
+		help="""If specified, output is in json.""")
+
+	return cmd_name
+
+
+def dispatch(args):
+	"""
+	Part of the command interface expected by shrinkwrap.py. Called to
+	execute the subcommand, with the arguments the user passed on the
+	command line. The arguments comply with those requested in add_parser().
+	"""
+	configs = config.load_all(args.configs)
+
+	cfgs = []
+	for c in sorted(configs, key=lambda c: c['fullname']):
+		if len(args.configs) == 0 and not args.all and not c['concrete']:
+			continue
+
+		cfgs.append({
+			'name': c['fullname'],
+			'description': c['description'],
+			'image': c['image'] if c['image'] is not None else '<none>',
+			'concrete': c['concrete'],
+			'btvars': {
+				k: _var_value(v['value'])
+				for k,v in c['buildex']['btvars'].items()
+			},
+			'rtvars': {
+				k: _var_value(v['value'])
+				for k,v in c['run']['rtvars'].items()
+			},
+			'components': _comp_revisions(c['build']),
+		})
+
+	if args.json:
+		print(json.dumps(cfgs, indent=4))
+		return
+
+	width = 80
+	indent = 21
+	vindent = 24
+
+	descs = []
+	for c in cfgs:
+		buf = io.StringIO()
+
+		buf.write(_text_wrap('name',
+				     c['name'],
+				     width=width,
+				     indent=indent,
+				     paraspace=1))
+		buf.write('\n')
+		buf.write(_text_wrap('description',
+				     c['description'],
+				     width=width,
+				     indent=indent,
+				     paraspace=1))
+		buf.write('\n')
+		buf.write(_text_wrap('image',
+				     c['image'],
+				     width=width,
+				     indent=indent,
+				     paraspace=1))
+		buf.write('\n')
+		buf.write(_text_wrap('concrete',
+				     c['concrete'],
+				     width=width,
+				     indent=indent,
+				     paraspace=1))
+		buf.write('\n')
+		buf.write(_dict_wrap('build-time vars',
+				     c['btvars'],
+				     width=width,
+				     kindent=indent,
+				     vindent=vindent))
+		buf.write('\n')
+		buf.write(_dict_wrap('run-time vars',
+				     c['rtvars'],
+				     width=width,
+				     kindent=indent,
+				     vindent=vindent))
+		buf.write('\n')
+		buf.write(_repo_wrap('components',
+				     c['components'],
+				     width=width,
+				     kindent=indent,
+				     vindent=vindent))
+
+		descs.append(buf.getvalue())
+
+	separator = '\n' + ('-' * width) + '\n\n'
+	all = separator.join(descs)
+	print(all)
+
+
+def _var_value(value):
+	if value is None:
+		return '<null>'
+	if value == '':
+		return '<empty>'
+	return str(value)
+
+
+def _comp_revisions(components):
+	revs = {}
+	for comp in sorted(components.keys()):
+		for repo in sorted(components[comp]['repo'].keys()):
+			name = comp if repo == '.' else f"{comp} ({repo})"
+			revision = components[comp]['repo'][repo]['revision']
+			remote = components[comp]['repo'][repo]['remote']
+			revs[name] = {
+				'repository': remote,
+				'revision': revision,
+			}
+	return revs
+
+
+def _text_wrap(tag, text, width=80, indent=0, paraspace=1, end='\n'):
+	text = str(text)
+	tag = str(tag)
+	indent_pattern = ' ' * indent
+
+	lines = [textwrap.fill(l,
+			       width=width,
+			       initial_indent=indent_pattern,
+			       subsequent_indent=indent_pattern)
+		for l in text.splitlines() if l]
+
+	wrapped = ('\n' * (paraspace + 1)).join(lines)
+
+	if tag:
+		if len(tag) > indent - 2:
+			wrapped = f'{tag}:\n' + wrapped
+		else:
+			wrapped = f'{tag}:' + wrapped[len(tag) + 1:]
+
+	return wrapped + end
+
+
+def _dict_wrap(tag, dictionary, width=80, kindent=0, vindent=0, end='\n'):
+	if len(dictionary) == 0:
+		lines = [str(None)]
+	else:
+		dwidth = width - kindent
+		lines = []
+
+		for k, v in dictionary.items():
+			line = _text_wrap(k,
+					  v,
+					  width=dwidth,
+					  indent=vindent,
+					  paraspace=0,
+					  end='')
+			lines.append(line)
+
+	dtext = '\n'.join(lines)
+
+	return _text_wrap(tag,
+			  dtext,
+			  width=width,
+			  indent=kindent,
+			  paraspace=0,
+			  end=end)
+
+
+def _repo_wrap(tag, components, width=80, kindent=0, vindent=0, end='\n'):
+	def is_git_sha(s):
+		return bool(re.fullmatch(r"[0-9a-f]{40}", s))
+
+	repo_indent = 0
+	for info in components.values():
+		rev = info['revision']
+		if is_git_sha(rev):
+			rev = rev[:12]
+		repo_indent = max(repo_indent, len(rev))
+	repo_indent += 2
+
+	dictionary = {}
+	for comp, info in components.items():
+		repo = info['repository']
+		rev = info['revision']
+		if is_git_sha(rev):
+			rev = rev[:12]
+		value = f"{rev}{' ' * (repo_indent - len(rev))}{repo}"
+		dictionary[comp] = value
+
+	return _dict_wrap(tag, dictionary, 10000, kindent, vindent, end)