taskcluster-taskgraph 24.4.0__tar.gz → 24.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/CHANGELOG.md +33 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/PKG-INFO +1 -1
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/use-fetches.rst +13 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.util.rst +8 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/pyproject.toml +2 -1
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/generator.py +236 -23
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/optimize/base.py +14 -7
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/optimize/strategies.py +10 -2
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/fetch-content +252 -18
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/run-task +3 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/taskgraph.py +10 -8
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/cached_tasks.py +12 -4
- taskcluster_taskgraph-24.5.0/src/taskgraph/util/memory.py +24 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/config.yml +1 -1
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/Dockerfile +1 -1
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/yarn.lock +9 -9
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/check/kind.yml +12 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/docker-image/kind.yml +1 -1
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_generator.py +320 -10
- taskcluster_taskgraph-24.5.0/test/test_scripts_fetch_content.py +963 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_scripts_run_task.py +93 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_cached_tasks.py +15 -0
- taskcluster_taskgraph-24.5.0/test/test_util_memory.py +38 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/uv.lock +1323 -1010
- taskcluster_taskgraph-24.4.0/test/test_scripts_fetch_content.py +0 -381
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.codespell-ignore-words.txt +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.github/CODEOWNERS +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.github/workflows/codeql-analysis.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.github/workflows/codspeed.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.github/workflows/pypi-publish.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.gitignore +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.pre-commit-config.yaml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.readthedocs.yaml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.taskcluster.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.yamllint +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/CODE_OF_CONDUCT.md +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/CONTRIBUTING.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/LICENSE +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/Makefile +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/README.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/index.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/kind.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/loading.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/optimization.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/overview.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/scopes.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/task-graphs.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/transforms.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/conf.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/contributing.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/glossary.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/bootstrap-taskgraph.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/create-actions.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/create-tasks.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/debugging.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/define-schemas.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/docker.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/index.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/load-task-locally.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/resolve-keyed-by.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/run-locally.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/send-notifications.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/index.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/cli.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/index.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/migrations.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/optimization-strategies.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/parameters.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/modules.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.actions.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.loader.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.optimize.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.transforms.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.transforms.run.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/chunking.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/from_deps.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/index.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/matrix.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/task_context.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/tutorials/connecting-taskcluster.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/tutorials/creating-a-task-graph.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/tutorials/example-taskcluster.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/tutorials/getting-started.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/tutorials/index.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/make.bat +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/README.md +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/pyproject.toml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/src/pytest_taskgraph/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/vcs.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/README.md +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/pyproject.toml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/src/sphinx_taskgraph/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/src/sphinx_taskgraph/autoschema.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/renovate.json +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/add_new_jobs.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/cancel.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/cancel_all.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/rebuild_cached_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/registry.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/retrigger.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/util.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/config.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/create.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/decision.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/docker.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/filter_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/graph.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/loader/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/loader/default.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/loader/transform.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/main.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/morph.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/optimize/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/parameters.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/hgrc +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/robustcheckout.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/target_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/task.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/base.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/cached_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/chunking.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/code_review.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/docker_image.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/fetch.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/from_deps.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/matrix.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/notify.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/common.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/index_search.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/run_task.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/toolchain.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/task.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/task_context.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/archive.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/attributes.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/caches.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/copy.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/dependencies.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/docker.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/hash.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/json.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/keyed_by.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/parameterization.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/path.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/python_path.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/readonlydict.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/schema.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/set_name.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/shell.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/task_context.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/taskcluster.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/taskgraph.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/templates.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/time.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/treeherder.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/vcs.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/verify.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/workertypes.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/yaml.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/REGISTRY +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/HASH +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/README.md +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/VERSION +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/system-setup.sh +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/fetch/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/image_builder/README.rst +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/README +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/insert-indexes.js +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/package.json +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/python/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/run-task/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/run-task/system-setup.sh +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/skopeo/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/skopeo/policy.json +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/skopeo/push_image.sh +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/codecov/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/complete/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/doc/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/fetch/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/push-image/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/test/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/test/linux.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/test/windows.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/scripts/codecov-upload.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/custom_parameters.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/custom_target_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/transforms/add_pr_route.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/transforms/push_image.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-pull-request-untrusted.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-pull-request.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-push.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-release-pytest-taskgraph.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-release-taskcluster-taskgraph.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/cookiecutter.json +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/hooks/post_gen_project.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/config.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/docker/linux/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/kinds/docker-image/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/kinds/hello/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/{{cookiecutter.project_slug}}_taskgraph/transforms/hello.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster.github.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/automationrelevance.json +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/conftest.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/task_context.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/config.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world-tag/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world-tag/REGISTRY +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world-tag/VERSION +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/kinds/docker-image/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/scripts/toolchain/run.ps1 +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/scripts/toolchain/run.sh +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/test_taskgraph/transforms/foo.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/testmod/thing.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/mockedopen.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_actions_rebuild_cached_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_actions_registry.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_config.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_create.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_decision.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_docker.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_graph.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_graph_perf.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_main.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_morph.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_optimize.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_optimize_strategies.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_parameters.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_target_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_taskgraph.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transform_chunking.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transform_docker_image.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transform_task_context.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_base.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_cached_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_fetch.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_from_deps.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_matrix.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_notify.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_run.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_run_run_task.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_run_toolchain.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_task.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_archive.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_attributes.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_copy.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_dependencies.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_docker.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_json.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_keyed_by.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_parameterization.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_path.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_python_path.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_readonlydict.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_schema.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_taskcluster.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_templates.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_time.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_treeherder.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_vcs.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_verify.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_workertypes.py +0 -0
- {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_yaml.py +0 -0
|
@@ -1,5 +1,37 @@
|
|
|
1
1
|
# Change Log
|
|
2
2
|
|
|
3
|
+
## [24.5.0] - 2026-10-21
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
- `taskgraph.generator.phase_finished` function, which can be replaced by
|
|
8
|
+
consumers and is called as each generation phase ends (#1073)
|
|
9
|
+
- Hook point in `run-task` at startup (#1077)
|
|
10
|
+
|
|
11
|
+
### Changed
|
|
12
|
+
|
|
13
|
+
- perf: on Linux, graph generation now uses a process per kind instead of a
|
|
14
|
+
process pool, to cut down on IPC overhead
|
|
15
|
+
(https://bugzilla.mozilla.org/show_bug.cgi?id=2079635)
|
|
16
|
+
- perf: pause the garbage collector during graph generation
|
|
17
|
+
(https://bugzilla.mozilla.org/show_bug.cgi?id=2079640)
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
|
|
21
|
+
- On Windows, fetch-content now reads tarballs until EOF, to be able to check
|
|
22
|
+
the stream's integrity (#1074)
|
|
23
|
+
|
|
24
|
+
## [24.4.1] - 2026-10-08
|
|
25
|
+
|
|
26
|
+
### Fixed
|
|
27
|
+
|
|
28
|
+
- `github-issue-comment` tasks now use the PR cache namespaces
|
|
29
|
+
- perf: look up kind dependency tasks by kind in the generator (#1054)
|
|
30
|
+
- perf: extract tar archives while they download in `fetch-content` (#1068)
|
|
31
|
+
- perf: let tar decompress zstd archives in `fetch-content` (#1068)
|
|
32
|
+
- perf: read downloads in 1MiB chunks in `fetch-content` (#1068)
|
|
33
|
+
- perf: resolve each dependent's deadline only once in `optimize` (#1056)
|
|
34
|
+
|
|
3
35
|
## [24.4.0] - 2026-10-05
|
|
4
36
|
|
|
5
37
|
### Added
|
|
@@ -11,6 +43,7 @@
|
|
|
11
43
|
### Changed
|
|
12
44
|
|
|
13
45
|
- `taskgraph init` template and tutorial docs now default to the `public_restricted` PR policy
|
|
46
|
+
- BREAKING CHANGE: `run-task` requires Python 3.8
|
|
14
47
|
|
|
15
48
|
### Fixed
|
|
16
49
|
|
|
@@ -105,3 +105,16 @@ There are a few differences from the earlier ``build`` examples here:
|
|
|
105
105
|
|
|
106
106
|
It is not possible to configure the ``dest`` or ``extract`` values when using
|
|
107
107
|
``fetch`` or ``toolchain`` kinds.
|
|
108
|
+
|
|
109
|
+
Tuning Download Performance
|
|
110
|
+
---------------------------
|
|
111
|
+
|
|
112
|
+
The following environment variable tunes how ``fetch-content`` downloads
|
|
113
|
+
artifacts.
|
|
114
|
+
|
|
115
|
+
``TASKGRAPH_FETCH_STREAM``
|
|
116
|
+
Tar archives that are extracted are extracted while they download, rather
|
|
117
|
+
than written to disk and read back, into a staging directory that is only
|
|
118
|
+
merged into place once the download is complete and verified. This is on
|
|
119
|
+
by default; set to ``0`` to download to a file first. Zip archives, and
|
|
120
|
+
fetches that aren't extracted, always go through a file.
|
|
@@ -60,6 +60,14 @@ taskgraph.util.keyed\_by module
|
|
|
60
60
|
:undoc-members:
|
|
61
61
|
:show-inheritance:
|
|
62
62
|
|
|
63
|
+
taskgraph.util.memory module
|
|
64
|
+
----------------------------
|
|
65
|
+
|
|
66
|
+
.. automodule:: taskgraph.util.memory
|
|
67
|
+
:members:
|
|
68
|
+
:undoc-members:
|
|
69
|
+
:show-inheritance:
|
|
70
|
+
|
|
63
71
|
taskgraph.util.parameterization module
|
|
64
72
|
--------------------------------------
|
|
65
73
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
### Project
|
|
2
2
|
[project]
|
|
3
3
|
name = "taskcluster-taskgraph"
|
|
4
|
-
version = "24.
|
|
4
|
+
version = "24.5.0"
|
|
5
5
|
description = "Build taskcluster taskgraphs"
|
|
6
6
|
readme = "README.rst"
|
|
7
7
|
authors = [
|
|
@@ -52,6 +52,7 @@ dev = [
|
|
|
52
52
|
"sphinx-book-theme >=1.1.4",
|
|
53
53
|
"sphinx-taskgraph>=0.1.0",
|
|
54
54
|
"sphinxcontrib-mermaid>=1.0.0",
|
|
55
|
+
"vermin>=1.8.0",
|
|
55
56
|
"zstandard>=0.25.0",
|
|
56
57
|
]
|
|
57
58
|
|
|
@@ -6,16 +6,21 @@ import copy
|
|
|
6
6
|
import inspect
|
|
7
7
|
import logging
|
|
8
8
|
import multiprocessing
|
|
9
|
+
import multiprocessing.connection
|
|
9
10
|
import os
|
|
11
|
+
import pickle
|
|
10
12
|
import platform
|
|
13
|
+
import signal
|
|
14
|
+
import sys
|
|
15
|
+
import time
|
|
16
|
+
import traceback
|
|
11
17
|
from concurrent.futures import (
|
|
12
18
|
FIRST_COMPLETED,
|
|
13
|
-
ProcessPoolExecutor,
|
|
14
19
|
ThreadPoolExecutor,
|
|
15
20
|
wait,
|
|
16
21
|
)
|
|
17
22
|
from dataclasses import dataclass
|
|
18
|
-
from typing import Callable, Optional, Union
|
|
23
|
+
from typing import Callable, Optional, Union, cast
|
|
19
24
|
|
|
20
25
|
from . import filter_tasks
|
|
21
26
|
from .config import GraphConfig, load_graph_config
|
|
@@ -26,6 +31,7 @@ from .parameters import Parameters, parameters_loader
|
|
|
26
31
|
from .task import Task
|
|
27
32
|
from .taskgraph import TaskGraph
|
|
28
33
|
from .transforms.base import TransformConfig, TransformSequence
|
|
34
|
+
from .util.memory import gc_disabled
|
|
29
35
|
from .util.python_path import find_object
|
|
30
36
|
from .util.schema import SchemaValidationError
|
|
31
37
|
from .util.verify import verifications
|
|
@@ -34,12 +40,31 @@ from .util.yaml import load_yaml
|
|
|
34
40
|
logger = logging.getLogger(__name__)
|
|
35
41
|
|
|
36
42
|
|
|
43
|
+
def phase_finished(name, start, end):
|
|
44
|
+
"""Called after each generation phase, with `time.monotonic()` values
|
|
45
|
+
taken around it. `name` is None if the phase raised an exception.
|
|
46
|
+
|
|
47
|
+
Projects can replace this from their `register` function; a replacement
|
|
48
|
+
made then also applies to the phase that is running.
|
|
49
|
+
"""
|
|
50
|
+
if name is None:
|
|
51
|
+
logger.debug(f"Generation phase failed after {end - start:.2f}s")
|
|
52
|
+
else:
|
|
53
|
+
logger.debug(f"Generated {name} in {end - start:.2f}s")
|
|
54
|
+
|
|
55
|
+
|
|
37
56
|
class KindNotFound(Exception):
|
|
38
57
|
"""
|
|
39
58
|
Raised when trying to load kind from a directory without a kind.yml.
|
|
40
59
|
"""
|
|
41
60
|
|
|
42
61
|
|
|
62
|
+
def _max_workers():
|
|
63
|
+
# os.process_cpu_count is new in Python 3.13.
|
|
64
|
+
cpu_count = getattr(os, "process_cpu_count", os.cpu_count)
|
|
65
|
+
return cpu_count() or 1
|
|
66
|
+
|
|
67
|
+
|
|
43
68
|
@dataclass(frozen=True)
|
|
44
69
|
class Kind:
|
|
45
70
|
name: str
|
|
@@ -124,6 +149,16 @@ class Kind:
|
|
|
124
149
|
return cls(kind_name, path, config, graph_config)
|
|
125
150
|
|
|
126
151
|
|
|
152
|
+
class _RemoteTraceback(Exception):
|
|
153
|
+
"""The traceback of an exception raised in a child process."""
|
|
154
|
+
|
|
155
|
+
def __init__(self, tb):
|
|
156
|
+
self.tb = tb
|
|
157
|
+
|
|
158
|
+
def __str__(self):
|
|
159
|
+
return f'\n"""\n{self.tb}"""'
|
|
160
|
+
|
|
161
|
+
|
|
127
162
|
class TaskGraphGenerator:
|
|
128
163
|
"""
|
|
129
164
|
The central controller for taskgraph. This handles all phases of graph
|
|
@@ -282,8 +317,21 @@ class TaskGraphGenerator:
|
|
|
282
317
|
except KindNotFound:
|
|
283
318
|
continue
|
|
284
319
|
|
|
320
|
+
@staticmethod
|
|
321
|
+
def _get_kind_dependencies_tasks(kind, tasks_by_kind):
|
|
322
|
+
"""Return the tasks of the kinds `kind` depends on, in the order in
|
|
323
|
+
which they were loaded."""
|
|
324
|
+
kind_dependencies = kind.config.get("kind-dependencies", [])
|
|
325
|
+
return {
|
|
326
|
+
label: task
|
|
327
|
+
for kind_name, tasks in tasks_by_kind.items()
|
|
328
|
+
if kind_name in kind_dependencies
|
|
329
|
+
for label, task in tasks.items()
|
|
330
|
+
}
|
|
331
|
+
|
|
285
332
|
def _load_tasks_serial(self, kinds, kind_graph, parameters):
|
|
286
333
|
all_tasks = {}
|
|
334
|
+
tasks_by_kind = {}
|
|
287
335
|
for kind_name in kind_graph.visit_postorder():
|
|
288
336
|
logger.debug(f"Loading tasks for kind {kind_name}")
|
|
289
337
|
|
|
@@ -297,11 +345,7 @@ class TaskGraphGenerator:
|
|
|
297
345
|
try:
|
|
298
346
|
new_tasks = kind.load_tasks(
|
|
299
347
|
parameters,
|
|
300
|
-
|
|
301
|
-
k: t
|
|
302
|
-
for k, t in all_tasks.items()
|
|
303
|
-
if t.kind in kind.config.get("kind-dependencies", [])
|
|
304
|
-
},
|
|
348
|
+
self._get_kind_dependencies_tasks(kind, tasks_by_kind),
|
|
305
349
|
self._write_artifacts,
|
|
306
350
|
)
|
|
307
351
|
except SchemaValidationError as exc:
|
|
@@ -310,15 +354,18 @@ class TaskGraphGenerator:
|
|
|
310
354
|
except Exception:
|
|
311
355
|
logger.exception(f"Error loading tasks for kind {kind_name}:")
|
|
312
356
|
raise
|
|
357
|
+
kind_tasks = tasks_by_kind.setdefault(kind_name, {})
|
|
313
358
|
for task in new_tasks:
|
|
314
359
|
if task.label in all_tasks:
|
|
315
360
|
raise Exception("duplicate tasks with label " + task.label)
|
|
316
361
|
all_tasks[task.label] = task
|
|
362
|
+
kind_tasks[task.label] = task
|
|
317
363
|
|
|
318
364
|
return all_tasks
|
|
319
365
|
|
|
320
366
|
def _load_tasks_parallel(self, kinds, kind_graph, parameters, executor):
|
|
321
367
|
all_tasks = {}
|
|
368
|
+
tasks_by_kind = {}
|
|
322
369
|
futures_to_kind = {}
|
|
323
370
|
futures = set()
|
|
324
371
|
edges = set(kind_graph.edges)
|
|
@@ -328,7 +375,6 @@ class TaskGraphGenerator:
|
|
|
328
375
|
def submit_ready_kinds():
|
|
329
376
|
"""Create the next batch of tasks for kinds without dependencies."""
|
|
330
377
|
nonlocal kinds, edges, futures
|
|
331
|
-
loaded_tasks = all_tasks.copy()
|
|
332
378
|
kinds_with_deps = {edge[0] for edge in edges}
|
|
333
379
|
ready_kinds = (
|
|
334
380
|
set(kinds) - kinds_with_deps - set(futures_to_kind.values())
|
|
@@ -346,11 +392,7 @@ class TaskGraphGenerator:
|
|
|
346
392
|
future = executor.submit(
|
|
347
393
|
kind.load_tasks,
|
|
348
394
|
dict(parameters),
|
|
349
|
-
|
|
350
|
-
k: t
|
|
351
|
-
for k, t in loaded_tasks.items()
|
|
352
|
-
if t.kind in kind.config.get("kind-dependencies", [])
|
|
353
|
-
},
|
|
395
|
+
self._get_kind_dependencies_tasks(kind, tasks_by_kind),
|
|
354
396
|
self._write_artifacts,
|
|
355
397
|
)
|
|
356
398
|
futures.add(future)
|
|
@@ -376,10 +418,12 @@ class TaskGraphGenerator:
|
|
|
376
418
|
kind = futures_to_kind.pop(future)
|
|
377
419
|
futures.remove(future)
|
|
378
420
|
|
|
421
|
+
kind_tasks = tasks_by_kind.setdefault(kind, {})
|
|
379
422
|
for task in future.result():
|
|
380
423
|
if task.label in all_tasks:
|
|
381
424
|
raise Exception("duplicate tasks with label " + task.label)
|
|
382
425
|
all_tasks[task.label] = task
|
|
426
|
+
kind_tasks[task.label] = task
|
|
383
427
|
|
|
384
428
|
# Update state for next batch of futures.
|
|
385
429
|
del kinds[kind]
|
|
@@ -390,6 +434,157 @@ class TaskGraphGenerator:
|
|
|
390
434
|
|
|
391
435
|
return all_tasks
|
|
392
436
|
|
|
437
|
+
def _load_tasks_forked(self, kinds, kind_graph, parameters):
|
|
438
|
+
"""Load each kind in a child process forked once the kinds it depends
|
|
439
|
+
on are loaded.
|
|
440
|
+
|
|
441
|
+
The child shares every task loaded so far with the parent, so the
|
|
442
|
+
tasks of the kinds it depends on don't need to be sent to it, and it
|
|
443
|
+
sends its own tasks back through a pipe of its own.
|
|
444
|
+
"""
|
|
445
|
+
all_tasks = {}
|
|
446
|
+
tasks_by_kind = {}
|
|
447
|
+
# Pipe to each running child -> (kind name, pid)
|
|
448
|
+
running = {}
|
|
449
|
+
remaining = kinds.copy()
|
|
450
|
+
edges = set(kind_graph.edges)
|
|
451
|
+
max_workers = _max_workers()
|
|
452
|
+
|
|
453
|
+
try:
|
|
454
|
+
while remaining or running:
|
|
455
|
+
kinds_with_deps = {edge[0] for edge in edges}
|
|
456
|
+
ready_kinds = sorted(set(remaining) - kinds_with_deps)
|
|
457
|
+
for name in ready_kinds[: max_workers - len(running)]:
|
|
458
|
+
kind = remaining.pop(name)
|
|
459
|
+
reader, pid = self._fork_kind(
|
|
460
|
+
kind,
|
|
461
|
+
parameters,
|
|
462
|
+
self._get_kind_dependencies_tasks(kind, tasks_by_kind),
|
|
463
|
+
)
|
|
464
|
+
running[reader] = (name, pid)
|
|
465
|
+
|
|
466
|
+
# Waiting on nothing would block forever. _run made sure there
|
|
467
|
+
# is no dependency loop, so some kind is always running here.
|
|
468
|
+
assert running
|
|
469
|
+
for ready in multiprocessing.connection.wait(list(running)):
|
|
470
|
+
# wait returns the objects it was given, but is typed as
|
|
471
|
+
# also returning sockets and file descriptors.
|
|
472
|
+
reader = cast(multiprocessing.connection.Connection, ready)
|
|
473
|
+
name, pid = running[reader]
|
|
474
|
+
try:
|
|
475
|
+
result = reader.recv()
|
|
476
|
+
except EOFError:
|
|
477
|
+
# The child exited without sending anything.
|
|
478
|
+
result = None
|
|
479
|
+
del running[reader]
|
|
480
|
+
reader.close()
|
|
481
|
+
_, status = os.waitpid(pid, 0)
|
|
482
|
+
new_tasks = self._kind_result(name, result, status)
|
|
483
|
+
|
|
484
|
+
kind_tasks = tasks_by_kind.setdefault(name, {})
|
|
485
|
+
for task in new_tasks:
|
|
486
|
+
if task.label in all_tasks:
|
|
487
|
+
raise Exception("duplicate tasks with label " + task.label)
|
|
488
|
+
all_tasks[task.label] = task
|
|
489
|
+
kind_tasks[task.label] = task
|
|
490
|
+
|
|
491
|
+
edges = {e for e in edges if e[1] != name}
|
|
492
|
+
finally:
|
|
493
|
+
# Only left with running children on error.
|
|
494
|
+
for reader, (_, pid) in running.items():
|
|
495
|
+
reader.close()
|
|
496
|
+
os.kill(pid, signal.SIGKILL)
|
|
497
|
+
os.waitpid(pid, 0)
|
|
498
|
+
|
|
499
|
+
return all_tasks
|
|
500
|
+
|
|
501
|
+
def _fork_kind(self, kind, parameters, kind_dependencies_tasks):
|
|
502
|
+
"""Fork a child process loading the tasks of `kind`, and return the
|
|
503
|
+
pipe its result comes back through, and its pid."""
|
|
504
|
+
reader, writer = multiprocessing.Pipe(duplex=False)
|
|
505
|
+
# The child inherits the disabled garbage collector, so it doesn't
|
|
506
|
+
# write to every page it shares with us traversing the objects in them.
|
|
507
|
+
with gc_disabled():
|
|
508
|
+
pid = os.fork()
|
|
509
|
+
if pid == 0:
|
|
510
|
+
# Never leave this block, or return into the caller's code, in
|
|
511
|
+
# the child.
|
|
512
|
+
status = 1
|
|
513
|
+
try:
|
|
514
|
+
reader.close()
|
|
515
|
+
status = self._load_kind_in_child(
|
|
516
|
+
kind, parameters, kind_dependencies_tasks, writer
|
|
517
|
+
)
|
|
518
|
+
finally:
|
|
519
|
+
try:
|
|
520
|
+
sys.stdout.flush()
|
|
521
|
+
sys.stderr.flush()
|
|
522
|
+
finally:
|
|
523
|
+
os._exit(status)
|
|
524
|
+
writer.close()
|
|
525
|
+
return reader, pid
|
|
526
|
+
|
|
527
|
+
def _load_kind_in_child(self, kind, parameters, kind_dependencies_tasks, writer):
|
|
528
|
+
"""Load the tasks of `kind` and send them, or the exception loading
|
|
529
|
+
them raised, through `writer`. Return the child's exit status."""
|
|
530
|
+
try:
|
|
531
|
+
tasks = kind.load_tasks(
|
|
532
|
+
parameters, kind_dependencies_tasks, self._write_artifacts
|
|
533
|
+
)
|
|
534
|
+
except BaseException as e:
|
|
535
|
+
tb = traceback.format_exc()
|
|
536
|
+
try:
|
|
537
|
+
# Make sure the parent will be able to unpickle it.
|
|
538
|
+
pickle.loads(pickle.dumps(e))
|
|
539
|
+
except Exception:
|
|
540
|
+
e = Exception(f"{type(e).__name__}: {e}")
|
|
541
|
+
writer.send((e, tb, None))
|
|
542
|
+
return 1
|
|
543
|
+
|
|
544
|
+
try:
|
|
545
|
+
writer.send((None, None, tasks))
|
|
546
|
+
except Exception as e:
|
|
547
|
+
writer.send(
|
|
548
|
+
(
|
|
549
|
+
Exception(
|
|
550
|
+
f"Could not send the tasks of kind {kind.name} "
|
|
551
|
+
f"to the parent process: {type(e).__name__}: {e}"
|
|
552
|
+
),
|
|
553
|
+
traceback.format_exc(),
|
|
554
|
+
None,
|
|
555
|
+
)
|
|
556
|
+
)
|
|
557
|
+
return 1
|
|
558
|
+
return 0
|
|
559
|
+
|
|
560
|
+
@staticmethod
|
|
561
|
+
def _kind_result(name, result, status):
|
|
562
|
+
"""Return the tasks the child loading kind `name` sent, given what it
|
|
563
|
+
sent and its exit status. Raise the exception loading them raised in
|
|
564
|
+
the child, if any."""
|
|
565
|
+
if result is None:
|
|
566
|
+
code = os.waitstatus_to_exitcode(status)
|
|
567
|
+
if code < 0:
|
|
568
|
+
how = f"was killed by signal {signal.Signals(-code).name}"
|
|
569
|
+
else:
|
|
570
|
+
how = f"exited with status {code}"
|
|
571
|
+
raise Exception(
|
|
572
|
+
f"Process loading tasks for kind {name} {how} without sending them"
|
|
573
|
+
)
|
|
574
|
+
|
|
575
|
+
exc, tb, tasks = result
|
|
576
|
+
if exc is not None:
|
|
577
|
+
exc.__cause__ = _RemoteTraceback(tb)
|
|
578
|
+
if isinstance(exc, SchemaValidationError):
|
|
579
|
+
logger.error(f"Error loading tasks for kind {name}:\n{exc}")
|
|
580
|
+
else:
|
|
581
|
+
logger.error(
|
|
582
|
+
f"Error loading tasks for kind {name}:",
|
|
583
|
+
exc_info=(type(exc), exc, exc.__traceback__),
|
|
584
|
+
)
|
|
585
|
+
raise exc
|
|
586
|
+
return tasks
|
|
587
|
+
|
|
393
588
|
def _run(self):
|
|
394
589
|
logger.info("Loading graph configuration.")
|
|
395
590
|
graph_config = load_graph_config(self.root_dir)
|
|
@@ -437,6 +632,12 @@ class TaskGraphGenerator:
|
|
|
437
632
|
for kind in kinds.values():
|
|
438
633
|
for dep in kind.config.get("kind-dependencies", []):
|
|
439
634
|
edges.add((kind.name, dep, "kind-dependency"))
|
|
635
|
+
for _, dep, _ in edges:
|
|
636
|
+
if dep not in kinds:
|
|
637
|
+
message = f'Could not find the kind "{dep}"\nAvailable kinds:\n'
|
|
638
|
+
for k in sorted(kinds):
|
|
639
|
+
message += f' - "{k}"\n'
|
|
640
|
+
raise Exception(message)
|
|
440
641
|
kind_graph = Graph(frozenset(kinds), frozenset(edges))
|
|
441
642
|
|
|
442
643
|
if target_kinds:
|
|
@@ -444,6 +645,11 @@ class TaskGraphGenerator:
|
|
|
444
645
|
set(target_kinds) | {"docker-image"}
|
|
445
646
|
)
|
|
446
647
|
|
|
648
|
+
# The task loaders wait for each kind's dependencies to be loaded
|
|
649
|
+
# before loading it, so a dependency loop would hang them or silently
|
|
650
|
+
# leave kinds out. This raises if there is one.
|
|
651
|
+
kind_graph.visit_postorder()
|
|
652
|
+
|
|
447
653
|
yield "kind_graph", kind_graph
|
|
448
654
|
|
|
449
655
|
logger.info("Generating full task set")
|
|
@@ -451,7 +657,7 @@ class TaskGraphGenerator:
|
|
|
451
657
|
# The next block deals with enabling parallel kind processing, which
|
|
452
658
|
# currently has different support on different platforms. In summary:
|
|
453
659
|
# * Parallel kind processing is supported and enabled by default on
|
|
454
|
-
# Linux. We
|
|
660
|
+
# Linux. We fork a process per kind by default, but experimental
|
|
455
661
|
# support for multiple threads can be enabled instead.
|
|
456
662
|
# * On other platforms, we have experimental support for parallel
|
|
457
663
|
# kind processing with multiple threads.
|
|
@@ -471,21 +677,19 @@ class TaskGraphGenerator:
|
|
|
471
677
|
if os.environ.get("TASKGRAPH_SERIAL"):
|
|
472
678
|
return self._load_tasks_serial(kinds, kind_graph, parameters)
|
|
473
679
|
elif os.environ.get("TASKGRAPH_USE_THREADS"):
|
|
474
|
-
executor = ThreadPoolExecutor(max_workers=
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
mp_context=multiprocessing.get_context("fork")
|
|
680
|
+
executor = ThreadPoolExecutor(max_workers=_max_workers())
|
|
681
|
+
return self._load_tasks_parallel(
|
|
682
|
+
kinds, kind_graph, parameters, executor
|
|
478
683
|
)
|
|
479
|
-
|
|
480
|
-
kinds, kind_graph, parameters
|
|
481
|
-
)
|
|
684
|
+
else:
|
|
685
|
+
return self._load_tasks_forked(kinds, kind_graph, parameters)
|
|
482
686
|
else:
|
|
483
687
|
if os.environ.get("TASKGRAPH_SERIAL") or not os.environ.get(
|
|
484
688
|
"TASKGRAPH_USE_THREADS"
|
|
485
689
|
):
|
|
486
690
|
return self._load_tasks_serial(kinds, kind_graph, parameters)
|
|
487
691
|
else:
|
|
488
|
-
executor = ThreadPoolExecutor(max_workers=
|
|
692
|
+
executor = ThreadPoolExecutor(max_workers=_max_workers())
|
|
489
693
|
return self._load_tasks_parallel(
|
|
490
694
|
kinds, kind_graph, parameters, executor
|
|
491
695
|
)
|
|
@@ -594,10 +798,19 @@ class TaskGraphGenerator:
|
|
|
594
798
|
|
|
595
799
|
def _run_until(self, name):
|
|
596
800
|
while name not in self._run_results:
|
|
801
|
+
start = time.monotonic()
|
|
597
802
|
try:
|
|
598
|
-
|
|
803
|
+
# Generation creates millions of objects that stay alive, so
|
|
804
|
+
# collecting garbage meanwhile only traverses them for nothing.
|
|
805
|
+
# The callers' code runs with the collector in its usual state.
|
|
806
|
+
with gc_disabled():
|
|
807
|
+
k, v = next(self._run) # type: ignore
|
|
599
808
|
except StopIteration:
|
|
600
809
|
raise AttributeError(f"No such run result {name}")
|
|
810
|
+
except BaseException:
|
|
811
|
+
phase_finished(None, start, time.monotonic())
|
|
812
|
+
raise
|
|
813
|
+
phase_finished(k, start, time.monotonic())
|
|
601
814
|
self._run_results[k] = v
|
|
602
815
|
return self._run_results[name]
|
|
603
816
|
|
{taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/optimize/base.py
RENAMED
|
@@ -301,6 +301,18 @@ def replace_tasks(
|
|
|
301
301
|
target_task_graph.graph.links_and_reverse_links_dict()
|
|
302
302
|
)
|
|
303
303
|
|
|
304
|
+
# Many tasks share dependents (e.g. docker images or toolchains), so
|
|
305
|
+
# resolve the deadline of each dependent only once.
|
|
306
|
+
now = datetime.datetime.now(datetime.timezone.utc)
|
|
307
|
+
deadlines = {}
|
|
308
|
+
|
|
309
|
+
def get_deadline(label):
|
|
310
|
+
if label not in deadlines:
|
|
311
|
+
deadlines[label] = resolve_timestamps(
|
|
312
|
+
now, target_task_graph.tasks[label].task["deadline"]
|
|
313
|
+
)
|
|
314
|
+
return deadlines[label]
|
|
315
|
+
|
|
304
316
|
for label in target_task_graph.graph.visit_postorder():
|
|
305
317
|
logger.debug(f"replace_tasks: {label}")
|
|
306
318
|
# if we're not allowed to optimize, that's easy..
|
|
@@ -331,14 +343,9 @@ def replace_tasks(
|
|
|
331
343
|
opt_by, opt, arg = optimizations(label)
|
|
332
344
|
|
|
333
345
|
# compute latest deadline of dependents (if any)
|
|
334
|
-
dependents = [target_task_graph.tasks[l] for l in dependents_of[label]]
|
|
335
346
|
deadline = None
|
|
336
|
-
if
|
|
337
|
-
|
|
338
|
-
deadline = max(
|
|
339
|
-
resolve_timestamps(now, task.task["deadline"])
|
|
340
|
-
for task in dependents # type: ignore
|
|
341
|
-
)
|
|
347
|
+
if dependents_of[label]:
|
|
348
|
+
deadline = max(get_deadline(l) for l in dependents_of[label])
|
|
342
349
|
|
|
343
350
|
if isinstance(opt, IndexSearch):
|
|
344
351
|
arg = arg, index_to_taskid, taskid_to_status
|
{taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/optimize/strategies.py
RENAMED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import functools
|
|
1
2
|
import logging
|
|
2
3
|
from datetime import datetime
|
|
3
4
|
|
|
@@ -10,6 +11,13 @@ from taskgraph.util.taskcluster import find_task_id, status_task
|
|
|
10
11
|
logger = logging.getLogger("optimization")
|
|
11
12
|
|
|
12
13
|
|
|
14
|
+
@functools.cache
|
|
15
|
+
def _parse_time(timestamp, fmt):
|
|
16
|
+
# Many tasks share the same deadline or replacement task, so avoid parsing
|
|
17
|
+
# the same timestamps over and over.
|
|
18
|
+
return datetime.strptime(timestamp, fmt)
|
|
19
|
+
|
|
20
|
+
|
|
13
21
|
@register_strategy("index-search")
|
|
14
22
|
class IndexSearch(OptimizationStrategy):
|
|
15
23
|
# A task with no dependencies remaining after optimization will be replaced
|
|
@@ -56,10 +64,10 @@ class IndexSearch(OptimizationStrategy):
|
|
|
56
64
|
)
|
|
57
65
|
continue
|
|
58
66
|
|
|
59
|
-
if deadline and
|
|
67
|
+
if deadline and _parse_time(
|
|
60
68
|
status["expires"], # type: ignore
|
|
61
69
|
self.fmt,
|
|
62
|
-
) <
|
|
70
|
+
) < _parse_time(deadline, self.fmt):
|
|
63
71
|
logger.debug(
|
|
64
72
|
f"not replacing {task.label} with {task_id} because it expires before {deadline}"
|
|
65
73
|
)
|