taskcluster-taskgraph 24.4.1__tar.gz → 24.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/CHANGELOG.md +21 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/PKG-INFO +1 -1
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.util.rst +8 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/pyproject.toml +1 -1
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/generator.py +216 -12
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/fetch-content +4 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/run-task +3 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/taskgraph.py +10 -8
- taskcluster_taskgraph-24.5.0/src/taskgraph/util/memory.py +24 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_generator.py +320 -10
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_scripts_run_task.py +93 -0
- taskcluster_taskgraph-24.5.0/test/test_util_memory.py +38 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/uv.lock +2 -2
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.codespell-ignore-words.txt +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.github/CODEOWNERS +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.github/workflows/codeql-analysis.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.github/workflows/codspeed.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.github/workflows/pypi-publish.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.gitignore +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.pre-commit-config.yaml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.readthedocs.yaml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.taskcluster.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.yamllint +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/CODE_OF_CONDUCT.md +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/CONTRIBUTING.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/LICENSE +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/Makefile +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/README.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/index.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/kind.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/loading.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/optimization.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/overview.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/scopes.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/task-graphs.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/transforms.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/conf.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/contributing.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/glossary.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/bootstrap-taskgraph.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/create-actions.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/create-tasks.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/debugging.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/define-schemas.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/docker.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/index.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/load-task-locally.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/resolve-keyed-by.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/run-locally.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/send-notifications.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/use-fetches.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/index.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/cli.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/index.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/migrations.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/optimization-strategies.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/parameters.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/modules.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.actions.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.loader.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.optimize.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.transforms.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.transforms.run.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/chunking.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/from_deps.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/index.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/matrix.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/task_context.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/tutorials/connecting-taskcluster.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/tutorials/creating-a-task-graph.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/tutorials/example-taskcluster.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/tutorials/getting-started.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/tutorials/index.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/make.bat +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/README.md +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/pyproject.toml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/src/pytest_taskgraph/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/vcs.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/README.md +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/pyproject.toml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/src/sphinx_taskgraph/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/src/sphinx_taskgraph/autoschema.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/renovate.json +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/add_new_jobs.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/cancel.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/cancel_all.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/rebuild_cached_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/registry.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/retrigger.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/util.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/config.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/create.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/decision.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/docker.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/filter_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/graph.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/loader/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/loader/default.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/loader/transform.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/main.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/morph.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/optimize/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/optimize/base.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/optimize/strategies.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/parameters.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/hgrc +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/robustcheckout.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/target_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/task.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/base.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/cached_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/chunking.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/code_review.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/docker_image.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/fetch.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/from_deps.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/matrix.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/notify.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/common.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/index_search.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/run_task.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/toolchain.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/task.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/task_context.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/archive.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/attributes.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/cached_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/caches.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/copy.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/dependencies.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/docker.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/hash.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/json.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/keyed_by.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/parameterization.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/path.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/python_path.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/readonlydict.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/schema.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/set_name.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/shell.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/task_context.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/taskcluster.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/taskgraph.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/templates.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/time.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/treeherder.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/vcs.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/verify.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/workertypes.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/yaml.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/config.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/REGISTRY +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/HASH +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/README.md +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/VERSION +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/system-setup.sh +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/fetch/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/image_builder/README.rst +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/README +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/insert-indexes.js +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/package.json +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/yarn.lock +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/python/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/run-task/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/run-task/system-setup.sh +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/skopeo/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/skopeo/policy.json +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/skopeo/push_image.sh +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/check/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/codecov/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/complete/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/doc/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/docker-image/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/fetch/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/push-image/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/test/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/test/linux.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/test/windows.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/scripts/codecov-upload.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/custom_parameters.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/custom_target_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/transforms/add_pr_route.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/transforms/push_image.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-pull-request-untrusted.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-pull-request.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-push.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-release-pytest-taskgraph.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-release-taskcluster-taskgraph.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/cookiecutter.json +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/hooks/post_gen_project.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/config.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/docker/linux/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/kinds/docker-image/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/kinds/hello/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/{{cookiecutter.project_slug}}_taskgraph/transforms/hello.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster.github.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/__init__.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/automationrelevance.json +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/conftest.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/task_context.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/config.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world-tag/Dockerfile +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world-tag/REGISTRY +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world-tag/VERSION +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/kinds/docker-image/kind.yml +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/scripts/toolchain/run.ps1 +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/scripts/toolchain/run.sh +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/test_taskgraph/transforms/foo.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/testmod/thing.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/mockedopen.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_actions_rebuild_cached_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_actions_registry.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_config.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_create.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_decision.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_docker.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_graph.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_graph_perf.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_main.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_morph.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_optimize.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_optimize_strategies.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_parameters.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_scripts_fetch_content.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_target_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_taskgraph.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transform_chunking.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transform_docker_image.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transform_task_context.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_base.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_cached_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_fetch.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_from_deps.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_matrix.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_notify.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_run.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_run_run_task.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_run_toolchain.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_task.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_archive.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_attributes.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_cached_tasks.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_copy.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_dependencies.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_docker.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_json.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_keyed_by.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_parameterization.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_path.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_python_path.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_readonlydict.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_schema.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_taskcluster.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_templates.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_time.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_treeherder.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_vcs.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_verify.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_workertypes.py +0 -0
- {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_yaml.py +0 -0
|
@@ -1,5 +1,26 @@
|
|
|
1
1
|
# Change Log
|
|
2
2
|
|
|
3
|
+
## [24.5.0] - 2026-10-21
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
- `taskgraph.generator.phase_finished` function, which can be replaced by
|
|
8
|
+
consumers and is called as each generation phase ends (#1073)
|
|
9
|
+
- Hook point in `run-task` at startup (#1077)
|
|
10
|
+
|
|
11
|
+
### Changed
|
|
12
|
+
|
|
13
|
+
- perf: on Linux, graph generation now uses a process per kind instead of a
|
|
14
|
+
process pool, to cut down on IPC overhead
|
|
15
|
+
(https://bugzilla.mozilla.org/show_bug.cgi?id=2079635)
|
|
16
|
+
- perf: pause the garbage collector during graph generation
|
|
17
|
+
(https://bugzilla.mozilla.org/show_bug.cgi?id=2079640)
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
|
|
21
|
+
- On Windows, fetch-content now reads tarballs until EOF, to be able to check
|
|
22
|
+
the stream's integrity (#1074)
|
|
23
|
+
|
|
3
24
|
## [24.4.1] - 2026-10-08
|
|
4
25
|
|
|
5
26
|
### Fixed
|
|
@@ -60,6 +60,14 @@ taskgraph.util.keyed\_by module
|
|
|
60
60
|
:undoc-members:
|
|
61
61
|
:show-inheritance:
|
|
62
62
|
|
|
63
|
+
taskgraph.util.memory module
|
|
64
|
+
----------------------------
|
|
65
|
+
|
|
66
|
+
.. automodule:: taskgraph.util.memory
|
|
67
|
+
:members:
|
|
68
|
+
:undoc-members:
|
|
69
|
+
:show-inheritance:
|
|
70
|
+
|
|
63
71
|
taskgraph.util.parameterization module
|
|
64
72
|
--------------------------------------
|
|
65
73
|
|
|
@@ -6,16 +6,21 @@ import copy
|
|
|
6
6
|
import inspect
|
|
7
7
|
import logging
|
|
8
8
|
import multiprocessing
|
|
9
|
+
import multiprocessing.connection
|
|
9
10
|
import os
|
|
11
|
+
import pickle
|
|
10
12
|
import platform
|
|
13
|
+
import signal
|
|
14
|
+
import sys
|
|
15
|
+
import time
|
|
16
|
+
import traceback
|
|
11
17
|
from concurrent.futures import (
|
|
12
18
|
FIRST_COMPLETED,
|
|
13
|
-
ProcessPoolExecutor,
|
|
14
19
|
ThreadPoolExecutor,
|
|
15
20
|
wait,
|
|
16
21
|
)
|
|
17
22
|
from dataclasses import dataclass
|
|
18
|
-
from typing import Callable, Optional, Union
|
|
23
|
+
from typing import Callable, Optional, Union, cast
|
|
19
24
|
|
|
20
25
|
from . import filter_tasks
|
|
21
26
|
from .config import GraphConfig, load_graph_config
|
|
@@ -26,6 +31,7 @@ from .parameters import Parameters, parameters_loader
|
|
|
26
31
|
from .task import Task
|
|
27
32
|
from .taskgraph import TaskGraph
|
|
28
33
|
from .transforms.base import TransformConfig, TransformSequence
|
|
34
|
+
from .util.memory import gc_disabled
|
|
29
35
|
from .util.python_path import find_object
|
|
30
36
|
from .util.schema import SchemaValidationError
|
|
31
37
|
from .util.verify import verifications
|
|
@@ -34,12 +40,31 @@ from .util.yaml import load_yaml
|
|
|
34
40
|
logger = logging.getLogger(__name__)
|
|
35
41
|
|
|
36
42
|
|
|
43
|
+
def phase_finished(name, start, end):
|
|
44
|
+
"""Called after each generation phase, with `time.monotonic()` values
|
|
45
|
+
taken around it. `name` is None if the phase raised an exception.
|
|
46
|
+
|
|
47
|
+
Projects can replace this from their `register` function; a replacement
|
|
48
|
+
made then also applies to the phase that is running.
|
|
49
|
+
"""
|
|
50
|
+
if name is None:
|
|
51
|
+
logger.debug(f"Generation phase failed after {end - start:.2f}s")
|
|
52
|
+
else:
|
|
53
|
+
logger.debug(f"Generated {name} in {end - start:.2f}s")
|
|
54
|
+
|
|
55
|
+
|
|
37
56
|
class KindNotFound(Exception):
|
|
38
57
|
"""
|
|
39
58
|
Raised when trying to load kind from a directory without a kind.yml.
|
|
40
59
|
"""
|
|
41
60
|
|
|
42
61
|
|
|
62
|
+
def _max_workers():
|
|
63
|
+
# os.process_cpu_count is new in Python 3.13.
|
|
64
|
+
cpu_count = getattr(os, "process_cpu_count", os.cpu_count)
|
|
65
|
+
return cpu_count() or 1
|
|
66
|
+
|
|
67
|
+
|
|
43
68
|
@dataclass(frozen=True)
|
|
44
69
|
class Kind:
|
|
45
70
|
name: str
|
|
@@ -124,6 +149,16 @@ class Kind:
|
|
|
124
149
|
return cls(kind_name, path, config, graph_config)
|
|
125
150
|
|
|
126
151
|
|
|
152
|
+
class _RemoteTraceback(Exception):
|
|
153
|
+
"""The traceback of an exception raised in a child process."""
|
|
154
|
+
|
|
155
|
+
def __init__(self, tb):
|
|
156
|
+
self.tb = tb
|
|
157
|
+
|
|
158
|
+
def __str__(self):
|
|
159
|
+
return f'\n"""\n{self.tb}"""'
|
|
160
|
+
|
|
161
|
+
|
|
127
162
|
class TaskGraphGenerator:
|
|
128
163
|
"""
|
|
129
164
|
The central controller for taskgraph. This handles all phases of graph
|
|
@@ -399,6 +434,157 @@ class TaskGraphGenerator:
|
|
|
399
434
|
|
|
400
435
|
return all_tasks
|
|
401
436
|
|
|
437
|
+
def _load_tasks_forked(self, kinds, kind_graph, parameters):
|
|
438
|
+
"""Load each kind in a child process forked once the kinds it depends
|
|
439
|
+
on are loaded.
|
|
440
|
+
|
|
441
|
+
The child shares every task loaded so far with the parent, so the
|
|
442
|
+
tasks of the kinds it depends on don't need to be sent to it, and it
|
|
443
|
+
sends its own tasks back through a pipe of its own.
|
|
444
|
+
"""
|
|
445
|
+
all_tasks = {}
|
|
446
|
+
tasks_by_kind = {}
|
|
447
|
+
# Pipe to each running child -> (kind name, pid)
|
|
448
|
+
running = {}
|
|
449
|
+
remaining = kinds.copy()
|
|
450
|
+
edges = set(kind_graph.edges)
|
|
451
|
+
max_workers = _max_workers()
|
|
452
|
+
|
|
453
|
+
try:
|
|
454
|
+
while remaining or running:
|
|
455
|
+
kinds_with_deps = {edge[0] for edge in edges}
|
|
456
|
+
ready_kinds = sorted(set(remaining) - kinds_with_deps)
|
|
457
|
+
for name in ready_kinds[: max_workers - len(running)]:
|
|
458
|
+
kind = remaining.pop(name)
|
|
459
|
+
reader, pid = self._fork_kind(
|
|
460
|
+
kind,
|
|
461
|
+
parameters,
|
|
462
|
+
self._get_kind_dependencies_tasks(kind, tasks_by_kind),
|
|
463
|
+
)
|
|
464
|
+
running[reader] = (name, pid)
|
|
465
|
+
|
|
466
|
+
# Waiting on nothing would block forever. _run made sure there
|
|
467
|
+
# is no dependency loop, so some kind is always running here.
|
|
468
|
+
assert running
|
|
469
|
+
for ready in multiprocessing.connection.wait(list(running)):
|
|
470
|
+
# wait returns the objects it was given, but is typed as
|
|
471
|
+
# also returning sockets and file descriptors.
|
|
472
|
+
reader = cast(multiprocessing.connection.Connection, ready)
|
|
473
|
+
name, pid = running[reader]
|
|
474
|
+
try:
|
|
475
|
+
result = reader.recv()
|
|
476
|
+
except EOFError:
|
|
477
|
+
# The child exited without sending anything.
|
|
478
|
+
result = None
|
|
479
|
+
del running[reader]
|
|
480
|
+
reader.close()
|
|
481
|
+
_, status = os.waitpid(pid, 0)
|
|
482
|
+
new_tasks = self._kind_result(name, result, status)
|
|
483
|
+
|
|
484
|
+
kind_tasks = tasks_by_kind.setdefault(name, {})
|
|
485
|
+
for task in new_tasks:
|
|
486
|
+
if task.label in all_tasks:
|
|
487
|
+
raise Exception("duplicate tasks with label " + task.label)
|
|
488
|
+
all_tasks[task.label] = task
|
|
489
|
+
kind_tasks[task.label] = task
|
|
490
|
+
|
|
491
|
+
edges = {e for e in edges if e[1] != name}
|
|
492
|
+
finally:
|
|
493
|
+
# Only left with running children on error.
|
|
494
|
+
for reader, (_, pid) in running.items():
|
|
495
|
+
reader.close()
|
|
496
|
+
os.kill(pid, signal.SIGKILL)
|
|
497
|
+
os.waitpid(pid, 0)
|
|
498
|
+
|
|
499
|
+
return all_tasks
|
|
500
|
+
|
|
501
|
+
def _fork_kind(self, kind, parameters, kind_dependencies_tasks):
|
|
502
|
+
"""Fork a child process loading the tasks of `kind`, and return the
|
|
503
|
+
pipe its result comes back through, and its pid."""
|
|
504
|
+
reader, writer = multiprocessing.Pipe(duplex=False)
|
|
505
|
+
# The child inherits the disabled garbage collector, so it doesn't
|
|
506
|
+
# write to every page it shares with us traversing the objects in them.
|
|
507
|
+
with gc_disabled():
|
|
508
|
+
pid = os.fork()
|
|
509
|
+
if pid == 0:
|
|
510
|
+
# Never leave this block, or return into the caller's code, in
|
|
511
|
+
# the child.
|
|
512
|
+
status = 1
|
|
513
|
+
try:
|
|
514
|
+
reader.close()
|
|
515
|
+
status = self._load_kind_in_child(
|
|
516
|
+
kind, parameters, kind_dependencies_tasks, writer
|
|
517
|
+
)
|
|
518
|
+
finally:
|
|
519
|
+
try:
|
|
520
|
+
sys.stdout.flush()
|
|
521
|
+
sys.stderr.flush()
|
|
522
|
+
finally:
|
|
523
|
+
os._exit(status)
|
|
524
|
+
writer.close()
|
|
525
|
+
return reader, pid
|
|
526
|
+
|
|
527
|
+
def _load_kind_in_child(self, kind, parameters, kind_dependencies_tasks, writer):
|
|
528
|
+
"""Load the tasks of `kind` and send them, or the exception loading
|
|
529
|
+
them raised, through `writer`. Return the child's exit status."""
|
|
530
|
+
try:
|
|
531
|
+
tasks = kind.load_tasks(
|
|
532
|
+
parameters, kind_dependencies_tasks, self._write_artifacts
|
|
533
|
+
)
|
|
534
|
+
except BaseException as e:
|
|
535
|
+
tb = traceback.format_exc()
|
|
536
|
+
try:
|
|
537
|
+
# Make sure the parent will be able to unpickle it.
|
|
538
|
+
pickle.loads(pickle.dumps(e))
|
|
539
|
+
except Exception:
|
|
540
|
+
e = Exception(f"{type(e).__name__}: {e}")
|
|
541
|
+
writer.send((e, tb, None))
|
|
542
|
+
return 1
|
|
543
|
+
|
|
544
|
+
try:
|
|
545
|
+
writer.send((None, None, tasks))
|
|
546
|
+
except Exception as e:
|
|
547
|
+
writer.send(
|
|
548
|
+
(
|
|
549
|
+
Exception(
|
|
550
|
+
f"Could not send the tasks of kind {kind.name} "
|
|
551
|
+
f"to the parent process: {type(e).__name__}: {e}"
|
|
552
|
+
),
|
|
553
|
+
traceback.format_exc(),
|
|
554
|
+
None,
|
|
555
|
+
)
|
|
556
|
+
)
|
|
557
|
+
return 1
|
|
558
|
+
return 0
|
|
559
|
+
|
|
560
|
+
@staticmethod
|
|
561
|
+
def _kind_result(name, result, status):
|
|
562
|
+
"""Return the tasks the child loading kind `name` sent, given what it
|
|
563
|
+
sent and its exit status. Raise the exception loading them raised in
|
|
564
|
+
the child, if any."""
|
|
565
|
+
if result is None:
|
|
566
|
+
code = os.waitstatus_to_exitcode(status)
|
|
567
|
+
if code < 0:
|
|
568
|
+
how = f"was killed by signal {signal.Signals(-code).name}"
|
|
569
|
+
else:
|
|
570
|
+
how = f"exited with status {code}"
|
|
571
|
+
raise Exception(
|
|
572
|
+
f"Process loading tasks for kind {name} {how} without sending them"
|
|
573
|
+
)
|
|
574
|
+
|
|
575
|
+
exc, tb, tasks = result
|
|
576
|
+
if exc is not None:
|
|
577
|
+
exc.__cause__ = _RemoteTraceback(tb)
|
|
578
|
+
if isinstance(exc, SchemaValidationError):
|
|
579
|
+
logger.error(f"Error loading tasks for kind {name}:\n{exc}")
|
|
580
|
+
else:
|
|
581
|
+
logger.error(
|
|
582
|
+
f"Error loading tasks for kind {name}:",
|
|
583
|
+
exc_info=(type(exc), exc, exc.__traceback__),
|
|
584
|
+
)
|
|
585
|
+
raise exc
|
|
586
|
+
return tasks
|
|
587
|
+
|
|
402
588
|
def _run(self):
|
|
403
589
|
logger.info("Loading graph configuration.")
|
|
404
590
|
graph_config = load_graph_config(self.root_dir)
|
|
@@ -446,6 +632,12 @@ class TaskGraphGenerator:
|
|
|
446
632
|
for kind in kinds.values():
|
|
447
633
|
for dep in kind.config.get("kind-dependencies", []):
|
|
448
634
|
edges.add((kind.name, dep, "kind-dependency"))
|
|
635
|
+
for _, dep, _ in edges:
|
|
636
|
+
if dep not in kinds:
|
|
637
|
+
message = f'Could not find the kind "{dep}"\nAvailable kinds:\n'
|
|
638
|
+
for k in sorted(kinds):
|
|
639
|
+
message += f' - "{k}"\n'
|
|
640
|
+
raise Exception(message)
|
|
449
641
|
kind_graph = Graph(frozenset(kinds), frozenset(edges))
|
|
450
642
|
|
|
451
643
|
if target_kinds:
|
|
@@ -453,6 +645,11 @@ class TaskGraphGenerator:
|
|
|
453
645
|
set(target_kinds) | {"docker-image"}
|
|
454
646
|
)
|
|
455
647
|
|
|
648
|
+
# The task loaders wait for each kind's dependencies to be loaded
|
|
649
|
+
# before loading it, so a dependency loop would hang them or silently
|
|
650
|
+
# leave kinds out. This raises if there is one.
|
|
651
|
+
kind_graph.visit_postorder()
|
|
652
|
+
|
|
456
653
|
yield "kind_graph", kind_graph
|
|
457
654
|
|
|
458
655
|
logger.info("Generating full task set")
|
|
@@ -460,7 +657,7 @@ class TaskGraphGenerator:
|
|
|
460
657
|
# The next block deals with enabling parallel kind processing, which
|
|
461
658
|
# currently has different support on different platforms. In summary:
|
|
462
659
|
# * Parallel kind processing is supported and enabled by default on
|
|
463
|
-
# Linux. We
|
|
660
|
+
# Linux. We fork a process per kind by default, but experimental
|
|
464
661
|
# support for multiple threads can be enabled instead.
|
|
465
662
|
# * On other platforms, we have experimental support for parallel
|
|
466
663
|
# kind processing with multiple threads.
|
|
@@ -480,21 +677,19 @@ class TaskGraphGenerator:
|
|
|
480
677
|
if os.environ.get("TASKGRAPH_SERIAL"):
|
|
481
678
|
return self._load_tasks_serial(kinds, kind_graph, parameters)
|
|
482
679
|
elif os.environ.get("TASKGRAPH_USE_THREADS"):
|
|
483
|
-
executor = ThreadPoolExecutor(max_workers=
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
mp_context=multiprocessing.get_context("fork")
|
|
680
|
+
executor = ThreadPoolExecutor(max_workers=_max_workers())
|
|
681
|
+
return self._load_tasks_parallel(
|
|
682
|
+
kinds, kind_graph, parameters, executor
|
|
487
683
|
)
|
|
488
|
-
|
|
489
|
-
kinds, kind_graph, parameters
|
|
490
|
-
)
|
|
684
|
+
else:
|
|
685
|
+
return self._load_tasks_forked(kinds, kind_graph, parameters)
|
|
491
686
|
else:
|
|
492
687
|
if os.environ.get("TASKGRAPH_SERIAL") or not os.environ.get(
|
|
493
688
|
"TASKGRAPH_USE_THREADS"
|
|
494
689
|
):
|
|
495
690
|
return self._load_tasks_serial(kinds, kind_graph, parameters)
|
|
496
691
|
else:
|
|
497
|
-
executor = ThreadPoolExecutor(max_workers=
|
|
692
|
+
executor = ThreadPoolExecutor(max_workers=_max_workers())
|
|
498
693
|
return self._load_tasks_parallel(
|
|
499
694
|
kinds, kind_graph, parameters, executor
|
|
500
695
|
)
|
|
@@ -603,10 +798,19 @@ class TaskGraphGenerator:
|
|
|
603
798
|
|
|
604
799
|
def _run_until(self, name):
|
|
605
800
|
while name not in self._run_results:
|
|
801
|
+
start = time.monotonic()
|
|
606
802
|
try:
|
|
607
|
-
|
|
803
|
+
# Generation creates millions of objects that stay alive, so
|
|
804
|
+
# collecting garbage meanwhile only traverses them for nothing.
|
|
805
|
+
# The callers' code runs with the collector in its usual state.
|
|
806
|
+
with gc_disabled():
|
|
807
|
+
k, v = next(self._run) # type: ignore
|
|
608
808
|
except StopIteration:
|
|
609
809
|
raise AttributeError(f"No such run result {name}")
|
|
810
|
+
except BaseException:
|
|
811
|
+
phase_finished(None, start, time.monotonic())
|
|
812
|
+
raise
|
|
813
|
+
phase_finished(k, start, time.monotonic())
|
|
610
814
|
self._run_results[k] = v
|
|
611
815
|
return self._run_results[name]
|
|
612
816
|
|
{taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/fetch-content
RENAMED
|
@@ -642,6 +642,10 @@ def extract_stream(reader, dest_dir):
|
|
|
642
642
|
# See extract_archive.
|
|
643
643
|
reader.forget()
|
|
644
644
|
TarFile.open(fileobj=ifh, mode="r|").extractall(str(dest_dir))
|
|
645
|
+
# tarfile stops at the end-of-archive marker, which can leave the
|
|
646
|
+
# end of the compressed stream, and its checksum, unread.
|
|
647
|
+
while ifh.read(CHUNK_SIZE):
|
|
648
|
+
pass
|
|
645
649
|
elif magic == ZSTD_MAGIC and shutil.which("zstd"):
|
|
646
650
|
# Not tar --use-compress-program: when reading from stdin, tar relays
|
|
647
651
|
# it to the decompressor in small blocks, which is several times
|
{taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/run-task
RENAMED
|
@@ -1720,6 +1720,9 @@ def main(args):
|
|
|
1720
1720
|
os.setresgid(gid, gid, gid)
|
|
1721
1721
|
os.setresuid(uid, uid, uid)
|
|
1722
1722
|
|
|
1723
|
+
if hook_path := os.environ.get("RUN_TASK_STARTUP_HOOK"):
|
|
1724
|
+
run_python_script(hook_path)
|
|
1725
|
+
|
|
1723
1726
|
for repo in repositories:
|
|
1724
1727
|
vcs_checkout_from_args(repo)
|
|
1725
1728
|
|
|
@@ -6,6 +6,7 @@ from dataclasses import dataclass
|
|
|
6
6
|
|
|
7
7
|
from .graph import Graph
|
|
8
8
|
from .task import Task
|
|
9
|
+
from .util.memory import gc_disabled
|
|
9
10
|
|
|
10
11
|
|
|
11
12
|
@dataclass(frozen=True)
|
|
@@ -58,13 +59,14 @@ class TaskGraph:
|
|
|
58
59
|
"""
|
|
59
60
|
tasks = {}
|
|
60
61
|
edges = set()
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
62
|
+
with gc_disabled():
|
|
63
|
+
for key, value in tasks_dict.items():
|
|
64
|
+
tasks[key] = Task.from_json(value)
|
|
65
|
+
if "task_id" in value:
|
|
66
|
+
tasks[key].task_id = value["task_id"]
|
|
67
|
+
for depname, dep in value["dependencies"].items():
|
|
68
|
+
# Task filtering can cause dependencies to be removed from the graph.
|
|
69
|
+
if dep in tasks_dict:
|
|
70
|
+
edges.add((key, dep, depname))
|
|
69
71
|
task_graph = cls(tasks, Graph(frozenset(tasks), edges))
|
|
70
72
|
return tasks, task_graph
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
4
|
+
|
|
5
|
+
import gc
|
|
6
|
+
from contextlib import contextmanager
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@contextmanager
|
|
10
|
+
def gc_disabled():
|
|
11
|
+
"""Disable the garbage collector for the duration of the block, restoring
|
|
12
|
+
its previous state afterwards.
|
|
13
|
+
|
|
14
|
+
Generating or loading a task graph creates millions of objects that stay
|
|
15
|
+
alive, so collections keep traversing a growing heap without freeing
|
|
16
|
+
anything.
|
|
17
|
+
"""
|
|
18
|
+
was_enabled = gc.isenabled()
|
|
19
|
+
gc.disable()
|
|
20
|
+
try:
|
|
21
|
+
yield
|
|
22
|
+
finally:
|
|
23
|
+
if was_enabled:
|
|
24
|
+
gc.enable()
|