taskcluster-taskgraph 19.2.1__tar.gz → 21.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (269) hide show
  1. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/.pre-commit-config.yaml +4 -4
  2. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/.taskcluster.yml +8 -1
  3. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/CHANGELOG.md +32 -0
  4. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/PKG-INFO +1 -1
  5. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/howto/bootstrap-taskgraph.rst +1 -1
  6. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/migrations.rst +17 -0
  7. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/pyproject.toml +1 -1
  8. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/decision.py +14 -0
  9. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/generator.py +46 -26
  10. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/main.py +6 -0
  11. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/run-task/run-task +18 -5
  12. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/task.py +8 -19
  13. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/schema.py +87 -24
  14. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/vcs.py +25 -0
  15. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/verify.py +12 -6
  16. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/config.yml +1 -0
  17. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/run-task/Dockerfile +1 -1
  18. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/kinds/test/linux.yml +20 -0
  19. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_decision.py +59 -0
  20. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_generator.py +30 -2
  21. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_scripts_run_task.py +48 -2
  22. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_schema.py +45 -23
  23. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_vcs.py +75 -0
  24. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_verify.py +33 -0
  25. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/uv.lock +2 -2
  26. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/.codespell-ignore-words.txt +0 -0
  27. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/.github/CODEOWNERS +0 -0
  28. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/.github/dependabot.yml +0 -0
  29. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/.github/workflows/codeql-analysis.yml +0 -0
  30. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/.github/workflows/pre-commit-autoupdate.yml +0 -0
  31. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/.github/workflows/pre-commit.yml +0 -0
  32. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/.github/workflows/pypi-publish.yml +0 -0
  33. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/.gitignore +0 -0
  34. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/.readthedocs.yaml +0 -0
  35. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/.yamllint +0 -0
  36. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/CODE_OF_CONDUCT.md +0 -0
  37. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/CONTRIBUTING.rst +0 -0
  38. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/LICENSE +0 -0
  39. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/Makefile +0 -0
  40. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/README.rst +0 -0
  41. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/concepts/index.rst +0 -0
  42. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/concepts/kind.rst +0 -0
  43. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/concepts/loading.rst +0 -0
  44. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/concepts/optimization.rst +0 -0
  45. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/concepts/overview.rst +0 -0
  46. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/concepts/scopes.rst +0 -0
  47. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/concepts/task-graphs.rst +0 -0
  48. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/concepts/transforms.rst +0 -0
  49. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/conf.py +0 -0
  50. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/contributing.rst +0 -0
  51. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/glossary.rst +0 -0
  52. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/howto/create-actions.rst +0 -0
  53. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/howto/create-tasks.rst +0 -0
  54. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/howto/debugging.rst +0 -0
  55. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/howto/docker.rst +0 -0
  56. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/howto/index.rst +0 -0
  57. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/howto/load-task-locally.rst +0 -0
  58. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/howto/resolve-keyed-by.rst +0 -0
  59. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/howto/run-locally.rst +0 -0
  60. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/howto/send-notifications.rst +0 -0
  61. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/howto/use-fetches.rst +0 -0
  62. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/index.rst +0 -0
  63. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/cli.rst +0 -0
  64. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/index.rst +0 -0
  65. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/optimization-strategies.rst +0 -0
  66. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/parameters.rst +0 -0
  67. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/source/modules.rst +0 -0
  68. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/source/taskgraph.actions.rst +0 -0
  69. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/source/taskgraph.loader.rst +0 -0
  70. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/source/taskgraph.optimize.rst +0 -0
  71. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/source/taskgraph.rst +0 -0
  72. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/source/taskgraph.transforms.rst +0 -0
  73. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/source/taskgraph.transforms.run.rst +0 -0
  74. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/source/taskgraph.util.rst +0 -0
  75. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/transforms/chunking.rst +0 -0
  76. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/transforms/from_deps.rst +0 -0
  77. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/transforms/index.rst +0 -0
  78. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/transforms/matrix.rst +0 -0
  79. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/reference/transforms/task_context.rst +0 -0
  80. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/tutorials/connecting-taskcluster.rst +0 -0
  81. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/tutorials/creating-a-task-graph.rst +0 -0
  82. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/tutorials/example-taskcluster.yml +0 -0
  83. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/tutorials/getting-started.rst +0 -0
  84. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/docs/tutorials/index.rst +0 -0
  85. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/make.bat +0 -0
  86. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/packages/pytest-taskgraph/README.md +0 -0
  87. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/packages/pytest-taskgraph/pyproject.toml +0 -0
  88. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/packages/pytest-taskgraph/src/pytest_taskgraph/__init__.py +0 -0
  89. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py +0 -0
  90. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/vcs.py +0 -0
  91. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/packages/sphinx-taskgraph/README.md +0 -0
  92. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/packages/sphinx-taskgraph/pyproject.toml +0 -0
  93. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/packages/sphinx-taskgraph/src/sphinx_taskgraph/__init__.py +0 -0
  94. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/packages/sphinx-taskgraph/src/sphinx_taskgraph/autoschema.py +0 -0
  95. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/__init__.py +0 -0
  96. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/actions/__init__.py +0 -0
  97. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/actions/add_new_jobs.py +0 -0
  98. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/actions/cancel.py +0 -0
  99. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/actions/cancel_all.py +0 -0
  100. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/actions/rebuild_cached_tasks.py +0 -0
  101. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/actions/registry.py +0 -0
  102. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/actions/retrigger.py +0 -0
  103. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/actions/util.py +0 -0
  104. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/config.py +0 -0
  105. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/create.py +0 -0
  106. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/docker.py +0 -0
  107. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/filter_tasks.py +0 -0
  108. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/graph.py +0 -0
  109. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/loader/__init__.py +0 -0
  110. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/loader/default.py +0 -0
  111. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/loader/transform.py +0 -0
  112. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/morph.py +0 -0
  113. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/optimize/__init__.py +0 -0
  114. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/optimize/base.py +0 -0
  115. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/optimize/strategies.py +0 -0
  116. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/parameters.py +0 -0
  117. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/run-task/fetch-content +0 -0
  118. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/run-task/hgrc +0 -0
  119. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/run-task/robustcheckout.py +0 -0
  120. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/target_tasks.py +0 -0
  121. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/task.py +0 -0
  122. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/taskgraph.py +0 -0
  123. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/__init__.py +0 -0
  124. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/base.py +0 -0
  125. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/cached_tasks.py +0 -0
  126. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/chunking.py +0 -0
  127. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/code_review.py +0 -0
  128. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/docker_image.py +0 -0
  129. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/fetch.py +0 -0
  130. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/from_deps.py +0 -0
  131. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/matrix.py +0 -0
  132. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/notify.py +0 -0
  133. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/run/__init__.py +0 -0
  134. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/run/common.py +0 -0
  135. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/run/index_search.py +0 -0
  136. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/run/run_task.py +0 -0
  137. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/run/toolchain.py +0 -0
  138. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/task_context.py +0 -0
  139. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/__init__.py +0 -0
  140. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/archive.py +0 -0
  141. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/attributes.py +0 -0
  142. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/cached_tasks.py +0 -0
  143. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/caches.py +0 -0
  144. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/copy.py +0 -0
  145. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/dependencies.py +0 -0
  146. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/docker.py +0 -0
  147. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/hash.py +0 -0
  148. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/json.py +0 -0
  149. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/keyed_by.py +0 -0
  150. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/parameterization.py +0 -0
  151. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/path.py +0 -0
  152. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/python_path.py +0 -0
  153. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/readonlydict.py +0 -0
  154. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/set_name.py +0 -0
  155. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/shell.py +0 -0
  156. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/taskcluster.py +0 -0
  157. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/taskgraph.py +0 -0
  158. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/templates.py +0 -0
  159. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/time.py +0 -0
  160. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/treeherder.py +0 -0
  161. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/workertypes.py +0 -0
  162. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/yaml.py +0 -0
  163. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/REGISTRY +0 -0
  164. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/decision/Dockerfile +0 -0
  165. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/decision/HASH +0 -0
  166. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/decision/README.md +0 -0
  167. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/decision/VERSION +0 -0
  168. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/decision/system-setup.sh +0 -0
  169. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/fetch/Dockerfile +0 -0
  170. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/image_builder/README.rst +0 -0
  171. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/index-task/Dockerfile +0 -0
  172. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/index-task/README +0 -0
  173. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/index-task/insert-indexes.js +0 -0
  174. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/index-task/package.json +0 -0
  175. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/index-task/yarn.lock +0 -0
  176. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/python/Dockerfile +0 -0
  177. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/run-task/system-setup.sh +0 -0
  178. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/skopeo/Dockerfile +0 -0
  179. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/skopeo/policy.json +0 -0
  180. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/skopeo/push_image.sh +0 -0
  181. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/kinds/check/kind.yml +0 -0
  182. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/kinds/codecov/kind.yml +0 -0
  183. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/kinds/complete/kind.yml +0 -0
  184. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/kinds/doc/kind.yml +0 -0
  185. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/kinds/docker-image/kind.yml +0 -0
  186. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/kinds/fetch/kind.yml +0 -0
  187. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/kinds/push-image/kind.yml +0 -0
  188. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/kinds/test/kind.yml +0 -0
  189. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/kinds/test/windows.yml +0 -0
  190. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/scripts/codecov-upload.py +0 -0
  191. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/scripts/external_tools/tooltool.py +0 -0
  192. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/self_taskgraph/__init__.py +0 -0
  193. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/self_taskgraph/custom_parameters.py +0 -0
  194. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/self_taskgraph/custom_target_tasks.py +0 -0
  195. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/self_taskgraph/transforms/add_pr_route.py +0 -0
  196. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/self_taskgraph/transforms/push_image.py +0 -0
  197. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/test/params/main-repo-pull-request-untrusted.yml +0 -0
  198. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/test/params/main-repo-pull-request.yml +0 -0
  199. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/test/params/main-repo-push.yml +0 -0
  200. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/test/params/main-repo-release-pytest-taskgraph.yml +0 -0
  201. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/taskcluster/test/params/main-repo-release-taskcluster-taskgraph.yml +0 -0
  202. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/template/cookiecutter.json +0 -0
  203. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/template/hooks/post_gen_project.py +0 -0
  204. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/template/{{cookiecutter.project_name}}/taskcluster/config.yml +0 -0
  205. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/template/{{cookiecutter.project_name}}/taskcluster/docker/linux/Dockerfile +0 -0
  206. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/template/{{cookiecutter.project_name}}/taskcluster/kinds/docker-image/kind.yml +0 -0
  207. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/template/{{cookiecutter.project_name}}/taskcluster/kinds/hello/kind.yml +0 -0
  208. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/template/{{cookiecutter.project_name}}/taskcluster/{{cookiecutter.project_slug}}_taskgraph/transforms/hello.py +0 -0
  209. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/template/{{cookiecutter.project_name}}/taskcluster.github.yml +0 -0
  210. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/__init__.py +0 -0
  211. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/automationrelevance.json +0 -0
  212. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/conftest.py +0 -0
  213. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/data/task_context.yml +0 -0
  214. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/data/taskcluster/config.yml +0 -0
  215. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/data/taskcluster/docker/hello-world/Dockerfile +0 -0
  216. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/data/taskcluster/docker/hello-world-tag/Dockerfile +0 -0
  217. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/data/taskcluster/docker/hello-world-tag/REGISTRY +0 -0
  218. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/data/taskcluster/docker/hello-world-tag/VERSION +0 -0
  219. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/data/taskcluster/kinds/docker-image/kind.yml +0 -0
  220. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/data/taskcluster/scripts/toolchain/run.ps1 +0 -0
  221. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/data/taskcluster/scripts/toolchain/run.sh +0 -0
  222. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/data/taskcluster/test_taskgraph/transforms/foo.py +0 -0
  223. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/data/testmod/thing.py +0 -0
  224. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/mockedopen.py +0 -0
  225. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_actions_rebuild_cached_tasks.py +0 -0
  226. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_actions_registry.py +0 -0
  227. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_config.py +0 -0
  228. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_create.py +0 -0
  229. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_docker.py +0 -0
  230. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_graph.py +0 -0
  231. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_main.py +0 -0
  232. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_morph.py +0 -0
  233. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_optimize.py +0 -0
  234. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_optimize_strategies.py +0 -0
  235. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_parameters.py +0 -0
  236. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_scripts_fetch_content.py +0 -0
  237. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_target_tasks.py +0 -0
  238. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_taskgraph.py +0 -0
  239. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_transform_chunking.py +0 -0
  240. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_transform_docker_image.py +0 -0
  241. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_transform_task_context.py +0 -0
  242. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_transforms_base.py +0 -0
  243. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_transforms_cached_tasks.py +0 -0
  244. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_transforms_fetch.py +0 -0
  245. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_transforms_from_deps.py +0 -0
  246. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_transforms_matrix.py +0 -0
  247. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_transforms_notify.py +0 -0
  248. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_transforms_run.py +0 -0
  249. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_transforms_run_run_task.py +0 -0
  250. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_transforms_run_toolchain.py +0 -0
  251. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_transforms_task.py +0 -0
  252. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_archive.py +0 -0
  253. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_attributes.py +0 -0
  254. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_cached_tasks.py +0 -0
  255. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_copy.py +0 -0
  256. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_dependencies.py +0 -0
  257. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_docker.py +0 -0
  258. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_json.py +0 -0
  259. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_keyed_by.py +0 -0
  260. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_parameterization.py +0 -0
  261. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_path.py +0 -0
  262. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_python_path.py +0 -0
  263. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_readonlydict.py +0 -0
  264. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_taskcluster.py +0 -0
  265. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_templates.py +0 -0
  266. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_time.py +0 -0
  267. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_treeherder.py +0 -0
  268. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_workertypes.py +0 -0
  269. {taskcluster_taskgraph-19.2.1 → taskcluster_taskgraph-21.0.0}/test/test_util_yaml.py +0 -0
@@ -13,13 +13,13 @@ repos:
13
13
  exclude: template
14
14
  - id: check-added-large-files
15
15
  - repo: https://github.com/astral-sh/ruff-pre-commit
16
- rev: v0.14.14
16
+ rev: v0.15.8
17
17
  hooks:
18
18
  - id: ruff
19
19
  args: [--fix, --exit-non-zero-on-fix]
20
20
  - id: ruff-format
21
21
  - repo: https://github.com/astral-sh/uv-pre-commit
22
- rev: 0.9.28
22
+ rev: 0.11.2
23
23
  hooks:
24
24
  - id: uv-lock
25
25
  - repo: https://github.com/adrienverge/yamllint.git
@@ -28,7 +28,7 @@ repos:
28
28
  - id: yamllint
29
29
  exclude: template
30
30
  - repo: https://github.com/codespell-project/codespell
31
- rev: v2.4.1
31
+ rev: v2.4.2
32
32
  hooks:
33
33
  - id: codespell
34
34
  entry: codespell -I .codespell-ignore-words.txt
@@ -37,7 +37,7 @@ repos:
37
37
  test/test_util_path.py
38
38
  )$
39
39
  - repo: https://github.com/compilerla/conventional-pre-commit
40
- rev: v4.3.0
40
+ rev: v4.4.0
41
41
  hooks:
42
42
  - id: conventional-pre-commit
43
43
  stages: [commit-msg]
@@ -226,6 +226,7 @@ tasks:
226
226
  - $if: 'isPullRequest'
227
227
  then:
228
228
  TASKGRAPH_PULL_REQUEST_NUMBER: '${event.pull_request.number}'
229
+ TASKGRAPH_EXTRA_REFS: {$json: ["refs/notes/decision-parameters"]}
229
230
  - $if: 'tasks_for == "action" || tasks_for == "pr-action"'
230
231
  then:
231
232
  ACTION_TASK_GROUP_ID: '${action.taskGroupId}' # taskGroupId of the target task
@@ -251,7 +252,13 @@ tasks:
251
252
  - bash
252
253
  - -cx
253
254
  - $let:
254
- extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
255
+ extraArgs:
256
+ $if: 'tasks_for == "cron"'
257
+ then: '${cron.quoted_args}'
258
+ else:
259
+ $if: 'tasks_for == "github-pull-request"'
260
+ then: '--allow-parameter-override'
261
+ else: ''
255
262
  in:
256
263
  $if: 'tasks_for == "action" || tasks_for == "pr-action"'
257
264
  then: >
@@ -1,5 +1,37 @@
1
1
  # Change Log
2
2
 
3
+ ## [21.0.0] - 2026-04-13
4
+
5
+ ### Added
6
+
7
+ - Ability for `run-task` to fetch arbitrary extra refs
8
+ - Ability for Decision task to override params via Git notes
9
+
10
+ ### Fixed
11
+
12
+ - Improved index route verification
13
+
14
+ ### Changed
15
+
16
+ - BREAKING CHANGE: run-task and decision images now use Debian 13
17
+
18
+ ### Removed
19
+
20
+ - BREAKING CHANGE: Removed docker-worker features: dind, privileged, loopback-audio
21
+
22
+ ## [20.0.0] - 2026-03-17
23
+
24
+ ### Added
25
+
26
+ - Support for generic-worker's `hide-cmd-window` feature
27
+ - Ability to process kinds concurrently with threading
28
+
29
+ ### Fixed
30
+
31
+ - BREAKING CHANGE: Binaries installed as part of `PIP_REQUIREMENTS` by `run-task` now end up in `~/.local/bin`
32
+ - Exception when base revision is the null revision
33
+ - Ability to use `optionally_keyed_by` with underlying dict using msgspec
34
+
3
35
  ## [19.2.1] - 2026-02-19
4
36
 
5
37
  ### Fixed
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: taskcluster-taskgraph
3
- Version: 19.2.1
3
+ Version: 21.0.0
4
4
  Summary: Build taskcluster taskgraphs
5
5
  Project-URL: Repository, https://github.com/taskcluster/taskgraph
6
6
  Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues
@@ -101,7 +101,7 @@ in addition to ``taskcluster-taskgraph`` we also want to depend on
101
101
  EOF
102
102
 
103
103
  # This works best if you use the same Python as the one used in the Decision
104
- # image (currently 3.11).
104
+ # image (currently 3.13).
105
105
  pip-compile --generate-hashes --output-file requirements.txt requirements.in
106
106
 
107
107
  .. note::
@@ -3,6 +3,23 @@ Migration Guide
3
3
 
4
4
  This page can help when migrating Taskgraph across major versions.
5
5
 
6
+ 20.x -> 21.x
7
+ ------------
8
+
9
+ * If you have tasks using `docker-worker`'s `dind`, `privileged` or `loopback-audio`
10
+ features, they are no longer supported. Either avoid using these features or migrate
11
+ to a `generic-worker` pool with D2G payloads enabled.
12
+ * Ensure any task's using the `run-task` or `decision` images work with the new
13
+ Debian 13 base.
14
+
15
+ 19.x -> 20.x
16
+ ------------
17
+
18
+ * Binaries belonging to packages that get installed by `run-task` as part of
19
+ the `PIP_REQUIREMENTS` variable, are now installed to `~/.local/bin` instead
20
+ of `~/.local/lib/<python>/site-packages/bin`. Update any task commands that hardcode
21
+ the binary path accordingly.
22
+
6
23
  18.x -> 19.x
7
24
  ------------
8
25
 
@@ -1,7 +1,7 @@
1
1
  ### Project
2
2
  [project]
3
3
  name = "taskcluster-taskgraph"
4
- version = "19.2.1"
4
+ version = "21.0.0"
5
5
  description = "Build taskcluster taskgraphs"
6
6
  readme = "README.rst"
7
7
  authors = [
@@ -264,6 +264,20 @@ def get_decision_parameters(graph_config, options):
264
264
  ] == "github-pull-request":
265
265
  set_try_config(parameters, task_config_file)
266
266
 
267
+ # load extra parameters from vcs note if able
268
+ note_ref = "refs/notes/decision-parameters"
269
+ if options.get("allow_parameter_override") and (
270
+ note_params := repo.get_note(note_ref)
271
+ ):
272
+ try:
273
+ note_params = json.loads(note_params)
274
+ logger.info(
275
+ f"Overriding parameters from {note_ref}:\n{json.dumps(note_params, indent=2)}"
276
+ )
277
+ parameters.update(note_params)
278
+ except ValueError as e:
279
+ raise Exception(f"Failed to parse {note_ref} as JSON: {e}") from e
280
+
267
281
  result = Parameters(**parameters)
268
282
  result.check()
269
283
  return result
@@ -11,6 +11,7 @@ import platform
11
11
  from concurrent.futures import (
12
12
  FIRST_COMPLETED,
13
13
  ProcessPoolExecutor,
14
+ ThreadPoolExecutor,
14
15
  wait,
15
16
  )
16
17
  from dataclasses import dataclass
@@ -312,15 +313,13 @@ class TaskGraphGenerator:
312
313
 
313
314
  return all_tasks
314
315
 
315
- def _load_tasks_parallel(self, kinds, kind_graph, parameters):
316
+ def _load_tasks_parallel(self, kinds, kind_graph, parameters, executor):
316
317
  all_tasks = {}
317
318
  futures_to_kind = {}
318
319
  futures = set()
319
320
  edges = set(kind_graph.edges)
320
321
 
321
- with ProcessPoolExecutor(
322
- mp_context=multiprocessing.get_context("fork")
323
- ) as executor:
322
+ with executor:
324
323
 
325
324
  def submit_ready_kinds():
326
325
  """Create the next batch of tasks for kinds without dependencies."""
@@ -434,29 +433,50 @@ class TaskGraphGenerator:
434
433
  yield "kind_graph", kind_graph
435
434
 
436
435
  logger.info("Generating full task set")
437
- # The short version of the below is: we only support parallel kind
438
- # processing on Linux.
436
+
437
+ # The next block deals with enabling parallel kind processing, which
438
+ # currently has different support on different platforms. In summary:
439
+ # * Parallel kind processing is supported and enabled by default on
440
+ # Linux. We use multiple processes by default, but experimental
441
+ # support for multiple threads can be enabled instead.
442
+ # * On other platforms, we have experimental support for parallel
443
+ # kind processing with multiple threads.
439
444
  #
440
- # Current parallel generation relies on multiprocessing, and more
441
- # specifically: the "fork" multiprocessing method. This is not supported
442
- # at all on Windows (it uses "spawn"). Forking is supported on macOS,
443
- # but no longer works reliably in all cases, and our usage of it here
444
- # causes crashes. See https://github.com/python/cpython/issues/77906
445
- # and http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html
446
- # for more details on that.
447
- # Other methods of multiprocessing (both "spawn" and "forkserver")
448
- # do not work for our use case, because they cause global variables
449
- # to be reinitialized, which are sometimes modified earlier in graph
450
- # generation. These issues can theoretically be worked around by
451
- # eliminating all reliance on globals as part of task generation, but
452
- # is far from a small amount of work in users like Gecko/Firefox.
453
- # In the long term, the better path forward is likely to be switching
454
- # to threading with a free-threaded python to achieve similar parallel
455
- # processing.
456
- if platform.system() != "Linux" or os.environ.get("TASKGRAPH_SERIAL"):
457
- all_tasks = self._load_tasks_serial(kinds, kind_graph, parameters)
458
- else:
459
- all_tasks = self._load_tasks_parallel(kinds, kind_graph, parameters)
445
+ # On all platforms serial kind processing can be enabled by setting
446
+ # TASKGRAPH_SERIAL in the environment.
447
+ #
448
+ # On all platforms, multiple threads can be enabled by setting
449
+ # TASKGRAPH_USE_THREADS in the environment. Taskgraph must be running
450
+ # from a free-threaded Python build to see any performance benefits.
451
+ #
452
+ # In the long term, the goal is turn enabled parallel kind processing for
453
+ # all platforms by default using threads, and remove support for multiple
454
+ # processes altogether.
455
+ def load_tasks():
456
+ if platform.system() == "Linux":
457
+ if os.environ.get("TASKGRAPH_SERIAL"):
458
+ return self._load_tasks_serial(kinds, kind_graph, parameters)
459
+ elif os.environ.get("TASKGRAPH_USE_THREADS"):
460
+ executor = ThreadPoolExecutor(max_workers=os.process_cpu_count())
461
+ else:
462
+ executor = ProcessPoolExecutor(
463
+ mp_context=multiprocessing.get_context("fork")
464
+ )
465
+ return self._load_tasks_parallel(
466
+ kinds, kind_graph, parameters, executor
467
+ )
468
+ else:
469
+ if os.environ.get("TASKGRAPH_SERIAL") or not os.environ.get(
470
+ "TASKGRAPH_USE_THREADS"
471
+ ):
472
+ return self._load_tasks_serial(kinds, kind_graph, parameters)
473
+ else:
474
+ executor = ThreadPoolExecutor(max_workers=os.process_cpu_count())
475
+ return self._load_tasks_parallel(
476
+ kinds, kind_graph, parameters, executor
477
+ )
478
+
479
+ all_tasks = load_tasks()
460
480
 
461
481
  full_task_set = TaskGraph(all_tasks, Graph(frozenset(all_tasks), frozenset()))
462
482
  yield self.verify("full_task_set", full_task_set, graph_config, parameters)
@@ -933,6 +933,12 @@ def load_task(args):
933
933
  @argument(
934
934
  "--verbose", "-v", action="store_true", help="include debug-level logging output"
935
935
  )
936
+ @argument(
937
+ "--allow-parameter-override",
938
+ default=False,
939
+ action="store_true",
940
+ help="Allow user to override computed decision task parameters.",
941
+ )
936
942
  def decision(options):
937
943
  from taskgraph.decision import taskgraph_decision # noqa: PLC0415
938
944
 
@@ -31,7 +31,7 @@ import time
31
31
  import urllib.error
32
32
  import urllib.request
33
33
  from pathlib import Path
34
- from typing import Dict, Optional
34
+ from typing import Dict, List, Optional
35
35
 
36
36
  SECRET_BASEURL_TPL = "{}/secrets/v1/secret/{{}}".format(
37
37
  os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster").rstrip("/")
@@ -640,6 +640,7 @@ def git_checkout(
640
640
  ssh_key_file: Optional[Path],
641
641
  ssh_known_hosts_file: Optional[Path],
642
642
  shallow: bool = False,
643
+ extra_refs: Optional[List[str]] = None,
643
644
  ):
644
645
  assert head_ref or head_rev
645
646
 
@@ -734,6 +735,14 @@ def git_checkout(
734
735
  env=env,
735
736
  )
736
737
 
738
+ if extra_refs:
739
+ git_fetch(
740
+ destination_path,
741
+ *[f"{ref}:{ref}" for ref in extra_refs],
742
+ remote=head_repo,
743
+ env=env,
744
+ )
745
+
737
746
  args = [
738
747
  "git",
739
748
  "checkout",
@@ -932,6 +941,8 @@ def collect_vcs_options(args, project, name):
932
941
  head_rev = os.environ.get(f"{env_prefix}_HEAD_REV")
933
942
  pip_requirements = os.environ.get(f"{env_prefix}_PIP_REQUIREMENTS")
934
943
  private_key_secret = os.environ.get(f"{env_prefix}_SSH_SECRET_NAME")
944
+ if extra_refs := os.environ.get(f"{env_prefix}_EXTRA_REFS"):
945
+ extra_refs = json.loads(extra_refs)
935
946
 
936
947
  store_path = os.environ.get("HG_STORE_PATH")
937
948
 
@@ -965,6 +976,7 @@ def collect_vcs_options(args, project, name):
965
976
  "ssh-secret-name": private_key_secret,
966
977
  "pip-requirements": pip_requirements,
967
978
  "shallow-clone": shallow_clone,
979
+ "extra-refs": extra_refs,
968
980
  }
969
981
 
970
982
 
@@ -1014,6 +1026,7 @@ def vcs_checkout_from_args(options):
1014
1026
  ssh_key_file,
1015
1027
  ssh_known_hosts_file,
1016
1028
  shallow=options.get("shallow-clone", False),
1029
+ extra_refs=options.get("extra-refs"),
1017
1030
  )
1018
1031
  elif options["repo-type"] == "hg":
1019
1032
  revision = hg_checkout(
@@ -1044,8 +1057,8 @@ def install_pip_requirements(repositories):
1044
1057
 
1045
1058
  # TODO: Stop using system Python (#381)
1046
1059
  if shutil.which("uv"):
1047
- user_site_dir = subprocess.run(
1048
- [sys.executable, "-msite", "--user-site"], capture_output=True, text=True
1060
+ user_base_dir = subprocess.run(
1061
+ [sys.executable, "-msite", "--user-base"], capture_output=True, text=True
1049
1062
  ).stdout.strip()
1050
1063
  cmd = [
1051
1064
  "uv",
@@ -1053,8 +1066,8 @@ def install_pip_requirements(repositories):
1053
1066
  "install",
1054
1067
  "--python",
1055
1068
  sys.executable,
1056
- "--target",
1057
- user_site_dir,
1069
+ "--prefix",
1070
+ user_base_dir,
1058
1071
  ]
1059
1072
  else:
1060
1073
  cmd = [sys.executable, "-mpip", "install", "--user", "--break-system-packages"]
@@ -273,9 +273,6 @@ class DockerWorkerPayloadSchema(Schema, forbid_unknown_fields=False, kw_only=Tru
273
273
  taskcluster_proxy: bool
274
274
  allow_ptrace: bool
275
275
  loopback_video: bool
276
- loopback_audio: bool
277
- docker_in_docker: bool # (aka 'dind')
278
- privileged: bool
279
276
  # environment variables
280
277
  env: dict[str, taskref_or_string_msgspec]
281
278
  # the maximum time to run, in seconds
@@ -356,9 +353,6 @@ def build_docker_worker_payload(config, task, task_def):
356
353
  if worker.get("chain-of-trust"):
357
354
  features["chainOfTrust"] = True
358
355
 
359
- if worker.get("docker-in-docker"):
360
- features["dind"] = True
361
-
362
356
  if task.get("needs-sccache"):
363
357
  features["taskclusterProxy"] = True
364
358
  task_def["scopes"].append(
@@ -375,16 +369,10 @@ def build_docker_worker_payload(config, task, task_def):
375
369
 
376
370
  capabilities = {}
377
371
 
378
- for lo in "audio", "video":
379
- if worker.get("loopback-" + lo):
380
- capitalized = "loopback" + lo.capitalize()
381
- devices = capabilities.setdefault("devices", {})
382
- devices[capitalized] = True
383
- task_def["scopes"].append("docker-worker:capability:device:" + capitalized)
384
-
385
- if worker.get("privileged"):
386
- capabilities["privileged"] = True
387
- task_def["scopes"].append("docker-worker:capability:privileged")
372
+ if worker.get("loopback-video"):
373
+ devices = capabilities.setdefault("devices", {})
374
+ devices["loopbackVideo"] = True
375
+ task_def["scopes"].append("docker-worker:capability:device:loopbackVideo")
388
376
 
389
377
  task_def["payload"] = payload = {
390
378
  "image": image,
@@ -569,6 +557,7 @@ class GenericWorkerPayloadSchema(Schema, forbid_unknown_fields=False, kw_only=Tr
569
557
  # feature for task to run as current OS user
570
558
  run_task_as_current_user: Optional[bool] = None
571
559
  taskcluster_proxy: Optional[bool] = None
560
+ hide_cmd_window: Optional[bool] = None
572
561
  # Whether any artifacts are assigned to this worker
573
562
  skip_artifacts: Optional[bool] = None
574
563
 
@@ -681,6 +670,9 @@ def build_generic_worker_payload(config, task, task_def):
681
670
  "generic-worker:run-task-as-current-user:{}".format(task["worker-type"]),
682
671
  )
683
672
 
673
+ if worker.get("hide-cmd-window"):
674
+ features["hideCmdWindow"] = True
675
+
684
676
  if features:
685
677
  task_def["payload"]["features"] = features
686
678
 
@@ -811,9 +803,6 @@ def set_defaults(config, tasks):
811
803
  worker.setdefault("taskcluster-proxy", False)
812
804
  worker.setdefault("allow-ptrace", False)
813
805
  worker.setdefault("loopback-video", False)
814
- worker.setdefault("loopback-audio", False)
815
- worker.setdefault("docker-in-docker", False)
816
- worker.setdefault("privileged", False)
817
806
  worker.setdefault("volumes", [])
818
807
  worker.setdefault("env", {})
819
808
  if "caches" in worker:
@@ -4,9 +4,9 @@
4
4
 
5
5
  import pprint
6
6
  import re
7
+ import threading
7
8
  from collections.abc import Mapping
8
- from functools import reduce
9
- from typing import Any, Literal, Optional, Union
9
+ from typing import Annotated, Any, Literal, Optional, Union, get_args, get_origin
10
10
 
11
11
  import msgspec
12
12
  import voluptuous
@@ -14,6 +14,8 @@ import voluptuous
14
14
  import taskgraph
15
15
  from taskgraph.util.keyed_by import evaluate_keyed_by, iter_dot_path
16
16
 
17
+ _schema_creation_lock = threading.RLock()
18
+
17
19
  # Common type definitions that are used across multiple schemas
18
20
  TaskPriority = Literal[
19
21
  "highest", "very-high", "high", "medium", "low", "very-low", "lowest"
@@ -67,11 +69,37 @@ def validate_schema(schema, obj, msg_prefix):
67
69
  raise Exception(f"{msg_prefix}\n{str(exc)}\n{pprint.pformat(obj)}")
68
70
 
69
71
 
70
- def UnionTypes(*types):
71
- """Use `functools.reduce` to simulate `Union[*allowed_types]` on older
72
- Python versions.
73
- """
74
- return reduce(lambda a, b: Union[a, b], types)
72
+ class OptionallyKeyedBy:
73
+ """Metadata class for optionally_keyed_by fields in msgspec schemas."""
74
+
75
+ def __init__(self, *fields, wrapped_type):
76
+ self.fields = {f"by-{field}" for field in fields}
77
+ self.wrapped_type = wrapped_type
78
+
79
+ def uses_keyed_by(self, obj) -> bool:
80
+ if not isinstance(obj, dict) or len(obj) != 1:
81
+ return False
82
+
83
+ key = list(obj)[0]
84
+ if key not in self.fields:
85
+ return False
86
+
87
+ return True
88
+
89
+ def validate(self, obj) -> None:
90
+ if not self.uses_keyed_by(obj):
91
+ # Not using keyed by, validate directly against wrapped type
92
+ msgspec.convert(obj, self.wrapped_type)
93
+ return
94
+
95
+ # First validate the outer keyed-by dict
96
+ msgspec.convert(obj, dict[str, dict])
97
+
98
+ # Next validate each inner value. We call self.validate recursively to
99
+ # support nested `by-*` keys.
100
+ keyed_by_dict = list(obj.values())[0]
101
+ for value in keyed_by_dict.values():
102
+ self.validate(value)
75
103
 
76
104
 
77
105
  def optionally_keyed_by(*arguments, use_msgspec=False):
@@ -83,13 +111,15 @@ def optionally_keyed_by(*arguments, use_msgspec=False):
83
111
  use_msgspec: If True, return msgspec type hints; if False, return voluptuous validator
84
112
  """
85
113
  if use_msgspec:
86
- # msgspec implementation - return type hints
114
+ # msgspec implementation - use Annotated[Any, OptionallyKeyedBy]
87
115
  _type = arguments[-1]
88
116
  if _type is object:
89
117
  return object
90
118
  fields = arguments[:-1]
91
- bykeys = [Literal[f"by-{field}"] for field in fields]
92
- return Union[_type, dict[UnionTypes(*bykeys), dict[str, Any]]]
119
+ wrapper = OptionallyKeyedBy(*fields, wrapped_type=_type)
120
+ # Annotating Any allows msgspec to accept any value without validation.
121
+ # The actual validation then happens in Schema.__post_init__
122
+ return Annotated[Any, wrapper]
93
123
  else:
94
124
  # voluptuous implementation - return validator function
95
125
  schema = arguments[-1]
@@ -253,28 +283,36 @@ class LegacySchema(voluptuous.Schema):
253
283
  """
254
284
  Operates identically to voluptuous.Schema, but applying some taskgraph-specific checks
255
285
  in the process.
286
+
287
+ voluptuous.Schema's `_compile` method is thread-unsafe. Any usage (whether direct or
288
+ indirect) of it must be protected by a lock.
256
289
  """
257
290
 
258
291
  def __init__(self, *args, check=True, **kwargs):
259
- super().__init__(*args, **kwargs)
292
+ with _schema_creation_lock:
293
+ # this constructor may call `_compile`
294
+ super().__init__(*args, **kwargs)
260
295
 
261
- self.check = check
262
- if not taskgraph.fast and self.check:
263
- check_schema(self)
296
+ self.check = check
297
+ if not taskgraph.fast and self.check:
298
+ check_schema(self)
264
299
 
265
300
  def extend(self, *args, **kwargs):
266
- schema = super().extend(*args, **kwargs)
301
+ with _schema_creation_lock:
302
+ # `extend` may create a new Schema object, which may call `_compile`
303
+ schema = super().extend(*args, **kwargs)
267
304
 
268
- if self.check:
269
- check_schema(schema)
270
- # We want twice extend schema to be checked too.
271
- schema.__class__ = LegacySchema
272
- return schema
305
+ if self.check:
306
+ check_schema(schema)
307
+ # We want twice extend schema to be checked too.
308
+ schema.__class__ = LegacySchema
309
+ return schema
273
310
 
274
311
  def _compile(self, schema):
275
- if taskgraph.fast:
276
- return
277
- return super()._compile(schema)
312
+ with _schema_creation_lock:
313
+ if taskgraph.fast:
314
+ return
315
+ return super()._compile(schema)
278
316
 
279
317
  def __getitem__(self, item):
280
318
  return self.schema[item] # type: ignore
@@ -307,6 +345,31 @@ class Schema(
307
345
  foo: str
308
346
  """
309
347
 
348
+ def __post_init__(self):
349
+ if taskgraph.fast:
350
+ return
351
+
352
+ # Validate fields that use optionally_keyed_by. We need to validate this
353
+ # manually because msgspec doesn't support union types with multiple
354
+ # dicts. Any fields that use `optionally_keyed_by("foo", dict)` would
355
+ # otherwise raise an exception.
356
+ for field_name, field_type in self.__class__.__annotations__.items():
357
+ origin = get_origin(field_type)
358
+ args = get_args(field_type)
359
+
360
+ if (
361
+ origin is not Annotated
362
+ or len(args) < 2
363
+ or not isinstance(args[1], OptionallyKeyedBy)
364
+ ):
365
+ # Not using `optionally_keyed_by`
366
+ continue
367
+
368
+ keyed_by = args[1]
369
+ obj = getattr(self, field_name)
370
+
371
+ keyed_by.validate(obj)
372
+
310
373
  @classmethod
311
374
  def validate(cls, data):
312
375
  """Validate data against this schema."""
@@ -314,7 +377,7 @@ class Schema(
314
377
  return data
315
378
 
316
379
  try:
317
- return msgspec.convert(data, cls)
380
+ msgspec.convert(data, cls)
318
381
  except (msgspec.ValidationError, msgspec.DecodeError) as e:
319
382
  raise msgspec.ValidationError(str(e))
320
383
 
@@ -215,6 +215,14 @@ class Repository(ABC):
215
215
  If this function returns an unexpected value, then make sure
216
216
  the revision was fetched from the remote repository."""
217
217
 
218
+ def get_note(self, note: str, revision: Optional[str] = None) -> Optional[str]:
219
+ """Read a note attached to the given revision (defaults to HEAD).
220
+
221
+ Returns the note content as a string, or ``None`` if no note exists.
222
+ Only supported by Git; returns ``None`` for all other VCS types.
223
+ """
224
+ return None
225
+
218
226
 
219
227
  class HgRepository(Repository):
220
228
  @property
@@ -505,6 +513,11 @@ class GitRepository(Repository):
505
513
  cmd.append("--cached")
506
514
  elif mode == "all":
507
515
  cmd.append("HEAD")
516
+ elif base == self.NULL_REVISION:
517
+ # When base is NULL_REVISION (e.g new branches on Github), diff
518
+ # from the empty tree to show all files present at rev. Use Git's
519
+ # well-known empty tree hash.
520
+ cmd = ["diff", "4b825dc642cb6eb9a060e54bf8d69288fbee4904", rev]
508
521
  elif self.is_shallow:
509
522
  # In shallow clones, `git log` won't have the history necessary to
510
523
  # determine the files changed. Using `git diff` finds the
@@ -581,6 +594,18 @@ class GitRepository(Repository):
581
594
  return False
582
595
  raise
583
596
 
597
+ def get_note(self, note: str, revision: Optional[str] = None) -> Optional[str]:
598
+ if not note.startswith("refs/notes/"):
599
+ note = f"refs/notes/{note}"
600
+
601
+ revision = revision or "HEAD"
602
+ try:
603
+ return self.run("notes", f"--ref={note}", "show", revision).strip()
604
+ except subprocess.CalledProcessError as e:
605
+ if e.returncode == 1:
606
+ return None
607
+ raise
608
+
584
609
 
585
610
  def get_repository(path: str):
586
611
  """Get a repository object for the repository at `path`.