taskcluster-taskgraph 24.4.0__tar.gz → 24.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (273) hide show
  1. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/CHANGELOG.md +33 -0
  2. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/PKG-INFO +1 -1
  3. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/use-fetches.rst +13 -0
  4. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.util.rst +8 -0
  5. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/pyproject.toml +2 -1
  6. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/generator.py +236 -23
  7. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/optimize/base.py +14 -7
  8. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/optimize/strategies.py +10 -2
  9. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/fetch-content +252 -18
  10. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/run-task +3 -0
  11. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/taskgraph.py +10 -8
  12. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/cached_tasks.py +12 -4
  13. taskcluster_taskgraph-24.5.0/src/taskgraph/util/memory.py +24 -0
  14. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/config.yml +1 -1
  15. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/Dockerfile +1 -1
  16. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/yarn.lock +9 -9
  17. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/check/kind.yml +12 -0
  18. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/docker-image/kind.yml +1 -1
  19. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_generator.py +320 -10
  20. taskcluster_taskgraph-24.5.0/test/test_scripts_fetch_content.py +963 -0
  21. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_scripts_run_task.py +93 -0
  22. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_cached_tasks.py +15 -0
  23. taskcluster_taskgraph-24.5.0/test/test_util_memory.py +38 -0
  24. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/uv.lock +1323 -1010
  25. taskcluster_taskgraph-24.4.0/test/test_scripts_fetch_content.py +0 -381
  26. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.codespell-ignore-words.txt +0 -0
  27. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.github/CODEOWNERS +0 -0
  28. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.github/workflows/codeql-analysis.yml +0 -0
  29. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.github/workflows/codspeed.yml +0 -0
  30. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.github/workflows/pypi-publish.yml +0 -0
  31. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.gitignore +0 -0
  32. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.pre-commit-config.yaml +0 -0
  33. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.readthedocs.yaml +0 -0
  34. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.taskcluster.yml +0 -0
  35. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/.yamllint +0 -0
  36. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/CODE_OF_CONDUCT.md +0 -0
  37. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/CONTRIBUTING.rst +0 -0
  38. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/LICENSE +0 -0
  39. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/Makefile +0 -0
  40. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/README.rst +0 -0
  41. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/index.rst +0 -0
  42. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/kind.rst +0 -0
  43. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/loading.rst +0 -0
  44. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/optimization.rst +0 -0
  45. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/overview.rst +0 -0
  46. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/scopes.rst +0 -0
  47. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/task-graphs.rst +0 -0
  48. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/concepts/transforms.rst +0 -0
  49. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/conf.py +0 -0
  50. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/contributing.rst +0 -0
  51. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/glossary.rst +0 -0
  52. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/bootstrap-taskgraph.rst +0 -0
  53. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/create-actions.rst +0 -0
  54. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/create-tasks.rst +0 -0
  55. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/debugging.rst +0 -0
  56. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/define-schemas.rst +0 -0
  57. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/docker.rst +0 -0
  58. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/index.rst +0 -0
  59. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/load-task-locally.rst +0 -0
  60. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/resolve-keyed-by.rst +0 -0
  61. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/run-locally.rst +0 -0
  62. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/howto/send-notifications.rst +0 -0
  63. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/index.rst +0 -0
  64. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/cli.rst +0 -0
  65. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/index.rst +0 -0
  66. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/migrations.rst +0 -0
  67. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/optimization-strategies.rst +0 -0
  68. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/parameters.rst +0 -0
  69. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/modules.rst +0 -0
  70. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.actions.rst +0 -0
  71. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.loader.rst +0 -0
  72. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.optimize.rst +0 -0
  73. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.rst +0 -0
  74. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.transforms.rst +0 -0
  75. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.transforms.run.rst +0 -0
  76. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/chunking.rst +0 -0
  77. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/from_deps.rst +0 -0
  78. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/index.rst +0 -0
  79. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/matrix.rst +0 -0
  80. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/task_context.rst +0 -0
  81. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/tutorials/connecting-taskcluster.rst +0 -0
  82. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/tutorials/creating-a-task-graph.rst +0 -0
  83. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/tutorials/example-taskcluster.yml +0 -0
  84. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/tutorials/getting-started.rst +0 -0
  85. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/docs/tutorials/index.rst +0 -0
  86. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/make.bat +0 -0
  87. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/README.md +0 -0
  88. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/pyproject.toml +0 -0
  89. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/src/pytest_taskgraph/__init__.py +0 -0
  90. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py +0 -0
  91. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/vcs.py +0 -0
  92. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/README.md +0 -0
  93. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/pyproject.toml +0 -0
  94. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/src/sphinx_taskgraph/__init__.py +0 -0
  95. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/src/sphinx_taskgraph/autoschema.py +0 -0
  96. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/renovate.json +0 -0
  97. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/__init__.py +0 -0
  98. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/__init__.py +0 -0
  99. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/add_new_jobs.py +0 -0
  100. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/cancel.py +0 -0
  101. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/cancel_all.py +0 -0
  102. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/rebuild_cached_tasks.py +0 -0
  103. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/registry.py +0 -0
  104. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/retrigger.py +0 -0
  105. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/util.py +0 -0
  106. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/config.py +0 -0
  107. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/create.py +0 -0
  108. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/decision.py +0 -0
  109. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/docker.py +0 -0
  110. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/filter_tasks.py +0 -0
  111. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/graph.py +0 -0
  112. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/loader/__init__.py +0 -0
  113. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/loader/default.py +0 -0
  114. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/loader/transform.py +0 -0
  115. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/main.py +0 -0
  116. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/morph.py +0 -0
  117. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/optimize/__init__.py +0 -0
  118. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/parameters.py +0 -0
  119. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/hgrc +0 -0
  120. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/robustcheckout.py +0 -0
  121. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/target_tasks.py +0 -0
  122. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/task.py +0 -0
  123. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/__init__.py +0 -0
  124. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/base.py +0 -0
  125. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/cached_tasks.py +0 -0
  126. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/chunking.py +0 -0
  127. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/code_review.py +0 -0
  128. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/docker_image.py +0 -0
  129. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/fetch.py +0 -0
  130. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/from_deps.py +0 -0
  131. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/matrix.py +0 -0
  132. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/notify.py +0 -0
  133. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/__init__.py +0 -0
  134. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/common.py +0 -0
  135. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/index_search.py +0 -0
  136. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/run_task.py +0 -0
  137. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/toolchain.py +0 -0
  138. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/task.py +0 -0
  139. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/task_context.py +0 -0
  140. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/__init__.py +0 -0
  141. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/archive.py +0 -0
  142. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/attributes.py +0 -0
  143. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/caches.py +0 -0
  144. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/copy.py +0 -0
  145. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/dependencies.py +0 -0
  146. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/docker.py +0 -0
  147. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/hash.py +0 -0
  148. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/json.py +0 -0
  149. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/keyed_by.py +0 -0
  150. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/parameterization.py +0 -0
  151. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/path.py +0 -0
  152. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/python_path.py +0 -0
  153. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/readonlydict.py +0 -0
  154. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/schema.py +0 -0
  155. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/set_name.py +0 -0
  156. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/shell.py +0 -0
  157. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/task_context.py +0 -0
  158. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/taskcluster.py +0 -0
  159. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/taskgraph.py +0 -0
  160. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/templates.py +0 -0
  161. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/time.py +0 -0
  162. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/treeherder.py +0 -0
  163. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/vcs.py +0 -0
  164. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/verify.py +0 -0
  165. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/workertypes.py +0 -0
  166. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/yaml.py +0 -0
  167. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/REGISTRY +0 -0
  168. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/Dockerfile +0 -0
  169. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/HASH +0 -0
  170. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/README.md +0 -0
  171. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/VERSION +0 -0
  172. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/system-setup.sh +0 -0
  173. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/fetch/Dockerfile +0 -0
  174. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/image_builder/README.rst +0 -0
  175. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/README +0 -0
  176. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/insert-indexes.js +0 -0
  177. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/package.json +0 -0
  178. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/python/Dockerfile +0 -0
  179. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/run-task/Dockerfile +0 -0
  180. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/run-task/system-setup.sh +0 -0
  181. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/skopeo/Dockerfile +0 -0
  182. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/skopeo/policy.json +0 -0
  183. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/skopeo/push_image.sh +0 -0
  184. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/codecov/kind.yml +0 -0
  185. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/complete/kind.yml +0 -0
  186. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/doc/kind.yml +0 -0
  187. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/fetch/kind.yml +0 -0
  188. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/push-image/kind.yml +0 -0
  189. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/test/kind.yml +0 -0
  190. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/test/linux.yml +0 -0
  191. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/test/windows.yml +0 -0
  192. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/scripts/codecov-upload.py +0 -0
  193. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/__init__.py +0 -0
  194. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/custom_parameters.py +0 -0
  195. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/custom_target_tasks.py +0 -0
  196. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/transforms/add_pr_route.py +0 -0
  197. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/transforms/push_image.py +0 -0
  198. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-pull-request-untrusted.yml +0 -0
  199. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-pull-request.yml +0 -0
  200. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-push.yml +0 -0
  201. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-release-pytest-taskgraph.yml +0 -0
  202. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-release-taskcluster-taskgraph.yml +0 -0
  203. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/cookiecutter.json +0 -0
  204. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/hooks/post_gen_project.py +0 -0
  205. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/config.yml +0 -0
  206. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/docker/linux/Dockerfile +0 -0
  207. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/kinds/docker-image/kind.yml +0 -0
  208. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/kinds/hello/kind.yml +0 -0
  209. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/{{cookiecutter.project_slug}}_taskgraph/transforms/hello.py +0 -0
  210. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster.github.yml +0 -0
  211. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/__init__.py +0 -0
  212. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/automationrelevance.json +0 -0
  213. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/conftest.py +0 -0
  214. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/task_context.yml +0 -0
  215. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/config.yml +0 -0
  216. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world/Dockerfile +0 -0
  217. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world-tag/Dockerfile +0 -0
  218. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world-tag/REGISTRY +0 -0
  219. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world-tag/VERSION +0 -0
  220. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/kinds/docker-image/kind.yml +0 -0
  221. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/scripts/toolchain/run.ps1 +0 -0
  222. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/scripts/toolchain/run.sh +0 -0
  223. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/test_taskgraph/transforms/foo.py +0 -0
  224. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/data/testmod/thing.py +0 -0
  225. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/mockedopen.py +0 -0
  226. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_actions_rebuild_cached_tasks.py +0 -0
  227. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_actions_registry.py +0 -0
  228. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_config.py +0 -0
  229. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_create.py +0 -0
  230. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_decision.py +0 -0
  231. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_docker.py +0 -0
  232. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_graph.py +0 -0
  233. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_graph_perf.py +0 -0
  234. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_main.py +0 -0
  235. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_morph.py +0 -0
  236. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_optimize.py +0 -0
  237. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_optimize_strategies.py +0 -0
  238. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_parameters.py +0 -0
  239. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_target_tasks.py +0 -0
  240. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_taskgraph.py +0 -0
  241. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transform_chunking.py +0 -0
  242. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transform_docker_image.py +0 -0
  243. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transform_task_context.py +0 -0
  244. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_base.py +0 -0
  245. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_cached_tasks.py +0 -0
  246. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_fetch.py +0 -0
  247. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_from_deps.py +0 -0
  248. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_matrix.py +0 -0
  249. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_notify.py +0 -0
  250. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_run.py +0 -0
  251. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_run_run_task.py +0 -0
  252. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_run_toolchain.py +0 -0
  253. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_transforms_task.py +0 -0
  254. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_archive.py +0 -0
  255. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_attributes.py +0 -0
  256. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_copy.py +0 -0
  257. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_dependencies.py +0 -0
  258. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_docker.py +0 -0
  259. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_json.py +0 -0
  260. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_keyed_by.py +0 -0
  261. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_parameterization.py +0 -0
  262. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_path.py +0 -0
  263. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_python_path.py +0 -0
  264. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_readonlydict.py +0 -0
  265. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_schema.py +0 -0
  266. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_taskcluster.py +0 -0
  267. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_templates.py +0 -0
  268. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_time.py +0 -0
  269. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_treeherder.py +0 -0
  270. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_vcs.py +0 -0
  271. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_verify.py +0 -0
  272. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_workertypes.py +0 -0
  273. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.5.0}/test/test_util_yaml.py +0 -0
@@ -1,5 +1,37 @@
1
1
  # Change Log
2
2
 
3
+ ## [24.5.0] - 2026-10-21
4
+
5
+ ### Added
6
+
7
+ - `taskgraph.generator.phase_finished` function, which can be replaced by
8
+ consumers and is called as each generation phase ends (#1073)
9
+ - Hook point in `run-task` at startup (#1077)
10
+
11
+ ### Changed
12
+
13
+ - perf: on Linux, graph generation now uses a process per kind instead of a
14
+ process pool, to cut down on IPC overhead
15
+ (https://bugzilla.mozilla.org/show_bug.cgi?id=2079635)
16
+ - perf: pause the garbage collector during graph generation
17
+ (https://bugzilla.mozilla.org/show_bug.cgi?id=2079640)
18
+
19
+ ### Fixed
20
+
21
+ - On Windows, fetch-content now reads tarballs until EOF, to be able to check
22
+ the stream's integrity (#1074)
23
+
24
+ ## [24.4.1] - 2026-10-08
25
+
26
+ ### Fixed
27
+
28
+ - `github-issue-comment` tasks now use the PR cache namespaces
29
+ - perf: look up kind dependency tasks by kind in the generator (#1054)
30
+ - perf: extract tar archives while they download in `fetch-content` (#1068)
31
+ - perf: let tar decompress zstd archives in `fetch-content` (#1068)
32
+ - perf: read downloads in 1MiB chunks in `fetch-content` (#1068)
33
+ - perf: resolve each dependent's deadline only once in `optimize` (#1056)
34
+
3
35
  ## [24.4.0] - 2026-10-05
4
36
 
5
37
  ### Added
@@ -11,6 +43,7 @@
11
43
  ### Changed
12
44
 
13
45
  - `taskgraph init` template and tutorial docs now default to the `public_restricted` PR policy
46
+ - BREAKING CHANGE: `run-task` requires Python 3.8
14
47
 
15
48
  ### Fixed
16
49
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: taskcluster-taskgraph
3
- Version: 24.4.0
3
+ Version: 24.5.0
4
4
  Summary: Build taskcluster taskgraphs
5
5
  Project-URL: Repository, https://github.com/taskcluster/taskgraph
6
6
  Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues
@@ -105,3 +105,16 @@ There are a few differences from the earlier ``build`` examples here:
105
105
 
106
106
  It is not possible to configure the ``dest`` or ``extract`` values when using
107
107
  ``fetch`` or ``toolchain`` kinds.
108
+
109
+ Tuning Download Performance
110
+ ---------------------------
111
+
112
+ The following environment variable tunes how ``fetch-content`` downloads
113
+ artifacts.
114
+
115
+ ``TASKGRAPH_FETCH_STREAM``
116
+ Tar archives that are extracted are extracted while they download, rather
117
+ than written to disk and read back, into a staging directory that is only
118
+ merged into place once the download is complete and verified. This is on
119
+ by default; set to ``0`` to download to a file first. Zip archives, and
120
+ fetches that aren't extracted, always go through a file.
@@ -60,6 +60,14 @@ taskgraph.util.keyed\_by module
60
60
  :undoc-members:
61
61
  :show-inheritance:
62
62
 
63
+ taskgraph.util.memory module
64
+ ----------------------------
65
+
66
+ .. automodule:: taskgraph.util.memory
67
+ :members:
68
+ :undoc-members:
69
+ :show-inheritance:
70
+
63
71
  taskgraph.util.parameterization module
64
72
  --------------------------------------
65
73
 
@@ -1,7 +1,7 @@
1
1
  ### Project
2
2
  [project]
3
3
  name = "taskcluster-taskgraph"
4
- version = "24.4.0"
4
+ version = "24.5.0"
5
5
  description = "Build taskcluster taskgraphs"
6
6
  readme = "README.rst"
7
7
  authors = [
@@ -52,6 +52,7 @@ dev = [
52
52
  "sphinx-book-theme >=1.1.4",
53
53
  "sphinx-taskgraph>=0.1.0",
54
54
  "sphinxcontrib-mermaid>=1.0.0",
55
+ "vermin>=1.8.0",
55
56
  "zstandard>=0.25.0",
56
57
  ]
57
58
 
@@ -6,16 +6,21 @@ import copy
6
6
  import inspect
7
7
  import logging
8
8
  import multiprocessing
9
+ import multiprocessing.connection
9
10
  import os
11
+ import pickle
10
12
  import platform
13
+ import signal
14
+ import sys
15
+ import time
16
+ import traceback
11
17
  from concurrent.futures import (
12
18
  FIRST_COMPLETED,
13
- ProcessPoolExecutor,
14
19
  ThreadPoolExecutor,
15
20
  wait,
16
21
  )
17
22
  from dataclasses import dataclass
18
- from typing import Callable, Optional, Union
23
+ from typing import Callable, Optional, Union, cast
19
24
 
20
25
  from . import filter_tasks
21
26
  from .config import GraphConfig, load_graph_config
@@ -26,6 +31,7 @@ from .parameters import Parameters, parameters_loader
26
31
  from .task import Task
27
32
  from .taskgraph import TaskGraph
28
33
  from .transforms.base import TransformConfig, TransformSequence
34
+ from .util.memory import gc_disabled
29
35
  from .util.python_path import find_object
30
36
  from .util.schema import SchemaValidationError
31
37
  from .util.verify import verifications
@@ -34,12 +40,31 @@ from .util.yaml import load_yaml
34
40
  logger = logging.getLogger(__name__)
35
41
 
36
42
 
43
+ def phase_finished(name, start, end):
44
+ """Called after each generation phase, with `time.monotonic()` values
45
+ taken around it. `name` is None if the phase raised an exception.
46
+
47
+ Projects can replace this from their `register` function; a replacement
48
+ made then also applies to the phase that is running.
49
+ """
50
+ if name is None:
51
+ logger.debug(f"Generation phase failed after {end - start:.2f}s")
52
+ else:
53
+ logger.debug(f"Generated {name} in {end - start:.2f}s")
54
+
55
+
37
56
  class KindNotFound(Exception):
38
57
  """
39
58
  Raised when trying to load kind from a directory without a kind.yml.
40
59
  """
41
60
 
42
61
 
62
+ def _max_workers():
63
+ # os.process_cpu_count is new in Python 3.13.
64
+ cpu_count = getattr(os, "process_cpu_count", os.cpu_count)
65
+ return cpu_count() or 1
66
+
67
+
43
68
  @dataclass(frozen=True)
44
69
  class Kind:
45
70
  name: str
@@ -124,6 +149,16 @@ class Kind:
124
149
  return cls(kind_name, path, config, graph_config)
125
150
 
126
151
 
152
+ class _RemoteTraceback(Exception):
153
+ """The traceback of an exception raised in a child process."""
154
+
155
+ def __init__(self, tb):
156
+ self.tb = tb
157
+
158
+ def __str__(self):
159
+ return f'\n"""\n{self.tb}"""'
160
+
161
+
127
162
  class TaskGraphGenerator:
128
163
  """
129
164
  The central controller for taskgraph. This handles all phases of graph
@@ -282,8 +317,21 @@ class TaskGraphGenerator:
282
317
  except KindNotFound:
283
318
  continue
284
319
 
320
+ @staticmethod
321
+ def _get_kind_dependencies_tasks(kind, tasks_by_kind):
322
+ """Return the tasks of the kinds `kind` depends on, in the order in
323
+ which they were loaded."""
324
+ kind_dependencies = kind.config.get("kind-dependencies", [])
325
+ return {
326
+ label: task
327
+ for kind_name, tasks in tasks_by_kind.items()
328
+ if kind_name in kind_dependencies
329
+ for label, task in tasks.items()
330
+ }
331
+
285
332
  def _load_tasks_serial(self, kinds, kind_graph, parameters):
286
333
  all_tasks = {}
334
+ tasks_by_kind = {}
287
335
  for kind_name in kind_graph.visit_postorder():
288
336
  logger.debug(f"Loading tasks for kind {kind_name}")
289
337
 
@@ -297,11 +345,7 @@ class TaskGraphGenerator:
297
345
  try:
298
346
  new_tasks = kind.load_tasks(
299
347
  parameters,
300
- {
301
- k: t
302
- for k, t in all_tasks.items()
303
- if t.kind in kind.config.get("kind-dependencies", [])
304
- },
348
+ self._get_kind_dependencies_tasks(kind, tasks_by_kind),
305
349
  self._write_artifacts,
306
350
  )
307
351
  except SchemaValidationError as exc:
@@ -310,15 +354,18 @@ class TaskGraphGenerator:
310
354
  except Exception:
311
355
  logger.exception(f"Error loading tasks for kind {kind_name}:")
312
356
  raise
357
+ kind_tasks = tasks_by_kind.setdefault(kind_name, {})
313
358
  for task in new_tasks:
314
359
  if task.label in all_tasks:
315
360
  raise Exception("duplicate tasks with label " + task.label)
316
361
  all_tasks[task.label] = task
362
+ kind_tasks[task.label] = task
317
363
 
318
364
  return all_tasks
319
365
 
320
366
  def _load_tasks_parallel(self, kinds, kind_graph, parameters, executor):
321
367
  all_tasks = {}
368
+ tasks_by_kind = {}
322
369
  futures_to_kind = {}
323
370
  futures = set()
324
371
  edges = set(kind_graph.edges)
@@ -328,7 +375,6 @@ class TaskGraphGenerator:
328
375
  def submit_ready_kinds():
329
376
  """Create the next batch of tasks for kinds without dependencies."""
330
377
  nonlocal kinds, edges, futures
331
- loaded_tasks = all_tasks.copy()
332
378
  kinds_with_deps = {edge[0] for edge in edges}
333
379
  ready_kinds = (
334
380
  set(kinds) - kinds_with_deps - set(futures_to_kind.values())
@@ -346,11 +392,7 @@ class TaskGraphGenerator:
346
392
  future = executor.submit(
347
393
  kind.load_tasks,
348
394
  dict(parameters),
349
- {
350
- k: t
351
- for k, t in loaded_tasks.items()
352
- if t.kind in kind.config.get("kind-dependencies", [])
353
- },
395
+ self._get_kind_dependencies_tasks(kind, tasks_by_kind),
354
396
  self._write_artifacts,
355
397
  )
356
398
  futures.add(future)
@@ -376,10 +418,12 @@ class TaskGraphGenerator:
376
418
  kind = futures_to_kind.pop(future)
377
419
  futures.remove(future)
378
420
 
421
+ kind_tasks = tasks_by_kind.setdefault(kind, {})
379
422
  for task in future.result():
380
423
  if task.label in all_tasks:
381
424
  raise Exception("duplicate tasks with label " + task.label)
382
425
  all_tasks[task.label] = task
426
+ kind_tasks[task.label] = task
383
427
 
384
428
  # Update state for next batch of futures.
385
429
  del kinds[kind]
@@ -390,6 +434,157 @@ class TaskGraphGenerator:
390
434
 
391
435
  return all_tasks
392
436
 
437
+ def _load_tasks_forked(self, kinds, kind_graph, parameters):
438
+ """Load each kind in a child process forked once the kinds it depends
439
+ on are loaded.
440
+
441
+ The child shares every task loaded so far with the parent, so the
442
+ tasks of the kinds it depends on don't need to be sent to it, and it
443
+ sends its own tasks back through a pipe of its own.
444
+ """
445
+ all_tasks = {}
446
+ tasks_by_kind = {}
447
+ # Pipe to each running child -> (kind name, pid)
448
+ running = {}
449
+ remaining = kinds.copy()
450
+ edges = set(kind_graph.edges)
451
+ max_workers = _max_workers()
452
+
453
+ try:
454
+ while remaining or running:
455
+ kinds_with_deps = {edge[0] for edge in edges}
456
+ ready_kinds = sorted(set(remaining) - kinds_with_deps)
457
+ for name in ready_kinds[: max_workers - len(running)]:
458
+ kind = remaining.pop(name)
459
+ reader, pid = self._fork_kind(
460
+ kind,
461
+ parameters,
462
+ self._get_kind_dependencies_tasks(kind, tasks_by_kind),
463
+ )
464
+ running[reader] = (name, pid)
465
+
466
+ # Waiting on nothing would block forever. _run made sure there
467
+ # is no dependency loop, so some kind is always running here.
468
+ assert running
469
+ for ready in multiprocessing.connection.wait(list(running)):
470
+ # wait returns the objects it was given, but is typed as
471
+ # also returning sockets and file descriptors.
472
+ reader = cast(multiprocessing.connection.Connection, ready)
473
+ name, pid = running[reader]
474
+ try:
475
+ result = reader.recv()
476
+ except EOFError:
477
+ # The child exited without sending anything.
478
+ result = None
479
+ del running[reader]
480
+ reader.close()
481
+ _, status = os.waitpid(pid, 0)
482
+ new_tasks = self._kind_result(name, result, status)
483
+
484
+ kind_tasks = tasks_by_kind.setdefault(name, {})
485
+ for task in new_tasks:
486
+ if task.label in all_tasks:
487
+ raise Exception("duplicate tasks with label " + task.label)
488
+ all_tasks[task.label] = task
489
+ kind_tasks[task.label] = task
490
+
491
+ edges = {e for e in edges if e[1] != name}
492
+ finally:
493
+ # Only left with running children on error.
494
+ for reader, (_, pid) in running.items():
495
+ reader.close()
496
+ os.kill(pid, signal.SIGKILL)
497
+ os.waitpid(pid, 0)
498
+
499
+ return all_tasks
500
+
501
+ def _fork_kind(self, kind, parameters, kind_dependencies_tasks):
502
+ """Fork a child process loading the tasks of `kind`, and return the
503
+ pipe its result comes back through, and its pid."""
504
+ reader, writer = multiprocessing.Pipe(duplex=False)
505
+ # The child inherits the disabled garbage collector, so it doesn't
506
+ # write to every page it shares with us traversing the objects in them.
507
+ with gc_disabled():
508
+ pid = os.fork()
509
+ if pid == 0:
510
+ # Never leave this block, or return into the caller's code, in
511
+ # the child.
512
+ status = 1
513
+ try:
514
+ reader.close()
515
+ status = self._load_kind_in_child(
516
+ kind, parameters, kind_dependencies_tasks, writer
517
+ )
518
+ finally:
519
+ try:
520
+ sys.stdout.flush()
521
+ sys.stderr.flush()
522
+ finally:
523
+ os._exit(status)
524
+ writer.close()
525
+ return reader, pid
526
+
527
+ def _load_kind_in_child(self, kind, parameters, kind_dependencies_tasks, writer):
528
+ """Load the tasks of `kind` and send them, or the exception loading
529
+ them raised, through `writer`. Return the child's exit status."""
530
+ try:
531
+ tasks = kind.load_tasks(
532
+ parameters, kind_dependencies_tasks, self._write_artifacts
533
+ )
534
+ except BaseException as e:
535
+ tb = traceback.format_exc()
536
+ try:
537
+ # Make sure the parent will be able to unpickle it.
538
+ pickle.loads(pickle.dumps(e))
539
+ except Exception:
540
+ e = Exception(f"{type(e).__name__}: {e}")
541
+ writer.send((e, tb, None))
542
+ return 1
543
+
544
+ try:
545
+ writer.send((None, None, tasks))
546
+ except Exception as e:
547
+ writer.send(
548
+ (
549
+ Exception(
550
+ f"Could not send the tasks of kind {kind.name} "
551
+ f"to the parent process: {type(e).__name__}: {e}"
552
+ ),
553
+ traceback.format_exc(),
554
+ None,
555
+ )
556
+ )
557
+ return 1
558
+ return 0
559
+
560
+ @staticmethod
561
+ def _kind_result(name, result, status):
562
+ """Return the tasks the child loading kind `name` sent, given what it
563
+ sent and its exit status. Raise the exception loading them raised in
564
+ the child, if any."""
565
+ if result is None:
566
+ code = os.waitstatus_to_exitcode(status)
567
+ if code < 0:
568
+ how = f"was killed by signal {signal.Signals(-code).name}"
569
+ else:
570
+ how = f"exited with status {code}"
571
+ raise Exception(
572
+ f"Process loading tasks for kind {name} {how} without sending them"
573
+ )
574
+
575
+ exc, tb, tasks = result
576
+ if exc is not None:
577
+ exc.__cause__ = _RemoteTraceback(tb)
578
+ if isinstance(exc, SchemaValidationError):
579
+ logger.error(f"Error loading tasks for kind {name}:\n{exc}")
580
+ else:
581
+ logger.error(
582
+ f"Error loading tasks for kind {name}:",
583
+ exc_info=(type(exc), exc, exc.__traceback__),
584
+ )
585
+ raise exc
586
+ return tasks
587
+
393
588
  def _run(self):
394
589
  logger.info("Loading graph configuration.")
395
590
  graph_config = load_graph_config(self.root_dir)
@@ -437,6 +632,12 @@ class TaskGraphGenerator:
437
632
  for kind in kinds.values():
438
633
  for dep in kind.config.get("kind-dependencies", []):
439
634
  edges.add((kind.name, dep, "kind-dependency"))
635
+ for _, dep, _ in edges:
636
+ if dep not in kinds:
637
+ message = f'Could not find the kind "{dep}"\nAvailable kinds:\n'
638
+ for k in sorted(kinds):
639
+ message += f' - "{k}"\n'
640
+ raise Exception(message)
440
641
  kind_graph = Graph(frozenset(kinds), frozenset(edges))
441
642
 
442
643
  if target_kinds:
@@ -444,6 +645,11 @@ class TaskGraphGenerator:
444
645
  set(target_kinds) | {"docker-image"}
445
646
  )
446
647
 
648
+ # The task loaders wait for each kind's dependencies to be loaded
649
+ # before loading it, so a dependency loop would hang them or silently
650
+ # leave kinds out. This raises if there is one.
651
+ kind_graph.visit_postorder()
652
+
447
653
  yield "kind_graph", kind_graph
448
654
 
449
655
  logger.info("Generating full task set")
@@ -451,7 +657,7 @@ class TaskGraphGenerator:
451
657
  # The next block deals with enabling parallel kind processing, which
452
658
  # currently has different support on different platforms. In summary:
453
659
  # * Parallel kind processing is supported and enabled by default on
454
- # Linux. We use multiple processes by default, but experimental
660
+ # Linux. We fork a process per kind by default, but experimental
455
661
  # support for multiple threads can be enabled instead.
456
662
  # * On other platforms, we have experimental support for parallel
457
663
  # kind processing with multiple threads.
@@ -471,21 +677,19 @@ class TaskGraphGenerator:
471
677
  if os.environ.get("TASKGRAPH_SERIAL"):
472
678
  return self._load_tasks_serial(kinds, kind_graph, parameters)
473
679
  elif os.environ.get("TASKGRAPH_USE_THREADS"):
474
- executor = ThreadPoolExecutor(max_workers=os.process_cpu_count())
475
- else:
476
- executor = ProcessPoolExecutor(
477
- mp_context=multiprocessing.get_context("fork")
680
+ executor = ThreadPoolExecutor(max_workers=_max_workers())
681
+ return self._load_tasks_parallel(
682
+ kinds, kind_graph, parameters, executor
478
683
  )
479
- return self._load_tasks_parallel(
480
- kinds, kind_graph, parameters, executor
481
- )
684
+ else:
685
+ return self._load_tasks_forked(kinds, kind_graph, parameters)
482
686
  else:
483
687
  if os.environ.get("TASKGRAPH_SERIAL") or not os.environ.get(
484
688
  "TASKGRAPH_USE_THREADS"
485
689
  ):
486
690
  return self._load_tasks_serial(kinds, kind_graph, parameters)
487
691
  else:
488
- executor = ThreadPoolExecutor(max_workers=os.process_cpu_count())
692
+ executor = ThreadPoolExecutor(max_workers=_max_workers())
489
693
  return self._load_tasks_parallel(
490
694
  kinds, kind_graph, parameters, executor
491
695
  )
@@ -594,10 +798,19 @@ class TaskGraphGenerator:
594
798
 
595
799
  def _run_until(self, name):
596
800
  while name not in self._run_results:
801
+ start = time.monotonic()
597
802
  try:
598
- k, v = next(self._run) # type: ignore
803
+ # Generation creates millions of objects that stay alive, so
804
+ # collecting garbage meanwhile only traverses them for nothing.
805
+ # The callers' code runs with the collector in its usual state.
806
+ with gc_disabled():
807
+ k, v = next(self._run) # type: ignore
599
808
  except StopIteration:
600
809
  raise AttributeError(f"No such run result {name}")
810
+ except BaseException:
811
+ phase_finished(None, start, time.monotonic())
812
+ raise
813
+ phase_finished(k, start, time.monotonic())
601
814
  self._run_results[k] = v
602
815
  return self._run_results[name]
603
816
 
@@ -301,6 +301,18 @@ def replace_tasks(
301
301
  target_task_graph.graph.links_and_reverse_links_dict()
302
302
  )
303
303
 
304
+ # Many tasks share dependents (e.g. docker images or toolchains), so
305
+ # resolve the deadline of each dependent only once.
306
+ now = datetime.datetime.now(datetime.timezone.utc)
307
+ deadlines = {}
308
+
309
+ def get_deadline(label):
310
+ if label not in deadlines:
311
+ deadlines[label] = resolve_timestamps(
312
+ now, target_task_graph.tasks[label].task["deadline"]
313
+ )
314
+ return deadlines[label]
315
+
304
316
  for label in target_task_graph.graph.visit_postorder():
305
317
  logger.debug(f"replace_tasks: {label}")
306
318
  # if we're not allowed to optimize, that's easy..
@@ -331,14 +343,9 @@ def replace_tasks(
331
343
  opt_by, opt, arg = optimizations(label)
332
344
 
333
345
  # compute latest deadline of dependents (if any)
334
- dependents = [target_task_graph.tasks[l] for l in dependents_of[label]]
335
346
  deadline = None
336
- if dependents:
337
- now = datetime.datetime.now(datetime.timezone.utc)
338
- deadline = max(
339
- resolve_timestamps(now, task.task["deadline"])
340
- for task in dependents # type: ignore
341
- )
347
+ if dependents_of[label]:
348
+ deadline = max(get_deadline(l) for l in dependents_of[label])
342
349
 
343
350
  if isinstance(opt, IndexSearch):
344
351
  arg = arg, index_to_taskid, taskid_to_status
@@ -1,3 +1,4 @@
1
+ import functools
1
2
  import logging
2
3
  from datetime import datetime
3
4
 
@@ -10,6 +11,13 @@ from taskgraph.util.taskcluster import find_task_id, status_task
10
11
  logger = logging.getLogger("optimization")
11
12
 
12
13
 
14
+ @functools.cache
15
+ def _parse_time(timestamp, fmt):
16
+ # Many tasks share the same deadline or replacement task, so avoid parsing
17
+ # the same timestamps over and over.
18
+ return datetime.strptime(timestamp, fmt)
19
+
20
+
13
21
  @register_strategy("index-search")
14
22
  class IndexSearch(OptimizationStrategy):
15
23
  # A task with no dependencies remaining after optimization will be replaced
@@ -56,10 +64,10 @@ class IndexSearch(OptimizationStrategy):
56
64
  )
57
65
  continue
58
66
 
59
- if deadline and datetime.strptime(
67
+ if deadline and _parse_time(
60
68
  status["expires"], # type: ignore
61
69
  self.fmt,
62
- ) < datetime.strptime(deadline, self.fmt):
70
+ ) < _parse_time(deadline, self.fmt):
63
71
  logger.debug(
64
72
  f"not replacing {task.label} with {task_id} because it expires before {deadline}"
65
73
  )