taskcluster-taskgraph 24.4.1__tar.gz → 24.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (272) hide show
  1. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/CHANGELOG.md +21 -0
  2. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/PKG-INFO +1 -1
  3. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.util.rst +8 -0
  4. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/pyproject.toml +1 -1
  5. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/generator.py +216 -12
  6. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/fetch-content +4 -0
  7. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/run-task +3 -0
  8. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/taskgraph.py +10 -8
  9. taskcluster_taskgraph-24.5.0/src/taskgraph/util/memory.py +24 -0
  10. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_generator.py +320 -10
  11. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_scripts_run_task.py +93 -0
  12. taskcluster_taskgraph-24.5.0/test/test_util_memory.py +38 -0
  13. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/uv.lock +2 -2
  14. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.codespell-ignore-words.txt +0 -0
  15. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.github/CODEOWNERS +0 -0
  16. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.github/workflows/codeql-analysis.yml +0 -0
  17. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.github/workflows/codspeed.yml +0 -0
  18. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.github/workflows/pypi-publish.yml +0 -0
  19. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.gitignore +0 -0
  20. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.pre-commit-config.yaml +0 -0
  21. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.readthedocs.yaml +0 -0
  22. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.taskcluster.yml +0 -0
  23. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/.yamllint +0 -0
  24. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/CODE_OF_CONDUCT.md +0 -0
  25. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/CONTRIBUTING.rst +0 -0
  26. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/LICENSE +0 -0
  27. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/Makefile +0 -0
  28. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/README.rst +0 -0
  29. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/index.rst +0 -0
  30. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/kind.rst +0 -0
  31. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/loading.rst +0 -0
  32. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/optimization.rst +0 -0
  33. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/overview.rst +0 -0
  34. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/scopes.rst +0 -0
  35. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/task-graphs.rst +0 -0
  36. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/concepts/transforms.rst +0 -0
  37. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/conf.py +0 -0
  38. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/contributing.rst +0 -0
  39. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/glossary.rst +0 -0
  40. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/bootstrap-taskgraph.rst +0 -0
  41. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/create-actions.rst +0 -0
  42. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/create-tasks.rst +0 -0
  43. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/debugging.rst +0 -0
  44. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/define-schemas.rst +0 -0
  45. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/docker.rst +0 -0
  46. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/index.rst +0 -0
  47. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/load-task-locally.rst +0 -0
  48. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/resolve-keyed-by.rst +0 -0
  49. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/run-locally.rst +0 -0
  50. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/send-notifications.rst +0 -0
  51. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/howto/use-fetches.rst +0 -0
  52. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/index.rst +0 -0
  53. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/cli.rst +0 -0
  54. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/index.rst +0 -0
  55. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/migrations.rst +0 -0
  56. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/optimization-strategies.rst +0 -0
  57. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/parameters.rst +0 -0
  58. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/modules.rst +0 -0
  59. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.actions.rst +0 -0
  60. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.loader.rst +0 -0
  61. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.optimize.rst +0 -0
  62. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.rst +0 -0
  63. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.transforms.rst +0 -0
  64. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/source/taskgraph.transforms.run.rst +0 -0
  65. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/chunking.rst +0 -0
  66. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/from_deps.rst +0 -0
  67. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/index.rst +0 -0
  68. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/matrix.rst +0 -0
  69. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/reference/transforms/task_context.rst +0 -0
  70. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/tutorials/connecting-taskcluster.rst +0 -0
  71. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/tutorials/creating-a-task-graph.rst +0 -0
  72. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/tutorials/example-taskcluster.yml +0 -0
  73. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/tutorials/getting-started.rst +0 -0
  74. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/docs/tutorials/index.rst +0 -0
  75. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/make.bat +0 -0
  76. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/README.md +0 -0
  77. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/pyproject.toml +0 -0
  78. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/src/pytest_taskgraph/__init__.py +0 -0
  79. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py +0 -0
  80. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/vcs.py +0 -0
  81. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/README.md +0 -0
  82. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/pyproject.toml +0 -0
  83. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/src/sphinx_taskgraph/__init__.py +0 -0
  84. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/packages/sphinx-taskgraph/src/sphinx_taskgraph/autoschema.py +0 -0
  85. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/renovate.json +0 -0
  86. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/__init__.py +0 -0
  87. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/__init__.py +0 -0
  88. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/add_new_jobs.py +0 -0
  89. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/cancel.py +0 -0
  90. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/cancel_all.py +0 -0
  91. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/rebuild_cached_tasks.py +0 -0
  92. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/registry.py +0 -0
  93. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/retrigger.py +0 -0
  94. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/actions/util.py +0 -0
  95. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/config.py +0 -0
  96. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/create.py +0 -0
  97. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/decision.py +0 -0
  98. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/docker.py +0 -0
  99. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/filter_tasks.py +0 -0
  100. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/graph.py +0 -0
  101. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/loader/__init__.py +0 -0
  102. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/loader/default.py +0 -0
  103. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/loader/transform.py +0 -0
  104. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/main.py +0 -0
  105. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/morph.py +0 -0
  106. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/optimize/__init__.py +0 -0
  107. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/optimize/base.py +0 -0
  108. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/optimize/strategies.py +0 -0
  109. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/parameters.py +0 -0
  110. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/hgrc +0 -0
  111. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/run-task/robustcheckout.py +0 -0
  112. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/target_tasks.py +0 -0
  113. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/task.py +0 -0
  114. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/__init__.py +0 -0
  115. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/base.py +0 -0
  116. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/cached_tasks.py +0 -0
  117. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/chunking.py +0 -0
  118. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/code_review.py +0 -0
  119. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/docker_image.py +0 -0
  120. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/fetch.py +0 -0
  121. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/from_deps.py +0 -0
  122. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/matrix.py +0 -0
  123. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/notify.py +0 -0
  124. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/__init__.py +0 -0
  125. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/common.py +0 -0
  126. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/index_search.py +0 -0
  127. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/run_task.py +0 -0
  128. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/run/toolchain.py +0 -0
  129. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/task.py +0 -0
  130. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/transforms/task_context.py +0 -0
  131. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/__init__.py +0 -0
  132. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/archive.py +0 -0
  133. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/attributes.py +0 -0
  134. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/cached_tasks.py +0 -0
  135. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/caches.py +0 -0
  136. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/copy.py +0 -0
  137. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/dependencies.py +0 -0
  138. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/docker.py +0 -0
  139. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/hash.py +0 -0
  140. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/json.py +0 -0
  141. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/keyed_by.py +0 -0
  142. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/parameterization.py +0 -0
  143. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/path.py +0 -0
  144. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/python_path.py +0 -0
  145. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/readonlydict.py +0 -0
  146. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/schema.py +0 -0
  147. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/set_name.py +0 -0
  148. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/shell.py +0 -0
  149. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/task_context.py +0 -0
  150. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/taskcluster.py +0 -0
  151. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/taskgraph.py +0 -0
  152. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/templates.py +0 -0
  153. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/time.py +0 -0
  154. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/treeherder.py +0 -0
  155. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/vcs.py +0 -0
  156. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/verify.py +0 -0
  157. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/workertypes.py +0 -0
  158. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/src/taskgraph/util/yaml.py +0 -0
  159. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/config.yml +0 -0
  160. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/REGISTRY +0 -0
  161. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/Dockerfile +0 -0
  162. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/HASH +0 -0
  163. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/README.md +0 -0
  164. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/VERSION +0 -0
  165. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/decision/system-setup.sh +0 -0
  166. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/fetch/Dockerfile +0 -0
  167. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/image_builder/README.rst +0 -0
  168. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/Dockerfile +0 -0
  169. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/README +0 -0
  170. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/insert-indexes.js +0 -0
  171. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/package.json +0 -0
  172. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/index-task/yarn.lock +0 -0
  173. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/python/Dockerfile +0 -0
  174. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/run-task/Dockerfile +0 -0
  175. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/run-task/system-setup.sh +0 -0
  176. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/skopeo/Dockerfile +0 -0
  177. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/skopeo/policy.json +0 -0
  178. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/docker/skopeo/push_image.sh +0 -0
  179. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/check/kind.yml +0 -0
  180. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/codecov/kind.yml +0 -0
  181. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/complete/kind.yml +0 -0
  182. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/doc/kind.yml +0 -0
  183. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/docker-image/kind.yml +0 -0
  184. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/fetch/kind.yml +0 -0
  185. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/push-image/kind.yml +0 -0
  186. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/test/kind.yml +0 -0
  187. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/test/linux.yml +0 -0
  188. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/kinds/test/windows.yml +0 -0
  189. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/scripts/codecov-upload.py +0 -0
  190. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/__init__.py +0 -0
  191. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/custom_parameters.py +0 -0
  192. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/custom_target_tasks.py +0 -0
  193. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/transforms/add_pr_route.py +0 -0
  194. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/self_taskgraph/transforms/push_image.py +0 -0
  195. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-pull-request-untrusted.yml +0 -0
  196. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-pull-request.yml +0 -0
  197. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-push.yml +0 -0
  198. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-release-pytest-taskgraph.yml +0 -0
  199. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/taskcluster/test/params/main-repo-release-taskcluster-taskgraph.yml +0 -0
  200. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/cookiecutter.json +0 -0
  201. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/hooks/post_gen_project.py +0 -0
  202. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/config.yml +0 -0
  203. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/docker/linux/Dockerfile +0 -0
  204. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/kinds/docker-image/kind.yml +0 -0
  205. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/kinds/hello/kind.yml +0 -0
  206. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster/{{cookiecutter.project_slug}}_taskgraph/transforms/hello.py +0 -0
  207. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/template/{{cookiecutter.project_name}}/taskcluster.github.yml +0 -0
  208. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/__init__.py +0 -0
  209. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/automationrelevance.json +0 -0
  210. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/conftest.py +0 -0
  211. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/task_context.yml +0 -0
  212. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/config.yml +0 -0
  213. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world/Dockerfile +0 -0
  214. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world-tag/Dockerfile +0 -0
  215. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world-tag/REGISTRY +0 -0
  216. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/docker/hello-world-tag/VERSION +0 -0
  217. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/kinds/docker-image/kind.yml +0 -0
  218. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/scripts/toolchain/run.ps1 +0 -0
  219. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/scripts/toolchain/run.sh +0 -0
  220. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/taskcluster/test_taskgraph/transforms/foo.py +0 -0
  221. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/data/testmod/thing.py +0 -0
  222. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/mockedopen.py +0 -0
  223. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_actions_rebuild_cached_tasks.py +0 -0
  224. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_actions_registry.py +0 -0
  225. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_config.py +0 -0
  226. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_create.py +0 -0
  227. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_decision.py +0 -0
  228. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_docker.py +0 -0
  229. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_graph.py +0 -0
  230. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_graph_perf.py +0 -0
  231. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_main.py +0 -0
  232. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_morph.py +0 -0
  233. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_optimize.py +0 -0
  234. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_optimize_strategies.py +0 -0
  235. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_parameters.py +0 -0
  236. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_scripts_fetch_content.py +0 -0
  237. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_target_tasks.py +0 -0
  238. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_taskgraph.py +0 -0
  239. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transform_chunking.py +0 -0
  240. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transform_docker_image.py +0 -0
  241. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transform_task_context.py +0 -0
  242. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_base.py +0 -0
  243. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_cached_tasks.py +0 -0
  244. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_fetch.py +0 -0
  245. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_from_deps.py +0 -0
  246. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_matrix.py +0 -0
  247. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_notify.py +0 -0
  248. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_run.py +0 -0
  249. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_run_run_task.py +0 -0
  250. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_run_toolchain.py +0 -0
  251. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_transforms_task.py +0 -0
  252. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_archive.py +0 -0
  253. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_attributes.py +0 -0
  254. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_cached_tasks.py +0 -0
  255. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_copy.py +0 -0
  256. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_dependencies.py +0 -0
  257. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_docker.py +0 -0
  258. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_json.py +0 -0
  259. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_keyed_by.py +0 -0
  260. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_parameterization.py +0 -0
  261. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_path.py +0 -0
  262. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_python_path.py +0 -0
  263. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_readonlydict.py +0 -0
  264. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_schema.py +0 -0
  265. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_taskcluster.py +0 -0
  266. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_templates.py +0 -0
  267. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_time.py +0 -0
  268. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_treeherder.py +0 -0
  269. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_vcs.py +0 -0
  270. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_verify.py +0 -0
  271. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_workertypes.py +0 -0
  272. {taskcluster_taskgraph-24.4.1 → taskcluster_taskgraph-24.5.0}/test/test_util_yaml.py +0 -0
@@ -1,5 +1,26 @@
1
1
  # Change Log
2
2
 
3
+ ## [24.5.0] - 2026-10-21
4
+
5
+ ### Added
6
+
7
+ - `taskgraph.generator.phase_finished` function, which can be replaced by
8
+ consumers and is called as each generation phase ends (#1073)
9
+ - Hook point in `run-task` at startup (#1077)
10
+
11
+ ### Changed
12
+
13
+ - perf: on Linux, graph generation now uses a process per kind instead of a
14
+ process pool, to cut down on IPC overhead
15
+ (https://bugzilla.mozilla.org/show_bug.cgi?id=2079635)
16
+ - perf: pause the garbage collector during graph generation
17
+ (https://bugzilla.mozilla.org/show_bug.cgi?id=2079640)
18
+
19
+ ### Fixed
20
+
21
+ - On Windows, fetch-content now reads tarballs until EOF, to be able to check
22
+ the stream's integrity (#1074)
23
+
3
24
  ## [24.4.1] - 2026-10-08
4
25
 
5
26
  ### Fixed
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: taskcluster-taskgraph
3
- Version: 24.4.1
3
+ Version: 24.5.0
4
4
  Summary: Build taskcluster taskgraphs
5
5
  Project-URL: Repository, https://github.com/taskcluster/taskgraph
6
6
  Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues
@@ -60,6 +60,14 @@ taskgraph.util.keyed\_by module
60
60
  :undoc-members:
61
61
  :show-inheritance:
62
62
 
63
+ taskgraph.util.memory module
64
+ ----------------------------
65
+
66
+ .. automodule:: taskgraph.util.memory
67
+ :members:
68
+ :undoc-members:
69
+ :show-inheritance:
70
+
63
71
  taskgraph.util.parameterization module
64
72
  --------------------------------------
65
73
 
@@ -1,7 +1,7 @@
1
1
  ### Project
2
2
  [project]
3
3
  name = "taskcluster-taskgraph"
4
- version = "24.4.1"
4
+ version = "24.5.0"
5
5
  description = "Build taskcluster taskgraphs"
6
6
  readme = "README.rst"
7
7
  authors = [
@@ -6,16 +6,21 @@ import copy
6
6
  import inspect
7
7
  import logging
8
8
  import multiprocessing
9
+ import multiprocessing.connection
9
10
  import os
11
+ import pickle
10
12
  import platform
13
+ import signal
14
+ import sys
15
+ import time
16
+ import traceback
11
17
  from concurrent.futures import (
12
18
  FIRST_COMPLETED,
13
- ProcessPoolExecutor,
14
19
  ThreadPoolExecutor,
15
20
  wait,
16
21
  )
17
22
  from dataclasses import dataclass
18
- from typing import Callable, Optional, Union
23
+ from typing import Callable, Optional, Union, cast
19
24
 
20
25
  from . import filter_tasks
21
26
  from .config import GraphConfig, load_graph_config
@@ -26,6 +31,7 @@ from .parameters import Parameters, parameters_loader
26
31
  from .task import Task
27
32
  from .taskgraph import TaskGraph
28
33
  from .transforms.base import TransformConfig, TransformSequence
34
+ from .util.memory import gc_disabled
29
35
  from .util.python_path import find_object
30
36
  from .util.schema import SchemaValidationError
31
37
  from .util.verify import verifications
@@ -34,12 +40,31 @@ from .util.yaml import load_yaml
34
40
  logger = logging.getLogger(__name__)
35
41
 
36
42
 
43
+ def phase_finished(name, start, end):
44
+ """Called after each generation phase, with `time.monotonic()` values
45
+ taken around it. `name` is None if the phase raised an exception.
46
+
47
+ Projects can replace this from their `register` function; a replacement
48
+ made then also applies to the phase that is running.
49
+ """
50
+ if name is None:
51
+ logger.debug(f"Generation phase failed after {end - start:.2f}s")
52
+ else:
53
+ logger.debug(f"Generated {name} in {end - start:.2f}s")
54
+
55
+
37
56
  class KindNotFound(Exception):
38
57
  """
39
58
  Raised when trying to load kind from a directory without a kind.yml.
40
59
  """
41
60
 
42
61
 
62
+ def _max_workers():
63
+ # os.process_cpu_count is new in Python 3.13.
64
+ cpu_count = getattr(os, "process_cpu_count", os.cpu_count)
65
+ return cpu_count() or 1
66
+
67
+
43
68
  @dataclass(frozen=True)
44
69
  class Kind:
45
70
  name: str
@@ -124,6 +149,16 @@ class Kind:
124
149
  return cls(kind_name, path, config, graph_config)
125
150
 
126
151
 
152
+ class _RemoteTraceback(Exception):
153
+ """The traceback of an exception raised in a child process."""
154
+
155
+ def __init__(self, tb):
156
+ self.tb = tb
157
+
158
+ def __str__(self):
159
+ return f'\n"""\n{self.tb}"""'
160
+
161
+
127
162
  class TaskGraphGenerator:
128
163
  """
129
164
  The central controller for taskgraph. This handles all phases of graph
@@ -399,6 +434,157 @@ class TaskGraphGenerator:
399
434
 
400
435
  return all_tasks
401
436
 
437
+ def _load_tasks_forked(self, kinds, kind_graph, parameters):
438
+ """Load each kind in a child process forked once the kinds it depends
439
+ on are loaded.
440
+
441
+ The child shares every task loaded so far with the parent, so the
442
+ tasks of the kinds it depends on don't need to be sent to it, and it
443
+ sends its own tasks back through a pipe of its own.
444
+ """
445
+ all_tasks = {}
446
+ tasks_by_kind = {}
447
+ # Pipe to each running child -> (kind name, pid)
448
+ running = {}
449
+ remaining = kinds.copy()
450
+ edges = set(kind_graph.edges)
451
+ max_workers = _max_workers()
452
+
453
+ try:
454
+ while remaining or running:
455
+ kinds_with_deps = {edge[0] for edge in edges}
456
+ ready_kinds = sorted(set(remaining) - kinds_with_deps)
457
+ for name in ready_kinds[: max_workers - len(running)]:
458
+ kind = remaining.pop(name)
459
+ reader, pid = self._fork_kind(
460
+ kind,
461
+ parameters,
462
+ self._get_kind_dependencies_tasks(kind, tasks_by_kind),
463
+ )
464
+ running[reader] = (name, pid)
465
+
466
+ # Waiting on nothing would block forever. _run made sure there
467
+ # is no dependency loop, so some kind is always running here.
468
+ assert running
469
+ for ready in multiprocessing.connection.wait(list(running)):
470
+ # wait returns the objects it was given, but is typed as
471
+ # also returning sockets and file descriptors.
472
+ reader = cast(multiprocessing.connection.Connection, ready)
473
+ name, pid = running[reader]
474
+ try:
475
+ result = reader.recv()
476
+ except EOFError:
477
+ # The child exited without sending anything.
478
+ result = None
479
+ del running[reader]
480
+ reader.close()
481
+ _, status = os.waitpid(pid, 0)
482
+ new_tasks = self._kind_result(name, result, status)
483
+
484
+ kind_tasks = tasks_by_kind.setdefault(name, {})
485
+ for task in new_tasks:
486
+ if task.label in all_tasks:
487
+ raise Exception("duplicate tasks with label " + task.label)
488
+ all_tasks[task.label] = task
489
+ kind_tasks[task.label] = task
490
+
491
+ edges = {e for e in edges if e[1] != name}
492
+ finally:
493
+ # Only left with running children on error.
494
+ for reader, (_, pid) in running.items():
495
+ reader.close()
496
+ os.kill(pid, signal.SIGKILL)
497
+ os.waitpid(pid, 0)
498
+
499
+ return all_tasks
500
+
501
+ def _fork_kind(self, kind, parameters, kind_dependencies_tasks):
502
+ """Fork a child process loading the tasks of `kind`, and return the
503
+ pipe its result comes back through, and its pid."""
504
+ reader, writer = multiprocessing.Pipe(duplex=False)
505
+ # The child inherits the disabled garbage collector, so it doesn't
506
+ # write to every page it shares with us traversing the objects in them.
507
+ with gc_disabled():
508
+ pid = os.fork()
509
+ if pid == 0:
510
+ # Never leave this block, or return into the caller's code, in
511
+ # the child.
512
+ status = 1
513
+ try:
514
+ reader.close()
515
+ status = self._load_kind_in_child(
516
+ kind, parameters, kind_dependencies_tasks, writer
517
+ )
518
+ finally:
519
+ try:
520
+ sys.stdout.flush()
521
+ sys.stderr.flush()
522
+ finally:
523
+ os._exit(status)
524
+ writer.close()
525
+ return reader, pid
526
+
527
+ def _load_kind_in_child(self, kind, parameters, kind_dependencies_tasks, writer):
528
+ """Load the tasks of `kind` and send them, or the exception loading
529
+ them raised, through `writer`. Return the child's exit status."""
530
+ try:
531
+ tasks = kind.load_tasks(
532
+ parameters, kind_dependencies_tasks, self._write_artifacts
533
+ )
534
+ except BaseException as e:
535
+ tb = traceback.format_exc()
536
+ try:
537
+ # Make sure the parent will be able to unpickle it.
538
+ pickle.loads(pickle.dumps(e))
539
+ except Exception:
540
+ e = Exception(f"{type(e).__name__}: {e}")
541
+ writer.send((e, tb, None))
542
+ return 1
543
+
544
+ try:
545
+ writer.send((None, None, tasks))
546
+ except Exception as e:
547
+ writer.send(
548
+ (
549
+ Exception(
550
+ f"Could not send the tasks of kind {kind.name} "
551
+ f"to the parent process: {type(e).__name__}: {e}"
552
+ ),
553
+ traceback.format_exc(),
554
+ None,
555
+ )
556
+ )
557
+ return 1
558
+ return 0
559
+
560
+ @staticmethod
561
+ def _kind_result(name, result, status):
562
+ """Return the tasks the child loading kind `name` sent, given what it
563
+ sent and its exit status. Raise the exception loading them raised in
564
+ the child, if any."""
565
+ if result is None:
566
+ code = os.waitstatus_to_exitcode(status)
567
+ if code < 0:
568
+ how = f"was killed by signal {signal.Signals(-code).name}"
569
+ else:
570
+ how = f"exited with status {code}"
571
+ raise Exception(
572
+ f"Process loading tasks for kind {name} {how} without sending them"
573
+ )
574
+
575
+ exc, tb, tasks = result
576
+ if exc is not None:
577
+ exc.__cause__ = _RemoteTraceback(tb)
578
+ if isinstance(exc, SchemaValidationError):
579
+ logger.error(f"Error loading tasks for kind {name}:\n{exc}")
580
+ else:
581
+ logger.error(
582
+ f"Error loading tasks for kind {name}:",
583
+ exc_info=(type(exc), exc, exc.__traceback__),
584
+ )
585
+ raise exc
586
+ return tasks
587
+
402
588
  def _run(self):
403
589
  logger.info("Loading graph configuration.")
404
590
  graph_config = load_graph_config(self.root_dir)
@@ -446,6 +632,12 @@ class TaskGraphGenerator:
446
632
  for kind in kinds.values():
447
633
  for dep in kind.config.get("kind-dependencies", []):
448
634
  edges.add((kind.name, dep, "kind-dependency"))
635
+ for _, dep, _ in edges:
636
+ if dep not in kinds:
637
+ message = f'Could not find the kind "{dep}"\nAvailable kinds:\n'
638
+ for k in sorted(kinds):
639
+ message += f' - "{k}"\n'
640
+ raise Exception(message)
449
641
  kind_graph = Graph(frozenset(kinds), frozenset(edges))
450
642
 
451
643
  if target_kinds:
@@ -453,6 +645,11 @@ class TaskGraphGenerator:
453
645
  set(target_kinds) | {"docker-image"}
454
646
  )
455
647
 
648
+ # The task loaders wait for each kind's dependencies to be loaded
649
+ # before loading it, so a dependency loop would hang them or silently
650
+ # leave kinds out. This raises if there is one.
651
+ kind_graph.visit_postorder()
652
+
456
653
  yield "kind_graph", kind_graph
457
654
 
458
655
  logger.info("Generating full task set")
@@ -460,7 +657,7 @@ class TaskGraphGenerator:
460
657
  # The next block deals with enabling parallel kind processing, which
461
658
  # currently has different support on different platforms. In summary:
462
659
  # * Parallel kind processing is supported and enabled by default on
463
- # Linux. We use multiple processes by default, but experimental
660
+ # Linux. We fork a process per kind by default, but experimental
464
661
  # support for multiple threads can be enabled instead.
465
662
  # * On other platforms, we have experimental support for parallel
466
663
  # kind processing with multiple threads.
@@ -480,21 +677,19 @@ class TaskGraphGenerator:
480
677
  if os.environ.get("TASKGRAPH_SERIAL"):
481
678
  return self._load_tasks_serial(kinds, kind_graph, parameters)
482
679
  elif os.environ.get("TASKGRAPH_USE_THREADS"):
483
- executor = ThreadPoolExecutor(max_workers=os.process_cpu_count())
484
- else:
485
- executor = ProcessPoolExecutor(
486
- mp_context=multiprocessing.get_context("fork")
680
+ executor = ThreadPoolExecutor(max_workers=_max_workers())
681
+ return self._load_tasks_parallel(
682
+ kinds, kind_graph, parameters, executor
487
683
  )
488
- return self._load_tasks_parallel(
489
- kinds, kind_graph, parameters, executor
490
- )
684
+ else:
685
+ return self._load_tasks_forked(kinds, kind_graph, parameters)
491
686
  else:
492
687
  if os.environ.get("TASKGRAPH_SERIAL") or not os.environ.get(
493
688
  "TASKGRAPH_USE_THREADS"
494
689
  ):
495
690
  return self._load_tasks_serial(kinds, kind_graph, parameters)
496
691
  else:
497
- executor = ThreadPoolExecutor(max_workers=os.process_cpu_count())
692
+ executor = ThreadPoolExecutor(max_workers=_max_workers())
498
693
  return self._load_tasks_parallel(
499
694
  kinds, kind_graph, parameters, executor
500
695
  )
@@ -603,10 +798,19 @@ class TaskGraphGenerator:
603
798
 
604
799
  def _run_until(self, name):
605
800
  while name not in self._run_results:
801
+ start = time.monotonic()
606
802
  try:
607
- k, v = next(self._run) # type: ignore
803
+ # Generation creates millions of objects that stay alive, so
804
+ # collecting garbage meanwhile only traverses them for nothing.
805
+ # The callers' code runs with the collector in its usual state.
806
+ with gc_disabled():
807
+ k, v = next(self._run) # type: ignore
608
808
  except StopIteration:
609
809
  raise AttributeError(f"No such run result {name}")
810
+ except BaseException:
811
+ phase_finished(None, start, time.monotonic())
812
+ raise
813
+ phase_finished(k, start, time.monotonic())
610
814
  self._run_results[k] = v
611
815
  return self._run_results[name]
612
816
 
@@ -642,6 +642,10 @@ def extract_stream(reader, dest_dir):
642
642
  # See extract_archive.
643
643
  reader.forget()
644
644
  TarFile.open(fileobj=ifh, mode="r|").extractall(str(dest_dir))
645
+ # tarfile stops at the end-of-archive marker, which can leave the
646
+ # end of the compressed stream, and its checksum, unread.
647
+ while ifh.read(CHUNK_SIZE):
648
+ pass
645
649
  elif magic == ZSTD_MAGIC and shutil.which("zstd"):
646
650
  # Not tar --use-compress-program: when reading from stdin, tar relays
647
651
  # it to the decompressor in small blocks, which is several times
@@ -1720,6 +1720,9 @@ def main(args):
1720
1720
  os.setresgid(gid, gid, gid)
1721
1721
  os.setresuid(uid, uid, uid)
1722
1722
 
1723
+ if hook_path := os.environ.get("RUN_TASK_STARTUP_HOOK"):
1724
+ run_python_script(hook_path)
1725
+
1723
1726
  for repo in repositories:
1724
1727
  vcs_checkout_from_args(repo)
1725
1728
 
@@ -6,6 +6,7 @@ from dataclasses import dataclass
6
6
 
7
7
  from .graph import Graph
8
8
  from .task import Task
9
+ from .util.memory import gc_disabled
9
10
 
10
11
 
11
12
  @dataclass(frozen=True)
@@ -58,13 +59,14 @@ class TaskGraph:
58
59
  """
59
60
  tasks = {}
60
61
  edges = set()
61
- for key, value in tasks_dict.items():
62
- tasks[key] = Task.from_json(value)
63
- if "task_id" in value:
64
- tasks[key].task_id = value["task_id"]
65
- for depname, dep in value["dependencies"].items():
66
- # Task filtering can cause dependencies to be removed from the graph.
67
- if dep in tasks_dict:
68
- edges.add((key, dep, depname))
62
+ with gc_disabled():
63
+ for key, value in tasks_dict.items():
64
+ tasks[key] = Task.from_json(value)
65
+ if "task_id" in value:
66
+ tasks[key].task_id = value["task_id"]
67
+ for depname, dep in value["dependencies"].items():
68
+ # Task filtering can cause dependencies to be removed from the graph.
69
+ if dep in tasks_dict:
70
+ edges.add((key, dep, depname))
69
71
  task_graph = cls(tasks, Graph(frozenset(tasks), edges))
70
72
  return tasks, task_graph
@@ -0,0 +1,24 @@
1
+ # This Source Code Form is subject to the terms of the Mozilla Public
2
+ # License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
+
5
+ import gc
6
+ from contextlib import contextmanager
7
+
8
+
9
+ @contextmanager
10
+ def gc_disabled():
11
+ """Disable the garbage collector for the duration of the block, restoring
12
+ its previous state afterwards.
13
+
14
+ Generating or loading a task graph creates millions of objects that stay
15
+ alive, so collections keep traversing a growing heap without freeing
16
+ anything.
17
+ """
18
+ was_enabled = gc.isenabled()
19
+ gc.disable()
20
+ try:
21
+ yield
22
+ finally:
23
+ if was_enabled:
24
+ gc.enable()