taskcluster-taskgraph 24.4.0__tar.gz → 24.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (271) hide show
  1. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/CHANGELOG.md +12 -0
  2. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/PKG-INFO +1 -1
  3. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/howto/use-fetches.rst +13 -0
  4. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/pyproject.toml +2 -1
  5. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/generator.py +20 -11
  6. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/optimize/base.py +14 -7
  7. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/optimize/strategies.py +10 -2
  8. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/run-task/fetch-content +248 -18
  9. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/cached_tasks.py +12 -4
  10. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/config.yml +1 -1
  11. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/index-task/Dockerfile +1 -1
  12. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/index-task/yarn.lock +9 -9
  13. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/kinds/check/kind.yml +12 -0
  14. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/kinds/docker-image/kind.yml +1 -1
  15. taskcluster_taskgraph-24.4.1/test/test_scripts_fetch_content.py +963 -0
  16. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_cached_tasks.py +15 -0
  17. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/uv.lock +1322 -1009
  18. taskcluster_taskgraph-24.4.0/test/test_scripts_fetch_content.py +0 -381
  19. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/.codespell-ignore-words.txt +0 -0
  20. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/.github/CODEOWNERS +0 -0
  21. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/.github/workflows/codeql-analysis.yml +0 -0
  22. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/.github/workflows/codspeed.yml +0 -0
  23. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/.github/workflows/pypi-publish.yml +0 -0
  24. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/.gitignore +0 -0
  25. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/.pre-commit-config.yaml +0 -0
  26. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/.readthedocs.yaml +0 -0
  27. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/.taskcluster.yml +0 -0
  28. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/.yamllint +0 -0
  29. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/CODE_OF_CONDUCT.md +0 -0
  30. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/CONTRIBUTING.rst +0 -0
  31. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/LICENSE +0 -0
  32. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/Makefile +0 -0
  33. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/README.rst +0 -0
  34. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/concepts/index.rst +0 -0
  35. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/concepts/kind.rst +0 -0
  36. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/concepts/loading.rst +0 -0
  37. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/concepts/optimization.rst +0 -0
  38. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/concepts/overview.rst +0 -0
  39. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/concepts/scopes.rst +0 -0
  40. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/concepts/task-graphs.rst +0 -0
  41. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/concepts/transforms.rst +0 -0
  42. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/conf.py +0 -0
  43. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/contributing.rst +0 -0
  44. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/glossary.rst +0 -0
  45. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/howto/bootstrap-taskgraph.rst +0 -0
  46. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/howto/create-actions.rst +0 -0
  47. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/howto/create-tasks.rst +0 -0
  48. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/howto/debugging.rst +0 -0
  49. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/howto/define-schemas.rst +0 -0
  50. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/howto/docker.rst +0 -0
  51. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/howto/index.rst +0 -0
  52. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/howto/load-task-locally.rst +0 -0
  53. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/howto/resolve-keyed-by.rst +0 -0
  54. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/howto/run-locally.rst +0 -0
  55. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/howto/send-notifications.rst +0 -0
  56. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/index.rst +0 -0
  57. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/cli.rst +0 -0
  58. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/index.rst +0 -0
  59. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/migrations.rst +0 -0
  60. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/optimization-strategies.rst +0 -0
  61. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/parameters.rst +0 -0
  62. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/source/modules.rst +0 -0
  63. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/source/taskgraph.actions.rst +0 -0
  64. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/source/taskgraph.loader.rst +0 -0
  65. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/source/taskgraph.optimize.rst +0 -0
  66. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/source/taskgraph.rst +0 -0
  67. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/source/taskgraph.transforms.rst +0 -0
  68. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/source/taskgraph.transforms.run.rst +0 -0
  69. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/source/taskgraph.util.rst +0 -0
  70. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/transforms/chunking.rst +0 -0
  71. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/transforms/from_deps.rst +0 -0
  72. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/transforms/index.rst +0 -0
  73. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/transforms/matrix.rst +0 -0
  74. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/reference/transforms/task_context.rst +0 -0
  75. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/tutorials/connecting-taskcluster.rst +0 -0
  76. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/tutorials/creating-a-task-graph.rst +0 -0
  77. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/tutorials/example-taskcluster.yml +0 -0
  78. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/tutorials/getting-started.rst +0 -0
  79. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/docs/tutorials/index.rst +0 -0
  80. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/make.bat +0 -0
  81. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/packages/pytest-taskgraph/README.md +0 -0
  82. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/packages/pytest-taskgraph/pyproject.toml +0 -0
  83. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/packages/pytest-taskgraph/src/pytest_taskgraph/__init__.py +0 -0
  84. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py +0 -0
  85. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/vcs.py +0 -0
  86. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/packages/sphinx-taskgraph/README.md +0 -0
  87. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/packages/sphinx-taskgraph/pyproject.toml +0 -0
  88. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/packages/sphinx-taskgraph/src/sphinx_taskgraph/__init__.py +0 -0
  89. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/packages/sphinx-taskgraph/src/sphinx_taskgraph/autoschema.py +0 -0
  90. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/renovate.json +0 -0
  91. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/__init__.py +0 -0
  92. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/actions/__init__.py +0 -0
  93. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/actions/add_new_jobs.py +0 -0
  94. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/actions/cancel.py +0 -0
  95. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/actions/cancel_all.py +0 -0
  96. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/actions/rebuild_cached_tasks.py +0 -0
  97. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/actions/registry.py +0 -0
  98. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/actions/retrigger.py +0 -0
  99. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/actions/util.py +0 -0
  100. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/config.py +0 -0
  101. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/create.py +0 -0
  102. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/decision.py +0 -0
  103. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/docker.py +0 -0
  104. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/filter_tasks.py +0 -0
  105. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/graph.py +0 -0
  106. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/loader/__init__.py +0 -0
  107. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/loader/default.py +0 -0
  108. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/loader/transform.py +0 -0
  109. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/main.py +0 -0
  110. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/morph.py +0 -0
  111. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/optimize/__init__.py +0 -0
  112. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/parameters.py +0 -0
  113. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/run-task/hgrc +0 -0
  114. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/run-task/robustcheckout.py +0 -0
  115. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/run-task/run-task +0 -0
  116. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/target_tasks.py +0 -0
  117. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/task.py +0 -0
  118. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/taskgraph.py +0 -0
  119. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/__init__.py +0 -0
  120. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/base.py +0 -0
  121. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/cached_tasks.py +0 -0
  122. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/chunking.py +0 -0
  123. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/code_review.py +0 -0
  124. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/docker_image.py +0 -0
  125. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/fetch.py +0 -0
  126. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/from_deps.py +0 -0
  127. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/matrix.py +0 -0
  128. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/notify.py +0 -0
  129. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/run/__init__.py +0 -0
  130. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/run/common.py +0 -0
  131. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/run/index_search.py +0 -0
  132. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/run/run_task.py +0 -0
  133. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/run/toolchain.py +0 -0
  134. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/task.py +0 -0
  135. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/transforms/task_context.py +0 -0
  136. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/__init__.py +0 -0
  137. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/archive.py +0 -0
  138. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/attributes.py +0 -0
  139. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/caches.py +0 -0
  140. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/copy.py +0 -0
  141. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/dependencies.py +0 -0
  142. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/docker.py +0 -0
  143. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/hash.py +0 -0
  144. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/json.py +0 -0
  145. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/keyed_by.py +0 -0
  146. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/parameterization.py +0 -0
  147. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/path.py +0 -0
  148. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/python_path.py +0 -0
  149. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/readonlydict.py +0 -0
  150. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/schema.py +0 -0
  151. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/set_name.py +0 -0
  152. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/shell.py +0 -0
  153. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/task_context.py +0 -0
  154. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/taskcluster.py +0 -0
  155. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/taskgraph.py +0 -0
  156. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/templates.py +0 -0
  157. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/time.py +0 -0
  158. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/treeherder.py +0 -0
  159. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/vcs.py +0 -0
  160. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/verify.py +0 -0
  161. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/workertypes.py +0 -0
  162. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/src/taskgraph/util/yaml.py +0 -0
  163. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/REGISTRY +0 -0
  164. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/decision/Dockerfile +0 -0
  165. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/decision/HASH +0 -0
  166. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/decision/README.md +0 -0
  167. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/decision/VERSION +0 -0
  168. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/decision/system-setup.sh +0 -0
  169. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/fetch/Dockerfile +0 -0
  170. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/image_builder/README.rst +0 -0
  171. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/index-task/README +0 -0
  172. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/index-task/insert-indexes.js +0 -0
  173. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/index-task/package.json +0 -0
  174. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/python/Dockerfile +0 -0
  175. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/run-task/Dockerfile +0 -0
  176. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/run-task/system-setup.sh +0 -0
  177. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/skopeo/Dockerfile +0 -0
  178. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/skopeo/policy.json +0 -0
  179. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/docker/skopeo/push_image.sh +0 -0
  180. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/kinds/codecov/kind.yml +0 -0
  181. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/kinds/complete/kind.yml +0 -0
  182. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/kinds/doc/kind.yml +0 -0
  183. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/kinds/fetch/kind.yml +0 -0
  184. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/kinds/push-image/kind.yml +0 -0
  185. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/kinds/test/kind.yml +0 -0
  186. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/kinds/test/linux.yml +0 -0
  187. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/kinds/test/windows.yml +0 -0
  188. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/scripts/codecov-upload.py +0 -0
  189. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/self_taskgraph/__init__.py +0 -0
  190. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/self_taskgraph/custom_parameters.py +0 -0
  191. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/self_taskgraph/custom_target_tasks.py +0 -0
  192. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/self_taskgraph/transforms/add_pr_route.py +0 -0
  193. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/self_taskgraph/transforms/push_image.py +0 -0
  194. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/test/params/main-repo-pull-request-untrusted.yml +0 -0
  195. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/test/params/main-repo-pull-request.yml +0 -0
  196. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/test/params/main-repo-push.yml +0 -0
  197. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/test/params/main-repo-release-pytest-taskgraph.yml +0 -0
  198. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/taskcluster/test/params/main-repo-release-taskcluster-taskgraph.yml +0 -0
  199. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/template/cookiecutter.json +0 -0
  200. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/template/hooks/post_gen_project.py +0 -0
  201. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/template/{{cookiecutter.project_name}}/taskcluster/config.yml +0 -0
  202. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/template/{{cookiecutter.project_name}}/taskcluster/docker/linux/Dockerfile +0 -0
  203. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/template/{{cookiecutter.project_name}}/taskcluster/kinds/docker-image/kind.yml +0 -0
  204. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/template/{{cookiecutter.project_name}}/taskcluster/kinds/hello/kind.yml +0 -0
  205. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/template/{{cookiecutter.project_name}}/taskcluster/{{cookiecutter.project_slug}}_taskgraph/transforms/hello.py +0 -0
  206. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/template/{{cookiecutter.project_name}}/taskcluster.github.yml +0 -0
  207. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/__init__.py +0 -0
  208. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/automationrelevance.json +0 -0
  209. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/conftest.py +0 -0
  210. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/data/task_context.yml +0 -0
  211. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/data/taskcluster/config.yml +0 -0
  212. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/data/taskcluster/docker/hello-world/Dockerfile +0 -0
  213. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/data/taskcluster/docker/hello-world-tag/Dockerfile +0 -0
  214. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/data/taskcluster/docker/hello-world-tag/REGISTRY +0 -0
  215. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/data/taskcluster/docker/hello-world-tag/VERSION +0 -0
  216. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/data/taskcluster/kinds/docker-image/kind.yml +0 -0
  217. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/data/taskcluster/scripts/toolchain/run.ps1 +0 -0
  218. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/data/taskcluster/scripts/toolchain/run.sh +0 -0
  219. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/data/taskcluster/test_taskgraph/transforms/foo.py +0 -0
  220. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/data/testmod/thing.py +0 -0
  221. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/mockedopen.py +0 -0
  222. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_actions_rebuild_cached_tasks.py +0 -0
  223. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_actions_registry.py +0 -0
  224. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_config.py +0 -0
  225. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_create.py +0 -0
  226. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_decision.py +0 -0
  227. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_docker.py +0 -0
  228. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_generator.py +0 -0
  229. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_graph.py +0 -0
  230. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_graph_perf.py +0 -0
  231. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_main.py +0 -0
  232. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_morph.py +0 -0
  233. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_optimize.py +0 -0
  234. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_optimize_strategies.py +0 -0
  235. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_parameters.py +0 -0
  236. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_scripts_run_task.py +0 -0
  237. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_target_tasks.py +0 -0
  238. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_taskgraph.py +0 -0
  239. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_transform_chunking.py +0 -0
  240. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_transform_docker_image.py +0 -0
  241. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_transform_task_context.py +0 -0
  242. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_transforms_base.py +0 -0
  243. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_transforms_cached_tasks.py +0 -0
  244. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_transforms_fetch.py +0 -0
  245. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_transforms_from_deps.py +0 -0
  246. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_transforms_matrix.py +0 -0
  247. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_transforms_notify.py +0 -0
  248. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_transforms_run.py +0 -0
  249. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_transforms_run_run_task.py +0 -0
  250. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_transforms_run_toolchain.py +0 -0
  251. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_transforms_task.py +0 -0
  252. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_archive.py +0 -0
  253. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_attributes.py +0 -0
  254. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_copy.py +0 -0
  255. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_dependencies.py +0 -0
  256. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_docker.py +0 -0
  257. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_json.py +0 -0
  258. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_keyed_by.py +0 -0
  259. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_parameterization.py +0 -0
  260. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_path.py +0 -0
  261. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_python_path.py +0 -0
  262. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_readonlydict.py +0 -0
  263. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_schema.py +0 -0
  264. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_taskcluster.py +0 -0
  265. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_templates.py +0 -0
  266. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_time.py +0 -0
  267. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_treeherder.py +0 -0
  268. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_vcs.py +0 -0
  269. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_verify.py +0 -0
  270. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_workertypes.py +0 -0
  271. {taskcluster_taskgraph-24.4.0 → taskcluster_taskgraph-24.4.1}/test/test_util_yaml.py +0 -0
@@ -1,5 +1,16 @@
1
1
  # Change Log
2
2
 
3
+ ## [24.4.1] - 2026-10-08
4
+
5
+ ### Fixed
6
+
7
+ - `github-issue-comment` tasks now use the PR cache namespaces
8
+ - perf: look up kind dependency tasks by kind in the generator (#1054)
9
+ - perf: extract tar archives while they download in `fetch-content` (#1068)
10
+ - perf: let tar decompress zstd archives in `fetch-content` (#1068)
11
+ - perf: read downloads in 1MiB chunks in `fetch-content` (#1068)
12
+ - perf: resolve each dependent's deadline only once in `optimize` (#1056)
13
+
3
14
  ## [24.4.0] - 2026-10-05
4
15
 
5
16
  ### Added
@@ -11,6 +22,7 @@
11
22
  ### Changed
12
23
 
13
24
  - `taskgraph init` template and tutorial docs now default to the `public_restricted` PR policy
25
+ - BREAKING CHANGE: `run-task` requires Python 3.8
14
26
 
15
27
  ### Fixed
16
28
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: taskcluster-taskgraph
3
- Version: 24.4.0
3
+ Version: 24.4.1
4
4
  Summary: Build taskcluster taskgraphs
5
5
  Project-URL: Repository, https://github.com/taskcluster/taskgraph
6
6
  Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues
@@ -105,3 +105,16 @@ There are a few differences from the earlier ``build`` examples here:
105
105
 
106
106
  It is not possible to configure the ``dest`` or ``extract`` values when using
107
107
  ``fetch`` or ``toolchain`` kinds.
108
+
109
+ Tuning Download Performance
110
+ ---------------------------
111
+
112
+ The following environment variable tunes how ``fetch-content`` downloads
113
+ artifacts.
114
+
115
+ ``TASKGRAPH_FETCH_STREAM``
116
+ Tar archives that are extracted are extracted while they download, rather
117
+ than written to disk and read back, into a staging directory that is only
118
+ merged into place once the download is complete and verified. This is on
119
+ by default; set to ``0`` to download to a file first. Zip archives, and
120
+ fetches that aren't extracted, always go through a file.
@@ -1,7 +1,7 @@
1
1
  ### Project
2
2
  [project]
3
3
  name = "taskcluster-taskgraph"
4
- version = "24.4.0"
4
+ version = "24.4.1"
5
5
  description = "Build taskcluster taskgraphs"
6
6
  readme = "README.rst"
7
7
  authors = [
@@ -52,6 +52,7 @@ dev = [
52
52
  "sphinx-book-theme >=1.1.4",
53
53
  "sphinx-taskgraph>=0.1.0",
54
54
  "sphinxcontrib-mermaid>=1.0.0",
55
+ "vermin>=1.8.0",
55
56
  "zstandard>=0.25.0",
56
57
  ]
57
58
 
@@ -282,8 +282,21 @@ class TaskGraphGenerator:
282
282
  except KindNotFound:
283
283
  continue
284
284
 
285
+ @staticmethod
286
+ def _get_kind_dependencies_tasks(kind, tasks_by_kind):
287
+ """Return the tasks of the kinds `kind` depends on, in the order in
288
+ which they were loaded."""
289
+ kind_dependencies = kind.config.get("kind-dependencies", [])
290
+ return {
291
+ label: task
292
+ for kind_name, tasks in tasks_by_kind.items()
293
+ if kind_name in kind_dependencies
294
+ for label, task in tasks.items()
295
+ }
296
+
285
297
  def _load_tasks_serial(self, kinds, kind_graph, parameters):
286
298
  all_tasks = {}
299
+ tasks_by_kind = {}
287
300
  for kind_name in kind_graph.visit_postorder():
288
301
  logger.debug(f"Loading tasks for kind {kind_name}")
289
302
 
@@ -297,11 +310,7 @@ class TaskGraphGenerator:
297
310
  try:
298
311
  new_tasks = kind.load_tasks(
299
312
  parameters,
300
- {
301
- k: t
302
- for k, t in all_tasks.items()
303
- if t.kind in kind.config.get("kind-dependencies", [])
304
- },
313
+ self._get_kind_dependencies_tasks(kind, tasks_by_kind),
305
314
  self._write_artifacts,
306
315
  )
307
316
  except SchemaValidationError as exc:
@@ -310,15 +319,18 @@ class TaskGraphGenerator:
310
319
  except Exception:
311
320
  logger.exception(f"Error loading tasks for kind {kind_name}:")
312
321
  raise
322
+ kind_tasks = tasks_by_kind.setdefault(kind_name, {})
313
323
  for task in new_tasks:
314
324
  if task.label in all_tasks:
315
325
  raise Exception("duplicate tasks with label " + task.label)
316
326
  all_tasks[task.label] = task
327
+ kind_tasks[task.label] = task
317
328
 
318
329
  return all_tasks
319
330
 
320
331
  def _load_tasks_parallel(self, kinds, kind_graph, parameters, executor):
321
332
  all_tasks = {}
333
+ tasks_by_kind = {}
322
334
  futures_to_kind = {}
323
335
  futures = set()
324
336
  edges = set(kind_graph.edges)
@@ -328,7 +340,6 @@ class TaskGraphGenerator:
328
340
  def submit_ready_kinds():
329
341
  """Create the next batch of tasks for kinds without dependencies."""
330
342
  nonlocal kinds, edges, futures
331
- loaded_tasks = all_tasks.copy()
332
343
  kinds_with_deps = {edge[0] for edge in edges}
333
344
  ready_kinds = (
334
345
  set(kinds) - kinds_with_deps - set(futures_to_kind.values())
@@ -346,11 +357,7 @@ class TaskGraphGenerator:
346
357
  future = executor.submit(
347
358
  kind.load_tasks,
348
359
  dict(parameters),
349
- {
350
- k: t
351
- for k, t in loaded_tasks.items()
352
- if t.kind in kind.config.get("kind-dependencies", [])
353
- },
360
+ self._get_kind_dependencies_tasks(kind, tasks_by_kind),
354
361
  self._write_artifacts,
355
362
  )
356
363
  futures.add(future)
@@ -376,10 +383,12 @@ class TaskGraphGenerator:
376
383
  kind = futures_to_kind.pop(future)
377
384
  futures.remove(future)
378
385
 
386
+ kind_tasks = tasks_by_kind.setdefault(kind, {})
379
387
  for task in future.result():
380
388
  if task.label in all_tasks:
381
389
  raise Exception("duplicate tasks with label " + task.label)
382
390
  all_tasks[task.label] = task
391
+ kind_tasks[task.label] = task
383
392
 
384
393
  # Update state for next batch of futures.
385
394
  del kinds[kind]
@@ -301,6 +301,18 @@ def replace_tasks(
301
301
  target_task_graph.graph.links_and_reverse_links_dict()
302
302
  )
303
303
 
304
+ # Many tasks share dependents (e.g. docker images or toolchains), so
305
+ # resolve the deadline of each dependent only once.
306
+ now = datetime.datetime.now(datetime.timezone.utc)
307
+ deadlines = {}
308
+
309
+ def get_deadline(label):
310
+ if label not in deadlines:
311
+ deadlines[label] = resolve_timestamps(
312
+ now, target_task_graph.tasks[label].task["deadline"]
313
+ )
314
+ return deadlines[label]
315
+
304
316
  for label in target_task_graph.graph.visit_postorder():
305
317
  logger.debug(f"replace_tasks: {label}")
306
318
  # if we're not allowed to optimize, that's easy..
@@ -331,14 +343,9 @@ def replace_tasks(
331
343
  opt_by, opt, arg = optimizations(label)
332
344
 
333
345
  # compute latest deadline of dependents (if any)
334
- dependents = [target_task_graph.tasks[l] for l in dependents_of[label]]
335
346
  deadline = None
336
- if dependents:
337
- now = datetime.datetime.now(datetime.timezone.utc)
338
- deadline = max(
339
- resolve_timestamps(now, task.task["deadline"])
340
- for task in dependents # type: ignore
341
- )
347
+ if dependents_of[label]:
348
+ deadline = max(get_deadline(l) for l in dependents_of[label])
342
349
 
343
350
  if isinstance(opt, IndexSearch):
344
351
  arg = arg, index_to_taskid, taskid_to_status
@@ -1,3 +1,4 @@
1
+ import functools
1
2
  import logging
2
3
  from datetime import datetime
3
4
 
@@ -10,6 +11,13 @@ from taskgraph.util.taskcluster import find_task_id, status_task
10
11
  logger = logging.getLogger("optimization")
11
12
 
12
13
 
14
+ @functools.cache
15
+ def _parse_time(timestamp, fmt):
16
+ # Many tasks share the same deadline or replacement task, so avoid parsing
17
+ # the same timestamps over and over.
18
+ return datetime.strptime(timestamp, fmt)
19
+
20
+
13
21
  @register_strategy("index-search")
14
22
  class IndexSearch(OptimizationStrategy):
15
23
  # A task with no dependencies remaining after optimization will be replaced
@@ -56,10 +64,10 @@ class IndexSearch(OptimizationStrategy):
56
64
  )
57
65
  continue
58
66
 
59
- if deadline and datetime.strptime(
67
+ if deadline and _parse_time(
60
68
  status["expires"], # type: ignore
61
69
  self.fmt,
62
- ) < datetime.strptime(deadline, self.fmt):
70
+ ) < _parse_time(deadline, self.fmt):
63
71
  logger.debug(
64
72
  f"not replacing {task.label} with {task_id} because it expires before {deadline}"
65
73
  )
@@ -8,9 +8,11 @@ import bz2
8
8
  import concurrent.futures
9
9
  import contextlib
10
10
  import datetime
11
+ import functools
11
12
  import gzip
12
13
  import hashlib
13
14
  import io
15
+ import itertools
14
16
  import json
15
17
  import lzma
16
18
  import multiprocessing
@@ -43,6 +45,12 @@ except ImportError:
43
45
 
44
46
  CONCURRENCY = multiprocessing.cpu_count()
45
47
 
48
+ # Whether to extract tar archives while they download, rather than writing
49
+ # them to disk and reading them back. TASKGRAPH_FETCH_STREAM=0 disables it.
50
+ DEFAULT_STREAMING = True
51
+
52
+ CHUNK_SIZE = 1024 * 1024
53
+
46
54
 
47
55
  def log(msg):
48
56
  print(msg, file=sys.stderr)
@@ -219,7 +227,7 @@ def stream_download(url, sha256=None, size=None, headers=None):
219
227
  size = content_length
220
228
 
221
229
  while True:
222
- chunk = fh.read(65536)
230
+ chunk = fh.read(CHUNK_SIZE)
223
231
  if not chunk:
224
232
  break
225
233
 
@@ -248,6 +256,13 @@ def stream_download(url, sha256=None, size=None, headers=None):
248
256
  )
249
257
 
250
258
 
259
+ def streaming_enabled():
260
+ value = os.environ.get("TASKGRAPH_FETCH_STREAM", "").strip().lower()
261
+ if not value:
262
+ return DEFAULT_STREAMING
263
+ return value in ("1", "true", "yes")
264
+
265
+
251
266
  def download_to_path(url, path, sha256=None, size=None, headers=None):
252
267
  """Download a URL to a filesystem path, possibly with verification."""
253
268
 
@@ -336,12 +351,19 @@ class ArchiveTypeNotSupported(Exception):
336
351
  super(Exception, self).__init__(f"Archive type not supported for {path}")
337
352
 
338
353
 
354
+ ZSTD_MAGIC = b"\x28\xb5\x2f\xfd"
355
+
356
+
339
357
  def open_stream(path: pathlib.Path):
340
358
  """Attempt to identify a path as an extractable archive by looking at its
341
359
  content."""
342
- fh = path.open(mode="rb")
343
- magic = fh.read(6)
344
- fh.seek(0)
360
+ return open_fileobj(path.open(mode="rb"), path)
361
+
362
+
363
+ def open_fileobj(fh, name):
364
+ """Like ``open_stream``, for a buffered file object that may not be
365
+ seekable."""
366
+ magic = fh.peek(6)[:6]
345
367
  if magic[:2] == b"PK":
346
368
  return "zip", fh
347
369
  if magic[:2] == b"\x1f\x8b":
@@ -350,7 +372,7 @@ def open_stream(path: pathlib.Path):
350
372
  fh = bz2.BZ2File(fh)
351
373
  elif magic == b"\xfd7zXZ\x00":
352
374
  fh = lzma.LZMAFile(fh)
353
- elif magic[:4] == b"\x28\xb5\x2f\xfd":
375
+ elif magic[:4] == ZSTD_MAGIC:
354
376
  fh = ZstdDecompressor().stream_reader(fh)
355
377
  fh = io.BufferedReader(fh)
356
378
  try:
@@ -365,7 +387,12 @@ def open_stream(path: pathlib.Path):
365
387
  return "tar", fh
366
388
  except Exception:
367
389
  pass
368
- raise ArchiveTypeNotSupported(path)
390
+ raise ArchiveTypeNotSupported(name)
391
+
392
+
393
+ def is_zstd(path: pathlib.Path):
394
+ with path.open(mode="rb") as fh:
395
+ return fh.read(4) == ZSTD_MAGIC
369
396
 
370
397
 
371
398
  def archive_type(path: pathlib.Path):
@@ -466,6 +493,13 @@ def extract_archive(path, dest_dir):
466
493
  tar = TarFile.open(fileobj=ifh, mode="r|")
467
494
  tar.extractall(str(dest_dir))
468
495
  args = []
496
+ elif is_zstd(path) and shutil.which("zstd"):
497
+ # Decompressing in a separate process keeps it from competing
498
+ # with concurrent downloads for the GIL.
499
+ ifh.close()
500
+ ifh = open(os.devnull, "rb")
501
+ args = ["tar", "--use-compress-program=zstd", "-xf", str(path)]
502
+ pipe_stdin = False
469
503
  else:
470
504
  args = ["tar", "xf", "-"]
471
505
  pipe_stdin = True
@@ -478,23 +512,211 @@ def extract_archive(path, dest_dir):
478
512
  raise ValueError(f"unknown archive format: {path}")
479
513
 
480
514
  if args:
481
- with ifh, subprocess.Popen(
482
- args, cwd=str(dest_dir), bufsize=0, stdin=subprocess.PIPE
483
- ) as p:
484
- while True:
485
- if not pipe_stdin:
486
- break
515
+ with ifh:
516
+ chunks = iter(functools.partial(ifh.read, 131072), b"")
517
+ run_pipeline([args], dest_dir, chunks if pipe_stdin else ())
518
+
519
+ log(f"{path} extracted in {time.time() - t0:.3f}s")
487
520
 
488
- chunk = ifh.read(131072)
489
- if not chunk:
490
- break
491
521
 
492
- p.stdin.write(chunk)
522
+ def run_pipeline(commands, cwd, chunks):
523
+ """Run ``commands`` in ``cwd``, each reading the output of the previous
524
+ one, and the first reading ``chunks``."""
525
+ procs = []
526
+ stdin = subprocess.PIPE
527
+ for i, args in enumerate(commands):
528
+ last = i == len(commands) - 1
529
+ p = subprocess.Popen(
530
+ args,
531
+ cwd=str(cwd),
532
+ bufsize=0,
533
+ stdin=stdin,
534
+ stdout=None if last else subprocess.PIPE,
535
+ )
536
+ if procs:
537
+ # Only the next process holds the pipe, so that the previous one
538
+ # gets SIGPIPE if it exits.
539
+ procs[-1].stdout.close()
540
+ stdin = p.stdout
541
+ procs.append(p)
542
+
543
+ try:
544
+ for chunk in chunks:
545
+ procs[0].stdin.write(chunk)
546
+ except BrokenPipeError:
547
+ # A process exited early; its exit status says why.
548
+ pass
549
+ finally:
550
+ try:
551
+ procs[0].stdin.close()
552
+ except BrokenPipeError:
553
+ pass
554
+ for p in procs:
555
+ p.wait()
493
556
 
557
+ for args, p in zip(commands, procs):
494
558
  if p.returncode:
495
559
  raise Exception(f"{args!r} exited {p.returncode}")
496
560
 
497
- log(f"{path} extracted in {time.time() - t0:.3f}s")
561
+
562
+ class IterReader(io.RawIOBase):
563
+ """A file object reading from an iterator of chunks.
564
+
565
+ Until ``iter_from_start`` or ``forget`` is called, it keeps what it has
566
+ read, so that the start of a stream can be inspected and then handed on
567
+ whole.
568
+ """
569
+
570
+ def __init__(self, chunks):
571
+ self._chunks = chunks
572
+ self._chunk = b""
573
+ self._pos = 0
574
+ self._recorded = []
575
+ # Whether reading from the iterator raised.
576
+ self.failed = False
577
+
578
+ def readable(self):
579
+ return True
580
+
581
+ def _next(self):
582
+ try:
583
+ chunk = next(self._chunks, b"")
584
+ except Exception:
585
+ self.failed = True
586
+ raise
587
+ if self._recorded is not None and chunk:
588
+ self._recorded.append(chunk)
589
+ return chunk
590
+
591
+ def readinto(self, b):
592
+ if self._pos == len(self._chunk):
593
+ self._chunk = memoryview(self._next())
594
+ self._pos = 0
595
+ n = min(len(b), len(self._chunk) - self._pos)
596
+ b[:n] = self._chunk[self._pos : self._pos + n]
597
+ self._pos += n
598
+ return n
599
+
600
+ def _rest(self):
601
+ while True:
602
+ chunk = self._next()
603
+ if not chunk:
604
+ return
605
+ yield chunk
606
+
607
+ def iter_from_start(self):
608
+ """Iterate over the whole stream, from the start."""
609
+ recorded, self._recorded = self._recorded, None
610
+ return itertools.chain(recorded, self._rest())
611
+
612
+ def forget(self):
613
+ """Stop keeping what has been read."""
614
+ self._recorded = None
615
+
616
+ def drain(self):
617
+ """Read the rest of the stream, discarding it."""
618
+ self.forget()
619
+ for _ in self._rest():
620
+ pass
621
+
622
+
623
+ def extract_stream(reader, dest_dir):
624
+ """Extract the tar archive read from ``reader`` into ``dest_dir``.
625
+
626
+ Returns False, having only read the start of the stream, if it isn't a
627
+ tar archive. Otherwise reads the stream to the end, so that the download
628
+ it comes from gets verified.
629
+ """
630
+ fh = io.BufferedReader(reader, CHUNK_SIZE)
631
+ magic = fh.peek(4)[:4]
632
+ try:
633
+ typ, ifh = open_fileobj(fh, "stream")
634
+ except (ArchiveTypeNotSupported, ValueError):
635
+ # ValueError is a zstd archive without the zstandard module, which
636
+ # extract_archive will report.
637
+ return False
638
+ if typ != "tar":
639
+ return False
640
+
641
+ if sys.platform == "win32":
642
+ # See extract_archive.
643
+ reader.forget()
644
+ TarFile.open(fileobj=ifh, mode="r|").extractall(str(dest_dir))
645
+ elif magic == ZSTD_MAGIC and shutil.which("zstd"):
646
+ # Not tar --use-compress-program: when reading from stdin, tar relays
647
+ # it to the decompressor in small blocks, which is several times
648
+ # slower.
649
+ run_pipeline(
650
+ [["zstd", "-dcq"], ["tar", "xf", "-"]],
651
+ dest_dir,
652
+ reader.iter_from_start(),
653
+ )
654
+ else:
655
+ reader.forget()
656
+ run_pipeline(
657
+ [["tar", "xf", "-"]],
658
+ dest_dir,
659
+ iter(functools.partial(ifh.read1, CHUNK_SIZE), b""),
660
+ )
661
+ reader.drain()
662
+ return True
663
+
664
+
665
+ def empty_directory(path):
666
+ """Remove everything in ``path``, including read-only directories."""
667
+ for root, dirs, _ in os.walk(path):
668
+ for d in dirs:
669
+ d = os.path.join(root, d)
670
+ if not os.path.islink(d):
671
+ os.chmod(d, stat.S_IRWXU)
672
+ for entry in os.scandir(path):
673
+ if entry.is_dir(follow_symlinks=False):
674
+ shutil.rmtree(entry.path)
675
+ else:
676
+ os.unlink(entry.path)
677
+
678
+
679
+ def stream_and_extract(url, dest_path, dest_dir, sha256=None, size=None, headers=None):
680
+ """Download ``url`` and extract it into ``dest_dir`` as it arrives.
681
+
682
+ If it turns out not to be a tar archive, it is downloaded to
683
+ ``dest_path`` instead, and False is returned.
684
+
685
+ With ``sha256`` or ``size``, content is extracted before it can be
686
+ verified. That's fine as long as ``dest_dir`` is a staging directory that
687
+ is only merged once this returns: on a mismatch it is emptied, the same
688
+ as a download that fails verification is deleted.
689
+ """
690
+ for _ in retrier(attempts=5, sleeptime=60):
691
+ chunks = stream_download(url, sha256=sha256, size=size, headers=headers)
692
+ reader = IterReader(chunks)
693
+ try:
694
+ t0 = time.time()
695
+ if extract_stream(reader, dest_dir):
696
+ log(f"{url} extracted to {dest_dir} in {time.time() - t0:.3f}s")
697
+ return True
698
+ log(f"{url} is not a tar archive; downloading it to {dest_path}")
699
+ with rename_after_close(dest_path, "wb") as fh:
700
+ for chunk in reader.iter_from_start():
701
+ fh.write(chunk)
702
+ return False
703
+ except Exception as e:
704
+ log(f"Download failed: {e}")
705
+ empty_directory(dest_dir)
706
+ if not reader.failed:
707
+ # The download didn't fail, so find out whether it is intact.
708
+ # If it is, the archive itself is broken and retrying won't
709
+ # help.
710
+ try:
711
+ reader.drain()
712
+ except Exception:
713
+ pass
714
+ else:
715
+ raise
716
+ finally:
717
+ chunks.close()
718
+
719
+ raise Exception("Download failed, no more retries!")
498
720
 
499
721
 
500
722
  def should_repack_archive(
@@ -719,12 +941,20 @@ def fetch_and_extract(url, dest_dir, extract=True, sha256=None, size=None):
719
941
  If the downloaded URL is an archive, it is extracted automatically
720
942
  and the archive is deleted. Otherwise the file remains in place in
721
943
  the destination directory.
944
+
945
+ When extracting, ``dest_dir`` must be a staging directory: a failed
946
+ attempt empties it.
722
947
  """
723
948
 
724
949
  basename = urllib.parse.unquote(urllib.parse.urlparse(url).path.split("/")[-1])
725
950
  dest_path = dest_dir / basename
726
951
 
727
- download_to_path(url, dest_path, sha256=sha256, size=size)
952
+ # Zip archives can't be streamed, so they keep going through a file.
953
+ if extract and streaming_enabled() and archive_type(dest_path) != "zip":
954
+ if stream_and_extract(url, dest_path, dest_dir, sha256=sha256, size=size):
955
+ return
956
+ else:
957
+ download_to_path(url, dest_path, sha256=sha256, size=size)
728
958
 
729
959
  if not extract:
730
960
  return
@@ -63,9 +63,14 @@ def add_optimization(
63
63
  # Pull requests use a different target cache index route. This way we can
64
64
  # be confident they won't be used by anything other than the pull request
65
65
  # that created the cache in the first place.
66
- if config.params["tasks_for"] == "github-pull-request" and config.graph_config[
67
- "taskgraph"
68
- ].get("cache-pull-requests", True):
66
+ #
67
+ # Comments on a pull request run against the PR's head, so they use the
68
+ # same cache namespace as the PR itself.
69
+ tasks_for = config.params["tasks_for"]
70
+ if tasks_for in (
71
+ "github-pull-request",
72
+ "github-issue-comment",
73
+ ) and config.graph_config["taskgraph"].get("cache-pull-requests", True):
69
74
  subs["head_ref"] = config.params["head_ref"]
70
75
  if subs["head_ref"].startswith("refs/heads/"):
71
76
  subs["head_ref"] = subs["head_ref"][11:]
@@ -76,7 +81,10 @@ def add_optimization(
76
81
  # ... and cache at the lowest level.
77
82
  subs["level"] = config.params["level"]
78
83
 
79
- if config.params["tasks_for"].startswith("github-pull-request"):
84
+ if (
85
+ tasks_for.startswith("github-pull-request")
86
+ or tasks_for == "github-issue-comment"
87
+ ):
80
88
  if config.graph_config["taskgraph"].get("cache-pull-requests", True):
81
89
  taskdesc.setdefault("routes", []).append(
82
90
  f"index.{TARGET_PR_CACHE_INDEX.format(**subs)}"
@@ -45,4 +45,4 @@ workers:
45
45
  provisioner: 'taskgraph-t'
46
46
  implementation: generic-worker
47
47
  os: windows
48
- worker-type: win11-64-24h2
48
+ worker-type: win11-64-25h2
@@ -1,4 +1,4 @@
1
- FROM node:24-alpine@sha256:e67514e5d0f6c46656005e1b693b2ec9d52e80b641307de684d4a015ba7a4eaf
1
+ FROM node:24-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1
2
2
 
3
3
  ENV NODE_ENV production
4
4
  RUN mkdir /app
@@ -139,9 +139,9 @@ hawk@^9.0.1:
139
139
  "@hapi/hoek" "9.x.x"
140
140
 
141
141
  http-cache-semantics@^4.2.0:
142
- version "4.2.0"
143
- resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz#205f4db64f8562b76a4ff9235aa5279839a09dd5"
144
- integrity sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==
142
+ version "4.3.0"
143
+ resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.3.0.tgz#09eead3b16c6d85552857cc3fbd888d8353a2aaf"
144
+ integrity sha512-M5t5LlJpS1UHMjvwRQVdFHvPISGeLAxNcrWuJkeGh0KxsqCHZ1O3NXZU/8x7cD0BDcGW8kapxMKTvwlqrNkHkA==
145
145
 
146
146
  http2-wrapper@^2.2.1:
147
147
  version "2.2.1"
@@ -228,16 +228,16 @@ taskcluster-lib-urls@^13.0.0:
228
228
  integrity sha512-IehEsRYfvTx8lvSEq7TaCn296+gSIf/3bOTHcm8gQ3ljna0OUvc/FEqMcCOPBG5gus9nh+L/ZOGCpXJi8tYf9w==
229
229
 
230
230
  type-fest@^5.6.0:
231
- version "5.9.0"
232
- resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-5.9.0.tgz#0a4dd554d397f1a44447477eaeb3d1f4d8c7680f"
233
- integrity sha512-yANm3Jr3GiJ1qgJlxGAVxTOIcEOk1rhQHamlXtnrCK7EHP4HeM9OGxtMg/W7HFdrVzw/ZWJKGVIJusVH85sLtw==
231
+ version "5.10.0"
232
+ resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-5.10.0.tgz#bf0131fdf662889c94c409d37402e364fb2016f7"
233
+ integrity sha512-NoSdpq/WEiAg5sjmBkmV/hfxv6HJH4NqPNrqjtSO5CwRmpsDfaf4begxW34KdJykH/l1yHtwBWQkCRdoXO8mPA==
234
234
  dependencies:
235
235
  tagged-tag "^1.0.0"
236
236
 
237
237
  uint8array-extras@^1.5.0:
238
- version "1.5.0"
239
- resolved "https://registry.yarnpkg.com/uint8array-extras/-/uint8array-extras-1.5.0.tgz#10d2a85213de3ada304fea1c454f635c73839e86"
240
- integrity sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==
238
+ version "1.6.0"
239
+ resolved "https://registry.yarnpkg.com/uint8array-extras/-/uint8array-extras-1.6.0.tgz#ab895526255bddd698e75a02992ee73f1b112ebe"
240
+ integrity sha512-8iAasVS4wUx0gPLjH8Xtz2PeDzTSiy8QiLGu2DT3X5GU+egFEQhpnbtkehbAwjvDcxIERmCYcysiODFmE3Ud0Q==
241
241
 
242
242
  uuid@^9.0.0:
243
243
  version "9.0.1"
@@ -39,3 +39,15 @@ tasks:
39
39
  run:
40
40
  command: >-
41
41
  uv run --all-extras pyright
42
+ scripts-min-python:
43
+ description: >-
44
+ Verify that the standalone scripts consumers run inside tasks
45
+ still support the oldest Python they're used with
46
+ treeherder:
47
+ symbol: check(scripts-min-python)
48
+ run:
49
+ command: >-
50
+ uv run vermin --target=3.8- --violations --eval-annotations --no-tips
51
+ src/taskgraph/run-task/run-task src/taskgraph/run-task/fetch-content &&
52
+ uv run --no-project --python 3.8 python src/taskgraph/run-task/run-task --help > /dev/null &&
53
+ uv run --no-project --python 3.8 python src/taskgraph/run-task/fetch-content --help > /dev/null