taskcluster-taskgraph 12.2.0__tar.gz → 13.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (254) hide show
  1. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/CHANGELOG.md +18 -0
  2. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/PKG-INFO +1 -1
  3. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/howto/docker.rst +59 -0
  4. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/migrations.rst +36 -1
  5. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py +7 -3
  6. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/__init__.py +1 -1
  7. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/config.py +13 -2
  8. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/run-task/run-task +17 -82
  9. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/run/__init__.py +1 -38
  10. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/run/common.py +78 -39
  11. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/run/run_task.py +12 -27
  12. taskcluster_taskgraph-13.0.0/src/taskgraph/util/caches.py +57 -0
  13. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/taskcluster.py +3 -3
  14. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/verify.py +1 -1
  15. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/python/Dockerfile +1 -1
  16. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/kinds/check/kind.yml +1 -1
  17. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/kinds/codecov/kind.yml +1 -0
  18. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/kinds/doc/kind.yml +1 -1
  19. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/kinds/test/kind.yml +1 -1
  20. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/self_taskgraph/custom_target_tasks.py +1 -1
  21. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_scripts_run_task.py +51 -0
  22. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_transforms_run.py +2 -63
  23. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_transforms_run_run_task.py +80 -9
  24. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_taskcluster.py +11 -11
  25. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/.codespell-ignore-words.txt +0 -0
  26. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/.github/workflows/codeql-analysis.yml +0 -0
  27. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/.github/workflows/pypi-publish.yml +0 -0
  28. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/.gitignore +0 -0
  29. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/.hatch_build.py +0 -0
  30. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/.pre-commit-config.yaml +0 -0
  31. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/.readthedocs.yaml +0 -0
  32. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/.taskcluster.yml +0 -0
  33. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/.yamllint +0 -0
  34. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/CODE_OF_CONDUCT.md +0 -0
  35. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/CONTRIBUTING.rst +0 -0
  36. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/LICENSE +0 -0
  37. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/Makefile +0 -0
  38. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/README.rst +0 -0
  39. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/concepts/index.rst +0 -0
  40. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/concepts/kind.rst +0 -0
  41. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/concepts/loading.rst +0 -0
  42. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/concepts/optimization.rst +0 -0
  43. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/concepts/scopes.rst +0 -0
  44. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/concepts/task-graphs.rst +0 -0
  45. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/concepts/taskcluster.rst +0 -0
  46. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/concepts/transforms.rst +0 -0
  47. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/conf.py +0 -0
  48. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/contributing.rst +0 -0
  49. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/glossary.rst +0 -0
  50. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/howto/bootstrap-taskgraph.rst +0 -0
  51. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/howto/create-actions.rst +0 -0
  52. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/howto/create-tasks.rst +0 -0
  53. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/howto/debugging.rst +0 -0
  54. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/howto/index.rst +0 -0
  55. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/howto/run-locally.rst +0 -0
  56. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/howto/send-notifications.rst +0 -0
  57. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/howto/use-fetches.rst +0 -0
  58. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/index.rst +0 -0
  59. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/cli.rst +0 -0
  60. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/index.rst +0 -0
  61. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/optimization-strategies.rst +0 -0
  62. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/parameters.rst +0 -0
  63. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/source/modules.rst +0 -0
  64. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/source/taskgraph.actions.rst +0 -0
  65. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/source/taskgraph.loader.rst +0 -0
  66. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/source/taskgraph.optimize.rst +0 -0
  67. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/source/taskgraph.rst +0 -0
  68. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/source/taskgraph.transforms.rst +0 -0
  69. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/source/taskgraph.transforms.run.rst +0 -0
  70. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/source/taskgraph.util.rst +0 -0
  71. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/transforms/chunking.rst +0 -0
  72. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/transforms/from_deps.rst +0 -0
  73. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/transforms/index.rst +0 -0
  74. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/transforms/matrix.rst +0 -0
  75. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/reference/transforms/task_context.rst +0 -0
  76. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/tutorials/connecting-taskcluster.rst +0 -0
  77. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/tutorials/creating-a-task-graph.rst +0 -0
  78. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/tutorials/example-taskcluster.yml +0 -0
  79. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/tutorials/getting-started.rst +0 -0
  80. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/docs/tutorials/index.rst +0 -0
  81. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/make.bat +0 -0
  82. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/packages/pytest-taskgraph/README.md +0 -0
  83. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/packages/pytest-taskgraph/pyproject.toml +0 -0
  84. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/packages/pytest-taskgraph/src/pytest_taskgraph/__init__.py +0 -0
  85. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/vcs.py +0 -0
  86. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/packages/pytest-taskgraph/uv.lock +0 -0
  87. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/pyproject.toml +0 -0
  88. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/actions/__init__.py +0 -0
  89. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/actions/add_new_jobs.py +0 -0
  90. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/actions/cancel.py +0 -0
  91. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/actions/cancel_all.py +0 -0
  92. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/actions/rebuild_cached_tasks.py +0 -0
  93. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/actions/registry.py +0 -0
  94. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/actions/retrigger.py +0 -0
  95. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/actions/util.py +0 -0
  96. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/create.py +0 -0
  97. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/decision.py +0 -0
  98. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/docker.py +0 -0
  99. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/filter_tasks.py +0 -0
  100. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/generator.py +0 -0
  101. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/graph.py +0 -0
  102. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/loader/__init__.py +0 -0
  103. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/loader/default.py +0 -0
  104. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/loader/transform.py +0 -0
  105. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/main.py +0 -0
  106. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/morph.py +0 -0
  107. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/optimize/__init__.py +0 -0
  108. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/optimize/base.py +0 -0
  109. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/optimize/strategies.py +0 -0
  110. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/parameters.py +0 -0
  111. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/run-task/fetch-content +0 -0
  112. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/run-task/hgrc +0 -0
  113. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/run-task/robustcheckout.py +0 -0
  114. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/target_tasks.py +0 -0
  115. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/task.py +0 -0
  116. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/taskgraph.py +0 -0
  117. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/__init__.py +0 -0
  118. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/base.py +0 -0
  119. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/cached_tasks.py +0 -0
  120. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/chunking.py +0 -0
  121. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/code_review.py +0 -0
  122. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/docker_image.py +0 -0
  123. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/fetch.py +0 -0
  124. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/from_deps.py +0 -0
  125. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/matrix.py +0 -0
  126. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/notify.py +0 -0
  127. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/run/index_search.py +0 -0
  128. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/run/toolchain.py +0 -0
  129. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/task.py +0 -0
  130. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/transforms/task_context.py +0 -0
  131. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/__init__.py +0 -0
  132. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/archive.py +0 -0
  133. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/attributes.py +0 -0
  134. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/cached_tasks.py +0 -0
  135. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/copy.py +0 -0
  136. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/dependencies.py +0 -0
  137. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/docker.py +0 -0
  138. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/hash.py +0 -0
  139. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/keyed_by.py +0 -0
  140. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/parameterization.py +0 -0
  141. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/path.py +0 -0
  142. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/python_path.py +0 -0
  143. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/readonlydict.py +0 -0
  144. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/schema.py +0 -0
  145. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/set_name.py +0 -0
  146. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/shell.py +0 -0
  147. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/taskgraph.py +0 -0
  148. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/templates.py +0 -0
  149. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/time.py +0 -0
  150. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/treeherder.py +0 -0
  151. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/vcs.py +0 -0
  152. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/workertypes.py +0 -0
  153. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/src/taskgraph/util/yaml.py +0 -0
  154. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/config.yml +0 -0
  155. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/REGISTRY +0 -0
  156. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/decision/Dockerfile +0 -0
  157. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/decision/HASH +0 -0
  158. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/decision/README.md +0 -0
  159. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/decision/VERSION +0 -0
  160. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/decision/system-setup.sh +0 -0
  161. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/fetch/Dockerfile +0 -0
  162. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/image_builder/README.rst +0 -0
  163. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/index-task/Dockerfile +0 -0
  164. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/index-task/README +0 -0
  165. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/index-task/insert-indexes.js +0 -0
  166. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/index-task/package.json +0 -0
  167. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/index-task/yarn.lock +0 -0
  168. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/run-task/Dockerfile +0 -0
  169. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/run-task/system-setup.sh +0 -0
  170. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/skopeo/Dockerfile +0 -0
  171. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/skopeo/policy.json +0 -0
  172. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/docker/skopeo/push_image.sh +0 -0
  173. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/kinds/docker-image/kind.yml +0 -0
  174. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/kinds/fetch/kind.yml +0 -0
  175. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/kinds/push-image/kind.yml +0 -0
  176. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/scripts/codecov-upload.py +0 -0
  177. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/scripts/external_tools/tooltool.py +0 -0
  178. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/self_taskgraph/__init__.py +0 -0
  179. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/self_taskgraph/custom_parameters.py +0 -0
  180. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/self_taskgraph/transforms/push_image.py +0 -0
  181. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/test/params/main-repo-pull-request-untrusted.yml +0 -0
  182. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/test/params/main-repo-pull-request.yml +0 -0
  183. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/test/params/main-repo-push.yml +0 -0
  184. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/test/params/main-repo-release-pytest-taskgraph.yml +0 -0
  185. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/taskcluster/test/params/main-repo-release-taskcluster-taskgraph.yml +0 -0
  186. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/template/cookiecutter.json +0 -0
  187. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/template/hooks/post_gen_project.py +0 -0
  188. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/template/{{cookiecutter.project_name}}/taskcluster/config.yml +0 -0
  189. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/template/{{cookiecutter.project_name}}/taskcluster/docker/linux/Dockerfile +0 -0
  190. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/template/{{cookiecutter.project_name}}/taskcluster/kinds/docker-image/kind.yml +0 -0
  191. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/template/{{cookiecutter.project_name}}/taskcluster/kinds/hello/kind.yml +0 -0
  192. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/template/{{cookiecutter.project_name}}/taskcluster/{{cookiecutter.project_slug}}_taskgraph/transforms/hello.py +0 -0
  193. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/template/{{cookiecutter.project_name}}/taskcluster.github.yml +0 -0
  194. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/template/{{cookiecutter.project_name}}/taskcluster.hgmo.yml +0 -0
  195. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/__init__.py +0 -0
  196. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/automationrelevance.json +0 -0
  197. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/conftest.py +0 -0
  198. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/data/task_context.yml +0 -0
  199. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/data/taskcluster/config.yml +0 -0
  200. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/data/taskcluster/docker/hello-world/Dockerfile +0 -0
  201. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/data/taskcluster/docker/hello-world-tag/Dockerfile +0 -0
  202. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/data/taskcluster/docker/hello-world-tag/REGISTRY +0 -0
  203. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/data/taskcluster/docker/hello-world-tag/VERSION +0 -0
  204. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/data/taskcluster/scripts/toolchain/run.ps1 +0 -0
  205. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/data/taskcluster/scripts/toolchain/run.sh +0 -0
  206. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/data/taskcluster/test_taskgraph/transforms/foo.py +0 -0
  207. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/data/testmod/thing.py +0 -0
  208. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/mockedopen.py +0 -0
  209. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_actions_rebuild_cached_tasks.py +0 -0
  210. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_actions_registry.py +0 -0
  211. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_config.py +0 -0
  212. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_create.py +0 -0
  213. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_decision.py +0 -0
  214. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_docker.py +0 -0
  215. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_generator.py +0 -0
  216. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_graph.py +0 -0
  217. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_main.py +0 -0
  218. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_morph.py +0 -0
  219. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_optimize.py +0 -0
  220. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_optimize_strategies.py +0 -0
  221. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_parameters.py +0 -0
  222. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_scripts_fetch_content.py +0 -0
  223. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_target_tasks.py +0 -0
  224. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_taskgraph.py +0 -0
  225. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_transform_chunking.py +0 -0
  226. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_transform_docker_image.py +0 -0
  227. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_transform_task_context.py +0 -0
  228. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_transforms_base.py +0 -0
  229. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_transforms_cached_tasks.py +0 -0
  230. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_transforms_fetch.py +0 -0
  231. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_transforms_from_deps.py +0 -0
  232. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_transforms_matrix.py +0 -0
  233. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_transforms_notify.py +0 -0
  234. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_transforms_run_toolchain.py +0 -0
  235. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_transforms_task.py +0 -0
  236. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_archive.py +0 -0
  237. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_attributes.py +0 -0
  238. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_cached_tasks.py +0 -0
  239. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_copy.py +0 -0
  240. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_dependencies.py +0 -0
  241. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_docker.py +0 -0
  242. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_parameterization.py +0 -0
  243. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_path.py +0 -0
  244. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_python_path.py +0 -0
  245. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_readonlydict.py +0 -0
  246. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_schema.py +0 -0
  247. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_templates.py +0 -0
  248. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_time.py +0 -0
  249. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_treeherder.py +0 -0
  250. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_vcs.py +0 -0
  251. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_verify.py +0 -0
  252. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_workertypes.py +0 -0
  253. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/test/test_util_yaml.py +0 -0
  254. {taskcluster_taskgraph-12.2.0 → taskcluster_taskgraph-13.0.0}/uv.lock +0 -0
@@ -1,5 +1,23 @@
1
1
  # Change Log
2
2
 
3
+ ## [13.0.0] - 2025-01-23
4
+
5
+ ### Added
6
+
7
+ - automatically interpolate {task_workdir} in environment (#630)
8
+ - add caches for common package managers (#623)
9
+ - support setting repo wide default values for cache selection (#623)
10
+ - support list of caches in `use-caches` key (#623)
11
+
12
+ ### Fixed
13
+
14
+ - `get_ancestors` now returns tasks keyed by taskid, to avoid missing tasks when multiple tasks exist with the same label in a graph (#633)
15
+
16
+ ### Removed
17
+
18
+ - unused resource-monitor code (#636)
19
+ - `cache-dotcache` key (#623)
20
+
3
21
  ## [12.2.0] - 2025-01-15
4
22
 
5
23
  ### Added
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: taskcluster-taskgraph
3
- Version: 12.2.0
3
+ Version: 13.0.0
4
4
  Summary: Build taskcluster taskgraphs
5
5
  Project-URL: Repository, https://github.com/taskcluster/taskgraph
6
6
  Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues
@@ -201,6 +201,65 @@ process the ``Dockerfile`` and handle the special syntax. Whereas the
201
201
  only generated once and then re-used by all subsequent pushes until the image
202
202
  is modified.
203
203
 
204
+ Layering images on top of each other
205
+ ....................................
206
+
207
+ The ``parent`` value on a docker-image task can refer to another image's ``name``.
208
+ The parent image can then be referenced from the dependent image's Dockerfile
209
+ as ``$DOCKER_IMAGE_PARENT``. Whenever the base image's definition (or
210
+ something it includes) changes, that automatically also triggers a
211
+ rebuild of the dependent image.
212
+
213
+ For example, ``kind.yml`` could include:
214
+
215
+ .. code-block:: yaml
216
+
217
+ tasks:
218
+ base:
219
+ symbol: I(base)
220
+ extras:
221
+ symbol: I(extras)
222
+ parent: base
223
+
224
+ And the ``extras`` image's Dockerfile:
225
+
226
+ .. code-block::
227
+
228
+ FROM $DOCKER_IMAGE_PARENT
229
+
230
+ RUN apt-get update && apt-get install -y extra software && apt-get clean
231
+
232
+
233
+ .. note::
234
+ The ``taskgraph build-image`` command doesn't handle this syntax, so it won't work for those images.
235
+ In taskcluster this is handled by the ``image_builder`` tool, see
236
+ `the image_builder docker image <https://hub.docker.com/r/mozillareleases/image_builder>`_ and
237
+ `its source code <https://searchfox.org/mozilla-central/source/taskcluster/docker/image_builder>`_.
238
+
239
+
240
+ Reusing image definitions
241
+ .........................
242
+
243
+ The same ``Dockerfile`` can be used for multiple images, by setting ``definition``, for example:
244
+
245
+ .. code-block:: yaml
246
+
247
+ tasks:
248
+ debian12:
249
+ symbol: I(deb12)
250
+ definition: debian
251
+ args:
252
+ BASE_IMAGE: debian:12
253
+ debian11:
254
+ symbol: I(deb11)
255
+ definition: debian
256
+ args:
257
+ BASE_IMAGE: debian:11
258
+
259
+ Both the ``debian11`` and ``debian12`` images will be built from the definition in
260
+ ``taskcluster/docker/debian``, but they'll be passed different ``BASE_IMAGE``
261
+ variables (as in ``docker build --build-arg``).
262
+
204
263
  Context Directory Hashing
205
264
  ~~~~~~~~~~~~~~~~~~~~~~~~~
206
265
 
@@ -3,6 +3,41 @@ Migration Guide
3
3
 
4
4
  This page can help when migrating Taskgraph across major versions.
5
5
 
6
+ 12.x -> 13.x
7
+ ------------
8
+
9
+ * Remove all ``run.cache-dotcache`` keys. If it was set to ``true``, replace it
10
+ with:
11
+
12
+ .. code-block:: yaml
13
+
14
+ run:
15
+ use-caches: [checkout, <caches>]
16
+
17
+ Where caches can be any of ``cargo``, ``pip``, ``uv`` or ``npm``. If the task
18
+ was setting up ``.cache`` for another tool, a mount will need to be created
19
+ for it manually. If ``use-caches`` was previously set to ``false``, omit
20
+ ``checkout`` in the example above. If ``use-caches`` was previously set to
21
+ ``true``, replace ``true`` with the value above (including ``checkout``).
22
+ * Invert any usage of the dict keys and values returned by `get_ancestors`:
23
+
24
+ For example, if you were using:
25
+
26
+ .. code-block:: python
27
+
28
+ for label, taskid in get_ancestors(...):
29
+ ...
30
+
31
+ Change it to:
32
+
33
+ .. code-block:: python
34
+
35
+ for taskid, label in get_ancestors(...):
36
+ ...
37
+
38
+ Note that due to this change `get_ancestors` may return multiple tasks with
39
+ the same label now, which your code may need to deal with.
40
+
6
41
  11.x -> 12.x
7
42
  ------------
8
43
 
@@ -110,7 +145,7 @@ This page can help when migrating Taskgraph across major versions.
110
145
  5.x -> 6.x
111
146
  ----------
112
147
 
113
- * Replace all uses of ``command-context` with the more generalized ``task-context``
148
+ * Replace all uses of ``command-context`` with the more generalized ``task-context``
114
149
 
115
150
  4.x -> 5.x
116
151
  ----------
@@ -204,7 +204,11 @@ def make_transform_config(parameters, graph_config):
204
204
  if extra_params:
205
205
  parameters.update(extra_params)
206
206
  if extra_graph_config:
207
- graph_config._config.update(extra_graph_config)
207
+ # We need this intermediate variable because `GraphConfig` is
208
+ # frozen and we can't set attributes on it.
209
+ new_graph_config = merge(graph_config._config, extra_graph_config)
210
+ graph_config._config.update(new_graph_config)
211
+
208
212
  return TransformConfig(
209
213
  "test",
210
214
  str(here),
@@ -220,12 +224,12 @@ def make_transform_config(parameters, graph_config):
220
224
 
221
225
  @pytest.fixture
222
226
  def run_transform(make_transform_config):
223
- def inner(func, tasks, config=None):
227
+ def inner(func, tasks, config=None, **kwargs):
224
228
  if not isinstance(tasks, list):
225
229
  tasks = [tasks]
226
230
 
227
231
  if not config:
228
- config = make_transform_config()
232
+ config = make_transform_config(**kwargs)
229
233
  return list(func(config, tasks))
230
234
 
231
235
  return inner
@@ -2,7 +2,7 @@
2
2
  # License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
4
 
5
- __version__ = "12.2.0"
5
+ __version__ = "13.0.0"
6
6
 
7
7
  # Maximum number of dependencies a single task can have
8
8
  # https://docs.taskcluster.net/docs/reference/platform/queue/api#createTask
@@ -12,6 +12,7 @@ from typing import Dict
12
12
  from voluptuous import All, Any, Extra, Length, Optional, Required
13
13
 
14
14
  from .util import path
15
+ from .util.caches import CACHES
15
16
  from .util.python_path import find_object
16
17
  from .util.schema import Schema, optionally_keyed_by, validate_schema
17
18
  from .util.yaml import load_yaml
@@ -74,6 +75,16 @@ graph_config_schema = Schema(
74
75
  "index-path-regexes",
75
76
  description="Regular expressions matching index paths to be summarized.",
76
77
  ): [str],
78
+ Optional(
79
+ "run",
80
+ description="Configuration related to the 'run' transforms.",
81
+ ): {
82
+ Optional(
83
+ "use-caches",
84
+ description="List of caches to enable, or a boolean to "
85
+ "enable/disable all of them.",
86
+ ): Any(bool, list(CACHES.keys())),
87
+ },
77
88
  Required("repositories"): All(
78
89
  {
79
90
  str: {
@@ -106,8 +117,8 @@ class GraphConfig:
106
117
  def __contains__(self, name):
107
118
  return name in self._config
108
119
 
109
- def get(self, name):
110
- return self._config.get(name)
120
+ def get(self, name, default=None):
121
+ return self._config.get(name, default)
111
122
 
112
123
  def register(self):
113
124
  """
@@ -1026,64 +1026,6 @@ def install_pip_requirements(repositories):
1026
1026
  run_required_command(b"pip-install", cmd)
1027
1027
 
1028
1028
 
1029
- def maybe_run_resource_monitoring():
1030
- """Run the resource monitor if available.
1031
-
1032
- Discussion in https://github.com/taskcluster/taskcluster-rfcs/pull/160
1033
- and https://bugzil.la/1648051
1034
-
1035
- """
1036
- if "MOZ_FETCHES" not in os.environ:
1037
- return
1038
- if "RESOURCE_MONITOR_OUTPUT" not in os.environ:
1039
- return
1040
-
1041
- prefix = b"resource_monitor"
1042
-
1043
- executable = "{}/resource-monitor/resource-monitor{}".format(
1044
- os.environ.get("MOZ_FETCHES_DIR"), ".exe" if IS_WINDOWS else ""
1045
- )
1046
-
1047
- if not os.path.exists(executable) or not os.access(executable, os.X_OK):
1048
- print_line(prefix, b"%s not executable\n" % executable.encode("utf-8"))
1049
- return
1050
- args = [
1051
- executable,
1052
- "-process",
1053
- str(os.getpid()),
1054
- "-output",
1055
- os.environ["RESOURCE_MONITOR_OUTPUT"],
1056
- ]
1057
- print_line(prefix, b"Resource monitor starting: %s\n" % str(args).encode("utf-8"))
1058
- # Avoid environment variables the payload doesn't need.
1059
- del os.environ["RESOURCE_MONITOR_OUTPUT"]
1060
-
1061
- # Without CREATE_NEW_PROCESS_GROUP Windows signals will attempt to kill run-task, too.
1062
- process = subprocess.Popen(
1063
- args,
1064
- # Disable buffering because we want to receive output
1065
- # as it is generated so timestamps in logs are
1066
- # accurate.
1067
- bufsize=0,
1068
- stdout=subprocess.PIPE,
1069
- stderr=subprocess.STDOUT,
1070
- creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if IS_WINDOWS else 0,
1071
- cwd=os.getcwd(),
1072
- )
1073
-
1074
- def capture_output():
1075
- fh = io.TextIOWrapper(process.stdout, encoding="latin1")
1076
- while True:
1077
- data = fh.readline().encode("latin1")
1078
- if data == b"":
1079
- break
1080
- print_line(prefix, data)
1081
-
1082
- monitor_process = Thread(target=capture_output)
1083
- monitor_process.start()
1084
- return process
1085
-
1086
-
1087
1029
  def _display_python_version():
1088
1030
  print_line(
1089
1031
  b"setup", b"Python version: %s\n" % platform.python_version().encode("utf-8")
@@ -1091,10 +1033,10 @@ def _display_python_version():
1091
1033
 
1092
1034
 
1093
1035
  def main(args):
1094
- os.environ["TASK_WORKDIR"] = os.getcwd()
1036
+ task_workdir = os.environ["TASK_WORKDIR"] = os.getcwd()
1095
1037
  print_line(
1096
1038
  b"setup",
1097
- b"run-task started in %s\n" % os.environ["TASK_WORKDIR"].encode("utf-8"),
1039
+ b"run-task started in %s\n" % task_workdir.encode("utf-8"),
1098
1040
  )
1099
1041
  print_line(
1100
1042
  b"setup",
@@ -1313,20 +1255,26 @@ def main(args):
1313
1255
  for repo in repositories:
1314
1256
  vcs_checkout_from_args(repo)
1315
1257
 
1316
- resource_process = None
1317
-
1318
- try:
1319
- for k in ["MOZ_FETCHES_DIR", "UPLOAD_DIR"] + [
1320
- "{}_PATH".format(repository["project"].upper())
1321
- for repository in repositories
1322
- ]:
1323
- if k in os.environ:
1324
- os.environ[k] = os.path.abspath(os.environ[k])
1258
+ # Interpolate environment variables with defined substitution patterns. For
1259
+ # example, the variable `CACHE_DIR={task_workdir}/.cache` will be
1260
+ # interpolated with the task's working directory.
1261
+ env_subs = {
1262
+ "task_workdir": task_workdir,
1263
+ }
1264
+ for k, v in os.environ.items():
1265
+ for subname, subval in env_subs.items():
1266
+ # We check for an exact match rather than using a format string to
1267
+ # avoid accidentally trying to interpolate environment variables
1268
+ # that happen to contain brackets for some other reason.
1269
+ pattern = f"{{{subname}}}"
1270
+ if pattern in v:
1271
+ os.environ[k] = v.replace(pattern, subval)
1325
1272
  print_line(
1326
1273
  b"setup",
1327
1274
  b"%s is %s\n" % (k.encode("utf-8"), os.environ[k].encode("utf-8")),
1328
1275
  )
1329
1276
 
1277
+ try:
1330
1278
  if "MOZ_FETCHES" in os.environ:
1331
1279
  fetch_artifacts()
1332
1280
 
@@ -1334,21 +1282,8 @@ def main(args):
1334
1282
  # fetches to grab dependencies.
1335
1283
  install_pip_requirements(repositories)
1336
1284
 
1337
- resource_process = maybe_run_resource_monitoring()
1338
-
1339
1285
  return run_command(b"task", task_args, cwd=args.task_cwd)
1340
1286
  finally:
1341
- if resource_process:
1342
- print_line(b"resource_monitor", b"terminating\n")
1343
- if IS_WINDOWS:
1344
- # .terminate() on Windows is not a graceful shutdown, due to
1345
- # differences in signals. CTRL_BREAK_EVENT will work provided
1346
- # the subprocess is in a different process group, so this script
1347
- # isn't also killed.
1348
- os.kill(resource_process.pid, signal.CTRL_BREAK_EVENT)
1349
- else:
1350
- resource_process.terminate()
1351
- resource_process.wait()
1352
1287
  fetches_dir = os.environ.get("MOZ_FETCHES_DIR")
1353
1288
  if fetches_dir and os.path.isdir(fetches_dir):
1354
1289
  print_line(b"fetches", b"removing %s\n" % fetches_dir.encode("utf-8"))
@@ -160,43 +160,6 @@ def set_label(config, tasks):
160
160
  yield task
161
161
 
162
162
 
163
- @transforms.add
164
- def add_resource_monitor(config, tasks):
165
- for task in tasks:
166
- if task.get("attributes", {}).get("resource-monitor"):
167
- worker_implementation, worker_os = worker_type_implementation(
168
- config.graph_config, task["worker-type"]
169
- )
170
- # Normalise worker os so that linux-bitbar and similar use linux tools.
171
- if worker_os:
172
- worker_os = worker_os.split("-")[0]
173
- if "win7" in task["worker-type"]:
174
- arch = "32"
175
- else:
176
- arch = "64"
177
- task.setdefault("fetches", {})
178
- task["fetches"].setdefault("toolchain", [])
179
- task["fetches"]["toolchain"].append(f"{worker_os}{arch}-resource-monitor")
180
-
181
- if worker_implementation == "docker-worker":
182
- artifact_source = "/builds/worker/monitoring/resource-monitor.json"
183
- else:
184
- artifact_source = "monitoring/resource-monitor.json"
185
- task["worker"].setdefault("artifacts", [])
186
- task["worker"]["artifacts"].append(
187
- {
188
- "name": "public/monitoring/resource-monitor.json",
189
- "type": "file",
190
- "path": artifact_source,
191
- }
192
- )
193
- # Set env for output file
194
- task["worker"].setdefault("env", {})
195
- task["worker"]["env"]["RESOURCE_MONITOR_OUTPUT"] = artifact_source
196
-
197
- yield task
198
-
199
-
200
163
  def get_attribute(dict, key, attributes, attribute_name):
201
164
  """Get `attribute_name` from the given `attributes` dict, and if there
202
165
  is a corresponding value, set `key` in `dict` to that value."""
@@ -367,7 +330,7 @@ def use_fetches(config, tasks):
367
330
  "task-reference": json.dumps(task_fetches, sort_keys=True)
368
331
  }
369
332
 
370
- env.setdefault("MOZ_FETCHES_DIR", "fetches")
333
+ env.setdefault("MOZ_FETCHES_DIR", "{task_workdir}/fetches")
371
334
 
372
335
  yield task
373
336
 
@@ -7,9 +7,12 @@ worker implementation they operate on, and take the same three parameters, for
7
7
  consistency.
8
8
  """
9
9
 
10
- import hashlib
11
10
  import json
11
+ from typing import Any, Dict, List, Union
12
12
 
13
+ from taskgraph.transforms.base import TransformConfig
14
+ from taskgraph.util import path
15
+ from taskgraph.util.caches import CACHES, get_checkout_dir
13
16
  from taskgraph.util.taskcluster import get_artifact_prefix
14
17
 
15
18
 
@@ -31,10 +34,10 @@ def add_cache(task, taskdesc, name, mount_point, skip_untrusted=False):
31
34
  skip_untrusted (bool): Whether cache is used in untrusted environments
32
35
  (default: False). Only applies to docker-worker.
33
36
  """
34
- if not task["run"].get("use-caches", True):
35
- return
36
-
37
37
  worker = task["worker"]
38
+ if worker["implementation"] not in ("docker-worker", "generic-worker"):
39
+ # caches support not implemented
40
+ return
38
41
 
39
42
  if worker["implementation"] == "docker-worker":
40
43
  taskdesc["worker"].setdefault("caches", []).append(
@@ -54,10 +57,6 @@ def add_cache(task, taskdesc, name, mount_point, skip_untrusted=False):
54
57
  }
55
58
  )
56
59
 
57
- else:
58
- # Caches not implemented
59
- pass
60
-
61
60
 
62
61
  def add_artifacts(config, task, taskdesc, path):
63
62
  taskdesc["worker"].setdefault("artifacts", []).append(
@@ -91,46 +90,19 @@ def support_vcs_checkout(config, task, taskdesc, repo_configs, sparse=False):
91
90
  reserved for ``run-task`` tasks.
92
91
  """
93
92
  worker = task["worker"]
94
- is_mac = worker["os"] == "macosx"
93
+ assert worker["os"] in ("linux", "macosx", "windows")
95
94
  is_win = worker["os"] == "windows"
96
- is_linux = worker["os"] == "linux"
97
95
  is_docker = worker["implementation"] == "docker-worker"
98
- assert is_mac or is_win or is_linux
99
96
 
97
+ checkoutdir = get_checkout_dir(task)
100
98
  if is_win:
101
- checkoutdir = "./build"
102
99
  hgstore = "y:/hg-shared"
103
100
  elif is_docker:
104
- checkoutdir = "{workdir}/checkouts".format(**task["run"])
105
101
  hgstore = f"{checkoutdir}/hg-store"
106
102
  else:
107
- checkoutdir = "./checkouts"
108
103
  hgstore = f"{checkoutdir}/hg-shared"
109
104
 
110
- vcsdir = checkoutdir + "/" + get_vcsdir_name(worker["os"])
111
- cache_name = "checkouts"
112
-
113
- # Robust checkout does not clean up subrepositories, so ensure that tasks
114
- # that checkout different sets of paths have separate caches.
115
- # See https://bugzilla.mozilla.org/show_bug.cgi?id=1631610
116
- if len(repo_configs) > 1:
117
- checkout_paths = {
118
- "\t".join([repo_config.path, repo_config.prefix])
119
- for repo_config in sorted(
120
- repo_configs.values(), key=lambda repo_config: repo_config.path
121
- )
122
- }
123
- checkout_paths_str = "\n".join(checkout_paths).encode("utf-8")
124
- digest = hashlib.sha256(checkout_paths_str).hexdigest()
125
- cache_name += f"-repos-{digest}"
126
-
127
- # Sparse checkouts need their own cache because they can interfere
128
- # with clients that aren't sparse aware.
129
- if sparse:
130
- cache_name += "-sparse"
131
-
132
- add_cache(task, taskdesc, cache_name, checkoutdir)
133
-
105
+ vcsdir = f"{checkoutdir}/{get_vcsdir_name(worker['os'])}"
134
106
  env = taskdesc["worker"].setdefault("env", {})
135
107
  env.update(
136
108
  {
@@ -138,7 +110,8 @@ def support_vcs_checkout(config, task, taskdesc, repo_configs, sparse=False):
138
110
  "REPOSITORIES": json.dumps(
139
111
  {repo.prefix: repo.name for repo in repo_configs.values()}
140
112
  ),
141
- "VCS_PATH": vcsdir,
113
+ # If vcsdir is already absolute this will return it unmodified.
114
+ "VCS_PATH": path.join("{task_workdir}", vcsdir),
142
115
  }
143
116
  )
144
117
  for repo_config in repo_configs.values():
@@ -162,3 +135,69 @@ def support_vcs_checkout(config, task, taskdesc, repo_configs, sparse=False):
162
135
  # only some worker platforms have taskcluster-proxy enabled
163
136
  if task["worker"]["implementation"] in ("docker-worker",):
164
137
  taskdesc["worker"]["taskcluster-proxy"] = True
138
+
139
+ return vcsdir
140
+
141
+
142
+ def should_use_cache(
143
+ name: str,
144
+ use_caches: Union[bool, List[str]],
145
+ has_checkout: bool,
146
+ ) -> bool:
147
+ # Never enable the checkout cache if there's no clone. This allows
148
+ # 'checkout' to be specified as a default cache without impacting
149
+ # irrelevant tasks.
150
+ if name == "checkout" and not has_checkout:
151
+ return False
152
+
153
+ if isinstance(use_caches, bool):
154
+ return use_caches
155
+
156
+ return name in use_caches
157
+
158
+
159
+ def support_caches(
160
+ config: TransformConfig, task: Dict[str, Any], taskdesc: Dict[str, Any]
161
+ ):
162
+ """Add caches for common tools."""
163
+ run = task["run"]
164
+ worker = task["worker"]
165
+ workdir = run.get("workdir")
166
+ base_cache_dir = ".task-cache"
167
+ if worker["implementation"] == "docker-worker":
168
+ workdir = workdir or "/builds/worker"
169
+ base_cache_dir = path.join(workdir, base_cache_dir)
170
+
171
+ use_caches = run.get("use-caches")
172
+ if use_caches is None:
173
+ # Use project default values for filtering caches, default to
174
+ # checkout cache if no selection is specified.
175
+ use_caches = (
176
+ config.graph_config.get("taskgraph", {})
177
+ .get("run", {})
178
+ .get("use-caches", ["checkout"])
179
+ )
180
+
181
+ for name, cache_cfg in CACHES.items():
182
+ if not should_use_cache(name, use_caches, run["checkout"]):
183
+ continue
184
+
185
+ if "cache_dir" in cache_cfg:
186
+ assert callable(cache_cfg["cache_dir"])
187
+ cache_dir = cache_cfg["cache_dir"](task)
188
+ else:
189
+ cache_dir = f"{base_cache_dir}/{name}"
190
+
191
+ if "cache_name" in cache_cfg:
192
+ assert callable(cache_cfg["cache_name"])
193
+ cache_name = cache_cfg["cache_name"](config, task)
194
+ else:
195
+ cache_name = name
196
+
197
+ if cache_cfg.get("env"):
198
+ env = taskdesc["worker"].setdefault("env", {})
199
+ # If cache_dir is already absolute, the `.join` call returns it as
200
+ # is. In that case, {task_workdir} will get interpolated by
201
+ # run-task.
202
+ env[cache_cfg["env"]] = path.join("{task_workdir}", cache_dir)
203
+ add_cache(task, taskdesc, cache_name, cache_dir)
@@ -11,9 +11,13 @@ import os
11
11
  from voluptuous import Any, Optional, Required
12
12
 
13
13
  from taskgraph.transforms.run import run_task_using
14
- from taskgraph.transforms.run.common import support_vcs_checkout
14
+ from taskgraph.transforms.run.common import (
15
+ support_caches,
16
+ support_vcs_checkout,
17
+ )
15
18
  from taskgraph.transforms.task import taskref_or_string
16
19
  from taskgraph.util import path, taskcluster
20
+ from taskgraph.util.caches import CACHES
17
21
  from taskgraph.util.schema import Schema
18
22
 
19
23
  EXEC_COMMANDS = {
@@ -24,12 +28,11 @@ EXEC_COMMANDS = {
24
28
  run_task_schema = Schema(
25
29
  {
26
30
  Required("using"): "run-task",
27
- # if true, add a cache at ~worker/.cache, which is where things like pip
28
- # tend to hide their caches. This cache is never added for level-1 tasks.
29
- # TODO Once bug 1526028 is fixed, this and 'use-caches' should be merged.
30
- Required("cache-dotcache"): bool,
31
- # Whether or not to use caches.
32
- Optional("use-caches"): bool,
31
+ # Which caches to use. May take a boolean in which case either all
32
+ # (True) or no (False) caches will be used. Alternatively, it can
33
+ # accept a list of caches to enable. Defaults to only the checkout cache
34
+ # enabled.
35
+ Optional("use-caches", "caches"): Any(bool, list(CACHES.keys())),
33
36
  # if true (the default), perform a checkout on the worker
34
37
  Required("checkout"): Any(bool, {str: dict}),
35
38
  Optional(
@@ -70,7 +73,7 @@ def common_setup(config, task, taskdesc, command):
70
73
  for (repo, config) in run["checkout"].items()
71
74
  }
72
75
 
73
- support_vcs_checkout(
76
+ vcs_path = support_vcs_checkout(
74
77
  config,
75
78
  task,
76
79
  taskdesc,
@@ -78,7 +81,6 @@ def common_setup(config, task, taskdesc, command):
78
81
  sparse=bool(run["sparse-profile"]),
79
82
  )
80
83
 
81
- vcs_path = taskdesc["worker"]["env"]["VCS_PATH"]
82
84
  for repo_config in repo_configs.values():
83
85
  checkout_path = path.join(vcs_path, repo_config.path)
84
86
  command.append(f"--{repo_config.prefix}-checkout={checkout_path}")
@@ -104,11 +106,11 @@ def common_setup(config, task, taskdesc, command):
104
106
  if "cwd" in run:
105
107
  command.extend(("--task-cwd", run["cwd"]))
106
108
 
109
+ support_caches(config, task, taskdesc)
107
110
  taskdesc["worker"].setdefault("env", {})["MOZ_SCM_LEVEL"] = config.params["level"]
108
111
 
109
112
 
110
113
  worker_defaults = {
111
- "cache-dotcache": False,
112
114
  "checkout": True,
113
115
  "sparse-profile": None,
114
116
  "run-as-root": False,
@@ -135,16 +137,6 @@ def docker_worker_run_task(config, task, taskdesc):
135
137
  command = run.pop("run-task-command", ["/usr/local/bin/run-task"])
136
138
  common_setup(config, task, taskdesc, command)
137
139
 
138
- if run.get("cache-dotcache"):
139
- worker["caches"].append(
140
- {
141
- "type": "persistent",
142
- "name": "{project}-dotcache".format(**config.params),
143
- "mount-point": "{workdir}/.cache".format(**run),
144
- "skip-untrusted": True,
145
- }
146
- )
147
-
148
140
  run_command = run["command"]
149
141
 
150
142
  # dict is for the case of `{'task-reference': str}`.
@@ -177,13 +169,6 @@ def generic_worker_run_task(config, task, taskdesc):
177
169
  common_setup(config, task, taskdesc, command)
178
170
 
179
171
  worker.setdefault("mounts", [])
180
- if run.get("cache-dotcache"):
181
- worker["mounts"].append(
182
- {
183
- "cache-name": "{project}-dotcache".format(**config.params),
184
- "directory": "{workdir}/.cache".format(**run),
185
- }
186
- )
187
172
  worker["mounts"].append(
188
173
  {
189
174
  "content": {