squidient 2.5.2__tar.gz → 2.6.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. {squidient-2.5.2 → squidient-2.6.1}/PKG-INFO +1 -1
  2. {squidient-2.5.2 → squidient-2.6.1}/pyproject.toml +1 -1
  3. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/configure.py +15 -1
  4. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/connection/job.py +16 -2
  5. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/btest.html +4 -0
  6. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/test.html +4 -0
  7. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/definitions.py +13 -1
  8. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/monitor/monitor.py +25 -1
  9. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/reports/report.py +9 -0
  10. squidient-2.6.1/src/squidient/testing/benchmarkfailurenotifier.py +146 -0
  11. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/testing/benchmarktester.py +25 -1
  12. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/testing/instance.py +30 -0
  13. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/testing/instancemerger.py +19 -0
  14. {squidient-2.5.2 → squidient-2.6.1}/.gitignore +0 -0
  15. {squidient-2.5.2 → squidient-2.6.1}/LICENSE.txt +0 -0
  16. {squidient-2.5.2 → squidient-2.6.1}/README.md +0 -0
  17. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/__init__.py +0 -0
  18. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/builds/benchmarkbuild.py +0 -0
  19. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/builds/benchmarkbuilder.py +0 -0
  20. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/builds/build.py +0 -0
  21. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/builds/builder.py +0 -0
  22. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/builds/executablebuild.py +0 -0
  23. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/builds/predefinedbuild.py +0 -0
  24. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/cli.py +0 -0
  25. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/codecoverage/codecoverage.py +0 -0
  26. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/codecoverage/gcovrcodecoverage.py +0 -0
  27. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/codecoverage/intelcodecoverage.py +0 -0
  28. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/compare.py +0 -0
  29. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/comparewithdifftable.py +0 -0
  30. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/methods/absolute.py +0 -0
  31. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/methods/close.py +0 -0
  32. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/methods/diff.py +0 -0
  33. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/methods/l1norm.py +0 -0
  34. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/methods/l1relative.py +0 -0
  35. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/methods/l2norm.py +0 -0
  36. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/methods/l2relative.py +0 -0
  37. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/methods/linfnorm.py +0 -0
  38. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/methods/linfrelative.py +0 -0
  39. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/methods/power.py +0 -0
  40. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/methods/relative.py +0 -0
  41. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/methods/scalartolerance.py +0 -0
  42. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/methods/tolerance.py +0 -0
  43. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compare/methods/vectortolerance.py +0 -0
  44. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/compilation/cmakeoptions.py +0 -0
  45. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/connection/buildlock.py +0 -0
  46. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/connection/ssh.py +0 -0
  47. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/connection/sshconfigchecker.py +0 -0
  48. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/.squidient-initialized +0 -0
  49. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/alya-config/cmake-alamak.json +0 -0
  50. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/alya-config/cmake-chemint.json +0 -0
  51. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/alya-config/cmake-clang.json +0 -0
  52. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/alya-config/cmake-commdom.json +0 -0
  53. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/alya-config/cmake-gnu.json +0 -0
  54. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/alya-config/cmake-intel.json +0 -0
  55. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/alya-config/cmake-mumps.json +0 -0
  56. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/alya-config/cmake-nvhpc.json +0 -0
  57. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/alya-config/cmake-talp.json +0 -0
  58. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/benchmarks/benchmarks.json +0 -0
  59. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/builds/builds.json +0 -0
  60. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/builds/cc.json +0 -0
  61. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/builds/defaults.json +0 -0
  62. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/builds/predefined-old.json +0 -0
  63. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/builds/predefined.json +0 -0
  64. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/configurations/projects.json +0 -0
  65. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/configurations/systems.json +0 -0
  66. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/configurations/tests.json +0 -0
  67. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/email/report.html +0 -0
  68. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/icons/big.png +0 -0
  69. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/icons/small.png +0 -0
  70. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/index.html +0 -0
  71. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/terraform/.gitignore +0 -0
  72. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/terraform/aws-ec2-t3.small/.gitignore +0 -0
  73. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/terraform/aws-ec2-t3.small/t3.small/cloud-init-slurm-fedora.sh +0 -0
  74. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/terraform/aws-ec2-t3.small/t3.small/t3.small.tf +0 -0
  75. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/terraform/aws-ec2-t3.small/t3.small/terraform.tfvars +0 -0
  76. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/terraform/aws-ec2-t3.small-x2/t3.small/.gitignore +0 -0
  77. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/terraform/aws-ec2-t3.small-x2/t3.small/cloud-init-slurm-fedora.sh +0 -0
  78. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/terraform/aws-ec2-t3.small-x2/t3.small/replace-var.sh +0 -0
  79. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/terraform/aws-ec2-t3.small-x2/t3.small/t3.small.tf +0 -0
  80. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/terraform/aws-ec2-t3.small-x2/t3.small/terraform.tfvars +0 -0
  81. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/terraform/aws-ec2-t3.small-x3/t3.small/.gitignore +0 -0
  82. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/terraform/aws-ec2-t3.small-x3/t3.small/cloud-init-slurm-fedora.sh +0 -0
  83. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/terraform/aws-ec2-t3.small-x3/t3.small/t3.small.tf +0 -0
  84. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/terraform/aws-ec2-t3.small-x3/t3.small/terraform.tfvars +0 -0
  85. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/version/version.json +0 -0
  86. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/bb.html +0 -0
  87. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/benchmarks.html +0 -0
  88. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/btests.html +0 -0
  89. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/builds.html +0 -0
  90. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/cc-gcovr.html +0 -0
  91. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/cc-intel.html +0 -0
  92. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/files.html +0 -0
  93. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/canceled.png +0 -0
  94. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/created.png +0 -0
  95. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/failed.png +0 -0
  96. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/failed_black.png +0 -0
  97. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/failed_green.png +0 -0
  98. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/file_cloud.png +0 -0
  99. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/file_config.png +0 -0
  100. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/file_ko.png +0 -0
  101. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/file_ok.png +0 -0
  102. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/skipped.png +0 -0
  103. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/squidient_logo.png +0 -0
  104. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/success.png +0 -0
  105. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/success_black.png +0 -0
  106. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/timeout.png +0 -0
  107. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/tolerance.png +0 -0
  108. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/warning.png +0 -0
  109. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/warning_black.png +0 -0
  110. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/png/warning_green.png +0 -0
  111. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/svg/canceled.svg +0 -0
  112. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/svg/created.svg +0 -0
  113. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/svg/failed.svg +0 -0
  114. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/svg/failed_black.svg +0 -0
  115. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/svg/failed_green.svg +0 -0
  116. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/svg/manual.svg +0 -0
  117. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/svg/running.svg +0 -0
  118. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/svg/skipped.svg +0 -0
  119. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/svg/success.svg +0 -0
  120. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/svg/success_black.svg +0 -0
  121. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/svg/warning.svg +0 -0
  122. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/svg/warning_black.svg +0 -0
  123. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/img/svg/warning_green.svg +0 -0
  124. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/index.html +0 -0
  125. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/jquery.dynatable.css +0 -0
  126. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/jquery.dynatable.js +0 -0
  127. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/perf.html +0 -0
  128. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/tests.html +0 -0
  129. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/testsuite.css +0 -0
  130. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/data/web/testsuite.js +0 -0
  131. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/deployment/modulegenerator.py +0 -0
  132. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/git.py +0 -0
  133. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/init.py +0 -0
  134. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/monitor/monitordb.py +0 -0
  135. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/reports/monitorreport.py +0 -0
  136. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/sqreport/__init__.py +0 -0
  137. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/sqreport/squidientbasereport.py +0 -0
  138. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/sqreport/squidientcasereport.py +0 -0
  139. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/sqreport/squidientdetailedreport.py +0 -0
  140. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/sqreport/squidientfullreport.py +0 -0
  141. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/squidient.py +0 -0
  142. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/squidientmonitor.py +0 -0
  143. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/staging/staging.py +0 -0
  144. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/systems/partition.py +0 -0
  145. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/systems/platform.py +0 -0
  146. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/systems/qos.py +0 -0
  147. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/systems/queuetester.py +0 -0
  148. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/systems/reservation.py +0 -0
  149. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/systems/system.py +0 -0
  150. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/systems/systemmanager.py +0 -0
  151. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/terraform/terraform.py +0 -0
  152. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/testing/benchmarkinstance.py +0 -0
  153. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/testing/benchmarktest.py +0 -0
  154. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/testing/test.py +0 -0
  155. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/testing/tester.py +0 -0
  156. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/timers/__init__.py +0 -0
  157. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/timers/timer.py +0 -0
  158. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/utils/arguments.py +0 -0
  159. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/utils/lock.py +0 -0
  160. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/utils/log.py +0 -0
  161. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/utils/message.py +0 -0
  162. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/utils/package.py +0 -0
  163. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/utils/utils.py +0 -0
  164. {squidient-2.5.2 → squidient-2.6.1}/src/squidient/utils/watchdog.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: squidient
3
- Version: 2.5.2
3
+ Version: 2.6.1
4
4
  Summary: CI/CD framework for alya
5
5
  Project-URL: Homepage, https://alya.gitlab.bsc.es/alya/ci-cd/squidient
6
6
  Project-URL: Bug Tracker, https://alya.gitlab.bsc.es/alya/ci-cd/squidient/issues
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "squidient"
7
- version = "2.5.2"
7
+ version = "2.6.1"
8
8
  authors = [
9
9
  { name="Damien Dosimont", email="damien.dosimont@gmail.com" },
10
10
  ]
@@ -27,6 +27,7 @@ from typing import Optional, Sequence, Dict, Any
27
27
 
28
28
  from .builds.predefinedbuild import *
29
29
  from .builds.build import *
30
+ from .definitions import benchmark_failure_copy, benchmark_failure_sender
30
31
  from .utils.message import console
31
32
 
32
33
  from pathlib import Path
@@ -364,6 +365,8 @@ def generate_json():
364
365
  config["benchmarks"]["builds"] = predefined_benchmarks["all"]
365
366
  config["benchmarks"]["tests"] = []
366
367
  config["benchmarks"]["lowDiskUsage"] = True
368
+ config["benchmarks"]["failureSender"] = benchmark_failure_sender
369
+ config["benchmarks"]["failureCopy"] = benchmark_failure_copy
367
370
  # staging
368
371
  config["staging"] = {}
369
372
  config["staging"]["builds"] = predefined_builds["fast"]["gpp"]
@@ -391,7 +394,7 @@ def generate_json():
391
394
 
392
395
  def gitlab_config(pipeline, directory, alya_ref, alya_branch, flex=False, system="bsc", platform="gpp", hosts=None, install=False,
393
396
  type=None, api=None, reservation=False, project=None, alamak=False, alamak_branch=None,
394
- alamak_revision=None):
397
+ alamak_revision=None, notify_failures=False):
395
398
 
396
399
  predefined_benchmarks = PredefinedBuild("benchmarks").get_pb()
397
400
  predefined_builds = PredefinedBuild("builds").get_pb()
@@ -443,6 +446,9 @@ def gitlab_config(pipeline, directory, alya_ref, alya_branch, flex=False, system
443
446
  # various
444
447
  console.section("Configuring flex")
445
448
  config["flex"] = flex
449
+ config["benchmarks"]["notifyFailures"] = notify_failures
450
+ config["benchmarks"]["failureSender"] = benchmark_failure_sender
451
+ config["benchmarks"]["failureCopy"] = benchmark_failure_copy
446
452
  console.section("Configuring clean log")
447
453
  config["cleanLog"] = False
448
454
  console.section("Configuring system")
@@ -590,6 +596,7 @@ def configure_help():
590
596
  console.write("\t--api:[api] Define GitLab API")
591
597
  console.write("\t--alamak-branch:[branch] Define which alamak branch to use")
592
598
  console.write("\t--alamak-revision:[revision] Define which alamak revision to use")
599
+ console.write("\t--notify-failures Notify benchmark authors when a benchmark fails")
593
600
  console.write("\t--reservation Enable reservation")
594
601
  line_jump()
595
602
 
@@ -643,6 +650,7 @@ def _run_cli(argv: Sequence[str]) -> int:
643
650
  alamak = False
644
651
  alamak_branch = None
645
652
  alamak_revision = None
653
+ notify_failures = False
646
654
 
647
655
  for arg in argv[0:]:
648
656
  if "--pipeline:" in arg:
@@ -686,8 +694,13 @@ def _run_cli(argv: Sequence[str]) -> int:
686
694
  console.step("Project " + project)
687
695
  if "--alamak-branch:" in arg:
688
696
  alamak_branch = arg.replace("--alamak-branch:", "")
697
+ console.step("Alamak branch " + alamak_branch)
689
698
  if "--alamak-revision:" in arg:
690
699
  alamak_revision = arg.replace("--alamak-revision:", "")
700
+ console.step("Alamak revision " + alamak_revision)
701
+ if "--notify-failures" in arg:
702
+ notify_failures = True
703
+ console.step("Benchmark failure notifications enabled")
691
704
 
692
705
  gitlab_config(
693
706
  pipeline=pipeline,
@@ -706,6 +719,7 @@ def _run_cli(argv: Sequence[str]) -> int:
706
719
  alamak=alamak,
707
720
  alamak_branch=alamak_branch,
708
721
  alamak_revision=alamak_revision,
722
+ notify_failures=notify_failures,
709
723
  )
710
724
  return 0
711
725
 
@@ -87,12 +87,16 @@ class JobThread (threading.Thread):
87
87
  self._current_jobs = self.get_job_numbers()
88
88
  if self._current_jobs > self._max_jobs:
89
89
  time.sleep(20)
90
- output = self._ssh.ssh(
90
+ result = self._ssh.ssh(
91
91
  cmd=job["command"],
92
92
  system=self._system,
93
93
  platform=self._platform,
94
94
  server_path=job["server_path"],
95
- ).stdout
95
+ )
96
+ output = result.stdout
97
+ job["stdout"] = output
98
+ job["stderr"] = getattr(result, "stderr", "")
99
+ job["returncode"] = getattr(result, "returncode", 0)
96
100
  try:
97
101
  m = re.search(r"Submitted batch job (\d+)", output)
98
102
  if not m:
@@ -128,6 +132,7 @@ class Job:
128
132
  self._jobs_not_actualized = {}
129
133
  self._jobs_launched = {}
130
134
  self._jobs_not_launched = {}
135
+ self._job_submissions = {}
131
136
  self._job_list = {}
132
137
  self._input_queues = {}
133
138
  self._output_queues = {}
@@ -138,6 +143,7 @@ class Job:
138
143
  self._jobs_not_actualized[system] = {}
139
144
  self._jobs_launched[system] = {}
140
145
  self._jobs_not_launched[system] = {}
146
+ self._job_submissions[system] = {}
141
147
  self._job_list[system] = {}
142
148
  self._input_queues[system] = {}
143
149
  self._output_queues[system] = {}
@@ -174,6 +180,7 @@ class Job:
174
180
  self._jobs_not_launched[system][platform] = {}
175
181
  self._jobs_launched[system][platform] = {}
176
182
  self._jobs_not_actualized[system][platform] = {}
183
+ self._job_submissions[system][platform] = {}
177
184
  self._jobs[system][platform] = {}
178
185
  job = {}
179
186
  job["id"] = id
@@ -281,11 +288,18 @@ class Job:
281
288
  job_id = -1
282
289
  if id in self._jobs_launched[system][platform]:
283
290
  job_id = self._jobs_launched[system][platform][id]["job_id"]
291
+ self._job_submissions[system][platform][job_id] = self._jobs_launched[system][platform][id]
284
292
  self._jobs[system][platform][job_id] = job_id
285
293
  self._jobs_not_actualized[system][platform][job_id] = job_id
286
294
  self._jobs_launched[system][platform].pop(id)
287
295
  return job_id
288
296
 
297
+ def get_job_submission(self, job_id, system, platform):
298
+ try:
299
+ return self._job_submissions[system][platform].get(job_id, {})
300
+ except:
301
+ return {}
302
+
289
303
  def kill_all(self):
290
304
  for system in self._job_threads:
291
305
  for platform in self._job_threads[system]:
@@ -36,6 +36,7 @@
36
36
  <th>Openmp</th>
37
37
  <th>Json</th>
38
38
  <th>Job</th>
39
+ <th>Sbatch</th>
39
40
  <th>Output</th>
40
41
  <th>Alya</th>
41
42
  <th>Postprocess</th>
@@ -77,6 +78,9 @@
77
78
  dir=testsuiteJSON.benchmark_tests + name + "/" + t[x].build + "/" + t[x].mpi + "_" + t[x].openmp + "/"
78
79
  t[x].job=testsuiteJSON.setUrl(testsuiteJSON.FILE_CLOUD, dir + "/JOB.SB", null)
79
80
  t[x].json=testsuiteJSON.setUrl(testsuiteJSON.FILE_CONFIG, dir + "/" + name + ".json", null)
81
+ t[x].sbatch_out=testsuiteJSON.setUrl(testsuiteJSON.FILE_OK, dir + "/sbatch.out", null)
82
+ t[x].sbatch_err=testsuiteJSON.setUrl(testsuiteJSON.FILE_KO, dir + "/sbatch.err", null)
83
+ t[x].sbatch=t[x].sbatch_out + " " + t[x].sbatch_err
80
84
  t[x].output_out=testsuiteJSON.setUrl(testsuiteJSON.FILE_OK, dir + "/test.out", null)
81
85
  t[x].output_err=testsuiteJSON.setUrl(testsuiteJSON.FILE_KO, dir + "/test.err", null)
82
86
  t[x].output=t[x].output_out + " " + t[x].output_err
@@ -36,6 +36,7 @@
36
36
  <th>Openmp</th>
37
37
  <th>Json</th>
38
38
  <th>Job</th>
39
+ <th>Sbatch</th>
39
40
  <th>Output</th>
40
41
  <th>Alya</th>
41
42
  <th>Postprocess</th>
@@ -78,6 +79,9 @@
78
79
  dir=testsuiteJSON.tests + name + "/" + t[x].build + "/" + t[x].mpi + "_" + t[x].openmp + "/"
79
80
  t[x].job=testsuiteJSON.setUrl(testsuiteJSON.FILE_CLOUD, dir + "/JOB.SB", null)
80
81
  t[x].json=testsuiteJSON.setUrl(testsuiteJSON.FILE_CONFIG, dir + "/" + name + ".json", null)
82
+ t[x].sbatch_out=testsuiteJSON.setUrl(testsuiteJSON.FILE_OK, dir + "/sbatch.out", null)
83
+ t[x].sbatch_err=testsuiteJSON.setUrl(testsuiteJSON.FILE_KO, dir + "/sbatch.err", null)
84
+ t[x].sbatch=t[x].sbatch_out + " " + t[x].sbatch_err
81
85
  t[x].output_out=testsuiteJSON.setUrl(testsuiteJSON.FILE_OK, dir + "/test.out", null)
82
86
  t[x].output_err=testsuiteJSON.setUrl(testsuiteJSON.FILE_KO, dir + "/test.err", null)
83
87
  t[x].output=t[x].output_out + " " + t[x].output_err
@@ -67,4 +67,16 @@ squidient_local = "squidient/"
67
67
  squidient_remote = "squidient"
68
68
  cmake_link_dir = "ALYA_CMAKE_INSTALL_DIR"
69
69
  max_advised_builds = 2
70
- database_name = {"test": "rooster_test", "production": "rooster"}
70
+ database_name = {"test": "rooster_test", "production": "rooster"}
71
+ benchmark_failure_sender = "damien.dosimont@bsc.es"
72
+ benchmark_failure_copy = ["damien.dosimont@bsc.es"]
73
+ benchmark_failure_log_files = [
74
+ "sbatch.out",
75
+ "sbatch.err",
76
+ "test.out",
77
+ "test.err",
78
+ "alya.out",
79
+ "alya.err",
80
+ "post.out",
81
+ "post.err",
82
+ ]
@@ -44,6 +44,7 @@ class Monitor(Staging):
44
44
  self.run_all_monitor_tests(terraform)
45
45
  status = self.finish_all_monitor_tests()
46
46
  self._report.write_tests(status, self._tester.get_total(), self._tester.get_failed())
47
+ self.write_notification_failures()
47
48
  jump()
48
49
  self._db.end_session()
49
50
  self._report.valid(True)
@@ -105,6 +106,23 @@ class Monitor(Staging):
105
106
  self._tester.wait_tests()
106
107
  return self._tester.build_report()
107
108
 
109
+ def write_notification_failures(self):
110
+ if not hasattr(self._tester, "get_notification_failures"):
111
+ return
112
+ failures = self._tester.get_notification_failures()
113
+ if len(failures) > 0:
114
+ self._report.write_notification_failures(failures)
115
+
116
+ def validate(self):
117
+ failures = []
118
+ if hasattr(self._report, "get_notification_failures"):
119
+ failures = self._report.get_notification_failures()
120
+ for failure in failures:
121
+ console.error(failure.get("message", str(failure)))
122
+ if len(failures) > 0:
123
+ exit(1)
124
+ super().validate()
125
+
108
126
  def build(self, terraform):
109
127
  with self.timer("monitor.build"):
110
128
  self._db = MonitorDB(self._config, enable=False)
@@ -208,4 +226,10 @@ class Monitor(Staging):
208
226
  revision=self._report.get_alya_revision(),
209
227
  db=self._db,
210
228
  directories=self._test_directories,
211
- test_list=self._test_list)
229
+ test_list=self._test_list,
230
+ notify_failures=(
231
+ getattr(self, "_gitlab", False)
232
+ and self._config.get("benchmarks", {}).get("notifyFailures", False)
233
+ ),
234
+ notification_sender=self._config.get("benchmarks", {}).get("failureSender"),
235
+ notification_copy=self._config.get("benchmarks", {}).get("failureCopy"))
@@ -61,6 +61,8 @@ class Report:
61
61
  self._valid = self._valid and self._data["builds"]["status"]
62
62
  if "tests" in self._data:
63
63
  self._valid = self._valid and self._data["tests"]["status"]
64
+ if len(self._data.get("notificationFailures", [])) > 0:
65
+ self._valid = False
64
66
  self._data["valid"] = self._valid
65
67
  self.save()
66
68
 
@@ -107,3 +109,10 @@ class Report:
107
109
 
108
110
  def get_valid(self):
109
111
  return self._valid
112
+
113
+ def get_notification_failures(self):
114
+ return self._data.get("notificationFailures", [])
115
+
116
+ def write_notification_failures(self, failures):
117
+ self._data["notificationFailures"] = failures
118
+ self.valid(len(failures) == 0)
@@ -0,0 +1,146 @@
1
+ #########################################################################
2
+ # #
3
+ # This file is part of squidient. #
4
+ # #
5
+ # squidient is free software: you can redistribute it and/or modify #
6
+ # it under the terms of the GNU General Public License as published by #
7
+ # the Free Software Foundation, either version 3 of the License, or #
8
+ # (at your option) any later version. #
9
+ # #
10
+ # squidient is distributed in the hope that it will be useful, #
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of #
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
13
+ # GNU General Public License for more details. #
14
+ # #
15
+ # You should have received a copy of the GNU General Public License #
16
+ # along with squidient. If not, see <https://www.gnu.org/licenses/>. #
17
+ # #
18
+ #########################################################################
19
+
20
+ from email.message import EmailMessage
21
+ from ..definitions import benchmark_failure_log_files
22
+ import os
23
+ import shutil
24
+ import smtplib
25
+ import subprocess
26
+
27
+
28
+ class BenchmarkFailureNotificationError(Exception):
29
+
30
+ def __init__(self, recipients, subject, body, error):
31
+ self.recipients = recipients
32
+ self.subject = subject
33
+ self.body = body
34
+ self.error = str(error)
35
+ super().__init__(self.message())
36
+
37
+ def message(self):
38
+ return (
39
+ "Benchmark failure email could not be sent.\n"
40
+ "Recipients: " + ", ".join(self.recipients) + "\n"
41
+ "Subject: " + self.subject + "\n"
42
+ "Transport error: " + self.error + "\n"
43
+ "\n"
44
+ "Mail body:\n"
45
+ + self.body
46
+ )
47
+
48
+ def as_dict(self):
49
+ return {
50
+ "recipients": self.recipients,
51
+ "subject": self.subject,
52
+ "body": self.body,
53
+ "error": self.error,
54
+ "message": self.message(),
55
+ }
56
+
57
+
58
+ class BenchmarkFailureNotifier:
59
+
60
+ def __init__(
61
+ self,
62
+ report_dir,
63
+ from_address=None,
64
+ copy_recipients=None,
65
+ mail_sender=None):
66
+ self._report_dir = report_dir
67
+ self._from_address = from_address
68
+ if copy_recipients is None:
69
+ copy_recipients = []
70
+ self._copy_recipients = copy_recipients
71
+ self._mail_sender = mail_sender or self.send_mail
72
+
73
+ def notify(self, report):
74
+ recipients = self.recipients(report.get("authors", []))
75
+ if len(recipients) == 0:
76
+ return
77
+ subject = "[squidient] Benchmark failed: " + report.get("name", "unknown")
78
+ body = self.mail_body(report)
79
+ try:
80
+ self._mail_sender(recipients, subject, body)
81
+ except Exception as error:
82
+ raise BenchmarkFailureNotificationError(recipients, subject, body, error)
83
+
84
+ def recipients(self, authors):
85
+ recipients = []
86
+ for author in authors:
87
+ if isinstance(author, str):
88
+ candidate = author
89
+ elif isinstance(author, dict):
90
+ candidate = author.get("email", "")
91
+ else:
92
+ candidate = ""
93
+ if "@" in candidate and candidate not in recipients:
94
+ recipients.append(candidate)
95
+ for copy in self._copy_recipients:
96
+ if "@" in copy and copy not in recipients:
97
+ recipients.append(copy)
98
+ return recipients
99
+
100
+ def mail_body(self, report):
101
+ lines = [
102
+ "Benchmark: " + report.get("name", "unknown"),
103
+ "Error: " + report.get("error", "unknown"),
104
+ "",
105
+ ]
106
+ for instance in report.get("instances", []):
107
+ if instance.get("status", True):
108
+ continue
109
+ instance_dir = os.path.join(
110
+ self._report_dir,
111
+ report.get("name", ""),
112
+ str(instance.get("build", "")),
113
+ str(instance.get("mpi", "")) + "_" + str(instance.get("openmp", "")),
114
+ )
115
+ lines.extend([
116
+ "Instance: " + instance.get("name", "unknown"),
117
+ "Build: " + str(instance.get("build", "")),
118
+ "MPI/OpenMP: " + str(instance.get("mpi", "")) + "/" + str(instance.get("openmp", "")),
119
+ "Job ID: " + str(instance.get("jobId", "")),
120
+ "Error: " + str(instance.get("error", "")),
121
+ "",
122
+ ])
123
+ for filename in benchmark_failure_log_files:
124
+ lines.append("----- " + filename + " -----")
125
+ try:
126
+ with open(os.path.join(instance_dir, filename), "r", errors="replace") as log:
127
+ content = log.read()
128
+ except Exception:
129
+ content = "<missing>"
130
+ lines.append(content)
131
+ lines.append("")
132
+ return "\n".join(lines)
133
+
134
+ def send_mail(self, recipients, subject, body):
135
+ message = EmailMessage()
136
+ if self._from_address is not None:
137
+ message["From"] = self._from_address
138
+ message["To"] = ", ".join(recipients)
139
+ message["Subject"] = subject
140
+ message.set_content(body)
141
+ sendmail = shutil.which("sendmail")
142
+ if sendmail is not None:
143
+ subprocess.run([sendmail, "-t"], input=message.as_string(), text=True, check=True)
144
+ return
145
+ with smtplib.SMTP("localhost") as smtp:
146
+ smtp.send_message(message)
@@ -23,6 +23,7 @@ from ..definitions import *
23
23
  from ..git import *
24
24
  from .tester import Tester
25
25
  from .benchmarktest import BenchmarkTest
26
+ from .benchmarkfailurenotifier import BenchmarkFailureNotificationError, BenchmarkFailureNotifier
26
27
  import os
27
28
  import shutil
28
29
 
@@ -33,7 +34,7 @@ logger = logging.getLogger(logging_context)
33
34
  class BenchmarkTester(Tester):
34
35
 
35
36
  def __init__(self, systemManager, hash, job, ssh, config, benchmarks, revision, db, directories=["."],
36
- test_list=[]):
37
+ test_list=[], notify_failures=False, notification_sender=None, notification_copy=None, mail_sender=None):
37
38
 
38
39
  super().__init__(systemManager, hash, job, ssh, config, revision, directories, test_list)
39
40
  self._merge = False
@@ -45,6 +46,14 @@ class BenchmarkTester(Tester):
45
46
  self._report_dir = benchmark_test_report_dir
46
47
  self._source_dir = self._remote_dir + "/.repositories"
47
48
  self._db = db
49
+ self._notify_failures = notify_failures
50
+ self._notification_failures = []
51
+ self._failure_notifier = BenchmarkFailureNotifier(
52
+ self._report_dir,
53
+ from_address=notification_sender,
54
+ copy_recipients=notification_copy,
55
+ mail_sender=mail_sender,
56
+ )
48
57
 
49
58
  def get_benchmark(self, benchmark, fake=False):
50
59
  b = self._benchmarks[benchmark]
@@ -155,6 +164,7 @@ class BenchmarkTester(Tester):
155
164
  report[t.get_name()] = t.get_report()
156
165
  if not report[t.get_name()]["status"]:
157
166
  self._failed += 1
167
+ self.notify_failure(report[t.get_name()])
158
168
  self._status = report[t.get_name()]["status"] and self._status
159
169
  if t.get_relaunch():
160
170
  self._relaunch = True
@@ -163,3 +173,17 @@ class BenchmarkTester(Tester):
163
173
  json2js("benchmark_tests", f)
164
174
  print_line()
165
175
  return self._status
176
+
177
+ def notify_failure(self, report):
178
+ if not getattr(self, "_notify_failures", False):
179
+ return
180
+ try:
181
+ self._failure_notifier.notify(report)
182
+ except BenchmarkFailureNotificationError as error:
183
+ self._notification_failures.append(error.as_dict())
184
+ console.warning(error.message())
185
+ except Exception as error:
186
+ console.warning("Cannot send benchmark failure email: " + str(error))
187
+
188
+ def get_notification_failures(self):
189
+ return self._notification_failures
@@ -29,6 +29,9 @@ from ..systems.queuetester import *
29
29
 
30
30
  logger = logging.getLogger(logging_context)
31
31
 
32
+ SBATCH_OUTPUT_FILE = "sbatch.out"
33
+ SBATCH_ERROR_FILE = "sbatch.err"
34
+
32
35
 
33
36
  def file_contains(path, text):
34
37
  try:
@@ -99,6 +102,7 @@ class Instance:
99
102
  self._job_terminated = False
100
103
  self._job_launched = False
101
104
  self._job_id = 0
105
+ self._job_submission = {}
102
106
  self._relaunch = False
103
107
  self._retry = 0
104
108
  self._build_report = True
@@ -395,6 +399,13 @@ class Instance:
395
399
  self._job_id = self._job.get_job_id(system=self._system,
396
400
  platform=self._platform,
397
401
  id=self._id + "_" + str(self._retry))
402
+ if hasattr(self._job, "get_job_submission"):
403
+ self._job_submission = self._job.get_job_submission(job_id=self._job_id,
404
+ system=self._system,
405
+ platform=self._platform)
406
+ else:
407
+ self._job_submission = {}
408
+ self.write_sbatch_outputs()
398
409
  else:
399
410
  return "Scheduled"
400
411
  if self._job_id == -1:
@@ -442,6 +453,8 @@ class Instance:
442
453
  self._report["relaunched"] = self._retry > 0
443
454
  self._report["retries"] = self._retry
444
455
  self._report["jobId"] = self._job_id
456
+ self._report["sbatchOutput"] = SBATCH_OUTPUT_FILE
457
+ self._report["sbatchError"] = SBATCH_ERROR_FILE
445
458
 
446
459
  if self._remove_env:
447
460
  self._report["env"] = {}
@@ -495,6 +508,15 @@ class Instance:
495
508
  self.backup_outputs()
496
509
  self._build_report = False
497
510
 
511
+ def write_sbatch_outputs(self):
512
+ os.makedirs(self._report_path, exist_ok=True)
513
+ stdout = self._job_submission.get("stdout", "")
514
+ stderr = self._job_submission.get("stderr", "")
515
+ with open(self._report_path + "/" + SBATCH_OUTPUT_FILE, "w") as output:
516
+ output.write(stdout)
517
+ with open(self._report_path + "/" + SBATCH_ERROR_FILE, "w") as error:
518
+ error.write(stderr)
519
+
498
520
  def build_performance(self):
499
521
 
500
522
  self._report["performance"] = {}
@@ -610,6 +632,8 @@ class Instance:
610
632
  open(self._report_path + "/alya.err", 'a+').close()
611
633
  open(self._report_path + "/post.out", 'a+').close()
612
634
  open(self._report_path + "/post.err", 'a+').close()
635
+ open(self._report_path + "/" + SBATCH_OUTPUT_FILE, 'a+').close()
636
+ open(self._report_path + "/" + SBATCH_ERROR_FILE, 'a+').close()
613
637
  open(self._report_path + "/time.out", 'a+').close()
614
638
 
615
639
  def backup_outputs(self):
@@ -621,6 +645,8 @@ class Instance:
621
645
  "alya.err",
622
646
  "post.out",
623
647
  "post.err",
648
+ SBATCH_OUTPUT_FILE,
649
+ SBATCH_ERROR_FILE,
624
650
  "time.out",
625
651
  ]:
626
652
  source = os.path.join(self._report_path, filename)
@@ -655,6 +681,10 @@ class Instance:
655
681
  def set_job_id(self, job_id):
656
682
  self._job_id = job_id
657
683
 
684
+ def set_job_submission(self, job_submission):
685
+ self._job_submission = job_submission
686
+ self.write_sbatch_outputs()
687
+
658
688
  def update_status(self, launched=None, valid=None, terminated=None):
659
689
  if launched is not None:
660
690
  self._job_launched = launched
@@ -57,6 +57,7 @@ class InstanceMerger:
57
57
  self._job_terminated = False
58
58
  self._job_launched = False
59
59
  self._job_id = 0
60
+ self._job_submission = {}
60
61
  self._waiting_job_id = False
61
62
  self._exec_modules = []
62
63
 
@@ -164,8 +165,17 @@ class InstanceMerger:
164
165
  self._job_id = self._job.get_job_id(system=self._system,
165
166
  platform=self._platform,
166
167
  id=self._id)
168
+ if hasattr(self._job, "get_job_submission"):
169
+ self._job_submission = self._job.get_job_submission(job_id=self._job_id,
170
+ system=self._system,
171
+ platform=self._platform)
172
+ else:
173
+ self._job_submission = {}
174
+ self.write_sbatch_outputs()
167
175
  for instance in self._instances:
168
176
  self._instances[instance].set_job_id(self._job_id)
177
+ if hasattr(self._instances[instance], "set_job_submission"):
178
+ self._instances[instance].set_job_submission(self._job_submission)
169
179
  self._instances[instance].update_status(launched=True)
170
180
  else:
171
181
  return "Scheduled"
@@ -184,6 +194,15 @@ class InstanceMerger:
184
194
  return "Finished"
185
195
  return self._job_status
186
196
 
197
+ def write_sbatch_outputs(self):
198
+ os.makedirs(self._report_path, exist_ok=True)
199
+ stdout = self._job_submission.get("stdout", "")
200
+ stderr = self._job_submission.get("stderr", "")
201
+ with open(self._report_path + "/" + SBATCH_OUTPUT_FILE, "w") as output:
202
+ output.write(stdout)
203
+ with open(self._report_path + "/" + SBATCH_ERROR_FILE, "w") as error:
204
+ error.write(stderr)
205
+
187
206
  def get_internalId(self):
188
207
  return self._internalId
189
208
 
File without changes
File without changes
File without changes