assemblyline-core 4.7.5.dev16__tar.gz → 4.7.5.dev18__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/PKG-INFO +1 -1
  2. assemblyline_core-4.7.5.dev18/assemblyline_core/VERSION +1 -0
  3. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/scaler/scaler_server.py +5 -1
  4. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core.egg-info/PKG-INFO +1 -1
  5. assemblyline_core-4.7.5.dev16/assemblyline_core/VERSION +0 -1
  6. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/LICENCE.md +0 -0
  7. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/README.md +0 -0
  8. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/__init__.py +0 -0
  9. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/alerter/__init__.py +0 -0
  10. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/alerter/processing.py +0 -0
  11. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/alerter/run_alerter.py +0 -0
  12. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/archiver/__init__.py +0 -0
  13. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/archiver/run_archiver.py +0 -0
  14. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/badlist_client.py +0 -0
  15. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/dispatching/__init__.py +0 -0
  16. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/dispatching/client.py +0 -0
  17. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/dispatching/dispatcher.py +0 -0
  18. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/dispatching/schedules.py +0 -0
  19. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/expiry/__init__.py +0 -0
  20. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/expiry/run_expiry.py +0 -0
  21. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/ingester/__init__.py +0 -0
  22. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/ingester/constants.py +0 -0
  23. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/ingester/ingester.py +0 -0
  24. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/metrics/__init__.py +0 -0
  25. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/metrics/es_metrics.py +0 -0
  26. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  27. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/metrics/helper.py +0 -0
  28. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/metrics/metrics_server.py +0 -0
  29. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  30. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  31. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  32. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/replay/__init__.py +0 -0
  33. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/replay/client.py +0 -0
  34. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/replay/creator/__init__.py +0 -0
  35. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/replay/creator/run.py +0 -0
  36. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/replay/creator/run_worker.py +0 -0
  37. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/replay/loader/__init__.py +0 -0
  38. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/replay/loader/run.py +0 -0
  39. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/replay/loader/run_worker.py +0 -0
  40. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/replay/replay.py +0 -0
  41. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/safelist_client.py +0 -0
  42. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/scaler/__init__.py +0 -0
  43. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/scaler/collection.py +0 -0
  44. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  45. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  46. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/scaler/controllers/interface.py +0 -0
  47. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  48. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/scaler/run_scaler.py +0 -0
  49. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/server_base.py +0 -0
  50. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/signature_client.py +0 -0
  51. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/submission_client.py +0 -0
  52. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/tasking_client.py +0 -0
  53. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/updater/__init__.py +0 -0
  54. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/updater/helper.py +0 -0
  55. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/updater/run_updater.py +0 -0
  56. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/vacuum/__init__.py +0 -0
  57. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/vacuum/crawler.py +0 -0
  58. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/vacuum/department_map.py +0 -0
  59. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/vacuum/safelist.py +0 -0
  60. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/vacuum/stream_map.py +0 -0
  61. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/vacuum/worker.py +0 -0
  62. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/workflow/__init__.py +0 -0
  63. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/workflow/run_workflow.py +0 -0
  64. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  65. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  66. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core.egg-info/requires.txt +0 -0
  67. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core.egg-info/top_level.txt +0 -0
  68. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/setup.cfg +0 -0
  69. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/setup.py +0 -0
  70. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/test/test_alerter.py +0 -0
  71. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/test/test_badlist_client.py +0 -0
  72. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/test/test_expiry.py +0 -0
  73. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/test/test_replay.py +0 -0
  74. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/test/test_safelist_client.py +0 -0
  75. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/test/test_scaler.py +0 -0
  76. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/test/test_scheduler.py +0 -0
  77. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/test/test_signature_client.py +0 -0
  78. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/test/test_tasking_client.py +0 -0
  79. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/test/test_vacuum.py +0 -0
  80. {assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/test/test_workflow.py +0 -0
{assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: assemblyline-core
3
- Version: 4.7.5.dev16
3
+ Version: 4.7.5.dev18
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
assemblyline_core-4.7.5.dev18/assemblyline_core/VERSION ADDED
@@ -0,0 +1 @@
1
+ 4.7.5.dev18
{assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core/scaler/scaler_server.py RENAMED
@@ -314,6 +314,7 @@ class ScalerServer(ThreadedCoreBase):
314
314
  priv_labels = {}
315
315
 
316
316
  service_defaults_config = self.config.core.scaler.service_defaults
317
+ self.max_pending = max(1, self.config.core.scaler.max_pending)
317
318
 
318
319
  # If Scaler has envs that set service-server env, then that should override configured values
319
320
  if SERVICE_API_HOST:
@@ -807,7 +808,10 @@ class ScalerServer(ThreadedCoreBase):
807
808
  # Make adjustments to the targets until everything is satisified
808
809
  # or we don't have the resouces to make more adjustments
809
810
  def trim(prof: list[ServiceProfile]):
811
+ # only consider services where more are desired
810
812
  prof = [_p for _p in prof if _p.desired_instances > targets[_p.name]]
813
+ # only consider services where there is room for more pending containers
814
+ prof = [_p for _p in prof if _p.running_instances + self.max_pending > targets[_p.name]]
811
815
  drop = [_p for _p in prof if _p.cpu > free_cpu or _p.ram > free_memory]
812
816
  if drop:
813
817
  summary = {_p.name: (_p.cpu, _p.ram) for _p in drop}
@@ -817,7 +821,7 @@ class ScalerServer(ThreadedCoreBase):
817
821
 
818
822
  remaining_profiles: list[ServiceProfile] = trim(list(all_profiles.values()))
819
823
  while remaining_profiles:
820
- # TODO do we need to add balancing metrics other than 'least running' for this? probably
824
+ # Prioritize by adding resources to services with the least instances requested first.
821
825
  remaining_profiles.sort(key=lambda _p: targets[_p.name])
822
826
 
823
827
  # Add one for the profile at the bottom
{assemblyline_core-4.7.5.dev16 → assemblyline_core-4.7.5.dev18}/assemblyline_core.egg-info/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: assemblyline-core
3
- Version: 4.7.5.dev16
3
+ Version: 4.7.5.dev18
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
assemblyline_core-4.7.5.dev16/assemblyline_core/VERSION DELETED
@@ -1 +0,0 @@
1
- 4.7.5.dev16