assemblyline-core 4.7.2.dev17__tar.gz → 4.7.2.dev18__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/PKG-INFO +1 -1
  2. assemblyline_core-4.7.2.dev18/assemblyline_core/VERSION +1 -0
  3. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +5 -5
  4. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/scaler/scaler_server.py +14 -12
  5. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core.egg-info/PKG-INFO +1 -1
  6. assemblyline_core-4.7.2.dev17/assemblyline_core/VERSION +0 -1
  7. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/LICENCE.md +0 -0
  8. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/README.md +0 -0
  9. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/__init__.py +0 -0
  10. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/alerter/__init__.py +0 -0
  11. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/alerter/processing.py +0 -0
  12. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/alerter/run_alerter.py +0 -0
  13. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/archiver/__init__.py +0 -0
  14. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/archiver/run_archiver.py +0 -0
  15. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/badlist_client.py +0 -0
  16. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/dispatching/__init__.py +0 -0
  17. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/dispatching/client.py +0 -0
  18. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/dispatching/dispatcher.py +0 -0
  19. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/dispatching/schedules.py +0 -0
  20. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/expiry/__init__.py +0 -0
  21. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/expiry/run_expiry.py +0 -0
  22. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/ingester/__init__.py +0 -0
  23. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/ingester/constants.py +0 -0
  24. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/ingester/ingester.py +0 -0
  25. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/metrics/__init__.py +0 -0
  26. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/metrics/es_metrics.py +0 -0
  27. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  28. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/metrics/helper.py +0 -0
  29. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/metrics/metrics_server.py +0 -0
  30. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  31. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  32. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  33. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/replay/__init__.py +0 -0
  34. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/replay/client.py +0 -0
  35. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/replay/creator/__init__.py +0 -0
  36. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/replay/creator/run.py +0 -0
  37. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/replay/creator/run_worker.py +0 -0
  38. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/replay/loader/__init__.py +0 -0
  39. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/replay/loader/run.py +0 -0
  40. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/replay/loader/run_worker.py +0 -0
  41. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/replay/replay.py +0 -0
  42. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/safelist_client.py +0 -0
  43. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/scaler/__init__.py +0 -0
  44. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/scaler/collection.py +0 -0
  45. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  46. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  47. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/scaler/controllers/interface.py +0 -0
  48. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/scaler/run_scaler.py +0 -0
  49. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/server_base.py +0 -0
  50. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/signature_client.py +0 -0
  51. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/submission_client.py +0 -0
  52. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/tasking_client.py +0 -0
  53. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/updater/__init__.py +0 -0
  54. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/updater/helper.py +0 -0
  55. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/updater/run_updater.py +0 -0
  56. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/vacuum/__init__.py +0 -0
  57. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/vacuum/crawler.py +0 -0
  58. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/vacuum/department_map.py +0 -0
  59. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/vacuum/safelist.py +0 -0
  60. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/vacuum/stream_map.py +0 -0
  61. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/vacuum/worker.py +0 -0
  62. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/workflow/__init__.py +0 -0
  63. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core/workflow/run_workflow.py +0 -0
  64. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  65. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  66. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core.egg-info/requires.txt +0 -0
  67. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/assemblyline_core.egg-info/top_level.txt +0 -0
  68. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/setup.cfg +0 -0
  69. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/setup.py +0 -0
  70. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/test/test_alerter.py +0 -0
  71. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/test/test_badlist_client.py +0 -0
  72. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/test/test_expiry.py +0 -0
  73. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/test/test_replay.py +0 -0
  74. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/test/test_safelist_client.py +0 -0
  75. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/test/test_scaler.py +0 -0
  76. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/test/test_scheduler.py +0 -0
  77. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/test/test_signature_client.py +0 -0
  78. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/test/test_tasking_client.py +0 -0
  79. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/test/test_vacuum.py +0 -0
  80. {assemblyline_core-4.7.2.dev17 → assemblyline_core-4.7.2.dev18}/test/test_workflow.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: assemblyline-core
3
- Version: 4.7.2.dev17
3
+ Version: 4.7.2.dev18
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.7.2.dev18
@@ -287,10 +287,9 @@ def parse_cpu(string: str) -> float:
287
287
 
288
288
  class KubernetesController(ControllerInterface):
289
289
  def __init__(self, logger, namespace: str, prefix: str, priority: str, dependency_priority: str,
290
- cpu_reservation: float, linux_node_selector: Selector, labels=None, log_level="INFO", core_env={},
291
- cluster_pod_list=True, enable_pod_security=False,
292
- default_service_tolerations=[],
293
- priv_labels=None):
290
+ cpu_reservation: float, cpu_slack: float, linux_node_selector: Selector, labels=None,
291
+ log_level="INFO", core_env={}, cluster_pod_list=True, enable_pod_security=False,
292
+ default_service_tolerations=[], priv_labels=None):
294
293
  # Try loading a kubernetes connection from either the fact that we are running
295
294
  # inside of a cluster, or have a config file that tells us how
296
295
  try:
@@ -316,6 +315,7 @@ class KubernetesController(ControllerInterface):
316
315
  self.priority: str = priority
317
316
  self.dependency_priority: str = dependency_priority
318
317
  self.cpu_reservation: float = max(0.0, min(cpu_reservation, 1.0))
318
+ self.cpu_slack: float = max(1.0, cpu_slack + 1.0)
319
319
  self.logger = logger
320
320
  self.log_level: str = log_level
321
321
  self._labels: dict[str, str] = labels or {}
@@ -770,7 +770,7 @@ class KubernetesController(ControllerInterface):
770
770
  volume_mounts=mounts,
771
771
  security_context=security_context,
772
772
  resources=V1ResourceRequirements(
773
- limits={'cpu': cores, 'memory': f'{memory}Mi'},
773
+ limits={'cpu': cores*self.cpu_slack, 'memory': f'{memory}Mi'},
774
774
  requests={'cpu': cores*self.cpu_reservation, 'memory': f'{min_memory}Mi'},
775
775
  ),
776
776
  liveness_probe=health_probe,
@@ -334,18 +334,20 @@ class ScalerServer(ThreadedCoreBase):
334
334
 
335
335
  if KUBERNETES_AL_CONFIG:
336
336
  self.log.info(f"Loading Kubernetes cluster interface on namespace: {NAMESPACE}")
337
- self.controller = KubernetesController(logger=self.log, prefix='alsvc_', labels=labels,
338
- namespace=NAMESPACE, priority='al-service-priority',
339
- dependency_priority='al-core-priority',
340
- cpu_reservation=self.config.services.cpu_reservation,
341
- linux_node_selector=self.config.core.scaler.linux_node_selector,
342
- log_level=self.config.logging.log_level,
343
- core_env=core_env,
344
- cluster_pod_list=self.config.core.scaler.cluster_pod_list,
345
- enable_pod_security=self.config.core.scaler.enable_pod_security,
346
- default_service_tolerations=service_defaults_config.tolerations,
347
- priv_labels=priv_labels
348
- )
337
+ self.controller = KubernetesController(
338
+ logger=self.log, prefix='alsvc_', labels=labels,
339
+ namespace=NAMESPACE, priority='al-service-priority',
340
+ dependency_priority='al-core-priority',
341
+ cpu_reservation=self.config.services.cpu_reservation,
342
+ cpu_slack=self.config.services.cpu_slack,
343
+ linux_node_selector=self.config.core.scaler.linux_node_selector,
344
+ log_level=self.config.logging.log_level,
345
+ core_env=core_env,
346
+ cluster_pod_list=self.config.core.scaler.cluster_pod_list,
347
+ enable_pod_security=self.config.core.scaler.enable_pod_security,
348
+ default_service_tolerations=service_defaults_config.tolerations,
349
+ priv_labels=priv_labels
350
+ )
349
351
 
350
352
  # Add global configuration for privileged services
351
353
  # Check if the ConfigMap already exists, if it does, update it
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: assemblyline-core
3
- Version: 4.7.2.dev17
3
+ Version: 4.7.2.dev18
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.7.2.dev17