assemblyline-core 4.5.1.dev307__tar.gz → 4.5.1.dev309__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of assemblyline-core might be problematic. Click here for more details.

Files changed (88) hide show
  1. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/PKG-INFO +1 -1
  2. assemblyline-core-4.5.1.dev309/assemblyline_core/VERSION +1 -0
  3. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +4 -2
  4. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/scaler/scaler_server.py +6 -1
  5. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/updater/run_updater.py +4 -1
  6. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core.egg-info/PKG-INFO +1 -1
  7. assemblyline-core-4.5.1.dev307/assemblyline_core/VERSION +0 -1
  8. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/LICENCE.md +0 -0
  9. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/README.md +0 -0
  10. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/__init__.py +0 -0
  11. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/alerter/__init__.py +0 -0
  12. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/alerter/processing.py +0 -0
  13. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/alerter/run_alerter.py +0 -0
  14. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/archiver/__init__.py +0 -0
  15. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/archiver/run_archiver.py +0 -0
  16. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/badlist_client.py +0 -0
  17. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/dispatching/__init__.py +0 -0
  18. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/dispatching/__main__.py +0 -0
  19. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/dispatching/client.py +0 -0
  20. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/dispatching/dispatcher.py +0 -0
  21. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/dispatching/schedules.py +0 -0
  22. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/dispatching/timeout.py +0 -0
  23. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/expiry/__init__.py +0 -0
  24. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/expiry/run_expiry.py +0 -0
  25. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/ingester/__init__.py +0 -0
  26. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/ingester/__main__.py +0 -0
  27. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/ingester/constants.py +0 -0
  28. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/ingester/ingester.py +0 -0
  29. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/metrics/__init__.py +0 -0
  30. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/metrics/es_metrics.py +0 -0
  31. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  32. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/metrics/helper.py +0 -0
  33. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/metrics/metrics_server.py +0 -0
  34. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  35. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  36. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  37. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/plumber/__init__.py +0 -0
  38. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/plumber/run_plumber.py +0 -0
  39. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/replay/__init__.py +0 -0
  40. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/replay/client.py +0 -0
  41. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/replay/creator/__init__.py +0 -0
  42. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/replay/creator/run.py +0 -0
  43. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/replay/creator/run_worker.py +0 -0
  44. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/replay/loader/__init__.py +0 -0
  45. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/replay/loader/run.py +0 -0
  46. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/replay/loader/run_worker.py +0 -0
  47. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/replay/replay.py +0 -0
  48. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/safelist_client.py +0 -0
  49. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/scaler/__init__.py +0 -0
  50. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/scaler/collection.py +0 -0
  51. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  52. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  53. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/scaler/controllers/interface.py +0 -0
  54. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/scaler/run_scaler.py +0 -0
  55. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/server_base.py +0 -0
  56. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/signature_client.py +0 -0
  57. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/submission_client.py +0 -0
  58. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/tasking_client.py +0 -0
  59. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/updater/__init__.py +0 -0
  60. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/updater/helper.py +0 -0
  61. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/vacuum/__init__.py +0 -0
  62. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/vacuum/crawler.py +0 -0
  63. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/vacuum/department_map.py +0 -0
  64. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/vacuum/safelist.py +0 -0
  65. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/vacuum/stream_map.py +0 -0
  66. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/vacuum/worker.py +0 -0
  67. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/workflow/__init__.py +0 -0
  68. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core/workflow/run_workflow.py +0 -0
  69. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  70. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  71. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core.egg-info/requires.txt +0 -0
  72. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/assemblyline_core.egg-info/top_level.txt +0 -0
  73. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/setup.cfg +0 -0
  74. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/setup.py +0 -0
  75. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/test/test_alerter.py +0 -0
  76. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/test/test_badlist_client.py +0 -0
  77. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/test/test_dispatcher.py +0 -0
  78. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/test/test_expiry.py +0 -0
  79. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/test/test_plumber.py +0 -0
  80. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/test/test_replay.py +0 -0
  81. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/test/test_safelist_client.py +0 -0
  82. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/test/test_scaler.py +0 -0
  83. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/test/test_scheduler.py +0 -0
  84. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/test/test_signature_client.py +0 -0
  85. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/test/test_simulation.py +0 -0
  86. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/test/test_vacuum.py +0 -0
  87. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/test/test_worker_ingest.py +0 -0
  88. {assemblyline-core-4.5.1.dev307 → assemblyline-core-4.5.1.dev309}/test/test_worker_submit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev307
3
+ Version: 4.5.1.dev309
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.5.1.dev309
@@ -241,7 +241,8 @@ def parse_cpu(string: str) -> float:
241
241
  class KubernetesController(ControllerInterface):
242
242
  def __init__(self, logger, namespace: str, prefix: str, priority: str, dependency_priority: str,
243
243
  cpu_reservation: float, linux_node_selector: Selector, labels=None, log_level="INFO", core_env={},
244
- default_service_account=None, cluster_pod_list=True, default_service_tolerations = []):
244
+ default_service_account=None, cluster_pod_list=True, default_service_tolerations = [],
245
+ priv_labels=None):
245
246
  # Try loading a kubernetes connection from either the fact that we are running
246
247
  # inside of a cluster, or have a config file that tells us how
247
248
  try:
@@ -270,6 +271,7 @@ class KubernetesController(ControllerInterface):
270
271
  self.logger = logger
271
272
  self.log_level: str = log_level
272
273
  self._labels: dict[str, str] = labels or {}
274
+ self._priv_labels: dict[str, str] = priv_labels or {}
273
275
  self.linux_node_selector = linux_node_selector
274
276
  self.apps_api = client.AppsV1Api()
275
277
  self.api = client.CoreV1Api()
@@ -800,7 +802,7 @@ class KubernetesController(ControllerInterface):
800
802
  all_labels['component'] = service_name
801
803
  if core_mounts:
802
804
  all_labels['privilege'] = 'core'
803
- all_labels.update({k: v for k, v in (_l.split("=") for _l in self.config.core.scaler.privileged_services_additional_labels)})
805
+ all_labels.update(self._priv_labels)
804
806
  all_labels.update(labels or {})
805
807
 
806
808
  # Build set of volumes, first the global mounts, then the core specific ones,
@@ -297,6 +297,7 @@ class ScalerServer(ThreadedCoreBase):
297
297
  'section': 'service',
298
298
  'privilege': 'service'
299
299
  }
300
+ priv_labels = {}
300
301
 
301
302
  service_defaults_config = self.config.core.scaler.service_defaults
302
303
 
@@ -309,6 +310,9 @@ class ScalerServer(ThreadedCoreBase):
309
310
  if self.config.core.scaler.additional_labels:
310
311
  labels.update({k: v for k, v in (_l.split("=") for _l in self.config.core.scaler.additional_labels)})
311
312
 
313
+ if self.config.core.scaler.privileged_services_additional_labels:
314
+ priv_labels.update({k: v for k, v in (_l.split("=") for _l in self.config.core.scaler.privileged_services_additional_labels)})
315
+
312
316
  if KUBERNETES_AL_CONFIG:
313
317
  self.log.info(f"Loading Kubernetes cluster interface on namespace: {NAMESPACE}")
314
318
  self.controller = KubernetesController(logger=self.log, prefix=SERVICE_PREFIX, labels=labels,
@@ -320,7 +324,8 @@ class ScalerServer(ThreadedCoreBase):
320
324
  core_env=core_env,
321
325
  cluster_pod_list=self.config.core.scaler.cluster_pod_list,
322
326
  default_service_account=self.config.services.service_account,
323
- default_service_tolerations=service_defaults_config.tolerations
327
+ default_service_tolerations=service_defaults_config.tolerations,
328
+ priv_labels=priv_labels
324
329
  )
325
330
 
326
331
  # Add global configuration for privileged services
@@ -457,7 +457,10 @@ class ServiceUpdater(ThreadedCoreBase):
457
457
  if 'KUBERNETES_SERVICE_HOST' in os.environ and NAMESPACE:
458
458
  extra_labels = {}
459
459
  if self.config.core.scaler.additional_labels:
460
- extra_labels = {k: v for k, v in (_l.split("=") for _l in self.config.core.scaler.additional_labels)}
460
+ extra_labels.update({k: v for k, v in (_l.split("=") for _l in self.config.core.scaler.additional_labels)})
461
+
462
+ if self.config.core.scaler.privileged_services_additional_labels:
463
+ extra_labels.update({k: v for k, v in (_l.split("=") for _l in self.config.core.scaler.privileged_services_additional_labels)})
461
464
 
462
465
  # If Updater has envs that set the service-server to use HTTPS, then assume a Root CA needs to be mounted
463
466
  if SERVICE_API_HOST and SERVICE_API_HOST.startswith('https'):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev307
3
+ Version: 4.5.1.dev309
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.5.1.dev307