assemblyline-core 4.5.1.dev441__tar.gz → 4.5.1.dev443__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/PKG-INFO +1 -1
  2. assemblyline_core-4.5.1.dev443/assemblyline_core/VERSION +1 -0
  3. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/scaler/controllers/docker_ctl.py +2 -2
  4. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/scaler/scaler_server.py +6 -0
  5. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core.egg-info/PKG-INFO +1 -1
  6. assemblyline_core-4.5.1.dev441/assemblyline_core/VERSION +0 -1
  7. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/LICENCE.md +0 -0
  8. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/README.md +0 -0
  9. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/__init__.py +0 -0
  10. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/alerter/__init__.py +0 -0
  11. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/alerter/processing.py +0 -0
  12. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/alerter/run_alerter.py +0 -0
  13. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/archiver/__init__.py +0 -0
  14. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/archiver/run_archiver.py +0 -0
  15. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/badlist_client.py +0 -0
  16. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/dispatching/__init__.py +0 -0
  17. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/dispatching/__main__.py +0 -0
  18. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/dispatching/client.py +0 -0
  19. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/dispatching/dispatcher.py +0 -0
  20. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/dispatching/schedules.py +0 -0
  21. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/dispatching/timeout.py +0 -0
  22. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/expiry/__init__.py +0 -0
  23. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/expiry/run_expiry.py +0 -0
  24. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/ingester/__init__.py +0 -0
  25. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/ingester/__main__.py +0 -0
  26. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/ingester/constants.py +0 -0
  27. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/ingester/ingester.py +0 -0
  28. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/metrics/__init__.py +0 -0
  29. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/metrics/es_metrics.py +0 -0
  30. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  31. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/metrics/helper.py +0 -0
  32. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/metrics/metrics_server.py +0 -0
  33. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  34. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  35. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  36. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/plumber/__init__.py +0 -0
  37. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/plumber/run_plumber.py +0 -0
  38. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/replay/__init__.py +0 -0
  39. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/replay/client.py +0 -0
  40. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/replay/creator/__init__.py +0 -0
  41. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/replay/creator/run.py +0 -0
  42. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/replay/creator/run_worker.py +0 -0
  43. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/replay/loader/__init__.py +0 -0
  44. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/replay/loader/run.py +0 -0
  45. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/replay/loader/run_worker.py +0 -0
  46. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/replay/replay.py +0 -0
  47. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/safelist_client.py +0 -0
  48. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/scaler/__init__.py +0 -0
  49. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/scaler/collection.py +0 -0
  50. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  51. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/scaler/controllers/interface.py +0 -0
  52. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +2 -2
  53. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/scaler/run_scaler.py +0 -0
  54. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/server_base.py +0 -0
  55. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/signature_client.py +0 -0
  56. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/submission_client.py +0 -0
  57. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/tasking_client.py +0 -0
  58. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/updater/__init__.py +0 -0
  59. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/updater/helper.py +0 -0
  60. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/updater/run_updater.py +0 -0
  61. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/vacuum/__init__.py +0 -0
  62. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/vacuum/crawler.py +0 -0
  63. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/vacuum/department_map.py +0 -0
  64. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/vacuum/safelist.py +0 -0
  65. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/vacuum/stream_map.py +0 -0
  66. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/vacuum/worker.py +0 -0
  67. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/workflow/__init__.py +0 -0
  68. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core/workflow/run_workflow.py +0 -0
  69. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  70. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  71. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core.egg-info/requires.txt +0 -0
  72. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/assemblyline_core.egg-info/top_level.txt +0 -0
  73. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/setup.cfg +0 -0
  74. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/setup.py +0 -0
  75. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/test/test_alerter.py +0 -0
  76. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/test/test_badlist_client.py +0 -0
  77. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/test/test_dispatcher.py +0 -0
  78. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/test/test_expiry.py +0 -0
  79. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/test/test_plumber.py +0 -0
  80. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/test/test_replay.py +0 -0
  81. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/test/test_safelist_client.py +0 -0
  82. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/test/test_scaler.py +0 -0
  83. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/test/test_scheduler.py +0 -0
  84. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/test/test_signature_client.py +0 -0
  85. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/test/test_simulation.py +0 -0
  86. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/test/test_vacuum.py +0 -0
  87. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/test/test_worker_ingest.py +0 -0
  88. {assemblyline_core-4.5.1.dev441 → assemblyline_core-4.5.1.dev443}/test/test_worker_submit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev441
3
+ Version: 4.5.1.dev443
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.5.1.dev443
@@ -189,10 +189,10 @@ class DockerController(ControllerInterface):
189
189
  volumes = {row[0]: {'bind': row[1], 'mode': 'ro'} for row in self.global_mounts}
190
190
 
191
191
  # Define environment variables
192
- env = [f'{_e.name}={_e.value}' for _e in cfg.environment]
193
- env += [f'{name}={os.environ[name]}' for name in INHERITED_VARIABLES if name in os.environ]
192
+ env = [f'{name}={os.environ[name]}' for name in INHERITED_VARIABLES if name in os.environ]
194
193
  env += [f'LOG_LEVEL={self.log_level}', f'AL_SERVICE_NAME={service_name}']
195
194
  env += [f'{_n}={_v}' for _n, _v in self._service_limited_env[service_name].items()]
195
+ env += [f'{_e.name}={_e.value}' for _e in cfg.environment]
196
196
  if prof.privileged:
197
197
  env.append('PRIVILEGED=true')
198
198
  volumes.update({row[0]: {'bind': row[1], 'mode': 'ro'} for row in self.core_mounts})
@@ -70,6 +70,7 @@ DOCKER_CONFIGURATION_PATH = os.getenv('DOCKER_CONFIGURATION_PATH', None)
70
70
  DOCKER_CONFIGURATION_VOLUME = os.getenv('DOCKER_CONFIGURATION_VOLUME', None)
71
71
 
72
72
  SERVICE_API_HOST = os.getenv('SERVICE_API_HOST', None)
73
+ SERVICE_API_KEY = os.getenv('SERVICE_API_KEY', None)
73
74
  INTERNAL_ENCRYPT = bool(SERVICE_API_HOST and SERVICE_API_HOST.startswith('https'))
74
75
 
75
76
 
@@ -303,6 +304,11 @@ class ScalerServer(ThreadedCoreBase):
303
304
  [EnvironmentVariable(dict(name="SERVICE_API_HOST", value=SERVICE_API_HOST))] + \
304
305
  [env for env in service_defaults_config.environment if env.name != "SERVICE_API_HOST"]
305
306
 
307
+ if SERVICE_API_KEY:
308
+ service_defaults_config.environment = \
309
+ [EnvironmentVariable(dict(name="SERVICE_API_KEY", value=SERVICE_API_KEY))] + \
310
+ [env for env in service_defaults_config.environment if env.name != "SERVICE_API_KEY"]
311
+
306
312
  if self.config.core.scaler.additional_labels:
307
313
  labels.update({k: v for k, v in (_l.split("=") for _l in self.config.core.scaler.additional_labels)})
308
314
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev441
3
+ Version: 4.5.1.dev443
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.5.1.dev441
@@ -713,8 +713,6 @@ class KubernetesController(ControllerInterface):
713
713
  environment_variables += [V1EnvVar(name=_n, value=_v) for _n, _v in self.core_env.items()]
714
714
  environment_variables.extend(self.core_secret_env)
715
715
  environment_variables.append(V1EnvVar(name='PRIVILEGED', value='true'))
716
- # Overwrite them with configured special environment variables
717
- environment_variables += [V1EnvVar(name=_e.name, value=_e.value) for _e in container_config.environment]
718
716
  # Overwrite those with special hard coded variables
719
717
  environment_variables += [
720
718
  V1EnvVar(name='AL_SERVICE_NAME', value=service_name),
@@ -723,6 +721,8 @@ class KubernetesController(ControllerInterface):
723
721
  # Overwrite ones defined dynamically by dependency container launches
724
722
  for name, value in self._service_limited_env[service_name].items():
725
723
  environment_variables.append(V1EnvVar(name=name, value=value))
724
+ # Overwrite them with configured special environment variables
725
+ environment_variables += [V1EnvVar(name=_e.name, value=_e.value) for _e in container_config.environment]
726
726
  image_pull_policy = 'Always' if DEV_MODE else 'IfNotPresent'
727
727
  return [V1Container(
728
728
  name=deployment_name,