assemblyline-core 4.7.2.dev46__tar.gz → 4.7.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/PKG-INFO +1 -1
  2. assemblyline_core-4.7.2.1/assemblyline_core/VERSION +1 -0
  3. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/dispatching/dispatcher.py +6 -0
  4. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/ingester/ingester.py +1 -0
  5. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/updater/run_updater.py +3 -2
  6. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core.egg-info/PKG-INFO +1 -1
  7. assemblyline_core-4.7.2.dev46/assemblyline_core/VERSION +0 -1
  8. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/LICENCE.md +0 -0
  9. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/README.md +0 -0
  10. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/__init__.py +0 -0
  11. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/alerter/__init__.py +0 -0
  12. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/alerter/processing.py +0 -0
  13. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/alerter/run_alerter.py +0 -0
  14. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/archiver/__init__.py +0 -0
  15. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/archiver/run_archiver.py +0 -0
  16. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/badlist_client.py +0 -0
  17. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/dispatching/__init__.py +0 -0
  18. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/dispatching/client.py +0 -0
  19. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/dispatching/schedules.py +0 -0
  20. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/expiry/__init__.py +0 -0
  21. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/expiry/run_expiry.py +0 -0
  22. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/ingester/__init__.py +0 -0
  23. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/ingester/constants.py +0 -0
  24. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/metrics/__init__.py +0 -0
  25. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/metrics/es_metrics.py +0 -0
  26. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  27. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/metrics/helper.py +0 -0
  28. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/metrics/metrics_server.py +0 -0
  29. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  30. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  31. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  32. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/replay/__init__.py +0 -0
  33. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/replay/client.py +0 -0
  34. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/replay/creator/__init__.py +0 -0
  35. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/replay/creator/run.py +0 -0
  36. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/replay/creator/run_worker.py +0 -0
  37. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/replay/loader/__init__.py +0 -0
  38. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/replay/loader/run.py +0 -0
  39. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/replay/loader/run_worker.py +0 -0
  40. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/replay/replay.py +0 -0
  41. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/safelist_client.py +0 -0
  42. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/scaler/__init__.py +0 -0
  43. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/scaler/collection.py +0 -0
  44. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  45. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  46. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/scaler/controllers/interface.py +0 -0
  47. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  48. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/scaler/run_scaler.py +0 -0
  49. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/scaler/scaler_server.py +0 -0
  50. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/server_base.py +0 -0
  51. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/signature_client.py +0 -0
  52. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/submission_client.py +0 -0
  53. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/tasking_client.py +0 -0
  54. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/updater/__init__.py +0 -0
  55. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/updater/helper.py +0 -0
  56. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/vacuum/__init__.py +0 -0
  57. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/vacuum/crawler.py +0 -0
  58. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/vacuum/department_map.py +0 -0
  59. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/vacuum/safelist.py +0 -0
  60. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/vacuum/stream_map.py +0 -0
  61. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/vacuum/worker.py +0 -0
  62. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/workflow/__init__.py +0 -0
  63. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core/workflow/run_workflow.py +0 -0
  64. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  65. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  66. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core.egg-info/requires.txt +0 -0
  67. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/assemblyline_core.egg-info/top_level.txt +0 -0
  68. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/setup.cfg +0 -0
  69. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/setup.py +0 -0
  70. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/test/test_alerter.py +0 -0
  71. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/test/test_badlist_client.py +0 -0
  72. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/test/test_expiry.py +0 -0
  73. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/test/test_replay.py +0 -0
  74. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/test/test_safelist_client.py +0 -0
  75. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/test/test_scaler.py +0 -0
  76. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/test/test_scheduler.py +0 -0
  77. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/test/test_signature_client.py +0 -0
  78. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/test/test_tasking_client.py +0 -0
  79. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/test/test_vacuum.py +0 -0
  80. {assemblyline_core-4.7.2.dev46 → assemblyline_core-4.7.2.1}/test/test_workflow.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: assemblyline-core
3
- Version: 4.7.2.dev46
3
+ Version: 4.7.2.1
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.7.2.1
@@ -27,6 +27,7 @@ from assemblyline.common.forge import (
27
27
  )
28
28
  from assemblyline.common.isotime import now_as_iso
29
29
  from assemblyline.common.metrics import MetricsFactory
30
+ from assemblyline.common.postprocess import ActionWorker
30
31
  from assemblyline.datastore.helper import AssemblylineDatastore
31
32
  from assemblyline.odm.messages.changes import Operation, ServiceChange
32
33
  from assemblyline.odm.messages.dispatcher_heartbeat import Metrics
@@ -233,6 +234,10 @@ class Dispatcher(ThreadedCoreBase):
233
234
  # already processed
234
235
  self.timeout_queue: Queue[DispatchAction] = Queue()
235
236
 
237
+ # Utility object to handle post-processing actions
238
+ self.postprocess_worker = ActionWorker(cache=False, config=self.config, datastore=self.datastore,
239
+ redis_persist=self.redis_persist)
240
+
236
241
  # Update bad sid list
237
242
  self.redis_bad_sids = Set(BAD_SID_HASH, host=self.redis_persist)
238
243
  self.bad_sids: set[str] = set(self.redis_bad_sids.members())
@@ -244,6 +249,7 @@ class Dispatcher(ThreadedCoreBase):
244
249
  def stop(self):
245
250
  super().stop()
246
251
  self.service_change_watcher.stop()
252
+ self.postprocess_worker.stop()
247
253
 
248
254
  def try_run(self):
249
255
  self.log.info(f'Using dispatcher id {self.instance_id}')
@@ -25,6 +25,7 @@ from assemblyline.common.exceptions import get_stacktrace_info
25
25
  from assemblyline.common.importing import load_module_by_path
26
26
  from assemblyline.common.isotime import now, now_as_iso
27
27
  from assemblyline.common.metrics import MetricsFactory
28
+ from assemblyline.common.postprocess import ActionWorker
28
29
  from assemblyline.common.str_utils import dotdump, safe_str
29
30
  from assemblyline.datastore.exceptions import DataStoreException
30
31
  from assemblyline.filestore import CorruptedFileStoreException, FileStoreException
@@ -12,6 +12,7 @@ from typing import Any, Dict, List, Optional
12
12
 
13
13
  import docker
14
14
  from assemblyline.common import isotime
15
+ from assemblyline.common.version import FRAMEWORK_VERSION, SYSTEM_VERSION
15
16
  from assemblyline.odm.messages.changes import Operation, ServiceChange
16
17
  from assemblyline.odm.models.config import Mount, Selector
17
18
  from assemblyline.odm.models.service import DockerConfig, Service
@@ -499,9 +500,9 @@ class ServiceUpdater(ThreadedCoreBase):
499
500
  # Install function for services
500
501
  def install_service(service_name: str, install_data: dict) -> str:
501
502
  if self.config.services.preferred_update_channel == 'stable':
502
- tag = 'stable'
503
+ tag = f'{FRAMEWORK_VERSION}.{SYSTEM_VERSION}.stable'
503
504
  else:
504
- tag = 'latest'
505
+ tag = f'{FRAMEWORK_VERSION}.{SYSTEM_VERSION}.latest'
505
506
  try:
506
507
  service = Service(
507
508
  {'name': service_name,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: assemblyline-core
3
- Version: 4.7.2.dev46
3
+ Version: 4.7.2.1
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.7.2.dev46