assemblyline-core 4.7.2.dev47__tar.gz → 4.7.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/PKG-INFO +1 -1
- assemblyline_core-4.7.2.2/assemblyline_core/VERSION +1 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/dispatching/dispatcher.py +6 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/ingester/ingester.py +1 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/updater/run_updater.py +3 -2
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core.egg-info/PKG-INFO +1 -1
- assemblyline_core-4.7.2.dev47/assemblyline_core/VERSION +0 -1
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/LICENCE.md +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/README.md +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/__init__.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/alerter/__init__.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/alerter/processing.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/alerter/run_alerter.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/archiver/__init__.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/archiver/run_archiver.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/badlist_client.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/dispatching/__init__.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/dispatching/client.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/dispatching/schedules.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/expiry/__init__.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/expiry/run_expiry.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/ingester/__init__.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/ingester/constants.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/metrics/__init__.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/metrics/es_metrics.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/metrics/helper.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/metrics/metrics_server.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/replay/__init__.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/replay/client.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/replay/creator/__init__.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/replay/creator/run.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/replay/creator/run_worker.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/replay/loader/__init__.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/replay/loader/run.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/replay/loader/run_worker.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/replay/replay.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/safelist_client.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/scaler/__init__.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/scaler/collection.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/scaler/controllers/__init__.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/scaler/controllers/interface.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/scaler/run_scaler.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/scaler/scaler_server.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/server_base.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/signature_client.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/submission_client.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/tasking_client.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/updater/__init__.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/updater/helper.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/vacuum/__init__.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/vacuum/crawler.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/vacuum/department_map.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/vacuum/safelist.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/vacuum/stream_map.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/vacuum/worker.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/workflow/__init__.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/workflow/run_workflow.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core.egg-info/SOURCES.txt +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core.egg-info/dependency_links.txt +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core.egg-info/requires.txt +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core.egg-info/top_level.txt +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/setup.cfg +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/setup.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/test/test_alerter.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/test/test_badlist_client.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/test/test_expiry.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/test/test_replay.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/test/test_safelist_client.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/test/test_scaler.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/test/test_scheduler.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/test/test_signature_client.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/test/test_tasking_client.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/test/test_vacuum.py +0 -0
- {assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/test/test_workflow.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
4.7.2.2
|
|
@@ -27,6 +27,7 @@ from assemblyline.common.forge import (
|
|
|
27
27
|
)
|
|
28
28
|
from assemblyline.common.isotime import now_as_iso
|
|
29
29
|
from assemblyline.common.metrics import MetricsFactory
|
|
30
|
+
from assemblyline.common.postprocess import ActionWorker
|
|
30
31
|
from assemblyline.datastore.helper import AssemblylineDatastore
|
|
31
32
|
from assemblyline.odm.messages.changes import Operation, ServiceChange
|
|
32
33
|
from assemblyline.odm.messages.dispatcher_heartbeat import Metrics
|
|
@@ -233,6 +234,10 @@ class Dispatcher(ThreadedCoreBase):
|
|
|
233
234
|
# already processed
|
|
234
235
|
self.timeout_queue: Queue[DispatchAction] = Queue()
|
|
235
236
|
|
|
237
|
+
# Utility object to handle post-processing actions
|
|
238
|
+
self.postprocess_worker = ActionWorker(cache=False, config=self.config, datastore=self.datastore,
|
|
239
|
+
redis_persist=self.redis_persist)
|
|
240
|
+
|
|
236
241
|
# Update bad sid list
|
|
237
242
|
self.redis_bad_sids = Set(BAD_SID_HASH, host=self.redis_persist)
|
|
238
243
|
self.bad_sids: set[str] = set(self.redis_bad_sids.members())
|
|
@@ -244,6 +249,7 @@ class Dispatcher(ThreadedCoreBase):
|
|
|
244
249
|
def stop(self):
|
|
245
250
|
super().stop()
|
|
246
251
|
self.service_change_watcher.stop()
|
|
252
|
+
self.postprocess_worker.stop()
|
|
247
253
|
|
|
248
254
|
def try_run(self):
|
|
249
255
|
self.log.info(f'Using dispatcher id {self.instance_id}')
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/ingester/ingester.py
RENAMED
|
@@ -25,6 +25,7 @@ from assemblyline.common.exceptions import get_stacktrace_info
|
|
|
25
25
|
from assemblyline.common.importing import load_module_by_path
|
|
26
26
|
from assemblyline.common.isotime import now, now_as_iso
|
|
27
27
|
from assemblyline.common.metrics import MetricsFactory
|
|
28
|
+
from assemblyline.common.postprocess import ActionWorker
|
|
28
29
|
from assemblyline.common.str_utils import dotdump, safe_str
|
|
29
30
|
from assemblyline.datastore.exceptions import DataStoreException
|
|
30
31
|
from assemblyline.filestore import CorruptedFileStoreException, FileStoreException
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/updater/run_updater.py
RENAMED
|
@@ -12,6 +12,7 @@ from typing import Any, Dict, List, Optional
|
|
|
12
12
|
|
|
13
13
|
import docker
|
|
14
14
|
from assemblyline.common import isotime
|
|
15
|
+
from assemblyline.common.version import FRAMEWORK_VERSION, SYSTEM_VERSION
|
|
15
16
|
from assemblyline.odm.messages.changes import Operation, ServiceChange
|
|
16
17
|
from assemblyline.odm.models.config import Mount, Selector
|
|
17
18
|
from assemblyline.odm.models.service import DockerConfig, Service
|
|
@@ -499,9 +500,9 @@ class ServiceUpdater(ThreadedCoreBase):
|
|
|
499
500
|
# Install function for services
|
|
500
501
|
def install_service(service_name: str, install_data: dict) -> str:
|
|
501
502
|
if self.config.services.preferred_update_channel == 'stable':
|
|
502
|
-
tag = 'stable'
|
|
503
|
+
tag = f'{FRAMEWORK_VERSION}.{SYSTEM_VERSION}.stable'
|
|
503
504
|
else:
|
|
504
|
-
tag = 'latest'
|
|
505
|
+
tag = f'{FRAMEWORK_VERSION}.{SYSTEM_VERSION}.latest'
|
|
505
506
|
try:
|
|
506
507
|
service = Service(
|
|
507
508
|
{'name': service_name,
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
4.7.2.dev47
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/alerter/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/alerter/processing.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/alerter/run_alerter.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/archiver/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/badlist_client.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/dispatching/client.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/expiry/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/expiry/run_expiry.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/ingester/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/ingester/constants.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/metrics/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/metrics/es_metrics.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/metrics/helper.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/replay/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/replay/client.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/replay/creator/run.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/replay/loader/run.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/replay/replay.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/safelist_client.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/scaler/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/scaler/collection.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/scaler/run_scaler.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/server_base.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/signature_client.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/submission_client.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/tasking_client.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/updater/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/updater/helper.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/vacuum/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/vacuum/crawler.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/vacuum/safelist.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/vacuum/stream_map.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/vacuum/worker.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core/workflow/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core.egg-info/SOURCES.txt
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core.egg-info/requires.txt
RENAMED
|
File without changes
|
{assemblyline_core-4.7.2.dev47 → assemblyline_core-4.7.2.2}/assemblyline_core.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|