assemblyline-core 4.5.1.dev356__tar.gz → 4.5.1.dev358__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of assemblyline-core might be problematic. Click here for more details.
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/PKG-INFO +1 -1
- assemblyline-core-4.5.1.dev358/assemblyline_core/VERSION +1 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/vacuum/worker.py +8 -3
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core.egg-info/PKG-INFO +1 -1
- assemblyline-core-4.5.1.dev356/assemblyline_core/VERSION +0 -1
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/LICENCE.md +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/README.md +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/alerter/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/alerter/processing.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/alerter/run_alerter.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/archiver/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/archiver/run_archiver.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/badlist_client.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/dispatching/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/dispatching/__main__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/dispatching/client.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/dispatching/dispatcher.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/dispatching/schedules.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/dispatching/timeout.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/expiry/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/expiry/run_expiry.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/ingester/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/ingester/__main__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/ingester/constants.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/ingester/ingester.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/metrics/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/metrics/es_metrics.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/metrics/helper.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/metrics/metrics_server.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/plumber/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/plumber/run_plumber.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/replay/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/replay/client.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/replay/creator/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/replay/creator/run.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/replay/creator/run_worker.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/replay/loader/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/replay/loader/run.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/replay/loader/run_worker.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/replay/replay.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/safelist_client.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/scaler/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/scaler/collection.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/scaler/controllers/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/scaler/controllers/interface.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/scaler/run_scaler.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/scaler/scaler_server.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/server_base.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/signature_client.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/submission_client.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/tasking_client.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/updater/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/updater/helper.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/updater/run_updater.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/vacuum/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/vacuum/crawler.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/vacuum/department_map.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/vacuum/safelist.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/vacuum/stream_map.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/workflow/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/workflow/run_workflow.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core.egg-info/SOURCES.txt +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core.egg-info/dependency_links.txt +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core.egg-info/requires.txt +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core.egg-info/top_level.txt +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/setup.cfg +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/setup.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_alerter.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_badlist_client.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_dispatcher.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_expiry.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_plumber.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_replay.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_safelist_client.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_scaler.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_scheduler.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_signature_client.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_simulation.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_vacuum.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_worker_ingest.py +0 -0
- {assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_worker_submit.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
4.5.1.dev358
|
{assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/vacuum/worker.py
RENAMED
|
@@ -37,6 +37,7 @@ from assemblyline.common.str_utils import safe_str
|
|
|
37
37
|
from assemblyline.remote.datatypes import get_client as get_redis_client
|
|
38
38
|
from assemblyline.odm.messages.submission import Submission
|
|
39
39
|
from assemblyline.remote.datatypes.queues.named import NamedQueue
|
|
40
|
+
from assemblyline.remote.datatypes.hash import Hash
|
|
40
41
|
|
|
41
42
|
from assemblyline_core.vacuum.crawler import VACUUM_BUFFER_NAME
|
|
42
43
|
|
|
@@ -164,12 +165,16 @@ class FileProcessor(threading.Thread):
|
|
|
164
165
|
# Anything that can't be copied easily should be initialized in 'run'.
|
|
165
166
|
self.config: Config = config
|
|
166
167
|
self.datastore = datastore
|
|
167
|
-
self.metadata_check = MetadataValidator(datastore)
|
|
168
|
+
self.metadata_check = MetadataValidator(datastore, Hash("metadata_suggestions", persistent_redis))
|
|
169
|
+
|
|
170
|
+
# Merge the default metadata required for ingestion with those that are required from vacuum
|
|
171
|
+
validation_scheme = config.submission.metadata.ingest.get('_default', {})
|
|
172
|
+
validation_scheme.update(config.submission.metadata.ingest.get(self.config.core.vacuum.ingest_type, {}))
|
|
173
|
+
|
|
168
174
|
self.metadata_check_kwargs = {
|
|
169
|
-
'validation_scheme':
|
|
175
|
+
'validation_scheme': validation_scheme,
|
|
170
176
|
'strict': self.config.core.vacuum.ingest_type in self.config.submission.metadata.strict_schemes
|
|
171
177
|
}
|
|
172
|
-
self.validation_scheme = self.config.submission.metadata.ingest.get(self.config.core.vacuum.ingest_type, {})
|
|
173
178
|
self.counter = counter
|
|
174
179
|
self.minimum_classification = self.config.core.vacuum.minimum_classification
|
|
175
180
|
logger.info("Connect to work queue")
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
4.5.1.dev356
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/replay/client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/replay/replay.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/assemblyline_core/server_base.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_badlist_client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_safelist_client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_signature_client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_worker_ingest.py
RENAMED
|
File without changes
|
{assemblyline-core-4.5.1.dev356 → assemblyline-core-4.5.1.dev358}/test/test_worker_submit.py
RENAMED
|
File without changes
|