assemblyline-core 4.7.4.dev9__tar.gz → 4.7.5.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/PKG-INFO +1 -1
- assemblyline_core-4.7.5.dev1/assemblyline_core/VERSION +1 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/ingester/ingester.py +3 -38
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core.egg-info/PKG-INFO +1 -1
- assemblyline_core-4.7.4.dev9/assemblyline_core/VERSION +0 -1
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/LICENCE.md +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/README.md +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/__init__.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/alerter/__init__.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/alerter/processing.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/alerter/run_alerter.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/archiver/__init__.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/archiver/run_archiver.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/badlist_client.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/dispatching/__init__.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/dispatching/client.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/dispatching/dispatcher.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/dispatching/schedules.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/expiry/__init__.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/expiry/run_expiry.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/ingester/__init__.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/ingester/constants.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/__init__.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/es_metrics.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/helper.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/metrics_server.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/__init__.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/client.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/creator/__init__.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/creator/run.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/creator/run_worker.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/loader/__init__.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/loader/run.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/loader/run_worker.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/replay.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/safelist_client.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/__init__.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/collection.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/controllers/__init__.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/controllers/interface.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/run_scaler.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/scaler_server.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/server_base.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/signature_client.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/submission_client.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/tasking_client.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/updater/__init__.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/updater/helper.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/updater/run_updater.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/__init__.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/crawler.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/department_map.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/safelist.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/stream_map.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/worker.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/workflow/__init__.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/workflow/run_workflow.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core.egg-info/SOURCES.txt +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core.egg-info/dependency_links.txt +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core.egg-info/requires.txt +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core.egg-info/top_level.txt +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/setup.cfg +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/setup.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_alerter.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_badlist_client.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_expiry.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_replay.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_safelist_client.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_scaler.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_scheduler.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_signature_client.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_tasking_client.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_vacuum.py +0 -0
- {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_workflow.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
4.7.5.dev1
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/ingester/ingester.py
RENAMED
|
@@ -9,51 +9,16 @@ score received, possibly sending a message to indicate that an alert should
|
|
|
9
9
|
be created.
|
|
10
10
|
"""
|
|
11
11
|
|
|
12
|
-
import logging
|
|
13
|
-
import threading
|
|
14
|
-
import time
|
|
15
12
|
from os import environ
|
|
16
13
|
from random import random
|
|
17
|
-
from typing import Any, Iterable, List, Optional, Tuple
|
|
18
|
-
|
|
19
|
-
import elasticapm
|
|
20
14
|
|
|
21
15
|
from assemblyline import odm
|
|
22
|
-
from assemblyline.common import exceptions, forge, isotime
|
|
23
|
-
from assemblyline.common.constants import DROP_PRIORITY
|
|
24
|
-
from assemblyline.common.exceptions import get_stacktrace_info
|
|
25
|
-
from assemblyline.common.importing import load_module_by_path
|
|
26
|
-
from assemblyline.common.isotime import now, now_as_iso
|
|
27
|
-
from assemblyline.common.metrics import MetricsFactory
|
|
28
|
-
from assemblyline.common.str_utils import dotdump, safe_str
|
|
29
|
-
from assemblyline.datastore.exceptions import DataStoreException
|
|
30
|
-
from assemblyline.filestore import CorruptedFileStoreException, FileStoreException
|
|
31
|
-
from assemblyline.odm.messages.ingest_heartbeat import Metrics
|
|
32
16
|
from assemblyline.odm.messages.submission import Submission as MessageSubmission
|
|
33
|
-
from assemblyline.odm.messages.submission import SubmissionMessage
|
|
34
17
|
from assemblyline.odm.models.alert import EXTENDED_SCAN_VALUES
|
|
35
|
-
from assemblyline.odm.models.filescore import FileScore
|
|
36
|
-
from assemblyline.odm.models.submission import Submission as DatabaseSubmission
|
|
37
18
|
from assemblyline.odm.models.submission import SubmissionParams
|
|
38
|
-
|
|
39
|
-
from
|
|
40
|
-
|
|
41
|
-
from assemblyline.remote.datatypes.queues.comms import CommsQueue
|
|
42
|
-
from assemblyline.remote.datatypes.queues.multi import MultiQueue
|
|
43
|
-
from assemblyline.remote.datatypes.queues.named import NamedQueue
|
|
44
|
-
from assemblyline.remote.datatypes.queues.priority import PriorityQueue
|
|
45
|
-
from assemblyline.remote.datatypes.user_quota_tracker import UserQuotaTracker
|
|
46
|
-
from assemblyline_core.dispatching.dispatcher import Dispatcher
|
|
47
|
-
from assemblyline_core.server_base import ThreadedCoreBase
|
|
48
|
-
from assemblyline_core.submission_client import SubmissionClient
|
|
49
|
-
|
|
50
|
-
from .constants import COMPLETE_QUEUE_NAME, INGEST_QUEUE_NAME, drop_chance
|
|
51
|
-
|
|
52
|
-
_dup_prefix = 'w-m-'
|
|
53
|
-
_notification_queue_prefix = 'nq-'
|
|
54
|
-
_max_retries = 10
|
|
55
|
-
_retry_delay = 60 * 4 # Wait 4 minutes to retry
|
|
56
|
-
_max_time = 2 * 24 * 60 * 60 # Wait 2 days for responses.
|
|
19
|
+
|
|
20
|
+
from .constants import drop_chance
|
|
21
|
+
|
|
57
22
|
HOUR_IN_SECONDS = 60 * 60
|
|
58
23
|
COMPLETE_THREADS = int(environ.get('INGESTER_COMPLETE_THREADS', 4))
|
|
59
24
|
INGEST_THREADS = int(environ.get('INGESTER_INGEST_THREADS', 1))
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
4.7.4.dev9
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/alerter/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/archiver/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/badlist_client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/expiry/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/expiry/run_expiry.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/ingester/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/helper.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/loader/run.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/replay.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/safelist_client.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/collection.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/run_scaler.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/server_base.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/signature_client.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/submission_client.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/tasking_client.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/updater/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/updater/helper.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/__init__.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/crawler.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/safelist.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/stream_map.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/worker.py
RENAMED
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/workflow/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core.egg-info/SOURCES.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|