assemblyline-core 4.7.4.dev9__tar.gz → 4.7.5.dev1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/PKG-INFO +1 -1
  2. assemblyline_core-4.7.5.dev1/assemblyline_core/VERSION +1 -0
  3. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/ingester/ingester.py +3 -38
  4. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core.egg-info/PKG-INFO +1 -1
  5. assemblyline_core-4.7.4.dev9/assemblyline_core/VERSION +0 -1
  6. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/LICENCE.md +0 -0
  7. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/README.md +0 -0
  8. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/__init__.py +0 -0
  9. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/alerter/__init__.py +0 -0
  10. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/alerter/processing.py +0 -0
  11. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/alerter/run_alerter.py +0 -0
  12. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/archiver/__init__.py +0 -0
  13. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/archiver/run_archiver.py +0 -0
  14. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/badlist_client.py +0 -0
  15. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/dispatching/__init__.py +0 -0
  16. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/dispatching/client.py +0 -0
  17. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/dispatching/dispatcher.py +0 -0
  18. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/dispatching/schedules.py +0 -0
  19. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/expiry/__init__.py +0 -0
  20. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/expiry/run_expiry.py +0 -0
  21. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/ingester/__init__.py +0 -0
  22. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/ingester/constants.py +0 -0
  23. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/__init__.py +0 -0
  24. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/es_metrics.py +0 -0
  25. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  26. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/helper.py +0 -0
  27. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/metrics_server.py +0 -0
  28. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  29. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  30. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  31. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/__init__.py +0 -0
  32. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/client.py +0 -0
  33. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/creator/__init__.py +0 -0
  34. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/creator/run.py +0 -0
  35. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/creator/run_worker.py +0 -0
  36. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/loader/__init__.py +0 -0
  37. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/loader/run.py +0 -0
  38. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/loader/run_worker.py +0 -0
  39. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/replay/replay.py +0 -0
  40. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/safelist_client.py +0 -0
  41. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/__init__.py +0 -0
  42. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/collection.py +0 -0
  43. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  44. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  45. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/controllers/interface.py +0 -0
  46. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  47. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/run_scaler.py +0 -0
  48. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/scaler/scaler_server.py +0 -0
  49. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/server_base.py +0 -0
  50. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/signature_client.py +0 -0
  51. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/submission_client.py +0 -0
  52. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/tasking_client.py +0 -0
  53. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/updater/__init__.py +0 -0
  54. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/updater/helper.py +0 -0
  55. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/updater/run_updater.py +0 -0
  56. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/__init__.py +0 -0
  57. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/crawler.py +0 -0
  58. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/department_map.py +0 -0
  59. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/safelist.py +0 -0
  60. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/stream_map.py +0 -0
  61. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/vacuum/worker.py +0 -0
  62. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/workflow/__init__.py +0 -0
  63. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core/workflow/run_workflow.py +0 -0
  64. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  65. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  66. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core.egg-info/requires.txt +0 -0
  67. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/assemblyline_core.egg-info/top_level.txt +0 -0
  68. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/setup.cfg +0 -0
  69. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/setup.py +0 -0
  70. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_alerter.py +0 -0
  71. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_badlist_client.py +0 -0
  72. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_expiry.py +0 -0
  73. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_replay.py +0 -0
  74. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_safelist_client.py +0 -0
  75. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_scaler.py +0 -0
  76. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_scheduler.py +0 -0
  77. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_signature_client.py +0 -0
  78. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_tasking_client.py +0 -0
  79. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_vacuum.py +0 -0
  80. {assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.5.dev1}/test/test_workflow.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: assemblyline-core
3
- Version: 4.7.4.dev9
3
+ Version: 4.7.5.dev1
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.7.5.dev1
@@ -9,51 +9,16 @@ score received, possibly sending a message to indicate that an alert should
9
9
  be created.
10
10
  """
11
11
 
12
- import logging
13
- import threading
14
- import time
15
12
  from os import environ
16
13
  from random import random
17
- from typing import Any, Iterable, List, Optional, Tuple
18
-
19
- import elasticapm
20
14
 
21
15
  from assemblyline import odm
22
- from assemblyline.common import exceptions, forge, isotime
23
- from assemblyline.common.constants import DROP_PRIORITY
24
- from assemblyline.common.exceptions import get_stacktrace_info
25
- from assemblyline.common.importing import load_module_by_path
26
- from assemblyline.common.isotime import now, now_as_iso
27
- from assemblyline.common.metrics import MetricsFactory
28
- from assemblyline.common.str_utils import dotdump, safe_str
29
- from assemblyline.datastore.exceptions import DataStoreException
30
- from assemblyline.filestore import CorruptedFileStoreException, FileStoreException
31
- from assemblyline.odm.messages.ingest_heartbeat import Metrics
32
16
  from assemblyline.odm.messages.submission import Submission as MessageSubmission
33
- from assemblyline.odm.messages.submission import SubmissionMessage
34
17
  from assemblyline.odm.models.alert import EXTENDED_SCAN_VALUES
35
- from assemblyline.odm.models.filescore import FileScore
36
- from assemblyline.odm.models.submission import Submission as DatabaseSubmission
37
18
  from assemblyline.odm.models.submission import SubmissionParams
38
- from assemblyline.odm.models.user import User
39
- from assemblyline.remote.datatypes.events import EventWatcher
40
- from assemblyline.remote.datatypes.hash import Hash
41
- from assemblyline.remote.datatypes.queues.comms import CommsQueue
42
- from assemblyline.remote.datatypes.queues.multi import MultiQueue
43
- from assemblyline.remote.datatypes.queues.named import NamedQueue
44
- from assemblyline.remote.datatypes.queues.priority import PriorityQueue
45
- from assemblyline.remote.datatypes.user_quota_tracker import UserQuotaTracker
46
- from assemblyline_core.dispatching.dispatcher import Dispatcher
47
- from assemblyline_core.server_base import ThreadedCoreBase
48
- from assemblyline_core.submission_client import SubmissionClient
49
-
50
- from .constants import COMPLETE_QUEUE_NAME, INGEST_QUEUE_NAME, drop_chance
51
-
52
- _dup_prefix = 'w-m-'
53
- _notification_queue_prefix = 'nq-'
54
- _max_retries = 10
55
- _retry_delay = 60 * 4 # Wait 4 minutes to retry
56
- _max_time = 2 * 24 * 60 * 60 # Wait 2 days for responses.
19
+
20
+ from .constants import drop_chance
21
+
57
22
  HOUR_IN_SECONDS = 60 * 60
58
23
  COMPLETE_THREADS = int(environ.get('INGESTER_COMPLETE_THREADS', 4))
59
24
  INGEST_THREADS = int(environ.get('INGESTER_INGEST_THREADS', 1))
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: assemblyline-core
3
- Version: 4.7.4.dev9
3
+ Version: 4.7.5.dev1
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.7.4.dev9