assemblyline-core 4.5.1.dev461__tar.gz → 4.5.1.dev462__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/PKG-INFO +1 -1
  2. assemblyline_core-4.5.1.dev462/assemblyline_core/VERSION +1 -0
  3. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/tasking_client.py +6 -1
  4. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core.egg-info/PKG-INFO +1 -1
  5. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/test/test_tasking_client.py +3 -17
  6. assemblyline_core-4.5.1.dev461/assemblyline_core/VERSION +0 -1
  7. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/LICENCE.md +0 -0
  8. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/README.md +0 -0
  9. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/__init__.py +0 -0
  10. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/alerter/__init__.py +0 -0
  11. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/alerter/processing.py +0 -0
  12. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/alerter/run_alerter.py +0 -0
  13. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/archiver/__init__.py +0 -0
  14. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/archiver/run_archiver.py +0 -0
  15. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/badlist_client.py +0 -0
  16. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/dispatching/__init__.py +0 -0
  17. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/dispatching/__main__.py +0 -0
  18. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/dispatching/client.py +0 -0
  19. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/dispatching/dispatcher.py +0 -0
  20. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/dispatching/schedules.py +0 -0
  21. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/dispatching/timeout.py +0 -0
  22. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/expiry/__init__.py +0 -0
  23. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/expiry/run_expiry.py +0 -0
  24. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/ingester/__init__.py +0 -0
  25. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/ingester/__main__.py +0 -0
  26. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/ingester/constants.py +0 -0
  27. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/ingester/ingester.py +0 -0
  28. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/metrics/__init__.py +0 -0
  29. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/metrics/es_metrics.py +0 -0
  30. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  31. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/metrics/helper.py +0 -0
  32. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/metrics/metrics_server.py +0 -0
  33. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  34. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  35. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  36. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/plumber/__init__.py +0 -0
  37. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/plumber/run_plumber.py +0 -0
  38. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/replay/__init__.py +0 -0
  39. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/replay/client.py +0 -0
  40. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/replay/creator/__init__.py +0 -0
  41. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/replay/creator/run.py +0 -0
  42. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/replay/creator/run_worker.py +0 -0
  43. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/replay/loader/__init__.py +0 -0
  44. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/replay/loader/run.py +0 -0
  45. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/replay/loader/run_worker.py +0 -0
  46. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/replay/replay.py +0 -0
  47. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/safelist_client.py +0 -0
  48. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/scaler/__init__.py +0 -0
  49. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/scaler/collection.py +0 -0
  50. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  51. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  52. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/scaler/controllers/interface.py +0 -0
  53. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  54. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/scaler/run_scaler.py +0 -0
  55. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/scaler/scaler_server.py +0 -0
  56. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/server_base.py +0 -0
  57. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/signature_client.py +0 -0
  58. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/submission_client.py +0 -0
  59. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/updater/__init__.py +0 -0
  60. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/updater/helper.py +0 -0
  61. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/updater/run_updater.py +0 -0
  62. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/vacuum/__init__.py +0 -0
  63. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/vacuum/crawler.py +0 -0
  64. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/vacuum/department_map.py +0 -0
  65. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/vacuum/safelist.py +0 -0
  66. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/vacuum/stream_map.py +0 -0
  67. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/vacuum/worker.py +0 -0
  68. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/workflow/__init__.py +0 -0
  69. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core/workflow/run_workflow.py +0 -0
  70. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  71. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  72. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core.egg-info/requires.txt +0 -0
  73. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/assemblyline_core.egg-info/top_level.txt +0 -0
  74. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/setup.cfg +0 -0
  75. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/setup.py +0 -0
  76. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/test/test_alerter.py +0 -0
  77. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/test/test_badlist_client.py +0 -0
  78. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/test/test_dispatcher.py +0 -0
  79. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/test/test_expiry.py +0 -0
  80. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/test/test_plumber.py +0 -0
  81. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/test/test_replay.py +0 -0
  82. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/test/test_safelist_client.py +0 -0
  83. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/test/test_scaler.py +0 -0
  84. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/test/test_scheduler.py +0 -0
  85. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/test/test_signature_client.py +0 -0
  86. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/test/test_simulation.py +0 -0
  87. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/test/test_vacuum.py +0 -0
  88. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/test/test_worker_ingest.py +0 -0
  89. {assemblyline_core-4.5.1.dev461 → assemblyline_core-4.5.1.dev462}/test/test_worker_submit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev461
3
+ Version: 4.5.1.dev462
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.5.1.dev462
@@ -49,6 +49,8 @@ class TaskingClient:
49
49
  self.event_sender = EventSender('changes', redis)
50
50
  self.cleanup = False
51
51
  self.event_listener = None
52
+ self.heuristics: dict[str, Heuristic] = {}
53
+ self.register_only = register_only
52
54
 
53
55
  # If we're performing service registration, we only need a connection to the datastore
54
56
  if not register_only:
@@ -56,7 +58,6 @@ class TaskingClient:
56
58
  self.event_listener = EventWatcher(redis)
57
59
  self.filestore = filestore or forge.get_filestore(self.config)
58
60
  self.heuristic_handler = HeuristicHandler(self.datastore)
59
- self.heuristics: dict[str, Heuristic] = {}
60
61
  self.reload_heuristics({})
61
62
  self.status_table = ExpiringHash(SERVICE_STATE_HASH, ttl=60*30, host=redis)
62
63
  self.tag_safelister = forge.CachedObject(forge.get_tag_safelister, kwargs=dict(
@@ -195,6 +196,10 @@ class TaskingClient:
195
196
  f"heuristic {item['update']['_id']}: {item['update']['result'].upper()}")
196
197
 
197
198
  # Look for heuristics that are no longer managed by the service and clean them up
199
+ if self.register_only:
200
+ # Fetch current heuristics for the service during registration
201
+ self.reload_heuristics({'service_name': service.name})
202
+
198
203
  all_heuristics = set(h_id for h_id in self.heuristics.keys()
199
204
  if h_id.startswith(f"{service.name.upper()}."))
200
205
  removed_heuristics = all_heuristics - set(h['heur_id'] for h in heuristics)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev461
3
+ Version: 4.5.1.dev462
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -5,26 +5,12 @@ from assemblyline_core.tasking_client import TaskingClient
5
5
  from assemblyline.odm.models.service import Service
6
6
  from assemblyline.odm.models.heuristic import Heuristic
7
7
  from assemblyline.odm.models.result import Result, Section, Heuristic as SectionHeuristic
8
- from assemblyline.odm.random_data import (
9
- create_badlists,
10
- create_users,
11
- wipe_badlist,
12
- wipe_users,
13
- )
14
- from assemblyline.odm.randomizer import random_minimal_obj
15
8
 
9
+ from assemblyline.odm.randomizer import random_minimal_obj
16
10
 
17
- @pytest.fixture(scope="module")
18
- def client(datastore_connection):
19
- try:
20
- create_users(datastore_connection)
21
- create_badlists(datastore_connection)
22
- yield TaskingClient(datastore_connection)
23
- finally:
24
- wipe_users(datastore_connection)
25
- wipe_badlist(datastore_connection)
11
+ def test_register_service(datastore_connection):
12
+ client = TaskingClient(datastore_connection, register_only=True)
26
13
 
27
- def test_register_service(client, datastore_connection):
28
14
  # Test service registration
29
15
  service = random_minimal_obj(Service).as_primitives()
30
16
  heuristics = [random_minimal_obj(Heuristic).as_primitives() for _ in range(2)]
@@ -1 +0,0 @@
1
- 4.5.1.dev461