assemblyline-core 4.7.5.dev102__tar.gz → 4.7.5.dev109__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/PKG-INFO +1 -1
  2. assemblyline_core-4.7.5.dev109/assemblyline_core/VERSION +1 -0
  3. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/__init__.py +0 -1
  4. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/scaler/collection.py +3 -4
  5. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/scaler/controllers/interface.py +0 -4
  6. assemblyline_core-4.7.5.dev102/assemblyline_core/scaler/controllers/kubernetes.py → assemblyline_core-4.7.5.dev109/assemblyline_core/scaler/controllers/kubernetes_ctl.py +412 -174
  7. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/scaler/scaler_server.py +322 -82
  8. assemblyline_core-4.7.5.dev109/assemblyline_core/tasking_client.py +537 -0
  9. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/updater/run_updater.py +1 -1
  10. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core.egg-info/PKG-INFO +1 -1
  11. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core.egg-info/SOURCES.txt +2 -6
  12. assemblyline_core-4.7.5.dev109/test/test_tasking_client.py +77 -0
  13. assemblyline_core-4.7.5.dev102/assemblyline_core/VERSION +0 -1
  14. assemblyline_core-4.7.5.dev102/assemblyline_core/scaler/base.py +0 -296
  15. assemblyline_core-4.7.5.dev102/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -391
  16. assemblyline_core-4.7.5.dev102/assemblyline_core/scaler_hpa/kubernetes.py +0 -381
  17. assemblyline_core-4.7.5.dev102/assemblyline_core/scaler_hpa/run_scaler.py +0 -7
  18. assemblyline_core-4.7.5.dev102/assemblyline_core/scaler_hpa/scaler_server.py +0 -533
  19. assemblyline_core-4.7.5.dev102/assemblyline_core/workflow/__init__.py +0 -0
  20. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/LICENCE.md +0 -0
  21. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/README.md +0 -0
  22. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/alerter/__init__.py +0 -0
  23. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/alerter/processing.py +0 -0
  24. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/alerter/run_alerter.py +0 -0
  25. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/archiver/__init__.py +0 -0
  26. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/archiver/run_archiver.py +0 -0
  27. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/badlist_client.py +0 -0
  28. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/dispatching/__init__.py +0 -0
  29. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/dispatching/client.py +0 -0
  30. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/dispatching/dispatcher.py +0 -0
  31. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/dispatching/schedules.py +0 -0
  32. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/expiry/__init__.py +0 -0
  33. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/expiry/run_expiry.py +0 -0
  34. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/ingester/__init__.py +0 -0
  35. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/ingester/constants.py +0 -0
  36. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/ingester/ingester.py +0 -0
  37. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/metrics/__init__.py +0 -0
  38. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/metrics/es_metrics.py +0 -0
  39. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  40. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/metrics/helper.py +0 -0
  41. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/metrics/metrics_server.py +0 -0
  42. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  43. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  44. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  45. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/replay/__init__.py +0 -0
  46. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/replay/client.py +0 -0
  47. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/replay/creator/__init__.py +0 -0
  48. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/replay/creator/run.py +0 -0
  49. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/replay/creator/run_worker.py +0 -0
  50. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/replay/loader/__init__.py +0 -0
  51. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/replay/loader/run.py +0 -0
  52. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/replay/loader/run_worker.py +0 -0
  53. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/replay/replay.py +0 -0
  54. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/safelist_client.py +0 -0
  55. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/scaler/__init__.py +0 -0
  56. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  57. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  58. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/scaler/run_scaler.py +0 -0
  59. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/server_base.py +0 -0
  60. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/signature_client.py +0 -0
  61. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/submission_client.py +0 -0
  62. {assemblyline_core-4.7.5.dev102/assemblyline_core/scaler_hpa → assemblyline_core-4.7.5.dev109/assemblyline_core/updater}/__init__.py +0 -0
  63. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/updater/helper.py +0 -0
  64. {assemblyline_core-4.7.5.dev102/assemblyline_core/updater → assemblyline_core-4.7.5.dev109/assemblyline_core/vacuum}/__init__.py +0 -0
  65. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/vacuum/crawler.py +0 -0
  66. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/vacuum/department_map.py +0 -0
  67. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/vacuum/safelist.py +0 -0
  68. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/vacuum/stream_map.py +0 -0
  69. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/vacuum/worker.py +0 -0
  70. {assemblyline_core-4.7.5.dev102/assemblyline_core/vacuum → assemblyline_core-4.7.5.dev109/assemblyline_core/workflow}/__init__.py +0 -0
  71. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core/workflow/run_workflow.py +0 -0
  72. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  73. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core.egg-info/requires.txt +0 -0
  74. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/assemblyline_core.egg-info/top_level.txt +0 -0
  75. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/setup.cfg +0 -0
  76. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/setup.py +0 -0
  77. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/test/test_alerter.py +0 -0
  78. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/test/test_badlist_client.py +0 -0
  79. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/test/test_expiry.py +0 -0
  80. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/test/test_replay.py +0 -0
  81. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/test/test_safelist_client.py +0 -0
  82. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/test/test_scaler.py +0 -0
  83. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/test/test_scheduler.py +0 -0
  84. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/test/test_signature_client.py +0 -0
  85. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/test/test_updater_helper.py +0 -0
  86. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/test/test_vacuum.py +0 -0
  87. {assemblyline_core-4.7.5.dev102 → assemblyline_core-4.7.5.dev109}/test/test_workflow.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: assemblyline-core
3
- Version: 4.7.5.dev102
3
+ Version: 4.7.5.dev109
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.7.5.dev109
@@ -1,6 +1,5 @@
1
1
  PAUSABLE_COMPONENTS = ['ingester', 'dispatcher']
2
2
 
3
-
4
3
  def normalize_hashlist_item(tag_type: str, tag_value: str) -> str:
5
4
  # Normalize tag data pertaining to domains or URIs
6
5
  if tag_type.endswith('.domain'):
@@ -12,9 +12,10 @@ Row = namedtuple('Row', ['timestamp', 'busy', 'throughput'])
12
12
 
13
13
 
14
14
  class Collection:
15
- """A buffer for metrics data from multiple instances of multiple services."""
16
- def __init__(self, period, ttl=None) -> None:
15
+ def __init__(self, period, ttl=None):
17
16
  """
17
+ A buffer for metrics data from multiple instances of multiple services.
18
+
18
19
  :param period: Expected seconds between updates
19
20
  :param ttl: Seconds before a message is dropped from the buffer
20
21
  """
@@ -23,7 +24,6 @@ class Collection:
23
24
  self.services: Dict[str, Dict[str, Row]] = {}
24
25
 
25
26
  def update(self, service, host, busy_seconds, throughput):
26
- """Store a new record for a given host."""
27
27
  # Load the sequence of data points that
28
28
  try:
29
29
  hosts = self.services[service]
@@ -34,7 +34,6 @@ class Collection:
34
34
  hosts[host] = Row(time.time(), busy_seconds, throughput)
35
35
 
36
36
  def read(self, service):
37
- """Read how many instances of a service are known and how busy they are."""
38
37
  now = time.time()
39
38
 
40
39
  # Load the last messages from this service
@@ -44,10 +44,6 @@ class ControllerInterface:
44
44
  """Set the target for running instances of a service."""
45
45
  raise NotImplementedError()
46
46
 
47
- def get_unavailable(self) -> dict[str, int]:
48
- """Report the number of pending containers from deployments monitored by the system."""
49
- raise NotImplementedError()
50
-
51
47
  def restart(self, service: ServiceProfile):
52
48
  raise NotImplementedError()
53
49