assemblyline-core 4.5.1.dev65__tar.gz → 4.5.1.dev67__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of assemblyline-core might be problematic. Click here for more details.

Files changed (88) hide show
  1. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/PKG-INFO +1 -1
  2. assemblyline-core-4.5.1.dev67/assemblyline_core/VERSION +1 -0
  3. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/updater/helper.py +31 -2
  4. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core.egg-info/PKG-INFO +1 -1
  5. assemblyline-core-4.5.1.dev65/assemblyline_core/VERSION +0 -1
  6. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/LICENCE.md +0 -0
  7. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/README.md +0 -0
  8. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/__init__.py +0 -0
  9. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/alerter/__init__.py +0 -0
  10. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/alerter/processing.py +0 -0
  11. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/alerter/run_alerter.py +0 -0
  12. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/archiver/__init__.py +0 -0
  13. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/archiver/run_archiver.py +0 -0
  14. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/badlist_client.py +0 -0
  15. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/dispatching/__init__.py +0 -0
  16. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/dispatching/__main__.py +0 -0
  17. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/dispatching/client.py +0 -0
  18. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/dispatching/dispatcher.py +0 -0
  19. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/dispatching/schedules.py +0 -0
  20. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/dispatching/timeout.py +0 -0
  21. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/expiry/__init__.py +0 -0
  22. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/expiry/run_expiry.py +0 -0
  23. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/ingester/__init__.py +0 -0
  24. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/ingester/__main__.py +0 -0
  25. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/ingester/constants.py +0 -0
  26. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/ingester/ingester.py +0 -0
  27. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/metrics/__init__.py +0 -0
  28. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/metrics/es_metrics.py +0 -0
  29. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  30. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/metrics/helper.py +0 -0
  31. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/metrics/metrics_server.py +0 -0
  32. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  33. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  34. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  35. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/plumber/__init__.py +0 -0
  36. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/plumber/run_plumber.py +0 -0
  37. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/replay/__init__.py +0 -0
  38. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/replay/client.py +0 -0
  39. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/replay/creator/__init__.py +0 -0
  40. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/replay/creator/run.py +0 -0
  41. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/replay/creator/run_worker.py +0 -0
  42. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/replay/loader/__init__.py +0 -0
  43. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/replay/loader/run.py +0 -0
  44. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/replay/loader/run_worker.py +0 -0
  45. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/replay/replay.py +0 -0
  46. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/safelist_client.py +0 -0
  47. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/scaler/__init__.py +0 -0
  48. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/scaler/collection.py +0 -0
  49. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  50. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  51. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/scaler/controllers/interface.py +0 -0
  52. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  53. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/scaler/run_scaler.py +0 -0
  54. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/scaler/scaler_server.py +0 -0
  55. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/server_base.py +0 -0
  56. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/signature_client.py +0 -0
  57. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/submission_client.py +0 -0
  58. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/tasking_client.py +0 -0
  59. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/updater/__init__.py +0 -0
  60. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/updater/run_updater.py +0 -0
  61. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/vacuum/__init__.py +0 -0
  62. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/vacuum/crawler.py +0 -0
  63. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/vacuum/department_map.py +0 -0
  64. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/vacuum/safelist.py +0 -0
  65. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/vacuum/stream_map.py +0 -0
  66. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/vacuum/worker.py +0 -0
  67. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/workflow/__init__.py +0 -0
  68. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core/workflow/run_workflow.py +0 -0
  69. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  70. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  71. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core.egg-info/requires.txt +0 -0
  72. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/assemblyline_core.egg-info/top_level.txt +0 -0
  73. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/setup.cfg +0 -0
  74. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/setup.py +0 -0
  75. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/test/test_alerter.py +0 -0
  76. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/test/test_badlist_client.py +0 -0
  77. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/test/test_dispatcher.py +0 -0
  78. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/test/test_expiry.py +0 -0
  79. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/test/test_plumber.py +0 -0
  80. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/test/test_replay.py +0 -0
  81. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/test/test_safelist_client.py +0 -0
  82. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/test/test_scaler.py +0 -0
  83. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/test/test_scheduler.py +0 -0
  84. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/test/test_signature_client.py +0 -0
  85. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/test/test_simulation.py +0 -0
  86. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/test/test_vacuum.py +0 -0
  87. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/test/test_worker_ingest.py +0 -0
  88. {assemblyline-core-4.5.1.dev65 → assemblyline-core-4.5.1.dev67}/test/test_worker_submit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev65
3
+ Version: 4.5.1.dev67
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.5.1.dev67
@@ -48,7 +48,7 @@ class ContainerRegistry():
48
48
  def get_image_os(self, image_name, image_tag) -> str:
49
49
  raise NotImplementedError()
50
50
 
51
-
51
+
52
52
  class DockerHub(ContainerRegistry):
53
53
  def __init__(self, update_channel, proxies: Dict[str, str] = None, *args, **kwargs):
54
54
  super().__init__(DEFAULT_DOCKER_REGISTRY, None, True, proxies)
@@ -69,6 +69,19 @@ class DockerHub(ContainerRegistry):
69
69
 
70
70
  # Ref: https://docs.docker.com/registry/spec/api/#detail
71
71
  class DockerRegistry(ContainerRegistry):
72
+ def __init__(self, server, headers: Dict[str, str] = None, verify: bool = True,
73
+ proxies: Dict[str, str] = None, token_server: str = None, *args, **kwargs):
74
+ super().__init__(server, headers, verify, proxies, *args, **kwargs)
75
+
76
+ if not self.session.headers.get('Authorization'):
77
+ # Retrieve token for authentication: https://distribution.github.io/distribution/spec/auth/token/
78
+
79
+ # Assume the token server is the same as the container image registry host if not explicitly set
80
+ token_server = token_server if token_server else server
81
+ token_url = f"https://{token_server}/token?scope=repository:{image_name}:pull"
82
+ token = requests.get(token_url).json().get('token')
83
+ self.session.headers["Authorization"] = f"Bearer {token}"
84
+
72
85
  def get_image_tags(self, image_name) -> List[str]:
73
86
  # Find latest tag for each types
74
87
  resp = self._make_request(f"/v2/{image_name}/tags/list")
@@ -88,6 +101,19 @@ class DockerRegistry(ContainerRegistry):
88
101
 
89
102
  # Ref: https://github.com/goharbor/harbor/blob/main/api/v2.0/swagger.yaml
90
103
  class HarborRegistry(ContainerRegistry):
104
+ def __init__(self, server, headers: Dict[str, str] = None, verify: bool = True,
105
+ proxies: Dict[str, str] = None, token_server: str = None, *args, **kwargs):
106
+ super().__init__(server, headers, verify, proxies, *args, **kwargs)
107
+
108
+ if not self.session.headers.get('Authorization'):
109
+ # Retrieve token for authentication: https://github.com/goharbor/harbor/wiki/Harbor-FAQs#api
110
+
111
+ # Assume the token server is the same as the container image registry host if not explicitly set
112
+ token_server = token_server if token_server else server
113
+ token_url = f"https://{server}/service/token?scope=repository:{image_name}:pull"
114
+ token = requests.get(token_url).json().get('token')
115
+ headers["Authorization"] = f"Bearer {token}"
116
+
91
117
  def _get_project_repo_ids(self, image_name) -> Tuple[str, str]:
92
118
  # Determine project/repo IDs from image name
93
119
  project_id, repo_id = image_name.split('/', 1)
@@ -180,10 +206,12 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
180
206
  # We're assuming that if only a password is given, then this is a token
181
207
  auth = f"Bearer {service_config.docker_config.registry_password}"
182
208
 
209
+ token_server = None
183
210
  proxies = None
184
211
  for reg_conf in system_config.core.updater.registry_configs:
185
212
  if reg_conf.name == server:
186
213
  proxies = reg_conf.proxies or None
214
+ token_server = reg_conf.token_server or None
187
215
  break
188
216
 
189
217
  registry_type = 'dockerhub' if server == DEFAULT_DOCKER_REGISTRY else service_config.docker_config.registry_type
@@ -192,7 +220,8 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
192
220
  'headers': {'Authorization': auth},
193
221
  'verify': not system_config.services.allow_insecure_registry,
194
222
  'proxies': proxies,
195
- 'update_channel': update_channel
223
+ 'update_channel': update_channel,
224
+ 'token_server': token_server
196
225
  }
197
226
 
198
227
  registry: ContainerRegistry = REGISTRY_TYPE_MAPPING[registry_type](**registry_args)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev65
3
+ Version: 4.5.1.dev67
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.5.1.dev65