assemblyline-core 4.5.0.56__tar.gz → 4.5.0.58__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of assemblyline-core might be problematic. Click here for more details.

Files changed (88) hide show
  1. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/PKG-INFO +1 -1
  2. assemblyline-core-4.5.0.58/assemblyline_core/VERSION +1 -0
  3. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/updater/helper.py +39 -9
  4. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core.egg-info/PKG-INFO +1 -1
  5. assemblyline-core-4.5.0.56/assemblyline_core/VERSION +0 -1
  6. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/LICENCE.md +0 -0
  7. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/README.md +0 -0
  8. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/__init__.py +0 -0
  9. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/alerter/__init__.py +0 -0
  10. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/alerter/processing.py +0 -0
  11. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/alerter/run_alerter.py +0 -0
  12. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/archiver/__init__.py +0 -0
  13. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/archiver/run_archiver.py +0 -0
  14. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/badlist_client.py +0 -0
  15. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/dispatching/__init__.py +0 -0
  16. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/dispatching/__main__.py +0 -0
  17. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/dispatching/client.py +0 -0
  18. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/dispatching/dispatcher.py +0 -0
  19. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/dispatching/schedules.py +0 -0
  20. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/dispatching/timeout.py +0 -0
  21. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/expiry/__init__.py +0 -0
  22. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/expiry/run_expiry.py +0 -0
  23. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/ingester/__init__.py +0 -0
  24. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/ingester/__main__.py +0 -0
  25. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/ingester/constants.py +0 -0
  26. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/ingester/ingester.py +0 -0
  27. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/metrics/__init__.py +0 -0
  28. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/metrics/es_metrics.py +0 -0
  29. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  30. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/metrics/helper.py +0 -0
  31. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/metrics/metrics_server.py +0 -0
  32. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  33. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  34. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  35. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/plumber/__init__.py +0 -0
  36. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/plumber/run_plumber.py +0 -0
  37. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/replay/__init__.py +0 -0
  38. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/replay/client.py +0 -0
  39. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/replay/creator/__init__.py +0 -0
  40. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/replay/creator/run.py +0 -0
  41. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/replay/creator/run_worker.py +0 -0
  42. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/replay/loader/__init__.py +0 -0
  43. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/replay/loader/run.py +0 -0
  44. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/replay/loader/run_worker.py +0 -0
  45. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/replay/replay.py +0 -0
  46. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/safelist_client.py +0 -0
  47. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/scaler/__init__.py +0 -0
  48. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/scaler/collection.py +0 -0
  49. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  50. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  51. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/scaler/controllers/interface.py +0 -0
  52. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  53. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/scaler/run_scaler.py +0 -0
  54. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/scaler/scaler_server.py +0 -0
  55. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/server_base.py +0 -0
  56. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/signature_client.py +0 -0
  57. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/submission_client.py +0 -0
  58. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/tasking_client.py +0 -0
  59. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/updater/__init__.py +0 -0
  60. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/updater/run_updater.py +0 -0
  61. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/vacuum/__init__.py +0 -0
  62. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/vacuum/crawler.py +0 -0
  63. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/vacuum/department_map.py +0 -0
  64. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/vacuum/safelist.py +0 -0
  65. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/vacuum/stream_map.py +0 -0
  66. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/vacuum/worker.py +0 -0
  67. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/workflow/__init__.py +0 -0
  68. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core/workflow/run_workflow.py +0 -0
  69. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  70. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  71. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core.egg-info/requires.txt +0 -0
  72. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/assemblyline_core.egg-info/top_level.txt +0 -0
  73. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/setup.cfg +0 -0
  74. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/setup.py +0 -0
  75. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/test/test_alerter.py +0 -0
  76. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/test/test_badlist_client.py +0 -0
  77. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/test/test_dispatcher.py +0 -0
  78. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/test/test_expiry.py +0 -0
  79. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/test/test_plumber.py +0 -0
  80. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/test/test_replay.py +0 -0
  81. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/test/test_safelist_client.py +0 -0
  82. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/test/test_scaler.py +0 -0
  83. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/test/test_scheduler.py +0 -0
  84. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/test/test_signature_client.py +0 -0
  85. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/test/test_simulation.py +0 -0
  86. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/test/test_vacuum.py +0 -0
  87. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/test/test_worker_ingest.py +0 -0
  88. {assemblyline-core-4.5.0.56 → assemblyline-core-4.5.0.58}/test/test_worker_submit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.0.56
3
+ Version: 4.5.0.58
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.5.0.58
@@ -1,3 +1,4 @@
1
+ import os
1
2
  import requests
2
3
  import re
3
4
  import socket
@@ -15,6 +16,9 @@ from typing import Dict, List
15
16
  from packaging.version import parse, Version
16
17
  from urllib.parse import urlencode
17
18
 
19
+ from azure.identity import DefaultAzureCredential
20
+
21
+
18
22
  DEFAULT_DOCKER_REGISTRY = "hub.docker.com"
19
23
 
20
24
 
@@ -146,7 +150,8 @@ def get_registry_config(docker_config: DockerConfig, system_config: SystemConfig
146
150
  for registry in registries:
147
151
  if server.startswith(registry.name):
148
152
  # Return authentication credentials and the type of registry
149
- return dict(username=registry.username, password=registry.password, type=registry.type)
153
+ return dict(username=registry.username, password=registry.password, type=registry.type,
154
+ use_fic=registry.use_fic)
150
155
 
151
156
  # Otherwise return what's configured for the service
152
157
  return dict(username=docker_config.registry_username, password=docker_config.registry_password,
@@ -204,6 +209,15 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
204
209
  auth = None
205
210
  server, image_name = process_image(searchable_image)
206
211
 
212
+ # Load in proxies and token server
213
+ token_server = None
214
+ proxies = None
215
+ for reg_conf in system_config.core.updater.registry_configs:
216
+ if reg_conf.name == server:
217
+ proxies = reg_conf.proxies or None
218
+ token_server = reg_conf.token_server or None
219
+ break
220
+
207
221
  # Generate 'Authenication' header value for pulling tag list from registry
208
222
  auth_config = get_registry_config(service_config.docker_config, system_config)
209
223
  registry_type = auth_config.pop('type')
@@ -215,17 +229,33 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
215
229
  auth = f"Bearer {auth_config['password']}"
216
230
 
217
231
  if server.endswith(".azurecr.io"):
232
+ if auth_config.get('use_fic', False):
233
+ # If the use of federated identity token is set, exchange said token to an ACR token
234
+ try:
235
+ credentials = DefaultAzureCredential()
236
+ aad_token = credentials.get_token('https://management.core.windows.net/.default').token
237
+
238
+ refresh_token = requests.post(
239
+ f"https://{server}/oauth2/exchange",
240
+ headers={"Content-Type": "application/x-www-form-urlencoded"},
241
+ data=f"grant_type=access_token&service={server}&access_token={aad_token}",
242
+ proxies=proxies).json()["refresh_token"]
243
+
244
+ token = requests.post(
245
+ f"https://{server}/oauth2/token",
246
+ headers={"Content-Type": "application/x-www-form-urlencoded"},
247
+ data=f"grant_type=refresh_token&service={server}&refresh_token={refresh_token}"
248
+ f"&scope=repository:{image_name}:metadata_read",
249
+ proxies=proxies).json()["access_token"]
250
+
251
+ auth = f"Bearer {token}"
252
+ except Exception as e:
253
+ logger.error(f"{prefix} Failed to acquire Azure credentials: {str(e)}")
254
+
218
255
  # This is an Azure Container Registry based on the server name
219
256
  registry = AzureContainerRegistry()
220
257
  else:
221
258
  registry = REGISTRY_TYPE_MAPPING[registry_type]
222
- token_server = None
223
- proxies = None
224
- for reg_conf in system_config.core.updater.registry_configs:
225
- if reg_conf.name == server:
226
- proxies = reg_conf.proxies or None
227
- token_server = reg_conf.token_server or None
228
- break
229
259
 
230
260
  if server == DEFAULT_DOCKER_REGISTRY:
231
261
  tags = _get_dockerhub_tags(image_name, update_channel, prefix, proxies, logger=logger)
@@ -235,7 +265,7 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
235
265
  proxies, token_server)
236
266
  # Pre-filter tags to only consider 'compatible' tags relative to the running system
237
267
  tags = [tag for tag in tags
238
- if re.match(f"({FRAMEWORK_VERSION})\.({SYSTEM_VERSION})\.\\d+\.({update_channel})\\d+", tag)]
268
+ if re.match(f"({FRAMEWORK_VERSION})\\.({SYSTEM_VERSION})\\.\\d+\\.({update_channel})\\d+", tag)]
239
269
  if not tags:
240
270
  logger.warning(f"{prefix} Cannot fetch latest tag for service {service_name} - {image_name}" \
241
271
  f" => [server: {server}, repo_name: {image_name}, channel: {update_channel}]")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.0.56
3
+ Version: 4.5.0.58
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.5.0.56