assemblyline-core 4.5.1.dev339__tar.gz → 4.5.1.dev340__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of assemblyline-core might be problematic. Click here for more details.
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/PKG-INFO +1 -1
- assemblyline-core-4.5.1.dev340/assemblyline_core/VERSION +1 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/updater/helper.py +43 -9
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core.egg-info/PKG-INFO +1 -1
- assemblyline-core-4.5.1.dev339/assemblyline_core/VERSION +0 -1
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/LICENCE.md +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/README.md +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/alerter/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/alerter/processing.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/alerter/run_alerter.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/archiver/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/archiver/run_archiver.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/badlist_client.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/dispatching/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/dispatching/__main__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/dispatching/client.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/dispatching/dispatcher.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/dispatching/schedules.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/dispatching/timeout.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/expiry/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/expiry/run_expiry.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/ingester/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/ingester/__main__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/ingester/constants.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/ingester/ingester.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/metrics/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/metrics/es_metrics.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/metrics/helper.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/metrics/metrics_server.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/plumber/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/plumber/run_plumber.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/replay/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/replay/client.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/replay/creator/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/replay/creator/run.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/replay/creator/run_worker.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/replay/loader/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/replay/loader/run.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/replay/loader/run_worker.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/replay/replay.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/safelist_client.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/scaler/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/scaler/collection.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/scaler/controllers/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/scaler/controllers/interface.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/scaler/run_scaler.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/scaler/scaler_server.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/server_base.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/signature_client.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/submission_client.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/tasking_client.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/updater/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/updater/run_updater.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/vacuum/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/vacuum/crawler.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/vacuum/department_map.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/vacuum/safelist.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/vacuum/stream_map.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/vacuum/worker.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/workflow/__init__.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/workflow/run_workflow.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core.egg-info/SOURCES.txt +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core.egg-info/dependency_links.txt +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core.egg-info/requires.txt +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core.egg-info/top_level.txt +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/setup.cfg +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/setup.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_alerter.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_badlist_client.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_dispatcher.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_expiry.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_plumber.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_replay.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_safelist_client.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_scaler.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_scheduler.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_signature_client.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_simulation.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_vacuum.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_worker_ingest.py +0 -0
- {assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_worker_submit.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
4.5.1.dev340
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import os
|
|
1
2
|
import requests
|
|
2
3
|
import re
|
|
3
4
|
import socket
|
|
@@ -15,6 +16,10 @@ from typing import Dict, List
|
|
|
15
16
|
from packaging.version import parse, Version
|
|
16
17
|
from urllib.parse import urlencode
|
|
17
18
|
|
|
19
|
+
from azure.identity import WorkloadIdentityCredential
|
|
20
|
+
from azure.core.exceptions import AzureError
|
|
21
|
+
|
|
22
|
+
|
|
18
23
|
DEFAULT_DOCKER_REGISTRY = "hub.docker.com"
|
|
19
24
|
|
|
20
25
|
|
|
@@ -145,7 +150,8 @@ def get_registry_config(docker_config: DockerConfig, system_config: SystemConfig
|
|
|
145
150
|
for registry in registries:
|
|
146
151
|
if server.startswith(registry.name):
|
|
147
152
|
# Return authentication credentials and the type of registry
|
|
148
|
-
return dict(username=registry.username, password=registry.password, type=registry.type
|
|
153
|
+
return dict(username=registry.username, password=registry.password, type=registry.type,
|
|
154
|
+
use_fic=registry.use_fic, fic_token_path=registry.fic_token_path)
|
|
149
155
|
|
|
150
156
|
# Otherwise return what's configured for the service
|
|
151
157
|
return dict(username=docker_config.registry_username, password=docker_config.registry_password,
|
|
@@ -203,6 +209,15 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
|
|
|
203
209
|
auth = None
|
|
204
210
|
server, image_name = process_image(searchable_image)
|
|
205
211
|
|
|
212
|
+
# Load in proxies and token server
|
|
213
|
+
token_server = None
|
|
214
|
+
proxies = None
|
|
215
|
+
for reg_conf in system_config.core.updater.registry_configs:
|
|
216
|
+
if reg_conf.name == server:
|
|
217
|
+
proxies = reg_conf.proxies or None
|
|
218
|
+
token_server = reg_conf.token_server or None
|
|
219
|
+
break
|
|
220
|
+
|
|
206
221
|
# Generate 'Authenication' header value for pulling tag list from registry
|
|
207
222
|
auth_config = get_registry_config(service_config.docker_config, system_config)
|
|
208
223
|
registry_type = auth_config.pop('type')
|
|
@@ -214,17 +229,36 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
|
|
|
214
229
|
auth = f"Bearer {auth_config['password']}"
|
|
215
230
|
|
|
216
231
|
if server.endswith(".azurecr.io"):
|
|
232
|
+
if auth_config.get('use_fic', False):
|
|
233
|
+
# If the use of federated identity token is set, exchange said token to an ACR token
|
|
234
|
+
try:
|
|
235
|
+
credentials = WorkloadIdentityCredential(
|
|
236
|
+
tenant_id = auth_config.get('fic_tenant_id', None),
|
|
237
|
+
client_id = auth_config.get('fic_client_id', None),
|
|
238
|
+
token_file_path = auth_config.get('fic_token_path', None))
|
|
239
|
+
aad_token = credentials.get_token('https://management.core.windows.net/.default').token
|
|
240
|
+
|
|
241
|
+
refresh_token = requests.post(
|
|
242
|
+
f"https://{server}/oauth2/exchange",
|
|
243
|
+
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
|
244
|
+
data=f"grant_type=access_token&service={server}&access_token={aad_token}",
|
|
245
|
+
proxies=proxies).json()["refresh_token"]
|
|
246
|
+
|
|
247
|
+
token = requests.post(
|
|
248
|
+
f"https://{server}/oauth2/token",
|
|
249
|
+
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
|
250
|
+
data=f"grant_type=refresh_token&service={server}&refresh_token={refresh_token}"
|
|
251
|
+
f"&scope=repository:{image_name}:metadata_read",
|
|
252
|
+
proxies=proxies).json()["access_token"]
|
|
253
|
+
|
|
254
|
+
auth = f"Bearer {token}"
|
|
255
|
+
except Exception as e:
|
|
256
|
+
logger.error(f"{prefix} Failed to acquire Azure credentials: {str(e)}")
|
|
257
|
+
|
|
217
258
|
# This is an Azure Container Registry based on the server name
|
|
218
259
|
registry = AzureContainerRegistry()
|
|
219
260
|
else:
|
|
220
261
|
registry = REGISTRY_TYPE_MAPPING[registry_type]
|
|
221
|
-
token_server = None
|
|
222
|
-
proxies = None
|
|
223
|
-
for reg_conf in system_config.core.updater.registry_configs:
|
|
224
|
-
if reg_conf.name == server:
|
|
225
|
-
proxies = reg_conf.proxies or None
|
|
226
|
-
token_server = reg_conf.token_server or None
|
|
227
|
-
break
|
|
228
262
|
|
|
229
263
|
if server == DEFAULT_DOCKER_REGISTRY:
|
|
230
264
|
tags = _get_dockerhub_tags(image_name, update_channel, prefix, proxies, logger=logger)
|
|
@@ -235,7 +269,7 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
|
|
|
235
269
|
|
|
236
270
|
# Pre-filter tags to only consider 'compatible' tags relative to the running system
|
|
237
271
|
tags = [tag for tag in tags
|
|
238
|
-
if re.match(f"({FRAMEWORK_VERSION})
|
|
272
|
+
if re.match(f"({FRAMEWORK_VERSION}).({SYSTEM_VERSION}).\\d+.({update_channel})\\d+", tag)]
|
|
239
273
|
if not tags:
|
|
240
274
|
logger.warning(f"{prefix} Cannot fetch latest tag for service {service_name} - {image_name}" \
|
|
241
275
|
f" => [server: {server}, repo_name: {image_name}, channel: {update_channel}]")
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
4.5.1.dev339
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/replay/client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/replay/replay.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/server_base.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/assemblyline_core/vacuum/worker.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_badlist_client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_safelist_client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_signature_client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_worker_ingest.py
RENAMED
|
File without changes
|
{assemblyline-core-4.5.1.dev339 → assemblyline-core-4.5.1.dev340}/test/test_worker_submit.py
RENAMED
|
File without changes
|