assemblyline-core 4.5.1.dev181__tar.gz → 4.5.1.dev182__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of assemblyline-core might be problematic. Click here for more details.

Files changed (88) hide show
  1. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/PKG-INFO +1 -1
  2. assemblyline-core-4.5.1.dev182/assemblyline_core/VERSION +1 -0
  3. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/updater/helper.py +42 -1
  4. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/updater/run_updater.py +1 -0
  5. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core.egg-info/PKG-INFO +1 -1
  6. assemblyline-core-4.5.1.dev181/assemblyline_core/VERSION +0 -1
  7. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/LICENCE.md +0 -0
  8. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/README.md +0 -0
  9. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/__init__.py +0 -0
  10. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/alerter/__init__.py +0 -0
  11. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/alerter/processing.py +0 -0
  12. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/alerter/run_alerter.py +0 -0
  13. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/archiver/__init__.py +0 -0
  14. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/archiver/run_archiver.py +0 -0
  15. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/badlist_client.py +0 -0
  16. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/dispatching/__init__.py +0 -0
  17. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/dispatching/__main__.py +0 -0
  18. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/dispatching/client.py +0 -0
  19. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/dispatching/dispatcher.py +0 -0
  20. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/dispatching/schedules.py +0 -0
  21. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/dispatching/timeout.py +0 -0
  22. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/expiry/__init__.py +0 -0
  23. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/expiry/run_expiry.py +0 -0
  24. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/ingester/__init__.py +0 -0
  25. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/ingester/__main__.py +0 -0
  26. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/ingester/constants.py +0 -0
  27. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/ingester/ingester.py +0 -0
  28. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/metrics/__init__.py +0 -0
  29. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/metrics/es_metrics.py +0 -0
  30. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  31. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/metrics/helper.py +0 -0
  32. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/metrics/metrics_server.py +0 -0
  33. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  34. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  35. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  36. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/plumber/__init__.py +0 -0
  37. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/plumber/run_plumber.py +0 -0
  38. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/replay/__init__.py +0 -0
  39. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/replay/client.py +0 -0
  40. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/replay/creator/__init__.py +0 -0
  41. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/replay/creator/run.py +0 -0
  42. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/replay/creator/run_worker.py +0 -0
  43. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/replay/loader/__init__.py +0 -0
  44. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/replay/loader/run.py +0 -0
  45. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/replay/loader/run_worker.py +0 -0
  46. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/replay/replay.py +0 -0
  47. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/safelist_client.py +0 -0
  48. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/scaler/__init__.py +0 -0
  49. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/scaler/collection.py +0 -0
  50. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  51. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  52. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/scaler/controllers/interface.py +0 -0
  53. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  54. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/scaler/run_scaler.py +0 -0
  55. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/scaler/scaler_server.py +0 -0
  56. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/server_base.py +0 -0
  57. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/signature_client.py +0 -0
  58. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/submission_client.py +0 -0
  59. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/tasking_client.py +0 -0
  60. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/updater/__init__.py +0 -0
  61. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/vacuum/__init__.py +0 -0
  62. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/vacuum/crawler.py +0 -0
  63. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/vacuum/department_map.py +0 -0
  64. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/vacuum/safelist.py +0 -0
  65. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/vacuum/stream_map.py +0 -0
  66. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/vacuum/worker.py +0 -0
  67. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/workflow/__init__.py +0 -0
  68. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core/workflow/run_workflow.py +0 -0
  69. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  70. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  71. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core.egg-info/requires.txt +0 -0
  72. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/assemblyline_core.egg-info/top_level.txt +0 -0
  73. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/setup.cfg +0 -0
  74. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/setup.py +0 -0
  75. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/test/test_alerter.py +0 -0
  76. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/test/test_badlist_client.py +0 -0
  77. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/test/test_dispatcher.py +0 -0
  78. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/test/test_expiry.py +0 -0
  79. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/test/test_plumber.py +0 -0
  80. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/test/test_replay.py +0 -0
  81. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/test/test_safelist_client.py +0 -0
  82. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/test/test_scaler.py +0 -0
  83. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/test/test_scheduler.py +0 -0
  84. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/test/test_signature_client.py +0 -0
  85. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/test/test_simulation.py +0 -0
  86. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/test/test_vacuum.py +0 -0
  87. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/test/test_worker_ingest.py +0 -0
  88. {assemblyline-core-4.5.1.dev181 → assemblyline-core-4.5.1.dev182}/test/test_worker_submit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev181
3
+ Version: 4.5.1.dev182
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.5.1.dev182
@@ -62,6 +62,43 @@ class DockerRegistry(ContainerRegistry):
62
62
  return resp_data['tags'] or []
63
63
  return []
64
64
 
65
+ class AzureContainerRegistry(ContainerRegistry):
66
+ def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None, token_server=None):
67
+ # Find latest tag for each types
68
+ url = f"https://{server}/v2/{image_name}/tags/list"
69
+
70
+ # Get tag list
71
+ headers = {}
72
+ if auth:
73
+ headers["Authorization"] = auth
74
+
75
+ # Attempt request with provided credentials alone
76
+ resp = self._perform_request(url, headers, verify, proxies)
77
+
78
+ if not resp:
79
+ # Authentication with just credentials failed, moving over to generating a bearer token
80
+
81
+ # Retrieve token for authentication: https://azure.github.io/acr/Token-BasicAuth.html#using-the-token-api
82
+ token_url = f"https://{server}/oauth2/token?scope=repository:{image_name}:metadata_read,pull&service={server}"
83
+ resp = self._perform_request(token_url, headers, verify, proxies)
84
+ if resp and resp.ok:
85
+ # Request to obtain token was successful, set Authorization header for registry API
86
+ token = resp.json().get('access_token')
87
+ headers["Authorization"] = f"Bearer {token}"
88
+
89
+ resp = self._perform_request(url, headers, verify, proxies)
90
+ if not resp:
91
+ # Fetching tag list failed
92
+ return []
93
+
94
+ # At this point, we should have a response from the API
95
+ if resp and resp.ok:
96
+ # Test for positive list of tags
97
+ resp_data = resp.json()
98
+ return resp_data['tags'] or []
99
+
100
+ return []
101
+
65
102
  class HarborRegistry(ContainerRegistry):
66
103
  def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None, token_server=None):
67
104
  # Determine project/repo IDs from image name
@@ -158,7 +195,11 @@ def get_latest_tag_for_service(
158
195
  # We're assuming that if only a password is given, then this is a token
159
196
  auth = f"Bearer {service_config.docker_config.registry_password}"
160
197
 
161
- registry = REGISTRY_TYPE_MAPPING[service_config.docker_config.registry_type]
198
+ if server.endswith(".azurecr.io"):
199
+ # This is an Azure Container Registry based on the server name
200
+ registry = AzureContainerRegistry()
201
+ else:
202
+ registry = REGISTRY_TYPE_MAPPING[service_config.docker_config.registry_type]
162
203
  token_server = None
163
204
  proxies = None
164
205
  for reg_conf in system_config.core.updater.registry_configs:
@@ -337,6 +337,7 @@ class KubernetesUpdateInterface:
337
337
  priority_class_name=self.priority_class,
338
338
  service_account_name=docker_config.service_account or self.default_service_account or PRIVILEGED_SERVICE_ACCOUNT_NAME,
339
339
  affinity=selector_to_node_affinity(self.linux_node_selector),
340
+ tolerations=self.default_service_tolerations
340
341
  )
341
342
 
342
343
  if use_pull_secret:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev181
3
+ Version: 4.5.1.dev182
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.5.1.dev181