assemblyline-core 4.5.0.10__tar.gz → 4.5.0.12__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of assemblyline-core might be problematic. Click here for more details.

Files changed (88) hide show
  1. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/PKG-INFO +1 -1
  2. assemblyline-core-4.5.0.12/assemblyline_core/VERSION +1 -0
  3. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/updater/helper.py +33 -5
  4. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core.egg-info/PKG-INFO +1 -1
  5. assemblyline-core-4.5.0.10/assemblyline_core/VERSION +0 -1
  6. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/LICENCE.md +0 -0
  7. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/README.md +0 -0
  8. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/__init__.py +0 -0
  9. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/alerter/__init__.py +0 -0
  10. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/alerter/processing.py +0 -0
  11. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/alerter/run_alerter.py +0 -0
  12. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/archiver/__init__.py +0 -0
  13. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/archiver/run_archiver.py +0 -0
  14. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/badlist_client.py +0 -0
  15. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/dispatching/__init__.py +0 -0
  16. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/dispatching/__main__.py +0 -0
  17. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/dispatching/client.py +0 -0
  18. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/dispatching/dispatcher.py +0 -0
  19. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/dispatching/schedules.py +0 -0
  20. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/dispatching/timeout.py +0 -0
  21. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/expiry/__init__.py +0 -0
  22. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/expiry/run_expiry.py +0 -0
  23. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/ingester/__init__.py +0 -0
  24. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/ingester/__main__.py +0 -0
  25. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/ingester/constants.py +0 -0
  26. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/ingester/ingester.py +0 -0
  27. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/metrics/__init__.py +0 -0
  28. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/metrics/es_metrics.py +0 -0
  29. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  30. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/metrics/helper.py +0 -0
  31. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/metrics/metrics_server.py +0 -0
  32. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  33. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  34. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  35. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/plumber/__init__.py +0 -0
  36. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/plumber/run_plumber.py +0 -0
  37. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/replay/__init__.py +0 -0
  38. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/replay/client.py +0 -0
  39. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/replay/creator/__init__.py +0 -0
  40. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/replay/creator/run.py +0 -0
  41. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/replay/creator/run_worker.py +0 -0
  42. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/replay/loader/__init__.py +0 -0
  43. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/replay/loader/run.py +0 -0
  44. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/replay/loader/run_worker.py +0 -0
  45. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/replay/replay.py +0 -0
  46. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/safelist_client.py +0 -0
  47. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/scaler/__init__.py +0 -0
  48. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/scaler/collection.py +0 -0
  49. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  50. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  51. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/scaler/controllers/interface.py +0 -0
  52. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  53. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/scaler/run_scaler.py +0 -0
  54. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/scaler/scaler_server.py +0 -0
  55. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/server_base.py +0 -0
  56. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/signature_client.py +0 -0
  57. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/submission_client.py +0 -0
  58. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/tasking_client.py +0 -0
  59. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/updater/__init__.py +0 -0
  60. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/updater/run_updater.py +0 -0
  61. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/vacuum/__init__.py +0 -0
  62. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/vacuum/crawler.py +0 -0
  63. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/vacuum/department_map.py +0 -0
  64. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/vacuum/safelist.py +0 -0
  65. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/vacuum/stream_map.py +0 -0
  66. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/vacuum/worker.py +0 -0
  67. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/workflow/__init__.py +0 -0
  68. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core/workflow/run_workflow.py +0 -0
  69. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  70. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  71. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core.egg-info/requires.txt +0 -0
  72. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/assemblyline_core.egg-info/top_level.txt +0 -0
  73. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/setup.cfg +0 -0
  74. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/setup.py +0 -0
  75. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/test/test_alerter.py +0 -0
  76. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/test/test_badlist_client.py +0 -0
  77. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/test/test_dispatcher.py +0 -0
  78. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/test/test_expiry.py +0 -0
  79. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/test/test_plumber.py +0 -0
  80. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/test/test_replay.py +0 -0
  81. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/test/test_safelist_client.py +0 -0
  82. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/test/test_scaler.py +0 -0
  83. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/test/test_scheduler.py +0 -0
  84. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/test/test_signature_client.py +0 -0
  85. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/test/test_simulation.py +0 -0
  86. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/test/test_vacuum.py +0 -0
  87. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/test/test_worker_ingest.py +0 -0
  88. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.12}/test/test_worker_submit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.0.10
3
+ Version: 4.5.0.12
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.5.0.12
@@ -18,12 +18,12 @@ DEFAULT_DOCKER_REGISTRY = "registry.hub.docker.com"
18
18
 
19
19
  class ContainerRegistry():
20
20
  # Provide a means of obtaining a list of tags from a container registry
21
- def _get_proprietary_registry_tags(self, server, image_name, auth, verify):
21
+ def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None, token_server=None):
22
22
  raise NotImplementedError()
23
23
 
24
24
 
25
25
  class DockerRegistry(ContainerRegistry):
26
- def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None):
26
+ def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None, token_server=None):
27
27
  # Find latest tag for each types
28
28
  url = f"https://{server}/v2/{image_name}/tags/list"
29
29
 
@@ -31,6 +31,17 @@ class DockerRegistry(ContainerRegistry):
31
31
  headers = {}
32
32
  if auth:
33
33
  headers["Authorization"] = auth
34
+ else:
35
+ # Retrieve token for authentication: https://distribution.github.io/distribution/spec/auth/token/
36
+
37
+ # Assume the token server is the same as the container image registry host if not explicitly set
38
+ token_server = token_server if token_server else server
39
+ token_url = f"https://{token_server}/token?scope=repository:{image_name}:pull"
40
+ resp = requests.get(token_url)
41
+ if resp.ok:
42
+ # Request to obtain token was successful, set Authorization header for registry API
43
+ token = resp.json().get('token')
44
+ headers["Authorization"] = f"Bearer {token}"
34
45
 
35
46
  resp = None
36
47
  try:
@@ -50,7 +61,7 @@ class DockerRegistry(ContainerRegistry):
50
61
 
51
62
 
52
63
  class HarborRegistry(ContainerRegistry):
53
- def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None):
64
+ def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None, token_server=None):
54
65
  # Determine project/repo IDs from image name
55
66
  project_id, repo_id = image_name.split('/', 1)
56
67
  repo_id = repo_id.replace('/', "%2F")
@@ -59,7 +70,17 @@ class HarborRegistry(ContainerRegistry):
59
70
  headers = {}
60
71
  if auth:
61
72
  headers["Authorization"] = auth
62
-
73
+ else:
74
+ # Retrieve token for authentication: https://github.com/goharbor/harbor/wiki/Harbor-FAQs#api
75
+
76
+ # Assume the token server is the same as the container image registry host if not explicitly set
77
+ token_server = token_server if token_server else server
78
+ token_url = f"https://{server}/service/token?scope=repository:{image_name}:pull"
79
+ resp = requests.get(token_url)
80
+ if resp.ok:
81
+ # Request to obtain token was successful, set Authorization header for registry API
82
+ token = resp.json().get('token')
83
+ headers["Authorization"] = f"Bearer {token}"
63
84
  resp = None
64
85
  try:
65
86
  resp = requests.get(url, headers=headers, verify=verify, proxies=proxies)
@@ -131,25 +152,32 @@ def get_latest_tag_for_service(
131
152
  break
132
153
 
133
154
  if service_config.docker_config.registry_username and service_config.docker_config.registry_password:
155
+ # We're authenticating using Basic Auth
134
156
  auth_config = {
135
157
  'username': service_config.docker_config.registry_username,
136
158
  'password': service_config.docker_config.registry_password
137
159
  }
138
160
  upass = f"{service_config.docker_config.registry_username}:{service_config.docker_config.registry_password}"
139
161
  auth = f"Basic {b64encode(upass.encode()).decode()}"
162
+ elif service_config.docker_config.registry_password:
163
+ # We're assuming that if only a password is given, then this is a token
164
+ auth = f"Bearer {service_config.docker_config.registry_password}"
140
165
 
141
166
  registry = REGISTRY_TYPE_MAPPING[service_config.docker_config.registry_type]
167
+ token_server = None
142
168
  proxies = None
143
169
  for reg_conf in system_config.core.updater.registry_configs:
144
170
  if reg_conf.name == server:
145
171
  proxies = reg_conf.proxies or None
172
+ token_server = reg_conf.token_server or None
146
173
  break
147
174
 
148
175
  if server == DEFAULT_DOCKER_REGISTRY:
149
176
  tags = _get_dockerhub_tags(image_name, update_channel, proxies)
150
177
  else:
151
178
  tags = registry._get_proprietary_registry_tags(server, image_name, auth,
152
- not system_config.services.allow_insecure_registry, proxies)
179
+ not system_config.services.allow_insecure_registry,
180
+ proxies, token_server)
153
181
 
154
182
  tag_name = None
155
183
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.0.10
3
+ Version: 4.5.0.12
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.5.0.10