assemblyline-core 4.5.0.10__tar.gz → 4.5.0.11__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of assemblyline-core might be problematic. Click here for more details.

Files changed (88) hide show
  1. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/PKG-INFO +1 -1
  2. assemblyline-core-4.5.0.11/assemblyline_core/VERSION +1 -0
  3. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/updater/helper.py +26 -4
  4. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core.egg-info/PKG-INFO +1 -1
  5. assemblyline-core-4.5.0.10/assemblyline_core/VERSION +0 -1
  6. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/LICENCE.md +0 -0
  7. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/README.md +0 -0
  8. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/__init__.py +0 -0
  9. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/alerter/__init__.py +0 -0
  10. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/alerter/processing.py +0 -0
  11. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/alerter/run_alerter.py +0 -0
  12. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/archiver/__init__.py +0 -0
  13. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/archiver/run_archiver.py +0 -0
  14. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/badlist_client.py +0 -0
  15. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/dispatching/__init__.py +0 -0
  16. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/dispatching/__main__.py +0 -0
  17. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/dispatching/client.py +0 -0
  18. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/dispatching/dispatcher.py +0 -0
  19. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/dispatching/schedules.py +0 -0
  20. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/dispatching/timeout.py +0 -0
  21. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/expiry/__init__.py +0 -0
  22. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/expiry/run_expiry.py +0 -0
  23. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/ingester/__init__.py +0 -0
  24. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/ingester/__main__.py +0 -0
  25. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/ingester/constants.py +0 -0
  26. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/ingester/ingester.py +0 -0
  27. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/metrics/__init__.py +0 -0
  28. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/metrics/es_metrics.py +0 -0
  29. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  30. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/metrics/helper.py +0 -0
  31. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/metrics/metrics_server.py +0 -0
  32. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  33. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  34. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  35. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/plumber/__init__.py +0 -0
  36. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/plumber/run_plumber.py +0 -0
  37. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/replay/__init__.py +0 -0
  38. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/replay/client.py +0 -0
  39. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/replay/creator/__init__.py +0 -0
  40. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/replay/creator/run.py +0 -0
  41. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/replay/creator/run_worker.py +0 -0
  42. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/replay/loader/__init__.py +0 -0
  43. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/replay/loader/run.py +0 -0
  44. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/replay/loader/run_worker.py +0 -0
  45. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/replay/replay.py +0 -0
  46. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/safelist_client.py +0 -0
  47. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/scaler/__init__.py +0 -0
  48. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/scaler/collection.py +0 -0
  49. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  50. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  51. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/scaler/controllers/interface.py +0 -0
  52. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  53. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/scaler/run_scaler.py +0 -0
  54. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/scaler/scaler_server.py +0 -0
  55. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/server_base.py +0 -0
  56. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/signature_client.py +0 -0
  57. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/submission_client.py +0 -0
  58. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/tasking_client.py +0 -0
  59. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/updater/__init__.py +0 -0
  60. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/updater/run_updater.py +0 -0
  61. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/vacuum/__init__.py +0 -0
  62. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/vacuum/crawler.py +0 -0
  63. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/vacuum/department_map.py +0 -0
  64. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/vacuum/safelist.py +0 -0
  65. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/vacuum/stream_map.py +0 -0
  66. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/vacuum/worker.py +0 -0
  67. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/workflow/__init__.py +0 -0
  68. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core/workflow/run_workflow.py +0 -0
  69. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  70. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  71. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core.egg-info/requires.txt +0 -0
  72. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/assemblyline_core.egg-info/top_level.txt +0 -0
  73. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/setup.cfg +0 -0
  74. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/setup.py +0 -0
  75. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/test/test_alerter.py +0 -0
  76. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/test/test_badlist_client.py +0 -0
  77. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/test/test_dispatcher.py +0 -0
  78. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/test/test_expiry.py +0 -0
  79. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/test/test_plumber.py +0 -0
  80. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/test/test_replay.py +0 -0
  81. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/test/test_safelist_client.py +0 -0
  82. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/test/test_scaler.py +0 -0
  83. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/test/test_scheduler.py +0 -0
  84. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/test/test_signature_client.py +0 -0
  85. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/test/test_simulation.py +0 -0
  86. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/test/test_vacuum.py +0 -0
  87. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/test/test_worker_ingest.py +0 -0
  88. {assemblyline-core-4.5.0.10 → assemblyline-core-4.5.0.11}/test/test_worker_submit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.0.10
3
+ Version: 4.5.0.11
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.5.0.11
@@ -18,12 +18,12 @@ DEFAULT_DOCKER_REGISTRY = "registry.hub.docker.com"
18
18
 
19
19
  class ContainerRegistry():
20
20
  # Provide a means of obtaining a list of tags from a container registry
21
- def _get_proprietary_registry_tags(self, server, image_name, auth, verify):
21
+ def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None, token_server=None):
22
22
  raise NotImplementedError()
23
23
 
24
24
 
25
25
  class DockerRegistry(ContainerRegistry):
26
- def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None):
26
+ def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None, token_server=None):
27
27
  # Find latest tag for each types
28
28
  url = f"https://{server}/v2/{image_name}/tags/list"
29
29
 
@@ -31,6 +31,14 @@ class DockerRegistry(ContainerRegistry):
31
31
  headers = {}
32
32
  if auth:
33
33
  headers["Authorization"] = auth
34
+ else:
35
+ # Retrieve token for authentication: https://distribution.github.io/distribution/spec/auth/token/
36
+
37
+ # Assume the token server is the same as the container image registry host if not explicitly set
38
+ token_server = token_server if token_server else server
39
+ token_url = f"https://{token_server}/token?scope=repository:{image_name}:pull"
40
+ token = requests.get(token_url).json().get('token')
41
+ headers["Authorization"] = f"Bearer {token}"
34
42
 
35
43
  resp = None
36
44
  try:
@@ -50,7 +58,7 @@ class DockerRegistry(ContainerRegistry):
50
58
 
51
59
 
52
60
  class HarborRegistry(ContainerRegistry):
53
- def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None):
61
+ def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None, token_server=None):
54
62
  # Determine project/repo IDs from image name
55
63
  project_id, repo_id = image_name.split('/', 1)
56
64
  repo_id = repo_id.replace('/', "%2F")
@@ -59,7 +67,14 @@ class HarborRegistry(ContainerRegistry):
59
67
  headers = {}
60
68
  if auth:
61
69
  headers["Authorization"] = auth
70
+ else:
71
+ # Retrieve token for authentication: https://github.com/goharbor/harbor/wiki/Harbor-FAQs#api
62
72
 
73
+ # Assume the token server is the same as the container image registry host if not explicitly set
74
+ token_server = token_server if token_server else server
75
+ token_url = f"https://{server}/service/token?scope=repository:{image_name}:pull"
76
+ token = requests.get(token_url).json().get('token')
77
+ headers["Authorization"] = f"Bearer {token}"
63
78
  resp = None
64
79
  try:
65
80
  resp = requests.get(url, headers=headers, verify=verify, proxies=proxies)
@@ -131,25 +146,32 @@ def get_latest_tag_for_service(
131
146
  break
132
147
 
133
148
  if service_config.docker_config.registry_username and service_config.docker_config.registry_password:
149
+ # We're authenticating using Basic Auth
134
150
  auth_config = {
135
151
  'username': service_config.docker_config.registry_username,
136
152
  'password': service_config.docker_config.registry_password
137
153
  }
138
154
  upass = f"{service_config.docker_config.registry_username}:{service_config.docker_config.registry_password}"
139
155
  auth = f"Basic {b64encode(upass.encode()).decode()}"
156
+ elif service_config.docker_config.registry_password:
157
+ # We're assuming that if only a password is given, then this is a token
158
+ auth = f"Bearer {service_config.docker_config.registry_password}"
140
159
 
141
160
  registry = REGISTRY_TYPE_MAPPING[service_config.docker_config.registry_type]
161
+ token_server = None
142
162
  proxies = None
143
163
  for reg_conf in system_config.core.updater.registry_configs:
144
164
  if reg_conf.name == server:
145
165
  proxies = reg_conf.proxies or None
166
+ token_server = reg_conf.token_server or None
146
167
  break
147
168
 
148
169
  if server == DEFAULT_DOCKER_REGISTRY:
149
170
  tags = _get_dockerhub_tags(image_name, update_channel, proxies)
150
171
  else:
151
172
  tags = registry._get_proprietary_registry_tags(server, image_name, auth,
152
- not system_config.services.allow_insecure_registry, proxies)
173
+ not system_config.services.allow_insecure_registry,
174
+ proxies, token_server)
153
175
 
154
176
  tag_name = None
155
177
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.0.10
3
+ Version: 4.5.0.11
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.5.0.10