assemblyline-core 4.5.1.dev67__tar.gz → 4.5.1.dev70__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of assemblyline-core might be problematic. Click here for more details.

Files changed (88) hide show
  1. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/PKG-INFO +1 -1
  2. assemblyline-core-4.5.1.dev70/assemblyline_core/VERSION +1 -0
  3. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/updater/helper.py +19 -11
  4. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core.egg-info/PKG-INFO +1 -1
  5. assemblyline-core-4.5.1.dev67/assemblyline_core/VERSION +0 -1
  6. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/LICENCE.md +0 -0
  7. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/README.md +0 -0
  8. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/__init__.py +0 -0
  9. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/alerter/__init__.py +0 -0
  10. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/alerter/processing.py +0 -0
  11. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/alerter/run_alerter.py +0 -0
  12. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/archiver/__init__.py +0 -0
  13. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/archiver/run_archiver.py +0 -0
  14. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/badlist_client.py +0 -0
  15. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/dispatching/__init__.py +0 -0
  16. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/dispatching/__main__.py +0 -0
  17. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/dispatching/client.py +0 -0
  18. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/dispatching/dispatcher.py +0 -0
  19. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/dispatching/schedules.py +0 -0
  20. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/dispatching/timeout.py +0 -0
  21. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/expiry/__init__.py +0 -0
  22. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/expiry/run_expiry.py +0 -0
  23. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/ingester/__init__.py +0 -0
  24. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/ingester/__main__.py +0 -0
  25. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/ingester/constants.py +0 -0
  26. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/ingester/ingester.py +0 -0
  27. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/metrics/__init__.py +0 -0
  28. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/metrics/es_metrics.py +0 -0
  29. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  30. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/metrics/helper.py +0 -0
  31. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/metrics/metrics_server.py +0 -0
  32. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  33. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  34. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  35. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/plumber/__init__.py +0 -0
  36. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/plumber/run_plumber.py +0 -0
  37. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/replay/__init__.py +0 -0
  38. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/replay/client.py +0 -0
  39. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/replay/creator/__init__.py +0 -0
  40. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/replay/creator/run.py +0 -0
  41. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/replay/creator/run_worker.py +0 -0
  42. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/replay/loader/__init__.py +0 -0
  43. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/replay/loader/run.py +0 -0
  44. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/replay/loader/run_worker.py +0 -0
  45. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/replay/replay.py +0 -0
  46. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/safelist_client.py +0 -0
  47. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/scaler/__init__.py +0 -0
  48. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/scaler/collection.py +0 -0
  49. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  50. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  51. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/scaler/controllers/interface.py +0 -0
  52. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  53. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/scaler/run_scaler.py +0 -0
  54. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/scaler/scaler_server.py +0 -0
  55. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/server_base.py +0 -0
  56. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/signature_client.py +0 -0
  57. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/submission_client.py +0 -0
  58. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/tasking_client.py +0 -0
  59. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/updater/__init__.py +0 -0
  60. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/updater/run_updater.py +0 -0
  61. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/vacuum/__init__.py +0 -0
  62. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/vacuum/crawler.py +0 -0
  63. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/vacuum/department_map.py +0 -0
  64. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/vacuum/safelist.py +0 -0
  65. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/vacuum/stream_map.py +0 -0
  66. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/vacuum/worker.py +0 -0
  67. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/workflow/__init__.py +0 -0
  68. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core/workflow/run_workflow.py +0 -0
  69. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  70. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  71. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core.egg-info/requires.txt +0 -0
  72. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/assemblyline_core.egg-info/top_level.txt +0 -0
  73. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/setup.cfg +0 -0
  74. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/setup.py +0 -0
  75. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/test/test_alerter.py +0 -0
  76. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/test/test_badlist_client.py +0 -0
  77. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/test/test_dispatcher.py +0 -0
  78. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/test/test_expiry.py +0 -0
  79. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/test/test_plumber.py +0 -0
  80. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/test/test_replay.py +0 -0
  81. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/test/test_safelist_client.py +0 -0
  82. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/test/test_scaler.py +0 -0
  83. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/test/test_scheduler.py +0 -0
  84. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/test/test_signature_client.py +0 -0
  85. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/test/test_simulation.py +0 -0
  86. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/test/test_vacuum.py +0 -0
  87. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/test/test_worker_ingest.py +0 -0
  88. {assemblyline-core-4.5.1.dev67 → assemblyline-core-4.5.1.dev70}/test/test_worker_submit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev67
3
+ Version: 4.5.1.dev70
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.5.1.dev70
@@ -48,7 +48,9 @@ class ContainerRegistry():
48
48
  def get_image_os(self, image_name, image_tag) -> str:
49
49
  raise NotImplementedError()
50
50
 
51
-
51
+ def get_token(self, image_name) -> None: ...
52
+
53
+
52
54
  class DockerHub(ContainerRegistry):
53
55
  def __init__(self, update_channel, proxies: Dict[str, str] = None, *args, **kwargs):
54
56
  super().__init__(DEFAULT_DOCKER_REGISTRY, None, True, proxies)
@@ -72,16 +74,18 @@ class DockerRegistry(ContainerRegistry):
72
74
  def __init__(self, server, headers: Dict[str, str] = None, verify: bool = True,
73
75
  proxies: Dict[str, str] = None, token_server: str = None, *args, **kwargs):
74
76
  super().__init__(server, headers, verify, proxies, *args, **kwargs)
75
-
77
+ self.token_server = token_server
78
+
79
+ def get_token(self, image_name) -> None:
76
80
  if not self.session.headers.get('Authorization'):
77
81
  # Retrieve token for authentication: https://distribution.github.io/distribution/spec/auth/token/
78
-
82
+
79
83
  # Assume the token server is the same as the container image registry host if not explicitly set
80
- token_server = token_server if token_server else server
84
+ token_server = self.token_server if self.token_server else self.server
81
85
  token_url = f"https://{token_server}/token?scope=repository:{image_name}:pull"
82
86
  token = requests.get(token_url).json().get('token')
83
- self.session.headers["Authorization"] = f"Bearer {token}"
84
-
87
+ self.session.headers["Authorization"] = f"Bearer {token}"
88
+
85
89
  def get_image_tags(self, image_name) -> List[str]:
86
90
  # Find latest tag for each types
87
91
  resp = self._make_request(f"/v2/{image_name}/tags/list")
@@ -93,7 +97,8 @@ class DockerRegistry(ContainerRegistry):
93
97
  resp = self._make_request(f"/v2/{image_name}/manifests/{image_tag}")
94
98
  if resp:
95
99
  # Retrieve OS compatibilty from historical record
96
- return json.loads(resp['history'][0]['v1Compatibility'])['os']
100
+ if resp['schemaVersion'] == 1:
101
+ return json.loads(resp['history'][0]['v1Compatibility'])['os']
97
102
 
98
103
  # Unable to determine the OS compatibility
99
104
  return None
@@ -104,16 +109,18 @@ class HarborRegistry(ContainerRegistry):
104
109
  def __init__(self, server, headers: Dict[str, str] = None, verify: bool = True,
105
110
  proxies: Dict[str, str] = None, token_server: str = None, *args, **kwargs):
106
111
  super().__init__(server, headers, verify, proxies, *args, **kwargs)
107
-
112
+ self.token_server = token_server
113
+
114
+ def get_token(self, image_name) -> None:
108
115
  if not self.session.headers.get('Authorization'):
109
116
  # Retrieve token for authentication: https://github.com/goharbor/harbor/wiki/Harbor-FAQs#api
110
117
 
111
118
  # Assume the token server is the same as the container image registry host if not explicitly set
112
- token_server = token_server if token_server else server
119
+ token_server = self.token_server if self.token_server else self.server
113
120
  token_url = f"https://{server}/service/token?scope=repository:{image_name}:pull"
114
121
  token = requests.get(token_url).json().get('token')
115
- headers["Authorization"] = f"Bearer {token}"
116
-
122
+ self.headers["Authorization"] = f"Bearer {token}"
123
+
117
124
  def _get_project_repo_ids(self, image_name) -> Tuple[str, str]:
118
125
  # Determine project/repo IDs from image name
119
126
  project_id, repo_id = image_name.split('/', 1)
@@ -225,6 +232,7 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
225
232
  }
226
233
 
227
234
  registry: ContainerRegistry = REGISTRY_TYPE_MAPPING[registry_type](**registry_args)
235
+ registry.get_token(image_name)
228
236
  tags = registry.get_image_tags(image_name)
229
237
 
230
238
  tag_name = None
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev67
3
+ Version: 4.5.1.dev70
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.5.1.dev67