ciocore 7.0.2b5__py2.py3-none-any.whl → 8.0.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ciocore might be problematic. Click here for more details.
- ciocore/VERSION +1 -1
- ciocore/__init__.py +23 -1
- ciocore/api_client.py +422 -156
- ciocore/cli.py +503 -0
- ciocore/common.py +10 -1
- ciocore/config.py +86 -53
- ciocore/data.py +20 -73
- ciocore/docsite/404.html +723 -0
- ciocore/docsite/apidoc/api_client/index.html +3203 -0
- ciocore/docsite/apidoc/apidoc/index.html +868 -0
- ciocore/docsite/apidoc/config/index.html +1591 -0
- ciocore/docsite/apidoc/data/index.html +1480 -0
- ciocore/docsite/apidoc/hardware_set/index.html +2367 -0
- ciocore/docsite/apidoc/package_environment/index.html +1450 -0
- ciocore/docsite/apidoc/package_tree/index.html +2310 -0
- ciocore/docsite/assets/_mkdocstrings.css +16 -0
- ciocore/docsite/assets/images/favicon.png +0 -0
- ciocore/docsite/assets/javascripts/bundle.4e31edb1.min.js +29 -0
- ciocore/docsite/assets/javascripts/bundle.4e31edb1.min.js.map +8 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.ar.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.da.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.de.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.du.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.es.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.fi.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.fr.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.hi.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.hu.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.hy.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.it.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.ja.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.jp.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.kn.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.ko.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.multi.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.nl.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.no.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.pt.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.ro.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.ru.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.sa.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.stemmer.support.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.sv.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.ta.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.te.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.th.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.tr.min.js +18 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.vi.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/min/lunr.zh.min.js +1 -0
- ciocore/docsite/assets/javascripts/lunr/tinyseg.js +206 -0
- ciocore/docsite/assets/javascripts/lunr/wordcut.js +6708 -0
- ciocore/docsite/assets/javascripts/workers/search.dfff1995.min.js +42 -0
- ciocore/docsite/assets/javascripts/workers/search.dfff1995.min.js.map +8 -0
- ciocore/docsite/assets/stylesheets/main.83068744.min.css +1 -0
- ciocore/docsite/assets/stylesheets/main.83068744.min.css.map +1 -0
- ciocore/docsite/assets/stylesheets/palette.ecc896b0.min.css +1 -0
- ciocore/docsite/assets/stylesheets/palette.ecc896b0.min.css.map +1 -0
- ciocore/docsite/cmdline/docs/index.html +834 -0
- ciocore/docsite/cmdline/downloader/index.html +897 -0
- ciocore/docsite/cmdline/packages/index.html +841 -0
- ciocore/docsite/cmdline/uploader/index.html +950 -0
- ciocore/docsite/how-to-guides/index.html +831 -0
- ciocore/docsite/index.html +853 -0
- ciocore/docsite/logo.png +0 -0
- ciocore/docsite/objects.inv +0 -0
- ciocore/docsite/search/search_index.json +1 -0
- ciocore/docsite/sitemap.xml +3 -0
- ciocore/docsite/sitemap.xml.gz +0 -0
- ciocore/docsite/stylesheets/extra.css +26 -0
- ciocore/docsite/stylesheets/tables.css +167 -0
- ciocore/downloader/__init__.py +0 -0
- ciocore/downloader/base_downloader.py +644 -0
- ciocore/downloader/download_runner_base.py +47 -0
- ciocore/downloader/job_downloader.py +119 -0
- ciocore/{downloader.py → downloader/legacy_downloader.py} +0 -1
- ciocore/downloader/log.py +73 -0
- ciocore/downloader/logging_download_runner.py +87 -0
- ciocore/downloader/perpetual_downloader.py +63 -0
- ciocore/downloader/registry.py +97 -0
- ciocore/downloader/reporter.py +135 -0
- ciocore/file_utils.py +3 -3
- ciocore/hardware_set.py +0 -4
- ciocore/package_environment.py +67 -75
- ciocore/package_query.py +171 -0
- ciocore/package_tree.py +300 -377
- ciocore/retry.py +0 -0
- ciocore/uploader/_uploader.py +205 -152
- {ciocore-7.0.2b5.dist-info → ciocore-8.0.0.dist-info}/METADATA +34 -16
- ciocore-8.0.0.dist-info/RECORD +127 -0
- {ciocore-7.0.2b5.dist-info → ciocore-8.0.0.dist-info}/WHEEL +1 -1
- ciocore-8.0.0.dist-info/entry_points.txt +2 -0
- tests/extra_env_fixtures.py +57 -0
- tests/instance_type_fixtures.py +42 -8
- tests/project_fixtures.py +8 -0
- tests/test_api_client.py +121 -2
- tests/test_base_downloader.py +104 -0
- tests/test_cli.py +163 -0
- tests/test_common.py +8 -8
- tests/test_config.py +23 -9
- tests/test_data.py +144 -160
- tests/test_downloader.py +118 -0
- tests/test_hardware_set.py +69 -20
- tests/test_job_downloader.py +213 -0
- ciocore/__about__.py +0 -10
- ciocore/cli/__init__.py +0 -3
- ciocore/cli/conductor.py +0 -210
- ciocore-7.0.2b5.data/scripts/conductor +0 -19
- ciocore-7.0.2b5.data/scripts/conductor.bat +0 -13
- ciocore-7.0.2b5.dist-info/RECORD +0 -51
- tests/mocks/api_client_mock.py +0 -31
- {ciocore-7.0.2b5.dist-info → ciocore-8.0.0.dist-info}/top_level.txt +0 -0
ciocore/api_client.py
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The api_client module is used to make requests to the Conductor API.
|
|
3
|
+
"""
|
|
4
|
+
|
|
1
5
|
import base64
|
|
2
6
|
import importlib
|
|
3
7
|
import json
|
|
@@ -9,21 +13,17 @@ import requests
|
|
|
9
13
|
import socket
|
|
10
14
|
import time
|
|
11
15
|
import sys
|
|
16
|
+
import platform
|
|
17
|
+
import hashlib
|
|
18
|
+
|
|
19
|
+
from urllib import parse
|
|
12
20
|
|
|
13
|
-
try:
|
|
14
|
-
from urllib import parse
|
|
15
|
-
except ImportError:
|
|
16
|
-
import urlparse as parse
|
|
17
|
-
|
|
18
21
|
import ciocore
|
|
19
22
|
|
|
20
23
|
from ciocore import config
|
|
21
24
|
from ciocore import common, auth
|
|
22
25
|
|
|
23
|
-
|
|
24
|
-
from ciocore.common import CONDUCTOR_LOGGER_NAME
|
|
25
|
-
|
|
26
|
-
logger = logging.getLogger(CONDUCTOR_LOGGER_NAME)
|
|
26
|
+
logger = logging.getLogger(__name__)
|
|
27
27
|
|
|
28
28
|
# A convenience tuple of network exceptions that can/should likely be retried by the retry decorator
|
|
29
29
|
try:
|
|
@@ -40,17 +40,43 @@ except AttributeError:
|
|
|
40
40
|
)
|
|
41
41
|
|
|
42
42
|
|
|
43
|
+
def truncate_middle(s, max_length):
|
|
44
|
+
"""
|
|
45
|
+
Truncate the string `s` to `max_length` by removing characters from the middle.
|
|
46
|
+
|
|
47
|
+
:param s: The original string to be truncated.
|
|
48
|
+
:type s: str
|
|
49
|
+
:param max_length: The maximum allowed length of the string after truncation.
|
|
50
|
+
:type max_length: int
|
|
51
|
+
:return: The truncated string.
|
|
52
|
+
:rtype: str
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
if len(s) <= max_length:
|
|
56
|
+
# String is already at or below the maximum length, return it as is
|
|
57
|
+
return s
|
|
58
|
+
|
|
59
|
+
# Calculate the number of characters to keep from the start and end of the string
|
|
60
|
+
num_keep_front = (max_length // 2)
|
|
61
|
+
num_keep_end = max_length - num_keep_front - 1 # -1 for the ellipsis
|
|
62
|
+
|
|
63
|
+
# Construct the truncated string
|
|
64
|
+
return s[:num_keep_front] + '~' + s[-num_keep_end:]
|
|
65
|
+
|
|
43
66
|
|
|
44
67
|
# TODO: appspot_dot_com_cert = os.path.join(common.base_dir(),'auth','appspot_dot_com_cert2') load
|
|
45
68
|
# appspot.com cert into requests lib verify = appspot_dot_com_cert
|
|
46
69
|
|
|
47
|
-
|
|
48
70
|
class ApiClient:
|
|
71
|
+
"""
|
|
72
|
+
The ApiClient class is a wrapper around the requests library that handles authentication and retries.
|
|
73
|
+
"""
|
|
74
|
+
|
|
49
75
|
http_verbs = ["PUT", "POST", "GET", "DELETE", "HEAD", "PATCH"]
|
|
50
|
-
|
|
76
|
+
|
|
51
77
|
USER_AGENT_TEMPLATE = "client {client_name}/{client_version} (ciocore {ciocore_version}; {runtime} {runtime_version}; {platform} {platform_details}; {hostname} {pid}; {python_path})"
|
|
52
78
|
USER_AGENT_MAX_PATH_LENGTH = 1024
|
|
53
|
-
|
|
79
|
+
|
|
54
80
|
user_agent_header = None
|
|
55
81
|
|
|
56
82
|
def __init__(self):
|
|
@@ -61,17 +87,17 @@ class ApiClient:
|
|
|
61
87
|
method=verb, url=conductor_url, headers=headers, params=params, data=data
|
|
62
88
|
)
|
|
63
89
|
|
|
64
|
-
logger.debug("verb:
|
|
65
|
-
logger.debug("conductor_url:
|
|
66
|
-
logger.debug("headers:
|
|
67
|
-
logger.debug("params:
|
|
68
|
-
logger.debug("data:
|
|
90
|
+
logger.debug(f"verb: {verb}")
|
|
91
|
+
logger.debug(f"conductor_url: {conductor_url}")
|
|
92
|
+
logger.debug(f"headers: {headers}")
|
|
93
|
+
logger.debug(f"params: {params}")
|
|
94
|
+
logger.debug(f"data: {data}")
|
|
69
95
|
|
|
70
96
|
# If we get 300s/400s debug out the response. TODO(lws): REMOVE THIS
|
|
71
|
-
if
|
|
97
|
+
if 300 <= response.status_code < 500:
|
|
72
98
|
logger.debug("***** ERROR!! *****")
|
|
73
|
-
logger.debug("Reason:
|
|
74
|
-
logger.debug("Text:
|
|
99
|
+
logger.debug(f"Reason: {response.reason}")
|
|
100
|
+
logger.debug(f"Text: {response.text}")
|
|
75
101
|
|
|
76
102
|
# trigger an exception to be raised for 4XX or 5XX http responses
|
|
77
103
|
if raise_on_error:
|
|
@@ -85,37 +111,38 @@ class ApiClient:
|
|
|
85
111
|
url,
|
|
86
112
|
headers=None,
|
|
87
113
|
params=None,
|
|
88
|
-
|
|
114
|
+
json_payload=None,
|
|
89
115
|
data=None,
|
|
90
116
|
stream=False,
|
|
91
117
|
remove_headers_list=None,
|
|
92
118
|
raise_on_error=True,
|
|
93
119
|
tries=5,
|
|
94
120
|
):
|
|
95
|
-
|
|
96
121
|
"""
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
122
|
+
Make a request to the Conductor API.
|
|
123
|
+
|
|
124
|
+
Deprecated:
|
|
125
|
+
Primarily used to removed enforced headers by requests.Request. Requests 2.x will add
|
|
126
|
+
Transfer-Encoding: chunked with file like object that is 0 bytes, causing s3 failures (501)
|
|
127
|
+
- https://github.com/psf/requests/issues/4215#issuecomment-319521235
|
|
128
|
+
|
|
129
|
+
To get around this bug make_prepared_request has functionality to remove the enforced header
|
|
130
|
+
that would occur when using requests.request(...). Requests 3.x resolves this issue, when
|
|
131
|
+
client is built to use Requests 3.x this function can be removed.
|
|
132
|
+
|
|
133
|
+
Returns:
|
|
134
|
+
requests.Response: The response object.
|
|
135
|
+
|
|
136
|
+
Args:
|
|
137
|
+
verb (str): The HTTP verb to use.
|
|
138
|
+
url (str): The URL to make the request to.
|
|
139
|
+
headers (dict): A dictionary of headers to send with the request.
|
|
140
|
+
params (dict): A dictionary of query parameters to send with the request.
|
|
141
|
+
json (dict): A JSON payload to send with the request.
|
|
142
|
+
stream (bool): Whether or not to stream the response.
|
|
143
|
+
remove_headers_list (list): A list of headers to remove from the request.
|
|
144
|
+
raise_on_error (bool): Whether or not to raise an exception if the request fails.
|
|
145
|
+
tries (int): The number of times to retry the request.
|
|
119
146
|
"""
|
|
120
147
|
|
|
121
148
|
req = requests.Request(
|
|
@@ -123,7 +150,7 @@ class ApiClient:
|
|
|
123
150
|
url=url,
|
|
124
151
|
headers=headers,
|
|
125
152
|
params=params,
|
|
126
|
-
json=
|
|
153
|
+
json=json_payload,
|
|
127
154
|
data=data,
|
|
128
155
|
)
|
|
129
156
|
prepped = req.prepare()
|
|
@@ -133,9 +160,11 @@ class ApiClient:
|
|
|
133
160
|
prepped.headers.pop(header, None)
|
|
134
161
|
|
|
135
162
|
# Create a retry wrapper function
|
|
136
|
-
retry_wrapper = common.DecRetry(
|
|
163
|
+
retry_wrapper = common.DecRetry(
|
|
164
|
+
retry_exceptions=CONNECTION_EXCEPTIONS, tries=tries
|
|
165
|
+
)
|
|
137
166
|
|
|
138
|
-
# requests sessions potentially not thread-safe, but need to
|
|
167
|
+
# requests sessions potentially not thread-safe, but need to removed enforced
|
|
139
168
|
# headers by using a prepared request.create which can only be done through an
|
|
140
169
|
# request.Session object. Create Session object per call of make_prepared_request, it will
|
|
141
170
|
# not benefit from connection pooling reuse. https://github.com/psf/requests/issues/1871
|
|
@@ -169,12 +198,26 @@ class ApiClient:
|
|
|
169
198
|
conductor_url=None,
|
|
170
199
|
raise_on_error=True,
|
|
171
200
|
tries=5,
|
|
172
|
-
use_api_key=False
|
|
201
|
+
use_api_key=False,
|
|
173
202
|
):
|
|
174
203
|
"""
|
|
175
|
-
|
|
204
|
+
Make a request to the Conductor API.
|
|
205
|
+
|
|
206
|
+
Args:
|
|
207
|
+
uri_path (str): The path to the resource to request.
|
|
208
|
+
headers (dict): A dictionary of headers to send with the request.
|
|
209
|
+
params (dict): A dictionary of query parameters to send with the request.
|
|
210
|
+
data (dict): A dictionary of data to send with the request.
|
|
211
|
+
verb (str): The HTTP verb to use.
|
|
212
|
+
conductor_url (str): The Conductor URL.
|
|
213
|
+
raise_on_error (bool): Whether or not to raise an exception if the request fails.
|
|
214
|
+
tries (int): The number of times to retry the request.
|
|
215
|
+
use`_api_key (bool): Whether or not to use the API key for authentication.
|
|
216
|
+
|
|
217
|
+
Returns:
|
|
218
|
+
tuple(str, int): The response text and status code.
|
|
176
219
|
"""
|
|
177
|
-
cfg = config.
|
|
220
|
+
cfg = config.get()
|
|
178
221
|
# TODO: set Content Content-Type to json if data arg
|
|
179
222
|
if not headers:
|
|
180
223
|
headers = {"Content-Type": "application/json", "Accept": "application/json"}
|
|
@@ -185,10 +228,10 @@ class ApiClient:
|
|
|
185
228
|
raise Exception("Error: Could not get conductor credentials!")
|
|
186
229
|
|
|
187
230
|
headers["Authorization"] = "Bearer %s" % bearer_token
|
|
188
|
-
|
|
231
|
+
|
|
189
232
|
if not ApiClient.user_agent_header:
|
|
190
233
|
self.register_client("ciocore")
|
|
191
|
-
|
|
234
|
+
|
|
192
235
|
headers["User-Agent"] = ApiClient.user_agent_header
|
|
193
236
|
|
|
194
237
|
# Construct URL
|
|
@@ -204,7 +247,9 @@ class ApiClient:
|
|
|
204
247
|
assert verb in self.http_verbs, "Invalid http verb: %s" % verb
|
|
205
248
|
|
|
206
249
|
# Create a retry wrapper function
|
|
207
|
-
retry_wrapper = common.DecRetry(
|
|
250
|
+
retry_wrapper = common.DecRetry(
|
|
251
|
+
retry_exceptions=CONNECTION_EXCEPTIONS, tries=tries
|
|
252
|
+
)
|
|
208
253
|
|
|
209
254
|
# wrap the request function with the retry wrapper
|
|
210
255
|
wrapped_func = retry_wrapper(self._make_request)
|
|
@@ -215,104 +260,56 @@ class ApiClient:
|
|
|
215
260
|
)
|
|
216
261
|
|
|
217
262
|
return response.text, response.status_code
|
|
218
|
-
|
|
219
|
-
@classmethod
|
|
220
|
-
def _get_user_agent_header(cls, client_name, client_version=None):
|
|
221
|
-
'''
|
|
222
|
-
Generates the http User Agent header that includes helpful debug info.
|
|
223
|
-
|
|
224
|
-
The final component is the path to the python executable (MD5 hex).
|
|
225
|
-
|
|
226
|
-
ex: 'ciomaya/0.3.7 (ciocore 4.3.2; python 3.9.5; linux 3.10.0-1160.53.1.el7.x86_64; 0ee7123c2365d7a0d126de5a70f19727)'
|
|
227
|
-
|
|
228
|
-
:param client_name: The name of the client to be used in the header. If it's importable it
|
|
229
|
-
will be queried for its __version__ (unless client_version is supplied)
|
|
230
|
-
:type client_name: str
|
|
231
|
-
|
|
232
|
-
:param client_version: The version to use in the header if client_name can't be queried for
|
|
233
|
-
__version__ (or it needs to be overridden)
|
|
234
|
-
:type client_version: str [default = None]
|
|
235
|
-
|
|
236
|
-
:return: The value for the User Agent header
|
|
237
|
-
:rtype: str
|
|
238
|
-
'''
|
|
239
|
-
|
|
240
|
-
try:
|
|
241
|
-
client_module = importlib.import_module(client_name)
|
|
242
|
-
|
|
243
|
-
except ImportError:
|
|
244
|
-
logger.warning("Unable to import module '%s'. Won't query for version details.", client_name)
|
|
245
|
-
client_version = 'unknown'
|
|
246
|
-
|
|
247
|
-
if not client_version:
|
|
248
|
-
try:
|
|
249
|
-
client_version = client_module.__version__
|
|
250
|
-
|
|
251
|
-
except AttributeError:
|
|
252
|
-
logger.warning("Module '%s' has no __version__ attribute. Setting version to 'N/A' in the user agent", client_name)
|
|
253
|
-
client_version = 'unknown'
|
|
254
|
-
|
|
255
|
-
ciocore_version = ciocore.__version__
|
|
256
|
-
|
|
257
|
-
python_path = sys.executable
|
|
258
|
-
|
|
259
|
-
# If the length of the path is longer than allowed, truncate the middle
|
|
260
|
-
if len(python_path) > cls.USER_AGENT_MAX_PATH_LENGTH:
|
|
261
|
-
|
|
262
|
-
first_half = int(cls.USER_AGENT_MAX_PATH_LENGTH/2)
|
|
263
|
-
second_half = len(python_path) - int(cls.USER_AGENT_MAX_PATH_LENGTH/2)
|
|
264
|
-
|
|
265
|
-
print(first_half, second_half)
|
|
266
|
-
python_path = "{}...{}".format(python_path[0:first_half], python_path[second_half:-1])
|
|
267
|
-
|
|
268
|
-
python_path_encoded = base64.b64encode(python_path.encode('utf-8')).decode('utf-8')
|
|
269
|
-
|
|
270
|
-
if platform.system() == "Linux":
|
|
271
|
-
platform_details = platform.release()
|
|
272
|
-
|
|
273
|
-
elif platform.system() == "Windows":
|
|
274
|
-
platform_details = platform.version()
|
|
275
|
-
|
|
276
|
-
elif platform.system() == "Darwin":
|
|
277
|
-
platform_details = platform.mac_ver()[0]
|
|
278
|
-
|
|
279
|
-
else:
|
|
280
|
-
raise ValueError("Unrecognized platform '{}'".format(platform.release()))
|
|
281
|
-
|
|
282
|
-
pid = base64.b64encode(str(os.getpid()).encode('utf-8')).decode('utf-8')
|
|
283
|
-
hostname = base64.b64encode(socket.gethostname().encode('utf-8')).decode('utf-8')
|
|
284
|
-
|
|
285
|
-
return cls.USER_AGENT_TEMPLATE.format(client_name=client_name,
|
|
286
|
-
client_version=client_version,
|
|
287
|
-
ciocore_version=ciocore_version,
|
|
288
|
-
runtime='python',
|
|
289
|
-
runtime_version=platform.python_version(),
|
|
290
|
-
platform=sys.platform,
|
|
291
|
-
platform_details=platform_details,
|
|
292
|
-
pid=pid,
|
|
293
|
-
hostname=hostname,
|
|
294
|
-
python_path=python_path_encoded
|
|
295
|
-
)
|
|
296
|
-
|
|
263
|
+
|
|
297
264
|
@classmethod
|
|
298
265
|
def register_client(cls, client_name, client_version=None):
|
|
299
|
-
|
|
266
|
+
"""
|
|
267
|
+
Generates the http User Agent header that includes helpful debug info.
|
|
268
|
+
"""
|
|
269
|
+
|
|
270
|
+
# Use the provided client_version.
|
|
271
|
+
if not client_version:
|
|
272
|
+
client_version = 'unknown'
|
|
273
|
+
|
|
274
|
+
python_version = platform.python_version()
|
|
275
|
+
system_info = platform.system()
|
|
276
|
+
release_info = platform.release()
|
|
300
277
|
|
|
278
|
+
|
|
279
|
+
# Get the MD5 hex digest of the path to the python executable
|
|
280
|
+
python_executable_path = truncate_middle(sys.executable.encode('utf-8'), cls.USER_AGENT_MAX_PATH_LENGTH)
|
|
281
|
+
md5_hash = hashlib.md5(python_executable_path).hexdigest()
|
|
282
|
+
|
|
283
|
+
user_agent = (
|
|
284
|
+
f"{client_name}/{client_version} "
|
|
285
|
+
f"(python {python_version}; {system_info} {release_info}; {md5_hash})"
|
|
286
|
+
)
|
|
287
|
+
cls.user_agent_header = user_agent
|
|
301
288
|
|
|
289
|
+
return user_agent
|
|
290
|
+
|
|
302
291
|
|
|
303
292
|
def read_conductor_credentials(use_api_key=False):
|
|
304
293
|
"""
|
|
305
|
-
Read the conductor credentials file
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
294
|
+
Read the conductor credentials file.
|
|
295
|
+
|
|
296
|
+
If the credentials file exists, it will contain a bearer token from either
|
|
297
|
+
the user or the API key.
|
|
298
|
+
|
|
299
|
+
If the credentials file doesn't exist, or is
|
|
300
|
+
expired, or is from a different domain, we try to fetch a new one in the API key scenario or
|
|
301
|
+
prompt the user to log in.
|
|
302
|
+
|
|
303
|
+
Args:
|
|
304
|
+
use_api_key (bool): Whether or not to try to use the API key
|
|
309
305
|
|
|
310
|
-
Returns:
|
|
306
|
+
Returns:
|
|
307
|
+
A Bearer token in the event of a success or None
|
|
311
308
|
|
|
312
309
|
"""
|
|
313
310
|
|
|
314
|
-
|
|
315
|
-
|
|
311
|
+
cfg = config.get()
|
|
312
|
+
|
|
316
313
|
logger.debug("Reading conductor credentials...")
|
|
317
314
|
if use_api_key:
|
|
318
315
|
if not cfg.get("api_key"):
|
|
@@ -336,38 +333,40 @@ def read_conductor_credentials(use_api_key=False):
|
|
|
336
333
|
|
|
337
334
|
else:
|
|
338
335
|
auth.run(creds_file, cfg["url"])
|
|
339
|
-
|
|
340
336
|
if not os.path.exists(creds_file):
|
|
341
337
|
return None
|
|
342
338
|
|
|
343
339
|
logger.debug("Reading credentials file...")
|
|
344
|
-
with open(creds_file) as fp:
|
|
340
|
+
with open(creds_file, "r", encoding="utf-8") as fp:
|
|
345
341
|
file_contents = json.loads(fp.read())
|
|
346
|
-
|
|
347
342
|
expiration = file_contents.get("expiration")
|
|
348
|
-
|
|
349
343
|
same_domain = creds_same_domain(file_contents)
|
|
350
|
-
|
|
351
344
|
if same_domain and expiration and expiration >= int(time.time()):
|
|
352
345
|
return file_contents["access_token"]
|
|
353
|
-
|
|
354
346
|
logger.debug("Credentials have expired or are from a different domain")
|
|
355
|
-
|
|
356
347
|
if use_api_key:
|
|
357
348
|
logger.debug("Refreshing API key bearer token!")
|
|
358
349
|
get_api_key_bearer_token(creds_file)
|
|
359
350
|
else:
|
|
360
351
|
logger.debug("Sending to auth page...")
|
|
361
352
|
auth.run(creds_file, cfg["url"])
|
|
362
|
-
|
|
363
353
|
# Re-read the creds file, since it has been re-upped
|
|
364
|
-
with open(creds_file) as fp:
|
|
354
|
+
with open(creds_file, "r", encoding="utf-8") as fp:
|
|
365
355
|
file_contents = json.loads(fp.read())
|
|
366
356
|
return file_contents["access_token"]
|
|
367
357
|
|
|
368
358
|
|
|
369
359
|
def get_api_key_bearer_token(creds_file=None):
|
|
370
|
-
|
|
360
|
+
"""
|
|
361
|
+
Get a bearer token from the API key.
|
|
362
|
+
|
|
363
|
+
Args:
|
|
364
|
+
creds_file (str): The path to the credentials file. If not provided, the bearer token will not be written to disk.
|
|
365
|
+
|
|
366
|
+
Returns:
|
|
367
|
+
A dictionary containing the bearer token and other information.
|
|
368
|
+
"""
|
|
369
|
+
cfg = config.get()
|
|
371
370
|
url = "{}/api/oauth_jwt".format(cfg["url"])
|
|
372
371
|
response = requests.get(
|
|
373
372
|
url,
|
|
@@ -399,7 +398,15 @@ def get_api_key_bearer_token(creds_file=None):
|
|
|
399
398
|
|
|
400
399
|
|
|
401
400
|
def get_creds_path(api_key=False):
|
|
402
|
-
|
|
401
|
+
"""
|
|
402
|
+
Get the path to the credentials file.
|
|
403
|
+
|
|
404
|
+
Args:
|
|
405
|
+
api_key (bool): Whether or not to use the API key.
|
|
406
|
+
|
|
407
|
+
Returns:
|
|
408
|
+
str: The path to the credentials file.
|
|
409
|
+
"""
|
|
403
410
|
creds_dir = os.path.join(os.path.expanduser("~"), ".config", "conductor")
|
|
404
411
|
if api_key:
|
|
405
412
|
creds_file = os.path.join(creds_dir, "api_key_credentials")
|
|
@@ -412,13 +419,25 @@ def get_bearer_token(refresh=False):
|
|
|
412
419
|
"""
|
|
413
420
|
Return the bearer token.
|
|
414
421
|
|
|
422
|
+
Args:
|
|
423
|
+
refresh (bool): Whether or not to refresh the token.
|
|
424
|
+
|
|
415
425
|
TODO: Thread safe multiproc caching, like it used to be pre-python3.7.
|
|
416
426
|
"""
|
|
417
427
|
return read_conductor_credentials(True)
|
|
418
428
|
|
|
419
429
|
|
|
420
430
|
def creds_same_domain(creds):
|
|
421
|
-
|
|
431
|
+
"""
|
|
432
|
+
Check if the creds are for the same domain as the config.
|
|
433
|
+
|
|
434
|
+
Args:
|
|
435
|
+
creds (dict): The credentials dictionary.
|
|
436
|
+
|
|
437
|
+
Returns:
|
|
438
|
+
bool: Whether or not the creds are for the same domain as the config.
|
|
439
|
+
"""
|
|
440
|
+
cfg = config.get()
|
|
422
441
|
"""Ensure the creds file refers to the domain in config"""
|
|
423
442
|
token = creds.get("access_token")
|
|
424
443
|
if not token:
|
|
@@ -435,6 +454,12 @@ def creds_same_domain(creds):
|
|
|
435
454
|
def account_id_from_jwt(token):
|
|
436
455
|
"""
|
|
437
456
|
Fetch the accounts id from a jwt token value.
|
|
457
|
+
|
|
458
|
+
Args:
|
|
459
|
+
token (str): The jwt token.
|
|
460
|
+
|
|
461
|
+
Returns:
|
|
462
|
+
str: The account id.
|
|
438
463
|
"""
|
|
439
464
|
payload = jwt.decode(token, verify=False)
|
|
440
465
|
return payload.get("account")
|
|
@@ -443,9 +468,15 @@ def account_id_from_jwt(token):
|
|
|
443
468
|
def account_name_from_jwt(token):
|
|
444
469
|
"""
|
|
445
470
|
Fetch the accounts name from a jwt token value.
|
|
471
|
+
|
|
472
|
+
Args:
|
|
473
|
+
token (str): The jwt token.
|
|
474
|
+
|
|
475
|
+
Returns:
|
|
476
|
+
str: The account name.
|
|
446
477
|
"""
|
|
447
|
-
cfg = config.config().config
|
|
448
478
|
account_id = account_id_from_jwt(token)
|
|
479
|
+
cfg = config.get()
|
|
449
480
|
if account_id:
|
|
450
481
|
url = "%s/api/v1/accounts/%s" % (cfg["api_url"], account_id)
|
|
451
482
|
response = requests.get(url, headers={"authorization": "Bearer %s" % token})
|
|
@@ -458,6 +489,12 @@ def account_name_from_jwt(token):
|
|
|
458
489
|
def request_instance_types(as_dict=False):
|
|
459
490
|
"""
|
|
460
491
|
Get the list of available instances types.
|
|
492
|
+
|
|
493
|
+
Args:
|
|
494
|
+
as_dict (bool): Whether or not to return the instance types as a dictionary.
|
|
495
|
+
|
|
496
|
+
Returns:
|
|
497
|
+
list: The list of instance types.
|
|
461
498
|
"""
|
|
462
499
|
api = ApiClient()
|
|
463
500
|
response, response_code = api.make_request(
|
|
@@ -472,13 +509,21 @@ def request_instance_types(as_dict=False):
|
|
|
472
509
|
logger.debug("Found available instance types: %s", instance_types)
|
|
473
510
|
|
|
474
511
|
if as_dict:
|
|
475
|
-
return dict(
|
|
512
|
+
return dict(
|
|
513
|
+
[(instance["description"], instance) for instance in instance_types]
|
|
514
|
+
)
|
|
476
515
|
return instance_types
|
|
477
516
|
|
|
478
517
|
|
|
479
518
|
def request_projects(statuses=("active",)):
|
|
480
519
|
"""
|
|
481
|
-
Query Conductor for all client Projects that are in the given
|
|
520
|
+
Query Conductor for all client Projects that are in the given status(es).
|
|
521
|
+
|
|
522
|
+
Args:
|
|
523
|
+
statuses (tuple): The statuses to filter for.
|
|
524
|
+
|
|
525
|
+
Returns:
|
|
526
|
+
list: The list of project names.
|
|
482
527
|
"""
|
|
483
528
|
api = ApiClient()
|
|
484
529
|
|
|
@@ -507,6 +552,9 @@ def request_projects(statuses=("active",)):
|
|
|
507
552
|
def request_software_packages():
|
|
508
553
|
"""
|
|
509
554
|
Query Conductor for all software packages for the currently available sidecar.
|
|
555
|
+
|
|
556
|
+
Returns:
|
|
557
|
+
list: The list of software packages.
|
|
510
558
|
"""
|
|
511
559
|
api = ApiClient()
|
|
512
560
|
|
|
@@ -519,4 +567,222 @@ def request_software_packages():
|
|
|
519
567
|
msg = "Failed to get software packages for latest sidecar"
|
|
520
568
|
msg += "\nError %s ...\n%s" % (response_code, response)
|
|
521
569
|
raise Exception(msg)
|
|
522
|
-
|
|
570
|
+
|
|
571
|
+
software = json.loads(response).get("data", [])
|
|
572
|
+
software = [sw for sw in software if not ("3dsmax" in sw["product"] and sw["platform"] == "linux")]
|
|
573
|
+
return software
|
|
574
|
+
|
|
575
|
+
def request_extra_environment():
|
|
576
|
+
"""
|
|
577
|
+
Query Conductor for extra environment.
|
|
578
|
+
"""
|
|
579
|
+
api = ApiClient()
|
|
580
|
+
|
|
581
|
+
uri = "api/v1/integrations/env-vars-configs"
|
|
582
|
+
response, response_code = api.make_request(
|
|
583
|
+
uri_path=uri, verb="GET", raise_on_error=False, use_api_key=True
|
|
584
|
+
)
|
|
585
|
+
|
|
586
|
+
if response_code not in [200]:
|
|
587
|
+
msg = "Failed to get extra environment"
|
|
588
|
+
msg += "\nError %s ...\n%s" % (response_code, response)
|
|
589
|
+
raise Exception(msg)
|
|
590
|
+
|
|
591
|
+
all_accounts = json.loads(response).get("data", [])
|
|
592
|
+
|
|
593
|
+
token = read_conductor_credentials(True)
|
|
594
|
+
if not token:
|
|
595
|
+
raise Exception("Error: Could not get conductor credentials!")
|
|
596
|
+
account_id = str(account_id_from_jwt(token))
|
|
597
|
+
|
|
598
|
+
if not account_id:
|
|
599
|
+
raise Exception("Error: Could not get account id from jwt!")
|
|
600
|
+
account_env = next((account for account in all_accounts if account["account_id"] == account_id), None)
|
|
601
|
+
if not account_env:
|
|
602
|
+
raise Exception("Error: Could not get account environment!")
|
|
603
|
+
return account_env.get("env", [])
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def get_jobs(first_jid, last_jid=None):
|
|
609
|
+
"""
|
|
610
|
+
Query Conductor for all jobs between the given job ids.
|
|
611
|
+
|
|
612
|
+
Returns:
|
|
613
|
+
list: The list of jobs.
|
|
614
|
+
|
|
615
|
+
Raises:
|
|
616
|
+
Exception: If the request fails.
|
|
617
|
+
|
|
618
|
+
Examples:
|
|
619
|
+
>>> from ciocore import api_client
|
|
620
|
+
>>> jobs = api_client.get_jobs(1959)
|
|
621
|
+
>>> len(jobs)
|
|
622
|
+
1
|
|
623
|
+
>>> jobs[0]["jid"]
|
|
624
|
+
'01959'
|
|
625
|
+
>>> jobs = api_client.get_jobs(1959, 1961)
|
|
626
|
+
>>> len(jobs)
|
|
627
|
+
3
|
|
628
|
+
"""
|
|
629
|
+
if last_jid is None:
|
|
630
|
+
last_jid = first_jid
|
|
631
|
+
low = str(int(first_jid) - 1).zfill(5)
|
|
632
|
+
high = str(int(last_jid) + 1).zfill(5)
|
|
633
|
+
api = ApiClient()
|
|
634
|
+
uri = "api/v1/jobs"
|
|
635
|
+
|
|
636
|
+
response, response_code = api.make_request(
|
|
637
|
+
uri_path=uri,
|
|
638
|
+
verb="GET",
|
|
639
|
+
raise_on_error=False,
|
|
640
|
+
use_api_key=True,
|
|
641
|
+
params={"filter": f"jid_gt_{low},jid_lt_{high}"},
|
|
642
|
+
)
|
|
643
|
+
|
|
644
|
+
if response_code not in [200]:
|
|
645
|
+
msg = f"Failed to get jobs {first_jid}-{last_jid}"
|
|
646
|
+
msg += "\nError %s ...\n%s" % (response_code, response)
|
|
647
|
+
raise Exception(msg)
|
|
648
|
+
jobs = json.loads(response).get("data")
|
|
649
|
+
return jobs
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
def get_log(job_id, task_id):
|
|
653
|
+
"""
|
|
654
|
+
Get the log for the given job and task.
|
|
655
|
+
|
|
656
|
+
Args:
|
|
657
|
+
job_id (str): The job id.
|
|
658
|
+
task_id (str): The task id.
|
|
659
|
+
|
|
660
|
+
Returns:
|
|
661
|
+
list: A list of logs.
|
|
662
|
+
|
|
663
|
+
Raises:
|
|
664
|
+
Exception: If the request fails.
|
|
665
|
+
|
|
666
|
+
Examples:
|
|
667
|
+
>>> from ciocore import api_client
|
|
668
|
+
>>> logs = api_client.get_log(1959, 0)
|
|
669
|
+
{
|
|
670
|
+
"logs": [
|
|
671
|
+
{
|
|
672
|
+
"container_id": "j-5669544198668288-5619559933149184-5095331660038144-stde",
|
|
673
|
+
"instance_name": "renderer-5669544198668288-170062309438-62994",
|
|
674
|
+
"log": [
|
|
675
|
+
"Blender 2.93.0 (hash 84da05a8b806 built 2021-06-02 11:29:24)",
|
|
676
|
+
...
|
|
677
|
+
...
|
|
678
|
+
"Saved: '/var/folders/8r/46lmjdmj50x_0swd9klwptzm0000gq/T/blender_bmw/renders/render_0001.png'",
|
|
679
|
+
" Time: 00:29.22 (Saving: 00:00.32)",
|
|
680
|
+
"",
|
|
681
|
+
"",
|
|
682
|
+
"Blender quit"
|
|
683
|
+
],
|
|
684
|
+
"timestamp": "1.700623521101516E9"
|
|
685
|
+
}
|
|
686
|
+
],
|
|
687
|
+
"new_num_lines": [
|
|
688
|
+
144
|
|
689
|
+
],
|
|
690
|
+
"status_description": "",
|
|
691
|
+
"task_status": "success"
|
|
692
|
+
}
|
|
693
|
+
"""
|
|
694
|
+
job_id = str(job_id).zfill(5)
|
|
695
|
+
task_id = str(task_id).zfill(3)
|
|
696
|
+
|
|
697
|
+
api = ApiClient()
|
|
698
|
+
uri = f"get_log_file?job={job_id}&task={task_id}&num_lines[]=0"
|
|
699
|
+
|
|
700
|
+
response, response_code = api.make_request(
|
|
701
|
+
uri_path=uri, verb="GET", raise_on_error=False, use_api_key=True
|
|
702
|
+
)
|
|
703
|
+
|
|
704
|
+
if response_code not in [200]:
|
|
705
|
+
msg = f"Failed to get log for job {job_id} task {task_id}"
|
|
706
|
+
msg += "\nError %s ...\n%s" % (response_code, response)
|
|
707
|
+
raise Exception(msg)
|
|
708
|
+
|
|
709
|
+
return response
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
def kill_jobs(*job_ids):
|
|
713
|
+
"""
|
|
714
|
+
Kill the given jobs.
|
|
715
|
+
|
|
716
|
+
Args:
|
|
717
|
+
job_ids (list): The list of job ids.
|
|
718
|
+
|
|
719
|
+
Returns:
|
|
720
|
+
dict: The response.
|
|
721
|
+
|
|
722
|
+
Examples:
|
|
723
|
+
>>> from ciocore import api_client
|
|
724
|
+
>>> api_client.kill_jobs("03095","03094")
|
|
725
|
+
{'body': 'success', 'message': "Jobs [u'03095', u'03094'] have been kill."}
|
|
726
|
+
|
|
727
|
+
"""
|
|
728
|
+
job_ids = [str(job_id).zfill(5) for job_id in job_ids]
|
|
729
|
+
api = ApiClient()
|
|
730
|
+
payload = {
|
|
731
|
+
"action": "kill",
|
|
732
|
+
"jobids": job_ids,
|
|
733
|
+
}
|
|
734
|
+
response, response_code = api.make_request(
|
|
735
|
+
uri_path="jobs_multi",
|
|
736
|
+
verb="PUT",
|
|
737
|
+
raise_on_error=False,
|
|
738
|
+
use_api_key=True,
|
|
739
|
+
data=json.dumps(payload)
|
|
740
|
+
)
|
|
741
|
+
|
|
742
|
+
if response_code not in [200]:
|
|
743
|
+
msg = f"Failed to kill jobs {job_ids}"
|
|
744
|
+
msg += "\nError %s ...\n%s" % (response_code, response)
|
|
745
|
+
raise Exception(msg)
|
|
746
|
+
|
|
747
|
+
return json.loads(response)
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
def kill_tasks(job_id, *task_ids):
|
|
751
|
+
"""
|
|
752
|
+
Kill the given tasks.
|
|
753
|
+
|
|
754
|
+
Args:
|
|
755
|
+
job_id (str): The job id.
|
|
756
|
+
task_ids (list): The list of task ids.
|
|
757
|
+
|
|
758
|
+
Returns:
|
|
759
|
+
dict: The response.
|
|
760
|
+
|
|
761
|
+
Examples:
|
|
762
|
+
>>> from ciocore import api_client
|
|
763
|
+
>>> api_client.kill_tasks("03096", *range(50,56))
|
|
764
|
+
{'body': 'success', 'message': ' 6 Tasks set to "kill"\n\t050\n\t051\n\t052\n\t053\n\t054\n\t055'}
|
|
765
|
+
"""
|
|
766
|
+
|
|
767
|
+
job_id = str(job_id).zfill(5)
|
|
768
|
+
task_ids = [str(task_id).zfill(3) for task_id in task_ids]
|
|
769
|
+
api = ApiClient()
|
|
770
|
+
payload = {
|
|
771
|
+
"action": "kill",
|
|
772
|
+
"jobid": job_id,
|
|
773
|
+
"taskids": task_ids,
|
|
774
|
+
}
|
|
775
|
+
response, response_code = api.make_request(
|
|
776
|
+
uri_path="tasks_multi",
|
|
777
|
+
verb="PUT",
|
|
778
|
+
raise_on_error=False,
|
|
779
|
+
use_api_key=True,
|
|
780
|
+
data=json.dumps(payload)
|
|
781
|
+
)
|
|
782
|
+
|
|
783
|
+
if response_code not in [200]:
|
|
784
|
+
msg = f"Failed to kill tasks {task_ids} of job {job_id}"
|
|
785
|
+
msg += "\nError %s ...\n%s" % (response_code, response)
|
|
786
|
+
raise Exception(msg)
|
|
787
|
+
|
|
788
|
+
return json.loads(response)
|