ciocore 7.0.2b5__py2.py3-none-any.whl → 8.0.0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ciocore might be problematic. Click here for more details.

Files changed (111) hide show
  1. ciocore/VERSION +1 -1
  2. ciocore/__init__.py +23 -1
  3. ciocore/api_client.py +422 -156
  4. ciocore/cli.py +503 -0
  5. ciocore/common.py +10 -1
  6. ciocore/config.py +86 -53
  7. ciocore/data.py +20 -73
  8. ciocore/docsite/404.html +723 -0
  9. ciocore/docsite/apidoc/api_client/index.html +3203 -0
  10. ciocore/docsite/apidoc/apidoc/index.html +868 -0
  11. ciocore/docsite/apidoc/config/index.html +1591 -0
  12. ciocore/docsite/apidoc/data/index.html +1480 -0
  13. ciocore/docsite/apidoc/hardware_set/index.html +2367 -0
  14. ciocore/docsite/apidoc/package_environment/index.html +1450 -0
  15. ciocore/docsite/apidoc/package_tree/index.html +2310 -0
  16. ciocore/docsite/assets/_mkdocstrings.css +16 -0
  17. ciocore/docsite/assets/images/favicon.png +0 -0
  18. ciocore/docsite/assets/javascripts/bundle.4e31edb1.min.js +29 -0
  19. ciocore/docsite/assets/javascripts/bundle.4e31edb1.min.js.map +8 -0
  20. ciocore/docsite/assets/javascripts/lunr/min/lunr.ar.min.js +1 -0
  21. ciocore/docsite/assets/javascripts/lunr/min/lunr.da.min.js +18 -0
  22. ciocore/docsite/assets/javascripts/lunr/min/lunr.de.min.js +18 -0
  23. ciocore/docsite/assets/javascripts/lunr/min/lunr.du.min.js +18 -0
  24. ciocore/docsite/assets/javascripts/lunr/min/lunr.es.min.js +18 -0
  25. ciocore/docsite/assets/javascripts/lunr/min/lunr.fi.min.js +18 -0
  26. ciocore/docsite/assets/javascripts/lunr/min/lunr.fr.min.js +18 -0
  27. ciocore/docsite/assets/javascripts/lunr/min/lunr.hi.min.js +1 -0
  28. ciocore/docsite/assets/javascripts/lunr/min/lunr.hu.min.js +18 -0
  29. ciocore/docsite/assets/javascripts/lunr/min/lunr.hy.min.js +1 -0
  30. ciocore/docsite/assets/javascripts/lunr/min/lunr.it.min.js +18 -0
  31. ciocore/docsite/assets/javascripts/lunr/min/lunr.ja.min.js +1 -0
  32. ciocore/docsite/assets/javascripts/lunr/min/lunr.jp.min.js +1 -0
  33. ciocore/docsite/assets/javascripts/lunr/min/lunr.kn.min.js +1 -0
  34. ciocore/docsite/assets/javascripts/lunr/min/lunr.ko.min.js +1 -0
  35. ciocore/docsite/assets/javascripts/lunr/min/lunr.multi.min.js +1 -0
  36. ciocore/docsite/assets/javascripts/lunr/min/lunr.nl.min.js +18 -0
  37. ciocore/docsite/assets/javascripts/lunr/min/lunr.no.min.js +18 -0
  38. ciocore/docsite/assets/javascripts/lunr/min/lunr.pt.min.js +18 -0
  39. ciocore/docsite/assets/javascripts/lunr/min/lunr.ro.min.js +18 -0
  40. ciocore/docsite/assets/javascripts/lunr/min/lunr.ru.min.js +18 -0
  41. ciocore/docsite/assets/javascripts/lunr/min/lunr.sa.min.js +1 -0
  42. ciocore/docsite/assets/javascripts/lunr/min/lunr.stemmer.support.min.js +1 -0
  43. ciocore/docsite/assets/javascripts/lunr/min/lunr.sv.min.js +18 -0
  44. ciocore/docsite/assets/javascripts/lunr/min/lunr.ta.min.js +1 -0
  45. ciocore/docsite/assets/javascripts/lunr/min/lunr.te.min.js +1 -0
  46. ciocore/docsite/assets/javascripts/lunr/min/lunr.th.min.js +1 -0
  47. ciocore/docsite/assets/javascripts/lunr/min/lunr.tr.min.js +18 -0
  48. ciocore/docsite/assets/javascripts/lunr/min/lunr.vi.min.js +1 -0
  49. ciocore/docsite/assets/javascripts/lunr/min/lunr.zh.min.js +1 -0
  50. ciocore/docsite/assets/javascripts/lunr/tinyseg.js +206 -0
  51. ciocore/docsite/assets/javascripts/lunr/wordcut.js +6708 -0
  52. ciocore/docsite/assets/javascripts/workers/search.dfff1995.min.js +42 -0
  53. ciocore/docsite/assets/javascripts/workers/search.dfff1995.min.js.map +8 -0
  54. ciocore/docsite/assets/stylesheets/main.83068744.min.css +1 -0
  55. ciocore/docsite/assets/stylesheets/main.83068744.min.css.map +1 -0
  56. ciocore/docsite/assets/stylesheets/palette.ecc896b0.min.css +1 -0
  57. ciocore/docsite/assets/stylesheets/palette.ecc896b0.min.css.map +1 -0
  58. ciocore/docsite/cmdline/docs/index.html +834 -0
  59. ciocore/docsite/cmdline/downloader/index.html +897 -0
  60. ciocore/docsite/cmdline/packages/index.html +841 -0
  61. ciocore/docsite/cmdline/uploader/index.html +950 -0
  62. ciocore/docsite/how-to-guides/index.html +831 -0
  63. ciocore/docsite/index.html +853 -0
  64. ciocore/docsite/logo.png +0 -0
  65. ciocore/docsite/objects.inv +0 -0
  66. ciocore/docsite/search/search_index.json +1 -0
  67. ciocore/docsite/sitemap.xml +3 -0
  68. ciocore/docsite/sitemap.xml.gz +0 -0
  69. ciocore/docsite/stylesheets/extra.css +26 -0
  70. ciocore/docsite/stylesheets/tables.css +167 -0
  71. ciocore/downloader/__init__.py +0 -0
  72. ciocore/downloader/base_downloader.py +644 -0
  73. ciocore/downloader/download_runner_base.py +47 -0
  74. ciocore/downloader/job_downloader.py +119 -0
  75. ciocore/{downloader.py → downloader/legacy_downloader.py} +0 -1
  76. ciocore/downloader/log.py +73 -0
  77. ciocore/downloader/logging_download_runner.py +87 -0
  78. ciocore/downloader/perpetual_downloader.py +63 -0
  79. ciocore/downloader/registry.py +97 -0
  80. ciocore/downloader/reporter.py +135 -0
  81. ciocore/file_utils.py +3 -3
  82. ciocore/hardware_set.py +0 -4
  83. ciocore/package_environment.py +67 -75
  84. ciocore/package_query.py +171 -0
  85. ciocore/package_tree.py +300 -377
  86. ciocore/retry.py +0 -0
  87. ciocore/uploader/_uploader.py +205 -152
  88. {ciocore-7.0.2b5.dist-info → ciocore-8.0.0.dist-info}/METADATA +34 -16
  89. ciocore-8.0.0.dist-info/RECORD +127 -0
  90. {ciocore-7.0.2b5.dist-info → ciocore-8.0.0.dist-info}/WHEEL +1 -1
  91. ciocore-8.0.0.dist-info/entry_points.txt +2 -0
  92. tests/extra_env_fixtures.py +57 -0
  93. tests/instance_type_fixtures.py +42 -8
  94. tests/project_fixtures.py +8 -0
  95. tests/test_api_client.py +121 -2
  96. tests/test_base_downloader.py +104 -0
  97. tests/test_cli.py +163 -0
  98. tests/test_common.py +8 -8
  99. tests/test_config.py +23 -9
  100. tests/test_data.py +144 -160
  101. tests/test_downloader.py +118 -0
  102. tests/test_hardware_set.py +69 -20
  103. tests/test_job_downloader.py +213 -0
  104. ciocore/__about__.py +0 -10
  105. ciocore/cli/__init__.py +0 -3
  106. ciocore/cli/conductor.py +0 -210
  107. ciocore-7.0.2b5.data/scripts/conductor +0 -19
  108. ciocore-7.0.2b5.data/scripts/conductor.bat +0 -13
  109. ciocore-7.0.2b5.dist-info/RECORD +0 -51
  110. tests/mocks/api_client_mock.py +0 -31
  111. {ciocore-7.0.2b5.dist-info → ciocore-8.0.0.dist-info}/top_level.txt +0 -0
ciocore/api_client.py CHANGED
@@ -1,3 +1,7 @@
1
+ """
2
+ The api_client module is used to make requests to the Conductor API.
3
+ """
4
+
1
5
  import base64
2
6
  import importlib
3
7
  import json
@@ -9,21 +13,17 @@ import requests
9
13
  import socket
10
14
  import time
11
15
  import sys
16
+ import platform
17
+ import hashlib
18
+
19
+ from urllib import parse
12
20
 
13
- try:
14
- from urllib import parse
15
- except ImportError:
16
- import urlparse as parse
17
-
18
21
  import ciocore
19
22
 
20
23
  from ciocore import config
21
24
  from ciocore import common, auth
22
25
 
23
-
24
- from ciocore.common import CONDUCTOR_LOGGER_NAME
25
-
26
- logger = logging.getLogger(CONDUCTOR_LOGGER_NAME)
26
+ logger = logging.getLogger(__name__)
27
27
 
28
28
  # A convenience tuple of network exceptions that can/should likely be retried by the retry decorator
29
29
  try:
@@ -40,17 +40,43 @@ except AttributeError:
40
40
  )
41
41
 
42
42
 
43
+ def truncate_middle(s, max_length):
44
+ """
45
+ Truncate the string `s` to `max_length` by removing characters from the middle.
46
+
47
+ :param s: The original string to be truncated.
48
+ :type s: str
49
+ :param max_length: The maximum allowed length of the string after truncation.
50
+ :type max_length: int
51
+ :return: The truncated string.
52
+ :rtype: str
53
+ """
54
+
55
+ if len(s) <= max_length:
56
+ # String is already at or below the maximum length, return it as is
57
+ return s
58
+
59
+ # Calculate the number of characters to keep from the start and end of the string
60
+ num_keep_front = (max_length // 2)
61
+ num_keep_end = max_length - num_keep_front - 1 # -1 for the ellipsis
62
+
63
+ # Construct the truncated string
64
+ return s[:num_keep_front] + '~' + s[-num_keep_end:]
65
+
43
66
 
44
67
  # TODO: appspot_dot_com_cert = os.path.join(common.base_dir(),'auth','appspot_dot_com_cert2') load
45
68
  # appspot.com cert into requests lib verify = appspot_dot_com_cert
46
69
 
47
-
48
70
  class ApiClient:
71
+ """
72
+ The ApiClient class is a wrapper around the requests library that handles authentication and retries.
73
+ """
74
+
49
75
  http_verbs = ["PUT", "POST", "GET", "DELETE", "HEAD", "PATCH"]
50
-
76
+
51
77
  USER_AGENT_TEMPLATE = "client {client_name}/{client_version} (ciocore {ciocore_version}; {runtime} {runtime_version}; {platform} {platform_details}; {hostname} {pid}; {python_path})"
52
78
  USER_AGENT_MAX_PATH_LENGTH = 1024
53
-
79
+
54
80
  user_agent_header = None
55
81
 
56
82
  def __init__(self):
@@ -61,17 +87,17 @@ class ApiClient:
61
87
  method=verb, url=conductor_url, headers=headers, params=params, data=data
62
88
  )
63
89
 
64
- logger.debug("verb: %s", verb)
65
- logger.debug("conductor_url: %s", conductor_url)
66
- logger.debug("headers: %s", headers)
67
- logger.debug("params: %s", params)
68
- logger.debug("data: %s", data)
90
+ logger.debug(f"verb: {verb}")
91
+ logger.debug(f"conductor_url: {conductor_url}")
92
+ logger.debug(f"headers: {headers}")
93
+ logger.debug(f"params: {params}")
94
+ logger.debug(f"data: {data}")
69
95
 
70
96
  # If we get 300s/400s debug out the response. TODO(lws): REMOVE THIS
71
- if response.status_code and 300 <= response.status_code < 500:
97
+ if 300 <= response.status_code < 500:
72
98
  logger.debug("***** ERROR!! *****")
73
- logger.debug("Reason: %s" % response.reason)
74
- logger.debug("Text: %s" % response.text)
99
+ logger.debug(f"Reason: {response.reason}")
100
+ logger.debug(f"Text: {response.text}")
75
101
 
76
102
  # trigger an exception to be raised for 4XX or 5XX http responses
77
103
  if raise_on_error:
@@ -85,37 +111,38 @@ class ApiClient:
85
111
  url,
86
112
  headers=None,
87
113
  params=None,
88
- json=None,
114
+ json_payload=None,
89
115
  data=None,
90
116
  stream=False,
91
117
  remove_headers_list=None,
92
118
  raise_on_error=True,
93
119
  tries=5,
94
120
  ):
95
-
96
121
  """
97
- Primarily used to removed enforced headers by requests.Request. Requests 2.x will add
98
- Transfer-Encoding: chunked with file like object that is 0 bytes, causing s3 failures (501)
99
- - https://github.com/psf/requests/issues/4215#issuecomment-319521235
100
-
101
- To get around this bug make_prepared_request has functionality to remove the enforced header
102
- that would occur when using requests.request(...). Requests 3.x resolves this issue, when
103
- client is built to use Requests 3.x this function can be deprecated.
104
-
105
- args:
106
- verb: (str) of HTTP verbs
107
- url: (str) url
108
- headers: (dict)
109
- params: (dict)
110
- json: (dict)
111
- data: (varies)
112
- stream: (bool)
113
- remove_headers_list: list of headers to remove i.e ["Transfer-Encoding"]
114
- raise_on_error: (bool)
115
- tries: (int) number of attempts to perform request
116
-
117
-
118
- return: request.Response
122
+ Make a request to the Conductor API.
123
+
124
+ Deprecated:
125
+ Primarily used to removed enforced headers by requests.Request. Requests 2.x will add
126
+ Transfer-Encoding: chunked with file like object that is 0 bytes, causing s3 failures (501)
127
+ - https://github.com/psf/requests/issues/4215#issuecomment-319521235
128
+
129
+ To get around this bug make_prepared_request has functionality to remove the enforced header
130
+ that would occur when using requests.request(...). Requests 3.x resolves this issue, when
131
+ client is built to use Requests 3.x this function can be removed.
132
+
133
+ Returns:
134
+ requests.Response: The response object.
135
+
136
+ Args:
137
+ verb (str): The HTTP verb to use.
138
+ url (str): The URL to make the request to.
139
+ headers (dict): A dictionary of headers to send with the request.
140
+ params (dict): A dictionary of query parameters to send with the request.
141
+ json (dict): A JSON payload to send with the request.
142
+ stream (bool): Whether or not to stream the response.
143
+ remove_headers_list (list): A list of headers to remove from the request.
144
+ raise_on_error (bool): Whether or not to raise an exception if the request fails.
145
+ tries (int): The number of times to retry the request.
119
146
  """
120
147
 
121
148
  req = requests.Request(
@@ -123,7 +150,7 @@ class ApiClient:
123
150
  url=url,
124
151
  headers=headers,
125
152
  params=params,
126
- json=json,
153
+ json=json_payload,
127
154
  data=data,
128
155
  )
129
156
  prepped = req.prepare()
@@ -133,9 +160,11 @@ class ApiClient:
133
160
  prepped.headers.pop(header, None)
134
161
 
135
162
  # Create a retry wrapper function
136
- retry_wrapper = common.DecRetry(retry_exceptions=CONNECTION_EXCEPTIONS, tries=tries)
163
+ retry_wrapper = common.DecRetry(
164
+ retry_exceptions=CONNECTION_EXCEPTIONS, tries=tries
165
+ )
137
166
 
138
- # requests sessions potentially not thread-safe, but need to potentially removed enforced
167
+ # requests sessions potentially not thread-safe, but need to removed enforced
139
168
  # headers by using a prepared request.create which can only be done through an
140
169
  # request.Session object. Create Session object per call of make_prepared_request, it will
141
170
  # not benefit from connection pooling reuse. https://github.com/psf/requests/issues/1871
@@ -169,12 +198,26 @@ class ApiClient:
169
198
  conductor_url=None,
170
199
  raise_on_error=True,
171
200
  tries=5,
172
- use_api_key=False
201
+ use_api_key=False,
173
202
  ):
174
203
  """
175
- verb: PUT, POST, GET, DELETE, HEAD, PATCH
204
+ Make a request to the Conductor API.
205
+
206
+ Args:
207
+ uri_path (str): The path to the resource to request.
208
+ headers (dict): A dictionary of headers to send with the request.
209
+ params (dict): A dictionary of query parameters to send with the request.
210
+ data (dict): A dictionary of data to send with the request.
211
+ verb (str): The HTTP verb to use.
212
+ conductor_url (str): The Conductor URL.
213
+ raise_on_error (bool): Whether or not to raise an exception if the request fails.
214
+ tries (int): The number of times to retry the request.
215
+ use`_api_key (bool): Whether or not to use the API key for authentication.
216
+
217
+ Returns:
218
+ tuple(str, int): The response text and status code.
176
219
  """
177
- cfg = config.config().config
220
+ cfg = config.get()
178
221
  # TODO: set Content Content-Type to json if data arg
179
222
  if not headers:
180
223
  headers = {"Content-Type": "application/json", "Accept": "application/json"}
@@ -185,10 +228,10 @@ class ApiClient:
185
228
  raise Exception("Error: Could not get conductor credentials!")
186
229
 
187
230
  headers["Authorization"] = "Bearer %s" % bearer_token
188
-
231
+
189
232
  if not ApiClient.user_agent_header:
190
233
  self.register_client("ciocore")
191
-
234
+
192
235
  headers["User-Agent"] = ApiClient.user_agent_header
193
236
 
194
237
  # Construct URL
@@ -204,7 +247,9 @@ class ApiClient:
204
247
  assert verb in self.http_verbs, "Invalid http verb: %s" % verb
205
248
 
206
249
  # Create a retry wrapper function
207
- retry_wrapper = common.DecRetry(retry_exceptions=CONNECTION_EXCEPTIONS, tries=tries)
250
+ retry_wrapper = common.DecRetry(
251
+ retry_exceptions=CONNECTION_EXCEPTIONS, tries=tries
252
+ )
208
253
 
209
254
  # wrap the request function with the retry wrapper
210
255
  wrapped_func = retry_wrapper(self._make_request)
@@ -215,104 +260,56 @@ class ApiClient:
215
260
  )
216
261
 
217
262
  return response.text, response.status_code
218
-
219
- @classmethod
220
- def _get_user_agent_header(cls, client_name, client_version=None):
221
- '''
222
- Generates the http User Agent header that includes helpful debug info.
223
-
224
- The final component is the path to the python executable (MD5 hex).
225
-
226
- ex: 'ciomaya/0.3.7 (ciocore 4.3.2; python 3.9.5; linux 3.10.0-1160.53.1.el7.x86_64; 0ee7123c2365d7a0d126de5a70f19727)'
227
-
228
- :param client_name: The name of the client to be used in the header. If it's importable it
229
- will be queried for its __version__ (unless client_version is supplied)
230
- :type client_name: str
231
-
232
- :param client_version: The version to use in the header if client_name can't be queried for
233
- __version__ (or it needs to be overridden)
234
- :type client_version: str [default = None]
235
-
236
- :return: The value for the User Agent header
237
- :rtype: str
238
- '''
239
-
240
- try:
241
- client_module = importlib.import_module(client_name)
242
-
243
- except ImportError:
244
- logger.warning("Unable to import module '%s'. Won't query for version details.", client_name)
245
- client_version = 'unknown'
246
-
247
- if not client_version:
248
- try:
249
- client_version = client_module.__version__
250
-
251
- except AttributeError:
252
- logger.warning("Module '%s' has no __version__ attribute. Setting version to 'N/A' in the user agent", client_name)
253
- client_version = 'unknown'
254
-
255
- ciocore_version = ciocore.__version__
256
-
257
- python_path = sys.executable
258
-
259
- # If the length of the path is longer than allowed, truncate the middle
260
- if len(python_path) > cls.USER_AGENT_MAX_PATH_LENGTH:
261
-
262
- first_half = int(cls.USER_AGENT_MAX_PATH_LENGTH/2)
263
- second_half = len(python_path) - int(cls.USER_AGENT_MAX_PATH_LENGTH/2)
264
-
265
- print(first_half, second_half)
266
- python_path = "{}...{}".format(python_path[0:first_half], python_path[second_half:-1])
267
-
268
- python_path_encoded = base64.b64encode(python_path.encode('utf-8')).decode('utf-8')
269
-
270
- if platform.system() == "Linux":
271
- platform_details = platform.release()
272
-
273
- elif platform.system() == "Windows":
274
- platform_details = platform.version()
275
-
276
- elif platform.system() == "Darwin":
277
- platform_details = platform.mac_ver()[0]
278
-
279
- else:
280
- raise ValueError("Unrecognized platform '{}'".format(platform.release()))
281
-
282
- pid = base64.b64encode(str(os.getpid()).encode('utf-8')).decode('utf-8')
283
- hostname = base64.b64encode(socket.gethostname().encode('utf-8')).decode('utf-8')
284
-
285
- return cls.USER_AGENT_TEMPLATE.format(client_name=client_name,
286
- client_version=client_version,
287
- ciocore_version=ciocore_version,
288
- runtime='python',
289
- runtime_version=platform.python_version(),
290
- platform=sys.platform,
291
- platform_details=platform_details,
292
- pid=pid,
293
- hostname=hostname,
294
- python_path=python_path_encoded
295
- )
296
-
263
+
297
264
  @classmethod
298
265
  def register_client(cls, client_name, client_version=None):
299
- cls.user_agent_header = cls._get_user_agent_header(client_name, client_version)
266
+ """
267
+ Generates the http User Agent header that includes helpful debug info.
268
+ """
269
+
270
+ # Use the provided client_version.
271
+ if not client_version:
272
+ client_version = 'unknown'
273
+
274
+ python_version = platform.python_version()
275
+ system_info = platform.system()
276
+ release_info = platform.release()
300
277
 
278
+
279
+ # Get the MD5 hex digest of the path to the python executable
280
+ python_executable_path = truncate_middle(sys.executable.encode('utf-8'), cls.USER_AGENT_MAX_PATH_LENGTH)
281
+ md5_hash = hashlib.md5(python_executable_path).hexdigest()
282
+
283
+ user_agent = (
284
+ f"{client_name}/{client_version} "
285
+ f"(python {python_version}; {system_info} {release_info}; {md5_hash})"
286
+ )
287
+ cls.user_agent_header = user_agent
301
288
 
289
+ return user_agent
290
+
302
291
 
303
292
  def read_conductor_credentials(use_api_key=False):
304
293
  """
305
- Read the conductor credentials file, if it exists. This will contain a bearer token from either
306
- the user or the API key (if that's desired). If the credentials file doesn't exist, or is
307
- expired, or is from a different domain, try and fetch a new one in the API key scenario or
308
- prompt the user to log in. Args: use_api_key: Whether or not to use the API key
294
+ Read the conductor credentials file.
295
+
296
+ If the credentials file exists, it will contain a bearer token from either
297
+ the user or the API key.
298
+
299
+ If the credentials file doesn't exist, or is
300
+ expired, or is from a different domain, we try to fetch a new one in the API key scenario or
301
+ prompt the user to log in.
302
+
303
+ Args:
304
+ use_api_key (bool): Whether or not to try to use the API key
309
305
 
310
- Returns: A Bearer token in the event of a success or None if things couldn't get figured out
306
+ Returns:
307
+ A Bearer token in the event of a success or None
311
308
 
312
309
  """
313
310
 
314
- # TODO: use config.get(). Below call is deprecated
315
- cfg = config.config().config
311
+ cfg = config.get()
312
+
316
313
  logger.debug("Reading conductor credentials...")
317
314
  if use_api_key:
318
315
  if not cfg.get("api_key"):
@@ -336,38 +333,40 @@ def read_conductor_credentials(use_api_key=False):
336
333
 
337
334
  else:
338
335
  auth.run(creds_file, cfg["url"])
339
-
340
336
  if not os.path.exists(creds_file):
341
337
  return None
342
338
 
343
339
  logger.debug("Reading credentials file...")
344
- with open(creds_file) as fp:
340
+ with open(creds_file, "r", encoding="utf-8") as fp:
345
341
  file_contents = json.loads(fp.read())
346
-
347
342
  expiration = file_contents.get("expiration")
348
-
349
343
  same_domain = creds_same_domain(file_contents)
350
-
351
344
  if same_domain and expiration and expiration >= int(time.time()):
352
345
  return file_contents["access_token"]
353
-
354
346
  logger.debug("Credentials have expired or are from a different domain")
355
-
356
347
  if use_api_key:
357
348
  logger.debug("Refreshing API key bearer token!")
358
349
  get_api_key_bearer_token(creds_file)
359
350
  else:
360
351
  logger.debug("Sending to auth page...")
361
352
  auth.run(creds_file, cfg["url"])
362
-
363
353
  # Re-read the creds file, since it has been re-upped
364
- with open(creds_file) as fp:
354
+ with open(creds_file, "r", encoding="utf-8") as fp:
365
355
  file_contents = json.loads(fp.read())
366
356
  return file_contents["access_token"]
367
357
 
368
358
 
369
359
  def get_api_key_bearer_token(creds_file=None):
370
- cfg = config.config().config
360
+ """
361
+ Get a bearer token from the API key.
362
+
363
+ Args:
364
+ creds_file (str): The path to the credentials file. If not provided, the bearer token will not be written to disk.
365
+
366
+ Returns:
367
+ A dictionary containing the bearer token and other information.
368
+ """
369
+ cfg = config.get()
371
370
  url = "{}/api/oauth_jwt".format(cfg["url"])
372
371
  response = requests.get(
373
372
  url,
@@ -399,7 +398,15 @@ def get_api_key_bearer_token(creds_file=None):
399
398
 
400
399
 
401
400
  def get_creds_path(api_key=False):
402
-
401
+ """
402
+ Get the path to the credentials file.
403
+
404
+ Args:
405
+ api_key (bool): Whether or not to use the API key.
406
+
407
+ Returns:
408
+ str: The path to the credentials file.
409
+ """
403
410
  creds_dir = os.path.join(os.path.expanduser("~"), ".config", "conductor")
404
411
  if api_key:
405
412
  creds_file = os.path.join(creds_dir, "api_key_credentials")
@@ -412,13 +419,25 @@ def get_bearer_token(refresh=False):
412
419
  """
413
420
  Return the bearer token.
414
421
 
422
+ Args:
423
+ refresh (bool): Whether or not to refresh the token.
424
+
415
425
  TODO: Thread safe multiproc caching, like it used to be pre-python3.7.
416
426
  """
417
427
  return read_conductor_credentials(True)
418
428
 
419
429
 
420
430
  def creds_same_domain(creds):
421
- cfg = config.config().config
431
+ """
432
+ Check if the creds are for the same domain as the config.
433
+
434
+ Args:
435
+ creds (dict): The credentials dictionary.
436
+
437
+ Returns:
438
+ bool: Whether or not the creds are for the same domain as the config.
439
+ """
440
+ cfg = config.get()
422
441
  """Ensure the creds file refers to the domain in config"""
423
442
  token = creds.get("access_token")
424
443
  if not token:
@@ -435,6 +454,12 @@ def creds_same_domain(creds):
435
454
  def account_id_from_jwt(token):
436
455
  """
437
456
  Fetch the accounts id from a jwt token value.
457
+
458
+ Args:
459
+ token (str): The jwt token.
460
+
461
+ Returns:
462
+ str: The account id.
438
463
  """
439
464
  payload = jwt.decode(token, verify=False)
440
465
  return payload.get("account")
@@ -443,9 +468,15 @@ def account_id_from_jwt(token):
443
468
  def account_name_from_jwt(token):
444
469
  """
445
470
  Fetch the accounts name from a jwt token value.
471
+
472
+ Args:
473
+ token (str): The jwt token.
474
+
475
+ Returns:
476
+ str: The account name.
446
477
  """
447
- cfg = config.config().config
448
478
  account_id = account_id_from_jwt(token)
479
+ cfg = config.get()
449
480
  if account_id:
450
481
  url = "%s/api/v1/accounts/%s" % (cfg["api_url"], account_id)
451
482
  response = requests.get(url, headers={"authorization": "Bearer %s" % token})
@@ -458,6 +489,12 @@ def account_name_from_jwt(token):
458
489
  def request_instance_types(as_dict=False):
459
490
  """
460
491
  Get the list of available instances types.
492
+
493
+ Args:
494
+ as_dict (bool): Whether or not to return the instance types as a dictionary.
495
+
496
+ Returns:
497
+ list: The list of instance types.
461
498
  """
462
499
  api = ApiClient()
463
500
  response, response_code = api.make_request(
@@ -472,13 +509,21 @@ def request_instance_types(as_dict=False):
472
509
  logger.debug("Found available instance types: %s", instance_types)
473
510
 
474
511
  if as_dict:
475
- return dict([(instance["description"], instance) for instance in instance_types])
512
+ return dict(
513
+ [(instance["description"], instance) for instance in instance_types]
514
+ )
476
515
  return instance_types
477
516
 
478
517
 
479
518
  def request_projects(statuses=("active",)):
480
519
  """
481
- Query Conductor for all client Projects that are in the given state(s)
520
+ Query Conductor for all client Projects that are in the given status(es).
521
+
522
+ Args:
523
+ statuses (tuple): The statuses to filter for.
524
+
525
+ Returns:
526
+ list: The list of project names.
482
527
  """
483
528
  api = ApiClient()
484
529
 
@@ -507,6 +552,9 @@ def request_projects(statuses=("active",)):
507
552
  def request_software_packages():
508
553
  """
509
554
  Query Conductor for all software packages for the currently available sidecar.
555
+
556
+ Returns:
557
+ list: The list of software packages.
510
558
  """
511
559
  api = ApiClient()
512
560
 
@@ -519,4 +567,222 @@ def request_software_packages():
519
567
  msg = "Failed to get software packages for latest sidecar"
520
568
  msg += "\nError %s ...\n%s" % (response_code, response)
521
569
  raise Exception(msg)
522
- return json.loads(response).get("data", [])
570
+
571
+ software = json.loads(response).get("data", [])
572
+ software = [sw for sw in software if not ("3dsmax" in sw["product"] and sw["platform"] == "linux")]
573
+ return software
574
+
575
+ def request_extra_environment():
576
+ """
577
+ Query Conductor for extra environment.
578
+ """
579
+ api = ApiClient()
580
+
581
+ uri = "api/v1/integrations/env-vars-configs"
582
+ response, response_code = api.make_request(
583
+ uri_path=uri, verb="GET", raise_on_error=False, use_api_key=True
584
+ )
585
+
586
+ if response_code not in [200]:
587
+ msg = "Failed to get extra environment"
588
+ msg += "\nError %s ...\n%s" % (response_code, response)
589
+ raise Exception(msg)
590
+
591
+ all_accounts = json.loads(response).get("data", [])
592
+
593
+ token = read_conductor_credentials(True)
594
+ if not token:
595
+ raise Exception("Error: Could not get conductor credentials!")
596
+ account_id = str(account_id_from_jwt(token))
597
+
598
+ if not account_id:
599
+ raise Exception("Error: Could not get account id from jwt!")
600
+ account_env = next((account for account in all_accounts if account["account_id"] == account_id), None)
601
+ if not account_env:
602
+ raise Exception("Error: Could not get account environment!")
603
+ return account_env.get("env", [])
604
+
605
+
606
+
607
+
608
+ def get_jobs(first_jid, last_jid=None):
609
+ """
610
+ Query Conductor for all jobs between the given job ids.
611
+
612
+ Returns:
613
+ list: The list of jobs.
614
+
615
+ Raises:
616
+ Exception: If the request fails.
617
+
618
+ Examples:
619
+ >>> from ciocore import api_client
620
+ >>> jobs = api_client.get_jobs(1959)
621
+ >>> len(jobs)
622
+ 1
623
+ >>> jobs[0]["jid"]
624
+ '01959'
625
+ >>> jobs = api_client.get_jobs(1959, 1961)
626
+ >>> len(jobs)
627
+ 3
628
+ """
629
+ if last_jid is None:
630
+ last_jid = first_jid
631
+ low = str(int(first_jid) - 1).zfill(5)
632
+ high = str(int(last_jid) + 1).zfill(5)
633
+ api = ApiClient()
634
+ uri = "api/v1/jobs"
635
+
636
+ response, response_code = api.make_request(
637
+ uri_path=uri,
638
+ verb="GET",
639
+ raise_on_error=False,
640
+ use_api_key=True,
641
+ params={"filter": f"jid_gt_{low},jid_lt_{high}"},
642
+ )
643
+
644
+ if response_code not in [200]:
645
+ msg = f"Failed to get jobs {first_jid}-{last_jid}"
646
+ msg += "\nError %s ...\n%s" % (response_code, response)
647
+ raise Exception(msg)
648
+ jobs = json.loads(response).get("data")
649
+ return jobs
650
+
651
+
652
+ def get_log(job_id, task_id):
653
+ """
654
+ Get the log for the given job and task.
655
+
656
+ Args:
657
+ job_id (str): The job id.
658
+ task_id (str): The task id.
659
+
660
+ Returns:
661
+ list: A list of logs.
662
+
663
+ Raises:
664
+ Exception: If the request fails.
665
+
666
+ Examples:
667
+ >>> from ciocore import api_client
668
+ >>> logs = api_client.get_log(1959, 0)
669
+ {
670
+ "logs": [
671
+ {
672
+ "container_id": "j-5669544198668288-5619559933149184-5095331660038144-stde",
673
+ "instance_name": "renderer-5669544198668288-170062309438-62994",
674
+ "log": [
675
+ "Blender 2.93.0 (hash 84da05a8b806 built 2021-06-02 11:29:24)",
676
+ ...
677
+ ...
678
+ "Saved: '/var/folders/8r/46lmjdmj50x_0swd9klwptzm0000gq/T/blender_bmw/renders/render_0001.png'",
679
+ " Time: 00:29.22 (Saving: 00:00.32)",
680
+ "",
681
+ "",
682
+ "Blender quit"
683
+ ],
684
+ "timestamp": "1.700623521101516E9"
685
+ }
686
+ ],
687
+ "new_num_lines": [
688
+ 144
689
+ ],
690
+ "status_description": "",
691
+ "task_status": "success"
692
+ }
693
+ """
694
+ job_id = str(job_id).zfill(5)
695
+ task_id = str(task_id).zfill(3)
696
+
697
+ api = ApiClient()
698
+ uri = f"get_log_file?job={job_id}&task={task_id}&num_lines[]=0"
699
+
700
+ response, response_code = api.make_request(
701
+ uri_path=uri, verb="GET", raise_on_error=False, use_api_key=True
702
+ )
703
+
704
+ if response_code not in [200]:
705
+ msg = f"Failed to get log for job {job_id} task {task_id}"
706
+ msg += "\nError %s ...\n%s" % (response_code, response)
707
+ raise Exception(msg)
708
+
709
+ return response
710
+
711
+
712
+ def kill_jobs(*job_ids):
713
+ """
714
+ Kill the given jobs.
715
+
716
+ Args:
717
+ job_ids (list): The list of job ids.
718
+
719
+ Returns:
720
+ dict: The response.
721
+
722
+ Examples:
723
+ >>> from ciocore import api_client
724
+ >>> api_client.kill_jobs("03095","03094")
725
+ {'body': 'success', 'message': "Jobs [u'03095', u'03094'] have been kill."}
726
+
727
+ """
728
+ job_ids = [str(job_id).zfill(5) for job_id in job_ids]
729
+ api = ApiClient()
730
+ payload = {
731
+ "action": "kill",
732
+ "jobids": job_ids,
733
+ }
734
+ response, response_code = api.make_request(
735
+ uri_path="jobs_multi",
736
+ verb="PUT",
737
+ raise_on_error=False,
738
+ use_api_key=True,
739
+ data=json.dumps(payload)
740
+ )
741
+
742
+ if response_code not in [200]:
743
+ msg = f"Failed to kill jobs {job_ids}"
744
+ msg += "\nError %s ...\n%s" % (response_code, response)
745
+ raise Exception(msg)
746
+
747
+ return json.loads(response)
748
+
749
+
750
+ def kill_tasks(job_id, *task_ids):
751
+ """
752
+ Kill the given tasks.
753
+
754
+ Args:
755
+ job_id (str): The job id.
756
+ task_ids (list): The list of task ids.
757
+
758
+ Returns:
759
+ dict: The response.
760
+
761
+ Examples:
762
+ >>> from ciocore import api_client
763
+ >>> api_client.kill_tasks("03096", *range(50,56))
764
+ {'body': 'success', 'message': ' 6 Tasks set to "kill"\n\t050\n\t051\n\t052\n\t053\n\t054\n\t055'}
765
+ """
766
+
767
+ job_id = str(job_id).zfill(5)
768
+ task_ids = [str(task_id).zfill(3) for task_id in task_ids]
769
+ api = ApiClient()
770
+ payload = {
771
+ "action": "kill",
772
+ "jobid": job_id,
773
+ "taskids": task_ids,
774
+ }
775
+ response, response_code = api.make_request(
776
+ uri_path="tasks_multi",
777
+ verb="PUT",
778
+ raise_on_error=False,
779
+ use_api_key=True,
780
+ data=json.dumps(payload)
781
+ )
782
+
783
+ if response_code not in [200]:
784
+ msg = f"Failed to kill tasks {task_ids} of job {job_id}"
785
+ msg += "\nError %s ...\n%s" % (response_code, response)
786
+ raise Exception(msg)
787
+
788
+ return json.loads(response)