anatools 5.1.28__py3-none-any.whl → 6.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. anatools/__init__.py +1 -1
  2. anatools/anaclient/anaclient.py +16 -15
  3. anatools/anaclient/api/api.py +7 -2
  4. anatools/anaclient/api/datasets.py +21 -3
  5. anatools/anaclient/api/handlers.py +1 -1
  6. anatools/anaclient/channels.py +50 -25
  7. anatools/anaclient/datasets.py +94 -7
  8. anatools/anaclient/helpers.py +11 -10
  9. anatools/anaclient/services.py +46 -20
  10. anatools/anaclient/volumes.py +19 -18
  11. anatools/annotations/annotations.py +39 -18
  12. anatools/annotations/draw.py +34 -18
  13. {anatools-5.1.28.data → anatools-6.0.1.data}/scripts/anadeploy +4 -0
  14. anatools-6.0.1.data/scripts/renderedai +4166 -0
  15. {anatools-5.1.28.dist-info → anatools-6.0.1.dist-info}/METADATA +1 -1
  16. {anatools-5.1.28.dist-info → anatools-6.0.1.dist-info}/RECORD +27 -26
  17. {anatools-5.1.28.dist-info → anatools-6.0.1.dist-info}/WHEEL +1 -1
  18. {anatools-5.1.28.data → anatools-6.0.1.data}/scripts/ana +0 -0
  19. {anatools-5.1.28.data → anatools-6.0.1.data}/scripts/anamount +0 -0
  20. {anatools-5.1.28.data → anatools-6.0.1.data}/scripts/anaprofile +0 -0
  21. {anatools-5.1.28.data → anatools-6.0.1.data}/scripts/anarules +0 -0
  22. {anatools-5.1.28.data → anatools-6.0.1.data}/scripts/anaserver +0 -0
  23. {anatools-5.1.28.data → anatools-6.0.1.data}/scripts/anatransfer +0 -0
  24. {anatools-5.1.28.data → anatools-6.0.1.data}/scripts/anautils +0 -0
  25. {anatools-5.1.28.dist-info → anatools-6.0.1.dist-info}/entry_points.txt +0 -0
  26. {anatools-5.1.28.dist-info → anatools-6.0.1.dist-info}/licenses/LICENSE +0 -0
  27. {anatools-5.1.28.dist-info → anatools-6.0.1.dist-info}/top_level.txt +0 -0
anatools/__init__.py CHANGED
@@ -16,4 +16,4 @@ View the `Introduction to Rendered.ai Documentation`_ to learn more.
16
16
  from .anaclient.anaclient import client
17
17
  from .annotations.annotations import annotations
18
18
 
19
- __version__ = '5.1.28'
19
+ __version__ = '6.0.1'
@@ -93,7 +93,7 @@ class client:
93
93
  if self.__bearer_token:
94
94
  initial_headers = {'Authorization': f'Bearer {self.__bearer_token}'}
95
95
 
96
- self.ana_api = api(self.__url, self.__status_url, initial_headers, self.verbose)
96
+ self.ana_api = api(self.__url, self.__status_url, initial_headers, self.verbose, self.interactive)
97
97
 
98
98
  # initialize client variables
99
99
  self.__logout = False
@@ -188,15 +188,15 @@ class client:
188
188
  if self.interactive: print_color(f'Failed to login to {self.__environment} with email {self.__email}.', 'ff0000')
189
189
  self.user = None
190
190
  except requests.exceptions.ConnectionError as e:
191
- print_color(f'Could not connect to API to login. Try again or contact support@rendered.ai for assistance.', 'ff0000')
191
+ if self.interactive: print_color(f'Could not connect to API to login. Try again or contact support@rendered.ai for assistance.', 'ff0000')
192
192
  raise AuthFailedError()
193
193
  except requests.exceptions.JSONDecodeError as e:
194
- print_color(f'Failed to login with email {self.__email} and endpoint {self.__url}. Please confirm this is the correct email and endpoint, contact support@rendered.ai for assistance.', 'ff0000')
194
+ if self.interactive: print_color(f'Failed to login with email {self.__email} and endpoint {self.__url}. Please confirm this is the correct email and endpoint, contact support@rendered.ai for assistance.', 'ff0000')
195
195
  raise AuthFailedError()
196
- self.ana_api = api(self.__url, self.__status_url, {'uid':self.user['uid'], 'Authorization': f'Bearer {self.user["idtoken"]}'}, self.verbose)
196
+ self.ana_api = api(self.__url, self.__status_url, {'uid':self.user['uid'], 'Authorization': f'Bearer {self.user["idtoken"]}'}, self.verbose, self.interactive)
197
197
  try:
198
198
  response = self.ana_api.getSDKCompatibility()
199
- if response['version'] != anatools.__version__: print_color(response['message'], 'ff0000')
199
+ if self.interactive and response['version'] != anatools.__version__: print_color(response['message'], 'ff0000')
200
200
  except:
201
201
  if self.verbose: print_color("Failed to check SDK compatibility.", 'ff0000')
202
202
  # ask to create an api key if using email/password
@@ -301,20 +301,20 @@ class client:
301
301
  import requests
302
302
 
303
303
  try:
304
- self.ana_api = api(self.__url, self.__status_url, {'apikey': self.__APIKey}, self.verbose)
304
+ self.ana_api = api(self.__url, self.__status_url, {'apikey': self.__APIKey}, self.verbose, self.interactive)
305
305
  apikeydata = self.ana_api.getAPIKeyContext(apiKey=self.__APIKey)
306
306
  if not apikeydata:
307
- print_color("Invalid API Key", 'ff0000')
307
+ if self.interactive: print_color("Invalid API Key", 'ff0000')
308
308
  raise AuthFailedError()
309
309
  if apikeydata.get('expiresAt'):
310
310
  apikey_date = datetime.strptime(apikeydata['expiresAt'], "%Y-%m-%dT%H:%M:%S.%fZ")
311
311
  current_date = datetime.now()
312
312
  if apikey_date < current_date:
313
- print_color(f"API Key expired at {apikey_date}", 'ff0000')
313
+ if self.interactive: print_color(f"API Key expired at {apikey_date}", 'ff0000')
314
314
  raise AuthFailedError()
315
315
  try:
316
316
  response = self.ana_api.getSDKCompatibility()
317
- if response['version'] != anatools.__version__: print_color(response['message'], 'ff0000')
317
+ if self.interactive and response['version'] != anatools.__version__: print_color(response['message'], 'ff0000')
318
318
  except:
319
319
  if self.verbose: print_color("Failed to check SDK compatibility.", 'ff0000')
320
320
  except requests.exceptions.ConnectionError as e:
@@ -383,15 +383,15 @@ class client:
383
383
  if self.user:
384
384
  if int(time.time()) > int(self.user['expiresAt']):
385
385
  self.user = self.ana_api.login(self.__email, self.__password)
386
- self.ana_api = api(self.__url, self.__status_url, {'uid': self.user['uid'], 'Authorization': f'Bearer {self.user["idtoken"]}'}, self.verbose)
386
+ self.ana_api = api(self.__url, self.__status_url, {'uid': self.user['uid'], 'Authorization': f'Bearer {self.user["idtoken"]}'}, self.verbose, self.interactive)
387
387
  try:
388
388
  notification = self.ana_api.getSystemNotifications()
389
389
  self.__notificationId = notification['notificationId']
390
390
  if notification and notification['notificationId'] != self.__notificationId:
391
391
  self.__notificationId = notification['notificationId']
392
- print_color(notification['message'], 'ffff00')
392
+ if self.interactive: print_color(notification['message'], 'ffff00')
393
393
  except requests.exceptions.ConnectionError as e:
394
- print_color(f"Could not get notifications: {e}", 'ffff00')
394
+ if self.interactive: print_color(f"Could not get notifications: {e}", 'ffff00')
395
395
 
396
396
 
397
397
  def check_logout(self):
@@ -462,8 +462,9 @@ class client:
462
462
  f"code_challenge={code_challenge}&"
463
463
  f"code_challenge_method=S256")
464
464
 
465
- print_color("No other authentication methods found, attempting to login via your browser.", 'ffff00')
466
- print_color(f"If your browser does not open automatically, please open this URL in your browser: \n{auth_url}", "91e600")
465
+ if self.interactive:
466
+ print_color("No other authentication methods found, attempting to login via your browser.", 'ffff00')
467
+ print_color(f"If your browser does not open automatically, please open this URL in your browser: \n{auth_url}", "91e600")
467
468
  webbrowser.open(auth_url)
468
469
 
469
470
  auth_code = [None]
@@ -561,7 +562,7 @@ class client:
561
562
  """
562
563
  from anatools.lib.print import print_color
563
564
  services = self.ana_api.getSystemStatus(serviceId)
564
- if services and display:
565
+ if services and display and self.interactive:
565
566
  spacing = max([len(service['serviceName']) for service in services])+4
566
567
  print('Service Name'.ljust(spacing, ' ')+'Status')
567
568
  for service in services:
@@ -2,12 +2,13 @@
2
2
 
3
3
  class api:
4
4
 
5
- def __init__(self, url, status_url, headers, verbose=False):
5
+ def __init__(self, url, status_url, headers, verbose=False, interactive=True):
6
6
  import requests
7
7
  self.url = url
8
8
  self.status_url = status_url
9
9
  self.headers = headers or {}
10
10
  self.verbose = verbose
11
+ self.interactive = interactive
11
12
  self.types = None
12
13
  self.fields = {}
13
14
  self.session = requests.Session()
@@ -116,8 +117,12 @@ class api:
116
117
  responsedata = response.json()
117
118
  if self.verbose == 'debug': print(responsedata)
118
119
  try:
120
+ # Check for errors FIRST, even if data exists (partial response scenario in GraphQL)
121
+ # This ensures errors like "The Datasets limit has been reached by this workspace"
122
+ # are properly raised instead of being silently ignored when data is also present
123
+ if 'errors' in responsedata and responsedata['errors']:
124
+ raise Exception(responsedata['errors'][-1]['message'])
119
125
  if 'data' in responsedata and responsedata['data'] is not None and call in responsedata['data']: return responsedata['data'][call]
120
- elif 'errors' in responsedata: raise Exception(responsedata['errors'][-1]['message'])
121
126
  else: raise Exception()
122
127
  except Exception as e:
123
128
  raise Exception(f'There was an issue with the {call} API call: {e}')
@@ -118,21 +118,39 @@ def editDataset(self, workspaceId, datasetId, name=None, description=None, pause
118
118
 
119
119
  def downloadDataset(self, workspaceId, datasetId):
120
120
  response = self.session.post(
121
- url = self.url,
122
- headers = self.headers,
121
+ url = self.url,
122
+ headers = self.headers,
123
123
  json = {
124
124
  "operationName": "downloadDataset",
125
125
  "variables": {
126
126
  "workspaceId": workspaceId,
127
127
  "datasetId": datasetId
128
128
  },
129
- "query": """mutation
129
+ "query": """mutation
130
130
  downloadDataset($workspaceId: String!, $datasetId: String!) {
131
131
  downloadDataset(workspaceId: $workspaceId, datasetId: $datasetId)
132
132
  }"""})
133
133
  return self.errorhandler(response, "downloadDataset")
134
134
 
135
135
 
136
+ def downloadDatasetFile(self, workspaceId, datasetId, filepath):
137
+ response = self.session.post(
138
+ url = self.url,
139
+ headers = self.headers,
140
+ json = {
141
+ "operationName": "downloadDatasetFile",
142
+ "variables": {
143
+ "workspaceId": workspaceId,
144
+ "datasetId": datasetId,
145
+ "filepath": filepath
146
+ },
147
+ "query": """mutation
148
+ downloadDatasetFile($workspaceId: String!, $datasetId: String!, $filepath: String!) {
149
+ downloadDatasetFile(workspaceId: $workspaceId, datasetId: $datasetId, filepath: $filepath)
150
+ }"""})
151
+ return self.errorhandler(response, "downloadDatasetFile")
152
+
153
+
136
154
  def cancelDataset(self, workspaceId, datasetId):
137
155
  response = self.session.post(
138
156
  url = self.url,
@@ -21,7 +21,7 @@ def errorhandler(self, response, call):
21
21
  'token is invalid' in lower_error_message or \
22
22
  'token expired' in lower_error_message or \
23
23
  'authentication failed' in lower_error_message:
24
- print_color("Authentication failed. The provided token may be invalid, expired, or you do not have permission for this operation.", 'ff0000')
24
+ if self.interactive: print_color("Authentication failed. The provided token may be invalid, expired, or you do not have permission for this operation.", 'ff0000')
25
25
  # Optionally, raise a more specific AuthFailedError if it exists and is appropriate here
26
26
  raise Exception(error_message)
27
27
  else: raise Exception(f'There was an issue with the {call} API call.')
@@ -182,15 +182,40 @@ def build_channel(self, channelfile, ignore=['data/', 'output/'], verify=False):
182
182
  disk_usage = shutil.disk_usage('/')
183
183
  gb_free = disk_usage.free / (1024**3) # Convert bytes to GB
184
184
  if self.verbose == 'debug': print(f"Disk space left: {gb_free:.3f}GB")
185
- if gb_free < 20: print_color(f'\nWarning: Low disk space detected! Only {gb_free:.1f}GB available.', 'ffff00')
185
+ if gb_free < 20 and self.interactive: print_color(f'\nWarning: Low disk space detected! Only {gb_free:.1f}GB available.', 'ffff00')
186
186
  docker_space_cmd = "docker system df --format json"
187
187
  result = subprocess.run(docker_space_cmd, shell=True, capture_output=True, text=True)
188
188
  if result.returncode == 0:
189
- docker_stats = json.loads(result.stdout)
190
- total_docker_size = sum(item.get('Size', 0) for item in docker_stats if 'Size' in item)
191
- docker_gb = total_docker_size / (1024**3) # Convert bytes to GB
189
+ # Parse Docker output which may be NDJSON (newline-delimited) or concatenated JSON objects
190
+ # Use JSONDecoder.raw_decode() to handle both formats robustly
191
+ import re
192
+ def parse_docker_json(data):
193
+ results = []
194
+ decoder = json.JSONDecoder()
195
+ idx = 0
196
+ data = data.strip()
197
+ while idx < len(data):
198
+ obj, end_idx = decoder.raw_decode(data, idx)
199
+ results.append(obj)
200
+ idx = end_idx
201
+ # Skip any whitespace between objects
202
+ while idx < len(data) and data[idx].isspace():
203
+ idx += 1
204
+ return results
205
+ docker_stats = parse_docker_json(result.stdout)
206
+ # Parse human-readable size strings (e.g., "85.19GB", "2.087MB", "0B") to GB
207
+ def parse_size_to_gb(size_str):
208
+ if not size_str or size_str == '0B':
209
+ return 0.0
210
+ match = re.match(r'^([\d.]+)([KMGT]?B)$', size_str)
211
+ if not match:
212
+ return 0.0
213
+ value, unit = float(match.group(1)), match.group(2)
214
+ multipliers = {'B': 1/(1024**3), 'KB': 1/(1024**2), 'MB': 1/1024, 'GB': 1, 'TB': 1024}
215
+ return value * multipliers.get(unit, 0)
216
+ docker_gb = sum(parse_size_to_gb(item.get('Size', '0B')) for item in docker_stats)
192
217
  if self.verbose == 'debug': print(f"Docker space used: {docker_gb:.3f}GB")
193
- if docker_gb > 20:
218
+ if docker_gb > 20 and self.interactive:
194
219
  print_color(f'\nWarning: Docker is using {docker_gb:.1f}GB of disk space!', 'ffff00')
195
220
  print_color('Consider running "docker system prune -a" to free up space.', 'ffff00')
196
221
  print_color('This will remove:', 'ffff00')
@@ -199,11 +224,11 @@ def build_channel(self, channelfile, ignore=['data/', 'output/'], verify=False):
199
224
  print_color(' - All dangling images', 'ffff00')
200
225
  print_color(' - All dangling build cache', 'ffff00')
201
226
  except Exception as e:
202
- print(e)
227
+ if self.verbose == 'debug': print(f"Docker space check failed: {e}")
203
228
  pass
204
229
 
205
230
  # build dockerfile
206
- print('Building Channel Image...', end='', flush=True)
231
+ if self.interactive: print('Building Channel Image...', end='', flush=True)
207
232
  if not os.path.isfile(channelfile): raise Exception(f'No channel file {channelfile} found.')
208
233
  channeldir, channelfile = os.path.split(channelfile)
209
234
  if channeldir == "": channeldir = "./"
@@ -249,10 +274,10 @@ RUN sudo /home/$USERNAME/miniconda3/envs/anatools/bin/pip install -r /ana/requir
249
274
  if 'error' in output: print_color(f'{output["error"]}', 'ff0000')
250
275
  if 'stream' in output: logfile.write(output['stream'])
251
276
  if 'error' in output: logfile.write(output["error"])
252
- print(f'\rBuilding Channel Image... [{time.time()-start:.3f}s]', end='', flush=True)
277
+ if self.interactive: print(f'\rBuilding Channel Image... [{time.time()-start:.3f}s]', end='', flush=True)
253
278
  except StopIteration as e:
254
279
  time.sleep(5)
255
- print(f"\rBuilding Channel Image...done. [{time.time()-start:.3f}s]", flush=True)
280
+ if self.interactive: print(f"\rBuilding Channel Image...done. [{time.time()-start:.3f}s]", flush=True)
256
281
  try:
257
282
  dockerclient = docker.from_env()
258
283
  dockerclient.images.get(channelfile.split('.')[0])
@@ -267,26 +292,26 @@ RUN sudo /home/$USERNAME/miniconda3/envs/anatools/bin/pip install -r /ana/requir
267
292
  # if verify, check that image is valid by running anautils command
268
293
  if verify:
269
294
  try:
270
- print(f'Verifying Channel Image...', end='', flush=True)
295
+ if self.interactive: print(f'Verifying Channel Image...', end='', flush=True)
271
296
  start = time.time()
272
297
 
273
298
  # Run anautils command in the Docker container to verify schema
274
299
  container_name = f"verify_{channelfile.split('.')[0]}_{int(time.time())}"
275
300
  cmd = [
276
- "docker", "run", "--name", container_name,
277
- channelfile.split('.')[0],
301
+ "docker", "run", "--name", container_name,
302
+ channelfile.split('.')[0],
278
303
  "anautils", "--mode=schema", "--output=/tmp"
279
304
  ]
280
-
305
+
281
306
  result = subprocess.run(cmd, capture_output=True, text=True, check=False)
282
307
  subprocess.run(["docker", "rm", container_name], capture_output=True)
283
308
  if result.returncode != 0:
284
- print_color(f"\n\n{result.stderr}\n", "ff0000")
309
+ if self.interactive: print_color(f"\n\n{result.stderr}\n", "ff0000")
285
310
  raise Exception(f'Error encountered while verifying Docker image with the anautils command.')
286
-
287
- print(f"\rVerifying Channel Image...done. [{time.time()-start:.3f}s]")
311
+
312
+ if self.interactive: print(f"\rVerifying Channel Image...done. [{time.time()-start:.3f}s]")
288
313
  status = True
289
-
314
+
290
315
  except Exception as e:
291
316
  raise Exception(f'Error encountered while verifying Docker image. Please check that you can generate the schema using the anautils command.')
292
317
 
@@ -339,7 +364,7 @@ def deploy_channel(self, channelId=None, channelfile=None, image=None):
339
364
  except: raise Exception('Error connecting to Docker.')
340
365
 
341
366
  # get repository info
342
- print(f"Pushing Channel Image...", end='', flush=True)
367
+ if self.interactive: print(f"Pushing Channel Image...", end='', flush=True)
343
368
  dockerinfo = self.ana_api.deployChannel(channelId, image)
344
369
  deploymentId = dockerinfo['deploymentId']
345
370
  reponame = dockerinfo['ecrEndpoint']
@@ -363,11 +388,11 @@ def deploy_channel(self, channelId=None, channelfile=None, image=None):
363
388
  progressDetail = line['progressDetail']
364
389
  if progressDetail['total'] >= largest:
365
390
  largest = progressDetail['total']
366
- print(f"\rPushing Channel Image... [{time.time()-start:.3f}s, {min(100,round((progressDetail['current']/progressDetail['total']) * 100))}%]", end='', flush=True)
391
+ if self.interactive: print(f"\rPushing Channel Image... [{time.time()-start:.3f}s, {min(100,round((progressDetail['current']/progressDetail['total']) * 100))}%]", end='', flush=True)
367
392
  logfile.write(str(line) + '\n')
368
393
  logfile.close()
369
394
  if self.verbose != 'debug': os.remove(os.path.join(channeldir, 'dockerpush.log'))
370
- print(f"\rPushing Channel Image...done. [{time.time()-start:.3f}s] ", flush=True)
395
+ if self.interactive: print(f"\rPushing Channel Image...done. [{time.time()-start:.3f}s] ", flush=True)
371
396
 
372
397
  # cleanup docker and update channels
373
398
  dockerclient.images.remove(reponame)
@@ -396,13 +421,13 @@ def get_deployment_status(self, deploymentId, stream=False):
396
421
  if deploymentId is None: raise Exception('DeploymentId must be specified.')
397
422
  if stream:
398
423
  data = self.ana_api.getChannelDeployment(deploymentId=deploymentId)
399
- print(f"\r\tStep {data['status']['step']} - {data['status']['message']}", end='', flush=True)
424
+ if self.interactive: print(f"\r\tStep {data['status']['step']} - {data['status']['message']}", end='', flush=True)
400
425
  while (data['status']['state'] not in ['Channel Deployment Complete','Channel Deployment Failed']):
401
426
  time.sleep(10)
402
- print(f"\r\tStep {data['status']['step']} - {data['status']['message']}", end='', flush=True)
427
+ if self.interactive: print(f"\r\tStep {data['status']['step']} - {data['status']['message']}", end='', flush=True)
403
428
  if self.check_logout(): return
404
429
  data = self.ana_api.getChannelDeployment(deploymentId=deploymentId)
405
- print(f"\r\tStep {data['status']['step']} - {data['status']['message']}", flush=True)
430
+ if self.interactive: print(f"\r\tStep {data['status']['step']} - {data['status']['message']}", flush=True)
406
431
  return data
407
432
  else: return self.ana_api.getChannelDeployment(deploymentId=deploymentId)
408
433
 
@@ -457,8 +482,8 @@ def upload_channel_documentation(self, channelId, mdfile):
457
482
  "X-Amz-Signature": fileinfo['fields']['signature'],
458
483
  }
459
484
  response = requests.post(fileinfo['url'], data=data, files=files)
460
- if response.status_code != 204:
461
- print(response.status_code)
485
+ if response.status_code != 204:
486
+ if self.verbose: print(response.status_code)
462
487
  raise Exception('Failed to upload channel documentation file.')
463
488
  return True
464
489
 
@@ -170,29 +170,116 @@ def delete_dataset(self, datasetId, workspaceId=None):
170
170
 
171
171
  def download_dataset(self, datasetId, workspaceId=None, localDir=None):
172
172
  """Download a dataset.
173
-
173
+
174
174
  Parameters
175
175
  ----------
176
176
  datasetId : str
177
177
  Dataset ID of dataset to download.
178
178
  workspaceId : str
179
- Workspace ID that the dataset is in. If none is provided, the default workspace will get used.
179
+ Workspace ID that the dataset is in. If none is provided, the default workspace will get used.
180
180
  localDir : str
181
181
  Path for where to download the dataset. If none is provided, current working directory will be used.
182
-
182
+
183
183
  Returns
184
184
  -------
185
185
  str
186
186
  Returns the path the dataset was downloaded to.
187
+
188
+ Raises
189
+ ------
190
+ ValueError
191
+ If datasetId is not provided.
192
+ Exception
193
+ If the dataset cannot be downloaded (e.g., still running, failed, or not found).
187
194
  """
188
195
  from anatools.lib.download import download_file
189
196
 
190
197
  self.check_logout()
191
198
  if datasetId is None: raise ValueError("The datasetId parameter is required!")
192
- if workspaceId is None: workspaceId = self.workspace
193
- url = self.ana_api.downloadDataset(workspaceId=workspaceId, datasetId=datasetId)
194
- fname = self.ana_api.getDatasets(workspaceId=workspaceId, datasetId=datasetId)[0]['name'] + '.zip'
195
- return download_file(url=url, fname=fname, localDir=localDir)
199
+ if workspaceId is None: workspaceId = self.workspace
200
+
201
+ # Get dataset info to provide better error messages
202
+ datasets = self.ana_api.getDatasets(workspaceId=workspaceId, datasetId=datasetId)
203
+ if not datasets:
204
+ raise Exception(f"Dataset with ID '{datasetId}' not found in workspace.")
205
+
206
+ dataset = datasets[0]
207
+ dataset_name = dataset.get('name', datasetId)
208
+ dataset_status = dataset.get('status', 'unknown')
209
+
210
+ url = self.ana_api.downloadDataset(workspaceId=workspaceId, datasetId=datasetId)
211
+
212
+ if url is None:
213
+ # Provide a helpful error message based on the dataset status
214
+ if dataset_status.lower() in ['running', 'pending', 'queued']:
215
+ raise Exception(f"Dataset '{dataset_name}' is still {dataset_status}. Please wait for the dataset to complete before downloading.")
216
+ elif dataset_status.lower() in ['failed', 'error', 'cancelled']:
217
+ raise Exception(f"Dataset '{dataset_name}' has status '{dataset_status}' and cannot be downloaded.")
218
+ else:
219
+ raise Exception(f"Dataset '{dataset_name}' is not available for download (status: {dataset_status}). The dataset may still be processing or may have failed.")
220
+
221
+ fname = dataset_name + '.zip'
222
+ return download_file(url=url, fname=fname, localDir=localDir)
223
+
224
+
225
+ def download_dataset_file(self, datasetId, filepath, workspaceId=None, localDir=None):
226
+ """Download a single file from a dataset.
227
+
228
+ This allows downloading individual files from a dataset rather than
229
+ downloading the entire dataset archive. Use get_dataset_files() to
230
+ list available files in a dataset.
231
+
232
+ Parameters
233
+ ----------
234
+ datasetId : str
235
+ Dataset ID of the dataset containing the file.
236
+ filepath : str
237
+ Relative path to the file within the dataset (e.g., "images/000000-1-image.png").
238
+ workspaceId : str
239
+ Workspace ID that the dataset is in. If none is provided, the default workspace will get used.
240
+ localDir : str
241
+ Path for where to download the file. If none is provided, current working directory will be used.
242
+
243
+ Returns
244
+ -------
245
+ str
246
+ Returns the path the file was downloaded to.
247
+
248
+ Raises
249
+ ------
250
+ ValueError
251
+ If datasetId or filepath is not provided.
252
+ Exception
253
+ If the file cannot be downloaded (e.g., not found).
254
+
255
+ Examples
256
+ --------
257
+ >>> # First, list available files
258
+ >>> files = ana.get_dataset_files(datasetId='abc123', path='images')
259
+ >>> print(files)
260
+ ['000000-1-image.png', '000001-1-image.png', ...]
261
+ >>>
262
+ >>> # Then download a specific file
263
+ >>> path = ana.download_dataset_file(datasetId='abc123', filepath='images/000000-1-image.png')
264
+ >>> print(path)
265
+ '/home/user/000000-1-image.png'
266
+ """
267
+ import os
268
+ from anatools.lib.download import download_file
269
+
270
+ self.check_logout()
271
+ if datasetId is None: raise ValueError("The datasetId parameter is required!")
272
+ if filepath is None or filepath.strip() == '': raise ValueError("The filepath parameter is required!")
273
+ if workspaceId is None: workspaceId = self.workspace
274
+
275
+ url = self.ana_api.downloadDatasetFile(workspaceId=workspaceId, datasetId=datasetId, filepath=filepath)
276
+
277
+ if url is None:
278
+ raise Exception(f"File '{filepath}' not found in dataset '{datasetId}'.")
279
+
280
+ # Extract the filename from the filepath
281
+ fname = os.path.basename(filepath)
282
+ return download_file(url=url, fname=fname, localDir=localDir)
196
283
 
197
284
 
198
285
  def cancel_dataset(self, datasetId, workspaceId=None):
@@ -16,10 +16,10 @@ def generate_etag(file_path, chunk_size=128 * hashlib.md5().block_size):
16
16
  raise IOError(f"Error reading file {file_path}: {e}")
17
17
 
18
18
 
19
- def upload_part(i, file_path, url, part_size, total_parts, progress_tracker, prefix_message, max_retries=3):
19
+ def upload_part(i, file_path, url, part_size, total_parts, progress_tracker, prefix_message, max_retries=3, interactive=True):
20
20
  """Uploads a single part, creating a new session for each request, updating progress with retry and delay."""
21
21
  percent_complete = (progress_tracker["completed_parts"] / total_parts) * 100
22
- print(f"\x1b[1K\r{prefix_message}: {percent_complete:.2f}% complete", end="", flush=True)
22
+ if interactive: print(f"\x1b[1K\r{prefix_message}: {percent_complete:.2f}% complete", end="", flush=True)
23
23
  retries = 0
24
24
 
25
25
  while retries <= max_retries:
@@ -36,21 +36,22 @@ def upload_part(i, file_path, url, part_size, total_parts, progress_tracker, pre
36
36
  e_tag = response.headers["ETag"]
37
37
  progress_tracker["completed_parts"] += 1
38
38
  percent_complete = (progress_tracker["completed_parts"] / total_parts) * 100
39
- print(f"\x1b[1K\r{prefix_message}: {percent_complete:.2f}% complete", end="", flush=True)
39
+ if interactive: print(f"\x1b[1K\r{prefix_message}: {percent_complete:.2f}% complete", end="", flush=True)
40
40
  return {"partNumber": i + 1, "eTag": e_tag}
41
41
 
42
42
  except requests.RequestException as e:
43
43
  if retries == max_retries:
44
44
  raise Exception(f"Failed to upload part {i+1} after {retries} attempts. Error: {e}")
45
45
 
46
- print(f"\n\033[91m{prefix_message}: Attempt {retries + 1} failed with error", flush=True)
47
- print(f"{e}. ", flush=True)
48
- print("Retrying in 30 seconds...\033[0m", flush=True)
46
+ if interactive:
47
+ print(f"\n\033[91m{prefix_message}: Attempt {retries + 1} failed with error", flush=True)
48
+ print(f"{e}. ", flush=True)
49
+ print("Retrying in 30 seconds...\033[0m", flush=True)
49
50
  time.sleep(30) # Wait 30 seconds before retrying
50
51
  retries += 1
51
52
 
52
53
 
53
- def multipart_upload_file(file_path, part_size, urls, prefix_message):
54
+ def multipart_upload_file(file_path, part_size, urls, prefix_message, interactive=True):
54
55
  if len(urls) > 0:
55
56
  parts = []
56
57
  total_parts = len(urls)
@@ -61,14 +62,14 @@ def multipart_upload_file(file_path, part_size, urls, prefix_message):
61
62
  for i, url in enumerate(urls):
62
63
  futures.append(
63
64
  executor.submit(
64
- upload_part, i, file_path, url, part_size, total_parts, progress_tracker, prefix_message
65
+ upload_part, i, file_path, url, part_size, total_parts, progress_tracker, prefix_message, interactive=interactive
65
66
  )
66
67
  )
67
68
 
68
69
  for future in as_completed(futures):
69
70
  parts.append(future.result())
70
71
 
71
- print(f"\x1b[1K\r{prefix_message}: upload complete.", end="", flush=True)
72
+ if interactive: print(f"\x1b[1K\r{prefix_message}: upload complete.", end="", flush=True)
72
73
  return parts
73
74
  else:
74
75
  with open(file_path, "rb") as filebytes:
@@ -77,5 +78,5 @@ def multipart_upload_file(file_path, part_size, urls, prefix_message):
77
78
  response = requests.put(url, files=files)
78
79
  if response.status_code != 204:
79
80
  raise Exception(f"Failed to upload, with status code: {response.status_code}")
80
- print(f"\x1b[1K\r{prefix_message}: upload complete.", end="", flush=True)
81
+ if interactive: print(f"\x1b[1K\r{prefix_message}: upload complete.", end="", flush=True)
81
82
  return []