NGPIris 5.1.1.dev3__tar.gz → 5.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris/cli/__init__.py +49 -6
  2. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris/hcp/hcp.py +32 -9
  3. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris.egg-info/PKG-INFO +2 -2
  4. ngpiris-5.2.0/NGPIris.egg-info/SOURCES.txt +52 -0
  5. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/PKG-INFO +2 -2
  6. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/pyproject.toml +5 -1
  7. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/tests/test_hci.py +5 -1
  8. ngpiris-5.2.0/tests/test_hcp.py +213 -0
  9. ngpiris-5.1.1.dev3/NGPIris.egg-info/SOURCES.txt +0 -127
  10. ngpiris-5.1.1.dev3/docs/_build/.DS_Store +0 -0
  11. ngpiris-5.1.1.dev3/docs/_build/doctrees/IRIS_5_changelog.doctree +0 -0
  12. ngpiris-5.1.1.dev3/docs/_build/doctrees/NGPIris.cli.doctree +0 -0
  13. ngpiris-5.1.1.dev3/docs/_build/doctrees/NGPIris.doctree +0 -0
  14. ngpiris-5.1.1.dev3/docs/_build/doctrees/NGPIris.hci.doctree +0 -0
  15. ngpiris-5.1.1.dev3/docs/_build/doctrees/NGPIris.hcp.doctree +0 -0
  16. ngpiris-5.1.1.dev3/docs/_build/doctrees/NGPIris.parse_credentials.doctree +0 -0
  17. ngpiris-5.1.1.dev3/docs/_build/doctrees/NGPIris.utils.doctree +0 -0
  18. ngpiris-5.1.1.dev3/docs/_build/doctrees/README.doctree +0 -0
  19. ngpiris-5.1.1.dev3/docs/_build/doctrees/Tutorial.doctree +0 -0
  20. ngpiris-5.1.1.dev3/docs/_build/doctrees/environment.pickle +0 -0
  21. ngpiris-5.1.1.dev3/docs/_build/doctrees/index.doctree +0 -0
  22. ngpiris-5.1.1.dev3/docs/_build/doctrees/modules.doctree +0 -0
  23. ngpiris-5.1.1.dev3/docs/_build/html/.buildinfo +0 -4
  24. ngpiris-5.1.1.dev3/docs/_build/html/IRIS_5_changelog.html +0 -462
  25. ngpiris-5.1.1.dev3/docs/_build/html/NGPIris.cli.html +0 -137
  26. ngpiris-5.1.1.dev3/docs/_build/html/NGPIris.hci.html +0 -280
  27. ngpiris-5.1.1.dev3/docs/_build/html/NGPIris.hcp.html +0 -357
  28. ngpiris-5.1.1.dev3/docs/_build/html/NGPIris.html +0 -236
  29. ngpiris-5.1.1.dev3/docs/_build/html/NGPIris.parse_credentials.html +0 -156
  30. ngpiris-5.1.1.dev3/docs/_build/html/NGPIris.utils.html +0 -143
  31. ngpiris-5.1.1.dev3/docs/_build/html/README.html +0 -681
  32. ngpiris-5.1.1.dev3/docs/_build/html/Tutorial.html +0 -796
  33. ngpiris-5.1.1.dev3/docs/_build/html/_downloads/ee00c4f6f22dc2671c766f728f00006f/credentials_template.json +0 -14
  34. ngpiris-5.1.1.dev3/docs/_build/html/_modules/NGPIris/cli.html +0 -338
  35. ngpiris-5.1.1.dev3/docs/_build/html/_modules/NGPIris/hci/exceptions.html +0 -106
  36. ngpiris-5.1.1.dev3/docs/_build/html/_modules/NGPIris/hci/hci.html +0 -250
  37. ngpiris-5.1.1.dev3/docs/_build/html/_modules/NGPIris/hci/helpers.html +0 -204
  38. ngpiris-5.1.1.dev3/docs/_build/html/_modules/NGPIris/hcp/exceptions.html +0 -117
  39. ngpiris-5.1.1.dev3/docs/_build/html/_modules/NGPIris/hcp/hcp.html +0 -655
  40. ngpiris-5.1.1.dev3/docs/_build/html/_modules/NGPIris/hcp/helpers.html +0 -148
  41. ngpiris-5.1.1.dev3/docs/_build/html/_modules/NGPIris/hcp/statistics.html +0 -137
  42. ngpiris-5.1.1.dev3/docs/_build/html/_modules/NGPIris/parse_credentials/parse_credentials.html +0 -170
  43. ngpiris-5.1.1.dev3/docs/_build/html/_modules/NGPIris/utils/utils.html +0 -115
  44. ngpiris-5.1.1.dev3/docs/_build/html/_modules/index.html +0 -108
  45. ngpiris-5.1.1.dev3/docs/_build/html/_sources/IRIS_5_changelog.md.txt +0 -43
  46. ngpiris-5.1.1.dev3/docs/_build/html/_sources/NGPIris.cli.rst.txt +0 -10
  47. ngpiris-5.1.1.dev3/docs/_build/html/_sources/NGPIris.hci.rst.txt +0 -37
  48. ngpiris-5.1.1.dev3/docs/_build/html/_sources/NGPIris.hcp.rst.txt +0 -45
  49. ngpiris-5.1.1.dev3/docs/_build/html/_sources/NGPIris.parse_credentials.rst.txt +0 -21
  50. ngpiris-5.1.1.dev3/docs/_build/html/_sources/NGPIris.rst.txt +0 -22
  51. ngpiris-5.1.1.dev3/docs/_build/html/_sources/NGPIris.utils.rst.txt +0 -21
  52. ngpiris-5.1.1.dev3/docs/_build/html/_sources/README.md.txt +0 -274
  53. ngpiris-5.1.1.dev3/docs/_build/html/_sources/Tutorial.md.txt +0 -326
  54. ngpiris-5.1.1.dev3/docs/_build/html/_sources/index.rst.txt +0 -21
  55. ngpiris-5.1.1.dev3/docs/_build/html/_sources/modules.rst.txt +0 -7
  56. ngpiris-5.1.1.dev3/docs/_build/html/_static/alabaster.css +0 -708
  57. ngpiris-5.1.1.dev3/docs/_build/html/_static/basic.css +0 -925
  58. ngpiris-5.1.1.dev3/docs/_build/html/_static/custom.css +0 -1
  59. ngpiris-5.1.1.dev3/docs/_build/html/_static/debug.css +0 -69
  60. ngpiris-5.1.1.dev3/docs/_build/html/_static/doctools.js +0 -156
  61. ngpiris-5.1.1.dev3/docs/_build/html/_static/documentation_options.js +0 -13
  62. ngpiris-5.1.1.dev3/docs/_build/html/_static/file.png +0 -0
  63. ngpiris-5.1.1.dev3/docs/_build/html/_static/language_data.js +0 -199
  64. ngpiris-5.1.1.dev3/docs/_build/html/_static/minus.png +0 -0
  65. ngpiris-5.1.1.dev3/docs/_build/html/_static/plus.png +0 -0
  66. ngpiris-5.1.1.dev3/docs/_build/html/_static/pygments.css +0 -83
  67. ngpiris-5.1.1.dev3/docs/_build/html/_static/scripts/furo-extensions.js +0 -0
  68. ngpiris-5.1.1.dev3/docs/_build/html/_static/scripts/furo.js +0 -3
  69. ngpiris-5.1.1.dev3/docs/_build/html/_static/scripts/furo.js.LICENSE.txt +0 -7
  70. ngpiris-5.1.1.dev3/docs/_build/html/_static/scripts/furo.js.map +0 -1
  71. ngpiris-5.1.1.dev3/docs/_build/html/_static/searchtools.js +0 -619
  72. ngpiris-5.1.1.dev3/docs/_build/html/_static/skeleton.css +0 -296
  73. ngpiris-5.1.1.dev3/docs/_build/html/_static/sphinx_highlight.js +0 -154
  74. ngpiris-5.1.1.dev3/docs/_build/html/_static/styles/furo-extensions.css +0 -2
  75. ngpiris-5.1.1.dev3/docs/_build/html/_static/styles/furo-extensions.css.map +0 -1
  76. ngpiris-5.1.1.dev3/docs/_build/html/_static/styles/furo.css +0 -2
  77. ngpiris-5.1.1.dev3/docs/_build/html/_static/styles/furo.css.map +0 -1
  78. ngpiris-5.1.1.dev3/docs/_build/html/genindex.html +0 -476
  79. ngpiris-5.1.1.dev3/docs/_build/html/index.html +0 -159
  80. ngpiris-5.1.1.dev3/docs/_build/html/modules.html +0 -148
  81. ngpiris-5.1.1.dev3/docs/_build/html/objects.inv +0 -0
  82. ngpiris-5.1.1.dev3/docs/_build/html/py-modindex.html +0 -188
  83. ngpiris-5.1.1.dev3/docs/_build/html/search.html +0 -120
  84. ngpiris-5.1.1.dev3/docs/_build/html/searchindex.js +0 -1
  85. ngpiris-5.1.1.dev3/tests/test_hcp.py +0 -216
  86. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/LICENSE +0 -0
  87. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/MANIFEST.in +0 -0
  88. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris/__init__.py +0 -0
  89. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris/hci/__init__.py +0 -0
  90. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris/hci/exceptions.py +0 -0
  91. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris/hci/hci.py +0 -0
  92. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris/hci/helpers.py +0 -0
  93. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris/hcp/__init__.py +0 -0
  94. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris/hcp/exceptions.py +0 -0
  95. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris/hcp/helpers.py +0 -0
  96. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris/hcp/statistics.py +0 -0
  97. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris/parse_credentials/__init__.py +0 -0
  98. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris/parse_credentials/parse_credentials.py +0 -0
  99. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris/utils/__init__.py +0 -0
  100. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris/utils/utils.py +0 -0
  101. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris.egg-info/dependency_links.txt +0 -0
  102. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris.egg-info/entry_points.txt +0 -0
  103. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris.egg-info/requires.txt +0 -0
  104. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/NGPIris.egg-info/top_level.txt +0 -0
  105. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/README.md +0 -0
  106. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/docs/IRIS_5_changelog.md +0 -0
  107. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/docs/Makefile +0 -0
  108. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/docs/NGPIris.cli.rst +0 -0
  109. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/docs/NGPIris.hci.rst +0 -0
  110. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/docs/NGPIris.hcp.rst +0 -0
  111. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/docs/NGPIris.parse_credentials.rst +0 -0
  112. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/docs/NGPIris.rst +0 -0
  113. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/docs/NGPIris.utils.rst +0 -0
  114. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/docs/Tutorial.md +0 -0
  115. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/docs/conf.py +0 -0
  116. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/docs/index.rst +0 -0
  117. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/docs/make.bat +0 -0
  118. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/docs/modules.rst +0 -0
  119. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/setup.cfg +0 -0
  120. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/tests/data/80MB_test_file +0 -0
  121. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/tests/data/a folder of data/80MB_test_file_0 +0 -0
  122. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/tests/data/a folder of data/80MB_test_file_1 +0 -0
  123. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/tests/data/complete_creds_0.json +0 -0
  124. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/tests/data/complete_creds_1.json +0 -0
  125. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/tests/data/complete_creds_2.json +0 -0
  126. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/tests/data/incomplete_creds_0.json +0 -0
  127. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/tests/data/incomplete_creds_1.json +0 -0
  128. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/tests/data/incomplete_creds_2.json +0 -0
  129. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/tests/data/incomplete_creds_3.json +0 -0
  130. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/tests/data/incomplete_creds_4.json +0 -0
  131. {ngpiris-5.1.1.dev3 → ngpiris-5.2.0}/tests/test_parse_credentials.py +0 -0
@@ -19,14 +19,14 @@ def format_list(list_of_things : list) -> str:
19
19
  list_of_buckets = list(map(lambda s : s + "\n", list_of_things))
20
20
  return "".join(list_of_buckets).strip("\n")
21
21
 
22
- def _list_objects_generator(hcph : HCPHandler, name_only : bool) -> Generator[str, Any, None]:
22
+ def _list_objects_generator(hcph : HCPHandler, path : str, name_only : bool, files_only : bool) -> Generator[str, Any, None]:
23
23
  """
24
24
  Handle object list as a paginator that `click` can handle. It works slightly
25
25
  different from `list_objects` in `hcp.py` in order to make the output
26
26
  printable in a terminal
27
27
  """
28
28
  paginator : Paginator = hcph.s3_client.get_paginator("list_objects_v2")
29
- pages : PageIterator = paginator.paginate(Bucket = hcph.bucket_name)
29
+ pages : PageIterator = paginator.paginate(Bucket = hcph.bucket_name, Prefix = path)
30
30
  (nb_of_cols, _) = get_terminal_size()
31
31
  max_width = floor(nb_of_cols / 5)
32
32
  if (not name_only):
@@ -36,7 +36,11 @@ def _list_objects_generator(hcph : HCPHandler, name_only : bool) -> Generator[st
36
36
  tablefmt = "plain",
37
37
  stralign = "center"
38
38
  ) + "\n" + "-"*nb_of_cols + "\n"
39
- for object in pages.search("Contents[?!ends_with(Key, '/')][]"): # filter objects that does not end with "/"
39
+ if files_only:
40
+ filter_string = "Contents[?!ends_with(Key, '/')][]" # filter objects that does not end with "/"
41
+ else:
42
+ filter_string = "Contents[*][]"
43
+ for object in pages.search(filter_string):
40
44
  if name_only:
41
45
  yield str(object["Key"]) + "\n"
42
46
  else:
@@ -55,6 +59,11 @@ def _list_objects_generator(hcph : HCPHandler, name_only : bool) -> Generator[st
55
59
  def object_is_folder(object_path : str, hcph : HCPHandler) -> bool:
56
60
  return (object_path[-1] == "/") and (hcph.get_object(object_path)["ContentLength"] == 0)
57
61
 
62
+ def add_trailing_slash(path : str) -> str:
63
+ if not path[-1] == "/":
64
+ path += "/"
65
+ return path
66
+
58
67
  @click.group()
59
68
  @click.argument("credentials")
60
69
  @click.version_option(package_name = "NGPIris")
@@ -85,9 +94,13 @@ def upload(context : Context, bucket : str, source : str, destination : str):
85
94
  """
86
95
  hcph : HCPHandler = get_HCPHandler(context)
87
96
  hcph.mount_bucket(bucket)
97
+ destination = add_trailing_slash(destination)
88
98
  if Path(source).is_dir():
99
+ source = add_trailing_slash(source)
89
100
  hcph.upload_folder(source, destination)
90
101
  else:
102
+ file_name = Path(source).name
103
+ destination += file_name
91
104
  hcph.upload_file(source, destination)
92
105
 
93
106
  @cli.command()
@@ -198,22 +211,52 @@ def list_buckets(context : Context):
198
211
 
199
212
  @cli.command()
200
213
  @click.argument("bucket")
214
+ @click.argument("path", required = False)
201
215
  @click.option(
202
216
  "-no",
203
217
  "--name-only",
204
218
  help = "Output only the name of the objects instead of all the associated metadata",
205
- default = False
219
+ default = False,
220
+ is_flag = True
221
+ )
222
+ @click.option(
223
+ "-p",
224
+ "--pagination",
225
+ help = "Output as a paginator",
226
+ default = False,
227
+ is_flag = True
228
+ )
229
+ @click.option(
230
+ "-fo",
231
+ "--files-only",
232
+ help = "Output only file objects",
233
+ default = False,
234
+ is_flag = True
206
235
  )
207
236
  @click.pass_context
208
- def list_objects(context : Context, bucket : str, name_only : bool):
237
+ def list_objects(context : Context, bucket : str, path : str, name_only : bool, pagination : bool, files_only : bool):
209
238
  """
210
239
  List the objects in a certain bucket/namespace on the HCP.
211
240
 
212
241
  BUCKET is the name of the bucket in which to list its objects.
242
+
243
+ PATH is an optional argument for where to list the objects
213
244
  """
214
245
  hcph : HCPHandler = get_HCPHandler(context)
215
246
  hcph.mount_bucket(bucket)
216
- click.echo_via_pager(_list_objects_generator(hcph, name_only))
247
+ if path:
248
+ path_with_slash = add_trailing_slash(path)
249
+
250
+ if not hcph.object_exists(path_with_slash):
251
+ raise RuntimeError("Path does not exist")
252
+ else:
253
+ path_with_slash = ""
254
+
255
+ if pagination:
256
+ click.echo_via_pager(_list_objects_generator(hcph, path_with_slash, name_only, files_only))
257
+ else:
258
+ for obj in hcph.list_objects(path_with_slash, name_only, files_only):
259
+ click.echo(obj)
217
260
 
218
261
  @cli.command()
219
262
  @click.argument("bucket")
@@ -178,6 +178,8 @@ class HCPHandler:
178
178
  pass
179
179
  else:
180
180
  raise RuntimeError("No bucket selected. Either use `mount_bucket` first or supply the optional `bucket_name` parameter for `test_connection`")
181
+
182
+ response = {}
181
183
  try:
182
184
  response = dict(self.s3_client.head_bucket(Bucket = bucket_name))
183
185
  except EndpointConnectionError as e: # pragma: no cover
@@ -234,7 +236,7 @@ class HCPHandler:
234
236
  return list_of_buckets
235
237
 
236
238
  @check_mounted
237
- def list_objects(self, path_key : str = "", name_only : bool = False) -> Generator:
239
+ def list_objects(self, path_key : str = "", name_only : bool = False, files_only : bool = False) -> Generator:
238
240
  """
239
241
  List all objects in the mounted bucket as a generator. If one wishes to
240
242
  get the result as a list, use :py:function:`list` to type cast the generator
@@ -243,16 +245,35 @@ class HCPHandler:
243
245
  :type path_key: str, optional
244
246
  :param name_only: If True, yield only a the object names. If False, yield the full metadata about each object. Defaults to False.
245
247
  :type name_only: bool, optional
248
+ :param files_only: If true, only yield file objects. Defaults to False
249
+ :type files_only: bool, optional
246
250
  :yield: A generator of all objects in a bucket
247
251
  :rtype: Generator
248
252
  """
249
253
  paginator : Paginator = self.s3_client.get_paginator("list_objects_v2")
250
- pages : PageIterator = paginator.paginate(Bucket = self.bucket_name)
251
- for object in pages.search("Contents[?starts_with(Key, '" + path_key + "')][]"):
252
- if name_only:
253
- yield str(object["Key"])
254
- else:
255
- yield object
254
+ pages : PageIterator = paginator.paginate(Bucket = self.bucket_name, Prefix = path_key)
255
+
256
+ if files_only:
257
+ filter_string = "Contents[?!ends_with(Key, '/')][]"
258
+ else:
259
+ filter_string = "Contents[*][]"
260
+
261
+ split_path_key = len(path_key.split("/")) + 1
262
+
263
+ pages_filtered = pages.search(filter_string)
264
+ for object in pages_filtered:
265
+ # Split the object key by "/"
266
+ split_object = object["Key"].split("/")
267
+ # Check if the object is within the specified path_key depth
268
+ if len(split_object) <= split_path_key:
269
+ # Skip objects that are not at the desired depth
270
+ if (len(split_object) == split_path_key) and split_object[-1]:
271
+ continue
272
+
273
+ if name_only:
274
+ yield str(object["Key"])
275
+ else:
276
+ yield object
256
277
 
257
278
  @check_mounted
258
279
  def get_object(self, key : str) -> dict:
@@ -382,6 +403,9 @@ class HCPHandler:
382
403
  file_name = Path(local_file_path).name
383
404
  key = file_name
384
405
 
406
+ if "\\" in local_file_path:
407
+ raise RuntimeError("The \"\\\" character is not allowed in the file path")
408
+
385
409
  if self.object_exists(key):
386
410
  raise ObjectAlreadyExist("The object \"" + key + "\" already exist in the mounted bucket")
387
411
  else:
@@ -510,7 +534,6 @@ class HCPHandler:
510
534
  """
511
535
  return self.fuzzy_search_in_bucket(search_string, name_only, case_sensitive, 100)
512
536
 
513
-
514
537
  @check_mounted
515
538
  def fuzzy_search_in_bucket(
516
539
  self,
@@ -556,7 +579,7 @@ class HCPHandler:
556
579
  if name_only:
557
580
  yield item
558
581
  else:
559
- yield full_list[index]
582
+ yield full_list[index] # type: ignore
560
583
 
561
584
  @check_mounted
562
585
  def get_object_acl(self, key : str) -> dict:
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: NGPIris
3
- Version: 5.1.1.dev3
3
+ Version: 5.2.0
4
4
  Author-email: Erik Brink <erik.brink@gu.se>, Isak Sylvin <isak.sylvin@gu.se>
5
5
  Description-Content-Type: text/markdown
6
6
  License-File: LICENSE
@@ -0,0 +1,52 @@
1
+ LICENSE
2
+ MANIFEST.in
3
+ README.md
4
+ pyproject.toml
5
+ NGPIris/__init__.py
6
+ NGPIris.egg-info/PKG-INFO
7
+ NGPIris.egg-info/SOURCES.txt
8
+ NGPIris.egg-info/dependency_links.txt
9
+ NGPIris.egg-info/entry_points.txt
10
+ NGPIris.egg-info/requires.txt
11
+ NGPIris.egg-info/top_level.txt
12
+ NGPIris/cli/__init__.py
13
+ NGPIris/hci/__init__.py
14
+ NGPIris/hci/exceptions.py
15
+ NGPIris/hci/hci.py
16
+ NGPIris/hci/helpers.py
17
+ NGPIris/hcp/__init__.py
18
+ NGPIris/hcp/exceptions.py
19
+ NGPIris/hcp/hcp.py
20
+ NGPIris/hcp/helpers.py
21
+ NGPIris/hcp/statistics.py
22
+ NGPIris/parse_credentials/__init__.py
23
+ NGPIris/parse_credentials/parse_credentials.py
24
+ NGPIris/utils/__init__.py
25
+ NGPIris/utils/utils.py
26
+ docs/IRIS_5_changelog.md
27
+ docs/Makefile
28
+ docs/NGPIris.cli.rst
29
+ docs/NGPIris.hci.rst
30
+ docs/NGPIris.hcp.rst
31
+ docs/NGPIris.parse_credentials.rst
32
+ docs/NGPIris.rst
33
+ docs/NGPIris.utils.rst
34
+ docs/Tutorial.md
35
+ docs/conf.py
36
+ docs/index.rst
37
+ docs/make.bat
38
+ docs/modules.rst
39
+ tests/test_hci.py
40
+ tests/test_hcp.py
41
+ tests/test_parse_credentials.py
42
+ tests/data/80MB_test_file
43
+ tests/data/complete_creds_0.json
44
+ tests/data/complete_creds_1.json
45
+ tests/data/complete_creds_2.json
46
+ tests/data/incomplete_creds_0.json
47
+ tests/data/incomplete_creds_1.json
48
+ tests/data/incomplete_creds_2.json
49
+ tests/data/incomplete_creds_3.json
50
+ tests/data/incomplete_creds_4.json
51
+ tests/data/a folder of data/80MB_test_file_0
52
+ tests/data/a folder of data/80MB_test_file_1
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: NGPIris
3
- Version: 5.1.1.dev3
3
+ Version: 5.2.0
4
4
  Author-email: Erik Brink <erik.brink@gu.se>, Isak Sylvin <isak.sylvin@gu.se>
5
5
  Description-Content-Type: text/markdown
6
6
  License-File: LICENSE
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "NGPIris"
3
- version = "5.1.1.dev3"
3
+ version = "5.2.0"
4
4
  readme = "README.md"
5
5
  dependencies = [
6
6
  "requests >= 2.31.0",
@@ -32,6 +32,10 @@ inputs = ["NGPIris"]
32
32
  pythonpath = [
33
33
  "."
34
34
  ]
35
+ testpaths = [
36
+ "tests"
37
+ ]
38
+ addopts = "--strict-markers" # Adds command-line options
35
39
  filterwarnings = "ignore::urllib3.connectionpool.InsecureRequestWarning"
36
40
 
37
41
  [project.scripts]
@@ -1,10 +1,14 @@
1
1
 
2
+ from configparser import ConfigParser
2
3
  from NGPIris.hci import HCIHandler
3
4
  from random import randint
4
5
  from json import dump
5
6
  from os import remove
6
7
 
7
- hci_h = HCIHandler("credentials/testCredentials.json")
8
+ ini_config = ConfigParser()
9
+ ini_config.read("tests/test_conf.ini")
10
+
11
+ hci_h = HCIHandler(ini_config.get("General", "credentials_path"))
8
12
  hci_h.request_token()
9
13
 
10
14
  def test_list_index_names_type() -> None:
@@ -0,0 +1,213 @@
1
+
2
+
3
+ from pathlib import Path
4
+ from filecmp import cmp
5
+ from typing import Any, Callable
6
+
7
+ from conftest import CustomConfig
8
+
9
+ from NGPIris.hcp import HCPHandler
10
+
11
+ # --------------------------- Helper fucntions ---------------------------------
12
+
13
+ def _without_mounting(hcp_h : HCPHandler, hcp_h_method : Callable[..., Any]) -> None:
14
+ hcp_h.bucket_name = None
15
+ try:
16
+ hcp_h_method(hcp_h)
17
+ except:
18
+ assert True
19
+ else: # pragma: no cover
20
+ assert False
21
+
22
+ # --------------------------- Test suite ---------------------------------------
23
+
24
+ def test_list_buckets(custom_config : CustomConfig) -> None:
25
+ assert custom_config.hcp_h.list_buckets()
26
+
27
+ def test_mount_bucket(custom_config : CustomConfig) -> None:
28
+ custom_config.hcp_h.mount_bucket(custom_config.test_bucket)
29
+
30
+ def test_mount_nonexisting_bucket(custom_config : CustomConfig) -> None:
31
+ try:
32
+ custom_config.hcp_h.mount_bucket("aBucketThatDoesNotExist")
33
+ except:
34
+ assert True
35
+ else: # pragma: no cover
36
+ assert False
37
+
38
+ def test_test_connection(custom_config : CustomConfig) -> None:
39
+ test_mount_bucket(custom_config)
40
+ custom_config.hcp_h.test_connection()
41
+
42
+ def test_test_connection_with_bucket_name(custom_config : CustomConfig) -> None:
43
+ custom_config.hcp_h.test_connection(bucket_name = custom_config.test_bucket)
44
+
45
+ def test_test_connection_without_mounting_bucket(custom_config : CustomConfig) -> None:
46
+ _hcp_h = custom_config.hcp_h
47
+ _without_mounting(_hcp_h, HCPHandler.test_connection)
48
+
49
+ def test_list_objects(custom_config : CustomConfig) -> None:
50
+ test_mount_bucket(custom_config)
51
+ assert type(list(custom_config.hcp_h.list_objects())) == list
52
+
53
+ def test_list_objects_without_mounting(custom_config : CustomConfig) -> None:
54
+ _hcp_h = custom_config.hcp_h
55
+ _without_mounting(_hcp_h, HCPHandler.list_objects)
56
+
57
+ def test_upload_file(custom_config : CustomConfig) -> None:
58
+ test_mount_bucket(custom_config)
59
+ custom_config.hcp_h.upload_file(custom_config.test_file_path)
60
+
61
+ def test_upload_file_without_mounting(custom_config : CustomConfig) -> None:
62
+ _hcp_h = custom_config.hcp_h
63
+ _without_mounting(_hcp_h, HCPHandler.upload_file)
64
+
65
+ def test_upload_file_in_sub_directory(custom_config : CustomConfig) -> None:
66
+ test_mount_bucket(custom_config)
67
+ custom_config.hcp_h.upload_file(custom_config.test_file_path, "a_sub_directory/a_file")
68
+
69
+ def test_upload_nonexistent_file(custom_config : CustomConfig) -> None:
70
+ test_mount_bucket(custom_config)
71
+ try:
72
+ custom_config.hcp_h.upload_file("tests/data/aTestFileThatDoesNotExist")
73
+ except:
74
+ assert True
75
+ else: # pragma: no cover
76
+ assert False
77
+
78
+ def test_upload_folder(custom_config : CustomConfig) -> None:
79
+ test_mount_bucket(custom_config)
80
+ custom_config.hcp_h.upload_folder("tests/data/a folder of data/", "a folder of data/")
81
+
82
+ def test_upload_folder_without_mounting(custom_config : CustomConfig) -> None:
83
+ _hcp_h = custom_config.hcp_h
84
+ _without_mounting(_hcp_h, HCPHandler.upload_folder)
85
+
86
+ def test_upload_nonexisting_folder(custom_config : CustomConfig) -> None:
87
+ test_mount_bucket(custom_config)
88
+ try:
89
+ custom_config.hcp_h.upload_folder("tests/data/aFolderOfFilesThatDoesNotExist")
90
+ except:
91
+ assert True
92
+ else: # pragma: no cover
93
+ assert False
94
+
95
+ def test_get_file(custom_config : CustomConfig) -> None:
96
+ test_mount_bucket(custom_config)
97
+ assert custom_config.hcp_h.object_exists("a_sub_directory/a_file")
98
+ assert custom_config.hcp_h.get_object("a_sub_directory/a_file")
99
+
100
+ def test_get_folder_without_mounting(custom_config : CustomConfig) -> None:
101
+ _hcp_h = custom_config.hcp_h
102
+ _without_mounting(_hcp_h, HCPHandler.object_exists)
103
+ _without_mounting(_hcp_h, HCPHandler.get_object)
104
+
105
+ def test_get_file_in_sub_directory(custom_config : CustomConfig) -> None:
106
+ test_mount_bucket(custom_config)
107
+ test_file = Path(custom_config.test_file_path).name
108
+ assert custom_config.hcp_h.object_exists(test_file)
109
+ assert custom_config.hcp_h.get_object(test_file)
110
+
111
+ def test_download_file(custom_config : CustomConfig) -> None:
112
+ test_mount_bucket(custom_config)
113
+ Path(custom_config.result_path).mkdir()
114
+ test_file = Path(custom_config.test_file_path).name
115
+ custom_config.hcp_h.download_file(test_file, custom_config.result_path + test_file)
116
+ assert cmp(custom_config.result_path + test_file, custom_config.test_file_path)
117
+
118
+ def test_download_file_without_mounting(custom_config : CustomConfig) -> None:
119
+ _hcp_h = custom_config.hcp_h
120
+ _without_mounting(_hcp_h, HCPHandler.download_file)
121
+
122
+ def test_download_nonexistent_file(custom_config : CustomConfig) -> None:
123
+ test_mount_bucket(custom_config)
124
+ try:
125
+ custom_config.hcp_h.download_file("aFileThatDoesNotExist", custom_config.result_path + "aFileThatDoesNotExist")
126
+ except:
127
+ assert True
128
+ else: # pragma: no cover
129
+ assert False
130
+
131
+ def test_download_folder(custom_config : CustomConfig) -> None:
132
+ test_mount_bucket(custom_config)
133
+ custom_config.hcp_h.download_folder("a folder of data/", custom_config.result_path)
134
+
135
+ def test_search_objects_in_bucket(custom_config : CustomConfig) -> None:
136
+ test_mount_bucket(custom_config)
137
+ test_file = Path(custom_config.test_file_path).name
138
+ custom_config.hcp_h.search_objects_in_bucket(test_file)
139
+
140
+ def test_search_objects_in_bucket_without_mounting(custom_config : CustomConfig) -> None:
141
+ _hcp_h = custom_config.hcp_h
142
+ _without_mounting(_hcp_h, HCPHandler.search_objects_in_bucket)
143
+
144
+ def test_get_object_acl(custom_config : CustomConfig) -> None:
145
+ test_mount_bucket(custom_config)
146
+ test_file = Path(custom_config.test_file_path).name
147
+ custom_config.hcp_h.get_object_acl(test_file)
148
+
149
+ def test_get_object_acl_without_mounting(custom_config : CustomConfig) -> None:
150
+ _hcp_h = custom_config.hcp_h
151
+ _without_mounting(_hcp_h, HCPHandler.get_object_acl)
152
+
153
+ def test_get_bucket_acl(custom_config : CustomConfig) -> None:
154
+ test_mount_bucket(custom_config)
155
+ custom_config.hcp_h.get_bucket_acl()
156
+
157
+ def test_get_bucket_acl_without_mounting(custom_config : CustomConfig) -> None:
158
+ _hcp_h = custom_config.hcp_h
159
+ _without_mounting(_hcp_h, HCPHandler.get_bucket_acl)
160
+
161
+ # ------------------ Possibly future ACL tests ---------------------------------
162
+
163
+ #def test_modify_single_object_acl(custom_config : CustomConfig) -> None:
164
+ # test_mount_bucket(custom_config)
165
+ # custom_config.hcp_h.modify_single_object_acl()
166
+ #
167
+ #def test_modify_single_bucket_acl(custom_config : CustomConfig) -> None:
168
+ # test_mount_bucket(custom_config)
169
+ # custom_config.hcp_h.modify_single_bucket_acl()
170
+ #
171
+ #def test_modify_object_acl(custom_config : CustomConfig) -> None:
172
+ # test_mount_bucket(custom_config)
173
+ # custom_config.hcp_h.modify_object_acl()
174
+ #
175
+ #def test_modify_bucket_acl(custom_config : CustomConfig) -> None:
176
+ # test_mount_bucket(custom_config)
177
+ # custom_config.hcp_h.modify_bucket_acl()
178
+
179
+ # ------------------------------------------------------------------------------
180
+
181
+ def test_delete_file(custom_config : CustomConfig) -> None:
182
+ test_mount_bucket(custom_config)
183
+ test_file = Path(custom_config.test_file_path).name
184
+ custom_config.hcp_h.delete_object(test_file)
185
+ custom_config.hcp_h.delete_object("a_sub_directory/a_file")
186
+ custom_config.hcp_h.delete_object("a_sub_directory")
187
+
188
+ def test_delete_file_without_mounting(custom_config : CustomConfig) -> None:
189
+ _hcp_h = custom_config.hcp_h
190
+ _without_mounting(_hcp_h, HCPHandler.delete_object)
191
+
192
+ def test_delete_folder_with_sub_directory(custom_config : CustomConfig) -> None:
193
+ test_mount_bucket(custom_config)
194
+ custom_config.hcp_h.upload_file(custom_config.test_file_path, "a folder of data/a sub dir/a file")
195
+ try:
196
+ custom_config.hcp_h.delete_folder("a folder of data/")
197
+ except:
198
+ assert True
199
+ else: # pragma: no cover
200
+ assert False
201
+ custom_config.hcp_h.delete_folder("a folder of data/a sub dir/")
202
+
203
+ def test_delete_folder(custom_config : CustomConfig) -> None:
204
+ test_mount_bucket(custom_config)
205
+ custom_config.hcp_h.delete_folder("a folder of data/")
206
+
207
+ def test_delete_folder_without_mounting(custom_config : CustomConfig) -> None:
208
+ _hcp_h = custom_config.hcp_h
209
+ _without_mounting(_hcp_h, HCPHandler.delete_folder)
210
+
211
+ def test_delete_nonexistent_files(custom_config : CustomConfig) -> None:
212
+ test_mount_bucket(custom_config)
213
+ custom_config.hcp_h.delete_objects(["some", "files", "that", "does", "not", "exist"])
@@ -1,127 +0,0 @@
1
- LICENSE
2
- MANIFEST.in
3
- README.md
4
- pyproject.toml
5
- NGPIris/__init__.py
6
- NGPIris.egg-info/PKG-INFO
7
- NGPIris.egg-info/SOURCES.txt
8
- NGPIris.egg-info/dependency_links.txt
9
- NGPIris.egg-info/entry_points.txt
10
- NGPIris.egg-info/requires.txt
11
- NGPIris.egg-info/top_level.txt
12
- NGPIris/cli/__init__.py
13
- NGPIris/hci/__init__.py
14
- NGPIris/hci/exceptions.py
15
- NGPIris/hci/hci.py
16
- NGPIris/hci/helpers.py
17
- NGPIris/hcp/__init__.py
18
- NGPIris/hcp/exceptions.py
19
- NGPIris/hcp/hcp.py
20
- NGPIris/hcp/helpers.py
21
- NGPIris/hcp/statistics.py
22
- NGPIris/parse_credentials/__init__.py
23
- NGPIris/parse_credentials/parse_credentials.py
24
- NGPIris/utils/__init__.py
25
- NGPIris/utils/utils.py
26
- docs/IRIS_5_changelog.md
27
- docs/Makefile
28
- docs/NGPIris.cli.rst
29
- docs/NGPIris.hci.rst
30
- docs/NGPIris.hcp.rst
31
- docs/NGPIris.parse_credentials.rst
32
- docs/NGPIris.rst
33
- docs/NGPIris.utils.rst
34
- docs/Tutorial.md
35
- docs/conf.py
36
- docs/index.rst
37
- docs/make.bat
38
- docs/modules.rst
39
- docs/_build/.DS_Store
40
- docs/_build/doctrees/IRIS_5_changelog.doctree
41
- docs/_build/doctrees/NGPIris.cli.doctree
42
- docs/_build/doctrees/NGPIris.doctree
43
- docs/_build/doctrees/NGPIris.hci.doctree
44
- docs/_build/doctrees/NGPIris.hcp.doctree
45
- docs/_build/doctrees/NGPIris.parse_credentials.doctree
46
- docs/_build/doctrees/NGPIris.utils.doctree
47
- docs/_build/doctrees/README.doctree
48
- docs/_build/doctrees/Tutorial.doctree
49
- docs/_build/doctrees/environment.pickle
50
- docs/_build/doctrees/index.doctree
51
- docs/_build/doctrees/modules.doctree
52
- docs/_build/html/.buildinfo
53
- docs/_build/html/IRIS_5_changelog.html
54
- docs/_build/html/NGPIris.cli.html
55
- docs/_build/html/NGPIris.hci.html
56
- docs/_build/html/NGPIris.hcp.html
57
- docs/_build/html/NGPIris.html
58
- docs/_build/html/NGPIris.parse_credentials.html
59
- docs/_build/html/NGPIris.utils.html
60
- docs/_build/html/README.html
61
- docs/_build/html/Tutorial.html
62
- docs/_build/html/genindex.html
63
- docs/_build/html/index.html
64
- docs/_build/html/modules.html
65
- docs/_build/html/objects.inv
66
- docs/_build/html/py-modindex.html
67
- docs/_build/html/search.html
68
- docs/_build/html/searchindex.js
69
- docs/_build/html/_downloads/ee00c4f6f22dc2671c766f728f00006f/credentials_template.json
70
- docs/_build/html/_modules/index.html
71
- docs/_build/html/_modules/NGPIris/cli.html
72
- docs/_build/html/_modules/NGPIris/hci/exceptions.html
73
- docs/_build/html/_modules/NGPIris/hci/hci.html
74
- docs/_build/html/_modules/NGPIris/hci/helpers.html
75
- docs/_build/html/_modules/NGPIris/hcp/exceptions.html
76
- docs/_build/html/_modules/NGPIris/hcp/hcp.html
77
- docs/_build/html/_modules/NGPIris/hcp/helpers.html
78
- docs/_build/html/_modules/NGPIris/hcp/statistics.html
79
- docs/_build/html/_modules/NGPIris/parse_credentials/parse_credentials.html
80
- docs/_build/html/_modules/NGPIris/utils/utils.html
81
- docs/_build/html/_sources/IRIS_5_changelog.md.txt
82
- docs/_build/html/_sources/NGPIris.cli.rst.txt
83
- docs/_build/html/_sources/NGPIris.hci.rst.txt
84
- docs/_build/html/_sources/NGPIris.hcp.rst.txt
85
- docs/_build/html/_sources/NGPIris.parse_credentials.rst.txt
86
- docs/_build/html/_sources/NGPIris.rst.txt
87
- docs/_build/html/_sources/NGPIris.utils.rst.txt
88
- docs/_build/html/_sources/README.md.txt
89
- docs/_build/html/_sources/Tutorial.md.txt
90
- docs/_build/html/_sources/index.rst.txt
91
- docs/_build/html/_sources/modules.rst.txt
92
- docs/_build/html/_static/alabaster.css
93
- docs/_build/html/_static/basic.css
94
- docs/_build/html/_static/custom.css
95
- docs/_build/html/_static/debug.css
96
- docs/_build/html/_static/doctools.js
97
- docs/_build/html/_static/documentation_options.js
98
- docs/_build/html/_static/file.png
99
- docs/_build/html/_static/language_data.js
100
- docs/_build/html/_static/minus.png
101
- docs/_build/html/_static/plus.png
102
- docs/_build/html/_static/pygments.css
103
- docs/_build/html/_static/searchtools.js
104
- docs/_build/html/_static/skeleton.css
105
- docs/_build/html/_static/sphinx_highlight.js
106
- docs/_build/html/_static/scripts/furo-extensions.js
107
- docs/_build/html/_static/scripts/furo.js
108
- docs/_build/html/_static/scripts/furo.js.LICENSE.txt
109
- docs/_build/html/_static/scripts/furo.js.map
110
- docs/_build/html/_static/styles/furo-extensions.css
111
- docs/_build/html/_static/styles/furo-extensions.css.map
112
- docs/_build/html/_static/styles/furo.css
113
- docs/_build/html/_static/styles/furo.css.map
114
- tests/test_hci.py
115
- tests/test_hcp.py
116
- tests/test_parse_credentials.py
117
- tests/data/80MB_test_file
118
- tests/data/complete_creds_0.json
119
- tests/data/complete_creds_1.json
120
- tests/data/complete_creds_2.json
121
- tests/data/incomplete_creds_0.json
122
- tests/data/incomplete_creds_1.json
123
- tests/data/incomplete_creds_2.json
124
- tests/data/incomplete_creds_3.json
125
- tests/data/incomplete_creds_4.json
126
- tests/data/a folder of data/80MB_test_file_0
127
- tests/data/a folder of data/80MB_test_file_1
Binary file