ibridges 0.2.2__tar.gz → 0.2.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. {ibridges-0.2.2 → ibridges-0.2.3}/PKG-INFO +2 -1
  2. {ibridges-0.2.2 → ibridges-0.2.3}/docs/source/cli.rst +33 -3
  3. {ibridges-0.2.2 → ibridges-0.2.3}/docs/source/faq.rst +12 -0
  4. {ibridges-0.2.2 → ibridges-0.2.3}/docs/source/quickstart.rst +3 -2
  5. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/__main__.py +80 -3
  6. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/_version.py +2 -2
  7. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/data_operations.py +10 -7
  8. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/resources.py +1 -4
  9. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/search.py +6 -6
  10. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/session.py +3 -0
  11. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/util.py +61 -0
  12. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges.egg-info/PKG-INFO +2 -1
  13. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges.egg-info/SOURCES.txt +1 -0
  14. {ibridges-0.2.2 → ibridges-0.2.3}/pyproject.toml +2 -0
  15. ibridges-0.2.3/tutorials/07-Streaming-iRODS-data.ipynb +353 -0
  16. {ibridges-0.2.2 → ibridges-0.2.3}/.dockerignore +0 -0
  17. {ibridges-0.2.2 → ibridges-0.2.3}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  18. {ibridges-0.2.2 → ibridges-0.2.3}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  19. {ibridges-0.2.2 → ibridges-0.2.3}/.github/dependabot.yml +0 -0
  20. {ibridges-0.2.2 → ibridges-0.2.3}/.github/workflows/integration-tests-irods.yml +0 -0
  21. {ibridges-0.2.2 → ibridges-0.2.3}/.github/workflows/integration-tests-yoda.yml +0 -0
  22. {ibridges-0.2.2 → ibridges-0.2.3}/.github/workflows/main.yml +0 -0
  23. {ibridges-0.2.2 → ibridges-0.2.3}/.github/workflows/pypi_release.yml +0 -0
  24. {ibridges-0.2.2 → ibridges-0.2.3}/.gitignore +0 -0
  25. {ibridges-0.2.2 → ibridges-0.2.3}/.readthedocs.yaml +0 -0
  26. {ibridges-0.2.2 → ibridges-0.2.3}/LICENSE +0 -0
  27. {ibridges-0.2.2 → ibridges-0.2.3}/README.md +0 -0
  28. {ibridges-0.2.2 → ibridges-0.2.3}/docker/.gitattributes +0 -0
  29. {ibridges-0.2.2 → ibridges-0.2.3}/docker/README.md +0 -0
  30. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_catalog/Dockerfile +0 -0
  31. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_catalog/init-user-db.sh +0 -0
  32. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_catalog_provider/Dockerfile +0 -0
  33. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_catalog_provider/entrypoint.sh +0 -0
  34. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_catalog_provider/setup-4.3.1.input +0 -0
  35. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_catalog_provider/setup-4.3.2.input +0 -0
  36. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/Dockerfile +0 -0
  37. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/entrypoint.sh +0 -0
  38. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/environments/plain-irods/config.toml +0 -0
  39. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/environments/plain-irods/irods_environment.json +0 -0
  40. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/environments/plain-irods/irods_environment_testuser.json +0 -0
  41. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/environments/yoda/config.toml +0 -0
  42. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/environments/yoda/irods_environment.json +0 -0
  43. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/testdata/beach.rtf +0 -0
  44. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/testdata/bunny.rtf +0 -0
  45. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/testdata/example.r +0 -0
  46. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/testdata/more_data/polarbear.txt +0 -0
  47. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/testdata/plant.rtf +0 -0
  48. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/testdata/sun.rtf +0 -0
  49. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/tests/conftest.py +0 -0
  50. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/tests/test_cli.py +0 -0
  51. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/tests/test_data_ops.py +0 -0
  52. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/tests/test_meta.py +0 -0
  53. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/tests/test_move.py +0 -0
  54. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/tests/test_permissions.py +0 -0
  55. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/tests/test_resources.py +0 -0
  56. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/tests/test_rules.py +0 -0
  57. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/tests/test_session.py +0 -0
  58. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/tests/test_sync.py +0 -0
  59. {ibridges-0.2.2 → ibridges-0.2.3}/docker/irods_client/tests/test_ticket.py +0 -0
  60. {ibridges-0.2.2 → ibridges-0.2.3}/docker-compose.yml +0 -0
  61. {ibridges-0.2.2 → ibridges-0.2.3}/docs/Makefile +0 -0
  62. {ibridges-0.2.2 → ibridges-0.2.3}/docs/ibridges-prc.png +0 -0
  63. {ibridges-0.2.2 → ibridges-0.2.3}/docs/logo.png +0 -0
  64. {ibridges-0.2.2 → ibridges-0.2.3}/docs/make.bat +0 -0
  65. {ibridges-0.2.2 → ibridges-0.2.3}/docs/requirements.txt +0 -0
  66. {ibridges-0.2.2 → ibridges-0.2.3}/docs/source/api/full_reference.rst +0 -0
  67. {ibridges-0.2.2 → ibridges-0.2.3}/docs/source/api/main.rst +0 -0
  68. {ibridges-0.2.2 → ibridges-0.2.3}/docs/source/api/user_reference.rst +0 -0
  69. {ibridges-0.2.2 → ibridges-0.2.3}/docs/source/conf.py +0 -0
  70. {ibridges-0.2.2 → ibridges-0.2.3}/docs/source/data_transfers.rst +0 -0
  71. {ibridges-0.2.2 → ibridges-0.2.3}/docs/source/index.rst +0 -0
  72. {ibridges-0.2.2 → ibridges-0.2.3}/docs/source/install.rst +0 -0
  73. {ibridges-0.2.2 → ibridges-0.2.3}/docs/source/ipath.rst +0 -0
  74. {ibridges-0.2.2 → ibridges-0.2.3}/docs/source/irods_search.rst +0 -0
  75. {ibridges-0.2.2 → ibridges-0.2.3}/docs/source/metadata.rst +0 -0
  76. {ibridges-0.2.2 → ibridges-0.2.3}/docs/source/session.rst +0 -0
  77. {ibridges-0.2.2 → ibridges-0.2.3}/docs/source/sync.rst +0 -0
  78. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/__init__.py +0 -0
  79. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/export_metadata.py +0 -0
  80. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/icat_columns.py +0 -0
  81. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/interactive.py +0 -0
  82. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/meta.py +0 -0
  83. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/path.py +0 -0
  84. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/permissions.py +0 -0
  85. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/rules.py +0 -0
  86. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges/tickets.py +0 -0
  87. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges.egg-info/dependency_links.txt +0 -0
  88. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges.egg-info/entry_points.txt +0 -0
  89. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges.egg-info/requires.txt +0 -0
  90. {ibridges-0.2.2 → ibridges-0.2.3}/ibridges.egg-info/top_level.txt +0 -0
  91. {ibridges-0.2.2 → ibridges-0.2.3}/setup.cfg +0 -0
  92. {ibridges-0.2.2 → ibridges-0.2.3}/tests/test_irodspath.py +0 -0
  93. {ibridges-0.2.2 → ibridges-0.2.3}/tests/testdata/bunny.txt +0 -0
  94. {ibridges-0.2.2 → ibridges-0.2.3}/tests/testdata/subfolder/sun.csv +0 -0
  95. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/00-FirstSteps.ipynb +0 -0
  96. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/01-Setup-and-connect.ipynb +0 -0
  97. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/02-iRODS-paths.ipynb +0 -0
  98. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/03-Working-with-data.ipynb +0 -0
  99. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/04-Metadata.ipynb +0 -0
  100. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/05-Data-Sharing.ipynb +0 -0
  101. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/06-Data-sync.ipynb +0 -0
  102. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/example_rules/example.r +0 -0
  103. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/img/DataObject1.png +0 -0
  104. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/img/DataObject2.png +0 -0
  105. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/img/DataObject3.png +0 -0
  106. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/img/DataObject4.png +0 -0
  107. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/img/DataObject5.png +0 -0
  108. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/img/DataObject6.png +0 -0
  109. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/img/Save_json.png +0 -0
  110. {ibridges-0.2.2 → ibridges-0.2.3}/tutorials/img/Yoda_environment.png +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: ibridges
3
- Version: 0.2.2
3
+ Version: 0.2.3
4
4
  Summary: Package for accessing data and metadata on iRods servers.
5
5
  Author-email: Christine Staiger <c.staiger@uu.nl>
6
6
  License: MIT License
@@ -32,6 +32,7 @@ Classifier: Programming Language :: Python :: 3.8
32
32
  Classifier: Programming Language :: Python :: 3.9
33
33
  Classifier: Programming Language :: Python :: 3.10
34
34
  Classifier: Programming Language :: Python :: 3.11
35
+ Classifier: Programming Language :: Python :: 3.12
35
36
  Classifier: Development Status :: 3 - Alpha
36
37
  Classifier: License :: OSI Approved :: MIT License
37
38
  Requires-Python: >=3.8
@@ -12,10 +12,40 @@ in your shell script without having to create a new python script.
12
12
  There are no CLI commands to add/change metadata, instead use the iBridges API for this.
13
13
 
14
14
 
15
- Setting up
16
- ----------
15
+ .. _cli-setup:
17
16
 
18
- As with the ibridges API, you will need to create an `irods_environment.json`. It is the easiest if you put this file
17
+ Setup
18
+ -----
19
+
20
+ As with the ibridges API, you will need to create an `irods_environment.json`. We have created a plugin system to automatically
21
+ create the environment file for you. Below are the currently (known) plugins, see the links for installation instructions:
22
+
23
+ .. list-table:: Server configuration plugins
24
+ :widths: 50 50
25
+ :header-rows: 1
26
+
27
+ * - Organization
28
+ - Link
29
+ * - Utrecht University
30
+ - https://github.com/UtrechtUniversity/ibridges-servers-uu
31
+
32
+ After installation, you will be able to create an `irods_environment.json` by simply answering questions like which email-address
33
+ you have. First find the server name with:
34
+
35
+ .. code:: shell
36
+
37
+ ibridges setup --list
38
+
39
+ Then finish the setup using the server name you just found:
40
+
41
+ .. code:: shell
42
+
43
+ ibridges setup server_name
44
+
45
+ If your organization does not provide a plugin, then you will have to create the `environment.json` yourself (with
46
+ the help of your iRODS administrator).
47
+
48
+ It is the easiest if you put this file
19
49
  in the default location: `~/.irods/irods_environment.json`, because then it will be automatically detected. However,
20
50
  if you have it in another location for some reason (let's say you have multiple environments), then you can tell the
21
51
  ibridges CLI where it is:
@@ -21,3 +21,15 @@ An advantage compared to the iCommands is that iBridges also works on Mac OS and
21
21
 
22
22
  Our development is done on `GitHub <https://github.com/UtrechtUniversity/iBridges>`__ We are welcoming contributions
23
23
  by pull requests. You can also ask for new features/ideas in our issue tracker.
24
+
25
+
26
+ **I have installed iBridges and now get "ibridges: command not found"**
27
+ -----------------------------------------------------------------------
28
+
29
+ This can happen for a variety for reasons, but the most common reason is that your PATH is not setup correctly on your system.
30
+ Often `pip` will complain about this when you install `ibridges`. To solve this, you must first find out where pip installs the
31
+ ibridges executable. Usually this will be something like `/home/your_username/.local/bin`, but this is dependent on your system. Then we must
32
+ add this to the path on the command line: `export PATH="${PATH}:/home/your_username/.local/bin"` (change the path according to your system). This should allow
33
+ your shell to find the `ibridges` command. You would have to type the previous command every time you start a new shell, which can be inconvenient.
34
+ To fix this permanently, add the command to your `.bashrc` or `.zshrc` file in your home directory at the end of the file
35
+ (depending on your shell, type `echo ${SHELL}` to find out).
@@ -15,8 +15,9 @@ iBridges requires Python version 3.8 or higher. You can install iBridges with pi
15
15
  Getting your iRODS environment file
16
16
  -----------------------------------
17
17
 
18
- To connect to an iRods server you need an iRods environment file (`irods_nevironment.json`).
19
- You can obtain this by asking your local iRods administrator. An example of an environment file:
18
+ To connect to an iRods server you need an iRods environment file (`irods_environment.json`).
19
+ If your organization provides automatic setup, you can create this file yourself using the :ref:`CLI <cli-setup>`.
20
+ Otherwise, you can obtain this by asking your local iRods administrator. An example of an environment file:
20
21
 
21
22
  .. code:: json
22
23
 
@@ -8,10 +8,15 @@ from pathlib import Path
8
8
  from typing import Union
9
9
 
10
10
  from ibridges.data_operations import download, sync, upload
11
- from ibridges.interactive import interactive_auth
11
+ from ibridges.interactive import DEFAULT_IENV_PATH, interactive_auth
12
12
  from ibridges.path import IrodsPath
13
13
  from ibridges.session import Session
14
- from ibridges.util import get_collection
14
+ from ibridges.util import (
15
+ find_environment_provider,
16
+ get_collection,
17
+ get_environment_providers,
18
+ print_environment_providers,
19
+ )
15
20
 
16
21
  try: # Python < 3.10 (backport)
17
22
  from importlib_metadata import version # type: ignore
@@ -40,6 +45,8 @@ Available subcommands:
40
45
  List a collection and subcollections in a hierarchical way.
41
46
  mkcoll:
42
47
  Create the collection and all its parent collections.
48
+ setup:
49
+ Create an iRODS environment file to connect to an iRODS server.
43
50
 
44
51
  The iBridges CLI does not implement the complete iBridges API. For example, there
45
52
  are no commands to modify metadata on the irods server.
@@ -53,6 +60,7 @@ ibridges sync ~/directory "irods:~/collection"
53
60
  ibridges list irods:~/collection
54
61
  ibridges mkcoll irods://~/bli/bla/blubb
55
62
  ibridges tree irods:~/collection
63
+ ibridges setup uu-its
56
64
 
57
65
 
58
66
  Program information:
@@ -88,6 +96,8 @@ def main() -> None:
88
96
  ibridges_mkcoll()
89
97
  elif subcommand == "tree":
90
98
  ibridges_tree()
99
+ elif subcommand == "setup":
100
+ ibridges_setup()
91
101
  else:
92
102
  print(f"Invalid subcommand ({subcommand}). For help see ibridges --help")
93
103
  sys.exit(1)
@@ -147,11 +157,78 @@ def _list_coll(session: Session, remote_path: IrodsPath):
147
157
  print(str(remote_path)+':')
148
158
  coll = get_collection(session, remote_path)
149
159
  print('\n'.join([' '+sub.path for sub in coll.data_objects]))
150
- print('\n'.join([' C- '+sub.path for sub in coll.subcollections]))
160
+ print('\n'.join([' C- '+sub.path for sub in coll.subcollections
161
+ if not str(remote_path) == sub.path]))
151
162
  else:
152
163
  raise ValueError(f"Irods path '{remote_path}' is not a collection.")
153
164
 
154
165
 
166
+ def ibridges_setup():
167
+ """Use templates to create an iRODS environment json."""
168
+ parser = argparse.ArgumentParser(
169
+ prog="ibridges setup",
170
+ description="Tool to create a valid irods_environment.json"
171
+ )
172
+ parser.add_argument(
173
+ "server_name",
174
+ help="Server name to create your irods_environment.json for.",
175
+ type=str,
176
+ default=None,
177
+ nargs="?"
178
+ )
179
+ parser.add_argument(
180
+ "--list",
181
+ help="List all available server names.",
182
+ action="store_true"
183
+ )
184
+ parser.add_argument(
185
+ "-o", "--output",
186
+ help="Store the environment to a file.",
187
+ type=Path,
188
+ default=DEFAULT_IENV_PATH,
189
+ required=False,
190
+ )
191
+ parser.add_argument(
192
+ "--overwrite",
193
+ help="Overwrite the irods environment file.",
194
+ action="store_true"
195
+ )
196
+ args = parser.parse_args()
197
+ env_providers = get_environment_providers()
198
+ if args.list:
199
+ if len(env_providers) == 0:
200
+ print("No server information was found. To use this function, please install a plugin"
201
+ " such as:\n\nhttps://github.com/UtrechtUniversity/ibridges-servers-uu"
202
+ "\n\nAlternatively create an irods_environment.json by yourself or with the help "
203
+ "of your iRODS administrator.")
204
+ print_environment_providers(env_providers)
205
+ return
206
+
207
+ try:
208
+ provider = find_environment_provider(env_providers, args.server_name)
209
+ except ValueError:
210
+ print(f"Error: Unknown server with name {args.server_name}.\n"
211
+ "Use `ibridges setup --list` to list all available server names.")
212
+ sys.exit(123)
213
+
214
+ user_answers = {}
215
+ for question in provider.questions:
216
+ user_answers[question] = input(question + ": ")
217
+
218
+ json_str = provider.environment_json(args.server_name, **user_answers)
219
+ if args.output.is_file() and not args.overwrite:
220
+ print(f"File {args.output} already exists, use --overwrite or copy the below manually.")
221
+ print("\n")
222
+ print(json_str)
223
+ if args.output.is_dir():
224
+ print(f"Output {args.output} is a directory, cannot export irods_environment"
225
+ " file.")
226
+ sys.exit(234)
227
+ else:
228
+ with open(args.output, "w", encoding="utf-8") as handle:
229
+ handle.write(json_str)
230
+
231
+
155
232
  def ibridges_list():
156
233
  """List a collection on iRODS."""
157
234
  parser = argparse.ArgumentParser(
@@ -12,5 +12,5 @@ __version__: str
12
12
  __version_tuple__: VERSION_TUPLE
13
13
  version_tuple: VERSION_TUPLE
14
14
 
15
- __version__ = version = '0.2.2'
16
- __version_tuple__ = version_tuple = (0, 2, 2)
15
+ __version__ = version = '0.2.3'
16
+ __version_tuple__ = version_tuple = (0, 2, 3)
@@ -15,7 +15,6 @@ import irods.collection
15
15
  import irods.data_object
16
16
  import irods.exception
17
17
  import irods.keywords as kw
18
- from irods import DEFAULT_CONNECTION_TIMEOUT
19
18
  from tqdm import tqdm
20
19
 
21
20
  from ibridges.path import CachedIrodsPath, IrodsPath
@@ -378,14 +377,18 @@ def perform_operations(session: Session, operations: dict, ignore_err: bool=Fals
378
377
  pbar = tqdm(total=sum(up_sizes) + sum(down_sizes), unit="B", unit_scale=True, unit_divisor=1024,
379
378
  disable=disable)
380
379
 
380
+ # The code below does not work as expected, since connections in the
381
+ # pool can be reused. Another solution for dynamic timeouts might be needed
382
+ # Leaving the previous solution in here for documentation.
383
+
381
384
  # For large files, the checksum computation might take too long, which can result in a timeout.
382
385
  # This is why we increase the time out from file sizes > 1 GB
383
386
  # This might still result in a time out if your server is very busy or a potato.
384
- max_size = max([*up_sizes, *down_sizes, 0])
385
- original_timeout = session.irods_session.pool.connection_timeout
386
- if max_size > 1e9 and original_timeout == DEFAULT_CONNECTION_TIMEOUT:
387
- session.irods_session.pool.connection_timeout = int(
388
- DEFAULT_CONNECTION_TIMEOUT*(max_size/1e9)+0.5)
387
+ # max_size = max([*up_sizes, *down_sizes, 0])
388
+ # original_timeout = session.irods_session.pool.connection_timeout
389
+ # if max_size > 1e9 and original_timeout == DEFAULT_CONNECTION_TIMEOUT:
390
+ # session.irods_session.pool.connection_timeout = int(
391
+ # DEFAULT_CONNECTION_TIMEOUT*(max_size/1e9)+0.5)
389
392
 
390
393
  for col in operations["create_collection"]:
391
394
  IrodsPath.create_collection(session, col)
@@ -406,7 +409,7 @@ def perform_operations(session: Session, operations: dict, ignore_err: bool=Fals
406
409
  _obj_get(session, ipath, lpath, overwrite=True, ignore_err=ignore_err, options=options,
407
410
  resc_name=resc_name)
408
411
  pbar.update(size)
409
- session.irods_session.pool.connection_timeout = original_timeout
412
+ # session.irods_session.pool.connection_timeout = original_timeout
410
413
 
411
414
 
412
415
  def sync(session: Session,
@@ -44,10 +44,7 @@ class Resources():
44
44
  If the resource does not exist.
45
45
 
46
46
  """
47
- try:
48
- return self.session.irods_session.resources.get(resc_name)
49
- except irods.exception.ResourceDoesNotExist as error:
50
- return {'successful': False, 'reason': repr(error)}
47
+ return self.session.irods_session.resources.get(resc_name)
51
48
 
52
49
  def get_free_space(self, resc_name: str) -> int:
53
50
  """Determine free space in a resource hierarchy.
@@ -60,17 +60,17 @@ def search_data(session: Session, path: Optional[Union[str, IrodsPath]] = None,
60
60
  # one data search in case path is a collection path and we want to retrieve all data there
61
61
  # one in case the path is or ends with a file name
62
62
  if path:
63
- path = IrodsPath(session, path)
64
- parent = path.parent
65
- name = path.name
63
+ path = str(path)
64
+ parent = path.rsplit("/", maxsplit=1)[0]
65
+ name = path.rsplit("/", maxsplit=1)[1]
66
66
  # all collections starting with path
67
- coll_query = coll_query.filter(icat.LIKE(icat.COLL_NAME, str(path)))
67
+ coll_query = coll_query.filter(icat.LIKE(icat.COLL_NAME, path))
68
68
 
69
69
  # all data objects in path
70
- data_query = data_query.filter(icat.LIKE(icat.COLL_NAME, str(path)))
70
+ data_query = data_query.filter(icat.LIKE(icat.COLL_NAME, path))
71
71
  # all data objects on path.parent with name
72
72
  data_name_query = data_name_query.filter(icat.LIKE(icat.DATA_NAME, name)).filter(
73
- icat.LIKE(icat.COLL_NAME, str(parent)))
73
+ icat.LIKE(icat.COLL_NAME, parent))
74
74
  if key_vals:
75
75
  for key in key_vals:
76
76
  data_query.filter(icat.LIKE(icat.META_DATA_ATTR_NAME, key))
@@ -72,6 +72,9 @@ class Session:
72
72
  raise TypeError(f"Error reading environment file '{irods_env_path}': "
73
73
  f"expected dictionary, got {type(irods_env)}.")
74
74
 
75
+ if "connection_timeout" not in irods_env:
76
+ irods_env["connection_timeout"] = 25000
77
+
75
78
  self._password = password
76
79
  self._irods_env: dict = irods_env
77
80
  self._irods_env_path = irods_env_path
@@ -2,12 +2,18 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
+ from collections.abc import Sequence
5
6
  from typing import Union
6
7
 
7
8
  import irods
8
9
 
9
10
  from ibridges.path import IrodsPath
10
11
 
12
+ try:
13
+ from importlib_metadata import entry_points
14
+ except ImportError:
15
+ from importlib.metadata import entry_points # type: ignore
16
+
11
17
 
12
18
  def get_dataobject(session,
13
19
  path: Union[str, IrodsPath]) -> irods.data_object.iRODSDataObject:
@@ -76,3 +82,58 @@ def obj_replicas(obj: irods.data_object.iRODSDataObject) -> list[tuple[int, str,
76
82
  r.size, repl_states.get(r.status, r.status)) for r in obj.replicas]
77
83
 
78
84
  return replicas
85
+
86
+ def get_environment_providers() -> list:
87
+ """Get a list of all environment template providers.
88
+
89
+ Returns
90
+ -------
91
+ The list that contains the providers.
92
+
93
+ """
94
+ return [entry.load() for entry in entry_points(group="ibridges_server_template")]
95
+
96
+
97
+ def print_environment_providers(env_providers: Sequence):
98
+ """Print the environment providers to the screen.
99
+
100
+ Parameters
101
+ ----------
102
+ env_providers
103
+ A list of all installed environment providers.
104
+
105
+ """
106
+ for provider in env_providers:
107
+ print(provider.name)
108
+ print("-"*len(provider.name))
109
+ print("\n")
110
+ max_len = max(len(x) for x in provider.descriptions)
111
+ for server_name, description in provider.descriptions.items():
112
+ print(f"{server_name: <{max_len+1}} - {description}")
113
+
114
+
115
+ def find_environment_provider(env_providers: list, server_name: str) -> object:
116
+ """Find the provider that provides the right template.
117
+
118
+ Parameters
119
+ ----------
120
+ env_providers
121
+ A list of all installed environment providers.
122
+ server_name
123
+ Name of the server for which the template is to be found.
124
+
125
+ Returns
126
+ -------
127
+ The provider that contains the template.
128
+
129
+ Raises
130
+ ------
131
+ ValueError
132
+ If the server_name identifier can't be found in the providers.
133
+
134
+ """
135
+ for provider in env_providers:
136
+ if provider.contains(server_name):
137
+ return provider
138
+ raise ValueError("Cannot find provider with name {server_name} ensure that the plugin is "
139
+ "installed.")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: ibridges
3
- Version: 0.2.2
3
+ Version: 0.2.3
4
4
  Summary: Package for accessing data and metadata on iRods servers.
5
5
  Author-email: Christine Staiger <c.staiger@uu.nl>
6
6
  License: MIT License
@@ -32,6 +32,7 @@ Classifier: Programming Language :: Python :: 3.8
32
32
  Classifier: Programming Language :: Python :: 3.9
33
33
  Classifier: Programming Language :: Python :: 3.10
34
34
  Classifier: Programming Language :: Python :: 3.11
35
+ Classifier: Programming Language :: Python :: 3.12
35
36
  Classifier: Development Status :: 3 - Alpha
36
37
  Classifier: License :: OSI Approved :: MIT License
37
38
  Requires-Python: >=3.8
@@ -96,6 +96,7 @@ tutorials/03-Working-with-data.ipynb
96
96
  tutorials/04-Metadata.ipynb
97
97
  tutorials/05-Data-Sharing.ipynb
98
98
  tutorials/06-Data-sync.ipynb
99
+ tutorials/07-Streaming-iRODS-data.ipynb
99
100
  tutorials/example_rules/example.r
100
101
  tutorials/img/DataObject1.png
101
102
  tutorials/img/DataObject2.png
@@ -18,6 +18,7 @@ classifiers = [
18
18
  "Programming Language :: Python :: 3.9",
19
19
  "Programming Language :: Python :: 3.10",
20
20
  "Programming Language :: Python :: 3.11",
21
+ "Programming Language :: Python :: 3.12",
21
22
  "Development Status :: 3 - Alpha",
22
23
  "License :: OSI Approved :: MIT License",
23
24
  ]
@@ -60,6 +61,7 @@ write_to = "ibridges/_version.py"
60
61
  [[tool.mypy.overrides]]
61
62
  module = [
62
63
  "irods.*",
64
+ "importlib_metadata.*",
63
65
  ]
64
66
  ignore_missing_imports = true
65
67
 
@@ -0,0 +1,353 @@
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "id": "d0ee0ad1",
6
+ "metadata": {},
7
+ "source": [
8
+ "# iBridges in compute tasks"
9
+ ]
10
+ },
11
+ {
12
+ "cell_type": "markdown",
13
+ "id": "c74d7aca",
14
+ "metadata": {},
15
+ "source": [
16
+ "In this notebook we show how to employ iBridges to find data by its metadata in iRODS and employ this data in a compute workflow.\n",
17
+ "\n",
18
+ "We do not store data on local disk in this example, we rather stream the content of the data objects into memory, i.e. a variable which we use in the compute workflow. Similarly we do not save the results in a file but directly stream them into a new data object on the iRODS server."
19
+ ]
20
+ },
21
+ {
22
+ "cell_type": "markdown",
23
+ "id": "d07fa233",
24
+ "metadata": {},
25
+ "source": [
26
+ "## Prerequisites"
27
+ ]
28
+ },
29
+ {
30
+ "cell_type": "markdown",
31
+ "id": "d514427e",
32
+ "metadata": {},
33
+ "source": [
34
+ "- Access to an iRODS instance\n",
35
+ "- Some textual data files labeled with the metadata key `author` and metadata value `Lewis Carroll`."
36
+ ]
37
+ },
38
+ {
39
+ "cell_type": "code",
40
+ "execution_count": 1,
41
+ "id": "9f19a37e",
42
+ "metadata": {},
43
+ "outputs": [
44
+ {
45
+ "name": "stdout",
46
+ "output_type": "stream",
47
+ "text": [
48
+ "/nluu12p/home/research-test-christine/my_books:\n",
49
+ " /nluu12p/home/research-test-christine/my_books/AdventuresSherlockHolmes.txt\n",
50
+ " /nluu12p/home/research-test-christine/my_books/AliceAdventuresInWonderLand.txt\n",
51
+ " /nluu12p/home/research-test-christine/my_books/DonQuixote.txt\n",
52
+ " /nluu12p/home/research-test-christine/my_books/Dracula.txt\n",
53
+ " /nluu12p/home/research-test-christine/my_books/Frankenstein.txt\n",
54
+ " /nluu12p/home/research-test-christine/my_books/Phantasmagoria.txt\n",
55
+ " /nluu12p/home/research-test-christine/my_books/RobinsonCrusoe.txt\n",
56
+ " /nluu12p/home/research-test-christine/my_books/TheHuntingOfTheSnark.txt\n",
57
+ " /nluu12p/home/research-test-christine/my_books/ThroughTheLookingGlass.txt\n",
58
+ " /nluu12p/home/research-test-christine/my_books/TravelsIntoSeveralRemoteNationsOfTheWorld.txt\n",
59
+ " /nluu12p/home/research-test-christine/my_books/TwentyThousandLeaguesUnderTheSea.txt\n",
60
+ "\n"
61
+ ]
62
+ }
63
+ ],
64
+ "source": [
65
+ "!ibridges list irods:my_books"
66
+ ]
67
+ },
68
+ {
69
+ "cell_type": "markdown",
70
+ "id": "999c2e2d",
71
+ "metadata": {},
72
+ "source": [
73
+ "## 1. Find the data in iRODS"
74
+ ]
75
+ },
76
+ {
77
+ "cell_type": "code",
78
+ "execution_count": null,
79
+ "id": "013aed4f",
80
+ "metadata": {},
81
+ "outputs": [],
82
+ "source": [
83
+ "from pprint import pprint\n",
84
+ "from ibridges.interactive import interactive_auth\n",
85
+ "from ibridges import search_data"
86
+ ]
87
+ },
88
+ {
89
+ "cell_type": "code",
90
+ "execution_count": null,
91
+ "id": "e4529a29",
92
+ "metadata": {},
93
+ "outputs": [],
94
+ "source": [
95
+ "session = interactive_auth()"
96
+ ]
97
+ },
98
+ {
99
+ "cell_type": "code",
100
+ "execution_count": null,
101
+ "id": "f8b2ee62",
102
+ "metadata": {},
103
+ "outputs": [],
104
+ "source": [
105
+ "KEY = 'author'\n",
106
+ "VALUE = 'Lewis Carroll'"
107
+ ]
108
+ },
109
+ {
110
+ "cell_type": "code",
111
+ "execution_count": null,
112
+ "id": "5b9f8974",
113
+ "metadata": {},
114
+ "outputs": [],
115
+ "source": [
116
+ "data = search_data(session, key_vals={KEY: VALUE})\n",
117
+ "pprint(data)"
118
+ ]
119
+ },
120
+ {
121
+ "cell_type": "markdown",
122
+ "id": "2c494e33",
123
+ "metadata": {},
124
+ "source": [
125
+ "## 2. Stream content into a variable"
126
+ ]
127
+ },
128
+ {
129
+ "cell_type": "code",
130
+ "execution_count": null,
131
+ "id": "1e41e3e7",
132
+ "metadata": {},
133
+ "outputs": [],
134
+ "source": [
135
+ "from ibridges import IrodsPath"
136
+ ]
137
+ },
138
+ {
139
+ "cell_type": "code",
140
+ "execution_count": null,
141
+ "id": "20ce2810",
142
+ "metadata": {},
143
+ "outputs": [],
144
+ "source": [
145
+ "text = \"\""
146
+ ]
147
+ },
148
+ {
149
+ "cell_type": "code",
150
+ "execution_count": null,
151
+ "id": "4277143f",
152
+ "metadata": {},
153
+ "outputs": [],
154
+ "source": [
155
+ "for item in data:\n",
156
+ " irods_path = IrodsPath(session, item['COLL_NAME'], item['DATA_NAME'])\n",
157
+ " with irods_path.dataobject.open('r') as objRead:\n",
158
+ " text = text + objRead.read().decode()"
159
+ ]
160
+ },
161
+ {
162
+ "cell_type": "code",
163
+ "execution_count": null,
164
+ "id": "7937bc29",
165
+ "metadata": {},
166
+ "outputs": [],
167
+ "source": [
168
+ "print(text[1700:1900])"
169
+ ]
170
+ },
171
+ {
172
+ "cell_type": "markdown",
173
+ "id": "e5571656",
174
+ "metadata": {},
175
+ "source": [
176
+ "## 3. Do your analysis"
177
+ ]
178
+ },
179
+ {
180
+ "cell_type": "code",
181
+ "execution_count": null,
182
+ "id": "188b78d4",
183
+ "metadata": {},
184
+ "outputs": [],
185
+ "source": [
186
+ "from collections import Counter\n",
187
+ "import string\n",
188
+ "\n",
189
+ "def wordcount(text):\n",
190
+ " # Convert to list of words, without punctuation\n",
191
+ " words = [''.join(char for char in word\n",
192
+ " if char not in string.punctuation) for word in text.split()]\n",
193
+ " print(\"Number of words:\", len(words))\n",
194
+ " unique_words_count = Counter(words)\n",
195
+ " return unique_words_count"
196
+ ]
197
+ },
198
+ {
199
+ "cell_type": "code",
200
+ "execution_count": null,
201
+ "id": "8d851666",
202
+ "metadata": {},
203
+ "outputs": [],
204
+ "source": [
205
+ "result = wordcount(text)\n",
206
+ "print(f\"Alice: {result['Alice']}\")"
207
+ ]
208
+ },
209
+ {
210
+ "cell_type": "markdown",
211
+ "id": "986587e3",
212
+ "metadata": {},
213
+ "source": [
214
+ "## 4. Write the results directly to iRODS"
215
+ ]
216
+ },
217
+ {
218
+ "cell_type": "markdown",
219
+ "id": "861f5936",
220
+ "metadata": {},
221
+ "source": [
222
+ "### Create a new empty data object"
223
+ ]
224
+ },
225
+ {
226
+ "cell_type": "code",
227
+ "execution_count": null,
228
+ "id": "424b1a74",
229
+ "metadata": {},
230
+ "outputs": [],
231
+ "source": [
232
+ "irods_path = IrodsPath(session, \"wordcount_result.json\")\n",
233
+ "obj = session.irods_session.data_objects.create(str(irods_path))\n",
234
+ "print(f\"New object of size {irods_path.size}\")"
235
+ ]
236
+ },
237
+ {
238
+ "cell_type": "code",
239
+ "execution_count": null,
240
+ "id": "16acf514",
241
+ "metadata": {},
242
+ "outputs": [],
243
+ "source": [
244
+ "import json\n",
245
+ "with obj.open('w') as obj_write:\n",
246
+ " obj_write.write(json.dumps(result).encode())"
247
+ ]
248
+ },
249
+ {
250
+ "cell_type": "code",
251
+ "execution_count": null,
252
+ "id": "421e2446",
253
+ "metadata": {},
254
+ "outputs": [],
255
+ "source": [
256
+ "print(f\"New object of size {irods_path.size}\")"
257
+ ]
258
+ },
259
+ {
260
+ "cell_type": "markdown",
261
+ "id": "90e6e21c",
262
+ "metadata": {},
263
+ "source": [
264
+ "### Add some descriptive metadata"
265
+ ]
266
+ },
267
+ {
268
+ "cell_type": "code",
269
+ "execution_count": null,
270
+ "id": "1780c671",
271
+ "metadata": {},
272
+ "outputs": [],
273
+ "source": [
274
+ "from ibridges import MetaData\n",
275
+ "from datetime import datetime"
276
+ ]
277
+ },
278
+ {
279
+ "cell_type": "code",
280
+ "execution_count": null,
281
+ "id": "01e832d5",
282
+ "metadata": {},
283
+ "outputs": [],
284
+ "source": [
285
+ "meta = MetaData(irods_path.dataobject)\n",
286
+ "print(meta)"
287
+ ]
288
+ },
289
+ {
290
+ "cell_type": "code",
291
+ "execution_count": null,
292
+ "id": "c127da96",
293
+ "metadata": {},
294
+ "outputs": [],
295
+ "source": [
296
+ "datetime.today()"
297
+ ]
298
+ },
299
+ {
300
+ "cell_type": "code",
301
+ "execution_count": null,
302
+ "id": "00c8076e",
303
+ "metadata": {},
304
+ "outputs": [],
305
+ "source": [
306
+ "meta.add('ISEARCH', KEY + '==' + VALUE)\n",
307
+ "meta.add('prov:SoftwareAgent', 'wordcount.py')\n",
308
+ "meta.add('prov:wasDerivedFrom', str(data))\n",
309
+ "meta.add('prov:actedOnBehalfOf', 'Christine')\n",
310
+ "meta.add('prov:generatedAtTime', datetime.now().strftime(\"%m/%d/%Y, %H:%M\"))"
311
+ ]
312
+ },
313
+ {
314
+ "cell_type": "code",
315
+ "execution_count": null,
316
+ "id": "16129b59",
317
+ "metadata": {},
318
+ "outputs": [],
319
+ "source": [
320
+ "print(meta)"
321
+ ]
322
+ },
323
+ {
324
+ "cell_type": "code",
325
+ "execution_count": null,
326
+ "id": "9dc02c64",
327
+ "metadata": {},
328
+ "outputs": [],
329
+ "source": []
330
+ }
331
+ ],
332
+ "metadata": {
333
+ "kernelspec": {
334
+ "display_name": "Python 3 (ipykernel)",
335
+ "language": "python",
336
+ "name": "python3"
337
+ },
338
+ "language_info": {
339
+ "codemirror_mode": {
340
+ "name": "ipython",
341
+ "version": 3
342
+ },
343
+ "file_extension": ".py",
344
+ "mimetype": "text/x-python",
345
+ "name": "python",
346
+ "nbconvert_exporter": "python",
347
+ "pygments_lexer": "ipython3",
348
+ "version": "3.10.13"
349
+ }
350
+ },
351
+ "nbformat": 4,
352
+ "nbformat_minor": 5
353
+ }
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes