ibridges 0.2.0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. {ibridges-0.2.0 → ibridges-0.2.2}/PKG-INFO +32 -17
  2. {ibridges-0.2.0 → ibridges-0.2.2}/README.md +31 -16
  3. ibridges-0.2.2/docs/ibridges-prc.png +0 -0
  4. ibridges-0.2.2/docs/logo.png +0 -0
  5. {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/cli.rst +28 -12
  6. ibridges-0.2.2/docs/source/data_transfers.rst +112 -0
  7. {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/index.rst +6 -1
  8. {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/ipath.rst +2 -1
  9. ibridges-0.2.2/docs/source/irods_search.rst +59 -0
  10. ibridges-0.2.2/docs/source/metadata.rst +120 -0
  11. {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/quickstart.rst +3 -3
  12. ibridges-0.2.2/docs/source/session.rst +94 -0
  13. {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/sync.rst +10 -6
  14. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/_version.py +2 -2
  15. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/data_operations.py +31 -7
  16. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges.egg-info/PKG-INFO +32 -17
  17. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges.egg-info/SOURCES.txt +6 -0
  18. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/02-iRODS-paths.ipynb +25 -134
  19. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/03-Working-with-data.ipynb +25 -106
  20. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/06-Data-sync.ipynb +5 -5
  21. {ibridges-0.2.0 → ibridges-0.2.2}/.dockerignore +0 -0
  22. {ibridges-0.2.0 → ibridges-0.2.2}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  23. {ibridges-0.2.0 → ibridges-0.2.2}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  24. {ibridges-0.2.0 → ibridges-0.2.2}/.github/dependabot.yml +0 -0
  25. {ibridges-0.2.0 → ibridges-0.2.2}/.github/workflows/integration-tests-irods.yml +0 -0
  26. {ibridges-0.2.0 → ibridges-0.2.2}/.github/workflows/integration-tests-yoda.yml +0 -0
  27. {ibridges-0.2.0 → ibridges-0.2.2}/.github/workflows/main.yml +0 -0
  28. {ibridges-0.2.0 → ibridges-0.2.2}/.github/workflows/pypi_release.yml +0 -0
  29. {ibridges-0.2.0 → ibridges-0.2.2}/.gitignore +0 -0
  30. {ibridges-0.2.0 → ibridges-0.2.2}/.readthedocs.yaml +0 -0
  31. {ibridges-0.2.0 → ibridges-0.2.2}/LICENSE +0 -0
  32. {ibridges-0.2.0 → ibridges-0.2.2}/docker/.gitattributes +0 -0
  33. {ibridges-0.2.0 → ibridges-0.2.2}/docker/README.md +0 -0
  34. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_catalog/Dockerfile +0 -0
  35. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_catalog/init-user-db.sh +0 -0
  36. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_catalog_provider/Dockerfile +0 -0
  37. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_catalog_provider/entrypoint.sh +0 -0
  38. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_catalog_provider/setup-4.3.1.input +0 -0
  39. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_catalog_provider/setup-4.3.2.input +0 -0
  40. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/Dockerfile +0 -0
  41. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/entrypoint.sh +0 -0
  42. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/environments/plain-irods/config.toml +0 -0
  43. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/environments/plain-irods/irods_environment.json +0 -0
  44. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/environments/plain-irods/irods_environment_testuser.json +0 -0
  45. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/environments/yoda/config.toml +0 -0
  46. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/environments/yoda/irods_environment.json +0 -0
  47. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/testdata/beach.rtf +0 -0
  48. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/testdata/bunny.rtf +0 -0
  49. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/testdata/example.r +0 -0
  50. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/testdata/more_data/polarbear.txt +0 -0
  51. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/testdata/plant.rtf +0 -0
  52. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/testdata/sun.rtf +0 -0
  53. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/conftest.py +0 -0
  54. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_cli.py +0 -0
  55. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_data_ops.py +0 -0
  56. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_meta.py +0 -0
  57. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_move.py +0 -0
  58. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_permissions.py +0 -0
  59. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_resources.py +0 -0
  60. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_rules.py +0 -0
  61. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_session.py +0 -0
  62. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_sync.py +0 -0
  63. {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_ticket.py +0 -0
  64. {ibridges-0.2.0 → ibridges-0.2.2}/docker-compose.yml +0 -0
  65. {ibridges-0.2.0 → ibridges-0.2.2}/docs/Makefile +0 -0
  66. {ibridges-0.2.0 → ibridges-0.2.2}/docs/make.bat +0 -0
  67. {ibridges-0.2.0 → ibridges-0.2.2}/docs/requirements.txt +0 -0
  68. {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/api/full_reference.rst +0 -0
  69. {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/api/main.rst +0 -0
  70. {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/api/user_reference.rst +0 -0
  71. {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/conf.py +0 -0
  72. {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/faq.rst +0 -0
  73. {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/install.rst +0 -0
  74. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/__init__.py +0 -0
  75. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/__main__.py +0 -0
  76. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/export_metadata.py +0 -0
  77. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/icat_columns.py +0 -0
  78. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/interactive.py +0 -0
  79. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/meta.py +0 -0
  80. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/path.py +0 -0
  81. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/permissions.py +0 -0
  82. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/resources.py +0 -0
  83. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/rules.py +0 -0
  84. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/search.py +0 -0
  85. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/session.py +0 -0
  86. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/tickets.py +0 -0
  87. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/util.py +0 -0
  88. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges.egg-info/dependency_links.txt +0 -0
  89. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges.egg-info/entry_points.txt +0 -0
  90. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges.egg-info/requires.txt +0 -0
  91. {ibridges-0.2.0 → ibridges-0.2.2}/ibridges.egg-info/top_level.txt +0 -0
  92. {ibridges-0.2.0 → ibridges-0.2.2}/pyproject.toml +0 -0
  93. {ibridges-0.2.0 → ibridges-0.2.2}/setup.cfg +0 -0
  94. {ibridges-0.2.0 → ibridges-0.2.2}/tests/test_irodspath.py +0 -0
  95. {ibridges-0.2.0 → ibridges-0.2.2}/tests/testdata/bunny.txt +0 -0
  96. {ibridges-0.2.0 → ibridges-0.2.2}/tests/testdata/subfolder/sun.csv +0 -0
  97. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/00-FirstSteps.ipynb +0 -0
  98. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/01-Setup-and-connect.ipynb +0 -0
  99. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/04-Metadata.ipynb +0 -0
  100. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/05-Data-Sharing.ipynb +0 -0
  101. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/example_rules/example.r +0 -0
  102. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/DataObject1.png +0 -0
  103. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/DataObject2.png +0 -0
  104. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/DataObject3.png +0 -0
  105. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/DataObject4.png +0 -0
  106. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/DataObject5.png +0 -0
  107. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/DataObject6.png +0 -0
  108. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/Save_json.png +0 -0
  109. {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/Yoda_environment.png +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: ibridges
3
- Version: 0.2.0
3
+ Version: 0.2.2
4
4
  Summary: Package for accessing data and metadata on iRods servers.
5
5
  Author-email: Christine Staiger <c.staiger@uu.nl>
6
6
  License: MIT License
@@ -55,13 +55,13 @@ Requires-Dist: sphinx_inline_tabs; extra == "test"
55
55
  Requires-Dist: sphinx_copybutton; extra == "test"
56
56
  Requires-Dist: ruff; extra == "test"
57
57
 
58
- # iBridges
58
+ # iBridges <img src="docs/logo.png" width="150" align="right">
59
+ iBridges is library for scientific programmers who are working with data in iRODS. We provide a wrapper around the [python-irodsclient](https://pypi.org/project/python-irodsclient/) to facilitate easy interaction with the iRODS server. iBridges is currently still in very active development.
60
+
59
61
  [![PyPI version](https://badge.fury.io/py/ibridges.svg)](https://badge.fury.io/py/ibridges)
60
62
  [![](https://github.com/UtrechtUniversity/iBridges/actions/workflows/integration-tests-irods.yml/badge.svg?branch=develop)](https://github.com/UtrechtUniversity/iBridges/actions/workflows/integration-tests-irods.yml) [![](https://github.com/UtrechtUniversity/iBridges/actions/workflows/main.yml/badge.svg?branch=develop)](https://github.com/UtrechtUniversity/iBridges/actions/workflows/main.yml)
61
63
  [![](https://github.com/UtrechtUniversity/iBridges/actions/workflows/integration-tests-yoda.yml/badge.svg)](https://github.com/UtrechtUniversity/iBridges/actions/workflows/integration-tests-yoda.yml) ![](https://readthedocs.org/projects/ibridges/badge/?version=latest&style=flat-default)
62
64
 
63
- iBridges is library for scientific programmers who are working with data in iRODS. We provide a wrapper around the [python-irodsclient](https://pypi.org/project/python-irodsclient/) to facilitate easy interaction with the iRODS server. iBridges is currently still in very active development.
64
-
65
65
  ## Highlights
66
66
 
67
67
  - Works on Windows, Mac OS and Linux
@@ -70,7 +70,7 @@ iBridges is library for scientific programmers who are working with data in iROD
70
70
  - **Interactive connection** to your iRods server.
71
71
  - **Upload** and **Download** your data.
72
72
  - Manipulate the **metadata** on the iRODS server.
73
- - **Synchronize** your data between your local computer and the iRODS server.
73
+ - **Synchronise** your data between your local computer and the iRODS server.
74
74
  - Create and manipulate **Tickets** to temporarily grant access to outside users.
75
75
  - **Search** through all metadata for your dataset or collection.
76
76
  - Small number of dependencies (`python-irodsclient` and `tqdm`)
@@ -104,6 +104,10 @@ If you want to install the unstable version to test out new features, you can in
104
104
  pip install git+https://github.com/UtrechtUniversity/iBridges.git@develop
105
105
  ```
106
106
 
107
+ ## Configuration
108
+
109
+ You will need to have a so-called `irods_environment.json`. Please follow the [documentation](https://ibridges.readthedocs.io/en/latest/cli.html) how to set it up.
110
+
107
111
  ## Usage
108
112
 
109
113
  Below are some basic examples of the features in iBridges.
@@ -127,7 +131,8 @@ download(session, "/irods/path", "/other/local/path")
127
131
  ```
128
132
 
129
133
  ## Command line interface
130
- To simply upload or download data you do not need to write full python program, we offer a command line interface
134
+
135
+ To simply upload or download data you do not need to write full python program, we offer a command line interface (see also [the documentation](https://ibridges.readthedocs.io/en/latest/)).
131
136
 
132
137
  - Establish a connection
133
138
 
@@ -142,45 +147,55 @@ To simply upload or download data you do not need to write full python program,
142
147
  ibridges list
143
148
 
144
149
  # list a different collection in your home
145
- ibridges list irods:~/<collection>
150
+ ibridges list "irods:~/<collection>"
146
151
 
147
152
  # list a collection on a different path than your home
148
- ibridges list irods:<full_irods_path>
153
+
154
+ ibridges list "irods:/full/irods/path"
149
155
  ```
150
156
 
151
157
  - Upload data
152
158
 
153
159
  ```bash
154
- ibridges upload my_file.json irods:~/some_collection
160
+ ibridges upload my_file.json "irods:~/some_collection"
155
161
  ```
156
162
 
157
163
  - Download data
158
164
 
159
165
  ```bash
160
- ibridges download irods:~/some_collection/some_object download_dir
166
+ ibridges download "irods:~/some_collection/some_object" download_dir
161
167
  ```
162
168
 
163
169
  - Synchronise data
164
170
 
165
171
  ```bash
166
- ibridges sync some_local_directory irods:~/some_collection
167
- ```
172
+
173
+ ibridges sync some_local_directory "irods:~/some_collection"
174
+
168
175
 
169
176
  ## Tutorials
170
177
  ### Documentation
171
178
  - **[ReadTheDocs](https://ibridges.readthedocs.io/en/latest/)**
172
179
 
173
180
  ### Guides
174
- - [Manual with examples](tutorials/QuickStart.ipynb)
175
- - [Tutorial for iRODS paths](tutorials/01-iRODS_paths.ipynb)
176
- - [Data synchronisation manual](tutorials/Data_sync.ipynb)
181
+
182
+ - [Manual with examples](tutorials/00-FirstSteps.ipynb)
177
183
 
178
184
  ### Beginners tutorials
179
185
  - [Setup client configuration](tutorials/01-Setup-and-connect.ipynb)
180
- - [iRODS Paths](tutorials/01-iRODS_paths.ipynb)
181
- - [Working with data](tutorials/02-Working-with-data.ipynb)
186
+ - [iRODS Paths](tutorials/02-iRODS-paths.ipynb)
187
+ - [Working with data](tutorials/03-Working-with-data.ipynb)
182
188
  - [Metadata](tutorials/04-Metadata.ipynb)
183
189
  - [Sharing data](tutorials/05-Data-Sharing.ipynb)
190
+ - [Synchronising data](tutorials/06-Data-sync.ipynb)
191
+
192
+
193
+ ## Do I have to choose between the *python-irodsclient* and *iBridges*?
194
+ No. *iBridges* itself is based on the *python-irodsclient*. When you connect to iRODS a *python-irodsclient* `irods.iRODSSession` is automatically created with which you can access all functionality of the *python-irodsclient*.
195
+
196
+ <p align="center">
197
+ <img src="docs/ibridges-prc.png" alt="ibridges and prc" style="width:75%;"/>
198
+ </p>
184
199
 
185
200
  ## Authors
186
201
 
@@ -1,10 +1,10 @@
1
- # iBridges
1
+ # iBridges <img src="docs/logo.png" width="150" align="right">
2
+ iBridges is library for scientific programmers who are working with data in iRODS. We provide a wrapper around the [python-irodsclient](https://pypi.org/project/python-irodsclient/) to facilitate easy interaction with the iRODS server. iBridges is currently still in very active development.
3
+
2
4
  [![PyPI version](https://badge.fury.io/py/ibridges.svg)](https://badge.fury.io/py/ibridges)
3
5
  [![](https://github.com/UtrechtUniversity/iBridges/actions/workflows/integration-tests-irods.yml/badge.svg?branch=develop)](https://github.com/UtrechtUniversity/iBridges/actions/workflows/integration-tests-irods.yml) [![](https://github.com/UtrechtUniversity/iBridges/actions/workflows/main.yml/badge.svg?branch=develop)](https://github.com/UtrechtUniversity/iBridges/actions/workflows/main.yml)
4
6
  [![](https://github.com/UtrechtUniversity/iBridges/actions/workflows/integration-tests-yoda.yml/badge.svg)](https://github.com/UtrechtUniversity/iBridges/actions/workflows/integration-tests-yoda.yml) ![](https://readthedocs.org/projects/ibridges/badge/?version=latest&style=flat-default)
5
7
 
6
- iBridges is library for scientific programmers who are working with data in iRODS. We provide a wrapper around the [python-irodsclient](https://pypi.org/project/python-irodsclient/) to facilitate easy interaction with the iRODS server. iBridges is currently still in very active development.
7
-
8
8
  ## Highlights
9
9
 
10
10
  - Works on Windows, Mac OS and Linux
@@ -13,7 +13,7 @@ iBridges is library for scientific programmers who are working with data in iROD
13
13
  - **Interactive connection** to your iRods server.
14
14
  - **Upload** and **Download** your data.
15
15
  - Manipulate the **metadata** on the iRODS server.
16
- - **Synchronize** your data between your local computer and the iRODS server.
16
+ - **Synchronise** your data between your local computer and the iRODS server.
17
17
  - Create and manipulate **Tickets** to temporarily grant access to outside users.
18
18
  - **Search** through all metadata for your dataset or collection.
19
19
  - Small number of dependencies (`python-irodsclient` and `tqdm`)
@@ -47,6 +47,10 @@ If you want to install the unstable version to test out new features, you can in
47
47
  pip install git+https://github.com/UtrechtUniversity/iBridges.git@develop
48
48
  ```
49
49
 
50
+ ## Configuration
51
+
52
+ You will need to have a so-called `irods_environment.json`. Please follow the [documentation](https://ibridges.readthedocs.io/en/latest/cli.html) how to set it up.
53
+
50
54
  ## Usage
51
55
 
52
56
  Below are some basic examples of the features in iBridges.
@@ -70,7 +74,8 @@ download(session, "/irods/path", "/other/local/path")
70
74
  ```
71
75
 
72
76
  ## Command line interface
73
- To simply upload or download data you do not need to write full python program, we offer a command line interface
77
+
78
+ To simply upload or download data you do not need to write full python program, we offer a command line interface (see also [the documentation](https://ibridges.readthedocs.io/en/latest/)).
74
79
 
75
80
  - Establish a connection
76
81
 
@@ -85,45 +90,55 @@ To simply upload or download data you do not need to write full python program,
85
90
  ibridges list
86
91
 
87
92
  # list a different collection in your home
88
- ibridges list irods:~/<collection>
93
+ ibridges list "irods:~/<collection>"
89
94
 
90
95
  # list a collection on a different path than your home
91
- ibridges list irods:<full_irods_path>
96
+
97
+ ibridges list "irods:/full/irods/path"
92
98
  ```
93
99
 
94
100
  - Upload data
95
101
 
96
102
  ```bash
97
- ibridges upload my_file.json irods:~/some_collection
103
+ ibridges upload my_file.json "irods:~/some_collection"
98
104
  ```
99
105
 
100
106
  - Download data
101
107
 
102
108
  ```bash
103
- ibridges download irods:~/some_collection/some_object download_dir
109
+ ibridges download "irods:~/some_collection/some_object" download_dir
104
110
  ```
105
111
 
106
112
  - Synchronise data
107
113
 
108
114
  ```bash
109
- ibridges sync some_local_directory irods:~/some_collection
110
- ```
115
+
116
+ ibridges sync some_local_directory "irods:~/some_collection"
117
+
111
118
 
112
119
  ## Tutorials
113
120
  ### Documentation
114
121
  - **[ReadTheDocs](https://ibridges.readthedocs.io/en/latest/)**
115
122
 
116
123
  ### Guides
117
- - [Manual with examples](tutorials/QuickStart.ipynb)
118
- - [Tutorial for iRODS paths](tutorials/01-iRODS_paths.ipynb)
119
- - [Data synchronisation manual](tutorials/Data_sync.ipynb)
124
+
125
+ - [Manual with examples](tutorials/00-FirstSteps.ipynb)
120
126
 
121
127
  ### Beginners tutorials
122
128
  - [Setup client configuration](tutorials/01-Setup-and-connect.ipynb)
123
- - [iRODS Paths](tutorials/01-iRODS_paths.ipynb)
124
- - [Working with data](tutorials/02-Working-with-data.ipynb)
129
+ - [iRODS Paths](tutorials/02-iRODS-paths.ipynb)
130
+ - [Working with data](tutorials/03-Working-with-data.ipynb)
125
131
  - [Metadata](tutorials/04-Metadata.ipynb)
126
132
  - [Sharing data](tutorials/05-Data-Sharing.ipynb)
133
+ - [Synchronising data](tutorials/06-Data-sync.ipynb)
134
+
135
+
136
+ ## Do I have to choose between the *python-irodsclient* and *iBridges*?
137
+ No. *iBridges* itself is based on the *python-irodsclient*. When you connect to iRODS a *python-irodsclient* `irods.iRODSSession` is automatically created with which you can access all functionality of the *python-irodsclient*.
138
+
139
+ <p align="center">
140
+ <img src="docs/ibridges-prc.png" alt="ibridges and prc" style="width:75%;"/>
141
+ </p>
127
142
 
128
143
  ## Authors
129
144
 
Binary file
Binary file
@@ -2,7 +2,7 @@ Command Line Interface
2
2
  ======================
3
3
 
4
4
  iBridges also has a Command Line Interface (CLI). The CLI provides an even more simplified
5
- interface for uploading, downloading and synchronizing. It does not provide (nor intends to) all the features
5
+ interface for uploading, downloading and synchronising. It does not provide (nor intends to) all the features
6
6
  that are available in the iBridges API library. It is mainly there for users that are not familiar with Python
7
7
  and still want to download or upload their data using the interface, or if you need a simple iBridges operation
8
8
  in your shell script without having to create a new python script.
@@ -15,22 +15,22 @@ in your shell script without having to create a new python script.
15
15
  Setting up
16
16
  ----------
17
17
 
18
- As with the ibridges API, you will need to create an irods environment file. It is the easiest if you put this file
18
+ As with the ibridges API, you will need to create an `irods_environment.json`. It is the easiest if you put this file
19
19
  in the default location: `~/.irods/irods_environment.json`, because then it will be automatically detected. However,
20
20
  if you have it in another location for some reason (let's say you have multiple environments), then you can tell the
21
- ibridges cli where it is:
21
+ ibridges CLI where it is:
22
22
 
23
23
  .. code:: shell
24
24
 
25
- ibridges init some_irods_env_file.json
25
+ ibridges init path/to/some_irods_env_file.json
26
26
 
27
27
  This will most likely ask for your password. After filling this in, iBridges will cache your password, so that
28
28
  you will not have to type it in every time you use an iBridges operation. This is especially useful if you want
29
29
  to execute scripts that run in the background. Note that the time your cached password is valid depends on the
30
30
  administrator settings of your iRODS server.
31
31
 
32
- iBridges stores the location of your irods environment file in `~/.ibridges/ibridges_cli.json`. You can safely delete
33
- this file if somehow it gets corrupted. If you have the irods environment in the default location, it can still be
32
+ iBridges stores the location of your iRODS environment file in `~/.ibridges/ibridges_cli.json`. You can safely delete
33
+ this file if somehow it gets corrupted. If you have the iRODS environment in the default location, it can still be
34
34
  useful to cache the password so that the next commands do not ask for your password anymore:
35
35
 
36
36
  .. code:: shell
@@ -61,8 +61,24 @@ Please try to avoid spaces in collection and data object names! If you really ne
61
61
 
62
62
  .. note::
63
63
 
64
- Note that all data objects and collections on the iRODS server are always preceded with "irods:". This is
65
- done to distinguish local and remote files.
64
+
65
+ Note that all data objects and collections on the iRODS server are always preceded with "irods:". This is done to distinguish local and remote files.
66
+
67
+
68
+ Creating a new collection
69
+ --------------------
70
+
71
+ To create a new collection in you iRODS home simply type:
72
+
73
+ .. code:: shell
74
+
75
+ ibridges mkcoll "irods:~/new_collection"
76
+
77
+ Or:
78
+
79
+ .. code:: shell
80
+
81
+ ibridges mkcoll "irods:/full/path/to/new_collection"
66
82
 
67
83
 
68
84
  Downloading data
@@ -74,7 +90,7 @@ The basic command to download a data object or collection is `ibridges download`
74
90
 
75
91
  ibridges download "irods:~/some_collection/some_object" download_dir
76
92
 
77
- The download_dir aregument is optional. If it is left out, it will be put in the current working directory.
93
+ The download_dir argument is optional. If it is left out, it will be put in the current working directory.
78
94
 
79
95
  There are two more options: `--overwrite` to allow the download command to overwrite a local file and
80
96
  `--resource` to set the resource to download the data from. See `ibridges download --help` for more details.
@@ -95,11 +111,11 @@ The command to upload files and directories to an iRODS server is similar to the
95
111
  destination collection or data object.
96
112
 
97
113
 
98
- Synchronizing data
114
+ Synchronising data
99
115
  ------------------
100
116
 
101
- In some cases, instead of downloading/uploading your data, you might want to synchronize data between local
102
- folders and collections. The `sync` command does this synchronization and only transfers files/directories
117
+ In some cases, instead of downloading/uploading your data, you might want to synchronise data between local
118
+ folders and collections. The `sync` command does this synchronisation and only transfers files/directories
103
119
  that are missing or have a different checksum (content).
104
120
 
105
121
  .. code:: shell
@@ -0,0 +1,112 @@
1
+ Data Transfers
2
+ ==============
3
+
4
+ Upload
5
+ ------
6
+ To upload data from your local file system to iRODS simply use the `upload` function.
7
+ To determine paths, we recommend to use `pathlib.Path` for local paths and `ibridges.IrodsPath` for iRODS paths.
8
+
9
+ In the example below we transfer a file or a folder to a new collection *new_coll*. The new collection will be created on the fly.
10
+
11
+ .. note::
12
+
13
+ If you transfer data to a destination folder or collection for which the path does not already exist, the missing folders or collections will NOT be created and the command will fail with the respective error message.
14
+ Please use `IrodsPath.create_collection` or `pathlib.Path.mkdir(parents=True, exist_ok=True)` to create the destination before the upload or download.
15
+
16
+ .. code-block:: python
17
+
18
+ from ibridges import upload
19
+ from ibridges import IrodsPath
20
+ from pathlib import Path
21
+
22
+ local_path = Path("/path/to the/data/to/upload")
23
+ irods_path = IrodsPath(session, '~', 'new_coll')
24
+ upload(session, local_path, irods_path)
25
+
26
+ The new collection will be created on the fly. Please note, that this is not true for new nested collections. Please note that the automatic creation of new collections does not work when they are nested. I.e.
27
+ you will receive the following exception.
28
+
29
+ .. code-block:: python
30
+
31
+ irods_path = IrodsPath(session, '~', 'new_coll', 'new_subcoll', 'new')
32
+
33
+ CAT_UNKNOWN_COLLECTION: collection '/nluu12p/home/research-test-christine/new_coll1/new_subcoll' is unknown
34
+
35
+
36
+ The output of a successful upload is:
37
+
38
+ .. code-block:: python
39
+
40
+ {
41
+ 'create_dir': set(),
42
+ 'create_collection': set(),
43
+ 'upload': [(PosixPath('/Users/christine/demofile.txt'),
44
+ IrodsPath(/, <zone_name>, home, <user or group>, new_coll1))],
45
+ 'download': [],
46
+ 'resc_name': '',
47
+ 'options': None
48
+ }
49
+
50
+ .. note::
51
+
52
+ All of the data transfer functions return a python dictionary summarising the changes.
53
+ With the option `dry_run=True` you can retrieve them before the actual data transfer.
54
+
55
+ The dictionary above summarises the changes. In case of an upload, it will list the created collections, in case you upload a directory with subdirectories; and it will list which file was uploaded to which iRODS path.
56
+
57
+ If you want to explicitly overwrite existing data, you need to set the `overwrite=True` parameter. **By default no existing data will be overwritten.**
58
+
59
+
60
+ Download
61
+ --------
62
+
63
+ The download function works similar to the upload function. Simply define your iRODS path you would like to download and a local destination path.
64
+
65
+ .. code-block:: python
66
+
67
+ from ibridges import download
68
+ from ibridges import IrodsPath
69
+ from pathlib import Path
70
+
71
+ local_path = Path("/destination/location/for/the/data")
72
+ irods_path = IrodsPath(session, '~', 'new_coll')
73
+ download(session, irods_path, local_path)
74
+
75
+
76
+ Again you will receive a dictionary with changes, which you can also retrieve beforehand with the option `dry_run=True`.
77
+
78
+ As above, existing local data will not be overwritten. Please use the option `overwrite=True` if you want to overwrite your local data.
79
+
80
+
81
+ Synchronisation
82
+ ---------------
83
+ Please see: :doc:`Synchronisation <sync>`.
84
+
85
+ Streaming data objects
86
+ ----------------------
87
+
88
+ With the `python-irodsclient` which `iBridges` is built on, we can open the file inside of a data object as a stream and process the content without downloading the data.
89
+ That works without any problems for textual data.
90
+
91
+ .. code-block:: python
92
+
93
+ from ibrigdes import IrodsPath
94
+
95
+ obj_path = IrodsPath(session, "path", "to", "object")
96
+
97
+ content = ''
98
+ with obj_path.dataobject.open('r') as stream:
99
+ content = stream.read().decode()
100
+
101
+
102
+ Some python libraries allow to be instantiated directly from such a stream. This is supported by e.g. `pandas`, `polars` and `whisper`.
103
+
104
+ .. code-block:: python
105
+
106
+ from io import StringIO
107
+ imort pandas as pd
108
+
109
+ df = None
110
+ with obj_path.dataobject.open('r') as stream:
111
+ df = pd.read_csv(StringIO(stream.read().decode()))
112
+ print(df)
@@ -28,8 +28,13 @@ We provide extensive tutorials on how to work with data in iRODS. Please consult
28
28
  quickstart
29
29
 
30
30
  install
31
- sync
31
+
32
+ session
32
33
  ipath
34
+ data_transfers
35
+ sync
36
+ metadata
37
+ irods_search
33
38
  cli
34
39
  faq
35
40
  api/main
@@ -1,7 +1,8 @@
1
1
  iRODS paths
2
2
  ===========
3
3
 
4
- iRODS paths follow a certain pattern just like the local paths of different operatings systems.
4
+
5
+ iRODS paths follow a certain pattern just like the local paths of different operating systems.
5
6
  To allow for similar path operations on iRODS paths, rather than using strings, `iBridges` offers an class with similar `pathlib`-like functionality.
6
7
 
7
8
  **Why prefer `pathlib.path` and `IrodsPath` over strings?**
@@ -0,0 +1,59 @@
1
+ iRODS Search
2
+ ============
3
+
4
+ `iBridges` offers an easy way to search for data. You can pass a combination of path, metadata keys and values and checksum. The output will be a list of dictionaries, one dictionary for each found item, which contain information where to find the item on the iRODS server.
5
+
6
+
7
+ Search data by Path
8
+ --------
9
+
10
+ In the example below we search for a data object by ites path.
11
+ The path can be an IrodsPath or a string:
12
+
13
+ .. code-block:: python
14
+
15
+ from ibridges import search_data
16
+ search_data(session, path=IrodsPath(session, "dataobj_name"))
17
+
18
+ The result is a list of dictionaries.
19
+
20
+ .. code-block:: python
21
+
22
+ [{'COLL_NAME': '/you/irods/home',
23
+ 'DATA_NAME': 'dataobj_name',
24
+ 'D_DATA_CHECKSUM': 'sha2:XGiECYZOtUfP9lnCGyZaBBkBGLaJJw1p6eoc0GxLeKU='}]
25
+
26
+ For collections the dictionary only contains the entry `COLL_NAME`.
27
+
28
+ To find all subcollections and dataobjects in a collection use the `%` as wildcard:
29
+
30
+ .. code-block:: python
31
+
32
+ search_data(session, path=IrodsPath(session, "subcoll/%"))
33
+
34
+
35
+ Search data by metadata
36
+ --------------------
37
+
38
+ We can also use the metadata we generated above to search for data.
39
+ We need to create a python dictionary which contains the metadata keys ad their values. The values are again optional.
40
+
41
+ .. code-block:: python
42
+
43
+ key_vals = {'key': 'value'}
44
+ search_data(session, key_vals = key_vals)
45
+ key_vals = {'key': ''}
46
+ search_data(session, key_vals = key_vals)
47
+
48
+ Use the `%` as a wild card again to match any combination of characters.
49
+
50
+
51
+ Search data by checksum
52
+ ------
53
+
54
+ The search by checksum is handy to find duplicate data. in *iBridges* we always create checksums when data is uploaded. The checksum is unique for the file we uploaded and can be used to identify the files. E.g. when you uploaded the same file twice, once as `file.txt` and once as `file1.txt` you can find the two duplicates by their checksum:
55
+
56
+ .. code-block:: python
57
+
58
+ obj = IrodsPath(session, "~", "dataobj_name").dataobject
59
+ search_data(session, checksum = obj.checksum)
@@ -0,0 +1,120 @@
1
+ Metadata
2
+ =========
3
+
4
+ iRODS offers metadata as key, value, units triplets. The type is always string. Below we show how to create an `ibridges.Metadata` object from a dataobject or collection.
5
+
6
+ The Metadata object
7
+ --------------------
8
+
9
+ .. code-block:: python
10
+
11
+ from ibridges.interactive import interactive_auth
12
+ from ibridges import Metadata
13
+ from ibridges import IrodsPath
14
+
15
+ session = interactive_auth()
16
+ obj = IrodsPath(session, "~", "dataobj_name").dataobject
17
+ meta = Metadata(obj)
18
+
19
+ With the object `meta` we can now access and manipulate the metadata of the data object.
20
+
21
+ Add, set and delete metadata
22
+ ----------------------------
23
+
24
+ - Add metadata to a collection or data object:
25
+
26
+ .. code-block:: python
27
+
28
+ meta.add('NewKey', 'NewValue', 'NewUnit')
29
+ print(meta)
30
+
31
+ - You always need to provide a key and a value, the unit is optional and can be left out.
32
+
33
+ - You can have several metadata entries with the same key but different values and units, i.e. metadata keys are not unique in iRODS but the combination of key, value and unit must be unique.
34
+
35
+ - Set metadata:
36
+
37
+ .. code-block:: python
38
+
39
+ meta.set('ExistingKey', 'Value', 'Unit')
40
+
41
+ The `set` function is a special function. It sets all metadata items with the key `ExistingKey` to the value and unit. It is the implementation of the *icommands* `imeta set`.
42
+
43
+ - Delete metadata of a collection or data object:
44
+
45
+ .. code-block:: python
46
+
47
+ meta.delete('NewKey', 'NewValue', 'NewUnit')
48
+
49
+ Export Metadata
50
+ ---------------
51
+
52
+ The function `Metadata.to_dict` will provide you with a python dictionary containing the user-defined metadata:
53
+
54
+ .. code-block:: python
55
+
56
+ meta.to_dict()
57
+
58
+ .. code-block:: python
59
+
60
+ {
61
+ 'name': '<obj_name>',
62
+ 'irods_id': 24490075,
63
+ 'checksum': 'sha2:XGiECYZOtUfP9lnCGyZaBBkBGLaJJw1p6eoc0GxLeKU=',
64
+ 'metadata': [('Key', 'Value', 'Unit'), ('Key', 'Value', 'Unit1')]
65
+ }
66
+
67
+ The dictionary contains the name of the data object or collection and its iRODS identifier.
68
+ For data objects the checksum is also listed under the key `checksum`. The checksum is not calculated, but extracted from the iRODS database.
69
+
70
+ The user-defined metadata can be accessed with the key `metadata`.
71
+
72
+ Export metadata of all member of a collection
73
+ ---------------------------------------------
74
+
75
+ Above we showed how to export the metadata of one collection or data object. In case you want to export
76
+ metadata of all memebers of a collection you can use:
77
+
78
+ .. code-block:: python
79
+
80
+ from ibridges.export_metadata import export_metadata_to_dict
81
+
82
+ coll = IrodsPath(session, <coll_path>).collection
83
+ meta = Metadata(coll)
84
+ export_metadata_to_dict(meta, session)
85
+
86
+ The resulting dictionary is built like above and contains two additional keys `subcollections` and `dataobjects`.
87
+
88
+ Under `subcollections` we will find the extracted metadata of all subcollection and under `dataobjects` the extracted metadata for all data objects.
89
+ Those extracted metadata are also represented by dictionaries as shown above, they only contain one extra key `rel_path` which denotes the path relative to the collection which we gave as input to the function.
90
+
91
+ .. code-block:: python
92
+
93
+ {
94
+ 'ibridges_metadata_version': 1.0,
95
+ 'name': 'Demo',
96
+ 'irods_id': 24484787,
97
+ 'metadata': [('Key', 'very_important', None)],
98
+ 'subcollections': [
99
+ {
100
+ 'name': 'subDemo',
101
+ 'irods_id': 24490064,
102
+ 'rel_path': 'Demo',
103
+ 'metadata': []
104
+ },
105
+ {
106
+ 'name': 'my_books',
107
+ 'irods_id': 24502538,
108
+ 'rel_path': 'my_books',
109
+ 'metadata': []}
110
+ ],
111
+ 'dataobjects': [
112
+ {
113
+ 'name': 'AliceInWonderland.txt',
114
+ 'irods_id': 24484789,
115
+ 'checksum': 'sha2:TQzOrHuw1qRQ6zh8xm5GEuVKGjs22STdgQCdezv8LY4=',
116
+ 'rel_path': 'my_books/AliceInWonderland.txt',
117
+ 'metadata': [('author', 'Lewis Carroll', None)]
118
+ }
119
+ ]
120
+ }
@@ -12,7 +12,7 @@ iBridges requires Python version 3.8 or higher. You can install iBridges with pi
12
12
  pip install ibridges
13
13
 
14
14
 
15
- Getting your irods environment file
15
+ Getting your iRODS environment file
16
16
  -----------------------------------
17
17
 
18
18
  To connect to an iRods server you need an iRods environment file (`irods_nevironment.json`).
@@ -43,7 +43,7 @@ but if needed (if you need access to more than one iRods instance for example) y
43
43
  replace that path in this quick start guide with the path you have chosen.
44
44
 
45
45
 
46
- Connecting to your iRods server
46
+ Connecting to your iRODS server
47
47
  -------------------------------
48
48
 
49
49
  To connect to your iRods server, we will create a session. The session is the central object in the iBridges library;
@@ -74,7 +74,7 @@ This upload function can upload both directories (collections in iRods) and file
74
74
  Add metadata
75
75
  ------------
76
76
 
77
- One of the powerful features of iRods is its ability to store metadata with your data in a consistent manner.
77
+ One of the powerful features of iRODS is its ability to store metadata with your data in a consistent manner.
78
78
  Let's add some metadata to a collection or data object:
79
79
 
80
80
  .. code:: python