ibridges 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ibridges-0.2.0 → ibridges-0.2.2}/PKG-INFO +32 -17
- {ibridges-0.2.0 → ibridges-0.2.2}/README.md +31 -16
- ibridges-0.2.2/docs/ibridges-prc.png +0 -0
- ibridges-0.2.2/docs/logo.png +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/cli.rst +28 -12
- ibridges-0.2.2/docs/source/data_transfers.rst +112 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/index.rst +6 -1
- {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/ipath.rst +2 -1
- ibridges-0.2.2/docs/source/irods_search.rst +59 -0
- ibridges-0.2.2/docs/source/metadata.rst +120 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/quickstart.rst +3 -3
- ibridges-0.2.2/docs/source/session.rst +94 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/sync.rst +10 -6
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/_version.py +2 -2
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/data_operations.py +31 -7
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges.egg-info/PKG-INFO +32 -17
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges.egg-info/SOURCES.txt +6 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/02-iRODS-paths.ipynb +25 -134
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/03-Working-with-data.ipynb +25 -106
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/06-Data-sync.ipynb +5 -5
- {ibridges-0.2.0 → ibridges-0.2.2}/.dockerignore +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/.github/dependabot.yml +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/.github/workflows/integration-tests-irods.yml +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/.github/workflows/integration-tests-yoda.yml +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/.github/workflows/main.yml +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/.github/workflows/pypi_release.yml +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/.gitignore +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/.readthedocs.yaml +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/LICENSE +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/.gitattributes +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/README.md +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_catalog/Dockerfile +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_catalog/init-user-db.sh +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_catalog_provider/Dockerfile +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_catalog_provider/entrypoint.sh +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_catalog_provider/setup-4.3.1.input +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_catalog_provider/setup-4.3.2.input +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/Dockerfile +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/entrypoint.sh +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/environments/plain-irods/config.toml +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/environments/plain-irods/irods_environment.json +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/environments/plain-irods/irods_environment_testuser.json +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/environments/yoda/config.toml +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/environments/yoda/irods_environment.json +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/testdata/beach.rtf +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/testdata/bunny.rtf +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/testdata/example.r +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/testdata/more_data/polarbear.txt +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/testdata/plant.rtf +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/testdata/sun.rtf +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/conftest.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_cli.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_data_ops.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_meta.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_move.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_permissions.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_resources.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_rules.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_session.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_sync.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker/irods_client/tests/test_ticket.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docker-compose.yml +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docs/Makefile +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docs/make.bat +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docs/requirements.txt +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/api/full_reference.rst +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/api/main.rst +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/api/user_reference.rst +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/conf.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/faq.rst +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/docs/source/install.rst +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/__init__.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/__main__.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/export_metadata.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/icat_columns.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/interactive.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/meta.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/path.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/permissions.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/resources.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/rules.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/search.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/session.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/tickets.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges/util.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges.egg-info/dependency_links.txt +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges.egg-info/entry_points.txt +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges.egg-info/requires.txt +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/ibridges.egg-info/top_level.txt +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/pyproject.toml +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/setup.cfg +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tests/test_irodspath.py +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tests/testdata/bunny.txt +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tests/testdata/subfolder/sun.csv +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/00-FirstSteps.ipynb +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/01-Setup-and-connect.ipynb +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/04-Metadata.ipynb +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/05-Data-Sharing.ipynb +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/example_rules/example.r +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/DataObject1.png +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/DataObject2.png +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/DataObject3.png +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/DataObject4.png +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/DataObject5.png +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/DataObject6.png +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/Save_json.png +0 -0
- {ibridges-0.2.0 → ibridges-0.2.2}/tutorials/img/Yoda_environment.png +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: ibridges
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Package for accessing data and metadata on iRods servers.
|
|
5
5
|
Author-email: Christine Staiger <c.staiger@uu.nl>
|
|
6
6
|
License: MIT License
|
|
@@ -55,13 +55,13 @@ Requires-Dist: sphinx_inline_tabs; extra == "test"
|
|
|
55
55
|
Requires-Dist: sphinx_copybutton; extra == "test"
|
|
56
56
|
Requires-Dist: ruff; extra == "test"
|
|
57
57
|
|
|
58
|
-
# iBridges
|
|
58
|
+
# iBridges <img src="docs/logo.png" width="150" align="right">
|
|
59
|
+
iBridges is library for scientific programmers who are working with data in iRODS. We provide a wrapper around the [python-irodsclient](https://pypi.org/project/python-irodsclient/) to facilitate easy interaction with the iRODS server. iBridges is currently still in very active development.
|
|
60
|
+
|
|
59
61
|
[](https://badge.fury.io/py/ibridges)
|
|
60
62
|
[](https://github.com/UtrechtUniversity/iBridges/actions/workflows/integration-tests-irods.yml) [](https://github.com/UtrechtUniversity/iBridges/actions/workflows/main.yml)
|
|
61
63
|
[](https://github.com/UtrechtUniversity/iBridges/actions/workflows/integration-tests-yoda.yml) 
|
|
62
64
|
|
|
63
|
-
iBridges is library for scientific programmers who are working with data in iRODS. We provide a wrapper around the [python-irodsclient](https://pypi.org/project/python-irodsclient/) to facilitate easy interaction with the iRODS server. iBridges is currently still in very active development.
|
|
64
|
-
|
|
65
65
|
## Highlights
|
|
66
66
|
|
|
67
67
|
- Works on Windows, Mac OS and Linux
|
|
@@ -70,7 +70,7 @@ iBridges is library for scientific programmers who are working with data in iROD
|
|
|
70
70
|
- **Interactive connection** to your iRods server.
|
|
71
71
|
- **Upload** and **Download** your data.
|
|
72
72
|
- Manipulate the **metadata** on the iRODS server.
|
|
73
|
-
- **
|
|
73
|
+
- **Synchronise** your data between your local computer and the iRODS server.
|
|
74
74
|
- Create and manipulate **Tickets** to temporarily grant access to outside users.
|
|
75
75
|
- **Search** through all metadata for your dataset or collection.
|
|
76
76
|
- Small number of dependencies (`python-irodsclient` and `tqdm`)
|
|
@@ -104,6 +104,10 @@ If you want to install the unstable version to test out new features, you can in
|
|
|
104
104
|
pip install git+https://github.com/UtrechtUniversity/iBridges.git@develop
|
|
105
105
|
```
|
|
106
106
|
|
|
107
|
+
## Configuration
|
|
108
|
+
|
|
109
|
+
You will need to have a so-called `irods_environment.json`. Please follow the [documentation](https://ibridges.readthedocs.io/en/latest/cli.html) how to set it up.
|
|
110
|
+
|
|
107
111
|
## Usage
|
|
108
112
|
|
|
109
113
|
Below are some basic examples of the features in iBridges.
|
|
@@ -127,7 +131,8 @@ download(session, "/irods/path", "/other/local/path")
|
|
|
127
131
|
```
|
|
128
132
|
|
|
129
133
|
## Command line interface
|
|
130
|
-
|
|
134
|
+
|
|
135
|
+
To simply upload or download data you do not need to write full python program, we offer a command line interface (see also [the documentation](https://ibridges.readthedocs.io/en/latest/)).
|
|
131
136
|
|
|
132
137
|
- Establish a connection
|
|
133
138
|
|
|
@@ -142,45 +147,55 @@ To simply upload or download data you do not need to write full python program,
|
|
|
142
147
|
ibridges list
|
|
143
148
|
|
|
144
149
|
# list a different collection in your home
|
|
145
|
-
ibridges list irods:~/<collection>
|
|
150
|
+
ibridges list "irods:~/<collection>"
|
|
146
151
|
|
|
147
152
|
# list a collection on a different path than your home
|
|
148
|
-
|
|
153
|
+
|
|
154
|
+
ibridges list "irods:/full/irods/path"
|
|
149
155
|
```
|
|
150
156
|
|
|
151
157
|
- Upload data
|
|
152
158
|
|
|
153
159
|
```bash
|
|
154
|
-
ibridges upload my_file.json irods:~/some_collection
|
|
160
|
+
ibridges upload my_file.json "irods:~/some_collection"
|
|
155
161
|
```
|
|
156
162
|
|
|
157
163
|
- Download data
|
|
158
164
|
|
|
159
165
|
```bash
|
|
160
|
-
ibridges download irods:~/some_collection/some_object download_dir
|
|
166
|
+
ibridges download "irods:~/some_collection/some_object" download_dir
|
|
161
167
|
```
|
|
162
168
|
|
|
163
169
|
- Synchronise data
|
|
164
170
|
|
|
165
171
|
```bash
|
|
166
|
-
|
|
167
|
-
|
|
172
|
+
|
|
173
|
+
ibridges sync some_local_directory "irods:~/some_collection"
|
|
174
|
+
|
|
168
175
|
|
|
169
176
|
## Tutorials
|
|
170
177
|
### Documentation
|
|
171
178
|
- **[ReadTheDocs](https://ibridges.readthedocs.io/en/latest/)**
|
|
172
179
|
|
|
173
180
|
### Guides
|
|
174
|
-
|
|
175
|
-
- [
|
|
176
|
-
- [Data synchronisation manual](tutorials/Data_sync.ipynb)
|
|
181
|
+
|
|
182
|
+
- [Manual with examples](tutorials/00-FirstSteps.ipynb)
|
|
177
183
|
|
|
178
184
|
### Beginners tutorials
|
|
179
185
|
- [Setup client configuration](tutorials/01-Setup-and-connect.ipynb)
|
|
180
|
-
- [iRODS Paths](tutorials/
|
|
181
|
-
- [Working with data](tutorials/
|
|
186
|
+
- [iRODS Paths](tutorials/02-iRODS-paths.ipynb)
|
|
187
|
+
- [Working with data](tutorials/03-Working-with-data.ipynb)
|
|
182
188
|
- [Metadata](tutorials/04-Metadata.ipynb)
|
|
183
189
|
- [Sharing data](tutorials/05-Data-Sharing.ipynb)
|
|
190
|
+
- [Synchronising data](tutorials/06-Data-sync.ipynb)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
## Do I have to choose between the *python-irodsclient* and *iBridges*?
|
|
194
|
+
No. *iBridges* itself is based on the *python-irodsclient*. When you connect to iRODS a *python-irodsclient* `irods.iRODSSession` is automatically created with which you can access all functionality of the *python-irodsclient*.
|
|
195
|
+
|
|
196
|
+
<p align="center">
|
|
197
|
+
<img src="docs/ibridges-prc.png" alt="ibridges and prc" style="width:75%;"/>
|
|
198
|
+
</p>
|
|
184
199
|
|
|
185
200
|
## Authors
|
|
186
201
|
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
# iBridges
|
|
1
|
+
# iBridges <img src="docs/logo.png" width="150" align="right">
|
|
2
|
+
iBridges is library for scientific programmers who are working with data in iRODS. We provide a wrapper around the [python-irodsclient](https://pypi.org/project/python-irodsclient/) to facilitate easy interaction with the iRODS server. iBridges is currently still in very active development.
|
|
3
|
+
|
|
2
4
|
[](https://badge.fury.io/py/ibridges)
|
|
3
5
|
[](https://github.com/UtrechtUniversity/iBridges/actions/workflows/integration-tests-irods.yml) [](https://github.com/UtrechtUniversity/iBridges/actions/workflows/main.yml)
|
|
4
6
|
[](https://github.com/UtrechtUniversity/iBridges/actions/workflows/integration-tests-yoda.yml) 
|
|
5
7
|
|
|
6
|
-
iBridges is library for scientific programmers who are working with data in iRODS. We provide a wrapper around the [python-irodsclient](https://pypi.org/project/python-irodsclient/) to facilitate easy interaction with the iRODS server. iBridges is currently still in very active development.
|
|
7
|
-
|
|
8
8
|
## Highlights
|
|
9
9
|
|
|
10
10
|
- Works on Windows, Mac OS and Linux
|
|
@@ -13,7 +13,7 @@ iBridges is library for scientific programmers who are working with data in iROD
|
|
|
13
13
|
- **Interactive connection** to your iRods server.
|
|
14
14
|
- **Upload** and **Download** your data.
|
|
15
15
|
- Manipulate the **metadata** on the iRODS server.
|
|
16
|
-
- **
|
|
16
|
+
- **Synchronise** your data between your local computer and the iRODS server.
|
|
17
17
|
- Create and manipulate **Tickets** to temporarily grant access to outside users.
|
|
18
18
|
- **Search** through all metadata for your dataset or collection.
|
|
19
19
|
- Small number of dependencies (`python-irodsclient` and `tqdm`)
|
|
@@ -47,6 +47,10 @@ If you want to install the unstable version to test out new features, you can in
|
|
|
47
47
|
pip install git+https://github.com/UtrechtUniversity/iBridges.git@develop
|
|
48
48
|
```
|
|
49
49
|
|
|
50
|
+
## Configuration
|
|
51
|
+
|
|
52
|
+
You will need to have a so-called `irods_environment.json`. Please follow the [documentation](https://ibridges.readthedocs.io/en/latest/cli.html) how to set it up.
|
|
53
|
+
|
|
50
54
|
## Usage
|
|
51
55
|
|
|
52
56
|
Below are some basic examples of the features in iBridges.
|
|
@@ -70,7 +74,8 @@ download(session, "/irods/path", "/other/local/path")
|
|
|
70
74
|
```
|
|
71
75
|
|
|
72
76
|
## Command line interface
|
|
73
|
-
|
|
77
|
+
|
|
78
|
+
To simply upload or download data you do not need to write full python program, we offer a command line interface (see also [the documentation](https://ibridges.readthedocs.io/en/latest/)).
|
|
74
79
|
|
|
75
80
|
- Establish a connection
|
|
76
81
|
|
|
@@ -85,45 +90,55 @@ To simply upload or download data you do not need to write full python program,
|
|
|
85
90
|
ibridges list
|
|
86
91
|
|
|
87
92
|
# list a different collection in your home
|
|
88
|
-
ibridges list irods:~/<collection>
|
|
93
|
+
ibridges list "irods:~/<collection>"
|
|
89
94
|
|
|
90
95
|
# list a collection on a different path than your home
|
|
91
|
-
|
|
96
|
+
|
|
97
|
+
ibridges list "irods:/full/irods/path"
|
|
92
98
|
```
|
|
93
99
|
|
|
94
100
|
- Upload data
|
|
95
101
|
|
|
96
102
|
```bash
|
|
97
|
-
ibridges upload my_file.json irods:~/some_collection
|
|
103
|
+
ibridges upload my_file.json "irods:~/some_collection"
|
|
98
104
|
```
|
|
99
105
|
|
|
100
106
|
- Download data
|
|
101
107
|
|
|
102
108
|
```bash
|
|
103
|
-
ibridges download irods:~/some_collection/some_object download_dir
|
|
109
|
+
ibridges download "irods:~/some_collection/some_object" download_dir
|
|
104
110
|
```
|
|
105
111
|
|
|
106
112
|
- Synchronise data
|
|
107
113
|
|
|
108
114
|
```bash
|
|
109
|
-
|
|
110
|
-
|
|
115
|
+
|
|
116
|
+
ibridges sync some_local_directory "irods:~/some_collection"
|
|
117
|
+
|
|
111
118
|
|
|
112
119
|
## Tutorials
|
|
113
120
|
### Documentation
|
|
114
121
|
- **[ReadTheDocs](https://ibridges.readthedocs.io/en/latest/)**
|
|
115
122
|
|
|
116
123
|
### Guides
|
|
117
|
-
|
|
118
|
-
- [
|
|
119
|
-
- [Data synchronisation manual](tutorials/Data_sync.ipynb)
|
|
124
|
+
|
|
125
|
+
- [Manual with examples](tutorials/00-FirstSteps.ipynb)
|
|
120
126
|
|
|
121
127
|
### Beginners tutorials
|
|
122
128
|
- [Setup client configuration](tutorials/01-Setup-and-connect.ipynb)
|
|
123
|
-
- [iRODS Paths](tutorials/
|
|
124
|
-
- [Working with data](tutorials/
|
|
129
|
+
- [iRODS Paths](tutorials/02-iRODS-paths.ipynb)
|
|
130
|
+
- [Working with data](tutorials/03-Working-with-data.ipynb)
|
|
125
131
|
- [Metadata](tutorials/04-Metadata.ipynb)
|
|
126
132
|
- [Sharing data](tutorials/05-Data-Sharing.ipynb)
|
|
133
|
+
- [Synchronising data](tutorials/06-Data-sync.ipynb)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
## Do I have to choose between the *python-irodsclient* and *iBridges*?
|
|
137
|
+
No. *iBridges* itself is based on the *python-irodsclient*. When you connect to iRODS a *python-irodsclient* `irods.iRODSSession` is automatically created with which you can access all functionality of the *python-irodsclient*.
|
|
138
|
+
|
|
139
|
+
<p align="center">
|
|
140
|
+
<img src="docs/ibridges-prc.png" alt="ibridges and prc" style="width:75%;"/>
|
|
141
|
+
</p>
|
|
127
142
|
|
|
128
143
|
## Authors
|
|
129
144
|
|
|
Binary file
|
|
Binary file
|
|
@@ -2,7 +2,7 @@ Command Line Interface
|
|
|
2
2
|
======================
|
|
3
3
|
|
|
4
4
|
iBridges also has a Command Line Interface (CLI). The CLI provides an even more simplified
|
|
5
|
-
interface for uploading, downloading and
|
|
5
|
+
interface for uploading, downloading and synchronising. It does not provide (nor intends to) all the features
|
|
6
6
|
that are available in the iBridges API library. It is mainly there for users that are not familiar with Python
|
|
7
7
|
and still want to download or upload their data using the interface, or if you need a simple iBridges operation
|
|
8
8
|
in your shell script without having to create a new python script.
|
|
@@ -15,22 +15,22 @@ in your shell script without having to create a new python script.
|
|
|
15
15
|
Setting up
|
|
16
16
|
----------
|
|
17
17
|
|
|
18
|
-
As with the ibridges API, you will need to create an
|
|
18
|
+
As with the ibridges API, you will need to create an `irods_environment.json`. It is the easiest if you put this file
|
|
19
19
|
in the default location: `~/.irods/irods_environment.json`, because then it will be automatically detected. However,
|
|
20
20
|
if you have it in another location for some reason (let's say you have multiple environments), then you can tell the
|
|
21
|
-
ibridges
|
|
21
|
+
ibridges CLI where it is:
|
|
22
22
|
|
|
23
23
|
.. code:: shell
|
|
24
24
|
|
|
25
|
-
ibridges init some_irods_env_file.json
|
|
25
|
+
ibridges init path/to/some_irods_env_file.json
|
|
26
26
|
|
|
27
27
|
This will most likely ask for your password. After filling this in, iBridges will cache your password, so that
|
|
28
28
|
you will not have to type it in every time you use an iBridges operation. This is especially useful if you want
|
|
29
29
|
to execute scripts that run in the background. Note that the time your cached password is valid depends on the
|
|
30
30
|
administrator settings of your iRODS server.
|
|
31
31
|
|
|
32
|
-
iBridges stores the location of your
|
|
33
|
-
this file if somehow it gets corrupted. If you have the
|
|
32
|
+
iBridges stores the location of your iRODS environment file in `~/.ibridges/ibridges_cli.json`. You can safely delete
|
|
33
|
+
this file if somehow it gets corrupted. If you have the iRODS environment in the default location, it can still be
|
|
34
34
|
useful to cache the password so that the next commands do not ask for your password anymore:
|
|
35
35
|
|
|
36
36
|
.. code:: shell
|
|
@@ -61,8 +61,24 @@ Please try to avoid spaces in collection and data object names! If you really ne
|
|
|
61
61
|
|
|
62
62
|
.. note::
|
|
63
63
|
|
|
64
|
-
|
|
65
|
-
done to distinguish local and remote files.
|
|
64
|
+
|
|
65
|
+
Note that all data objects and collections on the iRODS server are always preceded with "irods:". This is done to distinguish local and remote files.
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
Creating a new collection
|
|
69
|
+
--------------------
|
|
70
|
+
|
|
71
|
+
To create a new collection in you iRODS home simply type:
|
|
72
|
+
|
|
73
|
+
.. code:: shell
|
|
74
|
+
|
|
75
|
+
ibridges mkcoll "irods:~/new_collection"
|
|
76
|
+
|
|
77
|
+
Or:
|
|
78
|
+
|
|
79
|
+
.. code:: shell
|
|
80
|
+
|
|
81
|
+
ibridges mkcoll "irods:/full/path/to/new_collection"
|
|
66
82
|
|
|
67
83
|
|
|
68
84
|
Downloading data
|
|
@@ -74,7 +90,7 @@ The basic command to download a data object or collection is `ibridges download`
|
|
|
74
90
|
|
|
75
91
|
ibridges download "irods:~/some_collection/some_object" download_dir
|
|
76
92
|
|
|
77
|
-
The download_dir
|
|
93
|
+
The download_dir argument is optional. If it is left out, it will be put in the current working directory.
|
|
78
94
|
|
|
79
95
|
There are two more options: `--overwrite` to allow the download command to overwrite a local file and
|
|
80
96
|
`--resource` to set the resource to download the data from. See `ibridges download --help` for more details.
|
|
@@ -95,11 +111,11 @@ The command to upload files and directories to an iRODS server is similar to the
|
|
|
95
111
|
destination collection or data object.
|
|
96
112
|
|
|
97
113
|
|
|
98
|
-
|
|
114
|
+
Synchronising data
|
|
99
115
|
------------------
|
|
100
116
|
|
|
101
|
-
In some cases, instead of downloading/uploading your data, you might want to
|
|
102
|
-
folders and collections. The `sync` command does this
|
|
117
|
+
In some cases, instead of downloading/uploading your data, you might want to synchronise data between local
|
|
118
|
+
folders and collections. The `sync` command does this synchronisation and only transfers files/directories
|
|
103
119
|
that are missing or have a different checksum (content).
|
|
104
120
|
|
|
105
121
|
.. code:: shell
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
Data Transfers
|
|
2
|
+
==============
|
|
3
|
+
|
|
4
|
+
Upload
|
|
5
|
+
------
|
|
6
|
+
To upload data from your local file system to iRODS simply use the `upload` function.
|
|
7
|
+
To determine paths, we recommend to use `pathlib.Path` for local paths and `ibridges.IrodsPath` for iRODS paths.
|
|
8
|
+
|
|
9
|
+
In the example below we transfer a file or a folder to a new collection *new_coll*. The new collection will be created on the fly.
|
|
10
|
+
|
|
11
|
+
.. note::
|
|
12
|
+
|
|
13
|
+
If you transfer data to a destination folder or collection for which the path does not already exist, the missing folders or collections will NOT be created and the command will fail with the respective error message.
|
|
14
|
+
Please use `IrodsPath.create_collection` or `pathlib.Path.mkdir(parents=True, exist_ok=True)` to create the destination before the upload or download.
|
|
15
|
+
|
|
16
|
+
.. code-block:: python
|
|
17
|
+
|
|
18
|
+
from ibridges import upload
|
|
19
|
+
from ibridges import IrodsPath
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
local_path = Path("/path/to the/data/to/upload")
|
|
23
|
+
irods_path = IrodsPath(session, '~', 'new_coll')
|
|
24
|
+
upload(session, local_path, irods_path)
|
|
25
|
+
|
|
26
|
+
The new collection will be created on the fly. Please note, that this is not true for new nested collections. Please note that the automatic creation of new collections does not work when they are nested. I.e.
|
|
27
|
+
you will receive the following exception.
|
|
28
|
+
|
|
29
|
+
.. code-block:: python
|
|
30
|
+
|
|
31
|
+
irods_path = IrodsPath(session, '~', 'new_coll', 'new_subcoll', 'new')
|
|
32
|
+
|
|
33
|
+
CAT_UNKNOWN_COLLECTION: collection '/nluu12p/home/research-test-christine/new_coll1/new_subcoll' is unknown
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
The output of a successful upload is:
|
|
37
|
+
|
|
38
|
+
.. code-block:: python
|
|
39
|
+
|
|
40
|
+
{
|
|
41
|
+
'create_dir': set(),
|
|
42
|
+
'create_collection': set(),
|
|
43
|
+
'upload': [(PosixPath('/Users/christine/demofile.txt'),
|
|
44
|
+
IrodsPath(/, <zone_name>, home, <user or group>, new_coll1))],
|
|
45
|
+
'download': [],
|
|
46
|
+
'resc_name': '',
|
|
47
|
+
'options': None
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
.. note::
|
|
51
|
+
|
|
52
|
+
All of the data transfer functions return a python dictionary summarising the changes.
|
|
53
|
+
With the option `dry_run=True` you can retrieve them before the actual data transfer.
|
|
54
|
+
|
|
55
|
+
The dictionary above summarises the changes. In case of an upload, it will list the created collections, in case you upload a directory with subdirectories; and it will list which file was uploaded to which iRODS path.
|
|
56
|
+
|
|
57
|
+
If you want to explicitly overwrite existing data, you need to set the `overwrite=True` parameter. **By default no existing data will be overwritten.**
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
Download
|
|
61
|
+
--------
|
|
62
|
+
|
|
63
|
+
The download function works similar to the upload function. Simply define your iRODS path you would like to download and a local destination path.
|
|
64
|
+
|
|
65
|
+
.. code-block:: python
|
|
66
|
+
|
|
67
|
+
from ibridges import download
|
|
68
|
+
from ibridges import IrodsPath
|
|
69
|
+
from pathlib import Path
|
|
70
|
+
|
|
71
|
+
local_path = Path("/destination/location/for/the/data")
|
|
72
|
+
irods_path = IrodsPath(session, '~', 'new_coll')
|
|
73
|
+
download(session, irods_path, local_path)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
Again you will receive a dictionary with changes, which you can also retrieve beforehand with the option `dry_run=True`.
|
|
77
|
+
|
|
78
|
+
As above, existing local data will not be overwritten. Please use the option `overwrite=True` if you want to overwrite your local data.
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
Synchronisation
|
|
82
|
+
---------------
|
|
83
|
+
Please see: :doc:`Synchronisation <sync>`.
|
|
84
|
+
|
|
85
|
+
Streaming data objects
|
|
86
|
+
----------------------
|
|
87
|
+
|
|
88
|
+
With the `python-irodsclient` which `iBridges` is built on, we can open the file inside of a data object as a stream and process the content without downloading the data.
|
|
89
|
+
That works without any problems for textual data.
|
|
90
|
+
|
|
91
|
+
.. code-block:: python
|
|
92
|
+
|
|
93
|
+
from ibrigdes import IrodsPath
|
|
94
|
+
|
|
95
|
+
obj_path = IrodsPath(session, "path", "to", "object")
|
|
96
|
+
|
|
97
|
+
content = ''
|
|
98
|
+
with obj_path.dataobject.open('r') as stream:
|
|
99
|
+
content = stream.read().decode()
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
Some python libraries allow to be instantiated directly from such a stream. This is supported by e.g. `pandas`, `polars` and `whisper`.
|
|
103
|
+
|
|
104
|
+
.. code-block:: python
|
|
105
|
+
|
|
106
|
+
from io import StringIO
|
|
107
|
+
imort pandas as pd
|
|
108
|
+
|
|
109
|
+
df = None
|
|
110
|
+
with obj_path.dataobject.open('r') as stream:
|
|
111
|
+
df = pd.read_csv(StringIO(stream.read().decode()))
|
|
112
|
+
print(df)
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
iRODS paths
|
|
2
2
|
===========
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
|
|
5
|
+
iRODS paths follow a certain pattern just like the local paths of different operating systems.
|
|
5
6
|
To allow for similar path operations on iRODS paths, rather than using strings, `iBridges` offers an class with similar `pathlib`-like functionality.
|
|
6
7
|
|
|
7
8
|
**Why prefer `pathlib.path` and `IrodsPath` over strings?**
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
iRODS Search
|
|
2
|
+
============
|
|
3
|
+
|
|
4
|
+
`iBridges` offers an easy way to search for data. You can pass a combination of path, metadata keys and values and checksum. The output will be a list of dictionaries, one dictionary for each found item, which contain information where to find the item on the iRODS server.
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
Search data by Path
|
|
8
|
+
--------
|
|
9
|
+
|
|
10
|
+
In the example below we search for a data object by ites path.
|
|
11
|
+
The path can be an IrodsPath or a string:
|
|
12
|
+
|
|
13
|
+
.. code-block:: python
|
|
14
|
+
|
|
15
|
+
from ibridges import search_data
|
|
16
|
+
search_data(session, path=IrodsPath(session, "dataobj_name"))
|
|
17
|
+
|
|
18
|
+
The result is a list of dictionaries.
|
|
19
|
+
|
|
20
|
+
.. code-block:: python
|
|
21
|
+
|
|
22
|
+
[{'COLL_NAME': '/you/irods/home',
|
|
23
|
+
'DATA_NAME': 'dataobj_name',
|
|
24
|
+
'D_DATA_CHECKSUM': 'sha2:XGiECYZOtUfP9lnCGyZaBBkBGLaJJw1p6eoc0GxLeKU='}]
|
|
25
|
+
|
|
26
|
+
For collections the dictionary only contains the entry `COLL_NAME`.
|
|
27
|
+
|
|
28
|
+
To find all subcollections and dataobjects in a collection use the `%` as wildcard:
|
|
29
|
+
|
|
30
|
+
.. code-block:: python
|
|
31
|
+
|
|
32
|
+
search_data(session, path=IrodsPath(session, "subcoll/%"))
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
Search data by metadata
|
|
36
|
+
--------------------
|
|
37
|
+
|
|
38
|
+
We can also use the metadata we generated above to search for data.
|
|
39
|
+
We need to create a python dictionary which contains the metadata keys ad their values. The values are again optional.
|
|
40
|
+
|
|
41
|
+
.. code-block:: python
|
|
42
|
+
|
|
43
|
+
key_vals = {'key': 'value'}
|
|
44
|
+
search_data(session, key_vals = key_vals)
|
|
45
|
+
key_vals = {'key': ''}
|
|
46
|
+
search_data(session, key_vals = key_vals)
|
|
47
|
+
|
|
48
|
+
Use the `%` as a wild card again to match any combination of characters.
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
Search data by checksum
|
|
52
|
+
------
|
|
53
|
+
|
|
54
|
+
The search by checksum is handy to find duplicate data. in *iBridges* we always create checksums when data is uploaded. The checksum is unique for the file we uploaded and can be used to identify the files. E.g. when you uploaded the same file twice, once as `file.txt` and once as `file1.txt` you can find the two duplicates by their checksum:
|
|
55
|
+
|
|
56
|
+
.. code-block:: python
|
|
57
|
+
|
|
58
|
+
obj = IrodsPath(session, "~", "dataobj_name").dataobject
|
|
59
|
+
search_data(session, checksum = obj.checksum)
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
Metadata
|
|
2
|
+
=========
|
|
3
|
+
|
|
4
|
+
iRODS offers metadata as key, value, units triplets. The type is always string. Below we show how to create an `ibridges.Metadata` object from a dataobject or collection.
|
|
5
|
+
|
|
6
|
+
The Metadata object
|
|
7
|
+
--------------------
|
|
8
|
+
|
|
9
|
+
.. code-block:: python
|
|
10
|
+
|
|
11
|
+
from ibridges.interactive import interactive_auth
|
|
12
|
+
from ibridges import Metadata
|
|
13
|
+
from ibridges import IrodsPath
|
|
14
|
+
|
|
15
|
+
session = interactive_auth()
|
|
16
|
+
obj = IrodsPath(session, "~", "dataobj_name").dataobject
|
|
17
|
+
meta = Metadata(obj)
|
|
18
|
+
|
|
19
|
+
With the object `meta` we can now access and manipulate the metadata of the data object.
|
|
20
|
+
|
|
21
|
+
Add, set and delete metadata
|
|
22
|
+
----------------------------
|
|
23
|
+
|
|
24
|
+
- Add metadata to a collection or data object:
|
|
25
|
+
|
|
26
|
+
.. code-block:: python
|
|
27
|
+
|
|
28
|
+
meta.add('NewKey', 'NewValue', 'NewUnit')
|
|
29
|
+
print(meta)
|
|
30
|
+
|
|
31
|
+
- You always need to provide a key and a value, the unit is optional and can be left out.
|
|
32
|
+
|
|
33
|
+
- You can have several metadata entries with the same key but different values and units, i.e. metadata keys are not unique in iRODS but the combination of key, value and unit must be unique.
|
|
34
|
+
|
|
35
|
+
- Set metadata:
|
|
36
|
+
|
|
37
|
+
.. code-block:: python
|
|
38
|
+
|
|
39
|
+
meta.set('ExistingKey', 'Value', 'Unit')
|
|
40
|
+
|
|
41
|
+
The `set` function is a special function. It sets all metadata items with the key `ExistingKey` to the value and unit. It is the implementation of the *icommands* `imeta set`.
|
|
42
|
+
|
|
43
|
+
- Delete metadata of a collection or data object:
|
|
44
|
+
|
|
45
|
+
.. code-block:: python
|
|
46
|
+
|
|
47
|
+
meta.delete('NewKey', 'NewValue', 'NewUnit')
|
|
48
|
+
|
|
49
|
+
Export Metadata
|
|
50
|
+
---------------
|
|
51
|
+
|
|
52
|
+
The function `Metadata.to_dict` will provide you with a python dictionary containing the user-defined metadata:
|
|
53
|
+
|
|
54
|
+
.. code-block:: python
|
|
55
|
+
|
|
56
|
+
meta.to_dict()
|
|
57
|
+
|
|
58
|
+
.. code-block:: python
|
|
59
|
+
|
|
60
|
+
{
|
|
61
|
+
'name': '<obj_name>',
|
|
62
|
+
'irods_id': 24490075,
|
|
63
|
+
'checksum': 'sha2:XGiECYZOtUfP9lnCGyZaBBkBGLaJJw1p6eoc0GxLeKU=',
|
|
64
|
+
'metadata': [('Key', 'Value', 'Unit'), ('Key', 'Value', 'Unit1')]
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
The dictionary contains the name of the data object or collection and its iRODS identifier.
|
|
68
|
+
For data objects the checksum is also listed under the key `checksum`. The checksum is not calculated, but extracted from the iRODS database.
|
|
69
|
+
|
|
70
|
+
The user-defined metadata can be accessed with the key `metadata`.
|
|
71
|
+
|
|
72
|
+
Export metadata of all member of a collection
|
|
73
|
+
---------------------------------------------
|
|
74
|
+
|
|
75
|
+
Above we showed how to export the metadata of one collection or data object. In case you want to export
|
|
76
|
+
metadata of all memebers of a collection you can use:
|
|
77
|
+
|
|
78
|
+
.. code-block:: python
|
|
79
|
+
|
|
80
|
+
from ibridges.export_metadata import export_metadata_to_dict
|
|
81
|
+
|
|
82
|
+
coll = IrodsPath(session, <coll_path>).collection
|
|
83
|
+
meta = Metadata(coll)
|
|
84
|
+
export_metadata_to_dict(meta, session)
|
|
85
|
+
|
|
86
|
+
The resulting dictionary is built like above and contains two additional keys `subcollections` and `dataobjects`.
|
|
87
|
+
|
|
88
|
+
Under `subcollections` we will find the extracted metadata of all subcollection and under `dataobjects` the extracted metadata for all data objects.
|
|
89
|
+
Those extracted metadata are also represented by dictionaries as shown above, they only contain one extra key `rel_path` which denotes the path relative to the collection which we gave as input to the function.
|
|
90
|
+
|
|
91
|
+
.. code-block:: python
|
|
92
|
+
|
|
93
|
+
{
|
|
94
|
+
'ibridges_metadata_version': 1.0,
|
|
95
|
+
'name': 'Demo',
|
|
96
|
+
'irods_id': 24484787,
|
|
97
|
+
'metadata': [('Key', 'very_important', None)],
|
|
98
|
+
'subcollections': [
|
|
99
|
+
{
|
|
100
|
+
'name': 'subDemo',
|
|
101
|
+
'irods_id': 24490064,
|
|
102
|
+
'rel_path': 'Demo',
|
|
103
|
+
'metadata': []
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
'name': 'my_books',
|
|
107
|
+
'irods_id': 24502538,
|
|
108
|
+
'rel_path': 'my_books',
|
|
109
|
+
'metadata': []}
|
|
110
|
+
],
|
|
111
|
+
'dataobjects': [
|
|
112
|
+
{
|
|
113
|
+
'name': 'AliceInWonderland.txt',
|
|
114
|
+
'irods_id': 24484789,
|
|
115
|
+
'checksum': 'sha2:TQzOrHuw1qRQ6zh8xm5GEuVKGjs22STdgQCdezv8LY4=',
|
|
116
|
+
'rel_path': 'my_books/AliceInWonderland.txt',
|
|
117
|
+
'metadata': [('author', 'Lewis Carroll', None)]
|
|
118
|
+
}
|
|
119
|
+
]
|
|
120
|
+
}
|
|
@@ -12,7 +12,7 @@ iBridges requires Python version 3.8 or higher. You can install iBridges with pi
|
|
|
12
12
|
pip install ibridges
|
|
13
13
|
|
|
14
14
|
|
|
15
|
-
Getting your
|
|
15
|
+
Getting your iRODS environment file
|
|
16
16
|
-----------------------------------
|
|
17
17
|
|
|
18
18
|
To connect to an iRods server you need an iRods environment file (`irods_nevironment.json`).
|
|
@@ -43,7 +43,7 @@ but if needed (if you need access to more than one iRods instance for example) y
|
|
|
43
43
|
replace that path in this quick start guide with the path you have chosen.
|
|
44
44
|
|
|
45
45
|
|
|
46
|
-
Connecting to your
|
|
46
|
+
Connecting to your iRODS server
|
|
47
47
|
-------------------------------
|
|
48
48
|
|
|
49
49
|
To connect to your iRods server, we will create a session. The session is the central object in the iBridges library;
|
|
@@ -74,7 +74,7 @@ This upload function can upload both directories (collections in iRods) and file
|
|
|
74
74
|
Add metadata
|
|
75
75
|
------------
|
|
76
76
|
|
|
77
|
-
One of the powerful features of
|
|
77
|
+
One of the powerful features of iRODS is its ability to store metadata with your data in a consistent manner.
|
|
78
78
|
Let's add some metadata to a collection or data object:
|
|
79
79
|
|
|
80
80
|
.. code:: python
|