immlib 1.0.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- immlib/__init__.py +131 -0
- immlib/_init.py +108 -0
- immlib/_version.py +235 -0
- immlib/doc/__init__.py +38 -0
- immlib/doc/_core.py +311 -0
- immlib/iolib/__init__.py +29 -0
- immlib/iolib/_core.py +720 -0
- immlib/pathlib/__init__.py +69 -0
- immlib/pathlib/_cache.py +152 -0
- immlib/pathlib/_core.py +869 -0
- immlib/pathlib/_osf.py +538 -0
- immlib/test/__init__.py +16 -0
- immlib/test/__main__.py +10 -0
- immlib/test/doc/__init__.py +6 -0
- immlib/test/doc/test_core.py +91 -0
- immlib/test/iolib/__init__.py +7 -0
- immlib/test/iolib/test_core.py +81 -0
- immlib/test/pathlib/__init__.py +11 -0
- immlib/test/pathlib/test_core.py +146 -0
- immlib/test/pathlib/test_osf.py +54 -0
- immlib/test/types/__init__.py +5 -0
- immlib/test/types/test_core.py +110 -0
- immlib/test/util/__init__.py +11 -0
- immlib/test/util/test_core.py +681 -0
- immlib/test/util/test_numeric.py +1374 -0
- immlib/test/util/test_quantity.py +218 -0
- immlib/test/util/test_url.py +51 -0
- immlib/test/workflow/__init__.py +9 -0
- immlib/test/workflow/test_core.py +418 -0
- immlib/test/workflow/test_plantype.py +248 -0
- immlib/types/__init__.py +29 -0
- immlib/types/_core.py +333 -0
- immlib/util/__init__.py +283 -0
- immlib/util/_core.py +2524 -0
- immlib/util/_numeric.py +2651 -0
- immlib/util/_quantity.py +523 -0
- immlib/util/_url.py +114 -0
- immlib/workflow/__init__.py +48 -0
- immlib/workflow/_core.py +1635 -0
- immlib/workflow/_plantype.py +334 -0
- immlib-1.0.0.dev2.dist-info/METADATA +76 -0
- immlib-1.0.0.dev2.dist-info/RECORD +45 -0
- immlib-1.0.0.dev2.dist-info/WHEEL +5 -0
- immlib-1.0.0.dev2.dist-info/licenses/LICENSE +21 -0
- immlib-1.0.0.dev2.dist-info/top_level.txt +1 -0
immlib/pathlib/_osf.py
ADDED
|
@@ -0,0 +1,538 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
###############################################################################
|
|
3
|
+
# immlib/pathlib/_osf.py
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
# Dependencies ################################################################
|
|
7
|
+
|
|
8
|
+
import mimetypes, json, os
|
|
9
|
+
from pathlib import (PosixPath, PurePosixPath, Path)
|
|
10
|
+
from urllib.parse import urlparse
|
|
11
|
+
from functools import reduce
|
|
12
|
+
from datetime import datetime
|
|
13
|
+
|
|
14
|
+
from cloudpathlib.cloudpath import (
|
|
15
|
+
register_path_class, CloudPath, NoStatError)
|
|
16
|
+
from cloudpathlib.client import (register_client_class, Client)
|
|
17
|
+
from pcollections import (pdict, ldict, lazy)
|
|
18
|
+
|
|
19
|
+
from ..doc import docwrap
|
|
20
|
+
from ..util import (is_str, is_amap, is_url, url_download)
|
|
21
|
+
|
|
22
|
+
# Utility Functions ###########################################################
|
|
23
|
+
|
|
24
|
+
osf_basepath = 'https://api.osf.io/v2/nodes/%s/files/%s/'
|
|
25
|
+
osf_pagesize_format = 'page[size]='
|
|
26
|
+
osf_pagesize_check = 'page%5Bsize%5D='
|
|
27
|
+
osf_pagecache_filename = 'osf_treecache.json'
|
|
28
|
+
def _osf_pageload(proj, path,
|
|
29
|
+
pageno=0,
|
|
30
|
+
url=None,
|
|
31
|
+
storage='osfstorage',
|
|
32
|
+
cache_path=None,
|
|
33
|
+
mkdir_mode=0o775,
|
|
34
|
+
pagesize=100):
|
|
35
|
+
"""Loads a single page of the OSF contents for a specific OSF path."""
|
|
36
|
+
cache_path = None if cache_path is None else Path(cache_path)
|
|
37
|
+
cache_root = cache_path
|
|
38
|
+
if url is None:
|
|
39
|
+
path = '' if path is None else str(path).lstrip('/')
|
|
40
|
+
cache_path = (cache_root / path) if cache_root else None
|
|
41
|
+
url = (osf_basepath % (proj, storage)) + path
|
|
42
|
+
if pagesize is not None:
|
|
43
|
+
if osf_pagesize_format not in url and osf_pagesize_check not in url:
|
|
44
|
+
if '?page=' in url:
|
|
45
|
+
url = url + '&' + osf_pagesize_format + str(pagesize)
|
|
46
|
+
elif osf_pagesize_check not in url:
|
|
47
|
+
url = url + '?' + osf_pagesize_format + str(pagesize)
|
|
48
|
+
else:
|
|
49
|
+
pagesize = 0
|
|
50
|
+
# First step is to load the data url.
|
|
51
|
+
dat = None
|
|
52
|
+
fromcache = False
|
|
53
|
+
if cache_path is not None:
|
|
54
|
+
# See if the JSON is in the cache.
|
|
55
|
+
cache_flnm = f'.p{pageno}_{pagesize}.' + osf_pagecache_filename
|
|
56
|
+
cache_flnm = cache_path / cache_flnm
|
|
57
|
+
if cache_flnm.is_file():
|
|
58
|
+
try:
|
|
59
|
+
with cache_flnm.open('rt') as fl:
|
|
60
|
+
dat = json.load(fl)
|
|
61
|
+
fromcache = True
|
|
62
|
+
except Exception:
|
|
63
|
+
pass
|
|
64
|
+
if dat is None:
|
|
65
|
+
# We need to load the data from the OSF website.
|
|
66
|
+
dat = json.loads(url_download(url))
|
|
67
|
+
fromcache = False
|
|
68
|
+
# If there's no 'data' entry, we don't know what to do with it.
|
|
69
|
+
if 'data' not in dat:
|
|
70
|
+
raise ValueError(f'cannot detect kind of url entry for path:'
|
|
71
|
+
f' osf://{proj}/{path}')
|
|
72
|
+
is_file = is_amap(dat['data'])
|
|
73
|
+
if not (is_file or fromcache or cache_path is None):
|
|
74
|
+
# This is a directory that we didn't load from cache, but
|
|
75
|
+
# we have a cache path so we should save the url data. First,
|
|
76
|
+
# make sure the directory exists.
|
|
77
|
+
cache_flnm.parent.mkdir(mode=mkdir_mode, parents=True, exist_ok=True)
|
|
78
|
+
# Now write the file.
|
|
79
|
+
with cache_flnm.open('wt') as fl:
|
|
80
|
+
json.dump(dat, fl)
|
|
81
|
+
# At this point, the page has been loaded and cached; just return it.
|
|
82
|
+
return (dat, cache_path)
|
|
83
|
+
def _osf_cache_file(url, path, mkdir_mode=0o775):
|
|
84
|
+
"""Downloads the given URL to the given path then returns the path."""
|
|
85
|
+
# If there's no cache path, there's nothing to do here.
|
|
86
|
+
if path is None:
|
|
87
|
+
return None
|
|
88
|
+
# Make sure the path exists.
|
|
89
|
+
path = Path(path)
|
|
90
|
+
if path.is_file():
|
|
91
|
+
return path
|
|
92
|
+
if not path.parent.is_dir():
|
|
93
|
+
path.parent.mkdir(mode=mkdir_mode, parents=True)
|
|
94
|
+
# Download the file and save it.
|
|
95
|
+
url_download(url, destpath=path, mkdir_mode=mkdir_mode)
|
|
96
|
+
if not path.is_file():
|
|
97
|
+
raise RuntimeError(f"url failed to download: {url} -> {path}")
|
|
98
|
+
return path
|
|
99
|
+
def _osf_fileentry(name, json, cache_path=None, mkdir_mode=0o775):
|
|
100
|
+
if cache_path is None:
|
|
101
|
+
cp = None
|
|
102
|
+
else:
|
|
103
|
+
cp = lazy(_osf_cache_file,
|
|
104
|
+
json['links']['download'], cache_path/name,
|
|
105
|
+
mkdir_mode=mkdir_mode)
|
|
106
|
+
# Extract some meta-data.
|
|
107
|
+
attrs = json['attributes']
|
|
108
|
+
return ldict(
|
|
109
|
+
kind='file',
|
|
110
|
+
download_url=json['links']['download'],
|
|
111
|
+
cache_path=cp,
|
|
112
|
+
size=attrs.get('size'),
|
|
113
|
+
date_modified=attrs.get('date_modified'),
|
|
114
|
+
date_created=attrs.get('date_created'))
|
|
115
|
+
def _osf_crawl(proj, path=None,
|
|
116
|
+
storage='osfstorage',
|
|
117
|
+
cache_path=None,
|
|
118
|
+
mkdir_mode=0o775,
|
|
119
|
+
pagesize=100,
|
|
120
|
+
url=None):
|
|
121
|
+
"""Private implementation of the osf_contents function."""
|
|
122
|
+
if cache_path is None:
|
|
123
|
+
cache_root = None
|
|
124
|
+
else:
|
|
125
|
+
cache_path = Path(cache_path)/proj/storage
|
|
126
|
+
cache_root = cache_path
|
|
127
|
+
# First step is to load the data url.
|
|
128
|
+
(dat, cache_path) = _osf_pageload(proj, path=path, url=url,
|
|
129
|
+
storage=storage,
|
|
130
|
+
cache_path=cache_root,
|
|
131
|
+
mkdir_mode=mkdir_mode,
|
|
132
|
+
pagesize=pagesize)
|
|
133
|
+
# Is this a file?
|
|
134
|
+
if is_amap(dat['data']):
|
|
135
|
+
raise RuntimeError("_osf_crawl given file instead of dir")
|
|
136
|
+
# It's a directory, so we need to build up the list of directory contents.
|
|
137
|
+
ls = {}
|
|
138
|
+
pageno = 0
|
|
139
|
+
while dat is not None:
|
|
140
|
+
links = dat.get('links', {})
|
|
141
|
+
dat = dat['data']
|
|
142
|
+
for u in dat:
|
|
143
|
+
r = u['attributes']
|
|
144
|
+
rname = r['name']
|
|
145
|
+
if r['kind'] == 'file':
|
|
146
|
+
ls[rname] = _osf_fileentry(rname, u, cache_path, mkdir_mode)
|
|
147
|
+
else:
|
|
148
|
+
url = r['path'].lstrip('/')
|
|
149
|
+
url = (osf_basepath % (proj, storage)) + url
|
|
150
|
+
path = Path('/' if path is None else path)
|
|
151
|
+
cp = None if cache_root is None else cache_root/rname
|
|
152
|
+
ls[rname] = lazy(_osf_crawl,
|
|
153
|
+
proj, path/rname,
|
|
154
|
+
url=url,
|
|
155
|
+
storage=storage,
|
|
156
|
+
cache_path=cp,
|
|
157
|
+
mkdir_mode=mkdir_mode,
|
|
158
|
+
pagesize=pagesize)
|
|
159
|
+
nxt = links.get('next')
|
|
160
|
+
if nxt is None:
|
|
161
|
+
dat = None
|
|
162
|
+
else:
|
|
163
|
+
pageno += 1
|
|
164
|
+
(dat, cache_path) = _osf_pageload(proj, path=path,
|
|
165
|
+
pageno=pageno,
|
|
166
|
+
url=nxt,
|
|
167
|
+
storage=storage,
|
|
168
|
+
cache_path=cache_root,
|
|
169
|
+
mkdir_mode=mkdir_mode,
|
|
170
|
+
pagesize=pagesize)
|
|
171
|
+
return pdict(
|
|
172
|
+
kind='directory',
|
|
173
|
+
contents=ldict(ls),
|
|
174
|
+
cache_path=cache_path)
|
|
175
|
+
@docwrap
|
|
176
|
+
def osf_contents(proj,
|
|
177
|
+
storage='osfstorage',
|
|
178
|
+
cache_path=None,
|
|
179
|
+
mkdir_mode=0o775,
|
|
180
|
+
pagesize=100,
|
|
181
|
+
lazy=True):
|
|
182
|
+
"""Returns a dictionary of the contents of the given OSF project and path.
|
|
183
|
+
|
|
184
|
+
``osf_contents(project_name)`` returns a dictionary of the contents of the
|
|
185
|
+
OSF project with the given ``project_name``. These contents are represented
|
|
186
|
+
in a nested dictionary; each entry contains the keys ``'kind'`` (either
|
|
187
|
+
``'file'`` or ``'directory'``), ``'cache_path'`` (the directory or filename
|
|
188
|
+
of the associated cache), and either ``'contents'`` (for directories) or
|
|
189
|
+
``'download_url'`` (for files).
|
|
190
|
+
|
|
191
|
+
Parameters
|
|
192
|
+
----------
|
|
193
|
+
project : str
|
|
194
|
+
The OSF project ID.
|
|
195
|
+
storage : str, optional
|
|
196
|
+
The OSF storage type to extract. By default this is ``'osfstorage'``.
|
|
197
|
+
cache_path : path or None, optional
|
|
198
|
+
The cache directory in which to store the data downloaded.
|
|
199
|
+
mkdir_mode : int, optional
|
|
200
|
+
The mode to use when making directories in the cache. By default this
|
|
201
|
+
is ``0o775``.
|
|
202
|
+
pagesize : int, optional
|
|
203
|
+
The number of items to include in a single page when paging directory
|
|
204
|
+
contents from the OSF server. The default is 100.
|
|
205
|
+
lazy : bool, optional
|
|
206
|
+
If ``True`` (the default), then the returned dictionary is a lazy dict
|
|
207
|
+
representing the root of the project, but the OSF is not queried until
|
|
208
|
+
its contents are requested. If ``False``, then the directory contents
|
|
209
|
+
are built immediately.
|
|
210
|
+
|
|
211
|
+
Returns
|
|
212
|
+
-------
|
|
213
|
+
dict
|
|
214
|
+
A nested lazy persistent dictionary of the project's contents.
|
|
215
|
+
"""
|
|
216
|
+
if lazy:
|
|
217
|
+
# Prepare the crawl data-structure, but don't build it yet.
|
|
218
|
+
from pcollections import lazy
|
|
219
|
+
crawl = lazy(_osf_crawl, proj,
|
|
220
|
+
storage=storage,
|
|
221
|
+
cache_path=cache_path,
|
|
222
|
+
mkdir_mode=mkdir_mode,
|
|
223
|
+
pagesize=pagesize)
|
|
224
|
+
# Create a lazy-dict that just references crawl.
|
|
225
|
+
return ldict(kind='directory',
|
|
226
|
+
contents=lazy(lambda:crawl()['contents']),
|
|
227
|
+
cache_path=lazy(lambda:crawl()['cache_path']))
|
|
228
|
+
else:
|
|
229
|
+
return _osf_crawl(proj,
|
|
230
|
+
storage=storage,
|
|
231
|
+
cache_path=cache_path,
|
|
232
|
+
mkdir_mode=mkdir_mode,
|
|
233
|
+
pagesize=pagesize)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
# OSFClient ###################################################################
|
|
237
|
+
|
|
238
|
+
@register_client_class("osf")
|
|
239
|
+
class OSFClient(Client):
|
|
240
|
+
"""Client class for the OSF."""
|
|
241
|
+
@staticmethod
|
|
242
|
+
def _extract_path(contents, path):
|
|
243
|
+
path = str(path)
|
|
244
|
+
if path.startswith('osf://'):
|
|
245
|
+
path = path[6:]
|
|
246
|
+
path = path.lstrip('/')
|
|
247
|
+
parts = PurePosixPath(path).parts[1:]
|
|
248
|
+
for (ii,p) in enumerate(parts):
|
|
249
|
+
if 'contents' not in contents:
|
|
250
|
+
p = "/".join(parts[:ii+1])
|
|
251
|
+
raise NotADirectoryError(f"Not a directory: {repr(p)}")
|
|
252
|
+
contents = contents['contents']
|
|
253
|
+
if p not in contents:
|
|
254
|
+
p = "/".join(parts[:ii+1])
|
|
255
|
+
raise FileNotFoundError(f"No such directory: {repr(p)}")
|
|
256
|
+
contents = contents[p]
|
|
257
|
+
return contents
|
|
258
|
+
def __init__(self, project='xxxxx', storage='osfstorage',
|
|
259
|
+
file_cache_mode=None,
|
|
260
|
+
local_cache_dir=None,
|
|
261
|
+
content_type_method=mimetypes.guess_type,
|
|
262
|
+
pagesize=100,
|
|
263
|
+
mkdir_mode=0o775):
|
|
264
|
+
super().__init__(
|
|
265
|
+
file_cache_mode=file_cache_mode,
|
|
266
|
+
local_cache_dir=local_cache_dir,
|
|
267
|
+
content_type_method=content_type_method)
|
|
268
|
+
# Currently, root must always be '/' (it's not in the options list).
|
|
269
|
+
root = '/'
|
|
270
|
+
self.project_id = project
|
|
271
|
+
self.storage_provider = storage
|
|
272
|
+
self.root_path = root
|
|
273
|
+
self.mkdir_mode = mkdir_mode
|
|
274
|
+
self.pagesize = pagesize
|
|
275
|
+
self.file_cache_mode = file_cache_mode
|
|
276
|
+
# Get the contents of the OSF repository (lazily).
|
|
277
|
+
contents = osf_contents(
|
|
278
|
+
project,
|
|
279
|
+
storage=storage,
|
|
280
|
+
cache_path=local_cache_dir,
|
|
281
|
+
pagesize=pagesize,
|
|
282
|
+
mkdir_mode=mkdir_mode)
|
|
283
|
+
# Save these as the whole-project contents.
|
|
284
|
+
self.project_contents = contents
|
|
285
|
+
# Now extract the root of these contents; if the root is '/' or '',
|
|
286
|
+
# then we just use the project contents.
|
|
287
|
+
root = str(root).lstrip('/')
|
|
288
|
+
if root == '':
|
|
289
|
+
self.root_contents = contents
|
|
290
|
+
else:
|
|
291
|
+
rc = lazy(self._extract_path, contents, root)
|
|
292
|
+
self.root_contents = ldict(
|
|
293
|
+
kind='directory',
|
|
294
|
+
contents=lazy(lambda:rc()['contents']),
|
|
295
|
+
cache_path=lazy(lambda:rc()['cache_path']))
|
|
296
|
+
# Several of the abstract methods are non-operational for OSF, because all
|
|
297
|
+
# OSF operations are currently read-only.
|
|
298
|
+
def _move_file(self, src, dst, remove_src=True):
|
|
299
|
+
raise RuntimeError(f"OSF CloudPath operations are read-only")
|
|
300
|
+
def _remove(self, path, missing_ok=True):
|
|
301
|
+
raise RuntimeError(f"OSF CloudPath operations are read-only")
|
|
302
|
+
def _upload_file(self, local_path, cloud_path):
|
|
303
|
+
raise RuntimeError(f"OSF CloudPath operations are read-only")
|
|
304
|
+
# Other abstract methods are valid, however.
|
|
305
|
+
def _download_file(self, cloud_path, local_path, mkdir_mode=0o775):
|
|
306
|
+
if not isinstance(cloud_path, OSFPath):
|
|
307
|
+
raise TypeError("cannot download path that is not an OSFPath")
|
|
308
|
+
entry = self._extract_path(self.root_contents, cloud_path)
|
|
309
|
+
local_path = Path(local_path)
|
|
310
|
+
if entry['kind'] == 'directory':
|
|
311
|
+
# Recursively download everything.
|
|
312
|
+
for (name,ent) in entry['contents'].items():
|
|
313
|
+
p = local_path/name
|
|
314
|
+
if ent['kind'] == 'directory':
|
|
315
|
+
p.mkdir(mode=mkdir_mode, exist_ok=True)
|
|
316
|
+
self._download_file(cloud_path/name, p,
|
|
317
|
+
mkdir_mode=mkdir_mode)
|
|
318
|
+
else:
|
|
319
|
+
url_download(ent['download_url'], p)
|
|
320
|
+
else:
|
|
321
|
+
url_download(entry['download_url'], local_path)
|
|
322
|
+
return local_path
|
|
323
|
+
def _exists(self, cloud_path):
|
|
324
|
+
if not isinstance(cloud_path, OSFPath):
|
|
325
|
+
raise TypeError("cannot query path that is not an OSFPath")
|
|
326
|
+
try:
|
|
327
|
+
entry = self._extract_path(self.root_contents, cloud_path)
|
|
328
|
+
except (NotADirectoryError, FileNotFoundError):
|
|
329
|
+
return False
|
|
330
|
+
return True
|
|
331
|
+
def _list_dir(self, cloud_path, recursive=False):
|
|
332
|
+
if not isinstance(cloud_path, OSFPath):
|
|
333
|
+
raise TypeError("cannot list path that is not an OSFPath")
|
|
334
|
+
if recursive:
|
|
335
|
+
raise NotImplementedError(
|
|
336
|
+
"recursive listing of OSF projects is not supported")
|
|
337
|
+
entry = self._extract_path(self.root_contents, cloud_path)
|
|
338
|
+
return ((cloud_path/k, v['kind'] == 'directory')
|
|
339
|
+
for (k,v) in entry['contents'].items())
|
|
340
|
+
def _path_kind(self, cloud_path):
|
|
341
|
+
if not isinstance(cloud_path, OSFPath):
|
|
342
|
+
raise TypeError("cannot query path that is not an OSFPath")
|
|
343
|
+
entry = self._extract_path(self.root_contents, cloud_path)
|
|
344
|
+
return entry['kind']
|
|
345
|
+
def _path_entry(self, cloud_path):
|
|
346
|
+
if not isinstance(cloud_path, OSFPath):
|
|
347
|
+
raise TypeError("cannot query path that is not an OSFPath")
|
|
348
|
+
return self._extract_path(self.root_contents, cloud_path)
|
|
349
|
+
def _get_public_url(self, cloudpath):
|
|
350
|
+
raise TypeError(
|
|
351
|
+
f"{type(self)} does not support _generate_public_url")
|
|
352
|
+
def _generate_presigned_url(self, cloudpath, expire_seconds=60*60):
|
|
353
|
+
raise TypeError(
|
|
354
|
+
f"{type(self)} does not support _generate_presigned_url")
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
# OSFPath #####################################################################
|
|
358
|
+
|
|
359
|
+
@register_path_class("osf")
|
|
360
|
+
class OSFPath(CloudPath):
|
|
361
|
+
"""Class for representing and operating on OSF repositories.
|
|
362
|
+
|
|
363
|
+
``OSFPath(path)`` returns a path object representing an OSF path; OSF paths
|
|
364
|
+
use the format ``osf://<project-id>/<path>``. The project ID is derived
|
|
365
|
+
from the OSF tag; i.e., the website ``https://osf.io/<project-id>`` is the
|
|
366
|
+
primary website of the OSF project. The project-ID may be followed by a
|
|
367
|
+
colon and an OSF storage name (the project ID by itself is equivalent to
|
|
368
|
+
``osf://<project-ID>:osfstorage/``).
|
|
369
|
+
|
|
370
|
+
For example, the project found at the OSF website ``https://osf.io/bw9ec/``
|
|
371
|
+
has the URL ``osf://bw9ec/``.
|
|
372
|
+
|
|
373
|
+
Parameters
|
|
374
|
+
----------
|
|
375
|
+
cloud_path : str or path-like
|
|
376
|
+
The OSF path that the created ``OSFPath`` object is to represent.
|
|
377
|
+
client : OSFClient or None, optional
|
|
378
|
+
The ``OSFClient`` object to use. The ``OSFClient`` is responsible
|
|
379
|
+
primarily for the caching of data locally. If ``OSFClient`` is
|
|
380
|
+
``None``, then an ``OSFClient`` object is created for the project using
|
|
381
|
+
a temporary cache directory.
|
|
382
|
+
local_cache_dir : str or path-like, optional
|
|
383
|
+
The local directory in which cache files should be stored. This option
|
|
384
|
+
is ignored if ``client`` is not ``None``; otherwise it is passed to the
|
|
385
|
+
created client object. The cache directory is the root cache directory
|
|
386
|
+
for the entire OSF project.
|
|
387
|
+
file_cache_mode : cloudpathlib.enums.FileCacheMode, optional
|
|
388
|
+
How often to clear the file cache; see [cloudpathlib's caching
|
|
389
|
+
docs](https://cloudpathlib.drivendata.org/stable/caching/) for more
|
|
390
|
+
information about the options in ``cloudpathlib.enums.FileCacheMode``.
|
|
391
|
+
mkdir_mode : int, optional
|
|
392
|
+
The mode to use when making directories in the cache. By default this
|
|
393
|
+
is ``0o775``. This option is ignored if the ``client`` option is not
|
|
394
|
+
``None``.
|
|
395
|
+
pagesize : int, optional
|
|
396
|
+
The number of items to include in a single page when paging directory
|
|
397
|
+
contents from the OSF server. The default is 100. This option is
|
|
398
|
+
ignored if the ``client`` option is not ``None``.
|
|
399
|
+
"""
|
|
400
|
+
cloud_prefix = "osf://"
|
|
401
|
+
client = OSFClient
|
|
402
|
+
init_default_options = dict(
|
|
403
|
+
local_cache_dir=None,
|
|
404
|
+
file_cache_mode=None,
|
|
405
|
+
mkdir_mode=0o775,
|
|
406
|
+
pagesize=100)
|
|
407
|
+
def __init__(self, cloud_path, client=None,
|
|
408
|
+
local_cache_dir=Ellipsis,
|
|
409
|
+
file_cache_mode=Ellipsis,
|
|
410
|
+
mkdir_mode=Ellipsis,
|
|
411
|
+
pagesize=Ellipsis):
|
|
412
|
+
# Needed at the top of the init, see the CloudPath __init__ method in
|
|
413
|
+
# cloudpath.py in cloudpathlib.
|
|
414
|
+
self._handle = None
|
|
415
|
+
self.client = OSFClient.get_default_client()
|
|
416
|
+
# First validate the path.
|
|
417
|
+
if isinstance(cloud_path, OSFPath):
|
|
418
|
+
if client is None:
|
|
419
|
+
client = cloud_path.client
|
|
420
|
+
self.client = client
|
|
421
|
+
else:
|
|
422
|
+
# Go ahead and validate the url.
|
|
423
|
+
self.is_valid_cloudpath(cloud_path, raise_on_error=True)
|
|
424
|
+
# We'll also need to know the project and storage to create any new
|
|
425
|
+
# client object. To do that we parse the cloud path URL.
|
|
426
|
+
url = urlparse(str(cloud_path))
|
|
427
|
+
if url.scheme != 'osf' or not url.netloc:
|
|
428
|
+
raise ValueError(f"invalid OSF url: {repr(cloud_path)}")
|
|
429
|
+
if ':' in url.netloc:
|
|
430
|
+
(project, storage) = url.netloc.split(':')
|
|
431
|
+
else:
|
|
432
|
+
(project, storage) = (url.netloc, 'osfstorage')
|
|
433
|
+
# Now that we have the project and storage, we can figure out the
|
|
434
|
+
# client option, which we may be updating with options.
|
|
435
|
+
if client is None:
|
|
436
|
+
# No client was implied or given, so we make a new one from the
|
|
437
|
+
# path and the remaining options. Any Ellipsis options we look up
|
|
438
|
+
# in the default options dict (above).
|
|
439
|
+
if local_cache_dir is Ellipsis:
|
|
440
|
+
local_cache_dir = self.init_default_options['local_cache_dir']
|
|
441
|
+
if file_cache_mode is Ellipsis:
|
|
442
|
+
file_cache_mode = self.init_default_options['file_cache_mode']
|
|
443
|
+
if mkdir_mode is Ellipsis:
|
|
444
|
+
mkdir_mode = self.init_default_options['mkdir_mode']
|
|
445
|
+
if pagesize is Ellipsis:
|
|
446
|
+
pagesize = self.init_default_options['pagesize']
|
|
447
|
+
client = OSFClient(
|
|
448
|
+
project,
|
|
449
|
+
storage=storage,
|
|
450
|
+
local_cache_dir=local_cache_dir,
|
|
451
|
+
file_cache_mode=file_cache_mode,
|
|
452
|
+
mkdir_mode=mkdir_mode,
|
|
453
|
+
pagesize=pagesize)
|
|
454
|
+
elif isinstance(client, OSFClient):
|
|
455
|
+
# We can use the given client and just update any options whose
|
|
456
|
+
# values aren't Ellipsis.
|
|
457
|
+
change = False
|
|
458
|
+
if local_cache_dir is Ellipsis:
|
|
459
|
+
local_cache_dir = client._local_cache_dir
|
|
460
|
+
elif local_cache_dir != client._local_cache_dir:
|
|
461
|
+
change = True
|
|
462
|
+
if file_cache_mode is Ellipsis:
|
|
463
|
+
file_cache_mode = client.file_cache_mode
|
|
464
|
+
elif file_cache_mode != client.file_cache_mode:
|
|
465
|
+
change = True
|
|
466
|
+
if mkdir_mode is Ellipsis:
|
|
467
|
+
mkdir_mode = client.mkdir_mode
|
|
468
|
+
elif mkdir_mode != client.mkdir_mode:
|
|
469
|
+
change = True
|
|
470
|
+
if pagesize is Ellipsis:
|
|
471
|
+
pagesize = client.pagesize
|
|
472
|
+
elif pagesize != client.pagesize:
|
|
473
|
+
change = True
|
|
474
|
+
# Make sure the path's project and storage match the client.
|
|
475
|
+
if client.project_id != project: change = True
|
|
476
|
+
if client.storage_provider != storage: change = True
|
|
477
|
+
# If there's any change requested, we make a new client.
|
|
478
|
+
if change:
|
|
479
|
+
client = OSFClient(
|
|
480
|
+
project,
|
|
481
|
+
storage=storage,
|
|
482
|
+
local_cache_dir=local_cache_dir,
|
|
483
|
+
file_cache_mode=file_cache_mode,
|
|
484
|
+
mkdir_mode=mkdir_mode,
|
|
485
|
+
pagesize=pagesize)
|
|
486
|
+
else:
|
|
487
|
+
self.client = OSFClient()
|
|
488
|
+
raise TypeError("OSFPaths require OSFClient objects as clients")
|
|
489
|
+
self.client = client
|
|
490
|
+
# At this point we have a cloud path and client that are both valid.
|
|
491
|
+
super().__init__(cloud_path, client=client)
|
|
492
|
+
@property
|
|
493
|
+
def bucket(self):
|
|
494
|
+
return self.client.project_id
|
|
495
|
+
@property
|
|
496
|
+
def drive(self):
|
|
497
|
+
return self._no_prefix.split("/", 1)[0]
|
|
498
|
+
def is_dir(self):
|
|
499
|
+
return self.client._path_kind(self) == "directory"
|
|
500
|
+
def is_file(self):
|
|
501
|
+
return self.client._path_kind(self) == "file"
|
|
502
|
+
def mkdir(self, parents=False, exist_ok=False):
|
|
503
|
+
raise TypeError(f"OSF CloudPath operations are read-only")
|
|
504
|
+
def touch(self, exist_ok: bool = True):
|
|
505
|
+
raise TypeError(f"OSF CloudPath operations are read-only")
|
|
506
|
+
def stat(self):
|
|
507
|
+
ent = self.client._path_entry(self)
|
|
508
|
+
if ent['kind'] != 'file':
|
|
509
|
+
raise NoStatError(f"No stats available for directory: {self}")
|
|
510
|
+
mtime = ent.get('date_modified')
|
|
511
|
+
ctime = ent.get('date_created')
|
|
512
|
+
mtime = datetime.fromisoformat(mtime).timestamp() if mtime else 0
|
|
513
|
+
# The [:-1] chops off a character at the end not recognized by the ISO
|
|
514
|
+
# standard in all versions.
|
|
515
|
+
ctime = datetime.fromisoformat(ctime[:-1]).timestamp() if ctime else 0
|
|
516
|
+
stat = (
|
|
517
|
+
None, # mode
|
|
518
|
+
None, # ino
|
|
519
|
+
self.cloud_prefix, # dev,
|
|
520
|
+
None, # nlink,
|
|
521
|
+
None, # uid,
|
|
522
|
+
None, # gid,
|
|
523
|
+
ent.get("size", 0), # size,
|
|
524
|
+
None, # atime,
|
|
525
|
+
mtime, # mtime,
|
|
526
|
+
ctime) # ctime
|
|
527
|
+
return os.stat_result(stat)
|
|
528
|
+
@property
|
|
529
|
+
def project_id(self):
|
|
530
|
+
return self.client.project_id
|
|
531
|
+
@property
|
|
532
|
+
def _local(self):
|
|
533
|
+
lcd = self.client._local_cache_dir
|
|
534
|
+
base = lcd / self.project_id / self.client.storage_provider
|
|
535
|
+
return base / self.key.lstrip('/')
|
|
536
|
+
@property
|
|
537
|
+
def key(self):
|
|
538
|
+
return self._no_prefix_no_drive
|
immlib/test/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
################################################################################
|
|
3
|
+
# immlib/test/__init__.py
|
|
4
|
+
|
|
5
|
+
'''The core immlib test module.
|
|
6
|
+
|
|
7
|
+
The `immlib.test` package contains tests for the immlib library as well as
|
|
8
|
+
examples of the library's usage.
|
|
9
|
+
'''
|
|
10
|
+
|
|
11
|
+
from .doc import *
|
|
12
|
+
from .util import *
|
|
13
|
+
from .types import *
|
|
14
|
+
from .pathlib import *
|
|
15
|
+
from .iolib import *
|
|
16
|
+
from .workflow import *
|
immlib/test/__main__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
################################################################################
|
|
2
|
+
# immlib/test/__main__.py
|
|
3
|
+
#
|
|
4
|
+
# Main-function wrapper for the test package for immlib, using unittest.
|
|
5
|
+
|
|
6
|
+
def run_tests(verbosity=2, **kwargs):
|
|
7
|
+
from unittest import main
|
|
8
|
+
return main("immlib.test", verbosity=verbosity, **kwargs)
|
|
9
|
+
|
|
10
|
+
run_tests()
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
################################################################################
|
|
3
|
+
# immlib/test/doc/test_core.py
|
|
4
|
+
#
|
|
5
|
+
# Tests of the core documentation system in immlib: i.e., tests for the code in
|
|
6
|
+
# the immlib.doc._core module.
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
# Dependencies #################################################################
|
|
10
|
+
|
|
11
|
+
from unittest import TestCase
|
|
12
|
+
|
|
13
|
+
class TestDocCore(TestCase):
|
|
14
|
+
"""Tests the immlib.doc._core module.
|
|
15
|
+
|
|
16
|
+
The only public functions in the module are the `make_docproc` function and
|
|
17
|
+
the `docwrap` decorator.
|
|
18
|
+
"""
|
|
19
|
+
def test_make_docproc(self):
|
|
20
|
+
from immlib.doc._core import make_docproc
|
|
21
|
+
from immlib import docproc
|
|
22
|
+
from docrep import DocstringProcessor
|
|
23
|
+
# make_docproc() takes no arguments.
|
|
24
|
+
new_docproc = make_docproc()
|
|
25
|
+
# It makes a new DocstringProcessor.
|
|
26
|
+
self.assertIsInstance(new_docproc, DocstringProcessor)
|
|
27
|
+
# That DocstringProcessor isn't the same as the original processor.
|
|
28
|
+
self.assertIsNot(docproc, new_docproc)
|
|
29
|
+
def test_docwrap(self):
|
|
30
|
+
from immlib.doc._core import (docwrap, make_docproc)
|
|
31
|
+
# For this test we will use a custom docproc.
|
|
32
|
+
dp = make_docproc()
|
|
33
|
+
# First, make sure we can duplicate parameter documentation.
|
|
34
|
+
@docwrap('fn1', proc=dp)
|
|
35
|
+
def fn1(a, b, c=None):
|
|
36
|
+
"""Documentation test function 1.
|
|
37
|
+
|
|
38
|
+
This function tests the documentation formatter `@docwrap` of the
|
|
39
|
+
`immlib` library.
|
|
40
|
+
|
|
41
|
+
Parameters
|
|
42
|
+
----------
|
|
43
|
+
a : object
|
|
44
|
+
The first parameter to the function.
|
|
45
|
+
b : str
|
|
46
|
+
The second parameter to the function.
|
|
47
|
+
c : str or None, optional
|
|
48
|
+
The first optional parameter to the function. The default is
|
|
49
|
+
`None`. Must be a string or `None`.
|
|
50
|
+
|
|
51
|
+
Returns
|
|
52
|
+
-------
|
|
53
|
+
tuple
|
|
54
|
+
A tuple of `(a, b, c)`.
|
|
55
|
+
"""
|
|
56
|
+
return (a,b,c)
|
|
57
|
+
@docwrap('fn2', proc=dp)
|
|
58
|
+
def fn2(a, b, c=None, d=None):
|
|
59
|
+
"""Documentation test function 1.
|
|
60
|
+
|
|
61
|
+
This function tests the documentation formatter `@docwrap` of the
|
|
62
|
+
`immlib` library.
|
|
63
|
+
|
|
64
|
+
Parameters
|
|
65
|
+
----------
|
|
66
|
+
%(fn1.parameters.a)s
|
|
67
|
+
%(fn1.parameters.b)s
|
|
68
|
+
%(fn1.parameters.c)s
|
|
69
|
+
d : int or None, optional
|
|
70
|
+
The second optional parameter to the function. The default is
|
|
71
|
+
`None`. Must be an integer or `None`.
|
|
72
|
+
|
|
73
|
+
Returns
|
|
74
|
+
-------
|
|
75
|
+
tuple
|
|
76
|
+
A tuple of `(a, b, c, d)`.
|
|
77
|
+
"""
|
|
78
|
+
return (a,b,c,d)
|
|
79
|
+
# Make sure the appropriate text made it into the fn2 documentation.
|
|
80
|
+
for s in ["a : object",
|
|
81
|
+
"The first parameter to the function.",
|
|
82
|
+
"b : str",
|
|
83
|
+
"The second parameter to the function.",
|
|
84
|
+
"c : str or None, optional",
|
|
85
|
+
"The first optional parameter to the function.",
|
|
86
|
+
"d : int or None, optional",
|
|
87
|
+
"The second optional parameter to the function."]:
|
|
88
|
+
self.assertIn(s, fn2.__doc__)
|
|
89
|
+
self.assertEqual(fn1(1,2,3), (1,2,3))
|
|
90
|
+
self.assertEqual(fn2(1,2,3,4), (1,2,3,4))
|
|
91
|
+
|