h5rdmtoolbox 2.5.3__tar.gz → 2.5.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/PKG-INFO +7 -7
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/convention/core.py +14 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/gui/fairify.py +5 -5
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/user/groups.py +3 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/repository/utils.py +17 -5
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/repository/zenodo/core.py +69 -22
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/user.py +16 -2
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/utils.py +39 -4
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/wrapper/jsonld.py +58 -22
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox.egg-info/PKG-INFO +7 -7
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox.egg-info/SOURCES.txt +1 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox.egg-info/requires.txt +5 -5
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/setup.cfg +3 -3
- h5rdmtoolbox-2.5.4/tests/conftest.py +10 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/conventions/test_convention_vfuncs.py +3 -1
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/conventions/test_conventions.py +39 -5
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/conventions/test_references.py +12 -1
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/conventions/test_toolbox_validators.py +12 -2
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/ld/test_rdf.py +76 -15
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/repository/test_zenodo.py +40 -25
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/test_utils.py +11 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/wrapper/test_core.py +2 -2
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/wrapper/test_file.py +4 -3
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/LICENSE +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/README.md +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/_cfg.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/_repr.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/_version.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/cli.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/consts.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/convention/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/convention/cfg.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/convention/consts.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/convention/errors.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/convention/generate.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/convention/ontology/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/convention/ontology/h5ontocls.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/convention/ontology/hdf_datatypes.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/convention/references.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/convention/standard_attributes.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/convention/toolbox_validators.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/convention/utils.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/data/def_icon.png +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/data/fluid-v1.yml +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/data/h5tbx.yaml +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/data/id_icon.png +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/data/iri_icon.png +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/data/ld_icon.png +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/data/piv-v1.yml +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/data/style.css +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/data/tutorial_convention.yaml +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/data/tutorial_standard_name_table.yaml +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/data/type_icon.png +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/database/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/database/hdfdb/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/database/hdfdb/filedb.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/database/hdfdb/objdb.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/database/hdfdb/query.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/database/hdfdb/utils.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/database/interface.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/database/lazy.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/database/mongo.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/errors.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/extensions/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/extensions/magnitude.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/extensions/normalize.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/extensions/onto.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/extensions/units.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/extensions/vector.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/gui/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/gui/src/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/gui/src/main.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/identifiers.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/layout/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/layout/core.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/globals.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/hdf/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/hdf/attributes.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/hdf/datasets.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/hdf/file.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/hdf/groups.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/rdf.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/shacl.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/user/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/user/attributes.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/user/datasets.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/user/file.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/user/utils.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/ld/utils.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/plotting.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/protected_attributes.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/protocols.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/repository/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/repository/default_metamapper.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/repository/interface.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/repository/zenodo/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/repository/zenodo/metadata.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/repository/zenodo/tokens.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/repository/zenodo/utils.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/tutorial.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/wrapper/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/wrapper/accessor.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/wrapper/accessors/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/wrapper/accessors/provenance.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/wrapper/core.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/wrapper/ds_decoder.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/wrapper/h5attr.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/wrapper/h5ext.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/wrapper/h5utils.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/wrapper/h5yaml.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox/wrapper/xr2hdf.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox.egg-info/dependency_links.txt +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox.egg-info/entry_points.txt +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/h5rdmtoolbox.egg-info/top_level.txt +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/pyproject.toml +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/setup.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/clean_zenodo_sandbox.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/conventions/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/conventions/test_engmeta.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/conventions/test_utils.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/database/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/database/test_find_rdf.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/database/test_hdfDB.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/database/test_mongo.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/layouts/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/layouts/test_core.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/layouts/test_docs.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/ld/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/ld/test_ld.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/ld/test_shacl.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/ld/test_user_graph.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/ld/test_utils.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/repository/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/test_cli.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/test_config.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/test_extensions.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/test_identifiers.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/test_plotting.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/test_repr.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/test_user.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/test_version.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/test_xarray_export.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/wrapper/__init__.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/wrapper/test_dump.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/wrapper/test_h5ext.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/wrapper/test_lazy.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/wrapper/test_namespaces.py +0 -0
- {h5rdmtoolbox-2.5.3 → h5rdmtoolbox-2.5.4}/tests/wrapper/test_xr2df.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: h5rdmtoolbox
|
|
3
|
-
Version: 2.5.
|
|
3
|
+
Version: 2.5.4
|
|
4
4
|
Summary: Supporting a FAIR Research Data lifecycle using Python and HDF5.
|
|
5
5
|
Home-page: https://h5rdmtoolbox.readthedocs.io/en/latest/
|
|
6
6
|
Author: Matthias Probst
|
|
@@ -29,7 +29,7 @@ Requires-Dist: pint>=0.24.4
|
|
|
29
29
|
Requires-Dist: pint_xarray<=0.6.0,>=0.2.1
|
|
30
30
|
Requires-Dist: regex>=2020.7.9
|
|
31
31
|
Requires-Dist: packaging>=24.1
|
|
32
|
-
Requires-Dist: ontolutils<1.0.0,>=0.
|
|
32
|
+
Requires-Dist: ontolutils<1.0.0,>=0.25.1
|
|
33
33
|
Requires-Dist: python-forge==18.6.0
|
|
34
34
|
Requires-Dist: requests>=2.32.4
|
|
35
35
|
Requires-Dist: pydantic~=2.8
|
|
@@ -54,7 +54,7 @@ Provides-Extra: test
|
|
|
54
54
|
Requires-Dist: pytest>=8.3.3; extra == "test"
|
|
55
55
|
Requires-Dist: pytest-cov>=5.0.0; extra == "test"
|
|
56
56
|
Requires-Dist: pylint; extra == "test"
|
|
57
|
-
Requires-Dist: ssnolib<3.0.0,>=2.2.0.
|
|
57
|
+
Requires-Dist: ssnolib<3.0.0,>=2.2.0.2; extra == "test"
|
|
58
58
|
Requires-Dist: mongomock==4.1.2; extra == "test"
|
|
59
59
|
Requires-Dist: xmltodict<=0.13.0; extra == "test"
|
|
60
60
|
Requires-Dist: scipy>=1.10.1; extra == "test"
|
|
@@ -76,7 +76,7 @@ Requires-Dist: pymongo<=4.10.1,>=4.2.0; extra == "docs"
|
|
|
76
76
|
Requires-Dist: pytest>=8.3.3; extra == "docs"
|
|
77
77
|
Requires-Dist: pytest-cov>=5.0.0; extra == "docs"
|
|
78
78
|
Requires-Dist: pylint; extra == "docs"
|
|
79
|
-
Requires-Dist: ssnolib<3.0.0,>=2.2.0.
|
|
79
|
+
Requires-Dist: ssnolib<3.0.0,>=2.2.0.2; extra == "docs"
|
|
80
80
|
Requires-Dist: mongomock==4.1.2; extra == "docs"
|
|
81
81
|
Requires-Dist: xmltodict<=0.13.0; extra == "docs"
|
|
82
82
|
Requires-Dist: scipy>=1.10.1; extra == "docs"
|
|
@@ -102,7 +102,7 @@ Requires-Dist: PyQt5==5.15.10; extra == "complete"
|
|
|
102
102
|
Requires-Dist: pytest>=8.3.3; extra == "complete"
|
|
103
103
|
Requires-Dist: pytest-cov>=5.0.0; extra == "complete"
|
|
104
104
|
Requires-Dist: pylint; extra == "complete"
|
|
105
|
-
Requires-Dist: ssnolib<3.0.0,>=2.2.0.
|
|
105
|
+
Requires-Dist: ssnolib<3.0.0,>=2.2.0.2; extra == "complete"
|
|
106
106
|
Requires-Dist: mongomock==4.1.2; extra == "complete"
|
|
107
107
|
Requires-Dist: xmltodict<=0.13.0; extra == "complete"
|
|
108
108
|
Requires-Dist: scipy>=1.10.1; extra == "complete"
|
|
@@ -120,7 +120,7 @@ Requires-Dist: PyQt5==5.15.10; extra == "complete-with-docs"
|
|
|
120
120
|
Requires-Dist: pytest>=8.3.3; extra == "complete-with-docs"
|
|
121
121
|
Requires-Dist: pytest-cov>=5.0.0; extra == "complete-with-docs"
|
|
122
122
|
Requires-Dist: pylint; extra == "complete-with-docs"
|
|
123
|
-
Requires-Dist: ssnolib<3.0.0,>=2.2.0.
|
|
123
|
+
Requires-Dist: ssnolib<3.0.0,>=2.2.0.2; extra == "complete-with-docs"
|
|
124
124
|
Requires-Dist: mongomock==4.1.2; extra == "complete-with-docs"
|
|
125
125
|
Requires-Dist: xmltodict<=0.13.0; extra == "complete-with-docs"
|
|
126
126
|
Requires-Dist: scipy>=1.10.1; extra == "complete-with-docs"
|
|
@@ -141,7 +141,7 @@ Requires-Dist: pymongo<=4.10.1,>=4.2.0; extra == "complete-with-docs"
|
|
|
141
141
|
Requires-Dist: pytest>=8.3.3; extra == "complete-with-docs"
|
|
142
142
|
Requires-Dist: pytest-cov>=5.0.0; extra == "complete-with-docs"
|
|
143
143
|
Requires-Dist: pylint; extra == "complete-with-docs"
|
|
144
|
-
Requires-Dist: ssnolib<3.0.0,>=2.2.0.
|
|
144
|
+
Requires-Dist: ssnolib<3.0.0,>=2.2.0.2; extra == "complete-with-docs"
|
|
145
145
|
Requires-Dist: mongomock==4.1.2; extra == "complete-with-docs"
|
|
146
146
|
Requires-Dist: xmltodict<=0.13.0; extra == "complete-with-docs"
|
|
147
147
|
Requires-Dist: scipy>=1.10.1; extra == "complete-with-docs"
|
|
@@ -702,7 +702,21 @@ def from_repo(repo_interface: RepositoryInterface,
|
|
|
702
702
|
Name of the file to download
|
|
703
703
|
"""
|
|
704
704
|
logger.debug(f"Downloading file {name} from repository {repo_interface}")
|
|
705
|
+
|
|
705
706
|
filename = repo_interface.download_file(name)
|
|
707
|
+
_suffix = pathlib.Path(name).suffix
|
|
708
|
+
vfunc_filename = name.split(_suffix, 1)
|
|
709
|
+
has_vfunc = False
|
|
710
|
+
if len(vfunc_filename) == 2:
|
|
711
|
+
vfunc_filename = f'{vfunc_filename[0]}_vfuncs.py'
|
|
712
|
+
try:
|
|
713
|
+
downloaded_vfunc_filename = repo_interface.download_file(vfunc_filename)
|
|
714
|
+
has_vfunc = True
|
|
715
|
+
except Exception as e:
|
|
716
|
+
logger.debug(f'No vfuncs file found for {name}: {e}')
|
|
717
|
+
if has_vfunc:
|
|
718
|
+
shutil.copy(downloaded_vfunc_filename,
|
|
719
|
+
filename.parent / downloaded_vfunc_filename.name)
|
|
706
720
|
logger.debug(f"File downloaded to {filename}. Now loading convention from file.")
|
|
707
721
|
return from_file(filename)
|
|
708
722
|
|
|
@@ -10,7 +10,7 @@ import rdflib
|
|
|
10
10
|
from PyQt5 import QtWidgets
|
|
11
11
|
from PyQt5.QtGui import QFont
|
|
12
12
|
from ontolutils import M4I, SCHEMA, OBO, CODEMETA, QUDT_UNIT, QUDT_KIND
|
|
13
|
-
from ontolutils.classes.utils import
|
|
13
|
+
from ontolutils.classes.utils import split_uri
|
|
14
14
|
from pivmetalib.namespace import PIVMETA
|
|
15
15
|
from ssnolib.namespace import SSNO
|
|
16
16
|
|
|
@@ -204,7 +204,7 @@ class Ui(QtWidgets.QMainWindow, Ui_MainWindow):
|
|
|
204
204
|
if pred is None:
|
|
205
205
|
pred = self.lineEditIRI1.text()
|
|
206
206
|
self.log_message(f'updating prefix and suffix from text: {pred}')
|
|
207
|
-
ns, key =
|
|
207
|
+
ns, key = split_uri(pred)
|
|
208
208
|
if ns is not None:
|
|
209
209
|
if ns.startswith('http'):
|
|
210
210
|
for name, n in self.namespacelib.items():
|
|
@@ -221,7 +221,7 @@ class Ui(QtWidgets.QMainWindow, Ui_MainWindow):
|
|
|
221
221
|
if pred is None:
|
|
222
222
|
pred = self.lineEditIRI2.text()
|
|
223
223
|
self.log_message(f'updating prefix and suffix from text: {pred}')
|
|
224
|
-
ns, key =
|
|
224
|
+
ns, key = split_uri(pred)
|
|
225
225
|
self.log_message(f'Try to update combo boxes with ns={ns} and key={key}')
|
|
226
226
|
if ns is not None:
|
|
227
227
|
if ns.startswith('http'):
|
|
@@ -243,7 +243,7 @@ class Ui(QtWidgets.QMainWindow, Ui_MainWindow):
|
|
|
243
243
|
|
|
244
244
|
self.log_message(f'Updating combo box with data: {current_prefix}...')
|
|
245
245
|
self.comboBoxSuffix1.addItems(self.suffix_dict[current_prefix])
|
|
246
|
-
ns, key =
|
|
246
|
+
ns, key = split_uri(self.lineEditIRI1.text())
|
|
247
247
|
|
|
248
248
|
ns_uri = self.namespacelib[current_prefix]._NS
|
|
249
249
|
|
|
@@ -258,7 +258,7 @@ class Ui(QtWidgets.QMainWindow, Ui_MainWindow):
|
|
|
258
258
|
|
|
259
259
|
self.log_message(f'Updating combo box with data: {current_prefix}...')
|
|
260
260
|
self.comboBoxSuffix2.addItems(self.suffix_dict[current_prefix])
|
|
261
|
-
ns, key =
|
|
261
|
+
ns, key = split_uri(self.lineEditIRI2.text())
|
|
262
262
|
|
|
263
263
|
ns_uri = self.namespacelib[current_prefix]._NS
|
|
264
264
|
|
|
@@ -17,6 +17,7 @@ def process_group(group, graph, blank_node_iri_base: Optional[str] = None):
|
|
|
17
17
|
|
|
18
18
|
group_uri = get_obj_bnode(group, blank_node_iri_base)
|
|
19
19
|
rdf_manager = RDFManager(group.attrs)
|
|
20
|
+
parent_rdf_manager = RDFManager(group.parent.attrs) if group.parent is not None else None
|
|
20
21
|
rdf_type = rdf_manager.type
|
|
21
22
|
rdf_subject = rdf_manager.subject
|
|
22
23
|
|
|
@@ -27,6 +28,8 @@ def process_group(group, graph, blank_node_iri_base: Optional[str] = None):
|
|
|
27
28
|
rdf_predicate = rdf_manager.predicate
|
|
28
29
|
if rdf_predicate.get("SELF", None):
|
|
29
30
|
graph.add((group_uri, rdflib.URIRef(rdf_predicate["SELF"]), rdflib.URIRef(rdf_subject)))
|
|
31
|
+
if parent_rdf_manager.subject:
|
|
32
|
+
graph.add((rdflib.URIRef(parent_rdf_manager.subject), rdflib.URIRef(rdf_predicate["SELF"]), rdflib.URIRef(rdf_subject)))
|
|
30
33
|
rdf_file_predicate = rdf_manager.file_predicate
|
|
31
34
|
if rdf_file_predicate is not None:
|
|
32
35
|
file_uri = get_file_bnode(group.file, blank_node_iri_base)
|
|
@@ -26,8 +26,8 @@ def download_file(file_url,
|
|
|
26
26
|
return existing_filename
|
|
27
27
|
|
|
28
28
|
if target_folder is None:
|
|
29
|
-
from ..user import
|
|
30
|
-
target_folder =
|
|
29
|
+
from ..user import CACHE_DIR
|
|
30
|
+
target_folder = CACHE_DIR
|
|
31
31
|
else:
|
|
32
32
|
logger.debug(f'A target folder was specified. Downloading file to this folder: {target_folder}')
|
|
33
33
|
target_folder = pathlib.Path(target_folder)
|
|
@@ -53,7 +53,12 @@ def download_file(file_url,
|
|
|
53
53
|
except ValueError:
|
|
54
54
|
raise ValueError(f"Unsupported checksum algorithm: {checksum_algorithm}")
|
|
55
55
|
|
|
56
|
-
|
|
56
|
+
try:
|
|
57
|
+
response = requests.get(file_url, stream=True)
|
|
58
|
+
if response.status_code == 403:
|
|
59
|
+
response = requests.get(file_url, stream=True, params={'access_token': access_token})
|
|
60
|
+
except requests.RequestException as e:
|
|
61
|
+
response = requests.get(file_url, stream=True, params={'access_token': access_token})
|
|
57
62
|
response.raise_for_status()
|
|
58
63
|
|
|
59
64
|
chunk_size = 1024 # Define chunk size for download
|
|
@@ -101,7 +106,11 @@ def download_file(file_url,
|
|
|
101
106
|
# f'Expected {checksum_algorithm} checksum: {checksum}, '
|
|
102
107
|
# f'but got: {file_checksum}')
|
|
103
108
|
else:
|
|
104
|
-
|
|
109
|
+
try:
|
|
110
|
+
r = requests.get(file_url)
|
|
111
|
+
except requests.RequestException as e:
|
|
112
|
+
r = requests.get(file_url, params={'access_token': access_token})
|
|
113
|
+
|
|
105
114
|
r.raise_for_status()
|
|
106
115
|
|
|
107
116
|
try:
|
|
@@ -116,7 +125,10 @@ def download_file(file_url,
|
|
|
116
125
|
with open(target_filename, 'wb') as file:
|
|
117
126
|
file.write(r.content)
|
|
118
127
|
if links_content:
|
|
119
|
-
|
|
128
|
+
try:
|
|
129
|
+
_content_response = requests.get(links_content)
|
|
130
|
+
except requests.RequestException:
|
|
131
|
+
_content_response = requests.get(links_content, params={'access_token': access_token})
|
|
120
132
|
if _content_response.ok:
|
|
121
133
|
with open(target_filename, 'wb') as file:
|
|
122
134
|
file.write(_content_response.content)
|
|
@@ -203,15 +203,26 @@ class ZenodoRecord(RepositoryInterface):
|
|
|
203
203
|
"""Get the access token for the Zenodo API. This is needed to upload files."""
|
|
204
204
|
return get_api_token(sandbox=self.sandbox, env_var_name=self.env_name_for_token)
|
|
205
205
|
|
|
206
|
-
def _get(self):
|
|
206
|
+
def _get(self, authenticate: bool = False):
|
|
207
|
+
"""Get the deposit metadata as JSON. Using authenticate=True enforces authentication."""
|
|
207
208
|
url = f"{self.depositions_url}/{self.rec_id}"
|
|
208
|
-
|
|
209
|
-
|
|
209
|
+
|
|
210
|
+
if authenticate:
|
|
211
|
+
access_token = self.access_token
|
|
212
|
+
r = requests.get(url, params={"access_token": access_token})
|
|
213
|
+
r.raise_for_status()
|
|
214
|
+
return r.json()
|
|
215
|
+
|
|
216
|
+
record_url = url.replace("deposit/depositions/", "records/")
|
|
217
|
+
r = requests.get(record_url)
|
|
218
|
+
if r.status_code == 404:
|
|
219
|
+
access_token = self.access_token
|
|
220
|
+
r = requests.get(url, params={"access_token": access_token})
|
|
210
221
|
r.raise_for_status()
|
|
211
222
|
return r.json()
|
|
212
223
|
|
|
213
|
-
def get_metadata(self) -> Dict:
|
|
214
|
-
return self._get()['metadata']
|
|
224
|
+
def get_metadata(self, authenticate=False) -> Dict:
|
|
225
|
+
return self._get(authenticate=authenticate)['metadata']
|
|
215
226
|
|
|
216
227
|
def set_metadata(self, metadata: Union[Dict, Metadata]):
|
|
217
228
|
"""update the metadata of the deposit"""
|
|
@@ -247,8 +258,13 @@ class ZenodoRecord(RepositoryInterface):
|
|
|
247
258
|
|
|
248
259
|
def exists(self) -> bool:
|
|
249
260
|
"""Check if the deposit exists on Zenodo. Note, that only published records are detected!"""
|
|
250
|
-
|
|
251
|
-
|
|
261
|
+
url = f"{self.depositions_url}/{self.rec_id}"
|
|
262
|
+
record_url = url.replace("deposit/depositions/", "records/")
|
|
263
|
+
r = requests.get(record_url)
|
|
264
|
+
if r.status_code == 404:
|
|
265
|
+
access_token = self.access_token
|
|
266
|
+
r = requests.get(url, params={"access_token": access_token})
|
|
267
|
+
return r.ok
|
|
252
268
|
|
|
253
269
|
def is_published(self) -> bool:
|
|
254
270
|
"""Check if the deposit is published."""
|
|
@@ -268,6 +284,19 @@ class ZenodoRecord(RepositoryInterface):
|
|
|
268
284
|
return url
|
|
269
285
|
|
|
270
286
|
def _parse(data: Dict):
|
|
287
|
+
filename = data.get("filename", None)
|
|
288
|
+
if filename is None:
|
|
289
|
+
return dict(
|
|
290
|
+
download_url=_parse_download_url(data['links']['self'], data['key']),
|
|
291
|
+
access_url=f"https://doi.org/{self.get_doi()}",
|
|
292
|
+
name=data["key"],
|
|
293
|
+
media_type=_get_media_type(data.get('key')),
|
|
294
|
+
identifier=data.get('id', None),
|
|
295
|
+
identifier_url=None,
|
|
296
|
+
size=data.get('size', None),
|
|
297
|
+
checksum=data.get('checksum').strip("md5:") if data.get('checksum') else None,
|
|
298
|
+
checksum_algorithm=data.get('checksum_algorithm', "md5"),
|
|
299
|
+
)
|
|
271
300
|
return dict(
|
|
272
301
|
download_url=_parse_download_url(data['links']['download'], data['filename']),
|
|
273
302
|
access_url=f"https://doi.org/{self.get_doi()}",
|
|
@@ -343,10 +372,11 @@ class ZenodoRecord(RepositoryInterface):
|
|
|
343
372
|
if new_version_string is not None and increase_part is not None:
|
|
344
373
|
raise ValueError('Only one of new_version_string or increase_part must be provided.')
|
|
345
374
|
if increase_part is not None:
|
|
346
|
-
|
|
375
|
+
metadata = self._get(authenticate=True)["metadata"]
|
|
376
|
+
new_version_string = _bump_version(metadata['version'], increase_part)
|
|
347
377
|
|
|
348
378
|
self.unlock()
|
|
349
|
-
jdata = self._get()
|
|
379
|
+
jdata = self._get(authenticate=True)
|
|
350
380
|
|
|
351
381
|
curr_version = Version(jdata['metadata']['version'])
|
|
352
382
|
new_version = Version(new_version_string)
|
|
@@ -364,7 +394,7 @@ class ZenodoRecord(RepositoryInterface):
|
|
|
364
394
|
|
|
365
395
|
new_record = copy.deepcopy(self)
|
|
366
396
|
new_record.rec_id = _id
|
|
367
|
-
current_metadata = new_record.get_metadata()
|
|
397
|
+
current_metadata = new_record.get_metadata(authenticate=True)
|
|
368
398
|
current_metadata["version"] = str(new_version)
|
|
369
399
|
new_record.set_metadata(current_metadata)
|
|
370
400
|
return new_record
|
|
@@ -402,7 +432,7 @@ class ZenodoRecord(RepositoryInterface):
|
|
|
402
432
|
# Fallback: try to refresh deposit data (may still work) to get id
|
|
403
433
|
if rec_id is None:
|
|
404
434
|
try:
|
|
405
|
-
jdata = self._get()
|
|
435
|
+
jdata = self._get(authenticate=True)
|
|
406
436
|
rec_id = jdata.get("id")
|
|
407
437
|
except Exception:
|
|
408
438
|
rec_id = None
|
|
@@ -424,13 +454,15 @@ class ZenodoRecord(RepositoryInterface):
|
|
|
424
454
|
if not md:
|
|
425
455
|
raise ValueError("No metadata found in the record.")
|
|
426
456
|
|
|
427
|
-
license_ = md.get("license"
|
|
457
|
+
license_ = md.get("license", None)
|
|
458
|
+
if isinstance(license_, dict):
|
|
459
|
+
license_ = license_["id"]
|
|
428
460
|
creators = md.get("creators") or []
|
|
429
461
|
keywords = md.get("keywords") or []
|
|
430
462
|
doi = md.get("doi")
|
|
431
463
|
landing = record.get("links", {}).get("self")
|
|
432
464
|
|
|
433
|
-
publisher =
|
|
465
|
+
publisher = md.get("imprint_publisher", None)
|
|
434
466
|
if publisher:
|
|
435
467
|
if publisher.lower() == "zenodo":
|
|
436
468
|
publisher = foaf.Organization(
|
|
@@ -465,16 +497,17 @@ class ZenodoRecord(RepositoryInterface):
|
|
|
465
497
|
key = f.get("key") or f.get("filename") or "file"
|
|
466
498
|
doi = doi or str(record.get("doi"))
|
|
467
499
|
title = key
|
|
468
|
-
media_type = pathlib.Path(
|
|
469
|
-
size = f.get("filesize")
|
|
500
|
+
media_type = pathlib.Path(key).suffix
|
|
501
|
+
size = f.get("filesize", f.get("size", None))
|
|
470
502
|
links = f.get("links", {})
|
|
471
503
|
access_url = links.get("self") or record.get("links", {}).get("files")
|
|
472
|
-
download_url = links.get("download")
|
|
504
|
+
download_url = links.get("download", links.get("self"))
|
|
473
505
|
if record["submitted"]:
|
|
474
506
|
download_url = download_url.replace("/draft/files", "/files")
|
|
475
507
|
checksum_value = f.get("checksum")
|
|
476
508
|
if checksum_value:
|
|
477
|
-
|
|
509
|
+
if not checksum_value.startswith(("md5:", "sha1:", "sha256:", "sha512:")):
|
|
510
|
+
checksum_value = f"md5:{checksum_value}"
|
|
478
511
|
checksum_obj = _spdx_checksum_from_zenodo(checksum_value)
|
|
479
512
|
|
|
480
513
|
dist = {
|
|
@@ -507,7 +540,7 @@ class ZenodoRecord(RepositoryInterface):
|
|
|
507
540
|
"distribution": distributions,
|
|
508
541
|
}
|
|
509
542
|
dcat_dataset = dcat.Dataset.model_validate(
|
|
510
|
-
dataset
|
|
543
|
+
remove_none(dataset)
|
|
511
544
|
)
|
|
512
545
|
return dcat_dataset
|
|
513
546
|
|
|
@@ -516,7 +549,7 @@ class ZenodoRecord(RepositoryInterface):
|
|
|
516
549
|
|
|
517
550
|
def discard(self):
|
|
518
551
|
"""Discard the latest action, e.g. creating a new version"""
|
|
519
|
-
jdata = self._get()
|
|
552
|
+
jdata = self._get(authenticate=True)
|
|
520
553
|
r = requests.post(jdata['links']['discard'],
|
|
521
554
|
params={'access_token': self.access_token})
|
|
522
555
|
r.raise_for_status()
|
|
@@ -532,7 +565,7 @@ class ZenodoRecord(RepositoryInterface):
|
|
|
532
565
|
APIError
|
|
533
566
|
If the record cannot be unlocked because permission is missing.
|
|
534
567
|
"""
|
|
535
|
-
edit_url = self._get()['links'].get('edit', None)
|
|
568
|
+
edit_url = self._get(authenticate=True)['links'].get('edit', None)
|
|
536
569
|
if edit_url is None:
|
|
537
570
|
raise APIError('Unable to unlock the record. Please check your permission of the Zenodo API Token.')
|
|
538
571
|
|
|
@@ -567,7 +600,7 @@ class ZenodoRecord(RepositoryInterface):
|
|
|
567
600
|
# r.raise_for_status()
|
|
568
601
|
|
|
569
602
|
# https://developers.zenodo.org/?python#quickstart-upload
|
|
570
|
-
bucket_url = self._get()["links"]["bucket"]
|
|
603
|
+
bucket_url = self._get(authenticate=True)["links"]["bucket"]
|
|
571
604
|
logger.debug(f'adding file "{filename}" to record "{self.rec_id}"')
|
|
572
605
|
with open(filename, "rb") as fp:
|
|
573
606
|
r = requests.put(f"{bucket_url}/{filename.name}",
|
|
@@ -589,8 +622,14 @@ class ZenodoRecord(RepositoryInterface):
|
|
|
589
622
|
else:
|
|
590
623
|
target_filename = pathlib.Path(target_filename)
|
|
591
624
|
|
|
592
|
-
|
|
625
|
+
_links = self._get()["links"]
|
|
626
|
+
_html_url = _links.get("html", _links.get("self_html"))
|
|
627
|
+
export_url = f"{_html_url}/export/{fmt}"
|
|
628
|
+
|
|
593
629
|
r = requests.get(export_url)
|
|
630
|
+
if r.status_code == 404:
|
|
631
|
+
access_token = self.access_token
|
|
632
|
+
r = requests.get(export_url, params={"access_token": access_token})
|
|
594
633
|
r.raise_for_status()
|
|
595
634
|
with open(target_filename, 'wb') as f:
|
|
596
635
|
f.write(r.content)
|
|
@@ -614,3 +653,11 @@ class ZenodoRecord(RepositoryInterface):
|
|
|
614
653
|
'owl': 'http://www.w3.org/2002/07/owl#',
|
|
615
654
|
'dctype': 'http://purl.org/dc/dcmitype/'
|
|
616
655
|
})
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
def remove_none(obj):
|
|
659
|
+
if isinstance(obj, dict):
|
|
660
|
+
return {k: remove_none(v) for k, v in obj.items() if v is not None}
|
|
661
|
+
if isinstance(obj, list):
|
|
662
|
+
return [remove_none(v) for v in obj if v is not None]
|
|
663
|
+
return obj
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import os
|
|
1
2
|
import pathlib
|
|
2
3
|
import shutil
|
|
3
4
|
import time
|
|
@@ -11,10 +12,11 @@ from ._version import __version__
|
|
|
11
12
|
|
|
12
13
|
USER_LOG_DIR = pathlib.Path(appdirs.user_log_dir('h5rdmtoolbox', version=__version__))
|
|
13
14
|
USER_DATA_DIR = pathlib.Path(appdirs.user_data_dir('h5rdmtoolbox', version=__version__))
|
|
14
|
-
|
|
15
|
+
CACHE_DIR = pathlib.Path(appdirs.user_cache_dir('h5rdmtoolbox'))
|
|
15
16
|
USER_LOG_DIR.mkdir(parents=True, exist_ok=True)
|
|
16
17
|
USER_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
|
17
|
-
|
|
18
|
+
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
|
19
|
+
USER_CACHE_DIR = CACHE_DIR # deprecated name
|
|
18
20
|
|
|
19
21
|
_filecounter = count()
|
|
20
22
|
_dircounter = count()
|
|
@@ -22,6 +24,18 @@ _dircounter = count()
|
|
|
22
24
|
_now = time.time()
|
|
23
25
|
|
|
24
26
|
|
|
27
|
+
def default_cache_dir(__version__: str) -> pathlib.Path:
|
|
28
|
+
# Your current approach
|
|
29
|
+
return USER_DATA_DIR / "cache"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def cache_dir(__version__: str) -> pathlib.Path:
|
|
33
|
+
override = os.environ.get("H5RDMTOOLBOX_CACHE_DIR")
|
|
34
|
+
if override:
|
|
35
|
+
return pathlib.Path(override)
|
|
36
|
+
return default_cache_dir(__version__)
|
|
37
|
+
|
|
38
|
+
|
|
25
39
|
class DirManger:
|
|
26
40
|
"""Directory Manager class"""
|
|
27
41
|
|
|
@@ -6,6 +6,7 @@ import json
|
|
|
6
6
|
import logging
|
|
7
7
|
import os
|
|
8
8
|
import pathlib
|
|
9
|
+
import random
|
|
9
10
|
import re
|
|
10
11
|
import time
|
|
11
12
|
import uuid
|
|
@@ -25,10 +26,11 @@ from . import get_config, get_ureg
|
|
|
25
26
|
from . import user
|
|
26
27
|
from ._version import __version__
|
|
27
28
|
from .ld import rdf
|
|
28
|
-
from .user import
|
|
29
|
+
from .user import CACHE_DIR, USER_DATA_DIR
|
|
29
30
|
|
|
30
31
|
logger = logging.getLogger('h5rdmtoolbox')
|
|
31
32
|
DEFAULT_LOGGING_LEVEL = logging.INFO
|
|
33
|
+
RETRY_STATUS = {429, 500, 502, 503, 504}
|
|
32
34
|
|
|
33
35
|
|
|
34
36
|
def get_filesize(filename: Union[str, pathlib.Path]) -> int:
|
|
@@ -52,12 +54,44 @@ def has_internet_connection(timeout: int = 5) -> bool:
|
|
|
52
54
|
return False
|
|
53
55
|
|
|
54
56
|
|
|
57
|
+
def _request_with_backoff(method, url, session=None, max_retries=8, timeout=30, **kwargs):
|
|
58
|
+
r = None
|
|
59
|
+
s = session or requests.Session()
|
|
60
|
+
for attempt in range(max_retries + 1):
|
|
61
|
+
r = s.request(method, url, timeout=timeout, **kwargs)
|
|
62
|
+
|
|
63
|
+
if r.status_code not in RETRY_STATUS:
|
|
64
|
+
return r
|
|
65
|
+
|
|
66
|
+
# Prefer server guidance
|
|
67
|
+
retry_after = r.headers.get("Retry-After")
|
|
68
|
+
if retry_after is not None:
|
|
69
|
+
try:
|
|
70
|
+
sleep_s = float(retry_after)
|
|
71
|
+
except ValueError:
|
|
72
|
+
sleep_s = None
|
|
73
|
+
else:
|
|
74
|
+
sleep_s = None
|
|
75
|
+
|
|
76
|
+
if sleep_s is None:
|
|
77
|
+
# Exponential backoff with jitter (full jitter)
|
|
78
|
+
base = min(60.0, 0.5 * (2 ** attempt))
|
|
79
|
+
sleep_s = random.uniform(0, base)
|
|
80
|
+
|
|
81
|
+
if attempt == max_retries:
|
|
82
|
+
return r
|
|
83
|
+
|
|
84
|
+
time.sleep(sleep_s)
|
|
85
|
+
|
|
86
|
+
return r
|
|
87
|
+
|
|
88
|
+
|
|
55
89
|
def _download_file(url,
|
|
56
90
|
known_hash,
|
|
57
91
|
target: Optional[pathlib.Path] = None,
|
|
58
92
|
params: Optional[Dict] = None) -> pathlib.Path:
|
|
59
93
|
"""Download a file from a URL and check its hash"""
|
|
60
|
-
response =
|
|
94
|
+
response = _request_with_backoff('GET', url, params=params, stream=True)
|
|
61
95
|
if response.status_code == 200:
|
|
62
96
|
content = response.content
|
|
63
97
|
|
|
@@ -560,7 +594,8 @@ def download_context(url_source: Union[HttpUrl, List[HttpUrl]], force_download:
|
|
|
560
594
|
logger.debug(f'Downloading context file from {_url} to {context_file}')
|
|
561
595
|
try:
|
|
562
596
|
with open(context_file, 'wb') as f:
|
|
563
|
-
|
|
597
|
+
r = _request_with_backoff("GET", _url)
|
|
598
|
+
f.write(r.content)
|
|
564
599
|
except requests.RequestException:
|
|
565
600
|
raise RuntimeError(f'Failed to download context file from {_url}')
|
|
566
601
|
filenames.append(str(context_file))
|
|
@@ -608,7 +643,7 @@ class DownloadFileManager:
|
|
|
608
643
|
return cls._instance
|
|
609
644
|
|
|
610
645
|
def __init__(self):
|
|
611
|
-
self.file_directory =
|
|
646
|
+
self.file_directory = CACHE_DIR
|
|
612
647
|
self.file_directory.mkdir(parents=True, exist_ok=True)
|
|
613
648
|
self.registry: Dict[str, Dict[str, str]] = self.load_registry()
|
|
614
649
|
atexit.register(self.save_registry)
|