rudi-node-write 1.3.4__tar.gz → 1.3.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rudi_node_write-1.3.4/src/rudi_node_write.egg-info → rudi_node_write-1.3.6}/PKG-INFO +2 -3
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/pyproject.toml +2 -2
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/requirements-dev.txt +6 -8
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/conf/meta_defaults.py +1 -1
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/io_connector.py +4 -3
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/io_rudi_catalog_write.py +5 -5
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/io_rudi_manager_write.py +31 -7
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/io_rudi_manager_write_v3.py +50 -21
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_node_writer.py +13 -9
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_media.py +2 -2
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/err.py +3 -3
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/str_utils.py +5 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6/src/rudi_node_write.egg-info}/PKG-INFO +2 -3
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write.egg-info/requires.txt +1 -2
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/LICENCE.md +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/README.md +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/requirements.txt +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/setup.cfg +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/__init__.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/io_rudi_jwt_factory.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/io_rudi_manager_write_v2.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/io_rudi_storage_write.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/rudi_node_auth.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_const.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_contact.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_dates.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_dictionary_entry.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_geo.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_licence.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_meta.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_meta_misc.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_org.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/serializable.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/dict_utils.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/file_utils.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/html_utils.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/jwt.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/list_utils.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/log.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/type_date.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/typing_utils.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/url_utils.py +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write.egg-info/SOURCES.txt +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write.egg-info/dependency_links.txt +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write.egg-info/top_level.txt +0 -0
- {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/tests/test_rudi_node_write.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rudi-node-write
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.6
|
|
4
4
|
Summary: Use the internal API of a RUDI Producer node
|
|
5
5
|
Author-email: Olivier Martineau <olivier.martineau@irisa.fr>
|
|
6
6
|
Maintainer-email: Olivier Martineau <olivier.martineau@irisa.fr>
|
|
@@ -58,13 +58,12 @@ Requires-Dist: protobuf==6.31.1; extra == "dev"
|
|
|
58
58
|
Requires-Dist: py-env==0.0.1; extra == "dev"
|
|
59
59
|
Requires-Dist: pybind11==2.13.6; extra == "dev"
|
|
60
60
|
Requires-Dist: pyright==1.1.401; extra == "dev"
|
|
61
|
-
Requires-Dist: pyside6-addons==6.9.
|
|
61
|
+
Requires-Dist: pyside6-addons==6.9.1; extra == "dev"
|
|
62
62
|
Requires-Dist: pytest-check==2.5.3; extra == "dev"
|
|
63
63
|
Requires-Dist: pytest-cov==6.1.1; extra == "dev"
|
|
64
64
|
Requires-Dist: python-dotenv==1.1.0; extra == "dev"
|
|
65
65
|
Requires-Dist: python-multipart==0.0.20; extra == "dev"
|
|
66
66
|
Requires-Dist: regex==2024.11.6; extra == "dev"
|
|
67
|
-
Requires-Dist: rudi-node-write==1.3.3; extra == "dev"
|
|
68
67
|
Requires-Dist: timm==1.0.15; extra == "dev"
|
|
69
68
|
Requires-Dist: tomli==2.2.1; extra == "dev"
|
|
70
69
|
Requires-Dist: twine==6.1.0; extra == "dev"
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "rudi-node-write"
|
|
7
|
-
version = "1.3.
|
|
7
|
+
version = "1.3.6"
|
|
8
8
|
authors = [{ name = "Olivier Martineau", email = "olivier.martineau@irisa.fr" }]
|
|
9
9
|
maintainers = [
|
|
10
10
|
{ name = "Olivier Martineau", email = "olivier.martineau@irisa.fr" },
|
|
@@ -45,7 +45,7 @@ target-version = ['py311']
|
|
|
45
45
|
# ----- Tool: commitizen
|
|
46
46
|
[tool.commitizen]
|
|
47
47
|
name = "cz_conventional_commits"
|
|
48
|
-
version = "1.3.
|
|
48
|
+
version = "1.3.6"
|
|
49
49
|
version_files = ["pyproject.toml:version"]
|
|
50
50
|
|
|
51
51
|
# ----- Tool: pytest
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -r requirements.txt
|
|
2
1
|
backports.tarfile==1.2.0
|
|
3
2
|
black==25.1.0
|
|
4
3
|
build==1.2.2.post1
|
|
@@ -36,13 +35,12 @@ protobuf==6.31.1
|
|
|
36
35
|
py-env==0.0.1
|
|
37
36
|
pybind11==2.13.6
|
|
38
37
|
pyright==1.1.401
|
|
39
|
-
pyside6-addons==6.9.
|
|
38
|
+
pyside6-addons==6.9.1
|
|
40
39
|
pytest-check==2.5.3
|
|
41
40
|
pytest-cov==6.1.1
|
|
42
41
|
python-dotenv==1.1.0
|
|
43
42
|
python-multipart==0.0.20
|
|
44
43
|
regex==2024.11.6
|
|
45
|
-
rudi-node-write==1.3.3
|
|
46
44
|
timm==1.0.15
|
|
47
45
|
tomli==2.2.1
|
|
48
46
|
twine==6.1.0
|
|
@@ -87,7 +85,7 @@ websockets==15.0.1
|
|
|
87
85
|
# hf-xet==1.1.2 # Installed as dependency for huggingface-hub
|
|
88
86
|
# hidapi==0.14.0.post4 # Installed as dependency for onlykey
|
|
89
87
|
# httpcore==1.0.9 # Installed as dependency for httpx
|
|
90
|
-
# huggingface-hub==0.32.
|
|
88
|
+
# huggingface-hub==0.32.4 # Installed as dependency for timm
|
|
91
89
|
# id==1.5.0 # Installed as dependency for twine
|
|
92
90
|
# identify==2.6.12 # Installed as dependency for pre-commit
|
|
93
91
|
# idna==3.10 # Installed as dependency for anyio, email-validator, httpx, requests
|
|
@@ -143,13 +141,13 @@ websockets==15.0.1
|
|
|
143
141
|
# pydantic==2.11.5 # Installed as dependency for fastapi
|
|
144
142
|
# pydantic-core==2.34.1 # Installed as dependency for pydantic
|
|
145
143
|
# pyflakes==3.3.2 # Installed as dependency for flake8
|
|
146
|
-
# pygments==2.19.1 # Installed as dependency for ipython, ipython-pygments-lexers, readme-renderer, rich
|
|
144
|
+
# pygments==2.19.1 # Installed as dependency for ipython, ipython-pygments-lexers, pytest, readme-renderer, rich
|
|
147
145
|
# pymsgbox==1.0.9 # Installed as dependency for lib-agent
|
|
148
146
|
# pynacl==1.5.0 # Installed as dependency for lib-agent, onlykey
|
|
149
147
|
# pyproject-hooks==1.2.0 # Installed as dependency for build
|
|
150
148
|
# pyserial==3.5 # Installed as dependency for onlykey-solo-python
|
|
151
|
-
# pyside6-essentials==6.9.
|
|
152
|
-
# pytest==8.
|
|
149
|
+
# pyside6-essentials==6.9.1 # Installed as dependency for pyside6-addons
|
|
150
|
+
# pytest==8.4.0 # Installed as dependency for pytest-check, pytest-cov
|
|
153
151
|
# python-daemon==3.1.2 # Installed as dependency for lib-agent
|
|
154
152
|
# python-dateutil==2.9.0.post0 # Installed as dependency for jupyter-client
|
|
155
153
|
# pyusb==1.3.1 # Installed as dependency for onlykey-solo-python
|
|
@@ -169,7 +167,7 @@ websockets==15.0.1
|
|
|
169
167
|
# safetensors==0.5.3 # Installed as dependency for timm
|
|
170
168
|
# semver==3.0.4 # Installed as dependency for lib-agent
|
|
171
169
|
# shellingham==1.5.4 # Installed as dependency for typer
|
|
172
|
-
# shiboken6==6.9.
|
|
170
|
+
# shiboken6==6.9.1 # Installed as dependency for pyside6-addons, pyside6-essentials
|
|
173
171
|
# six==1.17.0 # Installed as dependency for cssbeautifier, ecdsa, jsbeautifier, onlykey, python-dateutil
|
|
174
172
|
# sniffio==1.3.1 # Installed as dependency for anyio
|
|
175
173
|
# stack-data==0.6.3 # Installed as dependency for ipython
|
|
@@ -6,7 +6,7 @@ from rudi_node_write.rudi_types.rudi_const import Language
|
|
|
6
6
|
DEFAULT_LANG: Language = "fr"
|
|
7
7
|
|
|
8
8
|
# Current version for RUDI Node metadata
|
|
9
|
-
RUDI_API_VERSION: Final[str] = "1.
|
|
9
|
+
RUDI_API_VERSION: Final[str] = "1.4.0"
|
|
10
10
|
|
|
11
11
|
# Current version for ODS
|
|
12
12
|
ODS_API_VERSION: str = "2.1"
|
{rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/io_connector.py
RENAMED
|
@@ -244,7 +244,7 @@ class Connector:
|
|
|
244
244
|
if not keep_alive and not self.keep_connection:
|
|
245
245
|
self.close_connection()
|
|
246
246
|
|
|
247
|
-
if
|
|
247
|
+
if isinstance(response_data, str):
|
|
248
248
|
log_d_if(self.should_log_response, here, "Response is a string", response_data)
|
|
249
249
|
if response.status < 400:
|
|
250
250
|
return rdata.decode("utf8")
|
|
@@ -259,10 +259,11 @@ class Connector:
|
|
|
259
259
|
|
|
260
260
|
if __name__ == "__main__": # pragma: no cover
|
|
261
261
|
tests = "Connector tests"
|
|
262
|
-
|
|
262
|
+
node_url = "https://bacasable.fenix.rudi-univ-rennes1.fr"
|
|
263
|
+
connector = Connector(f"{node_url}/catalog/version")
|
|
263
264
|
log_d(tests, "testing connection, got version", connector.test_connection())
|
|
264
265
|
|
|
265
|
-
connector = Connector("
|
|
266
|
+
connector = Connector(f"{node_url}/catalog/v1")
|
|
266
267
|
metadata_list = connector.request("resources")
|
|
267
268
|
if isinstance(metadata_list, dict) and metadata_list["total"] is not None:
|
|
268
269
|
log_d(tests, "number of resources declared", f"{metadata_list['total']}")
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
from os.path import isdir
|
|
1
|
+
from os.path import isdir
|
|
2
2
|
from time import time
|
|
3
3
|
from urllib.parse import quote
|
|
4
4
|
|
|
5
|
-
from _pytest.config.argparsing import ArgumentError
|
|
6
5
|
|
|
7
6
|
from rudi_node_write.connectors.io_connector import Connector, https_download
|
|
8
7
|
from rudi_node_write.connectors.io_rudi_jwt_factory import RudiNodeJwtFactory
|
|
@@ -28,8 +27,8 @@ from rudi_node_write.utils.file_utils import read_json_file, write_file
|
|
|
28
27
|
from rudi_node_write.utils.jwt import is_jwt_expired
|
|
29
28
|
from rudi_node_write.utils.list_utils import get_first_list_elt_or_none
|
|
30
29
|
from rudi_node_write.utils.log import log_d, log_e, log_w
|
|
31
|
-
from rudi_node_write.utils.str_utils import slash_join, uuid4_str, is_uuid_v4
|
|
32
|
-
from rudi_node_write.utils.typing_utils import get_type_name
|
|
30
|
+
from rudi_node_write.utils.str_utils import absolute_path, slash_join, uuid4_str, is_uuid_v4
|
|
31
|
+
from rudi_node_write.utils.typing_utils import get_type_name
|
|
33
32
|
from rudi_node_write.utils.url_utils import ensure_url_startswith
|
|
34
33
|
|
|
35
34
|
|
|
@@ -663,11 +662,12 @@ class RudiNodeCatalogConnector(Connector):
|
|
|
663
662
|
raise FileNotFoundError(f"The following folder does not exist: '{local_download_dir}'")
|
|
664
663
|
|
|
665
664
|
media_name = media.get("media_name")
|
|
665
|
+
assert isinstance(media_name, str)
|
|
666
666
|
media_url = safe_get_key(media, "connector", "url")
|
|
667
667
|
if not media_url:
|
|
668
668
|
raise FileNotFoundError(f"No URL was provided for the media {media_name}")
|
|
669
669
|
|
|
670
|
-
destination_path =
|
|
670
|
+
destination_path = absolute_path(local_download_dir, media_name)
|
|
671
671
|
try:
|
|
672
672
|
content = https_download(media_url)
|
|
673
673
|
if not content:
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
from base64 import urlsafe_b64decode, urlsafe_b64encode
|
|
2
1
|
from json import dumps
|
|
3
|
-
from os.path import
|
|
2
|
+
from os.path import exists, isdir
|
|
4
3
|
from ssl import SSLCertVerificationError
|
|
5
4
|
from time import time
|
|
6
5
|
from typing import Literal
|
|
@@ -22,10 +21,10 @@ from rudi_node_write.rudi_types.serializable import Serializable
|
|
|
22
21
|
from rudi_node_write.utils.dict_utils import check_is_dict, merge_dict_of_list, pick_in_dict, safe_get_key
|
|
23
22
|
from rudi_node_write.utils.err import HttpError, HttpErrorNotFound, UnexpectedValueException
|
|
24
23
|
from rudi_node_write.utils.file_utils import check_is_file, read_json_file, write_file
|
|
25
|
-
from rudi_node_write.utils.jwt import
|
|
24
|
+
from rudi_node_write.utils.jwt import is_jwt_expired
|
|
26
25
|
from rudi_node_write.utils.list_utils import get_first_list_elt_or_none
|
|
27
26
|
from rudi_node_write.utils.log import log_d, log_e, log_w
|
|
28
|
-
from rudi_node_write.utils.str_utils import check_is_uuid4, is_uuid_v4, slash_join, uuid4_str
|
|
27
|
+
from rudi_node_write.utils.str_utils import absolute_path, check_is_uuid4, is_uuid_v4, slash_join, uuid4_str
|
|
29
28
|
from rudi_node_write.utils.type_date import Date
|
|
30
29
|
from rudi_node_write.utils.typing_utils import check_is_int, get_type_name
|
|
31
30
|
from rudi_node_write.utils.url_utils import ensure_url_startswith
|
|
@@ -1027,7 +1026,7 @@ class RudiNodeManagerConnector(Connector):
|
|
|
1027
1026
|
if not media_url:
|
|
1028
1027
|
raise FileNotFoundError(f"No URL was provided for the media {media_name}")
|
|
1029
1028
|
|
|
1030
|
-
destination_path =
|
|
1029
|
+
destination_path = absolute_path(local_download_dir, media_name)
|
|
1031
1030
|
try:
|
|
1032
1031
|
content = https_download(media_url)
|
|
1033
1032
|
if not content:
|
|
@@ -1131,6 +1130,7 @@ class RudiNodeManagerConnector(Connector):
|
|
|
1131
1130
|
self,
|
|
1132
1131
|
file_local_path: str,
|
|
1133
1132
|
media_id: str = uuid4_str(),
|
|
1133
|
+
rudi_media: RudiMediaFile | None = None,
|
|
1134
1134
|
):
|
|
1135
1135
|
"""
|
|
1136
1136
|
:param file_local_path: the path of a local file we wish to send to a RUDI node Storage server
|
|
@@ -1139,9 +1139,20 @@ class RudiNodeManagerConnector(Connector):
|
|
|
1139
1139
|
"""
|
|
1140
1140
|
here = f"{self.class_name}.post_local_file"
|
|
1141
1141
|
|
|
1142
|
+
if rudi_media is None:
|
|
1143
|
+
rudi_media = RudiMediaFile.from_local_file(file_local_path, media_id)
|
|
1144
|
+
else:
|
|
1145
|
+
check_media_info = RudiMediaFile.from_local_file(file_local_path, media_id)
|
|
1146
|
+
rudi_media.file_size = check_media_info.file_size
|
|
1147
|
+
rudi_media.checksum = check_media_info.checksum
|
|
1148
|
+
|
|
1149
|
+
media_id = rudi_media.media_id
|
|
1150
|
+
|
|
1142
1151
|
# Posting the file as a binary directly to RUDI node Storage storage module
|
|
1143
1152
|
res = self.storage_connector.post_local_file(
|
|
1144
|
-
file_local_path=check_is_file(file_local_path),
|
|
1153
|
+
file_local_path=check_is_file(file_local_path),
|
|
1154
|
+
media_id=check_is_uuid4(media_id),
|
|
1155
|
+
rudi_media=rudi_media,
|
|
1145
1156
|
)
|
|
1146
1157
|
if res is None or not isinstance(res, dict) or res["media_info"] is None:
|
|
1147
1158
|
raise Exception("Upload to RUDI Media failed!")
|
|
@@ -1277,10 +1288,16 @@ class RudiNodeStorageConnector(Connector):
|
|
|
1277
1288
|
media_info = media_list.get("zone1") # What if there are other zones?
|
|
1278
1289
|
return [] if media_info is None else media_info.get("list")
|
|
1279
1290
|
|
|
1280
|
-
def post_local_file(
|
|
1291
|
+
def post_local_file(
|
|
1292
|
+
self,
|
|
1293
|
+
file_local_path: str,
|
|
1294
|
+
media_id: str = uuid4_str(),
|
|
1295
|
+
rudi_media: RudiMediaFile | None = None,
|
|
1296
|
+
):
|
|
1281
1297
|
"""
|
|
1282
1298
|
:param file_local_path: the path of a local file we wish to send to a RUDI node Media server
|
|
1283
1299
|
:param media_id: the UUIDv4 that identifies the media on the RUDI node
|
|
1300
|
+
:param rudi_media: an optional RudiMediaFile object (in the case of an update)
|
|
1284
1301
|
:return:
|
|
1285
1302
|
"""
|
|
1286
1303
|
# :param media_name: the original name of the file
|
|
@@ -1289,6 +1306,13 @@ class RudiNodeStorageConnector(Connector):
|
|
|
1289
1306
|
# :param charset: the encoding of the file
|
|
1290
1307
|
here = f"{self.class_name}.post_local_file"
|
|
1291
1308
|
|
|
1309
|
+
if rudi_media is None:
|
|
1310
|
+
rudi_media = RudiMediaFile.from_local_file(file_local_path, media_id)
|
|
1311
|
+
else:
|
|
1312
|
+
check_media_info = RudiMediaFile.from_local_file(file_local_path, media_id)
|
|
1313
|
+
rudi_media.file_size = check_media_info.file_size
|
|
1314
|
+
rudi_media.checksum = check_media_info.checksum
|
|
1315
|
+
|
|
1292
1316
|
rudi_media = RudiMediaFile.from_local_file(file_local_path, media_id)
|
|
1293
1317
|
if rudi_media.file_size > FileTooBigException.MAX_SIZE:
|
|
1294
1318
|
raise FileTooBigException(rudi_media.file_size)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from http.client import IncompleteRead
|
|
2
2
|
from json import dumps
|
|
3
|
-
from os.path import
|
|
3
|
+
from os.path import exists, isdir
|
|
4
4
|
from re import T
|
|
5
5
|
from time import time
|
|
6
6
|
from typing import Literal
|
|
@@ -26,7 +26,7 @@ from rudi_node_write.utils.file_utils import check_is_file, read_json_file, writ
|
|
|
26
26
|
from rudi_node_write.utils.jwt import is_jwt_expired
|
|
27
27
|
from rudi_node_write.utils.list_utils import get_first_list_elt_or_none
|
|
28
28
|
from rudi_node_write.utils.log import log_d, log_e, log_w
|
|
29
|
-
from rudi_node_write.utils.str_utils import check_is_uuid4, is_uuid_v4, slash_join, uuid4_str
|
|
29
|
+
from rudi_node_write.utils.str_utils import absolute_path, check_is_uuid4, is_uuid_v4, slash_join, uuid4_str
|
|
30
30
|
from rudi_node_write.utils.type_date import Date
|
|
31
31
|
from rudi_node_write.utils.typing_utils import check_is_int, get_type_name
|
|
32
32
|
from rudi_node_write.utils.url_utils import ensure_url_startswith
|
|
@@ -1006,7 +1006,7 @@ class RudiNodeManagerConnectorV3(RudiNodeManagerConnector):
|
|
|
1006
1006
|
if not media_url:
|
|
1007
1007
|
raise FileNotFoundError(f"No URL was provided for the media {media_name}")
|
|
1008
1008
|
|
|
1009
|
-
destination_path =
|
|
1009
|
+
destination_path = absolute_path(local_download_dir, media_name)
|
|
1010
1010
|
try:
|
|
1011
1011
|
content = https_download(media_url)
|
|
1012
1012
|
if not content:
|
|
@@ -1110,18 +1110,30 @@ class RudiNodeManagerConnectorV3(RudiNodeManagerConnector):
|
|
|
1110
1110
|
self,
|
|
1111
1111
|
file_local_path: str,
|
|
1112
1112
|
media_id: str = uuid4_str(),
|
|
1113
|
-
|
|
1113
|
+
rudi_media: RudiMediaFile | None = None,
|
|
1114
|
+
) -> RudiMediaFile:
|
|
1114
1115
|
"""
|
|
1115
1116
|
:param file_local_path: the path of a local file we wish to send to a RUDI node Storage server
|
|
1116
1117
|
:param media_id: the UUIDv4 that identifies the media on the RUDI node
|
|
1117
1118
|
:return:
|
|
1118
1119
|
"""
|
|
1119
|
-
here = f"{self.class_name}
|
|
1120
|
+
here = f"{self.class_name}.post_local_file"
|
|
1121
|
+
|
|
1122
|
+
if rudi_media is None:
|
|
1123
|
+
rudi_media = RudiMediaFile.from_local_file(file_local_path, media_id)
|
|
1124
|
+
else:
|
|
1125
|
+
check_media_info = RudiMediaFile.from_local_file(file_local_path, media_id)
|
|
1126
|
+
rudi_media.file_size = check_media_info.file_size
|
|
1127
|
+
rudi_media.checksum = check_media_info.checksum
|
|
1128
|
+
|
|
1129
|
+
media_id = rudi_media.media_id
|
|
1120
1130
|
|
|
1121
1131
|
# Posting the file as a binary directly to RUDI node Storage module
|
|
1122
1132
|
try:
|
|
1123
1133
|
res = self.storage_connector.post_local_file(
|
|
1124
|
-
file_local_path=check_is_file(file_local_path),
|
|
1134
|
+
file_local_path=check_is_file(file_local_path),
|
|
1135
|
+
media_id=rudi_media.media_id,
|
|
1136
|
+
rudi_media=rudi_media,
|
|
1125
1137
|
)
|
|
1126
1138
|
except IncompleteRead as e:
|
|
1127
1139
|
log_e(here, "Upload to RUDI Storage failed (IncompleteRead)", e)
|
|
@@ -1130,14 +1142,17 @@ class RudiNodeManagerConnectorV3(RudiNodeManagerConnector):
|
|
|
1130
1142
|
except Exception as e:
|
|
1131
1143
|
log_e(here, "Upload to RUDI Storage failed", e)
|
|
1132
1144
|
raise e
|
|
1133
|
-
if res is None or not isinstance(res, dict) or res
|
|
1145
|
+
if res is None or not isinstance(res, dict) or res.get("media_info") is None:
|
|
1134
1146
|
raise Exception("Upload to RUDI Storage failed!")
|
|
1135
1147
|
media_info, zone_name, commit_uuid = (
|
|
1136
1148
|
res["media_info"],
|
|
1137
1149
|
res["zone_name"],
|
|
1138
1150
|
res["commit_uuid"],
|
|
1139
1151
|
)
|
|
1152
|
+
assert isinstance(media_info, RudiMediaFile)
|
|
1140
1153
|
# log_d(here, "media_info", str(media_info))
|
|
1154
|
+
# log_d(here, "zone_name", zone_name)
|
|
1155
|
+
# log_d(here, "commit_uuid", commit_uuid)
|
|
1141
1156
|
|
|
1142
1157
|
# Posting the file metadata as a RudiMediaFile object to the RUDI Catalog module through the PM data API
|
|
1143
1158
|
if not self._gen == 1:
|
|
@@ -1154,19 +1169,19 @@ class RudiNodeManagerConnectorV3(RudiNodeManagerConnector):
|
|
|
1154
1169
|
return media_info
|
|
1155
1170
|
|
|
1156
1171
|
# Committing the file that has been uploaded
|
|
1157
|
-
|
|
1158
|
-
log_d(here, "commit_uuid", commit_uuid)
|
|
1159
|
-
res_commit = self.commit_media(media_id=media_info.media_id, zone_name=zone_name, commit_uuid=commit_uuid)
|
|
1172
|
+
res_commit = self.commit_media(media_id=media_id, zone_name=zone_name, commit_uuid=commit_uuid)
|
|
1160
1173
|
log_d(here, "res_commit", res_commit)
|
|
1161
1174
|
|
|
1162
1175
|
# Returning the updated RudiMedia information
|
|
1163
1176
|
if isinstance(api_media_info, dict):
|
|
1164
1177
|
updated_media_info = RudiMediaFile.from_json(api_media_info)
|
|
1165
|
-
return updated_media_info
|
|
1166
1178
|
else:
|
|
1167
|
-
log_w(here, f"unexpected result, should have a 'RudiMediaFile' compatible dict, got {
|
|
1179
|
+
log_w(here, f"unexpected result, should have a 'RudiMediaFile' compatible dict, got {api_media_info}")
|
|
1168
1180
|
return media_info
|
|
1169
1181
|
|
|
1182
|
+
# Returning the updated RudiMedia information
|
|
1183
|
+
return updated_media_info
|
|
1184
|
+
|
|
1170
1185
|
# TODO: PM API for commit only file / only API / both?
|
|
1171
1186
|
|
|
1172
1187
|
def commit_media(self, media_id: str, commit_uuid: str, zone_name: str, metadata_id: str | None = None):
|
|
@@ -1278,10 +1293,16 @@ class RudiNodeStorageConnectorV3(Connector):
|
|
|
1278
1293
|
|
|
1279
1294
|
return [] if media_info is None else media_info.get("list")
|
|
1280
1295
|
|
|
1281
|
-
def post_local_file(
|
|
1296
|
+
def post_local_file(
|
|
1297
|
+
self,
|
|
1298
|
+
file_local_path: str,
|
|
1299
|
+
media_id: str | None = None,
|
|
1300
|
+
rudi_media: RudiMediaFile | None = None,
|
|
1301
|
+
):
|
|
1282
1302
|
"""
|
|
1283
1303
|
:param file_local_path: the path of a local file we wish to send to a RUDI node Media server
|
|
1284
1304
|
:param media_id: the UUIDv4 that identifies the media on the RUDI node
|
|
1305
|
+
:param rudi_media: an optional RudiMediaFile object (in the case of an update)
|
|
1285
1306
|
:return:
|
|
1286
1307
|
"""
|
|
1287
1308
|
# :param media_name: the original name of the file
|
|
@@ -1290,12 +1311,19 @@ class RudiNodeStorageConnectorV3(Connector):
|
|
|
1290
1311
|
# :param charset: the encoding of the file
|
|
1291
1312
|
here = f"{self.class_name}.post_local_file"
|
|
1292
1313
|
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1314
|
+
if rudi_media is None:
|
|
1315
|
+
rudi_media = RudiMediaFile.from_local_file(file_local_path, media_id)
|
|
1316
|
+
else:
|
|
1317
|
+
check_media_info = RudiMediaFile.from_local_file(file_local_path, media_id)
|
|
1318
|
+
rudi_media.file_size = check_media_info.file_size
|
|
1319
|
+
rudi_media.checksum = check_media_info.checksum
|
|
1320
|
+
|
|
1321
|
+
if rudi_media.file_size > FileTooBigException.MAX_SIZE:
|
|
1322
|
+
raise FileTooBigException(rudi_media.file_size)
|
|
1323
|
+
media_id = rudi_media.media_id
|
|
1296
1324
|
|
|
1297
|
-
log_d(here, "sending as binary for file",
|
|
1298
|
-
headers = self.get_media_headers_for_file(
|
|
1325
|
+
log_d(here, "sending as binary for file", rudi_media)
|
|
1326
|
+
headers = self.get_media_headers_for_file(rudi_media) | {"Content-Type": "octet/stream"}
|
|
1299
1327
|
# log_d(here, "headers", headers)
|
|
1300
1328
|
with open(file_local_path, "rb") as bin_content:
|
|
1301
1329
|
try:
|
|
@@ -1315,17 +1343,18 @@ class RudiNodeStorageConnectorV3(Connector):
|
|
|
1315
1343
|
commit_ready = res[-2]
|
|
1316
1344
|
log_d(here, "commit_ready", commit_ready)
|
|
1317
1345
|
# Updating the connector.url information for the media
|
|
1318
|
-
|
|
1346
|
+
rudi_media.set_url(slash_join(self.base_url, "download", rudi_media.media_id))
|
|
1319
1347
|
# Updating the file storage status information for the media
|
|
1320
|
-
|
|
1348
|
+
rudi_media.set_status("available")
|
|
1321
1349
|
return {
|
|
1322
|
-
"media_info":
|
|
1350
|
+
"media_info": rudi_media,
|
|
1323
1351
|
"zone_name": commit_ready["zone_name"],
|
|
1324
1352
|
"commit_uuid": commit_ready["commit_uuid"],
|
|
1325
1353
|
}
|
|
1326
1354
|
if len(res) > 0 and res[0].get("status") == "error":
|
|
1327
1355
|
log_w(here, "ERR upload failed", res[0].get("msg"))
|
|
1328
1356
|
raise Exception("File upload to RUDI Media falied:" + res[0])
|
|
1357
|
+
|
|
1329
1358
|
return res
|
|
1330
1359
|
|
|
1331
1360
|
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
from json import dumps, loads
|
|
2
|
-
from os.path import
|
|
2
|
+
from os.path import isdir
|
|
3
3
|
from time import time
|
|
4
4
|
|
|
5
5
|
from rudi_node_write.connectors.io_connector import https_download
|
|
6
|
-
from rudi_node_write.connectors.io_rudi_manager_write import RudiNodeManagerConnector
|
|
7
6
|
from rudi_node_write.connectors.io_rudi_manager_write_v2 import RudiNodeManagerConnectorV2
|
|
8
7
|
from rudi_node_write.connectors.io_rudi_manager_write_v3 import RudiNodeManagerConnectorV3
|
|
9
8
|
from rudi_node_write.connectors.rudi_node_auth import RudiNodeAuth
|
|
10
|
-
from rudi_node_write.rudi_types.rudi_meta import RudiMetadata
|
|
9
|
+
from rudi_node_write.rudi_types.rudi_meta import RudiMediaFile, RudiMetadata
|
|
11
10
|
from rudi_node_write.utils.dict_utils import (
|
|
12
11
|
check_get_key,
|
|
13
12
|
filter_dict_list,
|
|
@@ -17,7 +16,7 @@ from rudi_node_write.utils.dict_utils import (
|
|
|
17
16
|
)
|
|
18
17
|
from rudi_node_write.utils.file_utils import read_json_file
|
|
19
18
|
from rudi_node_write.utils.log import log_d
|
|
20
|
-
from rudi_node_write.utils.str_utils import check_is_string, check_is_uuid4, slash_join, uuid4_str
|
|
19
|
+
from rudi_node_write.utils.str_utils import absolute_path, check_is_string, check_is_uuid4, slash_join, uuid4_str
|
|
21
20
|
from rudi_node_write.utils.type_date import Date
|
|
22
21
|
|
|
23
22
|
_USER_AGENT_DEFAULT = "RudiNodeWriter"
|
|
@@ -398,7 +397,7 @@ class RudiNodeWriter:
|
|
|
398
397
|
media_name = check_get_key(media, "media_name")
|
|
399
398
|
media_url = check_get_key(media, "connector", "url")
|
|
400
399
|
|
|
401
|
-
destination_path =
|
|
400
|
+
destination_path = absolute_path(local_download_dir, media_name)
|
|
402
401
|
content = https_download(media_url)
|
|
403
402
|
if content is None:
|
|
404
403
|
raise Exception(f"Could not download from {media_url}")
|
|
@@ -475,7 +474,7 @@ class RudiNodeWriter:
|
|
|
475
474
|
:param local_download_dir: the path to a local folder
|
|
476
475
|
:param file_name: the name of the file in which the JSON representation of the list of metadata will be saved
|
|
477
476
|
"""
|
|
478
|
-
file_path =
|
|
477
|
+
file_path = absolute_path(local_download_dir, file_name)
|
|
479
478
|
json_str = dumps(obj=self.metadata_list, ensure_ascii=False, indent=2).encode("utf-8")
|
|
480
479
|
open(file_path, "wb").write(json_str)
|
|
481
480
|
|
|
@@ -485,13 +484,18 @@ class RudiNodeWriter:
|
|
|
485
484
|
"""
|
|
486
485
|
return self.connector.put_metadata(metadata=metadata)
|
|
487
486
|
|
|
488
|
-
def post_local_file_and_media_info(
|
|
487
|
+
def post_local_file_and_media_info(
|
|
488
|
+
self,
|
|
489
|
+
file_local_path: str,
|
|
490
|
+
media_id: str = uuid4_str(),
|
|
491
|
+
rudi_media: RudiMediaFile | None = None,
|
|
492
|
+
) -> RudiMediaFile:
|
|
489
493
|
"""
|
|
490
494
|
Upload a local file on the RUDI node
|
|
491
|
-
Creates automatically the "media" part of the metadata
|
|
495
|
+
Creates automatically the "media" part of the metadata if it is not provided.
|
|
492
496
|
:return:
|
|
493
497
|
"""
|
|
494
|
-
return self.connector.post_local_file(file_local_path=file_local_path, media_id=media_id)
|
|
498
|
+
return self.connector.post_local_file(file_local_path=file_local_path, media_id=media_id, rudi_media=rudi_media)
|
|
495
499
|
|
|
496
500
|
|
|
497
501
|
if __name__ == "__main__": # pragma: no cover
|
{rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_media.py
RENAMED
|
@@ -413,7 +413,7 @@ class RudiMediaFile(RudiMedia):
|
|
|
413
413
|
self.file_status_update = Date.now_iso_str()
|
|
414
414
|
|
|
415
415
|
@staticmethod
|
|
416
|
-
def from_local_file(file_local_path: str, media_id: str = uuid4_str(), file_url: str = "to_be_provided"):
|
|
416
|
+
def from_local_file(file_local_path: str, media_id: str | None = uuid4_str(), file_url: str = "to_be_provided"):
|
|
417
417
|
here = f"{RudiMediaFile}.from_local_file"
|
|
418
418
|
file_info = FileDetails(file_local_path)
|
|
419
419
|
log_d(here, "file_info", file_info)
|
|
@@ -439,7 +439,7 @@ if __name__ == "__main__": # pragma: no cover
|
|
|
439
439
|
"updated": "2022-01-21T10:40:28.781+00:00",
|
|
440
440
|
},
|
|
441
441
|
"connector": {
|
|
442
|
-
"url": "https://bacasable.fenix.rudi-univ-rennes1.fr/
|
|
442
|
+
"url": "https://bacasable.fenix.rudi-univ-rennes1.fr/storage/download/2611547a-42f1-4d7c-b736-2fef5cca30fe",
|
|
443
443
|
"interface_contract": "dwnl",
|
|
444
444
|
"connector_parameters": [
|
|
445
445
|
{
|
|
@@ -68,12 +68,12 @@ class HttpError(Exception):
|
|
|
68
68
|
self.url = url
|
|
69
69
|
# print(here, f"http err {self.status}:", err)
|
|
70
70
|
if type(err) is dict and "error" in err and "message" in err:
|
|
71
|
-
|
|
71
|
+
self.status = err["status"] if "status" in err else err.get("statusCode")
|
|
72
72
|
err_type = err["error"]
|
|
73
73
|
err_msg = err["message"]
|
|
74
|
-
self.message = f"{
|
|
74
|
+
self.message = f"{err_type}: {err_msg}"
|
|
75
75
|
else:
|
|
76
|
-
self.message = f"{
|
|
76
|
+
self.message = f"{err}"
|
|
77
77
|
if req_method and base_url:
|
|
78
78
|
self.message = f"for request '{req_method} {slash_join(base_url, url)}' -> {err}"
|
|
79
79
|
super().__init__(self.message)
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from os.path import abspath, join
|
|
1
2
|
from re import compile
|
|
2
3
|
from typing import Callable
|
|
3
4
|
from uuid import UUID, uuid4
|
|
@@ -68,6 +69,10 @@ def check_is_uuid4(uuid) -> str:
|
|
|
68
69
|
raise ValueError(f"Input parameter is not a valid UUID v4: '{uuid}'")
|
|
69
70
|
|
|
70
71
|
|
|
72
|
+
def absolute_path(*args) -> str:
|
|
73
|
+
return abspath(join(*args))
|
|
74
|
+
|
|
75
|
+
|
|
71
76
|
def slash_join(*args) -> str:
|
|
72
77
|
"""
|
|
73
78
|
Joins a set of strings with a slash (/) between them (useful for merging URLs or paths fragments)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rudi-node-write
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.6
|
|
4
4
|
Summary: Use the internal API of a RUDI Producer node
|
|
5
5
|
Author-email: Olivier Martineau <olivier.martineau@irisa.fr>
|
|
6
6
|
Maintainer-email: Olivier Martineau <olivier.martineau@irisa.fr>
|
|
@@ -58,13 +58,12 @@ Requires-Dist: protobuf==6.31.1; extra == "dev"
|
|
|
58
58
|
Requires-Dist: py-env==0.0.1; extra == "dev"
|
|
59
59
|
Requires-Dist: pybind11==2.13.6; extra == "dev"
|
|
60
60
|
Requires-Dist: pyright==1.1.401; extra == "dev"
|
|
61
|
-
Requires-Dist: pyside6-addons==6.9.
|
|
61
|
+
Requires-Dist: pyside6-addons==6.9.1; extra == "dev"
|
|
62
62
|
Requires-Dist: pytest-check==2.5.3; extra == "dev"
|
|
63
63
|
Requires-Dist: pytest-cov==6.1.1; extra == "dev"
|
|
64
64
|
Requires-Dist: python-dotenv==1.1.0; extra == "dev"
|
|
65
65
|
Requires-Dist: python-multipart==0.0.20; extra == "dev"
|
|
66
66
|
Requires-Dist: regex==2024.11.6; extra == "dev"
|
|
67
|
-
Requires-Dist: rudi-node-write==1.3.3; extra == "dev"
|
|
68
67
|
Requires-Dist: timm==1.0.15; extra == "dev"
|
|
69
68
|
Requires-Dist: tomli==2.2.1; extra == "dev"
|
|
70
69
|
Requires-Dist: twine==6.1.0; extra == "dev"
|
|
@@ -41,13 +41,12 @@ protobuf==6.31.1
|
|
|
41
41
|
py-env==0.0.1
|
|
42
42
|
pybind11==2.13.6
|
|
43
43
|
pyright==1.1.401
|
|
44
|
-
pyside6-addons==6.9.
|
|
44
|
+
pyside6-addons==6.9.1
|
|
45
45
|
pytest-check==2.5.3
|
|
46
46
|
pytest-cov==6.1.1
|
|
47
47
|
python-dotenv==1.1.0
|
|
48
48
|
python-multipart==0.0.20
|
|
49
49
|
regex==2024.11.6
|
|
50
|
-
rudi-node-write==1.3.3
|
|
51
50
|
timm==1.0.15
|
|
52
51
|
tomli==2.2.1
|
|
53
52
|
twine==6.1.0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/rudi_node_auth.py
RENAMED
|
File without changes
|
{rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_const.py
RENAMED
|
File without changes
|
{rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_contact.py
RENAMED
|
File without changes
|
{rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_dates.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_licence.py
RENAMED
|
File without changes
|
|
File without changes
|
{rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_meta_misc.py
RENAMED
|
File without changes
|
|
File without changes
|
{rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/serializable.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|