rudi-node-write 1.3.4__tar.gz → 1.3.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {rudi_node_write-1.3.4/src/rudi_node_write.egg-info → rudi_node_write-1.3.6}/PKG-INFO +2 -3
  2. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/pyproject.toml +2 -2
  3. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/requirements-dev.txt +6 -8
  4. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/conf/meta_defaults.py +1 -1
  5. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/io_connector.py +4 -3
  6. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/io_rudi_catalog_write.py +5 -5
  7. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/io_rudi_manager_write.py +31 -7
  8. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/io_rudi_manager_write_v3.py +50 -21
  9. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_node_writer.py +13 -9
  10. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_media.py +2 -2
  11. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/err.py +3 -3
  12. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/str_utils.py +5 -0
  13. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6/src/rudi_node_write.egg-info}/PKG-INFO +2 -3
  14. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write.egg-info/requires.txt +1 -2
  15. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/LICENCE.md +0 -0
  16. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/README.md +0 -0
  17. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/requirements.txt +0 -0
  18. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/setup.cfg +0 -0
  19. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/__init__.py +0 -0
  20. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/io_rudi_jwt_factory.py +0 -0
  21. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/io_rudi_manager_write_v2.py +0 -0
  22. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/io_rudi_storage_write.py +0 -0
  23. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/connectors/rudi_node_auth.py +0 -0
  24. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_const.py +0 -0
  25. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_contact.py +0 -0
  26. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_dates.py +0 -0
  27. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_dictionary_entry.py +0 -0
  28. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_geo.py +0 -0
  29. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_licence.py +0 -0
  30. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_meta.py +0 -0
  31. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_meta_misc.py +0 -0
  32. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/rudi_org.py +0 -0
  33. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/rudi_types/serializable.py +0 -0
  34. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/dict_utils.py +0 -0
  35. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/file_utils.py +0 -0
  36. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/html_utils.py +0 -0
  37. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/jwt.py +0 -0
  38. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/list_utils.py +0 -0
  39. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/log.py +0 -0
  40. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/type_date.py +0 -0
  41. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/typing_utils.py +0 -0
  42. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write/utils/url_utils.py +0 -0
  43. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write.egg-info/SOURCES.txt +0 -0
  44. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write.egg-info/dependency_links.txt +0 -0
  45. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/src/rudi_node_write.egg-info/top_level.txt +0 -0
  46. {rudi_node_write-1.3.4 → rudi_node_write-1.3.6}/tests/test_rudi_node_write.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rudi-node-write
3
- Version: 1.3.4
3
+ Version: 1.3.6
4
4
  Summary: Use the internal API of a RUDI Producer node
5
5
  Author-email: Olivier Martineau <olivier.martineau@irisa.fr>
6
6
  Maintainer-email: Olivier Martineau <olivier.martineau@irisa.fr>
@@ -58,13 +58,12 @@ Requires-Dist: protobuf==6.31.1; extra == "dev"
58
58
  Requires-Dist: py-env==0.0.1; extra == "dev"
59
59
  Requires-Dist: pybind11==2.13.6; extra == "dev"
60
60
  Requires-Dist: pyright==1.1.401; extra == "dev"
61
- Requires-Dist: pyside6-addons==6.9.0; extra == "dev"
61
+ Requires-Dist: pyside6-addons==6.9.1; extra == "dev"
62
62
  Requires-Dist: pytest-check==2.5.3; extra == "dev"
63
63
  Requires-Dist: pytest-cov==6.1.1; extra == "dev"
64
64
  Requires-Dist: python-dotenv==1.1.0; extra == "dev"
65
65
  Requires-Dist: python-multipart==0.0.20; extra == "dev"
66
66
  Requires-Dist: regex==2024.11.6; extra == "dev"
67
- Requires-Dist: rudi-node-write==1.3.3; extra == "dev"
68
67
  Requires-Dist: timm==1.0.15; extra == "dev"
69
68
  Requires-Dist: tomli==2.2.1; extra == "dev"
70
69
  Requires-Dist: twine==6.1.0; extra == "dev"
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "rudi-node-write"
7
- version = "1.3.4"
7
+ version = "1.3.6"
8
8
  authors = [{ name = "Olivier Martineau", email = "olivier.martineau@irisa.fr" }]
9
9
  maintainers = [
10
10
  { name = "Olivier Martineau", email = "olivier.martineau@irisa.fr" },
@@ -45,7 +45,7 @@ target-version = ['py311']
45
45
  # ----- Tool: commitizen
46
46
  [tool.commitizen]
47
47
  name = "cz_conventional_commits"
48
- version = "1.3.4"
48
+ version = "1.3.6"
49
49
  version_files = ["pyproject.toml:version"]
50
50
 
51
51
  # ----- Tool: pytest
@@ -1,4 +1,3 @@
1
- # -r requirements.txt
2
1
  backports.tarfile==1.2.0
3
2
  black==25.1.0
4
3
  build==1.2.2.post1
@@ -36,13 +35,12 @@ protobuf==6.31.1
36
35
  py-env==0.0.1
37
36
  pybind11==2.13.6
38
37
  pyright==1.1.401
39
- pyside6-addons==6.9.0
38
+ pyside6-addons==6.9.1
40
39
  pytest-check==2.5.3
41
40
  pytest-cov==6.1.1
42
41
  python-dotenv==1.1.0
43
42
  python-multipart==0.0.20
44
43
  regex==2024.11.6
45
- rudi-node-write==1.3.3
46
44
  timm==1.0.15
47
45
  tomli==2.2.1
48
46
  twine==6.1.0
@@ -87,7 +85,7 @@ websockets==15.0.1
87
85
  # hf-xet==1.1.2 # Installed as dependency for huggingface-hub
88
86
  # hidapi==0.14.0.post4 # Installed as dependency for onlykey
89
87
  # httpcore==1.0.9 # Installed as dependency for httpx
90
- # huggingface-hub==0.32.3 # Installed as dependency for timm
88
+ # huggingface-hub==0.32.4 # Installed as dependency for timm
91
89
  # id==1.5.0 # Installed as dependency for twine
92
90
  # identify==2.6.12 # Installed as dependency for pre-commit
93
91
  # idna==3.10 # Installed as dependency for anyio, email-validator, httpx, requests
@@ -143,13 +141,13 @@ websockets==15.0.1
143
141
  # pydantic==2.11.5 # Installed as dependency for fastapi
144
142
  # pydantic-core==2.34.1 # Installed as dependency for pydantic
145
143
  # pyflakes==3.3.2 # Installed as dependency for flake8
146
- # pygments==2.19.1 # Installed as dependency for ipython, ipython-pygments-lexers, readme-renderer, rich
144
+ # pygments==2.19.1 # Installed as dependency for ipython, ipython-pygments-lexers, pytest, readme-renderer, rich
147
145
  # pymsgbox==1.0.9 # Installed as dependency for lib-agent
148
146
  # pynacl==1.5.0 # Installed as dependency for lib-agent, onlykey
149
147
  # pyproject-hooks==1.2.0 # Installed as dependency for build
150
148
  # pyserial==3.5 # Installed as dependency for onlykey-solo-python
151
- # pyside6-essentials==6.9.0 # Installed as dependency for pyside6-addons
152
- # pytest==8.3.5 # Installed as dependency for pytest-check, pytest-cov
149
+ # pyside6-essentials==6.9.1 # Installed as dependency for pyside6-addons
150
+ # pytest==8.4.0 # Installed as dependency for pytest-check, pytest-cov
153
151
  # python-daemon==3.1.2 # Installed as dependency for lib-agent
154
152
  # python-dateutil==2.9.0.post0 # Installed as dependency for jupyter-client
155
153
  # pyusb==1.3.1 # Installed as dependency for onlykey-solo-python
@@ -169,7 +167,7 @@ websockets==15.0.1
169
167
  # safetensors==0.5.3 # Installed as dependency for timm
170
168
  # semver==3.0.4 # Installed as dependency for lib-agent
171
169
  # shellingham==1.5.4 # Installed as dependency for typer
172
- # shiboken6==6.9.0 # Installed as dependency for pyside6-addons, pyside6-essentials
170
+ # shiboken6==6.9.1 # Installed as dependency for pyside6-addons, pyside6-essentials
173
171
  # six==1.17.0 # Installed as dependency for cssbeautifier, ecdsa, jsbeautifier, onlykey, python-dateutil
174
172
  # sniffio==1.3.1 # Installed as dependency for anyio
175
173
  # stack-data==0.6.3 # Installed as dependency for ipython
@@ -6,7 +6,7 @@ from rudi_node_write.rudi_types.rudi_const import Language
6
6
  DEFAULT_LANG: Language = "fr"
7
7
 
8
8
  # Current version for RUDI Node metadata
9
- RUDI_API_VERSION: Final[str] = "1.3.0"
9
+ RUDI_API_VERSION: Final[str] = "1.4.0"
10
10
 
11
11
  # Current version for ODS
12
12
  ODS_API_VERSION: str = "2.1"
@@ -244,7 +244,7 @@ class Connector:
244
244
  if not keep_alive and not self.keep_connection:
245
245
  self.close_connection()
246
246
 
247
- if type(response_data) is str:
247
+ if isinstance(response_data, str):
248
248
  log_d_if(self.should_log_response, here, "Response is a string", response_data)
249
249
  if response.status < 400:
250
250
  return rdata.decode("utf8")
@@ -259,10 +259,11 @@ class Connector:
259
259
 
260
260
  if __name__ == "__main__": # pragma: no cover
261
261
  tests = "Connector tests"
262
- connector = Connector("https://bacasable.fenix.rudi-univ-rennes1.fr/api/version")
262
+ node_url = "https://bacasable.fenix.rudi-univ-rennes1.fr"
263
+ connector = Connector(f"{node_url}/catalog/version")
263
264
  log_d(tests, "testing connection, got version", connector.test_connection())
264
265
 
265
- connector = Connector("https://bacasable.fenix.rudi-univ-rennes1.fr/api/v1")
266
+ connector = Connector(f"{node_url}/catalog/v1")
266
267
  metadata_list = connector.request("resources")
267
268
  if isinstance(metadata_list, dict) and metadata_list["total"] is not None:
268
269
  log_d(tests, "number of resources declared", f"{metadata_list['total']}")
@@ -1,8 +1,7 @@
1
- from os.path import isdir, abspath
1
+ from os.path import isdir
2
2
  from time import time
3
3
  from urllib.parse import quote
4
4
 
5
- from _pytest.config.argparsing import ArgumentError
6
5
 
7
6
  from rudi_node_write.connectors.io_connector import Connector, https_download
8
7
  from rudi_node_write.connectors.io_rudi_jwt_factory import RudiNodeJwtFactory
@@ -28,8 +27,8 @@ from rudi_node_write.utils.file_utils import read_json_file, write_file
28
27
  from rudi_node_write.utils.jwt import is_jwt_expired
29
28
  from rudi_node_write.utils.list_utils import get_first_list_elt_or_none
30
29
  from rudi_node_write.utils.log import log_d, log_e, log_w
31
- from rudi_node_write.utils.str_utils import slash_join, uuid4_str, is_uuid_v4
32
- from rudi_node_write.utils.typing_utils import get_type_name, check_type
30
+ from rudi_node_write.utils.str_utils import absolute_path, slash_join, uuid4_str, is_uuid_v4
31
+ from rudi_node_write.utils.typing_utils import get_type_name
33
32
  from rudi_node_write.utils.url_utils import ensure_url_startswith
34
33
 
35
34
 
@@ -663,11 +662,12 @@ class RudiNodeCatalogConnector(Connector):
663
662
  raise FileNotFoundError(f"The following folder does not exist: '{local_download_dir}'")
664
663
 
665
664
  media_name = media.get("media_name")
665
+ assert isinstance(media_name, str)
666
666
  media_url = safe_get_key(media, "connector", "url")
667
667
  if not media_url:
668
668
  raise FileNotFoundError(f"No URL was provided for the media {media_name}")
669
669
 
670
- destination_path = abspath(slash_join(local_download_dir, media_name))
670
+ destination_path = absolute_path(local_download_dir, media_name)
671
671
  try:
672
672
  content = https_download(media_url)
673
673
  if not content:
@@ -1,6 +1,5 @@
1
- from base64 import urlsafe_b64decode, urlsafe_b64encode
2
1
  from json import dumps
3
- from os.path import abspath, exists, isdir
2
+ from os.path import exists, isdir
4
3
  from ssl import SSLCertVerificationError
5
4
  from time import time
6
5
  from typing import Literal
@@ -22,10 +21,10 @@ from rudi_node_write.rudi_types.serializable import Serializable
22
21
  from rudi_node_write.utils.dict_utils import check_is_dict, merge_dict_of_list, pick_in_dict, safe_get_key
23
22
  from rudi_node_write.utils.err import HttpError, HttpErrorNotFound, UnexpectedValueException
24
23
  from rudi_node_write.utils.file_utils import check_is_file, read_json_file, write_file
25
- from rudi_node_write.utils.jwt import is_base64_url, is_jwt_expired, pad_b64_str
24
+ from rudi_node_write.utils.jwt import is_jwt_expired
26
25
  from rudi_node_write.utils.list_utils import get_first_list_elt_or_none
27
26
  from rudi_node_write.utils.log import log_d, log_e, log_w
28
- from rudi_node_write.utils.str_utils import check_is_uuid4, is_uuid_v4, slash_join, uuid4_str
27
+ from rudi_node_write.utils.str_utils import absolute_path, check_is_uuid4, is_uuid_v4, slash_join, uuid4_str
29
28
  from rudi_node_write.utils.type_date import Date
30
29
  from rudi_node_write.utils.typing_utils import check_is_int, get_type_name
31
30
  from rudi_node_write.utils.url_utils import ensure_url_startswith
@@ -1027,7 +1026,7 @@ class RudiNodeManagerConnector(Connector):
1027
1026
  if not media_url:
1028
1027
  raise FileNotFoundError(f"No URL was provided for the media {media_name}")
1029
1028
 
1030
- destination_path = abspath(slash_join(local_download_dir, media_name))
1029
+ destination_path = absolute_path(local_download_dir, media_name)
1031
1030
  try:
1032
1031
  content = https_download(media_url)
1033
1032
  if not content:
@@ -1131,6 +1130,7 @@ class RudiNodeManagerConnector(Connector):
1131
1130
  self,
1132
1131
  file_local_path: str,
1133
1132
  media_id: str = uuid4_str(),
1133
+ rudi_media: RudiMediaFile | None = None,
1134
1134
  ):
1135
1135
  """
1136
1136
  :param file_local_path: the path of a local file we wish to send to a RUDI node Storage server
@@ -1139,9 +1139,20 @@ class RudiNodeManagerConnector(Connector):
1139
1139
  """
1140
1140
  here = f"{self.class_name}.post_local_file"
1141
1141
 
1142
+ if rudi_media is None:
1143
+ rudi_media = RudiMediaFile.from_local_file(file_local_path, media_id)
1144
+ else:
1145
+ check_media_info = RudiMediaFile.from_local_file(file_local_path, media_id)
1146
+ rudi_media.file_size = check_media_info.file_size
1147
+ rudi_media.checksum = check_media_info.checksum
1148
+
1149
+ media_id = rudi_media.media_id
1150
+
1142
1151
  # Posting the file as a binary directly to RUDI node Storage storage module
1143
1152
  res = self.storage_connector.post_local_file(
1144
- file_local_path=check_is_file(file_local_path), media_id=check_is_uuid4(media_id)
1153
+ file_local_path=check_is_file(file_local_path),
1154
+ media_id=check_is_uuid4(media_id),
1155
+ rudi_media=rudi_media,
1145
1156
  )
1146
1157
  if res is None or not isinstance(res, dict) or res["media_info"] is None:
1147
1158
  raise Exception("Upload to RUDI Media failed!")
@@ -1277,10 +1288,16 @@ class RudiNodeStorageConnector(Connector):
1277
1288
  media_info = media_list.get("zone1") # What if there are other zones?
1278
1289
  return [] if media_info is None else media_info.get("list")
1279
1290
 
1280
- def post_local_file(self, file_local_path: str, media_id: str = uuid4_str()):
1291
+ def post_local_file(
1292
+ self,
1293
+ file_local_path: str,
1294
+ media_id: str = uuid4_str(),
1295
+ rudi_media: RudiMediaFile | None = None,
1296
+ ):
1281
1297
  """
1282
1298
  :param file_local_path: the path of a local file we wish to send to a RUDI node Media server
1283
1299
  :param media_id: the UUIDv4 that identifies the media on the RUDI node
1300
+ :param rudi_media: an optional RudiMediaFile object (in the case of an update)
1284
1301
  :return:
1285
1302
  """
1286
1303
  # :param media_name: the original name of the file
@@ -1289,6 +1306,13 @@ class RudiNodeStorageConnector(Connector):
1289
1306
  # :param charset: the encoding of the file
1290
1307
  here = f"{self.class_name}.post_local_file"
1291
1308
 
1309
+ if rudi_media is None:
1310
+ rudi_media = RudiMediaFile.from_local_file(file_local_path, media_id)
1311
+ else:
1312
+ check_media_info = RudiMediaFile.from_local_file(file_local_path, media_id)
1313
+ rudi_media.file_size = check_media_info.file_size
1314
+ rudi_media.checksum = check_media_info.checksum
1315
+
1292
1316
  rudi_media = RudiMediaFile.from_local_file(file_local_path, media_id)
1293
1317
  if rudi_media.file_size > FileTooBigException.MAX_SIZE:
1294
1318
  raise FileTooBigException(rudi_media.file_size)
@@ -1,6 +1,6 @@
1
1
  from http.client import IncompleteRead
2
2
  from json import dumps
3
- from os.path import abspath, exists, isdir
3
+ from os.path import exists, isdir
4
4
  from re import T
5
5
  from time import time
6
6
  from typing import Literal
@@ -26,7 +26,7 @@ from rudi_node_write.utils.file_utils import check_is_file, read_json_file, writ
26
26
  from rudi_node_write.utils.jwt import is_jwt_expired
27
27
  from rudi_node_write.utils.list_utils import get_first_list_elt_or_none
28
28
  from rudi_node_write.utils.log import log_d, log_e, log_w
29
- from rudi_node_write.utils.str_utils import check_is_uuid4, is_uuid_v4, slash_join, uuid4_str
29
+ from rudi_node_write.utils.str_utils import absolute_path, check_is_uuid4, is_uuid_v4, slash_join, uuid4_str
30
30
  from rudi_node_write.utils.type_date import Date
31
31
  from rudi_node_write.utils.typing_utils import check_is_int, get_type_name
32
32
  from rudi_node_write.utils.url_utils import ensure_url_startswith
@@ -1006,7 +1006,7 @@ class RudiNodeManagerConnectorV3(RudiNodeManagerConnector):
1006
1006
  if not media_url:
1007
1007
  raise FileNotFoundError(f"No URL was provided for the media {media_name}")
1008
1008
 
1009
- destination_path = abspath(slash_join(local_download_dir, media_name))
1009
+ destination_path = absolute_path(local_download_dir, media_name)
1010
1010
  try:
1011
1011
  content = https_download(media_url)
1012
1012
  if not content:
@@ -1110,18 +1110,30 @@ class RudiNodeManagerConnectorV3(RudiNodeManagerConnector):
1110
1110
  self,
1111
1111
  file_local_path: str,
1112
1112
  media_id: str = uuid4_str(),
1113
- ):
1113
+ rudi_media: RudiMediaFile | None = None,
1114
+ ) -> RudiMediaFile:
1114
1115
  """
1115
1116
  :param file_local_path: the path of a local file we wish to send to a RUDI node Storage server
1116
1117
  :param media_id: the UUIDv4 that identifies the media on the RUDI node
1117
1118
  :return:
1118
1119
  """
1119
- here = f"{self.class_name}_v3.post_local_file"
1120
+ here = f"{self.class_name}.post_local_file"
1121
+
1122
+ if rudi_media is None:
1123
+ rudi_media = RudiMediaFile.from_local_file(file_local_path, media_id)
1124
+ else:
1125
+ check_media_info = RudiMediaFile.from_local_file(file_local_path, media_id)
1126
+ rudi_media.file_size = check_media_info.file_size
1127
+ rudi_media.checksum = check_media_info.checksum
1128
+
1129
+ media_id = rudi_media.media_id
1120
1130
 
1121
1131
  # Posting the file as a binary directly to RUDI node Storage module
1122
1132
  try:
1123
1133
  res = self.storage_connector.post_local_file(
1124
- file_local_path=check_is_file(file_local_path), media_id=check_is_uuid4(media_id)
1134
+ file_local_path=check_is_file(file_local_path),
1135
+ media_id=rudi_media.media_id,
1136
+ rudi_media=rudi_media,
1125
1137
  )
1126
1138
  except IncompleteRead as e:
1127
1139
  log_e(here, "Upload to RUDI Storage failed (IncompleteRead)", e)
@@ -1130,14 +1142,17 @@ class RudiNodeManagerConnectorV3(RudiNodeManagerConnector):
1130
1142
  except Exception as e:
1131
1143
  log_e(here, "Upload to RUDI Storage failed", e)
1132
1144
  raise e
1133
- if res is None or not isinstance(res, dict) or res["media_info"] is None:
1145
+ if res is None or not isinstance(res, dict) or res.get("media_info") is None:
1134
1146
  raise Exception("Upload to RUDI Storage failed!")
1135
1147
  media_info, zone_name, commit_uuid = (
1136
1148
  res["media_info"],
1137
1149
  res["zone_name"],
1138
1150
  res["commit_uuid"],
1139
1151
  )
1152
+ assert isinstance(media_info, RudiMediaFile)
1140
1153
  # log_d(here, "media_info", str(media_info))
1154
+ # log_d(here, "zone_name", zone_name)
1155
+ # log_d(here, "commit_uuid", commit_uuid)
1141
1156
 
1142
1157
  # Posting the file metadata as a RudiMediaFile object to the RUDI Catalog module through the PM data API
1143
1158
  if not self._gen == 1:
@@ -1154,19 +1169,19 @@ class RudiNodeManagerConnectorV3(RudiNodeManagerConnector):
1154
1169
  return media_info
1155
1170
 
1156
1171
  # Committing the file that has been uploaded
1157
- log_d(here, "zone_name", zone_name)
1158
- log_d(here, "commit_uuid", commit_uuid)
1159
- res_commit = self.commit_media(media_id=media_info.media_id, zone_name=zone_name, commit_uuid=commit_uuid)
1172
+ res_commit = self.commit_media(media_id=media_id, zone_name=zone_name, commit_uuid=commit_uuid)
1160
1173
  log_d(here, "res_commit", res_commit)
1161
1174
 
1162
1175
  # Returning the updated RudiMedia information
1163
1176
  if isinstance(api_media_info, dict):
1164
1177
  updated_media_info = RudiMediaFile.from_json(api_media_info)
1165
- return updated_media_info
1166
1178
  else:
1167
- log_w(here, f"unexpected result, should have a 'RudiMediaFile' compatible dict, got {media_info}")
1179
+ log_w(here, f"unexpected result, should have a 'RudiMediaFile' compatible dict, got {api_media_info}")
1168
1180
  return media_info
1169
1181
 
1182
+ # Returning the updated RudiMedia information
1183
+ return updated_media_info
1184
+
1170
1185
  # TODO: PM API for commit only file / only API / both?
1171
1186
 
1172
1187
  def commit_media(self, media_id: str, commit_uuid: str, zone_name: str, metadata_id: str | None = None):
@@ -1278,10 +1293,16 @@ class RudiNodeStorageConnectorV3(Connector):
1278
1293
 
1279
1294
  return [] if media_info is None else media_info.get("list")
1280
1295
 
1281
- def post_local_file(self, file_local_path: str, media_id: str = uuid4_str()):
1296
+ def post_local_file(
1297
+ self,
1298
+ file_local_path: str,
1299
+ media_id: str | None = None,
1300
+ rudi_media: RudiMediaFile | None = None,
1301
+ ):
1282
1302
  """
1283
1303
  :param file_local_path: the path of a local file we wish to send to a RUDI node Media server
1284
1304
  :param media_id: the UUIDv4 that identifies the media on the RUDI node
1305
+ :param rudi_media: an optional RudiMediaFile object (in the case of an update)
1285
1306
  :return:
1286
1307
  """
1287
1308
  # :param media_name: the original name of the file
@@ -1290,12 +1311,19 @@ class RudiNodeStorageConnectorV3(Connector):
1290
1311
  # :param charset: the encoding of the file
1291
1312
  here = f"{self.class_name}.post_local_file"
1292
1313
 
1293
- media_info = RudiMediaFile.from_local_file(file_local_path, media_id)
1294
- if media_info.file_size > FileTooBigException.MAX_SIZE:
1295
- raise FileTooBigException(media_info.file_size)
1314
+ if rudi_media is None:
1315
+ rudi_media = RudiMediaFile.from_local_file(file_local_path, media_id)
1316
+ else:
1317
+ check_media_info = RudiMediaFile.from_local_file(file_local_path, media_id)
1318
+ rudi_media.file_size = check_media_info.file_size
1319
+ rudi_media.checksum = check_media_info.checksum
1320
+
1321
+ if rudi_media.file_size > FileTooBigException.MAX_SIZE:
1322
+ raise FileTooBigException(rudi_media.file_size)
1323
+ media_id = rudi_media.media_id
1296
1324
 
1297
- log_d(here, "sending as binary for file", media_info)
1298
- headers = self.get_media_headers_for_file(media_info) | {"Content-Type": "octet/stream"}
1325
+ log_d(here, "sending as binary for file", rudi_media)
1326
+ headers = self.get_media_headers_for_file(rudi_media) | {"Content-Type": "octet/stream"}
1299
1327
  # log_d(here, "headers", headers)
1300
1328
  with open(file_local_path, "rb") as bin_content:
1301
1329
  try:
@@ -1315,17 +1343,18 @@ class RudiNodeStorageConnectorV3(Connector):
1315
1343
  commit_ready = res[-2]
1316
1344
  log_d(here, "commit_ready", commit_ready)
1317
1345
  # Updating the connector.url information for the media
1318
- media_info.set_url(slash_join(self.base_url, "download", media_id))
1346
+ rudi_media.set_url(slash_join(self.base_url, "download", rudi_media.media_id))
1319
1347
  # Updating the file storage status information for the media
1320
- media_info.set_status("available")
1348
+ rudi_media.set_status("available")
1321
1349
  return {
1322
- "media_info": media_info,
1350
+ "media_info": rudi_media,
1323
1351
  "zone_name": commit_ready["zone_name"],
1324
1352
  "commit_uuid": commit_ready["commit_uuid"],
1325
1353
  }
1326
1354
  if len(res) > 0 and res[0].get("status") == "error":
1327
1355
  log_w(here, "ERR upload failed", res[0].get("msg"))
1328
1356
  raise Exception("File upload to RUDI Media falied:" + res[0])
1357
+
1329
1358
  return res
1330
1359
 
1331
1360
 
@@ -1,13 +1,12 @@
1
1
  from json import dumps, loads
2
- from os.path import abspath, isdir
2
+ from os.path import isdir
3
3
  from time import time
4
4
 
5
5
  from rudi_node_write.connectors.io_connector import https_download
6
- from rudi_node_write.connectors.io_rudi_manager_write import RudiNodeManagerConnector
7
6
  from rudi_node_write.connectors.io_rudi_manager_write_v2 import RudiNodeManagerConnectorV2
8
7
  from rudi_node_write.connectors.io_rudi_manager_write_v3 import RudiNodeManagerConnectorV3
9
8
  from rudi_node_write.connectors.rudi_node_auth import RudiNodeAuth
10
- from rudi_node_write.rudi_types.rudi_meta import RudiMetadata
9
+ from rudi_node_write.rudi_types.rudi_meta import RudiMediaFile, RudiMetadata
11
10
  from rudi_node_write.utils.dict_utils import (
12
11
  check_get_key,
13
12
  filter_dict_list,
@@ -17,7 +16,7 @@ from rudi_node_write.utils.dict_utils import (
17
16
  )
18
17
  from rudi_node_write.utils.file_utils import read_json_file
19
18
  from rudi_node_write.utils.log import log_d
20
- from rudi_node_write.utils.str_utils import check_is_string, check_is_uuid4, slash_join, uuid4_str
19
+ from rudi_node_write.utils.str_utils import absolute_path, check_is_string, check_is_uuid4, slash_join, uuid4_str
21
20
  from rudi_node_write.utils.type_date import Date
22
21
 
23
22
  _USER_AGENT_DEFAULT = "RudiNodeWriter"
@@ -398,7 +397,7 @@ class RudiNodeWriter:
398
397
  media_name = check_get_key(media, "media_name")
399
398
  media_url = check_get_key(media, "connector", "url")
400
399
 
401
- destination_path = abspath(slash_join(local_download_dir, media_name))
400
+ destination_path = absolute_path(local_download_dir, media_name)
402
401
  content = https_download(media_url)
403
402
  if content is None:
404
403
  raise Exception(f"Could not download from {media_url}")
@@ -475,7 +474,7 @@ class RudiNodeWriter:
475
474
  :param local_download_dir: the path to a local folder
476
475
  :param file_name: the name of the file in which the JSON representation of the list of metadata will be saved
477
476
  """
478
- file_path = abspath(slash_join(local_download_dir, file_name))
477
+ file_path = absolute_path(local_download_dir, file_name)
479
478
  json_str = dumps(obj=self.metadata_list, ensure_ascii=False, indent=2).encode("utf-8")
480
479
  open(file_path, "wb").write(json_str)
481
480
 
@@ -485,13 +484,18 @@ class RudiNodeWriter:
485
484
  """
486
485
  return self.connector.put_metadata(metadata=metadata)
487
486
 
488
- def post_local_file_and_media_info(self, file_local_path: str, media_id: str = uuid4_str()):
487
+ def post_local_file_and_media_info(
488
+ self,
489
+ file_local_path: str,
490
+ media_id: str = uuid4_str(),
491
+ rudi_media: RudiMediaFile | None = None,
492
+ ) -> RudiMediaFile:
489
493
  """
490
494
  Upload a local file on the RUDI node
491
- Creates automatically the "media" part of the metadata
495
+ Creates automatically the "media" part of the metadata if it is not provided.
492
496
  :return:
493
497
  """
494
- return self.connector.post_local_file(file_local_path=file_local_path, media_id=media_id)
498
+ return self.connector.post_local_file(file_local_path=file_local_path, media_id=media_id, rudi_media=rudi_media)
495
499
 
496
500
 
497
501
  if __name__ == "__main__": # pragma: no cover
@@ -413,7 +413,7 @@ class RudiMediaFile(RudiMedia):
413
413
  self.file_status_update = Date.now_iso_str()
414
414
 
415
415
  @staticmethod
416
- def from_local_file(file_local_path: str, media_id: str = uuid4_str(), file_url: str = "to_be_provided"):
416
+ def from_local_file(file_local_path: str, media_id: str | None = uuid4_str(), file_url: str = "to_be_provided"):
417
417
  here = f"{RudiMediaFile}.from_local_file"
418
418
  file_info = FileDetails(file_local_path)
419
419
  log_d(here, "file_info", file_info)
@@ -439,7 +439,7 @@ if __name__ == "__main__": # pragma: no cover
439
439
  "updated": "2022-01-21T10:40:28.781+00:00",
440
440
  },
441
441
  "connector": {
442
- "url": "https://bacasable.fenix.rudi-univ-rennes1.fr/media/download/2611547a-42f1-4d7c-b736-2fef5cca30fe",
442
+ "url": "https://bacasable.fenix.rudi-univ-rennes1.fr/storage/download/2611547a-42f1-4d7c-b736-2fef5cca30fe",
443
443
  "interface_contract": "dwnl",
444
444
  "connector_parameters": [
445
445
  {
@@ -68,12 +68,12 @@ class HttpError(Exception):
68
68
  self.url = url
69
69
  # print(here, f"http err {self.status}:", err)
70
70
  if type(err) is dict and "error" in err and "message" in err:
71
- err_status = err["status"] if "status" in err else err.get("statusCode")
71
+ self.status = err["status"] if "status" in err else err.get("statusCode")
72
72
  err_type = err["error"]
73
73
  err_msg = err["message"]
74
- self.message = f"{err_status} {err_type}: {err_msg}"
74
+ self.message = f"{err_type}: {err_msg}"
75
75
  else:
76
- self.message = f"{self.status} {err}"
76
+ self.message = f"{err}"
77
77
  if req_method and base_url:
78
78
  self.message = f"for request '{req_method} {slash_join(base_url, url)}' -> {err}"
79
79
  super().__init__(self.message)
@@ -1,3 +1,4 @@
1
+ from os.path import abspath, join
1
2
  from re import compile
2
3
  from typing import Callable
3
4
  from uuid import UUID, uuid4
@@ -68,6 +69,10 @@ def check_is_uuid4(uuid) -> str:
68
69
  raise ValueError(f"Input parameter is not a valid UUID v4: '{uuid}'")
69
70
 
70
71
 
72
+ def absolute_path(*args) -> str:
73
+ return abspath(join(*args))
74
+
75
+
71
76
  def slash_join(*args) -> str:
72
77
  """
73
78
  Joins a set of strings with a slash (/) between them (useful for merging URLs or paths fragments)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rudi-node-write
3
- Version: 1.3.4
3
+ Version: 1.3.6
4
4
  Summary: Use the internal API of a RUDI Producer node
5
5
  Author-email: Olivier Martineau <olivier.martineau@irisa.fr>
6
6
  Maintainer-email: Olivier Martineau <olivier.martineau@irisa.fr>
@@ -58,13 +58,12 @@ Requires-Dist: protobuf==6.31.1; extra == "dev"
58
58
  Requires-Dist: py-env==0.0.1; extra == "dev"
59
59
  Requires-Dist: pybind11==2.13.6; extra == "dev"
60
60
  Requires-Dist: pyright==1.1.401; extra == "dev"
61
- Requires-Dist: pyside6-addons==6.9.0; extra == "dev"
61
+ Requires-Dist: pyside6-addons==6.9.1; extra == "dev"
62
62
  Requires-Dist: pytest-check==2.5.3; extra == "dev"
63
63
  Requires-Dist: pytest-cov==6.1.1; extra == "dev"
64
64
  Requires-Dist: python-dotenv==1.1.0; extra == "dev"
65
65
  Requires-Dist: python-multipart==0.0.20; extra == "dev"
66
66
  Requires-Dist: regex==2024.11.6; extra == "dev"
67
- Requires-Dist: rudi-node-write==1.3.3; extra == "dev"
68
67
  Requires-Dist: timm==1.0.15; extra == "dev"
69
68
  Requires-Dist: tomli==2.2.1; extra == "dev"
70
69
  Requires-Dist: twine==6.1.0; extra == "dev"
@@ -41,13 +41,12 @@ protobuf==6.31.1
41
41
  py-env==0.0.1
42
42
  pybind11==2.13.6
43
43
  pyright==1.1.401
44
- pyside6-addons==6.9.0
44
+ pyside6-addons==6.9.1
45
45
  pytest-check==2.5.3
46
46
  pytest-cov==6.1.1
47
47
  python-dotenv==1.1.0
48
48
  python-multipart==0.0.20
49
49
  regex==2024.11.6
50
- rudi-node-write==1.3.3
51
50
  timm==1.0.15
52
51
  tomli==2.2.1
53
52
  twine==6.1.0