rudi-node-write 1.2.0__tar.gz → 1.2.2__tar.gz

{rudi_node_write-1.2.0/src/rudi_node_write.egg-info → rudi_node_write-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rudi-node-write
-Version: 1.2.0
+Version: 1.2.2
 Summary: Use the internal API of a RUDI Producer node
 Author-email: Olivier Martineau <olivier.martineau@irisa.fr>
 Maintainer-email: Olivier Martineau <olivier.martineau@irisa.fr>
@@ -17,6 +17,29 @@ Classifier: License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENCE.md
+Requires-Dist: beautifulsoup4==4.12.3
+Requires-Dist: chardet==5.2.0
+Requires-Dist: deepdiff==8.0.1
+Requires-Dist: puremagic==1.28
+Provides-Extra: dev
+Requires-Dist: black==24.10.0; extra == "dev"
+Requires-Dist: build==1.2.2.post1; extra == "dev"
+Requires-Dist: commitizen==3.29.1; extra == "dev"
+Requires-Dist: importlib-resources==6.4.5; extra == "dev"
+Requires-Dist: inflect==7.3.1; extra == "dev"
+Requires-Dist: ipykernel==6.29.5; extra == "dev"
+Requires-Dist: jaraco.collections==5.1.0; extra == "dev"
+Requires-Dist: nbstripout==0.7.1; extra == "dev"
+Requires-Dist: ordered-set==4.1.0; extra == "dev"
+Requires-Dist: pip-autoremove==0.10.0; extra == "dev"
+Requires-Dist: pip-chill==1.0.3; extra == "dev"
+Requires-Dist: pip-upgrade-outdated==1.5; extra == "dev"
+Requires-Dist: pip3-autoremove==1.2.2; extra == "dev"
+Requires-Dist: pre-commit==4.0.1; extra == "dev"
+Requires-Dist: pyright==1.1.386; extra == "dev"
+Requires-Dist: pytest-cov==5.0.0; extra == "dev"
+Requires-Dist: tomli==2.0.2; extra == "dev"
+Requires-Dist: twine==5.1.1; extra == "dev"
 
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Checked with pyright](https://microsoft.github.io/pyright/img/pyright_badge.svg)](https://microsoft.github.io/pyright/)

{rudi_node_write-1.2.0 → rudi_node_write-1.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rudi-node-write"
-version = "1.2.0"
+version = "1.2.2"
 authors = [{ name = "Olivier Martineau", email = "olivier.martineau@irisa.fr" }]
 maintainers = [{ name = "Olivier Martineau", email = "olivier.martineau@irisa.fr" }]
 description = "Use the internal API of a RUDI Producer node"
@@ -18,8 +18,8 @@ classifiers = ["Programming Language :: Python :: 3.11",
     "License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2)",
 ]
 keywords = ["rudi-node-write", "rudi-node-put", "RUDI", "producer node", "RUDI node", "open-data", "Univ. Rennes"]
-dynamic = ["dependencies"]
-# dependencies = ["beautifulsoup4", "chardet", "deepdiff", "defusedxml", "puremagic"]
+dynamic = ["dependencies", "optional-dependencies"]
+# dependencies = ["beautifulsoup4", "chardet", "deepdiff", "puremagic"]
 
 [project.urls]
 Homepage = "https://github.com/OlivierMartineau/rudi-node-write"
@@ -34,7 +34,7 @@ target-version = ['py311']
 # ----- Tool: commitizen
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "1.2.0"
+version = "1.2.2"
 version_files = ["pyproject.toml:version"]
 
 # ----- Tool: pytest
@@ -51,6 +51,7 @@ include = ["rudi_node_write*"]
 
 [tool.setuptools.dynamic]
 dependencies = {file = ["requirements.txt"]}
+optional-dependencies = {dev = { file = ["requirements-dev.txt"] }}
 
 # ----- Tool: pyright
 [tool.pyright]

rudi_node_write-1.2.2/requirements-dev.txt

@@ -0,0 +1,102 @@
+black==24.10.0
+build==1.2.2.post1
+commitizen==3.29.1
+importlib-resources==6.4.5
+inflect==7.3.1
+ipykernel==6.29.5
+jaraco.collections==5.1.0
+nbstripout==0.7.1
+ordered-set==4.1.0
+pip-autoremove==0.10.0
+pip-chill==1.0.3
+pip-upgrade-outdated==1.5
+pip3-autoremove==1.2.2
+pre-commit==4.0.1
+pyright==1.1.386
+pytest-cov==5.0.0
+tomli==2.0.2
+twine==5.1.1
+# appnope==0.1.4 # Installed as dependency for ipykernel
+# argcomplete==3.5.1 # Installed as dependency for commitizen
+# asttokens==2.4.1 # Installed as dependency for stack-data
+# attrs==24.2.0 # Installed as dependency for jsonschema, referencing
+# autocommand==2.2.2 # Installed as dependency for jaraco.text
+# backports.tarfile==1.2.0 # Installed as dependency for jaraco.context
+# certifi==2024.8.30 # Installed as dependency for requests
+# cfgv==3.4.0 # Installed as dependency for pre-commit
+# charset-normalizer==3.4.0 # Installed as dependency for commitizen, requests
+# click==8.1.7 # Installed as dependency for black
+# colorama==0.4.6 # Installed as dependency for commitizen
+# comm==0.2.2 # Installed as dependency for ipykernel
+# coverage==7.6.4 # Installed as dependency for pytest-cov
+# debugpy==1.8.7 # Installed as dependency for ipykernel
+# decli==0.6.2 # Installed as dependency for commitizen
+# decorator==5.1.1 # Installed as dependency for ipython
+# distlib==0.3.9 # Installed as dependency for virtualenv
+# docutils==0.21.2 # Installed as dependency for readme-renderer
+# executing==2.1.0 # Installed as dependency for stack-data
+# fastjsonschema==2.20.0 # Installed as dependency for nbformat
+# filelock==3.16.1 # Installed as dependency for virtualenv
+# identify==2.6.1 # Installed as dependency for pre-commit
+# idna==3.10 # Installed as dependency for requests
+# importlib-metadata==8.5.0 # Installed as dependency for keyring, twine
+# iniconfig==2.0.0 # Installed as dependency for pytest
+# ipython==8.28.0 # Installed as dependency for ipykernel
+# jaraco.classes==3.4.0 # Installed as dependency for keyring
+# jaraco.context==6.0.1 # Installed as dependency for jaraco.text, keyring
+# jaraco.functools==4.1.0 # Installed as dependency for jaraco.text, keyring
+# jaraco.text==4.0.0 # Installed as dependency for jaraco.collections
+# jedi==0.19.1 # Installed as dependency for ipython
+# jinja2==3.1.4 # Installed as dependency for commitizen
+# jsonschema==4.23.0 # Installed as dependency for nbformat
+# jsonschema-specifications==2024.10.1 # Installed as dependency for jsonschema
+# jupyter-client==8.6.3 # Installed as dependency for ipykernel
+# jupyter-core==5.7.2 # Installed as dependency for ipykernel, jupyter-client, nbformat
+# keyring==25.4.1 # Installed as dependency for twine
+# markdown-it-py==3.0.0 # Installed as dependency for rich
+# markupsafe==3.0.2 # Installed as dependency for jinja2
+# matplotlib-inline==0.1.7 # Installed as dependency for ipykernel, ipython
+# mdurl==0.1.2 # Installed as dependency for markdown-it-py
+# more-itertools==10.5.0 # Installed as dependency for inflect, jaraco.classes, jaraco.functools, jaraco.text
+# mypy-extensions==1.0.0 # Installed as dependency for black
+# nbformat==5.10.4 # Installed as dependency for nbstripout
+# nest-asyncio==1.6.0 # Installed as dependency for ipykernel
+# nh3==0.2.18 # Installed as dependency for readme-renderer
+# nodeenv==1.9.1 # Installed as dependency for pre-commit, pyright
+# packaging==24.1 # Installed as dependency for black, build, commitizen, ipykernel, pytest
+# parso==0.8.4 # Installed as dependency for jedi
+# pathspec==0.12.1 # Installed as dependency for black
+# pexpect==4.9.0 # Installed as dependency for ipython
+# pkginfo==1.11.2 # Installed as dependency for twine
+# platformdirs==4.3.6 # Installed as dependency for black, jupyter-core, virtualenv
+# pluggy==1.5.0 # Installed as dependency for pytest
+# prompt-toolkit==3.0.48 # Installed as dependency for ipython, questionary
+# psutil==6.1.0 # Installed as dependency for ipykernel
+# ptyprocess==0.7.0 # Installed as dependency for pexpect
+# pure-eval==0.2.3 # Installed as dependency for stack-data
+# pygments==2.18.0 # Installed as dependency for ipython, readme-renderer, rich
+# pyproject-hooks==1.2.0 # Installed as dependency for build
+# pytest==8.3.3 # Installed as dependency for pytest-cov
+# python-dateutil==2.9.0.post0 # Installed as dependency for jupyter-client
+# pyyaml==6.0.2 # Installed as dependency for commitizen, pre-commit
+# pyzmq==26.2.0 # Installed as dependency for ipykernel, jupyter-client
+# questionary==2.0.1 # Installed as dependency for commitizen
+# readme-renderer==44.0 # Installed as dependency for twine
+# referencing==0.35.1 # Installed as dependency for jsonschema, jsonschema-specifications
+# requests==2.32.3 # Installed as dependency for requests-toolbelt, twine
+# requests-toolbelt==1.0.0 # Installed as dependency for twine
+# rfc3986==2.0.0 # Installed as dependency for twine
+# rich==13.9.3 # Installed as dependency for twine
+# rpds-py==0.20.0 # Installed as dependency for jsonschema, referencing
+# six==1.16.0 # Installed as dependency for asttokens, python-dateutil
+# stack-data==0.6.3 # Installed as dependency for ipython
+# termcolor==2.5.0 # Installed as dependency for commitizen
+# tomlkit==0.13.2 # Installed as dependency for commitizen
+# tornado==6.4.1 # Installed as dependency for ipykernel, jupyter-client
+# traitlets==5.14.3 # Installed as dependency for comm, ipykernel, ipython, jupyter-client, jupyter-core, matplotlib-inline, nbformat
+# typeguard==4.3.0 # Installed as dependency for inflect
+# typing-extensions==4.12.2 # Installed as dependency for ipython, pyright, typeguard
+# urllib3==2.2.3 # Installed as dependency for requests, twine
+# virtualenv==20.27.0 # Installed as dependency for pre-commit
+# wcwidth==0.2.13 # Installed as dependency for prompt-toolkit
+# zipp==3.20.2 # Installed as dependency for importlib-metadata

rudi_node_write-1.2.2/requirements.txt

@@ -0,0 +1,6 @@
+beautifulsoup4==4.12.3
+chardet==5.2.0
+deepdiff==8.0.1
+puremagic==1.28
+# orderly-set==5.2.3 # Installed as dependency for deepdiff
+# soupsieve==2.6 # Installed as dependency for beautifulsoup4

{rudi_node_write-1.2.0 → rudi_node_write-1.2.2}/src/rudi_node_write/connectors/io_rudi_manager_write.py

@@ -55,6 +55,8 @@ def ensure_url_startswith_api(url):
 
 
 def ensure_url_startswith_api_data(url):
+    if url.startswith("data"):
+        return ensure_url_startswith_api(url)
     return ensure_url_startswith(url, "api/data")
 
 
@@ -367,7 +369,7 @@ class RudiNodeManagerConnector(Connector):
             if url_bit in RUDI_OBJECT_TYPES:
                 data: list[dict] = self._get_full_obj_list(url_bit)
             else:
-                data = self._get_api(url=url_bit, headers=self._pm_headers)  # type: ignore
+                data = self._get_api(url=url_bit, headers=self._pm_headers, should_log_request=True)  # type: ignore
 
             self._data_cache[url_bit] = {"data": data, _REFRESH_KEY: time()}
         obj_data = self._data_cache[url_bit]["data"]
@@ -394,7 +396,7 @@ class RudiNodeManagerConnector(Connector):
                 log_w(here, "Legacy node, request not available for 'media'")
                 return {}
         else:
-            return self._get_cache(slash_join("data", obj_type))
+            return self._get_cache(ensure_url_startswith(obj_type, "data"))
 
     # ----------[ Requesting data ]-------------------------------------------------------------------------------------
 
@@ -439,7 +441,7 @@ class RudiNodeManagerConnector(Connector):
 
     @property
     def last_metadata_update_date(self) -> Date | None:
-        res = self.get_data("resources/?sort=-updatedAt&limit=1&fields=updatedAt")
+        res = self.get_data("resources?sort=-updatedAt&limit=1&fields=updatedAt")
         if not isinstance(res, list):
             raise TypeError(f"The server should have returned a list, got:\n{res}")
         if len(res) == 0:
@@ -449,7 +451,7 @@ class RudiNodeManagerConnector(Connector):
 
     @property
     def last_data_update_date(self) -> Date | None:
-        res = self.get_data("resources/?sort=-dataset_dates.updated&limit=1&fields=dataset_dates.updated")
+        res = self.get_data("resources?sort=-dataset_dates.updated&limit=1&fields=dataset_dates.updated")
         print("last_data_update_date", res)
         if not isinstance(res, list):
             raise TypeError(f"The server should have returned a list, got:\n{res}")
@@ -1300,12 +1302,11 @@ class FileTooBigException(Exception):
 if __name__ == "__main__":  # pragma: no cover
     begin = time()
     tests = "RudiNodeManagerConnector tests"
-    creds_file = "../creds/creds_bas.json"
+    creds_file = "../creds/creds_release.json"
     rudi_node_creds = read_json_file(creds_file)
-    node_url = rudi_node_creds["url"]
-    log_d(tests, "node_url", node_url)
+    pm_url = rudi_node_creds["pm_url"]
+    log_d(tests, "node_url", pm_url)
     auth = RudiNodeAuth.from_json(rudi_node_creds)
-    pm_url = slash_join(node_url, "/prodmanager")
 
     pm_connector = RudiNodeManagerConnector(server_url=pm_url, auth=auth)
     pm_connector.test_connection()
@@ -1354,4 +1355,7 @@ if __name__ == "__main__":  # pragma: no cover
             )
         except HttpError as e:
             log_w(tests, e)
+
+    log_d(tests, "last_metadata_update_date", pm_connector.last_metadata_update_date)
+    log_d(tests, "last_data_update_date", pm_connector.last_data_update_date)
     log_d(tests, "exec. time", time() - begin)

{rudi_node_write-1.2.0 → rudi_node_write-1.2.2}/src/rudi_node_write/rudi_types/rudi_media.py

@@ -52,6 +52,13 @@ def check_is_accepted_value(value, accepted_values: list | None = None):
     return value
 
 
+def get_normalized_type_name(value):
+    if isinstance(value, str):
+        if value.lower() in ("true", "false"):
+            return "bool"
+    return get_type_name(value)
+
+
 def normalize_connector_values(value, value_type: str | None = None, accepted_values: list | None = None):
     if value_type is None:
         type_name = get_type_name(value)
@@ -66,7 +73,7 @@ def normalize_connector_values(value, value_type: str | None = None, accepted_va
     else:
         # log_d(here, "in", value, f"({value_type})")
         normalized_val_type = normalize_connector_parameter_type(value_type)
-        if normalize_connector_parameter_type(get_type_name(value)) != normalized_val_type:
+        if normalize_connector_parameter_type(get_normalized_type_name(value)) != normalized_val_type:
             raise ValueError(f"incoherence in connector parameter type: input '{value}' is not of type '{value_type}'")
         return [check_is_accepted_value(value, accepted_values), normalized_val_type, accepted_values]
 

{rudi_node_write-1.2.0 → rudi_node_write-1.2.2}/src/rudi_node_write/utils/file_utils.py

@@ -70,6 +70,8 @@ def get_file_mime(file_local_path: str) -> str:
     """
     file_info = magic_file(file_local_path)
     if not file_info:  # pragma: no cover
+        if get_file_extension(file_local_path) == ".csv":
+            return "text/csv"
         return "application/octet-stream"
     mime_type = file_info[0].mime_type
     if mime_type == "application/x-gzip":  # pragma: no cover
@@ -158,13 +160,17 @@ if __name__ == "__main__":  # pragma: no cover
     tests = "FileUtils"
     begin = time()
 
-    test_file_dir = "../../../tests/_test_files/"
+    test_file_dir = "../tests/_test_files/"
     yaml_file_path = test_file_dir + "RUDI producer internal API - 1.3.0.yml"
     right_path = test_file_dir + "unicorn.png"
     bin_file = test_file_dir + "WERTSTOFFE.m8s"
     tar_gz_file = test_file_dir + "rudi-node-read.tar.gz"
     txt_file = test_file_dir + "RUDI producer internal API - 1.3.0.yml"
+    csv_file = test_file_dir + "dummy.csv"
     wrong_path = test_file_dir + "toto"
+    unicode_file_path = test_file_dir + "unicode_chars.txt"
+    write_file(unicode_file_path, "tut0156êµîƒfiÌπÏ“{ëôøÇ¡¶{µœ≤é≤")
+
     log_d(tests, "exists file OK", exists_file(right_path))
     log_d(tests, "exists file KO", exists_file(wrong_path))
     log_d(tests, "is file OK", is_file(right_path))
@@ -172,22 +178,38 @@ if __name__ == "__main__":  # pragma: no cover
     log_d(tests, "exists file (dir)", exists_file(test_file_dir))
     log_d(tests, "is dir a file", is_file(test_file_dir))
     # log_d(here, 'check_is_file', check_is_file(test_file_dir))
-    log_d(tests, "file size", get_file_size(right_path))
-    log_d(tests, "file extension", get_file_extension(right_path))
-    log_d(tests, "file extension", get_file_extension("foobar.tar.gz"))
-    log_d(tests, "file extension", get_file_extension("foo.bar.tar.gz"))
-    log_d(tests, "file extension", get_file_extension(".bashrc"))
-    log_d(tests, "file extension", get_file_extension(yaml_file_path))
-    log_d(tests, "file mime", get_file_mime(right_path))
-    log_d(tests, "file charset", get_file_charset(yaml_file_path))
-    log_d(tests, "file info", FileDetails(right_path))
-    log_d(tests, "file info", FileDetails(yaml_file_path))
-    log_d(tests, "file info", FileDetails(bin_file))
-    log_d(tests, "file info", FileDetails(tar_gz_file))
-    log_d(tests, "file info", FileDetails(txt_file))
+    for file in [
+        right_path,
+        "foobar.tar.gz",
+        "foo.bar.tar.gz",
+        ".bashrc",
+        yaml_file_path,
+        csv_file,
+        bin_file,
+        tar_gz_file,
+        txt_file,
+        unicode_file_path,
+    ]:
+        if exists_file(file):
+            log_d(
+                tests,
+                f"file: '{file}' ",
+                f"\n- exists_file: {exists_file(file)}",
+                f"\n- extension: {get_file_extension(file)}",
+                f"\n- size: {get_file_size(file)}",
+                f"\n- MIME: {get_file_mime(file)}",
+                f"\n- charset: {get_file_charset(file)}",
+                f"\n- MD5: {get_file_hash(file)}",
+                f"\n- info: {FileDetails(file)}",
+            )
+        else:
+            log_d(
+                tests,
+                f"file: '{file}' ",
+                f"\n- exists_file: {exists_file(file)}",
+                f"\n- extension: {get_file_extension(file)}",
+            )
     # log_d(here, 'file info', get_file_info(wrong_path))
-    unicode_file_path = test_file_dir + "unicode_chars.txt"
-    write_file(unicode_file_path, "tut0156êµîƒfiÌπÏ“{ëôøÇ¡¶{µœ≤é≤")
 
     log_d(tests, "unicode file MD5 hash", get_file_hash(unicode_file_path))
     log_d(tests, "unicode file SHA-256 hash", get_file_hash(unicode_file_path, "sha256"))

{rudi_node_write-1.2.0 → rudi_node_write-1.2.2}/src/rudi_node_write/utils/list_utils.py

@@ -101,16 +101,16 @@ def clean_nones(value):
         return value
 
 
-#   if __name__ == "__main__":  # pragma: no cover
-#     tests = "tests"
-#     begin = time()
-#     a = [1, 2, 3, 4, 5]
-#     b = [9, 8, 7, 6, {"r": [5, 6]}]
-#     c = [8, 7, 6, {"r": [5, 6]}, 9]
-#     print(tests, f"{a} Δ {b}", list_diff(a, b))
-#     print(tests, f"{a} Δ {c}", list_diff(a, c))
-#     print(tests, f"{b} Δ {c}", list_diff(b, c))
-#     print(tests, "b == c", are_list_equal(b, c))
-#     # print(tests, f'{b} == {c} ->', are_list_equal(b, c))
-#     # print(tests, f'dict eq ->', {'r': [6, 5]} != {'r': [5, 6]})
-#     log_d(tests, "exec. time", time() - begin)
+if __name__ == "__main__":  # pragma: no cover
+    tests = "tests"
+    # begin = time()
+    a = [1, 2, 3, 4, 5]
+    b = [9, 8, 7, 6, {"r": [5, 6]}]
+    c = [8, 7, 6, {"r": [5, 6]}, 9]
+    print(tests, f"{a} Δ {b}", list_diff(a, b))
+    print(tests, f"{a} Δ {c}", list_diff(a, c))
+    print(tests, f"{b} Δ {c}", list_diff(b, c))
+    print(tests, "b == c", are_list_equal(b, c))
+    # print(tests, f'{b} == {c} ->', are_list_equal(b, c))
+    # print(tests, f'dict eq ->', {'r': [6, 5]} != {'r': [5, 6]})
+    # log_d(tests, "exec. time", time() - begin)

{rudi_node_write-1.2.0 → rudi_node_write-1.2.2/src/rudi_node_write.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rudi-node-write
-Version: 1.2.0
+Version: 1.2.2
 Summary: Use the internal API of a RUDI Producer node
 Author-email: Olivier Martineau <olivier.martineau@irisa.fr>
 Maintainer-email: Olivier Martineau <olivier.martineau@irisa.fr>
@@ -17,6 +17,29 @@ Classifier: License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENCE.md
+Requires-Dist: beautifulsoup4==4.12.3
+Requires-Dist: chardet==5.2.0
+Requires-Dist: deepdiff==8.0.1
+Requires-Dist: puremagic==1.28
+Provides-Extra: dev
+Requires-Dist: black==24.10.0; extra == "dev"
+Requires-Dist: build==1.2.2.post1; extra == "dev"
+Requires-Dist: commitizen==3.29.1; extra == "dev"
+Requires-Dist: importlib-resources==6.4.5; extra == "dev"
+Requires-Dist: inflect==7.3.1; extra == "dev"
+Requires-Dist: ipykernel==6.29.5; extra == "dev"
+Requires-Dist: jaraco.collections==5.1.0; extra == "dev"
+Requires-Dist: nbstripout==0.7.1; extra == "dev"
+Requires-Dist: ordered-set==4.1.0; extra == "dev"
+Requires-Dist: pip-autoremove==0.10.0; extra == "dev"
+Requires-Dist: pip-chill==1.0.3; extra == "dev"
+Requires-Dist: pip-upgrade-outdated==1.5; extra == "dev"
+Requires-Dist: pip3-autoremove==1.2.2; extra == "dev"
+Requires-Dist: pre-commit==4.0.1; extra == "dev"
+Requires-Dist: pyright==1.1.386; extra == "dev"
+Requires-Dist: pytest-cov==5.0.0; extra == "dev"
+Requires-Dist: tomli==2.0.2; extra == "dev"
+Requires-Dist: twine==5.1.1; extra == "dev"
 
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Checked with pyright](https://microsoft.github.io/pyright/img/pyright_badge.svg)](https://microsoft.github.io/pyright/)

{rudi_node_write-1.2.0 → rudi_node_write-1.2.2}/src/rudi_node_write.egg-info/SOURCES.txt

@@ -1,12 +1,14 @@
 LICENCE.md
 README.md
 pyproject.toml
+requirements-dev.txt
 requirements.txt
 src/rudi_node_write/__init__.py
 src/rudi_node_write/rudi_node_writer.py
 src/rudi_node_write.egg-info/PKG-INFO
 src/rudi_node_write.egg-info/SOURCES.txt
 src/rudi_node_write.egg-info/dependency_links.txt
+src/rudi_node_write.egg-info/requires.txt
 src/rudi_node_write.egg-info/top_level.txt
 src/rudi_node_write/conf/meta_defaults.py
 src/rudi_node_write/connectors/io_connector.py
@@ -37,6 +39,4 @@ src/rudi_node_write/utils/str_utils.py
 src/rudi_node_write/utils/type_date.py
 src/rudi_node_write/utils/typing_utils.py
 src/rudi_node_write/utils/url_utils.py
-src/rudi_node_write/wip/federation_backup.py
-src/rudi_node_write/wip/rudinode_federation.py
 tests/test_rudi_node_write.py

rudi_node_write-1.2.2/src/rudi_node_write.egg-info/requires.txt

@@ -0,0 +1,24 @@
+beautifulsoup4==4.12.3
+chardet==5.2.0
+deepdiff==8.0.1
+puremagic==1.28
+
+[dev]
+black==24.10.0
+build==1.2.2.post1
+commitizen==3.29.1
+importlib-resources==6.4.5
+inflect==7.3.1
+ipykernel==6.29.5
+jaraco.collections==5.1.0
+nbstripout==0.7.1
+ordered-set==4.1.0
+pip-autoremove==0.10.0
+pip-chill==1.0.3
+pip-upgrade-outdated==1.5
+pip3-autoremove==1.2.2
+pre-commit==4.0.1
+pyright==1.1.386
+pytest-cov==5.0.0
+tomli==2.0.2
+twine==5.1.1

rudi_node_write-1.2.0/src/rudi_node_write/wip/federation_backup.py

@@ -1,145 +0,0 @@
-from concurrent.futures import as_completed, ThreadPoolExecutor
-from os import cpu_count
-from time import sleep, time
-from urllib.request import urlopen
-
-from rudi_node_write.connectors.io_rudi_manager_write import RudiNodeManagerConnector
-from rudi_node_write.connectors.rudi_node_auth import RudiNodeAuth
-from rudi_node_write.rudi_node_writer import RudiNodeWriter
-from rudi_node_write.utils.file_utils import check_is_dir, check_is_file, read_json_file, write_json_file
-from rudi_node_write.utils.log import log_d, log_e, log_w
-from rudi_node_write.utils.str_utils import slash_join
-from rudi_node_write.utils.type_date import Date
-
-federation_data = {}
-
-
-MAX_WORKERS = 0
-DEFAULT_WAIT_TIME = 0.3
-
-
-def try_request(req, wait_time=DEFAULT_WAIT_TIME, quit_on_fail=True):
-    fun = "try_request"
-    sleep(wait_time)
-    try:
-        return req
-    except Exception as err:
-        if quit_on_fail:
-            return "FAIL"
-        # Let's try one more time
-        return try_request(req, wait_time, True)
-
-
-def backup_node(node_name, node_info):
-    node_url = node_info["url"]
-    pm_url = node_info["pmback"] if node_info.get("pmback") is not None else (slash_join(node_url, "prodmanager"))
-    if not pm_url.startswith("http"):
-        pm_url = "https://" + pm_url
-
-    node = RudiNodeWriter(pm_url=pm_url, auth=auth, keep_connection=True)
-
-    node_data = {"node_name": node_name, "pm_url": node.pm_url}
-    node_data["media_url"] = try_request(node.media_url)
-    node_data["init_data"] = try_request(node.init_data)
-    node_data["organization_list"] = try_request(node.organization_list)
-    node_data["contact_list"] = try_request(node.contact_list)
-    node_data["metadata_list"] = try_request(node.metadata_list)
-    node_data["enums"] = try_request(node.enums)
-    node_data["media_list"] = try_request(node.media_list)
-
-    node.close_connection()
-    return node_data
-
-
-if __name__ == "__main__":  # pragma: no cover
-    begin = time()
-    here = "RudiNodeFederation backup"
-
-    test_dir = check_is_dir("../dwnld")
-    data_dir = check_is_dir("../data")
-    creds_file = check_is_file("../creds/test_creds_manager.json")
-    servers_file = check_is_file("../creds/servers.json")
-
-    servers_list = read_json_file(servers_file)
-    nodes = servers_list.get("nodes")
-
-    # nodes = {"exatow": nodes["exatow"]}
-
-    nb_nodes = len(nodes.keys())
-
-    rudi_node_creds = read_json_file(creds_file)
-    auth = RudiNodeAuth.from_json(rudi_node_creds)
-    url = rudi_node_creds["url"]
-    pm_url = url if url.endswith("prodmanager") else slash_join(url, "prodmanager")
-    assert isinstance(auth, RudiNodeAuth)
-    errors = {}
-    if MAX_WORKERS:
-        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
-            futures = {executor.submit(backup_node, node_name, nodes[node_name]): node_name for node_name in nodes}
-            for future in as_completed(futures, timeout=120):
-                try:
-                    future_result = future.result()
-                except Exception as err:
-                    node_name = futures[future]
-                    log_e(here, "Backup failed for node:", futures[future])
-                    future_result = {}
-                    errors[node_name] = err
-                node_name = futures[future]
-                future_result["node_name"] = None
-                federation_data[node_name] = future_result
-    else:
-        # federation_info = {nodes[node_name] for node_name in nodes}
-
-        for node_name in nodes:
-            log_d(here, "----- node_name", node_name)
-            node_info = nodes[node_name]
-            try:
-                node_data = backup_node(node_name, node_info)
-                federation_data[node_name] = node_data
-
-                write_json_file(destination_file_path=slash_join(data_dir, node_name + ".json"), json_dict=node_data)
-            except Exception as e:
-                errors[node_name] = e
-                node_data = None
-
-            federation_data[node_name] = node_data
-
-    write_json_file(destination_file_path=slash_join(data_dir, "_federation.json"), json_dict=federation_data)
-    if len(errors.keys()) > 0:
-        log_e(here, "Backup failed for some nodes:", errors)
-    else:
-        log_d(here, "All nodes were successfully backed up")
-
-    log_d(here, "exec. time", time() - begin)
-
-# urls = [
-#     # "http://www.foxnews.com/",
-#     "http://www.cnn.com/",
-#     "http://europe.wsj.com/",
-#     "http://www.bbc.co.uk/",
-#     "http://some-made-up-domain.com/",
-# ]
-
-# url_dict = {url + "r": url for url in urls}
-# print(url_dict)
-
-
-# def load_url(url, timeout):
-#     with urlopen(url, timeout=timeout) as conn:
-#         return conn.read()
-
-
-# with ThreadPoolExecutor(max_workers=5) as executor:
-#     future_to_url = {executor.submit(load_url, url, 60): url for url in urls}
-#     for future in as_completed(future_to_url):
-#         url = future_to_url[future]
-#         try:
-#             data = future.result()
-#         except Exception as exc:
-#             print("%r generated an exception: %s" % (url, exc))
-#         else:
-#             print("%r page is %d bytes" % (url, len(data)))
-
-# print(cpu_count() * 5)
-# pool = ThreadPoolExecutor()
-# print(pool._max_workers)

rudi_node_write-1.2.0/src/rudi_node_write/wip/rudinode_federation.py

@@ -1,412 +0,0 @@
-from abc import ABC
-from concurrent.futures import as_completed, ThreadPoolExecutor
-from dataclasses import dataclass
-from os import cpu_count
-from time import sleep, time
-from typing import Callable
-from urllib.request import urlopen
-
-from rudi_node_write.connectors.io_connector import HTTP_REQUEST_METHODS
-from rudi_node_write.connectors.io_rudi_api_write import RudiNodeApiConnector
-from rudi_node_write.connectors.io_rudi_jwt_factory import RudiNodeJwtFactory
-from rudi_node_write.connectors.io_rudi_manager_write import RudiNodeManagerConnector
-from rudi_node_write.connectors.rudi_node_auth import RudiNodeAuth
-from rudi_node_write.rudi_node_writer import RudiNodeWriter
-from rudi_node_write.utils.file_utils import check_is_dir, check_is_file, read_json_file, write_json_file
-from rudi_node_write.utils.log import log_d, log_e, log_w
-from rudi_node_write.utils.str_utils import slash_join
-from rudi_node_write.utils.type_date import Date
-from rudi_node_write.utils.url_utils import ensure_http, ensure_url_startswith
-
-federation_data = {}
-
-DEFAULT_WAIT_TIME = 0.3
-
-NO_PORTAL = "No portal connected"
-
-
-def ensure_url_startswith_api(url):
-    return ensure_url_startswith(url, "api")
-
-
-class Module:
-    MANAGER = "manager"
-    CATALOG = "catalog"
-    STORAGE = "storage"
-
-
-class HttpMethods:
-    GET = "GET"
-    POST = "POST"
-    PUT = "PUT"
-    DELETE = "DELETE"
-
-
-@dataclass
-class ModuleRequest:
-    action_name: str
-    module_name: Module = Module.MANAGER
-    req_method: HttpMethods = HttpMethods.GET
-    url_bit: str = "/"
-    post_treatment: Callable = lambda x: x
-    can_fail: bool = False
-    keep_alive: bool = False
-    should_log_request: bool = False
-    should_log_response: bool = False
-
-
-# TODO: envoyer une suite/liste de traitements sur les nœuds:
-# - quel connecteur cible (node_write ? catalog ? media ?)
-# - quelle URL requête (url_bit )
-# - quel post-traitement
-#
-# => Input: objet à définir
-# #
-# [{
-#     target: Module.MANAGER | Module.CATALOG | MEDIA,
-#     url_bit,
-#     post_treatment on answer (defaulted to x=>x)
-#     opts: keep_alive, log request, log response (defaulted to False)
-# }]
-class NodeConnector:
-    def __init__(
-        self,
-        node_name,
-        catalog_url,
-        catalog_auth,
-        manager_url,
-        manager_auth,
-    ):
-        self._node_name = node_name
-        self._catalog = CatalogConnector(node_name=node_name, module_url=catalog_url, module_auth=catalog_auth)
-        self._manager = ManagerConnector(node_name=node_name, module_url=manager_url, module_auth=manager_auth)
-
-
-class ModuleConnector(ABC):
-    def __init__(self, node_name: str, get_connector: Callable, get_headers: Callable):
-        self._node_name = node_name
-        self._get_connector = get_connector
-        self._get_headers = get_headers
-
-    @property
-    def node_name(self):
-        return self._node_name
-
-    @property
-    def module_name(self):
-        raise NotImplementedError("This function should be implemented")
-
-    @property
-    def connector(self):
-        return self._get_connector()
-
-    @property
-    def headers(self):
-        return self._get_headers()
-
-    @staticmethod
-    def create_connector(node_name, module_name, module_url, module_auth):
-        if module_name == Module.MANAGER:
-            return ManagerConnector(node_name=node_name, module_url=module_url, module_auth=module_auth)
-        if module_name == Module.CATALOG:
-            return CatalogConnector(node_name=node_name, module_url=module_url, module_auth=module_auth)
-
-    def module_request(
-        self,
-        request: ModuleRequest,
-    ):
-        return self.request(
-            req_method=request.req_method,
-            url_bit=request.url_bit,
-            post_treatment=request.post_treatment,
-            can_fail=request.can_fail,
-            keep_alive=request.keep_alive,
-            should_log_request=request.should_log_request,
-            should_log_response=request.should_log_response,
-        )
-
-    def request(
-        self,
-        req_method: HttpMethods,
-        url_bit: str,
-        post_treatment=lambda x: x,
-        can_fail: bool = False,
-        keep_alive: bool = False,
-        should_log_request: bool = False,
-        should_log_response: bool = False,
-    ):
-        try:
-            node_res = self.connector.request(
-                req_method=req_method,
-                relative_url=ensure_url_startswith_api(url_bit),
-                headers=self.headers,
-                keep_alive=keep_alive,
-                should_log_request=should_log_request,
-                should_log_response=should_log_response,
-            )
-            return post_treatment(node_res)
-        except Exception as err:
-            if can_fail:
-                raise err
-            return f"ERR: {err}"
-
-
-class ManagerConnector(ModuleConnector):
-    def __init__(self, node_name, module_url, module_auth):
-        self._node_name = node_name
-        self._node_writer = RudiNodeWriter(pm_url=module_url, auth=module_auth, headers_user_agent="ManagerConnector")
-        super().__init__(
-            node_name=node_name,
-            get_connector=lambda: self._node_writer._pm_connector,
-            get_headers=lambda: self._node_writer._pm_connector._pm_headers,
-        )
-
-
-class CatalogConnector(ModuleConnector):
-    def __init__(self, node_name, module_url, module_auth):
-        self._node_name = node_name
-        self._catalog_connector = RudiNodeApiConnector(
-            server_url=module_url, jwt_factory=module_auth, headers_user_agent="CatalogConnector"
-        )
-        super().__init__(
-            node_name=node_name,
-            get_connector=lambda: self._catalog_connector,
-            get_headers=lambda: self._catalog_connector._headers,
-        )
-
-
-class RudiNodeFederation:
-    def __init__(self, config_file_path, credentials_file_path, local_jwt_factory):
-        self.jwt_factory = local_jwt_factory
-        self.load_federation_conf(config_file_path, credentials_file_path)
-
-    def load_federation_conf(self, config_file_path, credentials_file_path):
-        self._conf = read_json_file(config_file_path)
-        self._creds = read_json_file(credentials_file_path)
-        self._connectors = {}
-
-        # self._conf["nodes"] = {"exatow": self._conf["nodes"]["exatow"], "rm": self._conf["nodes"]["rm"]}
-
-        for node_name, node_conf in self.nodes_conf.items():
-            print(node_name)
-            node_url = ensure_http(node_conf["url"])
-
-            pm_url = ensure_http(
-                node_conf["pmback"] if node_conf.get("pmback") is not None else slash_join(node_url, "prodmanager")
-            )
-
-            self._connectors[node_name] = {
-                Module.MANAGER: ManagerConnector(
-                    node_name=node_name,
-                    module_url=pm_url,
-                    module_auth=RudiNodeAuth(b64url_auth=self.get_creds(node_name).get("b64auth")),
-                ),
-                Module.CATALOG: CatalogConnector(
-                    node_name=node_name, module_url=node_url, module_auth=self.jwt_factory
-                ),
-            }
-
-    @property
-    def nodes_conf(self):
-        return self.get_node_conf()
-
-    def get_node_conf(self, node_name=None):
-        if node_name is None:
-            return self._conf["nodes"]
-        return self._conf["nodes"].get(node_name)
-
-    def get_creds(self, node_name):
-        if self._creds.get(node_name):
-            return self._creds[node_name]
-        return self._creds["defaults"]
-
-    def request(
-        self,
-        node_name,
-        module_name,
-        req_method,
-        url_bit,
-        post_treatment=lambda x: x,
-        can_fail=False,
-        keep_alive=False,
-        should_log_request=False,
-        should_log_response=False,
-    ):
-        if module_name == Module.MANAGER:
-            connector = self._managers[node_name]
-            headers = connector._pm_headers
-            url_bit = ensure_url_startswith_api(url_bit)
-        elif module_name == Module.CATALOG:
-            connector = self._catalogs[node_name]
-            headers = connector._headers
-            url_bit = ensure_url_startswith_api(url_bit)
-        else:
-            raise NotImplementedError("Not yet available")
-        try:
-            node_res = connector.request(
-                req_method=req_method,
-                relative_url=url_bit,
-                headers=headers,
-                keep_alive=keep_alive,
-                should_log_request=should_log_request,
-                should_log_response=should_log_response,
-            )
-            return post_treatment(node_res)
-        except Exception as err:
-            if can_fail:
-                raise err
-            return f"ERR: {err}"
-
-    def batch_request_node(self, connectors, request_list, max_workers=0):
-        res = {}
-        err = {}
-        for request in request_list:
-            if isinstance(request, ModuleRequest):
-                res[request.action_name] = connectors[request.module_name].module_request(request)
-            else:
-                (action_name, module_name, req_method, url_bit, post_treatment) = request
-                res[action_name] = connectors[module_name].request(
-                    req_method=req_method,
-                    url_bit=url_bit,
-                    post_treatment=post_treatment,
-                )
-        return res
-
-    def batch_requests(self, request_list, max_workers=0):
-        """
-        @param request_dict: a dict with module names as keys and a list of actions {action_name: (module_name, req_method , url_bit , post_treatment)}}
-        """
-        res = {}
-        err = {}
-        if not max_workers:
-            for node_name, connectors in self._connectors.items():
-                res[node_name] = self.batch_request_node(connectors, request_list)
-        else:
-            with ThreadPoolExecutor(max_workers=max_workers) as executor:
-                futures = {
-                    executor.submit(connectors[request.module_name].module_request, request): request
-                    for request in request_list
-                }
-                for future in as_completed(futures, timeout=120):
-                    try:
-                        future_result = future.result()
-                    except Exception as err:
-                        node_name = futures[future]
-                        log_e(here, "Request failed:", futures[future])
-                        future_result = {}
-                        errors[request.node_name] = err
-                    node_name = futures[future]
-                    future_result["node_name"] = None
-                    federation_data[node_name] = future_result
-        return res
-
-    def try_request(req, wait_time=DEFAULT_WAIT_TIME, quit_on_fail=True):
-        fun = "try_request"
-        sleep(wait_time)
-        try:
-            return req
-        except Exception as err:
-            if quit_on_fail:
-                return "FAIL"
-            # Let's try one more time
-            return try_request(req, wait_time, True)
-
-    def backup_node(node_name, node_info):
-        node_url = node_info["url"]
-        pm_url = node_info["pmback"] if node_info.get("pmback") is not None else (slash_join(node_url, "prodmanager"))
-        ensure_http(pm_url)
-
-        node = RudiNodeWriter(pm_url=pm_url, auth=auth, keep_connection=True)
-
-        node_data = {"node_name": node_name, "pm_url": node.pm_url}
-        node_data["media_url"] = try_request(node.media_url)
-        node_data["init_data"] = try_request(node.init_data)
-        node_data["organization_list"] = try_request(node.organization_list)
-        node_data["contact_list"] = try_request(node.contact_list)
-        node_data["metadata_list"] = try_request(node.metadata_list)
-        node_data["enums"] = try_request(node.enums)
-        node_data["media_list"] = try_request(node.media_list)
-
-        node.close_connection()
-        return node_data
-
-
-if __name__ == "__main__":  # pragma: no cover
-    begin = time()
-    here = "RudiNodeFederation"
-
-    test_dir = check_is_dir("../dwnld")
-    data_dir = check_is_dir("../data")
-    creds_file = check_is_file("../creds/federation_creds.json")
-    servers_file = check_is_file("../creds/servers.json")
-
-    try:
-        local_jwt_factory = RudiNodeJwtFactory("http://localhost:4040", {"sub": "rudi_api_pm"})
-    except:
-        raise ConnectionError("Local crypto module is apparently not launched!")
-
-    rudi_federation = RudiNodeFederation(
-        config_file_path=servers_file, credentials_file_path=creds_file, local_jwt_factory=local_jwt_factory
-    )
-
-    def get_token(x):
-        if x == NO_PORTAL:
-            return NO_PORTAL
-        try:
-            return x["access_token"].replace(r"[\w-]+(\.[\w-]+){2}", "valid_jwt")
-        except:
-            return x
-
-    res = rudi_federation.batch_requests(
-        [
-            ModuleRequest(
-                action_name="portal_url",
-                module_name=Module.MANAGER,
-                url_bit="front/portal-url",
-                can_fail=True,
-                should_log_request=True,
-            ),
-            ModuleRequest(
-                action_name="portal_token",
-                module_name=Module.CATALOG,
-                url_bit="admin/portal/token",
-                post_treatment=get_token,
-            ),
-        ]
-    )
-    portal_connections_file = "rudi_node_write/wip/portal_connect.json"
-    reworked_portal_connect_file = "rudi_node_write/wip/portal_connect_final.json"
-    # res = read_json_file("rudi_node_write/wip/portal_connect.json")
-    print(res)
-    write_json_file(destination_file_path=portal_connections_file, json_dict=res)
-    final_res = {}
-    for node_name, node_res in res.items():
-        portal_url = node_res["portal_url"]
-        print(portal_url)
-        portal_connection = (
-            node_res["portal_token"]
-            if not node_res["portal_token"].startswith("eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9")
-            else "OK"
-        )
-        if final_res.get(portal_url) is None:
-            final_res[portal_url] = {node_name: portal_connection}
-        final_res[portal_url][node_name] = portal_connection
-    # print(res)
-    # portal_urls = rudi_federation.manager_get("front/portal-url")
-    # node_catalog_url = rudi_federation.manager_get("front/ext-api-url")
-    # catalog_portal_token = rudi_federation.catalog_get("portal/token", extract_token_from_http_res)
-    # node_portals = {}
-    # for node_name, portal_url in portal_urls.items():
-    #     node_info = {"catalog_url": node_catalog_url[node_name], "portal_token": catalog_portal_token}
-    #     if node_portals.get(portal_url) is None:
-    #         node_portals[portal_url] = {node_name: node_info}
-    #     else:
-    #         node_portals[portal_url][node_name] = node_info
-
-    # print(node_portals)
-    print(final_res)
-    write_json_file(destination_file_path=reworked_portal_connect_file, json_dict=final_res)
-
-print()
-
-# print(chr(sum(range(ord(min(str(not ())))))))