rudi-node-write 1.2.0__tar.gz → 1.2.1__tar.gz

{rudi_node_write-1.2.0/src/rudi_node_write.egg-info → rudi_node_write-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rudi-node-write
-Version: 1.2.0
+Version: 1.2.1
 Summary: Use the internal API of a RUDI Producer node
 Author-email: Olivier Martineau <olivier.martineau@irisa.fr>
 Maintainer-email: Olivier Martineau <olivier.martineau@irisa.fr>

{rudi_node_write-1.2.0 → rudi_node_write-1.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rudi-node-write"
-version = "1.2.0"
+version = "1.2.1"
 authors = [{ name = "Olivier Martineau", email = "olivier.martineau@irisa.fr" }]
 maintainers = [{ name = "Olivier Martineau", email = "olivier.martineau@irisa.fr" }]
 description = "Use the internal API of a RUDI Producer node"
@@ -34,7 +34,7 @@ target-version = ['py311']
 # ----- Tool: commitizen
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "1.2.0"
+version = "1.2.1"
 version_files = ["pyproject.toml:version"]
 
 # ----- Tool: pytest

{rudi_node_write-1.2.0 → rudi_node_write-1.2.1}/src/rudi_node_write/connectors/io_rudi_manager_write.py

@@ -55,6 +55,8 @@ def ensure_url_startswith_api(url):
 
 
 def ensure_url_startswith_api_data(url):
+    if url.startswith("data"):
+        return ensure_url_startswith_api(url)
     return ensure_url_startswith(url, "api/data")
 
 
@@ -367,7 +369,7 @@ class RudiNodeManagerConnector(Connector):
             if url_bit in RUDI_OBJECT_TYPES:
                 data: list[dict] = self._get_full_obj_list(url_bit)
             else:
-                data = self._get_api(url=url_bit, headers=self._pm_headers)  # type: ignore
+                data = self._get_api(url=url_bit, headers=self._pm_headers, should_log_request=True)  # type: ignore
 
             self._data_cache[url_bit] = {"data": data, _REFRESH_KEY: time()}
         obj_data = self._data_cache[url_bit]["data"]
@@ -394,7 +396,7 @@ class RudiNodeManagerConnector(Connector):
                 log_w(here, "Legacy node, request not available for 'media'")
                 return {}
         else:
-            return self._get_cache(slash_join("data", obj_type))
+            return self._get_cache(ensure_url_startswith(obj_type, "data"))
 
     # ----------[ Requesting data ]-------------------------------------------------------------------------------------
 
@@ -439,7 +441,7 @@ class RudiNodeManagerConnector(Connector):
 
     @property
     def last_metadata_update_date(self) -> Date | None:
-        res = self.get_data("resources/?sort=-updatedAt&limit=1&fields=updatedAt")
+        res = self.get_data("resources?sort=-updatedAt&limit=1&fields=updatedAt")
         if not isinstance(res, list):
             raise TypeError(f"The server should have returned a list, got:\n{res}")
         if len(res) == 0:
@@ -449,7 +451,7 @@ class RudiNodeManagerConnector(Connector):
 
     @property
     def last_data_update_date(self) -> Date | None:
-        res = self.get_data("resources/?sort=-dataset_dates.updated&limit=1&fields=dataset_dates.updated")
+        res = self.get_data("resources?sort=-dataset_dates.updated&limit=1&fields=dataset_dates.updated")
         print("last_data_update_date", res)
         if not isinstance(res, list):
             raise TypeError(f"The server should have returned a list, got:\n{res}")
@@ -1300,12 +1302,11 @@ class FileTooBigException(Exception):
 if __name__ == "__main__":  # pragma: no cover
     begin = time()
     tests = "RudiNodeManagerConnector tests"
-    creds_file = "../creds/creds_bas.json"
+    creds_file = "../creds/creds_release.json"
     rudi_node_creds = read_json_file(creds_file)
-    node_url = rudi_node_creds["url"]
-    log_d(tests, "node_url", node_url)
+    pm_url = rudi_node_creds["pm_url"]
+    log_d(tests, "node_url", pm_url)
     auth = RudiNodeAuth.from_json(rudi_node_creds)
-    pm_url = slash_join(node_url, "/prodmanager")
 
     pm_connector = RudiNodeManagerConnector(server_url=pm_url, auth=auth)
     pm_connector.test_connection()
@@ -1354,4 +1355,7 @@ if __name__ == "__main__":  # pragma: no cover
             )
         except HttpError as e:
             log_w(tests, e)
+
+    log_d(tests, "last_metadata_update_date", pm_connector.last_metadata_update_date)
+    log_d(tests, "last_data_update_date", pm_connector.last_data_update_date)
     log_d(tests, "exec. time", time() - begin)

{rudi_node_write-1.2.0 → rudi_node_write-1.2.1}/src/rudi_node_write/rudi_types/rudi_media.py

@@ -52,6 +52,13 @@ def check_is_accepted_value(value, accepted_values: list | None = None):
     return value
 
 
+def get_normalized_type_name(value):
+    if isinstance(value, str):
+        if value.lower() in ("true", "false"):
+            return "bool"
+    return get_type_name(value)
+
+
 def normalize_connector_values(value, value_type: str | None = None, accepted_values: list | None = None):
     if value_type is None:
         type_name = get_type_name(value)
@@ -66,7 +73,7 @@ def normalize_connector_values(value, value_type: str | None = None, accepted_va
     else:
         # log_d(here, "in", value, f"({value_type})")
         normalized_val_type = normalize_connector_parameter_type(value_type)
-        if normalize_connector_parameter_type(get_type_name(value)) != normalized_val_type:
+        if normalize_connector_parameter_type(get_normalized_type_name(value)) != normalized_val_type:
             raise ValueError(f"incoherence in connector parameter type: input '{value}' is not of type '{value_type}'")
         return [check_is_accepted_value(value, accepted_values), normalized_val_type, accepted_values]
 

{rudi_node_write-1.2.0 → rudi_node_write-1.2.1}/src/rudi_node_write/utils/file_utils.py

@@ -70,6 +70,8 @@ def get_file_mime(file_local_path: str) -> str:
     """
     file_info = magic_file(file_local_path)
     if not file_info:  # pragma: no cover
+        if get_file_extension(file_local_path) == ".csv":
+            return "text/csv"
         return "application/octet-stream"
     mime_type = file_info[0].mime_type
     if mime_type == "application/x-gzip":  # pragma: no cover
@@ -158,13 +160,17 @@ if __name__ == "__main__":  # pragma: no cover
     tests = "FileUtils"
     begin = time()
 
-    test_file_dir = "../../../tests/_test_files/"
+    test_file_dir = "../tests/_test_files/"
     yaml_file_path = test_file_dir + "RUDI producer internal API - 1.3.0.yml"
     right_path = test_file_dir + "unicorn.png"
     bin_file = test_file_dir + "WERTSTOFFE.m8s"
     tar_gz_file = test_file_dir + "rudi-node-read.tar.gz"
     txt_file = test_file_dir + "RUDI producer internal API - 1.3.0.yml"
+    csv_file = test_file_dir + "dummy.csv"
     wrong_path = test_file_dir + "toto"
+    unicode_file_path = test_file_dir + "unicode_chars.txt"
+    write_file(unicode_file_path, "tut0156êµîƒfiÌπÏ“{ëôøÇ¡¶{µœ≤é≤")
+
     log_d(tests, "exists file OK", exists_file(right_path))
     log_d(tests, "exists file KO", exists_file(wrong_path))
     log_d(tests, "is file OK", is_file(right_path))
@@ -172,22 +178,38 @@ if __name__ == "__main__":  # pragma: no cover
     log_d(tests, "exists file (dir)", exists_file(test_file_dir))
     log_d(tests, "is dir a file", is_file(test_file_dir))
     # log_d(here, 'check_is_file', check_is_file(test_file_dir))
-    log_d(tests, "file size", get_file_size(right_path))
-    log_d(tests, "file extension", get_file_extension(right_path))
-    log_d(tests, "file extension", get_file_extension("foobar.tar.gz"))
-    log_d(tests, "file extension", get_file_extension("foo.bar.tar.gz"))
-    log_d(tests, "file extension", get_file_extension(".bashrc"))
-    log_d(tests, "file extension", get_file_extension(yaml_file_path))
-    log_d(tests, "file mime", get_file_mime(right_path))
-    log_d(tests, "file charset", get_file_charset(yaml_file_path))
-    log_d(tests, "file info", FileDetails(right_path))
-    log_d(tests, "file info", FileDetails(yaml_file_path))
-    log_d(tests, "file info", FileDetails(bin_file))
-    log_d(tests, "file info", FileDetails(tar_gz_file))
-    log_d(tests, "file info", FileDetails(txt_file))
+    for file in [
+        right_path,
+        "foobar.tar.gz",
+        "foo.bar.tar.gz",
+        ".bashrc",
+        yaml_file_path,
+        csv_file,
+        bin_file,
+        tar_gz_file,
+        txt_file,
+        unicode_file_path,
+    ]:
+        if exists_file(file):
+            log_d(
+                tests,
+                f"file: '{file}' ",
+                f"\n- exists_file: {exists_file(file)}",
+                f"\n- extension: {get_file_extension(file)}",
+                f"\n- size: {get_file_size(file)}",
+                f"\n- MIME: {get_file_mime(file)}",
+                f"\n- charset: {get_file_charset(file)}",
+                f"\n- MD5: {get_file_hash(file)}",
+                f"\n- info: {FileDetails(file)}",
+            )
+        else:
+            log_d(
+                tests,
+                f"file: '{file}' ",
+                f"\n- exists_file: {exists_file(file)}",
+                f"\n- extension: {get_file_extension(file)}",
+            )
     # log_d(here, 'file info', get_file_info(wrong_path))
-    unicode_file_path = test_file_dir + "unicode_chars.txt"
-    write_file(unicode_file_path, "tut0156êµîƒfiÌπÏ“{ëôøÇ¡¶{µœ≤é≤")
 
     log_d(tests, "unicode file MD5 hash", get_file_hash(unicode_file_path))
     log_d(tests, "unicode file SHA-256 hash", get_file_hash(unicode_file_path, "sha256"))

rudi_node_write-1.2.1/src/rudi_node_write/wip/clean_node.py

@@ -0,0 +1,50 @@
+from abc import ABC
+from concurrent.futures import as_completed, ThreadPoolExecutor
+from dataclasses import dataclass
+from os import cpu_count
+from time import sleep, time
+from typing import Callable
+from urllib.request import urlopen
+
+from rudi_node_write.connectors.io_connector import HTTP_REQUEST_METHODS
+from rudi_node_write.connectors.io_rudi_api_write import RudiNodeApiConnector
+from rudi_node_write.connectors.io_rudi_jwt_factory import RudiNodeJwtFactory
+from rudi_node_write.connectors.io_rudi_manager_write import RudiNodeManagerConnector
+from rudi_node_write.connectors.rudi_node_auth import RudiNodeAuth
+from rudi_node_write.rudi_node_writer import RudiNodeWriter
+from rudi_node_write.utils.file_utils import check_is_dir, check_is_file, read_json_file, write_json_file
+from rudi_node_write.utils.log import log_d, log_e, log_w
+from rudi_node_write.utils.str_utils import slash_join
+from rudi_node_write.utils.type_date import Date
+from rudi_node_write.utils.url_utils import ensure_http, ensure_url_startswith
+
+if __name__ == "__main__":  # pragma: no cover
+    begin = time()
+    here = "RudiNodeClean"
+
+    test_dir = check_is_dir("../dwnld")
+    data_dir = check_is_dir("../data")
+    creds_file = read_json_file("../creds/creds_release.json")
+    auth = RudiNodeAuth(b64url_auth=creds_file["b64auth"])
+    node = RudiNodeWriter(creds_file["pm_url"], auth, keep_connection=True)
+
+    for meta in node.metadata_list:
+        changed = False
+        print(meta["global_id"], ":", meta["access_condition"]["confidentiality"]["gdpr_sensitive"])
+        if meta["access_condition"]["confidentiality"]["gdpr_sensitive"] == None:
+            meta["access_condition"]["confidentiality"]["gdpr_sensitive"] = False
+            changed = True
+        # try:
+        #     if meta["geography"]["geographic_distribution"]["geometry"] == None:
+        #         del meta["geography"]["geographic_distribution"]["geometry"]
+        #         changed = True
+        # except:
+        #     pass
+        # try:
+        #     if meta["metadata_info"]["metadata_dates"]["published"] == None:
+        #         del meta["metadata_info"]["metadata_dates"]["published"]
+        #         changed = True
+        # except:
+        #     pass
+        if changed:
+            node.put_metadata(meta)

{rudi_node_write-1.2.0 → rudi_node_write-1.2.1/src/rudi_node_write.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rudi-node-write
-Version: 1.2.0
+Version: 1.2.1
 Summary: Use the internal API of a RUDI Producer node
 Author-email: Olivier Martineau <olivier.martineau@irisa.fr>
 Maintainer-email: Olivier Martineau <olivier.martineau@irisa.fr>

{rudi_node_write-1.2.0 → rudi_node_write-1.2.1}/src/rudi_node_write.egg-info/SOURCES.txt

@@ -37,6 +37,7 @@ src/rudi_node_write/utils/str_utils.py
 src/rudi_node_write/utils/type_date.py
 src/rudi_node_write/utils/typing_utils.py
 src/rudi_node_write/utils/url_utils.py
+src/rudi_node_write/wip/clean_node.py
 src/rudi_node_write/wip/federation_backup.py
 src/rudi_node_write/wip/rudinode_federation.py
 tests/test_rudi_node_write.py