dcicutils 8.8.6__tar.gz → 8.8.6.1b2__tar.gz

{dcicutils-8.8.6 → dcicutils-8.8.6.1b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.8.6
+Version: 8.8.6.1b2
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.8.6 → dcicutils-8.8.6.1b2}/dcicutils/portal_utils.py

@@ -1,5 +1,6 @@
 from collections import deque
 from functools import lru_cache
+from dcicutils.function_cache_decorator import function_cache
 import io
 import json
 from pyramid.config import Configurator as PyramidConfigurator
@@ -18,6 +19,7 @@ from wsgiref.simple_server import make_server as wsgi_make_server
 from dcicutils.common import APP_SMAHT, OrchestratedApp, ORCHESTRATED_APPS
 from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
 from dcicutils.misc_utils import to_camel_case, VirtualApp
+from dcicutils.schema_utils import get_identifying_properties
 from dcicutils.tmpfile_utils import temporary_file
 
 Portal = Type["Portal"]  # Forward type reference for type hints.
@@ -416,6 +418,54 @@ class Portal:
             return []
         return schemas_super_type_map.get(type_name, [])
 
+    @function_cache(maxsize=100, serialize_key=True)
+    def get_identifying_paths(self, portal_object: dict, portal_type: Optional[str] = None) -> List[str]:
+        """
+        Returns the list of the identifying Portal (URL) paths for the given Portal object. Favors any
+        uuid based path and defavors aliases based paths (ala self.get_identifying_property_names);
+        no other ordering defined. Returns empty list of none or otherwise not found.
+        """
+        results = []
+        if not isinstance(portal_object, dict):
+            return results
+        if not isinstance(portal_type, str) or not portal_type:
+            if not (portal_type := self.get_schema_type(portal_object)):
+                return results
+        for identifying_property in self.get_identifying_property_names(portal_type):
+            if identifying_value := portal_object.get(identifying_property):
+                if isinstance(identifying_value, list):
+                    for identifying_value_item in identifying_value:
+                        results.append(f"/{portal_type}/{identifying_value_item}")
+                elif identifying_property == "uuid":
+                    results.append(f"/{identifying_value}")
+                else:
+                    results.append(f"/{portal_type}/{identifying_value}")
+        return results
+
+    @function_cache(maxsize=100, serialize_key=True)
+    def get_identifying_property_names(self, schema: Union[str, dict]) -> List[str]:
+        """
+        Returns the list of identifying property names for the given Portal schema, which may
+        be either a schema name or a schema object; empty list of none or otherwise not found.
+        """
+        results = []
+        if isinstance(schema, str):
+            try:
+                if not (schema := self.get_schema(schema)):
+                    return results
+            except Exception:
+                return results
+        elif not isinstance(schema, dict):
+            return results
+        if not (identifying_properties := get_identifying_properties(schema)):
+            return results
+        identifying_properties = [*identifying_properties]
+        for favored_identifying_property in reversed(["uuid", "identifier"]):
+            if favored_identifying_property in identifying_properties:
+                identifying_properties.remove(favored_identifying_property)
+                identifying_properties.insert(0, favored_identifying_property)
+        return identifying_properties
+
     def url(self, url: str, raw: bool = False, database: bool = False) -> str:
         if not isinstance(url, str) or not url:
             return "/"
@@ -516,6 +566,22 @@ class Portal:
             response = TestResponseWrapper(response)
         return response
 
+    @staticmethod
+    def _create_vapp(arg: Union[TestApp, VirtualApp, PyramidRouter, str] = None) -> TestApp:
+        if isinstance(arg, TestApp):
+            return arg
+        elif isinstance(arg, VirtualApp):
+            if not isinstance(arg.wrapped_app, TestApp):
+                raise Exception("Portal._create_vapp VirtualApp argument error.")
+            return arg.wrapped_app
+        if isinstance(arg, PyramidRouter):
+            router = arg
+        elif isinstance(arg, str) or not arg:
+            router = pyramid_get_app(arg or "development.ini", "app")
+        else:
+            raise Exception("Portal._create_vapp argument error.")
+        return TestApp(router, {"HTTP_ACCEPT": Portal.MIME_TYPE_JSON, "REMOTE_USER": "TEST"})
+
     @staticmethod
     def create_for_testing(arg: Optional[Union[str, bool, List[dict], dict, Callable]] = None) -> Portal:
         if isinstance(arg, list) or isinstance(arg, dict) or isinstance(arg, Callable):
@@ -547,22 +613,6 @@ class Portal:
         with temporary_file(content=minimal_ini_for_testing, suffix=".ini") as ini_file:
             return Portal(ini_file)
 
-    @staticmethod
-    def _create_vapp(arg: Union[TestApp, VirtualApp, PyramidRouter, str] = None) -> TestApp:
-        if isinstance(arg, TestApp):
-            return arg
-        elif isinstance(arg, VirtualApp):
-            if not isinstance(arg.wrapped_app, TestApp):
-                raise Exception("Portal._create_vapp VirtualApp argument error.")
-            return arg.wrapped_app
-        if isinstance(arg, PyramidRouter):
-            router = arg
-        elif isinstance(arg, str) or not arg:
-            router = pyramid_get_app(arg or "development.ini", "app")
-        else:
-            raise Exception("Portal._create_vapp argument error.")
-        return TestApp(router, {"HTTP_ACCEPT": Portal.MIME_TYPE_JSON, "REMOTE_USER": "TEST"})
-
     @staticmethod
     def _create_router_for_testing(endpoints: Optional[List[Dict[str, Union[str, Callable]]]] = None) -> PyramidRouter:
         if isinstance(endpoints, dict):

{dcicutils-8.8.6 → dcicutils-8.8.6.1b2}/dcicutils/structured_data.py

@@ -56,7 +56,7 @@ class StructuredDataSet:
                  remove_empty_objects_from_lists: bool = True,
                  ref_lookup_strategy: Optional[Callable] = None,
                  ref_lookup_nocache: bool = False,
-                 norefs: bool = False,
+                 norefs: bool = False, merge: bool = False,
                  progress: Optional[Callable] = None,
                  debug_sleep: Optional[str] = None) -> None:
         self._progress = progress if callable(progress) else None
@@ -75,6 +75,7 @@ class StructuredDataSet:
         self._nrows = 0
         self._autoadd_properties = autoadd if isinstance(autoadd, dict) and autoadd else None
         self._norefs = True if norefs is True else False
+        self._merge = True if merge is True else False
         self._debug_sleep = None
         if debug_sleep:
             try:
@@ -98,13 +99,13 @@ class StructuredDataSet:
              remove_empty_objects_from_lists: bool = True,
              ref_lookup_strategy: Optional[Callable] = None,
              ref_lookup_nocache: bool = False,
-             norefs: bool = False,
+             norefs: bool = False, merge: bool = False,
              progress: Optional[Callable] = None,
              debug_sleep: Optional[str] = None) -> StructuredDataSet:
         return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune,
                                  remove_empty_objects_from_lists=remove_empty_objects_from_lists,
                                  ref_lookup_strategy=ref_lookup_strategy, ref_lookup_nocache=ref_lookup_nocache,
-                                 norefs=norefs, progress=progress, debug_sleep=debug_sleep)
+                                 norefs=norefs, merge=merge, progress=progress, debug_sleep=debug_sleep)
 
     def validate(self, force: bool = False) -> None:
         def data_without_deleted_properties(data: dict) -> dict:
@@ -350,18 +351,23 @@ class StructuredDataSet:
 
     def _load_json_file(self, file: str) -> None:
         with open(file) as f:
-            file_json = json.load(f)
-            schema_inferred_from_file_name = Schema.type_name(file)
-            if self._portal.get_schema(schema_inferred_from_file_name) is not None:
+            item = json.load(f)
+            if ((schema_name_inferred_from_file_name := Schema.type_name(file)) and
+                (self._portal.get_schema(schema_name_inferred_from_file_name) is not None)):  # noqa
                 # If the JSON file name looks like a schema name then assume it
                 # contains an object or an array of object of that schema type.
-                self._add(Schema.type_name(file), file_json)
-            elif isinstance(file_json, dict):
+                if self._merge:
+                    item = self._merge_with_existing_portal_object(item, schema_name_inferred_from_file_name)
+                self._add(Schema.type_name(file), item)
+            elif isinstance(item, dict):
                 # Otherwise if the JSON file name does not look like a schema name then
                 # assume it a dictionary where each property is the name of a schema, and
                 # which (each property) contains a list of object of that schema type.
-                for schema_name in file_json:
-                    self._add(schema_name, file_json[schema_name])
+                for schema_name in item:
+                    item = item[schema_name]
+                    if self._merge:
+                        item = self._merge_with_existing_portal_object(item, schema_name)
+                    self._add(schema_name, item)
 
     def _load_reader(self, reader: RowReader, type_name: str) -> None:
         schema = None
@@ -383,11 +389,14 @@ class StructuredDataSet:
                 structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
                 if self._autoadd_properties:
                     self._add_properties(structured_row, self._autoadd_properties, schema)
+            # New merge functionality (2024-05-25).
+            if self._merge:
+                structured_row = self._merge_with_existing_portal_object(structured_row, schema_name)
             if (prune_error := self._prune_structured_row(structured_row)) is not None:
                 self._note_error({"src": create_dict(type=schema_name, row=reader.row_number),
                                   "error": prune_error}, "validation")
             else:
-                self._add(type_name, structured_row)
+                self._add(type_name, structured_row)  # TODO: why type_name and not schema_name?
             if self._progress:
                 self._progress({
                     PROGRESS.LOAD_ITEM: self._nrows,
@@ -428,6 +437,18 @@ class StructuredDataSet:
             if name not in structured_row and (not schema or schema.data.get("properties", {}).get(name)):
                 structured_row[name] = properties[name]
 
+    def _merge_with_existing_portal_object(self, portal_object: dict, portal_type: str) -> dict:
+        """
+        Given a Portal object (presumably/in-practice from the given metadata), if there is
+        an existing Portal item, identified by the identifying properties for the given object,
+        then merges the given object into the existing one and returns the result; otherwise
+        just returns the given object. Note that the given object may be CHANGED in place.
+        """
+        for identifying_path in self._portal.get_identifying_paths(portal_object, portal_type):
+            if existing_portal_object := self._portal.get_metadata(identifying_path, raw=True):
+                return merge_objects(existing_portal_object, portal_object)
+        return portal_object
+
     def _is_ref_lookup_specified_type(ref_lookup_flags: int) -> bool:
         return (ref_lookup_flags &
                 Portal.LOOKUP_SPECIFIED_TYPE) == Portal.LOOKUP_SPECIFIED_TYPE

{dcicutils-8.8.6 → dcicutils-8.8.6.1b2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.8.6"
+version = "8.8.6.1b2"
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"