dcicutils 8.8.6__py3-none-any.whl → 8.8.6.1b1__py3-none-any.whl

dcicutils/portal_utils.py

@@ -1,5 +1,6 @@
 from collections import deque
 from functools import lru_cache
+from dcicutils.function_cache_decorator import function_cache
 import io
 import json
 from pyramid.config import Configurator as PyramidConfigurator
@@ -18,6 +19,7 @@ from wsgiref.simple_server import make_server as wsgi_make_server
 from dcicutils.common import APP_SMAHT, OrchestratedApp, ORCHESTRATED_APPS
 from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
 from dcicutils.misc_utils import to_camel_case, VirtualApp
+from dcicutils.schema_utils import get_identifying_properties
 from dcicutils.tmpfile_utils import temporary_file
 
 Portal = Type["Portal"]  # Forward type reference for type hints.
@@ -441,6 +443,54 @@ class Portal:
             result_kwargs["timeout"] = timeout
         return result_kwargs
 
+    @function_cache(maxsize=100, serialize_key=True)
+    def get_identifying_paths(self, portal_object: dict, portal_type: Optional[str] = None) -> List[str]:
+        """
+        Returns the list of the identifying Portal (URL) paths for the given Portal object. Favors any
+        uuid based path and defavors aliases based paths (ala self.get_identifying_property_names);
+        no other ordering defined. Returns empty list of none or otherwise not found.
+        """
+        results = []
+        if not isinstance(portal_object, dict):
+            return results
+        if not isinstance(portal_type, str) or not portal_type:
+            if not (portal_type := self.get_schema_type(portal_object)):
+                return results
+        for identifying_property in self.get_identifying_property_names(portal_type):
+            if identifying_value := portal_object.get(identifying_property):
+                if isinstance(identifying_value, list):
+                    for identifying_value_item in identifying_value:
+                        results.append(f"/{portal_type}/{identifying_value_item}")
+                elif identifying_property == "uuid":
+                    results.append(f"/{identifying_value}")
+                else:
+                    results.append(f"/{portal_type}/{identifying_value}")
+        return results
+
+    @function_cache(maxsize=100, serialize_key=True)
+    def get_identifying_property_names(self, schema: Union[str, dict]) -> List[str]:
+        """
+        Returns the list of identifying property names for the given Portal schema, which may
+        be either a schema name or a schema object; empty list of none or otherwise not found.
+        """
+        results = []
+        if isinstance(schema, str):
+            try:
+                if not (schema := self.get_schema(schema)):
+                    return results
+            except Exception:
+                return results
+        elif not isinstance(schema, dict):
+            return results
+        if not (identifying_properties := get_identifying_properties(schema)):
+            return results
+        identifying_properties = [*identifying_properties]
+        for favored_identifying_property in reversed(["uuid", "identifier"]):
+            if favored_identifying_property in identifying_properties:
+                identifying_properties.remove(favored_identifying_property)
+                identifying_properties.insert(0, favored_identifying_property)
+        return identifying_properties
+
     @staticmethod
     def _default_keys_file(app: Optional[str], env: Optional[str], server: Optional[str]) -> Optional[str]:
         def infer_app_from_env(env: str) -> Optional[str]:  # noqa

dcicutils/structured_data.py

@@ -56,7 +56,7 @@ class StructuredDataSet:
                  remove_empty_objects_from_lists: bool = True,
                  ref_lookup_strategy: Optional[Callable] = None,
                  ref_lookup_nocache: bool = False,
-                 norefs: bool = False,
+                 norefs: bool = False, merge: bool = False,
                  progress: Optional[Callable] = None,
                  debug_sleep: Optional[str] = None) -> None:
         self._progress = progress if callable(progress) else None
@@ -75,6 +75,7 @@ class StructuredDataSet:
         self._nrows = 0
         self._autoadd_properties = autoadd if isinstance(autoadd, dict) and autoadd else None
         self._norefs = True if norefs is True else False
+        self._merge = True if merge is True else False
         self._debug_sleep = None
         if debug_sleep:
             try:
@@ -98,13 +99,13 @@ class StructuredDataSet:
              remove_empty_objects_from_lists: bool = True,
              ref_lookup_strategy: Optional[Callable] = None,
              ref_lookup_nocache: bool = False,
-             norefs: bool = False,
+             norefs: bool = False, merge: bool = False,
              progress: Optional[Callable] = None,
              debug_sleep: Optional[str] = None) -> StructuredDataSet:
         return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune,
                                  remove_empty_objects_from_lists=remove_empty_objects_from_lists,
                                  ref_lookup_strategy=ref_lookup_strategy, ref_lookup_nocache=ref_lookup_nocache,
-                                 norefs=norefs, progress=progress, debug_sleep=debug_sleep)
+                                 norefs=norefs, merge=merge, progress=progress, debug_sleep=debug_sleep)
 
     def validate(self, force: bool = False) -> None:
         def data_without_deleted_properties(data: dict) -> dict:
@@ -383,6 +384,11 @@ class StructuredDataSet:
                 structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
                 if self._autoadd_properties:
                     self._add_properties(structured_row, self._autoadd_properties, schema)
+            # New merge functionality (2024-05-25).
+            if self._merge:
+                for identifying_path in self.get_identifying_paths(self._portal, structured_row, type_name):
+                    if existing_portal_object := self._portal.get_metadata(identifying_path):
+                        structured_row = merge_objects(existing_portal_object, structured_row)
             if (prune_error := self._prune_structured_row(structured_row)) is not None:
                 self._note_error({"src": create_dict(type=schema_name, row=reader.row_number),
                                   "error": prune_error}, "validation")

{dcicutils-8.8.6.dist-info → dcicutils-8.8.6.1b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.8.6
+Version: 8.8.6.1b1
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.8.6.dist-info → dcicutils-8.8.6.1b1.dist-info}/RECORD

@@ -48,7 +48,7 @@ dcicutils/misc_utils.py,sha256=zHwsxxEn24muLBP7mDvMa8I9VdMejwW8HMuCL5xbhhw,10769
 dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
 dcicutils/portal_object_utils.py,sha256=gDXRgPsRvqCFwbC8WatsuflAxNiigOnqr0Hi93k3AgE,15422
-dcicutils/portal_utils.py,sha256=DYyE5o15GekDgzpJWas9iS7klAYbjJZUPW0G42McArk,30779
+dcicutils/portal_utils.py,sha256=_uLdB-ulmFqmClA_Dkpxe4gfEfzSMJSu4yDmtpmqXwQ,33402
 dcicutils/progress_bar.py,sha256=UT7lxb-rVF_gp4yjY2Tg4eun1naaH__hB4_v3O85bcE,19468
 dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
 dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
@@ -64,7 +64,7 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
 dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
-dcicutils/structured_data.py,sha256=XOMxrmkJohdCAyCJU09uI8ivthTKrtSSYReFbC9VYMs,63058
+dcicutils/structured_data.py,sha256=z0QMgbFafJWoOUMwJJDVufgeT7r7fTTasx1ry5axMVM,63546
 dcicutils/submitr/progress_constants.py,sha256=5bxyX77ql8qEJearfHEvsvXl7D0GuUODW0T65mbRmnE,2895
 dcicutils/submitr/ref_lookup_strategy.py,sha256=Js2cVznTmgjciLWBPLCvMiwLIHXjDn3jww-gJPjYuFw,3467
 dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
@@ -73,8 +73,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=_Y9EmL3D2dUZhxucxHvrtmmlbZmK4FpSsHEb7rGSJLU,3265
-dcicutils-8.8.6.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.8.6.dist-info/METADATA,sha256=7VdJjqhSwF0whauIEE6ky-Dh5EE9we5MdC_1GHnV8sA,3435
-dcicutils-8.8.6.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.8.6.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
-dcicutils-8.8.6.dist-info/RECORD,,
+dcicutils-8.8.6.1b1.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.8.6.1b1.dist-info/METADATA,sha256=5iidFphml1M_LwEmfPJdZnLcaSqT2FflV9Ld2JmeCWk,3439
+dcicutils-8.8.6.1b1.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.8.6.1b1.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
+dcicutils-8.8.6.1b1.dist-info/RECORD,,