dcicutils 8.8.6.1b1__py3-none-any.whl → 8.8.6.1b3__py3-none-any.whl

dcicutils/portal_utils.py

@@ -418,31 +418,6 @@ class Portal:
             return []
         return schemas_super_type_map.get(type_name, [])
 
-    def url(self, url: str, raw: bool = False, database: bool = False) -> str:
-        if not isinstance(url, str) or not url:
-            return "/"
-        elif (lowercase_url := url.lower()).startswith("http://") or lowercase_url.startswith("https://"):
-            return url
-        elif not (url := re.sub(r"/+", "/", url)).startswith("/"):
-            url = "/"
-        url = self.server + url if self.server else url
-        if isinstance(raw, bool) and raw:
-            url += ("&" if "?" in url else "?") + "frame=raw"
-        if isinstance(database, bool) and database:
-            url += ("&" if "?" in url else "?") + "datastore=database"
-        return url
-
-    def _kwargs(self, **kwargs) -> dict:
-        if "headers" in kwargs:
-            result_kwargs = {"headers": kwargs["headers"]}
-        else:
-            result_kwargs = {"headers": {"Content-type": Portal.MIME_TYPE_JSON, "Accept": Portal.MIME_TYPE_JSON}}
-        if self.key_pair:
-            result_kwargs["auth"] = self.key_pair
-        if isinstance(timeout := kwargs.get("timeout"), int):
-            result_kwargs["timeout"] = timeout
-        return result_kwargs
-
     @function_cache(maxsize=100, serialize_key=True)
     def get_identifying_paths(self, portal_object: dict, portal_type: Optional[str] = None) -> List[str]:
         """
@@ -491,6 +466,31 @@ class Portal:
                 identifying_properties.insert(0, favored_identifying_property)
         return identifying_properties
 
+    def url(self, url: str, raw: bool = False, database: bool = False) -> str:
+        if not isinstance(url, str) or not url:
+            return "/"
+        elif (lowercase_url := url.lower()).startswith("http://") or lowercase_url.startswith("https://"):
+            return url
+        elif not (url := re.sub(r"/+", "/", url)).startswith("/"):
+            url = "/"
+        url = self.server + url if self.server else url
+        if isinstance(raw, bool) and raw:
+            url += ("&" if "?" in url else "?") + "frame=raw"
+        if isinstance(database, bool) and database:
+            url += ("&" if "?" in url else "?") + "datastore=database"
+        return url
+
+    def _kwargs(self, **kwargs) -> dict:
+        if "headers" in kwargs:
+            result_kwargs = {"headers": kwargs["headers"]}
+        else:
+            result_kwargs = {"headers": {"Content-type": Portal.MIME_TYPE_JSON, "Accept": Portal.MIME_TYPE_JSON}}
+        if self.key_pair:
+            result_kwargs["auth"] = self.key_pair
+        if isinstance(timeout := kwargs.get("timeout"), int):
+            result_kwargs["timeout"] = timeout
+        return result_kwargs
+
     @staticmethod
     def _default_keys_file(app: Optional[str], env: Optional[str], server: Optional[str]) -> Optional[str]:
         def infer_app_from_env(env: str) -> Optional[str]:  # noqa
@@ -566,6 +566,22 @@ class Portal:
             response = TestResponseWrapper(response)
         return response
 
+    @staticmethod
+    def _create_vapp(arg: Union[TestApp, VirtualApp, PyramidRouter, str] = None) -> TestApp:
+        if isinstance(arg, TestApp):
+            return arg
+        elif isinstance(arg, VirtualApp):
+            if not isinstance(arg.wrapped_app, TestApp):
+                raise Exception("Portal._create_vapp VirtualApp argument error.")
+            return arg.wrapped_app
+        if isinstance(arg, PyramidRouter):
+            router = arg
+        elif isinstance(arg, str) or not arg:
+            router = pyramid_get_app(arg or "development.ini", "app")
+        else:
+            raise Exception("Portal._create_vapp argument error.")
+        return TestApp(router, {"HTTP_ACCEPT": Portal.MIME_TYPE_JSON, "REMOTE_USER": "TEST"})
+
     @staticmethod
     def create_for_testing(arg: Optional[Union[str, bool, List[dict], dict, Callable]] = None) -> Portal:
         if isinstance(arg, list) or isinstance(arg, dict) or isinstance(arg, Callable):
@@ -597,22 +613,6 @@ class Portal:
         with temporary_file(content=minimal_ini_for_testing, suffix=".ini") as ini_file:
             return Portal(ini_file)
 
-    @staticmethod
-    def _create_vapp(arg: Union[TestApp, VirtualApp, PyramidRouter, str] = None) -> TestApp:
-        if isinstance(arg, TestApp):
-            return arg
-        elif isinstance(arg, VirtualApp):
-            if not isinstance(arg.wrapped_app, TestApp):
-                raise Exception("Portal._create_vapp VirtualApp argument error.")
-            return arg.wrapped_app
-        if isinstance(arg, PyramidRouter):
-            router = arg
-        elif isinstance(arg, str) or not arg:
-            router = pyramid_get_app(arg or "development.ini", "app")
-        else:
-            raise Exception("Portal._create_vapp argument error.")
-        return TestApp(router, {"HTTP_ACCEPT": Portal.MIME_TYPE_JSON, "REMOTE_USER": "TEST"})
-
     @staticmethod
     def _create_router_for_testing(endpoints: Optional[List[Dict[str, Union[str, Callable]]]] = None) -> PyramidRouter:
         if isinstance(endpoints, dict):

dcicutils/structured_data.py

@@ -351,18 +351,23 @@ class StructuredDataSet:
 
     def _load_json_file(self, file: str) -> None:
         with open(file) as f:
-            file_json = json.load(f)
-            schema_inferred_from_file_name = Schema.type_name(file)
-            if self._portal.get_schema(schema_inferred_from_file_name) is not None:
+            item = json.load(f)
+            if ((schema_name_inferred_from_file_name := Schema.type_name(file)) and
+                (self._portal.get_schema(schema_name_inferred_from_file_name) is not None)):  # noqa
                 # If the JSON file name looks like a schema name then assume it
                 # contains an object or an array of object of that schema type.
-                self._add(Schema.type_name(file), file_json)
-            elif isinstance(file_json, dict):
+                if self._merge:
+                    item = self._merge_with_existing_portal_object(item, schema_name_inferred_from_file_name)
+                self._add(Schema.type_name(file), item)
+            elif isinstance(item, dict):
                 # Otherwise if the JSON file name does not look like a schema name then
                 # assume it a dictionary where each property is the name of a schema, and
                 # which (each property) contains a list of object of that schema type.
-                for schema_name in file_json:
-                    self._add(schema_name, file_json[schema_name])
+                for schema_name in item:
+                    item = item[schema_name]
+                    if self._merge:
+                        item = self._merge_with_existing_portal_object(item, schema_name)
+                    self._add(schema_name, item)
 
     def _load_reader(self, reader: RowReader, type_name: str) -> None:
         schema = None
@@ -386,14 +391,12 @@ class StructuredDataSet:
                     self._add_properties(structured_row, self._autoadd_properties, schema)
             # New merge functionality (2024-05-25).
             if self._merge:
-                for identifying_path in self.get_identifying_paths(self._portal, structured_row, type_name):
-                    if existing_portal_object := self._portal.get_metadata(identifying_path):
-                        structured_row = merge_objects(existing_portal_object, structured_row)
+                structured_row = self._merge_with_existing_portal_object(structured_row, schema_name)
             if (prune_error := self._prune_structured_row(structured_row)) is not None:
                 self._note_error({"src": create_dict(type=schema_name, row=reader.row_number),
                                   "error": prune_error}, "validation")
             else:
-                self._add(type_name, structured_row)
+                self._add(type_name, structured_row)  # TODO: why type_name and not schema_name?
             if self._progress:
                 self._progress({
                     PROGRESS.LOAD_ITEM: self._nrows,
@@ -434,6 +437,18 @@ class StructuredDataSet:
             if name not in structured_row and (not schema or schema.data.get("properties", {}).get(name)):
                 structured_row[name] = properties[name]
 
+    def _merge_with_existing_portal_object(self, portal_object: dict, portal_type: str) -> dict:
+        """
+        Given a Portal object (presumably/in-practice from the given metadata), if there is
+        an existing Portal item, identified by the identifying properties for the given object,
+        then merges the given object into the existing one and returns the result; otherwise
+        just returns the given object. Note that the given object may be CHANGED in place.
+        """
+        for identifying_path in self._portal.get_identifying_paths(portal_object, portal_type):
+            if existing_portal_object := self._portal.get_metadata(identifying_path, raw=True, raise_exception=False):
+                return merge_objects(existing_portal_object, portal_object)
+        return portal_object
+
     def _is_ref_lookup_specified_type(ref_lookup_flags: int) -> bool:
         return (ref_lookup_flags &
                 Portal.LOOKUP_SPECIFIED_TYPE) == Portal.LOOKUP_SPECIFIED_TYPE

{dcicutils-8.8.6.1b1.dist-info → dcicutils-8.8.6.1b3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.8.6.1b1
+Version: 8.8.6.1b3
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.8.6.1b1.dist-info → dcicutils-8.8.6.1b3.dist-info}/RECORD

@@ -48,7 +48,7 @@ dcicutils/misc_utils.py,sha256=zHwsxxEn24muLBP7mDvMa8I9VdMejwW8HMuCL5xbhhw,10769
 dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
 dcicutils/portal_object_utils.py,sha256=gDXRgPsRvqCFwbC8WatsuflAxNiigOnqr0Hi93k3AgE,15422
-dcicutils/portal_utils.py,sha256=_uLdB-ulmFqmClA_Dkpxe4gfEfzSMJSu4yDmtpmqXwQ,33402
+dcicutils/portal_utils.py,sha256=54e0utkLQxQv2_bD37P1ZGeyG63b2W7nCte6KT9eCY0,33402
 dcicutils/progress_bar.py,sha256=UT7lxb-rVF_gp4yjY2Tg4eun1naaH__hB4_v3O85bcE,19468
 dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
 dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
@@ -64,7 +64,7 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
 dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
-dcicutils/structured_data.py,sha256=z0QMgbFafJWoOUMwJJDVufgeT7r7fTTasx1ry5axMVM,63546
+dcicutils/structured_data.py,sha256=sm8x08ckPZcIcyBaSlQRGrOD3YL9d09gz-xB3_TAWGE,64516
 dcicutils/submitr/progress_constants.py,sha256=5bxyX77ql8qEJearfHEvsvXl7D0GuUODW0T65mbRmnE,2895
 dcicutils/submitr/ref_lookup_strategy.py,sha256=Js2cVznTmgjciLWBPLCvMiwLIHXjDn3jww-gJPjYuFw,3467
 dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
@@ -73,8 +73,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=_Y9EmL3D2dUZhxucxHvrtmmlbZmK4FpSsHEb7rGSJLU,3265
-dcicutils-8.8.6.1b1.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.8.6.1b1.dist-info/METADATA,sha256=5iidFphml1M_LwEmfPJdZnLcaSqT2FflV9Ld2JmeCWk,3439
-dcicutils-8.8.6.1b1.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.8.6.1b1.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
-dcicutils-8.8.6.1b1.dist-info/RECORD,,
+dcicutils-8.8.6.1b3.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.8.6.1b3.dist-info/METADATA,sha256=yvHwO7ib7srO20JLRYmG_InTcRd7LnrOZu1A6AYdLU0,3439
+dcicutils-8.8.6.1b3.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.8.6.1b3.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
+dcicutils-8.8.6.1b3.dist-info/RECORD,,