dcicutils 8.7.1.1b6__py3-none-any.whl → 8.7.1.1b9__py3-none-any.whl

dcicutils/portal_object_utils.py

@@ -137,23 +137,20 @@ class PortalObject:
             comparing_data = value
         else:
             return {}
-        return PortalObject._compare(this_data, comparing_data, self.type)
+        return PortalObject._compare(this_data, comparing_data)
 
     _ARRAY_KEY_REGULAR_EXPRESSION = re.compile(rf"^({Schema._ARRAY_NAME_SUFFIX_CHAR}\d+)$")
 
     @staticmethod
-    def _compare(a: Any, b: Any, value_type: str, _path: Optional[str] = None) -> dict:
+    def _compare(a: Any, b: Any, _path: Optional[str] = None) -> dict:
         def diff_creating(value: Any) -> object:  # noqa
-            nonlocal value_type
-            return create_readonly_object(value=value, type=value_type,
+            return create_readonly_object(value=value,
                                           creating_value=True, updating_value=None, deleting_value=False)
         def diff_updating(value: Any, updating_value: Any) -> object:  # noqa
-            nonlocal value_type
-            return create_readonly_object(value=value, type=value_type,
+            return create_readonly_object(value=value,
                                           creating_value=False, updating_value=updating_value, deleting_value=False)
         def diff_deleting(value: Any) -> object:  # noqa
-            nonlocal value_type
-            return create_readonly_object(value=value, type=value_type,
+            return create_readonly_object(value=value,
                                           creating_value=False, updating_value=None, deleting_value=True)
         diffs = {}
         if isinstance(a, dict) and isinstance(b, dict):
@@ -163,7 +160,7 @@ class PortalObject:
                     if a[key] != PortalObject._PROPERTY_DELETION_SENTINEL:
                         diffs[path] = diff_creating(a[key])
                 else:
-                    diffs.update(PortalObject._compare(a[key], b[key], type, _path=path))
+                    diffs.update(PortalObject._compare(a[key], b[key], _path=path))
         elif isinstance(a, list) and isinstance(b, list):
             # Ignore order of array elements; not absolutely technically correct but suits our purpose.
             for index in range(len(a)):
@@ -179,7 +176,7 @@ class PortalObject:
                             if index < len(b):
                                 diffs[path] = diff_deleting(b[index])
                 elif len(b) < index:
-                    diffs.update(PortalObject._compare(a[index], b[index], value_type, _path=path))
+                    diffs.update(PortalObject._compare(a[index], b[index], _path=path))
                 else:
                     diffs[path] = diff_creating(a[index])
         elif a != b:

dcicutils/portal_utils.py

@@ -15,7 +15,7 @@ from typing import Callable, Dict, List, Optional, Tuple, Type, Union
 from uuid import uuid4 as uuid
 from webtest.app import TestApp, TestResponse
 from wsgiref.simple_server import make_server as wsgi_make_server
-from dcicutils.common import OrchestratedApp, ORCHESTRATED_APPS
+from dcicutils.common import APP_SMAHT, OrchestratedApp, ORCHESTRATED_APPS
 from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
 from dcicutils.misc_utils import to_camel_case, VirtualApp
 from dcicutils.tmpfile_utils import temporary_file
@@ -42,6 +42,7 @@ class Portal:
        or a dcicutils.misc_utils.VirtualApp, or even a pyramid.router.Router.
     8. From another Portal object (i.e. copy constructor).
     """
+    DEFAULT_APP = APP_SMAHT
     KEYS_FILE_DIRECTORY = "~"
     MIME_TYPE_JSON = "application/json"
 
@@ -60,7 +61,7 @@ class Portal:
             self._vapp = None
             for arg in unspecified:
                 if arg is not None:
-                    raise Exception("Portal init error; extraneous args.")
+                    raise Exception("Portal initialization error; extraneous arguments.")
 
         def init_from_portal(portal: Portal, unspecified: Optional[list] = None) -> None:
             init(unspecified=unspecified)
@@ -93,13 +94,13 @@ class Portal:
                                 raise Exception(f"Portal server inconsistency: {server} vs {key_server}")
                         self._key["server"] = self._server = server
             if not self._key:
-                raise Exception("Portal init error; from key.")
+                raise Exception("Portal initialization error; from key.")
 
         def init_from_key_pair(key_pair: tuple, server: Optional[str], unspecified: Optional[list] = []) -> None:
             if len(key_pair) >= 2:
                 init_from_key({"key": key_pair[0], "secret": key_pair[1]}, server, unspecified=unspecified)
             else:
-                raise Exception("Portal init error; from key-pair.")
+                raise Exception("Portal initialization error; from key-pair.")
 
         def init_from_keys_file(keys_file: str, env: Optional[str], server: Optional[str],
                                 unspecified: Optional[list] = []) -> None:
@@ -107,7 +108,7 @@ class Portal:
                 with io.open(keys_file := os.path.expanduser(keys_file)) as f:
                     keys = json.load(f)
             except Exception:
-                raise Exception(f"Portal init error; cannot open keys-file: {keys_file}")
+                raise Exception(f"Portal initialization error; cannot open keys-file: {keys_file}")
             if isinstance(env, str) and env and isinstance(key := keys.get(env), dict):
                 init_from_key(key, server)
                 self._keys_file = keys_file
@@ -121,7 +122,8 @@ class Portal:
                 self._keys_file = keys_file
                 self._env = env
             else:
-                raise Exception(f"Portal init error; {env or server or None} not found in keys-file: {keys_file}")
+                raise Exception(f"Portal initialization error;"
+                                f" {env or server or None} not found in keys-file: {keys_file}")
 
         def init_from_env_server_app(env: str, server: str, app: Optional[str],
                                      unspecified: Optional[list] = None) -> None:
@@ -146,7 +148,7 @@ class Portal:
                 return prefix + server if server else None
 
         if (valid_app := app) and not (valid_app := Portal._valid_app(app)):
-            raise Exception(f"Portal init error; invalid app: {app}")
+            raise Exception(f"Portal initialization error; invalid app: {app}")
         self._app = valid_app
         if isinstance(arg, Portal):
             init_from_portal(arg, unspecified=[env, server, app])
@@ -164,12 +166,15 @@ class Portal:
             init_from_env_server_app(arg, server, app, unspecified=[env])
         elif (isinstance(env, str) and env) or (isinstance(server, str) and server):
             init_from_env_server_app(env, server, app, unspecified=[arg])
+        elif not arg and (keys_file := self._default_keys_file(app=self._app or Portal.DEFAULT_APP, env=env)):
+            # If no initial arg then look for default app keys file.
+            init_from_keys_file(keys_file, env, server)
         elif raise_exception:
-            raise Exception("Portal init error; insufficient args.")
+            raise Exception("Portal initialization error; insufficient arguments.")
         else:
             init()
         if not self.vapp and not self.key and raise_exception:
-            raise Exception("Portal init error; neither key nor vapp defined.")
+            raise Exception("Portal initialization error; neither key nor vapp defined.")
 
     @property
     def ini_file(self) -> Optional[str]:
@@ -443,7 +448,7 @@ class Portal:
         if isinstance(arg, list) or isinstance(arg, dict) or isinstance(arg, Callable):
             return Portal(Portal._create_router_for_testing(arg))
         elif isinstance(arg, str) and arg.endswith(".ini"):
-            return Portal(Portal._create_vapp(arg))
+            return Portal(arg)
         elif arg == "local" or arg is True:
             minimal_ini_for_testing = "\n".join([
                 "[app:app]\nuse = egg:encoded\nfile_upload_bucket = dummy",
@@ -467,7 +472,7 @@ class Portal:
         else:
             minimal_ini_for_testing = "[app:app]\nuse = egg:encoded\nsqlalchemy.url = postgresql://dummy\n"
         with temporary_file(content=minimal_ini_for_testing, suffix=".ini") as ini_file:
-            return Portal(Portal._create_vapp(ini_file))
+            return Portal(ini_file)
 
     @staticmethod
     def _create_vapp(arg: Union[TestApp, VirtualApp, PyramidRouter, str] = None) -> TestApp:

dcicutils/structured_data.py

@@ -11,8 +11,10 @@ from webtest.app import TestApp
 from dcicutils.common import OrchestratedApp
 from dcicutils.data_readers import CsvReader, Excel, RowReader
 from dcicutils.file_utils import search_for_file
-from dcicutils.misc_utils import (create_dict, load_json_if, merge_objects, remove_empty_properties, right_trim,
+from dcicutils.misc_utils import (create_dict, create_readonly_object, load_json_if,
+                                  merge_objects, remove_empty_properties, right_trim,
                                   split_string, to_boolean, to_enum, to_float, to_integer, VirtualApp)
+from dcicutils.portal_object_utils import PortalObject
 from dcicutils.portal_utils import Portal as PortalBase
 from dcicutils.zip_utils import unpack_gz_file_to_temporary_file, unpack_files
 
@@ -61,6 +63,10 @@ class StructuredDataSet:
     def data(self) -> dict:
         return self._data
 
+    @property
+    def portal(self) -> Optional[Portal]:
+        return self._portal
+
     @staticmethod
     def load(file: str, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
              schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
@@ -140,6 +146,27 @@ class StructuredDataSet:
                 upload_file["path"] = file_path
         return upload_files
 
+    def compare(self) -> dict:
+        diffs = {}
+        if self.data or self.portal:
+            refs = self.resolved_refs_with_uuids
+            for object_type in self.data:
+                if not diffs.get(object_type):
+                    diffs[object_type] = []
+                for portal_object in self.data[object_type]:
+                    portal_object = PortalObject(self.portal, portal_object, object_type)
+                    existing_object, identifying_path = portal_object.lookup(include_identifying_path=True, raw=True)
+                    if existing_object:
+                        object_diffs = portal_object.compare(existing_object, consider_refs=True, resolved_refs=refs)
+                        diffs[object_type].append(create_readonly_object(path=identifying_path,
+                                                                         uuid=existing_object.uuid,
+                                                                         diffs=object_diffs or None))
+                    else:
+                        # If there is no existing object we still create a record for this object
+                        # but with no uuid which will be the indication that it does not exist.
+                        diffs[object_type].append(create_readonly_object(path=identifying_path, uuid=None, diffs=None))
+        return diffs
+
     def _load_file(self, file: str) -> None:
         # Returns a dictionary where each property is the name (i.e. the type) of the data,
         # and the value is array of dictionaries for the data itself. Handle these kinds of files:

{dcicutils-8.7.1.1b6.dist-info → dcicutils-8.7.1.1b9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.7.1.1b6
+Version: 8.7.1.1b9
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.7.1.1b6.dist-info → dcicutils-8.7.1.1b9.dist-info}/RECORD

@@ -44,8 +44,8 @@ dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
 dcicutils/misc_utils.py,sha256=9JqdVjHLkZUDTryngF3Dbu0m7XcbitbR7izWnxUSWc4,101953
 dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
-dcicutils/portal_object_utils.py,sha256=Aa61u0o9OyWR7M26dGceMW9MSgDEZLqrwAkwu8FD228,12445
-dcicutils/portal_utils.py,sha256=8XVRCy882RrB8QT2gGvW36c4nT1RJCgUApt_wLP-EG8,26706
+dcicutils/portal_object_utils.py,sha256=KsOvrC2LM3OFX8GoHkqdO88td79v_Xvj1gVptZ5IiK4,12252
+dcicutils/portal_utils.py,sha256=VLTofvk9z5wAQg4PZUi9GmvMCcAeYAX1R3qU1H3CNjw,27065
 dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
 dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
 dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
@@ -59,15 +59,15 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
 dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
-dcicutils/structured_data.py,sha256=voFT8RqVXLX6-sZiEfhsgmwXEiMcJUrHneYUDXNm9wk,35647
+dcicutils/structured_data.py,sha256=M5y5z5rnD-rXN2Ynf3hoJ8kagABUmVX98BisXdHg1C8,37201
 dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
 dcicutils/tmpfile_utils.py,sha256=n95XF8dZVbQRSXBZTGToXXfSs3JUVRyN6c3ZZ0nhAWI,1403
 dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
-dcicutils-8.7.1.1b6.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.7.1.1b6.dist-info/METADATA,sha256=YLm-JVegaODLOnyLLQgUzFTwUPRlK-9dn0eZP1ANmHY,3314
-dcicutils-8.7.1.1b6.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.7.1.1b6.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
-dcicutils-8.7.1.1b6.dist-info/RECORD,,
+dcicutils-8.7.1.1b9.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.7.1.1b9.dist-info/METADATA,sha256=6NZZJZyyO_0mZfDU0xU948JwXitVSbKiup22bPx6be8,3314
+dcicutils-8.7.1.1b9.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.7.1.1b9.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
+dcicutils-8.7.1.1b9.dist-info/RECORD,,