dcicutils 8.5.0.1b3__tar.gz → 8.5.0.1b6__tar.gz

{dcicutils-8.5.0.1b3 → dcicutils-8.5.0.1b6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.5.0.1b3
+Version: 8.5.0.1b6
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

dcicutils-8.5.0.1b6/dcicutils/portal_utils.py

@@ -0,0 +1,375 @@
+from collections import deque
+import io
+import json
+from pyramid.paster import get_app
+from pyramid.router import Router
+import os
+import re
+import requests
+from requests.models import Response as RequestResponse
+from typing import Optional, Type, Union
+from webtest.app import TestApp, TestResponse
+from dcicutils.common import OrchestratedApp, ORCHESTRATED_APPS
+from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
+from dcicutils.misc_utils import to_camel_case, VirtualApp
+from dcicutils.zip_utils import temporary_file
+
+Portal = Type["Portal"]  # Forward type reference for type hints.
+
+
+class Portal:
+    """
+    This is meant to be an uber wrapper for Portal access. It can be created in a variety of ways:
+    1. From a (Portal) .ini file (e.g. development.ini)
+    2. From a key dictionary, containing "key" and "secret" property values.
+    3. From a key tuple, containing (in order) a key and secret values.
+    4. From a keys file assumed to reside in ~/.{app}-keys.json where the given "app" value is either "smaht", "cgap",
+       or "fourfront"; where is assumed to contain a dictionary with a key for the given "env" value, e.g. smaht-local;
+       and with a dictionary value containing "key" and "secret" property values, and an optional "server" property;
+       if an "app" value is not specified but the given "env" value begins with one of the app values then that value
+       will be used, i.e. e.g. if "env" is "smaht-local" and app is unspecified than it is assumed to be "smaht".
+    5. From a keys file as described above (#4) but rather than be identified by the given "env" value it
+       is looked up via the given "server" name and the "server" key dictionary value in the key file.
+    6. From a given "vapp" value (which is assumed to be a TestApp or VirtualApp).
+    7. From another Portal object; or from a a pyramid Router object.
+    """
+    FILE_SCHEMA_NAME = "File"
+    KEYS_FILE_DIRECTORY = os.path.expanduser(f"~")
+
+    def __init__(self,
+                 arg: Optional[Union[Portal, TestApp, VirtualApp, Router, dict, tuple, str]] = None,
+                 env: Optional[str] = None, server: Optional[str] = None,
+                 app: Optional[OrchestratedApp] = None) -> None:
+
+        def init(unspecified: Optional[list] = []) -> None:
+            self._ini_file = None
+            self._key = None
+            self._key_pair = None
+            self._key_id = None
+            self._secret = None
+            self._keys_file = None
+            self._env = None
+            self._server = None
+            self._app = None
+            self._vapp = None
+            for arg in unspecified:
+                if arg is not None:
+                    raise Exception("Portal init error; extraneous args.")
+
+        def init_from_portal(portal: Portal, unspecified: Optional[list] = None) -> None:
+            init(unspecified)
+            self._ini_file = portal._ini_file
+            self._key = portal._key
+            self._key_pair = portal._key_pair
+            self._key_id = portal._key_id
+            self._secret = portal._secret
+            self._keys_file = portal._keys_file
+            self._env = portal._env
+            self._server = portal._server
+            self._app = portal._app
+            self._vapp = portal._vapp
+
+        def init_from_vapp(vapp: Union[TestApp, VirtualApp, Router], unspecified: Optional[list] = []) -> None:
+            init(unspecified)
+            self._vapp = Portal._create_testapp(vapp)
+
+        def init_from_ini_file(ini_file: str, unspecified: Optional[list] = []) -> None:
+            init(unspecified)
+            self._ini_file = ini_file
+            self._vapp = Portal._create_testapp(ini_file)
+
+        def init_from_key(key: dict, server: Optional[str], unspecified: Optional[list] = []) -> None:
+            init(unspecified)
+            if (isinstance(key_id := key.get("key"), str) and key_id and
+                isinstance(secret := key.get("secret"), str) and secret):  # noqa
+                self._key = {"key": key_id, "secret": secret}
+                self._key_id = key_id
+                self._secret = secret
+                self._key_pair = (key_id, secret)
+                if ((isinstance(server, str) and server) or (isinstance(server := key.get("server"), str) and server)):
+                    if server := normalize_server(server):
+                        self._key["server"] = self._server = server
+            if not self._key:
+                raise Exception("Portal init error; from key.")
+
+        def init_from_key_pair(key_pair: tuple, server: Optional[str], unspecified: Optional[list] = []) -> None:
+            if len(key_pair) == 2:
+                init_from_key({"key": key_pair[0], "secret": key_pair[1]}, server, unspecified)
+            else:
+                raise Exception("Portal init error; from key-pair.")
+
+        def init_from_keys_file(keys_file: str, env: Optional[str], server: Optional[str],
+                                unspecified: Optional[list] = []) -> None:
+            try:
+                with io.open(keys_file) as f:
+                    keys = json.load(f)
+            except Exception:
+                raise Exception(f"Portal init error; cannot open keys-file: {keys_file}")
+            if isinstance(env, str) and env and isinstance(key := keys.get(env), dict):
+                init_from_key(key, server)
+                self._keys_file = keys_file
+                self._env = env
+            elif isinstance(server, str) and server and (key := [k for k in keys if keys[k].get("server") == server]):
+                init_from_key(key, server)
+                self._keys_file = keys_file
+            elif len(keys) == 1 and (env := next(iter(keys))) and isinstance(key := keys[env], dict) and key:
+                init_from_key(key, server)
+                self._keys_file = keys_file
+                self._env = env
+            else:
+                raise Exception(f"Portal init error; {env or server or None} not found in keys-file: {keys_file}")
+
+        def init_from_env_server_app(env: str, server: str, app: Optional[str],
+                                     unspecified: Optional[list] = None) -> None:
+            return init_from_keys_file(self._default_keys_file(app, env), env, server, unspecified)
+
+        def normalize_server(server: str) -> Optional[str]:
+            prefix = ""
+            if (lserver := server.lower()).startswith("http://"):
+                prefix = "http://"
+            elif lserver.startswith("https://"):
+                prefix = "https://"
+            if prefix:
+                if (server := re.sub(r"/+", "/", server[len(prefix):])).startswith("/"):
+                    server = server[1:]
+                if len(server) > 1 and server.endswith("/"):
+                    server = server[:-1]
+                return prefix + server if server else None
+
+        if isinstance(arg, Portal):
+            init_from_portal(arg, unspecified=[env, server, app])
+        elif isinstance(arg, (TestApp, VirtualApp, Router)):
+            init_from_vapp(arg, unspecified=[env, server, app])
+        elif isinstance(arg, str) and arg.endswith(".ini"):
+            init_from_ini_file(arg, unspecified=[env, server, app])
+        elif isinstance(arg, dict):
+            init_from_key(arg, server, unspecified=[env, app])
+        elif isinstance(arg, tuple):
+            init_from_key_pair(arg, server, unspecified=[env, app])
+        elif isinstance(arg, str) and arg.endswith(".json"):
+            init_from_keys_file(arg, env, server, unspecified=[app])
+        elif isinstance(arg, str) and arg:
+            init_from_env_server_app(arg, server, app, unspecified=[env])
+        elif isinstance(env, str) and env:
+            init_from_env_server_app(env, server, app, unspecified=[arg])
+        else:
+            raise Exception("Portal init error; invalid args.")
+
+    @property
+    def ini_file(self) -> Optional[str]:
+        return self._ini_file
+
+    @property
+    def key(self) -> Optional[dict]:
+        return self._key
+
+    @property
+    def key_pair(self) -> Optional[tuple]:
+        return self._key_pair
+
+    @property
+    def key_id(self) -> Optional[str]:
+        return self._key_id
+
+    @property
+    def secret(self) -> Optional[str]:
+        return self._secret
+
+    @property
+    def keys_file(self) -> Optional[str]:
+        return self._keys_file
+
+    @property
+    def env(self) -> Optional[str]:
+        return self._env
+
+    @property
+    def server(self) -> Optional[str]:
+        return self._server
+
+    @property
+    def app(self) -> Optional[str]:
+        return self._app
+
+    @property
+    def vapp(self) -> Optional[TestApp]:
+        return self._vapp
+
+    def get_metadata(self, object_id: str) -> Optional[dict]:
+        return get_metadata(obj_id=object_id, vapp=self._vapp, key=self._key)
+
+    def patch_metadata(self, object_id: str, data: str) -> Optional[dict]:
+        if self._key:
+            return patch_metadata(obj_id=object_id, patch_item=data, key=self._key)
+        return self.patch(f"/{object_id}", data)
+
+    def post_metadata(self, object_type: str, data: str) -> Optional[dict]:
+        if self._key:
+            return post_metadata(schema_name=object_type, post_item=data, key=self._key)
+        return self.post(f"/{object_type}", data)
+
+    def get(self, uri: str, follow: bool = True, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
+        if self._vapp:
+            response = self._vapp.get(self.url(uri), **self._kwargs(**kwargs))
+            if response and response.status_code in [301, 302, 303, 307, 308] and follow:
+                response = response.follow()
+            return self._response(response)
+        return requests.get(self.url(uri), allow_redirects=follow, **self._kwargs(**kwargs))
+
+    def patch(self, uri: str, data: Optional[dict] = None,
+              json: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
+        if self._vapp:
+            return self._vapp.patch_json(self.url(uri), json or data, **self._kwargs(**kwargs))
+        return requests.patch(self.url(uri), data=data, json=json, **self._kwargs(**kwargs))
+
+    def post(self, uri: str, data: Optional[dict] = None, json: Optional[dict] = None,
+             files: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
+        if self._vapp:
+            if files:
+                return self._vapp.post(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
+            else:
+                return self._vapp.post_json(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
+        return requests.post(self.url(uri), data=data, json=json, files=files, **self._kwargs(**kwargs))
+
+    def get_schema(self, schema_name: str) -> Optional[dict]:
+        return get_schema(self.schema_name(schema_name), portal_vapp=self._vapp, key=self._key)
+
+    def get_schemas(self) -> dict:
+        return self.get("/profiles/").json()
+
+    @staticmethod
+    def schema_name(name: str) -> str:
+        return to_camel_case(name)
+
+    def is_file_schema(self, schema_name: str) -> bool:
+        if super_type_map := self.get_schemas_super_type_map():
+            if file_super_type := super_type_map.get(Portal.FILE_SCHEMA_NAME):
+                return self.schema_name(schema_name) in file_super_type
+        return False
+
+    def get_schemas_super_type_map(self) -> dict:
+        """
+        Returns the "super type map" for all of the known schemas (via /profiles).
+        This is a dictionary of all types which have (one or more) sub-types whose value is
+        an array of all of those sub-types (direct and all descendents), in breadth first order.
+        """
+        def breadth_first(super_type_map: dict, super_type_name: str) -> dict:
+            result = []
+            queue = deque(super_type_map.get(super_type_name, []))
+            while queue:
+                result.append(sub_type_name := queue.popleft())
+                if sub_type_name in super_type_map:
+                    queue.extend(super_type_map[sub_type_name])
+            return result
+        if not (schemas := self.get_schemas()):
+            return {}
+        super_type_map = {}
+        for type_name in schemas:
+            if super_type_name := schemas[type_name].get("rdfs:subClassOf"):
+                super_type_name = super_type_name.replace("/profiles/", "").replace(".json", "")
+                if super_type_name != "Item":
+                    if not super_type_map.get(super_type_name):
+                        super_type_map[super_type_name] = [type_name]
+                    elif type_name not in super_type_map[super_type_name]:
+                        super_type_map[super_type_name].append(type_name)
+        super_type_map_flattened = {}
+        for super_type_name in super_type_map:
+            super_type_map_flattened[super_type_name] = breadth_first(super_type_map, super_type_name)
+        return super_type_map_flattened
+
+    def ping(self) -> bool:
+        try:
+            return self.get("/health").status_code == 200
+        except Exception:
+            return False
+
+    def url(self, uri: str) -> str:
+        if not isinstance(uri, str) or not uri:
+            return "/"
+        if (luri := uri.lower()).startswith("http://") or luri.startswith("https://"):
+            return uri
+        if not (uri := re.sub(r"/+", "/", uri)).startswith("/"):
+            uri = "/"
+        return self._server + uri if self._server else uri
+
+    def _kwargs(self, **kwargs) -> dict:
+        result_kwargs = {"headers":
+                         kwargs.get("headers", {"Content-type": "application/json", "Accept": "application/json"})}
+        if self._key_pair:
+            result_kwargs["auth"] = self._key_pair
+        if isinstance(timeout := kwargs.get("timeout"), int):
+            result_kwargs["timeout"] = timeout
+        return result_kwargs
+
+    def _default_keys_file(self, app: Optional[str], env: Optional[str] = None) -> Optional[str]:
+        def is_valid_app(app: Optional[str]) -> bool:  # noqa
+            return app and app.lower() in [name.lower() for name in ORCHESTRATED_APPS]
+        def infer_app_from_env(env: str) -> Optional[str]:  # noqa
+            if isinstance(env, str) and (lenv := env.lower()):
+                if app := [app for app in ORCHESTRATED_APPS if lenv.startswith(app.lower())]:
+                    return app[0]
+        if is_valid_app(app) or (app := infer_app_from_env(env)):
+            return os.path.join(Portal.KEYS_FILE_DIRECTORY, f".{app.lower()}-keys.json")
+
+    def _response(self, response) -> Optional[RequestResponse]:
+        if response and isinstance(getattr(response.__class__, "json"), property):
+            class RequestResponseWrapper:  # For consistency change json property to method.
+                def __init__(self, response, **kwargs):
+                    super().__init__(**kwargs)
+                    self._response = response
+                def __getattr__(self, attr):  # noqa
+                    return getattr(self._response, attr)
+                def json(self):  # noqa
+                    return self._response.json
+            response = RequestResponseWrapper(response)
+        return response
+
+    @staticmethod
+    def create_for_testing(ini_file: Optional[str] = None) -> Portal:
+        if isinstance(ini_file, str):
+            return Portal(Portal._create_testapp(ini_file))
+        minimal_ini_for_unit_testing = "[app:app]\nuse = egg:encoded\nsqlalchemy.url = postgresql://dummy\n"
+        with temporary_file(content=minimal_ini_for_unit_testing, suffix=".ini") as ini_file:
+            return Portal(Portal._create_testapp(ini_file))
+
+    @staticmethod
+    def create_for_testing_local(ini_file: Optional[str] = None) -> Portal:
+        if isinstance(ini_file, str) and ini_file:
+            return Portal(Portal._create_testapp(ini_file))
+        minimal_ini_for_testing_local = "\n".join([
+            "[app:app]\nuse = egg:encoded\nfile_upload_bucket = dummy",
+            "sqlalchemy.url = postgresql://postgres@localhost:5441/postgres?host=/tmp/snovault/pgdata",
+            "multiauth.groupfinder = encoded.authorization.smaht_groupfinder",
+            "multiauth.policies = auth0 session remoteuser accesskey",
+            "multiauth.policy.session.namespace = mailto",
+            "multiauth.policy.session.use = encoded.authentication.NamespacedAuthenticationPolicy",
+            "multiauth.policy.session.base = pyramid.authentication.SessionAuthenticationPolicy",
+            "multiauth.policy.remoteuser.namespace = remoteuser",
+            "multiauth.policy.remoteuser.use = encoded.authentication.NamespacedAuthenticationPolicy",
+            "multiauth.policy.remoteuser.base = pyramid.authentication.RemoteUserAuthenticationPolicy",
+            "multiauth.policy.accesskey.namespace = accesskey",
+            "multiauth.policy.accesskey.use = encoded.authentication.NamespacedAuthenticationPolicy",
+            "multiauth.policy.accesskey.base = encoded.authentication.BasicAuthAuthenticationPolicy",
+            "multiauth.policy.accesskey.check = encoded.authentication.basic_auth_check",
+            "multiauth.policy.auth0.use = encoded.authentication.NamespacedAuthenticationPolicy",
+            "multiauth.policy.auth0.namespace = auth0",
+            "multiauth.policy.auth0.base = encoded.authentication.Auth0AuthenticationPolicy"
+        ])
+        with temporary_file(content=minimal_ini_for_testing_local, suffix=".ini") as minimal_ini_file:
+            return Portal(Portal._create_testapp(minimal_ini_file))
+
+    @staticmethod
+    def _create_testapp(arg: Union[TestApp, VirtualApp, Router, str] = None, app_name: Optional[str] = None) -> TestApp:
+        if isinstance(arg, TestApp):
+            return arg
+        elif isinstance(arg, VirtualApp):
+            if not isinstance(arg.wrapped_app, TestApp):
+                raise Exception("Portal._create_testapp VirtualApp argument error.")
+            return arg.wrapped_app
+        if isinstance(arg, Router):
+            router = arg
+        elif isinstance(arg, str) or arg is None:
+            router = get_app(arg or "development.ini", app_name or "app")
+        else:
+            raise Exception("Portal._create_testapp argument error.")
+        return TestApp(router, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})

{dcicutils-8.5.0.1b3 → dcicutils-8.5.0.1b6}/dcicutils/structured_data.py

@@ -42,9 +42,9 @@ StructuredDataSet = Type["StructuredDataSet"]
 class StructuredDataSet:
 
     def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
-                 schemas: Optional[List[dict]] = None, data: Optional[List[dict]] = None,
+                 schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
                  order: Optional[List[str]] = None, prune: bool = True) -> None:
-        self.data = {} if not data else data  # If portal is None then no schemas nor refs.
+        self.data = {}
         self._portal = Portal(portal, data=self.data, schemas=schemas) if portal else None
         self._order = order
         self._prune = prune
@@ -52,13 +52,14 @@ class StructuredDataSet:
         self._errors = {}
         self._resolved_refs = set()
         self._validated = False
+        self._autoadd_properties = autoadd if isinstance(autoadd, dict) and autoadd else None
         self._load_file(file) if file else None
 
     @staticmethod
     def load(file: str, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
-             schemas: Optional[List[dict]] = None,
+             schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
              order: Optional[List[str]] = None, prune: bool = True) -> StructuredDataSet:
-        return StructuredDataSet(file=file, portal=portal, schemas=schemas, order=order, prune=prune)
+        return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune)
 
     def validate(self, force: bool = False) -> None:
         if self._validated and not force:
@@ -112,7 +113,7 @@ class StructuredDataSet:
     def _load_file(self, file: str) -> None:
         # Returns a dictionary where each property is the name (i.e. the type) of the data,
         # and the value is array of dictionaries for the data itself. Handle these kinds of files:
-        # 1.  Single CSV of JSON file, where the (base) name of the file is the data type name.
+        # 1.  Single CSV, TSV, or JSON file, where the (base) name of the file is the data type name.
         # 2.  Single Excel file containing one or more sheets, where each sheet
         #     represents (i.e. is named for, and contains data for) a different type.
         # 3.  Zip file (.zip or .tar.gz or .tgz or .tar), containing data files to load, where the
@@ -163,6 +164,8 @@ class StructuredDataSet:
             structured_row = structured_row_template.create_row()
             for column_name, value in row.items():
                 structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
+                if self._autoadd_properties:
+                    self._add_properties(structured_row, self._autoadd_properties, schema)
             self._add(type_name, structured_row)
         self._note_warning(reader.warnings, "reader")
         if schema:
@@ -177,6 +180,11 @@ class StructuredDataSet:
         else:
             self.data[type_name] = [data] if isinstance(data, dict) else data
 
+    def _add_properties(self, structured_row: dict, properties: dict, schema: Optional[dict] = None) -> None:
+        for name in properties:
+            if name not in structured_row and (not schema or schema.data.get("properties", {}).get(name)):
+                structured_row[name] = properties[name]
+
     def _note_warning(self, item: Optional[Union[dict, List[dict]]], group: str) -> None:
         self._note_issue(self._warnings, item, group)
 
@@ -475,7 +483,6 @@ class Schema:
                 if unique:
                     typeinfo[key]["unique"] = True
                 result.update(typeinfo)
-#               result.update(self._create_typeinfo(array_property_items, parent_key=key))
                 continue
             result[key] = {"type": property_value_type, "map": self._map_function({**property_value, "column": key})}
             if ARRAY_NAME_SUFFIX_CHAR in key:
@@ -543,16 +550,13 @@ class Portal(PortalBase):
 
     def __init__(self,
                  arg: Optional[Union[VirtualApp, TestApp, Router, Portal, dict, tuple, str]] = None,
-                 env: Optional[str] = None, app: OrchestratedApp = None, server: Optional[str] = None,
-                 key: Optional[Union[dict, tuple]] = None,
-                 portal: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None,
-                 data: Optional[dict] = None, schemas: Optional[List[dict]] = None) -> Optional[Portal]:
-        super().__init__(arg, env=env, app=app, server=server, key=key, portal=portal)
-        if isinstance(arg, Portal) and not portal:
-            portal = arg
-        if isinstance(portal, Portal):
-            self._schemas = schemas if schemas is not None else portal._schemas  # Explicitly specified/known schemas.
-            self._data = data if data is not None else portal._data  # Data set being loaded; e.g. by StructuredDataSet.
+                 env: Optional[str] = None, server: Optional[str] = None,
+                 app: Optional[OrchestratedApp] = None,
+                 data: Optional[dict] = None, schemas: Optional[List[dict]] = None) -> None:
+        super().__init__(arg, env=env, server=server, app=app)
+        if isinstance(arg, Portal):
+            self._schemas = schemas if schemas is not None else arg._schemas  # Explicitly specified/known schemas.
+            self._data = data if data is not None else arg._data  # Data set being loaded; e.g. by StructuredDataSet.
         else:
             self._schemas = schemas
             self._data = data

{dcicutils-8.5.0.1b3 → dcicutils-8.5.0.1b6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0.1b3"  # TODO: To become 8.6.0
+version = "8.5.0.1b6"  # TODO: To become 8.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

dcicutils-8.5.0.1b3/dcicutils/portal_utils.py

@@ -1,294 +0,0 @@
-from collections import deque
-from pyramid.paster import get_app
-from pyramid.router import Router
-import re
-import requests
-from requests.models import Response as RequestResponse
-from typing import Optional, Type, Union
-from webtest.app import TestApp, TestResponse
-from dcicutils.common import OrchestratedApp, APP_CGAP, APP_FOURFRONT, APP_SMAHT, ORCHESTRATED_APPS
-from dcicutils.creds_utils import CGAPKeyManager, FourfrontKeyManager, SMaHTKeyManager
-from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
-from dcicutils.misc_utils import to_camel_case, VirtualApp
-from dcicutils.zip_utils import temporary_file
-
-Portal = Type["Portal"]  # Forward type reference for type hints.
-FILE_SCHEMA_NAME = "File"
-
-
-class Portal:
-    """
-    This is meant to be an uber wrapper for Portal access. It can be created in a variety of ways:
-    1. From a (Portal) .ini file (e.g. development.ini)
-    2. From a key dictionary, containing "key" and "secret" property values.
-    3. From a key tuple, containing (in order) a key and secret values.
-    4. From a keys file assumed to reside in ~/.{app}-keys.json where the given "app" value is either "smaht", "cgap",
-       or "fourfront"; and where this file is assumed to contain a dictionary with a key equal to the given "env"
-       value (e.g. smaht-localhost) and with a dictionary value containing "key" and "secret" property values; if
-       an "app" value is not specified but the given "env" value begins with one of the app values then that value
-       will be used, i.e. e.g. if env is "smaht-localhost" and app is unspecified than it is assumed to be "smaht".
-    5. From a keys file as described above (#4) but rather than be identified by the given "env" value it
-       is looked up by the given "server" name and the "server" key dictionary value in the key file.
-    6. From a given "vapp" value (which is assumed to be a TestApp or VirtualApp).
-    7. From another Portal object.
-    8. From a a pyramid Router object.
-    """
-    def __init__(self,
-                 arg: Optional[Union[VirtualApp, TestApp, Router, Portal, dict, tuple, str]] = None,
-                 env: Optional[str] = None, app: Optional[OrchestratedApp] = None, server: Optional[str] = None,
-                 key: Optional[Union[dict, tuple]] = None,
-                 vapp: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None,
-                 portal: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None) -> Portal:
-        if vapp and not portal:
-            portal = vapp
-        if ((isinstance(arg, (VirtualApp, TestApp, Router, Portal)) or
-             isinstance(arg, str) and arg.endswith(".ini")) and not portal):
-            portal = arg
-        elif isinstance(arg, str) and not env:
-            env = arg
-        elif (isinstance(arg, dict) or isinstance(arg, tuple)) and not key:
-            key = arg
-        if not app and env:
-            if env.startswith(APP_SMAHT):
-                app = APP_SMAHT
-            elif env.startswith(APP_CGAP):
-                app = APP_CGAP
-            elif env.startswith(APP_FOURFRONT):
-                app = APP_FOURFRONT
-        if isinstance(portal, Portal):
-            self._vapp = portal._vapp
-            self._env = portal._env
-            self._app = portal._app
-            self._server = portal._server
-            self._key = portal._key
-            self._key_pair = portal._key_pair
-            self._key_id = portal._key_id
-            self._key_file = portal._key_file
-            return
-        self._vapp = None
-        self._env = env
-        self._app = app
-        self._server = server
-        self._key = None
-        self._key_pair = None
-        self._key_id = None
-        self._key_file = None
-        if isinstance(portal, (VirtualApp, TestApp)):
-            self._vapp = portal
-        elif isinstance(portal, (Router, str)):
-            self._vapp = Portal._create_vapp(portal)
-        elif isinstance(key, dict):
-            self._key = key
-            self._key_pair = (key.get("key"), key.get("secret")) if key else None
-            if key_server := key.get("server"):
-                self._server = key_server
-        elif isinstance(key, tuple) and len(key) >= 2:
-            self._key = {"key": key[0], "secret": key[1]}
-            self._key_pair = key
-        elif isinstance(env, str):
-            key_managers = {APP_CGAP: CGAPKeyManager, APP_FOURFRONT: FourfrontKeyManager, APP_SMAHT: SMaHTKeyManager}
-            if not (key_manager := key_managers.get(self._app)) or not (key_manager := key_manager()):
-                raise Exception(f"Invalid app name: {self._app} (valid: {', '.join(ORCHESTRATED_APPS)}).")
-            if isinstance(env, str):
-                self._key = key_manager.get_keydict_for_env(env)
-                if key_server := self._key.get("server"):
-                    self._server = key_server
-            elif isinstance(self._server, str):
-                self._key = key_manager.get_keydict_for_server(self._server)
-            self._key_pair = key_manager.keydict_to_keypair(self._key) if self._key else None
-            self._key_file = key_manager.keys_file
-        if self._key and (key_id := self._key.get("key")):
-            self._key_id = key_id
-        elif self._key_pair and (key_id := self._key_pair[1]):
-            self._key_id = key_id
-
-    @property
-    def env(self):
-        return self._env
-
-    @property
-    def app(self):
-        return self._app
-
-    @property
-    def server(self):
-        return self._server
-
-    @property
-    def key(self):
-        return self._key
-
-    @property
-    def key_pair(self):
-        return self._key_pair
-
-    @property
-    def key_id(self):
-        return self._key_id
-
-    @property
-    def key_file(self):
-        return self._key_file
-
-    @property
-    def vapp(self):
-        return self._vapp
-
-    def get_metadata(self, object_id: str) -> Optional[dict]:
-        return get_metadata(obj_id=object_id, vapp=self._vapp, key=self._key)
-
-    def patch_metadata(self, object_id: str, data: str) -> Optional[dict]:
-        if self._key:
-            return patch_metadata(obj_id=object_id, patch_item=data, key=self._key)
-        return self.patch(f"/{object_id}", data)
-
-    def post_metadata(self, object_type: str, data: str) -> Optional[dict]:
-        if self._key:
-            return post_metadata(schema_name=object_type, post_item=data, key=self._key)
-        return self.post(f"/{object_type}", data)
-
-    def get(self, uri: str, follow: bool = True, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
-        if isinstance(self._vapp, (VirtualApp, TestApp)):
-            response = self._vapp.get(self._uri(uri), **self._kwargs(**kwargs))
-            if response and response.status_code in [301, 302, 303, 307, 308] and follow:
-                response = response.follow()
-            return self._response(response)
-        return requests.get(self._uri(uri), allow_redirects=follow, **self._kwargs(**kwargs))
-
-    def patch(self, uri: str, data: Optional[dict] = None,
-              json: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
-        if isinstance(self._vapp, (VirtualApp, TestApp)):
-            return self._vapp.patch_json(self._uri(uri), json or data, **self._kwargs(**kwargs))
-        return requests.patch(self._uri(uri), json=json or data, **self._kwargs(**kwargs))
-
-    def post(self, uri: str, data: Optional[dict] = None, json: Optional[dict] = None,
-             files: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
-        if isinstance(self._vapp, (VirtualApp, TestApp)):
-            if files:
-                return self._vapp.post(self._uri(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
-            else:
-                return self._vapp.post_json(self._uri(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
-        return requests.post(self._uri(uri), json=json or data, files=files, **self._kwargs(**kwargs))
-
-    def get_schema(self, schema_name: str) -> Optional[dict]:
-        return get_schema(self.schema_name(schema_name), portal_vapp=self._vapp, key=self._key)
-
-    def get_schemas(self) -> dict:
-        return self.get("/profiles/").json()
-
-    @staticmethod
-    def schema_name(name: str) -> str:
-        return to_camel_case(name)
-
-    def is_file_schema(self, schema_name: str) -> bool:
-        if super_type_map := self.get_schemas_super_type_map():
-            if file_super_type := super_type_map.get(FILE_SCHEMA_NAME):
-                return self.schema_name(schema_name) in file_super_type
-        return False
-
-    def get_schemas_super_type_map(self) -> dict:
-        """
-        Returns the "super type map" for all of the known schemas (via /profiles).
-        This is a dictionary of all types which have (one or more) sub-types whose value is
-        an array of all of those sub-types (direct and all descendents), in breadth first order.
-        """
-        def breadth_first(super_type_map: dict, super_type_name: str) -> dict:
-            result = []
-            queue = deque(super_type_map.get(super_type_name, []))
-            while queue:
-                result.append(sub_type_name := queue.popleft())
-                if sub_type_name in super_type_map:
-                    queue.extend(super_type_map[sub_type_name])
-            return result
-        if not (schemas := self.get_schemas()):
-            return {}
-        super_type_map = {}
-        for type_name in schemas:
-            if super_type_name := schemas[type_name].get("rdfs:subClassOf"):
-                super_type_name = super_type_name.replace("/profiles/", "").replace(".json", "")
-                if super_type_name != "Item":
-                    if not super_type_map.get(super_type_name):
-                        super_type_map[super_type_name] = [type_name]
-                    elif type_name not in super_type_map[super_type_name]:
-                        super_type_map[super_type_name].append(type_name)
-        super_type_map_flattened = {}
-        for super_type_name in super_type_map:
-            super_type_map_flattened[super_type_name] = breadth_first(super_type_map, super_type_name)
-        return super_type_map_flattened
-
-    def ping(self) -> bool:
-        try:
-            return self.get("/health").status_code == 200
-        except Exception:
-            return False
-
-    def _uri(self, uri: str) -> str:
-        if not isinstance(uri, str) or not uri:
-            return "/"
-        if uri.lower().startswith("http://") or uri.lower().startswith("https://"):
-            return uri
-        uri = re.sub(r"/+", "/", uri)
-        return (self._server + ("/" if not uri.startswith("/") else "") + uri) if self._server else uri
-
-    def _kwargs(self, **kwargs) -> dict:
-        result_kwargs = {"headers":
-                         kwargs.get("headers", {"Content-type": "application/json", "Accept": "application/json"})}
-        if self._key_pair:
-            result_kwargs["auth"] = self._key_pair
-        if isinstance(timeout := kwargs.get("timeout"), int):
-            result_kwargs["timeout"] = timeout
-        return result_kwargs
-
-    def _response(self, response) -> Optional[RequestResponse]:
-        if response and isinstance(getattr(response.__class__, "json"), property):
-            class RequestResponseWrapper:  # For consistency change json property to method.
-                def __init__(self, response, **kwargs):
-                    super().__init__(**kwargs)
-                    self._response = response
-                def __getattr__(self, attr):  # noqa
-                    return getattr(self._response, attr)
-                def json(self):  # noqa
-                    return self._response.json
-            response = RequestResponseWrapper(response)
-        return response
-
-    @staticmethod
-    def create_for_testing(ini_file: Optional[str] = None) -> Portal:
-        if isinstance(ini_file, str):
-            return Portal(Portal._create_vapp(ini_file))
-        minimal_ini_for_unit_testing = "[app:app]\nuse = egg:encoded\nsqlalchemy.url = postgresql://dummy\n"
-        with temporary_file(content=minimal_ini_for_unit_testing, suffix=".ini") as ini_file:
-            return Portal(Portal._create_vapp(ini_file))
-
-    @staticmethod
-    def create_for_testing_local(ini_file: Optional[str] = None) -> Portal:
-        if isinstance(ini_file, str) and ini_file:
-            return Portal(Portal._create_vapp(ini_file))
-        minimal_ini_for_testing_local = "\n".join([
-            "[app:app]\nuse = egg:encoded\nfile_upload_bucket = dummy",
-            "sqlalchemy.url = postgresql://postgres@localhost:5441/postgres?host=/tmp/snovault/pgdata",
-            "multiauth.groupfinder = encoded.authorization.smaht_groupfinder",
-            "multiauth.policies = auth0 session remoteuser accesskey",
-            "multiauth.policy.session.namespace = mailto",
-            "multiauth.policy.session.use = encoded.authentication.NamespacedAuthenticationPolicy",
-            "multiauth.policy.session.base = pyramid.authentication.SessionAuthenticationPolicy",
-            "multiauth.policy.remoteuser.namespace = remoteuser",
-            "multiauth.policy.remoteuser.use = encoded.authentication.NamespacedAuthenticationPolicy",
-            "multiauth.policy.remoteuser.base = pyramid.authentication.RemoteUserAuthenticationPolicy",
-            "multiauth.policy.accesskey.namespace = accesskey",
-            "multiauth.policy.accesskey.use = encoded.authentication.NamespacedAuthenticationPolicy",
-            "multiauth.policy.accesskey.base = encoded.authentication.BasicAuthAuthenticationPolicy",
-            "multiauth.policy.accesskey.check = encoded.authentication.basic_auth_check",
-            "multiauth.policy.auth0.use = encoded.authentication.NamespacedAuthenticationPolicy",
-            "multiauth.policy.auth0.namespace = auth0",
-            "multiauth.policy.auth0.base = encoded.authentication.Auth0AuthenticationPolicy"
-        ])
-        with temporary_file(content=minimal_ini_for_testing_local, suffix=".ini") as minimal_ini_file:
-            return Portal(Portal._create_vapp(minimal_ini_file))
-
-    @staticmethod
-    def _create_vapp(value: Union[str, Router, TestApp] = "development.ini", app_name: str = "app") -> TestApp:
-        if isinstance(value, TestApp):
-            return value
-        app = value if isinstance(value, Router) else get_app(value, app_name)
-        return TestApp(app, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})