dcicutils 8.5.0.1b6__py3-none-any.whl → 8.6.0.0b0__py3-none-any.whl

dcicutils/misc_utils.py

@@ -982,11 +982,7 @@ def to_integer(value: str, fallback: Optional[Any] = None) -> Optional[Any]:
     try:
         return int(value)
     except Exception:
-        try:
-            return int(float(value))
-        except Exception:
-            pass
-    return fallback
+        return fallback
 
 
 def to_float(value: str, fallback: Optional[Any] = None) -> Optional[Any]:
@@ -1469,33 +1465,28 @@ def string_list(s):
     return [p for p in [part.strip() for part in s.split(",")] if p]
 
 
-def split_string(value: str, delimiter: str, escape: Optional[str] = None, unique: bool = False) -> List[str]:
+def split_string(value: str, delimiter: str, escape: Optional[str] = None) -> List[str]:
     """
     Splits the given string into an array of string based on the given delimiter, and an optional escape character.
     """
     if not isinstance(value, str) or not (value := value.strip()):
         return []
-    result = []
     if not isinstance(escape, str) or not escape:
-        for item in value.split(delimiter):
-            if (item := item.strip()) and (unique is not True or item not in result):
-                result.append(item)
-        return result
+        return [item.strip() for item in value.split(delimiter)]
+    result = []
     item = r""
     escaped = False
     for c in value:
         if c == delimiter and not escaped:
-            if (item := item.strip()) and (unique is not True or item not in result):
-                result.append(item)
+            result.append(item.strip())
             item = r""
         elif c == escape and not escaped:
             escaped = True
         else:
             item += c
             escaped = False
-    if (item := item.strip()) and (unique is not True or item not in result):
-        result.append(item)
-    return result
+    result.append(item.strip())
+    return [item for item in result if item]
 
 
 def right_trim(list_or_tuple: Union[List[Any], Tuple[Any]],

dcicutils/portal_utils.py

@@ -1,20 +1,19 @@
 from collections import deque
-import io
-import json
 from pyramid.paster import get_app
 from pyramid.router import Router
-import os
 import re
 import requests
 from requests.models import Response as RequestResponse
 from typing import Optional, Type, Union
 from webtest.app import TestApp, TestResponse
-from dcicutils.common import OrchestratedApp, ORCHESTRATED_APPS
+from dcicutils.common import OrchestratedApp, APP_CGAP, APP_FOURFRONT, APP_SMAHT, ORCHESTRATED_APPS
+from dcicutils.creds_utils import CGAPKeyManager, FourfrontKeyManager, SMaHTKeyManager
 from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
 from dcicutils.misc_utils import to_camel_case, VirtualApp
 from dcicutils.zip_utils import temporary_file
 
 Portal = Type["Portal"]  # Forward type reference for type hints.
+FILE_SCHEMA_NAME = "File"
 
 
 class Portal:
@@ -24,175 +23,105 @@ class Portal:
     2. From a key dictionary, containing "key" and "secret" property values.
     3. From a key tuple, containing (in order) a key and secret values.
     4. From a keys file assumed to reside in ~/.{app}-keys.json where the given "app" value is either "smaht", "cgap",
-       or "fourfront"; where is assumed to contain a dictionary with a key for the given "env" value, e.g. smaht-local;
-       and with a dictionary value containing "key" and "secret" property values, and an optional "server" property;
-       if an "app" value is not specified but the given "env" value begins with one of the app values then that value
-       will be used, i.e. e.g. if "env" is "smaht-local" and app is unspecified than it is assumed to be "smaht".
+       or "fourfront"; and where this file is assumed to contain a dictionary with a key equal to the given "env"
+       value (e.g. smaht-localhost) and with a dictionary value containing "key" and "secret" property values; if
+       an "app" value is not specified but the given "env" value begins with one of the app values then that value
+       will be used, i.e. e.g. if env is "smaht-localhost" and app is unspecified than it is assumed to be "smaht".
     5. From a keys file as described above (#4) but rather than be identified by the given "env" value it
-       is looked up via the given "server" name and the "server" key dictionary value in the key file.
+       is looked up by the given "server" name and the "server" key dictionary value in the key file.
     6. From a given "vapp" value (which is assumed to be a TestApp or VirtualApp).
-    7. From another Portal object; or from a a pyramid Router object.
+    7. From another Portal object.
+    8. From a a pyramid Router object.
     """
-    FILE_SCHEMA_NAME = "File"
-    KEYS_FILE_DIRECTORY = os.path.expanduser(f"~")
-
     def __init__(self,
-                 arg: Optional[Union[Portal, TestApp, VirtualApp, Router, dict, tuple, str]] = None,
-                 env: Optional[str] = None, server: Optional[str] = None,
-                 app: Optional[OrchestratedApp] = None) -> None:
-
-        def init(unspecified: Optional[list] = []) -> None:
-            self._ini_file = None
-            self._key = None
-            self._key_pair = None
-            self._key_id = None
-            self._secret = None
-            self._keys_file = None
-            self._env = None
-            self._server = None
-            self._app = None
-            self._vapp = None
-            for arg in unspecified:
-                if arg is not None:
-                    raise Exception("Portal init error; extraneous args.")
-
-        def init_from_portal(portal: Portal, unspecified: Optional[list] = None) -> None:
-            init(unspecified)
-            self._ini_file = portal._ini_file
-            self._key = portal._key
-            self._key_pair = portal._key_pair
-            self._key_id = portal._key_id
-            self._secret = portal._secret
-            self._keys_file = portal._keys_file
+                 arg: Optional[Union[VirtualApp, TestApp, Router, Portal, dict, tuple, str]] = None,
+                 env: Optional[str] = None, app: Optional[OrchestratedApp] = None, server: Optional[str] = None,
+                 key: Optional[Union[dict, tuple]] = None,
+                 vapp: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None,
+                 portal: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None) -> Portal:
+        if vapp and not portal:
+            portal = vapp
+        if ((isinstance(arg, (VirtualApp, TestApp, Router, Portal)) or
+             isinstance(arg, str) and arg.endswith(".ini")) and not portal):
+            portal = arg
+        elif isinstance(arg, str) and not env:
+            env = arg
+        elif (isinstance(arg, dict) or isinstance(arg, tuple)) and not key:
+            key = arg
+        if not app and env:
+            if env.startswith(APP_SMAHT):
+                app = APP_SMAHT
+            elif env.startswith(APP_CGAP):
+                app = APP_CGAP
+            elif env.startswith(APP_FOURFRONT):
+                app = APP_FOURFRONT
+        if isinstance(portal, Portal):
+            self._vapp = portal._vapp
             self._env = portal._env
-            self._server = portal._server
             self._app = portal._app
-            self._vapp = portal._vapp
-
-        def init_from_vapp(vapp: Union[TestApp, VirtualApp, Router], unspecified: Optional[list] = []) -> None:
-            init(unspecified)
-            self._vapp = Portal._create_testapp(vapp)
-
-        def init_from_ini_file(ini_file: str, unspecified: Optional[list] = []) -> None:
-            init(unspecified)
-            self._ini_file = ini_file
-            self._vapp = Portal._create_testapp(ini_file)
-
-        def init_from_key(key: dict, server: Optional[str], unspecified: Optional[list] = []) -> None:
-            init(unspecified)
-            if (isinstance(key_id := key.get("key"), str) and key_id and
-                isinstance(secret := key.get("secret"), str) and secret):  # noqa
-                self._key = {"key": key_id, "secret": secret}
-                self._key_id = key_id
-                self._secret = secret
-                self._key_pair = (key_id, secret)
-                if ((isinstance(server, str) and server) or (isinstance(server := key.get("server"), str) and server)):
-                    if server := normalize_server(server):
-                        self._key["server"] = self._server = server
-            if not self._key:
-                raise Exception("Portal init error; from key.")
-
-        def init_from_key_pair(key_pair: tuple, server: Optional[str], unspecified: Optional[list] = []) -> None:
-            if len(key_pair) == 2:
-                init_from_key({"key": key_pair[0], "secret": key_pair[1]}, server, unspecified)
-            else:
-                raise Exception("Portal init error; from key-pair.")
-
-        def init_from_keys_file(keys_file: str, env: Optional[str], server: Optional[str],
-                                unspecified: Optional[list] = []) -> None:
-            try:
-                with io.open(keys_file) as f:
-                    keys = json.load(f)
-            except Exception:
-                raise Exception(f"Portal init error; cannot open keys-file: {keys_file}")
-            if isinstance(env, str) and env and isinstance(key := keys.get(env), dict):
-                init_from_key(key, server)
-                self._keys_file = keys_file
-                self._env = env
-            elif isinstance(server, str) and server and (key := [k for k in keys if keys[k].get("server") == server]):
-                init_from_key(key, server)
-                self._keys_file = keys_file
-            elif len(keys) == 1 and (env := next(iter(keys))) and isinstance(key := keys[env], dict) and key:
-                init_from_key(key, server)
-                self._keys_file = keys_file
-                self._env = env
-            else:
-                raise Exception(f"Portal init error; {env or server or None} not found in keys-file: {keys_file}")
-
-        def init_from_env_server_app(env: str, server: str, app: Optional[str],
-                                     unspecified: Optional[list] = None) -> None:
-            return init_from_keys_file(self._default_keys_file(app, env), env, server, unspecified)
-
-        def normalize_server(server: str) -> Optional[str]:
-            prefix = ""
-            if (lserver := server.lower()).startswith("http://"):
-                prefix = "http://"
-            elif lserver.startswith("https://"):
-                prefix = "https://"
-            if prefix:
-                if (server := re.sub(r"/+", "/", server[len(prefix):])).startswith("/"):
-                    server = server[1:]
-                if len(server) > 1 and server.endswith("/"):
-                    server = server[:-1]
-                return prefix + server if server else None
-
-        if isinstance(arg, Portal):
-            init_from_portal(arg, unspecified=[env, server, app])
-        elif isinstance(arg, (TestApp, VirtualApp, Router)):
-            init_from_vapp(arg, unspecified=[env, server, app])
-        elif isinstance(arg, str) and arg.endswith(".ini"):
-            init_from_ini_file(arg, unspecified=[env, server, app])
-        elif isinstance(arg, dict):
-            init_from_key(arg, server, unspecified=[env, app])
-        elif isinstance(arg, tuple):
-            init_from_key_pair(arg, server, unspecified=[env, app])
-        elif isinstance(arg, str) and arg.endswith(".json"):
-            init_from_keys_file(arg, env, server, unspecified=[app])
-        elif isinstance(arg, str) and arg:
-            init_from_env_server_app(arg, server, app, unspecified=[env])
-        elif isinstance(env, str) and env:
-            init_from_env_server_app(env, server, app, unspecified=[arg])
-        else:
-            raise Exception("Portal init error; invalid args.")
-
-    @property
-    def ini_file(self) -> Optional[str]:
-        return self._ini_file
-
-    @property
-    def key(self) -> Optional[dict]:
-        return self._key
-
-    @property
-    def key_pair(self) -> Optional[tuple]:
-        return self._key_pair
+            self._server = portal._server
+            self._key = portal._key
+            self._key_pair = portal._key_pair
+            self._key_file = portal._key_file
+            return
+        self._vapp = None
+        self._env = env
+        self._app = app
+        self._server = server
+        self._key = None
+        self._key_pair = None
+        self._key_file = None
+        if isinstance(portal, (VirtualApp, TestApp)):
+            self._vapp = portal
+        elif isinstance(portal, (Router, str)):
+            self._vapp = Portal._create_vapp(portal)
+        elif isinstance(key, dict):
+            self._key = key
+            self._key_pair = (key.get("key"), key.get("secret")) if key else None
+            if key_server := key.get("server"):
+                self._server = key_server
+        elif isinstance(key, tuple) and len(key) >= 2:
+            self._key = {"key": key[0], "secret": key[1]}
+            self._key_pair = key
+        elif isinstance(env, str):
+            key_managers = {APP_CGAP: CGAPKeyManager, APP_FOURFRONT: FourfrontKeyManager, APP_SMAHT: SMaHTKeyManager}
+            if not (key_manager := key_managers.get(self._app)) or not (key_manager := key_manager()):
+                raise Exception(f"Invalid app name: {self._app} (valid: {', '.join(ORCHESTRATED_APPS)}).")
+            if isinstance(env, str):
+                self._key = key_manager.get_keydict_for_env(env)
+                if key_server := self._key.get("server"):
+                    self._server = key_server
+            elif isinstance(self._server, str):
+                self._key = key_manager.get_keydict_for_server(self._server)
+            self._key_pair = key_manager.keydict_to_keypair(self._key) if self._key else None
+            self._key_file = key_manager.keys_file
 
     @property
-    def key_id(self) -> Optional[str]:
-        return self._key_id
+    def env(self):
+        return self._env
 
     @property
-    def secret(self) -> Optional[str]:
-        return self._secret
+    def app(self):
+        return self._app
 
     @property
-    def keys_file(self) -> Optional[str]:
-        return self._keys_file
+    def server(self):
+        return self._server
 
     @property
-    def env(self) -> Optional[str]:
-        return self._env
+    def key(self):
+        return self._key
 
     @property
-    def server(self) -> Optional[str]:
-        return self._server
+    def key_pair(self):
+        return self._key_pair
 
     @property
-    def app(self) -> Optional[str]:
-        return self._app
+    def key_file(self):
+        return self._key_file
 
     @property
-    def vapp(self) -> Optional[TestApp]:
+    def vapp(self):
         return self._vapp
 
     def get_metadata(self, object_id: str) -> Optional[dict]:
@@ -209,27 +138,27 @@ class Portal:
         return self.post(f"/{object_type}", data)
 
     def get(self, uri: str, follow: bool = True, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
-        if self._vapp:
-            response = self._vapp.get(self.url(uri), **self._kwargs(**kwargs))
+        if isinstance(self._vapp, (VirtualApp, TestApp)):
+            response = self._vapp.get(self._uri(uri), **self._kwargs(**kwargs))
             if response and response.status_code in [301, 302, 303, 307, 308] and follow:
                 response = response.follow()
             return self._response(response)
-        return requests.get(self.url(uri), allow_redirects=follow, **self._kwargs(**kwargs))
+        return requests.get(self._uri(uri), allow_redirects=follow, **self._kwargs(**kwargs))
 
     def patch(self, uri: str, data: Optional[dict] = None,
               json: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
-        if self._vapp:
-            return self._vapp.patch_json(self.url(uri), json or data, **self._kwargs(**kwargs))
-        return requests.patch(self.url(uri), data=data, json=json, **self._kwargs(**kwargs))
+        if isinstance(self._vapp, (VirtualApp, TestApp)):
+            return self._vapp.patch_json(self._uri(uri), json or data, **self._kwargs(**kwargs))
+        return requests.patch(self._uri(uri), json=json or data, **self._kwargs(**kwargs))
 
     def post(self, uri: str, data: Optional[dict] = None, json: Optional[dict] = None,
              files: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
-        if self._vapp:
+        if isinstance(self._vapp, (VirtualApp, TestApp)):
             if files:
-                return self._vapp.post(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
+                return self._vapp.post(self._uri(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
             else:
-                return self._vapp.post_json(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
-        return requests.post(self.url(uri), data=data, json=json, files=files, **self._kwargs(**kwargs))
+                return self._vapp.post_json(self._uri(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
+        return requests.post(self._uri(uri), json=json or data, files=files, **self._kwargs(**kwargs))
 
     def get_schema(self, schema_name: str) -> Optional[dict]:
         return get_schema(self.schema_name(schema_name), portal_vapp=self._vapp, key=self._key)
@@ -243,7 +172,7 @@ class Portal:
 
     def is_file_schema(self, schema_name: str) -> bool:
         if super_type_map := self.get_schemas_super_type_map():
-            if file_super_type := super_type_map.get(Portal.FILE_SCHEMA_NAME):
+            if file_super_type := super_type_map.get(FILE_SCHEMA_NAME):
                 return self.schema_name(schema_name) in file_super_type
         return False
 
@@ -277,20 +206,13 @@ class Portal:
             super_type_map_flattened[super_type_name] = breadth_first(super_type_map, super_type_name)
         return super_type_map_flattened
 
-    def ping(self) -> bool:
-        try:
-            return self.get("/health").status_code == 200
-        except Exception:
-            return False
-
-    def url(self, uri: str) -> str:
+    def _uri(self, uri: str) -> str:
         if not isinstance(uri, str) or not uri:
             return "/"
-        if (luri := uri.lower()).startswith("http://") or luri.startswith("https://"):
+        if uri.lower().startswith("http://") or uri.lower().startswith("https://"):
             return uri
-        if not (uri := re.sub(r"/+", "/", uri)).startswith("/"):
-            uri = "/"
-        return self._server + uri if self._server else uri
+        uri = re.sub(r"/+", "/", uri)
+        return (self._server + ("/" if uri.startswith("/") else "") + uri) if self._server else uri
 
     def _kwargs(self, **kwargs) -> dict:
         result_kwargs = {"headers":
@@ -301,16 +223,6 @@ class Portal:
             result_kwargs["timeout"] = timeout
         return result_kwargs
 
-    def _default_keys_file(self, app: Optional[str], env: Optional[str] = None) -> Optional[str]:
-        def is_valid_app(app: Optional[str]) -> bool:  # noqa
-            return app and app.lower() in [name.lower() for name in ORCHESTRATED_APPS]
-        def infer_app_from_env(env: str) -> Optional[str]:  # noqa
-            if isinstance(env, str) and (lenv := env.lower()):
-                if app := [app for app in ORCHESTRATED_APPS if lenv.startswith(app.lower())]:
-                    return app[0]
-        if is_valid_app(app) or (app := infer_app_from_env(env)):
-            return os.path.join(Portal.KEYS_FILE_DIRECTORY, f".{app.lower()}-keys.json")
-
     def _response(self, response) -> Optional[RequestResponse]:
         if response and isinstance(getattr(response.__class__, "json"), property):
             class RequestResponseWrapper:  # For consistency change json property to method.
@@ -327,15 +239,15 @@ class Portal:
     @staticmethod
     def create_for_testing(ini_file: Optional[str] = None) -> Portal:
         if isinstance(ini_file, str):
-            return Portal(Portal._create_testapp(ini_file))
+            return Portal(Portal._create_vapp(ini_file))
         minimal_ini_for_unit_testing = "[app:app]\nuse = egg:encoded\nsqlalchemy.url = postgresql://dummy\n"
         with temporary_file(content=minimal_ini_for_unit_testing, suffix=".ini") as ini_file:
-            return Portal(Portal._create_testapp(ini_file))
+            return Portal(Portal._create_vapp(ini_file))
 
     @staticmethod
     def create_for_testing_local(ini_file: Optional[str] = None) -> Portal:
         if isinstance(ini_file, str) and ini_file:
-            return Portal(Portal._create_testapp(ini_file))
+            return Portal(Portal._create_vapp(ini_file))
         minimal_ini_for_testing_local = "\n".join([
             "[app:app]\nuse = egg:encoded\nfile_upload_bucket = dummy",
             "sqlalchemy.url = postgresql://postgres@localhost:5441/postgres?host=/tmp/snovault/pgdata",
@@ -356,20 +268,11 @@ class Portal:
             "multiauth.policy.auth0.base = encoded.authentication.Auth0AuthenticationPolicy"
         ])
         with temporary_file(content=minimal_ini_for_testing_local, suffix=".ini") as minimal_ini_file:
-            return Portal(Portal._create_testapp(minimal_ini_file))
+            return Portal(Portal._create_vapp(minimal_ini_file))
 
     @staticmethod
-    def _create_testapp(arg: Union[TestApp, VirtualApp, Router, str] = None, app_name: Optional[str] = None) -> TestApp:
-        if isinstance(arg, TestApp):
-            return arg
-        elif isinstance(arg, VirtualApp):
-            if not isinstance(arg.wrapped_app, TestApp):
-                raise Exception("Portal._create_testapp VirtualApp argument error.")
-            return arg.wrapped_app
-        if isinstance(arg, Router):
-            router = arg
-        elif isinstance(arg, str) or arg is None:
-            router = get_app(arg or "development.ini", app_name or "app")
-        else:
-            raise Exception("Portal._create_testapp argument error.")
-        return TestApp(router, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})
+    def _create_vapp(value: Union[str, Router, TestApp] = "development.ini", app_name: str = "app") -> TestApp:
+        if isinstance(value, TestApp):
+            return value
+        app = value if isinstance(value, Router) else get_app(value, app_name)
+        return TestApp(app, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})

dcicutils/schema_utils.py

@@ -0,0 +1,142 @@
+from typing import Any, Dict, List
+
+
+class JsonSchemaConstants:
+    ANY_OF = "anyOf"
+    ARRAY = "array"
+    BOOLEAN = "boolean"
+    ENUM = "enum"
+    INTEGER = "integer"
+    ITEMS = "items"
+    NUMBER = "number"
+    OBJECT = "object"
+    ONE_OF = "oneOf"
+    PATTERN = "pattern"
+    PROPERTIES = "properties"
+    REQUIRED = "required"
+    STRING = "string"
+    TYPE = "type"
+
+
+class EncodedSchemaConstants:
+    DEFAULT = "default"
+    FORMAT = "format"
+    IDENTIFYING_PROPERTIES = "identifyingProperties"
+    LINK_TO = "linkTo"
+    MERGE_REF = "$merge"
+    MIXIN_PROPERTIES = "mixinProperties"
+    REF = "$ref"
+    UNIQUE_KEY = "uniqueKey"
+
+
+class SchemaConstants(JsonSchemaConstants, EncodedSchemaConstants):
+    pass
+
+
+def get_properties(schema: Dict[str, Any]) -> Dict[str, Any]:
+    """Return the properties of a schema."""
+    return schema.get(SchemaConstants.PROPERTIES, {})
+
+
+def get_required(schema: Dict[str, Any]) -> List[str]:
+    """Return the required properties of a schema."""
+    return schema.get(SchemaConstants.REQUIRED, [])
+
+
+def get_any_of(schema: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Return the anyOf properties of a schema."""
+    return schema.get(SchemaConstants.ANY_OF, [])
+
+
+def get_one_of(schema: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Return the oneOf properties of a schema."""
+    return schema.get(SchemaConstants.ONE_OF, [])
+
+
+def get_conditionally_required_properties(schema: Dict[str, Any]) -> List[str]:
+    """Get required + possibly required properties.
+
+    Using heuristics here; update as needed.
+    """
+    return sorted(
+        list(
+            set(
+                get_required(schema)
+                + get_any_of_required_properties(schema)
+                + get_one_of_required_properties(schema)
+            )
+        )
+    )
+
+
+def get_any_of_required_properties(schema: Dict[str, Any]) -> List[str]:
+    """Get required properties from anyOf."""
+    return [
+        property_name
+        for any_of_schema in get_any_of(schema)
+        for property_name in get_required(any_of_schema)
+    ]
+
+
+def get_one_of_required_properties(schema: Dict[str, Any]) -> List[str]:
+    """Get required properties from oneOf."""
+    return [
+        property_name
+        for one_of_schema in get_one_of(schema)
+        for property_name in get_required(one_of_schema)
+    ]
+
+
+def get_mixin_properties(schema: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Return the mixin properties of a schema."""
+    return schema.get(EncodedSchemaConstants.MIXIN_PROPERTIES, [])
+
+
+def get_identifying_properties(schema: Dict[str, Any]) -> List[str]:
+    """Return the identifying properties of a schema."""
+    return schema.get(EncodedSchemaConstants.IDENTIFYING_PROPERTIES, [])
+
+
+def get_schema_type(schema: Dict[str, Any]) -> str:
+    """Return the type of a schema."""
+    return schema.get(SchemaConstants.TYPE, "")
+
+
+def is_array_schema(schema: Dict[str, Any]) -> bool:
+    """Return True if the schema is an array."""
+    return get_schema_type(schema) == SchemaConstants.ARRAY
+
+
+def is_object_schema(schema: Dict[str, Any]) -> bool:
+    """Return True if the schema is an object."""
+    return get_schema_type(schema) == SchemaConstants.OBJECT
+
+
+def is_string_schema(schema: Dict[str, Any]) -> bool:
+    """Return True if the schema is a string."""
+    return get_schema_type(schema) == SchemaConstants.STRING
+
+
+def is_number_schema(schema: Dict[str, Any]) -> bool:
+    """Return True if the schema is a number."""
+    return get_schema_type(schema) == SchemaConstants.NUMBER
+
+
+def is_integer_schema(schema: Dict[str, Any]) -> bool:
+    """Return True if the schema is an integer."""
+    return get_schema_type(schema) == SchemaConstants.INTEGER
+
+
+def is_boolean_schema(schema: Dict[str, Any]) -> bool:
+    """Return True if the schema is a boolean."""
+    return get_schema_type(schema) == SchemaConstants.BOOLEAN
+
+
+def get_items(schema: Dict[str, Any]) -> Dict[str, Any]:
+    """Return the items of a schema."""
+    return schema.get(SchemaConstants.ITEMS, {})
+
+
+def has_property(schema: Dict[str, Any], property_name: str) -> bool:
+    """Return True if the schema has the given property."""
+    return property_name in get_properties(schema)

dcicutils/structured_data.py

@@ -42,24 +42,23 @@ StructuredDataSet = Type["StructuredDataSet"]
 class StructuredDataSet:
 
     def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
-                 schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
+                 schemas: Optional[List[dict]] = None, data: Optional[List[dict]] = None,
                  order: Optional[List[str]] = None, prune: bool = True) -> None:
-        self.data = {}
+        self.data = {} if not data else data  # If portal is None then no schemas nor refs.
         self._portal = Portal(portal, data=self.data, schemas=schemas) if portal else None
         self._order = order
         self._prune = prune
         self._warnings = {}
         self._errors = {}
-        self._resolved_refs = set()
+        self._resolved_refs = []
         self._validated = False
-        self._autoadd_properties = autoadd if isinstance(autoadd, dict) and autoadd else None
         self._load_file(file) if file else None
 
     @staticmethod
     def load(file: str, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
-             schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
+             schemas: Optional[List[dict]] = None,
              order: Optional[List[str]] = None, prune: bool = True) -> StructuredDataSet:
-        return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune)
+        return StructuredDataSet(file=file, portal=portal, schemas=schemas, order=order, prune=prune)
 
     def validate(self, force: bool = False) -> None:
         if self._validated and not force:
@@ -97,7 +96,7 @@ class StructuredDataSet:
 
     @property
     def resolved_refs(self) -> List[str]:
-        return list(self._resolved_refs)
+        return self._resolved_refs
 
     @property
     def upload_files(self) -> List[str]:
@@ -113,7 +112,7 @@ class StructuredDataSet:
     def _load_file(self, file: str) -> None:
         # Returns a dictionary where each property is the name (i.e. the type) of the data,
         # and the value is array of dictionaries for the data itself. Handle these kinds of files:
-        # 1.  Single CSV, TSV, or JSON file, where the (base) name of the file is the data type name.
+        # 1.  Single CSV of JSON file, where the (base) name of the file is the data type name.
         # 2.  Single Excel file containing one or more sheets, where each sheet
         #     represents (i.e. is named for, and contains data for) a different type.
         # 3.  Zip file (.zip or .tar.gz or .tgz or .tar), containing data files to load, where the
@@ -164,13 +163,11 @@ class StructuredDataSet:
             structured_row = structured_row_template.create_row()
             for column_name, value in row.items():
                 structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
-                if self._autoadd_properties:
-                    self._add_properties(structured_row, self._autoadd_properties, schema)
             self._add(type_name, structured_row)
         self._note_warning(reader.warnings, "reader")
         if schema:
             self._note_error(schema._unresolved_refs, "ref")
-            self._resolved_refs.update(schema._resolved_refs)
+            self._resolved_refs = schema._resolved_refs
 
     def _add(self, type_name: str, data: Union[dict, List[dict]]) -> None:
         if self._prune:
@@ -180,11 +177,6 @@ class StructuredDataSet:
         else:
             self.data[type_name] = [data] if isinstance(data, dict) else data
 
-    def _add_properties(self, structured_row: dict, properties: dict, schema: Optional[dict] = None) -> None:
-        for name in properties:
-            if name not in structured_row and (not schema or schema.data.get("properties", {}).get(name)):
-                structured_row[name] = properties[name]
-
     def _note_warning(self, item: Optional[Union[dict, List[dict]]], group: str) -> None:
         self._note_issue(self._warnings, item, group)
 
@@ -245,7 +237,7 @@ class _StructuredRowTemplate:
             return {array_name: array} if array_name else {column_component: value}
 
         def set_value_internal(data: Union[dict, list], value: Optional[Any], src: Optional[str],
-                               path: List[Union[str, int]], typeinfo: Optional[dict], mapv: Optional[Callable]) -> None:
+                               path: List[Union[str, int]], mapv: Optional[Callable]) -> None:
 
             def set_value_backtrack_object(path_index: int, path_element: str) -> None:
                 nonlocal data, path, original_data
@@ -265,7 +257,7 @@ class _StructuredRowTemplate:
                     set_value_backtrack_object(i, p)
                 data = data[p]
             if (p := path[-1]) == -1 and isinstance(value, str):
-                values = _split_array_string(value, unique=typeinfo.get("unique") if typeinfo else False)
+                values = _split_array_string(value)
                 if mapv:
                     values = [mapv(value, src) for value in values]
                 merge_objects(data, values)
@@ -296,13 +288,11 @@ class _StructuredRowTemplate:
         for column_name in column_names or []:
             ensure_column_consistency(column_name)
             rational_column_name = self._schema.rationalize_column_name(column_name) if self._schema else column_name
-            column_typeinfo = self._schema.get_typeinfo(rational_column_name) if self._schema else None
-            map_value_function = column_typeinfo.get("map") if column_typeinfo else None
+            map_value_function = self._schema.get_map_value_function(rational_column_name) if self._schema else None
             if (column_components := _split_dotted_string(rational_column_name)):
                 merge_objects(structured_row_template, parse_components(column_components, path := []), True)
-                self._set_value_functions[column_name] = (
-                    lambda data, value, src, path=path, typeinfo=column_typeinfo, mapv=map_value_function:
-                        set_value_internal(data, value, src, path, typeinfo, mapv))
+                self._set_value_functions[column_name] = (lambda data, value, src, path=path, mapv=map_value_function:
+                                                          set_value_internal(data, value, src, path, mapv))
         return structured_row_template
 
 
@@ -325,8 +315,7 @@ class Schema:
 
     @staticmethod
     def load_by_name(name: str, portal: Portal) -> Optional[dict]:
-        schema_json = portal.get_schema(Schema.type_name(name)) if portal else None
-        return Schema(schema_json, portal) if schema_json else None
+        return Schema(portal.get_schema(Schema.type_name(name)), portal) if portal else None
 
     def validate(self, data: dict) -> List[str]:
         errors = []
@@ -342,7 +331,10 @@ class Schema:
     def resolved_refs(self) -> List[str]:
         return list(self._resolved_refs)
 
-    def get_typeinfo(self, column_name: str) -> Optional[dict]:
+    def get_map_value_function(self, column_name: str) -> Optional[Any]:
+        return (self._get_typeinfo(column_name) or {}).get("map")
+
+    def _get_typeinfo(self, column_name: str) -> Optional[dict]:
         if isinstance(info := self._typeinfo.get(column_name), str):
             info = self._typeinfo.get(info)
         if not info and isinstance(info := self._typeinfo.get(self.unadorn_column_name(column_name)), str):
@@ -475,14 +467,9 @@ class Schema:
                             raise Exception(f"Array of undefined or multiple types in JSON schema NOT supported: {key}")
                         raise Exception(f"Invalid array type specifier in JSON schema: {key}")
                     key = key + ARRAY_NAME_SUFFIX_CHAR
-                    if unique := (property_value.get("uniqueItems") is True):
-                        pass
                     property_value = array_property_items
                     property_value_type = property_value.get("type")
-                typeinfo = self._create_typeinfo(array_property_items, parent_key=key)
-                if unique:
-                    typeinfo[key]["unique"] = True
-                result.update(typeinfo)
+                result.update(self._create_typeinfo(array_property_items, parent_key=key))
                 continue
             result[key] = {"type": property_value_type, "map": self._map_function({**property_value, "column": key})}
             if ARRAY_NAME_SUFFIX_CHAR in key:
@@ -550,13 +537,16 @@ class Portal(PortalBase):
 
     def __init__(self,
                  arg: Optional[Union[VirtualApp, TestApp, Router, Portal, dict, tuple, str]] = None,
-                 env: Optional[str] = None, server: Optional[str] = None,
-                 app: Optional[OrchestratedApp] = None,
-                 data: Optional[dict] = None, schemas: Optional[List[dict]] = None) -> None:
-        super().__init__(arg, env=env, server=server, app=app)
-        if isinstance(arg, Portal):
-            self._schemas = schemas if schemas is not None else arg._schemas  # Explicitly specified/known schemas.
-            self._data = data if data is not None else arg._data  # Data set being loaded; e.g. by StructuredDataSet.
+                 env: Optional[str] = None, app: OrchestratedApp = None, server: Optional[str] = None,
+                 key: Optional[Union[dict, tuple]] = None,
+                 portal: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None,
+                 data: Optional[dict] = None, schemas: Optional[List[dict]] = None) -> Optional[Portal]:
+        super().__init__(arg, env=env, app=app, server=server, key=key, portal=portal)
+        if isinstance(arg, Portal) and not portal:
+            portal = arg
+        if isinstance(portal, Portal):
+            self._schemas = schemas if schemas is not None else portal._schemas  # Explicitly specified/known schemas.
+            self._data = data if data is not None else portal._data  # Data set being loaded; e.g. by StructuredDataSet.
         else:
             self._schemas = schemas
             self._data = data
@@ -570,9 +560,7 @@ class Portal(PortalBase):
 
     @lru_cache(maxsize=256)
     def get_schema(self, schema_name: str) -> Optional[dict]:
-        if not (schemas := self.get_schemas()):
-            return None
-        if schema := schemas.get(schema_name := Schema.type_name(schema_name)):
+        if (schemas := self.get_schemas()) and (schema := schemas.get(schema_name := Schema.type_name(schema_name))):
             return schema
         if schema_name == schema_name.upper() and (schema := schemas.get(schema_name.lower().title())):
             return schema
@@ -580,9 +568,8 @@ class Portal(PortalBase):
             return schema
 
     @lru_cache(maxsize=1)
-    def get_schemas(self) -> Optional[dict]:
-        if not (schemas := super().get_schemas()) or (schemas.get("status") == "error"):
-            return None
+    def get_schemas(self) -> dict:
+        schemas = super().get_schemas()
         if self._schemas:
             schemas = copy.deepcopy(schemas)
             for user_specified_schema in self._schemas:
@@ -628,5 +615,5 @@ def _split_dotted_string(value: str):
     return split_string(value, DOTTED_NAME_DELIMITER_CHAR)
 
 
-def _split_array_string(value: str, unique: bool = False):
-    return split_string(value, ARRAY_VALUE_DELIMITER_CHAR, ARRAY_VALUE_DELIMITER_ESCAPE_CHAR, unique=unique)
+def _split_array_string(value: str):
+    return split_string(value, ARRAY_VALUE_DELIMITER_CHAR, ARRAY_VALUE_DELIMITER_ESCAPE_CHAR)

{dcicutils-8.5.0.1b6.dist-info → dcicutils-8.6.0.0b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.5.0.1b6
+Version: 8.6.0.0b0
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.5.0.1b6.dist-info → dcicutils-8.6.0.0b0.dist-info}/RECORD

@@ -40,30 +40,31 @@ dcicutils/license_policies/park-lab-gpl-pipeline.jsonc,sha256=vLZkwm3Js-kjV44nug
 dcicutils/license_policies/park-lab-pipeline.jsonc,sha256=9qlY0ASy3iUMQlr3gorVcXrSfRHnVGbLhkS427UaRy4,283
 dcicutils/license_utils.py,sha256=d1cq6iwv5Ju-VjdoINi6q7CPNNL7Oz6rcJdLMY38RX0,46978
 dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
-dcicutils/misc_utils.py,sha256=nRjLEORY35YmJwTjO0fnauBPznaI_bkVasIW8PccDYM,100179
+dcicutils/misc_utils.py,sha256=slnJM87VrPj2gCiQR8Yrl8bwTjeynXpM1yKIjJg8NN4,99816
 dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
-dcicutils/portal_utils.py,sha256=PLIKONo_BhVrf-r5fkXAaxP5IQZRjvPjf83YxvRx0ZE,18393
+dcicutils/portal_utils.py,sha256=OF1JPoDgEbvCvPKz0egcnmqGEOYWkamuBLRN_RpZFtk,14073
 dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
 dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
 dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
 dcicutils/redis_tools.py,sha256=qkcSNMtvqkpvts-Cm9gWhneK523Q_oHwhNUud1be1qk,7055
 dcicutils/redis_utils.py,sha256=VJ-7g8pOZqR1ZCtdcjKz3-6as2DMUcs1b1zG6wSprH4,6462
 dcicutils/s3_utils.py,sha256=eN8lfDI8Yxqga7iuy_eOlmUJUEjewHho0szLmj2YmYI,28852
+dcicutils/schema_utils.py,sha256=xlvY7BuIhjJYk7GKYamhbHDw1751rmYcEM2pOaEH1BY,4203
 dcicutils/scripts/publish_to_pypi.py,sha256=qmWyjrg5bNQNfpNKFTZdyMXpRmrECnRV9VmNQddUPQA,13576
 dcicutils/scripts/run_license_checker.py,sha256=z2keYnRDZsHQbTeo1XORAXSXNJK5axVzL5LjiNqZ7jE,4184
 dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19745
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
 dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
-dcicutils/structured_data.py,sha256=mWjH6h9ARUvYMHI4ZiwbaoKtoeM0CAOz1UxCHh4HIHE,32874
+dcicutils/structured_data.py,sha256=bOzashl_cy5c8NWewcPpRPknQDbSPstL_3Cw2-KUSl0,32112
 dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
 dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=0OXR0aLNwyLIZOzIFTM_5DOun7dxIv6TIZbFiithkO0,3276
-dcicutils-8.5.0.1b6.dist-info/LICENSE.txt,sha256=t0_-jIjqxNnymZoNJe-OltRIuuF8qfhN0ATlHyrUJPk,1102
-dcicutils-8.5.0.1b6.dist-info/METADATA,sha256=wWiZixknI6Pne33sGS-Lxeh6hSKSScj44tPcctlpHJw,3314
-dcicutils-8.5.0.1b6.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.5.0.1b6.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
-dcicutils-8.5.0.1b6.dist-info/RECORD,,
+dcicutils-8.6.0.0b0.dist-info/LICENSE.txt,sha256=t0_-jIjqxNnymZoNJe-OltRIuuF8qfhN0ATlHyrUJPk,1102
+dcicutils-8.6.0.0b0.dist-info/METADATA,sha256=Wbkp0zSdhveaLePg7P-jFS06WfiHGMn30IQq2RxhN6g,3314
+dcicutils-8.6.0.0b0.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.6.0.0b0.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
+dcicutils-8.6.0.0b0.dist-info/RECORD,,