dcicutils 8.4.0.1b3__tar.gz → 8.4.0.1b5__tar.gz

{dcicutils-8.4.0.1b3 → dcicutils-8.4.0.1b5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.4.0.1b3
+Version: 8.4.0.1b5
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.4.0.1b3 → dcicutils-8.4.0.1b5}/dcicutils/data_readers.py

@@ -2,27 +2,30 @@ import abc
 import csv
 import openpyxl
 from typing import Any, Generator, Iterator, List, Optional, Type, Tuple, Union
-from dcicutils.misc_utils import right_trim
+from dcicutils.misc_utils import create_object, right_trim
+
+# Forward type references for type hints.
+Excel = Type["Excel"]
 
 
 class RowReader(abc.ABC):
 
     def __init__(self):
         self.header = None
-        self.location = 0
+        self.row_number = 0
         self._warning_empty_headers = False
         self._warning_extra_values = []  # Line numbers.
         self.open()
 
     def __iter__(self) -> Iterator:
         for row in self.rows:
-            self.location += 1
+            self.row_number += 1
             if self.is_comment_row(row):
                 continue
             if self.is_terminating_row(row):
                 break
             if len(self.header) < len(row):  # Row values beyond what there are headers for are ignored.
-                self._warning_extra_values.append(self.location)
+                self._warning_extra_values.append(self.row_number)
             yield {column: self.cell_value(value) for column, value in zip(self.header, row)}
 
     def _define_header(self, header: List[Optional[Any]]) -> None:
@@ -49,13 +52,20 @@ class RowReader(abc.ABC):
         pass
 
     @property
-    def issues(self) -> Optional[List[str]]:
-        issues = []
+    def file(self) -> Optional[str]:
+        return self._file if hasattr(self, "_file") else None
+
+    @property
+    def warnings(self) -> List[str]:
+        warnings = []
         if self._warning_empty_headers:
-            issues.append("Empty header column encountered; ignoring it and all subsequent columns.")
+            warnings.append({"src": create_object(file=self.file),
+                             "warning": "Empty header column encountered; ignoring it and all subsequent columns."})
         if self._warning_extra_values:
-            issues.extend([f"Extra column values on row [{row_number}]" for row_number in self._warning_extra_values])
-        return issues if issues else None
+            for row_number in self._warning_extra_values:
+                warnings.append({"src": create_object(file=self.file, row=row_number),
+                                 "warning": f"Extra row column values."})
+        return warnings
 
 
 class ListReader(RowReader):
@@ -101,9 +111,10 @@ class CsvReader(RowReader):
 
 class ExcelSheetReader(RowReader):
 
-    def __init__(self, sheet_name: str, workbook: openpyxl.workbook.workbook.Workbook) -> None:
+    def __init__(self, excel: Excel, sheet_name: str, workbook: openpyxl.workbook.workbook.Workbook) -> None:
         self.sheet_name = sheet_name or "Sheet1"
         self._workbook = workbook
+        self._file = excel._file
         self._rows = None
         super().__init__()
 
@@ -134,12 +145,13 @@ class Excel:
         self.open()
 
     def sheet_reader(self, sheet_name: str) -> ExcelSheetReader:
-        return self._reader_class(sheet_name=sheet_name, workbook=self._workbook)
+        return self._reader_class(self, sheet_name=sheet_name, workbook=self._workbook)
 
     def open(self) -> None:
         if self._workbook is None:
             self._workbook = openpyxl.load_workbook(self._file, data_only=True)
-            self.sheet_names = [(sheet_name or "").strip() for sheet_name in (self._workbook.sheetnames or [])]
+            self.sheet_names = [sheet_name for sheet_name in self._workbook.sheetnames
+                                if self._workbook[sheet_name].sheet_state != "hidden"]
 
     def __del__(self) -> None:
         if (workbook := self._workbook) is not None:

{dcicutils-8.4.0.1b3 → dcicutils-8.4.0.1b5}/dcicutils/misc_utils.py

@@ -1501,6 +1501,14 @@ def right_trim(list_or_tuple: Union[List[Any], Tuple[Any]],
     return list_or_tuple[:i + 1]
 
 
+def create_object(**kwargs) -> dict:
+    result = {}
+    for name in kwargs:
+        if kwargs[name]:
+            result[name] = kwargs[name]
+    return result
+
+
 def is_c4_arn(arn: str) -> bool:
     """
     Returns True iff the given (presumed) AWS ARN string value looks like it

{dcicutils-8.4.0.1b3 → dcicutils-8.4.0.1b5}/dcicutils/structured_data.py

@@ -1,3 +1,4 @@
+from collections import deque
 import copy
 from functools import lru_cache
 import json
@@ -15,8 +16,8 @@ from dcicutils.common import OrchestratedApp, APP_CGAP, APP_FOURFRONT, APP_SMAHT
 from dcicutils.creds_utils import CGAPKeyManager, FourfrontKeyManager, SMaHTKeyManager
 from dcicutils.data_readers import CsvReader, Excel, RowReader
 from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
-from dcicutils.misc_utils import (load_json_if, merge_objects, remove_empty_properties, right_trim, split_string,
-                                  to_boolean, to_camel_case, to_enum, to_float, to_integer, VirtualApp)
+from dcicutils.misc_utils import (create_object, load_json_if, merge_objects, remove_empty_properties, right_trim,
+                                  split_string, to_boolean, to_camel_case, to_enum, to_float, to_integer, VirtualApp)
 from dcicutils.zip_utils import temporary_file, unpack_gz_file_to_temporary_file, unpack_files
 
 
@@ -52,7 +53,10 @@ class StructuredDataSet:
         self._portal = Portal(portal, data=self.data, schemas=schemas) if portal else None
         self._order = order
         self._prune = prune
-        self._issues = None
+        self._warnings = {}
+        self._errors = {}
+        self._resolved_refs = []
+        self._validated = False
         self._load_file(file) if file else None
 
     @staticmethod
@@ -61,16 +65,85 @@ class StructuredDataSet:
              order: Optional[List[str]] = None, prune: bool = True) -> StructuredDataSet:
         return StructuredDataSet(file=file, portal=portal, schemas=schemas, order=order, prune=prune)
 
-    def validate(self) -> Optional[List[str]]:
-        issues = []
+    def validate(self, force: bool = False) -> None:
+        if self._validated and not force:
+            return
+        self._validated = True
         for type_name in self.data:
             if (schema := Schema.load_by_name(type_name, portal=self._portal)):
-                item_number = 0
+                row_number = 0
                 for data in self.data[type_name]:
-                    item_number += 1
-                    if (validation_issues := schema.validate(data)) is not None:
-                        issues.extend([f"{schema.name} [{item_number}]: {issue}" for issue in validation_issues])
-        return issues + (self._issues or [])
+                    row_number += 1
+                    if (validation_errors := schema.validate(data)) is not None:
+                        for validation_error in validation_errors:
+                            self._note_error({"src": create_object(type=schema.name, row=row_number),
+                                              "error": validation_error}, "validation")
+
+    @property
+    def warnings(self) -> dict:
+        return self._warnings
+
+    @property
+    def reader_warnings(self) -> List[dict]:
+        return self._warnings.get("reader") or []
+
+    @property
+    def errors(self) -> dict:
+        return self._errors
+
+    @property
+    def ref_errors(self) -> List[dict]:
+        return self._errors.get("ref") or []
+
+    @property
+    def validation_errors(self) -> List[dict]:
+        return self._errors.get("validation") or []
+
+    @property
+    def resolved_refs(self) -> List[str]:
+        return self._resolved_refs
+
+    @staticmethod
+    def format_issue(issue: dict, original_file: Optional[str] = None) -> str:
+        def src_string(issue: dict) -> str:
+            if not isinstance(issue, dict) or not isinstance(issue_src := issue.get("src"), dict):
+                return ""
+            show_file = original_file and (original_file.endswith(".zip") or
+                                           original_file.endswith(".tgz") or original_file.endswith(".gz"))
+            src_file = issue_src.get("file") if show_file else ""
+            src_type = issue_src.get("type")
+            src_column = issue_src.get("column")
+            src_row = issue_src.get("row", 0)
+            if src_file:
+                src = f"{os.path.basename(src_file)}"
+                sep = ":"
+            else:
+                src = ""
+                sep = "."
+            if src_type:
+                src += (sep if src else "") + src_type
+                sep = "."
+            if src_column:
+                src += (sep if src else "") + src_column
+            if src_row > 0:
+                src += (" " if src else "") + f"[{src_row}]"
+            if not src:
+                if issue.get("warning"):
+                    src = "Warning"
+                elif issue.get("error"):
+                    src = "Error"
+                else:
+                    src = "Issue"
+            return src
+        issue_message = None
+        if issue:
+            if error := issue.get("error"):
+                issue_message = error
+            elif warning := issue.get("warning"):
+                issue_message = warning
+            elif issue.get("truncated"):
+                return f"Truncated result set | More: {issue.get('more')} | See: {issue.get('details')}"
+        return f"{src_string(issue)}: {issue_message}" if issue_message else ""
 
     def _load_file(self, file: str) -> None:
         # Returns a dictionary where each property is the name (i.e. the type) of the data,
@@ -100,15 +173,13 @@ class StructuredDataSet:
             self._load_file(file)
 
     def _load_csv_file(self, file: str) -> None:
-        self._load_reader(reader := CsvReader(file), type_name=Schema.type_name(file))
-        self._note_issues(reader.issues, os.path.basename(file))
+        self._load_reader(CsvReader(file), type_name=Schema.type_name(file))
 
     def _load_excel_file(self, file: str) -> None:
         excel = Excel(file)  # Order the sheet names by any specified ordering (e.g. ala snovault.loadxl).
         order = {Schema.type_name(key): index for index, key in enumerate(self._order)} if self._order else {}
         for sheet_name in sorted(excel.sheet_names, key=lambda key: order.get(Schema.type_name(key), sys.maxsize)):
-            self._load_reader(reader := excel.sheet_reader(sheet_name), type_name=Schema.type_name(sheet_name))
-            self._note_issues(reader.issues, f"{file}:{sheet_name}")
+            self._load_reader(excel.sheet_reader(sheet_name), type_name=Schema.type_name(sheet_name))
 
     def _load_json_file(self, file: str) -> None:
         with open(file) as f:
@@ -119,16 +190,20 @@ class StructuredDataSet:
         noschema = False
         structured_row_template = None
         for row in reader:
-            if not structured_row_template:  # Delay creation just so we don't create it if there are no rows.
+            if not structured_row_template:  # Delay schema creation so we don't reference it if there are no rows.
                 if not schema and not noschema and not (schema := Schema.load_by_name(type_name, portal=self._portal)):
                     noschema = True
+                elif schema and (schema_name := schema.name):
+                    type_name = schema_name
                 structured_row_template = _StructuredRowTemplate(reader.header, schema)
             structured_row = structured_row_template.create_row()
             for column_name, value in row.items():
-                structured_row_template.set_value(structured_row, column_name, value, reader.location)
-            if schema and (schema_name := schema.name):
-                type_name = schema_name
+                structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
             self._add(type_name, structured_row)
+        self._note_warning(reader.warnings, "reader")
+        if schema:
+            self._note_error(schema._unresolved_refs, "ref")
+            self._resolved_refs = schema._resolved_refs
 
     def _add(self, type_name: str, data: Union[dict, List[dict]]) -> None:
         if self._prune:
@@ -138,11 +213,19 @@ class StructuredDataSet:
         else:
             self.data[type_name] = [data] if isinstance(data, dict) else data
 
-    def _note_issues(self, issues: Optional[List[str]], source: str) -> None:
-        if issues:
-            if not self._issues:
-                self._issues = []
-            self._issues.append({source: issues})
+    def _note_warning(self, item: Optional[Union[dict, List[dict]]], group: str) -> None:
+        self._note_issue(self._warnings, item, group)
+
+    def _note_error(self, item: Optional[Union[dict, List[dict]]], group: str) -> None:
+        self._note_issue(self._errors, item, group)
+
+    def _note_issue(self, issues: dict, item: Optional[Union[dict, List[dict]]], group: str) -> None:
+        if isinstance(item, dict) and item:
+            item = [item]
+        if isinstance(item, list) and item:
+            if not issues.get(group):
+                issues[group] = []
+            issues[group].extend(item)
 
 
 class _StructuredRowTemplate:
@@ -155,10 +238,10 @@ class _StructuredRowTemplate:
     def create_row(self) -> dict:
         return copy.deepcopy(self._template)
 
-    def set_value(self, data: dict, column_name: str, value: str, loc: int = -1) -> None:
+    def set_value(self, data: dict, column_name: str, value: str, file: Optional[str], row_number: int = -1) -> None:
         if (set_value_function := self._set_value_functions.get(column_name)):
-            src = (f"{f'{self._schema.name}.' if self._schema else ''}" +
-                   f"{f'{column_name}' if column_name else ''}{f' [{loc}]' if loc else ''}")
+            src = create_object(type=self._schema.name if self._schema else None,
+                                column=column_name, file=file, row=row_number)
             set_value_function(data, value, src)
 
     def _create_row_template(self, column_names: List[str]) -> dict:  # Surprisingly tricky code here.
@@ -262,17 +345,27 @@ class Schema:
             "number": self._map_function_number,
             "string": self._map_function_string
         }
+        self._resolved_refs = set()
+        self._unresolved_refs = []
         self._typeinfo = self._create_typeinfo(schema_json)
 
     @staticmethod
     def load_by_name(name: str, portal: Portal) -> Optional[dict]:
         return Schema(portal.get_schema(Schema.type_name(name)), portal) if portal else None
 
-    def validate(self, data: dict) -> Optional[List[str]]:
-        issues = []
-        for issue in SchemaValidator(self.data, format_checker=SchemaValidator.FORMAT_CHECKER).iter_errors(data):
-            issues.append(issue.message)
-        return issues if issues else None
+    def validate(self, data: dict) -> List[str]:
+        errors = []
+        for error in SchemaValidator(self.data, format_checker=SchemaValidator.FORMAT_CHECKER).iter_errors(data):
+            errors.append(error.message)
+        return errors
+
+    @property
+    def unresolved_refs(self) -> List[dict]:
+        return self._unresolved_refs
+
+    @property
+    def resolved_refs(self) -> List[str]:
+        return list(self._resolved_refs)
 
     def get_map_value_function(self, column_name: str) -> Optional[Any]:
         return (self._get_typeinfo(column_name) or {}).get("map")
@@ -333,14 +426,16 @@ class Schema:
     def _map_function_ref(self, typeinfo: dict) -> Callable:
         def map_ref(value: str, link_to: str, portal: Optional[Portal], src: Optional[str]) -> Any:
             nonlocal self, typeinfo
-            exception = None
             if not value:
                 if (column := typeinfo.get("column")) and column in self.data.get("required", []):
-                    exception = f"No required reference (linkTo) value for: {link_to}"
-            elif portal and not portal.ref_exists(link_to, value):
-                exception = f"Cannot resolve reference (linkTo) for: {link_to}"
-            if exception:
-                raise Exception(exception + f"{f'/{value}' if value else ''}{f' from {src}' if src else ''}")
+                    self._unresolved_refs.append({"src": src, "error": f"/{link_to}/<null>"})
+            elif portal:
+                if not (resolved := portal.ref_exists(link_to, value)):
+                    self._unresolved_refs.append({"src": src, "error": f"/{link_to}/{value}"})
+                elif len(resolved) > 1:
+                    self._unresolved_refs.append({"src": src, "error": f"/{link_to}/{value}", "types": resolved})
+                else:
+                    self._resolved_refs.add(f"/{link_to}/{value}")
             return value
         return lambda value, src: map_ref(value, typeinfo.get("linkTo"), self._portal, src)
 
@@ -481,15 +576,8 @@ class PortalBase:
                  env: Optional[str] = None, app: OrchestratedApp = APP_SMAHT, server: Optional[str] = None,
                  key: Optional[Union[dict, tuple]] = None,
                  portal: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None) -> PortalBase:
-        if isinstance(arg, VirtualApp) and not portal:
-            portal = arg
-        elif isinstance(arg, TestApp) and not portal:
-            portal = arg
-        elif isinstance(arg, Router) and not portal:
-            portal = arg
-        elif isinstance(arg, Portal) and not portal:
-            portal = arg
-        elif isinstance(arg, str) and arg.endswith(".ini") and not portal:
+        if ((isinstance(arg, (VirtualApp, TestApp, Router, Portal)) or
+             isinstance(arg, str) and arg.endswith(".ini")) and not portal):
             portal = arg
         elif isinstance(arg, str) and not env:
             env = arg
@@ -539,15 +627,12 @@ class PortalBase:
             return post_metadata(schema_name=object_type, post_item=data, key=self._key)
         return self.post(f"/{object_type}", data)
 
-    def get_schema(self, schema_name: str) -> Optional[dict]:
-        return get_schema(schema_name, portal_vapp=self._vapp, key=self._key)
-
     def get(self, uri: str, follow: bool = True, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
         if isinstance(self._vapp, (VirtualApp, TestApp)):
             response = self._vapp.get(self._uri(uri), **self._kwargs(**kwargs))
             if response and response.status_code in [301, 302, 303, 307, 308] and follow:
                 response = response.follow()
-            return response
+            return self._response(response)
         return requests.get(self._uri(uri), allow_redirects=follow, **self._kwargs(**kwargs))
 
     def patch(self, uri: str, data: Optional[dict] = None,
@@ -565,6 +650,12 @@ class PortalBase:
                 return self._vapp.post_json(self._uri(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
         return requests.post(self._uri(uri), json=json or data, files=files, **self._kwargs(**kwargs))
 
+    def get_schema(self, schema_name: str) -> Optional[dict]:
+        return get_schema(schema_name, portal_vapp=self._vapp, key=self._key)
+
+    def get_schemas(self) -> dict:
+        return self.get("/profiles/").json()
+
     def _uri(self, uri: str) -> str:
         if not isinstance(uri, str) or not uri:
             return "/"
@@ -582,6 +673,19 @@ class PortalBase:
             result_kwargs["timeout"] = timeout
         return result_kwargs
 
+    def _response(self, response) -> Optional[RequestResponse]:
+        if response and isinstance(getattr(response.__class__, "json"), property):
+            class RequestResponseWrapper:  # For consistency change json property to method.
+                def __init__(self, respnose, **kwargs):
+                    super().__init__(**kwargs)
+                    self._response = response
+                def __getattr__(self, attr):  # noqa
+                    return getattr(self._response, attr)
+                def json(self):  # noqa
+                    return self._response.json
+            response = RequestResponseWrapper(response)
+        return response
+
     @staticmethod
     def create_for_testing(ini_file: Optional[str] = None) -> PortalBase:
         if isinstance(ini_file, str):
@@ -592,7 +696,7 @@ class PortalBase:
 
     @staticmethod
     def create_for_testing_local(ini_file: Optional[str] = None) -> Portal:
-        if isinstance(ini_file, str):
+        if isinstance(ini_file, str) and ini_file:
             return Portal(Portal._create_testapp(ini_file))
         minimal_ini_for_testing_local = "\n".join([
             "[app:app]\nuse = egg:encoded\nfile_upload_bucket = dummy",
@@ -655,23 +759,67 @@ class Portal(PortalBase):
 
     @lru_cache(maxsize=256)
     def get_schema(self, schema_name: str) -> Optional[dict]:
-        def get_schema_exact(schema_name: str) -> Optional[dict]:  # noqa
-            return (next((schema for schema in self._schemas or []
-                         if Schema.type_name(schema.get("title")) == Schema.type_name(schema_name)), None) or
-                    super(Portal, self).get_schema(schema_name))
-        try:
-            if (schema := get_schema_exact(schema_name)):
-                return schema
-        except Exception:  # Try/force camel-case if all upper/lower-case.
-            if schema_name == schema_name.upper():
-                if (schema := get_schema_exact(schema_name.lower().title())):
-                    return schema
-            elif schema_name == schema_name.lower():
-                if (schema := get_schema_exact(schema_name.title())):
-                    return schema
-            raise
-
-    def ref_exists(self, type_name: str, value: str) -> bool:
+        if (schemas := self.get_schemas()) and (schema := schemas.get(schema_name := Schema.type_name(schema_name))):
+            return schema
+        if schema_name == schema_name.upper() and (schema := schemas.get(schema_name.lower().title())):
+            return schema
+        if schema_name == schema_name.lower() and (schema := schemas.get(schema_name.title())):
+            return schema
+
+    @lru_cache(maxsize=1)
+    def get_schemas(self) -> dict:
+        schemas = super(Portal, self).get_schemas()
+        if self._schemas:
+            schemas = copy.deepcopy(schemas)
+            for user_specified_schema in self._schemas:
+                if user_specified_schema.get("title"):
+                    schemas[user_specified_schema["title"]] = user_specified_schema
+        return schemas
+
+    @lru_cache(maxsize=1)
+    def get_schemas_super_type_map(self) -> dict:
+        """
+        Returns the "super type map" for all of the known schemas (via /profiles).
+        This is a dictionary of all types which have (one or more) sub-types whose value is
+        an array of all of those sub-types (direct and all descendents), in breadth first order.
+        """
+        def breadth_first(super_type_map: dict, super_type_name: str) -> dict:
+            result = []
+            queue = deque(super_type_map.get(super_type_name, []))
+            while queue:
+                result.append(sub_type_name := queue.popleft())
+                if sub_type_name in super_type_map:
+                    queue.extend(super_type_map[sub_type_name])
+            return result
+        if not (schemas := self.get_schemas()):
+            return {}
+        super_type_map = {}
+        for type_name in schemas:
+            if super_type_name := schemas[type_name].get("rdfs:subClassOf"):
+                super_type_name = super_type_name.replace("/profiles/", "").replace(".json", "")
+                if super_type_name != "Item":
+                    if not super_type_map.get(super_type_name):
+                        super_type_map[super_type_name] = [type_name]
+                    elif type_name not in super_type_map[super_type_name]:
+                        super_type_map[super_type_name].append(type_name)
+        super_type_map_flattened = {}
+        for super_type_name in super_type_map:
+            super_type_map_flattened[super_type_name] = breadth_first(super_type_map, super_type_name)
+        return super_type_map_flattened
+
+    def ref_exists(self, type_name: str, value: str) -> List[str]:
+        resolved = []
+        if self._ref_exists_single(type_name, value):
+            resolved.append(type_name)
+        # Check for the given ref in all sub-types of the given type.
+        if (schemas_super_type_map := self.get_schemas_super_type_map()):
+            if (sub_type_names := schemas_super_type_map.get(type_name)):
+                for sub_type_name in sub_type_names:
+                    if self._ref_exists_single(sub_type_name, value):
+                        resolved.append(type_name)
+        return resolved
+
+    def _ref_exists_single(self, type_name: str, value: str) -> bool:
         if self._data and (items := self._data.get(type_name)) and (schema := self.get_schema(type_name)):
             iproperties = set(schema.get("identifyingProperties", [])) | {"identifier", "uuid"}
             for item in items:

{dcicutils-8.4.0.1b3 → dcicutils-8.4.0.1b5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.4.0.1b3"  # TODO: To become 8.4.1
+version = "8.4.0.1b5"  # TODO: To become 8.4.1
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"