dcicutils 8.4.0.1b4__py3-none-any.whl → 8.4.0.1b5__py3-none-any.whl

dcicutils/data_readers.py

@@ -2,27 +2,30 @@ import abc
 import csv
 import openpyxl
 from typing import Any, Generator, Iterator, List, Optional, Type, Tuple, Union
-from dcicutils.misc_utils import right_trim
+from dcicutils.misc_utils import create_object, right_trim
+
+# Forward type references for type hints.
+Excel = Type["Excel"]
 
 
 class RowReader(abc.ABC):
 
     def __init__(self):
         self.header = None
-        self.location = 0
+        self.row_number = 0
         self._warning_empty_headers = False
         self._warning_extra_values = []  # Line numbers.
         self.open()
 
     def __iter__(self) -> Iterator:
         for row in self.rows:
-            self.location += 1
+            self.row_number += 1
             if self.is_comment_row(row):
                 continue
             if self.is_terminating_row(row):
                 break
             if len(self.header) < len(row):  # Row values beyond what there are headers for are ignored.
-                self._warning_extra_values.append(self.location)
+                self._warning_extra_values.append(self.row_number)
             yield {column: self.cell_value(value) for column, value in zip(self.header, row)}
 
     def _define_header(self, header: List[Optional[Any]]) -> None:
@@ -49,13 +52,20 @@ class RowReader(abc.ABC):
         pass
 
     @property
-    def issues(self) -> Optional[List[str]]:
-        issues = []
+    def file(self) -> Optional[str]:
+        return self._file if hasattr(self, "_file") else None
+
+    @property
+    def warnings(self) -> List[str]:
+        warnings = []
         if self._warning_empty_headers:
-            issues.append("Empty header column encountered; ignoring it and all subsequent columns.")
+            warnings.append({"src": create_object(file=self.file),
+                             "warning": "Empty header column encountered; ignoring it and all subsequent columns."})
         if self._warning_extra_values:
-            issues.extend([f"Extra column values on row [{row_number}]" for row_number in self._warning_extra_values])
-        return issues if issues else None
+            for row_number in self._warning_extra_values:
+                warnings.append({"src": create_object(file=self.file, row=row_number),
+                                 "warning": f"Extra row column values."})
+        return warnings
 
 
 class ListReader(RowReader):
@@ -101,9 +111,10 @@ class CsvReader(RowReader):
 
 class ExcelSheetReader(RowReader):
 
-    def __init__(self, sheet_name: str, workbook: openpyxl.workbook.workbook.Workbook) -> None:
+    def __init__(self, excel: Excel, sheet_name: str, workbook: openpyxl.workbook.workbook.Workbook) -> None:
         self.sheet_name = sheet_name or "Sheet1"
         self._workbook = workbook
+        self._file = excel._file
         self._rows = None
         super().__init__()
 
@@ -134,7 +145,7 @@ class Excel:
         self.open()
 
     def sheet_reader(self, sheet_name: str) -> ExcelSheetReader:
-        return self._reader_class(sheet_name=sheet_name, workbook=self._workbook)
+        return self._reader_class(self, sheet_name=sheet_name, workbook=self._workbook)
 
     def open(self) -> None:
         if self._workbook is None:

dcicutils/misc_utils.py

@@ -1501,6 +1501,14 @@ def right_trim(list_or_tuple: Union[List[Any], Tuple[Any]],
     return list_or_tuple[:i + 1]
 
 
+def create_object(**kwargs) -> dict:
+    result = {}
+    for name in kwargs:
+        if kwargs[name]:
+            result[name] = kwargs[name]
+    return result
+
+
 def is_c4_arn(arn: str) -> bool:
     """
     Returns True iff the given (presumed) AWS ARN string value looks like it

dcicutils/structured_data.py

@@ -16,8 +16,8 @@ from dcicutils.common import OrchestratedApp, APP_CGAP, APP_FOURFRONT, APP_SMAHT
 from dcicutils.creds_utils import CGAPKeyManager, FourfrontKeyManager, SMaHTKeyManager
 from dcicutils.data_readers import CsvReader, Excel, RowReader
 from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
-from dcicutils.misc_utils import (load_json_if, merge_objects, remove_empty_properties, right_trim, split_string,
-                                  to_boolean, to_camel_case, to_enum, to_float, to_integer, VirtualApp)
+from dcicutils.misc_utils import (create_object, load_json_if, merge_objects, remove_empty_properties, right_trim,
+                                  split_string, to_boolean, to_camel_case, to_enum, to_float, to_integer, VirtualApp)
 from dcicutils.zip_utils import temporary_file, unpack_gz_file_to_temporary_file, unpack_files
 
 
@@ -53,9 +53,10 @@ class StructuredDataSet:
         self._portal = Portal(portal, data=self.data, schemas=schemas) if portal else None
         self._order = order
         self._prune = prune
-        self._issues = None
-        self._refs_resolved = set()
-        self._refs_unresolved = set()
+        self._warnings = {}
+        self._errors = {}
+        self._resolved_refs = []
+        self._validated = False
         self._load_file(file) if file else None
 
     @staticmethod
@@ -64,19 +65,85 @@ class StructuredDataSet:
              order: Optional[List[str]] = None, prune: bool = True) -> StructuredDataSet:
         return StructuredDataSet(file=file, portal=portal, schemas=schemas, order=order, prune=prune)
 
-    def validate(self) -> Optional[List[str]]:
-        issues = []
+    def validate(self, force: bool = False) -> None:
+        if self._validated and not force:
+            return
+        self._validated = True
         for type_name in self.data:
             if (schema := Schema.load_by_name(type_name, portal=self._portal)):
-                item_number = 0
+                row_number = 0
                 for data in self.data[type_name]:
-                    item_number += 1
-                    if (validation_issues := schema.validate(data)) is not None:
-                        issues.extend([f"{schema.name} [{item_number}]: {issue}" for issue in validation_issues])
-        return issues + (self._issues or [])
+                    row_number += 1
+                    if (validation_errors := schema.validate(data)) is not None:
+                        for validation_error in validation_errors:
+                            self._note_error({"src": create_object(type=schema.name, row=row_number),
+                                              "error": validation_error}, "validation")
 
-    def refs(self) -> Tuple[List[str], List[str]]:
-        return (sorted(self._refs_resolved), sorted(self._refs_unresolved))
+    @property
+    def warnings(self) -> dict:
+        return self._warnings
+
+    @property
+    def reader_warnings(self) -> List[dict]:
+        return self._warnings.get("reader") or []
+
+    @property
+    def errors(self) -> dict:
+        return self._errors
+
+    @property
+    def ref_errors(self) -> List[dict]:
+        return self._errors.get("ref") or []
+
+    @property
+    def validation_errors(self) -> List[dict]:
+        return self._errors.get("validation") or []
+
+    @property
+    def resolved_refs(self) -> List[str]:
+        return self._resolved_refs
+
+    @staticmethod
+    def format_issue(issue: dict, original_file: Optional[str] = None) -> str:
+        def src_string(issue: dict) -> str:
+            if not isinstance(issue, dict) or not isinstance(issue_src := issue.get("src"), dict):
+                return ""
+            show_file = original_file and (original_file.endswith(".zip") or
+                                           original_file.endswith(".tgz") or original_file.endswith(".gz"))
+            src_file = issue_src.get("file") if show_file else ""
+            src_type = issue_src.get("type")
+            src_column = issue_src.get("column")
+            src_row = issue_src.get("row", 0)
+            if src_file:
+                src = f"{os.path.basename(src_file)}"
+                sep = ":"
+            else:
+                src = ""
+                sep = "."
+            if src_type:
+                src += (sep if src else "") + src_type
+                sep = "."
+            if src_column:
+                src += (sep if src else "") + src_column
+            if src_row > 0:
+                src += (" " if src else "") + f"[{src_row}]"
+            if not src:
+                if issue.get("warning"):
+                    src = "Warning"
+                elif issue.get("error"):
+                    src = "Error"
+                else:
+                    src = "Issue"
+            return src
+        issue_message = None
+        if issue:
+            if error := issue.get("error"):
+                issue_message = error
+            elif warning := issue.get("warning"):
+                issue_message = warning
+            elif issue.get("truncated"):
+                return f"Truncated result set | More: {issue.get('more')} | See: {issue.get('details')}"
+        return f"{src_string(issue)}: {issue_message}" if issue_message else ""
 
     def _load_file(self, file: str) -> None:
         # Returns a dictionary where each property is the name (i.e. the type) of the data,
@@ -106,15 +173,13 @@ class StructuredDataSet:
             self._load_file(file)
 
     def _load_csv_file(self, file: str) -> None:
-        self._load_reader(reader := CsvReader(file), type_name=Schema.type_name(file))
-        self._note_issues(reader.issues, os.path.basename(file))
+        self._load_reader(CsvReader(file), type_name=Schema.type_name(file))
 
     def _load_excel_file(self, file: str) -> None:
         excel = Excel(file)  # Order the sheet names by any specified ordering (e.g. ala snovault.loadxl).
         order = {Schema.type_name(key): index for index, key in enumerate(self._order)} if self._order else {}
         for sheet_name in sorted(excel.sheet_names, key=lambda key: order.get(Schema.type_name(key), sys.maxsize)):
-            self._load_reader(reader := excel.sheet_reader(sheet_name), type_name=Schema.type_name(sheet_name))
-            self._note_issues(reader.issues, f"{file}:{sheet_name}")
+            self._load_reader(excel.sheet_reader(sheet_name), type_name=Schema.type_name(sheet_name))
 
     def _load_json_file(self, file: str) -> None:
         with open(file) as f:
@@ -125,18 +190,20 @@ class StructuredDataSet:
         noschema = False
         structured_row_template = None
         for row in reader:
-            if not structured_row_template:  # Delay creation just so we don't create it if there are no rows.
+            if not structured_row_template:  # Delay schema creation so we don't reference it if there are no rows.
                 if not schema and not noschema and not (schema := Schema.load_by_name(type_name, portal=self._portal)):
                     noschema = True
+                elif schema and (schema_name := schema.name):
+                    type_name = schema_name
                 structured_row_template = _StructuredRowTemplate(reader.header, schema)
             structured_row = structured_row_template.create_row()
             for column_name, value in row.items():
-                structured_row_template.set_value(structured_row, column_name, value, reader.location)
-            if schema and (schema_name := schema.name):
-                type_name = schema_name
-                self._refs_resolved = self._refs_resolved | schema._refs_resolved
-                self._refs_unresolved = self._refs_unresolved | schema._refs_unresolved
+                structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
             self._add(type_name, structured_row)
+        self._note_warning(reader.warnings, "reader")
+        if schema:
+            self._note_error(schema._unresolved_refs, "ref")
+            self._resolved_refs = schema._resolved_refs
 
     def _add(self, type_name: str, data: Union[dict, List[dict]]) -> None:
         if self._prune:
@@ -146,11 +213,19 @@ class StructuredDataSet:
         else:
             self.data[type_name] = [data] if isinstance(data, dict) else data
 
-    def _note_issues(self, issues: Optional[List[str]], source: str) -> None:
-        if issues:
-            if not self._issues:
-                self._issues = []
-            self._issues.append({source: issues})
+    def _note_warning(self, item: Optional[Union[dict, List[dict]]], group: str) -> None:
+        self._note_issue(self._warnings, item, group)
+
+    def _note_error(self, item: Optional[Union[dict, List[dict]]], group: str) -> None:
+        self._note_issue(self._errors, item, group)
+
+    def _note_issue(self, issues: dict, item: Optional[Union[dict, List[dict]]], group: str) -> None:
+        if isinstance(item, dict) and item:
+            item = [item]
+        if isinstance(item, list) and item:
+            if not issues.get(group):
+                issues[group] = []
+            issues[group].extend(item)
 
 
 class _StructuredRowTemplate:
@@ -163,10 +238,10 @@ class _StructuredRowTemplate:
     def create_row(self) -> dict:
         return copy.deepcopy(self._template)
 
-    def set_value(self, data: dict, column_name: str, value: str, loc: int = -1) -> None:
+    def set_value(self, data: dict, column_name: str, value: str, file: Optional[str], row_number: int = -1) -> None:
         if (set_value_function := self._set_value_functions.get(column_name)):
-            src = (f"{f'{self._schema.name}.' if self._schema else ''}" +
-                   f"{f'{column_name}' if column_name else ''}{f' [{loc}]' if loc else ''}")
+            src = create_object(type=self._schema.name if self._schema else None,
+                                column=column_name, file=file, row=row_number)
             set_value_function(data, value, src)
 
     def _create_row_template(self, column_names: List[str]) -> dict:  # Surprisingly tricky code here.
@@ -270,19 +345,27 @@ class Schema:
             "number": self._map_function_number,
             "string": self._map_function_string
         }
-        self._refs_resolved = set()
-        self._refs_unresolved = set()
+        self._resolved_refs = set()
+        self._unresolved_refs = []
         self._typeinfo = self._create_typeinfo(schema_json)
 
     @staticmethod
     def load_by_name(name: str, portal: Portal) -> Optional[dict]:
         return Schema(portal.get_schema(Schema.type_name(name)), portal) if portal else None
 
-    def validate(self, data: dict) -> Optional[List[str]]:
-        issues = []
-        for issue in SchemaValidator(self.data, format_checker=SchemaValidator.FORMAT_CHECKER).iter_errors(data):
-            issues.append(issue.message)
-        return issues if issues else None
+    def validate(self, data: dict) -> List[str]:
+        errors = []
+        for error in SchemaValidator(self.data, format_checker=SchemaValidator.FORMAT_CHECKER).iter_errors(data):
+            errors.append(error.message)
+        return errors
+
+    @property
+    def unresolved_refs(self) -> List[dict]:
+        return self._unresolved_refs
+
+    @property
+    def resolved_refs(self) -> List[str]:
+        return list(self._resolved_refs)
 
     def get_map_value_function(self, column_name: str) -> Optional[Any]:
         return (self._get_typeinfo(column_name) or {}).get("map")
@@ -343,17 +426,16 @@ class Schema:
     def _map_function_ref(self, typeinfo: dict) -> Callable:
         def map_ref(value: str, link_to: str, portal: Optional[Portal], src: Optional[str]) -> Any:
             nonlocal self, typeinfo
-            exception = None
             if not value:
                 if (column := typeinfo.get("column")) and column in self.data.get("required", []):
-                    self._refs_unresolved.add(f"/{link_to}/<null>")
-                    exception = f"No required reference (linkTo) value for: /{link_to}"
-            elif portal and not portal.ref_exists(link_to, value):
-                self._refs_unresolved.add(f"/{link_to}/{value}")
-                exception = f"Cannot resolve reference (linkTo) for: /{link_to}"
-            if exception:
-                raise Exception(exception + f"{f'/{value}' if value else ''}{f' from {src}' if src else ''}")
-            self._refs_resolved.add(f"/{link_to}/{value}")
+                    self._unresolved_refs.append({"src": src, "error": f"/{link_to}/<null>"})
+            elif portal:
+                if not (resolved := portal.ref_exists(link_to, value)):
+                    self._unresolved_refs.append({"src": src, "error": f"/{link_to}/{value}"})
+                elif len(resolved) > 1:
+                    self._unresolved_refs.append({"src": src, "error": f"/{link_to}/{value}", "types": resolved})
+                else:
+                    self._resolved_refs.add(f"/{link_to}/{value}")
             return value
         return lambda value, src: map_ref(value, typeinfo.get("linkTo"), self._portal, src)
 
@@ -725,16 +807,17 @@ class Portal(PortalBase):
             super_type_map_flattened[super_type_name] = breadth_first(super_type_map, super_type_name)
         return super_type_map_flattened
 
-    def ref_exists(self, type_name: str, value: str) -> bool:
+    def ref_exists(self, type_name: str, value: str) -> List[str]:
+        resolved = []
         if self._ref_exists_single(type_name, value):
-            return True
+            resolved.append(type_name)
         # Check for the given ref in all sub-types of the given type.
         if (schemas_super_type_map := self.get_schemas_super_type_map()):
             if (sub_type_names := schemas_super_type_map.get(type_name)):
                 for sub_type_name in sub_type_names:
                     if self._ref_exists_single(sub_type_name, value):
-                        return True
-        return False
+                        resolved.append(type_name)
+        return resolved
 
     def _ref_exists_single(self, type_name: str, value: str) -> bool:
         if self._data and (items := self._data.get(type_name)) and (schema := self.get_schema(type_name)):

{dcicutils-8.4.0.1b4.dist-info → dcicutils-8.4.0.1b5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.4.0.1b4
+Version: 8.4.0.1b5
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.4.0.1b4.dist-info → dcicutils-8.4.0.1b5.dist-info}/RECORD

@@ -9,7 +9,7 @@ dcicutils/common.py,sha256=YE8Mt5-vaZWWz4uaChSVhqGFbFtW5QKtnIyOr4zG4vM,3955
 dcicutils/contribution_scripts.py,sha256=0k5Gw1TumcD5SAcXVkDd6-yvuMEw-jUp5Kfb7FJH6XQ,2015
 dcicutils/contribution_utils.py,sha256=vYLS1JUB3sKd24BUxZ29qUBqYeQBLK9cwo8x3k64uPg,25653
 dcicutils/creds_utils.py,sha256=xrLekD49Ex0GOpL9n7LlJA4gvNcY7txTVFOSYD7LvEU,11113
-dcicutils/data_readers.py,sha256=6uespLpwHyC7RmR-WJDM_zsvFH5uZkEi11a9ujNv24I,5263
+dcicutils/data_readers.py,sha256=qoyCnXeQa9hIxlDvZ1JeUyADet7rb6dQcG4ZEZ8r1_k,5674
 dcicutils/data_utils.py,sha256=k2OxOlsx7AJ6jF-YNlMyGus_JqSUBe4_n1s65Mv1gQQ,3098
 dcicutils/deployment_utils.py,sha256=rcNUFMe_tsrG4CHEtgBe41cZx4Pk4JqISPsjrJRMoEs,68891
 dcicutils/diff_utils.py,sha256=sQx-yz56DHAcQWOChYbAG3clXu7TbiZKlw-GggeveO0,8118
@@ -40,7 +40,7 @@ dcicutils/license_policies/park-lab-gpl-pipeline.jsonc,sha256=vLZkwm3Js-kjV44nug
 dcicutils/license_policies/park-lab-pipeline.jsonc,sha256=9qlY0ASy3iUMQlr3gorVcXrSfRHnVGbLhkS427UaRy4,283
 dcicutils/license_utils.py,sha256=d1cq6iwv5Ju-VjdoINi6q7CPNNL7Oz6rcJdLMY38RX0,46978
 dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
-dcicutils/misc_utils.py,sha256=n4h8kNuRpIlHez_WiCd-tHFy3Mmf1d5ezI4w3sIShg8,99656
+dcicutils/misc_utils.py,sha256=jfyWDrHAlx2REun51i3igEApfEMAsmakRDo2VKUr0LQ,99818
 dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
 dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
@@ -55,14 +55,14 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
 dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
-dcicutils/structured_data.py,sha256=Z7CuwtDG0ac4z1kfgAQ5mhdmEwRQdYxtoSxxmEbUBU8,41516
+dcicutils/structured_data.py,sha256=q3IaTkXkVlFXkKbwmCSbDRN9oQp5DwFkmHz0kWvb_W4,44447
 dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
 dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=0OXR0aLNwyLIZOzIFTM_5DOun7dxIv6TIZbFiithkO0,3276
-dcicutils-8.4.0.1b4.dist-info/LICENSE.txt,sha256=t0_-jIjqxNnymZoNJe-OltRIuuF8qfhN0ATlHyrUJPk,1102
-dcicutils-8.4.0.1b4.dist-info/METADATA,sha256=6i1VEXvhivHopvz9XgI6k1ykUOSxcE-H_G8iuFQgET4,3314
-dcicutils-8.4.0.1b4.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.4.0.1b4.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
-dcicutils-8.4.0.1b4.dist-info/RECORD,,
+dcicutils-8.4.0.1b5.dist-info/LICENSE.txt,sha256=t0_-jIjqxNnymZoNJe-OltRIuuF8qfhN0ATlHyrUJPk,1102
+dcicutils-8.4.0.1b5.dist-info/METADATA,sha256=CpxQgbbWpPqaWG4bW9slXIOLl82aO2F3KzBf2PuaaP8,3314
+dcicutils-8.4.0.1b5.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.4.0.1b5.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
+dcicutils-8.4.0.1b5.dist-info/RECORD,,