dcicutils 8.8.1.1b7__py3-none-any.whl → 8.8.1.1b10__py3-none-any.whl

dcicutils/progress_constants.py

@@ -0,0 +1,53 @@
+from enum import Enum
+
+
+# Constants for progress tracking for smaht-submitr.
+# Here only to share between smaht-portal, snovault, and smaht-submitr.
+
+class PROGRESS_INGESTER(Enum):
+    VALIDATION = "ingester_validation"
+    INITIATE = "ingester_initiate"
+    PARSE_LOAD_INITIATE = "ingester_parse_initiate"
+    PARSE_LOAD_DONE = "ingester_parse_done"
+    VALIDATE_LOAD_INITIATE = "ingester_validate_initiate"
+    VALIDATE_LOAD_DONE = "ingester_validate_done"
+    LOADXL_INITIATE = "ingester_loadxl_initiate"
+    LOADXL_DONE = "ingester_loadxl_done"
+
+
+class PROGRESS_PARSE(Enum):
+    LOAD_START = "start"
+    LOAD_ITEM = "parse"
+    LOAD_DONE = "finish"
+    LOAD_COUNT_SHEETS = "sheets"
+    LOAD_COUNT_ROWS = "rows"
+    LOAD_COUNT_REFS = "refs"
+    LOAD_COUNT_REFS_FOUND = "refs_found"
+    LOAD_COUNT_REFS_NOT_FOUND = "refs_not_found"
+    LOAD_COUNT_REFS_LOOKUP = "refs_lookup"
+    LOAD_COUNT_REFS_LOOKUP_CACHE_HIT = "refs_lookup_cache_hit"
+    LOAD_COUNT_REFS_EXISTS_CACHE_HIT = "refs_exists_cache_hit"
+    LOAD_COUNT_REFS_INVALID = "refs_invalid"
+    ANALYZE_START = "start"
+    ANALYZE_COUNT_TYPES = "types"
+    ANALYZE_COUNT_ITEMS = "objects"
+    ANALYZE_CREATE = "create"
+    ANALYZE_COUNT_LOOKUP = "lookups"
+    ANALYZE_UPDATE = "update"
+    ANALYZE_DONE = "finish"
+
+
+class PROGRESS_LOADXL(Enum):
+    START = "loadxl_start"
+    START_SECOND_ROUND = "loadxl_start_second_round"
+    ITEM = "loadxl_item"
+    ITEM_SECOND_ROUND = "loadxl_item_second_round"
+    GET = "loadxl_lookup"
+    POST = "loadxl_post"
+    PATCH = "loadxl_patch"
+    ERROR = "loadxl_error"
+    DONE = "loadxl_done"
+    TOTAL = "loadxl_total"
+    MESSAGE = "loadxl_message"
+    MESSAGE_VERBOSE = "loadxl_message_verbose"
+    MESSAGE_DEBUG = "loadxl_message_debug"

dcicutils/structured_data.py

@@ -17,6 +17,7 @@ from dcicutils.misc_utils import (create_dict, create_readonly_object, is_uuid,
                                   to_boolean, to_enum, to_float, to_integer, VirtualApp)
 from dcicutils.portal_object_utils import PortalObject
 from dcicutils.portal_utils import Portal as PortalBase
+from dcicutils.progress_constants import PROGRESS_PARSE as PROGRESS
 from dcicutils.schema_utils import Schema as SchemaBase
 from dcicutils.zip_utils import unpack_gz_file_to_temporary_file, unpack_files
 
@@ -37,31 +38,10 @@ ARRAY_NAME_SUFFIX_CHAR = "#"
 ARRAY_NAME_SUFFIX_REGEX = re.compile(rf"{ARRAY_NAME_SUFFIX_CHAR}\d+")
 DOTTED_NAME_DELIMITER_CHAR = "."
 
+
 # TODO: Should probably pass this knowledge in from callers.
 FILE_TYPE_NAME = "File"
 FILE_TYPE_PROPERTY_NAME = "filename"
-# This ExtraFile is a pseudo-type to handle extra_files in smaht-submitr.
-EXTRA_FILE_TYPE_NAME = "ExtraFile"
-EXTRA_FILE_TYPE_PROPERTY_NAME = "extra_files"
-
-ENABLE_ARRAY_SHEET_REFS = False
-
-# The ExtraFile pseudo-type schema.
-EXTRA_FILE_SCHEMA = {
-    "title": "ExtraFile",
-    "type": "object",
-    "required": [
-        "filename"
-    ],
-    "identifyingProperties": [
-        "filename"
-    ],
-    "properties": {
-        "filename": {
-            "type": "string"
-        }
-    }
-}
 
 # Forward type references for type hints.
 Portal = Type["Portal"]
@@ -212,7 +192,8 @@ class StructuredDataSet:
         diffs = {}
         if callable(progress):
             ntypes, nobjects = get_counts()
-            progress({"start": True, "types": ntypes, "objects": nobjects})
+            progress({PROGRESS.ANALYZE_START: True,
+                      PROGRESS.ANALYZE_COUNT_TYPES: ntypes, PROGRESS.ANALYZE_COUNT_ITEMS: nobjects})
         if self.data or self.portal:  # TODO: what is this OR biz?
             refs = self.resolved_refs_with_uuids
             # TODO: Need feedback/progress tracking mechanism here.
@@ -231,18 +212,19 @@ class StructuredDataSet:
                                                                        uuid=existing_object.uuid,
                                                                        diffs=object_diffs or None))
                         if callable(progress):
-                            progress({"update": True, "lookups": nlookups + nlookups_compare})
+                            progress({PROGRESS.ANALYZE_UPDATE: True,
+                                      PROGRESS.ANALYZE_COUNT_LOOKUP: nlookups + nlookups_compare})
                     elif identifying_path:
                         # If there is no existing object we still create a record for this object
                         # but with no uuid which will be the indication that it does not exist.
                         diffs[type_name].append(create_readonly_object(path=identifying_path, uuid=None, diffs=None))
                         if callable(progress):
-                            progress({"create": True, "lookups": nlookups})
+                            progress({PROGRESS.ANALYZE_CREATE: True, PROGRESS.ANALYZE_COUNT_LOOKUP: nlookups})
                     else:
                         if callable(progress):
-                            progress({"lookups": nlookups})
+                            progress({PROGRESS.ANALYZE_COUNT_LOOKUP: nlookups})
         if callable(progress):
-            progress({"finish": True})
+            progress({PROGRESS.ANALYZE_DONE: True})
         return diffs
 
     def load_file(self, file: str) -> None:
@@ -287,9 +269,10 @@ class StructuredDataSet:
                 for row in excel.sheet_reader(sheet_name):
                     nrows += 1
             return nrows, len(excel.sheet_names)
-        if self._progress:
+        if self._progress:  # TODO: Move to _load_reader
             nrows, nsheets = get_counts()
-            self._progress({"start": True, "sheets": nsheets, "rows": nrows})
+            self._progress({PROGRESS.LOAD_START: True,
+                            PROGRESS.LOAD_COUNT_SHEETS: nsheets, PROGRESS.LOAD_COUNT_ROWS: nrows})
         excel = Excel(file)  # Order the sheet names by any specified ordering (e.g. ala snovault.loadxl).
         order = {Schema.type_name(key): index for index, key in enumerate(self._order)} if self._order else {}
         for sheet_name in sorted(excel.sheet_names, key=lambda key: order.get(Schema.type_name(key), sys.maxsize)):
@@ -313,16 +296,15 @@ class StructuredDataSet:
             else:
                 del self._errors["ref"]
         if self._progress:
-            # TODO: Refactor with same thing below in _load_reader.
-            self._progress({
-                "finish": True,
-                "refs": self.ref_total_count,
-                "refs_found": self.ref_total_found_count,
-                "refs_not_found": self.ref_total_notfound_count,
-                "refs_lookup": self.ref_lookup_count,
-                "refs_lookup_cache_hit": self.ref_lookup_cache_hit_count,
-                "refs_exists_cache_hit": self.ref_exists_cache_hit_count,
-                "refs_invalid": self.ref_invalid_identifying_property_count
+            self._progress({   # TODO: Refactor with same thing below in _load_reader.
+                PROGRESS.LOAD_DONE: True,
+                PROGRESS.LOAD_COUNT_REFS: self.ref_total_count,
+                PROGRESS.LOAD_COUNT_REFS_FOUND: self.ref_total_found_count,
+                PROGRESS.LOAD_COUNT_REFS_NOT_FOUND: self.ref_total_notfound_count,
+                PROGRESS.LOAD_COUNT_REFS_LOOKUP: self.ref_lookup_count,
+                PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT: self.ref_lookup_cache_hit_count,
+                PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT: self.ref_exists_cache_hit_count,
+                PROGRESS.LOAD_COUNT_REFS_INVALID: self.ref_invalid_identifying_property_count
             })
 
     def _load_json_file(self, file: str) -> None:
@@ -352,14 +334,14 @@ class StructuredDataSet:
             self._add(type_name, structured_row)
             if self._progress:
                 self._progress({
-                    "parse": True,
-                    "refs": self.ref_total_count,
-                    "refs_found": self.ref_total_found_count,
-                    "refs_not_found": self.ref_total_notfound_count,
-                    "refs_lookup": self.ref_lookup_count,
-                    "refs_lookup_cache_hit": self.ref_lookup_cache_hit_count,
-                    "refs_exists_cache_hit": self.ref_exists_cache_hit_count,
-                    "refs_invalid": self.ref_invalid_identifying_property_count
+                    PROGRESS.LOAD_ITEM: True,
+                    PROGRESS.LOAD_COUNT_REFS: self.ref_total_count,
+                    PROGRESS.LOAD_COUNT_REFS_FOUND: self.ref_total_found_count,
+                    PROGRESS.LOAD_COUNT_REFS_NOT_FOUND: self.ref_total_notfound_count,
+                    PROGRESS.LOAD_COUNT_REFS_LOOKUP: self.ref_lookup_count,
+                    PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT: self.ref_lookup_cache_hit_count,
+                    PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT: self.ref_exists_cache_hit_count,
+                    PROGRESS.LOAD_COUNT_REFS_INVALID: self.ref_invalid_identifying_property_count
                 })
         self._note_warning(reader.warnings, "reader")
         if schema:
@@ -465,12 +447,10 @@ class StructuredDataSet:
 
 class _StructuredRowTemplate:
 
-    def __init__(self, column_names: List[str], schema: Optional[Schema] = None,
-                 obtain_array_values: Optional[Callable] = None) -> None:
+    def __init__(self, column_names: List[str], schema: Optional[Schema] = None) -> None:
         self._schema = schema
         self._set_value_functions = {}
         self._template = self._create_row_template(column_names)
-        self._obtain_array_values = obtain_array_values if callable(obtain_array_values) else None
 
     def create_row(self) -> dict:
         return copy.deepcopy(self._template)
@@ -530,16 +510,6 @@ class _StructuredRowTemplate:
                     set_value_backtrack_object(i, p)
                 data = data[p]
             if (p := path[-1]) == -1 and isinstance(value, str):
-                if ENABLE_ARRAY_SHEET_REFS and False:
-                    # TODO: IN PROGRESS. DISABLED FOR NOW.
-                    if isinstance(value, str) and value.lower().startswith("[ref:") and value.endswith("]"):
-                        if self._obtain_array_values:
-                            values = self._obtain_array_values(value)
-                        if sheet_name_containing_array := value[5:].strip():
-                            if dot := sheet_name_containing_array.find(".") > 0:
-                                if sheet_name_containing_array := sheet_name_containing_array[0:dot].strip():
-                                    pass
-                                    # sheet_column_containing_array = sheet_name_containing_array[dot + 1:].strip()
                 values = _split_array_string(value, unique=typeinfo.get("unique") if typeinfo else False)
                 if mapv:
                     values = [mapv(value, src) for value in values]

{dcicutils-8.8.1.1b7.dist-info → dcicutils-8.8.1.1b10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.8.1.1b7
+Version: 8.8.1.1b10
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.8.1.1b7.dist-info → dcicutils-8.8.1.1b10.dist-info}/RECORD

@@ -48,6 +48,7 @@ dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmj
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
 dcicutils/portal_object_utils.py,sha256=MF6MTZ6yxakZFDjbkTKCsF4q4p11dLDVvT5JBV9m6RQ,15408
 dcicutils/portal_utils.py,sha256=oBoI3KWRp6YrbsuVGbmPQ3kATB5cVVsQo7-qmnYXWqg,30260
+dcicutils/progress_constants.py,sha256=1N3BOnViX4AlrNK4lq7O-zRBqoZ0BTvC_vpFk2rOxD4,1736
 dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
 dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
 dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
@@ -62,15 +63,15 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
 dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
-dcicutils/structured_data.py,sha256=tnxilT_CBN58hEmvH-wMRM47RJiQTPQYiP2O_JrO41Q,59323
+dcicutils/structured_data.py,sha256=1guVNDzIVxJkQA_m0jSh9xI2FB5oVXR4m7sqrqF8A5w,58559
 dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
 dcicutils/tmpfile_utils.py,sha256=n95XF8dZVbQRSXBZTGToXXfSs3JUVRyN6c3ZZ0nhAWI,1403
 dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
-dcicutils-8.8.1.1b7.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.8.1.1b7.dist-info/METADATA,sha256=0bVW0YQ9uJnm4u110RRhzXeCQQ0V1N3v42jxYTnsNGU,3356
-dcicutils-8.8.1.1b7.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.8.1.1b7.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
-dcicutils-8.8.1.1b7.dist-info/RECORD,,
+dcicutils-8.8.1.1b10.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.8.1.1b10.dist-info/METADATA,sha256=ZpO-aF0tzLRyns4vaIGBO5VsQq1hX8bw4rWDv8_v5Hc,3357
+dcicutils-8.8.1.1b10.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.8.1.1b10.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
+dcicutils-8.8.1.1b10.dist-info/RECORD,,