dcicutils 8.8.0.1b13__py3-none-any.whl → 8.8.0.1b15__py3-none-any.whl

dcicutils/data_readers.py

@@ -168,7 +168,12 @@ class Excel:
 
     def open(self) -> None:
         if self._workbook is None:
-            self._workbook = openpyxl.load_workbook(self._file, data_only=True)
+            import warnings
+            with warnings.catch_warnings():
+                # Without this warning suppression thing, for some spreadsheets we get this stdout warning:
+                # UserWarning: data validation extension is not supported and will be removed
+                warnings.filterwarnings("ignore", category=UserWarning)
+                self._workbook = openpyxl.load_workbook(self._file, data_only=True)
             self.sheet_names = [sheet_name for sheet_name in self._workbook.sheetnames
                                 if not self.is_hidden_sheet(self._workbook[sheet_name])]
 

dcicutils/structured_data.py

@@ -7,7 +7,6 @@ from pyramid.router import Router
 import re
 import sys
 import time
-from tqdm import tqdm
 from typing import Any, Callable, List, Optional, Tuple, Type, Union
 from webtest.app import TestApp
 from dcicutils.common import OrchestratedApp
@@ -70,10 +69,9 @@ class StructuredDataSet:
                  order: Optional[List[str]] = None, prune: bool = True,
                  ref_lookup_strategy: Optional[Callable] = None,
                  ref_lookup_nocache: bool = False,
-                 progress: bool = False,
+                 progress: Optional[Callable] = None,
                  debug_sleep: Optional[str] = None) -> None:
-        progress = False
-        self._progress = progress
+        self._progress = progress if callable(progress) else None
         self._data = {}
         self._portal = Portal(portal, data=self._data, schemas=schemas,
                               ref_lookup_strategy=ref_lookup_strategy,
@@ -93,20 +91,15 @@ class StructuredDataSet:
                 self._debug_sleep = None
         self._load_file(file) if file else None
 
-    def _progress_add(self, amount: Union[int, Callable]) -> None:
-        if self._progress is not False and self._progress is not None:
-            if callable(amount):
-                amount = amount()
-            if not isinstance(amount, int):
-                return
-            if self._progress is True:
-                if amount > 0:
-                    self._progress = tqdm(total=amount)
-            elif isinstance(self._progress, tqdm):
-                if amount > 0:
-                    self._progress.total += amount
-                elif amount < 0:
-                    self._progress.update(-amount)
+    def _progress_update(self, nrows: Union[int, Callable],
+                         nrefs_resolved: Optional[int] = None,
+                         nrefs_unresolved: Optional[int] = None,
+                         nlookups: Optional[int] = None) -> None:
+        if self._progress:
+            if callable(nrows):
+                nrows = nrows()
+            if isinstance(nrows, int) and nrows != 0:
+                self._progress(nrows, nrefs_resolved, nrefs_unresolved, nlookups)
 
     @property
     def data(self) -> dict:
@@ -259,7 +252,8 @@ class StructuredDataSet:
                 for row in excel.sheet_reader(sheet_name):
                     nrows += 1
             return nrows
-        self._progress_add(calculate_total_rows_to_process)
+        if self._progress:
+            self._progress_update(calculate_total_rows_to_process)
         excel = Excel(file)  # Order the sheet names by any specified ordering (e.g. ala snovault.loadxl).
         order = {Schema.type_name(key): index for index, key in enumerate(self._order)} if self._order else {}
         for sheet_name in sorted(excel.sheet_names, key=lambda key: order.get(Schema.type_name(key), sys.maxsize)):
@@ -301,7 +295,8 @@ class StructuredDataSet:
                 if self._autoadd_properties:
                     self._add_properties(structured_row, self._autoadd_properties, schema)
             self._add(type_name, structured_row)
-            self._progress_add(-1)
+            if self._progress:
+                self._progress_update(-1, len(self._resolved_refs), len(self.ref_errors), self.ref_lookup_count)
         self._note_warning(reader.warnings, "reader")
         if schema:
             self._note_error(schema._unresolved_refs, "ref")

{dcicutils-8.8.0.1b13.dist-info → dcicutils-8.8.0.1b15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.8.0.1b13
+Version: 8.8.0.1b15
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.8.0.1b13.dist-info → dcicutils-8.8.0.1b15.dist-info}/RECORD

@@ -10,7 +10,7 @@ dcicutils/common.py,sha256=YE8Mt5-vaZWWz4uaChSVhqGFbFtW5QKtnIyOr4zG4vM,3955
 dcicutils/contribution_scripts.py,sha256=0k5Gw1TumcD5SAcXVkDd6-yvuMEw-jUp5Kfb7FJH6XQ,2015
 dcicutils/contribution_utils.py,sha256=vYLS1JUB3sKd24BUxZ29qUBqYeQBLK9cwo8x3k64uPg,25653
 dcicutils/creds_utils.py,sha256=xrLekD49Ex0GOpL9n7LlJA4gvNcY7txTVFOSYD7LvEU,11113
-dcicutils/data_readers.py,sha256=A48f7H7upkePpRKfq8bEv1EGM5Jjdes1pyjnreatjh0,6666
+dcicutils/data_readers.py,sha256=4VqjTbmm6YtyUQBuxG8J0cpns1fNtzkfo611RmI0Atk,7016
 dcicutils/data_utils.py,sha256=k2OxOlsx7AJ6jF-YNlMyGus_JqSUBe4_n1s65Mv1gQQ,3098
 dcicutils/datetime_utils.py,sha256=EODDGAngp1yh2ZlDIuI7tB74JBJucw2DljqfPknzK0Y,4666
 dcicutils/deployment_utils.py,sha256=rcNUFMe_tsrG4CHEtgBe41cZx4Pk4JqISPsjrJRMoEs,68891
@@ -62,15 +62,15 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
 dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
-dcicutils/structured_data.py,sha256=72nX3ijfc2QACqfyjWFlQIX-VX1B8RPRc4GA-Ly2G5E,52998
+dcicutils/structured_data.py,sha256=Xoyz-ig6N6LKkuYP_RNJ-ZTS1BVNX1DnpHe13q_aV5A,53005
 dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
 dcicutils/tmpfile_utils.py,sha256=n95XF8dZVbQRSXBZTGToXXfSs3JUVRyN6c3ZZ0nhAWI,1403
 dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
-dcicutils-8.8.0.1b13.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.8.0.1b13.dist-info/METADATA,sha256=_QuHO_XVPM0VCTHDGz89YJGi4cAf4ApoaPZCGJW8rgU,3357
-dcicutils-8.8.0.1b13.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.8.0.1b13.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
-dcicutils-8.8.0.1b13.dist-info/RECORD,,
+dcicutils-8.8.0.1b15.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.8.0.1b15.dist-info/METADATA,sha256=LYKE0PGwS9Rrc3wl6M8cIreHJj5sMYHSAm2wSSRbmQ8,3357
+dcicutils-8.8.0.1b15.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.8.0.1b15.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
+dcicutils-8.8.0.1b15.dist-info/RECORD,,