dcicutils 7.10.0.2b13__py3-none-any.whl → 7.11.0.1b9__py3-none-any.whl

dcicutils/sheet_utils.py

@@ -8,18 +8,19 @@ import json
 import openpyxl
 import os
 import re
+import subprocess
 import uuid
 import yaml
 
 from openpyxl.worksheet.worksheet import Worksheet
 from openpyxl.workbook.workbook import Workbook
-from tempfile import TemporaryFile
+from tempfile import TemporaryFile, TemporaryDirectory
 from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union
 from .common import AnyJsonData
 from .env_utils import public_env_name, EnvUtils
 from .ff_utils import get_schema
 from .lang_utils import conjoined_list, disjoined_list, maybe_pluralize, there_are
-from .misc_utils import ignored, PRINT, pad_to, JsonLinesReader, AbstractVirtualApp
+from .misc_utils import ignored, PRINT, pad_to, JsonLinesReader, AbstractVirtualApp, remove_suffix
 from .task_utils import pmap
 
 
@@ -1011,6 +1012,53 @@ class TsvItemManager(ItemManagerMixin, TsvManager):
     pass
 
 
+def _do_shell_command(command, cwd=None):
+    # This might need to be more elaborate, but hopefully it will do for now. -kmp 11-Sep-2023
+    subprocess.check_output(command, cwd=cwd)
+
+
+@contextlib.contextmanager
+def maybe_unpack(filename):  # Maybe move to another module
+    """
+    If necessary, unpack a file that is zipped and/or tarred, yielding the name of the file (unpacked or not).
+    """
+    unpackables = ['.tar.gz', '.tar', '.tgz', '.gz', '.zip']
+    ext = None
+    for unpackable in unpackables:
+        if filename.endswith(unpackable):
+            ext = unpackable
+            break
+    if not ext:
+        yield filename
+        return
+    if not os.path.exists(filename):
+        # We don't bother to raise this error if we're not planning to do any unpacking.
+        # The caller can decide if/when such errors are needed in that case.
+        # But if we are going to have to move bits around, they'll need to actually be there.
+        # -kmp 12-Sep-2023
+        raise ValueError(f"The file {filename!r} does not exist.")
+    target_base_part = remove_suffix(ext, os.path.basename(filename), required=True)
+    target_ext = '.tar.gz' if ext == '.tgz' else ext
+    with TemporaryDirectory() as temp_dir:
+        temp_base = os.path.join(temp_dir, target_base_part)
+        temp_filename = temp_base + target_ext
+        _do_shell_command(['cp', filename, temp_filename])
+        if temp_filename.endswith('.gz'):
+            _do_shell_command(['gunzip', temp_filename], cwd=temp_dir)
+            temp_filename = remove_suffix('.gz', temp_filename)
+        elif temp_filename.endswith(".zip"):
+            _do_shell_command(['unzip', temp_filename], cwd=temp_dir)
+            temp_filename = remove_suffix('.zip', temp_filename)
+        if temp_filename.endswith(".tar"):
+            _do_shell_command(['tar', '-xf', temp_filename], cwd=temp_dir)
+            tar_file = temp_filename
+            temp_filename = remove_suffix(".tar", temp_filename, required=True)
+            if not os.path.isdir(temp_filename):
+                raise Exception(f"{tar_file} didn't unpack to a dir: {temp_filename}")
+        # print(f"Unpacked {filename} to {temp_filename}")
+        yield temp_filename
+
+
 class TableSetManager(AbstractTableSetManager):
     """
     This class will open a .xlsx or .csv file and load its content in our standard format.
@@ -1031,8 +1079,10 @@ class TableSetManager(AbstractTableSetManager):
         """
         Given a filename and various options
         """
-        manager = cls.create_implementation_manager(filename=filename, tab_name=tab_name, escaping=escaping, **kwargs)
-        return manager.load_content()
+        with maybe_unpack(filename) as filename:
+            manager = cls.create_implementation_manager(filename=filename, tab_name=tab_name, escaping=escaping,
+                                                        **kwargs)
+            return manager.load_content()
 
 
 class ItemManager(AbstractTableSetManager):
@@ -1067,11 +1117,14 @@ class ItemManager(AbstractTableSetManager):
         :param portal_env: A portal to consult to find schemas (usually if calling from the outside of a portal).
         :param portal_vapp: A vapp to use (usually if calling from within a portal).
         """
-        manager = cls.create_implementation_manager(filename=filename, tab_name=tab_name, escaping=escaping,
-                                                    schemas=schemas, autoload_schemas=autoload_schemas,
-                                                    portal_env=portal_env, portal_vapp=portal_vapp,
-                                                    **kwargs)
-        return manager.load_content()
+
+        with maybe_unpack(filename) as filename:
+
+            manager = cls.create_implementation_manager(filename=filename, tab_name=tab_name, escaping=escaping,
+                                                        schemas=schemas, autoload_schemas=autoload_schemas,
+                                                        portal_env=portal_env, portal_vapp=portal_vapp,
+                                                        **kwargs)
+            return manager.load_content()
 
 
 load_table_set = TableSetManager.load

{dcicutils-7.10.0.2b13.dist-info → dcicutils-7.11.0.1b9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 7.10.0.2b13
+Version: 7.11.0.1b9
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT
@@ -21,7 +21,7 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Topic :: Database :: Database Engines/Servers
 Requires-Dist: PyJWT (>=2.6.0,<3.0.0)
-Requires-Dist: PyYAML (==5.3.1)
+Requires-Dist: PyYAML (>=5.1,<5.5)
 Requires-Dist: aws-requests-auth (>=0.4.2,<1)
 Requires-Dist: boto3 (>=1.17.39,<2.0.0)
 Requires-Dist: botocore (>=1.20.39,<2.0.0)

{dcicutils-7.10.0.2b13.dist-info → dcicutils-7.11.0.1b9.dist-info}/RECORD

@@ -43,14 +43,14 @@ dcicutils/redis_utils.py,sha256=VJ-7g8pOZqR1ZCtdcjKz3-6as2DMUcs1b1zG6wSprH4,6462
 dcicutils/s3_utils.py,sha256=a9eU3Flh8Asc8xPWLGP16A6UQ_FVwhoFQNqm4ZYgSQ4,28852
 dcicutils/scripts/publish_to_pypi.py,sha256=qmWyjrg5bNQNfpNKFTZdyMXpRmrECnRV9VmNQddUPQA,13576
 dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19745
-dcicutils/sheet_utils.py,sha256=YUJ6AVr3OlWGxo6dEpwHq4bCJSdA0SU_nfFVfDCQZhs,44594
+dcicutils/sheet_utils.py,sha256=bnnefjeTUL4ES7gtqThISXJKeli1AIFryu4h7Dt9dxw,47040
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
 dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
 dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
 dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
-dcicutils-7.10.0.2b13.dist-info/LICENSE.txt,sha256=t0_-jIjqxNnymZoNJe-OltRIuuF8qfhN0ATlHyrUJPk,1102
-dcicutils-7.10.0.2b13.dist-info/METADATA,sha256=ilmBL4J-rU8A74M_a8lmXduk84O4grnkg3U13DV93qU,3082
-dcicutils-7.10.0.2b13.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-7.10.0.2b13.dist-info/entry_points.txt,sha256=Z3vezbXsTpTIY4N2F33c5e-WDVQxgz_Vsk1oV_JBN7A,146
-dcicutils-7.10.0.2b13.dist-info/RECORD,,
+dcicutils-7.11.0.1b9.dist-info/LICENSE.txt,sha256=t0_-jIjqxNnymZoNJe-OltRIuuF8qfhN0ATlHyrUJPk,1102
+dcicutils-7.11.0.1b9.dist-info/METADATA,sha256=MER7N-gDAB5nz6YT51jT7aIu8_rHT2x65FBF5x3DN70,3084
+dcicutils-7.11.0.1b9.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-7.11.0.1b9.dist-info/entry_points.txt,sha256=Z3vezbXsTpTIY4N2F33c5e-WDVQxgz_Vsk1oV_JBN7A,146
+dcicutils-7.11.0.1b9.dist-info/RECORD,,