sertit 1.44.1.dev0__tar.gz → 1.44.1.dev1__tar.gz

{sertit-1.44.1.dev0/sertit.egg-info → sertit-1.44.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sertit
-Version: 1.44.1.dev0
+Version: 1.44.1.dev1
 Summary: ('SERTIT python library for generic tools',)
 Home-page: UNKNOWN
 Author: ICube-SERTIT

{sertit-1.44.1.dev0 → sertit-1.44.1.dev1}/sertit/__meta__.py

@@ -17,7 +17,7 @@
 """
 Sertit Utils library
 """
-__version__ = "1.44.1.dev0"
+__version__ = "1.44.1.dev1"
 __title__ = "sertit"
 __description__ = ("SERTIT python library for generic tools",)
 __author__ = "ICube-SERTIT"

{sertit-1.44.1.dev0 → sertit-1.44.1.dev1}/sertit/files.py

@@ -377,7 +377,9 @@ def get_archived_rio_path(
     return path.get_archived_rio_path(archive_path, file_regex, as_list)
 
 
-def read_archived_file(archive_path: AnyPathStrType, regex: str) -> bytes:
+def read_archived_file(
+    archive_path: AnyPathStrType, regex: str, file_list: list = None
+) -> bytes:
     """
     Read archived file (in bytes) from :code:`zip` or :code:`tar` archives.
 
@@ -386,6 +388,7 @@ def read_archived_file(archive_path: AnyPathStrType, regex: str) -> bytes:
     Args:
         archive_path (AnyPathStrType): Archive path
         regex (str): Regex (used by re) as it can be found in the getmembers() list
+        file_list (list): List of files contained in the archive. Optional, if not given it will be re-computed.
 
     Returns:
          bytes: Archived file in bytes
@@ -399,16 +402,19 @@ def read_archived_file(archive_path: AnyPathStrType, regex: str) -> bytes:
     try:
         if archive_path.suffix == ".tar":
             with tarfile.open(archive_path) as tar_ds:
-                tar_mb = tar_ds.getmembers()
-                name_list = [mb.name for mb in tar_mb]
-                band_name = list(filter(regex.match, name_list))[0]
-                tarinfo = [mb for mb in tar_mb if mb.name == band_name][0]
+                # file_list is not very useful for TAR files...
+                if file_list is None:
+                    tar_mb = tar_ds.getmembers()
+                    file_list = [mb.name for mb in tar_mb]
+                name = list(filter(regex.match, file_list))[0]
+                tarinfo = tar_ds.getmember(name)
                 file_str = tar_ds.extractfile(tarinfo).read()
         elif archive_path.suffix == ".zip":
             with zipfile.ZipFile(archive_path) as zip_ds:
-                name_list = [f.filename for f in zip_ds.filelist]
-                band_name = list(filter(regex.match, name_list))[0]
-                file_str = zip_ds.read(band_name)
+                if file_list is None:
+                    file_list = [f.filename for f in zip_ds.filelist]
+                name = list(filter(regex.match, file_list))[0]
+                file_str = zip_ds.read(name)
 
         elif archive_path.suffix == ".tar.gz":
             raise TypeError(
@@ -426,7 +432,9 @@ def read_archived_file(archive_path: AnyPathStrType, regex: str) -> bytes:
     return file_str
 
 
-def read_archived_xml(archive_path: AnyPathStrType, xml_regex: str) -> etree._Element:
+def read_archived_xml(
+    archive_path: AnyPathStrType, xml_regex: str, file_list: list = None
+) -> etree._Element:
     """
     Read archived XML from :code:`zip` or :code:`tar` archives.
 
@@ -435,6 +443,7 @@ def read_archived_xml(archive_path: AnyPathStrType, xml_regex: str) -> etree._El
     Args:
         archive_path (AnyPathStrType): Archive path
         xml_regex (str): XML regex (used by re) as it can be found in the getmembers() list
+        file_list (list): List of files contained in the archive. Optional, if not given it will be re-computed.
 
     Returns:
          etree._Element: XML file
@@ -445,12 +454,14 @@ def read_archived_xml(archive_path: AnyPathStrType, xml_regex: str) -> etree._El
         >>> read_archived_xml(arch_path, file_regex)
         <Element LANDSAT_METADATA_FILE at 0x1c90007f8c8>
     """
-    xml_bytes = read_archived_file(archive_path, xml_regex)
+    xml_bytes = read_archived_file(archive_path, xml_regex, file_list=file_list)
 
     return etree.fromstring(xml_bytes)
 
 
-def read_archived_html(archive_path: AnyPathStrType, regex: str) -> html.HtmlElement:
+def read_archived_html(
+    archive_path: AnyPathStrType, regex: str, file_list: list = None
+) -> html.HtmlElement:
     """
     Read archived HTML from :code:`zip` or :code:`tar` archives.
 
@@ -459,6 +470,7 @@ def read_archived_html(archive_path: AnyPathStrType, regex: str) -> html.HtmlEle
     Args:
         archive_path (AnyPathStrType): Archive path
         regex (str): HTML regex (used by re) as it can be found in the getmembers() list
+        file_list (list): List of files contained in the archive. Optional, if not given it will be re-computed.
 
     Returns:
          html._Element: HTML file
@@ -469,7 +481,7 @@ def read_archived_html(archive_path: AnyPathStrType, regex: str) -> html.HtmlEle
         >>> read_archived_html(arch_path, file_regex)
         <Element html at 0x1c90007f8c8>
     """
-    html_bytes = read_archived_file(archive_path, regex)
+    html_bytes = read_archived_file(archive_path, regex, file_list=file_list)
 
     return html.fromstring(html_bytes)
 

{sertit-1.44.1.dev0 → sertit-1.44.1.dev1}/sertit/path.py

@@ -188,6 +188,7 @@ def get_archived_path(
     file_regex: str,
     as_list: bool = False,
     case_sensitive: bool = False,
+    file_list: list = None,
 ) -> Union[list, AnyPathType]:
     """
     Get archived file path from inside the archive.
@@ -202,6 +203,7 @@ def get_archived_path(
         file_regex (str): File regex (used by re) as it can be found in the getmembers() list
         as_list (bool): If true, returns a list (including all found files). If false, returns only the first match
         case_sensitive (bool): If true, the regex is case-sensitive.
+        file_list (list): List of files to get archived from. Optional, if not given it will be re-computed.
 
     Returns:
         Union[list, str]: Path from inside the zipfile
@@ -214,7 +216,10 @@ def get_archived_path(
     """
     # Get file list
     archive_path = AnyPath(archive_path)
-    file_list = get_archived_file_list(archive_path)
+
+    # Offer the ability to give the file list directly, as this operation is expensive when done with large archives stored on the cloud
+    if file_list is None:
+        file_list = get_archived_file_list(archive_path)
 
     # Search for file
     regex = (
@@ -236,7 +241,10 @@ def get_archived_path(
 
 
 def get_archived_rio_path(
-    archive_path: AnyPathStrType, file_regex: str, as_list: bool = False
+    archive_path: AnyPathStrType,
+    file_regex: str,
+    as_list: bool = False,
+    file_list: list = None,
 ) -> Union[list, AnyPathType]:
     """
     Get archived file path from inside the archive, to be read with rasterio:
@@ -260,6 +268,7 @@ def get_archived_rio_path(
         archive_path (AnyPathStrType): Archive path
         file_regex (str): File regex (used by re) as it can be found in the getmembers() list
         as_list (bool): If true, returns a list (including all found files). If false, returns only the first match
+        file_list (list): List of files contained in the archive. Optional, if not given it will be re-computed.
 
     Returns:
         Union[list, str]: Band path that can be read by rasterio
@@ -283,7 +292,9 @@ def get_archived_rio_path(
         raise TypeError("Only .zip and .tar files can be read from inside its archive.")
 
     # Search for file
-    archived_band_paths = get_archived_path(archive_path, file_regex, as_list=True)
+    archived_band_paths = get_archived_path(
+        archive_path, file_regex, as_list=True, file_list=file_list
+    )
 
     # Convert to rio path
     if is_cloud_path(archive_path):

{sertit-1.44.1.dev0 → sertit-1.44.1.dev1}/sertit/vectors.py

@@ -422,7 +422,8 @@ def read(
         archive_regex (str): [Archive only] Regex for the wanted vector inside the archive
         window (Any): Anything that can be returned as a bbox (i.e. path, gpd.GeoPandas, Iterable, ...).
             In case of an iterable, assumption is made it corresponds to geographic bounds. Mimics :code:`rasters.read(..., window=)`. If given, :code:`bbox` is ignored.
-        **kwargs: Additional arguments used in gpd.read_file
+        **kwargs: Additional arguments used in gpd.read_file.
+            You can also give :code:`file_list`, the list of files of the archive to get the vector from, as this operation is expensive when done with large archives stored on the cloud.
 
     Returns:
         gpd.GeoDataFrame: Read vector as a GeoDataFrame
@@ -477,7 +478,9 @@ def read(
         # Manage archive case
         if vector_path.suffix in [".tar", ".zip"]:
             prefix = vector_path.suffix[-3:]
-            file_list = path.get_archived_file_list(vector_path)
+            file_list = kwargs.pop(
+                "file_list", path.get_archived_file_list(vector_path)
+            )
 
             try:
                 regex = re.compile(archive_regex)

{sertit-1.44.1.dev0 → sertit-1.44.1.dev1/sertit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sertit
-Version: 1.44.1.dev0
+Version: 1.44.1.dev1
 Summary: ('SERTIT python library for generic tools',)
 Home-page: UNKNOWN
 Author: ICube-SERTIT