plexus-python-common 1.0.31__tar.gz → 1.0.33__tar.gz

{plexus_python_common-1.0.31 → plexus_python_common-1.0.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: plexus-python-common
-Version: 1.0.31
+Version: 1.0.33
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13

{plexus_python_common-1.0.31 → plexus_python_common-1.0.33}/src/plexus/common/utils/s3utils.py

@@ -37,10 +37,14 @@ __all__ = [
     "s3_pull_text",
     "s3_push_text",
     "S3TransferCallbackClient",
+    "ArchiveMemberChunk",
     "s3_make_progress_callback",
     "s3_make_progressed_client",
+    "s3_archive_member_tree",
     "s3_archive_listfile",
     "s3_archive_open_member",
+    "s3_archive_use_ranged_requests",
+    "s3_archive_use_chunked_reads",
     "s3_archive_open_members",
 ]
 
@@ -88,7 +92,12 @@ def s3_make_client(
             client.close()
 
 
-def s3_list_objects(client: S3Client, bucket: str, prefix: str, limit: int = None) -> Generator[S3ObjectMeta]:
+def s3_list_objects(
+    client: S3Client,
+    bucket: str,
+    prefix: str,
+    limit: int = None,
+) -> Generator[S3ObjectMeta, None, None]:
     """
     Lists all objects from the given S3 ``bucket`` and ``prefix``.
 
@@ -130,7 +139,7 @@ def s3_listfile(
     include_patterns: list[str] | None = None,
     exclude_patterns: list[str] | None = None,
     depth: int = 0,
-) -> Generator[S3ObjectMeta]:
+) -> Generator[S3ObjectMeta, None, None]:
     """
     Lists all objects from the given S3 ``bucket`` and ``prefix``, filtered by patterns and directory depth.
 
@@ -426,7 +435,7 @@ def s3_make_progressed_client(
     secret_access_key: str = None,
     region_name: str = None,
     endpoint_url: str = None,
-) -> Generator[S3Client]:
+) -> Generator[S3Client, None, None]:
     """
     Creates an S3 client with progress callback as a context manager for safe resource handling.
 
@@ -483,9 +492,65 @@ def s3_options_from_s3_client(client: S3Client) -> dict[str, Any]:
     return s3_options
 
 
+type ArchiveMemberTree = dict[str, tuple[zipfile.ZipInfo, ArchiveMemberTree | None]]
+
+
+def s3_archive_member_tree(
+    client: S3Client,
+    bucket: str,
+    key: str,
+) -> ArchiveMemberTree:
+    """
+    Builds a tree structure of members in a ZIP archive stored in S3 for efficient lookup.
+    Directories have ZipInfo and a nested dict; files have ZipInfo and None.
+    Directory members are recognized by names ending with a trailing slash ("/").
+
+    Example:
+    {
+      "dir1/": (ZipInfo, {
+          "file1.txt": (ZipInfo, None),
+          "subdir/": (ZipInfo, {
+              "file2.txt": (ZipInfo, None)
+          })
+      }),
+      "file3.txt": (ZipInfo, None)
+    }
+
+    :param client: An instance of ``S3Client``.
+    :param bucket: Bucket name.
+    :param key: Object key of the ZIP archive.
+    :return: A tree structure of members in the ZIP archive.
+    """
+    s3_options = s3_options_from_s3_client(client)
+
+    archive_url = f"s3://{bucket}/{key}"
+
+    with fsspec.open(archive_url, "rb", s3=s3_options) as s3_fh, zipfile.ZipFile(s3_fh) as archive:
+        member_zip_infos = archive.infolist()
+
+    root_member_tree: ArchiveMemberTree = {}
+
+    def build_member_tree(info: zipfile.ZipInfo):
+        *parts, last_part = info.filename.rstrip("/").split("/")
+        current_member_tree = root_member_tree
+        for part in parts:
+            _, current_member_tree = current_member_tree.setdefault(part + "/", (None, {}))
+        if info.is_dir():
+            current_member_tree[last_part + "/"] = info, {}
+        else:
+            current_member_tree[last_part] = info, None
+
+    # Sort by filename to ensure directories are created before their contents
+    for info in sorted(member_zip_infos, key=lambda x: x.filename):
+        build_member_tree(info)
+
+    return root_member_tree
+
+
 def s3_archive_listfile(
     client: S3Client,
-    archive_url: str,
+    bucket: str,
+    key: str,
     members: list[str] | None = None,
 ) -> tuple[int, list[zipfile.ZipInfo], list[str]]:
     """
@@ -494,14 +559,15 @@ def s3_archive_listfile(
     under that directory will be included in the results.
 
     Example usage:
-    >>> archive_size, member_infos, missed_members = s3_archive_listfile(client, archive_url, members=["file1.txt", "dir1/"])
-    >>> for info in member_infos:
+    >>> archive_size, member_zip_infos, missed_members = s3_archive_listfile(client, bucket, key, members=["file1.txt", "dir1/"])
+    >>> for info in member_zip_infos:
     ...     print(info.filename, info.file_size)
     >>> if missed_members:
     ...     print("Members not found:", missed_members)
 
     :param client: An instance of ``S3Client``.
-    :param archive_url: The URL of the ZIP archive in S3.
+    :param bucket: Bucket name.
+    :param key: Object key of the ZIP archive.
     :param members: Optional list of member names to filter; if ``None``, all members are returned.
     :return: A tuple containing:
              - The size of the archive in bytes.
@@ -510,82 +576,49 @@ def s3_archive_listfile(
     """
     s3_options = s3_options_from_s3_client(client)
 
-    storage_opts = fsspec.utils.infer_storage_options(archive_url)
-    protocol = storage_opts.get("protocol")
-    if protocol != "s3":
-        raise ValueError(f"unsupported protocol '{protocol}', only 's3' is supported")
+    archive_url = f"s3://{bucket}/{key}"
 
     fs = fsspec.filesystem("s3", **s3_options)
     archive_size = fs.size(archive_url)
 
-    with fsspec.open(archive_url, "rb", s3=s3_options) as s3_fh, zipfile.ZipFile(s3_fh) as archive:
-        member_zip_infos = archive.infolist()
-
-    if members is None:
-        return archive_size, [info for info in member_zip_infos if not info.is_dir()], []
-
-    # Build a tree structure of members for efficient lookup
-    # Directories have ZipInfo and a nested dict; files have ZipInfo and None
-    # Directory members are recognized by names ending with a trailing slash ("/")
-    # Example:
-    # {
-    #   "dir1/": (ZipInfo, {
-    #       "file1.txt": (ZipInfo, None),
-    #       "subdir/": (ZipInfo, {
-    #           "file2.txt": (ZipInfo, None)
-    #       })
-    #   }),
-    #   "file3.txt": (ZipInfo, None)
-    # }
-    members_tree: dict[str, tuple[zipfile.ZipInfo, dict | None]] = {}
-
-    def build_members_tree(info: zipfile.ZipInfo):
-        *parts, last_part = info.filename.rstrip("/").split("/")
-        current = members_tree
-        for part in parts:
-            _, current = current.setdefault(part + "/", (None, {}))
-        if info.is_dir():
-            current[last_part + "/"] = info, {}
-        else:
-            current[last_part] = info, None
-
-    # Sort by filename to ensure directories are created before their contents
-    for info in sorted(member_zip_infos, key=lambda x: x.filename):
-        build_members_tree(info)
+    root_member_tree = s3_archive_member_tree(client, bucket, key)
 
     def search_members_tree(member: str) -> tuple[zipfile.ZipInfo | None, dict | None]:
         *parts, last_part = member.rstrip("/").split("/")
-        current = members_tree
+        current_member_tree = root_member_tree
         for part in parts:
-            _, current = current.get(part + "/", (None, None))
-            if current is None:
+            _, current_member_tree = current_member_tree.get(part + "/", (None, None))
+            if current_member_tree is None:
                 return None, None
         if member.endswith("/"):  # Directory member recognized by trailing slash
-            return current.get(last_part + "/", (None, None))
+            return current_member_tree.get(last_part + "/", (None, None))
         else:
-            return current.get(last_part, (None, None))
+            return current_member_tree.get(last_part, (None, None))
 
-    def collect_member_infos(tree: dict[str, tuple[zipfile.ZipInfo, dict | None]]) -> Generator[zipfile.ZipInfo]:
-        for _, (member_info, member_tree) in tree.items():
-            if member_info is None:
+    def collect_member_zip_infos(tree: ArchiveMemberTree) -> Generator[zipfile.ZipInfo, None, None]:
+        for member_zip_info, member_tree in tree.values():
+            if member_zip_info is None:
                 continue
-            if member_info.is_dir():
-                yield from collect_member_infos(member_tree)
+            if member_zip_info.is_dir():
+                yield from collect_member_zip_infos(member_tree)
             else:
-                yield member_info
+                yield member_zip_info
+
+    if members is None:
+        return archive_size, list(collect_member_zip_infos(root_member_tree)), []
 
     included_member_zip_infos = []
     missed_members = []
 
     for member in members:
-        member_info, member_tree = search_members_tree(member)
-        if member_info is None:
+        member_zip_info, member_tree = search_members_tree(member)
+        if member_zip_info is None:
             missed_members.append(member)
             continue
-        if not member_info.is_dir():
-            included_member_zip_infos.append(member_info)
+        if not member_zip_info.is_dir():
+            included_member_zip_infos.append(member_zip_info)
         else:
-            included_member_zip_infos.extend(collect_member_infos(member_tree or {}))
+            included_member_zip_infos.extend(collect_member_zip_infos(member_tree or {}))
 
     return archive_size, included_member_zip_infos, missed_members
 
@@ -593,7 +626,8 @@ def s3_archive_listfile(
 if typing.TYPE_CHECKING:
     def s3_archive_open_member(
         client: S3Client,
-        archive_url: str,
+        bucket: str,
+        key: str,
         member: str,
         mode: Literal["r", "rb"] = "r",
     ) -> contextlib.AbstractContextManager[typing.IO]: ...
@@ -602,7 +636,8 @@ if typing.TYPE_CHECKING:
 @contextlib.contextmanager
 def s3_archive_open_member(
     client: S3Client,
-    archive_url: str,
+    bucket: str,
+    key: str,
     member: str,
     mode: Literal["r", "rb"] = "r",
 ) -> Generator[typing.IO, None, None]:
@@ -610,7 +645,8 @@ def s3_archive_open_member(
     Opens a specific member file from a ZIP archive stored in S3.
 
     :param client: An instance of ``S3Client``.
-    :param archive_url: The URL of the ZIP archive in S3.
+    :param bucket: Bucket name.
+    :param key: Object key of the ZIP archive.
     :param member: The member file name to open from the archive.
     :param mode: File mode for opening the member ("r" for text, "rb" for binary).
 
@@ -621,38 +657,101 @@ def s3_archive_open_member(
 
     s3_options = s3_options_from_s3_client(client)
 
-    storage_opts = fsspec.utils.infer_storage_options(archive_url)
-    protocol = storage_opts.get("protocol")
-    if protocol != "s3":
-        raise ValueError(f"unsupported protocol '{protocol}', only 's3' is supported")
-
-    with fsspec.open(f"zip://{member}::{archive_url}", mode, s3=s3_options) as s3_fh:
+    with fsspec.open(f"zip://{member}::s3://{bucket}/{key}", mode, s3=s3_options) as s3_fh:
         yield s3_fh
 
 
-ZIPFILE_HEADER_MIN_SIZE = 30
-ZIPFILE_HEADER_FN_LEN_OFFSET = 26
-ZIPFILE_HEADER_EX_LEN_OFFSET = 28
+ZIP_CENTRAL_DIR_ESTIMATED_SIZE = 64 * 1024
+ZIP_INFO_HDR_MIN_SIZE = 30
+ZIP_INFO_HDR_ESTIMATED_SIZE = 128
+ZIP_INFO_HDR_FN_LEN_OFFSET = 26
+ZIP_INFO_HDR_EX_LEN_OFFSET = 28
+
+
+@dataclasses.dataclass(frozen=True)
+class ArchiveMemberChunk(object):
+    name: str
+    header_offset: int
+    compress_size: int
+    compress_type: int
+    header_overhead: int = ZIP_INFO_HDR_ESTIMATED_SIZE
+
+    @property
+    def begin(self) -> int:
+        return self.header_offset
+
+    @property
+    def end(self) -> int:
+        return self.header_offset + self.header_overhead + self.compress_size
+
+
+def s3_archive_use_ranged_requests(
+    threshold: float = 0.5,
+    central_directory_overhead: int = 64 * ZIP_CENTRAL_DIR_ESTIMATED_SIZE,
+    zip_info_header_overhead: int = ZIP_INFO_HDR_ESTIMATED_SIZE,
+) -> Callable[[int, list[zipfile.ZipInfo]], bool]:
+    """
+    Decide whether to use ranged requests for accessing members of a ZIP archive in S3
+    based on estimated transfer size ratio.
+
+    :param threshold: If (estimated ranged transfer bytes / archive bytes) <= threshold, use ranged per-member access;
+                      otherwise download the whole archive.
+    :param central_directory_overhead: Estimated overhead size for the central directory in bytes.
+    :param zip_info_header_overhead: Estimated overhead size for each member's ``ZipInfo`` header in bytes.
+    :return: A callable that takes (``archive_size``, ``member_zip_infos``) and returns a boolean
+             indicating whether to use ranged requests.
+    """
+
+    if zip_info_header_overhead < ZIP_INFO_HDR_MIN_SIZE:
+        raise ValueError(f"zip_info_header_overhead must be at least {ZIP_INFO_HDR_MIN_SIZE} bytes")
+
+    def use_ranged_requests(
+        archive_size: int,
+        member_zip_infos: list[zipfile.ZipInfo],
+    ) -> bool:
+        estimated_ranged_total_size = central_directory_overhead + sum(info.compress_size + zip_info_header_overhead
+                                                                       for info in member_zip_infos)
+
+        # Avoid division by zero; prefer ranged if archive size is zero (degenerate case)
+        return (estimated_ranged_total_size / archive_size) <= threshold if archive_size > 0 else True
+
+    return use_ranged_requests
+
+
+def s3_archive_use_chunked_reads(
+    zip_info_header_overhead: int = ZIP_INFO_HDR_ESTIMATED_SIZE,
+) -> Callable[[zipfile.ZipInfo], ArchiveMemberChunk]:
+    """
+    Map each ``ZipInfo`` to an ``ArchiveMemberChunk`` for grouping adjacent members into single ranged reads.
 
+    :param zip_info_header_overhead: Estimated overhead size for each member's ``ZipInfo`` header in bytes.
+    :return: A callable that takes a ``ZipInfo`` and returns an ``ArchiveMemberChunk``.
+    """
 
-def zipfile_hdr_fn_len_slice(offset: int) -> slice:
-    return slice(offset + ZIPFILE_HEADER_FN_LEN_OFFSET, offset + ZIPFILE_HEADER_FN_LEN_OFFSET + 2)
+    if zip_info_header_overhead < ZIP_INFO_HDR_MIN_SIZE:
+        raise ValueError(f"zip_info_header_overhead must be at least {ZIP_INFO_HDR_MIN_SIZE} bytes")
 
+    def use_chunked_reads(
+        zip_info: zipfile.ZipInfo,
+    ) -> ArchiveMemberChunk:
+        return ArchiveMemberChunk(zip_info.filename,
+                                  zip_info.header_offset,
+                                  zip_info.compress_size,
+                                  zip_info.compress_type,
+                                  header_overhead=zip_info_header_overhead)
 
-def zipfile_hdr_ex_len_slice(offset: int) -> slice:
-    return slice(offset + ZIPFILE_HEADER_EX_LEN_OFFSET, offset + ZIPFILE_HEADER_EX_LEN_OFFSET + 2)
+    return use_chunked_reads
 
 
 def s3_archive_open_members(
     client: S3Client,
-    archive_url: str,
+    bucket: str,
+    key: str,
     members: list[str] | None = None,
     mode: Literal["r", "rb"] = "r",
     *,
-    threshold: float = 0.5,
-    central_directory_overhead: int = 64 * 1024,
-    member_header_overhead: int = 128,
-    use_chunked_reads: bool = False,
+    use_ranged_requests: bool | Callable[[int, list[zipfile.ZipInfo]], bool] = True,
+    use_chunked_reads: bool | Callable[[zipfile.ZipInfo], ArchiveMemberChunk] = False,
 ) -> Generator[tuple[str, Callable[[], typing.IO]], None, None]:
     """
     Choose the best transfer strategy (ranged requests per-member vs full archive transfer)
@@ -665,71 +764,72 @@ def s3_archive_open_members(
 
     Example usage:
 
-    >>> for member, opener in s3_archive_open_members(client, archive_url, members):
+    >>> for member, opener in s3_archive_open_members(client, bucket, key, members):
     ...     with opener() as fh:
     ...         data = fh.read()
 
     Incorrect usage that may lead to errors due to temporary file cleanup:
 
     >>> openers = []
-    >>> for member, opener in s3_archive_open_members(client, archive_url, members):
+    >>> for member, opener in s3_archive_open_members(client, bucket, key, members):
     ...     openers.append((member, opener))
     >>> for member, opener in openers:
     ...     with opener() as fh:  # May fail if temporary files have been cleaned up
     ...         data = fh.read()
 
     :param client: An instance of ``S3Client``.
-    :param archive_url: S3 URL to the ZIP archive.
+    :param bucket: Bucket name.
+    :param key: Object key of the ZIP archive.
     :param members: List of member names to stream.
     :param mode: File mode for opening members ("r" for text, "rb" for binary).
-    :param threshold: If (estimated ranged transfer bytes / archive bytes) <= threshold,
-                      use ranged per-member access; otherwise download the whole archive.
-    :param central_directory_overhead: Passed to s3_estimate_archive_ranged_requests.
-    :param member_header_overhead: Passed to s3_estimate_archive_ranged_requests.
-    :param use_chunked_reads: If ``True`` and ranged access is chosen, group adjacent members into single ranged reads.
+    :param use_ranged_requests: If ``True``, always use ranged requests to access the archive; if callable, use it as
+                                custom logic to decide based on the archive size and member infos; if ``False``, always
+                                download the whole archive.
+    :param use_chunked_reads: If ``True``, group adjacent members into single ranged reads using default ``ZipInfo``
+                              to ``ArchiveMemberChunk`` mapping; if callable, use it as custom mapping from
+                              ``ZipInfo`` to ``ArchiveMemberChunk`` for grouping adjacent members; if ``False``,
+                              read each member with individual ranged requests. If ``use_ranged_requests`` is ``False``,
+                              this parameter is ignored.
 
     :return: An iterable of callables that return file-like objects for each requested member.
     """
     if mode not in ("r", "rb"):
         raise ValueError("mode must be either 'r' or 'rb'")
-    if member_header_overhead < ZIPFILE_HEADER_MIN_SIZE:
-        raise ValueError(f"member_header_overhead must be at least {ZIPFILE_HEADER_MIN_SIZE} bytes")
 
     s3_options = s3_options_from_s3_client(client)
 
-    archive_size, member_zip_infos, missed_members = s3_archive_listfile(client, archive_url, members)
+    archive_size, member_zip_infos, missed_members = s3_archive_listfile(client, bucket, key, members)
+    if missed_members:
+        raise FileNotFoundError(f"Archive members not found: {', '.join(missed_members)}")
 
-    if len(missed_members) > 0:
-        raise FileNotFoundError(f"members not found in archive '{missed_members}'")
+    if callable(use_ranged_requests):
+        use_ranged_requests = use_ranged_requests(archive_size, member_zip_infos)
 
-    estimated_ranged_total_size = central_directory_overhead + sum(info.compress_size + member_header_overhead
-                                                                   for info in member_zip_infos)
+    if use_ranged_requests and not use_chunked_reads:
+        for info in member_zip_infos:
+            opener = functools.partial(s3_archive_open_member, client, bucket, key, info.filename, mode)
+            yield info.filename, opener
+        return
 
-    # Avoid division by zero; prefer ranged if archive size is zero (degenerate case)
-    use_ranged = (estimated_ranged_total_size / archive_size) <= threshold if archive_size > 0 else True
+    archive_url = f"s3://{bucket}/{key}"
 
-    @dataclasses.dataclass(frozen=True)
-    class MemberChunk(object):
-        name: str
-        header_offset: int
-        compress_size: int
-        compress_type: int
+    if use_ranged_requests and use_chunked_reads:
 
-        @property
-        def end(self) -> int:
-            return self.header_offset + self.compress_size + member_header_overhead
+        fn_len_slice = lambda index: slice(index + ZIP_INFO_HDR_FN_LEN_OFFSET, index + ZIP_INFO_HDR_FN_LEN_OFFSET + 2)
+        ex_len_slice = lambda index: slice(index + ZIP_INFO_HDR_EX_LEN_OFFSET, index + ZIP_INFO_HDR_EX_LEN_OFFSET + 2)
 
-    if use_ranged and use_chunked_reads:
         # Open archive once to read central directory and gather ZipInfo for requested members.
         # We will group adjacent members (by local header offsets) and issue one ranged read per group,
         # then extract each member from the group's bytes to avoid many small ranged requests.
         with fsspec.open(archive_url, "rb", s3=s3_options) as s3_fh, zipfile.ZipFile(s3_fh) as archive:
+            if callable(use_chunked_reads):
+                chunks = [use_chunked_reads(info) for info in member_zip_infos]
+            else:
+                chunks = [ArchiveMemberChunk(info.filename, info.header_offset, info.compress_size, info.compress_type)
+                          for info in member_zip_infos]
 
-            chunks = [MemberChunk(info.filename, info.header_offset, info.compress_size, info.compress_type)
-                      for info in member_zip_infos]
-
-            chunks_groups = chunk_between(sorted(chunks, key=lambda x: x.header_offset),
-                                          chunk_func=lambda x, y: y.header_offset > x.end + member_header_overhead)
+            chunks_groups = chunk_between(sorted(chunks, key=lambda x: x.begin),
+                                          chunk_func=lambda x, y: y.begin > x.end)
 
             # For each group, create openers for members inside the group.
             for group in chunks_groups:
@@ -740,18 +840,18 @@ def s3_archive_open_members(
                 s3_fh.seek(group_offset)
                 group_bytes = s3_fh.read(group_size)
 
-                def make_opener(chunk: MemberChunk) -> Callable[[], typing.IO]:
+                def make_opener(chunk: ArchiveMemberChunk) -> Callable[[], typing.IO]:
 
                     def opener() -> typing.IO:
                         index = chunk.header_offset - group_offset
 
-                        if index + ZIPFILE_HEADER_MIN_SIZE > len(group_bytes):
+                        if index + ZIP_INFO_HDR_MIN_SIZE > len(group_bytes):
                             raise IOError("unexpected short read of member header")
 
-                        fn_len = int.from_bytes(group_bytes[zipfile_hdr_fn_len_slice(index)], "little")
-                        ex_len = int.from_bytes(group_bytes[zipfile_hdr_ex_len_slice(index)], "little")
+                        fn_len = int.from_bytes(group_bytes[fn_len_slice(index)], "little")
+                        ex_len = int.from_bytes(group_bytes[ex_len_slice(index)], "little")
 
-                        raw_data_begin = index + ZIPFILE_HEADER_MIN_SIZE + fn_len + ex_len
+                        raw_data_begin = index + ZIP_INFO_HDR_MIN_SIZE + fn_len + ex_len
                         raw_data_end = raw_data_begin + chunk.compress_size
 
                         if raw_data_end > len(group_bytes):
@@ -776,12 +876,6 @@ def s3_archive_open_members(
                 yield from ((chunk.name, make_opener(chunk)) for chunk in group)
         return
 
-    if use_ranged:
-        for info in member_zip_infos:
-            opener = functools.partial(s3_archive_open_member, client, archive_url, info.filename, mode)
-            yield info.filename, opener
-        return
-
     # Download full archive once and serve members from it (read member bytes into memory)
     with fsspec.open(archive_url, "rb", s3=s3_options) as s3_fh, tempfile.TemporaryFile() as temp_fh:
         shutil.copyfileobj(s3_fh, temp_fh)

{plexus_python_common-1.0.31 → plexus_python_common-1.0.33}/src/plexus/common/utils/strutils.py

@@ -22,6 +22,10 @@ __all__ = [
     "strict_relpath_parser",
     "strict_abspath_pattern",
     "strict_abspath_parser",
+    "strict_archive_relpath_pattern",
+    "strict_archive_relpath_parser",
+    "strict_archive_abspath_pattern",
+    "strict_archive_abspath_parser",
     "email_address_pattern",
     "email_address_parser",
     "semver_pattern",
@@ -82,6 +86,7 @@ period_token: pp.ParserElement = pp.Char(".")
 colon_token: pp.ParserElement = pp.Char(":")
 slash_token: pp.ParserElement = pp.Char("/")
 plus_token: pp.ParserElement = pp.Char("+")
+sharp_token: pp.ParserElement = pp.Char("#")
 
 lower_regexp: re.Pattern[str] = re.compile(r"[a-z]")
 upper_regexp: re.Pattern[str] = re.compile(r"[A-Z]")
@@ -155,6 +160,10 @@ strict_relpath_regexp: re.Pattern[str] = re.compile(
     rf"(?!.*(^|/)\.+($|/))(?:{strict_chars_regexp.pattern}/)*(?:{strict_chars_regexp.pattern})?")
 strict_abspath_regexp: re.Pattern[str] = re.compile(
     rf"(?!.*(^|/)\.+($|/))/(?:{strict_chars_regexp.pattern}/)*(?:{strict_chars_regexp.pattern})?")
+strict_archive_relpath_regexp: re.Pattern[str] = re.compile(
+    rf"(?!.*(^|/)\.+($|/))(?:{strict_chars_regexp.pattern}/)*({strict_chars_regexp.pattern})(?:#({strict_relpath_regexp.pattern}))?")
+strict_archive_abspath_regexp: re.Pattern[str] = re.compile(
+    rf"(?!.*(^|/)\.+($|/))/(?:{strict_chars_regexp.pattern}/)*({strict_chars_regexp.pattern})(?:#({strict_relpath_regexp.pattern}))?")
 
 strict_path_chars_element = strict_chars_element.copy()
 strict_path_chars_element.add_condition(token_reparse(period_token[1, ...], negate=True),
@@ -164,6 +173,12 @@ strict_relpath_element: pp.ParserElement = pp.Combine(
     (strict_path_chars_element + slash_token)[...] + strict_path_chars_element[0, 1])
 strict_abspath_element: pp.ParserElement = pp.Combine(
     slash_token + (strict_path_chars_element + slash_token)[...] + strict_path_chars_element[0, 1])
+strict_archive_relpath_element: pp.ParserElement = pp.Combine(
+    (strict_path_chars_element + slash_token)[...] + strict_path_chars_element +
+    (sharp_token + strict_relpath_element)[0, 1])
+strict_archive_abspath_element: pp.ParserElement = pp.Combine(
+    slash_token + (strict_path_chars_element + slash_token)[...] + strict_path_chars_element +
+    (sharp_token + strict_relpath_element)[0, 1])
 
 email_address_regexp: re.Pattern[str] = re.compile(
     rf"(({lower_digit_regexp.pattern}|[_-])+)(?:\.({lower_digit_regexp.pattern}|[_-])+)*@(?:{kebab_case_regexp.pattern}\.)+({lower_digit_regexp.pattern}{{2,63}})")
@@ -206,6 +221,10 @@ strict_relpath_pattern = make_string_pattern(strict_relpath_regexp)
 strict_relpath_parser = make_string_parser(strict_relpath_element)
 strict_abspath_pattern = make_string_pattern(strict_abspath_regexp)
 strict_abspath_parser = make_string_parser(strict_abspath_element)
+strict_archive_relpath_pattern = make_string_pattern(strict_archive_relpath_regexp)
+strict_archive_relpath_parser = make_string_parser(strict_archive_relpath_element)
+strict_archive_abspath_pattern = make_string_pattern(strict_archive_abspath_regexp)
+strict_archive_abspath_parser = make_string_parser(strict_archive_abspath_element)
 
 email_address_pattern = make_string_pattern(email_address_regexp)
 email_address_parser = make_string_parser(email_address_element)

{plexus_python_common-1.0.31 → plexus_python_common-1.0.33}/src/plexus_python_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: plexus-python-common
-Version: 1.0.31
+Version: 1.0.33
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13

{plexus_python_common-1.0.31 → plexus_python_common-1.0.33}/test/plexus_tests/common/utils/s3utils_test.py

@@ -488,21 +488,21 @@ class S3UtilsTest(unittest.TestCase):
             local_root = os.path.join(resources_directory, "unittest", "s3utils")
             local_members = [os.path.relpath(file_path, local_root) for file_path in listfile(local_root)]
 
-            for archive_url, members in itertools.product(
-                ["s3://dummy-bucket/s3utils_archive/archive.uncompressed.zip",
-                 "s3://dummy-bucket/s3utils_archive/archive.compressed.zip"],
+            for archive_key, members in itertools.product(
+                ["s3utils_archive/archive.uncompressed.zip", "s3utils_archive/archive.compressed.zip"],
                 [local_members, None],
             ):
-                archive_size, member_zip_infos, missed_members = s3_archive_listfile(client, archive_url, members)
+                archive_size, member_zip_infos, missed_members = s3_archive_listfile(client,
+                                                                                     "dummy-bucket",
+                                                                                     archive_key,
+                                                                                     members)
 
                 self.assertEqual(archive_size - sum(info.compress_size for info in member_zip_infos), 2470)
                 self.assertEqual(set(info.filename for info in member_zip_infos), set(local_members))
                 self.assertEqual(len(missed_members), 0)
 
-            for archive_url, (members, members_expect) in itertools.product(
-                ["s3://dummy-bucket/s3utils_archive/archive.uncompressed.zip",
-                 "s3://dummy-bucket/s3utils_archive/archive.compressed.zip",
-                 ],
+            for archive_key, (members, members_expect) in itertools.product(
+                ["s3utils_archive/archive.uncompressed.zip", "s3utils_archive/archive.compressed.zip"],
                 [(local_members, local_members),
                  (None, local_members),
                  (["dir.baz/", "dir.foo/"],
@@ -524,7 +524,10 @@ class S3UtilsTest(unittest.TestCase):
                   ),
                  ],
             ):
-                archive_size, member_zip_infos, missed_members = s3_archive_listfile(client, archive_url, members)
+                archive_size, member_zip_infos, missed_members = s3_archive_listfile(client,
+                                                                                     "dummy-bucket",
+                                                                                     archive_key,
+                                                                                     members)
 
                 self.assertEqual(sorted(list(info.filename for info in member_zip_infos)), sorted(members_expect))
                 self.assertEqual(len(missed_members), 0)
@@ -545,16 +548,14 @@ class S3UtilsTest(unittest.TestCase):
             local_root = os.path.join(resources_directory, "unittest", "s3utils")
             local_members = [os.path.relpath(file_path, local_root) for file_path in listfile(local_root)]
 
-            for archive_url, mode in itertools.product(
-                ["s3://dummy-bucket/s3utils_archive/archive.uncompressed.zip",
-                 "s3://dummy-bucket/s3utils_archive/archive.compressed.zip",
-                 ],
+            for archive_key, mode in itertools.product(
+                ["s3utils_archive/archive.uncompressed.zip", "s3utils_archive/archive.compressed.zip"],
                 ["r", "rb"],
             ):
                 for local_member in local_members:
                     with (
                         open(os.path.join(local_root, local_member), mode) as local_fh,
-                        s3_archive_open_member(client, archive_url, local_member, mode) as s3_fh,
+                        s3_archive_open_member(client, "dummy-bucket", archive_key, local_member, mode) as s3_fh,
                     ):
                         self.assertEqual(local_fh.read(), s3_fh.read())
 
@@ -574,22 +575,20 @@ class S3UtilsTest(unittest.TestCase):
             local_root = os.path.join(resources_directory, "unittest", "s3utils")
             local_members = [os.path.relpath(file_path, local_root) for file_path in listfile(local_root)]
 
-            for archive_url, members, mode, threshold, use_chunked_reads in itertools.product(
-                ["s3://dummy-bucket/s3utils_archive/archive.uncompressed.zip",
-                 "s3://dummy-bucket/s3utils_archive/archive.compressed.zip",
-                 ],
+            for archive_key, members, mode, use_ranged_requests, use_chunked_reads in itertools.product(
+                ["s3utils_archive/archive.uncompressed.zip", "s3utils_archive/archive.compressed.zip"],
                 [local_members, None],
                 ["r", "rb"],
-                [1.0, 0.0],
-                [False, True],
+                [False, True, s3_archive_use_ranged_requests()],
+                [False, True, s3_archive_use_chunked_reads()],
             ):
                 for member, opener in s3_archive_open_members(
                     client,
-                    archive_url,
+                    "dummy-bucket",
+                    archive_key,
                     members,
                     mode,
-                    threshold=threshold,
-                    central_directory_overhead=0,
+                    use_ranged_requests=use_ranged_requests,
                     use_chunked_reads=use_chunked_reads,
                 ):
                     with open(os.path.join(local_root, member), mode) as local_fh, opener() as s3_fh:

{plexus_python_common-1.0.31 → plexus_python_common-1.0.33}/test/plexus_tests/common/utils/strutils_test.py

@@ -13,6 +13,8 @@ from plexus.common.utils.strutils import kebab_case_parser, kebab_case_pattern
 from plexus.common.utils.strutils import parse_bag_name, parse_user_email, parse_user_name, parse_vehicle_name
 from plexus.common.utils.strutils import snake_case_parser, snake_case_pattern
 from plexus.common.utils.strutils import strict_abspath_parser, strict_abspath_pattern
+from plexus.common.utils.strutils import strict_archive_abspath_parser, strict_archive_abspath_pattern
+from plexus.common.utils.strutils import strict_archive_relpath_parser, strict_archive_relpath_pattern
 from plexus.common.utils.strutils import strict_relpath_parser, strict_relpath_pattern
 from plexus.common.utils.strutils import topic_parser, topic_pattern
 from plexus.common.utils.strutils import vin_code_parser, vin_code_pattern
@@ -377,6 +379,154 @@ class StrUtilsTest(unittest.TestCase):
         with self.assertRaises(pp.ParseException):
             strict_abspath_parser.parse_string(data, parse_all=True)
 
+    data_strict_archive_relpath_pattern = [
+        ("archive.zip",),
+        ("directory/archive.zip",),
+        ("directory/dummy/archive.zip",),
+        ("directory/dummy/path/archive.zip",),
+        ("directory/dummy/path/to/archive.zip",),
+        ("directory/dummy/path_to-archive.zip",),
+        ("directory/dummy/.path_to-archive.zip",),
+        ("archive.zip#",),
+        ("archive.zip#directory",),
+        ("archive.zip#directory/",),
+        ("directory/archive.zip#directory/dummy",),
+        ("directory/archive.zip#directory/dummy/",),
+        ("directory/dummy/archive.zip#directory/dummy/path",),
+        ("directory/dummy/path/archive.zip#directory/dummy/path/to",),
+        ("directory/dummy/path/to/archive.zip#directory/dummy/path/to/file.txt",),
+        ("directory/dummy/path_to-archive.zip#directory/dummy/path_to-file.txt.",),
+        ("directory/dummy/.path_to-archive.zip#directory/dummy/.path_to-file.txt",),
+    ]
+
+    @ddt.idata(data_strict_archive_relpath_pattern)
+    @ddt.unpack
+    def test_strict_archive_relpath_pattern(self, data):
+        self.assertIsNotNone(strict_archive_relpath_pattern.match(data))
+        self.assertIsNotNone(strict_archive_relpath_parser.parse_string(data, parse_all=True))
+
+    data_strict_archive_relpath_pattern__bad_cases = [
+        ("/archive.zip",),
+        ("/directory/archive.zip",),
+        ("/directory/dummy/archive.zip",),
+        ("/directory/dummy/path/archive.zip",),
+        ("/directory/dummy/path/to/archive.zip",),
+        ("/directory/dummy/path_to-archive.zip",),
+        ("/directory/dummy/.path_to-archive.zip",),
+        ("/archive.zip#directory",),
+        ("/archive.zip#directory/",),
+        ("/directory/archive.zip#directory/dummy",),
+        ("/directory/archive.zip#directory/dummy/",),
+        ("/directory/dummy/archive.zip#directory/dummy/path",),
+        ("/directory/dummy/path/archive.zip#directory/dummy/path/to",),
+        ("/directory/dummy/path/to/archive.zip#directory/dummy/path/to/file.txt",),
+        ("/directory/dummy/path_to-archive.zip#directory/dummy/path_to-file.txt.",),
+        ("/directory/dummy/.path_to-archive.zip#directory/dummy/.path_to-file.txt",),
+        ("archive.zip/",),
+        ("directory/archive.zip/",),
+        ("directory/dummy/archive.zip/",),
+        ("directory/dummy/path/archive.zip/",),
+        ("directory/dummy/path/to/archive.zip/",),
+        ("directory/dummy/path_to-archive.zip/",),
+        ("directory/dummy/.path_to-archive.zip/",),
+        ("archive.zip#/directory",),
+        ("archive.zip#/directory/",),
+        ("directory/archive.zip#/directory/dummy",),
+        ("directory/archive.zip#/directory/dummy/",),
+        ("directory/dummy/archive.zip#/directory/dummy/path",),
+        ("directory/dummy/path/archive.zip#/directory/dummy/path/to",),
+        ("directory/dummy/path/to/archive.zip#/directory/dummy/path/to/file.txt",),
+        ("directory/dummy/path_to-archive.zip#/directory/dummy/path_to-file.txt.",),
+        ("directory/dummy/.path_to-archive.zip#/directory/dummy/.path_to-file.txt",),
+        ("directory/archive.zip#directory/./dummy",),
+        ("directory/archive.zip#directory/../dummy",),
+        ("directory/archive.zip#directory/.../dummy",),
+        ("directory/archive.zip#directory/\t/dummy",),
+        ("directory/archive.zip#directory/\r/dummy",),
+        ("directory/archive.zip#directory/\n/dummy",),
+    ]
+
+    @ddt.idata(data_strict_archive_relpath_pattern__bad_cases)
+    @ddt.unpack
+    def test_strict_archive_relpath_pattern__bad_cases(self, data):
+        self.assertIsNone(strict_archive_relpath_pattern.match(data))
+        with self.assertRaises(pp.ParseException):
+            strict_archive_relpath_parser.parse_string(data, parse_all=True)
+
+    data_strict_archive_abspath_pattern = [
+        ("/archive.zip",),
+        ("/directory/archive.zip",),
+        ("/directory/dummy/archive.zip",),
+        ("/directory/dummy/path/archive.zip",),
+        ("/directory/dummy/path/to/archive.zip",),
+        ("/directory/dummy/path_to-archive.zip",),
+        ("/directory/dummy/.path_to-archive.zip",),
+        ("/archive.zip#",),
+        ("/archive.zip#directory",),
+        ("/archive.zip#directory/",),
+        ("/directory/archive.zip#directory/dummy",),
+        ("/directory/archive.zip#directory/dummy/",),
+        ("/directory/dummy/archive.zip#directory/dummy/path",),
+        ("/directory/dummy/path/archive.zip#directory/dummy/path/to",),
+        ("/directory/dummy/path/to/archive.zip#directory/dummy/path/to/file.txt",),
+        ("/directory/dummy/path_to-archive.zip#directory/dummy/path_to-file.txt.",),
+        ("/directory/dummy/.path_to-archive.zip#directory/dummy/.path_to-file.txt",),
+    ]
+
+    @ddt.idata(data_strict_archive_abspath_pattern)
+    @ddt.unpack
+    def test_strict_archive_abspath_pattern(self, data):
+        self.assertIsNotNone(strict_archive_abspath_pattern.match(data))
+        self.assertIsNotNone(strict_archive_abspath_parser.parse_string(data, parse_all=True))
+
+    data_strict_archive_abspath_pattern__bad_cases = [
+        ("archive.zip",),
+        ("directory/archive.zip",),
+        ("directory/dummy/archive.zip",),
+        ("directory/dummy/path/archive.zip",),
+        ("directory/dummy/path/to/archive.zip",),
+        ("directory/dummy/path_to-archive.zip",),
+        ("directory/dummy/.path_to-archive.zip",),
+        ("archive.zip#directory",),
+        ("archive.zip#directory/",),
+        ("directory/archive.zip#directory/dummy",),
+        ("directory/archive.zip#directory/dummy/",),
+        ("directory/dummy/archive.zip#directory/dummy/path",),
+        ("directory/dummy/path/archive.zip#directory/dummy/path/to",),
+        ("directory/dummy/path/to/archive.zip#directory/dummy/path/to/file.txt",),
+        ("directory/dummy/path_to-archive.zip#directory/dummy/path_to-file.txt.",),
+        ("directory/dummy/.path_to-archive.zip#directory/dummy/.path_to-file.txt",),
+        ("/archive.zip/",),
+        ("/directory/archive.zip/",),
+        ("/directory/dummy/archive.zip/",),
+        ("/directory/dummy/path/archive.zip/",),
+        ("/directory/dummy/path/to/archive.zip/",),
+        ("/directory/dummy/path_to-archive.zip/",),
+        ("/directory/dummy/.path_to-archive.zip/",),
+        ("/archive.zip#/directory",),
+        ("/archive.zip#/directory/",),
+        ("/directory/archive.zip#/directory/dummy",),
+        ("/directory/archive.zip#/directory/dummy/",),
+        ("/directory/dummy/archive.zip#/directory/dummy/path",),
+        ("/directory/dummy/path/archive.zip#/directory/dummy/path/to",),
+        ("/directory/dummy/path/to/archive.zip#/directory/dummy/path/to/file.txt",),
+        ("/directory/dummy/path_to-archive.zip#/directory/dummy/path_to-file.txt.",),
+        ("/directory/dummy/.path_to-archive.zip#/directory/dummy/.path_to-file.txt",),
+        ("/directory/archive.zip#directory/./dummy",),
+        ("/directory/archive.zip#directory/../dummy",),
+        ("/directory/archive.zip#directory/.../dummy",),
+        ("/directory/archive.zip#directory/\t/dummy",),
+        ("/directory/archive.zip#directory/\r/dummy",),
+        ("/directory/archive.zip#directory/\n/dummy",),
+    ]
+
+    @ddt.idata(data_strict_archive_abspath_pattern__bad_cases)
+    @ddt.unpack
+    def test_strict_archive_abspath_pattern__bad_cases(self, data):
+        self.assertIsNone(strict_archive_abspath_pattern.match(data))
+        with self.assertRaises(pp.ParseException):
+            strict_archive_abspath_parser.parse_string(data, parse_all=True)
+
     data_email_address_pattern = [
         ("someone@dummy.com",),
         ("some.one@dummy.com",),