plexus-python-common 1.0.46__tar.gz → 1.0.47__tar.gz

{plexus_python_common-1.0.46 → plexus_python_common-1.0.47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: plexus-python-common
-Version: 1.0.46
+Version: 1.0.47
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13

{plexus_python_common-1.0.46 → plexus_python_common-1.0.47}/src/plexus/common/utils/bagutils.py

@@ -5,6 +5,7 @@ from typing import Literal
 
 import sqlalchemy as sa
 import sqlalchemy.orm as sa_orm
+from iker.common.utils.pathutils import make_path
 
 __all__ = [
     "SerializationFormat",
@@ -77,12 +78,20 @@ class BagMessage(BagBase):
 
 
 class BagReader(AbstractContextManager):
-    def __init__(self, db_dir: str, db_file: str = default_bag_db_file):
-        self.db_path = os.path.join(db_dir, db_file)
-        if not os.path.exists(self.db_path):
-            raise FileNotFoundError(f"could not find SQLite DB at '{db_dir}'")
-
-        self.engine = sa.create_engine(f"sqlite:///{self.db_path}")
+    def __init__(self, db_dir: str | os.PathLike[str], db_file: str | os.PathLike[str] = default_bag_db_file):
+        """
+        Creates a BagReader instance to read messages from a ROS2 bag file.
+
+        :param db_dir: directory containing the SQLite DB file.
+        :param db_file: name of the SQLite DB file.
+        :return: BagReader instance.
+        """
+        self.db_dir = make_path(db_dir)
+        self.db_path = self.db_dir / db_file
+        if not self.db_path.exists():
+            raise FileNotFoundError(f"could not find SQLite DB at '{str(self.db_path)}'")
+
+        self.engine = sa.create_engine(f"sqlite:///{str(self.db_path.absolute())}")
         self.session = sa_orm.Session(self.engine)
         self.topic_map = None
 
@@ -121,12 +130,22 @@ class BagReader(AbstractContextManager):
 
 
 class BagWriter(AbstractContextManager):
-    def __init__(self, db_dir: str, db_file: str = default_bag_db_file):
-        self.db_path = os.path.join(db_dir, db_file)
-        if os.path.exists(self.db_path):
-            os.remove(self.db_path)
-
-        self.engine = sa.create_engine(f"sqlite:///{self.db_path}")
+    def __init__(self, db_dir: str | os.PathLike[str], db_file: str | os.PathLike[str] = default_bag_db_file):
+        """
+        Creates a BagWriter instance to write messages to a ROS2 bag file.
+
+        :param db_dir: directory to store the SQLite DB file.
+        :param db_file: name of the SQLite DB file.
+        :return: BagWriter instance.
+        """
+        self.db_dir = make_path(db_dir)
+        if not self.db_dir.exists():
+            self.db_dir.mkdir(parents=True, exist_ok=True)
+        self.db_path = self.db_dir / db_file
+        if self.db_path.exists():
+            self.db_path.unlink()
+
+        self.engine = sa.create_engine(f"sqlite:///{str(self.db_path.absolute())}")
         self.session = sa_orm.Session(self.engine)
         self.topic_map = {}
 
@@ -191,25 +210,5 @@ class BagWriter(AbstractContextManager):
         return db_message
 
 
-def bag_reader(db_dir: str, db_file: str = default_bag_db_file) -> BagReader:
-    """
-    Creates a BagReader instance to read messages from a ROS2 bag file.
-
-    :param db_dir: directory containing the SQLite DB file.
-    :param db_file: name of the SQLite DB file.
-    :return: BagReader instance.
-    """
-    return BagReader(db_dir, db_file)
-
-
-def bag_writer(db_dir: str, db_file: str = default_bag_db_file) -> BagWriter:
-    """
-    Creates a BagWriter instance to write messages to a ROS2 bag file.
-
-    :param db_dir: directory to store the SQLite DB file.
-    :param db_file: name of the SQLite DB file.
-    :return: BagWriter instance.
-    """
-    if not os.path.exists(db_dir):
-        os.makedirs(db_dir, exist_ok=True)
-    return BagWriter(db_dir, db_file)
+bag_reader = BagReader
+bag_writer = BagWriter

{plexus_python_common-1.0.46 → plexus_python_common-1.0.47}/src/plexus/common/utils/config.py

@@ -1,17 +1,19 @@
+import os
+
 from iker.common.utils import logger
 from iker.common.utils.config import Config
 from iker.common.utils.funcutils import memorized
-from iker.common.utils.shutils import expanded_path
+from iker.common.utils.pathutils import make_path
 
 
 @memorized()
-def config(config_path: str = "") -> Config:
+def config(config_path: str | os.PathLike[str] = "") -> Config:
     default_items: list[tuple[str, str, str]] = [
         ("plexus", "logging.level", "INFO"),
         ("plexus", "logging.format", "%(asctime)s [%(levelname)s] %(name)s: %(message)s"),
     ]
 
-    instance = Config(config_path or expanded_path("~/.plexus.cfg"))
+    instance = Config(config_path or make_path("~/.plexus.cfg", expand=True, normalize=True, absolute=True))
     instance.restore()
     instance.update(default_items, overwrite=False)
 

{plexus_python_common-1.0.46 → plexus_python_common-1.0.47}/src/plexus/common/utils/jsonutils.py

@@ -9,7 +9,7 @@ from iker.common.utils.iterutils import batched
 from iker.common.utils.jsonutils import JsonType, JsonValueCompatible
 from iker.common.utils.jsonutils import json_reformat
 
-from plexus.common.utils.shutils import collect_volumed_filenames, populate_volumed_filenames
+from plexus.common.utils.pathutils import collect_volumed_filenames, populate_volumed_filenames
 
 __all__ = [
     "json_datetime_decoder",

{plexus_python_common-1.0.46 → plexus_python_common-1.0.47}/src/plexus/common/utils/s3utils.py

@@ -6,7 +6,7 @@ import functools
 import io
 import mimetypes
 import os
-import os.path
+import pathlib
 import shutil
 import tempfile
 import typing
@@ -20,7 +20,7 @@ import fsspec
 import fsspec.utils
 from iker.common.utils.iterutils import chunk_between, head, last
 from iker.common.utils.jsonutils import JsonObject
-from iker.common.utils.shutils import glob_match, listfile, path_depth
+from iker.common.utils.pathutils import glob_match, make_path, path_depth, scan_files
 from iker.common.utils.strutils import is_empty, trim_to_none
 from mypy_boto3_s3 import S3Client
 from rich.progress import BarColumn, DownloadColumn, Progress, TextColumn, TransferSpeedColumn
@@ -30,7 +30,7 @@ __all__ = [
     "s3_make_client",
     "s3_head_object",
     "s3_list_objects",
-    "s3_listfile",
+    "s3_list_files",
     "s3_cp_download",
     "s3_cp_upload",
     "s3_sync_download",
@@ -40,7 +40,7 @@ __all__ = [
     "s3_make_progressed_client",
     "ArchiveMemberChunk",
     "s3_archive_member_tree",
-    "s3_archive_listfile",
+    "s3_archive_list_files",
     "s3_archive_open_member",
     "s3_archive_use_ranged_requests",
     "s3_archive_use_chunked_reads",
@@ -147,7 +147,7 @@ def s3_list_objects(
         continuation_token = response.get("NextContinuationToken")
 
 
-def s3_listfile(
+def s3_list_files(
     client: S3Client,
     bucket: str,
     prefix: str,
@@ -173,9 +173,10 @@ def s3_listfile(
         prefix = prefix + "/"
 
     def filter_object_meta(object_meta: S3ObjectMeta) -> bool:
-        if 0 < depth <= path_depth(prefix, os.path.dirname(object_meta.key)):
+        object_path = make_path(object_meta.key)
+        if 0 < depth <= path_depth(prefix, object_path.parent):
             return False
-        if len(glob_match([os.path.basename(object_meta.key)], include_patterns, exclude_patterns)) == 0:
+        if len(glob_match([object_path.name], include_patterns, exclude_patterns)) == 0:
             return False
         return True
 
@@ -238,16 +239,18 @@ def s3_sync_download(
     if not prefix.endswith("/"):
         prefix = prefix + "/"
 
-    objects = s3_listfile(client,
-                          bucket,
-                          prefix,
-                          include_patterns=include_patterns,
-                          exclude_patterns=exclude_patterns,
-                          depth=depth)
+    dir_path = make_path(dir_path)
+
+    objects = s3_list_files(client,
+                            bucket,
+                            prefix,
+                            include_patterns=include_patterns,
+                            exclude_patterns=exclude_patterns,
+                            depth=depth)
 
     def download_file(key: str):
-        file_path = os.path.join(dir_path, key[len(prefix):])
-        os.makedirs(os.path.dirname(file_path), exist_ok=True)
+        file_path = dir_path / key[len(prefix):]
+        file_path.parent.mkdir(parents=True, exist_ok=True)
         s3_cp_download(client, bucket, key, file_path)
 
     with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
@@ -293,13 +296,15 @@ def s3_sync_upload(
     if not prefix.endswith("/"):
         prefix = prefix + "/"
 
-    file_paths = listfile(dir_path,
-                          include_patterns=include_patterns,
-                          exclude_patterns=exclude_patterns,
-                          depth=depth)
+    dir_path = make_path(dir_path)
+
+    file_paths = scan_files(dir_path,
+                            include_patterns=include_patterns,
+                            exclude_patterns=exclude_patterns,
+                            depth=depth)
 
-    def upload_file(file_path: str):
-        s3_cp_upload(client, file_path, bucket, prefix + os.path.relpath(file_path, dir_path))
+    def upload_file(file_path: pathlib.Path):
+        s3_cp_upload(client, file_path, bucket, prefix + str(file_path.relative_to(dir_path)))
 
     with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
         futures = [executor.submit(upload_file, file_path) for file_path in file_paths]
@@ -415,7 +420,7 @@ class S3ClientProgressProxy(object):
         Callback=None,
         Config=None,
     ):
-        bytes_total = os.path.getsize(Filename)
+        bytes_total = make_path(Filename).stat().st_size
         with (
             contextlib.nullcontext(Callback) if Callback is not None
             else self.make_transfer_callback(Key, bytes_total, "upload")
@@ -585,7 +590,7 @@ def s3_archive_member_tree(
     return root_member_tree
 
 
-def s3_archive_listfile(
+def s3_archive_list_files(
     client: S3Client,
     bucket: str,
     key: str,
@@ -597,7 +602,7 @@ def s3_archive_listfile(
     under that directory will be included in the results.
 
     Example usage:
-    >>> archive_size, member_zip_infos, missed_members = s3_archive_listfile(client, bucket, key, members=["file1.txt", "dir1/"])
+    >>> archive_size, member_zip_infos, missed_members = s3_archive_list_files(client, bucket, key, members=["file1.txt", "dir1/"])
     >>> for info in member_zip_infos:
     ...     print(info.filename, info.file_size)
     >>> if missed_members:
@@ -836,7 +841,7 @@ def s3_archive_open_members(
 
     s3_options = s3_options_from_s3_client(client)
 
-    archive_size, member_zip_infos, missed_members = s3_archive_listfile(client, bucket, key, members)
+    archive_size, member_zip_infos, missed_members = s3_archive_list_files(client, bucket, key, members)
     if missed_members:
         raise FileNotFoundError(f"Archive members not found: {', '.join(missed_members)}")
 

{plexus_python_common-1.0.46 → plexus_python_common-1.0.47}/src/plexus/common/utils/tagutils.py

@@ -1,6 +1,7 @@
 import dataclasses
 import datetime
-import os.path
+import os
+import pathlib
 import textwrap
 import typing
 from collections.abc import Generator, Mapping, Sequence
@@ -324,18 +325,17 @@ def render_tagset_markdown_readme(tagset: Tagset) -> str:
 
 
 @singleton
-def tag_cache_file_path() -> str:
-    return os.path.expanduser(f"~/.local/plus/datahub/tag_cache/{randomizer().random_alphanumeric(7)}.db")
+def tag_cache_file_path() -> pathlib.Path:
+    return pathlib.Path.home() / ".local" / "plus" / "datahub" / "tag_cache" / f"{randomizer().random_alphanumeric(7)}.db"
 
 
 class TagCache(object):
-    def __init__(self, *, file_path: str | None = None, fail_if_exists: bool = False):
-        self.file_path = file_path or tag_cache_file_path()
-        if fail_if_exists and os.path.exists(self.file_path):
-            raise FileExistsError(f"tag cache file '{self.file_path}' already exists")
-        if not os.path.exists(os.path.dirname(self.file_path)):
-            os.makedirs(os.path.dirname(self.file_path), exist_ok=True)
-        self.conn_maker = ConnectionMaker.from_url(f"sqlite:///{self.file_path}",
+    def __init__(self, *, file_path: str | os.PathLike[str] | None = None, fail_if_exists: bool = False):
+        self.file_path = pathlib.Path(file_path or tag_cache_file_path())
+        if fail_if_exists and self.file_path.exists():
+            raise FileExistsError(f"tag cache file '{str(self.file_path)}' already exists")
+        self.file_path.parent.mkdir(parents=True, exist_ok=True)
+        self.conn_maker = ConnectionMaker.from_url(f"sqlite:///{str(self.file_path.absolute())}",
                                                    engine_opts=dict(connect_args={"check_same_thread": False}))
         BaseModel.metadata.create_all(self.conn_maker.engine)
 

{plexus_python_common-1.0.46 → plexus_python_common-1.0.47}/src/plexus_python_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: plexus-python-common
-Version: 1.0.46
+Version: 1.0.47
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13

{plexus_python_common-1.0.46 → plexus_python_common-1.0.47}/src/plexus_python_common.egg-info/SOURCES.txt

@@ -10,6 +10,22 @@ setup.py
 resources/unittest/jsonutils/dummy.0.jsonl
 resources/unittest/jsonutils/dummy.1.jsonl
 resources/unittest/jsonutils/dummy.2.jsonl
+resources/unittest/pathutils/0-dummy
+resources/unittest/pathutils/1-dummy
+resources/unittest/pathutils/2-dummy
+resources/unittest/pathutils/dummy.0.0.jsonl
+resources/unittest/pathutils/dummy.0.0.vol-0.jsonl
+resources/unittest/pathutils/dummy.0.jsonl
+resources/unittest/pathutils/dummy.1.1.jsonl
+resources/unittest/pathutils/dummy.1.1.vol-1.jsonl
+resources/unittest/pathutils/dummy.1.jsonl
+resources/unittest/pathutils/dummy.2.2.jsonl
+resources/unittest/pathutils/dummy.2.2.vol-2.jsonl
+resources/unittest/pathutils/dummy.2.jsonl
+resources/unittest/pathutils/dummy.csv.part0
+resources/unittest/pathutils/dummy.csv.part1
+resources/unittest/pathutils/dummy.csv.part2
+resources/unittest/pathutils/dummy.txt
 resources/unittest/s3utils/dir.baz/file.bar.baz
 resources/unittest/s3utils/dir.baz/file.foo.bar
 resources/unittest/s3utils/dir.baz/file.foo.baz
@@ -22,22 +38,6 @@ resources/unittest/s3utils/dir.foo/dir.foo.bar/file.foo.baz
 resources/unittest/s3utils/dir.foo/dir.foo.bar/dir.foo.bar.baz/file.foo.bar.baz
 resources/unittest/s3utils_archive/archive.compressed.zip
 resources/unittest/s3utils_archive/archive.uncompressed.zip
-resources/unittest/shutils/0-dummy
-resources/unittest/shutils/1-dummy
-resources/unittest/shutils/2-dummy
-resources/unittest/shutils/dummy.0.0.jsonl
-resources/unittest/shutils/dummy.0.0.vol-0.jsonl
-resources/unittest/shutils/dummy.0.jsonl
-resources/unittest/shutils/dummy.1.1.jsonl
-resources/unittest/shutils/dummy.1.1.vol-1.jsonl
-resources/unittest/shutils/dummy.1.jsonl
-resources/unittest/shutils/dummy.2.2.jsonl
-resources/unittest/shutils/dummy.2.2.vol-2.jsonl
-resources/unittest/shutils/dummy.2.jsonl
-resources/unittest/shutils/dummy.csv.part0
-resources/unittest/shutils/dummy.csv.part1
-resources/unittest/shutils/dummy.csv.part2
-resources/unittest/shutils/dummy.txt
 src/plexus/common/__init__.py
 src/plexus/common/pose.py
 src/plexus/common/proj.py
@@ -57,8 +57,8 @@ src/plexus/common/utils/datautils.py
 src/plexus/common/utils/dockerutils.py
 src/plexus/common/utils/jsonutils.py
 src/plexus/common/utils/ormutils.py
+src/plexus/common/utils/pathutils.py
 src/plexus/common/utils/s3utils.py
-src/plexus/common/utils/shutils.py
 src/plexus/common/utils/sqlutils.py
 src/plexus/common/utils/strutils.py
 src/plexus/common/utils/tagutils.py
@@ -69,19 +69,22 @@ src/plexus_python_common.egg-info/dependency_links.txt
 src/plexus_python_common.egg-info/not-zip-safe
 src/plexus_python_common.egg-info/requires.txt
 src/plexus_python_common.egg-info/top_level.txt
-test/plexus_test.py
+test/testenv.py
 test/plexus_tests/__init__.py
+test/plexus_tests/common/__init__.py
 test/plexus_tests/common/pose_test.py
 test/plexus_tests/common/proj_test.py
+test/plexus_tests/common/carto/__init__.py
 test/plexus_tests/common/carto/osm_file_test.py
 test/plexus_tests/common/carto/osm_tags_test.py
+test/plexus_tests/common/utils/__init__.py
 test/plexus_tests/common/utils/bagutils_test.py
 test/plexus_tests/common/utils/datautils_test.py
 test/plexus_tests/common/utils/dockerutils_test.py
 test/plexus_tests/common/utils/jsonutils_test.py
 test/plexus_tests/common/utils/ormutils_test.py
+test/plexus_tests/common/utils/pathutils_test.py
 test/plexus_tests/common/utils/s3utils_test.py
-test/plexus_tests/common/utils/shutils_test.py
 test/plexus_tests/common/utils/strutils_test.py
 test/plexus_tests/common/utils/tagutils_test.py
 test/plexus_tests/common/utils/testutils_test.py

{plexus_python_common-1.0.46 → plexus_python_common-1.0.47}/test/plexus_tests/common/utils/jsonutils_test.py

@@ -4,13 +4,12 @@ import unittest
 import ddt
 
 from plexus.common.utils.jsonutils import read_chunked_jsonl
-from plexus_test import resources_directory
+from testenv import resources_directory
 
 
 @ddt.ddt
 class JsonUtilsTest(unittest.TestCase):
 
     def test_read_chunked_jsonl(self):
-        for data, path in read_chunked_jsonl(
-            os.path.join(resources_directory, "unittest/jsonutils", "dummy.{{}}.jsonl")):
+        for data, path in read_chunked_jsonl(str(resources_directory / "unittest" / "jsonutils" / "dummy.{{}}.jsonl")):
             self.assertEqual(data["file"], os.path.basename(path))

plexus_python_common-1.0.46/test/plexus_tests/common/utils/shutils_test.py → plexus_python_common-1.0.47/test/plexus_tests/common/utils/pathutils_test.py

@@ -1,16 +1,15 @@
-import os
 import unittest
 from collections.abc import Callable, Generator
 
 import ddt
 from iker.common.utils.dtutils import dt_parse_iso
 
-from plexus.common.utils.shutils import collect_volumed_filenames, populate_volumed_filenames
-from plexus_tests import resources_directory
+from plexus.common.utils.pathutils import collect_volumed_filenames, populate_volumed_filenames
+from testenv import resources_directory
 
 
 @ddt.ddt
-class ShUtilsTest(unittest.TestCase):
+class PathUtilsTest(unittest.TestCase):
     data_collect_volumed_filenames = [
         ("dummy.txt", [("dummy.txt", {})]),
         (
@@ -75,10 +74,8 @@ class ShUtilsTest(unittest.TestCase):
     @ddt.unpack
     def test_collect_volumed_filenames(self, template, names):
         self.assertEqual(
-            list((os.path.join(resources_directory, "unittest/shutils", name), volumes)
-                 for name, volumes in names),
-            list(sorted(
-                collect_volumed_filenames(os.path.join(resources_directory, "unittest/shutils", template)))),
+            list((str(resources_directory / "unittest" / "pathutils" / name), volumes) for name, volumes in names),
+            list(sorted(collect_volumed_filenames(resources_directory / "unittest" / "pathutils" / template))),
         )
 
     data_populate_volumed_filenames = [
@@ -144,10 +141,9 @@ class ShUtilsTest(unittest.TestCase):
     @ddt.idata(data_populate_volumed_filenames)
     @ddt.unpack
     def test_populate_volumed_filenames(self, template, names):
-        generator = populate_volumed_filenames(os.path.join(resources_directory, "unittest/shutils", template))
+        generator = populate_volumed_filenames(resources_directory / "unittest" / "pathutils" / template)
         self.assertEqual(
-            list((os.path.join(resources_directory, "unittest/shutils", name), volumes)
-                 for name, volumes in names),
+            list((str(resources_directory / "unittest" / "pathutils" / name), volumes) for name, volumes in names),
             list(next(generator) for _ in names),
         )
 
@@ -258,10 +254,9 @@ class ShUtilsTest(unittest.TestCase):
 
             return volume_func
 
-        generator = populate_volumed_filenames(os.path.join(resources_directory, "unittest/shutils", template),
+        generator = populate_volumed_filenames(resources_directory / "unittest" / "pathutils" / template,
                                                volume_func=make_custom_volume_func())
         self.assertEqual(
-            list((os.path.join(resources_directory, "unittest/shutils", name), volumes)
-                 for name, volumes in names),
+            list((str(resources_directory / "unittest" / "pathutils" / name), volumes) for name, volumes in names),
             list(next(generator) for _ in names),
         )

{plexus_python_common-1.0.46 → plexus_python_common-1.0.47}/test/plexus_tests/common/utils/s3utils_test.py

@@ -1,18 +1,16 @@
 import contextlib
 import itertools
-import os
 import tempfile
 import unittest
 
 import ddt
 import moto
 import moto.server
+from iker.common.utils.pathutils import make_path, scan_files
 from iker.common.utils.randutils import randomizer
-from iker.common.utils.shutils import listfile
-from iker.common.utils.testutils import norm_path
 
 from plexus.common.utils.s3utils import *
-from plexus_tests import resources_directory
+from testenv import resources_directory
 
 
 @contextlib.contextmanager
@@ -66,7 +64,7 @@ class S3UtilsTest(unittest.TestCase):
                 self.assertTrue(any(o.key == key for o in result))
                 self.assertEqual(s3_pull_text(client, "dummy-bucket", key), text)
 
-    def test_s3_listfile__random_text_files(self):
+    def test_s3_list_files__random_text_files(self):
         with moto.mock_aws(), s3_make_client() as client:
             client.create_bucket(Bucket="dummy-bucket")
 
@@ -79,7 +77,7 @@ class S3UtilsTest(unittest.TestCase):
             for key, text in data:
                 s3_push_text(client, text, "dummy-bucket", key)
 
-            result = list(s3_listfile(client, "dummy-bucket", "dummy_prefix/"))
+            result = list(s3_list_files(client, "dummy-bucket", "dummy_prefix/"))
 
             for key, text in data:
                 self.assertTrue(any(o.key == key for o in result))
@@ -97,13 +95,13 @@ class S3UtilsTest(unittest.TestCase):
             for key, text in data:
                 s3_push_text(client, text, "dummy-bucket", key)
 
-            result = list(s3_listfile(client, "dummy-bucket", "dummy_prefix/"))
+            result = list(s3_list_files(client, "dummy-bucket", "dummy_prefix/"))
 
             for key, text in data:
                 self.assertTrue(any(o.key == key for o in result))
                 self.assertEqual(s3_pull_text(client, "dummy-bucket", key), text)
 
-    data_s3_listfile = [
+    data_s3_list_files = [
         (
             "unittest/s3utils/",
             "dummy-bucket",
@@ -232,26 +230,26 @@ class S3UtilsTest(unittest.TestCase):
         ),
     ]
 
-    @ddt.idata(data_s3_listfile)
+    @ddt.idata(data_s3_list_files)
     @ddt.unpack
-    def test_s3_listfile(self, src, bucket, prefix, include_patterns, exclude_patterns, depth, expect):
+    def test_s3_list_files(self, src, bucket, prefix, include_patterns, exclude_patterns, depth, expect):
         with moto.mock_aws(), s3_make_client() as client:
             client.create_bucket(Bucket=bucket)
 
             s3_sync_upload(client,
-                           os.path.join(resources_directory, src),
+                           resources_directory / src,
                            bucket,
                            prefix)
 
-            object_metas = s3_listfile(client,
-                                       bucket,
-                                       prefix,
-                                       include_patterns=include_patterns,
-                                       exclude_patterns=exclude_patterns,
-                                       depth=depth)
+            object_metas = s3_list_files(client,
+                                         bucket,
+                                         prefix,
+                                         include_patterns=include_patterns,
+                                         exclude_patterns=exclude_patterns,
+                                         depth=depth)
 
-            self.assertSetEqual(set(map(lambda x: norm_path(x.key), object_metas)),
-                                set(map(lambda x: norm_path(x), expect)))
+            self.assertSetEqual(set(map(lambda x: make_path(x.key, normalize=True), object_metas)),
+                                set(map(lambda x: make_path(x, normalize=True), expect)))
 
     data_s3_sync = [
         (
@@ -397,7 +395,7 @@ class S3UtilsTest(unittest.TestCase):
             client.create_bucket(Bucket=bucket)
 
             s3_sync_upload(client,
-                           os.path.join(resources_directory, src),
+                           resources_directory / src,
                            bucket,
                            prefix,
                            include_patterns=include_patterns,
@@ -405,54 +403,63 @@ class S3UtilsTest(unittest.TestCase):
                            depth=depth)
 
             with tempfile.TemporaryDirectory() as temp_directory:
+                temp_directory = make_path(temp_directory)
                 s3_sync_download(client,
                                  bucket,
                                  prefix,
-                                 os.path.join(temp_directory, dst))
+                                 temp_directory / dst)
 
-                self.assertSetEqual(set(map(lambda x: norm_path(x), listfile(os.path.join(temp_directory, dst)))),
-                                    set(map(lambda x: norm_path(os.path.join(temp_directory, x)), expect)))
+                self.assertSetEqual(
+                    set(map(lambda x: make_path(x, normalize=True), scan_files(temp_directory / dst))),
+                    set(map(lambda x: make_path(temp_directory / x, normalize=True), expect)),
+                )
 
         with moto.mock_aws(), s3_make_client() as client:
             client.create_bucket(Bucket=bucket)
 
             s3_sync_upload(client,
-                           os.path.join(resources_directory, src),
+                           resources_directory / src,
                            bucket,
                            prefix)
 
             with tempfile.TemporaryDirectory() as temp_directory:
+                temp_directory = make_path(temp_directory)
                 s3_sync_download(client,
                                  bucket,
                                  prefix,
-                                 os.path.join(temp_directory, dst),
+                                 temp_directory / dst,
                                  include_patterns=include_patterns,
                                  exclude_patterns=exclude_patterns,
                                  depth=depth)
 
-                self.assertSetEqual(set(map(lambda x: norm_path(x), listfile(os.path.join(temp_directory, dst)))),
-                                    set(map(lambda x: norm_path(os.path.join(temp_directory, x)), expect)))
+                self.assertSetEqual(
+                    set(map(lambda x: make_path(x, normalize=True), scan_files(temp_directory / dst))),
+                    set(map(lambda x: make_path(temp_directory / x, normalize=True), expect)),
+                )
 
         with moto.mock_aws(), s3_make_client() as client:
             client.create_bucket(Bucket=bucket)
 
             s3_sync_upload(client,
-                           os.path.join(resources_directory, src),
+                           resources_directory / src,
                            bucket,
                            prefix,
                            include_patterns=include_patterns,
                            depth=depth)
 
             with tempfile.TemporaryDirectory() as temp_directory:
+                temp_directory = make_path(temp_directory)
                 s3_sync_download(client,
                                  bucket,
                                  prefix,
-                                 os.path.join(temp_directory, dst),
+                                 temp_directory / dst,
                                  exclude_patterns=exclude_patterns,
                                  depth=depth)
 
-                self.assertSetEqual(set(map(lambda x: norm_path(x), listfile(os.path.join(temp_directory, dst)))),
-                                    set(map(lambda x: norm_path(os.path.join(temp_directory, x)), expect)))
+                self.assertSetEqual(
+                    set(map(lambda x: make_path(x, normalize=True), scan_files(temp_directory / dst))),
+                    set(map(lambda x: make_path(temp_directory / x, normalize=True), expect)),
+                )
 
     data_s3_text = [
         ("dummy-bucket", "dummy/key", "dummy content", None),
@@ -472,7 +479,7 @@ class S3UtilsTest(unittest.TestCase):
             s3_push_text(client, text, bucket, key, encoding=encoding)
             self.assertEqual(s3_pull_text(client, bucket, key, encoding=encoding), text)
 
-    def test_s3_archive_listfile(self):
+    def test_s3_archive_list_files(self):
         with (
             moto.mock_aws(),
             moto_server_endpoint_url() as endpoint_url,
@@ -481,21 +488,21 @@ class S3UtilsTest(unittest.TestCase):
             client.create_bucket(Bucket="dummy-bucket")
 
             s3_sync_upload(client,
-                           os.path.join(resources_directory, "unittest", "s3utils_archive"),
+                           resources_directory / "unittest" / "s3utils_archive",
                            "dummy-bucket",
                            "s3utils_archive")
 
-            local_root = os.path.join(resources_directory, "unittest", "s3utils")
-            local_members = [os.path.relpath(file_path, local_root) for file_path in listfile(local_root)]
+            local_root = resources_directory / "unittest" / "s3utils"
+            local_members = [str(file_path.relative_to(local_root)) for file_path in scan_files(local_root)]
 
             for archive_key, members in itertools.product(
                 ["s3utils_archive/archive.uncompressed.zip", "s3utils_archive/archive.compressed.zip"],
                 [local_members, None],
             ):
-                archive_size, member_zip_infos, missed_members = s3_archive_listfile(client,
-                                                                                     "dummy-bucket",
-                                                                                     archive_key,
-                                                                                     members)
+                archive_size, member_zip_infos, missed_members = s3_archive_list_files(client,
+                                                                                       "dummy-bucket",
+                                                                                       archive_key,
+                                                                                       members)
 
                 self.assertEqual(archive_size - sum(info.compress_size for info in member_zip_infos), 2470)
                 self.assertEqual(set(info.filename for info in member_zip_infos), set(local_members))
@@ -506,28 +513,31 @@ class S3UtilsTest(unittest.TestCase):
                 [(local_members, local_members),
                  (None, local_members),
                  (["dir.baz/", "dir.foo/"],
-                  [os.path.relpath(file_path, local_root)
+                  [str(file_path.relative_to(local_root))
                    for file_path in
-                   listfile(os.path.join(local_root, "dir.baz")) + listfile(os.path.join(local_root, "dir.foo"))],
+                   itertools.chain(scan_files(local_root / "dir.baz"),
+                                   scan_files(local_root / "dir.foo"))],
                   ),
                  (["dir.foo/"],
-                  [os.path.relpath(file_path, local_root)
-                   for file_path in listfile(os.path.join(local_root, "dir.foo"))],
+                  [str(file_path.relative_to(local_root))
+                   for file_path in scan_files(local_root / "dir.foo")],
                   ),
                  (["dir.foo/"] * 3,  # Duplicate entries, the result should repeat them as well
-                  [os.path.relpath(file_path, local_root)
-                   for file_path in (listfile(os.path.join(local_root, "dir.foo")) * 3)],
+                  [str(file_path.relative_to(local_root))
+                   for file_path in itertools.chain(scan_files(local_root / "dir.foo"),
+                                                    scan_files(local_root / "dir.foo"),
+                                                    scan_files(local_root / "dir.foo"))],
                   ),
                  (["dir.foo/dir.foo.bar/"],
-                  [os.path.relpath(file_path, local_root)
-                   for file_path in listfile(os.path.join(local_root, "dir.foo", "dir.foo.bar"))],
+                  [str(file_path.relative_to(local_root))
+                   for file_path in scan_files(local_root / "dir.foo" / "dir.foo.bar")],
                   ),
                  ],
             ):
-                archive_size, member_zip_infos, missed_members = s3_archive_listfile(client,
-                                                                                     "dummy-bucket",
-                                                                                     archive_key,
-                                                                                     members)
+                archive_size, member_zip_infos, missed_members = s3_archive_list_files(client,
+                                                                                       "dummy-bucket",
+                                                                                       archive_key,
+                                                                                       members)
 
                 self.assertEqual(sorted(list(info.filename for info in member_zip_infos)), sorted(members_expect))
                 self.assertEqual(len(missed_members), 0)
@@ -541,12 +551,12 @@ class S3UtilsTest(unittest.TestCase):
             client.create_bucket(Bucket="dummy-bucket")
 
             s3_sync_upload(client,
-                           os.path.join(resources_directory, "unittest", "s3utils_archive"),
+                           resources_directory / "unittest" / "s3utils_archive",
                            "dummy-bucket",
                            "s3utils_archive")
 
-            local_root = os.path.join(resources_directory, "unittest", "s3utils")
-            local_members = [os.path.relpath(file_path, local_root) for file_path in listfile(local_root)]
+            local_root = resources_directory / "unittest" / "s3utils"
+            local_members = [str(file_path.relative_to(local_root)) for file_path in scan_files(local_root)]
 
             for archive_key, mode in itertools.product(
                 ["s3utils_archive/archive.uncompressed.zip", "s3utils_archive/archive.compressed.zip"],
@@ -554,7 +564,7 @@ class S3UtilsTest(unittest.TestCase):
             ):
                 for local_member in local_members:
                     with (
-                        open(os.path.join(local_root, local_member), mode) as local_fh,
+                        open(local_root / local_member, mode) as local_fh,
                         s3_archive_open_member(client, "dummy-bucket", archive_key, local_member, mode) as s3_fh,
                     ):
                         self.assertEqual(local_fh.read(), s3_fh.read())
@@ -568,12 +578,12 @@ class S3UtilsTest(unittest.TestCase):
             client.create_bucket(Bucket="dummy-bucket")
 
             s3_sync_upload(client,
-                           os.path.join(resources_directory, "unittest", "s3utils_archive"),
+                           resources_directory / "unittest" / "s3utils_archive",
                            "dummy-bucket",
                            "s3utils_archive")
 
-            local_root = os.path.join(resources_directory, "unittest", "s3utils")
-            local_members = [os.path.relpath(file_path, local_root) for file_path in listfile(local_root)]
+            local_root = resources_directory / "unittest" / "s3utils"
+            local_members = [str(file_path.relative_to(local_root)) for file_path in scan_files(local_root)]
 
             for archive_key, members, mode, use_ranged_requests, use_chunked_reads in itertools.product(
                 ["s3utils_archive/archive.uncompressed.zip", "s3utils_archive/archive.compressed.zip"],
@@ -591,5 +601,5 @@ class S3UtilsTest(unittest.TestCase):
                     use_ranged_requests=use_ranged_requests,
                     use_chunked_reads=use_chunked_reads,
                 ):
-                    with open(os.path.join(local_root, member), mode) as local_fh, opener() as s3_fh:
+                    with open(local_root / member, mode) as local_fh, opener() as s3_fh:
                         self.assertEqual(local_fh.read(), s3_fh.read())

{plexus_python_common-1.0.46 → plexus_python_common-1.0.47}/test/plexus_tests/common/utils/tagutils_test.py

@@ -1,5 +1,4 @@
 import datetime
-import os.path
 import unittest
 
 import ddt
@@ -33,7 +32,7 @@ class TagUtilsTest(unittest.TestCase):
         tag_cache = TagCache()
 
         self.assertEqual(tag_cache.file_path, tag_cache_file_path())
-        self.assertTrue(os.path.exists(tag_cache.file_path))
+        self.assertTrue(tag_cache.file_path.exists())
 
         self.assertEqual(len(tag_cache.query("dummy_vehicle")), 0)
 

plexus_python_common-1.0.47/test/testenv.py

@@ -0,0 +1,15 @@
+import pathlib
+
+__all__ = [
+    "module_directory",
+    "source_directory",
+    "test_directory",
+    "resources_directory",
+    "temporary_directory",
+]
+
+module_directory: pathlib.Path = pathlib.Path(__file__).absolute().parent.parent
+source_directory: pathlib.Path = module_directory / "src"
+test_directory: pathlib.Path = module_directory / "test"
+resources_directory: pathlib.Path = module_directory / "resources"
+temporary_directory: pathlib.Path = module_directory / "tmp"

plexus_python_common-1.0.46/test/plexus_test.py

@@ -1,13 +0,0 @@
-import unittest
-
-from plexus_tests import *
-
-
-class Test(unittest.TestCase):
-
-    def test(self):
-        self.assertIsNotNone(module_directory)
-        self.assertIsNotNone(source_directory)
-        self.assertIsNotNone(test_directory)
-        self.assertIsNotNone(resources_directory)
-        self.assertIsNotNone(temporary_directory)

plexus_python_common-1.0.46/test/plexus_tests/__init__.py

@@ -1,15 +0,0 @@
-import os
-
-__all__ = [
-    "module_directory",
-    "source_directory",
-    "test_directory",
-    "resources_directory",
-    "temporary_directory",
-]
-
-module_directory: str = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
-source_directory: str = os.path.abspath(os.path.join(module_directory, "src"))
-test_directory: str = os.path.abspath(os.path.join(module_directory, "test"))
-resources_directory: str = os.path.abspath(os.path.join(module_directory, "resources"))
-temporary_directory: str = os.path.abspath(os.path.join(module_directory, "tmp"))