pos3 0.2.0__tar.gz → 0.2.2__tar.gz

{pos3-0.2.0 → pos3-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pos3
-Version: 0.2.0
+Version: 0.2.2
 Summary: S3 Simple Sync - Make using S3 as simple as using local files
 Author-email: Positronic Robotics <hi@positronic.ro>
 License: Apache-2.0

{pos3-0.2.0 → pos3-0.2.2}/pos3/__init__.py

@@ -142,6 +142,17 @@ def _s3_paths_conflict(left: str, right: str) -> bool:
     return left_norm.startswith(right_norm + "/") or right_norm.startswith(left_norm + "/")
 
 
+def _make_s3_key(prefix: str, info: FileInfo) -> str:
+    """Build the canonical S3 key from a prefix and FileInfo, including trailing '/' for directories."""
+    if info.relative_path:
+        key = prefix + "/" + info.relative_path if prefix else info.relative_path
+    else:
+        key = prefix
+    if info.is_dir and not key.endswith("/"):
+        key += "/"
+    return key
+
+
 def _process_futures(futures, operation: str) -> None:
     for future in futures:
         try:
@@ -618,12 +629,12 @@ class _Mirror:
             )
 
             for info in to_copy:
-                s3_key = prefix + ("/" + info.relative_path if info.relative_path else "")
+                s3_key = _make_s3_key(prefix, info)
                 to_put.append((info, local_path, bucket, s3_key, profile))
                 total_bytes += info.size
 
             for info in to_delete if delete else []:
-                s3_key = prefix + ("/" + info.relative_path if info.relative_path else "")
+                s3_key = _make_s3_key(prefix, info)
                 to_remove.append((bucket, s3_key, profile))
 
         if to_put:
@@ -679,7 +690,7 @@ class _Mirror:
         total_bytes = 0
 
         for info in to_copy:
-            s3_key = prefix + ("/" + info.relative_path if info.relative_path else "")
+            s3_key = _make_s3_key(prefix, info)
             to_put.append((info, bucket, s3_key, local_path))
             total_bytes += info.size
 
@@ -709,26 +720,35 @@ class _Mirror:
     def _list_s3_objects(self, bucket: str, key: str, profile: Profile | None = None) -> Iterator[dict]:
         logger.debug("Listing S3 objects: bucket=%s, key=%s", bucket, key)
         client = self._get_client(profile)
-        # Skip head_object for directory-like keys ending with '/'
-        # as we want to list contents, not check if the directory marker exists
-        if not key.endswith("/"):
+
+        # Determine the listing prefix - ensure it ends with "/" for directory-like operations
+        # This prevents "droid/recovery" from matching "droid/recovery_towels"
+        list_prefix = key
+
+        # If key doesn't end with "/", try to fetch it as a single object first
+        if key and not key.endswith("/"):
             try:
                 obj = client.head_object(Bucket=bucket, Key=key)
             except ClientError as exc:
                 error_code = exc.response["Error"]["Code"]
                 if error_code != "404":
                     raise
+                # Not a single file - treat as directory by adding "/"
+                list_prefix = key + "/"
             else:
+                # Found single object
                 logger.debug("Found single object via head_object: %s", key)
                 if "ContentLength" in obj and "Size" not in obj:
                     obj["Size"] = obj["ContentLength"]
                 yield {**obj, "Key": key}
                 return
+        # If key already ends with "/", skip head_object - it's clearly a directory prefix
 
+        # List with the directory prefix (guaranteed to end with "/")
         paginator = client.get_paginator("list_objects_v2")
-        for page in paginator.paginate(Bucket=bucket, Prefix=key):
+        for page in paginator.paginate(Bucket=bucket, Prefix=list_prefix):
             objects = page.get("Contents", [])
-            logger.debug("Listed %d objects with prefix %s", len(objects), key)
+            logger.debug("Listed %d objects with prefix %s", len(objects), list_prefix)
             yield from objects
 
     def _scan_s3(self, bucket: str, prefix: str, profile: Profile | None = None) -> Iterator[FileInfo]:
@@ -773,7 +793,6 @@ class _Mirror:
         try:
             client = self._get_client(profile)
             if info.is_dir:
-                key += "/" if not key.endswith("/") else ""
                 client.put_object(Bucket=bucket, Key=key, Body=b"")
             else:
                 file_path = local_path / info.relative_path if info.relative_path else local_path

{pos3-0.2.0 → pos3-0.2.2}/pos3.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pos3
-Version: 0.2.0
+Version: 0.2.2
 Summary: S3 Simple Sync - Make using S3 as simple as using local files
 Author-email: Positronic Robotics <hi@positronic.ro>
 License: Apache-2.0

{pos3-0.2.0 → pos3-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pos3"
-version = "0.2.0"
+version = "0.2.2"
 description = "S3 Simple Sync - Make using S3 as simple as using local files"
 readme = "README.md"
 requires-python = ">=3.9"

{pos3-0.2.0 → pos3-0.2.2}/tests/test_s3.py

@@ -29,6 +29,43 @@ def _setup_s3_mock(mock_boto_client, paginate_return_value=None):
     return mock_s3
 
 
+class TestMakeS3Key:
+    def test_file_with_prefix(self):
+        info = s3.FileInfo(relative_path="file.txt", size=100, is_dir=False)
+        assert s3._make_s3_key("data", info) == "data/file.txt"
+
+    def test_dir_with_prefix(self):
+        info = s3.FileInfo(relative_path="subdir", size=0, is_dir=True)
+        assert s3._make_s3_key("data", info) == "data/subdir/"
+
+    def test_root_dir(self):
+        info = s3.FileInfo(relative_path="", size=0, is_dir=True)
+        assert s3._make_s3_key("data", info) == "data/"
+
+    def test_empty_prefix_file(self):
+        info = s3.FileInfo(relative_path="file.txt", size=100, is_dir=False)
+        assert s3._make_s3_key("", info) == "file.txt"
+
+    def test_empty_prefix_dir(self):
+        info = s3.FileInfo(relative_path="subdir", size=0, is_dir=True)
+        assert s3._make_s3_key("", info) == "subdir/"
+
+    def test_nested_path(self):
+        info = s3.FileInfo(relative_path="a/b/c.txt", size=50, is_dir=False)
+        assert s3._make_s3_key("prefix", info) == "prefix/a/b/c.txt"
+
+    def test_nested_dir(self):
+        info = s3.FileInfo(relative_path="a/b", size=0, is_dir=True)
+        assert s3._make_s3_key("prefix", info) == "prefix/a/b/"
+
+    def test_dir_already_trailing_slash_prefix(self):
+        """Prefix with trailing slash in relative_path shouldn't double-slash."""
+        info = s3.FileInfo(relative_path="sub/", size=0, is_dir=True)
+        result = s3._make_s3_key("data", info)
+        assert result == "data/sub/"
+        assert "//" not in result
+
+
 class TestS3URLParsing:
     def test_parse_s3_url_valid(self):
         assert s3._parse_s3_url("s3://bucket/path/to/data") == (
@@ -159,6 +196,36 @@ class TestUpload:
         assert mock_s3.upload_file.call_count >= 1
         assert mock_s3.delete_object.call_count == 1
 
+    @patch(BOTO3_PATCH_TARGET)
+    def test_upload_delete_directory_marker_trailing_slash(self, mock_boto_client):
+        """Test that deleting a directory from S3 uses trailing slash to match directory markers."""
+        # S3 has a directory marker "output/subdir/" and a file "output/file.txt"
+        paginate = [
+            {
+                "Contents": [
+                    {"Key": "output/subdir/", "Size": 0},  # Directory marker with trailing slash
+                    {"Key": "output/file.txt", "Size": 5},
+                ]
+            }
+        ]
+        mock_s3 = _setup_s3_mock(mock_boto_client, paginate)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            output = Path(tmpdir) / "output"
+            output.mkdir()
+            # Local only has file.txt - subdir was deleted locally
+            (output / "file.txt").write_text("content")
+
+            with s3.mirror(cache_root=tmpdir, show_progress=False):
+                s3.upload("s3://bucket/output", local=output, interval=None)
+
+        # The directory marker should be deleted with trailing slash
+        delete_calls = mock_s3.delete_object.call_args_list
+        deleted_keys = [call[1]["Key"] for call in delete_calls]
+        assert "output/subdir/" in deleted_keys, (
+            f"Expected delete of 'output/subdir/' but got: {deleted_keys}"
+        )
+
     @patch(BOTO3_PATCH_TARGET)
     def test_background_sync_uploads_repeatedly(self, mock_boto_client):
         mock_s3 = _setup_s3_mock(mock_boto_client)
@@ -871,6 +938,116 @@ class TestExclude:
             assert "file.txt" in call_args[1]
 
 
+class TestPrefixBoundaryMatching:
+    @patch(BOTO3_PATCH_TARGET)
+    def test_prefix_boundary_prevents_spurious_matches(self, mock_boto_client):
+        """Test that S3 prefix matching respects path boundaries.
+
+        When downloading s3://bucket/data/, should NOT match s3://bucket/data_backup/
+        This is a regression test for the bug where "droid/recovery" matched "droid/recovery_towels"
+        """
+        mock_s3 = _setup_s3_mock(mock_boto_client)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            with s3.mirror(cache_root=tmpdir, show_progress=False):
+                mirror_obj = s3._require_active_mirror()
+
+                # Simulate listing objects - should add "/" to prefix when key doesn't end with "/"
+                _ = list(mirror_obj._list_s3_objects("bucket", "data", None))
+
+                # Verify that paginate was called with "data/" (with trailing slash)
+                paginator_calls = mock_s3.get_paginator.return_value.paginate.call_args_list
+                assert len(paginator_calls) == 1
+                call_kwargs = paginator_calls[0][1]
+                assert (
+                    call_kwargs["Prefix"] == "data/"
+                ), f"Expected Prefix='data/' but got Prefix='{call_kwargs['Prefix']}'"
+
+    @patch(BOTO3_PATCH_TARGET)
+    def test_prefix_boundary_with_trailing_slash(self, mock_boto_client):
+        """Test that keys already ending with '/' don't get double slashes."""
+        mock_s3 = _setup_s3_mock(mock_boto_client)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            with s3.mirror(cache_root=tmpdir, show_progress=False):
+                mirror_obj = s3._require_active_mirror()
+
+                # List with trailing slash already present
+                _ = list(mirror_obj._list_s3_objects("bucket", "data/", None))
+
+                # Should use "data/" as-is, not "data//"
+                paginator_calls = mock_s3.get_paginator.return_value.paginate.call_args_list
+                assert len(paginator_calls) == 1
+                call_kwargs = paginator_calls[0][1]
+                assert call_kwargs["Prefix"] == "data/"
+
+    @patch(BOTO3_PATCH_TARGET)
+    def test_single_file_download_bypasses_list(self, mock_boto_client):
+        """Test that single file downloads use head_object and don't list with prefix."""
+        mock_s3 = Mock()
+        mock_boto_client.return_value = mock_s3
+
+        # Mock head_object to return a valid file
+        mock_s3.head_object.return_value = {"ContentLength": 1234, "ETag": "abc123"}
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            with s3.mirror(cache_root=tmpdir, show_progress=False):
+                mirror_obj = s3._require_active_mirror()
+
+                # List a single file (no trailing slash)
+                results = list(mirror_obj._list_s3_objects("bucket", "data/file.txt", None))
+
+                # Should have called head_object and returned the file
+                assert mock_s3.head_object.call_count == 1
+                assert len(results) == 1
+                assert results[0]["Key"] == "data/file.txt"
+                assert results[0]["Size"] == 1234
+
+                # Should NOT have called paginate
+                assert mock_s3.get_paginator.call_count == 0
+
+    @patch(BOTO3_PATCH_TARGET)
+    def test_directory_without_trailing_slash_gets_slash_added(self, mock_boto_client):
+        """Test that downloading a directory without trailing slash still works correctly.
+
+        User scenario: download('s3://bucket/my_dir') where my_dir is a directory.
+        The fix should:
+        1. Try head_object('my_dir') first
+        2. Get 404 (not a single file)
+        3. Add trailing slash and list with Prefix='my_dir/'
+        4. Only match 'my_dir/*', NOT 'my_dir_backup/*'
+        """
+        mock_s3 = _setup_s3_mock(mock_boto_client)
+
+        # Mock paginator to return directory contents
+        mock_paginator = Mock()
+        mock_s3.get_paginator.return_value = mock_paginator
+        mock_paginator.paginate.return_value = [
+            {"Contents": [{"Key": "my_dir/file1.txt", "Size": 100}, {"Key": "my_dir/file2.txt", "Size": 200}]}
+        ]
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            with s3.mirror(cache_root=tmpdir, show_progress=False):
+                mirror_obj = s3._require_active_mirror()
+
+                # User downloads directory without trailing slash (after normalization: key="my_dir")
+                results = list(mirror_obj._list_s3_objects("bucket", "my_dir", None))
+
+                # Should have tried head_object first
+                assert mock_s3.head_object.call_count == 1
+                head_call_key = mock_s3.head_object.call_args[1]["Key"]
+                assert head_call_key == "my_dir"
+
+                # After getting 404, should have listed with trailing slash
+                paginate_calls = mock_paginator.paginate.call_args_list
+                assert len(paginate_calls) == 1
+                prefix_used = paginate_calls[0][1]["Prefix"]
+                assert prefix_used == "my_dir/", f"Expected 'my_dir/' but got '{prefix_used}'"
+
+                # Should have returned the directory contents
+                assert len(results) == 2
+
+
 class TestProfile:
     def setup_method(self):
         """Clear registered profiles before each test."""