ondio 0.2.3__tar.gz → 0.2.4__tar.gz

{ondio-0.2.3 → ondio-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ondio
-Version: 0.2.3
+Version: 0.2.4
 Summary: Uniform IO of audio data and bioacoustic model results across S3, GCS, HTTP, and local filesystems
 Author-email: Michael Catchen <mdcatchen@gmail.com>
 License-Expression: BSD-3-Clause

{ondio-0.2.3 → ondio-0.2.4}/ondio/backends/aws.py

@@ -18,6 +18,9 @@ def _split(uri: str) -> tuple[str, str]:
     parsed = urlparse(uri)
     if parsed.scheme != "s3" or not parsed.netloc:
         raise ValueError(f"not an s3:// URI: {uri!r}")
+    # urlparse would drop the text after ? or #, so the key would not be the one given.
+    if "?" in uri or "#" in uri:
+        raise ValueError(f"s3:// URIs with a query or fragment are not supported: {uri!r}")
     return parsed.netloc, parsed.path.lstrip("/")
 
 

{ondio-0.2.3 → ondio-0.2.4}/ondio/backends/gcs.py

@@ -15,6 +15,9 @@ def _split(uri: str) -> tuple[str, str]:
     parsed = urlparse(uri)
     if parsed.scheme != "gs" or not parsed.netloc:
         raise ValueError(f"not a gs:// URI: {uri!r}")
+    # urlparse would drop the text after ? or #, so the key would not be the one given.
+    if "?" in uri or "#" in uri:
+        raise ValueError(f"gs:// URIs with a query or fragment are not supported: {uri!r}")
     return parsed.netloc, parsed.path.lstrip("/")
 
 

{ondio-0.2.3 → ondio-0.2.4}/ondio/backends/local.py

@@ -12,13 +12,23 @@ from urllib.parse import unquote, urlparse
 from ondio.backends.protocol import _matches_filters
 from ondio.types import AuthError, ObjectExistsError, ObjectNotFoundError, OndioError
 
+def _is_file_uri(uri: str) -> bool:
+    return urlparse(uri).scheme.lower() == "file"
+
+
 def _to_path(uri: str) -> Path:
-    if uri.startswith("file://"):
-        parsed = urlparse(uri)
-        if parsed.netloc not in ("", "localhost"):
-            raise ValueError(f"file:// URIs with a remote host are not supported: {uri!r}")
-        return Path(unquote(parsed.path))
-    return Path(uri)
+    if not _is_file_uri(uri):
+        return Path(uri)
+    # The forms below would be read as a different path: urlparse drops a query or
+    # fragment, and without `//` the rest of the URI is a relative path.
+    if not uri.lower().startswith("file://"):
+        raise ValueError(f"file: URIs must start with file://: {uri!r}")
+    if "?" in uri or "#" in uri:
+        raise ValueError(f"file:// URIs with a query or fragment are not supported: {uri!r}")
+    parsed = urlparse(uri)
+    if parsed.netloc not in ("", "localhost"):
+        raise ValueError(f"file:// URIs with a remote host are not supported: {uri!r}")
+    return Path(unquote(parsed.path))
 
 def _wrap_oserror(uri: str, exc: OSError) -> Exception:
     if isinstance(exc, PermissionError):
@@ -118,7 +128,7 @@ class LocalBackend:
         required_prefix: str | None = None,
         required_ext: str | None = None,
     ) -> list[str]:
-        as_file_uri = uri_prefix.startswith("file://")
+        as_file_uri = _is_file_uri(uri_prefix)
         matches = (
             p
             for p in self._iter_matches(uri_prefix, recursive=recursive)

{ondio-0.2.3 → ondio-0.2.4}/ondio/dispatcher.py

@@ -3,7 +3,11 @@
 Every function accepts `**kwargs`, forwarded to the backend constructor (see
 [`get_backend`][ondio.registry.get_backend]) — not to the individual operation.
 URIs are fully qualified: `s3://…`, `gs://…`, `http(s)://…`, `file://…`, or a
-plain local path.
+plain local path. A `file:`, `s3://` or `gs://` URI with `?` or `#` is refused with
+`ValueError`, since the text after it would be dropped. A local file whose name has
+`?` or `#` is reached with `%3F` and `%23` in a `file://` URI, or with a plain local
+path, which is used as written. S3 and GCS keys are not decoded, so a key with `?`
+or `#` cannot be reached.
 """
 
 from __future__ import annotations

{ondio-0.2.3 → ondio-0.2.4}/ondio/registry.py

@@ -49,8 +49,9 @@ def locate(uri: str) -> Location:
     are used as written.
 
     Raises:
-        ValueError: For a relative local path, a `file://` URI with a remote host,
-            or an `s3://` or `gs://` URI with no bucket.
+        ValueError: For a relative local path, a `file:` URI that does not start
+            with `file://`, a `file://` URI with a remote host, an `s3://` or `gs://`
+            URI with no bucket, or a `file:`, `s3://` or `gs://` URI with `?` or `#`.
         UnsupportedOperationError: For an HTTP URL, which ondio cannot write.
         UnknownPlatformError: For an unknown scheme.
     """

{ondio-0.2.3 → ondio-0.2.4}/pyproject.toml

@@ -13,7 +13,7 @@ license = "BSD-3-Clause"
 license-files = ["LICENSE.md"]
 authors = [{ name = "Michael Catchen", email = "mdcatchen@gmail.com" }]
 requires-python = ">= 3.11"
-version = "0.2.3"
+version = "0.2.4"
 
 [project.optional-dependencies]
 gcs = ["google-cloud-storage"]

{ondio-0.2.3 → ondio-0.2.4}/tests/unit/test_aws_backend.py

@@ -292,3 +292,24 @@ def test_an_s3_location_is_the_key_ondio_writes(prefix):
     client = boto3.client("s3", region_name="us-east-1")
     body = client.get_object(Bucket=store.removeprefix("s3://"), Key=key)["Body"].read()
     assert body == b"payload"
+
+
+class TestRefusedUris:
+    @pytest.fixture(params=["?b", "#b", "?", "#"])
+    def uri(self, prefix, request):
+        return f"{prefix}/a{request.param}"
+
+    def test_read_refuses_a_query_or_fragment(self, prefix, uri):
+        ondio.write(f"{prefix}/a", b"payload")
+        with pytest.raises(ValueError):
+            ondio.read(uri)
+
+    @pytest.mark.parametrize("operation", [ondio.create, ondio.write])
+    def test_writes_refuse_a_query_or_fragment(self, prefix, uri, operation):
+        with pytest.raises(ValueError):
+            operation(uri, b"payload")
+        assert ondio.list_files(f"{prefix}/") == []
+
+    def test_an_encoded_question_mark_is_part_of_the_key(self, prefix):
+        ondio.write(f"{prefix}/a%3Fb", b"payload")
+        assert ondio.list_files(f"{prefix}/") == [f"{prefix}/a%3Fb"]

{ondio-0.2.3 → ondio-0.2.4}/tests/unit/test_gcs_backend.py

@@ -268,3 +268,14 @@ class TestErrors:
 def test_create_is_unsupported(backend):
     with pytest.raises(UnsupportedOperationError):
         backend.create(f"{PREFIX}/result.json", b"x")
+
+
+@pytest.mark.parametrize("suffix", ["?b", "#b", "?", "#"])
+def test_a_query_or_fragment_is_refused(backend, store, suffix):
+    store["a"] = b"payload"
+    uri = f"{PREFIX}/a{suffix}"
+    with pytest.raises(ValueError):
+        backend.read(uri)
+    with pytest.raises(ValueError):
+        backend.write(uri, b"other")
+    assert store["a"] == b"payload"

{ondio-0.2.3 → ondio-0.2.4}/tests/unit/test_local_dispatcher.py

@@ -63,6 +63,53 @@ class TestReadWrite:
         assert ondio.locate(uri) == Location("file://", str(target).removeprefix("/"))
 
 
+
+class TestRefusedUris:
+    @pytest.fixture(params=["?x", "#x", "?", "#"])
+    def suffix(self, request):
+        return request.param
+
+    def test_read_refuses_a_query_or_fragment(self, tmp_path, suffix):
+        target = tmp_path / "out"
+        target.write_bytes(b"payload")
+        with pytest.raises(ValueError):
+            ondio.read(target.as_uri() + suffix)
+
+    @pytest.mark.parametrize("operation", [ondio.create, ondio.write])
+    def test_writes_refuse_a_query_or_fragment(self, tmp_path, suffix, operation):
+        with pytest.raises(ValueError):
+            operation((tmp_path / "out").as_uri() + suffix, b"payload")
+        assert list(tmp_path.iterdir()) == []
+
+    @pytest.mark.parametrize("scheme", ["file:", "FILE:"])
+    def test_a_file_uri_without_two_slashes_is_refused(self, tmp_path, monkeypatch, scheme):
+        monkeypatch.chdir(tmp_path)
+        uri = scheme + str(tmp_path / "out")
+        for operation in (ondio.read, ondio.locate):
+            with pytest.raises(ValueError):
+                operation(uri)
+        for operation in (ondio.create, ondio.write):
+            with pytest.raises(ValueError):
+                operation(uri, b"payload")
+        assert list(tmp_path.iterdir()) == []
+
+    def test_a_bare_path_keeps_question_mark_and_hash(self, tmp_path):
+        target = tmp_path / "a?b#c"
+        ondio.create(str(target), b"payload")
+        assert target.read_bytes() == b"payload"
+        assert ondio.read(str(target)) == b"payload"
+        assert ondio.locate(str(target)) == Location("file://", str(target).removeprefix("/"))
+
+    def test_an_encoded_question_mark_reaches_the_file(self, tmp_path):
+        target = tmp_path / "a?b"
+        target.write_bytes(b"payload")
+        assert ondio.read(target.as_uri()) == b"payload"
+
+    def test_an_uppercase_scheme_lists_file_uris(self, tree):
+        uri = "FILE://" + tree.as_uri().removeprefix("file://")
+        assert ondio.list_files(uri) == ondio.list_files(tree.as_uri())
+
+
 class TestReadRange:
     @pytest.fixture
     def uri(self, tmp_path):

{ondio-0.2.3 → ondio-0.2.4}/tests/unit/test_registry.py

@@ -75,6 +75,14 @@ def test_create_refuses_unsupported_schemes_before_backend_setup(uri, setup_erro
         ("file:///data/%6fut", Location("file://", "data/out")),
         ("file:///data/%2e%2e/other", Location("file://", "data/../other")),
         ("file:///data/a%2F..%2Fother", Location("file://", "data/a/../other")),
+        ("file:///data/a%20b", Location("file://", "data/a b")),
+        # A ? or # reaches a local file name only percent-encoded, or in a bare path.
+        ("file:///data/a%3Fb%23c", Location("file://", "data/a?b#c")),
+        ("FILE:///data/out", Location("file://", "data/out")),
+        ("/data/a?b#c", Location("file://", "data/a?b#c")),
+        # The same text in a cloud key stays as written.
+        ("s3://bucket/a%3Fb", Location("s3://bucket", "a%3Fb")),
+        ("gs://bucket/a%3Fb", Location("gs://bucket", "a%3Fb")),
     ],
 )
 def test_locate_names_the_store_and_the_path_ondio_reads(uri, location):
@@ -99,7 +107,9 @@ def test_forms_of_one_local_folder_locate_the_same_place(uri):
     ("uri", "error"),
     [
         ("relative/path", ValueError),
-        ("file:/data/out", ValueError),  # ondio reads this as a relative path
+        ("file:/data/out", ValueError),
+        ("file:data/out", ValueError),
+        ("FILE:/data/out", ValueError),
         ("file://host/data/out", ValueError),
         ("s3:///key", ValueError),
         ("gs:///key", ValueError),
@@ -110,3 +120,28 @@ def test_forms_of_one_local_folder_locate_the_same_place(uri):
 def test_locate_refuses_a_location_with_no_fixed_place(uri, error):
     with pytest.raises(error):
         locate(uri)
+
+
+@pytest.mark.parametrize(
+    "uri",
+    [
+        "file:///data/out?x",
+        "file:///data/out#x",
+        "file:///data/out?",
+        "file:///data/out#",
+        "file://localhost/data/out?x",
+        "FILE:///data/out?x",
+        "s3://bucket/a?b",
+        "s3://bucket/a#b",
+        "s3://bucket/a?",
+        "s3://bucket/a#",
+        "gs://bucket/a?b",
+        "gs://bucket/a#b",
+        "gs://bucket/a?",
+        "gs://bucket/a#",
+    ],
+)
+def test_locate_refuses_a_query_or_fragment(uri):
+    # urlparse would drop the text after ? or #, so ondio would name another path.
+    with pytest.raises(ValueError):
+        locate(uri)