vecorel-cli 0.4.0__tar.gz → 0.4.1__tar.gz

{vecorel_cli-0.4.0/vecorel_cli.egg-info → vecorel_cli-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vecorel-cli
-Version: 0.4.0
+Version: 0.4.1
 Summary: CLI tools such as validation and file format conversion for vecorel.
 Author: Matthias Mohr
 License-Expression: Apache-2.0

{vecorel_cli-0.4.0 → vecorel_cli-0.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vecorel-cli"
-version = "0.4.0"
+version = "0.4.1"
 description = "CLI tools such as validation and file format conversion for vecorel."
 readme = "README.md"
 license = "Apache-2.0"

{vecorel_cli-0.4.0 → vecorel_cli-0.4.1}/tests/test_convert_duckdb.py

@@ -1,6 +1,7 @@
 import json
 import re
 import sys
+import zipfile
 
 import geopandas as gpd
 import numpy as np
@@ -109,6 +110,48 @@ def test_duckdb_converter(tmp_folder):
     assert validation.errors == []
 
 
+def test_duckdb_converter_extracts_an_archive_without_a_cache(tmp_folder):
+    src = _source_file(tmp_folder)
+    archive = tmp_folder / "source.zip"
+    with zipfile.ZipFile(archive, "w") as zf:
+        zf.write(src, "source.parquet")
+    dest = tmp_folder / "converted.parquet"
+
+    Converter().convert(dest, input_files={str(archive): ["source.parquet"]})
+
+    assert sorted(gpd.read_parquet(dest)["id"]) == ["bowtie", "multi", "square", "with_z"]
+
+
+def test_duckdb_converter_ignores_hive_style_folders(tmp_folder):
+    source_folder = tmp_folder / "region=north"
+    source_folder.mkdir()
+    src = _source_file(source_folder)
+    dest_folder = tmp_folder / "year=2024"
+    dest_folder.mkdir()
+    dest = dest_folder / "converted.parquet"
+
+    Converter().convert(dest, input_files={src: "source.parquet"})
+
+    with pq.ParquetFile(dest) as pf:
+        names = pf.schema_arrow.names
+    assert "year" not in names
+    assert "region" not in names
+
+
+@pytest.mark.parametrize(
+    "urls, in_place",
+    [
+        ("https://host/data.parquet?token=abc", True),
+        ("https://host/data.geoparquet", True),
+        ({"https://host/x?id=1": "fields.parquet"}, True),
+        ({"https://host/data.zip": ["fields.parquet"]}, False),
+        ("https://host/data.fgb", False),
+    ],
+)
+def test_only_parquet_is_read_in_place(urls, in_place):
+    assert duckdb_module._readable_in_place(urls) is in_place
+
+
 def test_duckdb_converter_numbers_rows_without_an_id_mapping(tmp_folder):
     src = _source_file(tmp_folder)
     dest = tmp_folder / "converted.parquet"

{vecorel_cli-0.4.0 → vecorel_cli-0.4.1}/tests/test_download.py

@@ -7,7 +7,7 @@ from fsspec.implementations.local import LocalFileSystem
 
 import vecorel_cli.conversion.base as base
 from vecorel_cli.conversion.base import BaseConverter
-from vecorel_cli.vecorel.util import stream_file
+from vecorel_cli.vecorel.util import get_fs, merge_headers, stream_file
 
 URI = "https://example.invalid/data/fields.gml"
 
@@ -192,3 +192,56 @@ def test_multivolume_7z_is_downloaded_and_extracted(explicit_cache, tmp_folder,
         # every volume was cached under its own name
         for part in parts:
             assert (tmp_folder / part).exists()
+
+
+def test_request_headers_go_with_every_download(tmp_folder, monkeypatch):
+    seen = []
+    source_fs = _FakeSourceFS(b"body", {})
+
+    def fake_get_fs(uri, **kwargs):
+        if str(uri).startswith(("http://", "https://")):
+            seen.append(kwargs.get("client_kwargs"))
+            return source_fs
+        return LocalFileSystem()
+
+    monkeypatch.setattr(base, "get_fs", fake_get_fs)
+    HeaderConverter = type(
+        "HeaderConverter", (BaseConverter,), {"request_headers": {"X-Test": "1"}}
+    )
+
+    HeaderConverter().download_files(URI, cache_folder=str(tmp_folder))
+
+    assert seen == [{"headers": {"X-Test": "1"}}]
+
+
+def test_request_headers_merge_with_explicit_client_options(tmp_folder, monkeypatch):
+    seen = []
+    source_fs = _FakeSourceFS(b"body", {})
+
+    def fake_get_fs(uri, **kwargs):
+        if str(uri).startswith(("http://", "https://")):
+            seen.append(kwargs.get("client_kwargs"))
+            return source_fs
+        return LocalFileSystem()
+
+    monkeypatch.setattr(base, "get_fs", fake_get_fs)
+    HeaderConverter = type(
+        "HeaderConverter", (BaseConverter,), {"request_headers": {"X-Test": "1", "X-Kept": "k"}}
+    )
+    client_kwargs = {"trust_env": True, "headers": {"x-test": "2"}}
+
+    HeaderConverter().download_files(URI, cache_folder=str(tmp_folder), client_kwargs=client_kwargs)
+
+    assert seen == [{"trust_env": True, "headers": {"X-Kept": "k", "x-test": "2"}}]
+
+
+def test_headers_merge_case_insensitively():
+    merged = merge_headers(
+        {"User-Agent": "default", "X-Test": "1"}, {"user-agent": "mine", "x-test": "2"}
+    )
+    assert merged == {"user-agent": "mine", "x-test": "2"}
+
+
+def test_a_converter_user_agent_replaces_the_default():
+    fs = get_fs("https://example.invalid/x", client_kwargs={"headers": {"user-agent": "browser"}})
+    assert fs.client_kwargs["headers"] == {"user-agent": "browser"}

{vecorel_cli-0.4.0 → vecorel_cli-0.4.1}/vecorel_cli/conversion/base.py

@@ -33,7 +33,7 @@ from ..vecorel.collection import Collection
 from ..vecorel.hilbert import hilbert_sort_geodataframe
 from ..vecorel.schemas import Schemas
 from ..vecorel.typing import Sources
-from ..vecorel.util import get_fs, name_from_uri, stream_file, suffix_duplicate_ids
+from ..vecorel.util import get_fs, merge_headers, name_from_uri, stream_file, suffix_duplicate_ids
 from .deflate64 import deflate64_zip
 from .flatdict import FlatDict
 
@@ -58,6 +58,8 @@ class BaseConverter(LoggerMixin):
     data_access: str = ""
     open_options: dict = {}
     avoid_range_request: bool = False
+    # Sent with every download, e.g. a browser User-Agent for a server that refuses others
+    request_headers: dict[str, str] = {}
     variants: dict[str, Sources] = {}
     variant: Optional[str] = None
 
@@ -365,6 +367,11 @@ class BaseConverter(LoggerMixin):
 
         if self.avoid_range_request and "block_size" not in kwargs:
             kwargs["block_size"] = 0
+        if self.request_headers:
+            # headers passed in explicitly win over the converter's
+            client_kwargs = kwargs.get("client_kwargs") or {}
+            headers = merge_headers(self.request_headers, client_kwargs.get("headers", {}))
+            kwargs["client_kwargs"] = {**client_kwargs, "headers": headers}
 
         # Multi-volume 7z archives (.7z.001, .7z.002, ...) are one 7z stream split
         # across several URIs; they must be downloaded and extracted together.

{vecorel_cli-0.4.0 → vecorel_cli-0.4.1}/vecorel_cli/conversion/duckdb.py

@@ -21,7 +21,7 @@ from ..vecorel.ops import (
     report,
     warn_missing_required,
 )
-from ..vecorel.util import find_differing_crs
+from ..vecorel.util import find_differing_crs, name_from_uri
 from .base import BaseConverter
 
 
@@ -32,6 +32,15 @@ def _sql_path(path) -> str:
     return f"'{escaped}'"
 
 
+def _readable_in_place(urls) -> bool:
+    if isinstance(urls, str):
+        urls = {urls: name_from_uri(urls)}
+    return all(
+        isinstance(target, str) and target.lower().endswith((".parquet", ".geoparquet"))
+        for target in urls.values()
+    )
+
+
 def _sql_name(name) -> str:
     escaped = str(name).replace('"', '""')
     return f'"{escaped}"'
@@ -116,7 +125,9 @@ class DuckDBBaseConverter(BaseConverter):
                 raise ValueError("No input files provided")
 
         self.info("Getting file(s) if not cached yet")
-        if cache:
+        # Without a cache DuckDB reads remote parquet files in place; anything else, such as
+        # an archive, is downloaded to a temporary folder as the GeoDataFrame codepath does
+        if cache or not _readable_in_place(urls):
             request_args = {}
             if self.avoid_range_request:
                 request_args["block_size"] = 0
@@ -140,7 +151,8 @@ class DuckDBBaseConverter(BaseConverter):
         available = {
             row[0]: row[1]
             for row in con.execute(
-                "DESCRIBE SELECT * FROM read_parquet(?, union_by_name=true)", [sources]
+                "DESCRIBE SELECT * FROM read_parquet(?, union_by_name=true, hive_partitioning=false)",
+                [sources],
             ).fetchall()
         }
 
@@ -190,7 +202,7 @@ class DuckDBBaseConverter(BaseConverter):
                     # migrated type so integer-valued floats format the same way
                     expr = f"({self.column_migrations[c]})"
                     sql_type = con.execute(
-                        f"DESCRIBE SELECT {expr} AS part FROM read_parquet(?, union_by_name=true)",
+                        f"DESCRIBE SELECT {expr} AS part FROM read_parquet(?, union_by_name=true, hive_partitioning=false)",
                         [sources],
                     ).fetchone()[1]
                     parts.append(self._stringify_sql(expr, sql_type, c))
@@ -260,7 +272,7 @@ class DuckDBBaseConverter(BaseConverter):
             sources_sql = "[" + ",".join(_sql_path(path) for path in sources) + "]"
         source_query = f"""
             SELECT {selection}
-            FROM read_parquet({sources_sql}, union_by_name=true)
+            FROM read_parquet({sources_sql}, union_by_name=true, hive_partitioning=false)
             {where}
         """
 
@@ -618,7 +630,7 @@ class DuckDBBaseConverter(BaseConverter):
                 f"""
                 SELECT coalesce(sum(n), 0), coalesce(sum(n) FILTER (WHERE n > 1), 0)
                 FROM (
-                    SELECT count(*) AS n FROM read_parquet({_sql_path(output_file)})
+                    SELECT count(*) AS n FROM read_parquet({_sql_path(output_file)}, hive_partitioning=false)
                     WHERE "id" IS NOT NULL GROUP BY {key}
                 )
                 """
@@ -648,7 +660,7 @@ class DuckDBBaseConverter(BaseConverter):
                                   AS VARCHAR)
                              ELSE CAST("id" AS VARCHAR)
                         END AS "id")
-                      FROM read_parquet({_sql_path(output_file)}, file_row_number=true)
+                      FROM read_parquet({_sql_path(output_file)}, file_row_number=true, hive_partitioning=false)
                     ) TO ? (
                         FORMAT parquet,
                         ROW_GROUP_SIZE {row_group_size},
@@ -765,7 +777,7 @@ class DuckDBBaseConverter(BaseConverter):
                     columns.append(_sql_name(name))
             per_part = per_part or coalesce or bool(constants)
             query = f"SELECT {', '.join(columns)}"
-            source = f"read_parquet({_sql_path(path)})"
+            source = f"read_parquet({_sql_path(path)}, hive_partitioning=false)"
             if constants:
                 # A registered table rather than SQL literals, so arrays, objects and
                 # temporal values keep their types
@@ -783,7 +795,7 @@ class DuckDBBaseConverter(BaseConverter):
             # nothing to add to any part, so a single scan over all of them
             sources = "[" + ",".join(_sql_path(path) for path in paths) + "]"
             columns = ", ".join(_sql_name(name) for name in union)
-            source_query = f"SELECT {columns} FROM read_parquet({sources}, union_by_name=true)"
+            source_query = f"SELECT {columns} FROM read_parquet({sources}, union_by_name=true, hive_partitioning=false)"
         targets = [row[0] for row in con.execute(f"DESCRIBE {source_query}").fetchall()]
 
         return self.write_query(
@@ -850,10 +862,10 @@ class DuckDBBaseConverter(BaseConverter):
     ):
         directory = os.path.dirname(output_file) or "."
         tmp_path = None
-        query = f"SELECT d.* FROM read_parquet({_sql_path(output_file)}) d"
+        query = f"SELECT d.* FROM read_parquet({_sql_path(output_file)}, hive_partitioning=false) d"
         if keys_path:
             query += f"""
-              POSITIONAL JOIN read_parquet({_sql_path(keys_path)}) k
+              POSITIONAL JOIN read_parquet({_sql_path(keys_path)}, hive_partitioning=false) k
               ORDER BY k.hilbert, k.ordinal
             """
         try:

{vecorel_cli-0.4.0 → vecorel_cli-0.4.1}/vecorel_cli/vecorel/util.py

@@ -120,6 +120,17 @@ def _expected_content_length(f) -> Optional[int]:
         return None
 
 
+def merge_headers(*headers: dict) -> dict:
+    """Merge HTTP headers; names are case-insensitive, and a later one replaces an earlier."""
+    merged = {}
+    for group in headers:
+        for name, value in group.items():
+            for existing in [k for k in merged if k.lower() == name.lower()]:
+                del merged[existing]
+            merged[name] = value
+    return merged
+
+
 def get_fs(url_or_path: Union[str, Path, URL], **kwargs) -> AbstractFileSystem:
     """Choose fsspec filesystem by sniffing input url"""
     if isinstance(url_or_path, Path):
@@ -130,7 +141,7 @@ def get_fs(url_or_path: Union[str, Path, URL], **kwargs) -> AbstractFileSystem:
 
     if parsed.scheme in ("http", "https"):
         client_kwargs = kwargs.pop("client_kwargs", {})
-        headers = {"User-Agent": USER_AGENT, **client_kwargs.get("headers", {})}
+        headers = merge_headers({"User-Agent": USER_AGENT}, client_kwargs.get("headers", {}))
         return HTTPFileSystem(client_kwargs={**client_kwargs, "headers": headers}, **kwargs)
 
     if parsed.scheme == "s3":

{vecorel_cli-0.4.0 → vecorel_cli-0.4.1/vecorel_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vecorel-cli
-Version: 0.4.0
+Version: 0.4.1
 Summary: CLI tools such as validation and file format conversion for vecorel.
 Author: Matthias Mohr
 License-Expression: Apache-2.0