vecorel-cli 0.4.0__tar.gz → 0.4.2__tar.gz

{vecorel_cli-0.4.0/vecorel_cli.egg-info → vecorel_cli-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vecorel-cli
-Version: 0.4.0
+Version: 0.4.2
 Summary: CLI tools such as validation and file format conversion for vecorel.
 Author: Matthias Mohr
 License-Expression: Apache-2.0

{vecorel_cli-0.4.0 → vecorel_cli-0.4.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vecorel-cli"
-version = "0.4.0"
+version = "0.4.2"
 description = "CLI tools such as validation and file format conversion for vecorel."
 readme = "README.md"
 license = "Apache-2.0"

{vecorel_cli-0.4.0 → vecorel_cli-0.4.2}/tests/test_convert_duckdb.py

@@ -1,6 +1,7 @@
 import json
 import re
 import sys
+import zipfile
 
 import geopandas as gpd
 import numpy as np
@@ -9,6 +10,7 @@ import pytest
 import shapely
 from loguru import logger
 
+from vecorel_cli.const import COMPRESSION_METHODS
 from vecorel_cli.conversion import duckdb as duckdb_module
 from vecorel_cli.conversion.base import BaseConverter
 from vecorel_cli.conversion.duckdb import DuckDBBaseConverter, _constants_table
@@ -109,6 +111,48 @@ def test_duckdb_converter(tmp_folder):
     assert validation.errors == []
 
 
+def test_duckdb_converter_extracts_an_archive_without_a_cache(tmp_folder):
+    src = _source_file(tmp_folder)
+    archive = tmp_folder / "source.zip"
+    with zipfile.ZipFile(archive, "w") as zf:
+        zf.write(src, "source.parquet")
+    dest = tmp_folder / "converted.parquet"
+
+    Converter().convert(dest, input_files={str(archive): ["source.parquet"]})
+
+    assert sorted(gpd.read_parquet(dest)["id"]) == ["bowtie", "multi", "square", "with_z"]
+
+
+def test_duckdb_converter_ignores_hive_style_folders(tmp_folder):
+    source_folder = tmp_folder / "region=north"
+    source_folder.mkdir()
+    src = _source_file(source_folder)
+    dest_folder = tmp_folder / "year=2024"
+    dest_folder.mkdir()
+    dest = dest_folder / "converted.parquet"
+
+    Converter().convert(dest, input_files={src: "source.parquet"})
+
+    with pq.ParquetFile(dest) as pf:
+        names = pf.schema_arrow.names
+    assert "year" not in names
+    assert "region" not in names
+
+
+@pytest.mark.parametrize(
+    "urls, in_place",
+    [
+        ("https://host/data.parquet?token=abc", True),
+        ("https://host/data.geoparquet", True),
+        ({"https://host/x?id=1": "fields.parquet"}, True),
+        ({"https://host/data.zip": ["fields.parquet"]}, False),
+        ("https://host/data.fgb", False),
+    ],
+)
+def test_only_parquet_is_read_in_place(urls, in_place):
+    assert duckdb_module._readable_in_place(urls) is in_place
+
+
 def test_duckdb_converter_numbers_rows_without_an_id_mapping(tmp_folder):
     src = _source_file(tmp_folder)
     dest = tmp_folder / "converted.parquet"
@@ -520,6 +564,20 @@ def test_merge_parquet(tmp_folder):
     assert ValidateData().validate(dest, num=100, schema_map={}).errors == []
 
 
+@pytest.mark.parametrize("compression", COMPRESSION_METHODS)
+def test_every_compression_method_is_written(tmp_folder, compression):
+    src = _source_file(tmp_folder)
+    part = tmp_folder / "part.parquet"
+    Converter().convert(part, input_files={src: "source.parquet"}, compression=compression)
+    dest = tmp_folder / "merged.parquet"
+    Converter().merge_parquet([part], dest, compression=compression)
+
+    for path in (part, dest):
+        with pq.ParquetFile(path) as pf:
+            codec = pf.metadata.row_group(0).column(0).compression
+        assert codec == ("UNCOMPRESSED" if compression == "none" else compression.upper())
+
+
 def test_merge_parquet_sees_an_id_that_repeats_across_parts(tmp_folder, capsys):
     """A per-part check cannot see this; a check over the merge can."""
     parts = []

{vecorel_cli-0.4.0 → vecorel_cli-0.4.2}/tests/test_download.py

@@ -7,7 +7,7 @@ from fsspec.implementations.local import LocalFileSystem
 
 import vecorel_cli.conversion.base as base
 from vecorel_cli.conversion.base import BaseConverter
-from vecorel_cli.vecorel.util import stream_file
+from vecorel_cli.vecorel.util import get_fs, merge_headers, stream_file
 
 URI = "https://example.invalid/data/fields.gml"
 
@@ -192,3 +192,56 @@ def test_multivolume_7z_is_downloaded_and_extracted(explicit_cache, tmp_folder,
         # every volume was cached under its own name
         for part in parts:
             assert (tmp_folder / part).exists()
+
+
+def test_request_headers_go_with_every_download(tmp_folder, monkeypatch):
+    seen = []
+    source_fs = _FakeSourceFS(b"body", {})
+
+    def fake_get_fs(uri, **kwargs):
+        if str(uri).startswith(("http://", "https://")):
+            seen.append(kwargs.get("client_kwargs"))
+            return source_fs
+        return LocalFileSystem()
+
+    monkeypatch.setattr(base, "get_fs", fake_get_fs)
+    HeaderConverter = type(
+        "HeaderConverter", (BaseConverter,), {"request_headers": {"X-Test": "1"}}
+    )
+
+    HeaderConverter().download_files(URI, cache_folder=str(tmp_folder))
+
+    assert seen == [{"headers": {"X-Test": "1"}}]
+
+
+def test_request_headers_merge_with_explicit_client_options(tmp_folder, monkeypatch):
+    seen = []
+    source_fs = _FakeSourceFS(b"body", {})
+
+    def fake_get_fs(uri, **kwargs):
+        if str(uri).startswith(("http://", "https://")):
+            seen.append(kwargs.get("client_kwargs"))
+            return source_fs
+        return LocalFileSystem()
+
+    monkeypatch.setattr(base, "get_fs", fake_get_fs)
+    HeaderConverter = type(
+        "HeaderConverter", (BaseConverter,), {"request_headers": {"X-Test": "1", "X-Kept": "k"}}
+    )
+    client_kwargs = {"trust_env": True, "headers": {"x-test": "2"}}
+
+    HeaderConverter().download_files(URI, cache_folder=str(tmp_folder), client_kwargs=client_kwargs)
+
+    assert seen == [{"trust_env": True, "headers": {"X-Kept": "k", "x-test": "2"}}]
+
+
+def test_headers_merge_case_insensitively():
+    merged = merge_headers(
+        {"User-Agent": "default", "X-Test": "1"}, {"user-agent": "mine", "x-test": "2"}
+    )
+    assert merged == {"user-agent": "mine", "x-test": "2"}
+
+
+def test_a_converter_user_agent_replaces_the_default():
+    fs = get_fs("https://example.invalid/x", client_kwargs={"headers": {"user-agent": "browser"}})
+    assert fs.client_kwargs["headers"] == {"user-agent": "browser"}

vecorel_cli-0.4.2/tests/test_per_file.py

@@ -0,0 +1,95 @@
+import io
+import sys
+
+import geopandas as gpd
+import multivolumefile
+import py7zr
+import shapely
+from loguru import logger
+
+from vecorel_cli.conversion.base import BaseConverter
+from vecorel_cli.conversion.per_file import PerFileConverterMixin
+
+
+class Converter(PerFileConverterMixin, BaseConverter):
+    id = "test"
+    short_name = "Test"
+    title = "Test dataset"
+    description = "Test dataset"
+    license = "CC0-1.0"
+    variants = {"2024": None, "2025": None}
+    columns = {"geometry": "geometry", "id": "id", "region": "region"}
+    missing_schemas = {"properties": {"region": {"type": "string"}}}
+
+
+def _source(folder, region):
+    gdf = gpd.GeoDataFrame(
+        {
+            "id": [f"{region}-1", f"{region}-2"],
+            "region": [region, region],
+            "geometry": [shapely.box(0, 0, 1, 1), shapely.box(2, 0, 3, 1)],
+        },
+        crs="EPSG:4326",
+    )
+    path = folder / f"{region}.parquet"
+    gdf.to_parquet(path)
+    return path
+
+
+def _capture_log():
+    # loguru's default sink is bound to stderr at import, so point it at stdout
+    logger.remove()
+    logger.add(sys.stdout, format="{message}", level="DEBUG", colorize=False)
+
+
+def test_each_source_is_converted_on_its_own_and_merged(tmp_path, capsys):
+    sources = [_source(tmp_path, region) for region in ("a", "b")]
+    output = tmp_path / "out" / "merged.parquet"
+    _capture_log()
+
+    Converter().convert(str(output), input_files={str(s): s.name for s in sources})
+
+    merged = gpd.read_parquet(output)
+    assert sorted(merged["id"]) == ["a-1", "a-2", "b-1", "b-2"]
+    # constant within each part, so it only survives the merge as a column
+    assert sorted(merged["region"].unique()) == ["a", "b"]
+    assert [p.name for p in output.parent.iterdir()] == ["merged.parquet"]  # the parts are gone
+
+    log = capsys.readouterr().out
+    assert log.count("Using user provided input file(s)") == 1
+    assert log.count("No variant given") == 1
+
+
+def test_a_single_source_is_converted_directly(tmp_path):
+    source = _source(tmp_path, "a")
+    output = tmp_path / "single.parquet"
+
+    Converter().convert(str(output), input_files={str(source): source.name})
+
+    assert sorted(gpd.read_parquet(output)["id"]) == ["a-1", "a-2"]
+
+
+def test_the_volumes_of_a_7z_archive_are_converted_together(tmp_path):
+    archived = _source(tmp_path, "b")
+    measure = io.BytesIO()
+    with py7zr.SevenZipFile(measure, "w") as archive:
+        archive.write(archived, archived.name)
+    volume_size = len(measure.getvalue()) // 3 + 1
+    with multivolumefile.MultiVolume(
+        str(tmp_path / "b.7z"), mode="wb", volume=volume_size, ext_digits=3
+    ) as volume:
+        with py7zr.SevenZipFile(volume, "w") as archive:
+            archive.write(archived, archived.name)
+    volumes = sorted(tmp_path.glob("b.7z.*"))
+    assert len(volumes) > 1, "the archive must span several volumes for this test"
+    archived.unlink()
+
+    source = _source(tmp_path, "a")
+    output = tmp_path / "merged.parquet"
+    # only the first volume names the target file
+    input_files = {str(source): source.name}
+    input_files |= {str(v): ([archived.name] if i == 0 else []) for i, v in enumerate(volumes)}
+
+    Converter().convert(str(output), input_files=input_files)
+
+    assert sorted(gpd.read_parquet(output)["id"]) == ["a-1", "a-2", "b-1", "b-2"]

{vecorel_cli-0.4.0 → vecorel_cli-0.4.2}/vecorel_cli/conversion/base.py

@@ -22,6 +22,7 @@ import py7zr
 import rarfile
 import shapely
 from fsspec import AbstractFileSystem
+from fsspec.exceptions import FSTimeoutError
 from fsspec.implementations.local import LocalFileSystem
 from geopandas import GeoDataFrame
 
@@ -33,7 +34,7 @@ from ..vecorel.collection import Collection
 from ..vecorel.hilbert import hilbert_sort_geodataframe
 from ..vecorel.schemas import Schemas
 from ..vecorel.typing import Sources
-from ..vecorel.util import get_fs, name_from_uri, stream_file, suffix_duplicate_ids
+from ..vecorel.util import get_fs, merge_headers, name_from_uri, stream_file, suffix_duplicate_ids
 from .deflate64 import deflate64_zip
 from .flatdict import FlatDict
 
@@ -58,6 +59,8 @@ class BaseConverter(LoggerMixin):
     data_access: str = ""
     open_options: dict = {}
     avoid_range_request: bool = False
+    # Sent with every download, e.g. a browser User-Agent for a server that refuses others
+    request_headers: dict[str, str] = {}
     variants: dict[str, Sources] = {}
     variant: Optional[str] = None
 
@@ -365,6 +368,11 @@ class BaseConverter(LoggerMixin):
 
         if self.avoid_range_request and "block_size" not in kwargs:
             kwargs["block_size"] = 0
+        if self.request_headers:
+            # headers passed in explicitly win over the converter's
+            client_kwargs = kwargs.get("client_kwargs") or {}
+            headers = merge_headers(self.request_headers, client_kwargs.get("headers", {}))
+            kwargs["client_kwargs"] = {**client_kwargs, "headers": headers}
 
         # Multi-volume 7z archives (.7z.001, .7z.002, ...) are one 7z stream split
         # across several URIs; they must be downloaded and extracted together.
@@ -436,11 +444,13 @@ class BaseConverter(LoggerMixin):
         try:
             with cache_fs.open(part_file, mode="wb") as file:
                 stream_file(source_fs, uri, file)
-        except BaseException:
+        except BaseException as e:
             try:
                 cache_fs.rm(part_file)
             except FileNotFoundError:
                 pass
+            if isinstance(e, FSTimeoutError):
+                raise TimeoutError(f"Download timed out: {uri}") from e
             raise
         cache_fs.mv(part_file, cache_file)
 
@@ -506,6 +516,15 @@ class BaseConverter(LoggerMixin):
                 raise ValueError(f"Unknown variant '{self.variant}', choose from {opts}")
         return urls
 
+    def get_input_urls(self, input_files=None):
+        if input_files is not None and isinstance(input_files, dict) and len(input_files) > 0:
+            self.warning("Using user provided input file(s) instead of the pre-defined file(s)")
+            return input_files
+        urls = self.get_urls()
+        if urls is None:
+            raise ValueError("No input files provided")
+        return urls
+
     def get_data(
         self, paths: list[tuple[str, str]], **kwargs
     ) -> Generator[tuple[GeoDataFrame, str, str, Optional[str]]]:
@@ -648,13 +667,7 @@ class BaseConverter(LoggerMixin):
         if directory:
             os.makedirs(directory, exist_ok=True)
 
-        if input_files is not None and isinstance(input_files, dict) and len(input_files) > 0:
-            self.warning("Using user provided input file(s) instead of the pre-defined file(s)")
-            urls = input_files
-        else:
-            urls = self.get_urls()
-            if urls is None:
-                raise ValueError("No input files provided")
+        urls = self.get_input_urls(input_files)
 
         self.info("Getting file(s) if not cached yet")
         paths = self.download_files(urls, cache)

{vecorel_cli-0.4.0 → vecorel_cli-0.4.2}/vecorel_cli/conversion/duckdb.py

@@ -21,7 +21,7 @@ from ..vecorel.ops import (
     report,
     warn_missing_required,
 )
-from ..vecorel.util import find_differing_crs
+from ..vecorel.util import find_differing_crs, name_from_uri
 from .base import BaseConverter
 
 
@@ -32,6 +32,15 @@ def _sql_path(path) -> str:
     return f"'{escaped}'"
 
 
+def _readable_in_place(urls) -> bool:
+    if isinstance(urls, str):
+        urls = {urls: name_from_uri(urls)}
+    return all(
+        isinstance(target, str) and target.lower().endswith((".parquet", ".geoparquet"))
+        for target in urls.values()
+    )
+
+
 def _sql_name(name) -> str:
     escaped = str(name).replace('"', '""')
     return f'"{escaped}"'
@@ -107,16 +116,12 @@ class DuckDBBaseConverter(BaseConverter):
         if directory:
             os.makedirs(directory, exist_ok=True)
 
-        if input_files is not None and isinstance(input_files, dict) and len(input_files) > 0:
-            self.warning("Using user provided input file(s) instead of the pre-defined file(s)")
-            urls = input_files
-        else:
-            urls = self.get_urls()
-            if urls is None:
-                raise ValueError("No input files provided")
+        urls = self.get_input_urls(input_files)
 
         self.info("Getting file(s) if not cached yet")
-        if cache:
+        # Without a cache DuckDB reads remote parquet files in place; anything else, such as
+        # an archive, is downloaded to a temporary folder as the GeoDataFrame codepath does
+        if cache or not _readable_in_place(urls):
             request_args = {}
             if self.avoid_range_request:
                 request_args["block_size"] = 0
@@ -140,7 +145,8 @@ class DuckDBBaseConverter(BaseConverter):
         available = {
             row[0]: row[1]
             for row in con.execute(
-                "DESCRIBE SELECT * FROM read_parquet(?, union_by_name=true)", [sources]
+                "DESCRIBE SELECT * FROM read_parquet(?, union_by_name=true, hive_partitioning=false)",
+                [sources],
             ).fetchall()
         }
 
@@ -190,7 +196,7 @@ class DuckDBBaseConverter(BaseConverter):
                     # migrated type so integer-valued floats format the same way
                     expr = f"({self.column_migrations[c]})"
                     sql_type = con.execute(
-                        f"DESCRIBE SELECT {expr} AS part FROM read_parquet(?, union_by_name=true)",
+                        f"DESCRIBE SELECT {expr} AS part FROM read_parquet(?, union_by_name=true, hive_partitioning=false)",
                         [sources],
                     ).fetchone()[1]
                     parts.append(self._stringify_sql(expr, sql_type, c))
@@ -260,7 +266,7 @@ class DuckDBBaseConverter(BaseConverter):
             sources_sql = "[" + ",".join(_sql_path(path) for path in sources) + "]"
         source_query = f"""
             SELECT {selection}
-            FROM read_parquet({sources_sql}, union_by_name=true)
+            FROM read_parquet({sources_sql}, union_by_name=true, hive_partitioning=false)
             {where}
         """
 
@@ -356,6 +362,8 @@ class DuckDBBaseConverter(BaseConverter):
         compression = compression or "zstd"
         if compression == "zstd" and compression_level is None:
             compression_level = 15
+        # DuckDB calls "none" "uncompressed"; the postprocess (Arrow) takes the original name
+        copy_compression = "uncompressed" if compression == "none" else compression
         geoparquet_version = geoparquet_version or "1.1.0"
         row_group_size = GeoParquet.row_group_size
         if isinstance(output_file, Path):
@@ -549,12 +557,12 @@ class DuckDBBaseConverter(BaseConverter):
                 }}
             )
         """,
-            [output_file, compression, collection_json],
+            [output_file, copy_compression, collection_json],
         )
 
         if "id" in selected_targets and suffix_duplicate_ids:
             self._suffix_duplicate_ids_in_file(
-                con, output_file, compression, collection_json, row_group_size, per_collection
+                con, output_file, copy_compression, collection_json, row_group_size, per_collection
             )
 
         # Sort against the same CRS-derived Hilbert grid as the
@@ -579,7 +587,7 @@ class DuckDBBaseConverter(BaseConverter):
                     con,
                     output_file,
                     keys_path,
-                    compression,
+                    copy_compression,
                     collection_json,
                     final_group_size,
                 )
@@ -618,7 +626,7 @@ class DuckDBBaseConverter(BaseConverter):
                 f"""
                 SELECT coalesce(sum(n), 0), coalesce(sum(n) FILTER (WHERE n > 1), 0)
                 FROM (
-                    SELECT count(*) AS n FROM read_parquet({_sql_path(output_file)})
+                    SELECT count(*) AS n FROM read_parquet({_sql_path(output_file)}, hive_partitioning=false)
                     WHERE "id" IS NOT NULL GROUP BY {key}
                 )
                 """
@@ -648,7 +656,7 @@ class DuckDBBaseConverter(BaseConverter):
                                   AS VARCHAR)
                              ELSE CAST("id" AS VARCHAR)
                         END AS "id")
-                      FROM read_parquet({_sql_path(output_file)}, file_row_number=true)
+                      FROM read_parquet({_sql_path(output_file)}, file_row_number=true, hive_partitioning=false)
                     ) TO ? (
                         FORMAT parquet,
                         ROW_GROUP_SIZE {row_group_size},
@@ -765,7 +773,7 @@ class DuckDBBaseConverter(BaseConverter):
                     columns.append(_sql_name(name))
             per_part = per_part or coalesce or bool(constants)
             query = f"SELECT {', '.join(columns)}"
-            source = f"read_parquet({_sql_path(path)})"
+            source = f"read_parquet({_sql_path(path)}, hive_partitioning=false)"
             if constants:
                 # A registered table rather than SQL literals, so arrays, objects and
                 # temporal values keep their types
@@ -783,7 +791,7 @@ class DuckDBBaseConverter(BaseConverter):
             # nothing to add to any part, so a single scan over all of them
             sources = "[" + ",".join(_sql_path(path) for path in paths) + "]"
             columns = ", ".join(_sql_name(name) for name in union)
-            source_query = f"SELECT {columns} FROM read_parquet({sources}, union_by_name=true)"
+            source_query = f"SELECT {columns} FROM read_parquet({sources}, union_by_name=true, hive_partitioning=false)"
         targets = [row[0] for row in con.execute(f"DESCRIBE {source_query}").fetchall()]
 
         return self.write_query(
@@ -850,10 +858,10 @@ class DuckDBBaseConverter(BaseConverter):
     ):
         directory = os.path.dirname(output_file) or "."
         tmp_path = None
-        query = f"SELECT d.* FROM read_parquet({_sql_path(output_file)}) d"
+        query = f"SELECT d.* FROM read_parquet({_sql_path(output_file)}, hive_partitioning=false) d"
         if keys_path:
             query += f"""
-              POSITIONAL JOIN read_parquet({_sql_path(keys_path)}) k
+              POSITIONAL JOIN read_parquet({_sql_path(keys_path)}, hive_partitioning=false) k
               ORDER BY k.hilbert, k.ordinal
             """
         try:

vecorel_cli-0.4.2/vecorel_cli/conversion/per_file.py

@@ -0,0 +1,53 @@
+import os
+from tempfile import TemporaryDirectory
+
+from .duckdb import DuckDBBaseConverter
+
+
+class PerFileConverterMixin:
+    """Converts each source on its own and merges the parts, for sources that do not fit
+    into memory together. The ids must be unique across the sources, and the parts need
+    about the size of the output in free disk space next to it."""
+
+    _part_urls = None
+
+    def get_input_urls(self, input_files=None):
+        if self._part_urls is not None:
+            return self._part_urls
+        return super().get_input_urls(input_files)
+
+    def convert(self, output_file, cache=None, input_files=None, variant=None, **kwargs) -> str:
+        self.select_variant(variant)
+        urls = self.get_input_urls(input_files)
+        sources = self._split_sources(urls)
+        # the variant is passed on as selected, so that its default is not reported per part
+        try:
+            if len(sources) <= 1:
+                self._part_urls = urls
+                return super().convert(output_file, cache=cache, variant=self.variant, **kwargs)
+
+            directory = os.path.dirname(output_file) or "."
+            os.makedirs(directory, exist_ok=True)
+            with TemporaryDirectory(dir=directory) as parts_folder:
+                parts = []
+                for index, source in enumerate(sources):
+                    self.info(f"Converting source {index + 1}/{len(sources)}: {next(iter(source))}")
+                    part = os.path.join(parts_folder, f"part_{index}.parquet")
+                    self._part_urls = source
+                    super().convert(part, cache=cache, variant=self.variant, **kwargs)
+                    parts.append(part)
+
+                self.info(f"Merging {len(parts)} parts into {output_file}")
+                merge_options = ("compression", "compression_level", "geoparquet_version")
+                return DuckDBBaseConverter().merge_parquet(
+                    parts, output_file, **{k: kwargs[k] for k in merge_options if k in kwargs}
+                )
+        finally:
+            self._part_urls = None
+
+    def _split_sources(self, urls) -> list:
+        if isinstance(urls, str):
+            return [urls]
+        # the volumes of a multi-volume 7z archive can only be extracted together
+        regular, volume_groups = self._group_multivolume_7z(urls)
+        return [{uri: target} for uri, target in regular.items()] + list(volume_groups.values())

{vecorel_cli-0.4.0 → vecorel_cli-0.4.2}/vecorel_cli/vecorel/util.py

@@ -5,6 +5,7 @@ from pathlib import Path
 from typing import Optional, Union
 from urllib.parse import urlparse
 
+import aiohttp
 import yaml
 from fsspec import AbstractFileSystem
 from fsspec.implementations.http import HTTPFileSystem
@@ -120,6 +121,21 @@ def _expected_content_length(f) -> Optional[int]:
         return None
 
 
+def merge_headers(*headers: dict) -> dict:
+    """Merge HTTP headers; names are case-insensitive, and a later one replaces an earlier."""
+    merged = {}
+    for group in headers:
+        for name, value in group.items():
+            for existing in [k for k in merged if k.lower() == name.lower()]:
+                del merged[existing]
+            merged[name] = value
+    return merged
+
+
+HTTP_CONNECT_TIMEOUT = 60
+HTTP_READ_TIMEOUT = 300
+
+
 def get_fs(url_or_path: Union[str, Path, URL], **kwargs) -> AbstractFileSystem:
     """Choose fsspec filesystem by sniffing input url"""
     if isinstance(url_or_path, Path):
@@ -130,8 +146,13 @@ def get_fs(url_or_path: Union[str, Path, URL], **kwargs) -> AbstractFileSystem:
 
     if parsed.scheme in ("http", "https"):
         client_kwargs = kwargs.pop("client_kwargs", {})
-        headers = {"User-Agent": USER_AGENT, **client_kwargs.get("headers", {})}
-        return HTTPFileSystem(client_kwargs={**client_kwargs, "headers": headers}, **kwargs)
+        headers = merge_headers({"User-Agent": USER_AGENT}, client_kwargs.get("headers", {}))
+        timeout = client_kwargs.get("timeout") or aiohttp.ClientTimeout(
+            total=None, sock_connect=HTTP_CONNECT_TIMEOUT, sock_read=HTTP_READ_TIMEOUT
+        )
+        return HTTPFileSystem(
+            client_kwargs={**client_kwargs, "headers": headers, "timeout": timeout}, **kwargs
+        )
 
     if parsed.scheme == "s3":
         from s3fs import S3FileSystem

{vecorel_cli-0.4.0 → vecorel_cli-0.4.2/vecorel_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vecorel-cli
-Version: 0.4.0
+Version: 0.4.2
 Summary: CLI tools such as validation and file format conversion for vecorel.
 Author: Matthias Mohr
 License-Expression: Apache-2.0

{vecorel_cli-0.4.0 → vecorel_cli-0.4.2}/vecorel_cli.egg-info/SOURCES.txt

@@ -18,6 +18,7 @@ tests/test_improve.py
 tests/test_jsonschema.py
 tests/test_merge.py
 tests/test_ops.py
+tests/test_per_file.py
 tests/test_rename_extension.py
 tests/test_validate.py
 tests/test_validate_schema.py
@@ -55,6 +56,7 @@ vecorel_cli/conversion/base.py
 vecorel_cli/conversion/deflate64.py
 vecorel_cli/conversion/duckdb.py
 vecorel_cli/conversion/flatdict.py
+vecorel_cli/conversion/per_file.py
 vecorel_cli/datasets/__init__.py
 vecorel_cli/datasets/template.py
 vecorel_cli/encoding/__init__.py