earthforge-vector 0.1.0__tar.gz

earthforge_vector-0.1.0/.gitignore

@@ -0,0 +1,31 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+*.egg
+dist/
+build/
+*.whl
+
+# Virtual environments
+.venv/
+venv/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.mypy_cache/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Claude
+.claude/

earthforge_vector-0.1.0/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.4
+Name: earthforge-vector
+Version: 0.1.0
+Summary: EarthForge vector format inspection (GeoParquet, FlatGeobuf, GeoJSON).
+License-Expression: GPL-3.0-or-later
+Requires-Python: >=3.11
+Requires-Dist: earthforge-core>=0.1.0
+Requires-Dist: pyarrow>=14.0
+Description-Content-Type: text/markdown
+
+# earthforge-vector
+
+Vector format inspection for EarthForge. Part of the [EarthForge](../../README.md) toolkit.

earthforge_vector-0.1.0/README.md

@@ -0,0 +1,3 @@
+# earthforge-vector
+
+Vector format inspection for EarthForge. Part of the [EarthForge](../../README.md) toolkit.

earthforge_vector-0.1.0/pyproject.toml

@@ -0,0 +1,18 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "earthforge-vector"
+version = "0.1.0"
+description = "EarthForge vector format inspection (GeoParquet, FlatGeobuf, GeoJSON)."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "GPL-3.0-or-later"
+dependencies = [
+    "earthforge-core>=0.1.0",
+    "pyarrow>=14.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/earthforge"]

earthforge_vector-0.1.0/src/earthforge/vector/__init__.py

@@ -0,0 +1,6 @@
+"""EarthForge vector format inspection.
+
+Provides deep metadata extraction for vector geospatial formats including
+GeoParquet, FlatGeobuf, and GeoJSON. Uses pyarrow for Parquet/GeoParquet
+introspection without loading full datasets into memory.
+"""

earthforge_vector-0.1.0/src/earthforge/vector/convert.py

@@ -0,0 +1,356 @@
+"""Vector format conversion.
+
+Converts between vector geospatial formats with a focus on producing valid
+GeoParquet output. Supports Shapefile, GeoJSON, and other OGR-readable
+formats as input. Writes GeoParquet with proper ``geo`` metadata including
+CRS, geometry types, encoding, and bounding box.
+
+Uses GDAL/OGR for reading source formats and pyarrow for writing Parquet.
+Falls back to geopandas if available, but does not require it.
+
+Usage::
+
+    from earthforge.vector.convert import convert_vector
+
+    result = await convert_vector("buildings.shp", output="buildings.parquet")
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from functools import partial
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from earthforge.vector.errors import VectorError
+
+
+class ConvertResult(BaseModel):
+    """Structured result from a vector format conversion.
+
+    Attributes:
+        source: Input file path.
+        output: Output file path.
+        input_format: Source format name (e.g. ``"ESRI Shapefile"``).
+        output_format: Target format (e.g. ``"geoparquet"``).
+        feature_count: Number of features converted.
+        geometry_type: Geometry type (e.g. ``"Polygon"``).
+        crs: CRS identifier string.
+        bbox: Bounding box ``[west, south, east, north]``.
+        file_size_bytes: Output file size in bytes.
+    """
+
+    source: str = Field(title="Source")
+    output: str = Field(title="Output")
+    input_format: str = Field(title="Input Format")
+    output_format: str = Field(title="Output Format")
+    feature_count: int = Field(title="Features")
+    geometry_type: str | None = Field(default=None, title="Geometry Type")
+    crs: str | None = Field(default=None, title="CRS")
+    bbox: list[float] | None = Field(default=None, title="BBox")
+    file_size_bytes: int | None = Field(default=None, title="Size (bytes)")
+
+
+def _ogr_type_to_arrow(ogr_type: int) -> str:
+    """Map an OGR field type to a pyarrow type string.
+
+    Parameters:
+        ogr_type: OGR field type constant.
+
+    Returns:
+        Arrow type name string.
+    """
+    from osgeo import ogr
+
+    mapping = {
+        ogr.OFTInteger: "int32",
+        ogr.OFTInteger64: "int64",
+        ogr.OFTReal: "float64",
+        ogr.OFTString: "string",
+        ogr.OFTDate: "string",
+        ogr.OFTDateTime: "string",
+        ogr.OFTBinary: "binary",
+    }
+    return mapping.get(ogr_type, "string")
+
+
+def _ogr_geom_type_name(ogr_geom_type: int) -> str:
+    """Convert OGR geometry type constant to human-readable name.
+
+    Parameters:
+        ogr_geom_type: OGR geometry type constant.
+
+    Returns:
+        Geometry type name.
+    """
+    from osgeo import ogr
+
+    mapping = {
+        ogr.wkbPoint: "Point",
+        ogr.wkbLineString: "LineString",
+        ogr.wkbPolygon: "Polygon",
+        ogr.wkbMultiPoint: "MultiPoint",
+        ogr.wkbMultiLineString: "MultiLineString",
+        ogr.wkbMultiPolygon: "MultiPolygon",
+        ogr.wkbGeometryCollection: "GeometryCollection",
+        ogr.wkbPoint25D: "Point",
+        ogr.wkbLineString25D: "LineString",
+        ogr.wkbPolygon25D: "Polygon",
+        ogr.wkbMultiPoint25D: "MultiPoint",
+        ogr.wkbMultiLineString25D: "MultiLineString",
+        ogr.wkbMultiPolygon25D: "MultiPolygon",
+    }
+    return mapping.get(ogr_geom_type, "Unknown")
+
+
+def _extract_crs_info(spatial_ref: Any) -> tuple[str | None, dict[str, Any] | None]:
+    """Extract CRS identifier and PROJJSON from an OGR SpatialReference.
+
+    Parameters:
+        spatial_ref: OGR SpatialReference object.
+
+    Returns:
+        Tuple of (crs_string, projjson_dict).
+    """
+    if spatial_ref is None:
+        return None, None
+
+    # Try to get authority:code
+    auth_name = spatial_ref.GetAuthorityName(None)
+    auth_code = spatial_ref.GetAuthorityCode(None)
+    crs_string = f"{auth_name}:{auth_code}" if auth_name and auth_code else None
+
+    # Build PROJJSON for GeoParquet metadata
+    projjson: dict[str, Any] | None = None
+    try:
+        projjson_str = spatial_ref.ExportToPROJJSON()
+        if projjson_str:
+            projjson = json.loads(projjson_str)
+    except Exception:
+        # Fall back to building minimal PROJJSON
+        if crs_string:
+            projjson = {
+                "type": "GeographicCRS" if spatial_ref.IsGeographic() else "ProjectedCRS",
+                "name": spatial_ref.GetName() or crs_string,
+                "id": {"authority": auth_name, "code": int(auth_code) if auth_code else 0},
+            }
+
+    if crs_string is None and spatial_ref.GetName():
+        crs_string = spatial_ref.GetName()
+
+    return crs_string, projjson
+
+
+def _convert_vector_sync(
+    source: str,
+    *,
+    output: str | None = None,
+    target_format: str = "geoparquet",
+    compression: str = "snappy",
+) -> ConvertResult:
+    """Convert a vector file to GeoParquet synchronously.
+
+    Parameters:
+        source: Path to the input vector file (Shapefile, GeoJSON, etc.).
+        output: Output file path. If ``None``, derives from source name.
+        target_format: Target format (currently only ``"geoparquet"``).
+        compression: Parquet compression codec (``"snappy"``, ``"zstd"``, ``"gzip"``).
+
+    Returns:
+        Structured conversion result.
+
+    Raises:
+        VectorError: If the source cannot be read or conversion fails.
+    """
+    if target_format != "geoparquet":
+        raise VectorError(f"Unsupported target format: {target_format}")
+
+    try:
+        from osgeo import ogr
+    except ImportError as exc:
+        raise VectorError(
+            "GDAL/OGR is required for vector conversion: install GDAL Python bindings"
+        ) from exc
+
+    try:
+        import pyarrow as pa
+        import pyarrow.parquet as pq
+    except ImportError as exc:
+        raise VectorError(
+            "pyarrow is required for GeoParquet output: pip install earthforge[vector]"
+        ) from exc
+
+    # Open source
+    try:
+        ds = ogr.Open(source)
+    except RuntimeError as exc:
+        raise VectorError(f"Failed to open vector file '{source}'") from exc
+    if ds is None:
+        raise VectorError(f"Failed to open vector file '{source}'")
+
+    layer = ds.GetLayer(0)
+    if layer is None:
+        raise VectorError(f"No layers found in '{source}'")
+
+    input_format = ds.GetDriver().GetName()
+    layer_defn = layer.GetLayerDefn()
+    feature_count = layer.GetFeatureCount()
+    geom_type = _ogr_geom_type_name(layer.GetGeomType())
+
+    # Extract CRS
+    spatial_ref = layer.GetSpatialRef()
+    crs_string, projjson = _extract_crs_info(spatial_ref)
+
+    # Get extent
+    extent = layer.GetExtent()  # (xmin, xmax, ymin, ymax)
+    bbox = [extent[0], extent[2], extent[1], extent[3]] if extent else None
+
+    # Build field definitions
+    field_names: list[str] = []
+    field_types: list[str] = []
+    for i in range(layer_defn.GetFieldCount()):
+        field_def = layer_defn.GetFieldDefn(i)
+        field_names.append(field_def.GetName())
+        field_types.append(_ogr_type_to_arrow(field_def.GetType()))
+
+    # Read all features
+    arrays: dict[str, list[Any]] = {name: [] for name in field_names}
+    geometries: list[bytes] = []
+
+    layer.ResetReading()
+    feature = layer.GetNextFeature()
+    actual_count = 0
+    while feature is not None:
+        # Read attribute fields
+        for i, name in enumerate(field_names):
+            if not feature.IsFieldSet(i) or feature.IsFieldNull(i):
+                arrays[name].append(None)
+            elif field_types[i] == "int32":
+                arrays[name].append(feature.GetFieldAsInteger(i))
+            elif field_types[i] == "int64":
+                arrays[name].append(feature.GetFieldAsInteger64(i))
+            elif field_types[i] == "float64":
+                arrays[name].append(feature.GetFieldAsDouble(i))
+            else:
+                arrays[name].append(feature.GetFieldAsString(i))
+
+        # Read geometry as WKB
+        geom = feature.GetGeometryRef()
+        if geom is not None:
+            geometries.append(bytes(geom.ExportToWkb()))
+        else:
+            geometries.append(b"")
+
+        actual_count += 1
+        feature = layer.GetNextFeature()
+
+    ds = None  # Close OGR dataset
+
+    if feature_count < 0:
+        feature_count = actual_count
+
+    # Build pyarrow table
+    pa_columns: dict[str, Any] = {}
+    for name, arrow_type in zip(field_names, field_types, strict=True):
+        type_map = {
+            "int32": pa.int32(),
+            "int64": pa.int64(),
+            "float64": pa.float64(),
+            "string": pa.string(),
+            "binary": pa.binary(),
+        }
+        pa_type = type_map.get(arrow_type, pa.string())
+        pa_columns[name] = pa.array(arrays[name], type=pa_type)
+
+    pa_columns["geometry"] = pa.array(geometries, type=pa.binary())
+    table = pa.table(pa_columns)
+
+    # Build GeoParquet metadata
+    geo_metadata: dict[str, Any] = {
+        "version": "1.1.0",
+        "primary_column": "geometry",
+        "columns": {
+            "geometry": {
+                "encoding": "WKB",
+                "geometry_types": [geom_type],
+            }
+        },
+    }
+
+    if bbox:
+        geo_metadata["columns"]["geometry"]["bbox"] = bbox
+    if projjson:
+        geo_metadata["columns"]["geometry"]["crs"] = projjson
+
+    # Attach geo metadata to schema
+    existing = table.schema.metadata or {}
+    existing[b"geo"] = json.dumps(geo_metadata).encode("utf-8")
+    table = table.replace_schema_metadata(existing)
+
+    # Determine output path
+    if output is None:
+        output = str(Path(source).with_suffix(".parquet"))
+
+    # Write GeoParquet
+    try:
+        pq.write_table(table, output, compression=compression)
+    except Exception as exc:
+        raise VectorError(f"Failed to write GeoParquet '{output}': {exc}") from exc
+
+    file_size: int | None = None
+    try:
+        file_size = Path(output).stat().st_size
+    except OSError:
+        pass
+
+    return ConvertResult(
+        source=source,
+        output=output,
+        input_format=input_format,
+        output_format="geoparquet",
+        feature_count=actual_count,
+        geometry_type=geom_type,
+        crs=crs_string,
+        bbox=bbox,
+        file_size_bytes=file_size,
+    )
+
+
+async def convert_vector(
+    source: str,
+    *,
+    output: str | None = None,
+    target_format: str = "geoparquet",
+    compression: str = "snappy",
+) -> ConvertResult:
+    """Convert a vector file to GeoParquet.
+
+    Reads the source using GDAL/OGR and writes GeoParquet with proper ``geo``
+    metadata. Supports Shapefile, GeoJSON, GPKG, and any OGR-supported format.
+
+    Parameters:
+        source: Path to the input vector file.
+        output: Output file path. If ``None``, replaces extension with ``.parquet``.
+        target_format: Target format (default: ``"geoparquet"``).
+        compression: Parquet compression codec (default: ``"snappy"``).
+
+    Returns:
+        Structured conversion result.
+
+    Raises:
+        VectorError: If the conversion fails.
+    """
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(
+        None,
+        partial(
+            _convert_vector_sync,
+            source,
+            output=output,
+            target_format=target_format,
+            compression=compression,
+        ),
+    )

earthforge_vector-0.1.0/src/earthforge/vector/errors.py

@@ -0,0 +1,21 @@
+"""Vector-specific error types.
+
+All exceptions inherit from :class:`~earthforge.core.errors.EarthForgeError`
+so the CLI can catch them uniformly and map to appropriate exit codes.
+"""
+
+from __future__ import annotations
+
+from earthforge.core.errors import EarthForgeError
+
+
+class VectorError(EarthForgeError):
+    """Base error for vector operations.
+
+    Parameters:
+        message: Human-readable error description.
+        exit_code: Process exit code (default: 20).
+    """
+
+    def __init__(self, message: str, *, exit_code: int = 20) -> None:
+        super().__init__(message, exit_code=exit_code)

earthforge_vector-0.1.0/src/earthforge/vector/info.py

@@ -0,0 +1,245 @@
+"""Deep metadata extraction for vector geospatial formats.
+
+Reads Parquet/GeoParquet file metadata via pyarrow without loading data into
+memory. Extracts schema, row counts, geometry columns, CRS, bounding box, and
+encoding information from GeoParquet ``geo`` metadata.
+
+For non-Parquet vector formats (GeoJSON, FlatGeobuf), provides basic file-level
+metadata. Deep inspection of those formats may be added in later milestones.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from functools import partial
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from earthforge.vector.errors import VectorError
+
+
+class ColumnInfo(BaseModel):
+    """Metadata for a single column in a vector dataset.
+
+    Attributes:
+        name: Column name.
+        type: Arrow type string (e.g. ``"int64"``, ``"binary"``).
+        is_geometry: Whether this column contains geometry data.
+    """
+
+    name: str = Field(title="Column")
+    type: str = Field(title="Type")
+    is_geometry: bool = Field(default=False, title="Geometry")
+
+
+class VectorInfo(BaseModel):
+    """Structured metadata for a vector geospatial file.
+
+    Attributes:
+        source: The file path that was inspected.
+        format: Detected vector format (e.g. ``"geoparquet"``, ``"parquet"``).
+        row_count: Total number of rows/features.
+        num_columns: Total number of columns.
+        columns: Per-column metadata.
+        geometry_column: Name of the primary geometry column, if any.
+        geometry_types: List of geometry types found (e.g. ``["Point"]``).
+        crs: CRS string from GeoParquet metadata, if available.
+        bbox: Bounding box ``[west, south, east, north]``, if available.
+        encoding: Geometry encoding (e.g. ``"WKB"``), if available.
+        num_row_groups: Number of Parquet row groups.
+        compression: Parquet compression codec, if applicable.
+        file_size_bytes: File size in bytes.
+    """
+
+    source: str = Field(title="Source")
+    format: str = Field(title="Format")
+    row_count: int = Field(title="Rows")
+    num_columns: int = Field(title="Columns")
+    columns: list[ColumnInfo] = Field(title="Column Details")
+    geometry_column: str | None = Field(default=None, title="Geometry Column")
+    geometry_types: list[str] = Field(default_factory=list, title="Geometry Types")
+    crs: str | None = Field(default=None, title="CRS")
+    bbox: list[float] | None = Field(default=None, title="Bounding Box")
+    encoding: str | None = Field(default=None, title="Encoding")
+    num_row_groups: int | None = Field(default=None, title="Row Groups")
+    compression: str | None = Field(default=None, title="Compression")
+    file_size_bytes: int | None = Field(default=None, title="Size (bytes)")
+
+
+def _read_parquet_info(source: str) -> VectorInfo:
+    """Read metadata from a Parquet/GeoParquet file synchronously.
+
+    Uses pyarrow to read only the file metadata and schema — no row data
+    is loaded into memory. Parses the ``geo`` metadata key for GeoParquet-
+    specific information (geometry column, CRS, bbox, encoding).
+
+    Parameters:
+        source: Path to a Parquet file.
+
+    Returns:
+        Structured vector metadata.
+
+    Raises:
+        VectorError: If the file cannot be read or is not a valid Parquet file.
+    """
+    try:
+        import pyarrow.parquet as pq
+    except ImportError as exc:
+        msg = "pyarrow is required for Parquet inspection: pip install pyarrow"
+        raise VectorError(msg) from exc
+
+    try:
+        pf = pq.ParquetFile(source)
+    except Exception as exc:
+        msg = f"Failed to read Parquet file '{source}': {exc}"
+        raise VectorError(msg) from exc
+
+    schema = pf.schema_arrow
+    metadata = schema.metadata or {}
+    num_rows = pf.metadata.num_rows
+    num_row_groups = pf.metadata.num_row_groups
+
+    # Detect compression from the first row group's first column chunk
+    compression: str | None = None
+    if num_row_groups > 0 and schema:
+        try:
+            rg = pf.metadata.row_group(0)
+            if rg.num_columns > 0:
+                compression = rg.column(0).compression
+        except Exception:  # noqa: S110 — best-effort metadata extraction
+            pass
+
+    # Parse GeoParquet metadata
+    geo_meta = _parse_geo_metadata(metadata)
+    geometry_column = geo_meta.get("primary_column")
+    geometry_columns: set[str] = set()
+    geometry_types: list[str] = []
+    crs: str | None = None
+    bbox: list[float] | None = None
+    encoding: str | None = None
+
+    if geometry_column and "columns" in geo_meta:
+        geometry_columns.add(geometry_column)
+        col_meta = geo_meta["columns"].get(geometry_column, {})
+        geometry_types = col_meta.get("geometry_types", [])
+        encoding = col_meta.get("encoding")
+        bbox_raw = col_meta.get("bbox")
+        if isinstance(bbox_raw, list) and len(bbox_raw) == 4:
+            bbox = [float(v) for v in bbox_raw]
+
+        crs_obj = col_meta.get("crs")
+        if isinstance(crs_obj, dict):
+            # GeoParquet stores CRS as PROJJSON — extract the name or ID
+            crs = _extract_crs_string(crs_obj)
+        elif isinstance(crs_obj, str):
+            crs = crs_obj
+
+    # Also check for additional geometry columns
+    if "columns" in geo_meta:
+        geometry_columns.update(geo_meta["columns"].keys())
+
+    # Build column info
+    columns: list[ColumnInfo] = []
+    for i in range(len(schema)):
+        field = schema.field(i)
+        columns.append(
+            ColumnInfo(
+                name=field.name,
+                type=str(field.type),
+                is_geometry=field.name in geometry_columns,
+            )
+        )
+
+    # File size
+    file_size: int | None = None
+    try:
+        file_size = Path(source).stat().st_size
+    except OSError:
+        pass
+
+    fmt = "geoparquet" if geometry_column else "parquet"
+
+    return VectorInfo(
+        source=source,
+        format=fmt,
+        row_count=num_rows,
+        num_columns=len(schema),
+        columns=columns,
+        geometry_column=geometry_column,
+        geometry_types=geometry_types,
+        crs=crs,
+        bbox=bbox,
+        encoding=encoding,
+        num_row_groups=num_row_groups,
+        compression=compression,
+        file_size_bytes=file_size,
+    )
+
+
+def _parse_geo_metadata(metadata: dict[bytes, bytes]) -> dict[str, Any]:
+    """Parse the ``geo`` key from Parquet file metadata.
+
+    Parameters:
+        metadata: Raw Parquet schema metadata (bytes keys and values).
+
+    Returns:
+        Parsed GeoParquet metadata dict, or empty dict if not present.
+    """
+    geo_bytes = metadata.get(b"geo")
+    if geo_bytes is None:
+        return {}
+    try:
+        result: dict[str, Any] = json.loads(geo_bytes)
+        return result
+    except (json.JSONDecodeError, UnicodeDecodeError):
+        return {}
+
+
+def _extract_crs_string(crs_obj: dict[str, Any]) -> str:
+    """Extract a human-readable CRS identifier from PROJJSON.
+
+    Tries ``id.code`` (e.g. ``"EPSG:4326"``), then ``name``, then falls
+    back to a truncated JSON representation.
+
+    Parameters:
+        crs_obj: PROJJSON CRS object.
+
+    Returns:
+        CRS identifier string.
+    """
+    # Try EPSG-style authority:code
+    crs_id = crs_obj.get("id", {})
+    if isinstance(crs_id, dict):
+        authority = crs_id.get("authority")
+        code = crs_id.get("code")
+        if authority and code:
+            return f"{authority}:{code}"
+
+    # Fall back to name
+    name = crs_obj.get("name")
+    if isinstance(name, str):
+        return name
+
+    return json.dumps(crs_obj)[:100]
+
+
+async def inspect_vector(source: str) -> VectorInfo:
+    """Inspect a vector file and return structured metadata.
+
+    Runs the synchronous pyarrow read in a thread executor to avoid blocking
+    the event loop. Currently supports Parquet and GeoParquet files.
+
+    Parameters:
+        source: Path to a vector file.
+
+    Returns:
+        Structured vector metadata.
+
+    Raises:
+        VectorError: If the file cannot be read or format is unsupported.
+    """
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(None, partial(_read_parquet_info, source))