rhinestone 0.1.0__py3-none-any.whl

rhinestone/__init__.py

@@ -0,0 +1,60 @@
+"""Rhinestone public package."""
+
+from . import sources
+from .api import Rhinestone, configure
+from .catalogs import Catalog
+from .models import (
+    AccessPlan,
+    Config,
+    Dependencies,
+    FileAccessPlan,
+    LibraryName,
+    Metadata,
+    Provenance,
+    Provider,
+    RemoteDatasetPlan,
+    Resource,
+    ResourceCandidate,
+    Result,
+    Runtime,
+    RuntimeFactory,
+    SearchDiagnostic,
+    SearchQuery,
+    SearchResult,
+    ServiceQueryPlan,
+    Source,
+    SourceDefinition,
+)
+from .search import SearchResults
+from .security import DestinationPolicy, DestinationRule, NetworkPolicyLevel
+
+__all__ = [
+    "AccessPlan",
+    "Catalog",
+    "Config",
+    "DestinationPolicy",
+    "DestinationRule",
+    "Dependencies",
+    "FileAccessPlan",
+    "LibraryName",
+    "Metadata",
+    "NetworkPolicyLevel",
+    "Provenance",
+    "Provider",
+    "RemoteDatasetPlan",
+    "Rhinestone",
+    "Resource",
+    "ResourceCandidate",
+    "Result",
+    "Runtime",
+    "RuntimeFactory",
+    "SearchDiagnostic",
+    "SearchQuery",
+    "SearchResult",
+    "SearchResults",
+    "ServiceQueryPlan",
+    "Source",
+    "SourceDefinition",
+    "configure",
+    "sources",
+]

rhinestone/_http.py

@@ -0,0 +1,152 @@
+"""Built-in standard-library HTTP transport."""
+
+import json
+from typing import Any, Mapping, Optional, cast
+from urllib.error import HTTPError
+from urllib.parse import urlencode, urlsplit, urlunsplit
+from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen
+
+from .errors import ProviderResponseError
+
+_TIMEOUT_SECONDS = 30
+_DEFAULT_HEADERS = {
+    "Accept": "application/json",
+    "User-Agent": "rhinestone",
+}
+
+
+class _JsonIntegerDecodeError(ValueError):
+    """JSON integer conversion failed inside the decoder."""
+
+
+def _parse_json_int(value: str) -> int:
+    try:
+        return int(value)
+    except ValueError as error:
+        raise _JsonIntegerDecodeError from error
+
+
+class JsonDocument(dict[str, Any]):
+    """Decoded JSON object together with the URI that supplied it."""
+
+    def __init__(self, value: Mapping[str, Any], response_uri: str) -> None:
+        super().__init__(value)
+        self.response_uri = response_uri
+
+
+def get_json(
+    url: str,
+    params: Mapping[str, Any],
+    headers: Optional[Mapping[str, str]] = None,
+) -> Any:
+    """Fetch and decode JSON over HTTP using the standard library."""
+    request = _request(url, params, headers)
+    opener = (
+        build_opener(_NoRedirectHandler())
+        if getattr(headers, "_rhinestone_no_redirects", False)
+        else None
+    )
+    open_request = opener.open if opener is not None else urlopen
+    with open_request(request, timeout=_TIMEOUT_SECONDS) as response:
+        try:
+            decoded = json.load(response, parse_int=_parse_json_int)
+        except (
+            UnicodeDecodeError,
+            json.JSONDecodeError,
+            _JsonIntegerDecodeError,
+        ):
+            raise ProviderResponseError("Provider response is not valid JSON") from None
+        if isinstance(decoded, Mapping):
+            response_uri = getattr(response, "geturl", lambda: request.full_url)()
+            return JsonDocument(
+                cast(Mapping[str, Any], decoded), response_uri or request.full_url
+            )
+        return decoded
+
+
+def get_text(url: str) -> str:
+    """Fetch a text document without following HTTP redirects."""
+    request = Request(url, headers={"User-Agent": "rhinestone"})
+    opener = build_opener(_NoRedirectHandler())
+    with opener.open(request, timeout=_TIMEOUT_SECONDS) as response:
+        charset = response.headers.get_content_charset() or "utf-8"
+        return response.read().decode(charset)
+
+
+class JsonServiceRuntime:
+    """Small requests-compatible runtime used by ``JsonServiceAdapter``."""
+
+    def get(
+        self,
+        url: str,
+        *,
+        params: Mapping[str, Any],
+        headers: Mapping[str, str],
+        timeout: int,
+        allow_redirects: bool,
+    ) -> "JsonResponse":
+        request = _request(url, params, headers)
+        opener = build_opener(_NoRedirectHandler())
+        try:
+            response = opener.open(request, timeout=timeout)
+        except HTTPError as error:
+            response = error
+        with response:
+            charset = response.headers.get_content_charset() or "utf-8"
+            status_code = cast(int, response.getcode())
+            return JsonResponse(status_code, response.read(), charset)
+
+
+class JsonResponse:
+    """Subset of the requests response contract used by ``JsonServiceAdapter``."""
+
+    def __init__(self, status_code: int, body: bytes, charset: str) -> None:
+        self.status_code = status_code
+        self._body = body
+        self._charset = charset
+
+    def raise_for_status(self) -> None:
+        if self.status_code >= 400:
+            raise OSError(f"HTTP request failed with status {self.status_code}")
+
+    def json(self) -> Any:
+        return json.loads(self._body.decode(self._charset))
+
+
+class _NoRedirectHandler(HTTPRedirectHandler):
+    def redirect_request(
+        self,
+        req: Request,
+        fp: Any,
+        code: int,
+        msg: str,
+        headers: Any,
+        newurl: str,
+    ) -> Optional[Request]:
+        return None
+
+
+def _request(
+    url: str,
+    params: Mapping[str, Any],
+    headers: Optional[Mapping[str, str]] = None,
+) -> Request:
+    request_url = _append_query(url, params)
+    return Request(
+        request_url,
+        headers={**_DEFAULT_HEADERS, **dict(headers or {})},
+    )
+
+
+def _append_query(url: str, params: Mapping[str, Any]) -> str:
+    """Append parameters without replacing or normalizing an existing query."""
+    additional_query = urlencode(params, doseq=True)
+    if not additional_query:
+        return url
+    components = urlsplit(url)
+    query = (
+        f"{components.query}&{additional_query}"
+        if components.query
+        else additional_query
+    )
+    return urlunsplit(components._replace(query=query))

rhinestone/adapters/__init__.py

@@ -0,0 +1,33 @@
+"""Compatibility exports for built-in source adapters."""
+
+from .execution import ExecutionAdapter
+from .source import (
+    CkanAdapter,
+    DcatAdapter,
+    DirectAdapter,
+    GsiFundamentalAdapter,
+    JsonGetter,
+    JsonObject,
+    OdptAdapter,
+    OgcFeaturesAdapter,
+    PlateauAdapter,
+    ProviderAdapter,
+    StacAdapter,
+    StaticAdapter,
+)
+
+__all__ = [
+    "CkanAdapter",
+    "DirectAdapter",
+    "DcatAdapter",
+    "GsiFundamentalAdapter",
+    "JsonGetter",
+    "JsonObject",
+    "OdptAdapter",
+    "OgcFeaturesAdapter",
+    "PlateauAdapter",
+    "ProviderAdapter",
+    "StacAdapter",
+    "StaticAdapter",
+    "ExecutionAdapter",
+]

rhinestone/adapters/execution/__init__.py

@@ -0,0 +1,17 @@
+"""Built-in execution adapters."""
+
+from ._resource import resource_attributes
+from .base import ExecutionAdapter
+from .gdal import GdalAdapter
+from .json_service import JsonServiceAdapter
+from .pyogrio import PyogrioAdapter
+from .rasterio import RasterioAdapter
+
+__all__ = [
+    "ExecutionAdapter",
+    "GdalAdapter",
+    "JsonServiceAdapter",
+    "PyogrioAdapter",
+    "RasterioAdapter",
+    "resource_attributes",
+]

rhinestone/adapters/execution/_resource.py

@@ -0,0 +1,14 @@
+"""Shared helpers for selected execution adapters."""
+
+from typing import Any, Mapping
+
+from ...models import Resource
+
+
+def resource_attributes(resource: Resource) -> Mapping[str, Any]:
+    for candidate in resource.source.candidates:
+        if candidate.uri == resource.uri and candidate.attributes.get(
+            "matches_config", True
+        ):
+            return candidate.attributes
+    return {}

rhinestone/adapters/execution/base.py

@@ -0,0 +1,47 @@
+"""Public base class for execution adapters."""
+
+from abc import ABC, abstractmethod
+from typing import Any, FrozenSet
+
+from ...models import Resource
+from ...security import DestinationPolicy
+
+__all__ = ["ExecutionAdapter"]
+
+
+class ExecutionAdapter(ABC):
+    """Base for adapters that translate a selected ``Resource`` to an OSS call."""
+
+    name: str
+    """Stable dependency and selection name."""
+
+    priority: int
+    """Deterministic selection priority."""
+
+    def __init__(self, destination_policy: DestinationPolicy | None = None) -> None:
+        self._destination_policy = (
+            destination_policy or DestinationPolicy.unrestricted()
+        )
+
+    @abstractmethod
+    def supports(self, resource: Resource, dependencies: FrozenSet[str]) -> bool:
+        """Whether this adapter can open the already selected resource."""
+
+    @abstractmethod
+    def open(
+        self,
+        resource: Resource,
+        runtime: Any,
+        *,
+        destination_policy: DestinationPolicy | None = None,
+    ) -> Any:
+        """Delegate the already selected resource to the supplied runtime."""
+
+    def authorize(
+        self,
+        resource: Resource,
+        *,
+        destination_policy: DestinationPolicy | None = None,
+    ) -> None:
+        """Authorize a resource before resolving its user-owned runtime."""
+        (destination_policy or self._destination_policy).authorize(resource.uri)

rhinestone/adapters/execution/gdal/README.md

@@ -0,0 +1,10 @@
+# GDAL Execution Adapter
+
+`GdalAdapter` は選択済み Resource を利用者提供の GDAL runtime へ渡します。
+
+- `name`: `gdal`、優先度: `20`
+- 対応形式: Shapefile, GeoTIFF, COG, NetCDF, WMS, GML, CityGML、または XYZ tile
+- 翻訳: ZIP の `/vsizip/` URI、remote ZIP の `/vsicurl/`、encoding の open option、XYZ の GDAL WMS XML
+- runtime: `OpenEx(uri, open_options=...)`
+
+Resource の再選択、データ解析、形式・CRS 変換は行いません。

rhinestone/adapters/execution/gdal/__init__.py

@@ -0,0 +1,114 @@
+"""Built-in adapters that delegate selected Resources to user-owned runtimes."""
+
+from typing import Any, FrozenSet, List, Mapping, cast
+from xml.etree.ElementTree import Element, SubElement, tostring
+
+from ....errors import ResourceAccessError
+from ....models import FileAccessPlan, Resource
+from ....security import DestinationPolicy
+from .._resource import resource_attributes
+from ..base import ExecutionAdapter
+
+
+class GdalAdapter(ExecutionAdapter):
+    """Translate Resources into calls understood by a supplied GDAL module."""
+
+    name = "gdal"
+    priority = 20
+    _formats = frozenset(
+        {"shapefile", "geotiff", "cog", "netcdf", "wms", "gml", "citygml"}
+    )
+
+    def __init__(self, destination_policy: DestinationPolicy | None = None) -> None:
+        super().__init__(destination_policy)
+
+    def supports(self, resource: Resource, dependencies: FrozenSet[str]) -> bool:
+        tile = resource.access_plan.options.get("tile")
+        return self.name in dependencies and (
+            (resource.format or "").lower() in self._formats
+            or (
+                isinstance(tile, Mapping)
+                and cast(Mapping[str, Any], tile).get("scheme") == "xyz"
+            )
+        )
+
+    def open(
+        self,
+        resource: Resource,
+        runtime: Any,
+        *,
+        destination_policy: DestinationPolicy | None = None,
+    ) -> Any:
+        uri = self._runtime_uri(resource)
+        attributes = resource_attributes(resource)
+        options: List[str] = []
+        encoding = attributes.get("encoding")
+        if isinstance(encoding, str):
+            options.append("ENCODING=" + encoding.upper())
+        try:
+            runtime_options: dict[str, Any] = {"open_options": tuple(options)}
+            result = runtime.OpenEx(uri, **runtime_options)
+            if result is None:
+                raise ResourceAccessError("GDAL returned no dataset")
+            return result
+        except Exception as error:
+            raise ResourceAccessError(
+                f"GDAL could not open {resource.uri!r}"
+            ) from error
+
+    @staticmethod
+    def _runtime_uri(resource: Resource) -> str:
+        attributes = resource_attributes(resource)
+        uri = resource.uri
+        tile = resource.access_plan.options.get("tile")
+        if tile is not None:
+            if not isinstance(tile, Mapping):
+                raise ResourceAccessError("GDAL tile options must be an object")
+            tile = cast(Mapping[str, Any], tile)
+            tile_url = tile.get("url")
+            if not isinstance(tile_url, str):
+                raise ResourceAccessError("GDAL tile URL must be a string")
+            uri = GdalAdapter.tile_xml(tile)
+        archive = attributes.get("archive")
+        if isinstance(resource.access_plan, FileAccessPlan):
+            archive = resource.access_plan.archive or archive
+        if archive == "zip":
+            uri = (
+                "/vsizip//vsicurl/" + uri
+                if uri.startswith(("http://", "https://"))
+                else "/vsizip/" + uri
+            )
+            member = resource.access_plan.options.get("entry_point")
+            if member:
+                uri += "/" + member
+        return uri
+
+    @staticmethod
+    def tile_xml(tile: Mapping[str, Any]) -> str:
+        root = Element("GDAL_WMS")
+        service = SubElement(root, "Service", name="TMS")
+        template = tile["url"]
+        for name in ("x", "y", "z"):
+            template = template.replace("{" + name + "}", "${" + name + "}")
+        SubElement(service, "ServerUrl").text = template
+        window = SubElement(root, "DataWindow")
+        for key, value in {
+            "UpperLeftX": "-20037508.342789244",
+            "UpperLeftY": "20037508.342789244",
+            "LowerRightX": "20037508.342789244",
+            "LowerRightY": "-20037508.342789244",
+            "TileLevel": str(tile["max_zoom"]),
+            "TileCountX": "1",
+            "TileCountY": "1",
+            "YOrigin": "top",
+        }.items():
+            SubElement(window, key).text = value
+        for key, value in {
+            "Projection": "EPSG:3857",
+            "BlockSizeX": "256",
+            "BlockSizeY": "256",
+            "BandsCount": "3",
+            "OverviewCount": str(tile["max_zoom"] - tile["min_zoom"]),
+        }.items():
+            SubElement(root, key).text = value
+        return tostring(root, encoding="unicode")

rhinestone/adapters/execution/json_service/README.md

@@ -0,0 +1,10 @@
+# JSON Service Execution Adapter
+
+`JsonServiceAdapter` は選択済み JSON service query をHTTP runtimeへ渡します。標準の`configure()`経路ではRhinestoneの組み込みruntimeを使用します。
+
+- `name`: `json-service`、優先度: `10`
+- 対応条件: `ServiceQueryPlan`、`application/json`、一致する service 名
+- 注入: request を作る callback、service 名、任意の `CredentialRegistry`
+- runtime contract: `get(uri, params=..., headers=..., timeout=30, allow_redirects=False)`
+
+redirect は拒否し、secret を含む可能性のある下位例外は公開しません。JSON schema の検証以外の変換は行いません。Adapter単体では同じcontractを満たすruntimeを直接渡せます。

rhinestone/adapters/execution/json_service/__init__.py

@@ -0,0 +1,103 @@
+"""Selected JSON service requests delegated to a requests-compatible runtime."""
+
+from typing import Any, Callable, FrozenSet, List, Mapping, Optional, Tuple, cast
+
+from ....errors import ProviderResponseError, ResourceAccessError
+from ....models import AccessPlan, Resource, ServiceQueryPlan
+from ....registry import CredentialRegistry
+from ....security import DestinationPolicy
+from ..base import ExecutionAdapter
+
+RequestPreparer = Callable[
+    [AccessPlan, CredentialRegistry], Tuple[Mapping[str, Any], Mapping[str, str]]
+]
+
+
+class JsonServiceAdapter(ExecutionAdapter):
+    name = "json-service"
+    priority = 10
+
+    def __init__(
+        self,
+        prepare_request: RequestPreparer,
+        service: str,
+        credentials: Optional[CredentialRegistry] = None,
+        destination_policy: Optional[DestinationPolicy] = None,
+    ) -> None:
+        super().__init__(destination_policy)
+        self._prepare_request = prepare_request
+        self._service = service
+        self._credentials = credentials or CredentialRegistry({})
+
+    def bind_credentials(self, credentials: CredentialRegistry) -> "JsonServiceAdapter":
+        return JsonServiceAdapter(
+            self._prepare_request,
+            self._service,
+            credentials,
+            self._destination_policy,
+        )
+
+    def supports(self, resource: Resource, dependencies: FrozenSet[str]) -> bool:
+        return (
+            self.name in dependencies
+            and isinstance(resource.access_plan, ServiceQueryPlan)
+            and resource.media_type == "application/json"
+            and resource.access_plan.options.get("service") == self._service
+        )
+
+    def open(
+        self,
+        resource: Resource,
+        runtime: Any,
+        *,
+        destination_policy: DestinationPolicy | None = None,
+    ) -> Any:
+        self._authorize_resource(resource, destination_policy)
+        params, headers = self._prepare_request(resource.access_plan, self._credentials)
+        try:
+            response = runtime.get(
+                resource.uri,
+                params=params,
+                headers=headers,
+                timeout=30,
+                allow_redirects=False,
+            )
+            if 300 <= response.status_code < 400:
+                raise ResourceAccessError("Service redirects are not supported")
+            response.raise_for_status()
+        except Exception:
+            # requests exceptions may include a URL containing the secret query.
+            raise ResourceAccessError("JSON service request failed") from None
+        try:
+            data = response.json()
+        except Exception:
+            raise ProviderResponseError("Service returned invalid JSON") from None
+        if resource.access_plan.options.get("response_type") == "array" and (
+            not isinstance(data, list)
+            or any(not isinstance(item, Mapping) for item in cast(List[Any], data))
+        ):
+            raise ProviderResponseError("Service must return an array of objects")
+        return cast(Any, data)
+
+    def authorize(
+        self,
+        resource: Resource,
+        *,
+        destination_policy: DestinationPolicy | None = None,
+    ) -> None:
+        """Authorize credential release before resolving the service runtime."""
+        self._authorize_resource(resource, destination_policy)
+
+    def _authorize_resource(
+        self,
+        resource: Resource,
+        destination_policy: DestinationPolicy | None,
+    ) -> None:
+        credential = resource.access_plan.options.get("credential")
+        (destination_policy or self._destination_policy).authorize(
+            resource.uri,
+            credentialed=True,
+            provider=resource.provenance.provider,
+            service=self._service,
+            credential=credential if isinstance(credential, str) else None,
+        )

rhinestone/adapters/execution/pyogrio/README.md

@@ -0,0 +1,10 @@
+# pyogrio Execution Adapter
+
+`PyogrioAdapter` は選択済み vector Resource を利用者提供の pyogrio runtime へ渡します。
+
+- `name`: `pyogrio`、優先度: `10`
+- 対応形式: Shapefile, GeoJSON, GeoPackage, FlatGeobuf
+- 翻訳: 確定済み `encoding` を `read_dataframe()` option へ渡します。
+- runtime: `read_dataframe(uri, **options)`
+
+Resource の再選択、データ解析、形式・CRS 変換は行いません。

rhinestone/adapters/execution/pyogrio/__init__.py

@@ -0,0 +1,44 @@
+"""Pyogrio execution adapter."""
+
+from typing import Any, Dict, FrozenSet
+
+from ....errors import ResourceAccessError
+from ....models import Resource
+from ....security import DestinationPolicy
+from .._resource import resource_attributes
+from ..base import ExecutionAdapter
+
+
+class PyogrioAdapter(ExecutionAdapter):
+    """Delegate vector Resources to a supplied pyogrio module."""
+
+    name = "pyogrio"
+    priority = 10
+    _formats = frozenset({"shapefile", "geojson", "gpkg", "flatgeobuf"})
+
+    def __init__(self, destination_policy: DestinationPolicy | None = None) -> None:
+        super().__init__(destination_policy)
+
+    def supports(self, resource: Resource, dependencies: FrozenSet[str]) -> bool:
+        return (
+            resource.format or ""
+        ).lower() in self._formats and self.name in dependencies
+
+    def open(
+        self,
+        resource: Resource,
+        runtime: Any,
+        *,
+        destination_policy: DestinationPolicy | None = None,
+    ) -> Any:
+        attributes = resource_attributes(resource)
+        options: Dict[str, Any] = {}
+        encoding = attributes.get("encoding")
+        if isinstance(encoding, str):
+            options["encoding"] = encoding
+        try:
+            return runtime.read_dataframe(resource.uri, **options)
+        except Exception as error:
+            raise ResourceAccessError(
+                f"pyogrio could not open {resource.uri!r}"
+            ) from error

rhinestone/adapters/execution/rasterio/README.md

@@ -0,0 +1,9 @@
+# Rasterio Execution Adapter
+
+`RasterioAdapter` は選択済み raster Resource を利用者提供の Rasterio runtime へ渡します。
+
+- `name`: `rasterio`、優先度: `15`
+- 対応形式: COG, GeoTIFF
+- runtime: `open(uri)`
+
+URI は選択済み Resource の値をそのまま使用します。ダウンロード、Resource の再選択、形式・CRS 変換は行いません。

rhinestone/adapters/execution/rasterio/__init__.py

@@ -0,0 +1,38 @@
+"""Rasterio execution adapter."""
+
+from typing import Any, FrozenSet
+
+from ....errors import ResourceAccessError
+from ....models import Resource
+from ....security import DestinationPolicy
+from ..base import ExecutionAdapter
+
+
+class RasterioAdapter(ExecutionAdapter):
+    """Delegate raster Resources to a supplied Rasterio module."""
+
+    name = "rasterio"
+    priority = 15
+    _formats = frozenset({"cog", "geotiff"})
+
+    def __init__(self, destination_policy: DestinationPolicy | None = None) -> None:
+        super().__init__(destination_policy)
+
+    def supports(self, resource: Resource, dependencies: FrozenSet[str]) -> bool:
+        return (
+            resource.format or ""
+        ).lower() in self._formats and self.name in dependencies
+
+    def open(
+        self,
+        resource: Resource,
+        runtime: Any,
+        *,
+        destination_policy: DestinationPolicy | None = None,
+    ) -> Any:
+        try:
+            return runtime.open(resource.uri)
+        except Exception as error:
+            raise ResourceAccessError(
+                f"Rasterio could not open {resource.uri!r}"
+            ) from error