httk-store 2.1.0__py3-none-any.whl

httk/registry/cli/store/__init__.py

@@ -0,0 +1,5 @@
+"""Register httk-store's command-line namespace."""
+
+from httk.core import register_cli_command
+
+register_cli_command("store", "httk.store.export:command", "export and inspect httk stores")

httk/registry/entries/store/__init__.py

@@ -0,0 +1,13 @@
+"""Register entry providers implemented by :mod:`httk.store`."""
+
+from httk.core.register import register_entry_provider
+
+register_entry_provider(name="store-references", factory="httk.store.entry_providers:ReferenceEntryProvider")
+register_entry_provider(name="store-files", factory="httk.store.entry_providers:FileEntryProvider")
+register_entry_provider(name="store-calculations", factory="httk.store.entry_providers:CalculationEntryProvider")
+register_entry_provider(name="store-runs", factory="httk.store.entry_providers:RunEntryProvider")
+register_entry_provider(name="store-records", factory="httk.store.entry_providers:DataRecordEntryProvider")
+
+# The database-backed provider (requires the httk-store[db] extra); the factory
+# reference is lazy, so registration itself never imports sqlalchemy.
+register_entry_provider(name="store-db-store", factory="httk.store.backend.sql.entry_provider:StoreEntryProvider")

httk/store/__init__.py

@@ -0,0 +1,171 @@
+"""Provide httk-store's data-management capability layer for httk v2.
+
+Built on the stdlib-only *contracts and models* in *httk-core*, httk-store
+supplies *capabilities*:
+
+- in-memory :class:`~httk.core.EntryProvider` implementations for the standard
+  OPTIMADE entry types (:class:`ReferenceEntryProvider`,
+  :class:`FileEntryProvider`, :class:`CalculationEntryProvider`), serving
+  httk-core's record models through the neutral provider contract; and
+- **property-definition validation** (:func:`validate_property`,
+  :func:`validate_record`, :class:`PropertyValidationError`) built on
+  ``jsonschema`` (Draft 2020-12), checking record values against their OPTIMADE
+  property definitions fully offline; and
+- the **store/searcher query protocols** (:mod:`httk.store.query`) — the
+  backend-agnostic query contract implemented by httk data stores and consumed
+  by serving modules; and
+- the **federated store** (:mod:`httk.store.federated_store`) — ordered, immutable
+  source and target bindings plus lazy sequential union query execution; and
+- the **generic OPTIMADE filter translation** (:mod:`httk.store.query.optimade_filters`)
+  — turning filter syntax trees parsed by
+  :func:`httk.core.optimade.parse_optimade_filter` into search expressions over the
+  query protocols (the machinery in :mod:`httk.store.query.optimade_filters`, including
+  :func:`~httk.store.query.optimade_filters.filter_searcher`), with
+  neutral :class:`~httk.store.query.optimade_filters.FilterTranslationError` categories; and
+
+- the **database storage layer** (:mod:`httk.store.backend.sql`, requiring the
+  ``httk-store[db]`` extra) — relational storage and querying of plain frozen
+  dataclasses (:class:`~httk.store.backend.sql.store.SqlStore` over SQLite, DuckDB,
+  PostgreSQL, or ClickHouse),
+  served through the provider contract by
+  :class:`~httk.store.backend.sql.entry_provider.StoreEntryProvider`.
+
+The providers self-register (under ``httk.registry.entries.store``, as
+``store-references``/``store-files``/``store-calculations``/``store-db-store``)
+when ``httk.core`` discovers the module, so a serving module (such as
+*httk-serve*) can find them through the registry.
+
+.. py:class:: StandardEntryProvider
+   :canonical: httk.store.entry_providers.StandardEntryProvider
+"""
+
+import importlib
+from typing import Any
+
+from .entry_providers import (
+    CalculationEntryProvider,
+    DataRecordEntryProvider,
+    FileEntryProvider,
+    ReferenceEntryProvider,
+    RunEntryProvider,
+    product_relationships,
+)
+from .export import export_dataset
+from .federated_store import (
+    FederatedResultSet,
+    FederatedSearcher,
+    FederatedSourceError,
+    FederatedStore,
+    FederatedStoreError,
+    FederatedTarget,
+)
+from .id_ledger import IdLedger, IdLedgerError, check_ledger_key
+from .query import (
+    ContinuationToken,
+    CountUnavailableError,
+    MultipleResultsError,
+    NoResultError,
+    PageableResultSetLike,
+    PageOrder,
+    PaginationCursorError,
+    PortableQueryCapabilities,
+    ResultPage,
+    ResultRow,
+    ResultRowLike,
+    ResultSetLike,
+    Searcher,
+    SearchExpression,
+    SearchField,
+    SearchResult,
+    SearchVariable,
+    Store,
+    UnsupportedQueryError,
+    portable_query_capabilities,
+    portable_query_fields,
+)
+from .query.optimade_filters import (
+    FilterTranslationCategory,
+    FilterTranslationError,
+    filter_searcher,
+)
+from .storage_layout import EntryFamilyDeclaration, EntryLayoutBindingError, EntryRecordDeclaration
+from .store_common import EntryIdConflictError, EntryIdScheme, EntryStore
+from .validation import PropertyValidationError, validate_property, validate_record
+
+__all__ = [
+    "Backend",  # pyright: ignore[reportUnsupportedDunderAll]  (provided lazily via __getattr__)
+    "CalculationEntryProvider",
+    "ContinuationToken",
+    "CountUnavailableError",
+    "DataRecordEntryProvider",
+    "EntryFamilyDeclaration",
+    "EntryIdConflictError",
+    "EntryIdScheme",
+    "EntryLayoutBindingError",
+    "EntryRecordDeclaration",
+    "EntryStore",
+    "FederatedResultSet",
+    "FederatedSearcher",
+    "FederatedSourceError",
+    "FederatedStore",
+    "FederatedStoreError",
+    "FederatedTarget",
+    "FileEntryProvider",
+    "FilterTranslationCategory",
+    "FilterTranslationError",
+    "IdLedger",
+    "IdLedgerError",
+    "MongoStore",  # pyright: ignore[reportUnsupportedDunderAll]  (provided lazily via __getattr__)
+    "MultipleResultsError",
+    "NoResultError",
+    "PageOrder",
+    "PageableResultSetLike",
+    "PaginationCursorError",
+    "PortableQueryCapabilities",
+    "PropertyValidationError",
+    "ReferenceEntryProvider",
+    "ResultPage",
+    "ResultRow",
+    "ResultRowLike",
+    "ResultSetLike",
+    "RunEntryProvider",
+    "SearchExpression",
+    "SearchField",
+    "SearchResult",
+    "SearchVariable",
+    "Searcher",
+    "SqlStore",  # pyright: ignore[reportUnsupportedDunderAll]  (provided lazily via __getattr__)
+    "Store",
+    "UnsupportedQueryError",
+    "check_ledger_key",
+    "export_dataset",
+    "filter_searcher",
+    "portable_query_capabilities",
+    "portable_query_fields",
+    "product_relationships",
+    "validate_property",
+    "validate_record",
+]
+
+# The storage-backend engines are re-exported lazily: importing ``httk.store``
+# must stay free of both ``sqlalchemy`` and ``pymongo``, so ``Backend`` and
+# ``SqlStore`` load the SQL layer, and ``MongoStore`` the MongoDB layer, only on
+# first attribute access.
+_LAZY_EXPORTS = {
+    "Backend": "httk.store.backend.sql.engine",
+    "SqlStore": "httk.store.backend.sql.store",
+    "MongoStore": "httk.store.backend.mongo",
+}
+
+
+def __getattr__(name: str) -> Any:
+    """Import a storage-backend engine lazily on first access.
+
+    :param name: The module attribute to import.
+    :return: The requested storage-backend export.
+    :raises AttributeError: If ``name`` is not a lazily re-exported attribute.
+    """
+    module_name = _LAZY_EXPORTS.get(name)
+    if module_name is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    return getattr(importlib.import_module(module_name), name)

httk/store/backend/__init__.py

@@ -0,0 +1,24 @@
+"""Storage backends for *httk-store*.
+
+Each subpackage is a self-contained storage backend:
+
+- :mod:`httk.store.backend.sql` — the SQLAlchemy-backed relational layer
+  (:class:`~httk.store.backend.sql.engine.Backend`,
+  :class:`~httk.store.backend.sql.store.SqlStore`) shared by every SQL dialect;
+- :mod:`httk.store.backend.sqlite`, :mod:`httk.store.backend.duckdb`,
+  :mod:`httk.store.backend.clickhouse`, :mod:`httk.store.backend.postgresql` —
+  the per-dialect constructor and quirk code the SQL layer delegates to; and
+- :mod:`httk.store.backend.mongo` — the MongoDB-backed layer
+  (:class:`~httk.store.backend.mongo.store.MongoStore`).
+
+Two backend-neutral modules sit alongside the subpackages, driver-free and
+shared by every backend:
+
+- :mod:`httk.store.backend.schema` — :func:`~httk.store.backend.schema.resolve_schema`
+  and the schema IR that drives storage; and
+- :mod:`httk.store.backend.codecs` — the :class:`~httk.store.backend.codecs.ValueCodec`
+  registry with exact, round-trippable encodings.
+
+Importing this package pulls in neither ``sqlalchemy`` nor ``pymongo``; the
+driver-backed names load lazily on first use of the relevant subpackage.
+"""

httk/store/backend/clickhouse/__init__.py

@@ -0,0 +1,8 @@
+"""ClickHouse-specific support for the SQL storage backend.
+
+The dialect-agnostic engine wrapper and store live in
+:mod:`httk.store.backend.sql`; this package holds the ClickHouse constructor
+(:func:`httk.store.backend.clickhouse.engine.database`) and the Keeper-backed
+schema, lease, and connection-guard machinery in
+:mod:`httk.store.backend.clickhouse.support`.
+"""

httk/store/backend/clickhouse/engine.py

@@ -0,0 +1,61 @@
+"""ClickHouse database construction for :class:`~httk.store.backend.sql.engine.Backend`."""
+
+import importlib
+from typing import TYPE_CHECKING
+
+import sqlalchemy
+
+if TYPE_CHECKING:
+    from httk.store.backend.sql.engine import Backend
+
+
+def database(
+    cls: "type[Backend]",
+    url: str | sqlalchemy.URL,
+    *,
+    database: str | None = None,
+) -> "Backend":
+    """Build a ClickHouse-backed backend from a ``clickhousedb://`` URL.
+
+    The URL uses the SQLAlchemy ``clickhouse-connect`` dialect, for example
+    ``clickhousedb://default:@host:8123/my_database``.  ``database`` replaces the
+    URL path when supplied.  The constructor always merges ``join_use_nulls=1``
+    into the URL query and selects the ``bulk-fenced`` storage profile before any
+    :class:`~httk.store.backend.sql.store.SqlStore` initialization occurs.
+
+    :param cls: The backend class to instantiate.
+    :param url: ClickHouse SQLAlchemy URL or URL string.
+    :param database: The database name overriding the URL path, if supplied.
+    :return: Connected ClickHouse backend wrapper using the bulk-fenced profile.
+    :raises ImportError: If ``clickhouse-connect`` is not installed; install the
+        ``httk-store[clickhouse]`` extra.
+    :raises RuntimeError: If Keeper is unavailable, the server is too old, or
+        ``join_use_nulls`` cannot be enforced.
+    """
+    try:
+        importlib.import_module("clickhouse_connect")
+        # Importing this module registers the clickhousedb SQLAlchemy URL.
+        importlib.import_module("clickhouse_connect.cc_sqlalchemy")
+    except ImportError as error:
+        raise ImportError(
+            "the ClickHouse backend needs clickhouse-connect; install the 'httk-store[clickhouse]' extra "
+            "to use Backend.clickhouse()"
+        ) from error
+    from sqlalchemy.engine import make_url
+
+    clickhouse_url = make_url(url) if isinstance(url, str) else url
+    if clickhouse_url.drivername.split("+")[0] != "clickhousedb":
+        raise ValueError("Backend.clickhouse() requires a clickhousedb:// SQLAlchemy URL")
+    if database is not None:
+        clickhouse_url = clickhouse_url.set(database=database)
+    clickhouse_url = clickhouse_url.update_query_dict({"join_use_nulls": "1"})
+    engine = sqlalchemy.create_engine(clickhouse_url)
+    try:
+        result = cls(engine, write_profile="bulk-fenced")
+        from httk.store.backend.clickhouse.support import ensure_bootstrap_table
+
+        ensure_bootstrap_table(result.engine)
+        return result
+    except BaseException:
+        engine.dispose()
+        raise