httk-core 2.0.0__py3-none-any.whl

httk/core/__init__.py

@@ -0,0 +1,112 @@
+#
+#    The high-throughput toolkit (httk)
+#    Copyright (C) 2012-2024 the httk AUTHORS
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import pkgutil
+
+import httk
+
+from . import _discover
+from .dataloader import DataLoader, DataRecord, DatasetMeta, DecodeObjectCallback
+from .datastream import (
+    BytestreamBackend,
+    BytestreamBytes,
+    BytestreamBytesView,
+    BytestreamCommon,
+    BytestreamFile,
+    BytestreamFilename,
+    BytestreamFilenameView,
+    BytestreamFileView,
+    BytestreamLike,
+    BytestreamRequest,
+    BytestreamRequestView,
+    BytestreamURL,
+    BytestreamURLView,
+    BytestreamView,
+    CompressionCodec,
+    TextstreamBackend,
+    TextstreamCommon,
+    TextstreamFile,
+    TextstreamFilename,
+    TextstreamFilenameView,
+    TextstreamFileView,
+    TextstreamLike,
+    TextstreamRequest,
+    TextstreamRequestView,
+    TextstreamString,
+    TextstreamStringView,
+    TextstreamURL,
+    TextstreamURLView,
+    TextstreamView,
+    known_compressions,
+    register_compression,
+)
+from .loading import load
+from .views import Backend, View, unwrap
+
+_discover.discover_and_register()
+
+
+def _discover_modules():
+    prefix = httk.__name__ + "."
+    names = [m.name for m in pkgutil.iter_modules(httk.__path__, prefix) if m.ispkg]
+    return names
+
+
+subpackages = _discover_modules()
+
+__all__ = [
+    "load",
+    "subpackages",
+    "DataLoader",
+    "DataRecord",
+    "DatasetMeta",
+    "DecodeObjectCallback",
+    "Backend",
+    "View",
+    "unwrap",
+    "BytestreamView",
+    "BytestreamFileView",
+    "BytestreamFilenameView",
+    "BytestreamBytesView",
+    "BytestreamRequestView",
+    "BytestreamURLView",
+    "BytestreamBackend",
+    "BytestreamCommon",
+    "BytestreamFile",
+    "BytestreamFilename",
+    "BytestreamBytes",
+    "BytestreamRequest",
+    "BytestreamURL",
+    "BytestreamLike",
+    "TextstreamView",
+    "TextstreamFileView",
+    "TextstreamFilenameView",
+    "TextstreamStringView",
+    "TextstreamRequestView",
+    "TextstreamURLView",
+    "TextstreamBackend",
+    "TextstreamCommon",
+    "TextstreamFile",
+    "TextstreamFilename",
+    "TextstreamString",
+    "TextstreamRequest",
+    "TextstreamURL",
+    "TextstreamLike",
+    "CompressionCodec",
+    "register_compression",
+    "known_compressions",
+]

httk/core/_discover.py

@@ -0,0 +1,35 @@
+#
+#    The high-throughput toolkit (httk)
+#    Copyright (C) 2012-2024 the httk AUTHORS
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import importlib
+import importlib.util
+import pkgutil
+
+
+def discover_and_register():
+    import httk.handlers
+
+    prefix = "httk.handlers."
+    for m in pkgutil.iter_modules(httk.handlers.__path__, prefix):
+        if not m.ispkg:
+            continue
+
+        spec = importlib.util.find_spec(m.name)
+        if spec is None:
+            continue
+
+        _mod = importlib.import_module(m.name)  # imports only that handler package chain

httk/core/_plugins.py

@@ -0,0 +1,95 @@
+#
+#    The high-throughput toolkit (httk)
+#    Copyright (C) 2012-2024 the httk AUTHORS
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from dataclasses import dataclass
+from importlib import import_module
+from typing import Any, Callable
+
+CallableRef = str | Callable[..., Any]
+
+
+def resolve_callable(ref: CallableRef) -> Callable[..., Any]:
+    """
+    Resolve a callable reference.
+
+    Accepts:
+      - a callable object
+      - a string of form "module.submodule:callable_name"
+    """
+    if callable(ref):
+        return ref
+    if not isinstance(ref, str):
+        raise TypeError(f"Expected callable or str reference, got {type(ref)!r}")
+
+    module_name, _, attr = ref.partition(":")
+    if not module_name or not attr:
+        raise ValueError(f"Invalid reference {ref!r}; expected 'module:callable'.")
+
+    obj = getattr(import_module(module_name), attr)
+    if not callable(obj):
+        raise TypeError(f"Resolved {ref!r} to non-callable object {obj!r}")
+    return obj
+
+
+@dataclass(frozen=True)
+class PluginSpec:
+    """
+    A minimal plugin spec.
+
+    - `key`: selection key (e.g., a file extension like ".cif", or a format name like "cif")
+    - `handler`: callable or "module:callable" reference (lazy)
+    """
+
+    key: str
+    handler: CallableRef
+    name: str | None = None  # optional display name
+
+
+class PluginRegistry:
+    """
+    Registry mapping keys -> plugin specs.
+
+    Intended use:
+      - loaders: key is file extension ".cif"
+      - savers: key is file extension ".cif" or format name
+      - show/visualization: key is format/backend name
+    """
+
+    def __init__(self) -> None:
+        self._by_key: dict[str, PluginSpec] = {}
+
+    def register(self, *, key: str, handler: CallableRef, name: str | None = None) -> None:
+        k = key
+        self._by_key[k] = PluginSpec(key=k, handler=handler, name=name)
+
+    def keys(self) -> list[str]:
+        return sorted(self._by_key.keys())
+
+    def get(self, key: str) -> PluginSpec | None:
+        return self._by_key.get(key)
+
+    def require(self, key: str) -> PluginSpec:
+        spec = self.get(key)
+        if spec is None:
+            known = ", ".join(self.keys()) or "(none)"
+            raise ValueError(f"No plugin registered for {key!r}. Known: {known}")
+        return spec
+
+    def dispatch(self, key: str, *args: Any, **kwargs: Any) -> Any:
+        spec = self.require(key)
+        fn = resolve_callable(spec.handler)
+        return fn(*args, **kwargs)

httk/core/dataloader.py

@@ -0,0 +1,263 @@
+#
+#    The high-throughput toolkit (httk)
+#    Copyright (C) 2012-2024 the httk AUTHORS
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+from collections.abc import Callable, Iterator, KeysView
+from dataclasses import dataclass
+from functools import cached_property
+from pathlib import Path
+from typing import Any, ClassVar
+
+from .datastream import TextstreamFileView, TextstreamLike
+from .datastream.compression import split_compression_suffix
+
+type DecodeObjectCallback = Callable[[dict[str, Any], str], Any]
+"""Callback invoked as ``(dict_obj, jsonld_url)`` that returns the value to use in place of
+``dict_obj`` (return the input unchanged to decline)."""
+
+
+class DataRecord:
+    """Read-only attribute and mapping view over a ``dict[str, Any]``.
+
+    Top-level keys are reachable both as attributes (``record.name``) and as items
+    (``record["name"]``); the wrapped values are the plain parsed JSON and are not
+    themselves wrapped. Supports iteration over keys, ``len()``, ``in``, and ``keys()``.
+    """
+
+    def __init__(self, data: dict[str, Any]) -> None:
+        self._data = data
+
+    def __getattr__(self, name: str) -> Any:
+        # Underscored names are never data keys; failing fast here also avoids recursing
+        # into self._data lookups when _data itself is not yet set (copy/unpickling).
+        if name.startswith("_"):
+            raise AttributeError(name)
+        try:
+            return self._data[name]
+        except KeyError:
+            raise AttributeError(name) from None
+
+    def __getitem__(self, key: str) -> Any:
+        return self._data[key]
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._data)
+
+    def __len__(self) -> int:
+        return len(self._data)
+
+    def __contains__(self, key: object) -> bool:
+        return key in self._data
+
+    def keys(self) -> KeysView[str]:
+        return self._data.keys()
+
+    def __repr__(self) -> str:
+        return f"DataRecord({self._data!r})"
+
+
+@dataclass(frozen=True)
+class DatasetMeta:
+    """Header metadata extracted from a structured JSON-LD dataset document."""
+
+    context: dict[str, Any]
+    """The raw ``@context`` object."""
+
+    id: str | None
+    """The document ``@id``, or ``None`` if absent."""
+
+    type_: str | None
+    """The document ``@type``, or ``None`` if absent (trailing underscore avoids the builtin ``type``)."""
+
+    header: dict[str, Any]
+    """All remaining top-level keys except ``data``, ``indicies``, and ``@``-keys (titles, creator, license, provenance, ...)."""
+
+    dataset_ids: dict[str, str]
+    """Mapping of dataset name to its ``@id``."""
+
+    fields: dict[str, dict[str, str]]
+    """Mapping of dataset name to a mapping of field name to its property URL."""
+
+
+@dataclass(frozen=True)
+class _LoadedData:
+    data: Any
+    meta: DatasetMeta | None
+    index: DataRecord | None
+
+
+def _build_meta(doc: dict[str, Any]) -> DatasetMeta:
+    context = doc.get("@context", {})
+    if not isinstance(context, dict):
+        context = {}
+
+    header = {key: value for key, value in doc.items() if key not in ("data", "indicies") and not key.startswith("@")}
+
+    dataset_ids: dict[str, str] = {}
+    fields: dict[str, dict[str, str]] = {}
+
+    data_context = context.get("data", {})
+    nested = data_context.get("@context", {}) if isinstance(data_context, dict) else {}
+    if isinstance(nested, dict):
+        for dataset_name, spec in nested.items():
+            if not isinstance(spec, dict):
+                continue
+            dataset_id = spec.get("@id")
+            if isinstance(dataset_id, str):
+                dataset_ids[dataset_name] = dataset_id
+            field_context = spec.get("@context")
+            if isinstance(field_context, dict):
+                field_urls = {field: url for field, url in field_context.items() if isinstance(url, str)}
+                if field_urls:
+                    fields[dataset_name] = field_urls
+
+    return DatasetMeta(
+        context=context,
+        id=doc.get("@id"),
+        type_=doc.get("@type"),
+        header=header,
+        dataset_ids=dataset_ids,
+        fields=fields,
+    )
+
+
+def _decode_entry(
+    entry: dict[str, Any],
+    field_urls: dict[str, str],
+    dataset_id: str | None,
+    decode: DecodeObjectCallback,
+) -> Any:
+    for field_name, url in field_urls.items():
+        value = entry.get(field_name)
+        if isinstance(value, dict):
+            entry[field_name] = decode(value, url)
+    if dataset_id is None:
+        return entry
+    return decode(entry, dataset_id)
+
+
+def _apply_decode(data: dict[str, Any], meta: DatasetMeta, decode: DecodeObjectCallback) -> None:
+    for dataset_name, entries in data.items():
+        dataset_id = meta.dataset_ids.get(dataset_name)
+        field_urls = meta.fields.get(dataset_name, {})
+        if dataset_id is None and not field_urls:
+            continue
+        if isinstance(entries, dict):
+            data[dataset_name] = _decode_entry(entries, field_urls, dataset_id, decode)
+        elif isinstance(entries, list):
+            data[dataset_name] = [
+                _decode_entry(entry, field_urls, dataset_id, decode) if isinstance(entry, dict) else entry
+                for entry in entries
+            ]
+
+
+class DataLoader:
+    """Lazy loader for httk dataset files, resolved only when data is first accessed.
+
+    A ``DataLoader`` is a declare-time placeholder: constructing it records its arguments
+    and performs no I/O. The source is read the first time ``data``, ``meta``, or ``index``
+    is accessed. Files are either plain JSON (any JSON value is exposed as ``data`` with
+    ``meta``/``index`` set to ``None``) or a structured JSON-LD document (with ``@context``,
+    header fields, ``data``, and optional ``indicies``) whose header is exposed via ``meta``,
+    datasets via ``data.<name>``, and lookup indices via ``index.<name>``.
+
+    Loaders that share an ``identifier`` deduplicate through a class-level registry: the
+    first load wins, and later loaders reusing that identifier return the same result while
+    their ``source`` and ``decode_object`` arguments are ignored. Keeping identifiers unique
+    is the caller's responsibility. Not thread-safe.
+
+    Format is resolved from the source name after stripping any compression suffix: a ``.json``
+    name (e.g. ``data.json`` or ``data.json.gz``) is parsed as JSON; any other recognizable
+    suffix raises ``ValueError``; a source with no determinable name is treated as JSON.
+    Compression is handled transparently by the stream layer, so ``.json.gz`` and similar load
+    directly. A ``str``/``Path`` source is interpreted as a filename unless its scheme marks it
+    as a URL (``http``, ``https``, ``ftp``, ``file``); pass ``kind="content"`` for literal
+    content or ``kind="filename"``/``kind="url"`` to force an interpretation.
+
+    Example:
+        symmetry_basics = DataLoader("symmetry_basics", "data/spacegroup_symbols.json")
+        spacegroups = symmetry_basics.data.spacegroups  # first access triggers the load
+    """
+
+    _loaded: ClassVar[dict[str, _LoadedData]] = {}
+
+    def __init__(
+        self,
+        identifier: str,
+        source: TextstreamLike,
+        decode_object: DecodeObjectCallback | None = None,
+        **hints: Any,
+    ) -> None:
+        self._identifier = identifier
+        self._source = source
+        self._decode_object = decode_object
+        self._hints = hints
+
+    def _resolve_name(self) -> str | None:
+        source = self._source
+        if isinstance(source, (str, Path)):
+            # With kind="content" the string is the data itself, not a name.
+            if self._hints.get("kind") == "content":
+                return None
+            return str(source)
+        url = getattr(source, "url", None)
+        if isinstance(url, str):
+            return url
+        name = getattr(source, "name", None)
+        if isinstance(name, str):
+            return name
+        return None
+
+    def _load(self) -> _LoadedData:
+        if self._identifier in DataLoader._loaded:
+            return DataLoader._loaded[self._identifier]
+
+        name = self._resolve_name()
+        if name is not None:
+            base, _ = split_compression_suffix(name)
+            suffix = Path(base).suffix.lower()
+            if suffix not in ("", ".json"):
+                raise ValueError(f"unsupported data format: {suffix}")
+
+        doc = json.load(TextstreamFileView(self._source, **self._hints))
+
+        if isinstance(doc, dict) and "@context" in doc:
+            meta = _build_meta(doc)
+            data_section = doc.get("data")
+            if self._decode_object is not None and isinstance(data_section, dict):
+                _apply_decode(data_section, meta, self._decode_object)
+            data: Any = DataRecord(data_section) if isinstance(data_section, dict) else data_section
+            index_section = doc.get("indicies")
+            index = DataRecord(index_section) if isinstance(index_section, dict) else None
+            loaded = _LoadedData(data=data, meta=meta, index=index)
+        else:
+            loaded = _LoadedData(data=doc, meta=None, index=None)
+
+        DataLoader._loaded[self._identifier] = loaded
+        return loaded
+
+    @cached_property
+    def data(self) -> Any:
+        return self._load().data
+
+    @cached_property
+    def meta(self) -> DatasetMeta | None:
+        return self._load().meta
+
+    @cached_property
+    def index(self) -> DataRecord | None:
+        return self._load().index

httk/core/datastream/__init__.py

@@ -0,0 +1,80 @@
+from .bytestream_backend import BytestreamBackend
+from .bytestream_bytes import BytestreamBytes
+from .bytestream_bytes_view import BytestreamBytesView
+from .bytestream_common import BytestreamCommon
+from .bytestream_file import BytestreamFile
+from .bytestream_file_view import BytestreamFileView
+from .bytestream_filename import BytestreamFilename
+from .bytestream_filename_view import BytestreamFilenameView
+from .bytestream_like import BytestreamLike
+from .bytestream_request import BytestreamRequest
+from .bytestream_request_view import BytestreamRequestView
+from .bytestream_url import BytestreamURL
+from .bytestream_url_view import BytestreamURLView
+from .bytestream_view import BytestreamView
+from .compression import CompressionCodec, known_compressions, register_compression
+from .textstream_backend import TextstreamBackend
+from .textstream_common import TextstreamCommon
+from .textstream_file import TextstreamFile
+from .textstream_file_view import TextstreamFileView
+from .textstream_filename import TextstreamFilename
+from .textstream_filename_view import TextstreamFilenameView
+from .textstream_like import TextstreamLike
+from .textstream_request import TextstreamRequest
+from .textstream_request_view import TextstreamRequestView
+from .textstream_string import TextstreamString
+from .textstream_string_view import TextstreamStringView
+from .textstream_url import TextstreamURL
+from .textstream_url_view import TextstreamURLView
+from .textstream_view import TextstreamView
+
+# *URL backends precede *Filename/*String so a bare scheme'd string dispatches to a URL;
+# an explicit kind= hint still forces the intended interpretation either way.
+BytestreamBackend.backend_classes = [
+    BytestreamFile,
+    BytestreamRequest,
+    BytestreamURL,
+    BytestreamFilename,
+    BytestreamBytes,
+]
+TextstreamBackend.backend_classes = [
+    TextstreamFile,
+    TextstreamRequest,
+    TextstreamURL,
+    TextstreamFilename,
+    TextstreamString,
+]
+
+__all__ = [
+    "BytestreamView",
+    "BytestreamFileView",
+    "BytestreamFilenameView",
+    "BytestreamBytesView",
+    "BytestreamRequestView",
+    "BytestreamURLView",
+    "BytestreamBackend",
+    "BytestreamCommon",
+    "BytestreamFile",
+    "BytestreamFilename",
+    "BytestreamBytes",
+    "BytestreamRequest",
+    "BytestreamURL",
+    "BytestreamLike",
+    "TextstreamView",
+    "TextstreamFileView",
+    "TextstreamFilenameView",
+    "TextstreamStringView",
+    "TextstreamRequestView",
+    "TextstreamURLView",
+    "TextstreamBackend",
+    "TextstreamCommon",
+    "TextstreamFile",
+    "TextstreamFilename",
+    "TextstreamString",
+    "TextstreamRequest",
+    "TextstreamURL",
+    "TextstreamLike",
+    "CompressionCodec",
+    "register_compression",
+    "known_compressions",
+]

httk/core/datastream/bytestream_api.py

@@ -0,0 +1,33 @@
+from abc import ABC, abstractmethod
+
+
+class BytestreamAPI(ABC):
+    """
+    Abstract base class for a bare minimum API for streamable byte data.
+
+    Supports:
+    * read, close, name, and closed with the meanings defined by io.IOBase
+
+    Since it is a *minimal* streaming data API it deliberately omits:
+    seek, tell, etc.; there should be no assumption that the underlying
+    data source is seekable. However, many backend implementations
+    may choose to support them.
+    """
+
+    @abstractmethod
+    def read(self, size: int = -1) -> bytes:
+        raise NotImplementedError
+
+    @abstractmethod
+    def close(self) -> None:
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def name(self) -> str | None:
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def closed(self) -> bool:
+        raise NotImplementedError

httk/core/datastream/bytestream_backend.py

@@ -0,0 +1,12 @@
+from typing import Any, ClassVar
+
+from ..views import Backend
+from .bytestream_api import BytestreamAPI
+
+
+class BytestreamBackend(Backend["BytestreamBackend"], BytestreamAPI):
+    """
+    Abstract base class for all backends of streaming byte data.
+    """
+
+    backend_classes: ClassVar[list[type[Backend[Any]]]]

httk/core/datastream/bytestream_bytes.py

@@ -0,0 +1,52 @@
+import io
+from typing import Any
+
+from .bytestream_backend import BytestreamBackend
+from .bytestream_common import BytestreamCommon
+from .compression import open_compressed, validate_compression
+
+
+class BytestreamBytes(BytestreamCommon, BytestreamBackend):
+    """
+    Backend for streaming byte data backed by an actual bytes object.
+    """
+
+    b: bytes
+    _compression: str
+    _f: io.IOBase | None
+    _underlying: io.IOBase | None
+    _closed: bool
+
+    # Cannot type annotate __new__ as `Self | None` for some reason
+    def __new__(cls, content: bytes | bytearray, **hints: Any) -> Any:
+        if not isinstance(content, bytes | bytearray):
+            return None
+        if hints and hints.get("kind", "content") != "content":
+            return None
+        return super().__new__(cls)
+
+    def __init__(self, content: bytes | bytearray, **hints: Any) -> None:
+        self.b = bytes(content)
+        self._compression = hints.get("compression", "auto")
+        validate_compression(self._compression)
+        self._f = None
+        self._underlying = None
+        self._closed = False
+
+    def _ensure_f(self) -> io.IOBase:
+        if self._closed:
+            raise ValueError("I/O operation on closed stream")
+        if self._f is None:
+            raw: io.IOBase = io.BytesIO(self.b)
+            opened = open_compressed(raw, compression=self._compression, name=None)
+            self._underlying = raw if opened is not raw else None
+            self._f = opened
+        return self._f
+
+    @property
+    def name(self) -> str | None:
+        return None
+
+    @property
+    def closed(self) -> bool:
+        return self._closed