pygeodetic-engine 0.2.0__py3-none-any.whl

geodetic_engine/__init__.py

@@ -0,0 +1,14 @@
+"""Geodetic transformation engine built on PROJ."""
+
+from importlib.metadata import version as _version
+
+from geodetic_engine._bundled_database import activate_at_import as _activate
+from geodetic_engine._environment import require_pinned_proj as _require_pinned_proj
+
+_require_pinned_proj()
+# Before any other code of this package's touches PROJ's search path, so that a
+# set_data_dir() call of the caller's own is still recognised as one.
+_activate()
+
+# The PyPI name, which differs from the import name.
+__version__ = _version("pygeodetic-engine")

geodetic_engine/_bundled_database.py

@@ -0,0 +1,418 @@
+"""Make the OSDU database shipped with this package the one PROJ reads.
+
+A release wheel ships ``geodetic_engine/osdudb/_data/proj_data/proj.db``: the
+proj.db of the pinned pyproj wheel with the official OSDU catalogue built into
+it. When ``geodetic_engine`` is first imported, the first of these rules that
+matches decides whether PROJ reads it:
+
+a. ``GEODETIC_ENGINE_OSDU_DB=0``: nothing is changed.
+b. ``pyproj.datadir.set_data_dir()`` or ``append_data_dir()`` was called
+   before the import: nothing is changed.
+c. ``PROJ_DATA``, or ``PROJ_LIB`` when ``PROJ_DATA`` is unset, names a
+   directory that holds a ``proj.db``: PROJ's search path becomes the
+   variable's directories, the first that holds a ``proj.db`` first. That is
+   pyproj's search path already when pyproj read the variable on import and
+   its first directory holds the ``proj.db``, and nothing is changed. Any
+   other search path, say for a value set after pyproj was imported, is
+   replaced, with a warning; but if the variable's ``proj.db`` records no
+   database layout, another major one, or an older minor one than the
+   ``proj.db`` PROJ reads, which PROJ would refuse, nothing is changed, with a
+   warning.
+d. Otherwise PROJ's search path becomes the bundled database's directory, then
+   the ``PROJ_DATA``/``PROJ_LIB`` directories, then pyproj's own data
+   directory, so grids and ``proj.ini`` are still found where they were.
+
+That is the precedence of this project's pyproj wheels (``set_data_dir``, then
+a ``PROJ_DATA`` holding a database, then the database the wheel bundles) with
+the OSDU database in place of the wheel's, applied to the environment as it is
+on import: the environment a child process starts with. No file is written and
+the environment is left alone, so a child process that imports the package
+applies the same rules.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import sqlite3
+from contextlib import closing, suppress
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final
+
+from pyproj import datadir
+from pyproj.exceptions import DataDirError
+
+from geodetic_engine import _search_path
+from geodetic_engine._sqlite import read_only
+
+logger = logging.getLogger(__name__)
+
+ENVIRONMENT_VARIABLE: Final = "GEODETIC_ENGINE_OSDU_DB"
+
+# A directory of its own: PROJ reads the file called proj.db in each directory
+# of its search path, so nothing else may be called that here.
+BUNDLED_DIRECTORY: Final = (
+    Path(__file__).resolve().parent / "osdudb" / "_data" / "proj_data"
+)
+
+_DATABASE: Final = "proj.db"
+_LAYOUT: Final = ("DATABASE.LAYOUT.VERSION.MAJOR", "DATABASE.LAYOUT.VERSION.MINOR")
+_OFF: Final = frozenset({"0", "false", "no", "off"})
+_ON: Final = frozenset({"", "1", "true", "yes", "on"})
+_CHANGED: Final = (
+    "pyproj.datadir.set_data_dir() changed PROJ's search path after "
+    "geodetic_engine chose the database"
+)
+
+
+@dataclass(frozen=True, slots=True)
+class ActiveDatabase:
+    """The ``proj.db`` PROJ reads, and why it is that one.
+
+    Attributes:
+        path: The database PROJ reads, the ``proj.db`` in the first directory
+            of its search path. None when that directory holds none.
+        bundled: Whether it is the OSDU database shipped with this package.
+        reason: Why it is in use, or why the bundled database is not.
+        search_path: Every directory PROJ searches, in order, for the
+            database, grids and ``proj.ini``.
+    """
+
+    path: Path | None
+    bundled: bool
+    reason: str
+    search_path: tuple[Path, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class _Choice:
+    reason: str
+    # PROJ's search path as the choice left it; a different one now means the
+    # caller has set another since.
+    seen: str | None
+    # The search path before the bundled database was put first, to go back
+    # to when it is switched off.
+    restore: str | None = None
+
+
+_choice: _Choice | None = None
+
+
+def activate_at_import() -> None:
+    """Apply rules a to d, once, when ``geodetic_engine`` is first imported."""
+    global _choice
+    # Logged once the lock is released; see _search_path.
+    warnings: list[str] = []
+    with _search_path.lock:
+        if _choice is not None:
+            return
+        reason, search = _decide(warnings)
+        before = _data_dir()
+        if search is not None:
+            _search_path.set_search_path(os.pathsep.join(search))
+        _choice = _Choice(
+            reason, _data_dir(), restore=before if search is not None else None
+        )
+        active = _active()
+    for warning in warnings:
+        logger.warning("%s", warning)
+    _log(active)
+
+
+def active_database() -> ActiveDatabase:
+    """Report the ``proj.db`` PROJ reads and why.
+
+    Returns:
+        The database, whether it is the bundled OSDU database, and the reason:
+        the rule that chose it on import, a call of
+        :func:`use_bundled_database`, or a later
+        ``pyproj.datadir.set_data_dir()``.
+
+    Example:
+        >>> from geodetic_engine.osdudb import active_database
+        >>> active_database().bundled  # doctest: +SKIP
+        True
+    """
+    return _reading()
+
+
+def use_bundled_database(enabled: bool = True) -> ActiveDatabase:
+    """Switch the OSDU database shipped with this package on or off.
+
+    Overrides the choice made on import, whatever made it. Switched on, the
+    bundled database's directory is put first on PROJ's search path and the
+    other directories stay, so grids are still found. Switched off, PROJ's
+    search path returns to what it was before the bundled database was put
+    first, or else loses the bundled directory.
+
+    A later ``pyproj.datadir.set_data_dir()`` overrides this in turn. pyproj
+    keeps one PROJ context per thread: the calling thread switches at once,
+    and another thread the next time it calls :func:`active_database` or
+    resolves a CRS, operation or identifier through this package. A thread
+    that only calls pyproj directly keeps the database it had.
+
+    Args:
+        enabled: True to read the bundled database, False to stop.
+
+    Returns:
+        The database PROJ now reads, as :func:`active_database` reports it.
+
+    Raises:
+        FileNotFoundError: If switching on, and this installation has no
+            bundled database; or switching off, and no other directory on
+            PROJ's search path holds a ``proj.db``.
+    """
+    global _choice
+    with _search_path.lock:
+        database = _bundled_database()
+        current = _data_dir()
+        directories = _split(current)
+        restore: str | None = None
+        if enabled:
+            if database is None:
+                raise FileNotFoundError(
+                    f"this installation has no bundled OSDU database at "
+                    f"{BUNDLED_DIRECTORY / _DATABASE}; the wheels on PyPI ship "
+                    "one. In a source checkout, run "
+                    "python scripts/build_bundled_osdudb.py, or build a "
+                    "database of your own with geodetic-osdudb build --official"
+                )
+            bundled = str(database.parent)
+            if not directories or not _same(directories[0], bundled):
+                restore = current
+            elif _choice is not None and current == _choice.seen:
+                restore = _choice.restore
+            _search_path.set_search_path(
+                os.pathsep.join(_unique([bundled, *directories]))
+            )
+            reason = "use_bundled_database() switched it on"
+        else:
+            if database is not None and any(
+                _same(entry, database.parent) for entry in directories
+            ):
+                if (
+                    _choice is not None
+                    and _choice.restore is not None
+                    and current == _choice.seen
+                ):
+                    search = _database_first(_split(_choice.restore))
+                else:
+                    search = _database_first(
+                        [e for e in directories if not _same(e, database.parent)]
+                    )
+                _search_path.set_search_path(os.pathsep.join(search))
+            reason = "use_bundled_database(False) switched it off"
+        _choice = _Choice(reason, _data_dir(), restore=restore)
+    active = _reading()
+    _log(active)
+    return active
+
+
+def _reading() -> ActiveDatabase:
+    """Re-point the calling thread's PROJ context, then report what it reads.
+
+    The thread re-points without the lock, so the report is taken under it
+    only once no switch has landed in between.
+    """
+    while True:
+        with suppress(DataDirError):
+            _search_path.database_identity()
+        with _search_path.lock:
+            if _search_path.followed():
+                return _active()
+
+
+def _decide(warnings: list[str]) -> tuple[str, list[str] | None]:
+    """Why the bundled database is or is not used, and the search path to set.
+
+    What to warn about is added to ``warnings``, for the caller to log.
+    """
+    # Read first: once this package has called set_data_dir() itself, a call
+    # of the caller's own can no longer be told apart from it.
+    explicit = _user_data_dir()
+    raw = os.environ.get(ENVIRONMENT_VARIABLE)
+    setting = (raw or "").strip().casefold()
+    if setting in _OFF:
+        return f"{ENVIRONMENT_VARIABLE}={raw} turns it off", None
+    if setting not in _ON:
+        warnings.append(
+            f"{ENVIRONMENT_VARIABLE}={raw!r} is not a boolean; the bundled OSDU "
+            "database stays on by default"
+        )
+    if explicit is not None:
+        return (
+            "pyproj.datadir.set_data_dir() or append_data_dir() was called "
+            "before geodetic_engine was imported",
+            None,
+        )
+    name, value = _environment_data_dir()
+    entries = _split(value)
+    for index, entry in enumerate(entries):
+        if (Path(entry) / _DATABASE).is_file():
+            if len(entries) == 1:
+                reason = f"{name}={entry} holds a proj.db"
+            else:
+                reason = f"{name} lists {entry}, which holds a proj.db"
+            # pyproj passes PROJ the proj.db of the first directory only.
+            search = [entry, *entries[:index], *entries[index + 1 :]]
+            # pyproj keeps the value verbatim when it reads it, and reads it
+            # once: a value it has not read, or read with no proj.db first, is
+            # applied, so that this process reads what a new one would.
+            current = _split(_data_dir())
+            if current == search:
+                return reason, None
+            if (refused := _refusal(name, entry, current)) is not None:
+                warnings.append(refused)
+                return refused, None
+            warnings.append(
+                f"PROJ's search path becomes {os.pathsep.join(search)}, from "
+                f"{name}; it was {os.pathsep.join(current) or 'unset'}"
+            )
+            return reason, search
+    database = _bundled_database()
+    if database is None:
+        return (
+            f"this installation has no bundled OSDU database at "
+            f"{BUNDLED_DIRECTORY / _DATABASE}",
+            None,
+        )
+    search = _unique([str(database.parent), *entries, *_split(_data_dir())])
+    if entries:
+        return f"it is the default, and {name} names no proj.db", search
+    return "it is the default, and nothing chose another database", search
+
+
+def _active() -> ActiveDatabase:
+    current = _data_dir()
+    directories = tuple(Path(entry) for entry in _split(current))
+    path = None
+    if directories and (candidate := directories[0] / _DATABASE).is_file():
+        path = candidate.resolve()
+    database = _bundled_database()
+    bundled = path is not None and database is not None and path == database.resolve()
+    if _choice is None:
+        reason = "geodetic_engine has not chosen a database"
+    elif current != _choice.seen:
+        reason = _CHANGED
+    else:
+        reason = _choice.reason
+    return ActiveDatabase(path, bundled, reason, directories)
+
+
+def _log(active: ActiveDatabase) -> None:
+    if active.bundled:
+        logger.info(
+            "PROJ reads the bundled OSDU database %s: %s", active.path, active.reason
+        )
+    else:
+        logger.info(
+            "PROJ reads %s, not the bundled OSDU database: %s",
+            active.path or "no proj.db",
+            active.reason,
+        )
+
+
+def _bundled_database() -> Path | None:
+    path = BUNDLED_DIRECTORY / _DATABASE
+    return path if path.is_file() else None
+
+
+def _user_data_dir() -> str | None:
+    # pyproj keeps the set_data_dir() argument in a private global.
+    # tests/test_bundled_database.py fails if a pyproj upgrade renames it.
+    value = getattr(datadir, "_USER_PROJ_DATA", None)
+    return None if value is None else str(value)
+
+
+def _environment_data_dir() -> tuple[str, str]:
+    """The variable pyproj reads its data directories from, and its value."""
+    if "PROJ_DATA" in os.environ:
+        return "PROJ_DATA", os.environ["PROJ_DATA"]
+    return "PROJ_LIB", os.environ.get("PROJ_LIB", "")
+
+
+def _data_dir() -> str | None:
+    try:
+        return datadir.get_data_dir()
+    except DataDirError:
+        return None
+
+
+def _split(value: str | None) -> list[str]:
+    return [entry for entry in (value or "").split(os.pathsep) if entry]
+
+
+def _key(entry: str) -> str:
+    return os.path.normcase(os.path.realpath(entry))
+
+
+def _same(first: str | Path, second: str | Path) -> bool:
+    return _key(str(first)) == _key(str(second))
+
+
+def _unique(entries: list[str]) -> list[str]:
+    seen: set[str] = set()
+    found = []
+    for entry in entries:
+        if (key := _key(entry)) not in seen:
+            seen.add(key)
+            found.append(entry)
+    return found
+
+
+def _database_first(entries: list[str]) -> list[str]:
+    """Move the first directory holding a proj.db to the front, as pyproj needs."""
+    for index, entry in enumerate(entries):
+        if (Path(entry) / _DATABASE).is_file():
+            return [entry, *entries[:index], *entries[index + 1 :]]
+    raise FileNotFoundError(
+        "no directory on PROJ's search path but the bundled database's holds "
+        "a proj.db to switch to"
+    )
+
+
+def _refusal(name: str, entry: str, current: list[str]) -> str | None:
+    """Why the proj.db a variable names is not applied, or None to apply it.
+
+    PROJ refuses a database that records no layout version, another major one,
+    or a minor one older than it was built for. The layout of the proj.db PROJ
+    reads now stands in for that, being at least as new.
+    """
+    reading = next(
+        (
+            Path(directory) / _DATABASE
+            for directory in current
+            if (Path(directory) / _DATABASE).is_file()
+        ),
+        None,
+    )
+    expected = None if reading is None else _layout(reading)
+    if reading is None or expected is None:
+        return None
+    candidate = Path(entry) / _DATABASE
+    found = _layout(candidate)
+    if found is not None and found[0] == expected[0] and found[1] >= expected[1]:
+        return None
+    layout = "no database layout" if found is None else _layout_name(found)
+    return (
+        f"{name} names {candidate}, with {layout}, so PROJ keeps reading "
+        f"{reading.resolve()}, with {_layout_name(expected)}"
+    )
+
+
+def _layout(database: Path) -> tuple[int, int] | None:
+    """The database layout version a proj.db records, or None if none."""
+    try:
+        with closing(read_only(database)) as connection:
+            rows = connection.execute(
+                "SELECT key, value FROM metadata WHERE key IN (?, ?)", _LAYOUT
+            ).fetchall()
+        found = dict(rows)
+        return int(found[_LAYOUT[0]]), int(found[_LAYOUT[1]])
+    except (sqlite3.Error, KeyError, TypeError, ValueError):
+        return None
+
+
+def _layout_name(layout: tuple[int, int]) -> str:
+    return f"database layout {layout[0]}.{layout[1]}"

geodetic_engine/_environment.py

@@ -0,0 +1,25 @@
+"""Verify that pyproj uses the PROJ release this package was tested against."""
+
+from typing import Final
+
+import pyproj
+
+from geodetic_engine._pins import read
+
+_PINS: Final = read()
+PINNED_PROJ_VERSION: Final = _PINS.proj_version
+PINNED_PYPROJ_VERSION: Final = _PINS.pyproj_version
+PYPROJ_WHEEL_INDEX: Final = _PINS.pyproj_wheel_index
+PYPROJ_WHEEL_REQUIREMENT: Final = _PINS.pyproj_requirement
+
+
+def require_pinned_proj() -> None:
+    """Refuse to run against a different PROJ and its EPSG dataset."""
+    if pyproj.proj_version_str != PINNED_PROJ_VERSION:
+        raise ImportError(
+            f"geodetic-engine requires PROJ {PINNED_PROJ_VERSION}, but "
+            f"pyproj {pyproj.__version__} uses PROJ {pyproj.proj_version_str}. "
+            f'Install the pinned wheel with: pip install "{PYPROJ_WHEEL_REQUIREMENT}" '
+            f"--extra-index-url {PYPROJ_WHEEL_INDEX}. If no wheel is available "
+            "for your platform, build pyproj from source against the pinned PROJ."
+        )

geodetic_engine/_pins.env

@@ -0,0 +1,15 @@
+# The PROJ, proj-data and pyproj releases geodetic-engine is built, tested and
+# released against, and where the matching pyproj wheels are published. This
+# is the only place they are stated. To change them, follow
+# runbooks/upgrading-proj-and-pyproj.md.
+#
+# KEY=VALUE lines only: no quotes, spaces or shell syntax; it is parsed, never
+# sourced. Read by install-proj.sh, the workflows, scripts/sync_pins.py and the
+# release scripts, and shipped in the wheel as geodetic_engine/_pins.env, where
+# the package reads it on import.
+PROJ_VERSION=9.9.0
+PROJ_SHA256=791a0610547eeabb17006cfd49cdbd2034f3240f47ed5e88a1031811f4e2bcf3
+PROJ_DATA_VERSION=1.25
+PROJ_DATA_SHA256=16d65cb5d25b7d8492dbde4d39cafcc8c42f3e55e6715bac4d87519d6d08f4c8
+PYPROJ_VERSION=3.8.0
+PYPROJ_WHEEL_INDEX=https://equinor.github.io/geodetic-engine/simple/

geodetic_engine/_pins.py

@@ -0,0 +1,97 @@
+"""The PROJ and pyproj releases this package pins, read from one file.
+
+A checkout states them in ``.devcontainer/pins.env``; the build backend ships
+that file in the wheel as ``geodetic_engine/_pins.env``. This module imports
+nothing outside the standard library, so the release scripts load it by path
+without importing the package, and with it pyproj.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final
+
+PACKAGED: Final = Path(__file__).with_name("_pins.env")
+CHECKOUT: Final = Path(__file__).resolve().parents[1] / ".devcontainer" / "pins.env"
+
+_VERSION: Final = r"\d+(?:\.\d+)+"
+_SHA256: Final = r"[0-9a-f]{64}"
+# Every key the file must set, with the form its value must take.
+_FORMS: Final = {
+    "PROJ_VERSION": re.compile(r"\d+\.\d+\.\d+"),
+    "PROJ_SHA256": re.compile(_SHA256),
+    "PROJ_DATA_VERSION": re.compile(_VERSION),
+    "PROJ_DATA_SHA256": re.compile(_SHA256),
+    "PYPROJ_VERSION": re.compile(_VERSION + r"(?:(?:a|b|rc)\d+)?(?:\.post\d+)?"),
+    "PYPROJ_WHEEL_INDEX": re.compile(r"https://[^\s\"'`\\]+/"),
+}
+KEYS: Final = tuple(_FORMS)
+
+
+@dataclass(frozen=True, slots=True)
+class Pins:
+    """The pinned releases.
+
+    Attributes:
+        proj_version: The PROJ release, for example ``"9.9.0"``.
+        proj_sha256: SHA-256 of its source archive.
+        proj_data_version: The proj-data grid package matching that release.
+        proj_data_sha256: SHA-256 of the proj-data archive.
+        pyproj_version: The pyproj release built against that PROJ.
+        pyproj_wheel_index: The package index serving those pyproj wheels.
+    """
+
+    proj_version: str
+    proj_sha256: str
+    proj_data_version: str
+    proj_data_sha256: str
+    pyproj_version: str
+    pyproj_wheel_index: str
+
+    @property
+    def pyproj_requirement(self) -> str:
+        """The pyproj wheel this project builds against the pinned PROJ."""
+        return f"pyproj=={self.pyproj_version}+proj{self.proj_version}"
+
+    def get(self, key: str) -> str:
+        """A value by its key in the file, or ``PYPROJ_REQUIREMENT``."""
+        if key == "PYPROJ_REQUIREMENT":
+            return self.pyproj_requirement
+        if key not in _FORMS:
+            raise ValueError(f"unknown pin {key!r}; known: {', '.join(KEYS)}")
+        return str(getattr(self, key.lower()))
+
+
+def parse(text: str, source: str = "pins.env") -> Pins:
+    """Read the pins, refusing anything but the expected keys in their forms.
+
+    Raises:
+        ValueError: On an unknown, repeated, missing or malformed setting.
+    """
+    found: dict[str, str] = {}
+    for number, line in enumerate(text.splitlines(), 1):
+        if not line.strip() or line.lstrip().startswith("#"):
+            continue
+        key, separator, value = line.partition("=")
+        if not separator or key not in _FORMS:
+            raise ValueError(
+                f"{source}:{number}: expected KEY=VALUE with KEY one of "
+                f"{', '.join(KEYS)}, got {line!r}"
+            )
+        if key in found:
+            raise ValueError(f"{source}:{number}: {key} is set twice")
+        if not _FORMS[key].fullmatch(value):
+            raise ValueError(f"{source}:{number}: {key}={value!r} is malformed")
+        found[key] = value
+    missing = [key for key in KEYS if key not in found]
+    if missing:
+        raise ValueError(f"{source} does not set {', '.join(missing)}")
+    return Pins(*(found[key] for key in KEYS))
+
+
+def read() -> Pins:
+    """The pins of this installation, or of the checkout it is run from."""
+    path = PACKAGED if PACKAGED.is_file() else CHECKOUT
+    return parse(path.read_text(encoding="utf-8"), str(path))

geodetic_engine/_referencing/__init__.py

@@ -0,0 +1,8 @@
+"""The private layer below ``geodesy`` and ``persistablereference``.
+
+What both need and neither owns: reading proj.db, OSDU record ids, the
+PROJJSON of an operation, the ``xy`` axis-order model, the Helmert algebra,
+and the exceptions raised here. It imports neither package, which lets
+``persistablereference`` read payloads without importing ``geodesy``.
+``geodesy`` re-exports what is public. ``lint-imports`` enforces the layering.
+"""