pytypehintstore 0.0.4__py3-none-any.whl

pytypehintstore/__init__.py

@@ -0,0 +1,11 @@
+from pytypehintstore.errors import StoreError, StoreLoadError, StoreLockedError
+from pytypehintstore.store import store_of
+
+__version__ = "0.0.4"
+
+__all__ = [
+    "store_of",
+    "StoreError",
+    "StoreLockedError",
+    "StoreLoadError",
+]

pytypehintstore/codec.py

@@ -0,0 +1,249 @@
+"""The transport form of a row: ISO text for a date, the member name for an
+enum, an object for a nested dataclass.
+
+Nothing here validates. `decode` converts only where the shape is the single
+possible reading; anything else travels intact for the core to reject.
+"""
+
+from datetime import date, time
+
+from pytypehint import Date, EnumShape, Float, Int, List, Str, Struct, Time
+
+# Reserved keys of the discriminated transport. A field name is an identifier,
+# so it can never collide with either.
+_TYPE = "$type"
+_VALUE = "$value"
+
+
+# The JSON type a value is written as: a date, a time and an enum member all
+# arrive as text and compete with a str. This decides whether an option needs
+# naming on the wire.
+def _wire_type(shape):
+    if type(shape) is Struct:
+        return dict
+
+    if type(shape) in (Date, Time, EnumShape):
+        return str
+
+    return shape.pytype
+
+
+# The type the core routes on, once decode has turned text back into values:
+# this decides whether the core still wants the wrapper.
+def _core_type(shape):
+    return dict if type(shape) is Struct else shape.pytype
+
+
+def _field(fields, name):
+    return next((f for f in fields if f.name == name), None)
+
+
+# Two lists share a Python type and differ only in what they hold, so
+# list[str] and list[int] are settled by reading their items.
+def _branch_of(shapes, value):
+    candidates = [s for s in shapes if type(value) is s.pytype]
+
+    if len(candidates) < 2:
+        return candidates[0] if candidates else None
+
+    return next((s for s in candidates if _accepts(s, value)), candidates[0])
+
+
+def _accepts(shape, value) -> bool:
+    if type(shape) is not List:
+        return True
+
+    for item in value:
+        branch = _branch_of(shape.item, item)
+
+        if branch is None or not _accepts(branch, item):
+            return False
+
+    return True
+
+
+def encode(struct, instance) -> dict:
+    return {f.name: _encode_options(f.shape, getattr(instance, f.name))
+            for f in struct.fields}
+
+
+def _encode_options(shapes, value):
+    shape = _branch_of(shapes, value)
+
+    if shape is None:
+        # A field mutated by hand: no option owns its Python type. It travels
+        # intact for the core to judge. Where the transport is wider than the
+        # type — ISO text under a Date — the round trip rewrites the field
+        # instead of refusing it; see README, "Known limits".
+        return value
+
+    written = _encode_value(shape, value)
+
+    # The core names a dataclass variant inside the object itself.
+    if type(shape) is Struct:
+        if sum(1 for s in shapes if type(s) is Struct) > 1:
+            return {_TYPE: shape.cls.__name__, **written}
+
+        return written
+
+    # One option per JSON type needs no discriminator; when several share one,
+    # the option gets named.
+    if sum(1 for s in shapes if _wire_type(s) is _wire_type(shape)) > 1:
+        return {_TYPE: shape.option_id(), _VALUE: written}
+
+    return written
+
+
+def _encode_value(shape, value):
+    if type(shape) is Struct:
+        return encode(shape, value)
+
+    if type(shape) is EnumShape:
+        # The name, not the value: the stable, readable half of a member.
+        return value.name
+
+    if type(shape) in (Date, Time):
+        return value.isoformat()
+
+    if type(shape) is List:
+        return [_encode_options(shape.item, item) for item in value]
+
+    return value
+
+
+def decode(struct, data):
+    if type(data) is not dict:
+        return data
+
+    result = {}
+
+    for key, value in data.items():
+        field = _field(struct.fields, key)
+
+        # An unknown key travels intact; rejecting it is build()'s job.
+        result[key] = (value if field is None
+                       else _decode_options(field.shape, value))
+
+    return result
+
+
+def _decode_options(shapes, value):
+    # A None is a value, not a path to descend; an omitted key never gets here
+    # at all and takes its default.
+    if value is None:
+        return None
+
+    if type(value) is dict:
+        return _decode_dict(shapes, value)
+
+    if type(value) is list:
+        return _decode_list(shapes, value)
+
+    if type(value) is str:
+        return _decode_str(shapes, value)
+
+    # `type(value) is int` excludes bool, so a JSON true is never a number here
+    # despite bool subclassing int. Coerce only where Float is the single
+    # numeric reading.
+    if type(value) is int:
+        if (any(type(s) is Float for s in shapes)
+                and not any(type(s) is Int for s in shapes)):
+            return float(value)
+
+    return value
+
+
+def _decode_dict(shapes, value):
+    if _TYPE in value and _VALUE in value:
+        # The wrapper is those two keys and nothing else. A key beside them, or
+        # a $value that is itself a wrapper, is a hand edit the core refuses —
+        # and reading it here would erase it from the next dump without a word.
+        if len(value) > 2 or (type(value[_VALUE]) is dict and _TYPE in value[_VALUE]):
+            return value
+
+        # A Struct never travels this way, and its option_id is a bare class
+        # name an enum is allowed to share: reading one here would hand a str
+        # to a dataclass branch and lose the enum.
+        shape = next((s for s in shapes if type(s) is not Struct
+                      and s.option_id() == value[_TYPE]), None)
+
+        if shape is None:
+            return value
+
+        inner = _decode_options((shape,), value[_VALUE])
+
+        # The text did not read as the option the file names. The wrapper is
+        # what says so; consuming it would file a broken date under str.
+        if type(inner) is not _core_type(shape):
+            return {_TYPE: value[_TYPE], _VALUE: inner}
+
+        # The core wants the wrapper only where two options share a Python
+        # type. A collision that was the wire's alone is already resolved.
+        if sum(1 for s in shapes if _core_type(s) is _core_type(shape)) > 1:
+            return {_TYPE: value[_TYPE], _VALUE: inner}
+
+        return inner
+
+    if _TYPE in value:
+        struct = next((s for s in shapes if type(s) is Struct
+                       and s.cls.__name__ == value[_TYPE]), None)
+
+        if struct is None:
+            return value
+
+        inner = {k: v for k, v in value.items() if k != _TYPE}
+        return {_TYPE: value[_TYPE], **decode(struct, inner)}
+
+    structs = [s for s in shapes if type(s) is Struct]
+
+    if len(structs) == 1:
+        return decode(structs[0], value)
+
+    return value
+
+
+def _decode_list(shapes, value):
+    lists = [s for s in shapes if type(s) is List]
+
+    # A union of lists collides on the transport type and travels wrapped, so
+    # a bare list means a single List branch.
+    if len(lists) == 1:
+        return [_decode_options(lists[0].item, item) for item in value]
+
+    return value
+
+
+def _decode_str(shapes, value):
+    # A Str reading keeps a string a string. Otherwise convert only where a
+    # single Date, Time or enum is the one reading.
+    if any(type(s) is Str for s in shapes):
+        return value
+
+    dates = [s for s in shapes if type(s) is Date]
+    times = [s for s in shapes if type(s) is Time]
+    enums = [s for s in shapes if type(s) is EnumShape]
+
+    if (bool(dates) + bool(times) + bool(enums)) != 1:
+        return value
+
+    if len(dates) == 1:
+        return _convert(date.fromisoformat, value)
+
+    if len(times) == 1:
+        return _convert(time.fromisoformat, value)
+
+    if len(enums) == 1:
+        # __members__ resolves an alias to its canonical member, and keeps a
+        # mixin out of the way: a StrEnum inherits str.__getitem__, which
+        # cls[name] would reach instead of the member lookup.
+        return _convert(enums[0].cls.__members__.__getitem__, value)
+
+    return value
+
+
+def _convert(fn, value):
+    try:
+        return fn(value)
+    except (KeyError, ValueError):
+        # Not a reading decode can make: build() rejects it.
+        return value

pytypehintstore/errors.py

@@ -0,0 +1,19 @@
+"""What a store raises on its own behalf.
+
+A validation failure is not one of these: `add` and `put` let the core's
+SchemaTypeError / SchemaValueError travel out untouched, so the caller sees the
+same line pytypehint would have produced anywhere else. These three cover what
+only a store can go wrong at: the file and the lock.
+"""
+
+
+class StoreError(Exception):
+    """Base of every failure a store reports for itself."""
+
+
+class StoreLockedError(StoreError):
+    """Another live process already owns this file."""
+
+
+class StoreLoadError(StoreError):
+    """The file on disk could not be read back into rows."""

pytypehintstore/fingerprint.py

@@ -0,0 +1,80 @@
+"""The identity of a schema: eight hex that move when the contract moves."""
+
+import hashlib
+import json
+from dataclasses import fields, is_dataclass
+from datetime import date, time
+from enum import Enum
+
+from pytypehint import MISSING
+
+_LENGTH = 8
+
+
+def fingerprint(schema) -> str:
+    """Eight hex of the sha256 of `schema` written as deterministic text."""
+    text = json.dumps(_plain(schema, (), {}), sort_keys=True)
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()[:_LENGTH]
+
+
+def _plain(value, seen, done):
+    """`value` as text json can render.
+
+    `seen` is the chain of dataclasses above it, so a recursive schema stops
+    instead of descending forever. `done` is every dataclass already written,
+    because the core compiles a class once and shares that object between every
+    field naming it: the schema is a graph, and walking it as a tree costs a
+    path per branch — minutes for a class the core compiles in a millisecond.
+    """
+    if value is MISSING:
+        return "MISSING"
+
+    if value is None or type(value) in (bool, int, float, str):
+        return value
+
+    if is_dataclass(value) and not isinstance(value, type):
+        if id(value) in seen:
+            return f"<{type(value).__name__}>"
+
+        if id(value) in done:
+            return {"$ref": done[id(value)]}
+
+        done[id(value)] = len(done)
+
+        # Only the fields the core compares. The rest are derived — a compiled
+        # regex, the recipe behind a default that is already materialised — and
+        # a compiled regex has no text to read.
+        return {type(value).__name__: {
+            f.name: _plain(getattr(value, f.name), (*seen, id(value)), done)
+            for f in fields(value) if f.compare}}
+
+    if isinstance(value, type):
+        if issubclass(value, Enum):
+            # The member names are contract: renaming one is another database.
+            return [value.__name__, list(value.__members__)]
+
+        return value.__name__
+
+    if isinstance(value, Enum):
+        return value.name
+
+    if isinstance(value, (date, time)):
+        return value.isoformat()
+
+    if isinstance(value, dict):
+        return {str(key): _plain(item, seen, done) for key, item in value.items()}
+
+    if isinstance(value, (set, frozenset)):
+        return sorted(repr(item) for item in value)
+
+    if isinstance(value, (list, tuple)):
+        return [_plain(item, seen, done) for item in value]
+
+    if callable(value):
+        return _plain(value(), seen, done)
+
+    # No repr() of courtesy: a repr carries memory addresses, and a hash built
+    # on one renames every database on the next run. If the core grows a type
+    # this does not know, the store must fail loudly at open.
+    raise TypeError(
+        f"cannot fingerprint a {type(value).__name__}: it has no stable text")

pytypehintstore/lockfile.py

@@ -0,0 +1,199 @@
+"""One process owns a store.
+
+The lockfile holds the owner's PID, and who holds it is settled by the operating
+system rather than by agreement between readers: an open handle on Windows, an
+advisory `flock` on POSIX. A crash leaves an orphan the next process takes over,
+and two processes racing for that orphan cannot both end up owning it.
+
+The contract is the same on both: one owner, orphans reclaimed, and the same
+error naming the PID that holds it.
+"""
+
+import ctypes
+import os
+import re
+import sys
+from pathlib import Path
+
+from pytypehintstore.errors import StoreLockedError
+
+# str.isdigit() is not this question: it says yes to a superscript, which int()
+# then refuses, and to an Arabic-Indic digit, which int() reads as a number no
+# process here ever had.
+_PID = re.compile(r"[0-9]{1,9}")
+
+
+def lock_path_of(path: Path) -> Path:
+    return path.with_name(path.name + ".lock")
+
+
+def acquire(path: Path, name: str):
+    """Take the lock for `path`, or say who owns it. Returns the held lock."""
+    lock = lock_path_of(path)
+
+    # Two rounds at most: claim, and if something stale was in the way, clear it
+    # and claim again. A third failure means someone else won the race, and that
+    # someone is a live owner worth reporting.
+    for _ in range(2):
+        fd = _claim(lock)
+
+        if fd is not None:
+            os.ftruncate(fd, 0)
+            os.write(fd, str(os.getpid()).encode("utf-8"))
+            return lock, fd
+
+        pid = _owner(lock)
+
+        if pid is not None and alive(pid):
+            raise StoreLockedError(
+                f"{name} is owned by process {pid}. A store belongs to one "
+                f"process; you are probably running several workers. Run a "
+                f"single worker, or reach for a database server — this is not one.")
+
+        _clear(lock)
+
+    raise StoreLockedError(
+        f"{name} is locked by another process that keeps retaking "
+        f"{lock.name}. Stop it, or delete {lock.name} by hand.")
+
+
+def release(held) -> None:
+    lock, fd = held
+
+    try:
+        _let_go(lock, fd)
+    except OSError:
+        # Removed by hand, or the disk is gone: neither is worth failing a
+        # close over.
+        pass
+
+
+def _owner(lock: Path):
+    try:
+        text = lock.read_bytes().decode("utf-8", "replace").strip()
+    except OSError:
+        return None
+
+    return int(text) if _PID.fullmatch(text) else None
+
+
+def alive(pid: int) -> bool:
+    """Whether a process with this id is running.
+
+    A id of zero or less is nobody: on POSIX it would reach a process group
+    instead of a process.
+    """
+    return pid > 0 and _running(pid)
+
+
+# The two halves below hold one contract each way. They are declared under
+# `sys.platform` rather than `os.name` because that is the form a type checker
+# reads as "this is for that platform only", so neither half is looked up on
+# the system that does not have it.
+
+if sys.platform == "win32":
+    # os.kill is not a probe on Windows: signal 0 reaches TerminateProcess.
+    # Ask the kernel instead — and tell ctypes what it is calling, because a
+    # handle is a pointer and the default return type would cut a large one in
+    # half, silently, since the halves of a handle look like a handle.
+    _PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
+    _SYNCHRONIZE = 0x00100000
+    _ERROR_ACCESS_DENIED = 5
+    _WAIT_OBJECT_0 = 0
+
+    _kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+    _kernel32.OpenProcess.argtypes = (ctypes.c_ulong, ctypes.c_int, ctypes.c_ulong)
+    _kernel32.OpenProcess.restype = ctypes.c_void_p
+    _kernel32.WaitForSingleObject.argtypes = (ctypes.c_void_p, ctypes.c_ulong)
+    _kernel32.WaitForSingleObject.restype = ctypes.c_ulong
+    _kernel32.CloseHandle.argtypes = (ctypes.c_void_p,)
+    _kernel32.CloseHandle.restype = ctypes.c_int
+
+    def _running(pid: int) -> bool:
+        handle = _kernel32.OpenProcess(
+            _PROCESS_QUERY_LIMITED_INFORMATION | _SYNCHRONIZE, False, pid)
+
+        if not handle:
+            # Denied means the process is there and belongs to someone else;
+            # anything else (no such PID) means it is gone.
+            return ctypes.get_last_error() == _ERROR_ACCESS_DENIED
+
+        try:
+            # A process that has ended is signalled, and waiting zero
+            # milliseconds asks that without waiting. Its exit code cannot
+            # answer: 259 is both "exited with 259" and "still running", and
+            # that lock would never be recovered.
+            return _kernel32.WaitForSingleObject(handle, 0) != _WAIT_OBJECT_0
+        finally:
+            _kernel32.CloseHandle(handle)
+
+    def _claim(lock: Path):
+        """O_EXCL decides, and the handle stays open for as long as the lock is
+        held: Windows refuses to unlink an open file, so the loser of a race
+        cannot delete the winner's brand new lockfile."""
+        try:
+            return os.open(lock, os.O_CREAT | os.O_EXCL | os.O_RDWR)
+        except FileExistsError:
+            return None
+
+    def _clear(lock: Path) -> None:
+        try:
+            os.unlink(lock)
+        except OSError:
+            # Gone already, or held open by the process that took it while we
+            # were deciding it was abandoned. O_EXCL settles the next round.
+            pass
+
+    def _let_go(lock: Path, fd: int) -> None:
+        # The handle first: the file cannot be unlinked while it is open, and a
+        # lock nobody can remove outlives its owner.
+        os.close(fd)
+        os.unlink(lock)
+
+else:
+    import fcntl
+
+    def _running(pid: int) -> bool:
+        try:
+            os.kill(pid, 0)
+        except ProcessLookupError:
+            return False
+        except PermissionError:
+            # Someone else's process, and it exists.
+            return True
+
+        return True
+
+    def _claim(lock: Path):
+        """`flock` decides, and the kernel drops it when the process dies.
+
+        An orphan therefore needs no stealing: it is simply a file nobody
+        holds, and the next claim takes it. The inode is checked afterwards
+        because another process may have unlinked the file between the open and
+        the lock, and a lock on a file with no name guards nothing.
+        """
+        fd = os.open(lock, os.O_CREAT | os.O_RDWR)
+
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+
+            if os.fstat(fd).st_ino != os.stat(lock).st_ino:
+                raise OSError("the lockfile was replaced while it was claimed")
+        except OSError:
+            os.close(fd)
+            return None
+
+        return fd
+
+    def _clear(lock: Path) -> None:
+        # Nothing to clear: what makes an orphan an orphan is that its flock
+        # died with its owner, and unlinking here would only race with whoever
+        # is about to claim it.
+        pass
+
+    def _let_go(lock: Path, fd: int) -> None:
+        # The name first, while the lock is still ours: between the unlink and
+        # the close, a claim can only create a file of its own and win it
+        # outright, which is exactly what should happen.
+        os.unlink(lock)
+        os.close(fd)