pyoq-sql 1.0.2__py3-none-any.whl

pyoq/fetching/tables.py

@@ -0,0 +1,110 @@
+"""One appearance of a table in a statement, and the name it answers to.
+
+A statement can read one table more than once: two relations reaching the same
+table, or a relation reaching the table it starts from. What qualifies a
+column is therefore the appearance rather than the table, and an appearance
+that shares its table's name with another is what makes a column ambiguous.
+
+Every appearance but the one a statement starts from is given a name of its
+own. They are numbered in the order the plan is walked, which the plan fixes,
+so one plan always compiles to the same SQL.
+
+A name is only its own if nothing else already answers to it. `pyoq_t1` is a
+legal table name and a legal column name, and a database is entitled to use
+both, so the names already in the schema are reserved and skipped rather than
+assumed not to exist.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from pyoq.query.nodes import FieldNode
+
+if TYPE_CHECKING:
+    from pyoq.query.nodes import ScalarFamily
+    from pyoq.schema.models import Identifier, ObjectReference
+
+_PREFIX = "pyoq_t"
+
+
+@dataclass(frozen=True, slots=True)
+class TableBinding:
+    """A table, and what a column of it is written under in this statement."""
+
+    table: ObjectReference
+    alias: str | None = None
+
+    @property
+    def qualifier(self) -> str:
+        """What a column of this appearance is qualified by."""
+        return self.table.name.value if self.alias is None else self.alias
+
+    @property
+    def schema(self) -> str | None:
+        """A name of its own stands alone, so it carries no schema."""
+        return None if self.alias is not None else _named(self.table.schema)
+
+    @property
+    def catalog(self) -> str | None:
+        return None if self.alias is not None else _named(self.table.catalog)
+
+    def field(self, column: Identifier, family: ScalarFamily, /) -> FieldNode:
+        return FieldNode(
+            column.value,
+            self.qualifier,
+            self.schema,
+            self.catalog,
+            family,
+        )
+
+
+class TableNames:
+    """Names for the appearances one statement reads, each different.
+
+    Numbered rather than derived from the table, because two appearances of
+    one table would derive the same name, which is the problem being solved.
+
+    ``reserved`` is what the schema already calls things. A generated name
+    that matched one of those would shadow it: a collection correlated to a
+    table whose name it had taken would compare that table with itself and
+    gather nothing, which is a wrong answer rather than an error.
+    """
+
+    __slots__ = ("_reserved", "_taken")
+
+    def __init__(self, reserved: frozenset[str] = frozenset()) -> None:
+        self._reserved = reserved
+        self._taken = 0
+
+    def next(self, table: ObjectReference, /) -> TableBinding:
+        return TableBinding(table, self._unused())
+
+    def _unused(self) -> str:
+        """The next name nothing else answers to."""
+        while True:
+            self._taken += 1
+            candidate = f"{_PREFIX}{self._taken}"
+            if not self._claimed(candidate):
+                return candidate
+
+    def _claimed(self, candidate: str) -> bool:
+        """Whether the schema already calls something this.
+
+        A name a collection carries a joined value under is written as the
+        appearance and the column with a dot between them, and a column may be
+        named with a dot in it, so a name that any reserved one begins with is
+        taken as well.
+        """
+        if candidate in self._reserved:
+            return True
+        beginning = f"{candidate}."
+        return any(name.startswith(beginning) for name in self._reserved)
+
+
+def _named(value: Identifier | None) -> str | None:
+    return None if value is None else value.value
+
+
+__all__ = ("TableBinding", "TableNames")

pyoq/generation/__init__.py

@@ -0,0 +1,54 @@
+"""Public schema generation pipeline."""
+
+from pyoq.generation.cleanup import ManifestOwnedCleanup
+from pyoq.generation.contracts import (
+    MANIFEST_FORMAT_VERSION,
+    MANIFEST_NAME,
+    ConcernRenderer,
+    DriftChecker,
+    DriftReport,
+    GenerationMode,
+    GenerationPlan,
+    ManifestStore,
+    OwnedFile,
+    OwnershipManifest,
+    PlanValidator,
+    PlanWriter,
+    ProjectLock,
+    RenderedFile,
+)
+from pyoq.generation.drift import FileDriftChecker
+from pyoq.generation.lock import DirectoryProjectLock
+from pyoq.generation.manifest import JsonManifestStore
+from pyoq.generation.pipeline import GenerationPipeline, GenerationPlanner
+from pyoq.generation.rendering import GeneratedTypesRenderer
+from pyoq.generation.validation import PythonSyntaxValidator
+from pyoq.generation.writer import AtomicGenerationWriter
+from pyoq.schema import SchemaSource
+
+__all__ = (
+    "MANIFEST_FORMAT_VERSION",
+    "MANIFEST_NAME",
+    "AtomicGenerationWriter",
+    "ConcernRenderer",
+    "DirectoryProjectLock",
+    "DriftChecker",
+    "DriftReport",
+    "FileDriftChecker",
+    "GeneratedTypesRenderer",
+    "GenerationMode",
+    "GenerationPipeline",
+    "GenerationPlan",
+    "GenerationPlanner",
+    "JsonManifestStore",
+    "ManifestOwnedCleanup",
+    "ManifestStore",
+    "OwnedFile",
+    "OwnershipManifest",
+    "PlanValidator",
+    "PlanWriter",
+    "ProjectLock",
+    "PythonSyntaxValidator",
+    "RenderedFile",
+    "SchemaSource",
+)

pyoq/generation/cleanup.py

@@ -0,0 +1,44 @@
+"""Checksum-guarded cleanup of manifest-owned files."""
+
+from __future__ import annotations
+
+from hashlib import sha256
+from pathlib import Path, PurePosixPath
+
+from pyoq.errors import GenerationOwnershipError
+from pyoq.generation.contracts import OwnershipManifest, output_path
+
+
+class ManifestOwnedCleanup:
+    def remove_stale(
+        self,
+        staging_root: Path,
+        current_manifest: OwnershipManifest | None,
+        retained_paths: frozenset[PurePosixPath],
+    ) -> None:
+        if current_manifest is None:
+            return
+        for owned_file in current_manifest.files:
+            if owned_file.path in retained_paths:
+                continue
+            _require_safe_parent(staging_root, owned_file.path)
+            stale_path = output_path(staging_root, owned_file.path)
+            if not stale_path.exists() and not stale_path.is_symlink():
+                continue
+            if (
+                not stale_path.is_file()
+                or stale_path.is_symlink()
+                or sha256(stale_path.read_bytes()).hexdigest() != owned_file.digest
+            ):
+                message = f"refusing to remove modified owned file: {owned_file.path}"
+                raise GenerationOwnershipError(message)
+            stale_path.unlink()
+
+
+def _require_safe_parent(root: Path, relative_path: PurePosixPath) -> None:
+    current = root
+    for part in relative_path.parts[:-1]:
+        current /= part
+        if current.is_symlink() or (current.exists() and not current.is_dir()):
+            message = f"unsafe parent for owned file: {relative_path}"
+            raise GenerationOwnershipError(message)

pyoq/generation/contracts.py

@@ -0,0 +1,248 @@
+"""Generation models and component protocols."""
+
+from __future__ import annotations
+
+from contextlib import AbstractContextManager
+from dataclasses import dataclass
+from enum import StrEnum
+from hashlib import sha256
+from pathlib import Path, PurePosixPath
+from typing import Protocol
+from unicodedata import normalize
+
+from pyoq.config import Configuration
+from pyoq.errors import GenerationValidationError
+from pyoq.schema import SchemaSnapshot
+
+MANIFEST_FORMAT_VERSION = 1
+MANIFEST_NAME = ".pyoq-manifest.json"
+_WINDOWS_INVALID_CHARACTERS = frozenset('<>:"|?*')
+_WINDOWS_RESERVED_STEMS = frozenset(
+    {"con", "prn", "aux", "nul"}
+    | {f"com{number}" for number in range(1, 10)}
+    | {f"lpt{number}" for number in range(1, 10)}
+)
+
+
+class GenerationMode(StrEnum):
+    WRITE = "write"
+    CHECK = "check"
+    DRY_RUN = "dry-run"
+
+
+@dataclass(frozen=True, slots=True, order=True)
+class RenderedFile:
+    path: PurePosixPath
+    content: str
+
+    def __post_init__(self) -> None:
+        _validate_output_path(self.path)
+        canonical_content = self.content.replace("\r\n", "\n").replace("\r", "\n")
+        object.__setattr__(self, "content", canonical_content)
+
+    @property
+    def encoded_content(self) -> bytes:
+        return self.content.encode("utf-8")
+
+    @property
+    def digest(self) -> str:
+        return sha256(self.encoded_content).hexdigest()
+
+
+@dataclass(frozen=True, slots=True)
+class GenerationPlan:
+    files: tuple[RenderedFile, ...]
+
+    def __post_init__(self) -> None:
+        ordered_files = tuple(
+            sorted(self.files, key=lambda generated: str(generated.path))
+        )
+        duplicate_paths = _duplicate_paths(ordered_files)
+        if duplicate_paths:
+            joined_paths = ", ".join(repr(str(path)) for path in duplicate_paths)
+            message = f"rendering concerns produced duplicate paths: {joined_paths}"
+            raise GenerationValidationError(message)
+        object.__setattr__(self, "files", ordered_files)
+
+
+@dataclass(frozen=True, slots=True, order=True)
+class OwnedFile:
+    path: PurePosixPath
+    digest: str
+
+    def __post_init__(self) -> None:
+        _validate_output_path(self.path)
+        valid_digest = (
+            len(self.digest) == 64
+            and self.digest == self.digest.lower()
+            and all(character in "0123456789abcdef" for character in self.digest)
+        )
+        if not valid_digest:
+            message = f"invalid ownership digest for {str(self.path)!r}"
+            raise GenerationValidationError(message)
+
+
+@dataclass(frozen=True, slots=True)
+class OwnershipManifest:
+    package: str
+    files: tuple[OwnedFile, ...]
+    format_version: int = MANIFEST_FORMAT_VERSION
+
+    def __post_init__(self) -> None:
+        _require_text(self.package, "generated package")
+        if self.format_version != MANIFEST_FORMAT_VERSION:
+            message = f"unsupported ownership manifest version: {self.format_version}"
+            raise GenerationValidationError(message)
+        ordered_files = tuple(sorted(self.files))
+        duplicate_paths = _duplicate_owned_paths(ordered_files)
+        if duplicate_paths:
+            joined_paths = ", ".join(repr(str(path)) for path in duplicate_paths)
+            message = f"ownership manifest contains duplicate paths: {joined_paths}"
+            raise GenerationValidationError(message)
+        object.__setattr__(self, "files", ordered_files)
+
+    @classmethod
+    def from_plan(cls, package: str, plan: GenerationPlan) -> OwnershipManifest:
+        return cls(
+            package=package,
+            files=tuple(OwnedFile(file.path, file.digest) for file in plan.files),
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class DriftReport:
+    created: tuple[PurePosixPath, ...] = ()
+    updated: tuple[PurePosixPath, ...] = ()
+    removed: tuple[PurePosixPath, ...] = ()
+    conflicts: tuple[PurePosixPath, ...] = ()
+    manifest_changed: bool = False
+
+    @property
+    def is_current(self) -> bool:
+        return not (
+            self.created
+            or self.updated
+            or self.removed
+            or self.conflicts
+            or self.manifest_changed
+        )
+
+    @property
+    def change_count(self) -> int:
+        return len(self.created) + len(self.updated) + len(self.removed)
+
+    def summary(self) -> str:
+        return (
+            f"create {len(self.created)}, update {len(self.updated)}, "
+            f"remove {len(self.removed)}, conflict {len(self.conflicts)}"
+        )
+
+
+class ConcernRenderer(Protocol):
+    @property
+    def concern(self) -> str: ...
+
+    def render(
+        self,
+        snapshot: SchemaSnapshot,
+        configuration: Configuration,
+    ) -> tuple[RenderedFile, ...]: ...
+
+
+class PlanValidator(Protocol):
+    def validate(self, plan: GenerationPlan) -> None: ...
+
+
+class ManifestStore(Protocol):
+    def load(self, package_root: Path) -> OwnershipManifest | None: ...
+
+    def encode(self, manifest: OwnershipManifest) -> bytes: ...
+
+
+class DriftChecker(Protocol):
+    def compare(
+        self,
+        package_root: Path,
+        plan: GenerationPlan,
+        current_manifest: OwnershipManifest | None,
+        desired_manifest: OwnershipManifest,
+    ) -> DriftReport: ...
+
+
+class PlanWriter(Protocol):
+    def write(
+        self,
+        package_root: Path,
+        plan: GenerationPlan,
+        current_manifest: OwnershipManifest | None,
+        desired_manifest: OwnershipManifest,
+    ) -> None: ...
+
+
+class ProjectLock(Protocol):
+    def acquire(self, project_root: Path) -> AbstractContextManager[None]: ...
+
+
+def output_path(root: Path, relative_path: PurePosixPath) -> Path:
+    return root.joinpath(*relative_path.parts)
+
+
+def _validate_output_path(path: PurePosixPath) -> None:
+    invalid = (
+        path.is_absolute()
+        or not path.parts
+        or path == PurePosixPath(".")
+        or ".." in path.parts
+        or "\\" in str(path)
+        or path.name.casefold() == MANIFEST_NAME
+        or any(_invalid_portable_part(part) for part in path.parts)
+    )
+    if invalid:
+        message = f"unsafe generated path: {str(path)!r}"
+        raise GenerationValidationError(message)
+
+
+def _duplicate_paths(files: tuple[RenderedFile, ...]) -> tuple[PurePosixPath, ...]:
+    grouped_paths: dict[str, list[PurePosixPath]] = {}
+    for generated_file in files:
+        key = _portable_path_key(generated_file.path)
+        grouped_paths.setdefault(key, []).append(generated_file.path)
+    return _colliding_paths(grouped_paths)
+
+
+def _duplicate_owned_paths(files: tuple[OwnedFile, ...]) -> tuple[PurePosixPath, ...]:
+    grouped_paths: dict[str, list[PurePosixPath]] = {}
+    for owned_file in files:
+        key = _portable_path_key(owned_file.path)
+        grouped_paths.setdefault(key, []).append(owned_file.path)
+    return _colliding_paths(grouped_paths)
+
+
+def _colliding_paths(
+    grouped_paths: dict[str, list[PurePosixPath]],
+) -> tuple[PurePosixPath, ...]:
+    collisions: set[PurePosixPath] = set()
+    for paths in grouped_paths.values():
+        if len(paths) > 1:
+            collisions.update(paths)
+    return tuple(sorted(collisions))
+
+
+def _portable_path_key(path: PurePosixPath) -> str:
+    return normalize("NFKC", str(path)).casefold()
+
+
+def _invalid_portable_part(part: str) -> bool:
+    stem = part.split(".", maxsplit=1)[0].casefold()
+    return (
+        part.endswith((" ", "."))
+        or stem in _WINDOWS_RESERVED_STEMS
+        or any(character in _WINDOWS_INVALID_CHARACTERS for character in part)
+        or any(ord(character) < 32 for character in part)
+    )
+
+
+def _require_text(value: object, label: str) -> None:
+    if not isinstance(value, str) or not value:
+        message = f"{label} must be a non-empty string"
+        raise GenerationValidationError(message)

pyoq/generation/drift.py

@@ -0,0 +1,169 @@
+"""Side-effect-free generated package drift detection."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from hashlib import sha256
+from pathlib import Path, PurePosixPath
+
+from pyoq.errors import GenerationDriftError
+from pyoq.generation.contracts import (
+    DriftReport,
+    GenerationPlan,
+    OwnedFile,
+    OwnershipManifest,
+    output_path,
+)
+
+FileDigestReader = Callable[[Path], str]
+
+
+def _file_digest(path: Path) -> str:
+    return sha256(path.read_bytes()).hexdigest()
+
+
+@dataclass(frozen=True, slots=True)
+class FileDriftChecker:
+    digest_reader: FileDigestReader = _file_digest
+
+    def compare(
+        self,
+        package_root: Path,
+        plan: GenerationPlan,
+        current_manifest: OwnershipManifest | None,
+        desired_manifest: OwnershipManifest,
+    ) -> DriftReport:
+        try:
+            return _compare_files(
+                package_root,
+                plan,
+                current_manifest,
+                desired_manifest,
+                self.digest_reader,
+            )
+        except OSError as error:
+            message = f"unable to inspect generated package: {error}"
+            raise GenerationDriftError(message) from error
+
+
+def _compare_files(
+    package_root: Path,
+    plan: GenerationPlan,
+    current_manifest: OwnershipManifest | None,
+    desired_manifest: OwnershipManifest,
+    digest_reader: FileDigestReader,
+) -> DriftReport:
+    current_files = _owned_files(current_manifest)
+    desired_files = {generated.path: generated for generated in plan.files}
+    created: list[PurePosixPath] = []
+    updated: list[PurePosixPath] = []
+    removed: list[PurePosixPath] = []
+    conflicts: list[PurePosixPath] = []
+    for path, generated_file in desired_files.items():
+        destination = output_path(package_root, path)
+        owned_file = current_files.get(path)
+        state = _planned_file_state(
+            package_root,
+            path,
+            destination,
+            generated_file.digest,
+            owned_file,
+            digest_reader,
+        )
+        _record_state(path, state, created, updated, conflicts)
+    for path, owned_file in current_files.items():
+        if path not in desired_files:
+            state = _stale_file_state(
+                package_root,
+                path,
+                output_path(package_root, path),
+                owned_file,
+                digest_reader,
+            )
+            _record_stale_state(path, state, removed, conflicts)
+    return DriftReport(
+        created=tuple(sorted(created)),
+        updated=tuple(sorted(updated)),
+        removed=tuple(sorted(removed)),
+        conflicts=tuple(sorted(conflicts)),
+        manifest_changed=current_manifest != desired_manifest,
+    )
+
+
+def _owned_files(manifest: OwnershipManifest | None) -> dict[PurePosixPath, OwnedFile]:
+    if manifest is None:
+        return {}
+    return {owned_file.path: owned_file for owned_file in manifest.files}
+
+
+def _planned_file_state(
+    package_root: Path,
+    relative_path: PurePosixPath,
+    destination: Path,
+    desired_digest: str,
+    owned_file: OwnedFile | None,
+    digest_reader: FileDigestReader,
+) -> str:
+    if _has_unsafe_parent(package_root, relative_path):
+        return "conflict"
+    if not destination.exists() and not destination.is_symlink():
+        return "created"
+    if owned_file is None or not destination.is_file() or destination.is_symlink():
+        return "conflict"
+    current_digest = digest_reader(destination)
+    if current_digest != owned_file.digest:
+        return "conflict"
+    return "current" if current_digest == desired_digest else "updated"
+
+
+def _stale_file_state(
+    package_root: Path,
+    relative_path: PurePosixPath,
+    destination: Path,
+    owned_file: OwnedFile,
+    digest_reader: FileDigestReader,
+) -> str:
+    if _has_unsafe_parent(package_root, relative_path):
+        return "conflict"
+    if not destination.exists() and not destination.is_symlink():
+        return "removed"
+    if not destination.is_file() or destination.is_symlink():
+        return "conflict"
+    return "removed" if digest_reader(destination) == owned_file.digest else "conflict"
+
+
+def _record_state(
+    path: PurePosixPath,
+    state: str,
+    created: list[PurePosixPath],
+    updated: list[PurePosixPath],
+    conflicts: list[PurePosixPath],
+) -> None:
+    if state == "created":
+        created.append(path)
+    elif state == "updated":
+        updated.append(path)
+    elif state == "conflict":
+        conflicts.append(path)
+
+
+def _record_stale_state(
+    path: PurePosixPath,
+    state: str,
+    removed: list[PurePosixPath],
+    conflicts: list[PurePosixPath],
+) -> None:
+    if state == "removed":
+        removed.append(path)
+    else:
+        conflicts.append(path)
+
+
+def _has_unsafe_parent(root: Path, relative_path: PurePosixPath) -> bool:
+    current = root
+    for part in relative_path.parts[:-1]:
+        current /= part
+        if current.is_symlink() or (current.exists() and not current.is_dir()):
+            return True
+    return False

pyoq/generation/lock.py

@@ -0,0 +1,33 @@
+"""Fail-closed generation project lock."""
+
+from __future__ import annotations
+
+from collections.abc import Generator
+from contextlib import contextmanager
+from pathlib import Path
+
+from pyoq.errors import GenerationLockError
+
+LOCK_DIRECTORY_NAME = ".pyoq-generation-lock"
+
+
+class DirectoryProjectLock:
+    @contextmanager
+    def acquire(self, project_root: Path) -> Generator[None, None, None]:
+        lock_path = project_root / LOCK_DIRECTORY_NAME
+        try:
+            lock_path.mkdir(mode=0o700)
+        except FileExistsError as error:
+            message = f"generation lock is already held: {lock_path}"
+            raise GenerationLockError(message) from error
+        except OSError as error:
+            message = f"unable to acquire generation lock: {error}"
+            raise GenerationLockError(message) from error
+        try:
+            yield
+        finally:
+            try:
+                lock_path.rmdir()
+            except OSError as error:
+                message = f"unable to release generation lock: {error}"
+                raise GenerationLockError(message) from error