shikiru-cli 0.2.0__py3-none-any.whl

shikiru/engine/pull_request_closing.py

@@ -0,0 +1,117 @@
+"""Off-loop, retried closing of the pull requests of cancelled tasks."""
+
+from __future__ import annotations
+
+import asyncio
+from contextlib import suppress
+from typing import TYPE_CHECKING
+
+from shikiru.domain.faults import next_backoff
+from shikiru.domain.tasks import Status
+from shikiru.engine.fault_reporter import RECOVERABLE_FAULTS
+
+if TYPE_CHECKING:
+    from logging import Logger
+
+    from shikiru.clock import Clock
+    from shikiru.config.settings import OrchestrationConfig
+    from shikiru.engine.fault_reporter import FaultReporter
+    from shikiru.engine.integration import PullRequestCloser
+    from shikiru.store.tasks import TaskStore
+
+CLOSE_FAULT_SCOPE = "pull request close"
+
+
+class PullRequestClosing:
+    """Closes cancelled tasks' open pull requests on its own task, never on the command loop.
+
+    A failed close is recorded through the `FaultReporter` and retried on a doubling backoff for
+    `network.pr_close_attempts` tries; after that it waits for the next engine start, whose
+    `restore` queues every cancelled task whose pull request is still open.
+    """
+
+    def __init__(
+        self,
+        *,
+        closer: PullRequestCloser,
+        tasks: TaskStore,
+        faults: FaultReporter,
+        clock: Clock,
+        settings: OrchestrationConfig,
+    ) -> None:
+        self._closer = closer
+        self._tasks = tasks
+        self._faults = faults
+        self._clock = clock
+        self._settings = settings
+        self._queue: dict[str, None] = {}
+        self._pending = asyncio.Event()
+        self._task: asyncio.Task[None] | None = None
+
+    def start(self) -> None:
+        """Start the closing task."""
+        self._task = asyncio.create_task(self._run())
+
+    def request(self, task_id: str) -> None:
+        """Queue closing the open pull request of cancelled `task_id`, if it has one."""
+        self._queue[task_id] = None
+        self._pending.set()
+
+    def has_open(self, task_id: str) -> bool:
+        """Return whether `task_id` has a pull request that was neither closed nor withdrawn."""
+        return self._closer.has_open(task_id)
+
+    def restore(self) -> None:
+        """Queue every cancelled task whose pull request is still open."""
+        for task in self._tasks.list_all():
+            if task.status is Status.CANCELLED and self.has_open(task.id):
+                self.request(task.id)
+
+    def supervise(self, logger: Logger) -> None:
+        """Log a closing task failure and start a replacement task."""
+        task = self._task
+        if task is None or not task.done():
+            return
+        if not task.cancelled() and (error := task.exception()) is not None:
+            logger.error(
+                "pull request closing failed", exc_info=(type(error), error, error.__traceback__)
+            )
+        self.start()
+
+    async def shutdown(self) -> None:
+        """Cancel the closing task; queued closes are repeated by the next engine start."""
+        task, self._task = self._task, None
+        if task is None:
+            return
+        task.cancel()
+        with suppress(asyncio.CancelledError):
+            await task
+
+    async def _run(self) -> None:
+        while True:
+            if not self._queue:
+                self._pending.clear()
+                await self._pending.wait()
+                continue
+            task_id = next(iter(self._queue))
+            del self._queue[task_id]
+            await self._close(task_id)
+
+    async def _close(self, task_id: str) -> None:
+        faults = self._settings.faults
+        delay = 0.0
+        for attempt in range(1, self._settings.network.pr_close_attempts + 1):
+            try:
+                await self._closer.close_open(task_id)
+            except RECOVERABLE_FAULTS as error:
+                self._faults.report(error, scope=CLOSE_FAULT_SCOPE, task_id=task_id)
+                delay = next_backoff(
+                    delay,
+                    initial=faults.backoff_initial_seconds,
+                    maximum=faults.backoff_max_seconds,
+                )
+                if attempt < self._settings.network.pr_close_attempts:
+                    await self._clock.sleep(delay)
+            else:
+                self._faults.resolve(CLOSE_FAULT_SCOPE, task_id=task_id)
+                return

shikiru/engine/push_service.py

@@ -0,0 +1,177 @@
+"""Off-loop publication of locally integrated base branches to the remote."""
+
+from __future__ import annotations
+
+import asyncio
+from contextlib import suppress
+from datetime import timedelta
+from typing import TYPE_CHECKING
+
+from shikiru.domain.pushes import is_due
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from datetime import datetime
+    from logging import Logger
+    from pathlib import Path
+
+    from shikiru.clock import Clock
+    from shikiru.domain.pushes import PushTarget
+    from shikiru.engine.tick_guard import TickGuard
+    from shikiru.store.pushes import PushTargets
+    from shikiru.workspace.git import Git
+
+PUSH_FAULT_SCOPE = "base push"
+REMOTE_CHECK_FAULT_SCOPE = "remote check"
+
+
+class PushService:
+    """Registers integrated bases and pushes each when its deadline comes, on its own task.
+
+    Each base's reconciliation and push use its own `TickGuard`, while remote configuration uses
+    `remote_guard`. A failed base check, rejected push or timeout is recorded and retried on fault
+    backoff without holding back other bases. While work is pending, the service refreshes remote
+    refs every `remote_recheck_seconds`; matching local and remote SHAs clear only the registrations
+    observed by that check. A remote that is not configured is not a fault: work stays pending and
+    the remote is checked again after `remote_recheck_seconds`. A push acknowledges only the
+    integrations that were registered when it started.
+    """
+
+    def __init__(
+        self,
+        *,
+        targets: PushTargets,
+        git: Git,
+        guard_for: Callable[[str], TickGuard],
+        remote_guard: TickGuard,
+        clock: Clock,
+        project_root: Path,
+        remote: str,
+        interval_seconds: int,
+        remote_recheck_seconds: int,
+        fetch_timeout_seconds: float,
+    ) -> None:
+        """Push from `project_root` to `remote`, an interval of `interval_seconds` apart."""
+        self._targets = targets
+        self._git = git
+        self._guard_for = guard_for
+        self._guards: dict[str, TickGuard] = {}
+        self._remote_guard = remote_guard
+        self._clock = clock
+        self._project_root = project_root
+        self._remote = remote
+        self._interval = interval_seconds
+        self._remote_recheck = timedelta(seconds=remote_recheck_seconds)
+        self._fetch_timeout = fetch_timeout_seconds
+        self._task: asyncio.Task[None] | None = None
+        self._next_remote_check: datetime | None = None
+        self._remote_available: bool | None = None
+
+    def register(self, base: str) -> None:
+        """Record an integration of `base` that must reach the remote."""
+        current = self._targets.get(base)
+        self._targets.register(base, now=self._clock.now(), interval=self._interval)
+        if current is None or not current.pending:
+            self._remote_available = None
+            self._next_remote_check = None
+
+    def poll(self, logger: Logger) -> None:
+        """Log a failed push task and start one when work is due and none is running."""
+        task = self._task
+        if task is not None:
+            if not task.done():
+                return
+            self._task = None
+            if not task.cancelled() and (error := task.exception()) is not None:
+                logger.error("base push failed", exc_info=(type(error), error, error.__traceback__))
+        if self._due():
+            self._task = asyncio.create_task(self._push_due())
+
+    async def finish_outstanding(self) -> None:
+        """Wait for the push in flight, if there is one."""
+        task = self._task
+        if task is not None:
+            await asyncio.wait((task,))
+
+    async def shutdown(self) -> None:
+        """Cancel an outstanding push; its pending work is sent by the next engine start."""
+        task, self._task = self._task, None
+        if task is None:
+            return
+        task.cancel()
+        with suppress(asyncio.CancelledError):
+            await task
+
+    def _due(self) -> bool:
+        now = self._clock.now()
+        pending = self._targets.pending()
+        if not pending:
+            return False
+        remote_check_due = self._next_remote_check is None or now >= self._next_remote_check
+        if remote_check_due:
+            return True
+        if self._remote_available is False:
+            return False
+        return any(is_due(target, now=now, interval=self._interval) for target in pending)
+
+    async def _push_due(self) -> None:
+        pending = self._targets.pending()
+        if not pending:
+            return
+        now = self._clock.now()
+        remote_check_due = self._next_remote_check is None or now >= self._next_remote_check
+        push_due = any(is_due(target, now=now, interval=self._interval) for target in pending)
+        if not remote_check_due and not push_due:
+            return
+        configured = await self._remote_guard.attempt(self._remote_configured)
+        if configured is None:
+            if remote_check_due:
+                self._next_remote_check = self._clock.now() + self._remote_recheck
+            return
+        self._remote_available = configured
+        if not configured:
+            self._next_remote_check = self._clock.now() + self._remote_recheck
+            return
+        if remote_check_due:
+            self._next_remote_check = self._clock.now() + self._remote_recheck
+            for target in pending:
+                await self._guard(target.base).attempt(
+                    lambda target=target: self._reconcile(target)
+                )
+        for target in self._targets.pending():
+            if is_due(target, now=self._clock.now(), interval=self._interval):
+                await self._guard(target.base).attempt(lambda target=target: self._push(target))
+
+    async def _reconcile(self, target: PushTarget) -> None:
+        remote_sha = await self._git.fetch_remote_branch(
+            self._project_root,
+            self._remote,
+            target.base,
+            timeout=self._fetch_timeout,
+        )
+        local_sha = await self._git.local_branch_sha(self._project_root, target.base)
+        if remote_sha is not None and remote_sha == local_sha:
+            self._targets.reconcile(
+                target.base,
+                registered=target.registered,
+                now=self._clock.now(),
+                interval=self._interval,
+            )
+
+    async def _remote_configured(self) -> bool:
+        return await self._git.remote_configured(self._project_root, self._remote)
+
+    def _guard(self, base: str) -> TickGuard:
+        if base not in self._guards:
+            self._guards[base] = self._guard_for(base)
+        return self._guards[base]
+
+    async def _push(self, target: PushTarget) -> bool:
+        await self._git.push_base(self._project_root, self._remote, target.base)
+        self._targets.acknowledge(
+            target.base,
+            registered=target.registered,
+            now=self._clock.now(),
+            interval=self._interval,
+        )
+        return True

shikiru/engine/quota_probe_threads.py

@@ -0,0 +1,79 @@
+"""Owned threads for blocking quota reads."""
+
+from __future__ import annotations
+
+import asyncio
+import threading
+from typing import TYPE_CHECKING
+
+from shikiru.errors import ShikiruError
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from datetime import datetime
+
+    from shikiru.agents.contracts import QuotaReading
+
+THREAD_NAME_PREFIX = "quota-probe-"
+
+
+def _deliver(
+    future: asyncio.Future[QuotaReading | None],
+    reading: QuotaReading | None,
+    error: Exception | None,
+) -> None:
+    if future.done():
+        return
+    if error is not None:
+        future.set_exception(error)
+    else:
+        future.set_result(reading)
+
+
+class QuotaProbeThreads:
+    """Runs blocking quota reads, each agent's on at most one daemon thread at a time.
+
+    A blocked read cannot be interrupted, so it is never abandoned and replaced: while an
+    agent's thread has not returned, a new read of that agent is refused. The threads are
+    daemons, so a read that never returns delays neither shutdown nor process exit.
+    """
+
+    def __init__(self) -> None:
+        """Start with no thread running."""
+        self._threads: dict[str, threading.Thread] = {}
+
+    def busy(self, agent: str) -> bool:
+        """Return whether `agent`'s previous read is still running."""
+        thread = self._threads.get(agent)
+        return thread is not None and thread.is_alive()
+
+    def start(
+        self,
+        agent: str,
+        read: Callable[[datetime], QuotaReading | None],
+        since: datetime,
+    ) -> asyncio.Future[QuotaReading | None]:
+        """Run `read(since)` on a new thread and return a future for its outcome.
+
+        Raises `ShikiruError` while `agent`'s previous read is still running.
+        """
+        if self.busy(agent):
+            message = f"the previous quota probe of {agent} is still waiting for its source"
+            raise ShikiruError(message)
+        loop = asyncio.get_running_loop()
+        future: asyncio.Future[QuotaReading | None] = loop.create_future()
+
+        def run() -> None:
+            reading: QuotaReading | None = None
+            failure: Exception | None = None
+            try:
+                reading = read(since)
+            except Exception as error:  # noqa: BLE001
+                failure = error
+            if not loop.is_closed():
+                loop.call_soon_threadsafe(_deliver, future, reading, failure)
+
+        thread = threading.Thread(target=run, name=f"{THREAD_NAME_PREFIX}{agent}", daemon=True)
+        self._threads[agent] = thread
+        thread.start()
+        return future

shikiru/engine/quota_reconciler.py

@@ -0,0 +1,210 @@
+"""Revalidates persisted quota blocks against fresh evidence from the agent plugins."""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from datetime import timedelta
+from typing import TYPE_CHECKING
+
+from shikiru.agents.contracts import quota_probe_for
+from shikiru.domain.events import EventKind
+from shikiru.domain.quota_blocks import block_released
+from shikiru.domain.quota_waits import QuotaAction, ScheduledQuotaAction
+from shikiru.engine.quota_probe_threads import QuotaProbeThreads
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+    from datetime import datetime
+
+    from shikiru.agents.contracts import QuotaProbe, QuotaReading, QuotaWindow
+    from shikiru.clock import Clock
+    from shikiru.config.settings import QuotaRefreshConfig
+    from shikiru.engine.agents import AgentPool
+    from shikiru.engine.fault_reporter import FaultReporter
+    from shikiru.engine.quota_schedule import QuotaSchedule
+    from shikiru.store.events import EventLog
+    from shikiru.store.quotas import AgentBlock, QuotaStore
+
+QUOTA_REFRESH_SCOPE = "quota refresh"
+QUOTA_RELEASE_REASON = "newer quota reading shows available quota"
+
+
+@dataclass(frozen=True, slots=True)
+class _Probe:
+    blocked_at: datetime
+    task: asyncio.Task[None]
+    demanded: bool
+
+
+class QuotaReconciler:
+    """Releases a quota block when a newer applicable reading shows its quota is available.
+
+    A reading counts only when it was observed after the block and covers every window the
+    block was exhausted on. Out-of-turn probes run beside the caller on a bounded interval, one
+    per block at a time and each for a bounded time; an explicit requeue forces one, and a probe
+    that fails or times out is recorded as an engine fault. The result of a probe whose block
+    has since changed is dropped.
+    """
+
+    def __init__(
+        self,
+        *,
+        quotas: QuotaStore,
+        agents: AgentPool,
+        events: EventLog,
+        clock: Clock,
+        settings: QuotaRefreshConfig,
+        faults: FaultReporter,
+        schedule: QuotaSchedule,
+    ) -> None:
+        """Probe through `agents`, persist in `quotas`, record outcomes in `events`.
+
+        The next examination or retry of each block is recorded in `schedule`.
+        """
+        self._quotas = quotas
+        self._agents = agents
+        self._events = events
+        self._clock = clock
+        self._settings = settings
+        self._faults = faults
+        self._probed_at: dict[str, datetime] = {}
+        self._probes: dict[str, _Probe] = {}
+        self._threads = QuotaProbeThreads()
+        self._schedule = schedule
+
+    def observe(self, agent: str, windows: Sequence[QuotaWindow]) -> None:
+        """Record `windows` for `agent`, first releasing its block when they prove it obsolete."""
+        now = self._clock.now()
+        block = self._quotas.block_for(agent)
+        if (
+            block is not None
+            and block.until > now
+            and block_released(
+                blocked_at=block.blocked_at,
+                stored=self._quotas.windows_for(agent),
+                reading=windows,
+                now=now,
+            )
+        ):
+            self._quotas.clear_block(agent)
+            self._events.append(
+                EventKind.QUOTA_CLEARED,
+                message=f"{agent} quota block cleared",
+                data={"agent": agent, "reason": QUOTA_RELEASE_REASON},
+            )
+        for window in windows:
+            self._quotas.record(agent, window)
+
+    def request_refresh(
+        self, agent: str, *, demanded_at: datetime | None = None
+    ) -> asyncio.Task[None] | None:
+        """Start a supervised probe of `agent`'s plugin while its block is in effect.
+
+        Never waits for the probe. Probes at most once per interval, unless `demanded_at`, such
+        as a task requeue, is later than the last probe; a demanded probe is not delayed by the
+        interval, and a requeue that arrives during a running probe of the same block makes that
+        probe demanded. Never twice at once for the same block; a probe of a block since replaced
+        is cancelled. Returns the running probe, or `None` when nothing was started or the block is
+        not examined.
+        """
+        now = self._clock.now()
+        block = self._quotas.block_for(agent)
+        if block is None or block.until <= now:
+            return None
+        last = self._probed_at.get(agent)
+        demanded = demanded_at is not None and (last is None or demanded_at > last)
+        running = self._probes.get(agent)
+        if running is not None and not running.task.done():
+            if running.blocked_at == block.blocked_at:
+                if demanded and not running.demanded:
+                    self._probes[agent] = _Probe(running.blocked_at, running.task, demanded=True)
+                return running.task
+            running.task.cancel()
+        interval = timedelta(seconds=self._settings.interval_seconds)
+        if last is not None and now - last < interval and not demanded:
+            return None
+        self._probed_at[agent] = now
+        task = asyncio.Task(self._probe(agent, block, now + interval), eager_start=True)
+        self._probes[agent] = _Probe(block.blocked_at, task, demanded=demanded)
+        return task
+
+    async def refresh(self, agent: str, *, demanded_at: datetime | None = None) -> None:
+        """Probe `agent`'s plugin as `request_refresh` does and wait until the probe ends."""
+        task = self.request_refresh(agent, demanded_at=demanded_at)
+        if task is not None:
+            await asyncio.wait({task})
+
+    @property
+    def probing(self) -> bool:
+        """Return whether any probe is still running."""
+        return any(not probe.task.done() for probe in self._probes.values())
+
+    def awaiting_demanded_probe(self, block: AgentBlock) -> bool:
+        """Return whether a requeue-demanded probe of `block` is still running."""
+        probe = self._probes.get(block.agent)
+        return (
+            probe is not None
+            and probe.demanded
+            and probe.blocked_at == block.blocked_at
+            and not probe.task.done()
+        )
+
+    async def shutdown(self) -> None:
+        """Cancel every probe in flight and wait until each has ended."""
+        tasks = [probe.task for probe in self._probes.values()]
+        self._probes.clear()
+        for task in tasks:
+            task.cancel()
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+    async def _probe(self, agent: str, block: AgentBlock, next_check: datetime) -> None:
+        async with self._faults.isolate(f"{QUOTA_REFRESH_SCOPE} {agent}"):
+            self.expect_check(agent, next_check)
+            probe = quota_probe_for(self._agents.get(agent))
+            if probe is None:
+                return
+            reading = await self._read(agent, probe, block.blocked_at)
+            current = self._quotas.block_for(agent)
+            if (
+                reading is not None
+                and current is not None
+                and current.blocked_at == block.blocked_at
+            ):
+                self.observe(agent, reading.windows)
+
+    async def _read(
+        self, agent: str, probe: QuotaProbe, blocked_at: datetime
+    ) -> QuotaReading | None:
+        timeout = self._settings.timeout_seconds
+        read = self._threads.start(agent, probe.read_quota, blocked_at)
+        timer = asyncio.create_task(self._clock.sleep(timeout))
+        try:
+            await asyncio.wait({read, timer}, return_when=asyncio.FIRST_COMPLETED)
+            if not read.done():
+                message = f"quota probe did not answer within {timeout} seconds"
+                raise TimeoutError(message)
+            return read.result()
+        finally:
+            read.cancel()
+            timer.cancel()
+            await asyncio.gather(read, timer, return_exceptions=True)
+
+    def expect_check(self, agent: str, at: datetime) -> None:
+        """Record that `agent`'s block is next examined at `at`, or retried when it lifts first.
+
+        An agent whose plugin cannot probe is not examined: its block is retried when it lifts.
+        """
+        block = self._quotas.block_for(agent)
+        if block is None:
+            return
+        if block.until <= at or quota_probe_for(self._agents.get(agent)) is None:
+            scheduled = ScheduledQuotaAction(QuotaAction.RETRY, block.until, block.blocked_at)
+        else:
+            scheduled = ScheduledQuotaAction(QuotaAction.NEXT_CHECK, at, block.blocked_at)
+        self._schedule.record(agent, scheduled)
+
+    @property
+    def interval_seconds(self) -> float:
+        """Return the longest an obsolete block can go unnoticed while someone waits on it."""
+        return self._settings.interval_seconds

shikiru/engine/quota_schedule.py

@@ -0,0 +1,24 @@
+"""What the engine will next do about each quota-blocked agent, for the views that show it."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from shikiru.domain.quota_waits import ScheduledQuotaAction
+
+
+class QuotaSchedule:
+    """The next scheduled action per agent, in memory: it is rebuilt as the engine acts again."""
+
+    def __init__(self) -> None:
+        """Start with no action scheduled."""
+        self._actions: dict[str, ScheduledQuotaAction] = {}
+
+    def record(self, agent: str, action: ScheduledQuotaAction) -> None:
+        """Remember `action` as `agent`'s next one."""
+        self._actions[agent] = action
+
+    def get(self, agent: str) -> ScheduledQuotaAction | None:
+        """Return `agent`'s recorded action, or none when none was recorded."""
+        return self._actions.get(agent)