taskflow-meter 1.0.0__py3-none-any.whl

taskflow_meter/collect/progress.py

@@ -0,0 +1,170 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Capture the progress a task reports about itself.
+
+The gap a listener cannot close.  When a task calls
+``self.update_progress(0.4)``, taskflow's ``TaskAction`` writes the
+number to storage and **never re-emits it on the engine's atom
+notifier** -- so a listener sees state transitions and nothing in
+between.
+
+The number is published on the *task's own* notifier, one per atom, so
+this walks the compiled graph and registers there.  Registering is
+symmetrical: whatever is bound is unbound again, because leaving a
+callback on somebody's task object outlives the monitoring that wanted
+it.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from collections.abc import Callable
+from typing import Any
+
+from taskflow import task as tf_task
+from taskflow.engines.action_engine import compiler as tf_compiler
+
+from taskflow_meter.events import Event
+from taskflow_meter.events import EventKind
+from taskflow_meter.events import SequenceAllocator
+from taskflow_meter.models import TASK
+
+LOG = logging.getLogger(__name__)
+
+Emit = Callable[[Event], Any]
+
+
+class ProgressTap:
+    """Registers for ``EVENT_UPDATE_PROGRESS`` on every task in a flow."""
+
+    def __init__(
+        self,
+        engine: Any,
+        emit: Emit,
+        *,
+        allocator: SequenceAllocator | None = None,
+        clock: Callable[[], float] = time.time,
+        book_id: str | None = None,
+    ) -> None:
+        self._engine = engine
+        self._emit = emit
+        self._allocator = allocator or SequenceAllocator()
+        self._clock = clock
+        self._book_id = book_id
+        self._bound: list[tuple[Any, Callable[..., None]]] = []
+
+    @property
+    def bound(self) -> int:
+        """How many tasks are currently being listened to."""
+        return len(self._bound)
+
+    def register(self) -> None:
+        """Bind to every task in the flow.  Idempotent."""
+        if self._bound:
+            return
+        for atom in atoms_of(self._engine):
+            if not isinstance(atom, tf_task.Task):
+                # Only tasks report progress; a retry controller has no
+                # notifier to bind to.
+                continue
+            if not atom.notifier.can_be_registered(
+                tf_task.EVENT_UPDATE_PROGRESS
+            ):
+                LOG.debug(
+                    "task %r does not accept progress listeners", atom.name
+                )
+                continue
+            callback = self._callback_for(atom)
+            atom.notifier.register(tf_task.EVENT_UPDATE_PROGRESS, callback)
+            self._bound.append((atom, callback))
+
+    def deregister(self) -> None:
+        """Unbind everything.  Idempotent, and never raises."""
+        while self._bound:
+            atom, callback = self._bound.pop()
+            try:
+                atom.notifier.deregister(
+                    tf_task.EVENT_UPDATE_PROGRESS, callback
+                )
+            except Exception:
+                LOG.warning(
+                    "could not deregister from task %r",
+                    getattr(atom, "name", atom),
+                    exc_info=True,
+                )
+
+    def __enter__(self) -> ProgressTap:
+        self.register()
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.deregister()
+
+    def _callback_for(self, atom: Any) -> Callable[..., None]:
+        def on_progress(
+            event_type: str,  # noqa: ARG001 - taskflow's callback shape
+            details: Any,
+        ) -> None:
+            # Runs on the thread executing the task.  Nothing may
+            # escape: a monitoring bug must not fail somebody's task.
+            try:
+                self._emit(self._build(atom, details))
+            except Exception:
+                LOG.exception("could not emit progress for task %r", atom.name)
+
+        return on_progress
+
+    def _build(self, atom: Any, details: Any) -> Event:
+        payload = dict(details or {})
+        progress = payload.pop("progress", None)
+        run_id = str(self._engine.storage.flow_uuid)
+        return Event(
+            run_id=run_id,
+            seq=self._allocator.allocate(run_id),
+            ts=self._clock(),
+            kind=EventKind.ATOM_PROGRESS,
+            book_id=self._book_id,
+            atom_name=atom.name,
+            atom_uuid=_atom_uuid(self._engine, atom.name),
+            atom_type=TASK,
+            progress=float(progress) if progress is not None else None,
+            details={"progress_details": payload} if payload else {},
+        )
+
+
+def atoms_of(engine: Any) -> list[Any]:
+    """Every atom in the flow, from the compiled graph.
+
+    Compiling here rather than waiting for ``run()`` is what lets the
+    tap bind before the first task starts -- and it is idempotent, so
+    an engine that was already compiled is untouched.
+    """
+    engine.compile()
+    compilation = engine.compilation
+    if compilation is None:  # pragma: no cover - compile() just ran
+        return []
+    graph = compilation.execution_graph
+    return [
+        node
+        for node, data in graph.nodes(data=True)
+        if data.get("kind") in tf_compiler.ATOMS
+    ]
+
+
+def _atom_uuid(engine: Any, name: str) -> str | None:
+    try:
+        return str(engine.storage.get_atom_uuid(name))
+    except Exception:
+        # Before the flow is prepared the atom may have no detail yet.
+        return None

taskflow_meter/conf.py

@@ -0,0 +1,173 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Configuration, read from the host service's own oslo.config.
+
+The point is that an operator adds the meter to their service's
+pipeline and it works, without writing a second config file.  If
+``[taskflow_meter] connection`` is unset we fall back to ``[database]
+connection`` -- the option those services already set -- so the common
+case needs no configuration at all.
+
+That fallback is a convenience, not a guarantee: a deployment whose
+taskflow logbooks live somewhere other than the service's main database
+has to say so.  It is spelled out in the option help rather than left
+for someone to discover from an empty flow list.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from oslo_config import cfg
+
+from taskflow_meter.api.wsgi import WSGIApp
+from taskflow_meter.datasource.persistence import PersistenceDataSource
+from taskflow_meter.meter import Meter
+from taskflow_meter.poller import DEFAULT_INTERVAL
+
+GROUP_NAME = "taskflow_meter"
+
+OPT_GROUP = cfg.OptGroup(
+    name=GROUP_NAME,
+    title="TaskFlow monitoring",
+    help="Read-only monitoring of taskflow flow execution progress.",
+)
+
+OPTS = [
+    cfg.StrOpt(
+        "connection",
+        secret=True,
+        help=(
+            "Connection string for the taskflow persistence backend, in "
+            "the form the flows themselves use. Defaults to [database] "
+            "connection, which is right when taskflow shares this "
+            "service's database and wrong when its logbooks live "
+            "elsewhere -- set this explicitly if they do."
+        ),
+    ),
+    cfg.BoolOpt(
+        "poll",
+        default=True,
+        help=(
+            "Poll the backend and keep an event history. Turn this off "
+            "in API workers when a separate collector process keeps the "
+            "store warm, so N workers do not become N pollers on the "
+            "same database. With it off, the event and stream endpoints "
+            "report 501."
+        ),
+    ),
+    cfg.FloatOpt(
+        "poll_interval",
+        default=DEFAULT_INTERVAL,
+        min=0.05,
+        help=(
+            "Seconds between polls. This is also the resolution of what "
+            "can be observed: a state a flow entered and left within one "
+            "interval was never visible, and no amount of polling "
+            "afterwards recovers it."
+        ),
+    ),
+    cfg.IntOpt(
+        "max_events_per_run",
+        default=1000,
+        min=1,
+        help=(
+            "Events retained per flow before the oldest are dropped. A "
+            "client that falls behind further than this is told its "
+            "history has a hole rather than being handed a stream that "
+            "silently skips events."
+        ),
+    ),
+]
+
+
+def register_opts(conf: cfg.ConfigOpts | None = None) -> cfg.ConfigOpts:
+    """Register our options on ``conf``.  Safe to call repeatedly.
+
+    Registered by group *name*, not by passing :data:`OPT_GROUP`.
+    oslo.config keeps each option's state -- including overrides -- on
+    the OptGroup object, so handing the same one to two ConfigOpts makes
+    them share values.  Nothing notices while there is only the global
+    CONF, and everything notices the moment there is a second one.
+    """
+    target = cfg.CONF if conf is None else conf
+    target.register_opts(OPTS, group=GROUP_NAME)
+    return target
+
+
+def list_opts() -> list[tuple[cfg.OptGroup, list[cfg.Opt]]]:
+    """Entry point for ``oslo-config-generator``.
+
+    Returns the group object so the generated sample carries its title
+    and help; registration deliberately does not.
+    """
+    return [(OPT_GROUP, OPTS)]
+
+
+def resolve_connection(conf: cfg.ConfigOpts) -> str:
+    """Find the taskflow connection, or explain what is missing."""
+    configured = conf[GROUP_NAME].connection
+    if configured:
+        return str(configured)
+
+    inherited = _host_database_connection(conf)
+    if inherited:
+        return inherited
+
+    msg = (
+        "no taskflow persistence connection configured: set "
+        f"[{GROUP_NAME}] connection, or run inside a service that sets "
+        "[database] connection"
+    )
+    raise cfg.RequiredOptError("connection", GROUP_NAME) from ValueError(msg)
+
+
+def _host_database_connection(conf: cfg.ConfigOpts) -> str | None:
+    """Read ``[database] connection`` if the host service has one.
+
+    Read rather than imported: oslo.db is not our dependency, and the
+    services that set this option have already registered it.
+    """
+    try:
+        return str(conf.database.connection) or None
+    except (cfg.NoSuchOptError, cfg.NoSuchGroupError, AttributeError):
+        return None
+
+
+def meter_from_config(conf: cfg.ConfigOpts | None = None) -> Meter:
+    """Build a meter from configuration alone."""
+    target = register_opts(conf)
+    settings = target[GROUP_NAME]
+    source = PersistenceDataSource(
+        conf={"connection": resolve_connection(target)}
+    )
+    store = None
+    if settings.poll:
+        from taskflow_meter.datasource.memory import MemoryDataSource
+
+        store = MemoryDataSource(
+            max_events_per_run=settings.max_events_per_run
+        )
+    return Meter(
+        source,
+        store=store,
+        poll=settings.poll,
+        interval=settings.poll_interval,
+    )
+
+
+def wsgi_app_from_config(
+    conf: cfg.ConfigOpts | None = None, **overrides: Any
+) -> WSGIApp:
+    """Build the WSGI callable a paste pipeline or Pecan tree can host."""
+    return WSGIApp(meter_from_config(conf), **overrides)

taskflow_meter/contrib/__init__.py

@@ -0,0 +1,18 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Adapters for hosting the meter inside somebody else's application.
+
+Nothing here is imported by the package itself, and none of it adds a
+runtime dependency: a deployment that hosts the meter in Pecan already
+has Pecan.
+"""

taskflow_meter/contrib/django.py

@@ -0,0 +1,160 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Register the meter's routes in a Django project's URL conf.
+
+Mounting the WSGI callable beside Django works too; reach for this when
+the routes need to be *inside* the project, so its middleware,
+authentication and permission decorators apply to them::
+
+    from django.urls import include, path
+    from taskflow_meter.contrib.django import meter_urlpatterns
+
+    urlpatterns = [
+        path("taskflow/", include(meter_urlpatterns(meter))),
+    ]
+
+Django's converter syntax differs from ours -- ``<str:run_id>`` rather
+than ``{run_id}`` -- so the templates are translated on the way in,
+from the same table the router matches on.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from typing import Any
+
+from django.http import HttpRequest
+from django.http import HttpResponse
+from django.http import StreamingHttpResponse
+from django.urls import URLPattern
+from django.urls import path as django_path
+
+from taskflow_meter.api import routes as route_table
+from taskflow_meter.api.dispatch import Dispatcher
+from taskflow_meter.api.http import MeterRequest
+from taskflow_meter.api.http import MeterResponse
+from taskflow_meter.api.http import mount_prefix
+from taskflow_meter.api.router import Route
+from taskflow_meter.api.service import MeterService
+from taskflow_meter.api.sse import StreamResponse
+from taskflow_meter.api.sse import iter_frames
+from taskflow_meter.meter import Meter
+
+DEFAULT_STREAM_INTERVAL = 1.0
+DEFAULT_HEARTBEAT = 15.0
+
+_PARAM = re.compile(r"\{([a-zA-Z_][a-zA-Z0-9_]*)\}")
+
+
+def meter_urlpatterns(
+    meter: Meter,
+    *,
+    service: MeterService | None = None,
+    stream_interval: float = DEFAULT_STREAM_INTERVAL,
+    heartbeat: float = DEFAULT_HEARTBEAT,
+) -> list[URLPattern]:
+    """Build URL patterns serving every endpoint the callables serve."""
+    resolved = service or MeterService(meter)
+    dispatcher = Dispatcher(resolved)
+
+    return [
+        django_path(
+            to_django_route(route.template),
+            _view(meter, dispatcher, route, stream_interval, heartbeat),
+            name=route.name,
+        )
+        for route in route_table.build_routes(resolved)
+    ]
+
+
+def to_django_route(template: str) -> str:
+    """``/flows/{run_id}`` -> ``flows/<str:run_id>``.
+
+    Django patterns are relative to wherever they are included, so the
+    leading slash goes too.
+    """
+    return _PARAM.sub(r"<str:\1>", template).lstrip("/")
+
+
+def _view(
+    meter: Meter,
+    dispatcher: Dispatcher,
+    route: Route,
+    stream_interval: float,
+    heartbeat: float,
+) -> Callable[..., HttpResponse | StreamingHttpResponse]:
+    def view(
+        request: HttpRequest, **params: Any
+    ) -> HttpResponse | StreamingHttpResponse:
+        meter.ensure_started()
+
+        result = dispatcher.run(route, _build_request(request, route, params))
+        if isinstance(result, StreamResponse):
+            streaming = StreamingHttpResponse(
+                iter_frames(
+                    result.cursor,
+                    interval=stream_interval,
+                    heartbeat=heartbeat,
+                ),
+                status=result.status,
+            )
+            _apply_headers(streaming, result.headers)
+            return streaming
+        return _response(result)
+
+    return view
+
+
+def _build_request(
+    request: HttpRequest, route: Route, params: dict[str, Any]
+) -> MeterRequest:
+    """Translate a Django request, recovering where it was included.
+
+    ``path_info`` excludes ``SCRIPT_NAME`` but includes the prefix the
+    patterns were included under, so the prefix is what is left when
+    our own template is taken off the end.
+    """
+    text_params = {key: str(value) for key, value in params.items()}
+    sub_path = route.template.format(**text_params)
+    script_name = request.META.get("SCRIPT_NAME", "")
+
+    return MeterRequest(
+        method=request.method or "GET",
+        path=sub_path,
+        prefix=script_name + mount_prefix(request.path_info, sub_path),
+        query=dict(request.GET.lists()),
+        headers={key.lower(): value for key, value in request.headers.items()},
+        path_params=text_params,
+    )
+
+
+def _response(result: MeterResponse) -> HttpResponse:
+    response = HttpResponse(result.body, status=result.status)
+    _apply_headers(response, result.headers)
+    return response
+
+
+def _apply_headers(
+    response: HttpResponse | StreamingHttpResponse,
+    headers: tuple[tuple[str, str], ...],
+) -> None:
+    """Copy our headers onto a Django response.
+
+    Content-Length included: Django does not compute one itself, and
+    its header mapping overwrites rather than appends, so there is no
+    duplicate for a later middleware to trip over.  A streaming
+    response never carries one to begin with.
+    """
+    for key, value in headers:
+        response[key] = value

taskflow_meter/contrib/fastapi.py

@@ -0,0 +1,149 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Register the meter's routes in a FastAPI application's own router.
+
+Mounting the raw ASGI callable is simpler and works everywhere, so
+reach for this when the routes need to be *inside* the host app: its
+authentication dependencies, its middleware, its exception handlers and
+its OpenAPI schema all apply to a router, and none of them apply to a
+mount::
+
+    from taskflow_meter.contrib.fastapi import meter_router
+
+    app.include_router(
+        meter_router(meter),
+        prefix="/taskflow",
+        dependencies=[Depends(require_admin)],
+    )
+
+Our path templates already use FastAPI's ``{name}`` syntax, so they are
+registered verbatim and the host's ``prefix`` is recovered from the
+request rather than configured twice.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable
+from collections.abc import Callable
+from collections.abc import Sequence
+
+from fastapi import APIRouter
+from fastapi import Request
+from fastapi import Response
+from fastapi.responses import StreamingResponse
+
+from taskflow_meter.api import routes as route_table
+from taskflow_meter.api.dispatch import Dispatcher
+from taskflow_meter.api.http import MeterRequest
+from taskflow_meter.api.http import MeterResponse
+from taskflow_meter.api.http import mount_prefix
+from taskflow_meter.api.http import split_path
+from taskflow_meter.api.router import Route
+from taskflow_meter.api.service import MeterService
+from taskflow_meter.api.sse import StreamResponse
+from taskflow_meter.api.sse import aiter_frames
+from taskflow_meter.meter import Meter
+
+DEFAULT_STREAM_INTERVAL = 1.0
+DEFAULT_HEARTBEAT = 15.0
+
+
+def meter_router(
+    meter: Meter,
+    *,
+    service: MeterService | None = None,
+    stream_interval: float = DEFAULT_STREAM_INTERVAL,
+    heartbeat: float = DEFAULT_HEARTBEAT,
+    tags: Sequence[str] | None = None,
+) -> APIRouter:
+    """Build a router serving every endpoint the callables serve."""
+    resolved = service or MeterService(meter)
+    dispatcher = Dispatcher(resolved)
+    router = APIRouter(tags=list(tags or ["taskflow-meter"]))
+
+    for route in route_table.build_routes(resolved):
+        router.add_api_route(
+            route.template,
+            _endpoint(meter, dispatcher, route, stream_interval, heartbeat),
+            methods=[route.method],
+            name=route.name,
+            # The payloads are plain dictionaries built by our own
+            # serialisers; letting FastAPI infer a response model would
+            # only give it something to validate them against twice.
+            response_class=Response,
+        )
+    return router
+
+
+def _endpoint(
+    meter: Meter,
+    dispatcher: Dispatcher,
+    route: Route,
+    stream_interval: float,
+    heartbeat: float,
+) -> Callable[[Request], Awaitable[Response]]:
+    async def endpoint(request: Request) -> Response:
+        # A mounted app is never told the server started, and an
+        # included router is no different.
+        await asyncio.to_thread(meter.ensure_started)
+
+        meter_request = _build_request(request, route)
+        result = await asyncio.to_thread(dispatcher.run, route, meter_request)
+
+        if isinstance(result, StreamResponse):
+            return StreamingResponse(
+                aiter_frames(
+                    result.cursor,
+                    interval=stream_interval,
+                    heartbeat=heartbeat,
+                ),
+                status_code=result.status,
+                headers=dict(result.headers),
+            )
+        return _response(result)
+
+    return endpoint
+
+
+def _build_request(request: Request, route: Route) -> MeterRequest:
+    """Translate a Starlette request, recovering the router's prefix.
+
+    ``include_router(prefix=...)`` puts the prefix in the path rather
+    than in ``root_path``, so it is found by taking our own template off
+    the end of what arrived.
+    """
+    params = {key: str(value) for key, value in request.path_params.items()}
+    root_path = request.scope.get("root_path", "")
+    path = split_path(request.scope.get("path", "/"), root_path)
+    sub_path = route.template.format(**params)
+
+    return MeterRequest(
+        method=request.method,
+        path=sub_path,
+        prefix=root_path + mount_prefix(path, sub_path),
+        query={
+            key: request.query_params.getlist(key)
+            for key in request.query_params
+        },
+        headers={key.lower(): value for key, value in request.headers.items()},
+        path_params=params,
+    )
+
+
+def _response(result: MeterResponse) -> Response:
+    return Response(
+        content=result.body,
+        status_code=result.status,
+        headers=dict(result.headers),
+    )