taskflow-meter 1.0.0__py3-none-any.whl

taskflow_meter/contrib/flask.py

@@ -0,0 +1,140 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Register the meter's routes in a Flask application's own blueprint.
+
+Mounting the WSGI callable with ``DispatcherMiddleware`` is simpler and
+works everywhere; reach for this when the routes need to be *inside*
+the host app, so its ``before_request`` hooks, error handlers and
+authentication apply to them::
+
+    from taskflow_meter.contrib.flask import meter_blueprint
+
+    app.register_blueprint(meter_blueprint(meter), url_prefix="/taskflow")
+
+Flask's rule syntax differs from ours -- ``<run_id>`` rather than
+``{run_id}`` -- so the templates are translated on the way in, from the
+same table the router matches on.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from typing import Any
+
+import flask
+from flask import Blueprint
+from flask import Response
+
+from taskflow_meter.api import routes as route_table
+from taskflow_meter.api.dispatch import Dispatcher
+from taskflow_meter.api.http import MeterRequest
+from taskflow_meter.api.http import MeterResponse
+from taskflow_meter.api.http import mount_prefix
+from taskflow_meter.api.router import Route
+from taskflow_meter.api.service import MeterService
+from taskflow_meter.api.sse import StreamResponse
+from taskflow_meter.api.sse import iter_frames
+from taskflow_meter.meter import Meter
+
+DEFAULT_STREAM_INTERVAL = 1.0
+DEFAULT_HEARTBEAT = 15.0
+
+_PARAM = re.compile(r"\{([a-zA-Z_][a-zA-Z0-9_]*)\}")
+
+
+def meter_blueprint(
+    meter: Meter,
+    *,
+    service: MeterService | None = None,
+    name: str = "taskflow_meter",
+    stream_interval: float = DEFAULT_STREAM_INTERVAL,
+    heartbeat: float = DEFAULT_HEARTBEAT,
+) -> Blueprint:
+    """Build a blueprint serving every endpoint the callables serve."""
+    resolved = service or MeterService(meter)
+    dispatcher = Dispatcher(resolved)
+    blueprint = Blueprint(name, __name__)
+
+    for route in route_table.build_routes(resolved):
+        blueprint.add_url_rule(
+            to_flask_rule(route.template),
+            endpoint=route.name,
+            view_func=_view(
+                meter, dispatcher, route, stream_interval, heartbeat
+            ),
+            methods=[route.method],
+        )
+    return blueprint
+
+
+def to_flask_rule(template: str) -> str:
+    """``/flows/{run_id}`` -> ``/flows/<run_id>``."""
+    return _PARAM.sub(r"<\1>", template)
+
+
+def _view(
+    meter: Meter,
+    dispatcher: Dispatcher,
+    route: Route,
+    stream_interval: float,
+    heartbeat: float,
+) -> Callable[..., Response]:
+    def view(**params: Any) -> Response:
+        # Flask has no lifespan either, so the first request is where a
+        # meter nobody started gets started.
+        meter.ensure_started()
+
+        result = dispatcher.run(route, _build_request(route, params))
+        if isinstance(result, StreamResponse):
+            return Response(
+                iter_frames(
+                    result.cursor,
+                    interval=stream_interval,
+                    heartbeat=heartbeat,
+                ),
+                status=result.status,
+                headers=list(result.headers),
+            )
+        return _response(result)
+
+    return view
+
+
+def _build_request(route: Route, params: dict[str, Any]) -> MeterRequest:
+    """Translate the active Flask request, recovering the url_prefix.
+
+    ``request.path`` excludes ``SCRIPT_NAME`` but includes whatever
+    prefix the blueprint was registered under, so the prefix is what is
+    left when our own template is taken off the end.
+    """
+    request = flask.request
+    text_params = {key: str(value) for key, value in params.items()}
+    sub_path = route.template.format(**text_params)
+
+    return MeterRequest(
+        method=request.method,
+        path=sub_path,
+        prefix=request.script_root + mount_prefix(request.path, sub_path),
+        query={key: request.args.getlist(key) for key in request.args},
+        headers={key.lower(): value for key, value in request.headers.items()},
+        path_params=text_params,
+    )
+
+
+def _response(result: MeterResponse) -> Response:
+    return Response(
+        result.body,
+        status=result.status,
+        headers=list(result.headers),
+    )

taskflow_meter/contrib/paste.py

@@ -0,0 +1,96 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Host the meter in a paste pipeline.
+
+A service that composes its WSGI stack from ``api-paste.ini`` wants a
+``paste.app_factory``.  Dispatch a prefix to the app and it is served
+alongside the API it is monitoring, on the same port, behind the same
+middleware::
+
+    [composite:main]
+    use = egg:Paste#urlmap
+    /: your_api
+    /taskflow-meter: taskflow_meter
+
+    [app:taskflow_meter]
+    paste.app_factory = taskflow_meter.contrib.paste:app_factory
+
+Configuration comes from the ``[taskflow_meter]`` group in the
+service's own oslo.config file, so there is no second config file to
+deploy.  Anything in the paste stanza
+overrides it, for the deployments that would rather keep it all in
+``api-paste.ini``::
+
+    [app:taskflow_meter]
+    paste.app_factory = taskflow_meter.contrib.paste:app_factory
+    connection = mysql+pymysql://user:pass@host/taskflow
+    poll_interval = 5
+
+urlmap gives the app a ``SCRIPT_NAME`` of the prefix it was mounted at,
+which is exactly what the WSGI callable builds its links from, so the
+mount point needs no configuring.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from oslo_config import cfg
+
+from taskflow_meter.api.wsgi import WSGIApp
+from taskflow_meter.conf import GROUP_NAME
+from taskflow_meter.conf import meter_from_config
+from taskflow_meter.conf import register_opts
+
+#: Settings a paste stanza may override, and how to read them.
+_OVERRIDES: dict[str, Any] = {
+    "connection": str,
+    "poll": lambda value: str(value).lower() in {"1", "true", "yes", "on"},
+    "poll_interval": float,
+    "max_events_per_run": int,
+}
+
+
+def app_factory(
+    global_config: dict[str, Any] | None = None,  # noqa: ARG001
+    **local_conf: str,
+) -> WSGIApp:
+    """Build the WSGI app.  Signature fixed by paste.
+
+    ``global_config`` is paste's ``[DEFAULT]`` section, which we do not
+    read: the service's oslo.config is the source of truth, and the
+    paste stanza is the override.
+    """
+    conf = register_opts()
+    apply_overrides(conf, local_conf)
+    return WSGIApp(meter_from_config(conf))
+
+
+def apply_overrides(conf: cfg.ConfigOpts, local_conf: dict[str, str]) -> None:
+    """Fold ``api-paste.ini`` values into the config.
+
+    Paste hands everything over as a string, so each one is coerced by
+    the type its option expects.  An unusable value is rejected here,
+    naming the setting, rather than surfacing later as an odd failure
+    from somewhere else.
+    """
+    for name, coerce in _OVERRIDES.items():
+        if name not in local_conf:
+            continue
+        raw = local_conf[name]
+        try:
+            value = coerce(raw)
+        except (TypeError, ValueError) as exc:
+            msg = f"{name}={raw!r} in the paste stanza is not usable"
+            raise ValueError(msg) from exc
+        conf.set_override(name, value, group=GROUP_NAME)

taskflow_meter/contrib/pecan.py

@@ -0,0 +1,84 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Host the meter inside a Pecan controller tree.
+
+For services built on Pecan, when the meter should live in the
+application's own routing rather than beside it in a paste pipeline::
+
+    from taskflow_meter.contrib.pecan import MeterController
+
+
+    class RootController:
+        v1 = V1Controller()
+        taskflow_meter = MeterController()
+
+Everything under that path is handed to the WSGI callable.  Pecan is
+built on WebOb, so the request already carries the environ the callable
+wants; the only work is moving the part of the path Pecan has already
+consumed out of ``PATH_INFO`` and into ``SCRIPT_NAME``, which is what
+makes the generated links point back at the right place.
+
+Mounting a sub-application means the host's own hooks and middleware do
+not run for these requests.  For deployments where they must, wire the
+route table into the host's router instead -- ``api.routes.build_routes``
+is that table, and it is data.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pecan
+from webob import Request
+
+from taskflow_meter.api.http import mount_prefix
+from taskflow_meter.api.wsgi import WSGIApp
+from taskflow_meter.conf import wsgi_app_from_config
+
+
+class MeterController:
+    """A Pecan controller that delegates to the WSGI callable."""
+
+    def __init__(self, app: WSGIApp | None = None) -> None:
+        """Wrap ``app``, or build one from the service's config."""
+        self.app = app if app is not None else wsgi_app_from_config()
+
+    @pecan.expose()
+    def index(self) -> Any:
+        """The controller's own path, with nothing after it."""
+        return self._delegate(())
+
+    @pecan.expose()
+    def _default(self, *remainder: str) -> Any:
+        """Everything below it."""
+        return self._delegate(remainder)
+
+    def _delegate(self, remainder: tuple[str, ...]) -> Any:
+        request = pecan.request
+        environ = dict(request.environ)
+        consumed, sub_path = _split(request.path_info, remainder)
+        environ["SCRIPT_NAME"] = request.script_name + consumed
+        environ["PATH_INFO"] = sub_path
+        # Pecan uses a returned WebOb response as-is, so the callable's
+        # status, headers and body reach the client untouched.
+        return Request(environ).get_response(self.app)
+
+
+def _split(path_info: str, remainder: tuple[str, ...]) -> tuple[str, str]:
+    """Split the path into what Pecan consumed and what is left for us.
+
+    Pecan does not rewrite ``PATH_INFO`` as it routes, so the mount
+    point has to be recovered by taking the remainder off the end.
+    """
+    sub_path = "/" + "/".join(remainder) if remainder else "/"
+    return mount_prefix(path_info, sub_path), sub_path

taskflow_meter/datasource/__init__.py

@@ -0,0 +1,33 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Datasources: where flow state is read from, and optionally written to."""
+
+from __future__ import annotations
+
+from taskflow_meter.datasource.base import DataSource
+from taskflow_meter.datasource.base import EventPage
+from taskflow_meter.datasource.base import FlowPage
+from taskflow_meter.datasource.base import UnknownMarkerError
+from taskflow_meter.datasource.base import WritableDataSource
+from taskflow_meter.datasource.memory import MemoryDataSource
+from taskflow_meter.datasource.persistence import PersistenceDataSource
+
+__all__ = [
+    "DataSource",
+    "EventPage",
+    "FlowPage",
+    "MemoryDataSource",
+    "PersistenceDataSource",
+    "UnknownMarkerError",
+    "WritableDataSource",
+]

taskflow_meter/datasource/base.py

@@ -0,0 +1,154 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""The contract every datasource implements.
+
+Read and write are separate on purpose.  The primary datasource reads
+taskflow's own persistence and can never accept events, while the in-memory
+and SQL ones are fed by a producer; making that a type distinction stops a
+read-only source from being wired up as a sink by mistake.
+"""
+
+from __future__ import annotations
+
+import abc
+from collections.abc import Iterable
+from dataclasses import dataclass
+from types import TracebackType
+from typing import Self
+
+from taskflow_meter.events import Event
+from taskflow_meter.models import AtomSnapshot
+from taskflow_meter.models import FlowSnapshot
+
+#: Flows returned by a single unqualified listing.
+DEFAULT_FLOW_LIMIT = 50
+
+#: Events returned by a single :meth:`DataSource.events_since` call.
+DEFAULT_EVENT_LIMIT = 500
+
+
+class UnknownMarkerError(LookupError):
+    """A paging marker refers to a flow the datasource cannot place.
+
+    Usually means the run expired between pages.  Raised rather than
+    silently restarting from the top, which would make a client loop over
+    the same first page forever.
+    """
+
+
+@dataclass(frozen=True, slots=True)
+class FlowPage:
+    """One page of flows, newest observation first."""
+
+    items: tuple[FlowSnapshot, ...] = ()
+    next_marker: str | None = None
+
+    @property
+    def has_more(self) -> bool:
+        return self.next_marker is not None
+
+
+@dataclass(frozen=True, slots=True)
+class EventPage:
+    """One page of a run's event stream.
+
+    ``truncated`` is the important field: it says the caller asked for
+    events that have already been evicted, so there is a hole between what
+    it last saw and what it is being given.  A client that cares must
+    re-read the snapshot instead of assuming continuity.
+    """
+
+    events: tuple[Event, ...] = ()
+    next_seq: int = 0
+    oldest_seq: int | None = None
+    truncated: bool = False
+
+
+class DataSource(abc.ABC):
+    """Read access to observed flows."""
+
+    #: Stevedore plugin name, set by subclasses.
+    name: str = ""
+
+    #: Whether :meth:`events_since` can actually return a history.  A
+    #: source reading a store that only keeps current state sets this
+    #: False, so an API can decline to advertise a stream it cannot serve
+    #: rather than handing clients an empty one that is indistinguishable
+    #: from silence.
+    supports_events: bool = True
+
+    def start(self) -> None:  # noqa: B027 - optional hook, not abstract
+        """Acquire whatever the source needs.  Idempotent.
+
+        Sources holding nothing (the in-memory one) need not override it.
+        """
+
+    def stop(self) -> None:  # noqa: B027 - optional hook, not abstract
+        """Release it again.  Idempotent, and safe to call unstarted."""
+
+    def __enter__(self) -> Self:
+        self.start()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self.stop()
+
+    @abc.abstractmethod
+    def list_flows(
+        self,
+        *,
+        state: str | None = None,
+        book_id: str | None = None,
+        limit: int = DEFAULT_FLOW_LIMIT,
+        marker: str | None = None,
+    ) -> FlowPage:
+        """Return flows, newest observation first."""
+
+    @abc.abstractmethod
+    def get_flow(self, run_id: str) -> FlowSnapshot | None:
+        """Return one flow, or ``None`` if the source has never seen it."""
+
+    @abc.abstractmethod
+    def events_since(
+        self,
+        run_id: str,
+        *,
+        since_seq: int = 0,
+        limit: int = DEFAULT_EVENT_LIMIT,
+    ) -> EventPage:
+        """Return events for ``run_id`` with ``seq`` greater than
+        ``since_seq``."""
+
+    def get_atoms(self, run_id: str) -> tuple[AtomSnapshot, ...] | None:
+        """Return a flow's atoms in name order, or ``None`` if unknown."""
+        flow = self.get_flow(run_id)
+        if flow is None:
+            return None
+        return tuple(flow.atoms[name] for name in flow.atom_names)
+
+
+class WritableDataSource(DataSource):
+    """A datasource that is fed by a producer."""
+
+    @abc.abstractmethod
+    def apply(self, event: Event) -> None:
+        """Fold one event into the stored state."""
+
+    def apply_many(self, events: Iterable[Event]) -> None:
+        for event in events:
+            self.apply(event)