keepup-tasks 0.5.0__tar.gz

keepup_tasks-0.5.0/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: keepup-tasks
+Version: 0.5.0
+Summary: The one scheduler, the declared jobs and the distributed locks they run under.
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: keepup-admin>=0.4.0
+Requires-Dist: keepup-db>=0.4.0
+Requires-Dist: keepup-auth>=0.4.0
+Requires-Dist: apscheduler<4,>=3.10
+Requires-Dist: fastapi<1,>=0.128
+
+# keepup-tasks
+
+The one scheduler, the declared jobs and the distributed locks they run under.
+
+One capability of [keepup](https://github.com/FedorSergeev/keepup-admin), the framework over FastAPI for admin
+panels: the base is installed as `keepup-admin` and imported as `keepup`, and
+this distribution carries one part of it. It is installed on its own
+(`pip install keepup-tasks`) or together with the base (`pip install "keepup-admin[panel]"`),
+and it declares itself to the kernel through its
+`[project.entry-points."keepup.plugins"]` entry point.

keepup_tasks-0.5.0/README.md

@@ -0,0 +1,10 @@
+# keepup-tasks
+
+The one scheduler, the declared jobs and the distributed locks they run under.
+
+One capability of [keepup](https://github.com/FedorSergeev/keepup-admin), the framework over FastAPI for admin
+panels: the base is installed as `keepup-admin` and imported as `keepup`, and
+this distribution carries one part of it. It is installed on its own
+(`pip install keepup-tasks`) or together with the base (`pip install "keepup-admin[panel]"`),
+and it declares itself to the kernel through its
+`[project.entry-points."keepup.plugins"]` entry point.

keepup_tasks-0.5.0/keepup_tasks/__init__.py

@@ -0,0 +1,31 @@
+"""Background tasks: the one scheduler, and the locks the work runs under.
+
+This distribution provides `scheduler` and `locks` and declares one plugin,
+`tasks`, in the `keepup.plugins` group. It depends on `keepup-admin` for the
+constructor, on `keepup-auth` for the panel guard of its routes -- the temporary
+edge the route bridge carries until `keepup-http` mounts what a plugin declares
+(keepup-121) -- and on `keepup-db`, because the locks it offers write to a table.
+"""
+
+from keepup_tasks.scheduler import (
+    JOB_DEFAULTS,
+    get_scheduler,
+    init_scheduler,
+    new_scheduler,
+)
+
+__version__ = "0.5.0"
+
+__all__ = ["JOB_DEFAULTS", "get_scheduler", "init_scheduler", "new_scheduler"]
+
+#: What the entry point asks for, looked up on access: a deployment that only
+#: wants the scheduler does not pay for the plugin's panel wiring.
+_PLUGIN_NAMES = {"TasksPlugin", "SERVICE_SCHEDULER", "SERVICE_LOCKS"}
+
+
+def __getattr__(name):
+    if name in _PLUGIN_NAMES:
+        from keepup_tasks import plugin
+
+        return getattr(plugin, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

keepup_tasks-0.5.0/keepup_tasks/plugin.py

@@ -0,0 +1,197 @@
+"""Background tasks as a capability: the one scheduler and the locks under it.
+
+Task keepup-109. The scheduler used to be the kernel's (`keepup/scheduler.py`):
+the application's lifespan created it before the plugins and started it after
+them, so a deployment that wanted no background work paid for a scheduler, the
+panel's "Background tasks" section listed whatever the kernel happened to start
+(nothing), and every loop of the framework was an `asyncio.create_task` in that
+same lifespan instead of a job somebody could see.
+
+The capability owns it now:
+
+* it starts the scheduler when the application comes up -- with the job defaults
+  the deployment named -- and shuts it down with the application;
+* it publishes the `scheduler` service, and consumes the `jobs` contribution
+  (`(job_id, callable, trigger)`, the contract in doc/plugin_constructor.md), so
+  a capability declares its work instead of starting a loop of its own, and the
+  section shows every declared job with its next run;
+* it publishes the `locks` service. The lock machinery is kernel infrastructure
+  -- the kernel's own plugin runtime and the capabilities that outlived it need
+  it -- and the capability is what offers it to everyone else.
+
+The routes are declared in `keepup_tasks.scheduler` and registered by the
+application until the transport that mounts what a plugin declares exists
+(`keepup-http`, keepup-121).
+"""
+
+import logging
+from typing import Any, List
+
+from keepup.kernel.datasource import SERVICE_DATASOURCE
+from keepup.kernel.descriptor import KIND_OPTIONAL, PluginDescriptor
+from keepup.plugins.base import BasePlugin
+from keepup_tasks.scheduler import (
+    close_scheduler,
+    init_scheduler,
+    jobs_payload,
+    restart_scheduler,
+    stats_payload,
+    trigger_job,
+)
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["TasksPlugin", "SERVICE_LOCKS", "SERVICE_SCHEDULER"]
+
+#: What another plugin asks for to have its work scheduled.
+SERVICE_SCHEDULER = "scheduler"
+
+#: What another plugin asks for to run its work once across the replicas.
+SERVICE_LOCKS = "locks"
+
+#: The metrics capability's service, asked for by name: the locks' own numbers
+#: are published as a sample instead of being written into its table.
+SERVICE_METRICS = "metrics"
+
+#: The panel's section: the jobs the scheduler holds and the locks in force.
+SECTION = {
+    "id": "background_tasks",
+    "name": "Background tasks",
+    "description": "Scheduled jobs and the distributed locks they hold",
+    "js": "/keepup-static/modules/js/background_tasks.js",
+    "css": "/keepup-static/modules/css/background_tasks.css",
+    "initFunction": "initBackgroundTasks",
+    "icon": "clock",
+    "version": "1.0.0",
+}
+
+
+class TasksPlugin(BasePlugin):
+    """The scheduler and the locks, and the section that shows both."""
+
+    descriptor = PluginDescriptor(
+        id="tasks",
+        name="Background tasks",
+        version="0.5.0",
+        distribution="keepup-tasks",
+        kind=KIND_OPTIONAL,
+        priority=80,
+        provides=(f"{SERVICE_SCHEDULER}>=1", f"{SERVICE_LOCKS}>=1"),
+        requires=("datasource>=1",),
+        contributions=("routes", "sections", "tables", "jobs"),
+    )
+
+    def __init__(self, config=None):
+        super().__init__("tasks", "Background tasks", config)
+        self.scheduler = None
+
+    def register(self, services):
+        """Offer the locks and the scheduler.
+
+        The locks are published here because a plugin may ask for them while it
+        registers -- the runtime's own post-construct lock does -- and the
+        scheduler is published once it exists (``initialize``), since a
+        scheduler that is not started yet is not something to hand out.
+        """
+        if services.has(SERVICE_DATASOURCE):
+            from keepup import locks
+
+            services.provide(SERVICE_LOCKS, locks, version=1, plugin_id="tasks")
+            # The locks' own numbers, when something is collecting them: the
+            # metrics capability is optional, and the lock machinery does not
+            # write into its table (keepup-109).
+            if services.has(SERVICE_METRICS):
+                services.require(SERVICE_METRICS).collect(locks.samples)
+
+    async def initialize(self):
+        """Create the scheduler and start it; the declared jobs join at the freeze."""
+        settings = getattr(self.plugin_manager, "settings", None)
+        self.scheduler = init_scheduler(getattr(settings, "scheduler_job_defaults", None))
+        self.scheduler.start()
+        if self.services is not None:
+            self.services.provide(SERVICE_SCHEDULER, self.scheduler, version=1,
+                                  plugin_id="tasks")
+        return True
+
+    async def post_construct(self):
+        """Put the declared jobs in, now that every plugin has declared them.
+
+        Contributions are mounted after the plugins initialise (step 8 of the
+        lifecycle), so a job another capability declared is known here and not
+        in `initialize` -- and a running scheduler takes a job added to it.
+        """
+        for job in self._declared_jobs():
+            self._put(job)
+        logger.info("The tasks capability holds %s job(s)",
+                    len(self.scheduler.get_jobs()) if self.scheduler else 0)
+
+    def _declared_jobs(self) -> List[Any]:
+        """Every job the other plugins declared, in the order they initialised.
+
+        Read here rather than collected by the kernel: the contribution is the
+        contract, and consuming it is this capability's work (keepup-109).
+        """
+        contributions = getattr(self.plugin_manager, "contributions", None)
+        if contributions is None:
+            return []
+        return list(contributions.of("jobs"))
+
+    def _put(self, job) -> None:
+        """One declared job: `(job_id, callable, trigger)`, intervals by default."""
+        job_id, work, trigger = job
+        self.scheduler.add_job(work, id=job_id, trigger="interval", **trigger)
+
+    async def cleanup(self):
+        """Stop the scheduler with the application that started it."""
+        if self.scheduler is not None:
+            close_scheduler()
+            self.scheduler = None
+
+    def get_declared_tables(self) -> List[Any]:
+        """The lock table: the service published here is the one that uses it.
+
+        Declared since keepup-158, when the 0.3.0 path that created it went
+        away. A deployment that assembles its runtime from plugins had no
+        `distributed_locks` table and failed on its first background sweep.
+        """
+        from keepup_tasks.tables import DISTRIBUTED_LOCKS
+
+        return [DISTRIBUTED_LOCKS]
+
+    def get_api_routes(self):
+        """The section's three doors, declared as data for the transport.
+
+        The application registers the same three through the bridge until
+        `keepup-http` mounts what a plugin declares (keepup-121); both answer
+        with the payload functions of `keepup_tasks.scheduler`, so there is one
+        implementation of each answer.
+        """
+        return [
+            {"path": "/api/admin/scheduler/jobs", "methods": ["GET"],
+             "handler": self.jobs},
+            {"path": "/api/admin/scheduler/jobs/{job_id}/trigger", "methods": ["POST"],
+             "handler": self.trigger},
+            {"path": "/api/admin/scheduler/stats", "methods": ["GET"],
+             "handler": self.stats},
+            {"path": "/api/admin/scheduler/restart", "methods": ["POST"],
+             "handler": self.restart},
+        ]
+
+    async def jobs(self, current_user=None):
+        return jobs_payload()
+
+    async def trigger(self, job_id: str, current_user=None):
+        return trigger_job(job_id)
+
+    async def stats(self, current_user=None):
+        return stats_payload()
+
+    async def restart(self, current_user=None):
+        return restart_scheduler()
+
+    def get_panel_sections(self):
+        """The jobs and the locks, as the panel's section."""
+        return [dict(SECTION)]
+
+    def get_handlers(self):
+        return {"scheduler": self.scheduler}

keepup_tasks-0.5.0/keepup_tasks/scheduler.py

@@ -0,0 +1,181 @@
+"""One scheduler for the whole process.
+
+Plugins are meant to register their jobs against it rather than each starting
+a scheduler of its own, which is what keeps the jobs visible in one place -- and
+what lets an administrator see and trigger them. A plugin that does need its
+own takes it from new_scheduler(), so it gets the same job defaults. Running the
+same job in every replica is prevented one level up, by the distributed lock
+around the job itself, not here.
+
+The job defaults are what APScheduler leaves open and a replica set cannot
+leave open (keepup-46):
+
+- ``coalesce``: a job that missed several runs -- the process was paused, the
+  loop was busy, the machine slept -- runs once on waking, not once per missed
+  slot. Without it a sync job that fell an hour behind fires sixty times in a
+  row, on every replica that woke up.
+- ``max_instances = 1``: a run that has not finished is not started again
+  beside itself. Two passes of the same job in one process would contend for
+  the same distributed lock and one of them would only log that it lost.
+- ``misfire_grace_time``: how late a run may still start. APScheduler's default
+  is one second, so a loop held for longer than that silently drops the run.
+"""
+
+import asyncio
+import logging
+from typing import Any, Dict, Optional
+
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from fastapi import Depends, HTTPException
+
+from keepup import clock
+from keepup_auth.dependencies import get_current_admin
+
+#: What an application may import from this module. Everything else is
+#: internal and may change without notice -- see doc/keepup.md.
+__all__ = [
+    "JOB_DEFAULTS",
+    "init_scheduler",
+    "new_scheduler",
+]
+
+logger = logging.getLogger(__name__)
+
+#: The process-wide scheduler, created by init_scheduler().
+scheduler = None
+
+#: The job defaults the application named, kept so a restart starts the same way.
+_job_defaults: Optional[Dict[str, Any]] = None
+
+
+def get_scheduler():
+    """Return the process-wide scheduler, or None before it is created."""
+    return scheduler
+
+
+#: The job defaults every scheduler of the framework starts with; an
+#: application overrides them with ``KeepupSettings.scheduler_job_defaults``.
+JOB_DEFAULTS = {
+    "coalesce": True,
+    "max_instances": 1,
+    "misfire_grace_time": 60,
+}
+
+
+def new_scheduler(job_defaults: Optional[Dict[str, Any]] = None) -> AsyncIOScheduler:
+    """A scheduler with the framework's job defaults, overridden by ``job_defaults``."""
+    return AsyncIOScheduler(job_defaults={**JOB_DEFAULTS, **(job_defaults or {})})
+
+
+def init_scheduler(job_defaults: Optional[Dict[str, Any]] = None):
+    """Create and configure the task scheduler."""
+    global scheduler, _job_defaults
+
+    _job_defaults = job_defaults
+    scheduler = new_scheduler(job_defaults)
+
+    logger.info("Global scheduler initialized - tasks will be registered by plugins")
+    return scheduler
+
+
+def restart_scheduler() -> Dict[str, Any]:
+    """Stop the scheduler and start a fresh one holding the same jobs.
+
+    What the panel's button asks for: the jobs are the contract, and the clock
+    that runs them is replaceable. A job is re-added by its function, id and
+    trigger, so nothing has to be declared twice (keepup-109).
+    """
+    global scheduler
+
+    previous = scheduler
+    declared = []
+    if previous is not None:
+        declared = [{"func": job.func, "id": job.id, "name": job.name,
+                     "trigger": job.trigger}
+                    for job in previous.get_jobs()]
+        previous.shutdown(wait=False)
+
+    fresh = init_scheduler(_job_defaults)
+    for spec in declared:
+        fresh.add_job(spec["func"], id=spec["id"], name=spec["name"], trigger=spec["trigger"])
+    fresh.start()
+    logger.info("Scheduler restarted with %s job(s)", len(fresh.get_jobs()))
+    return {"restarted": True, "jobs": [job.id for job in fresh.get_jobs()]}
+
+
+def close_scheduler() -> None:
+    """Stop the process-wide scheduler and forget it.
+
+    Called by the tasks capability when its application goes down: the name is
+    the process's, and a scheduler that stopped is not one to answer a route
+    with (keepup-109).
+    """
+    global scheduler
+
+    if scheduler is not None:
+        scheduler.shutdown()
+        scheduler = None
+
+
+def jobs_payload() -> Dict[str, Any]:
+    """Every job the scheduler holds, as the panel's section reads them."""
+    if not scheduler:
+        return {"error": "Scheduler not initialized"}
+    return {"jobs": [{"id": job.id, "name": job.name,
+                      "next_run_time": job.next_run_time,
+                      "trigger": str(job.trigger)}
+                     for job in scheduler.get_jobs()]}
+
+
+def stats_payload() -> Dict[str, Any]:
+    """How the scheduler is doing, as the panel's section reads it."""
+    if not scheduler:
+        return {"error": "Scheduler not initialized"}
+    jobs = scheduler.get_jobs()
+    next_run_times = [job.next_run_time for job in jobs if job.next_run_time]
+    return {
+        "scheduler_running": scheduler.running,
+        "total_jobs": len(jobs),
+        "next_wakeup": min(next_run_times) if next_run_times else None,
+        "pending_tasks": len(asyncio.all_tasks()) if asyncio.get_event_loop() else 0,
+    }
+
+
+def trigger_job(job_id: str) -> Dict[str, Any]:
+    """Run one job by hand, now; refuses a job nobody declared."""
+    if not scheduler:
+        raise HTTPException(status_code=500, detail="Scheduler not initialized")
+    job = scheduler.get_job(job_id)
+    if not job:
+        raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
+    job.modify(next_run_time=clock.utcnow())
+    return {"success": True, "message": f"Job {job_id} triggered manually"}
+
+
+def register_scheduler_routes(app):
+    """Register the scheduler endpoints on the application.
+
+    The application's bridge until the transport mounts what a plugin declares
+    (keepup-121); the plugin contributes the same three routes as data, and both
+    answer with the payload functions above.
+    """
+
+    @app.get("/api/admin/scheduler/jobs")
+    async def get_scheduler_jobs(admin: dict = Depends(get_current_admin)):
+        """Return the scheduler's jobs (administrators only)."""
+        return jobs_payload()
+
+    @app.post("/api/admin/scheduler/jobs/{job_id}/trigger")
+    async def trigger_scheduler_job(job_id: str, admin: dict = Depends(get_current_admin)):
+        """Run a scheduler job by hand (administrators only)."""
+        return trigger_job(job_id)
+
+    @app.get("/api/admin/scheduler/stats")
+    async def get_scheduler_stats(admin: dict = Depends(get_current_admin)):
+        """Return scheduler statistics (administrators only)."""
+        return stats_payload()
+
+    @app.post("/api/admin/scheduler/restart")
+    async def restart_the_scheduler(admin: dict = Depends(get_current_admin)):
+        """Stop the scheduler and start a fresh one with the same jobs."""
+        return restart_scheduler()

keepup_tasks-0.5.0/keepup_tasks/tables.py

@@ -0,0 +1,28 @@
+"""The table the locks service keeps its rows in.
+
+Task keepup-158. The declaration used to live in `keepup_db.schema`, on the
+0.3.0 path an application called by hand -- which meant a runtime assembled from
+plugins, the way the kernel does it, created no `distributed_locks` table at all:
+a project built that way served its panel and then reported `no such table:
+distributed_locks` on every background sweep.
+
+The table belongs to the capability that provides the service that uses it.
+`keepup.locks` reaches it with plain SQL, so the declaration here is the only
+thing that has to exist.
+"""
+
+from sqlalchemy import Column, DateTime, Text
+
+from keepup_db import tables
+
+__all__ = ["DISTRIBUTED_LOCKS"]
+
+
+DISTRIBUTED_LOCKS = tables.table(
+    "distributed_locks",
+    # A text key written as ``TEXT PRIMARY KEY``: SQLite leaves such a column
+    # nullable, and the declaration keeps it that way.
+    Column("lock_name", Text, primary_key=True, nullable=True),
+    Column("acquired_at", DateTime, nullable=False),
+    Column("instance_id", Text, nullable=False),
+)

keepup_tasks-0.5.0/keepup_tasks.egg-info/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: keepup-tasks
+Version: 0.5.0
+Summary: The one scheduler, the declared jobs and the distributed locks they run under.
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: keepup-admin>=0.4.0
+Requires-Dist: keepup-db>=0.4.0
+Requires-Dist: keepup-auth>=0.4.0
+Requires-Dist: apscheduler<4,>=3.10
+Requires-Dist: fastapi<1,>=0.128
+
+# keepup-tasks
+
+The one scheduler, the declared jobs and the distributed locks they run under.
+
+One capability of [keepup](https://github.com/FedorSergeev/keepup-admin), the framework over FastAPI for admin
+panels: the base is installed as `keepup-admin` and imported as `keepup`, and
+this distribution carries one part of it. It is installed on its own
+(`pip install keepup-tasks`) or together with the base (`pip install "keepup-admin[panel]"`),
+and it declares itself to the kernel through its
+`[project.entry-points."keepup.plugins"]` entry point.

keepup_tasks-0.5.0/keepup_tasks.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+README.md
+pyproject.toml
+keepup_tasks/__init__.py
+keepup_tasks/plugin.py
+keepup_tasks/scheduler.py
+keepup_tasks/tables.py
+keepup_tasks.egg-info/PKG-INFO
+keepup_tasks.egg-info/SOURCES.txt
+keepup_tasks.egg-info/dependency_links.txt
+keepup_tasks.egg-info/entry_points.txt
+keepup_tasks.egg-info/requires.txt
+keepup_tasks.egg-info/top_level.txt

keepup_tasks-0.5.0/keepup_tasks.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

keepup_tasks-0.5.0/keepup_tasks.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[keepup.plugins]
+tasks = keepup_tasks:TasksPlugin

keepup_tasks-0.5.0/keepup_tasks.egg-info/requires.txt

@@ -0,0 +1,5 @@
+keepup-admin>=0.4.0
+keepup-db>=0.4.0
+keepup-auth>=0.4.0
+apscheduler<4,>=3.10
+fastapi<1,>=0.128

keepup_tasks-0.5.0/keepup_tasks.egg-info/top_level.txt

@@ -0,0 +1 @@
+keepup_tasks

keepup_tasks-0.5.0/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "keepup-tasks"
+version = "0.5.0"
+description = "The one scheduler, the declared jobs and the distributed locks they run under."
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "keepup-admin>=0.4.0",
+    "keepup-db>=0.4.0",
+    "keepup-auth>=0.4.0",
+    "apscheduler>=3.10,<4",
+    "fastapi>=0.128,<1",
+]
+
+[project.entry-points."keepup.plugins"]
+tasks = "keepup_tasks:TasksPlugin"
+
+[tool.setuptools]
+packages = ["keepup_tasks"]

keepup_tasks-0.5.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+