velocity-python 0.1.105__tar.gz → 0.1.106__tar.gz

{velocity_python-0.1.105 → velocity_python-0.1.106}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: velocity-python
-Version: 0.1.105
+Version: 0.1.106
 Summary: A rapid application development library for interfacing with data storage
 Author-email: Velocity Team <info@codeclubs.org>
 License-Expression: MIT

{velocity_python-0.1.105 → velocity_python-0.1.106}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "velocity-python"
-version = "0.1.105"
+version = "0.1.106"
 authors = [
   { name="Velocity Team", email="info@codeclubs.org" },
 ]

{velocity_python-0.1.105 → velocity_python-0.1.106}/src/velocity/__init__.py

@@ -1,4 +1,4 @@
-__version__ = version = "0.1.105"
+__version__ = version = "0.1.106"
 
 import importlib as _importlib
 

velocity_python-0.1.106/src/velocity/aws/capabilities/__init__.py

@@ -0,0 +1,43 @@
+"""System-level capabilities a site turns on: declared once in code, toggled
+per site/environment by the control plane, materialised by
+:func:`provisioning.provision` at deploy time and run by
+:class:`dispatch.CapabilityDispatcher` at runtime. See :mod:`base`.
+
+Importing this package registers the built-in capabilities.
+"""
+
+from .base import (  # noqa: F401
+    EVERY_15_MINUTES,
+    HOURLY,
+    NIGHTLY,
+    REGISTRY,
+    Capability,
+    capability,
+    get,
+    resolve_config,
+)
+from .dispatch import CapabilityDispatcher, run_capability  # noqa: F401
+from .provisioning import provision, trigger_key  # noqa: F401
+
+# built-ins (importing registers them)
+from . import retention  # noqa: E402,F401
+from . import security  # noqa: E402,F401
+from . import infra  # noqa: E402,F401
+from . import single_session  # noqa: E402,F401
+from . import jobs  # noqa: E402,F401
+from .security import SecurityMixin  # noqa: E402,F401
+
+__all__ = [
+    "Capability",
+    "CapabilityDispatcher",
+    "REGISTRY",
+    "capability",
+    "get",
+    "provision",
+    "resolve_config",
+    "run_capability",
+    "trigger_key",
+    "NIGHTLY",
+    "HOURLY",
+    "EVERY_15_MINUTES",
+]

velocity_python-0.1.106/src/velocity/aws/capabilities/base.py

@@ -0,0 +1,79 @@
+"""Capabilities: self-describing system-level features a site can turn on.
+
+A whole class of features has the same shape — some schema, a recurring
+schedule, a handler, a few config knobs, maybe a handler mixin: retention
+pruning, session reaping, the feature-flag mirror, mail harvest, demo-safety
+scrubs, ops metrics. A :class:`Capability` declares all of that in ONE object,
+so the deploy pipeline can materialise it (see :mod:`provisioning`) and the
+queue handler can run it (see :mod:`dispatch`) without a bespoke migration or
+wiring per feature.
+
+Division of labour:
+
+* **code** (a subclass registered with :func:`capability`) says what the
+  feature *is* — its schema, default cadence, default knobs, and the work;
+* the **control plane** (the site registry) says which sites/environments run
+  it and with what policy, delivered to the site as an entry that
+  :func:`provisioning.provision` materialises.
+
+Config resolution (:func:`resolve_config`): the capability's ``defaults`` are
+the floor; the environment's feature-flag row overrides the knobs the
+capability declares (so they are edited on the Feature Flags page and mirrored
+between databases); the site's registry ``config`` overrides everything (the
+per-site policy — e.g. *which* tables retention may touch).
+"""
+
+from __future__ import annotations
+
+# cron expressions the scheduler (croniter) accepts
+NIGHTLY = "0 3 * * *"
+HOURLY = "0 * * * *"
+EVERY_15_MINUTES = "*/15 * * * *"
+
+REGISTRY: dict = {}
+
+
+class Capability:
+    name: str = ""
+    schema: dict = {}  # {table: {column: sql type}} -> ensure_schema
+    defaults: dict = {}  # flag name -> default; seeded as TEXT feature flags
+    schedules: tuple = ()  # ((action, cron), ...)
+    mixins: tuple = ()  # handler mixins the site attaches at runtime
+    applies_to: tuple = ("demo", "production")  # default stages; registry can force
+
+    def handle(self, action, tx, context, config):
+        """Do the work for one scheduled/queued ``action``. Return a JSON-able
+        summary (it lands in the queue response and the job log)."""
+        raise NotImplementedError(f"{self.name}: no handler for action {action!r}")
+
+    def on_refresh(self, tx, config):
+        """Hook run after a demo refresh (optional) — scrub/neutralise here."""
+        return None
+
+
+def capability(cls):
+    """Class decorator: instantiate and register under ``cls.name``."""
+    if not getattr(cls, "name", ""):
+        raise ValueError(f"{cls.__name__} must set a non-empty `name`")
+    REGISTRY[cls.name] = cls()
+    return cls
+
+
+def get(name):
+    try:
+        return REGISTRY[name]
+    except KeyError:
+        known = ", ".join(sorted(REGISTRY)) or "(none registered)"
+        raise KeyError(f"unknown capability {name!r}; registered: {known}") from None
+
+
+def resolve_config(cap, flags=None, row_config=None) -> dict:
+    """defaults <- feature flags (declared knobs only) <- site registry config."""
+    config = dict(cap.defaults)
+    flags = flags or {}
+    for key in cap.defaults:
+        value = flags.get(key)
+        if value not in (None, ""):
+            config[key] = value
+    config.update(row_config or {})
+    return config

velocity_python-0.1.106/src/velocity/aws/capabilities/dispatch.py

@@ -0,0 +1,50 @@
+"""Runtime dispatch: route the ``run-capability`` queue action to a capability.
+
+A site's queue handler mixes in :class:`CapabilityDispatcher`; the scheduler
+stubs seeded by :mod:`provision` enqueue ``run-capability`` with
+``{"capability": name, "action": action}`` and this routes it. Config is
+resolved locally — the site's own ``sys_capabilities`` row plus its feature
+flags — so the runtime never reaches into the control plane.
+"""
+
+from __future__ import annotations
+
+import json
+
+from velocity.aws import feature_flags
+
+from .base import REGISTRY, resolve_config
+from .provisioning import SITE_TABLE
+
+
+def run_capability(tx, context, name, action):
+    """Resolve config and run one capability action. Returns the response body."""
+    cap = REGISTRY.get(name)
+    if cap is None:
+        return {"error": f"unknown capability {name!r}", "capability": name}
+
+    row = tx.table(SITE_TABLE).find({"name": name}) or {}
+    if row and row.get("enabled") is False:
+        return {
+            "capability": name,
+            "action": action,
+            "skipped": "disabled for this site",
+        }
+
+    raw = row.get("config") if row else None
+    row_config = json.loads(raw) if isinstance(raw, str) and raw else (raw or {})
+    config = resolve_config(cap, flags=feature_flags.get(tx), row_config=row_config)
+
+    result = cap.handle(action, tx, context, config)
+    return {"capability": name, "action": action, "result": result}
+
+
+class CapabilityDispatcher:
+    """Queue-handler mixin: ``run-capability`` -> the registered capability."""
+
+    def OnActionRunCapability(self, tx, context):
+        payload = context.payload() or {}
+        body = run_capability(
+            tx, context, payload.get("capability"), payload.get("action")
+        )
+        context.response().set_body(body)

velocity_python-0.1.106/src/velocity/aws/capabilities/infra.py

@@ -0,0 +1,43 @@
+"""Infra-tier capabilities: rendered into a site's CloudFormation stacks by
+the sitefactory templates rather than run by a Lambda.
+
+They are registered here so the registry recognises their names (the
+provisioner would otherwise refuse an unknown capability) and so they sit
+on the same Site Capabilities page as everything else; their ``config`` is
+the template's policy. Nothing here executes at runtime.
+
+* ``security_headers`` — the CloudFront response-headers policy on the web
+  distribution (HSTS, nosniff, frame options, referrer policy, XSS-off, and
+  an optional ``csp``). Free; ON by default for every factory site — a row
+  is only needed to turn it OFF or to set a Content-Security-Policy.
+* ``api_throttling`` — API Gateway stage throttling (``rate`` requests/s,
+  ``burst``). Free; the template defaults apply without a row.
+"""
+
+from __future__ import annotations
+
+from .base import Capability, capability
+
+
+class InfraCapability(Capability):
+    schedules = ()
+    schema = {}
+
+    def handle(self, action, tx, context, config):
+        raise RuntimeError(
+            f"{self.name} is an infra capability rendered into the site's stacks "
+            f"by sitefactory; it has no runtime action ({action!r})"
+        )
+
+
+@capability
+class SecurityHeaders(InfraCapability):
+    name = "security_headers"
+    # policy lives in the row's config: {"csp": "...", "frame_options": "DENY"}
+    defaults = {}
+
+
+@capability
+class ApiThrottling(InfraCapability):
+    name = "api_throttling"
+    defaults = {}  # {"rate": 100, "burst": 200}

velocity_python-0.1.106/src/velocity/aws/capabilities/jobs.py

@@ -0,0 +1,53 @@
+"""jobs: the scheduled half of asynchronous-job hygiene.
+
+``JobHandler`` only discovers a stale job (worker died: Lambda timeout / OOM)
+when someone polls its status — an abandoned job nobody polls stays "Running"
+forever, and ``aws_job_activity`` is never pruned. This capability sweeps on
+a schedule with the SAME predicate the web tier uses
+(``velocity.aws.job_lifecycle.job_is_stale``): stale jobs are persisted as
+Failed with the standard message, and old activity rows are purged.
+"""
+
+from __future__ import annotations
+
+import datetime
+import json
+
+from velocity.aws.job_lifecycle import (
+    JOB_ACTIVITY_TABLE,
+    STALE_FAILURE,
+    STALE_JOB_STATUSES,
+    job_is_stale,
+)
+
+from .base import EVERY_15_MINUTES, Capability, capability
+
+
+@capability
+class Jobs(Capability):
+    name = "jobs"
+    defaults = {"job_stale_minutes": "16", "job_retention_days": "30"}
+    schedules = (("expire_stale", EVERY_15_MINUTES),)
+
+    def handle(self, action, tx, context, config):
+        if action != "expire_stale":
+            raise ValueError(f"jobs: unknown action {action!r}")
+        now = datetime.datetime.now(datetime.timezone.utc)
+        stale_after = datetime.timedelta(
+            minutes=int(config.get("job_stale_minutes") or 16)
+        )
+        jobs = tx.table(JOB_ACTIVITY_TABLE)
+
+        expired = 0
+        for status in sorted(STALE_JOB_STATUSES):
+            for job in jobs.select(where={"status": status}):
+                if job_is_stale(job, now=now, stale_after=stale_after):
+                    jobs.update(
+                        {"status": "Failed", "message": json.dumps(STALE_FAILURE)},
+                        where={"sys_id": job["sys_id"]},
+                    )
+                    expired += 1
+
+        days = int(config.get("job_retention_days") or 30)
+        purged = jobs.purge({"<sys_created": now - datetime.timedelta(days=days)})
+        return {"expired": expired, "purged": purged}

velocity_python-0.1.106/src/velocity/aws/capabilities/provisioning.py

@@ -0,0 +1,180 @@
+"""Deploy-time provisioning: materialise a site's enabled capabilities.
+
+Called by the sitefactory ``provision-capabilities`` step (part of
+``deploy-backend``) with the ``capabilities`` block of ``site.json``, which is
+the site registry rendered for one environment::
+
+    [{"name": "retention", "enabled": True, "config": {"tables": {...}}}, ...]
+
+For each entry this (1) records it in the site's own ``sys_capabilities``
+table — the runtime reads config from there, never from the control plane —
+(2) ensures the capability's schema, (3) seeds its feature-flag knobs (filling
+only what is unset, never clobbering an edit), and (4) upserts one
+``sys_task_triggers`` row + one ``program_objects`` stub per schedule.
+
+Everything is keyed deterministically (``cap:<name>:<action>``), so running it
+on every deploy is idempotent — and that is the point: a demo refresh wipes
+the triggers, the next deploy puts them back. A disabled entry disables its
+triggers rather than deleting them, so the ledger of runs survives.
+"""
+
+from __future__ import annotations
+
+import datetime
+import json
+
+from velocity.aws import feature_flags
+
+from .base import get
+
+SITE_TABLE = "sys_capabilities"  # per-site materialised registry
+TRIGGERS_TABLE = "sys_task_triggers"
+PROGRAMS_TABLE = "program_objects"
+FLAGS_TABLE = "sys_feature_flags"
+RUN_ACTION = "run-capability"
+SEEDED_BY = "provision-capabilities"
+EXECUTION_ENVIRONMENT = "SQS Event"
+
+# The scheduler runs program source with exec() and only {tx, handler,
+# context, attrs, postdata} in scope — so the stub is a single enqueue and the
+# real logic stays in git-tracked capability code.
+PROGRAM_SOURCE = (
+    "result = context.enqueue(action=%r, payload={%r: %r, %r: %r}, user=%r)\n"
+    'print("Enqueued", %r, result.get("batch_id"))\n'
+)
+
+
+def trigger_key(cap_name: str, action: str) -> str:
+    return f"cap:{cap_name}:{action}"
+
+
+def program_source(cap_name: str, action: str) -> str:
+    return PROGRAM_SOURCE % (
+        RUN_ACTION,
+        "capability",
+        cap_name,
+        "action",
+        action,
+        "Scheduler",
+        RUN_ACTION,
+    )
+
+
+def provision(tx, stage: str, entries, *, now=None) -> list:
+    """Materialise ``entries`` for ``stage``. Returns [(name, outcome), ...]."""
+    now = now or datetime.datetime.utcnow()
+    tx.table(SITE_TABLE).ensure_schema(
+        {"name": "TEXT", "enabled": "BOOLEAN", "config": "TEXT"}
+    )
+    report = []
+    for entry in entries or []:
+        name = entry["name"]
+        cap = get(name)
+        enabled = bool(entry.get("enabled", True))
+        applies = stage in cap.applies_to or bool(entry.get("force"))
+        active = enabled and applies
+
+        tx.table(SITE_TABLE).merge(
+            {
+                "name": name,
+                "enabled": active,
+                "config": json.dumps(entry.get("config") or {}),
+                "sys_modified_by": SEEDED_BY,
+            },
+            pk={"name": name},
+        )
+        if not active:
+            _set_schedules_enabled(tx, cap, False)
+            report.append((name, "disabled" if not enabled else f"not for {stage}"))
+            continue
+
+        for table, columns in cap.schema.items():
+            tx.table(table).ensure_schema(columns)
+
+        if cap.defaults:
+            _seed_flag_defaults(tx, stage, cap)
+
+        for action, cron in cap.schedules:
+            _upsert_schedule(tx, cap, action, cron, now)
+
+        report.append((name, "provisioned"))
+    return report
+
+
+def _seed_flag_defaults(tx, stage, cap):
+    """Ensure the knob columns exist and fill only the ones that are unset."""
+    tx.table(FLAGS_TABLE).ensure_schema({key: "TEXT" for key in cap.defaults})
+    row = feature_flags.get(tx, stage)
+    if not row:
+        return  # the app's own seed creates the stage row; nothing to fill yet
+    missing = {
+        key: str(value)
+        for key, value in cap.defaults.items()
+        if row.get(key) in (None, "")
+    }
+    if missing:
+        tx.table(FLAGS_TABLE).update(
+            missing, where={feature_flags.SYSTEM_COLUMN: stage}
+        )
+
+
+def _upsert_schedule(tx, cap, action, cron, now):
+    key = trigger_key(cap.name, action)
+    triggers = tx.table(TRIGGERS_TABLE)
+    if triggers.find({"trigger_name": key}):
+        # keep next_run / last_run / run_count — only the definition refreshes
+        triggers.update(
+            {"cron_expression": cron, "enabled": True, "sys_modified_by": SEEDED_BY},
+            where={"trigger_name": key},
+        )
+    else:
+        triggers.insert(
+            {
+                "name": f"{cap.name}: {action}",
+                "description": f"Capability '{cap.name}' -> '{action}' ({cron}).",
+                "trigger_name": key,
+                "cron_expression": cron,
+                # due immediately: backdated a day so it fires on the next tick
+                # regardless of the session time zone (next_run is zone-less)
+                "next_run": now - datetime.timedelta(days=1),
+                "enabled": True,
+                "run_count": 0,
+                "sys_modified_by": SEEDED_BY,
+            }
+        )
+
+    source = program_source(cap.name, action)
+    row = {
+        "program_name": f"{cap.name}: {action}",
+        "program_handle": key,
+        "program_description": (
+            f"Enqueues '{RUN_ACTION}' for capability '{cap.name}' action "
+            f"'{action}'. Provisioned by sitefactory; do not hand-edit."
+        ),
+        "source_code": source,
+        "tags": f"capability,{cap.name}",
+        "execution_environment": EXECUTION_ENVIRONMENT,
+        "trigger": key,
+        "last_saved_date": now,
+        "num_lines": len(source.splitlines()),
+        "num_chars": len(source),
+        "version": 1,
+        "sys_modified_by": SEEDED_BY,
+    }
+    programs = tx.table(PROGRAMS_TABLE)
+    where = {"execution_environment": EXECUTION_ENVIRONMENT, "trigger": key}
+    if programs.find(where):
+        programs.update(row, where=where)
+    else:
+        programs.insert(row)
+
+
+def _set_schedules_enabled(tx, cap, enabled: bool):
+    triggers = tx.table(TRIGGERS_TABLE)
+    for action, _cron in cap.schedules:
+        key = trigger_key(cap.name, action)
+        if triggers.find({"trigger_name": key}):
+            triggers.update(
+                {"enabled": enabled, "sys_modified_by": SEEDED_BY},
+                where={"trigger_name": key},
+            )

velocity_python-0.1.106/src/velocity/aws/capabilities/retention.py

@@ -0,0 +1,65 @@
+"""Retention: delete rows older than a per-table window.
+
+The *policy* — which tables, how many days — is the site's registry config,
+never guessed here; this capability only knows how to purge safely. A big
+table is not necessarily a prunable one (an email archive is not a log), so
+nothing is touched unless it is named in ``config["tables"]``::
+
+    {"tables": {"aws_api_activity": 90, "sys_log_email": 30, "filebuffer": 14}}
+
+A window of ``null``/``0`` in that map falls back to the ``purge_default_days``
+feature flag. The delete goes through velocity's :meth:`Table.purge` — batch-
+bounded and committed per batch, so a run stays inside the Lambda budget and
+never holds a long lock; an unfinished table just continues next run.
+"""
+
+from __future__ import annotations
+
+import datetime
+import re
+
+from .base import NIGHTLY, Capability, capability
+
+IDENTIFIER = re.compile(r"^[a-z_][a-z0-9_]*$")
+TIMESTAMP_COLUMN = "sys_created"
+BATCH_SIZE = 5000
+MAX_BATCHES = 200  # 1M rows per run — well inside the Lambda budget
+
+
+@capability
+class Retention(Capability):
+    name = "retention"
+    defaults = {"purge_default_days": "90"}
+    schedules = (("purge", NIGHTLY),)
+
+    def handle(self, action, tx, context, config):
+        if action != "purge":
+            raise ValueError(f"retention: unknown action {action!r}")
+        default_days = int(config.get("purge_default_days") or 90)
+        results = {}
+        for table, days in (config.get("tables") or {}).items():
+            results[table] = purge_table(tx, table, int(days or default_days))
+        return results
+
+
+def purge_table(
+    tx, table, days, *, batch=BATCH_SIZE, max_batches=MAX_BATCHES, now=None
+):
+    """Delete rows of ``table`` whose ``sys_created`` is older than ``days``."""
+    if not IDENTIFIER.match(str(table)):
+        raise ValueError(f"retention: refusing unsafe table name {table!r}")
+    if days < 1:
+        raise ValueError(f"retention: window must be >= 1 day for {table!r}")
+
+    target = tx.table(table)
+    if not target.exists():
+        return {"skipped": "table does not exist"}
+    if TIMESTAMP_COLUMN not in target.sys_columns():
+        return {"skipped": f"no {TIMESTAMP_COLUMN} column"}
+
+    now = now or datetime.datetime.now(datetime.timezone.utc)
+    cutoff = now - datetime.timedelta(days=days)
+    deleted = target.purge(
+        {f"<{TIMESTAMP_COLUMN}": cutoff}, batch=batch, max_batches=max_batches
+    )
+    return {"deleted": deleted, "days": days}