velocity-python 0.2.9__tar.gz → 0.3.0__tar.gz

{velocity_python-0.2.9 → velocity_python-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: velocity-python
-Version: 0.2.9
+Version: 0.3.0
 Summary: A rapid application development library for interfacing with data storage
 Author-email: Velocity Team <info@codeclubs.org>
 License-Expression: MIT
@@ -57,8 +57,10 @@ Requires-Dist: braintree<5,>=4.30.0; extra == "payment"
 Requires-Dist: xmltodict<2,>=0.13.0; extra == "payment"
 Provides-Extra: plaid
 Requires-Dist: ecdsa>=0.19.0; extra == "plaid"
+Provides-Extra: rules
+Requires-Dist: cel-python<0.6,>=0.5.0; extra == "rules"
 Provides-Extra: all
-Requires-Dist: velocity-python[aws,excel,http,payment,pdf,plaid,postgres,templates]; extra == "all"
+Requires-Dist: velocity-python[aws,excel,http,payment,pdf,plaid,postgres,rules,templates]; extra == "all"
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0.0; extra == "dev"
 Requires-Dist: pytest-cov>=6.0.0; extra == "dev"
@@ -81,6 +83,7 @@ Requires-Dist: xmltodict<2,>=0.13.0; extra == "test"
 Requires-Dist: authorizenet>=1.1.3; extra == "test"
 Requires-Dist: ecdsa>=0.19.0; extra == "test"
 Requires-Dist: openpyxl>=3.1.0; extra == "test"
+Requires-Dist: cel-python<0.6,>=0.5.0; extra == "test"
 Requires-Dist: redis>=5.0.0; extra == "test"
 Requires-Dist: fakeredis>=2.23.0; extra == "test"
 Requires-Dist: pymongo>=4.6.0; extra == "test"

{velocity_python-0.2.9 → velocity_python-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "velocity-python"
-version = "0.2.9"
+version = "0.3.0"
 authors = [
   { name="Velocity Team", email="info@codeclubs.org" },
 ]
@@ -101,8 +101,15 @@ payment = [
 plaid = [
   "ecdsa>=0.19.0",
 ]
+# Expression conditions for velocity.rules (CEL, pure Python; pulls lark,
+# pendulum and google-re2). Decision tables need nothing extra. Tested on
+# cel-python 0.5.x; raise the bound only after `make test-unit` passes on the
+# next minor (pre-1.0, so a minor may change the API).
+rules = [
+  "cel-python>=0.5.0,<0.6",
+]
 all = [
-  "velocity-python[postgres,aws,excel,templates,http,payment,plaid,pdf]",
+  "velocity-python[postgres,aws,excel,templates,http,payment,plaid,pdf,rules]",
 ]
 dev = [
   "pytest>=8.0.0",
@@ -130,6 +137,7 @@ test = [
   "authorizenet>=1.1.3",
   "ecdsa>=0.19.0",
   "openpyxl>=3.1.0",
+  "cel-python>=0.5.0,<0.6",
   # NoSQL adapter unit tests run against in-process fakes (real clients for
   # the wire formats, fakes for the servers). ES/Cassandra/Couchbase have
   # translation-only unit tests and need no driver here.

{velocity_python-0.2.9 → velocity_python-0.3.0}/src/velocity/__init__.py

@@ -1,4 +1,4 @@
-__version__ = version = "0.2.9"
+__version__ = version = "0.3.0"
 
 import importlib as _importlib
 

{velocity_python-0.2.9 → velocity_python-0.3.0}/src/velocity/aws/capabilities/__init__.py

@@ -13,10 +13,13 @@ from .base import (  # noqa: F401
     POLICY_KINDS,
     REGISTRY,
     Capability,
+    CapabilityDependencyError,
     capability,
     catalog,
+    dependency_order,
     describe,
     get,
+    missing_dependencies,
     load_config,
     policy_field,
     resolve_config,
@@ -30,6 +33,7 @@ from .dispatch import (  # noqa: F401
 )
 from .provisioning import (  # noqa: F401
     as_bool,
+    check_dependencies,
     provision,
     row_enabled,
     trigger_key,
@@ -47,15 +51,19 @@ from .security import SecurityMixin  # noqa: E402,F401
 __all__ = [
     "Capability",
     "CapabilityCatalogMixin",
+    "CapabilityDependencyError",
     "CapabilityDispatcher",
     "POLICY_KINDS",
     "REGISTRY",
     "as_bool",
     "capability",
     "catalog",
+    "check_dependencies",
+    "dependency_order",
     "describe",
     "get",
     "load_config",
+    "missing_dependencies",
     "policy_field",
     "provision",
     "resolve_config",

{velocity_python-0.2.9 → velocity_python-0.3.0}/src/velocity/aws/capabilities/base.py

@@ -72,6 +72,11 @@ COUNTRY_CODE = re.compile(r"^[A-Z]{2}$")
 REGISTRY: dict = {}
 
 
+class CapabilityDependencyError(ValueError):
+    """A capability is enabled without one it requires, requires an unknown
+    one, or the requirements form a cycle."""
+
+
 def policy_field(name, kind, label, help="", **extra) -> dict:
     """One entry of a capability's ``policy`` schema. ``extra`` carries
     kind-specific hints an editor may use (``options`` for a choice,
@@ -92,6 +97,7 @@ class Capability:
     mixins: tuple = ()  # handler mixins the site attaches at runtime
     applies_to: tuple = ("demo", "production")  # default stages; registry can force
     policy: tuple = ()  # (policy_field(...), ...) — the per-site config shape
+    requires_capabilities: tuple = ()  # names that must be enabled alongside
 
     def handle(self, action, tx, context, config):
         """Do the work for one scheduled/queued ``action``. Return a JSON-able
@@ -119,6 +125,61 @@ def get(name):
         raise KeyError(f"unknown capability {name!r}; registered: {known}") from None
 
 
+def requirements(name) -> tuple:
+    """The capabilities ``name`` declares it requires (unknown name: KeyError)."""
+    return tuple(getattr(get(name), "requires_capabilities", ()) or ())
+
+
+def dependency_order(names) -> list:
+    """``names`` plus everything they require (transitively), each after the
+    capabilities it requires; otherwise in the order given.
+
+    Raises :class:`CapabilityDependencyError` for a cycle (naming it) or a
+    requirement that is not a registered capability.
+    """
+    ordered: list = []
+    done: set = set()
+    visiting: list = []
+
+    def visit(name, needed_by=None):
+        if name in done:
+            return
+        if name in visiting:
+            cycle = visiting[visiting.index(name) :] + [name]
+            raise CapabilityDependencyError(
+                "capability requirements form a cycle: " + " -> ".join(cycle)
+            )
+        if name not in REGISTRY:
+            if needed_by is None:
+                get(name)  # the usual "unknown capability" KeyError
+            raise CapabilityDependencyError(
+                f"capability {needed_by!r} requires {name!r}, which is not registered"
+            )
+        visiting.append(name)
+        for required in requirements(name):
+            visit(required, name)
+        visiting.pop()
+        done.add(name)
+        ordered.append(name)
+
+    for name in names:
+        visit(name)
+    return ordered
+
+
+def missing_dependencies(enabled) -> dict:
+    """``{name: [required names not in enabled]}`` for each name in
+    ``enabled`` that lacks one; empty when every requirement is met."""
+    enabled = list(enabled)
+    present = set(enabled)
+    missing = {}
+    for name in enabled:
+        absent = [req for req in requirements(name) if req not in present]
+        if absent:
+            missing[name] = absent
+    return missing
+
+
 def resolve_config(cap, flags=None, row_config=None) -> dict:
     """defaults <- feature flags (declared knobs only) <- site registry config."""
     config = dict(cap.defaults)
@@ -148,6 +209,7 @@ def describe(cap) -> dict:
         ],
         "flags": list(cap.defaults),
         "policy": [dict(field) for field in cap.policy],
+        "requires": list(cap.requires_capabilities),
     }
 
 

velocity_python-0.3.0/src/velocity/aws/capabilities/dispatch.py

@@ -0,0 +1,124 @@
+"""Runtime dispatch: route the ``run-capability`` queue action to a capability.
+
+A site's queue handler mixes in :class:`CapabilityDispatcher`; the scheduler
+stubs seeded by :mod:`provision` enqueue ``run-capability`` with
+``{"capability": name, "action": action}`` and this routes it. Config is
+resolved locally — the site's own ``sys_capabilities`` row plus its feature
+flags — so the runtime never reaches into the control plane. A capability
+runs only with an enabled row of its own and enabled rows for every
+capability it requires (``requires_capabilities``, transitively); otherwise
+it is skipped and the skip is reported (:mod:`velocity.aws.failures`).
+"""
+
+from __future__ import annotations
+
+import json
+
+from velocity.aws import failures, feature_flags
+
+from .base import (
+    REGISTRY,
+    CapabilityDependencyError,
+    catalog,
+    dependency_order,
+    resolve_config,
+)
+from .provisioning import SITE_TABLE, row_enabled
+
+
+def _skip(tx, name, action, reason, *, report=True):
+    """The skipped response; with ``report``, a failure event (scrubbed,
+    never raising) once the transaction commits — a skip that should not
+    happen must reach an operator, not just a queue response."""
+    if report:
+
+        def send():
+            failures.report(
+                None,
+                None,
+                source="queue",
+                action="run-capability",
+                extra={
+                    "capability": name,
+                    "capability_action": action,
+                    "skipped": reason,
+                },
+            )
+
+        on_commit = getattr(tx, "on_commit", None)
+        if callable(on_commit):
+            on_commit(send)
+        else:
+            send()
+    return {"capability": name, "action": action, "skipped": reason}
+
+
+def run_capability(tx, context, name, action):
+    """Resolve config and run one capability action. Returns the response body.
+
+    Runs only when the site has an enabled ``sys_capabilities`` row for the
+    capability (provisioning wrote it) and an enabled row for every
+    capability it requires, directly or through another requirement
+    (:func:`dependency_order`). Otherwise the action is skipped, and — except
+    for a plain "disabled for this site" — reported through
+    :mod:`velocity.aws.failures`: a scheduled run for a capability that was
+    never provisioned, or whose requirement is off, is a misconfiguration.
+    """
+    cap = REGISTRY.get(name)
+    if cap is None:
+        return {"error": f"unknown capability {name!r}", "capability": name}
+
+    site = tx.table(SITE_TABLE)
+    row = site.find({"name": name})
+    if not row:
+        return _skip(tx, name, action, "not provisioned for this site")
+    if not row_enabled(row):
+        return _skip(tx, name, action, "disabled for this site", report=False)
+    try:
+        # every requirement, transitively (a -> b -> c needs c on too)
+        chain = [n for n in dependency_order([name]) if n != name]
+    except (CapabilityDependencyError, KeyError) as exc:
+        return _skip(tx, name, action, f"dependency error: {exc}")
+    missing = []
+    for required in chain:
+        required_row = site.find({"name": required})
+        if not required_row or not row_enabled(required_row):
+            missing.append(required)
+    if missing:
+        return _skip(
+            tx, name, action, f"requires {', '.join(missing)}, not enabled here"
+        )
+
+    raw = row.get("config")
+    row_config = json.loads(raw) if isinstance(raw, str) and raw else (raw or {})
+    config = resolve_config(cap, flags=feature_flags.get(tx), row_config=row_config)
+
+    result = cap.handle(action, tx, context, config)
+    return {"capability": name, "action": action, "result": result}
+
+
+class CapabilityDispatcher:
+    """Queue-handler mixin: ``run-capability`` -> the registered capability."""
+
+    def OnActionRunCapability(self, tx, context):
+        # Queue records and scheduled ticks only: a handler that mixes this in
+        # must not let an HTTP caller run a capability's actions.
+        from velocity.aws.dirty_pipeline import refuse_http
+
+        refuse_http(self, "run-capability")
+        payload = context.payload() or {}
+        body = run_capability(
+            tx, context, payload.get("capability"), payload.get("action")
+        )
+        context.response().set_body(body)
+
+
+class CapabilityCatalogMixin:
+    """Web-handler mixin for the control plane: ``capability-catalog`` answers
+    with every registered capability described (``{"rows": [...]}`` — the
+    shape velocity-react's ApiSelect and option hooks consume), so the admin
+    UI's pick-list and policy editors come from the code, not a copy."""
+
+    def OnActionCapabilityCatalog(self, tx, context):
+        del tx
+        context.response().set_body({"rows": catalog()})

{velocity_python-0.2.9 → velocity_python-0.3.0}/src/velocity/aws/capabilities/jobs.py

@@ -3,9 +3,11 @@
 ``JobHandler`` only discovers a stale job (worker died: Lambda timeout / OOM)
 when someone polls its status — an abandoned job nobody polls stays "Running"
 forever, and ``aws_job_activity`` is never pruned. This capability sweeps on
-a schedule with the SAME predicate the web tier uses
-(``velocity.aws.job_lifecycle.job_is_stale``): stale jobs are persisted as
-Failed with the standard message, and old activity rows are purged.
+a schedule with the same rule the web tier uses (``Running``/``Initialized``
+with no progress for the window), judged by the database clock
+(:func:`stale_jobs`): stale jobs are persisted as Failed with the standard
+message plus the status they expired from (``expired_from``), and old
+activity rows are purged.
 """
 
 from __future__ import annotations
@@ -17,7 +19,6 @@ from velocity.aws.job_lifecycle import (
     JOB_ACTIVITY_TABLE,
     STALE_FAILURE,
     STALE_JOB_STATUSES,
-    job_is_stale,
 )
 
 from .base import EVERY_15_MINUTES, Capability, capability
@@ -39,27 +40,51 @@ class Jobs(Capability):
         if action != "expire_stale":
             raise ValueError(f"jobs: unknown action {action!r}")
         now = datetime.datetime.now(datetime.timezone.utc)
-        stale_after = datetime.timedelta(
-            minutes=int(config.get("job_stale_minutes") or 16)
-        )
+        minutes = int(config.get("job_stale_minutes") or 16)
         jobs = tx.table(JOB_ACTIVITY_TABLE)
 
         expired = 0
-        for status in sorted(STALE_JOB_STATUSES):
-            for job in jobs.select(where={"status": status}):
-                if job_is_stale(job, now=now, stale_after=stale_after):
-                    jobs.update(
-                        {"status": "Failed", "message": json.dumps(STALE_FAILURE)},
-                        where={"sys_id": job["sys_id"]},
-                    )
-                    expired += 1
-                    _report_after_commit(tx, job, status)
+        for job in stale_jobs(tx, minutes):
+            status = job.get("status")
+            # the status in the where: a job that progressed since the select
+            # (or that another sweep expired) is left alone
+            changed = jobs.update(
+                {
+                    "status": "Failed",
+                    "message": json.dumps({**STALE_FAILURE, "expired_from": status}),
+                },
+                where={"sys_id": job["sys_id"], "status": status},
+            )
+            if changed:
+                expired += 1
+                _report_after_commit(tx, job, status)
 
         days = int(config.get("job_retention_days") or 30)
         purged = jobs.purge({"<sys_created": now - datetime.timedelta(days=days)})
         return {"expired": expired, "purged": purged}
 
 
+def stale_jobs(tx, minutes):
+    """``Running``/``Initialized`` jobs with no progress for ``minutes``,
+    judged by the database clock: ``sys_modified`` is written by the row
+    trigger in the session's time zone, so it is compared with
+    ``LOCALTIMESTAMP`` there, never with a Python clock (which disagrees with
+    it whenever the database does not run in UTC)."""
+    statuses = sorted(STALE_JOB_STATUSES)
+    placeholders = ", ".join(["%s"] * len(statuses))
+    return (
+        tx.execute(
+            f"SELECT sys_id, job_id, action, status FROM {JOB_ACTIVITY_TABLE} "
+            f"WHERE status IN ({placeholders}) "
+            "AND sys_modified < LOCALTIMESTAMP - %s::interval "
+            "ORDER BY sys_id",
+            [*statuses, f"{int(minutes)} minutes"],
+        )
+        .as_dict()
+        .all()
+    )
+
+
 def _report_after_commit(tx, job, status):
     """A job whose worker died never reached onError, so nothing reported it:
     publish a ``source="job"`` failure event once the sweep commits."""

{velocity_python-0.2.9 → velocity_python-0.3.0}/src/velocity/aws/capabilities/provisioning.py

@@ -12,6 +12,12 @@ table — the runtime reads config from there, never from the control plane —
 only what is unset, never clobbering an edit), and (4) upserts one
 ``sys_task_triggers`` row + one ``program_objects`` stub per schedule.
 
+Before anything is written, every capability this run leaves enabled must
+have the capabilities it requires (``requires_capabilities``) enabled too —
+in ``entries``, or (when ``entries`` does not name them) already enabled in
+the site's ``sys_capabilities`` — else :class:`CapabilityDependencyError`.
+Entries are then provisioned in dependency order (required ones first).
+
 Everything is keyed deterministically (``cap:<name>:<action>``), so running it
 on every deploy is idempotent — and that is the point: a demo refresh wipes
 the triggers, the next deploy puts them back. A disabled entry disables its
@@ -25,7 +31,13 @@ import json
 
 from velocity.aws import feature_flags
 
-from .base import get
+from .base import (
+    REGISTRY,
+    CapabilityDependencyError,
+    dependency_order,
+    get,
+    requirements,
+)
 
 SITE_TABLE = "sys_capabilities"  # per-site materialised registry
 TRIGGERS_TABLE = "sys_task_triggers"
@@ -202,17 +214,72 @@ def program_source(cap_name: str, action: str) -> str:
     )
 
 
+def _entry_active(entry, stage) -> bool:
+    cap = get(entry["name"])
+    enabled = as_bool(entry.get("enabled", True), default=False)
+    return enabled and (stage in cap.applies_to or bool(entry.get("force")))
+
+
+def check_dependencies(tx, stage: str, entries) -> list:
+    """The entries in dependency order, after checking that every entry this
+    run leaves active has what it requires active too. A requirement
+    ``entries`` does not mention is looked up in the site's own
+    ``sys_capabilities`` (an earlier provision); one ``entries`` disables or
+    keeps out of ``stage`` is missing. Raises
+    :class:`CapabilityDependencyError` listing every gap, before any write."""
+    entries = list(entries or [])
+    by_name = {entry["name"]: entry for entry in entries}
+    active = {name for name, entry in by_name.items() if _entry_active(entry, stage)}
+    order = dependency_order(list(by_name))  # unknown names / cycles raise here
+    site = tx.table(SITE_TABLE)
+    problems = []
+    for name in order:
+        if name not in active:
+            continue
+        for required in requirements(name):
+            if required in active:
+                continue
+            if required in by_name:
+                entry = by_name[required]
+                on = as_bool(entry.get("enabled", True), default=False)
+                why = f"not for {stage}" if on else "disabled"
+            else:
+                row = site.find({"name": required})
+                if row and row_enabled(row):
+                    continue
+                why = "not enabled for this site"
+            problems.append(f"{name} requires {required} ({why})")
+    # turning one off must not strand a capability left on from before
+    for name in by_name:
+        if name in active:
+            continue
+        for other, cap in sorted(REGISTRY.items()):
+            if other in by_name or name not in (cap.requires_capabilities or ()):
+                continue
+            row = site.find({"name": other})
+            if row and row_enabled(row):
+                problems.append(
+                    f"{other} requires {name}, which this run turns off "
+                    f"(disable {other} too, or keep {name})"
+                )
+    if problems:
+        raise CapabilityDependencyError(
+            "capability dependencies not met: " + "; ".join(problems)
+        )
+    return [by_name[name] for name in order if name in by_name]
+
+
 def provision(tx, stage: str, entries, *, now=None) -> list:
-    """Materialise ``entries`` for ``stage``. Returns [(name, outcome), ...]."""
+    """Materialise ``entries`` for ``stage``. Returns [(name, outcome), ...]
+    in dependency order (required capabilities first)."""
     now = now or datetime.datetime.utcnow()
     tx.table(SITE_TABLE).ensure_schema(**SCHEMA[SITE_TABLE])
     report = []
-    for entry in entries or []:
+    for entry in check_dependencies(tx, stage, entries):
         name = entry["name"]
         cap = get(name)
+        active = _entry_active(entry, stage)
         enabled = as_bool(entry.get("enabled", True), default=False)
-        applies = stage in cap.applies_to or bool(entry.get("force"))
-        active = enabled and applies
 
         tx.table(SITE_TABLE).merge(
             {