pushmetrics-cli 0.1.0__py3-none-any.whl

pm/validators.py

@@ -0,0 +1,499 @@
+"""Per-block-type content_json schema validation (pm-tool plan §2.7).
+
+Lightweight first-pass validation at the CLI layer — catches obvious
+mistakes (missing required field, wrong type) with a clear error +
+example body so the agent can recover in one round trip. The backend
+still does authoritative validation; this is the friendly front door.
+
+**Critical contract** (codex review 2026-05-22): the schemas below MUST
+match the backend's ``content_json_attr`` lists at
+``prefect/queryme/models/block/parameters/*.py``. If they drift, the CLI
+silently accepts bodies the backend can't execute (or rejects valid
+ones). ``cli/tests/test_validators.py`` has a cross-reference check that
+parses those parameters files and fails if a schema key isn't in the
+corresponding ``content_json_attr``.
+
+**Scope**:
+
+- Only validates the *shape* of a body (required fields present, types
+  plausible). Semantic checks (does database_id exist? is the integration
+  authorized?) happen server-side.
+- Required fields must be present AND match the expected type. Optional
+  fields, if present, must match. Extra fields are allowed (the backend
+  may use or ignore them).
+- Missing/unknown ``block_type`` → no-op (the type canonicalizer in
+  ``cli/pm/types.py`` is the gatekeeper for unknown types; this module
+  trusts that and lets unrecognized types through to the backend).
+
+**Modes** (pm-tool only ships ``create``):
+
+- ``mode='create'`` — all required fields must be present.
+- ``mode='update'`` — fields present must match types; missing fields OK.
+  Not used in A.3b; documented for the factory to wire later.
+
+Schemas are intentionally conservative — when in doubt, accept and let
+the backend error. The point of this module is the *friendly* path, not
+exhaustive coverage.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+from pm.errors import PMError
+
+
+# Per-block-type schemas. ``required`` and ``optional`` map field name →
+# accepted Python type (or tuple of types for unions). Example body is
+# returned verbatim in the structured error envelope so the agent can
+# copy it as a starting point.
+#
+# EVERY field name below MUST appear in the corresponding
+# ``content_json_attr`` list in
+# ``prefect/queryme/models/block/parameters/<type>.py``. The cross-
+# reference test in cli/tests/test_validators.py enforces this.
+BLOCK_SCHEMAS: dict[str, dict[str, Any]] = {
+    "sql": {
+        "required": {
+            "sql": str,
+            "database_id": str,
+        },
+        "optional": {
+            "limit": int,
+            "schema": str,
+            "export_options": dict,
+        },
+        "example": {
+            "sql": "SELECT 1",
+            "database_id": "<db_uid>",
+            "limit": 1000,
+        },
+    },
+    "email": {
+        # Matches EmailParameters.content_json_attr in
+        # prefect/queryme/models/block/parameters/email.py:
+        # body, subject, to_, attachments, cc, bcc, merge_pdf_config,
+        # merge_csv_config, smtp_from, smtp_from_fullname, integration_id.
+        #
+        # NOTE: `to_` (trailing underscore) — Python-style mangling that
+        # exists in the backend model because `to` is reserved-ish in
+        # several ORMs. Don't drop it. The agent's existing send_email
+        # tool already uses this exact key (block_tools.py:967).
+        "required": {
+            "subject": str,
+            "body": str,
+            "to_": (str, list),
+            "integration_id": str,
+        },
+        "optional": {
+            "cc": (str, list),
+            "bcc": (str, list),
+            "chart_block_uids": (str, list),
+            "attachments": (str, list),
+            "merge_pdf_config": dict,
+            "merge_csv_config": dict,
+            "smtp_from": str,
+            "smtp_from_fullname": str,
+        },
+        "example": {
+            "subject": "Daily orders",
+            "body": "<p>See attached.</p>",
+            "integration_id": "<email_integration_uid>",
+            "to_": "finance-team@example.com",
+        },
+    },
+    "slack": {
+        # Matches SlackParameters.content_json_attr:
+        # integration_id, channels, attachments, thread_ts,
+        # attachments_to_thread, merge_pdf_config, merge_csv_config,
+        # include_reload_button.
+        #
+        # NOTE: `channels` is a LIST of channel ids/names — see
+        # template_block/slack.py:471 which reads it as iterable. The
+        # agent's existing send_slack tool already uses this shape
+        # (block_tools.py:1041).
+        #
+        # `message` is the body content. NOT in content_json_attr (the
+        # parameters layer reads body via the `value` property), but the
+        # agent's send_slack tool writes it as `message` in the override
+        # content_json (block_tools.py:1040) — that's the path agents
+        # actually use, so we accept it as the user-facing key.
+        "required": {
+            "integration_id": str,
+            "channels": (str, list),
+            "message": str,
+        },
+        "optional": {
+            "attachments": (str, list),
+            "thread_ts": str,
+            "attachments_to_thread": bool,
+            "merge_pdf_config": dict,
+            "merge_csv_config": dict,
+            "include_reload_button": bool,
+        },
+        "example": {
+            "integration_id": "<slack_integration_uid>",
+            "channels": ["#data-alerts"],
+            "message": "Daily report is ready.",
+        },
+    },
+    "sftp": {
+        # Matches SftpParameters.content_json_attr:
+        # path, integration_id, attachments, merge_pdf_config,
+        # merge_csv_config.
+        #
+        # `path` is the REMOTE destination on the SFTP server. The data
+        # to upload comes from `attachments` (URLs / refs to upstream
+        # block outputs). There is NO inline file_contents or
+        # source_block_uid in the backend schema — those were earlier
+        # assumptions on the CLI side that didn't match reality.
+        "required": {
+            "integration_id": str,
+            "path": str,
+        },
+        "optional": {
+            "attachments": (str, list),
+            "merge_pdf_config": dict,
+            "merge_csv_config": dict,
+        },
+        "example": {
+            "integration_id": "<sftp_integration_uid>",
+            "path": "/uploads/daily/{date}.csv",
+            "attachments": ["<csv_url_from_run_sql>"],
+        },
+    },
+    "plotly": {
+        # Matches ChartParameters.content_json_attr: just `properties`.
+        # `properties` holds the full Plotly spec ({data, layout}). The
+        # agent's generate_chart tool writes it as
+        # {"properties": spec, "image_filename": <name>} —
+        # block_tools.py:1193.
+        "required": {
+            "properties": dict,
+        },
+        "optional": {
+            "image_filename": str,
+        },
+        "example": {
+            "properties": {
+                "data": [{"x": [1, 2, 3], "y": [4, 5, 6], "type": "bar"}],
+                "layout": {"title": "Example"},
+            },
+            "image_filename": "chart.png",
+        },
+    },
+    "metrics_view": {
+        # Metrics views are content blocks (not runnable). content_json
+        # holds the semantic layer definition. Fields below are what
+        # the semantic layer reads — keep this list in sync with
+        # ``superset/pushmetrics/semantic``.
+        # Codex pm-consolidation pass 2 (2026-05-22) flagged this drift:
+        # an earlier draft advertised ``database_id`` + ``source_sql``
+        # but the engine reads ``database`` + ``model`` — agents
+        # following the validator built content_json that the engine
+        # then rejected with "no database configured". 2026-06-05 fix.
+        "required": {
+            "name": str,
+            "measures": list,
+            "database": str,  # database UID hashid (not the human name —
+                              # the FE YAML export uses the human name for
+                              # portability and resolves on import, but
+                              # ``pm metrics-view create`` writes directly
+                              # to content_json so it must be the UID).
+            "model": str,     # SQL string. The engine wraps it as
+                              # ``FROM ({model}) AS model`` (see
+                              # ``semantic/sql_builder.py``) — a bare
+                              # table reference like ``"orders"`` fails.
+                              # FE "table mode" generates
+                              # ``SELECT * FROM <schema>.<table>``
+                              # before persisting; do the same.
+        },
+        "optional": {
+            "description": str,
+            "dimensions": list,
+            "timeseries": str,        # ts column name
+            "smallest_time_grain": str,  # day | week | month | etc.
+            # ``parameters`` is a LIST of parameter defs
+            # ``[{key, type, default_value, label?}]`` per the FE
+            # ``MetricsParameter`` interface and the SQL builder. Codex
+            # round-1 P1 (2026-06-05): an earlier draft typed this as
+            # ``dict`` — FE-saved metrics views would have failed
+            # validator round-trip.
+            "parameters": list,
+            "filters": list,
+            "joins": list,
+            "sources": list,
+        },
+        "example": {
+            "name": "Daily Orders",
+            "database": "<database_uid>",
+            "model": "SELECT * FROM orders",
+            # Measures use ``expression`` (any SQL agg) and an optional
+            # ``additive`` flag. Today ``additive`` is FE metadata only
+            # (the Explore UI uses it to choose chart aggregation hints);
+            # the semantic engine's symmetric aggregation is keyed on
+            # source_table + join relationships at
+            # ``semantic/sql_builder.py``, not on this flag. Set it
+            # truthfully (false for COUNT DISTINCT / SUM-of-distinct
+            # things) so downstream consumers can rely on it later.
+            "measures": [
+                {
+                    "name": "order_count",
+                    "expression": "COUNT(*)",
+                    "display_name": "Orders",
+                },
+                {
+                    "name": "unique_customers",
+                    "expression": "COUNT(DISTINCT customer_id)",
+                    "additive": False,
+                },
+            ],
+            # Dimensions use ``column`` (the SQL column name in the
+            # model) — NOT ``expression``. The SQL builder at
+            # ``superset/pushmetrics/semantic/sql_builder.py`` reads it
+            # via ``dim_def.get("column", dim_name)`` (e.g. lines 292/346),
+            # falling back to ``name`` when ``column`` is absent.
+            "dimensions": [
+                {"name": "order_date", "column": "created_at", "type": "temporal"},
+                {"name": "region", "column": "region_code"},
+            ],
+        },
+    },
+    "tableau": {
+        # Matches TableauParameters.content_json_attr:
+        # integration_id, workbook_id, workbook_name, view_id, view_name,
+        # custom_view_id, action, filters, resolution, orientation,
+        # page_type, viz_height, viz_width, export_options.
+        "required": {
+            "integration_id": str,
+            "action": str,
+        },
+        "optional": {
+            # ``view_id`` is required for view_* actions (view_image,
+            # view_csv, view_pdf, ...); ``workbook_id`` is required
+            # instead for workbook_* actions (workbook_pdf,
+            # workbook_powerpoint). Codex round-1 P2 (2026-06-05):
+            # earlier validator forced view_id for ALL actions,
+            # blocking workbook exports the worker accepts. We mark
+            # both optional + rely on the worker to reject the
+            # missing-id case with the action-specific error.
+            "view_id": str,
+            "workbook_id": str,
+            "workbook_name": str,
+            "view_name": str,
+            "custom_view_id": str,
+            # Tableau filter string. Comma-separates multiple values
+            # (``value1,value2``); embedded commas escape as ``%5C%2C``
+            # per Salesforce docs. The worker calls ``.strip()`` and
+            # ``re.sub()`` on this directly — a dict would raise
+            # ``AttributeError`` (no ``.strip()``) on the first call.
+            # Codex pm-consolidation pass 2 (2026-05-22) flagged this;
+            # 2026-06-05 fix restricts to ``str`` only.
+            "filters": str,
+            "resolution": str,
+            "orientation": str,
+            "page_type": str,
+            "viz_height": (int, str),
+            "viz_width": (int, str),
+            "export_options": dict,
+        },
+        "example": {
+            "integration_id": "<tableau_integration_uid>",
+            "view_id": "<tableau_view_id>",
+            "action": "view_image",
+            # Comma-separated; escape embedded commas as ``%5C%2C``.
+            "filters": "Region=EMEA",
+        },
+    },
+    "google_slides": {
+        # Matches GoogleSlidesParameters.content_json_attr:
+        # integration_id, action, source_file_id, presentation_id, title,
+        # folder_id, replacements, share_with, requests, page_id,
+        # page_object_id, image_url, chart_block_uid, workspace_id,
+        # position_x, position_y, width, height, composition_json,
+        # template_definition, export_options.
+        #
+        # Different `action` values require different additional fields
+        # (compose_from_template needs source_file_id + composition_json;
+        # export_pdf needs presentation_id; batch_update needs requests +
+        # presentation_id). The validator only requires `integration_id`
+        # + `action` — the runner produces clear errors for missing
+        # action-specific fields.
+        "required": {
+            "integration_id": str,
+            "action": str,
+        },
+        "optional": {
+            "source_file_id": str,
+            "presentation_id": str,
+            "title": str,
+            "folder_id": str,
+            "replacements": dict,
+            "share_with": (str, list),
+            "requests": list,
+            "page_id": str,
+            "page_object_id": str,
+            "image_url": str,
+            "chart_block_uid": str,
+            "position_x": (int, float),
+            "position_y": (int, float),
+            "width": (int, float),
+            "height": (int, float),
+            "composition_json": dict,
+            "template_definition": dict,
+            "export_options": dict,
+        },
+        "example": {
+            "integration_id": "<google_integration_uid>",
+            "action": "compose_from_template",
+            "source_file_id": "<slides_template_id>",
+            "composition_json": {"slides": []},
+        },
+    },
+    # agent_skill / agent_memory / explore — minimal schemas. The bodies are
+    # mostly free-form (markdown for skills/memory; explore is a saved view).
+    # Leave them out of BLOCK_SCHEMAS — validate_content_json no-ops for
+    # unrecognized types, which is what we want here.
+}
+
+
+def validate_content_json(
+    block_type: str,
+    body: dict[str, Any],
+    *,
+    mode: str = "create",
+) -> None:
+    """Raise :class:`pm.errors.PMError` if ``body`` doesn't match the schema
+    for ``block_type``. No-op for types not in ``BLOCK_SCHEMAS`` (lets them
+    through to the backend, which is authoritative).
+
+    Args:
+        block_type: canonical block type (already canonicalized via
+            ``cli/pm/types.py``).
+        body: parsed JSON dict.
+        mode: ``'create'`` (default — required fields must be present) or
+            ``'update'`` (only check types of fields that are present).
+    """
+    if not isinstance(body, dict):
+        raise PMError(
+            error_code="block_schema_validation_failed",
+            message=f"Body must be a JSON object; got {type(body).__name__}",
+            received=type(body).__name__,
+            help=f"pm help {block_type}",
+        )
+
+    schema = BLOCK_SCHEMAS.get(block_type)
+    if schema is None:
+        # No schema for this type — trust the backend to validate.
+        return
+
+    required = schema.get("required", {})
+    optional = schema.get("optional", {})
+    example = schema.get("example")
+
+    # Pass 1 (create mode only): required fields present?
+    if mode == "create":
+        missing = [name for name in required if name not in body]
+        if missing:
+            raise PMError(
+                error_code="block_schema_validation_failed",
+                message=(
+                    f"Missing required field(s) for type={block_type!r}: "
+                    f"{', '.join(missing)}"
+                ),
+                received=", ".join(sorted(body.keys())) if body else "(empty)",
+                allowed=sorted(set(required) | set(optional)),
+                example=example,
+                help=f"pm help {block_type}",
+            )
+
+    # Pass 2: type-check every field present in body against required+optional
+    type_map = {**required, **optional}
+    for field_name, value in body.items():
+        expected = type_map.get(field_name)
+        if expected is None:
+            # Unknown field — allow; backend may use or ignore. Conservative
+            # choice to keep the CLI welcoming.
+            continue
+        if not isinstance(value, expected):
+            raise PMError(
+                error_code="block_schema_validation_failed",
+                message=(
+                    f"Field {field_name!r} for type={block_type!r}: expected "
+                    f"{_type_name(expected)}, got {type(value).__name__}"
+                ),
+                received=f"{field_name}={type(value).__name__}",
+                allowed=[_type_name(expected)],
+                example=example,
+                help=f"pm help {block_type}",
+            )
+
+
+def _type_name(t: Any) -> str:
+    """Human-friendly name for a type or tuple of types."""
+    if isinstance(t, tuple):
+        return " | ".join(_type_name(x) for x in t)
+    return t.__name__
+
+
+# ── User-facing → backend-key translation ─────────────────────────────
+
+
+def normalize_content_json(block_type: str, body: dict[str, Any]) -> dict[str, Any]:
+    """Translate CLI user-facing keys to the backend's expected content_json
+    shape. Run after ``validate_content_json`` and before sending to the
+    BlocksClient.
+
+    The CLI persists ``block.content_json`` directly (no override
+    middleware between persist and run), so the persisted shape MUST
+    match what the noteflow worker reads. The agent's existing
+    ``send_email`` / ``send_slack`` tools take user-friendly keys
+    (``body``, ``message``) and translate at the override layer in
+    ``prefect/queryme/agent/block_runner.py``; the CLI bypasses that
+    path, so the translation has to happen here instead.
+
+    Codex review pass 2 (2026-05-22) caught the cases handled below.
+
+    The function never raises — input bodies validated under
+    ``validate_content_json`` are well-shaped enough that translation
+    is mechanical. Unknown block types pass through unchanged.
+
+    Translations applied:
+
+    - **email**: ``body`` → ``value`` (renamed key — the runner at
+      ``template_block/email.py:94`` reads ``content.get("value")``).
+
+    - **slack**:
+
+        - ``message`` → ``value_markdown`` (renamed key — the
+          ``SlackParameters.value_wysiwyg`` property reads
+          ``content_json.get("value_markdown")``). Also sets
+          ``editor_mode = "wysiwyg_mode"`` so the parameters layer
+          picks the markdown branch.
+        - ``channels`` as ``str`` → ``[str]`` (list). The runner iterates
+          ``channels`` at ``template_block/slack.py:471``; a string would
+          be split character-by-character into single-char channel names.
+
+    Returns a new dict (does not mutate the caller's body).
+    """
+    if not isinstance(body, dict):
+        return body
+    out = dict(body)
+
+    if block_type == "email":
+        if "body" in out:
+            # Pop + reassign so a body=... payload becomes value=...
+            out["value"] = out.pop("body")
+
+    elif block_type == "slack":
+        if "message" in out:
+            out["value_markdown"] = out.pop("message")
+            # The Slack parameters layer picks between value_json and
+            # value_markdown based on editor_mode. Default to the
+            # markdown branch (the common authoring path); callers who
+            # need value_json can set editor_mode explicitly.
+            out.setdefault("editor_mode", "wysiwyg_mode")
+        ch = out.get("channels")
+        if isinstance(ch, str):
+            out["channels"] = [ch]
+
+    return out