pytest-casebook 0.1.0__py3-none-any.whl

pytest_casebook/__init__.py

@@ -0,0 +1,43 @@
+"""
+pytest-casebook: a pytest plugin that removes the need to maintain test specification
+sheets and automated tests separately.
+
+- Write cases in YAML as "preconditions -> action -> expected result". The action is one of
+  running a CLI, calling a Python function, or sending a request to an HTTP API
+- Cases can be classified by groups into any number of levels (e.g. category / subcategory / item),
+  and `--casebook-html` outputs the results as a table in that hierarchy (a test specification
+  sheet)
+
+pytest-casebook: 試験項目表と自動テストの二重管理をなくす pytest プラグイン。
+
+- YAML に「前提条件 → 操作 → 期待結果」の形でケースを書く。操作は CLI の実行・
+  Python 関数の呼び出し・HTTP の API へのリクエストのどれか
+- ケースはグループで大項目・中項目・小項目のように何階層でも分類でき、
+  `--casebook-html` でその階層の表（試験項目表）として結果を出力できる
+"""
+
+from pathlib import Path
+
+from .http import HttpRequest, HttpResponse
+from .model import Case, CasebookDoc, CasebookError, Group, load
+from .runner import CommandResult, run_in_process
+
+__version__ = "0.1.0"
+
+# JSON Schema for case files (casebook_*.yaml), used for editor completion and validation.
+# ケースファイル（casebook_*.yaml）の JSON Schema。エディタの補完と検証に使う
+SCHEMA_PATH = Path(__file__).resolve().parent / "casebook.schema.json"
+
+__all__ = [
+    "SCHEMA_PATH",
+    "Case",
+    "CasebookDoc",
+    "CasebookError",
+    "CommandResult",
+    "Group",
+    "HttpRequest",
+    "HttpResponse",
+    "__version__",
+    "load",
+    "run_in_process",
+]

pytest_casebook/casebook.schema.json

@@ -0,0 +1,401 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "pytest-casebook のケースファイル",
+  "description": "casebook_*.yaml の書き方。エディタ（VS Code の YAML 拡張など）でキーの補完と検証に使う。",
+  "type": "object",
+  "required": [
+    "cases"
+  ],
+  "additionalProperties": false,
+  "properties": {
+    "title": {
+      "type": "string",
+      "description": "レポートの見出し（省略時はファイル名）"
+    },
+    "description": {
+      "type": "string",
+      "description": "レポートに出す説明"
+    },
+    "levels": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "description": "階層の呼び名（例: [大項目, 中項目, 小項目]）"
+    },
+    "states": {
+      "type": "object",
+      "description": "状態名 -> 内容（名前を付けた setup:）。ケースやグループの state: で選ぶ",
+      "additionalProperties": {
+        "$ref": "#/$defs/setup"
+      }
+    },
+    "command_prefix": {
+      "description": "どの command にも先頭に付ける引数。{casebook_dir} は YAML のディレクトリになる",
+      "oneOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      ]
+    },
+    "golden_dir": {
+      "type": "string",
+      "description": "期待出力の置き場所（このファイルからの相対パス）"
+    },
+    "check_orphans": {
+      "type": "boolean",
+      "description": "ケースがない期待出力が残っていたら失敗にする"
+    },
+    "normalize": {
+      "type": "array",
+      "description": "出力に当てる置き換え",
+      "items": {
+        "type": "object",
+        "required": [
+          "pattern"
+        ],
+        "additionalProperties": false,
+        "properties": {
+          "pattern": {
+            "type": "string",
+            "description": "正規表現"
+          },
+          "replace": {
+            "type": "string",
+            "description": "置き換え後"
+          }
+        }
+      }
+    },
+    "request_defaults": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "headers": {
+          "$ref": "#/$defs/headers"
+        }
+      },
+      "description": "どの request: にも重ねる既定の値"
+    },
+    "meta": {
+      "type": "object",
+      "description": "自由に使う情報（pytest-casebook は見ない）"
+    },
+    "cases": {
+      "$ref": "#/$defs/items"
+    }
+  },
+  "$defs": {
+    "items": {
+      "type": "array",
+      "description": "ケースまたはグループのリスト",
+      "items": {
+        "oneOf": [
+          {
+            "$ref": "#/$defs/group"
+          },
+          {
+            "$ref": "#/$defs/case"
+          }
+        ]
+      }
+    },
+    "reason": {
+      "type": "string",
+      "minLength": 1
+    },
+    "group": {
+      "type": "object",
+      "required": [
+        "group"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "group": {
+          "type": "string",
+          "minLength": 1,
+          "description": "グループの名前（入れ子の深さが階層になる）"
+        },
+        "description": {
+          "type": "string"
+        },
+        "state": {
+          "type": "string",
+          "description": "中のケースの既定の状態"
+        },
+        "marks": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "meta": {
+          "type": "object"
+        },
+        "skip": {
+          "$ref": "#/$defs/reason",
+          "description": "中のケースすべてを、この理由でスキップする"
+        },
+        "xfail": {
+          "$ref": "#/$defs/reason",
+          "description": "中のケースすべてが、この理由で失敗すると分かっている"
+        },
+        "cases": {
+          "$ref": "#/$defs/items"
+        }
+      }
+    },
+    "case": {
+      "type": "object",
+      "required": [
+        "name"
+      ],
+      "additionalProperties": false,
+      "oneOf": [
+        {
+          "required": [
+            "command"
+          ]
+        },
+        {
+          "required": [
+            "call"
+          ]
+        },
+        {
+          "required": [
+            "request"
+          ]
+        }
+      ],
+      "properties": {
+        "name": {
+          "type": "string",
+          "pattern": "^[\\w.\\-]+$",
+          "description": "ケース名（期待出力のファイル名になる）"
+        },
+        "description": {
+          "type": "string",
+          "description": "確認する内容"
+        },
+        "command": {
+          "description": "CLI として実行する引数。リストで書くと区切らずにそのまま渡す（Windows のパス向け）",
+          "oneOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          ]
+        },
+        "stdin": {
+          "type": "string"
+        },
+        "timeout": {
+          "type": "number",
+          "exclusiveMinimum": 0
+        },
+        "call": {
+          "type": "string",
+          "pattern": "^[^:\\s]+:[^:\\s]+$",
+          "description": "モジュール:関数"
+        },
+        "args": {
+          "type": "array"
+        },
+        "kwargs": {
+          "type": "object"
+        },
+        "request": {
+          "$ref": "#/$defs/request"
+        },
+        "state": {
+          "type": "string"
+        },
+        "setup": {
+          "$ref": "#/$defs/setup"
+        },
+        "observe": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "実行後に取り出す対象（files など）"
+        },
+        "expect": {
+          "$ref": "#/$defs/expect"
+        },
+        "marks": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "meta": {
+          "type": "object"
+        },
+        "skip": {
+          "$ref": "#/$defs/reason"
+        },
+        "xfail": {
+          "$ref": "#/$defs/reason"
+        },
+        "golden": {
+          "type": "string",
+          "description": "期待出力のファイル名（省略時は <name>.txt）"
+        }
+      }
+    },
+    "setup": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "description": "前提条件。files は作業ディレクトリに置くファイル、それ以外は pytest_casebook_setup に渡す",
+      "properties": {
+        "files": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          }
+        }
+      }
+    },
+    "headers": {
+      "type": "object",
+      "additionalProperties": {
+        "type": [
+          "string",
+          "number",
+          "boolean"
+        ]
+      }
+    },
+    "request": {
+      "oneOf": [
+        {
+          "type": "string",
+          "pattern": "^\\s*(([Gg][Ee][Tt]|[Pp][Oo][Ss][Tt]|[Pp][Uu][Tt]|[Pp][Aa][Tt][Cc][Hh]|[Dd][Ee][Ll][Ee][Tt][Ee]|[Hh][Ee][Aa][Dd]|[Oo][Pp][Tt][Ii][Oo][Nn][Ss])\\s+)?/\\S*\\s*$",
+          "description": "\"メソッド パス\"（メソッドを省くと GET）"
+        },
+        {
+          "type": "object",
+          "required": [
+            "path"
+          ],
+          "additionalProperties": false,
+          "not": {
+            "required": [
+              "json",
+              "body"
+            ]
+          },
+          "properties": {
+            "method": {
+              "type": "string",
+              "pattern": "^([Gg][Ee][Tt]|[Pp][Oo][Ss][Tt]|[Pp][Uu][Tt]|[Pp][Aa][Tt][Cc][Hh]|[Dd][Ee][Ll][Ee][Tt][Ee]|[Hh][Ee][Aa][Dd]|[Oo][Pp][Tt][Ii][Oo][Nn][Ss])$"
+            },
+            "path": {
+              "type": "string",
+              "pattern": "^/"
+            },
+            "query": {
+              "type": "object"
+            },
+            "headers": {
+              "$ref": "#/$defs/headers"
+            },
+            "json": {
+              "description": "JSON の本文"
+            },
+            "body": {
+              "type": "string",
+              "description": "JSON 以外の本文"
+            }
+          }
+        }
+      ]
+    },
+    "text": {
+      "description": "文字列なら完全一致、{contains: ...} なら含む、{regex: ...} なら正規表現",
+      "oneOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "object",
+          "minProperties": 1,
+          "maxProperties": 1,
+          "additionalProperties": false,
+          "properties": {
+            "contains": {
+              "type": "string"
+            },
+            "regex": {
+              "type": "string"
+            },
+            "$contains": {
+              "type": "string"
+            },
+            "$regex": {
+              "type": "string"
+            }
+          }
+        }
+      ]
+    },
+    "expect": {
+      "type": "object",
+      "minProperties": 1,
+      "description": "期待結果。ここに書いた項目だけを比べる。下の項目以外のキーは observe の対象（db.users など）",
+      "not": {
+        "required": [
+          "return",
+          "raises"
+        ]
+      },
+      "properties": {
+        "exit_code": {
+          "type": "integer"
+        },
+        "stdout": {
+          "$ref": "#/$defs/text"
+        },
+        "stderr": {
+          "$ref": "#/$defs/text"
+        },
+        "return": {
+          "description": "辞書は書いたキーだけ、リストは件数と各要素を比べる。$contains・$regex・$unordered・$includes が使える"
+        },
+        "raises": {
+          "$ref": "#/$defs/text"
+        },
+        "status": {
+          "type": "integer"
+        },
+        "json": {
+          "description": "return と同じ比べ方"
+        },
+        "body": {
+          "$ref": "#/$defs/text"
+        },
+        "headers": {
+          "type": "object",
+          "additionalProperties": {
+            "$ref": "#/$defs/text"
+          }
+        }
+      }
+    }
+  }
+}

pytest_casebook/expect.py

@@ -0,0 +1,208 @@
+"""
+YAML の `expect:` に書いた期待値と、実際の結果を比べる。
+
+- 文字列（stdout / stderr / raises / body）: そのまま書けば完全一致（末尾の改行は無視）、
+  `{contains: ...}` なら含まれていればよい、`{regex: ...}` なら正規表現に合えばよい
+- 値（return / json / observe の対象）: 辞書は書いたキーだけを比べ、リストは件数と各要素を比べ、
+  それ以外は完全一致。値の中では、`$` で始まる1つのキーだけの辞書を「比べ方」として扱う
+  （データのキーと紛れないように `$` を付ける）
+
+  - `{$contains: ...}` / `{$regex: ...}`: 文字列が含む / 正規表現に合う
+  - `{$unordered: [...]}`: 順序を問わず、件数と各要素が一致する
+  - `{$includes: [...]}`: 書いた要素がすべて含まれていればよい（ほかの要素があってもよい）
+"""
+
+from __future__ import annotations
+
+import difflib
+import re
+from collections.abc import Callable
+from typing import Any
+
+from .messages import MESSAGES
+from .serialize import compact
+
+# 値の中に書ける比べ方
+VALUE_MATCHERS = ("$contains", "$regex", "$unordered", "$includes")
+
+
+def _show(value: Any) -> str:
+    return compact(value)
+
+
+def value_matcher(expected: Any) -> tuple[str, Any] | None:
+    """`{$contains: ...}` のような比べ方なら `(名前, 中身)` を返す。そうでなければ None。"""
+    if isinstance(expected, dict) and len(expected) == 1:
+        ((key, body),) = expected.items()
+        if isinstance(key, str) and key.startswith("$"):
+            return key, body
+    return None
+
+
+def match_text(expected: Any, actual: str, label: str, lang: str = "ja") -> list[str]:
+    """
+    文字列の期待値と比べ、違いの説明のリストを返す（一致すれば空）。
+
+    expected は YAML を読むときに確かめ済みのもの（文字列か {contains} / {regex}）。
+    """
+    m = MESSAGES[lang]
+    actual = actual.rstrip("\n")
+    if isinstance(expected, str):
+        expected = expected.rstrip("\n")
+        if expected == actual:
+            return []
+        if "\n" in expected or "\n" in actual:
+            # 末尾の改行は取り除いてあるので、行は改行なしで比べて、あとで改行でつなぐ
+            # （keepends で比べると、最後の行どうしが1行につながってしまう）
+            diff = "\n".join(
+                difflib.unified_diff(
+                    expected.splitlines(),
+                    actual.splitlines(),
+                    fromfile=m["diff_expected"].format(label=label),
+                    tofile=m["diff_actual"].format(label=label),
+                    lineterm="",
+                )
+            )
+            return [m["text_differs"].format(label=label) + f"\n{diff.rstrip()}"]
+        return [_mismatch(m, label, expected, actual)]
+
+    ((matcher, pattern),) = expected.items()
+    matcher = matcher.lstrip("$")  # {contains} と {$contains} は同じ
+    if matcher == "contains" and pattern not in actual:
+        return [
+            m["not_contained"].format(label=label, pattern=_show(pattern), actual=_show(actual))
+        ]
+    if matcher == "regex" and not re.search(pattern, actual, re.MULTILINE):
+        return [
+            m["no_regex_match"].format(label=label, pattern=_show(pattern), actual=_show(actual))
+        ]
+    return []
+
+
+def _mismatch(m: dict[str, str], label: str, expected: Any, actual: Any) -> str:
+    return m["mismatch"].format(label=label, expected=_show(expected), actual=_show(actual))
+
+
+def match_value(expected: Any, actual: Any, label: str, lang: str = "ja") -> list[str]:
+    """
+    値の期待値と比べ、違いの説明のリストを返す（一致すれば空）。
+
+    :param expected: YAML に書いた期待値
+    :param actual: 実際の値（`to_jsonable` で変換し、正規化したもの）
+    :param label: 説明に使う場所の名前（例: `db.users[1].name`）
+    :param lang: 説明の言語（`ja` か `en`）
+    :return: 違いの説明のリスト（一致すれば空）
+    """
+    m = MESSAGES[lang]
+    matcher = value_matcher(expected)
+    if matcher is not None:
+        return _match_with(matcher[0], matcher[1], actual, label, lang)
+    if isinstance(expected, dict):
+        if not isinstance(actual, dict):
+            return [_mismatch(m, label, expected, actual)]
+        problems: list[str] = []
+        for key, value in expected.items():
+            if key not in actual:
+                problems.append(m["missing_key"].format(label=f"{label}.{key}"))
+            else:
+                problems += match_value(value, actual[key], f"{label}.{key}", lang)
+        return problems
+    if isinstance(expected, list):
+        if not isinstance(actual, list):
+            return [_mismatch(m, label, expected, actual)]
+        problems = []
+        if len(expected) != len(actual):
+            problems.append(
+                m["count_differs"].format(label=label, expected=len(expected), actual=len(actual))
+            )
+        for i, (e, a) in enumerate(zip(expected, actual, strict=False)):
+            problems += match_value(e, a, f"{label}[{i}]", lang)
+        return problems
+    if isinstance(expected, str) and isinstance(actual, str):
+        # ファイルの内容などは末尾の改行の有無を問わない（YAML の書き方で変わりやすいため）
+        if expected.rstrip("\n") != actual.rstrip("\n"):
+            return [_mismatch(m, label, expected, actual)]
+        return []
+    # True == 1 にならないよう、真偽値は型も比べる
+    if isinstance(expected, bool) != isinstance(actual, bool) or expected != actual:
+        return [_mismatch(m, label, expected, actual)]
+    return []
+
+
+def _match_with(name: str, body: Any, actual: Any, label: str, lang: str) -> list[str]:
+    """`$` で始まる比べ方で比べる。"""
+    m = MESSAGES[lang]
+    if name in ("$contains", "$regex"):
+        if not isinstance(actual, str):
+            return [m["not_string"].format(label=label, actual=_show(actual))]
+        return match_text({name: body}, actual, label, lang)
+    if not isinstance(actual, list):
+        return [m["not_list"].format(label=label, actual=_show(actual))]
+    if name == "$unordered" and len(body) != len(actual):
+        return [m["count_differs"].format(label=label, expected=len(body), actual=len(actual))]
+    # 期待の要素と実際の要素を、1対1で、できるだけ多く組み合わせる（二部グラフの最大マッチング）。
+    # 先頭から順に貪欲に組むと、{type: A} のような広い条件が先にあるときに、
+    # 後ろの細かい条件に合うはずの要素を取ってしまい、書く順番によって失敗してしまう
+    fits = [
+        [j for j, value in enumerate(actual) if not match_value(item, value, label, lang)]
+        for item in body
+    ]
+    owner: dict[int, int] = {}  # 実際の要素 -> 組んだ期待の要素
+
+    def assign(i: int, seen: set[int]) -> bool:
+        """期待の i 番目を、空いている（または付け替えられる）実際の要素に割り当てる。"""
+        for j in fits[i]:
+            if j in seen:
+                continue
+            seen.add(j)
+            if j not in owner or assign(owner[j], seen):
+                owner[j] = i
+                return True
+        return False
+
+    matched = {i for i in range(len(body)) if assign(i, set())}
+    return [
+        m["no_element"].format(label=label, item=_show(item), index=i)
+        for i, item in enumerate(body)
+        if i not in matched
+    ]
+
+
+def find_matcher_errors(value: Any, label: str) -> list[str]:
+    """期待値の中の `$` で始まる比べ方の書き間違いを探す（YAML を読むときに確かめる）。"""
+    matcher = value_matcher(value)
+    if matcher is not None:
+        name, body = matcher
+        if name not in VALUE_MATCHERS:
+            known = " / ".join(VALUE_MATCHERS)
+            return [f"{label}: 知らない比べ方です: {name}（使えるのは {known}）"]
+        if name in ("$contains", "$regex") and not isinstance(body, str):
+            return [f"{label}: {name} には文字列を書いてください"]
+        if name == "$regex":
+            try:
+                re.compile(body)
+            except re.error as e:
+                return [f"{label}: 正規表現の誤りです: {e}"]
+        if name in ("$unordered", "$includes"):
+            if not isinstance(body, list):
+                return [f"{label}: {name} にはリストを書いてください"]
+            return [
+                err for i, v in enumerate(body) for err in find_matcher_errors(v, f"{label}[{i}]")
+            ]
+        return []
+    if isinstance(value, dict):
+        return [err for k, v in value.items() for err in find_matcher_errors(v, f"{label}.{k}")]
+    if isinstance(value, list):
+        return [err for i, v in enumerate(value) for err in find_matcher_errors(v, f"{label}[{i}]")]
+    return []
+
+
+def map_strings(value: Any, func: Callable[[str], str]) -> Any:
+    """JSON にできる値の中の文字列すべてに func を当てる（正規化に使う）。"""
+    if isinstance(value, str):
+        return func(value)
+    if isinstance(value, list):
+        return [map_strings(v, func) for v in value]
+    if isinstance(value, dict):
+        return {k: map_strings(v, func) for k, v in value.items()}
+    return value