bb-harness 4.0.1__py3-none-any.whl

bb_harness/__init__.py

@@ -0,0 +1,3 @@
+"""Manual black-box test harness CLI tool."""
+
+__version__ = "4.0.1"

bb_harness/__main__.py

@@ -0,0 +1,8 @@
+"""Entry point for python -m bb_harness."""
+
+import sys
+
+from bb_harness.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

bb_harness/artifact_migration.py

@@ -0,0 +1,141 @@
+"""旧artifactを上書きせず構造情報だけを移行する。"""
+
+import copy
+
+from bb_harness.coverage_engine import ALIASES, TECHNIQUES, VERSION
+from bb_harness.schema_validation import validate_artifact
+from bb_harness.techniques.common import ModelError
+
+ROOT_ADDITIONS = {"schema_version", "generation", "migration", "evidence_binding"}
+CASE_ADDITIONS = {
+    "technique_refs",
+    "coverage_obligation_ids",
+    "coverage_inputs",
+    "test_data",
+    "case_revision",
+}
+CHARTER_ADDITIONS = {
+    "case_revision",
+    "technique_refs",
+    "basis_refs",
+    "mission",
+    "resources",
+    "entry_criteria",
+    "exit_criteria",
+    "environment",
+    "limitations",
+    "history_refs",
+}
+OBS_ADDITIONS = {"technique_refs", "model_refs", "coverage_obligation_ids", "basis_refs"}
+EVIDENCE_ADDITIONS = {
+    "case_revision",
+    "model_hash",
+    "coverage_obligation_ids",
+    "relation_evaluation",
+    "session_log",
+    "model_ref",
+    "sample_id",
+    "observed_inputs",
+    "observed_outputs",
+}
+GATE_ADDITIONS = {
+    "evidence_binding_mode",
+    "required_obligation_design_rate",
+    "required_obligation_execution_rate",
+    "coverage_exit_criteria_met",
+    "coverage_report_id",
+    "coverage_mode",
+    "coverage_unknown_count",
+}
+
+
+def migrate_artifact(
+    value: dict, artifact_type: str, *, artifact_version: str = "enhanced"
+) -> dict:
+    schema = f"{artifact_type}.schema.json"
+    validate_artifact(value, schema)
+    result = copy.deepcopy(value)
+    if artifact_version == "legacy":
+        if artifact_type in {"technique_plan", "coverage_report"}:
+            raise ModelError("legacyには被覆artifactの表現がない")
+        for key in ROOT_ADDITIONS:
+            result.pop(key, None)
+        if artifact_type == "test_model":
+            for key in {*TECHNIQUES, "parameters", "integration_paths"}:
+                result.pop(key, None)
+        if artifact_type == "manual_case_set":
+            for item in result["manual_cases"]:
+                for key in CASE_ADDITIONS:
+                    item.pop(key, None)
+            for item in result.get("exploratory_charters", []):
+                for key in CHARTER_ADDITIONS:
+                    item.pop(key, None)
+        if artifact_type == "observation_set":
+            allowed = {
+                "equivalence_partitioning",
+                "boundary_value",
+                "decision_table",
+                "state_transition",
+                "exploratory",
+                "error_guessing",
+                "use_case",
+                "user_scenario",
+            }
+            for item in result["observations"]:
+                if set(item["techniques"]) - allowed:
+                    raise ModelError("新技法を旧observation enumへ損失なく変換できない")
+                for key in OBS_ADDITIONS:
+                    item.pop(key, None)
+        if artifact_type == "execution_evidence":
+            for key in EVIDENCE_ADDITIONS:
+                result.pop(key, None)
+        if artifact_type == "gate_decision":
+            for key in GATE_ADDITIONS:
+                result.get("evidence_summary", {}).pop(key, None)
+        validate_artifact(result, schema)
+        return result
+    if artifact_version != "enhanced":
+        raise ModelError("unknown artifact version")
+    if result.get("schema_version") == VERSION:
+        return result
+    result["schema_version"] = VERSION
+    references = {
+        name: ("ISTQB CTAL-TA v4.0", reference) for name, reference, _ in TECHNIQUES.values()
+    }
+    references.update(
+        {
+            "equivalence_partitioning": ("ISTQB CTFL v4.0.1", "4.2.1"),
+            "boundary_value_analysis": ("ISTQB CTFL v4.0.1", "4.2.2"),
+            "error_guessing": ("ISTQB CTFL v4.0.1", "4.4.1"),
+            "exploratory_testing": ("ISTQB CTFL v4.0.1", "4.4.2"),
+        }
+    )
+    for item in result.get("observations", []) + result.get("manual_cases", []):
+        refs = []
+        for alias in item.get("techniques", []):
+            key = ALIASES.get(alias, alias)
+            if key in references:
+                standard, reference = references[key]
+                refs.append(
+                    {
+                        "key": key,
+                        "standard": standard,
+                        "reference": reference,
+                        "legacy_alias": alias,
+                    }
+                )
+        if refs:
+            item["technique_refs"] = refs
+    status = (
+        "unmapped"
+        if artifact_type == "manual_case_set"
+        else "needs_review"
+        if artifact_type == "test_model"
+        else "migrated"
+    )
+    result["migration"] = {
+        "status": status,
+        "reason": "旧IDと本文を保持。文字列から精度・条件式・正式被覆を推測しない",
+    }
+    validate_artifact(result, schema)
+    return result

bb_harness/batched_generation.py

@@ -0,0 +1,272 @@
+"""完全JSON単位の有限な分割生成。途中文字列の継ぎ足しはしない。"""
+
+import copy
+import json
+
+from bb_harness.coverage_engine import TECHNIQUES, validate_case_coverage
+from bb_harness.efficient_generation import apply_review_patch, remaining_work, review_patch_prompt
+from bb_harness.schema_validation import validate_artifact
+from bb_harness.techniques.common import ModelError
+
+MAX_CASE_BATCHES = 24
+BATCH_SIZE = 2
+CORE_FIELDS = (
+    "feature_id",
+    "flows",
+    "data_partitions",
+    "boundaries",
+    "rule_columns",
+    "states",
+    "valid_transitions",
+    "invalid_transitions",
+    "role_matrix",
+    "regression_edges",
+    "quality_lenses",
+    "parameters",
+)
+
+
+def select_schema(schema: dict, properties: list | tuple) -> dict:
+    """必要なpropertyと参照先のdefsだけを、再帰参照も保持して選択する。"""
+    result = {
+        "type": "object",
+        "additionalProperties": False,
+        "properties": {key: copy.deepcopy(schema["properties"][key]) for key in properties},
+        "required": list(properties),
+    }
+    pending = []
+
+    def visit(node):
+        if isinstance(node, dict):
+            if "$ref" in node:
+                pending.append(node["$ref"])
+            for value in node.values():
+                visit(value)
+        elif isinstance(node, list):
+            for value in node:
+                visit(value)
+
+    visit(result)
+    seen = set()
+    while pending:
+        ref = pending.pop()
+        if ref in seen:
+            continue
+        if not ref.startswith("#/$defs/"):
+            raise ModelError("partition schema requires portable local references")
+        seen.add(ref)
+        parts = [part.replace("~1", "/").replace("~0", "~") for part in ref[2:].split("/")]
+        source, target = schema, result
+        for part in parts:
+            source = source[part]
+        for part in parts[:-1]:
+            target = target.setdefault(part, {})
+        target[parts[-1]] = copy.deepcopy(source)
+        visit(source)
+    return result
+
+
+def _checkpoint(pipeline, name: str, value: dict) -> None:
+    from bb_harness.local_pipeline import _write_json
+
+    pipeline.checkpoint_dir.mkdir(exist_ok=True)
+    _write_json(pipeline.checkpoint_dir / f"{name}.json", value)
+
+
+def model_core_request(feature: dict) -> tuple[dict, str]:
+    from bb_harness.local_pipeline import (
+        _test_model_prompt,
+        portable_schema,
+    )
+
+    whole = portable_schema("test_model.schema.json")
+    schema = select_schema(whole, CORE_FIELDS)
+    for field in ("boundaries", "data_partitions", "invalid_transitions"):
+        schema["properties"][field]["minItems"] = 1
+    schema["properties"]["selected_techniques"] = {
+        "type": "array",
+        "uniqueItems": True,
+        "items": {"enum": list(TECHNIQUES)},
+    }
+    schema["required"].append("selected_techniques")
+    prompt = (
+        "モデルの概要と共有parameterだけを返します。型付きモデル本文は次の呼出で生成します。\n"
+        "selected_techniquesには、この仕様で根拠を持って適用する技法だけを列挙します。\n"
+        "不要な技法を選択せず、不明な値を発明しません。JSONは余分な空白を省いてください。\n"
+        "boundariesには数値境界と状態境界を含めます。仕様の許可状態／拒否状態の切替点も境界です。\n"
+        + _test_model_prompt(feature)
+    )
+
+    return schema, prompt
+
+
+def generate_model(pipeline, feature: dict) -> dict:
+    from bb_harness.local_pipeline import (
+        _normalize_test_model,
+        _validate_test_model_semantics,
+        portable_schema,
+    )
+
+    whole = portable_schema("test_model.schema.json")
+    schema, prompt = model_core_request(feature)
+
+    def validate_core(value):
+        base = {key: item for key, item in value.items() if key != "selected_techniques"}
+        validate_artifact(base, "test_model.schema.json")
+        _validate_test_model_semantics(base, feature=None)
+
+    core = pipeline._generate_custom(
+        "test_model__core",
+        schema,
+        prompt,
+        normalize=lambda value: _normalize_test_model(value, feature),
+        semantic_validate=validate_core,
+    )
+    _checkpoint(pipeline, "test_model-core", core)
+    selected = core.pop("selected_techniques")
+    result = copy.deepcopy(core)
+    for field in selected:
+        part_schema = select_schema(whole, [field])
+        prompt = (
+            f"{field}だけを完全なJSONで返してください。他技法や概要の再出力は不要です。\n"
+            "共有parameterを使い、根拠source_refsはfeature_specの実在オブジェクトを保持します。\n"
+            "まだ存在しないobservation/risk IDを参照しません。\n"
+            "モデルIDは技法名を含め一意にします。\n"
+            "条件や精度が不明なら発明せず空配列にしてください。JSONの余分な空白は省きます。\n"
+            + json.dumps({"feature_spec": feature, "model_core": core}, ensure_ascii=False)
+        )
+        value = pipeline._generate_custom(f"test_model__{field}", part_schema, prompt)
+        result[field] = value[field]
+        _checkpoint(pipeline, f"test_model-{field}", value)
+    identifiers = [item["id"] for field in TECHNIQUES for item in result.get(field, [])]
+    if len(set(identifiers)) != len(identifiers):
+        raise ModelError("duplicate model IDs across generated partitions")
+    parameter_ids = [item["id"] for item in result.get("parameters", [])]
+    if len(set(parameter_ids)) != len(parameter_ids):
+        raise ModelError("duplicate shared parameter IDs")
+    validate_artifact(result, "test_model.schema.json")
+    _validate_test_model_semantics(result, feature=None)
+    return result
+
+
+def pending_keys(pending: dict) -> set[str]:
+    return (
+        {"obligation:" + item["id"] for item in pending["obligations"]}
+        | {"risk:" + key for key in pending["risk_ids"]}
+        | {"observation:" + key for key in pending["observation_ids"]}
+    )
+
+
+def generate_cases(pipeline, feature: dict, model: dict, observations: dict, risks: dict) -> dict:
+    from bb_harness.local_pipeline import (
+        _merge_case_sets,
+        _normalize_cases,
+        _validate_source_grounding,
+        portable_schema,
+    )
+
+    cases = {"feature_id": feature["feature_id"], "manual_cases": [], "exploratory_charters": []}
+    schema = portable_schema("manual_case_set.schema.json")
+    schema["properties"]["manual_cases"].update(minItems=0, maxItems=BATCH_SIZE)
+    schema["properties"]["exploratory_charters"].update(maxItems=BATCH_SIZE)
+    for number in range(MAX_CASE_BATCHES):
+        pending = remaining_work(feature, model, observations, risks, cases)
+        before = pending_keys(pending)
+        if not before:
+            break
+        focused = {
+            "obligations": pending["obligations"][:BATCH_SIZE],
+            "risk_ids": pending["risk_ids"][:BATCH_SIZE],
+            "observation_ids": pending["observation_ids"][:BATCH_SIZE],
+        }
+        prompt = (
+            "追加ケースを最大2件、探索チャーターを最大2件、完全JSONで返します。\n"
+            "focusの入力・経路をcoverage_inputsへ保持し、step_refs/expected_result_refsは1始まり。\n"
+            "指定したrisk/観点の未被覆を優先し、既存ケースの再出力は不要です。\n"
+            "oracle/source_refとtrace_toには実在する根拠/OBS/RISK IDだけを使います。\n"
+            "仕様にない表示文言・内部実装を発明しません。期待値は具体的に観測可能にします。\n"
+            "必要な追加がなければ空配列。JSONは余分な空白を省きます。\n"
+            + json.dumps(
+                {
+                    "feature_spec": feature,
+                    "test_model": model,
+                    "observations": observations,
+                    "risks": risks,
+                    "focus": focused,
+                    "existing": [
+                        {"tc_id": c["tc_id"], "title": c["title"], "trace_to": c["trace_to"]}
+                        for c in cases["manual_cases"]
+                    ],
+                },
+                ensure_ascii=False,
+            )
+        )
+
+        def validate(value):
+            validate_artifact(value, "manual_case_set.schema.json")
+            _validate_source_grounding(value, pipeline.source_ids)
+            checked = validate_case_coverage(value, model, pipeline.technique_plan)
+            if checked["errors"]:
+                raise ModelError("; ".join(checked["errors"][:5]))
+
+        batch = pipeline._generate_custom(
+            f"manual_case_set__{number + 1}",
+            schema,
+            prompt,
+            normalize=lambda value: _normalize_cases(value, feature, observations, risks),
+            semantic_validate=validate,
+        )
+        cases = _merge_case_sets(cases, batch, feature, observations, risks)
+        _checkpoint(pipeline, f"cases-{number + 1:02d}", cases)
+        after = pending_keys(remaining_work(feature, model, observations, risks, cases))
+        if not before - after:
+            break
+    return cases
+
+
+def review_cases(
+    pipeline, feature: dict, model: dict, observations: dict, risks: dict, cases: dict
+) -> dict:
+    from bb_harness.local_pipeline import _validate_source_grounding, portable_schema
+
+    result = copy.deepcopy(cases)
+    size = max(len(cases["manual_cases"]), len(cases.get("exploratory_charters", [])))
+    for offset in range(0, size, BATCH_SIZE):
+        subset = {
+            "feature_id": feature["feature_id"],
+            "manual_cases": result["manual_cases"][offset : offset + BATCH_SIZE],
+            "exploratory_charters": result.get("exploratory_charters", [])[
+                offset : offset + BATCH_SIZE
+            ],
+        }
+
+        def validate(patch, subset=subset, result=result):
+            apply_review_patch(subset, patch)  # 対象外IDもここで拒否する。
+            updated = apply_review_patch(result, patch)
+            _validate_source_grounding(updated, pipeline.source_ids)
+            checked = validate_case_coverage(updated, model, pipeline.technique_plan)
+            if checked["errors"]:
+                raise ModelError("; ".join(checked["errors"][:5]))
+
+        patch = pipeline._generate_custom(
+            f"manual_case_review__{offset // BATCH_SIZE + 1}",
+            portable_schema("case_review_patch.schema.json"),
+            review_patch_prompt(feature, model, observations, risks, subset),
+            semantic_validate=validate,
+        )
+        result = apply_review_patch(result, patch)
+        _checkpoint(pipeline, f"review-{offset // BATCH_SIZE + 1:02d}", patch)
+    return result
+
+
+def design_status(lint: dict, coverage: dict) -> str:
+    if lint["errors"] or coverage["errors"]:
+        return "blocked"
+    if (
+        coverage["blocked_selections"]
+        or coverage["unknown_ids"]
+        or coverage["design"]["uncovered_ids"]
+        or coverage["design"]["rate"] is None
+    ):
+        return "degraded"
+    return "ready"

bb_harness/cli.py

@@ -0,0 +1,120 @@
+"""CLI entry point for bb-harness."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from bb_harness import __version__
+from bb_harness.commands import (
+    coverage,
+    evaluate,
+    export,
+    gate,
+    heatmap,
+    import_results,
+    ingest,
+    regression_graph,
+    run,
+    state_diagram,
+    validate,
+)
+
+
+def create_parser() -> argparse.ArgumentParser:
+    """Create main CLI parser with subcommands."""
+    parser = argparse.ArgumentParser(
+        prog="bb-harness",
+        description="Manual black-box test harness CLI tool",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"bb-harness {__version__}",
+    )
+
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        dest="global_verbose",
+        help="Enable verbose output",
+    )
+
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        dest="global_dry_run",
+        help="Run without making changes (for API operations)",
+    )
+
+    # Create subparsers
+    subparsers = parser.add_subparsers(
+        title="subcommands",
+        dest="command",
+        help="Available commands",
+    )
+
+    # Add subcommands
+    validate.add_subparser(subparsers)
+    ingest.add_subparser(subparsers)
+    state_diagram.add_subparser(subparsers)
+    regression_graph.add_subparser(subparsers)
+    heatmap.add_subparser(subparsers)
+    gate.add_subparser(subparsers)
+    export.add_subparser(subparsers)
+    import_results.add_subparser(subparsers)
+    run.add_subparser(subparsers)
+    coverage.add_subparser(subparsers)
+    evaluate.add_subparser(subparsers)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Main entry point."""
+    parser = create_parser()
+    args = parser.parse_args(argv)
+
+    if args.command is None:
+        parser.print_help()
+        return 0
+
+    # Propagate top-level --dry-run and --verbose to subcommands
+    # This allows both `bb-harness --dry-run export ...` and `bb-harness export ... --dry-run`
+    top_dry_run = getattr(args, "global_dry_run", False)
+    top_verbose = getattr(args, "global_verbose", False)
+
+    # Dispatch to subcommand
+    dispatch_map = {
+        "validate": validate.run,
+        "ingest": ingest.run,
+        "state-diagram": state_diagram.run,
+        "regression-graph": regression_graph.run,
+        "heatmap": heatmap.run,
+        "gate": gate.run,
+        "export": export.run,
+        "import": import_results.run,
+        "run": run.run,
+        "coverage": coverage.run,
+        "evaluate": evaluate.run,
+        "migrate": coverage.run,
+        "bind-cases": coverage.run,
+    }
+
+    handler = dispatch_map.get(args.command)
+    if handler:
+        # Ensure subcommand sees top-level flags (logical OR with subcommand's own flags)
+        if top_dry_run:
+            args.dry_run = True
+        if top_verbose:
+            args.verbose = True
+        return handler(args)
+
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

bb_harness/commands/__init__.py

@@ -0,0 +1,25 @@
+"""bb-harness command modules."""
+
+from bb_harness.commands import (
+    export,
+    gate,
+    heatmap,
+    import_results,
+    ingest,
+    regression_graph,
+    run,
+    state_diagram,
+    validate,
+)
+
+__all__ = [
+    "export",
+    "gate",
+    "heatmap",
+    "import_results",
+    "ingest",
+    "regression_graph",
+    "run",
+    "state_diagram",
+    "validate",
+]

bb_harness/commands/_invoke.py

@@ -0,0 +1,26 @@
+"""Invoke argparse-based packaged tools without spawning a subprocess."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Callable
+
+
+def invoke_tool(
+    main: Callable[[], int],
+    argv: list[str],
+    args: argparse.Namespace | None = None,
+) -> int:
+    effective = list(argv)
+    if args is not None and getattr(args, "dry_run", False):
+        effective.append("--dry-run")
+    if args is not None and getattr(args, "verbose", False):
+        command = " ".join(effective)
+        print(f"[verbose] Running: {main.__module__} {command}", file=sys.stderr)
+    previous = sys.argv
+    sys.argv = [main.__module__, *effective]
+    try:
+        return main()
+    finally:
+        sys.argv = previous

bb_harness/commands/coverage.py

@@ -0,0 +1,86 @@
+"""被覆検証と非破壊artifact移行のCLI。"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from bb_harness.artifact_migration import migrate_artifact
+from bb_harness.coverage_engine import build_coverage_report, build_technique_plan
+from bb_harness.evidence_revisions import bind_case_set
+from bb_harness.gate_engine import load_evidence_files
+from bb_harness.tools.validate_artifact import ARTIFACT_SCHEMA_MAP
+
+
+def add_subparser(subparsers: argparse._SubParsersAction) -> None:
+    parser = subparsers.add_parser("coverage", help="型付きモデルの設計・実施被覆を検証する")
+    for name in ("feature", "test-model", "observations", "risk", "cases"):
+        parser.add_argument("--" + name, type=Path, required=True)
+    parser.add_argument("--evidence", type=Path)
+    parser.add_argument("--build-id", default="unexecuted")
+    parser.add_argument("--output", type=Path, required=True, help="新規出力ディレクトリ")
+    parser = subparsers.add_parser("migrate", help="artifactを別ファイルへ非破壊移行する")
+    parser.add_argument("--input", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--type", choices=sorted(ARTIFACT_SCHEMA_MAP), required=True)
+    parser.add_argument("--artifact-version", choices=["legacy", "enhanced"], default="enhanced")
+    parser = subparsers.add_parser(
+        "bind-cases", help="現在のケースとモデルの版を別ファイルへ固定する"
+    )
+    parser.add_argument("--input", type=Path, required=True)
+    parser.add_argument("--test-model", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+
+
+def _read(path: Path) -> dict:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _write(path: Path, value: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("x", encoding="utf-8") as handle:
+        json.dump(value, handle, ensure_ascii=False, indent=2, allow_nan=False)
+        handle.write("\n")
+
+
+def run(args: argparse.Namespace) -> int:
+    try:
+        if args.command == "bind-cases":
+            if args.output.resolve() in {args.input.resolve(), args.test_model.resolve()}:
+                raise ValueError("入力ファイルの上書きは禁止")
+            _write(args.output, bind_case_set(_read(args.input), _read(args.test_model)))
+        elif args.command == "migrate":
+            if args.input.resolve() == args.output.resolve():
+                raise ValueError("移行元の上書きは禁止")
+            result = migrate_artifact(
+                _read(args.input), args.type, artifact_version=args.artifact_version
+            )
+            _write(args.output, result)
+        else:
+            feature, model, observations, risks, cases = (
+                _read(path)
+                for path in (
+                    args.feature,
+                    args.test_model,
+                    args.observations,
+                    args.risk,
+                    args.cases,
+                )
+            )
+            plan = build_technique_plan(feature, model, observations, risks)
+            report = build_coverage_report(
+                model,
+                plan,
+                cases,
+                load_evidence_files(args.evidence) if args.evidence else [],
+                build_id=args.build_id,
+            )
+            if args.output.exists():
+                raise ValueError("被覆出力には未存在のディレクトリを指定する")
+            _write(args.output / "technique_plan.json", plan)
+            _write(args.output / "coverage_report.json", report)
+        print(f"Generated: {args.output}")
+        return 0
+    except (OSError, ValueError, TypeError, KeyError) as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        return 1