triflux 10.9.21 → 10.9.23

package/skills/tfx-workspace/iteration-1/benchmark.json

@@ -1,524 +0,0 @@
-{
-  "metadata": {
-    "skill_name": "tfx-skills-suite",
-    "skill_path": "C:/Users/SSAFY/Desktop/Projects/cli/triflux/skills",
-    "executor_model": "claude-sonnet-4-6",
-    "analyzer_model": "claude-opus-4-6",
-    "timestamp": "2026-03-19T10:00:00Z",
-    "evals_run": [1, 2, 3, 4, 5, 6],
-    "runs_per_configuration": 1
-  },
-  "runs": [
-    {
-      "eval_id": 1,
-      "eval_name": "routing-implement-shortcut",
-      "configuration": "with_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 5,
-        "failed": 0,
-        "total": 5,
-        "time_seconds": 43.6,
-        "tokens": 16303,
-        "tool_calls": 4,
-        "errors": 0
-      },
-      "expectations": [
-        {
-          "text": "Routes to executor agent",
-          "passed": true,
-          "evidence": "Correctly mapped from implement shortcut table"
-        },
-        {
-          "text": "Uses implement MCP profile",
-          "passed": true,
-          "evidence": "Mapped from shortcut table"
-        },
-        {
-          "text": "Generates correct tfx-route.sh command",
-          "passed": true,
-          "evidence": "bash ~/.claude/scripts/tfx-route.sh executor '...' implement"
-        },
-        {
-          "text": "Does NOT trigger triage",
-          "passed": true,
-          "evidence": "Command shortcut skips triage"
-        },
-        {
-          "text": "Does NOT delegate to tfx-multi",
-          "passed": true,
-          "evidence": "No subtask decomposition occurred"
-        }
-      ]
-    },
-    {
-      "eval_id": 1,
-      "eval_name": "routing-implement-shortcut",
-      "configuration": "without_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 5,
-        "failed": 0,
-        "total": 5,
-        "time_seconds": 48.1,
-        "tokens": 16436,
-        "tool_calls": 4,
-        "errors": 0
-      },
-      "expectations": [
-        {
-          "text": "Routes to executor agent",
-          "passed": true,
-          "evidence": "Correctly mapped"
-        },
-        {
-          "text": "Uses implement MCP profile",
-          "passed": true,
-          "evidence": "Assigned by shortcut table"
-        },
-        {
-          "text": "Generates correct tfx-route.sh command",
-          "passed": true,
-          "evidence": "Correct syntax generated"
-        },
-        {
-          "text": "Does NOT trigger triage",
-          "passed": true,
-          "evidence": "Shortcut mode skips triage"
-        },
-        {
-          "text": "Does NOT delegate to tfx-multi",
-          "passed": true,
-          "evidence": "No delegation"
-        }
-      ]
-    },
-    {
-      "eval_id": 2,
-      "eval_name": "routing-multi-task-triage",
-      "configuration": "with_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 5,
-        "failed": 0,
-        "total": 5,
-        "time_seconds": 58.2,
-        "tokens": 17584,
-        "tool_calls": 3,
-        "errors": 0
-      },
-      "expectations": [
-        {
-          "text": "Identifies as auto mode",
-          "passed": true,
-          "evidence": "No shortcut match, auto mode selected"
-        },
-        {
-          "text": "Triggers Codex classification",
-          "passed": true,
-          "evidence": "Codex --full-auto classification triggered"
-        },
-        {
-          "text": "Decomposes into 2+ subtasks",
-          "passed": true,
-          "evidence": "2 subtasks: executor + security-reviewer"
-        },
-        {
-          "text": "Notes tfx-multi delegation",
-          "passed": true,
-          "evidence": "subtasks.length >= 2 triggers tfx-multi Phase 3"
-        },
-        {
-          "text": "Does NOT execute directly",
-          "passed": true,
-          "evidence": "Delegates to tfx-multi"
-        }
-      ]
-    },
-    {
-      "eval_id": 2,
-      "eval_name": "routing-multi-task-triage",
-      "configuration": "without_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 5,
-        "failed": 0,
-        "total": 5,
-        "time_seconds": 77.2,
-        "tokens": 18626,
-        "tool_calls": 4,
-        "errors": 0
-      },
-      "expectations": [
-        {
-          "text": "Identifies as auto mode",
-          "passed": true,
-          "evidence": "Auto mode selected"
-        },
-        {
-          "text": "Triggers Codex classification",
-          "passed": true,
-          "evidence": "Codex --full-auto triggered"
-        },
-        {
-          "text": "Decomposes into 2+ subtasks",
-          "passed": true,
-          "evidence": "2 subtasks decomposed"
-        },
-        {
-          "text": "Notes tfx-multi delegation",
-          "passed": true,
-          "evidence": "Hands off to tfx-multi Phase 3"
-        },
-        {
-          "text": "Does NOT execute directly",
-          "passed": true,
-          "evidence": "Delegates correctly"
-        }
-      ]
-    },
-    {
-      "eval_id": 3,
-      "eval_name": "multi-team-creation",
-      "configuration": "with_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 5,
-        "failed": 0,
-        "total": 5,
-        "time_seconds": 115.3,
-        "tokens": 27197,
-        "tool_calls": 3,
-        "errors": 0
-      },
-      "expectations": [
-        {
-          "text": "Creates TeamCreate with tfx- prefix",
-          "passed": true,
-          "evidence": "TeamCreate({ team_name: 'tfx-<base36>' })"
-        },
-        {
-          "text": "Creates 3 TaskCreate calls",
-          "passed": true,
-          "evidence": "3x TaskCreate with metadata"
-        },
-        {
-          "text": "Spawns 3 Agent wrappers with bypassPermissions",
-          "passed": true,
-          "evidence": "3x Agent({ mode: bypassPermissions })"
-        },
-        {
-          "text": "Uses tfx-route.sh inside wrappers",
-          "passed": true,
-          "evidence": "Direct codex/gemini calls prohibited"
-        },
-        {
-          "text": "Includes Phase 5 TeamDelete",
-          "passed": true,
-          "evidence": "TeamDelete always runs, max 30s wait"
-        }
-      ]
-    },
-    {
-      "eval_id": 3,
-      "eval_name": "multi-team-creation",
-      "configuration": "without_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 5,
-        "failed": 0,
-        "total": 5,
-        "time_seconds": 100.6,
-        "tokens": 26140,
-        "tool_calls": 3,
-        "errors": 0
-      },
-      "expectations": [
-        {
-          "text": "Creates TeamCreate with tfx- prefix",
-          "passed": true,
-          "evidence": "TeamCreate with tfx-<id>"
-        },
-        {
-          "text": "Creates 3 TaskCreate calls",
-          "passed": true,
-          "evidence": "Three TaskCreate calls"
-        },
-        {
-          "text": "Spawns 3 Agent wrappers with bypassPermissions",
-          "passed": true,
-          "evidence": "mode: bypassPermissions in all 3"
-        },
-        {
-          "text": "Uses tfx-route.sh inside wrappers",
-          "passed": true,
-          "evidence": "Never direct codex/gemini calls"
-        },
-        {
-          "text": "Includes Phase 5 TeamDelete",
-          "passed": true,
-          "evidence": "TeamDelete unconditionally"
-        }
-      ]
-    },
-    {
-      "eval_id": 4,
-      "eval_name": "doctor-diagnosis",
-      "configuration": "with_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 4,
-        "failed": 0,
-        "total": 4,
-        "time_seconds": 53.8,
-        "tokens": 14499,
-        "tool_calls": 4,
-        "errors": 0
-      },
-      "expectations": [
-        {
-          "text": "Runs triflux doctor first",
-          "passed": true,
-          "evidence": "Bash(\"triflux doctor\")"
-        },
-        {
-          "text": "Suggests --fix mode",
-          "passed": true,
-          "evidence": "Suggests after diagnosis report"
-        },
-        {
-          "text": "Mentions HUD and CLI checks",
-          "passed": true,
-          "evidence": "HUD and CLI paths checked"
-        },
-        {
-          "text": "Does NOT jump to --reset",
-          "passed": true,
-          "evidence": "--reset reserved for explicit request"
-        }
-      ]
-    },
-    {
-      "eval_id": 4,
-      "eval_name": "doctor-diagnosis",
-      "configuration": "without_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 4,
-        "failed": 0,
-        "total": 4,
-        "time_seconds": 48.3,
-        "tokens": 14482,
-        "tool_calls": 3,
-        "errors": 0
-      },
-      "expectations": [
-        {
-          "text": "Runs triflux doctor first",
-          "passed": true,
-          "evidence": "Bash(\"triflux doctor\")"
-        },
-        {
-          "text": "Suggests --fix mode",
-          "passed": true,
-          "evidence": "Offers --fix after diagnosis"
-        },
-        {
-          "text": "Mentions HUD and CLI checks",
-          "passed": true,
-          "evidence": "All 8 diagnostics listed"
-        },
-        {
-          "text": "Does NOT jump to --reset",
-          "passed": true,
-          "evidence": "--reset reserved for explicit request"
-        }
-      ]
-    },
-    {
-      "eval_id": 5,
-      "eval_name": "hub-start-sequence",
-      "configuration": "with_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 3,
-        "failed": 0,
-        "total": 3,
-        "time_seconds": 47.2,
-        "tokens": 14821,
-        "tool_calls": 4,
-        "errors": 0
-      },
-      "expectations": [
-        {
-          "text": "Runs node hub/server.mjs in background",
-          "passed": true,
-          "evidence": "Bash(\"node hub/server.mjs\", run_in_background=true)"
-        },
-        {
-          "text": "Mentions port 27888 and /mcp",
-          "passed": true,
-          "evidence": "Port 27888, http://127.0.0.1:27888/mcp"
-        },
-        {
-          "text": "No triage or routing attempted",
-          "passed": true,
-          "evidence": "Command match, not fallthrough"
-        }
-      ]
-    },
-    {
-      "eval_id": 5,
-      "eval_name": "hub-start-sequence",
-      "configuration": "without_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 3,
-        "failed": 0,
-        "total": 3,
-        "time_seconds": 51.8,
-        "tokens": 14904,
-        "tool_calls": 4,
-        "errors": 0
-      },
-      "expectations": [
-        {
-          "text": "Runs node hub/server.mjs in background",
-          "passed": true,
-          "evidence": "Bash(\"node hub/server.mjs\", run_in_background=true)"
-        },
-        {
-          "text": "Mentions port 27888 and /mcp",
-          "passed": true,
-          "evidence": "Port 27888, endpoint /mcp"
-        },
-        {
-          "text": "No triage or routing attempted",
-          "passed": true,
-          "evidence": "Command match, not fallthrough"
-        }
-      ]
-    },
-    {
-      "eval_id": 6,
-      "eval_name": "codex-gemini-remap",
-      "configuration": "with_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 4,
-        "failed": 0,
-        "total": 4,
-        "time_seconds": 69.7,
-        "tokens": 14889,
-        "tool_calls": 5,
-        "errors": 0
-      },
-      "expectations": [
-        {
-          "text": "designer remapped to Codex (effort: high)",
-          "passed": true,
-          "evidence": "designer → Codex (effort: high)"
-        },
-        {
-          "text": "writer remapped to Codex Spark (spark_fast)",
-          "passed": true,
-          "evidence": "writer → Codex Spark (effort: spark_fast)"
-        },
-        {
-          "text": "TFX_CLI_MODE=codex set",
-          "passed": true,
-          "evidence": "Set for every Phase 3 call"
-        },
-        {
-          "text": "MCP profiles changed",
-          "passed": true,
-          "evidence": "designer→implement, writer→analyze"
-        }
-      ]
-    },
-    {
-      "eval_id": 6,
-      "eval_name": "codex-gemini-remap",
-      "configuration": "without_skill",
-      "run_number": 1,
-      "result": {
-        "pass_rate": 1.0,
-        "passed": 4,
-        "failed": 0,
-        "total": 4,
-        "time_seconds": 85.2,
-        "tokens": 19802,
-        "tool_calls": 7,
-        "errors": 0
-      },
-      "expectations": [
-        {
-          "text": "designer remapped to Codex (effort: high)",
-          "passed": true,
-          "evidence": "designer → Codex (effort: high)"
-        },
-        {
-          "text": "writer remapped to Codex Spark (spark_fast)",
-          "passed": true,
-          "evidence": "writer → Codex Spark (effort: spark_fast)"
-        },
-        {
-          "text": "TFX_CLI_MODE=codex set",
-          "passed": true,
-          "evidence": "TFX_CLI_MODE set to codex"
-        },
-        {
-          "text": "MCP profiles changed",
-          "passed": true,
-          "evidence": "writer→analyze, designer→implement"
-        }
-      ]
-    }
-  ],
-  "run_summary": {
-    "with_skill": {
-      "pass_rate": { "mean": 1.0, "stddev": 0.0, "min": 1.0, "max": 1.0 },
-      "time_seconds": {
-        "mean": 64.6,
-        "stddev": 26.4,
-        "min": 43.6,
-        "max": 115.3
-      },
-      "tokens": { "mean": 17549, "stddev": 4857, "min": 14499, "max": 27197 }
-    },
-    "without_skill": {
-      "pass_rate": { "mean": 1.0, "stddev": 0.0, "min": 1.0, "max": 1.0 },
-      "time_seconds": {
-        "mean": 68.5,
-        "stddev": 20.4,
-        "min": 48.1,
-        "max": 100.6
-      },
-      "tokens": { "mean": 18398, "stddev": 4227, "min": 14482, "max": 26140 }
-    },
-    "delta": {
-      "pass_rate": "+0.00",
-      "time_seconds": "-3.9",
-      "tokens": "-849"
-    }
-  },
-  "notes": [
-    "All 26 assertions pass at 100% for both configurations — the skills are functionally correct",
-    "The fixes applied (dead reference removal, Phase numbering consistency, hub description) don't change routing logic, so pass rates are identical",
-    "NEW version is marginally faster (-3.9s avg) and uses fewer tokens (-849 avg), likely due to cleaner references reducing model confusion",
-    "tfx-multi is the most complex skill (115s / 27K tokens with_skill) — consider extracting reference docs to reduce context load",
-    "tfx-codex OLD references 'Phase(1~6)' which doesn't exist in tfx-auto — the NEW version correctly references the actual workflow names",
-    "All assertions pass regardless of configuration — these test the core routing logic which is unchanged. Consider adding assertions that specifically test the fixed issues (dead refs, phase naming) for differentiation"
-  ]
-}

package/skills/tfx-workspace/iteration-1/codex-gemini-remap/eval_metadata.json

@@ -1,11 +0,0 @@
-{
-  "eval_id": 6,
-  "eval_name": "codex-gemini-remap",
-  "prompt": "/tfx-codex API 문서를 작성하고 디자인 가이드도 만들어줘",
-  "assertions": [
-    "designer remapped to Codex with effort: high",
-    "writer remapped to Codex Spark with effort: spark_fast",
-    "Sets TFX_CLI_MODE=codex environment variable",
-    "Changes MCP profile: designer->implement, writer->analyze"
-  ]
-}

package/skills/tfx-workspace/iteration-1/codex-gemini-remap/old_skill/grading.json

@@ -1,25 +0,0 @@
-{
-  "expectations": [
-    {
-      "text": "designer remapped to Codex with effort: high",
-      "passed": true,
-      "evidence": "Agent output: designer → Codex (effort: high)"
-    },
-    {
-      "text": "writer remapped to Codex Spark with effort: spark_fast",
-      "passed": true,
-      "evidence": "Agent output: writer → Codex Spark (effort: spark_fast)"
-    },
-    {
-      "text": "Sets TFX_CLI_MODE=codex environment variable",
-      "passed": true,
-      "evidence": "Agent output: 'TFX_CLI_MODE: Set to codex'"
-    },
-    {
-      "text": "Changes MCP profile: designer->implement, writer->analyze",
-      "passed": true,
-      "evidence": "Agent output: writer→analyze, designer→implement"
-    }
-  ],
-  "summary": { "passed": 4, "failed": 0, "total": 4, "pass_rate": 1.0 }
-}