nasde-toolkit 0.3.2__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/.claude/skills/nasde-benchmark-creator/SKILL.md +58 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/.claude/skills/nasde-benchmark-from-history/SKILL.md +9 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/.claude/skills/nasde-benchmark-from-public-repos/SKILL.md +9 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/.claude/skills/nasde-benchmark-runner/SKILL.md +66 -38
- nasde_toolkit-0.3.3/.claude/skills/nasde-benchmark-runner/scripts/export_codex_oauth_token.ps1 +61 -0
- nasde_toolkit-0.3.3/.claude/skills/nasde-benchmark-runner/scripts/export_gemini_oauth_token.ps1 +59 -0
- nasde_toolkit-0.3.3/.claude/skills/nasde-benchmark-runner/scripts/export_oauth_token.ps1 +40 -0
- nasde_toolkit-0.3.3/.claude/skills/nasde-benchmark-runner/scripts/export_oauth_token.sh +46 -0
- nasde_toolkit-0.3.3/.gitattributes +46 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/.github/workflows/publish.yml +64 -25
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/.github/workflows/quality-gate.yml +8 -3
- nasde_toolkit-0.3.3/.python-version +1 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/CHANGELOG.md +78 -1
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/PKG-INFO +25 -11
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/README.md +22 -8
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/pyproject.toml +5 -2
- nasde_toolkit-0.3.3/scripts/export_codex_oauth_token.ps1 +61 -0
- nasde_toolkit-0.3.3/scripts/export_codex_oauth_token.sh +57 -0
- nasde_toolkit-0.3.3/scripts/export_gemini_oauth_token.ps1 +59 -0
- nasde_toolkit-0.3.3/scripts/export_gemini_oauth_token.sh +52 -0
- nasde_toolkit-0.3.3/scripts/export_oauth_token.ps1 +40 -0
- nasde_toolkit-0.3.3/scripts/export_oauth_token.sh +46 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/_version.py +2 -2
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/runner.py +2 -2
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/scaffold/__init__.py +45 -1
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/tests/test_evaluator_backends.py +5 -5
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/tests/test_skills_installer.py +30 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/uv.lock +676 -703
- nasde_toolkit-0.3.2/scripts/export_oauth_token.sh +0 -32
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/.claude/settings.json +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3/.claude/skills/nasde-benchmark-runner}/scripts/export_codex_oauth_token.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3/.claude/skills/nasde-benchmark-runner}/scripts/export_gemini_oauth_token.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/.claude/skills/nasde-dev/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/.dockerignore +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/.github/CODEOWNERS +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/.github/workflows/dogfooding.yml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/.github/workflows/example-validation.yml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/.gitignore +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/ARCHITECTURE.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/CLAUDE.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/LICENSE +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/SECURITY.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/RELEASING.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/adr/001-open-host-service.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/adr/002-all-deps-as-core.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/adr/003-eval-on-by-default.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/adr/004-pass-through-cli-delegation.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/adr/005-repo-root-chdir.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/adr/006-opik-deferred-metrics-patch.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/adr/007-git-tag-versioning.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/adr/008-independent-dimension-scales.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/benchmark-results.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/ci-setup.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/superpowers/plans/2026-03-20-ci-github-actions.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/superpowers/plans/2026-03-23-nasde-dev-skill-benchmark.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/superpowers/plans/2026-04-16-subprocess-evaluator-backend.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/superpowers/plans/2026-04-16-trajectory-aware-evaluator.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/superpowers/plans/2026-04-22-dependency-modernization.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/superpowers/specs/2026-03-20-nasde-dev-skill-benchmark-design.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/superpowers/specs/2026-04-16-trajectory-aware-evaluator-design.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/superpowers/specs/2026-04-22-dependency-modernization-design.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/docs/use-cases.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/.gitignore +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/assessment_dimensions.json +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/evaluator_skills/code-review/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/nasde.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/csharp-anemic-to-rich-domain/assessment_criteria.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/csharp-anemic-to-rich-domain/environment/Dockerfile +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/csharp-anemic-to-rich-domain/instruction.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/csharp-anemic-to-rich-domain/task.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/csharp-anemic-to-rich-domain/tests/test.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/ddd-threshold-discount/assessment_criteria.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/ddd-threshold-discount/environment/Dockerfile +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/ddd-threshold-discount/instruction.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/ddd-threshold-discount/solution/solve.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/ddd-threshold-discount/task.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/ddd-threshold-discount/tests/test.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/ddd-weather-discount/assessment_criteria.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/ddd-weather-discount/environment/Dockerfile +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/ddd-weather-discount/instruction.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/ddd-weather-discount/solution/solve.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/ddd-weather-discount/task.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/ddd-weather-discount/tests/test.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/java-order-dispatch-domain-entities/assessment_criteria.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/java-order-dispatch-domain-entities/environment/Dockerfile +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/java-order-dispatch-domain-entities/instruction.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/java-order-dispatch-domain-entities/task.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/tasks/java-order-dispatch-domain-entities/tests/test.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/claude-guided/CLAUDE.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/claude-guided/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/claude-ntcoding-tactical-ddd/CLAUDE.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/claude-ntcoding-tactical-ddd/skills/tactical-ddd/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/claude-ntcoding-tactical-ddd/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/claude-vanilla/CLAUDE.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/claude-vanilla/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/codex-guided/AGENTS.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/codex-guided/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/codex-ntcoding-tactical-ddd/AGENTS.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/codex-ntcoding-tactical-ddd/agents_skills/tactical-ddd/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/codex-ntcoding-tactical-ddd/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/codex-vanilla/AGENTS.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/codex-vanilla/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/gemini-ntcoding-tactical-ddd/GEMINI.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/gemini-ntcoding-tactical-ddd/gemini_skills/tactical-ddd/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/gemini-ntcoding-tactical-ddd/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/gemini-vanilla/GEMINI.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/ddd-architectural-challenges/variants/gemini-vanilla/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/.gitignore +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/assessment_dimensions.json +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/nasde.codex.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/nasde.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/tasks/add-multi-attempt-support/assessment_criteria.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/tasks/add-multi-attempt-support/instruction.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/tasks/add-multi-attempt-support/solution/solve.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/tasks/add-multi-attempt-support/task.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/tasks/add-multi-attempt-support/tests/test.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/claude-nasde-dev-full-stack/CLAUDE.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/claude-nasde-dev-full-stack/skills/nasde-dev/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/claude-nasde-dev-full-stack/skills/python-best-practices/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/claude-nasde-dev-full-stack/skills/python-testing/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/claude-nasde-dev-full-stack/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/claude-nasde-dev-with-arch/CLAUDE.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/claude-nasde-dev-with-arch/skills/nasde-dev/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/claude-nasde-dev-with-arch/skills/python-best-practices/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/claude-nasde-dev-with-arch/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/claude-nasde-dev-with-testing/CLAUDE.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/claude-nasde-dev-with-testing/skills/nasde-dev/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/claude-nasde-dev-with-testing/skills/python-testing/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/claude-nasde-dev-with-testing/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/claude-vanilla/CLAUDE.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/claude-vanilla/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/gemini-vanilla/GEMINI.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/nasde-dev-skill/variants/gemini-vanilla/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/.gitignore +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/assessment_dimensions.json +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/evaluator_skills/code-review/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/nasde.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/java-expense-report-extract-hierarchy/assessment_criteria.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/java-expense-report-extract-hierarchy/environment/Dockerfile +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/java-expense-report-extract-hierarchy/instruction.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/java-expense-report-extract-hierarchy/task.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/java-expense-report-extract-hierarchy/tests/test.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/java-trip-service-break-dependency/assessment_criteria.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/java-trip-service-break-dependency/environment/Dockerfile +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/java-trip-service-break-dependency/instruction.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/java-trip-service-break-dependency/task.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/java-trip-service-break-dependency/tests/test.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/python-gilded-rose-polymorphism/assessment_criteria.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/python-gilded-rose-polymorphism/environment/Dockerfile +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/python-gilded-rose-polymorphism/instruction.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/python-gilded-rose-polymorphism/task.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/python-gilded-rose-polymorphism/tests/test.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/python-theatrical-players-extract-method/assessment_criteria.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/python-theatrical-players-extract-method/environment/Dockerfile +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/python-theatrical-players-extract-method/instruction.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/python-theatrical-players-extract-method/task.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/tasks/python-theatrical-players-extract-method/tests/test.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/variants/claude-refactoring-skill/CLAUDE.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/variants/claude-refactoring-skill/skills/refactor/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/variants/claude-refactoring-skill/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/variants/claude-vanilla/CLAUDE.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/variants/claude-vanilla/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/variants/codex-refactoring-skill/AGENTS.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/variants/codex-refactoring-skill/agents_skills/refactor/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/variants/codex-refactoring-skill/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/variants/codex-vanilla/AGENTS.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/variants/codex-vanilla/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/variants/gemini-refactoring-skill/GEMINI.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/variants/gemini-refactoring-skill/gemini_skills/refactor/SKILL.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/variants/gemini-refactoring-skill/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/variants/gemini-vanilla/GEMINI.md +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/examples/refactoring-skill/variants/gemini-vanilla/variant.toml +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/nasde-toolkit-logo.png +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/scripts/validate_example_task.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/scripts/validate_gemini_auth.sh +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/__init__.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/agents/__init__.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/agents/configurable_claude.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/agents/configurable_codex.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/agents/configurable_gemini.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/banner.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/cli.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/config.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/docker.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/evaluator.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/evaluator_backends/__init__.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/evaluator_backends/claude_subprocess.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/evaluator_backends/codex_subprocess.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/evaluator_backends/protocol.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/skills_installer.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/src/nasde_toolkit/update_check.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/tests/__init__.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/tests/test_cli.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/tests/test_config.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/tests/test_configurable_codex.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/tests/test_configurable_gemini.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/tests/test_docker.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/tests/test_evaluator.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/tests/test_runner.py +0 -0
- {nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/tests/test_update_check.py +0 -0
|
@@ -14,6 +14,52 @@ description: |
|
|
|
14
14
|
|
|
15
15
|
Create and configure coding agent benchmarks for evaluation with `nasde`. A benchmark is a set of coding tasks that AI agents solve inside isolated Docker containers, scored both by functional tests (pass/fail) and by an LLM-as-a-Judge architecture assessment.
|
|
16
16
|
|
|
17
|
+
## Critical: line endings on Windows (read this first)
|
|
18
|
+
|
|
19
|
+
Benchmark scripts execute inside **Linux** sandboxes (Docker, Daytona). If `tests/test.sh`, `solution/solve.sh`, or `environment/Dockerfile` are checked out with **CRLF** line endings (the Windows git default when `core.autocrlf=true` and there is no `.gitattributes`), every trial fails immediately with:
|
|
20
|
+
|
|
21
|
+
```
|
|
22
|
+
bash: line 1: /tests/test.sh: cannot execute: required file not found
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
…because the kernel reads the shebang as `#!/bin/bash\r` and tries to execute a non-existent `/bin/bash\r`. The agent finishes its work, but the verifier never runs and Harbor reports `RewardFileNotFoundError`.
|
|
26
|
+
|
|
27
|
+
**Mitigation (always do this for a new benchmark — `nasde init` does it for you, but verify):**
|
|
28
|
+
|
|
29
|
+
1. The benchmark repo MUST have a `.gitattributes` file enforcing LF for shell scripts and Dockerfiles. The minimum content:
|
|
30
|
+
```gitattributes
|
|
31
|
+
* text=auto eol=lf
|
|
32
|
+
*.sh text eol=lf
|
|
33
|
+
*.bash text eol=lf
|
|
34
|
+
Dockerfile text eol=lf
|
|
35
|
+
*.dockerfile text eol=lf
|
|
36
|
+
docker-compose.yaml text eol=lf
|
|
37
|
+
docker-compose.yml text eol=lf
|
|
38
|
+
|
|
39
|
+
*.ps1 text eol=crlf
|
|
40
|
+
*.bat text eol=crlf
|
|
41
|
+
*.cmd text eol=crlf
|
|
42
|
+
```
|
|
43
|
+
`nasde init` writes this automatically. If you are adding a benchmark to an existing repo without `.gitattributes`, create one before adding any task.
|
|
44
|
+
|
|
45
|
+
2. When **writing** `.sh` or `Dockerfile` content programmatically on Windows, write with explicit LF — not `path.write_text(content)` (which translates `\n`→`\r\n` on Windows), but `path.write_text(content, encoding="utf-8", newline="")` or open the file in binary mode.
|
|
46
|
+
|
|
47
|
+
3. After committing on Windows for the first time, run:
|
|
48
|
+
```bash
|
|
49
|
+
git add --renormalize .
|
|
50
|
+
git commit -m "normalize line endings"
|
|
51
|
+
```
|
|
52
|
+
to fix any files that landed before `.gitattributes` was in place.
|
|
53
|
+
|
|
54
|
+
4. Sanity check before pushing a new task:
|
|
55
|
+
```bash
|
|
56
|
+
file tasks/<task>/tests/test.sh
|
|
57
|
+
# MUST say "with LF line terminators" or omit line-terminator info entirely.
|
|
58
|
+
# If it says "with CRLF line terminators" — fix it (`sed -i 's/\r$//' file`).
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
This applies equally when you're **adding tasks to a benchmark someone else created** — if their repo has no `.gitattributes` and you're on Windows, your contribution will silently break for them on Linux CI and vice versa.
|
|
62
|
+
|
|
17
63
|
## Step 1: Understand what to evaluate
|
|
18
64
|
|
|
19
65
|
Before creating files, clarify with the user:
|
|
@@ -116,6 +162,8 @@ What the agent must NOT do (e.g., don't modify existing tests).
|
|
|
116
162
|
|
|
117
163
|
### environment/Dockerfile (required)
|
|
118
164
|
|
|
165
|
+
> **Reminder for Windows authors:** the Dockerfile and any helper scripts it `COPY`s in must have LF line endings — Docker tolerates CRLF in some commands but not in `RUN` shell snippets, and any shell script copied with CRLF will hit the same shebang failure as `test.sh`.
|
|
166
|
+
|
|
119
167
|
```dockerfile
|
|
120
168
|
FROM <base-image>
|
|
121
169
|
|
|
@@ -137,6 +185,8 @@ The Dockerfile MUST be self-contained — the agent starts working immediately.
|
|
|
137
185
|
|
|
138
186
|
### tests/test.sh (required — Harbor verifier)
|
|
139
187
|
|
|
188
|
+
> **Reminder for Windows authors:** this file MUST be saved with LF line endings. See "Critical: line endings on Windows" at the top of this skill. CRLF here = `bash: required file not found` and a wasted trial.
|
|
189
|
+
|
|
140
190
|
```bash
|
|
141
191
|
#!/bin/bash
|
|
142
192
|
cd /app
|
|
@@ -319,3 +369,11 @@ Before running with a real agent:
|
|
|
319
369
|
```bash
|
|
320
370
|
nasde run --variant vanilla --tasks <task-name> --without-eval -C .
|
|
321
371
|
```
|
|
372
|
+
|
|
373
|
+
4. **Final pre-flight on Windows authors** — verify no CRLF leaked in:
|
|
374
|
+
```bash
|
|
375
|
+
find tasks -name '*.sh' -exec sh -c 'file "$1" | grep -q CRLF && echo "BAD: $1"' _ {} \;
|
|
376
|
+
find tasks -name 'Dockerfile' -exec sh -c 'file "$1" | grep -q CRLF && echo "BAD: $1"' _ {} \;
|
|
377
|
+
# Both should print nothing.
|
|
378
|
+
```
|
|
379
|
+
If anything prints, fix with `sed -i 's/\r$//' <file>` and re-commit.
|
{nasde_toolkit-0.3.2 → nasde_toolkit-0.3.3}/.claude/skills/nasde-benchmark-from-history/SKILL.md
RENAMED
|
@@ -19,6 +19,15 @@ Generate NASDE benchmark tasks by mining git history. You analyze commits, diffs
|
|
|
19
19
|
- An existing NASDE benchmark project (run `nasde init` first, or use the `nasde-benchmark-creator` skill)
|
|
20
20
|
- If the benchmark project doesn't exist yet, create it first — this skill generates tasks, not the project scaffold
|
|
21
21
|
|
|
22
|
+
## Critical: line endings on Windows (read this first)
|
|
23
|
+
|
|
24
|
+
When generating `tests/test.sh`, `solution/solve.sh`, or `environment/Dockerfile` on a Windows host, write them with **LF** line endings or every trial fails with `bash: required file not found` (the kernel reads `#!/bin/bash\r` as the shebang). See the full explanation and `.gitattributes` template in the `nasde-benchmark-creator` skill.
|
|
25
|
+
|
|
26
|
+
Quick rules:
|
|
27
|
+
- The benchmark project MUST have a `.gitattributes` enforcing `*.sh text eol=lf` and `Dockerfile text eol=lf`. `nasde init` creates this. If the existing project lacks it, **create `.gitattributes` before generating any task files**.
|
|
28
|
+
- When writing files programmatically, use `path.write_text(content, encoding="utf-8", newline="")` — never the bare default which translates `\n`→`\r\n` on Windows.
|
|
29
|
+
- Sanity-check after generation: `find tasks/<new-task> -name '*.sh' -o -name 'Dockerfile' | xargs file | grep CRLF` should print nothing.
|
|
30
|
+
|
|
22
31
|
## Step 1: Identify the source repository and commit range
|
|
23
32
|
|
|
24
33
|
Ask the user:
|
|
@@ -19,6 +19,15 @@ Build a diverse NASDE benchmark by curating tasks from multiple public GitHub re
|
|
|
19
19
|
- A clear description of the skill being evaluated (what it does, what kinds of tasks it helps with)
|
|
20
20
|
- Internet access (to browse and clone public repositories)
|
|
21
21
|
|
|
22
|
+
## Critical: line endings on Windows (read this first)
|
|
23
|
+
|
|
24
|
+
When generating `tests/test.sh`, `solution/solve.sh`, or `environment/Dockerfile` on a Windows host, write them with **LF** line endings or every trial fails with `bash: required file not found` (the kernel reads `#!/bin/bash\r` as the shebang). See the full explanation and `.gitattributes` template in the `nasde-benchmark-creator` skill.
|
|
25
|
+
|
|
26
|
+
Quick rules:
|
|
27
|
+
- The benchmark project MUST have a `.gitattributes` enforcing `*.sh text eol=lf` and `Dockerfile text eol=lf`. `nasde init` creates this. If the existing project lacks it, **create `.gitattributes` before generating any task files**.
|
|
28
|
+
- When writing files programmatically, use `path.write_text(content, encoding="utf-8", newline="")` — never the bare default which translates `\n`→`\r\n` on Windows.
|
|
29
|
+
- Sanity-check after generation: `find tasks/<new-task> -name '*.sh' -o -name 'Dockerfile' | xargs file | grep CRLF` should print nothing.
|
|
30
|
+
|
|
22
31
|
## Step 1: Understand the skill under test
|
|
23
32
|
|
|
24
33
|
Ask the user:
|
|
@@ -17,69 +17,95 @@ Run coding agent benchmarks with `nasde` and verify results. The two-stage pipel
|
|
|
17
17
|
|
|
18
18
|
## Authentication setup
|
|
19
19
|
|
|
20
|
-
Before running any benchmark, set up authentication tokens. Both
|
|
20
|
+
Before running any benchmark, set up authentication tokens for the agents you plan to run. Both OS and auth method matter — pick the right command per row.
|
|
21
21
|
|
|
22
|
-
###
|
|
22
|
+
### Step 1 — Ask the user which auth they prefer
|
|
23
23
|
|
|
24
|
-
|
|
25
|
-
source scripts/export_oauth_token.sh
|
|
26
|
-
```
|
|
24
|
+
**Always ask the user before running, never assume.** Two questions:
|
|
27
25
|
|
|
28
|
-
|
|
26
|
+
1. **Which agents will you run?** (Claude / Codex / Gemini, any combination)
|
|
27
|
+
2. **For each agent, OAuth (subscription) or API key (per-token billing)?** Default recommendation: OAuth where available — no per-token cost, no env vars to manage.
|
|
29
28
|
|
|
30
|
-
|
|
29
|
+
Then detect their OS and pick the matching script row from the table below. On Windows, also ask whether they're in **PowerShell** or **WSL** (cmd.exe is not directly supported — see "Windows: cmd.exe" below).
|
|
31
30
|
|
|
32
|
-
###
|
|
31
|
+
### Where the auth scripts live
|
|
33
32
|
|
|
34
|
-
|
|
33
|
+
The OAuth scripts ship inside this skill. After `nasde install-skills` they are at:
|
|
35
34
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
```
|
|
35
|
+
- **User scope** (default): `~/.claude/skills/nasde-benchmark-runner/scripts/` (macOS/Linux/WSL) or `%USERPROFILE%\.claude\skills\nasde-benchmark-runner\scripts\` (Windows PowerShell)
|
|
36
|
+
- **Project scope**: `<project>/.claude/skills/nasde-benchmark-runner/scripts/` (if installed with `nasde install-skills --scope project`)
|
|
37
|
+
- **Editable nasde checkout** (devs only): `<repo>/scripts/` — same files, mirrored from the skill bundle
|
|
40
38
|
|
|
41
|
-
|
|
39
|
+
Below, `<SKILL_SCRIPTS>` is shorthand for whichever absolute path applies. Resolve it once, then substitute it in every command. Verify the path with `ls <SKILL_SCRIPTS>` before telling the user to source anything — if the directory is missing, they need to run `nasde install-skills` first.
|
|
42
40
|
|
|
43
|
-
|
|
41
|
+
### Step 2 — Run the right script per agent × OS
|
|
44
42
|
|
|
45
|
-
|
|
46
|
-
export $(grep CODEX_API_KEY .env)
|
|
47
|
-
```
|
|
43
|
+
Priority order: **Claude → Codex → Gemini.** Claude is required even for non-Claude variants when `[evaluation] backend = "claude"` (default), because the assessment evaluator spawns `claude` CLI as a subprocess.
|
|
48
44
|
|
|
49
|
-
|
|
45
|
+
#### Claude Code
|
|
50
46
|
|
|
51
|
-
|
|
47
|
+
| OS / shell | OAuth (subscription) | API key |
|
|
48
|
+
|---|---|---|
|
|
49
|
+
| macOS | `source <SKILL_SCRIPTS>/export_oauth_token.sh` (reads Keychain entry "Claude Code-credentials") | `export ANTHROPIC_API_KEY=sk-ant-...` |
|
|
50
|
+
| Linux | `source <SKILL_SCRIPTS>/export_oauth_token.sh` (reads `~/.claude/.credentials.json`) | `export ANTHROPIC_API_KEY=sk-ant-...` |
|
|
51
|
+
| Windows PowerShell | `. <SKILL_SCRIPTS>\export_oauth_token.ps1` (reads `%USERPROFILE%\.claude\.credentials.json`) | `$env:ANTHROPIC_API_KEY = 'sk-ant-...'` |
|
|
52
|
+
| Windows WSL (Ubuntu) | `source <SKILL_SCRIPTS>/export_oauth_token.sh` (Linux path; resolve `<SKILL_SCRIPTS>` from your WSL home, not the Windows host's) | `export ANTHROPIC_API_KEY=sk-ant-...` |
|
|
52
53
|
|
|
53
|
-
|
|
54
|
+
Prerequisite for OAuth: `claude` CLI installed and `claude` ran once to log in.
|
|
54
55
|
|
|
55
|
-
|
|
56
|
+
The script exports `CLAUDE_CODE_OAUTH_TOKEN`. This is required for both Claude variant runs AND assessment evaluation (when `[evaluation] backend = "claude"` — the default).
|
|
56
57
|
|
|
57
|
-
|
|
58
|
-
gemini login # one-time: authenticate via Google account
|
|
59
|
-
source scripts/export_gemini_oauth_token.sh # validate tokens are present
|
|
60
|
-
```
|
|
58
|
+
#### Codex
|
|
61
59
|
|
|
62
|
-
|
|
60
|
+
| OS / shell | OAuth (ChatGPT subscription) | API key |
|
|
61
|
+
|---|---|---|
|
|
62
|
+
| macOS | `codex login` once, then `source <SKILL_SCRIPTS>/export_codex_oauth_token.sh` | `export CODEX_API_KEY=sk-proj-...` (or `OPENAI_API_KEY`) |
|
|
63
|
+
| Linux | `codex login` once, then `source <SKILL_SCRIPTS>/export_codex_oauth_token.sh` | `export CODEX_API_KEY=sk-proj-...` |
|
|
64
|
+
| Windows PowerShell | `codex login` once, then `. <SKILL_SCRIPTS>\export_codex_oauth_token.ps1` | `$env:CODEX_API_KEY = 'sk-proj-...'` |
|
|
65
|
+
| Windows WSL (Ubuntu) | `codex login` once, then `source <SKILL_SCRIPTS>/export_codex_oauth_token.sh` | `export CODEX_API_KEY=sk-proj-...` |
|
|
63
66
|
|
|
64
|
-
**
|
|
67
|
+
The OAuth scripts only **validate** `~/.codex/auth.json` (or `%USERPROFILE%\.codex\auth.json`) — Harbor injects the file into the sandbox automatically. API key always takes priority over OAuth when both are present.
|
|
65
68
|
|
|
66
|
-
|
|
67
|
-
export GEMINI_API_KEY=your-key
|
|
68
|
-
```
|
|
69
|
+
#### Gemini CLI
|
|
69
70
|
|
|
70
|
-
|
|
71
|
+
| OS / shell | OAuth (Google account) | API key |
|
|
72
|
+
|---|---|---|
|
|
73
|
+
| macOS | `gemini login` once, then `source <SKILL_SCRIPTS>/export_gemini_oauth_token.sh` | `export GEMINI_API_KEY=...` |
|
|
74
|
+
| Linux | `gemini login` once, then `source <SKILL_SCRIPTS>/export_gemini_oauth_token.sh` | `export GEMINI_API_KEY=...` |
|
|
75
|
+
| Windows PowerShell | `gemini login` once, then `. <SKILL_SCRIPTS>\export_gemini_oauth_token.ps1` | `$env:GEMINI_API_KEY = '...'` |
|
|
76
|
+
| Windows WSL (Ubuntu) | `gemini login` once, then `source <SKILL_SCRIPTS>/export_gemini_oauth_token.sh` | `export GEMINI_API_KEY=...` |
|
|
71
77
|
|
|
72
|
-
API key always takes priority over OAuth
|
|
78
|
+
The OAuth scripts export `GEMINI_OAUTH_CREDS` (the raw JSON) — `ConfigurableGemini` reads that env var and injects credentials into the sandbox. API key always takes priority over OAuth.
|
|
73
79
|
|
|
74
80
|
### Combined setup for cross-agent runs
|
|
75
81
|
|
|
82
|
+
Resolve `<SKILL_SCRIPTS>` first, then run all three.
|
|
83
|
+
|
|
84
|
+
**macOS / Linux / Windows WSL:**
|
|
76
85
|
```bash
|
|
77
|
-
|
|
78
|
-
#
|
|
79
|
-
|
|
80
|
-
source
|
|
86
|
+
SKILL_SCRIPTS=~/.claude/skills/nasde-benchmark-runner/scripts # adjust if --scope project
|
|
87
|
+
source $SKILL_SCRIPTS/export_oauth_token.sh # Claude (subscription)
|
|
88
|
+
source $SKILL_SCRIPTS/export_codex_oauth_token.sh # Codex (subscription) — or: export CODEX_API_KEY=...
|
|
89
|
+
source $SKILL_SCRIPTS/export_gemini_oauth_token.sh # Gemini (Google account) — or: export GEMINI_API_KEY=...
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
**Windows PowerShell:**
|
|
93
|
+
```powershell
|
|
94
|
+
$SkillScripts = "$env:USERPROFILE\.claude\skills\nasde-benchmark-runner\scripts"
|
|
95
|
+
. "$SkillScripts\export_oauth_token.ps1"
|
|
96
|
+
. "$SkillScripts\export_codex_oauth_token.ps1"
|
|
97
|
+
. "$SkillScripts\export_gemini_oauth_token.ps1"
|
|
81
98
|
```
|
|
82
99
|
|
|
100
|
+
### Windows: cmd.exe
|
|
101
|
+
|
|
102
|
+
cmd.exe is **not supported directly** — `.ps1` requires PowerShell, `.sh` requires bash. Two workarounds:
|
|
103
|
+
|
|
104
|
+
1. **Open PowerShell** (`powershell.exe`) and dot-source the `.ps1` script. This is the simplest path on a vanilla Windows install.
|
|
105
|
+
2. **Use WSL** (`wsl -d Ubuntu`) and source the `.sh` script. This is the recommended path if you also want Docker Desktop with the WSL2 backend, which is the most common dev setup.
|
|
106
|
+
|
|
107
|
+
If a user is in cmd.exe, point them to one of these two — don't try to extract the token manually.
|
|
108
|
+
|
|
83
109
|
## Running benchmarks
|
|
84
110
|
|
|
85
111
|
All commands assume `-C` points to the benchmark project directory.
|
|
@@ -118,7 +144,7 @@ nasde run --variant baseline -C path/to/benchmark --without-eval
|
|
|
118
144
|
|
|
119
145
|
### Parallel runs (multiple variants)
|
|
120
146
|
|
|
121
|
-
|
|
147
|
+
**Do not use `--all-variants` when you want parallelism.** `--all-variants` runs variants **sequentially** in a single process (one variant after another). To run two or more variants in parallel, launch separate `nasde run` processes with `&` and `wait` — each job directory gets a unique random suffix, so concurrent runs are collision-safe:
|
|
122
148
|
|
|
123
149
|
```bash
|
|
124
150
|
nasde run --variant vanilla --tasks my-task -C path/to/benchmark &
|
|
@@ -126,6 +152,8 @@ nasde run --variant guided --tasks my-task -C path/to/benchmark &
|
|
|
126
152
|
wait
|
|
127
153
|
```
|
|
128
154
|
|
|
155
|
+
Use `--all-variants` only when you want one variant after another (e.g. to limit total resource use, or when running Claude variants where parallel runs risk Docker OOM — see warning below).
|
|
156
|
+
|
|
129
157
|
For deterministic job names, use `--job-suffix`:
|
|
130
158
|
|
|
131
159
|
```bash
|
nasde_toolkit-0.3.3/.claude/skills/nasde-benchmark-runner/scripts/export_codex_oauth_token.ps1
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Validate Codex OAuth auth.json for ChatGPT subscription-based authentication.
|
|
2
|
+
#
|
|
3
|
+
# Windows-equivalent of scripts/export_codex_oauth_token.sh.
|
|
4
|
+
#
|
|
5
|
+
# Dot-source this script before running nasde to verify your ChatGPT
|
|
6
|
+
# subscription credentials:
|
|
7
|
+
#
|
|
8
|
+
# . .\scripts\export_codex_oauth_token.ps1
|
|
9
|
+
# nasde run --variant codex-vanilla -C my-benchmark
|
|
10
|
+
#
|
|
11
|
+
# Prerequisites: run `codex login` to authenticate via ChatGPT.
|
|
12
|
+
# Reads %USERPROFILE%\.codex\auth.json. NASDE injects this file into the
|
|
13
|
+
# sandbox automatically — this script only validates it.
|
|
14
|
+
|
|
15
|
+
$ErrorActionPreference = 'Stop'
|
|
16
|
+
|
|
17
|
+
$authPath = Join-Path $env:USERPROFILE '.codex\auth.json'
|
|
18
|
+
|
|
19
|
+
if (-not (Test-Path $authPath)) {
|
|
20
|
+
Write-Error "$authPath not found. Run 'codex login' to authenticate via ChatGPT subscription."
|
|
21
|
+
return
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
try {
|
|
25
|
+
$auth = Get-Content $authPath -Raw | ConvertFrom-Json
|
|
26
|
+
} catch {
|
|
27
|
+
Write-Error "Failed to parse $authPath : $_"
|
|
28
|
+
return
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
if ($auth.auth_mode -ne 'chatgpt') {
|
|
32
|
+
Write-Error "auth_mode is '$($auth.auth_mode)', expected 'chatgpt'. Run 'codex login' to authenticate via ChatGPT subscription."
|
|
33
|
+
return
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
$accessToken = $auth.tokens.access_token
|
|
37
|
+
if ([string]::IsNullOrEmpty($accessToken)) {
|
|
38
|
+
Write-Error "access_token is empty in $authPath. Run 'codex login' to re-authenticate."
|
|
39
|
+
return
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
# Decode JWT payload (middle segment) to read exp claim. Pad base64url to
|
|
43
|
+
# multiple of 4 with '=' before decoding.
|
|
44
|
+
try {
|
|
45
|
+
$payloadSegment = $accessToken.Split('.')[1]
|
|
46
|
+
$padded = $payloadSegment.Replace('-', '+').Replace('_', '/')
|
|
47
|
+
switch ($padded.Length % 4) { 2 { $padded += '==' } 3 { $padded += '=' } }
|
|
48
|
+
$payloadJson = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($padded))
|
|
49
|
+
$exp = ($payloadJson | ConvertFrom-Json).exp
|
|
50
|
+
} catch {
|
|
51
|
+
$exp = 0
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
$now = [int][double]::Parse((Get-Date -UFormat %s))
|
|
55
|
+
if ($exp -gt 0 -and $now -gt $exp) {
|
|
56
|
+
Write-Warning "access_token expired. Run 'codex login' to refresh."
|
|
57
|
+
Write-Warning "Proceeding anyway -- Codex CLI may auto-refresh via refresh_token."
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
$preview = $accessToken.Substring(0, [Math]::Min(20, $accessToken.Length))
|
|
61
|
+
Write-Host "OK Codex OAuth validated (auth_mode=chatgpt, token=$preview...)" -ForegroundColor Green
|
nasde_toolkit-0.3.3/.claude/skills/nasde-benchmark-runner/scripts/export_gemini_oauth_token.ps1
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Export Gemini CLI OAuth credentials for subscription-based authentication.
|
|
2
|
+
#
|
|
3
|
+
# Windows-equivalent of scripts/export_gemini_oauth_token.sh.
|
|
4
|
+
#
|
|
5
|
+
# Dot-source this script before running nasde to authenticate via your
|
|
6
|
+
# Google account instead of GEMINI_API_KEY:
|
|
7
|
+
#
|
|
8
|
+
# . .\scripts\export_gemini_oauth_token.ps1
|
|
9
|
+
# nasde run --variant gemini-vanilla -C my-benchmark
|
|
10
|
+
#
|
|
11
|
+
# Prerequisites: run `gemini login` to authenticate via Google.
|
|
12
|
+
# Reads %USERPROFILE%\.gemini\oauth_creds.json and exports the raw JSON as
|
|
13
|
+
# $env:GEMINI_OAUTH_CREDS. ConfigurableGemini injects this into the sandbox.
|
|
14
|
+
|
|
15
|
+
$ErrorActionPreference = 'Stop'
|
|
16
|
+
|
|
17
|
+
$credPath = Join-Path $env:USERPROFILE '.gemini\oauth_creds.json'
|
|
18
|
+
|
|
19
|
+
if (-not (Test-Path $credPath)) {
|
|
20
|
+
Write-Error "$credPath not found. Run 'gemini login' to authenticate via Google."
|
|
21
|
+
return
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
try {
|
|
25
|
+
$rawJson = Get-Content $credPath -Raw
|
|
26
|
+
$parsed = $rawJson | ConvertFrom-Json
|
|
27
|
+
} catch {
|
|
28
|
+
Write-Error "$credPath does not contain valid JSON credentials: $_"
|
|
29
|
+
return
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
if (-not $parsed) {
|
|
33
|
+
Write-Error "$credPath contains empty credentials."
|
|
34
|
+
return
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
$accessToken = $parsed.access_token
|
|
38
|
+
if (-not [string]::IsNullOrEmpty($accessToken)) {
|
|
39
|
+
try {
|
|
40
|
+
$payloadSegment = $accessToken.Split('.')[1]
|
|
41
|
+
$padded = $payloadSegment.Replace('-', '+').Replace('_', '/')
|
|
42
|
+
switch ($padded.Length % 4) { 2 { $padded += '==' } 3 { $padded += '=' } }
|
|
43
|
+
$payloadJson = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($padded))
|
|
44
|
+
$exp = ($payloadJson | ConvertFrom-Json).exp
|
|
45
|
+
} catch {
|
|
46
|
+
$exp = 0
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
$now = [int][double]::Parse((Get-Date -UFormat %s))
|
|
50
|
+
if ($exp -gt 0 -and $now -gt $exp) {
|
|
51
|
+
Write-Warning "access_token appears expired. Run 'gemini login' to refresh."
|
|
52
|
+
Write-Warning "Proceeding anyway -- Gemini CLI may auto-refresh via refresh_token."
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
$env:GEMINI_OAUTH_CREDS = $rawJson
|
|
57
|
+
|
|
58
|
+
$preview = $rawJson.Substring(0, [Math]::Min(20, $rawJson.Length))
|
|
59
|
+
Write-Host "OK GEMINI_OAUTH_CREDS exported ($preview...)" -ForegroundColor Green
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Extract Claude Code OAuth token from %USERPROFILE%\.claude\.credentials.json
|
|
2
|
+
# and export it as $env:CLAUDE_CODE_OAUTH_TOKEN.
|
|
3
|
+
#
|
|
4
|
+
# Windows-equivalent of scripts/export_oauth_token.sh (which reads the macOS
|
|
5
|
+
# Keychain). On Windows, Claude Code stores credentials as plain JSON.
|
|
6
|
+
#
|
|
7
|
+
# Dot-source this script before running nasde so the env var persists in the
|
|
8
|
+
# current PowerShell session:
|
|
9
|
+
#
|
|
10
|
+
# . .\scripts\export_oauth_token.ps1
|
|
11
|
+
# nasde run --variant baseline -C my-benchmark
|
|
12
|
+
#
|
|
13
|
+
# Running without dot-source (.\scripts\export_oauth_token.ps1) sets the var
|
|
14
|
+
# only inside the script's child scope and it disappears on return.
|
|
15
|
+
|
|
16
|
+
$ErrorActionPreference = 'Stop'
|
|
17
|
+
|
|
18
|
+
$credPath = Join-Path $env:USERPROFILE '.claude\.credentials.json'
|
|
19
|
+
|
|
20
|
+
if (-not (Test-Path $credPath)) {
|
|
21
|
+
Write-Error "Could not find '$credPath'. Run 'claude' CLI and log in first."
|
|
22
|
+
return
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
try {
|
|
26
|
+
$token = (Get-Content $credPath -Raw | ConvertFrom-Json).claudeAiOauth.accessToken
|
|
27
|
+
} catch {
|
|
28
|
+
Write-Error "Failed to parse OAuth token from '$credPath': $_"
|
|
29
|
+
return
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
if ([string]::IsNullOrEmpty($token)) {
|
|
33
|
+
Write-Error "claudeAiOauth.accessToken is empty in '$credPath'."
|
|
34
|
+
return
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
$env:CLAUDE_CODE_OAUTH_TOKEN = $token
|
|
38
|
+
|
|
39
|
+
$preview = $token.Substring(0, [Math]::Min(20, $token.Length))
|
|
40
|
+
Write-Host "OK CLAUDE_CODE_OAUTH_TOKEN exported ($preview...)" -ForegroundColor Green
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
# Extract Claude Code OAuth token and export it as CLAUDE_CODE_OAUTH_TOKEN.
|
|
4
|
+
#
|
|
5
|
+
# Source this script before running nasde to authenticate via your
|
|
6
|
+
# Claude Pro/Max subscription instead of ANTHROPIC_API_KEY:
|
|
7
|
+
#
|
|
8
|
+
# source scripts/export_oauth_token.sh
|
|
9
|
+
# nasde run --variant baseline -C my-benchmark
|
|
10
|
+
#
|
|
11
|
+
# Storage backend depends on OS:
|
|
12
|
+
# - macOS: Keychain entry "Claude Code-credentials" (written by `claude` CLI)
|
|
13
|
+
# - Linux: plain JSON at ~/.claude/.credentials.json
|
|
14
|
+
# Windows users: see scripts/export_oauth_token.ps1 (PowerShell).
|
|
15
|
+
|
|
16
|
+
# NOTE: Do NOT use `set -e` here -- this script is sourced into the user's
|
|
17
|
+
# shell, so errexit would persist and kill the terminal on any later non-zero
|
|
18
|
+
# exit code. Each command already has its own `|| { ... }` error handling.
|
|
19
|
+
|
|
20
|
+
_raw_creds=""
|
|
21
|
+
|
|
22
|
+
if command -v security >/dev/null 2>&1; then
|
|
23
|
+
_raw_creds="$(security find-generic-password -s "Claude Code-credentials" -w 2>/dev/null)"
|
|
24
|
+
fi
|
|
25
|
+
|
|
26
|
+
if [ -z "$_raw_creds" ] && [ -f "$HOME/.claude/.credentials.json" ]; then
|
|
27
|
+
_raw_creds="$(cat "$HOME/.claude/.credentials.json")"
|
|
28
|
+
fi
|
|
29
|
+
|
|
30
|
+
if [ -z "$_raw_creds" ]; then
|
|
31
|
+
echo "ERROR: Could not read Claude Code credentials." >&2
|
|
32
|
+
echo " - macOS: keychain entry 'Claude Code-credentials' missing" >&2
|
|
33
|
+
echo " - Linux: ~/.claude/.credentials.json missing" >&2
|
|
34
|
+
echo "Run 'claude' CLI and log in first." >&2
|
|
35
|
+
return 1 2>/dev/null || exit 1
|
|
36
|
+
fi
|
|
37
|
+
|
|
38
|
+
CLAUDE_CODE_OAUTH_TOKEN="$(echo "$_raw_creds" | python3 -c "import sys,json; print(json.load(sys.stdin)['claudeAiOauth']['accessToken'])")" || {
|
|
39
|
+
echo "ERROR: Failed to parse OAuth token from Claude credentials." >&2
|
|
40
|
+
return 1 2>/dev/null || exit 1
|
|
41
|
+
}
|
|
42
|
+
export CLAUDE_CODE_OAUTH_TOKEN
|
|
43
|
+
|
|
44
|
+
unset _raw_creds
|
|
45
|
+
|
|
46
|
+
echo "✓ CLAUDE_CODE_OAUTH_TOKEN exported (${CLAUDE_CODE_OAUTH_TOKEN:0:20}...)"
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Default: let Git detect text vs binary, force LF in working tree.
|
|
2
|
+
# Critical: shell scripts and Dockerfiles MUST be LF — they are executed by
|
|
3
|
+
# Linux interpreters in benchmark sandboxes (Daytona, Docker). CRLF causes
|
|
4
|
+
# `bash: required file not found` because the shebang becomes `#!/bin/bash\r`.
|
|
5
|
+
* text=auto eol=lf
|
|
6
|
+
|
|
7
|
+
# Source code that runs in Linux containers / cross-platform tooling: force LF.
|
|
8
|
+
*.sh text eol=lf
|
|
9
|
+
*.bash text eol=lf
|
|
10
|
+
*.py text eol=lf
|
|
11
|
+
Dockerfile text eol=lf
|
|
12
|
+
*.dockerfile text eol=lf
|
|
13
|
+
docker-compose.yaml text eol=lf
|
|
14
|
+
docker-compose.yml text eol=lf
|
|
15
|
+
*.toml text eol=lf
|
|
16
|
+
*.yaml text eol=lf
|
|
17
|
+
*.yml text eol=lf
|
|
18
|
+
*.json text eol=lf
|
|
19
|
+
*.md text eol=lf
|
|
20
|
+
Makefile text eol=lf
|
|
21
|
+
*.mk text eol=lf
|
|
22
|
+
|
|
23
|
+
# PowerShell expects CRLF on Windows. Keep as-is so PS5.1 parses cleanly.
|
|
24
|
+
*.ps1 text eol=crlf
|
|
25
|
+
*.psd1 text eol=crlf
|
|
26
|
+
*.psm1 text eol=crlf
|
|
27
|
+
|
|
28
|
+
# Windows batch files require CRLF.
|
|
29
|
+
*.bat text eol=crlf
|
|
30
|
+
*.cmd text eol=crlf
|
|
31
|
+
|
|
32
|
+
# Binary assets: never touch line endings.
|
|
33
|
+
*.png binary
|
|
34
|
+
*.jpg binary
|
|
35
|
+
*.jpeg binary
|
|
36
|
+
*.gif binary
|
|
37
|
+
*.ico binary
|
|
38
|
+
*.pdf binary
|
|
39
|
+
*.zip binary
|
|
40
|
+
*.gz binary
|
|
41
|
+
*.tgz binary
|
|
42
|
+
*.tar binary
|
|
43
|
+
*.whl binary
|
|
44
|
+
*.so binary
|
|
45
|
+
*.dll binary
|
|
46
|
+
*.exe binary
|