sanityops-cli 0.1.5__tar.gz → 0.1.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/PKG-INFO +38 -14
  2. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/README.md +36 -12
  3. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/pyproject.toml +2 -2
  4. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/__init__.py +1 -1
  5. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/agents/scanner_agent/agent.py +98 -2
  6. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/agents/scanner_agent/models/finding.py +56 -7
  7. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/agents/scanner_agent/prompts.py +13 -6
  8. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/defect_checker/checker.py +8 -3
  9. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/defect_checker/markdown_reporter.py +42 -0
  10. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/defect_checker/renderer.py +74 -6
  11. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/agents/scanner_agent/models/test_finding.py +38 -0
  12. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/agents/scanner_agent/test_agent.py +9 -1
  13. sanityops_cli-0.1.7/tests/unit/agents/scanner_agent/test_deterministic_capture.py +284 -0
  14. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/defect_checker/test_checker.py +2 -1
  15. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/defect_checker/test_markdown_reporter.py +226 -0
  16. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/defect_checker/test_renderer.py +296 -0
  17. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
  18. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/.github/ISSUE_TEMPLATE/config.yml +0 -0
  19. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
  20. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
  21. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/.github/dependabot.yml +0 -0
  22. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/.github/scripts/pr_review_bot.py +0 -0
  23. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/.github/workflows/ai-review.yml +0 -0
  24. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/.github/workflows/ci.yml +0 -0
  25. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/.github/workflows/release.yml +0 -0
  26. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/.gitignore +0 -0
  27. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/CHANGELOG.md +0 -0
  28. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/CODE_OF_CONDUCT.md +0 -0
  29. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/CONTRIBUTING.md +0 -0
  30. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/LICENSE +0 -0
  31. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/NOTICE +0 -0
  32. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/SECURITY.md +0 -0
  33. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/plans/2026-08-13-model-config-comments-in-init-template.md +0 -0
  34. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/plans/2026-08-14-loading-animations.md +0 -0
  35. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/plans/2026-08-14-skills-files-banner-readme.md +0 -0
  36. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/plans/2026-08-28-copyright-compliance.md +0 -0
  37. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/plans/2026-09-08-config-command.md +0 -0
  38. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/plans/2026-09-20-cross-artifact-rendering.md +0 -0
  39. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/plans/2026-09-20-help-panel-migration.md +0 -0
  40. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/plans/2026-09-20-repair-feature.md +0 -0
  41. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/specs/2026-08-13-model-config-comments-in-init-template-design.md +0 -0
  42. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/specs/2026-08-14-loading-animations-design.md +0 -0
  43. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/specs/2026-08-14-skills-files-banner-readme-design.md +0 -0
  44. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/specs/2026-08-28-copyright-compliance-design.md +0 -0
  45. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/specs/2026-09-08-config-command-design.md +0 -0
  46. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/specs/2026-09-20-cross-artifact-rendering-design.md +0 -0
  47. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/specs/2026-09-20-help-panel-migration-design.md +0 -0
  48. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/docs/superpowers/specs/2026-09-20-repair-feature-design.md +0 -0
  49. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/entry.py +0 -0
  50. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/release.sh +0 -0
  51. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/scripts/generate-checksums.ps1 +0 -0
  52. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/scripts/install.ps1 +0 -0
  53. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/scripts/install.sh +0 -0
  54. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/agents/__init__.py +0 -0
  55. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/agents/repair_agent/__init__.py +0 -0
  56. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/agents/repair_agent/agent.py +0 -0
  57. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/agents/repair_agent/prompts.py +0 -0
  58. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/agents/repair_agent/tools/__init__.py +0 -0
  59. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/agents/repair_agent/tools/store_repairs_tool.py +0 -0
  60. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/agents/scanner_agent/hooks/progress_hook.py +0 -0
  61. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/agents/scanner_agent/tools/grep_tool.py +0 -0
  62. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/agents/scanner_agent/tools/listfiles_tool.py +0 -0
  63. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/agents/scanner_agent/tools/readfile_tool.py +0 -0
  64. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/agents/scanner_agent/tools/storefindings_tool.py +0 -0
  65. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/api/__init__.py +0 -0
  66. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/api/client.py +0 -0
  67. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/commands/__init__.py +0 -0
  68. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/commands/config.py +0 -0
  69. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/commands/init.py +0 -0
  70. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/commands/inspect.py +0 -0
  71. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/constants/__init__.py +0 -0
  72. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/constants/config_defaults.py +0 -0
  73. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/constants/exit_codes.py +0 -0
  74. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/defect_checker/__init__.py +0 -0
  75. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/defect_checker/llm_config.py +0 -0
  76. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/exceptions/__init__.py +0 -0
  77. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/exceptions/api_exceptions.py +0 -0
  78. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/exceptions/base_exceptions.py +0 -0
  79. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/help_panel.py +0 -0
  80. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/logging/__init__.py +0 -0
  81. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/logging/logger.py +0 -0
  82. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/main.py +0 -0
  83. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/progress/__init__.py +0 -0
  84. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/progress/tracker.py +0 -0
  85. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/renderers/__init__.py +0 -0
  86. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/renderers/command_renderer/inspect_command_renderer.py +0 -0
  87. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/templates/__init__.py +0 -0
  88. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/templates/inspect_config.yaml +0 -0
  89. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/utils/__init__.py +0 -0
  90. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/utils/artifact_packer.py +0 -0
  91. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/utils/config_loader.py +0 -0
  92. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/utils/config_resolver.py +0 -0
  93. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/src/sanityops_cli/utils/validators.py +0 -0
  94. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/__init__.py +0 -0
  95. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/commands/__init__.py +0 -0
  96. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/commands/test_config.py +0 -0
  97. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/commands/test_init.py +0 -0
  98. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/commands/test_inspect_repair.py +0 -0
  99. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/test_help_panel.py +0 -0
  100. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/__init__.py +0 -0
  101. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/agents/__init__.py +0 -0
  102. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/agents/repair_agent/__init__.py +0 -0
  103. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/agents/repair_agent/test_agent.py +0 -0
  104. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/agents/repair_agent/test_store_repairs_tool.py +0 -0
  105. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/agents/scanner_agent/__init__.py +0 -0
  106. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/agents/scanner_agent/models/__init__.py +0 -0
  107. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/agents/scanner_agent/test_progress_hook.py +0 -0
  108. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/agents/scanner_agent/test_prompts.py +0 -0
  109. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/api/__init__.py +0 -0
  110. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/api/test_client.py +0 -0
  111. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/commands/test_inspect.py +0 -0
  112. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/commands/test_version.py +0 -0
  113. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/defect_checker/__init__.py +0 -0
  114. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/defect_checker/test_llm_config.py +0 -0
  115. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/logging/__init__.py +0 -0
  116. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/logging/test_logger.py +0 -0
  117. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/progress/__init__.py +0 -0
  118. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/progress/test_tracker.py +0 -0
  119. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/utils/test_artifact_packer.py +0 -0
  120. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/unit/utils/test_config_loader.py +0 -0
  121. {sanityops_cli-0.1.5 → sanityops_cli-0.1.7}/tests/utils/test_config_resolver.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: sanityops-cli
3
- Version: 0.1.5
3
+ Version: 0.1.7
4
4
  Summary: CLI tool for static defect inspection of AI logical artifacts
5
5
  Project-URL: Homepage, https://sanityops.org
6
6
  Project-URL: Documentation, https://sanityops.org
@@ -23,7 +23,7 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
23
23
  Classifier: Topic :: Software Development :: Quality Assurance
24
24
  Requires-Python: >=3.11
25
25
  Requires-Dist: anyio>=4.0
26
- Requires-Dist: defect-check
26
+ Requires-Dist: defect-check>=0.0.1
27
27
  Requires-Dist: httpx>=0.27
28
28
  Requires-Dist: pyyaml>=6.0
29
29
  Requires-Dist: rich>=13.0
@@ -48,9 +48,15 @@ A command-line tool for static defect inspection of AI logical artifacts (System
48
48
 
49
49
  - **Multi-level inspection depth** — Choose between L1 (fast), L2 (standard), or L3 (deep) check levels
50
50
  - **Multiple artifact types** — Inspect System Prompts, Skills, and Tool Schemas
51
- - **CI/CD ready** — Designed for automated pipeline integration
51
+ - **CI/CD friendly** — Non-interactive runs, configuration via environment variables, dated markdown reports for pipeline artifacts
52
52
  - **Configurable rules** — Customize inspection behavior via YAML configuration
53
53
 
54
+ ## Prerequisites
55
+
56
+ - **Python >= 3.11** — only needed when installing from PyPI or source; the curl/PowerShell installers ship a standalone binary with no Python requirement.
57
+ - **An LLM API key** — artifact analysis and defect checking are LLM-powered. Use Anthropic, OpenAI, or any compatible provider, configured either in the `model:` section of the config file or via `LLM_*` environment variables (see [Environment Variables](#environment-variables)).
58
+ - **Optional: a Sanityops server API key** — only if you want scanned artifacts uploaded to the Sanityops web dashboard. See [Data Handling](#data-handling) before enabling this.
59
+
54
60
  ## Installation
55
61
 
56
62
  ### From Curl (Recommended)
@@ -99,9 +105,9 @@ pip install -e .
99
105
  sanityops-cli init
100
106
  ```
101
107
 
102
- This creates `.sanityops/inspect_config.yaml` in your current directory with a
103
- freshly generated project UUID. The generated file matches the template below —
104
- edit the artifact paths to point at your prompts, tools, and skills.
108
+ This creates `.sanityops/inspect_config.yaml` in your current directory from the built-in template. The `project.id` starts as a placeholder UUID (`00000000-0000-0000-0000-000000000000`); on the first `inspect` run with a server API key configured, the project is created on the Sanityops server and the real id/name are written back to this file. `init` also writes a `.gitignore` inside `.sanityops/` so the config file (which may hold API keys) and scan results never get committed.
109
+
110
+ Edit the artifact paths to point at your prompts, tools, and skills.
105
111
 
106
112
  ### 2. Configure Artifacts
107
113
 
@@ -109,7 +115,10 @@ Edit `.sanityops/inspect_config.yaml` to specify your artifacts:
109
115
 
110
116
  ```yaml
111
117
  project:
112
- id: <your-project-id> # required: UUID
118
+ # id starts as a placeholder; on the first inspect run with a server API key,
119
+ # the project is created automatically and the real id/name are written back.
120
+ id: 00000000-0000-0000-0000-000000000000
121
+ name: ""
113
122
 
114
123
  # Optional: omit to use LLM_* environment variables
115
124
  # model:
@@ -147,6 +156,10 @@ sanityops-cli inspect --skip-defect-check
147
156
  sanityops-cli inspect --config path/to/config.yaml
148
157
  ```
149
158
 
159
+ > **Data upload notice:** if a Sanityops server API key is configured (`server.api_key` or `SANITYOPS_API_KEY`), `inspect` automatically uploads the scanned artifacts to the Sanityops server after analysis. Without an API key, everything stays on your machine. See [Data Handling](#data-handling).
160
+
161
+ `inspect` exits `0` when the run completes — including when defects are found — and non-zero on failure. CI pipelines should therefore gate on the generated report (`.sanityops/results/inspect-*.md`), not on the exit code.
162
+
150
163
  ### 4. Generate Repairs (Optional)
151
164
 
152
165
  After inspection, generate repaired artifacts from the report:
@@ -164,6 +177,20 @@ sanityops-cli inspect repair --timeout 3600 --token-budget 500000
164
177
 
165
178
  Repairs are written to `.sanityops/repairs/repair-<timestamp>.md` without modifying source files.
166
179
 
180
+ ## Data Handling
181
+
182
+ What leaves your machine, and when:
183
+
184
+ - **Local by default.** Artifact analysis and defect checking run on your machine and send artifact content only to the LLM provider you configure (`model:` section or `LLM_*` environment variables). Without a Sanityops server API key, nothing is uploaded to the Sanityops server.
185
+ - **Auto-upload is keyed to the server API key.** If `server.api_key` (or `SANITYOPS_API_KEY`) is set, `inspect` uploads the scanned artifacts — System Prompts, Tool Schemas, Skills — to the Sanityops server after analysis, so they appear in the web dashboard. The first such run also creates the project on the server and binds `project.id`.
186
+ - **Server destination.** `server.base_url` defaults to the Sanityops SaaS service (`https://demo.sanityops.org`). Only change it if you run a private deployment:
187
+
188
+ ```bash
189
+ sanityops-cli config server.base_url https://api.your-deployment.com
190
+ ```
191
+
192
+ - **Secrets are protected locally.** API keys are masked in CLI output and redacted from logs, and the `.sanityops/` directory is excluded from git by the `.gitignore` that `init` writes.
193
+
167
194
  ## Configuration
168
195
 
169
196
  ### Config Command
@@ -190,6 +217,8 @@ sanityops-cli config --list
190
217
  sanityops-cli config --unset server.base_url
191
218
  ```
192
219
 
220
+ `server.base_url` defaults to the Sanityops SaaS service (`https://demo.sanityops.org`). Change it only if you use a private deployment.
221
+
193
222
  ### Configuration Precedence
194
223
 
195
224
  Values are resolved in this priority order (highest to lowest):
@@ -205,8 +234,8 @@ This ordering ensures CI/CD pipelines can override any project config by injecti
205
234
 
206
235
  | Variable | Description |
207
236
  |----------|-------------|
208
- | `SANITYOPS_BASE_URL` | Override `server.base_url` |
209
- | `SANITYOPS_API_KEY` | Override `server.api_key` |
237
+ | `SANITYOPS_BASE_URL` | Override `server.base_url` (default: `https://demo.sanityops.org`) |
238
+ | `SANITYOPS_API_KEY` | Override `server.api_key`; when set, `inspect` auto-uploads artifacts to the server |
210
239
  | `LLM_PROVIDER` | LLM provider (e.g., `anthropic`, `openai`) |
211
240
  | `LLM_API_KEY` | LLM API key for defect checking |
212
241
  | `LLM_MODEL_ID` | LLM model ID (e.g., `claude-sonnet-4-20250514`) |
@@ -243,11 +272,6 @@ All CLI operations are logged to `~/.sanityops/logs/sanityops-cli-<YYYY-MM-DD>.l
243
272
 
244
273
  Full documentation is available at [sanityops.org](https://sanityops.org).
245
274
 
246
- ## Requirements
247
-
248
- - Python >= 3.11
249
- - OpenAI API key (or compatible LLM provider) for defect checking
250
-
251
275
  ## Development
252
276
 
253
277
  ### Setup
@@ -8,9 +8,15 @@ A command-line tool for static defect inspection of AI logical artifacts (System
8
8
 
9
9
  - **Multi-level inspection depth** — Choose between L1 (fast), L2 (standard), or L3 (deep) check levels
10
10
  - **Multiple artifact types** — Inspect System Prompts, Skills, and Tool Schemas
11
- - **CI/CD ready** — Designed for automated pipeline integration
11
+ - **CI/CD friendly** — Non-interactive runs, configuration via environment variables, dated markdown reports for pipeline artifacts
12
12
  - **Configurable rules** — Customize inspection behavior via YAML configuration
13
13
 
14
+ ## Prerequisites
15
+
16
+ - **Python >= 3.11** — only needed when installing from PyPI or source; the curl/PowerShell installers ship a standalone binary with no Python requirement.
17
+ - **An LLM API key** — artifact analysis and defect checking are LLM-powered. Use Anthropic, OpenAI, or any compatible provider, configured either in the `model:` section of the config file or via `LLM_*` environment variables (see [Environment Variables](#environment-variables)).
18
+ - **Optional: a Sanityops server API key** — only if you want scanned artifacts uploaded to the Sanityops web dashboard. See [Data Handling](#data-handling) before enabling this.
19
+
14
20
  ## Installation
15
21
 
16
22
  ### From Curl (Recommended)
@@ -59,9 +65,9 @@ pip install -e .
59
65
  sanityops-cli init
60
66
  ```
61
67
 
62
- This creates `.sanityops/inspect_config.yaml` in your current directory with a
63
- freshly generated project UUID. The generated file matches the template below —
64
- edit the artifact paths to point at your prompts, tools, and skills.
68
+ This creates `.sanityops/inspect_config.yaml` in your current directory from the built-in template. The `project.id` starts as a placeholder UUID (`00000000-0000-0000-0000-000000000000`); on the first `inspect` run with a server API key configured, the project is created on the Sanityops server and the real id/name are written back to this file. `init` also writes a `.gitignore` inside `.sanityops/` so the config file (which may hold API keys) and scan results never get committed.
69
+
70
+ Edit the artifact paths to point at your prompts, tools, and skills.
65
71
 
66
72
  ### 2. Configure Artifacts
67
73
 
@@ -69,7 +75,10 @@ Edit `.sanityops/inspect_config.yaml` to specify your artifacts:
69
75
 
70
76
  ```yaml
71
77
  project:
72
- id: <your-project-id> # required: UUID
78
+ # id starts as a placeholder; on the first inspect run with a server API key,
79
+ # the project is created automatically and the real id/name are written back.
80
+ id: 00000000-0000-0000-0000-000000000000
81
+ name: ""
73
82
 
74
83
  # Optional: omit to use LLM_* environment variables
75
84
  # model:
@@ -107,6 +116,10 @@ sanityops-cli inspect --skip-defect-check
107
116
  sanityops-cli inspect --config path/to/config.yaml
108
117
  ```
109
118
 
119
+ > **Data upload notice:** if a Sanityops server API key is configured (`server.api_key` or `SANITYOPS_API_KEY`), `inspect` automatically uploads the scanned artifacts to the Sanityops server after analysis. Without an API key, everything stays on your machine. See [Data Handling](#data-handling).
120
+
121
+ `inspect` exits `0` when the run completes — including when defects are found — and non-zero on failure. CI pipelines should therefore gate on the generated report (`.sanityops/results/inspect-*.md`), not on the exit code.
122
+
110
123
  ### 4. Generate Repairs (Optional)
111
124
 
112
125
  After inspection, generate repaired artifacts from the report:
@@ -124,6 +137,20 @@ sanityops-cli inspect repair --timeout 3600 --token-budget 500000
124
137
 
125
138
  Repairs are written to `.sanityops/repairs/repair-<timestamp>.md` without modifying source files.
126
139
 
140
+ ## Data Handling
141
+
142
+ What leaves your machine, and when:
143
+
144
+ - **Local by default.** Artifact analysis and defect checking run on your machine and send artifact content only to the LLM provider you configure (`model:` section or `LLM_*` environment variables). Without a Sanityops server API key, nothing is uploaded to the Sanityops server.
145
+ - **Auto-upload is keyed to the server API key.** If `server.api_key` (or `SANITYOPS_API_KEY`) is set, `inspect` uploads the scanned artifacts — System Prompts, Tool Schemas, Skills — to the Sanityops server after analysis, so they appear in the web dashboard. The first such run also creates the project on the server and binds `project.id`.
146
+ - **Server destination.** `server.base_url` defaults to the Sanityops SaaS service (`https://demo.sanityops.org`). Only change it if you run a private deployment:
147
+
148
+ ```bash
149
+ sanityops-cli config server.base_url https://api.your-deployment.com
150
+ ```
151
+
152
+ - **Secrets are protected locally.** API keys are masked in CLI output and redacted from logs, and the `.sanityops/` directory is excluded from git by the `.gitignore` that `init` writes.
153
+
127
154
  ## Configuration
128
155
 
129
156
  ### Config Command
@@ -150,6 +177,8 @@ sanityops-cli config --list
150
177
  sanityops-cli config --unset server.base_url
151
178
  ```
152
179
 
180
+ `server.base_url` defaults to the Sanityops SaaS service (`https://demo.sanityops.org`). Change it only if you use a private deployment.
181
+
153
182
  ### Configuration Precedence
154
183
 
155
184
  Values are resolved in this priority order (highest to lowest):
@@ -165,8 +194,8 @@ This ordering ensures CI/CD pipelines can override any project config by injecti
165
194
 
166
195
  | Variable | Description |
167
196
  |----------|-------------|
168
- | `SANITYOPS_BASE_URL` | Override `server.base_url` |
169
- | `SANITYOPS_API_KEY` | Override `server.api_key` |
197
+ | `SANITYOPS_BASE_URL` | Override `server.base_url` (default: `https://demo.sanityops.org`) |
198
+ | `SANITYOPS_API_KEY` | Override `server.api_key`; when set, `inspect` auto-uploads artifacts to the server |
170
199
  | `LLM_PROVIDER` | LLM provider (e.g., `anthropic`, `openai`) |
171
200
  | `LLM_API_KEY` | LLM API key for defect checking |
172
201
  | `LLM_MODEL_ID` | LLM model ID (e.g., `claude-sonnet-4-20250514`) |
@@ -203,11 +232,6 @@ All CLI operations are logged to `~/.sanityops/logs/sanityops-cli-<YYYY-MM-DD>.l
203
232
 
204
233
  Full documentation is available at [sanityops.org](https://sanityops.org).
205
234
 
206
- ## Requirements
207
-
208
- - Python >= 3.11
209
- - OpenAI API key (or compatible LLM provider) for defect checking
210
-
211
235
  ## Development
212
236
 
213
237
  ### Setup
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "sanityops-cli"
7
- version = "0.1.5"
7
+ version = "0.1.7"
8
8
  description = "CLI tool for static defect inspection of AI logical artifacts"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -31,7 +31,7 @@ dependencies = [
31
31
  "httpx>=0.27",
32
32
  "tomli>=2.0",
33
33
  "tomli-w>=1.0",
34
- "defect-check",
34
+ "defect-check>=0.0.1",
35
35
  "pyyaml>=6.0",
36
36
  "sanityops-agent>=0.0.4",
37
37
  "anyio>=4.0",
@@ -13,4 +13,4 @@
13
13
  # limitations under the License.
14
14
  #
15
15
 
16
- __version__ = "0.1.5"
16
+ __version__ = "0.1.7"
@@ -13,12 +13,21 @@
13
13
  # limitations under the License.
14
14
  #
15
15
 
16
+ import re
16
17
  from pathlib import Path
17
18
 
18
19
  import anyio
20
+ import yaml
19
21
  from rich.console import Console
20
22
 
21
- from sanityops_cli.agents.scanner_agent.models.finding import FindingsResult
23
+ from sanityops_cli.agents.scanner_agent.models.finding import (
24
+ Finding,
25
+ FindingsResult,
26
+ FindingType,
27
+ PromptContent,
28
+ Section,
29
+ SkillContent,
30
+ )
22
31
  from sanityops_cli.agents.scanner_agent.prompts import (
23
32
  # New analyzer prompts
24
33
  ANALYZE_PARENT_PROMPT,
@@ -239,7 +248,94 @@ class ScannerAgent:
239
248
  error_msg += f"\nError Detail:\n{error_detail_str}"
240
249
  raise ValidationError(f"Agent execution failed: {error_msg}")
241
250
 
242
- return self._build_result("", result)
251
+ findings_result = self._build_result("", result)
252
+
253
+ # Deterministic artifact capture: LLM transcription of prompt and
254
+ # skill files is lossy (random truncation/frontmatter loss), so
255
+ # replace the LLM's findings for those artifacts with content read
256
+ # verbatim from disk.
257
+ prompt_findings: list[Finding] = []
258
+ for prompt_path in prompts:
259
+ try:
260
+ content = Path(prompt_path).read_text(encoding="utf-8")
261
+ except (FileNotFoundError, PermissionError, UnicodeDecodeError, IsADirectoryError) as e:
262
+ raise ValidationError(f"Failed to read prompt file {prompt_path}: {e}") from e
263
+ prompt_findings.append(
264
+ Finding(
265
+ type=FindingType.PROMPT,
266
+ relative=prompt_path, # Absolute path (validated upstream)
267
+ content=PromptContent(content=content),
268
+ )
269
+ )
270
+ findings_result.prompts = prompt_findings
271
+
272
+ skill_findings: list[Finding] = []
273
+ for skill_path in skills:
274
+ try:
275
+ raw = Path(skill_path).read_text(encoding="utf-8")
276
+ except (FileNotFoundError, PermissionError, UnicodeDecodeError, IsADirectoryError) as e:
277
+ raise ValidationError(f"Failed to read skill file {skill_path}: {e}") from e
278
+
279
+ # Parse frontmatter: handle both standard format and EOF edge case
280
+ # Standard: "---\n<yaml>\n---\n<body>"
281
+ # EOF case: "---\n<yaml>\n---" (no trailing newline)
282
+ frontmatter: dict = {}
283
+ body = raw
284
+ if raw.startswith("---"):
285
+ # Try standard pattern first, then EOF pattern
286
+ match = re.match(r"^---\s*\n(.*?)\n---\s*\n?", raw, re.DOTALL)
287
+ if match:
288
+ yaml_content = match.group(1)
289
+ try:
290
+ parsed = yaml.safe_load(yaml_content)
291
+ if isinstance(parsed, dict):
292
+ frontmatter = parsed
293
+ except yaml.YAMLError as e:
294
+ # Log warning but continue with empty frontmatter
295
+ if self.logger:
296
+ self.logger.warning(f"YAML parse error in {skill_path}: {e}")
297
+ frontmatter = {}
298
+ body = raw[match.end():]
299
+
300
+ sections: list[Section] = []
301
+ current_title = "Overview"
302
+ current_lines: list[str] = []
303
+ for line in body.split("\n"):
304
+ if line.startswith("## "):
305
+ if current_lines:
306
+ sections.append(
307
+ Section(
308
+ title=current_title,
309
+ content="\n".join(current_lines).strip(),
310
+ )
311
+ )
312
+ current_title = line[3:].strip()
313
+ current_lines = []
314
+ else:
315
+ current_lines.append(line)
316
+ if current_lines:
317
+ sections.append(
318
+ Section(
319
+ title=current_title,
320
+ content="\n".join(current_lines).strip(),
321
+ )
322
+ )
323
+
324
+ skill_findings.append(
325
+ Finding(
326
+ type=FindingType.SKILL,
327
+ relative=skill_path, # Absolute path (validated upstream)
328
+ content=SkillContent(
329
+ name=str(frontmatter.get("name", "")),
330
+ description=str(frontmatter.get("description", "")),
331
+ frontmatter=frontmatter,
332
+ sections=sections,
333
+ ),
334
+ )
335
+ )
336
+ findings_result.skills = skill_findings
337
+
338
+ return findings_result
243
339
 
244
340
  def analyze_files_sync(
245
341
  self,
@@ -18,6 +18,7 @@ from enum import StrEnum
18
18
  from pathlib import Path
19
19
  from typing import Any
20
20
 
21
+ import yaml
21
22
  from pydantic import BaseModel, Field, ValidationInfo, field_validator, model_validator
22
23
 
23
24
 
@@ -41,22 +42,70 @@ class ToolSchema(BaseModel):
41
42
 
42
43
  class SkillContent(BaseModel):
43
44
  """Extracted content from a skill.md file."""
45
+
44
46
  name: str = Field(description="Skill name from frontmatter")
45
47
  description: str = Field(description="Skill description from frontmatter")
48
+ frontmatter: dict[str, Any] = Field(
49
+ default_factory=dict,
50
+ description="Raw YAML frontmatter fields (preserved for QDS-0 gate)",
51
+ )
46
52
  sections: list[Section] = Field(default_factory=list, description="Parsed markdown sections")
47
53
 
54
+ #: Frontmatter fields that must be positive integers (QDS-0.4). LLM-based
55
+ #: extraction can stringify numeric values, so coerce them back to int.
56
+ _POSITIVE_INT_FIELDS: tuple[str, ...] = (
57
+ "max_items", "max_chars", "timeout_seconds", "max_tool_calls", "max_retries",
58
+ )
59
+
60
+ @model_validator(mode="after")
61
+ def _normalize_frontmatter_types(self) -> "SkillContent":
62
+ # Create a normalized copy to avoid in-place mutation
63
+ normalized = dict(self.frontmatter)
64
+ for field_name in self._POSITIVE_INT_FIELDS:
65
+ value = normalized.get(field_name)
66
+ if isinstance(value, str) and value.strip().isdigit():
67
+ normalized[field_name] = int(value.strip())
68
+ elif isinstance(value, float) and not isinstance(value, bool):
69
+ # YAML may parse integers as floats (e.g., "5" -> 5.0)
70
+ normalized[field_name] = int(value)
71
+ elif isinstance(value, bool):
72
+ # bool is a subclass of int; reject explicit true/false for int fields
73
+ normalized.pop(field_name, None)
74
+ # int values are already correct, no conversion needed
75
+ boolean_fields = ("partial_result_allowed",)
76
+ for field_name in boolean_fields:
77
+ value = normalized.get(field_name)
78
+ if isinstance(value, str):
79
+ lowered = value.strip().lower()
80
+ if lowered in ("true", "false"):
81
+ normalized[field_name] = lowered == "true"
82
+ self.frontmatter = normalized
83
+ return self
84
+
48
85
  def to_markdown(self) -> str:
49
- """Reconstruct sections back to markdown string.
86
+ """Reconstruct the full skill document: frontmatter plus sections.
50
87
 
51
- Returns:
52
- Markdown string with all sections joined by double newlines.
88
+ The frontmatter is re-serialized ahead of the body so downstream
89
+ consumers (e.g. the QDS-0 gate) can re-parse the declared metadata.
53
90
  """
91
+ parts: list[str] = []
92
+ if self.frontmatter:
93
+ # yaml is imported at module level; safe_dump preserves frontmatter.
94
+ parts.append(
95
+ "---\n"
96
+ + yaml.safe_dump(self.frontmatter, sort_keys=False, allow_unicode=True)
97
+ + "---\n"
98
+ )
54
99
  if not self.sections:
55
- return ""
56
- return "\n\n".join(
57
- f"## {section.title}\n\n{section.content}"
58
- for section in self.sections
100
+ if not parts:
101
+ return ""
102
+ parts.append(
103
+ "\n\n".join(
104
+ f"## {section.title}\n\n{section.content}"
105
+ for section in self.sections
106
+ )
59
107
  )
108
+ return "\n".join(parts)
60
109
 
61
110
 
62
111
  class ToolContent(BaseModel):
@@ -190,7 +190,8 @@ For each skill file:
190
190
  1. Read the file content using `read` tool
191
191
  2. Extract from YAML frontmatter: `name` and `description` fields
192
192
  3. Parse markdown sections (## Headings) and capture their content
193
- 4. Report immediately via store_findings
193
+ 4. Capture ALL frontmatter fields verbatim into the `frontmatter` object
194
+ 5. Report immediately via store_findings
194
195
 
195
196
  ## Output Schema
196
197
  store_findings({{
@@ -201,6 +202,11 @@ store_findings({{
201
202
  "content": {{
202
203
  "name": "<from_frontmatter>",
203
204
  "description": "<from_frontmatter>",
205
+ "frontmatter": {{
206
+ "<frontmatter_key>": "<value>",
207
+ "name": "<from_frontmatter>",
208
+ "description": "<from_frontmatter>"
209
+ }},
204
210
  "sections": [
205
211
  {{"title": "Section Title", "content": "Section content..."}}
206
212
  ]
@@ -257,15 +263,16 @@ Return: "Completed: analyzed N tools"
257
263
 
258
264
  PROMPT_ANALYZER_PROMPT = """\
259
265
  You are a prompt analyzer. Analyze the following prompt files and extract full content.
260
-
261
266
  ## Input Files
262
267
  {prompt_files}
263
268
 
264
269
  ## Extraction Rules
265
270
  For each prompt file:
266
- 1. Read the file content using `read` tool
267
- 2. Capture full content verbatim
268
- 3. Report immediately via store_findings
271
+ 1. Read the file content using `read` tool (use multiple offset/limit calls if needed so EVERY line is read)
272
+ 2. Capture full content VERBATIM — copy every line exactly, including all markdown headers, tables, code blocks, and examples, from the first line to the last line of the file
273
+ 3. NEVER summarize, truncate, abbreviate, or paraphrase any part of the prompt; the content field must be byte-identical to the file (minus line numbers)
274
+ 4. Before reporting, verify your captured content ends with the file's actual last line; if it does not, re-read the missing portion and include it
275
+ 5. Report immediately via store_findings
269
276
 
270
277
  ## Output Schema
271
278
  store_findings({{
@@ -279,7 +286,7 @@ store_findings({{
279
286
  }}
280
287
  }})
281
288
 
282
- **CRITICAL**: Report each file immediately after analysis. Do not batch.
289
+ **CRITICAL**: Report each file immediately after analysis. Do not batch. The prompt `content` you report is the artifact that gets inspected — a truncated or summarized copy produces false defects.
283
290
 
284
291
  ## Completion
285
292
  Return: "Completed: analyzed N prompts"
@@ -70,11 +70,16 @@ class DefectChecker:
70
70
  content = finding.content
71
71
  if content is None or finding.type != FindingType.TOOL:
72
72
  continue
73
- result.append({
73
+ # NOTE: inputSchema is the canonical key for the tool input schema.
74
+ # We also include "parameters" as an alias for backward compatibility
75
+ # with older consumers. This alias will be deprecated in a future release.
76
+ schema = {
74
77
  "name": content.name,
75
78
  "description": content.description,
76
- "parameters": content.parameters,
77
- })
79
+ "inputSchema": content.parameters,
80
+ "parameters": content.parameters, # Deprecated: use inputSchema
81
+ }
82
+ result.append(schema)
78
83
  return result
79
84
 
80
85
  def _convert_prompts(self, prompts: list[Finding]) -> list[dict]:
@@ -77,6 +77,40 @@ def _aggregate_cross_defects(cross_results: list[dict]) -> list[dict]:
77
77
  return defects
78
78
 
79
79
 
80
+ def _format_permission_table(defects: list[dict]) -> list[str]:
81
+ """Generate extended markdown table for QD-PM permission defects.
82
+
83
+ Columns: ID | Name | Severity | Description | Action | Permission Side | Duty Side | Fix
84
+
85
+ Note: Missing fields display as em dash (—) via _esc_md_cell(None).
86
+ This differs from terminal renderer which silently omits missing fields.
87
+ """
88
+ lines = []
89
+ lines.append("| ID | Name | Severity | Description | Action | Permission Side | Duty Side | Fix |")
90
+ lines.append("|---|---|---|---|---|---|---|---|")
91
+
92
+ for defect in defects:
93
+ details = defect.get("details", {})
94
+ if not isinstance(details, dict):
95
+ details = {}
96
+
97
+ defect_id = _esc_md_cell(defect.get("id") or "unknown-permission-defect")
98
+ name = _esc_md_cell(defect.get("name"))
99
+ severity = _esc_md_cell(defect.get("severity"))
100
+ description = _esc_md_cell(defect.get("description"))
101
+ action = _esc_md_cell(details.get("action"))
102
+ permission_side = _esc_md_cell(details.get("permission_side"))
103
+ duty_side = _esc_md_cell(details.get("duty_side"))
104
+ fix = _esc_md_cell(defect.get("fix_suggestion"))
105
+
106
+ lines.append(
107
+ f"| {defect_id} | {name} | {severity} | {description} | {action} | "
108
+ f"{permission_side} | {duty_side} | {fix} |"
109
+ )
110
+
111
+ return lines
112
+
113
+
80
114
  def _calculate_cross_score(
81
115
  defects: list[dict],
82
116
  check_level: str,
@@ -188,6 +222,14 @@ def _build_report(
188
222
  lines.append("")
189
223
  continue
190
224
 
225
+ # Use permission-specific table for QD-PM module.
226
+ # Renderer also detects via module (primary), category, or ID prefix for consistency.
227
+ if module == "QD-PM":
228
+ lines.extend(_format_permission_table(defects))
229
+ lines.append("")
230
+ continue
231
+
232
+ # Standard defect table for other modules
191
233
  lines.append("| ID | Name | Severity | Description | Location | Impact | Fix |")
192
234
  lines.append("|---|---|---|---|---|---|---|")
193
235
  for defect in defects: