skill-lab 0.1.0__tar.gz

skill_lab-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 8ddieHu0314
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

skill_lab-0.1.0/PKG-INFO

@@ -0,0 +1,257 @@
+Metadata-Version: 2.4
+Name: skill-lab
+Version: 0.1.0
+Summary: A Python CLI tool for evaluating agent skills through static analysis and quality checks
+Author-email: Eddie Hu <eddiehu0314@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/8ddieHu0314/Skill-Lab
+Project-URL: Documentation, https://github.com/8ddieHu0314/Skill-Lab#readme
+Project-URL: Repository, https://github.com/8ddieHu0314/Skill-Lab
+Project-URL: Issues, https://github.com/8ddieHu0314/Skill-Lab/issues
+Project-URL: Releases, https://github.com/8ddieHu0314/Skill-Lab/releases
+Keywords: agent,skills,evaluation,cli,static-analysis,quality,SKILL.md,ai-agents
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer>=0.9.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0; extra == "dev"
+Requires-Dist: mypy>=1.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Dynamic: license-file
+
+# Skill Lab
+
+[![PyPI version](https://badge.fury.io/py/skill-lab.svg)](https://badge.fury.io/py/skill-lab)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Tests](https://github.com/8ddieHu0314/Skill-Lab/actions/workflows/test.yml/badge.svg)](https://github.com/8ddieHu0314/Skill-Lab/actions/workflows/test.yml)
+
+A Python CLI tool for evaluating agent skills through static analysis and quality checks.
+
+## Features
+
+- **SKILL.md Parsing**: Parse YAML frontmatter and markdown body from skill definitions
+- **18 Static Checks**: Comprehensive checks across 4 dimensions
+  - Structure: File existence, folder organization, frontmatter validation
+  - Naming: Format, directory matching
+  - Description: Length, trigger information
+  - Content: Examples, line budget, reference depth
+- **Quality Scoring**: Weighted 0-100 score based on check results
+- **Multiple Output Formats**: Console (rich formatting) and JSON
+- **Trace Evaluation**: Analyze execution traces against defined checks
+- **Trigger Testing**: Verify skill activation with different prompt types
+
+## Installation
+
+```bash
+# From PyPI
+pip install skill-lab
+
+# From source
+pip install -e .
+
+# With development dependencies
+pip install -e ".[dev]"
+```
+
+## Quick Start
+
+```bash
+# Evaluate a skill
+sklab evaluate ./my-skill
+
+# Quick validation (pass/fail)
+sklab validate ./my-skill
+
+# List available checks
+sklab list-checks
+```
+
+## Usage
+
+### Evaluate a Skill
+
+```bash
+# Console output (default)
+sklab evaluate ./my-skill
+
+# JSON output
+sklab evaluate ./my-skill --format json
+
+# Save to file
+sklab evaluate ./my-skill --output report.json
+
+# Verbose (show all checks, not just failures)
+sklab evaluate ./my-skill --verbose
+
+# Spec-only (skip quality suggestions)
+sklab evaluate ./my-skill --spec-only
+```
+
+### Quick Validation
+
+```bash
+# Returns exit code 0 if valid, 1 if invalid
+sklab validate ./my-skill
+```
+
+### List Available Checks
+
+```bash
+# List all checks
+sklab list-checks
+
+# Filter by dimension
+sklab list-checks --dimension structure
+
+# Show only spec-required checks
+sklab list-checks --spec-only
+```
+
+### Test Triggers
+
+```bash
+# Run trigger tests
+sklab test-triggers ./my-skill
+
+# Filter by trigger type
+sklab test-triggers ./my-skill --type explicit
+```
+
+### Evaluate Traces
+
+```bash
+# Evaluate an execution trace
+sklab eval-trace ./my-skill --trace ./trace.jsonl
+```
+
+## Check Categories
+
+### Structure Checks
+| Check ID | Severity | Description |
+|----------|----------|-------------|
+| `structure.skill-md-exists` | ERROR | SKILL.md file exists |
+| `structure.valid-frontmatter` | ERROR | YAML frontmatter is parseable |
+| `frontmatter.compatibility-length` | ERROR | Compatibility under 500 chars |
+| `frontmatter.metadata-format` | ERROR | Metadata is string-to-string map |
+| `structure.scripts-valid` | WARNING | /scripts contains valid files |
+| `structure.references-valid` | WARNING | /references contains valid files |
+
+### Naming Checks
+| Check ID | Severity | Description |
+|----------|----------|-------------|
+| `naming.required` | ERROR | Name field is present |
+| `naming.format` | ERROR | Lowercase, hyphens only, max 64 chars |
+| `naming.matches-directory` | ERROR | Name matches parent directory |
+
+### Description Checks
+| Check ID | Severity | Description |
+|----------|----------|-------------|
+| `description.required` | ERROR | Description field is present |
+| `description.not-empty` | ERROR | Description is not empty |
+| `description.max-length` | ERROR | Max 1024 characters |
+| `description.includes-triggers` | INFO | Describes when to use |
+
+### Content Checks
+| Check ID | Severity | Description |
+|----------|----------|-------------|
+| `content.body-not-empty` | WARNING | Body has content (min 50 chars) |
+| `content.line-budget` | WARNING | Under 500 lines |
+| `content.has-examples` | INFO | Contains code examples |
+| `content.reference-depth` | WARNING | References max 1 level deep |
+
+## Output Format (JSON)
+
+```json
+{
+  "skill_path": "/path/to/skill",
+  "skill_name": "my-skill",
+  "timestamp": "2026-01-25T14:30:00Z",
+  "duration_ms": 45.3,
+  "quality_score": 87.5,
+  "overall_pass": true,
+  "checks_run": 18,
+  "checks_passed": 19,
+  "checks_failed": 2,
+  "results": [...],
+  "summary": {
+    "by_severity": {...},
+    "by_dimension": {...}
+  }
+}
+```
+
+## Development
+
+```bash
+# Install dev dependencies
+pip install -e ".[dev]"
+
+# Run tests
+pytest tests/ -v
+
+# Run with coverage
+pytest tests/ --cov=skill_lab
+
+# Type checking
+mypy src/
+
+# Linting
+ruff check src/
+
+# Format code
+ruff format src/
+```
+
+## Project Structure
+
+```
+sklab/
+├── src/skill_lab/
+│   ├── cli.py                    # CLI interface (Typer)
+│   ├── core/
+│   │   ├── models.py             # Data models
+│   │   ├── registry.py           # Check registration
+│   │   └── scoring.py            # Quality scoring
+│   ├── parsers/
+│   │   └── skill_parser.py       # SKILL.md parsing
+│   ├── checks/
+│   │   ├── base.py               # Base check class
+│   │   └── static/               # Static checks
+│   ├── evaluators/
+│   │   ├── static_evaluator.py   # Static analysis
+│   │   └── trace_evaluator.py    # Trace evaluation
+│   ├── triggers/
+│   │   └── trigger_evaluator.py  # Trigger testing
+│   └── reporters/
+│       ├── json_reporter.py
+│       └── console_reporter.py
+├── tests/
+├── docs/
+└── examples/
+```
+
+## Related
+
+- [Agent Skills Specification](https://agentskills.io/specification) - The specification this tool validates against
+- [SkillsMP](https://skillsmp.com) - Marketplace for agent skills
+
+## License
+
+MIT

skill_lab-0.1.0/README.md

@@ -0,0 +1,220 @@
+# Skill Lab
+
+[![PyPI version](https://badge.fury.io/py/skill-lab.svg)](https://badge.fury.io/py/skill-lab)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Tests](https://github.com/8ddieHu0314/Skill-Lab/actions/workflows/test.yml/badge.svg)](https://github.com/8ddieHu0314/Skill-Lab/actions/workflows/test.yml)
+
+A Python CLI tool for evaluating agent skills through static analysis and quality checks.
+
+## Features
+
+- **SKILL.md Parsing**: Parse YAML frontmatter and markdown body from skill definitions
+- **18 Static Checks**: Comprehensive checks across 4 dimensions
+  - Structure: File existence, folder organization, frontmatter validation
+  - Naming: Format, directory matching
+  - Description: Length, trigger information
+  - Content: Examples, line budget, reference depth
+- **Quality Scoring**: Weighted 0-100 score based on check results
+- **Multiple Output Formats**: Console (rich formatting) and JSON
+- **Trace Evaluation**: Analyze execution traces against defined checks
+- **Trigger Testing**: Verify skill activation with different prompt types
+
+## Installation
+
+```bash
+# From PyPI
+pip install skill-lab
+
+# From source
+pip install -e .
+
+# With development dependencies
+pip install -e ".[dev]"
+```
+
+## Quick Start
+
+```bash
+# Evaluate a skill
+sklab evaluate ./my-skill
+
+# Quick validation (pass/fail)
+sklab validate ./my-skill
+
+# List available checks
+sklab list-checks
+```
+
+## Usage
+
+### Evaluate a Skill
+
+```bash
+# Console output (default)
+sklab evaluate ./my-skill
+
+# JSON output
+sklab evaluate ./my-skill --format json
+
+# Save to file
+sklab evaluate ./my-skill --output report.json
+
+# Verbose (show all checks, not just failures)
+sklab evaluate ./my-skill --verbose
+
+# Spec-only (skip quality suggestions)
+sklab evaluate ./my-skill --spec-only
+```
+
+### Quick Validation
+
+```bash
+# Returns exit code 0 if valid, 1 if invalid
+sklab validate ./my-skill
+```
+
+### List Available Checks
+
+```bash
+# List all checks
+sklab list-checks
+
+# Filter by dimension
+sklab list-checks --dimension structure
+
+# Show only spec-required checks
+sklab list-checks --spec-only
+```
+
+### Test Triggers
+
+```bash
+# Run trigger tests
+sklab test-triggers ./my-skill
+
+# Filter by trigger type
+sklab test-triggers ./my-skill --type explicit
+```
+
+### Evaluate Traces
+
+```bash
+# Evaluate an execution trace
+sklab eval-trace ./my-skill --trace ./trace.jsonl
+```
+
+## Check Categories
+
+### Structure Checks
+| Check ID | Severity | Description |
+|----------|----------|-------------|
+| `structure.skill-md-exists` | ERROR | SKILL.md file exists |
+| `structure.valid-frontmatter` | ERROR | YAML frontmatter is parseable |
+| `frontmatter.compatibility-length` | ERROR | Compatibility under 500 chars |
+| `frontmatter.metadata-format` | ERROR | Metadata is string-to-string map |
+| `structure.scripts-valid` | WARNING | /scripts contains valid files |
+| `structure.references-valid` | WARNING | /references contains valid files |
+
+### Naming Checks
+| Check ID | Severity | Description |
+|----------|----------|-------------|
+| `naming.required` | ERROR | Name field is present |
+| `naming.format` | ERROR | Lowercase, hyphens only, max 64 chars |
+| `naming.matches-directory` | ERROR | Name matches parent directory |
+
+### Description Checks
+| Check ID | Severity | Description |
+|----------|----------|-------------|
+| `description.required` | ERROR | Description field is present |
+| `description.not-empty` | ERROR | Description is not empty |
+| `description.max-length` | ERROR | Max 1024 characters |
+| `description.includes-triggers` | INFO | Describes when to use |
+
+### Content Checks
+| Check ID | Severity | Description |
+|----------|----------|-------------|
+| `content.body-not-empty` | WARNING | Body has content (min 50 chars) |
+| `content.line-budget` | WARNING | Under 500 lines |
+| `content.has-examples` | INFO | Contains code examples |
+| `content.reference-depth` | WARNING | References max 1 level deep |
+
+## Output Format (JSON)
+
+```json
+{
+  "skill_path": "/path/to/skill",
+  "skill_name": "my-skill",
+  "timestamp": "2026-01-25T14:30:00Z",
+  "duration_ms": 45.3,
+  "quality_score": 87.5,
+  "overall_pass": true,
+  "checks_run": 18,
+  "checks_passed": 19,
+  "checks_failed": 2,
+  "results": [...],
+  "summary": {
+    "by_severity": {...},
+    "by_dimension": {...}
+  }
+}
+```
+
+## Development
+
+```bash
+# Install dev dependencies
+pip install -e ".[dev]"
+
+# Run tests
+pytest tests/ -v
+
+# Run with coverage
+pytest tests/ --cov=skill_lab
+
+# Type checking
+mypy src/
+
+# Linting
+ruff check src/
+
+# Format code
+ruff format src/
+```
+
+## Project Structure
+
+```
+sklab/
+├── src/skill_lab/
+│   ├── cli.py                    # CLI interface (Typer)
+│   ├── core/
+│   │   ├── models.py             # Data models
+│   │   ├── registry.py           # Check registration
+│   │   └── scoring.py            # Quality scoring
+│   ├── parsers/
+│   │   └── skill_parser.py       # SKILL.md parsing
+│   ├── checks/
+│   │   ├── base.py               # Base check class
+│   │   └── static/               # Static checks
+│   ├── evaluators/
+│   │   ├── static_evaluator.py   # Static analysis
+│   │   └── trace_evaluator.py    # Trace evaluation
+│   ├── triggers/
+│   │   └── trigger_evaluator.py  # Trigger testing
+│   └── reporters/
+│       ├── json_reporter.py
+│       └── console_reporter.py
+├── tests/
+├── docs/
+└── examples/
+```
+
+## Related
+
+- [Agent Skills Specification](https://agentskills.io/specification) - The specification this tool validates against
+- [SkillsMP](https://skillsmp.com) - Marketplace for agent skills
+
+## License
+
+MIT

skill_lab-0.1.0/pyproject.toml

@@ -0,0 +1,128 @@
+[project]
+name = "skill-lab"
+version = "0.1.0"
+description = "A Python CLI tool for evaluating agent skills through static analysis and quality checks"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [
+    { name = "Eddie Hu", email = "eddiehu0314@gmail.com" }
+]
+keywords = [
+    "agent",
+    "skills",
+    "evaluation",
+    "cli",
+    "static-analysis",
+    "quality",
+    "SKILL.md",
+    "ai-agents",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Quality Assurance",
+    "Topic :: Software Development :: Testing",
+    "Typing :: Typed",
+]
+dependencies = [
+    "typer>=0.9.0",
+    "rich>=13.0.0",
+    "pyyaml>=6.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "pytest-cov>=4.0",
+    "mypy>=1.0",
+    "ruff>=0.1.0",
+]
+
+[project.scripts]
+sklab = "skill_lab.cli:app"
+
+[project.urls]
+Homepage = "https://github.com/8ddieHu0314/Skill-Lab"
+Documentation = "https://github.com/8ddieHu0314/Skill-Lab#readme"
+Repository = "https://github.com/8ddieHu0314/Skill-Lab"
+Issues = "https://github.com/8ddieHu0314/Skill-Lab/issues"
+Releases = "https://github.com/8ddieHu0314/Skill-Lab/releases"
+
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+# Ruff - Linting and formatting
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+src = ["src"]
+
+[tool.ruff.lint]
+select = [
+    "E",      # pycodestyle errors
+    "F",      # pyflakes
+    "I",      # isort
+    "W",      # pycodestyle warnings
+    "UP",     # pyupgrade
+    "B",      # flake8-bugbear
+    "SIM",    # flake8-simplify
+]
+ignore = [
+    "E501",   # line too long (handled by formatter)
+]
+
+[tool.ruff.lint.isort]
+known-first-party = ["skill_lab"]
+
+# MyPy - Type checking
+[tool.mypy]
+python_version = "3.10"
+strict = true
+warn_return_any = true
+warn_unused_ignores = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+no_implicit_optional = true
+show_error_codes = true
+
+[[tool.mypy.overrides]]
+module = ["yaml.*"]
+ignore_missing_imports = true
+
+# Pytest - Testing
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = "test_*.py"
+python_functions = "test_*"
+addopts = "-v --tb=short"
+filterwarnings = [
+    "ignore::DeprecationWarning",
+]
+
+# Coverage
+[tool.coverage.run]
+source = ["src/skill_lab"]
+branch = true
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise NotImplementedError",
+    "if TYPE_CHECKING:",
+    "if __name__ == .__main__.:",
+]
+show_missing = true

skill_lab-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

skill_lab-0.1.0/src/skill_lab/__init__.py

@@ -0,0 +1,8 @@
+"""Skill Lab - Agent Skills Evaluation Framework.
+
+A Python CLI tool for evaluating agent skills through static analysis and quality checks.
+"""
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__"]

skill_lab-0.1.0/src/skill_lab/__main__.py

@@ -0,0 +1,6 @@
+"""Entry point for python -m skill_lab."""
+
+from skill_lab.cli import app
+
+if __name__ == "__main__":
+    app()

skill_lab-0.1.0/src/skill_lab/checks/__init__.py

@@ -0,0 +1,5 @@
+"""Check implementations for skill evaluation."""
+
+from skill_lab.checks.base import StaticCheck
+
+__all__ = ["StaticCheck"]