skillnet-ai 0.0.1__py3-none-any.whl → 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- skillnet_ai/__init__.py +23 -0
- skillnet_ai/analyzer.py +222 -0
- skillnet_ai/cli.py +577 -0
- skillnet_ai/client.py +316 -0
- skillnet_ai/creator.py +1026 -0
- skillnet_ai/downloader.py +156 -0
- skillnet_ai/evaluator.py +1006 -0
- skillnet_ai/models.py +41 -0
- skillnet_ai/prompts.py +885 -0
- skillnet_ai/searcher.py +100 -0
- skillnet_ai-0.0.2.dist-info/METADATA +361 -0
- skillnet_ai-0.0.2.dist-info/RECORD +16 -0
- {skillnet_ai-0.0.1.dist-info → skillnet_ai-0.0.2.dist-info}/WHEEL +1 -1
- skillnet_ai-0.0.2.dist-info/entry_points.txt +2 -0
- skillnet_ai-0.0.2.dist-info/licenses/LICENSE +21 -0
- skillnet_ai-0.0.1.dist-info/METADATA +0 -20
- skillnet_ai-0.0.1.dist-info/RECORD +0 -5
- {skillnet_ai-0.0.1.dist-info → skillnet_ai-0.0.2.dist-info}/top_level.txt +0 -0
skillnet_ai/cli.py
ADDED
|
@@ -0,0 +1,577 @@
|
|
|
1
|
+
import typer
|
|
2
|
+
import os
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from rich.console import Console
|
|
5
|
+
from rich.table import Table
|
|
6
|
+
from rich.panel import Panel
|
|
7
|
+
from rich.columns import Columns
|
|
8
|
+
from skillnet_ai.creator import SkillCreator
|
|
9
|
+
from skillnet_ai.downloader import SkillDownloader
|
|
10
|
+
from skillnet_ai.evaluator import SkillEvaluator, EvaluatorConfig
|
|
11
|
+
from skillnet_ai.searcher import SkillNetSearcher
|
|
12
|
+
from skillnet_ai.analyzer import SkillRelationshipAnalyzer
|
|
13
|
+
|
|
14
|
+
app = typer.Typer(help="SkillNet AI CLI Tool")
|
|
15
|
+
console = Console()
|
|
16
|
+
|
|
17
|
+
API_KEY = os.getenv("API_KEY")
|
|
18
|
+
BASE_URL = os.getenv("BASE_URL") or "https://api.openai.com/v1"
|
|
19
|
+
|
|
20
|
+
@app.command()
|
|
21
|
+
def search(
|
|
22
|
+
q: str = typer.Argument(..., help="The search query (keywords or natural language description)."),
|
|
23
|
+
mode: str = typer.Option("keyword", help="Search mode: 'keyword' (exact/fuzzy) or 'vector' (semantic AI)."),
|
|
24
|
+
category: str = typer.Option(None, help="Filter results by category (e.g., 'Development')."),
|
|
25
|
+
limit: int = typer.Option(20, help="Maximum number of results to return."),
|
|
26
|
+
# Keyword specific options
|
|
27
|
+
page: int = typer.Option(1, help="Page number (only for keyword mode)."),
|
|
28
|
+
min_stars: int = typer.Option(0, help="Minimum star rating (only for keyword mode)."),
|
|
29
|
+
sort_by: str = typer.Option("stars", help="Sort criteria: 'stars' or 'recent' (only for keyword mode)."),
|
|
30
|
+
# Vector specific options
|
|
31
|
+
threshold: float = typer.Option(0.8, help="Similarity threshold 0.0-1.0 (only for vector mode)."),
|
|
32
|
+
):
|
|
33
|
+
"""
|
|
34
|
+
Search for skills on SkillNet using Keyword match or Vector (AI) semantic search.
|
|
35
|
+
"""
|
|
36
|
+
try:
|
|
37
|
+
# Initialize Searcher (Ensure URL points to your actual API)
|
|
38
|
+
searcher = SkillNetSearcher()
|
|
39
|
+
|
|
40
|
+
# Visual feedback during API call
|
|
41
|
+
with console.status(f"[bold green]Searching SkillNet ({mode} mode)..."):
|
|
42
|
+
# Assuming searcher.search returns List[SkillModel]
|
|
43
|
+
results = searcher.search(
|
|
44
|
+
q=q,
|
|
45
|
+
mode=mode, # type: ignore
|
|
46
|
+
category=category,
|
|
47
|
+
limit=limit,
|
|
48
|
+
page=page,
|
|
49
|
+
min_stars=min_stars,
|
|
50
|
+
sort_by=sort_by,
|
|
51
|
+
threshold=threshold
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
# Handle Empty Results
|
|
55
|
+
if not results:
|
|
56
|
+
console.print(f"[yellow]No results found for query: '{q}'[/yellow]")
|
|
57
|
+
return
|
|
58
|
+
|
|
59
|
+
# Build Output Table
|
|
60
|
+
table = Table(title=f"Search Results: {q} ({len(results)} items)")
|
|
61
|
+
|
|
62
|
+
# Define Columns
|
|
63
|
+
table.add_column("Name", style="cyan", no_wrap=True)
|
|
64
|
+
table.add_column("Category", style="magenta")
|
|
65
|
+
table.add_column("Stars", justify="right", style="green")
|
|
66
|
+
table.add_column("Description", style="white")
|
|
67
|
+
table.add_column("URL", style="dim blue", overflow="fold") # Added URL column
|
|
68
|
+
|
|
69
|
+
# Note: 'score' was removed because SkillModel definition does not have a 'score' field.
|
|
70
|
+
# If your API returns a score for vector search, you must add it to SkillModel first.
|
|
71
|
+
|
|
72
|
+
for item in results:
|
|
73
|
+
# 1. Access fields using Pydantic dot notation
|
|
74
|
+
# 2. Use correct field names from SkillModel (e.g. skill_description)
|
|
75
|
+
name = item.skill_name
|
|
76
|
+
cat = item.category if item.category else "N/A"
|
|
77
|
+
stars = str(item.stars)
|
|
78
|
+
desc = item.skill_description if item.skill_description else ""
|
|
79
|
+
url = item.skill_url if item.skill_url else "N/A"
|
|
80
|
+
|
|
81
|
+
# Truncate long descriptions for display
|
|
82
|
+
short_desc = (desc[:100] + '...') if len(desc) > 100 else desc
|
|
83
|
+
|
|
84
|
+
# Prepare row data
|
|
85
|
+
row_data = [
|
|
86
|
+
name,
|
|
87
|
+
cat,
|
|
88
|
+
stars,
|
|
89
|
+
short_desc,
|
|
90
|
+
url
|
|
91
|
+
]
|
|
92
|
+
|
|
93
|
+
table.add_row(*row_data)
|
|
94
|
+
|
|
95
|
+
console.print(table)
|
|
96
|
+
|
|
97
|
+
# Suggest next step
|
|
98
|
+
console.print("\n[dim]Tip: Use 'skillnet download <skill_url>' to get a skill.[/dim]")
|
|
99
|
+
|
|
100
|
+
except Exception as e:
|
|
101
|
+
console.print(f"[bold red]Error during search:[/bold red] {str(e)}")
|
|
102
|
+
# Optional: Print full traceback for debugging
|
|
103
|
+
# console.print_exception()
|
|
104
|
+
raise typer.Exit(code=1)
|
|
105
|
+
|
|
106
|
+
@app.command()
|
|
107
|
+
def download(
|
|
108
|
+
url: str = typer.Argument(..., help="The GitHub URL of the specific skill folder (e.g., https://github.com/owner/repo/tree/main/skills/math_solver)."),
|
|
109
|
+
target_dir: str = typer.Option(".", "--target-dir", "-d", help="Local directory to install the skill into."),
|
|
110
|
+
token: str = typer.Option(None, "--token", "-t", envvar="GITHUB_TOKEN", help="GitHub Personal Access Token (for private repos or higher rate limits)."),
|
|
111
|
+
):
|
|
112
|
+
"""
|
|
113
|
+
Download and install a specific skill directly from a GitHub repository subdirectory.
|
|
114
|
+
"""
|
|
115
|
+
# 1. Initialize Downloader
|
|
116
|
+
# Checks CLI option first, then environment variable GITHUB_TOKEN
|
|
117
|
+
downloader = SkillDownloader(api_token=token)
|
|
118
|
+
|
|
119
|
+
try:
|
|
120
|
+
# 2. Visual Feedback
|
|
121
|
+
console.print(f"[dim]Target directory: {os.path.abspath(target_dir)}[/dim]")
|
|
122
|
+
|
|
123
|
+
with console.status(f"[bold green]Downloading skill from GitHub...[/bold green]", spinner="dots"):
|
|
124
|
+
installed_path = downloader.download(folder_url=url, target_dir=target_dir)
|
|
125
|
+
|
|
126
|
+
# 3. Handle Results
|
|
127
|
+
if installed_path:
|
|
128
|
+
# Success
|
|
129
|
+
folder_name = os.path.basename(installed_path)
|
|
130
|
+
|
|
131
|
+
table = Table(title="Installation Successful", show_header=False, box=None)
|
|
132
|
+
table.add_row("[bold cyan]Skill:[/bold cyan]", folder_name)
|
|
133
|
+
table.add_row("[bold cyan]Location:[/bold cyan]", installed_path)
|
|
134
|
+
|
|
135
|
+
console.print(table)
|
|
136
|
+
console.print(f"\n[green]✓ {folder_name} is ready to use.[/green]")
|
|
137
|
+
else:
|
|
138
|
+
# Failure (Logic handled inside class, but we catch the None return)
|
|
139
|
+
console.print("[bold red]Download Failed.[/bold red]")
|
|
140
|
+
console.print("Possible reasons:")
|
|
141
|
+
console.print("1. The URL format is incorrect (must point to a specific folder, not just the repo root).")
|
|
142
|
+
console.print("2. The repository is private and no token was provided.")
|
|
143
|
+
console.print("3. GitHub API rate limits exceeded (try providing a token).")
|
|
144
|
+
raise typer.Exit(code=1)
|
|
145
|
+
|
|
146
|
+
except Exception as e:
|
|
147
|
+
console.print(f"[bold red]An unexpected error occurred:[/bold red] {str(e)}")
|
|
148
|
+
raise typer.Exit(code=1)
|
|
149
|
+
|
|
150
|
+
@app.command()
|
|
151
|
+
def create(
|
|
152
|
+
# Input sources (mutually exclusive)
|
|
153
|
+
trajectory_file: Path = typer.Argument(None, exists=True, readable=True, help="Path to trajectory/log file."),
|
|
154
|
+
github: str = typer.Option(None, "--github", "-g", help="GitHub repository URL (e.g., https://github.com/owner/repo)."),
|
|
155
|
+
office: Path = typer.Option(None, "--office", "-o", exists=True, readable=True, help="Path to office document (PDF, PPT, Word)."),
|
|
156
|
+
prompt: str = typer.Option(None, "--prompt", "-p", help="Direct description to generate skill from."),
|
|
157
|
+
# Output options
|
|
158
|
+
output_dir: Path = typer.Option(Path("./generated_skills"), "--output-dir", "-d", help="Directory to save generated skills."),
|
|
159
|
+
# Model options
|
|
160
|
+
model: str = typer.Option("gpt-4o", "--model", "-m", help="LLM model to use (e.g., gpt-4o, gpt-3.5-turbo)."),
|
|
161
|
+
max_files: int = typer.Option(20, "--max-files", help="Max Python files to analyze (--github only)."),
|
|
162
|
+
):
|
|
163
|
+
"""
|
|
164
|
+
Create executable Skill packages using AI.
|
|
165
|
+
|
|
166
|
+
Supports four modes:
|
|
167
|
+
- From trajectory: skillnet create trajectory.txt
|
|
168
|
+
- From GitHub: skillnet create --github https://github.com/owner/repo
|
|
169
|
+
- From Office doc: skillnet create --office document.pdf
|
|
170
|
+
- From prompt: skillnet create --prompt "Create a skill for..."
|
|
171
|
+
"""
|
|
172
|
+
# 1. Validate Environment
|
|
173
|
+
if not API_KEY:
|
|
174
|
+
console.print("[bold red]Error:[/bold red] API_KEY environment variable is not set.")
|
|
175
|
+
console.print("Please export API_KEY or set it in your environment.")
|
|
176
|
+
raise typer.Exit(code=1)
|
|
177
|
+
|
|
178
|
+
# 2. Determine mode based on provided options
|
|
179
|
+
mode_count = sum([
|
|
180
|
+
bool(github),
|
|
181
|
+
bool(trajectory_file),
|
|
182
|
+
bool(office),
|
|
183
|
+
bool(prompt)
|
|
184
|
+
])
|
|
185
|
+
|
|
186
|
+
if mode_count == 0:
|
|
187
|
+
console.print("[bold red]Error:[/bold red] Must specify one input source.")
|
|
188
|
+
console.print("\nUsage examples:")
|
|
189
|
+
console.print(" skillnet create trajectory.txt")
|
|
190
|
+
console.print(" skillnet create --github https://github.com/owner/repo")
|
|
191
|
+
console.print(" skillnet create --office document.pdf")
|
|
192
|
+
console.print(' skillnet create --prompt "Create a skill for web scraping"')
|
|
193
|
+
raise typer.Exit(code=1)
|
|
194
|
+
|
|
195
|
+
if mode_count > 1:
|
|
196
|
+
console.print("[bold red]Error:[/bold red] Only one input source can be specified at a time.")
|
|
197
|
+
raise typer.Exit(code=1)
|
|
198
|
+
|
|
199
|
+
# 3. Route to appropriate handler
|
|
200
|
+
if github:
|
|
201
|
+
_create_from_github(github, output_dir, model, max_files)
|
|
202
|
+
elif trajectory_file:
|
|
203
|
+
_create_from_trajectory(trajectory_file, output_dir, model)
|
|
204
|
+
elif office:
|
|
205
|
+
_create_from_office(office, output_dir, model)
|
|
206
|
+
elif prompt:
|
|
207
|
+
_create_from_prompt(prompt, output_dir, model)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _create_from_trajectory(trajectory_file: Path, output_dir: Path, model: str):
|
|
211
|
+
"""Internal function to create skill from trajectory file."""
|
|
212
|
+
try:
|
|
213
|
+
# Read Trajectory Content
|
|
214
|
+
console.print(f"[dim]Reading trajectory from: {trajectory_file}[/dim]")
|
|
215
|
+
with open(trajectory_file, "r", encoding="utf-8") as f:
|
|
216
|
+
trajectory_content = f.read()
|
|
217
|
+
|
|
218
|
+
if not trajectory_content.strip():
|
|
219
|
+
console.print("[bold red]Error:[/bold red] Trajectory file is empty.")
|
|
220
|
+
raise typer.Exit(code=1)
|
|
221
|
+
|
|
222
|
+
# Initialize Creator
|
|
223
|
+
creator = SkillCreator(
|
|
224
|
+
api_key=API_KEY,
|
|
225
|
+
base_url=BASE_URL,
|
|
226
|
+
model=model
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
# Run Generation with Spinner
|
|
230
|
+
with console.status("[bold green]AI is analyzing trajectory and generating skills...[/bold green]", spinner="dots"):
|
|
231
|
+
created_paths = creator.create_from_trajectory(
|
|
232
|
+
trajectory=trajectory_content,
|
|
233
|
+
output_dir=str(output_dir)
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
# Report Results
|
|
237
|
+
if created_paths:
|
|
238
|
+
console.print(f"\n[bold green]Success! Generated {len(created_paths)} skill(s):[/bold green]")
|
|
239
|
+
|
|
240
|
+
table = Table(show_header=True, header_style="bold magenta")
|
|
241
|
+
table.add_column("Skill Name", style="cyan")
|
|
242
|
+
table.add_column("Location", style="white")
|
|
243
|
+
|
|
244
|
+
for path in created_paths:
|
|
245
|
+
skill_name = os.path.basename(path)
|
|
246
|
+
table.add_row(skill_name, str(path))
|
|
247
|
+
|
|
248
|
+
console.print(table)
|
|
249
|
+
console.print(f"\n[dim]Files saved to: {os.path.abspath(output_dir)}[/dim]")
|
|
250
|
+
else:
|
|
251
|
+
console.print("\n[yellow]Analysis complete, but no clear skills were identified in this trajectory.[/yellow]")
|
|
252
|
+
|
|
253
|
+
except Exception as e:
|
|
254
|
+
console.print(f"\n[bold red]Creation Failed:[/bold red] {str(e)}")
|
|
255
|
+
raise typer.Exit(code=1)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _create_from_github(github_url: str, output_dir: Path, model: str, max_files: int):
|
|
259
|
+
"""Internal function to create skill from GitHub repository."""
|
|
260
|
+
try:
|
|
261
|
+
console.print(f"[dim]Creating skill from GitHub: {github_url}[/dim]")
|
|
262
|
+
|
|
263
|
+
# Initialize Creator
|
|
264
|
+
creator = SkillCreator(
|
|
265
|
+
api_key=API_KEY,
|
|
266
|
+
base_url=BASE_URL,
|
|
267
|
+
model=model
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
# Run Generation with Spinner
|
|
271
|
+
with console.status("[bold green]Fetching repository and generating skill...[/bold green]", spinner="dots"):
|
|
272
|
+
created_paths = creator.create_from_github(
|
|
273
|
+
github_url=github_url,
|
|
274
|
+
output_dir=str(output_dir),
|
|
275
|
+
api_token=os.getenv("GITHUB_TOKEN"),
|
|
276
|
+
max_files=max_files
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
# Report Results
|
|
280
|
+
if created_paths:
|
|
281
|
+
console.print(f"\n[bold green]Success! Generated {len(created_paths)} skill(s) from GitHub:[/bold green]")
|
|
282
|
+
|
|
283
|
+
table = Table(show_header=True, header_style="bold magenta")
|
|
284
|
+
table.add_column("Skill Name", style="cyan")
|
|
285
|
+
table.add_column("Location", style="white")
|
|
286
|
+
|
|
287
|
+
for path in created_paths:
|
|
288
|
+
skill_name = os.path.basename(path)
|
|
289
|
+
table.add_row(skill_name, str(path))
|
|
290
|
+
|
|
291
|
+
console.print(table)
|
|
292
|
+
console.print(f"\n[dim]Files saved to: {os.path.abspath(output_dir)}[/dim]")
|
|
293
|
+
console.print("\n[dim]Tip: Use 'skillnet evaluate <skill_path>' to evaluate the generated skill.[/dim]")
|
|
294
|
+
else:
|
|
295
|
+
console.print("\n[yellow]Failed to generate skill from the GitHub repository.[/yellow]")
|
|
296
|
+
|
|
297
|
+
except Exception as e:
|
|
298
|
+
console.print(f"\n[bold red]GitHub Skill Creation Failed:[/bold red] {str(e)}")
|
|
299
|
+
raise typer.Exit(code=1)
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _create_from_office(office_file: Path, output_dir: Path, model: str):
|
|
303
|
+
"""Internal function to create skill from office document."""
|
|
304
|
+
try:
|
|
305
|
+
console.print(f"[dim]Creating skill from office document: {office_file}[/dim]")
|
|
306
|
+
|
|
307
|
+
# Initialize Creator
|
|
308
|
+
creator = SkillCreator(
|
|
309
|
+
api_key=API_KEY,
|
|
310
|
+
base_url=BASE_URL,
|
|
311
|
+
model=model
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
# Run Generation with Spinner
|
|
315
|
+
with console.status("[bold green]Extracting content and generating skill...[/bold green]", spinner="dots"):
|
|
316
|
+
created_paths = creator.create_from_office(
|
|
317
|
+
file_path=str(office_file),
|
|
318
|
+
output_dir=str(output_dir)
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
# Report Results
|
|
322
|
+
if created_paths:
|
|
323
|
+
console.print(f"\n[bold green]Success! Generated {len(created_paths)} skill(s) from document:[/bold green]")
|
|
324
|
+
|
|
325
|
+
table = Table(show_header=True, header_style="bold magenta")
|
|
326
|
+
table.add_column("Skill Name", style="cyan")
|
|
327
|
+
table.add_column("Location", style="white")
|
|
328
|
+
|
|
329
|
+
for path in created_paths:
|
|
330
|
+
skill_name = os.path.basename(path)
|
|
331
|
+
table.add_row(skill_name, str(path))
|
|
332
|
+
|
|
333
|
+
console.print(table)
|
|
334
|
+
console.print(f"\n[dim]Files saved to: {os.path.abspath(output_dir)}[/dim]")
|
|
335
|
+
console.print("\n[dim]Tip: Use 'skillnet evaluate <skill_path>' to evaluate the generated skill.[/dim]")
|
|
336
|
+
else:
|
|
337
|
+
console.print("\n[yellow]Failed to generate skill from the document.[/yellow]")
|
|
338
|
+
|
|
339
|
+
except ImportError as e:
|
|
340
|
+
console.print(f"\n[bold red]Missing Dependency:[/bold red] {str(e)}")
|
|
341
|
+
console.print("\n[dim]Install office document support with:[/dim]")
|
|
342
|
+
console.print(" pip install PyPDF2 pycryptodome python-docx python-pptx")
|
|
343
|
+
raise typer.Exit(code=1)
|
|
344
|
+
except Exception as e:
|
|
345
|
+
console.print(f"\n[bold red]Office Skill Creation Failed:[/bold red] {str(e)}")
|
|
346
|
+
raise typer.Exit(code=1)
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _create_from_prompt(user_prompt: str, output_dir: Path, model: str):
|
|
350
|
+
"""Internal function to create skill from user's prompt description."""
|
|
351
|
+
try:
|
|
352
|
+
console.print(f"[dim]Creating skill from user prompt...[/dim]")
|
|
353
|
+
|
|
354
|
+
# Initialize Creator
|
|
355
|
+
creator = SkillCreator(
|
|
356
|
+
api_key=API_KEY,
|
|
357
|
+
base_url=BASE_URL,
|
|
358
|
+
model=model
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
# Run Generation with Spinner
|
|
362
|
+
with console.status("[bold green]AI is generating your custom skill...[/bold green]", spinner="dots"):
|
|
363
|
+
created_paths = creator.create_from_prompt(
|
|
364
|
+
user_input=user_prompt,
|
|
365
|
+
output_dir=str(output_dir)
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
# Report Results
|
|
369
|
+
if created_paths:
|
|
370
|
+
console.print(f"\n[bold green]Success! Generated {len(created_paths)} skill(s) from your description:[/bold green]")
|
|
371
|
+
|
|
372
|
+
table = Table(show_header=True, header_style="bold magenta")
|
|
373
|
+
table.add_column("Skill Name", style="cyan")
|
|
374
|
+
table.add_column("Location", style="white")
|
|
375
|
+
|
|
376
|
+
for path in created_paths:
|
|
377
|
+
skill_name = os.path.basename(path)
|
|
378
|
+
table.add_row(skill_name, str(path))
|
|
379
|
+
|
|
380
|
+
console.print(table)
|
|
381
|
+
console.print(f"\n[dim]Files saved to: {os.path.abspath(output_dir)}[/dim]")
|
|
382
|
+
console.print("\n[dim]Tip: Use 'skillnet evaluate <skill_path>' to evaluate the generated skill.[/dim]")
|
|
383
|
+
else:
|
|
384
|
+
console.print("\n[yellow]Failed to generate skill from your description.[/yellow]")
|
|
385
|
+
|
|
386
|
+
except Exception as e:
|
|
387
|
+
console.print(f"\n[bold red]Prompt-based Skill Creation Failed:[/bold red] {str(e)}")
|
|
388
|
+
raise typer.Exit(code=1)
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
@app.command()
|
|
392
|
+
def evaluate(
|
|
393
|
+
target: str = typer.Argument(..., help="Path to a local skill directory OR a GitHub URL."),
|
|
394
|
+
|
|
395
|
+
# Optional metadata overrides (useful if not auto-detected)
|
|
396
|
+
name: str = typer.Option(None, help="Name of the skill (overrides auto-detection)."),
|
|
397
|
+
category: str = typer.Option(None, help="Category of the skill (e.g., 'Data Analysis')."),
|
|
398
|
+
description: str = typer.Option(None, help="Short description of what the skill does."),
|
|
399
|
+
|
|
400
|
+
# Config options
|
|
401
|
+
model: str = typer.Option("gpt-4o", "--model", "-m", help="LLM model to use."),
|
|
402
|
+
max_workers: int = typer.Option(5, help="Concurrency for batch operations (not used for single eval)."),
|
|
403
|
+
):
|
|
404
|
+
"""
|
|
405
|
+
Evaluate the quality, safety, and completeness of a skill using AI.
|
|
406
|
+
|
|
407
|
+
Target can be a local folder path or a GitHub URL (e.g., https://github.com/user/repo/tree/main/skill).
|
|
408
|
+
"""
|
|
409
|
+
# 1. Validate Environment
|
|
410
|
+
if not API_KEY:
|
|
411
|
+
console.print("[bold red]Error:[/bold red] API_KEY environment variable is not set.")
|
|
412
|
+
raise typer.Exit(code=1)
|
|
413
|
+
|
|
414
|
+
# 2. Configure Evaluator
|
|
415
|
+
config = EvaluatorConfig(
|
|
416
|
+
api_key=API_KEY,
|
|
417
|
+
base_url=BASE_URL,
|
|
418
|
+
model=model,
|
|
419
|
+
max_workers=max_workers
|
|
420
|
+
)
|
|
421
|
+
evaluator = SkillEvaluator(config)
|
|
422
|
+
|
|
423
|
+
try:
|
|
424
|
+
# 3. Determine Mode (URL vs Local Path) and Run Evaluation
|
|
425
|
+
is_url = target.startswith("http://") or target.startswith("https://")
|
|
426
|
+
|
|
427
|
+
with console.status(f"[bold green]Evaluating skill ({'Remote' if is_url else 'Local'})...[/bold green]", spinner="dots"):
|
|
428
|
+
if is_url:
|
|
429
|
+
result = evaluator.evaluate_from_url(
|
|
430
|
+
url=target,
|
|
431
|
+
name=name,
|
|
432
|
+
category=category,
|
|
433
|
+
description=description
|
|
434
|
+
)
|
|
435
|
+
else:
|
|
436
|
+
result = evaluator.evaluate_from_path(
|
|
437
|
+
path=target,
|
|
438
|
+
name=name,
|
|
439
|
+
category=category,
|
|
440
|
+
description=description
|
|
441
|
+
)
|
|
442
|
+
|
|
443
|
+
# 4. Display Results
|
|
444
|
+
if "error" in result:
|
|
445
|
+
console.print(f"[bold red]Evaluation Failed:[/bold red] {result['error']}")
|
|
446
|
+
raise typer.Exit(code=1)
|
|
447
|
+
|
|
448
|
+
_display_evaluation_report(target, result)
|
|
449
|
+
|
|
450
|
+
except Exception as e:
|
|
451
|
+
console.print(f"[bold red]An unexpected error occurred:[/bold red] {str(e)}")
|
|
452
|
+
raise typer.Exit(code=1)
|
|
453
|
+
|
|
454
|
+
def _display_evaluation_report(target_name: str, data: dict):
|
|
455
|
+
"""Helper to render the JSON evaluation result into a nice Rich UI."""
|
|
456
|
+
console.print(f"\n[bold underline]Evaluation Report: {os.path.basename(target_name)}[/bold underline]\n")
|
|
457
|
+
|
|
458
|
+
# Dimensions to display
|
|
459
|
+
dimensions = ["safety", "completeness", "executability", "modifiability", "cost_awareness"]
|
|
460
|
+
|
|
461
|
+
# Create a grid of panels
|
|
462
|
+
panels = []
|
|
463
|
+
for dim in dimensions:
|
|
464
|
+
info = data.get(dim, {})
|
|
465
|
+
level = info.get("level", "Unknown")
|
|
466
|
+
reason = info.get("reason", "No details provided.")
|
|
467
|
+
|
|
468
|
+
# Color coding based on level
|
|
469
|
+
color = "white"
|
|
470
|
+
if "Excellent" in level: color = "green"
|
|
471
|
+
elif "Good" in level: color = "blue"
|
|
472
|
+
elif "Fair" in level: color = "yellow"
|
|
473
|
+
elif "Poor" in level: color = "red"
|
|
474
|
+
|
|
475
|
+
panel_content = f"[bold]{level}[/bold]\n\n[dim]{reason}[/dim]"
|
|
476
|
+
panels.append(Panel(panel_content, title=f"[{color}]{dim.title()}[/{color}]", expand=True))
|
|
477
|
+
|
|
478
|
+
console.print(Columns(panels, equal=True, expand=True))
|
|
479
|
+
|
|
480
|
+
# Display Score if present
|
|
481
|
+
overall_score = data.get("overall_score")
|
|
482
|
+
if overall_score:
|
|
483
|
+
score_color = "green" if overall_score >= 8 else "yellow" if overall_score >= 5 else "red"
|
|
484
|
+
console.print(f"\n[bold]Overall Score:[/bold] [{score_color}]{overall_score}/10[/{score_color}]")
|
|
485
|
+
|
|
486
|
+
# Summary
|
|
487
|
+
summary = data.get("summary")
|
|
488
|
+
if summary:
|
|
489
|
+
console.print(Panel(summary, title="Executive Summary", border_style="cyan"))
|
|
490
|
+
|
|
491
|
+
@app.command()
|
|
492
|
+
def analyze(
|
|
493
|
+
skills_dir: Path = typer.Argument(..., exists=True, file_okay=False, help="Directory containing multiple skill folders to analyze."),
|
|
494
|
+
save: bool = typer.Option(True, "--save/--no-save", help="Save the result to relationships.json in the directory."),
|
|
495
|
+
model: str = typer.Option("gpt-4o", "--model", "-m", help="LLM model to use."),
|
|
496
|
+
):
|
|
497
|
+
"""
|
|
498
|
+
Analyze and map relationships (similar_to, belong_to, compose_with, depend_on) between local skills.
|
|
499
|
+
|
|
500
|
+
This command scans all subdirectories in the target folder, reads their descriptions,
|
|
501
|
+
and uses AI to build a knowledge graph of how the skills relate to each other.
|
|
502
|
+
"""
|
|
503
|
+
# 1. Validate Environment
|
|
504
|
+
if not API_KEY:
|
|
505
|
+
console.print("[bold red]Error:[/bold red] API_KEY environment variable is not set.")
|
|
506
|
+
raise typer.Exit(code=1)
|
|
507
|
+
|
|
508
|
+
try:
|
|
509
|
+
# 2. Initialize Analyzer
|
|
510
|
+
analyzer = SkillRelationshipAnalyzer(
|
|
511
|
+
api_key=API_KEY,
|
|
512
|
+
base_url=BASE_URL,
|
|
513
|
+
model=model
|
|
514
|
+
)
|
|
515
|
+
|
|
516
|
+
# 3. Visual Feedback & Execution
|
|
517
|
+
console.print(f"[dim]Scanning directory: {os.path.abspath(skills_dir)}[/dim]")
|
|
518
|
+
|
|
519
|
+
results = []
|
|
520
|
+
with console.status("[bold green]Reading skills and analyzing relationships...[/bold green]", spinner="earth"):
|
|
521
|
+
results = analyzer.analyze_local_skills(
|
|
522
|
+
skills_dir=str(skills_dir),
|
|
523
|
+
save_to_file=save
|
|
524
|
+
)
|
|
525
|
+
|
|
526
|
+
# 4. Handle Empty Results
|
|
527
|
+
if not results:
|
|
528
|
+
console.print("\n[yellow]No strong relationships detected among the skills found.[/yellow]")
|
|
529
|
+
console.print("[dim]Make sure the directory contains subfolders with valid SKILL.md or README.md files.[/dim]")
|
|
530
|
+
return
|
|
531
|
+
|
|
532
|
+
# 5. Render Results Table
|
|
533
|
+
console.print(f"\n[bold green]Analysis Complete! Found {len(results)} relationships:[/bold green]\n")
|
|
534
|
+
|
|
535
|
+
table = Table(show_header=True, header_style="bold magenta", title="Skill Relationship Graph")
|
|
536
|
+
table.add_column("Source Skill", style="cyan", no_wrap=True)
|
|
537
|
+
table.add_column("Relationship", style="bold white", justify="center")
|
|
538
|
+
table.add_column("Target Skill", style="cyan", no_wrap=True)
|
|
539
|
+
table.add_column("Reasoning", style="dim")
|
|
540
|
+
|
|
541
|
+
for edge in results:
|
|
542
|
+
# Color code the relationship types for better readability
|
|
543
|
+
rel_type = edge.get('type', 'unknown')
|
|
544
|
+
rel_style = "white"
|
|
545
|
+
arrow = "->"
|
|
546
|
+
|
|
547
|
+
if rel_type == "depend_on":
|
|
548
|
+
rel_style = "red" # Critical dependency
|
|
549
|
+
arrow = "DEPEND ON"
|
|
550
|
+
elif rel_type == "belong_to":
|
|
551
|
+
rel_style = "blue" # Hierarchy
|
|
552
|
+
arrow = "BELONG TO"
|
|
553
|
+
elif rel_type == "compose_with": # pairs_with
|
|
554
|
+
rel_style = "green" # Collaboration
|
|
555
|
+
arrow = "COMPOSE WITH"
|
|
556
|
+
elif rel_type == "similar_to":
|
|
557
|
+
rel_style = "yellow" # Alternative
|
|
558
|
+
arrow = "SIMILAR TO"
|
|
559
|
+
|
|
560
|
+
table.add_row(
|
|
561
|
+
edge.get('source', 'Unknown'),
|
|
562
|
+
f"[{rel_style}]{arrow}[/{rel_style}]",
|
|
563
|
+
edge.get('target', 'Unknown'),
|
|
564
|
+
edge.get('reason', '')
|
|
565
|
+
)
|
|
566
|
+
|
|
567
|
+
console.print(table)
|
|
568
|
+
|
|
569
|
+
if save:
|
|
570
|
+
console.print(f"\n[dim]Relationship data saved to: {os.path.join(skills_dir, 'relationships.json')}[/dim]")
|
|
571
|
+
|
|
572
|
+
except Exception as e:
|
|
573
|
+
console.print(f"\n[bold red]Analysis Failed:[/bold red] {str(e)}")
|
|
574
|
+
raise typer.Exit(code=1)
|
|
575
|
+
|
|
576
|
+
if __name__ == "__main__":
|
|
577
|
+
app()
|