skillnet-ai 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

skillnet_ai/evaluator.py

@@ -5,7 +5,7 @@ import os
 import shlex
 import subprocess
 import time
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import ThreadPoolExecutor, as_completed, wait, FIRST_COMPLETED
 from dataclasses import dataclass
 from typing import Dict, Any, List, Optional, Tuple, Callable, Iterator
 
@@ -69,6 +69,9 @@ class Skill:
         normalized_url = cls._normalize_url(url)
         if not normalized_url:
             return None, f"Invalid GitHub URL: {url}"
+        # Derive skill name from URL if not provided
+        name = kwargs.get('name') or normalized_url.rstrip('/').split('/')[-1]
+
         # Download to local cache (with retries in evaluator)
         local_path = None
         for attempt in range(max_retries):
@@ -83,8 +86,6 @@ class Skill:
                 time.sleep(retry_delay)
         if not local_path:
             return None, f"Failed to download after {max_retries} retries: {url}"
-        # Derive skill name from URL if not provided
-        name = kwargs.get('name') or normalized_url.rstrip('/').split('/')[-1]
 
         return cls(
             path=local_path,
@@ -835,26 +836,55 @@ class SkillEvaluator:
         Returns:
             List of evaluation results in the same order as input.
         """
-        results = [None] * len(skills)
-        
-        with ThreadPoolExecutor(max_workers=self.config.max_workers) as executor:
-            future_to_idx = {
-                executor.submit(self.evaluate, skill): idx 
-                for idx, skill in enumerate(skills)
-            }
-            
+        results: List[Optional[Dict[str, Any]]] = [None] * len(skills)
+
+        max_workers = max(1, int(self.config.max_workers))
+        max_in_flight = max_workers * 2
+
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            it = iter(enumerate(skills))
+            in_flight: set = set()
+            future_to_idx: Dict[Any, int] = {}
+
+            def _submit_next() -> bool:
+                try:
+                    idx, skill = next(it)
+                except StopIteration:
+                    return False
+                fut = executor.submit(self.evaluate, skill)
+                in_flight.add(fut)
+                future_to_idx[fut] = idx
+                return True
+
+            for _ in range(min(max_in_flight, len(skills))):
+                _submit_next()
+
             with tqdm(total=len(skills), desc="Evaluating skills") as pbar:
-                for future in as_completed(future_to_idx):
-                    idx = future_to_idx[future]
-                    results[idx] = future.result()
-                    pbar.update(1)
-        
-        return results
+                try:
+                    while in_flight:
+                        done, pending = wait(in_flight, return_when=FIRST_COMPLETED)
+                        in_flight = set(pending)
+                        for fut in done:
+                            idx = future_to_idx.pop(fut)
+                            results[idx] = fut.result()
+                            pbar.update(1)
+                            if len(in_flight) < max_in_flight:
+                                _submit_next()
+                except KeyboardInterrupt:
+                    for fut in in_flight:
+                        fut.cancel()
+                    executor.shutdown(wait=False, cancel_futures=True)
+                    raise
+
+        return [r if r is not None else self._create_error_result("Missing result") for r in results]
     
     def evaluate_from_url(self, url: str, **kwargs) -> Dict[str, Any]:
         """Convenience helper: create and evaluate a skill from a URL."""
         skill, err = Skill.from_url(
-            url, self.downloader, self.config.cache_dir, **kwargs
+            url,
+            self.downloader,
+            self.config.cache_dir,
+            **kwargs
         )
         if err:
             return self._create_error_result(err)
@@ -896,9 +926,11 @@ if __name__ == '__main__':
     parser.add_argument('--output', required=True, help='Output JSONL file')
     parser.add_argument('--api-key', help='OpenAI API key')
     parser.add_argument('--base-url', help='OpenAI API base URL')
+    parser.add_argument('--github-token', help='GitHub token for downloading private repos or avoiding rate limits')
     parser.add_argument('--model', default='gpt-4o', help='Model name')
     parser.add_argument('--max-workers', type=int, default=5, help='Max workers')
-    parser.add_argument('--cache-dir', default='./evaluate_cache_dir', help='Cache directory')
+    parser.add_argument('--cache-dir', default='skill_downloads', help='Cache directory')
+    parser.add_argument('--limit', type=int, default=-1, help='Limit number of input records to process (0 = all)')
     parser.add_argument('--run-scripts', action='store_true',
                         help='Execute python scripts under scripts/')
     parser.add_argument('--script-timeout', type=int, default=8,
@@ -917,15 +949,18 @@ if __name__ == '__main__':
     def _load_records(jsonl_path: str) -> List[Dict[str, Any]]:
         with open(jsonl_path, 'r', encoding='utf-8') as f:
             return [json.loads(line) for line in f if line.strip()]
-    
-    def _build_skills(
+
+    def _evaluate_records_streaming(
         records: List[Dict[str, Any]],
         evaluator: 'SkillEvaluator',
         config: EvaluatorConfig,
-    ) -> Tuple[List[Optional[Skill]], Dict[int, str]]:
-        skills: List[Optional[Skill]] = []
-        errors: Dict[int, str] = {}
-        for idx, rec in enumerate(records):
+    ) -> List[Dict[str, Any]]:
+        results: List[Optional[Dict[str, Any]]] = [None] * len(records)
+
+        max_workers = max(1, int(config.max_workers))
+        max_in_flight = max_workers * 2
+
+        def _run_one(idx: int, rec: Dict[str, Any]) -> Dict[str, Any]:
             if 'skill_url' in rec:
                 skill, err = Skill.from_url(
                     rec['skill_url'],
@@ -933,43 +968,61 @@ if __name__ == '__main__':
                     config.cache_dir,
                     name=rec.get('skill_name'),
                     description=rec.get('skill_description'),
-                    category=rec.get('category')
+                    category=rec.get('category'),
                 )
-            elif 'skill_path' in rec:
+                if err:
+                    return evaluator._create_error_result(err)
+                return evaluator.evaluate(skill)
+
+            if 'skill_path' in rec:
                 skill, err = Skill.from_path(
                     rec['skill_path'],
                     name=rec.get('skill_name'),
                     description=rec.get('skill_description'),
-                    category=rec.get('category')
+                    category=rec.get('category'),
                 )
-            else:
-                raise ValueError("Record must have 'skill_url' or 'skill_path'")
+                if err:
+                    return evaluator._create_error_result(err)
+                return evaluator.evaluate(skill)
 
-            if err:
-                errors[idx] = err
-                skills.append(None)
-            else:
-                skills.append(skill)
-        return skills, errors
+            return evaluator._create_error_result("Record must have 'skill_url' or 'skill_path'")
 
-    def _evaluate_records(
-        records: List[Dict[str, Any]],
-        evaluator: 'SkillEvaluator',
-        skills: List[Optional[Skill]],
-        errors: Dict[int, str],
-    ) -> List[Dict[str, Any]]:
-        skills_to_eval = [(idx, s) for idx, s in enumerate(skills) if s is not None]
-        idx_to_result: Dict[int, Dict[str, Any]] = {}
-        if skills_to_eval:
-            indices, valid_skills = zip(*skills_to_eval)
-            batch_results = evaluator.evaluate_batch(list(valid_skills))
-            idx_to_result = dict(zip(indices, batch_results))
-        return [
-            evaluator._create_error_result(errors[idx])
-            if idx in errors
-            else idx_to_result[idx]
-            for idx in range(len(records))
-        ]
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            it = iter(enumerate(records))
+            in_flight: set = set()
+            future_to_idx: Dict[Any, int] = {}
+
+            def _submit_next() -> bool:
+                try:
+                    idx, rec = next(it)
+                except StopIteration:
+                    return False
+                fut = executor.submit(_run_one, idx, rec)
+                in_flight.add(fut)
+                future_to_idx[fut] = idx
+                return True
+
+            for _ in range(min(max_in_flight, len(records))):
+                _submit_next()
+
+            with tqdm(total=len(records), desc="Downloading & evaluating") as pbar:
+                try:
+                    while in_flight:
+                        done, pending = wait(in_flight, return_when=FIRST_COMPLETED)
+                        in_flight = set(pending)
+                        for fut in done:
+                            idx = future_to_idx.pop(fut)
+                            results[idx] = fut.result()
+                            pbar.update(1)
+                            if len(in_flight) < max_in_flight:
+                                _submit_next()
+                except KeyboardInterrupt:
+                    for fut in in_flight:
+                        fut.cancel()
+                    executor.shutdown(wait=False, cancel_futures=True)
+                    raise
+
+        return [r if r is not None else evaluator._create_error_result("Missing result") for r in results]
     
     def _write_outputs(records: List[Dict[str, Any]], output_jsonl_path: str) -> str:
         with open(output_jsonl_path, 'w', encoding='utf-8') as f:
@@ -992,12 +1045,15 @@ if __name__ == '__main__':
         max_script_runs=args.max_script_runs,
         script_python=args.script_python,
         include_script_results=args.include_script_results,
-        max_script_output_chars=args.max_script_output_chars
+        max_script_output_chars=args.max_script_output_chars,
+        github_token=args.github_token or os.getenv('GITHUB_TOKEN'),
     )
     evaluator = SkillEvaluator(config)
     records = _load_records(args.input)
-    skills, download_errors = _build_skills(records, evaluator, config)
-    results = _evaluate_records(records, evaluator, skills, download_errors)
+    if args.limit and args.limit > 0:
+        records = records[: args.limit]
+
+    results = _evaluate_records_streaming(records, evaluator, config)
     for rec, result in zip(records, results):
         rec['evaluation'] = result
     json_path = _write_outputs(records, args.output)

{skillnet_ai-0.0.4.dist-info → skillnet_ai-0.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skillnet-ai
-Version: 0.0.4
+Version: 0.0.6
 Summary: Official Python SDK for SkillNet: Create, Evaluate, and Connect AI Skills.
 Author-email: SkillNet Team <liangyuannnnn@gmail.com>
 License: MIT
@@ -9,26 +9,26 @@ Project-URL: Source, https://github.com/zjunlp/SkillNet
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: requests>=2.25.0
-Requires-Dist: openai<2.0,>=1.0
-Requires-Dist: pydantic>=2.0.0
-Requires-Dist: tqdm>=4.0.0
-Requires-Dist: typer>=0.9.0
-Requires-Dist: rich>=13.0.0
-Requires-Dist: PyPDF2>=3.0.0
-Requires-Dist: pycryptodome>=3.20.0
-Requires-Dist: python-docx>=0.8.11
-Requires-Dist: python-pptx>=0.6.21
+Requires-Dist: requests==2.32.5
+Requires-Dist: openai==1.109.1
+Requires-Dist: pydantic==2.12.5
+Requires-Dist: tqdm==4.67.1
+Requires-Dist: typer==0.21.1
+Requires-Dist: rich==13.9.4
+Requires-Dist: PyPDF2==3.0.1
+Requires-Dist: pycryptodome==3.23.0
+Requires-Dist: python-docx==1.2.0
+Requires-Dist: python-pptx==1.0.2
 Dynamic: license-file
 
 # skillnet-ai
 
 [![PyPI version](https://badge.fury.io/py/skillnet-ai.svg)](https://badge.fury.io/py/skillnet-ai)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
-[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/)
+[![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
 
 **skillnet-ai** is the official Python Toolkit for interacting with the SkillNet platform. It allows AI Agents to **Create**, **Evaluate** and **Organize** AI skills at scale. It functions seamlessly as both a powerful Python Library and a feature-rich Command Line Interface (CLI).
 
@@ -265,12 +265,25 @@ skillnet download <private_url> --token <your_github_token>
 
 Generate structured Skill Packages from various sources using LLMs.
 
+**Linux/macOS:**
+
 ```bash
 # Requirement: Ensure API_KEY is set in your environment variables.
 export API_KEY=sk-xxxxx
-export BASE_URL= xxxxxx # Optional custom LLM base URL
+export BASE_URL=xxxxxx  # Optional custom LLM base URL
+```
+
+**Windows PowerShell:**
+
+```powershell
+# Requirement: Ensure API_KEY is set in your environment variables.
+$env:API_KEY = "sk-xxxxx"
+$env:BASE_URL = "xxxxxx"  # Optional custom LLM base URL
+```
 
+**Usage Examples:**
 
+```bash
 # From a trajectory file
 skillnet create ./logs/trajectory.txt -d ./generated_skills
 
@@ -290,11 +303,26 @@ skillnet create --office report.pdf --model gpt-4o
 ### 4. Evaluate Skills (`evaluate`)
 Generate a comprehensive quality report (Safety, Completeness, Executability, Modifiability, Cost Awareness) for a skill.
 
+**Linux/macOS:**
+
 ```bash
 # Requirement: Ensure API_KEY is set in your environment variables.
 export API_KEY=sk-xxxxx
-export BASE_URL= xxxxxx # Optional custom LLM base URL
+export BASE_URL=xxxxxx  # Optional custom LLM base URL
+```
 
+**Windows PowerShell:**
+
+```powershell
+# Requirement: Ensure API_KEY is set in your environment variables.
+$env:API_KEY = "sk-xxxxx"
+$env:BASE_URL = "xxxxxx"  # Optional custom LLM base URL
+```
+
+**Usage Examples:**
+
+```bash
+# Requirement: Ensure API_KEY is set in your environment variables.
 # Evaluate a remote skill via GitHub URL
 skillnet evaluate https://github.com/anthropics/skills/tree/main/skills/algorithmic-art
 
@@ -308,12 +336,25 @@ skillnet evaluate ./my_skills/tool --category "Development" --model gpt-4o
 ### 5. Analyze Relationships (`analyze`)
 Scan a directory of skills to analyze their connections using AI.
 
+**Linux/macOS:**
 
 ```bash
 # Requirement: Ensure API_KEY is set in your environment variables.
 export API_KEY=sk-xxxxx
-export BASE_URL= xxxxxx # Optional custom LLM base URL
+export BASE_URL=xxxxxx  # Optional custom LLM base URL
+```
+
+**Windows PowerShell:**
 
+```powershell
+# Requirement: Ensure API_KEY is set in your environment variables.
+$env:API_KEY = "sk-xxxxx"
+$env:BASE_URL = "xxxxxx"  # Optional custom LLM base URL
+```
+
+**Usage Examples:**
+
+```bash
 # Analyze a directory containing multiple skill folders
 skillnet analyze ./my_agent_skills
 

{skillnet_ai-0.0.4.dist-info → skillnet_ai-0.0.6.dist-info}/RECORD

@@ -4,13 +4,13 @@ skillnet_ai/cli.py,sha256=g_geqiDJU3-JKHTKzfN0U_Mx3_Ye38Dk2VAv3pqEOwo,24443
 skillnet_ai/client.py,sha256=1vKuBUQMCSAybYddsuXlkix3sGW1KD-gqeFuTcVuzHM,11491
 skillnet_ai/creator.py,sha256=srJn_sV7mDWwTb1AG9QrXf2fDeA5pGBUCDXIYxkdQk0,39130
 skillnet_ai/downloader.py,sha256=egP7b1ovL4pbNuAqn8y17-6YseyPpJzWlJAohH9XeWE,6358
-skillnet_ai/evaluator.py,sha256=_x_9ijlF_R-doK2kiyIpRfVSdwQ2rly7seDXGmGNwpE,36446
+skillnet_ai/evaluator.py,sha256=cB3-2YDxUIrT9Wq5eSAPnHxVziSN03gqOGIWcn15n0w,38876
 skillnet_ai/models.py,sha256=2af5PnXhkzhRKUUc02KMr5U77KbhlTyxSlJETmWdmZY,1162
 skillnet_ai/prompts.py,sha256=afxSxV9b_UPhTCQdahgCAnQoRVwMNweOmT3r9aVwxY0,40377
 skillnet_ai/searcher.py,sha256=MvinkWR8omSCMXKSdZTSBBsifxQ13ep4hpnd4ql2w7U,3170
-skillnet_ai-0.0.4.dist-info/licenses/LICENSE,sha256=N5w66QPiGia1KA-Drt_75rmKLsyVt5GJUu-j6Gxpihs,1063
-skillnet_ai-0.0.4.dist-info/METADATA,sha256=UKNgnLUWsxy_OFfbx2gvybsckdi7EasUuQemSw-iKVc,12771
-skillnet_ai-0.0.4.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-skillnet_ai-0.0.4.dist-info/entry_points.txt,sha256=8JDzqrJcxks90e4ifiivf2KNeGMGLW-f7nRNi_f1bQQ,49
-skillnet_ai-0.0.4.dist-info/top_level.txt,sha256=0gHowx1tlCRYxEbdoQcH4nPoMZGWUAUX6wjSNv3VUoI,12
-skillnet_ai-0.0.4.dist-info/RECORD,,
+skillnet_ai-0.0.6.dist-info/licenses/LICENSE,sha256=N5w66QPiGia1KA-Drt_75rmKLsyVt5GJUu-j6Gxpihs,1063
+skillnet_ai-0.0.6.dist-info/METADATA,sha256=AQxxcvQiXa8C4GJm9IWVqU8nJx9blzQnkREm_Y7tIgA,13575
+skillnet_ai-0.0.6.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+skillnet_ai-0.0.6.dist-info/entry_points.txt,sha256=8JDzqrJcxks90e4ifiivf2KNeGMGLW-f7nRNi_f1bQQ,49
+skillnet_ai-0.0.6.dist-info/top_level.txt,sha256=0gHowx1tlCRYxEbdoQcH4nPoMZGWUAUX6wjSNv3VUoI,12
+skillnet_ai-0.0.6.dist-info/RECORD,,