tribunal-kit 9.2.3 → 9.2.4

package/skills/huggingface-tool-builder/references/find_models_by_paper.sh

@@ -0,0 +1,230 @@
+#!/bin/bash
+
+# Find models associated with papers on Hugging Face
+# Usage: ./find_models_by_paper.sh [arXiv_id|search_term]
+# Optional: Set HF_TOKEN environment variable for private/gated models
+
+set -e
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Help function
+show_help() {
+    echo -e "${BLUE}Find models associated with papers on Hugging Face${NC}"
+    echo ""
+    echo -e "${YELLOW}Usage:${NC}"
+    echo "  $0 [OPTIONS] [search_term|arXiv_id]"
+    echo ""
+    echo -e "${YELLOW}Options:${NC}"
+    echo "  --help    Show this help message"
+    echo "  --token   Use HF_TOKEN environment variable (if set)"
+    echo ""
+    echo -e "${YELLOW}Environment:${NC}"
+    echo "  HF_TOKEN  Optional: Hugging Face token for private/gated models"
+    echo ""
+    echo -e "${YELLOW}Examples:${NC}"
+    echo "  $0 1910.01108                    # Search by arXiv ID"
+    echo "  $0 distilbert                     # Search by model name"
+    echo "  $0 transformer                    # Search by keyword"
+    echo "  HF_TOKEN=your_token $0 1910.01108  # Use authentication"
+    echo ""
+    echo -e "${YELLOW}Description:${NC}"
+    echo "This script finds Hugging Face models that are associated with research papers."
+    echo "It searches for models that have arXiv IDs in their tags or mentions papers in their metadata."
+    echo ""
+    echo -e "${YELLOW}Notes:${NC}"
+    echo "• HF_TOKEN is optional for public models"
+    echo "• Use HF_TOKEN for private repositories or gated models"
+    echo "• HF_TOKEN enables higher rate limits for heavy usage"
+}
+
+# Parse arguments
+USE_TOKEN=false
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --help)
+            show_help
+            exit 0
+            ;;
+        --token)
+            USE_TOKEN=true
+            shift
+            ;;
+        -*)
+            echo -e "${RED}Unknown option: $1${NC}"
+            show_help
+            exit 1
+            ;;
+        *)
+            POSITIONAL_ARGS+=("$1")
+            shift
+            ;;
+    esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+if [[ $# -eq 0 ]]; then
+    echo -e "${RED}Error: Please provide a search term or arXiv ID${NC}"
+    echo -e "Use ${YELLOW}$0 --help${NC} for usage information"
+    exit 1
+fi
+
+SEARCH_TERM="$1"
+
+# Set up authentication header if HF_TOKEN is available
+if [[ -n "$HF_TOKEN" ]] && [[ "$USE_TOKEN" == true || -n "$HF_TOKEN" ]]; then
+    AUTH_HEADER="-H \"Authorization: Bearer $HF_TOKEN\""
+    echo -e "${BLUE}Using HF_TOKEN for authentication${NC}"
+else
+    AUTH_HEADER=""
+    if [[ -n "$HF_TOKEN" ]]; then
+        echo -e "${YELLOW}HF_TOKEN found but not using it (add --token flag to use)${NC}"
+    fi
+fi
+
+# Check if the input looks like an arXiv ID (format: YYYY.NNNNN or YYYY.NNNNNNN)
+if [[ "$SEARCH_TERM" =~ ^[0-9]{4}\.[0-9]{4,7}$ ]]; then
+    echo -e "${BLUE}Searching for models associated with arXiv paper: $SEARCH_TERM${NC}"
+    SEARCH_QUERY="arxiv%3A$SEARCH_TERM"
+    IS_ARXIV_SEARCH=true
+else
+    echo -e "${BLUE}Searching for models related to: $SEARCH_TERM${NC}"
+    SEARCH_QUERY="$SEARCH_TERM"
+    IS_ARXIV_SEARCH=false
+fi
+
+# Function to extract arXiv IDs from tags
+extract_arxiv_ids() {
+    local tags="$1"
+    echo "$tags" | jq -r '.[] | select(. | startswith("arxiv:")) | split(":")[1]' 2>/dev/null || true
+}
+
+# Function to get paper title from arXiv ID
+get_paper_title() {
+    local arxiv_id="$1"
+    # Try to get paper title from Hugging Face tags if available
+    # This is a simplified approach - in practice, you might want to call arXiv API
+    echo "Paper Title (arXiv:$arxiv_id)"
+}
+
+# Search for models
+API_URL="https://huggingface.co/api/models"
+echo -e "${YELLOW}Searching Hugging Face API...${NC}"
+
+# Build curl command with authentication if available
+CURL_CMD="curl -s $AUTH_HEADER \"$API_URL?search=$SEARCH_QUERY&limit=50\""
+echo -e "${BLUE}API Query: $API_URL?search=$SEARCH_QUERY&limit=50${NC}"
+
+# Execute the API call
+if [[ -n "$HF_TOKEN" ]]; then
+    RESPONSE=$(curl -s -H "Authorization: Bearer $HF_TOKEN" "$API_URL?search=$SEARCH_QUERY&limit=50" || true)
+else
+    RESPONSE=$(curl -s "$API_URL?search=$SEARCH_QUERY&limit=50" || true)
+fi
+
+# Check if we got a valid response
+if [[ -z "$RESPONSE" ]] || [[ "$RESPONSE" == "[]" ]]; then
+    echo -e "${RED}No models found for search term: $SEARCH_TERM${NC}"
+    
+    # If arXiv search failed, try without arxiv: prefix
+    if [[ "$IS_ARXIV_SEARCH" == true ]]; then
+        echo -e "${YELLOW}Trying broader search without arxiv: prefix...${NC}"
+        SEARCH_QUERY="$SEARCH_TERM"
+        IS_ARXIV_SEARCH=false
+        
+        if [[ -n "$HF_TOKEN" ]]; then
+            RESPONSE=$(curl -s -H "Authorization: Bearer $HF_TOKEN" "$API_URL?search=$SEARCH_QUERY&limit=50" || true)
+        else
+            RESPONSE=$(curl -s "$API_URL?search=$SEARCH_QUERY&limit=50" || true)
+        fi
+        
+        if [[ -z "$RESPONSE" ]] || [[ "$RESPONSE" == "[]" ]]; then
+            echo -e "${RED}Still no results found. Try a different search term.${NC}"
+            exit 1
+        fi
+    else
+        exit 1
+    fi
+fi
+
+# Process the results
+echo -e "${GREEN}Found models! Processing results...${NC}"
+
+# Use jq to process the JSON response and find models with paper associations
+MODELS_WITH_PAPERS=$(echo "$RESPONSE" | jq -r '
+  .[] |
+  select(.id != null) |
+  {
+    id: .id,
+    arxiv_tags: [.tags[] | select(. | startswith("arxiv:"))] | join("; "),
+    downloads: (.downloads // 0),
+    likes: (.likes // 0),
+    task: (.pipeline_tag // "unknown"),
+    library: (.library_name // "unknown")
+  }
+  | @base64' 2>/dev/null || true)
+
+# Count total results
+TOTAL_MODELS=$(echo "$RESPONSE" | jq 'length' 2>/dev/null || echo "0")
+MODELS_WITH_PAPERS_COUNT=$(echo "$MODELS_WITH_PAPERS" | wc -l)
+
+echo -e "${BLUE}Results Summary:${NC}"
+echo -e "  Total models found: $TOTAL_MODELS"
+echo -e "  Models with paper associations: $MODELS_WITH_PAPERS_COUNT"
+echo ""
+
+if [[ -z "$MODELS_WITH_PAPERS" ]]; then
+    # Show all models even if no paper associations found
+    echo -e "${YELLOW}No explicit paper associations found. Showing all matching models:${NC}"
+    echo "$RESPONSE" | jq -r '
+      .[] |
+      select(.id != null) |
+      "📦 \(.id)
+   Task: \(.pipeline_tag // "unknown")
+   Downloads: \(.downloads // 0)
+   Likes: \(.likes // 0)
+   Library: \(.library_name // "unknown")
+   ---"
+    ' 2>/dev/null || echo "Failed to parse response"
+else
+    # Show models with paper associations
+    echo -e "${GREEN}Models with paper associations:${NC}"
+    echo "$MODELS_WITH_PAPERS" | while read -r model_data; do
+        if [[ -n "$model_data" ]]; then
+            # Decode base64 and show formatted
+            echo "$model_data" | base64 -d | jq -r '
+                "📄 \(.id)
+   arXiv: \(.arxiv_tags)
+   Task: \(.task)
+   Downloads: \(.downloads)
+   Likes: \(.likes)
+   Library: \(.library)
+   ---"
+            ' 2>/dev/null || echo "Failed to parse model data"
+        fi
+    done
+fi
+
+# Additional search tips
+echo ""
+echo -e "${BLUE}Search Tips:${NC}"
+echo "• Try searching with the full arXiv ID (e.g., 1910.01108)"
+echo "• Try searching with the paper title keywords"
+echo "• Try searching with the model name"
+echo "• Use HF_TOKEN for private models or higher rate limits"
+echo ""
+echo -e "${BLUE}Examples to try:${NC}"
+echo "  $0 1910.01108                    # DistilBERT paper"
+echo "  $0 1810.04805                    # BERT paper" 
+echo "  $0 1706.03762                    # Attention is All You Need paper"
+echo "  $0 roberta                       # RoBERTa models"
+echo "  $0 transformer                   # Transformer models"
+echo "  HF_TOKEN=your_token $0 1910.01108  # Use authentication"

package/skills/huggingface-tool-builder/references/hf_enrich_models.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+show_help() {
+    cat << 'USAGE'
+Stream model IDs on stdin, emit one JSON object per line (NDJSON).
+
+Usage:
+  hf_enrich_models.sh [MODEL_ID ...]
+  cat ids.txt | hf_enrich_models.sh
+  baseline_hf_api.sh 50 | jq -r '.[].id' | hf_enrich_models.sh
+
+Description:
+  Reads newline-separated model IDs and fetches basic metadata for each.
+  Outputs NDJSON with id, downloads, likes, pipeline_tag, tags.
+  Uses HF_TOKEN for auth if the environment variable is set.
+
+Examples:
+  hf_enrich_models.sh gpt2 distilbert-base-uncased
+  baseline_hf_api.sh 50 | jq -r '.[].id' | hf_enrich_models.sh | jq -s 'sort_by(.downloads)'
+  HF_TOKEN=your_token hf_enrich_models.sh microsoft/DialoGPT-medium
+USAGE
+}
+
+if [[ "${1:-}" == "--help" ]]; then
+    show_help
+    exit 0
+fi
+
+if ! command -v jq >/dev/null 2>&1; then
+    echo "Error: jq is required but not installed" >&2
+    exit 1
+fi
+
+headers=()
+if [[ -n "${HF_TOKEN:-}" ]]; then
+    headers=(-H "Authorization: Bearer ${HF_TOKEN}")
+fi
+
+emit_error() {
+    local model_id="$1"
+    local message="$2"
+    jq -cn --arg id "$model_id" --arg error "$message" '{id: $id, error: $error}'
+}
+
+process_id() {
+    local model_id="$1"
+
+    if [[ -z "$model_id" ]]; then
+        return 0
+    fi
+
+    local url="https://huggingface.co/api/models/${model_id}"
+    local response
+    response=$(curl -s "${headers[@]}" "$url" 2>/dev/null || true)
+
+    if [[ -z "$response" ]]; then
+        emit_error "$model_id" "request_failed"
+        return 0
+    fi
+
+    if ! jq -e . >/dev/null 2>&1 <<<"$response"; then
+        emit_error "$model_id" "invalid_json"
+        return 0
+    fi
+
+    if jq -e '.error' >/dev/null 2>&1 <<<"$response"; then
+        emit_error "$model_id" "not_found"
+        return 0
+    fi
+
+    jq -c --arg id "$model_id" '{
+        id: (.id // $id),
+        downloads: (.downloads // 0),
+        likes: (.likes // 0),
+        pipeline_tag: (.pipeline_tag // "unknown"),
+        tags: (.tags // [])
+    }' <<<"$response" 2>/dev/null || emit_error "$model_id" "parse_failed"
+}
+
+if [[ $# -gt 0 ]]; then
+    for model_id in "$@"; do
+        process_id "$model_id"
+    done
+    exit 0
+fi
+
+if [[ -t 0 ]]; then
+    show_help
+    exit 1
+fi
+
+while IFS= read -r model_id; do
+    process_id "$model_id"
+done

package/skills/huggingface-tool-builder/references/hf_model_card_frontmatter.sh

@@ -0,0 +1,188 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+show_help() {
+    cat << 'USAGE'
+Fetch Hugging Face model cards via the hf CLI and summarize frontmatter.
+
+Usage:
+  hf_model_card_frontmatter.sh [MODEL_ID ...]
+  cat ids.txt | hf_model_card_frontmatter.sh
+
+Description:
+  Downloads README.md for each model via `hf download`, extracts YAML
+  frontmatter, and emits one JSON object per line (NDJSON) with key fields.
+  Uses HF_TOKEN if set (passed to the hf CLI).
+
+Output fields:
+  id, license, pipeline_tag, library_name, tags, language,
+  new_version, has_extra_gated_prompt
+
+Examples:
+  hf_model_card_frontmatter.sh openai/gpt-oss-120b
+  cat ids.txt | hf_model_card_frontmatter.sh | jq -s '.'
+  hf_model_card_frontmatter.sh meta-llama/Meta-Llama-3-8B \
+    | jq -s 'map({id, license, has_extra_gated_prompt})'
+USAGE
+}
+
+if [[ "${1:-}" == "--help" ]]; then
+    show_help
+    exit 0
+fi
+
+if ! command -v hf >/dev/null 2>&1; then
+    echo "Error: hf CLI is required but not installed" >&2
+    exit 1
+fi
+
+if ! command -v python3 >/dev/null 2>&1; then
+    echo "Error: python3 is required but not installed" >&2
+    exit 1
+fi
+
+token_args=()
+if [[ -n "${HF_TOKEN:-}" ]]; then
+    token_args=(--token "$HF_TOKEN")
+fi
+
+tmp_dir=$(mktemp -d)
+cleanup() {
+    rm -rf "$tmp_dir"
+}
+trap cleanup EXIT
+
+emit_error() {
+    local model_id="$1"
+    local message="$2"
+    python3 - << 'PY' "$model_id" "$message"
+import json
+import sys
+
+model_id = sys.argv[1]
+message = sys.argv[2]
+print(json.dumps({"id": model_id, "error": message}))
+PY
+}
+
+parse_readme() {
+    local model_id="$1"
+    local readme_path="$2"
+
+    MODEL_ID="$model_id" README_PATH="$readme_path" python3 - << 'PY'
+import json
+import os
+import sys
+
+model_id = os.environ.get("MODEL_ID", "")
+readme_path = os.environ.get("README_PATH", "")
+
+try:
+    with open(readme_path, "r", encoding="utf-8") as f:
+        lines = f.read().splitlines()
+except OSError:
+    print(json.dumps({"id": model_id, "error": "readme_missing"}))
+    sys.exit(0)
+
+frontmatter = []
+in_block = False
+for line in lines:
+    if line.strip() == "---":
+        if in_block:
+            break
+        in_block = True
+        continue
+    if in_block:
+        frontmatter.append(line)
+
+if not frontmatter:
+    print(json.dumps({"id": model_id, "error": "frontmatter_missing"}))
+    sys.exit(0)
+
+key = None
+out = {}
+
+for line in frontmatter:
+    stripped = line.strip()
+    if not stripped or line.lstrip().startswith("#"):
+        continue
+
+    if ":" in line and not line.lstrip().startswith("- "):
+        key_candidate, value = line.split(":", 1)
+        key_candidate = key_candidate.strip()
+        value = value.strip()
+        if key_candidate and all(c.isalnum() or c in "_-" for c in key_candidate):
+            key = key_candidate
+            if value in ("|", "|-", ">", ">-") or value == "":
+                out[key] = None
+                continue
+            if value.startswith("[") and value.endswith("]"):
+                items = [v.strip() for v in value.strip("[]").split(",") if v.strip()]
+                out[key] = items
+            else:
+                out[key] = value
+            continue
+
+    if line.lstrip().startswith("- ") and key:
+        item = line.strip()[2:]
+        if key not in out or out[key] is None:
+            out[key] = []
+        if isinstance(out[key], list):
+            out[key].append(item)
+
+result = {
+    "id": model_id,
+    "license": out.get("license"),
+    "pipeline_tag": out.get("pipeline_tag"),
+    "library_name": out.get("library_name"),
+    "tags": out.get("tags", []),
+    "language": out.get("language", []),
+    "new_version": out.get("new_version"),
+    "has_extra_gated_prompt": "extra_gated_prompt" in out,
+}
+
+print(json.dumps(result))
+PY
+}
+
+process_id() {
+    local model_id="$1"
+
+    if [[ -z "$model_id" ]]; then
+        return 0
+    fi
+
+    local safe_id
+    safe_id=$(printf '%s' "$model_id" | tr '/' '_')
+    local local_dir="$tmp_dir/$safe_id"
+
+    if ! hf download "$model_id" README.md --repo-type model --local-dir "$local_dir" "${token_args[@]}" >/dev/null 2>&1; then
+        emit_error "$model_id" "download_failed"
+        return 0
+    fi
+
+    local readme_path="$local_dir/README.md"
+    if [[ ! -f "$readme_path" ]]; then
+        emit_error "$model_id" "readme_missing"
+        return 0
+    fi
+
+    parse_readme "$model_id" "$readme_path"
+}
+
+if [[ $# -gt 0 ]]; then
+    for model_id in "$@"; do
+        process_id "$model_id"
+    done
+    exit 0
+fi
+
+if [[ -t 0 ]]; then
+    show_help
+    exit 1
+fi
+
+while IFS= read -r model_id; do
+    process_id "$model_id"
+done

package/skills/huggingface-tool-builder/references/hf_model_papers_auth.sh

@@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+
+# Hugging Face Model Papers Tool with Authentication
+# Fetches papers referenced by Hugging Face models using HF_TOKEN if available
+
+set -euo pipefail
+
+# Help function
+show_help() {
+    cat << EOF
+Hugging Face Model Papers Tool with Authentication
+
+This tool fetches papers referenced by Hugging Face models.
+Supports authentication via HF_TOKEN environment variable.
+
+Usage:
+    $0 [OPTIONS]
+
+Options:
+    MODEL_ID            Specific model to analyze (e.g., microsoft/DialoGPT-medium)
+    --trending [N]      Show papers for top N trending models (default: 5)
+    --help              Show this help message
+
+Environment Variables:
+    HF_TOKEN            Hugging Face API token (optional, for private models)
+
+Examples:
+    # Get papers for a specific model
+    $0 microsoft/DialoGPT-medium
+
+    # Get papers with authentication
+    HF_TOKEN=your_token_here $0 your-private-model
+
+    # Get papers for top 3 trending models
+    $0 --trending 3
+
+EOF
+}
+
+# Function to make authenticated API calls
+hf_api_call() {
+    local url="$1"
+    local headers=()
+    
+    # Add authentication header if HF_TOKEN is set
+    if [[ -n "${HF_TOKEN:-}" ]]; then
+        headers+=(-H "Authorization: Bearer $HF_TOKEN")
+    fi
+    
+    curl -s "${headers[@]}" "$url" 2>/dev/null || echo '{"error": "Network error"}'
+}
+
+# Function to extract papers from text
+extract_papers() {
+    local text="$1"
+    local title="$2"
+    
+    echo "$title"
+    
+    # Find ArXiv URLs
+    local arxiv_urls=$(echo "$text" | grep -oE 'https?://arxiv\.org/[^[:space:]\])]+' | head -5)
+    if [[ -n "$arxiv_urls" ]]; then
+        echo "ArXiv Papers:"
+        echo "$arxiv_urls" | sed 's/^/  • /'
+    fi
+    
+    # Find DOI URLs
+    local doi_urls=$(echo "$text" | grep -oE 'https?://doi\.org/[^[:space:]\])]+' | head -3)
+    if [[ -n "$doi_urls" ]]; then
+        echo "DOI Papers:"
+        echo "$doi_urls" | sed 's/^/  • /'
+    fi
+    
+    # Find arxiv IDs in format YYYY.NNNNN
+    local arxiv_ids=$(echo "$text" | grep -oE 'arXiv:[0-9]{4}\.[0-9]{4,5}' | head -5)
+    if [[ -n "$arxiv_ids" ]]; then
+        echo "ArXiv IDs:"
+        echo "$arxiv_ids" | sed 's/^/  • /'
+    fi
+    
+    # Check for paper mentions
+    if echo "$text" | grep -qi "paper\|publication\|citation"; then
+        local paper_mentions=$(echo "$text" | grep -i -A1 -B1 "paper\|publication" | head -6)
+        if [[ -n "$paper_mentions" ]]; then
+            echo "Paper mentions:"
+            echo "$paper_mentions" | sed 's/^/  /'
+        fi
+    fi
+    
+    if [[ -z "$arxiv_urls" && -z "$doi_urls" && -z "$arxiv_ids" ]]; then
+        echo "No papers found in model card"
+    fi
+}
+
+# Function to get model papers
+get_model_papers() {
+    local model_id="$1"
+    
+    echo "=== $model_id ==="
+    
+    # Get model info from API with authentication
+    local api_url="https://huggingface.co/api/models/$model_id"
+    local response=$(hf_api_call "$api_url")
+    
+    if echo "$response" | grep -q '"error"'; then
+        echo "Error: Could not fetch model '$model_id'"
+        if [[ -z "${HF_TOKEN:-}" ]]; then
+            echo "Note: This might be a private model. Try setting HF_TOKEN environment variable."
+        fi
+        return 1
+    fi
+    
+    # Parse basic info
+    local downloads=$(echo "$response" | jq -r '.downloads // 0')
+    local likes=$(echo "$response" | jq -r '.likes // 0')
+    echo "Downloads: $downloads | Likes: $likes"
+    
+    # Get model card
+    local card_url="https://huggingface.co/$model_id/raw/main/README.md"
+    local card_content=$(curl -s "$card_url" 2>/dev/null || echo "")
+    
+    if [[ -n "$card_content" ]]; then
+        extract_papers "$card_content" "Papers from model card:"
+    else
+        echo "Could not fetch model card"
+    fi
+    
+    # Check tags for arxiv references
+    local arxiv_tag=$(echo "$response" | jq -r '.tags[]' 2>/dev/null | grep arxiv || true)
+    if [[ -n "$arxiv_tag" ]]; then
+        echo "ArXiv from tags: $arxiv_tag"
+    fi
+    
+    echo
+}
+
+# Function to get trending models
+get_trending_models() {
+    local limit="${1:-5}"
+    
+    echo "Fetching top $limit trending models..."
+    
+    local trending_url="https://huggingface.co/api/trending?type=model&limit=$limit"
+    local response=$(hf_api_call "$trending_url")
+    
+    echo "$response" | jq -r '.recentlyTrending[] | .repoData.id' | head -"$limit" | while read -r model_id; do
+        if [[ -n "$model_id" ]]; then
+            get_model_papers "$model_id"
+        fi
+    done
+}
+
+# Main
+if [[ $# -eq 0 ]]; then
+    echo "Error: No arguments provided"
+    show_help
+    exit 1
+fi
+
+if [[ "$1" == "--help" ]]; then
+    show_help
+    exit 0
+elif [[ "$1" == "--trending" ]]; then
+    if [[ -n "${2:-}" ]] && [[ "$2" =~ ^[0-9]+$ ]]; then
+        get_trending_models "$2"
+    else
+        get_trending_models 5
+    fi
+else
+    get_model_papers "$1"
+fi