cecli-dev 0.97.3__tar.gz → 0.97.5__tar.gz

{cecli_dev-0.97.3 → cecli_dev-0.97.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cecli-dev
-Version: 0.97.3
+Version: 0.97.5
 Summary: we also can't pronounce cecli
 Project-URL: Homepage, https://github.com/dwash96/cecli
 Project-URL: Documentation, https://github.com/dwash96/cecli
@@ -84,12 +84,12 @@ Dynamic: license-file
 
 ## Why `cecli`?
 
-`cecli` (probably pronounced like "Cecily", aka `aider-ce`) is a community-driven fork of the [Aider](https://cecli.dev/) AI pair programming tool.
-Aider is a fantastic piece of software with a wonderful community but it has been painfully slow in receiving updates in the quickly evolving AI tooling space.
+`cecli` (probably pronounced like "Cecily") is yet another cli agent crafted for extensibility and customization. Originally a fork of the [Aider](https://cecli.dev/) AI pair programming tool, we aim to make agentic coding as maximally effective as it can be based on the growing capabilities of large language models.
 
-We aim to foster an open, collaborative ecosystem where new features, experiments, and improvements can be developed and shared rapidly. We believe in genuine FOSS principles and actively welcome contributors of all skill levels.
+We aim to foster an open, collaborative ecosystem where new features, experiments, and improvements can be developed and shared rapidly. We believe in the principles of FOSS and actively welcome contributors of all skill levels.
 
 If you are looking for bleeding-edge features or want to get your hands dirty with the internals of an AI coding agent, here's your sign.
+
 LLMs are a part of our lives from here on out so join us in learning about and crafting the future.
 
 ### Links

{cecli_dev-0.97.3 → cecli_dev-0.97.5}/README.md

@@ -1,11 +1,11 @@
 ## Why `cecli`?
 
-`cecli` (probably pronounced like "Cecily", aka `aider-ce`) is a community-driven fork of the [Aider](https://cecli.dev/) AI pair programming tool.
-Aider is a fantastic piece of software with a wonderful community but it has been painfully slow in receiving updates in the quickly evolving AI tooling space.
+`cecli` (probably pronounced like "Cecily") is yet another cli agent crafted for extensibility and customization. Originally a fork of the [Aider](https://cecli.dev/) AI pair programming tool, we aim to make agentic coding as maximally effective as it can be based on the growing capabilities of large language models.
 
-We aim to foster an open, collaborative ecosystem where new features, experiments, and improvements can be developed and shared rapidly. We believe in genuine FOSS principles and actively welcome contributors of all skill levels.
+We aim to foster an open, collaborative ecosystem where new features, experiments, and improvements can be developed and shared rapidly. We believe in the principles of FOSS and actively welcome contributors of all skill levels.
 
 If you are looking for bleeding-edge features or want to get your hands dirty with the internals of an AI coding agent, here's your sign.
+
 LLMs are a part of our lives from here on out so join us in learning about and crafting the future.
 
 ### Links

{cecli_dev-0.97.3 → cecli_dev-0.97.5}/benchmark/benchmark.py

@@ -515,7 +515,8 @@ def load_results(results_dir, stats_languages=None):
                 results["test_cases_total"] = total_cases
             if passed_cases is not None:
                 results["test_cases_passed"] = passed_cases
-
+            if passed_cases is not None and total_cases is not None:
+                results["test_cases_pass_ratio"] = passed_cases / total_cases
             # Update the JSON file immediately to keep data fresh
             fname.write_text(json.dumps(results, indent=4))
             logger.debug(f"Updated {fname} with test case counts")
@@ -583,7 +584,8 @@ def summarize_results(results_dir, verbose, stats_languages=None):
     res.completion_tokens = 0
     res.test_cases_total = 0
     res.test_cases_passed = 0
-
+    res.test_cases_pass_ratio_sum = 0.0
+    res.test_cases_pass_ratio_count = 0
     res.reasoning_effort = None
     res.thinking_tokens = None
     res.map_tokens = None
@@ -621,6 +623,8 @@ def summarize_results(results_dir, verbose, stats_languages=None):
         lang_stats.completion_tokens = 0
         lang_stats.test_cases_total = 0
         lang_stats.test_cases_passed = 0
+        lang_stats.test_cases_pass_ratio_sum = 0.0
+        lang_stats.test_cases_pass_ratio_count = 0
         lang_to_stats[lang] = lang_stats
         lang_to_passed_tests[lang] = [0] * tries
 
@@ -682,6 +686,11 @@ def summarize_results(results_dir, verbose, stats_languages=None):
             if passed_cases is not None:
                 add("test_cases_passed", passed_cases, res, lang_stats)
 
+            pass_ratio = results.get("test_cases_pass_ratio")
+            if pass_ratio is not None:
+                add("test_cases_pass_ratio_sum", pass_ratio, res, lang_stats)
+                add("test_cases_pass_ratio_count", 1, res, lang_stats)
+
             res.reasoning_effort = results.get("reasoning_effort")
             res.thinking_tokens = results.get("thinking_tokens")
             res.map_tokens = results.get("map_tokens")
@@ -764,6 +773,12 @@ def summarize_results(results_dir, verbose, stats_languages=None):
         res.test_cases_percentage = 100 * res.test_cases_passed / res.test_cases_total
         print(f"  test_cases_percentage: {res.test_cases_percentage:.1f}")
 
+    if res.test_cases_pass_ratio_count > 0:
+        res.test_cases_pass_ratio_avg = (
+            100 * res.test_cases_pass_ratio_sum / res.test_cases_pass_ratio_count
+        )
+        print(f"  test_cases_pass_ratio_avg: {res.test_cases_pass_ratio_avg:.1f}")
+
     if variants["model"]:
         a_model = set(variants["model"]).pop()
         command = f"cecli --model {a_model}"
@@ -801,6 +816,18 @@ def summarize_results(results_dir, verbose, stats_languages=None):
                 pass_rate = 100 * num_passed / float(lang_stats.completed_tests)
                 setattr(lang_stats, f"pass_rate_{i + 1}", pass_rate)
 
+            if lang_stats.test_cases_pass_ratio_count > 0:
+                lang_stats.test_cases_pass_ratio_avg = (
+                    100
+                    * lang_stats.test_cases_pass_ratio_sum
+                    / lang_stats.test_cases_pass_ratio_count
+                )
+            else:
+                lang_stats.test_cases_pass_ratio_avg = 0.0
+
+            del lang_stats.test_cases_pass_ratio_sum
+            del lang_stats.test_cases_pass_ratio_count
+
             # Then format attributes into ready-to-print strings
             for attr in lang_stats.__dict__:
                 val = getattr(lang_stats, attr)
@@ -870,6 +897,11 @@ def summarize_results(results_dir, verbose, stats_languages=None):
     console.rule()
 
     # print(json.dumps(vars(res), indent=4, sort_keys=True))
+    if hasattr(res, "test_cases_pass_ratio_sum"):
+        del res.test_cases_pass_ratio_sum
+    if hasattr(res, "test_cases_pass_ratio_count"):
+        del res.test_cases_pass_ratio_count
+
     return res
 
 
@@ -1156,6 +1188,7 @@ async def run_test_real(
             verbose=verbose,
             yes_always_commands=True,
             max_reflections=0,
+            file_diffs=False,
         ),
         map_mul_no_files=4,
         mcp_manager=None,

cecli_dev-0.97.5/benchmark/variations.1.sh

@@ -0,0 +1,203 @@
+#!/bin/bash
+# Benchmark runner script for testing multiple OpenRouter models
+# Usage: ./run_benchmark_variations.sh [OPTIONS]
+
+set -e  # Exit on error
+
+# Default values
+BASE_NAME="cecli-little-guys-h6"
+EDIT_FORMAT="hashline"
+MAP_TOKENS="512"
+THREADS="1"
+LANGUAGES="javascript,python,rust,go,java"
+HASH_RE="^.[15ef]"
+NUM_TESTS="72"
+EXERCISES_DIR="polyglot-benchmark"
+OUTPUT_DIR="tmp.benchmarks"
+SLEEP_BETWEEN=30  # Seconds to sleep between runs
+
+# List of models to test
+# RERUN
+#    "openrouter/minimax/minimax-m2.1"
+#    "openrouter/qwen/qwen3-vl-235b-a22b-thinking"
+MODELS=(
+#    "openrouter/qwen/qwen3.5-35b-a3b"
+#    "openrouter/xiaomi/mimo-v2-flash"
+#    "openrouter/moonshotai/kimi-k2.5"
+#    "openrouter/minimax/minimax-m2.5"       
+    "openrouter/anthropic/claude-haiku-4.5"
+    "openrouter/openai/gpt-oss-120b"
+    "openrouter/openai/gpt-5-mini"
+    "openrouter/google/gemini-3-flash-preview"
+    "openrouter/deepseek/deepseek-v3.2-exp"
+)
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --base-name)
+            BASE_NAME="$2"
+            shift 2
+            ;;
+        --edit-format)
+            EDIT_FORMAT="$2"
+            shift 2
+            ;;
+        --map-tokens)
+            MAP_TOKENS="$2"
+            shift 2
+            ;;
+        --threads)
+            THREADS="$2"
+            shift 2
+            ;;
+        --hash-re)
+            HASH_RE="$2"
+            shift 2
+            ;;
+        --num-tests)
+            NUM_TESTS="$2"
+            shift 2
+            ;;
+        --exercises-dir)
+            EXERCISES_DIR="$2"
+            shift 2
+            ;;
+        --output-dir)
+            OUTPUT_DIR="$2"
+            shift 2
+            ;;
+        --sleep)
+            SLEEP_BETWEEN="$2"
+            shift 2
+            ;;
+        --help)
+            echo "Usage: $0 [OPTIONS]"
+            echo ""
+            echo "Options:"
+            echo "  --base-name NAME      Base name for benchmark runs (default: $BASE_NAME)"
+            echo "  --edit-format FORMAT  Edit format to use (default: $EDIT_FORMAT)"
+            echo "  --map-tokens TOKENS   Map tokens (default: $MAP_TOKENS)"
+            echo "  --threads N           Number of threads (default: $THREADS)"
+            echo "  --hash-re REGEX       Hash regex filter (default: $HASH_RE)"
+            echo "  --num-tests N         Number of tests to run (default: $NUM_TESTS)"
+            echo "  --exercises-dir DIR   Exercises directory (default: $EXERCISES_DIR)"
+            echo "  --output-dir DIR      Output directory (default: $OUTPUT_DIR)"
+            echo "  --sleep SECONDS       Sleep between runs in seconds (default: $SLEEP_BETWEEN)"
+            echo "  --help                Show this help message"
+            echo ""
+            echo "Example:"
+            echo "  $0 --threads 2 --num-tests 5"
+            exit 0
+            ;;
+        *)
+            echo "Unknown option: $1"
+            echo "Use --help for usage information"
+            exit 1
+            ;;
+    esac
+done
+
+# Function to run a single benchmark
+run_benchmark() {
+    local model="$1"
+    local run_name="$2"
+
+    echo "========================================================================"
+    echo "Starting benchmark: $run_name"
+    echo "Model: $model"
+    echo "Time: $(date)"
+    echo "========================================================================"
+
+    # Create the benchmark command
+    ./benchmark/benchmark.py "$run_name" \
+        --new \
+        --model "$model" \
+        --edit-format "$EDIT_FORMAT" \
+        --map-tokens "$MAP_TOKENS" \
+        --threads "$THREADS" \
+        --hash-re "$HASH_RE" \
+        --num-tests "$NUM_TESTS" \
+        --languages "$LANGUAGES" \
+        --tries 2 \
+        --exercises-dir "$EXERCISES_DIR"
+
+    echo "Benchmark completed: $run_name"
+    echo "Results directory: $OUTPUT_DIR/$(ls -t $OUTPUT_DIR | grep "$run_name" | head -1)"
+    echo ""
+}
+
+# Function to generate statistics for all completed runs
+generate_stats() {
+    echo "========================================================================"
+    echo "Generating statistics for all completed runs"
+    echo "========================================================================"
+
+    for dir in "$OUTPUT_DIR"/*; do
+        if [ -d "$dir" ] && [ -f "$dir/.cecli.results.json" ]; then
+            echo "Processing: $(basename "$dir")"
+            ./benchmark/benchmark.py --stats "$dir" || true
+            echo ""
+        fi
+    done
+}
+
+# Main execution
+main() {
+    echo "========================================================================"
+    echo "OpenRouter Model Benchmark Runner"
+    echo "========================================================================"
+    echo "Configuration:"
+    echo "  Base name:      $BASE_NAME"
+    echo "  Edit format:    $EDIT_FORMAT"
+    echo "  Map tokens:     $MAP_TOKENS"
+    echo "  Threads:        $THREADS"
+    echo "  Hash regex:     $HASH_RE"
+    echo "  Num tests:      $NUM_TESTS"
+    echo "  Exercises dir:  $EXERCISES_DIR"
+    echo "  Output dir:     $OUTPUT_DIR"
+    echo "  Sleep between:  ${SLEEP_BETWEEN}s"
+    echo "  Models to test: ${#MODELS[@]}"
+    echo ""
+
+    # Create output directory if it doesn't exist
+    mkdir -p "$OUTPUT_DIR"
+
+    # Run benchmarks for each model
+    for model in "${MODELS[@]}"; do
+        # Create a run name by replacing slashes with hyphens
+        local model_slug=$(echo "$model" | sed 's|/|-|g')
+        local run_name="${BASE_NAME}-${model_slug}"
+
+        run_benchmark "$model" "$run_name"
+
+        # Sleep between runs to avoid rate limiting
+        if [ "$SLEEP_BETWEEN" -gt 0 ]; then
+            echo "Sleeping for ${SLEEP_BETWEEN} seconds before next run..."
+            sleep "$SLEEP_BETWEEN"
+            echo ""
+        fi
+    done
+
+    # Generate statistics
+    generate_stats
+
+    echo "========================================================================"
+    echo "All benchmarks completed!"
+    echo "========================================================================"
+    echo ""
+    echo "Summary of results directories:"
+    ls -la "$OUTPUT_DIR" | grep "$BASE_NAME"
+    echo ""
+    echo "To view statistics for a specific run:"
+    echo "  ./benchmark/benchmark.py --stats $OUTPUT_DIR/<run-directory>"
+    echo ""
+    echo "To compare all results:"
+    echo "  for dir in $OUTPUT_DIR/*$BASE_NAME*; do"
+    echo "    echo \"=== \$(basename \$dir) ===\""
+    echo "    ./benchmark/benchmark.py --stats \"\$dir\" 2>/dev/null | grep -E '(pass_rate|total_cost|completed_tests)' || true"
+    echo "  done"
+}
+
+# Run main function
+main

cecli_dev-0.97.5/benchmark/variations.2.sh

@@ -0,0 +1,203 @@
+#!/bin/bash
+# Benchmark runner script for testing multiple OpenRouter models
+# Usage: ./run_benchmark_variations.sh [OPTIONS]
+
+set -e  # Exit on error
+
+# Default values
+BASE_NAME="cecli-little-guys-h6"
+EDIT_FORMAT="hashline"
+MAP_TOKENS="512"
+THREADS="1"
+LANGUAGES="javascript,python,rust,go,java"
+HASH_RE="^.[15ef]"
+NUM_TESTS="72"
+EXERCISES_DIR="polyglot-benchmark"
+OUTPUT_DIR="tmp.benchmarks"
+SLEEP_BETWEEN=30  # Seconds to sleep between runs
+
+# List of models to test
+# RERUN
+#    "openrouter/minimax/minimax-m2.1"
+#    "openrouter/qwen/qwen3-vl-235b-a22b-thinking"
+MODELS=(
+    "openrouter/qwen/qwen3.5-35b-a3b"
+    "openrouter/xiaomi/mimo-v2-flash"
+    "openrouter/moonshotai/kimi-k2.5"
+    "openrouter/minimax/minimax-m2.5"       
+#    "openrouter/anthropic/claude-haiku-4.5"
+#    "openrouter/openai/gpt-oss-120b"
+#    "openrouter/openai/gpt-5-mini"
+#    "openrouter/google/gemini-3-flash-preview"
+#    "openrouter/deepseek/deepseek-v3.2-exp"
+)
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --base-name)
+            BASE_NAME="$2"
+            shift 2
+            ;;
+        --edit-format)
+            EDIT_FORMAT="$2"
+            shift 2
+            ;;
+        --map-tokens)
+            MAP_TOKENS="$2"
+            shift 2
+            ;;
+        --threads)
+            THREADS="$2"
+            shift 2
+            ;;
+        --hash-re)
+            HASH_RE="$2"
+            shift 2
+            ;;
+        --num-tests)
+            NUM_TESTS="$2"
+            shift 2
+            ;;
+        --exercises-dir)
+            EXERCISES_DIR="$2"
+            shift 2
+            ;;
+        --output-dir)
+            OUTPUT_DIR="$2"
+            shift 2
+            ;;
+        --sleep)
+            SLEEP_BETWEEN="$2"
+            shift 2
+            ;;
+        --help)
+            echo "Usage: $0 [OPTIONS]"
+            echo ""
+            echo "Options:"
+            echo "  --base-name NAME      Base name for benchmark runs (default: $BASE_NAME)"
+            echo "  --edit-format FORMAT  Edit format to use (default: $EDIT_FORMAT)"
+            echo "  --map-tokens TOKENS   Map tokens (default: $MAP_TOKENS)"
+            echo "  --threads N           Number of threads (default: $THREADS)"
+            echo "  --hash-re REGEX       Hash regex filter (default: $HASH_RE)"
+            echo "  --num-tests N         Number of tests to run (default: $NUM_TESTS)"
+            echo "  --exercises-dir DIR   Exercises directory (default: $EXERCISES_DIR)"
+            echo "  --output-dir DIR      Output directory (default: $OUTPUT_DIR)"
+            echo "  --sleep SECONDS       Sleep between runs in seconds (default: $SLEEP_BETWEEN)"
+            echo "  --help                Show this help message"
+            echo ""
+            echo "Example:"
+            echo "  $0 --threads 2 --num-tests 5"
+            exit 0
+            ;;
+        *)
+            echo "Unknown option: $1"
+            echo "Use --help for usage information"
+            exit 1
+            ;;
+    esac
+done
+
+# Function to run a single benchmark
+run_benchmark() {
+    local model="$1"
+    local run_name="$2"
+
+    echo "========================================================================"
+    echo "Starting benchmark: $run_name"
+    echo "Model: $model"
+    echo "Time: $(date)"
+    echo "========================================================================"
+
+    # Create the benchmark command
+    ./benchmark/benchmark.py "$run_name" \
+        --new \
+        --model "$model" \
+        --edit-format "$EDIT_FORMAT" \
+        --map-tokens "$MAP_TOKENS" \
+        --threads "$THREADS" \
+        --hash-re "$HASH_RE" \
+        --num-tests "$NUM_TESTS" \
+        --languages "$LANGUAGES" \
+        --tries 2 \
+        --exercises-dir "$EXERCISES_DIR"
+
+    echo "Benchmark completed: $run_name"
+    echo "Results directory: $OUTPUT_DIR/$(ls -t $OUTPUT_DIR | grep "$run_name" | head -1)"
+    echo ""
+}
+
+# Function to generate statistics for all completed runs
+generate_stats() {
+    echo "========================================================================"
+    echo "Generating statistics for all completed runs"
+    echo "========================================================================"
+
+    for dir in "$OUTPUT_DIR"/*; do
+        if [ -d "$dir" ] && [ -f "$dir/.cecli.results.json" ]; then
+            echo "Processing: $(basename "$dir")"
+            ./benchmark/benchmark.py --stats "$dir" || true
+            echo ""
+        fi
+    done
+}
+
+# Main execution
+main() {
+    echo "========================================================================"
+    echo "OpenRouter Model Benchmark Runner"
+    echo "========================================================================"
+    echo "Configuration:"
+    echo "  Base name:      $BASE_NAME"
+    echo "  Edit format:    $EDIT_FORMAT"
+    echo "  Map tokens:     $MAP_TOKENS"
+    echo "  Threads:        $THREADS"
+    echo "  Hash regex:     $HASH_RE"
+    echo "  Num tests:      $NUM_TESTS"
+    echo "  Exercises dir:  $EXERCISES_DIR"
+    echo "  Output dir:     $OUTPUT_DIR"
+    echo "  Sleep between:  ${SLEEP_BETWEEN}s"
+    echo "  Models to test: ${#MODELS[@]}"
+    echo ""
+
+    # Create output directory if it doesn't exist
+    mkdir -p "$OUTPUT_DIR"
+
+    # Run benchmarks for each model
+    for model in "${MODELS[@]}"; do
+        # Create a run name by replacing slashes with hyphens
+        local model_slug=$(echo "$model" | sed 's|/|-|g')
+        local run_name="${BASE_NAME}-${model_slug}"
+
+        run_benchmark "$model" "$run_name"
+
+        # Sleep between runs to avoid rate limiting
+        if [ "$SLEEP_BETWEEN" -gt 0 ]; then
+            echo "Sleeping for ${SLEEP_BETWEEN} seconds before next run..."
+            sleep "$SLEEP_BETWEEN"
+            echo ""
+        fi
+    done
+
+    # Generate statistics
+    generate_stats
+
+    echo "========================================================================"
+    echo "All benchmarks completed!"
+    echo "========================================================================"
+    echo ""
+    echo "Summary of results directories:"
+    ls -la "$OUTPUT_DIR" | grep "$BASE_NAME"
+    echo ""
+    echo "To view statistics for a specific run:"
+    echo "  ./benchmark/benchmark.py --stats $OUTPUT_DIR/<run-directory>"
+    echo ""
+    echo "To compare all results:"
+    echo "  for dir in $OUTPUT_DIR/*$BASE_NAME*; do"
+    echo "    echo \"=== \$(basename \$dir) ===\""
+    echo "    ./benchmark/benchmark.py --stats \"\$dir\" 2>/dev/null | grep -E '(pass_rate|total_cost|completed_tests)' || true"
+    echo "  done"
+}
+
+# Run main function
+main

cecli_dev-0.97.3/benchmark/primary_variations.sh → cecli_dev-0.97.5/benchmark/variations.3.sh

@@ -5,7 +5,7 @@
 set -e  # Exit on error
 
 # Default values
-BASE_NAME="cecli-base-d-big-3"
+BASE_NAME="cecli-little-guys-d6"
 EDIT_FORMAT="diff"
 MAP_TOKENS="512"
 THREADS="1"
@@ -21,14 +21,15 @@ SLEEP_BETWEEN=30  # Seconds to sleep between runs
 #    "openrouter/minimax/minimax-m2.1"
 #    "openrouter/qwen/qwen3-vl-235b-a22b-thinking"
 MODELS=(
-#    "openrouter/deepseek/deepseek-v3.2-exp"
-    "openrouter/moonshotai/kimi-k2.5"
+#    "openrouter/qwen/qwen3.5-35b-a3b"
+#    "openrouter/xiaomi/mimo-v2-flash"
+#    "openrouter/moonshotai/kimi-k2.5"
+#    "openrouter/minimax/minimax-m2.5"       
+    "openrouter/anthropic/claude-haiku-4.5"
     "openrouter/openai/gpt-oss-120b"
-    "openrouter/openai/gpt-5.2"   
+    "openrouter/openai/gpt-5-mini"
     "openrouter/google/gemini-3-flash-preview"
-    "openrouter/google/gemini-3-pro-preview"
-    "openrouter/anthropic/claude-haiku-4.5"
-    "openrouter/anthropic/claude-sonnet-4.5" 
+    "openrouter/deepseek/deepseek-v3.2-exp"
 )
 
 # Parse command line arguments
@@ -118,6 +119,7 @@ run_benchmark() {
         --hash-re "$HASH_RE" \
         --num-tests "$NUM_TESTS" \
         --languages "$LANGUAGES" \
+        --tries 2 \
         --exercises-dir "$EXERCISES_DIR"
 
     echo "Benchmark completed: $run_name"