python-harness 0.0.6__tar.gz → 0.0.10__tar.gz

{python_harness-0.0.6/python_harness.egg-info → python_harness-0.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-harness
-Version: 0.0.6
+Version: 0.0.10
 Summary: An agentic codebase evaluation and evolution tool for Python projects.
 Author-email: Mingli Yuan <mingli.yuan@gmail.com>
 License: MIT

{python_harness-0.0.6 → python_harness-0.0.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "python-harness"
-version = "0.0.6"
+version = "0.0.10"
 description = "An agentic codebase evaluation and evolution tool for Python projects."
 requires-python = ">=3.10"
 readme = "README.md"
@@ -62,4 +62,3 @@ addopts = "-ra -q --cov=python_harness --cov-report=term-missing --cov-report=ht
 testpaths = [
     "tests",
 ]
-

{python_harness-0.0.6 → python_harness-0.0.10}/python_harness/__init__.py

@@ -2,4 +2,4 @@
 Python Harness - An agentic evaluation tool for codebases.
 """
 
-__version__ = "0.0.1"
+__version__ = "0.0.10"

python_harness-0.0.10/python_harness/cli.py

@@ -0,0 +1,312 @@
+"""
+Command-line interface for python-harness.
+"""
+
+import os
+import sys
+from typing import Any
+
+import typer
+from dotenv import load_dotenv
+from rich.console import Console
+
+from python_harness.evaluator import Evaluator
+
+# Try to find .env file explicitly before anything else executes
+env_path = os.path.join(os.getcwd(), '.env')
+if os.path.exists(env_path):
+    load_dotenv(dotenv_path=env_path)
+else:
+    load_dotenv() # Fallback to default search
+
+app = typer.Typer(help="Agentic harness tool for universal Python codebase evaluation.")
+console = Console()
+
+
+def _print_detail_block(title: str, details: str, color: str) -> None:
+    normalized_details = [
+        line.rstrip() for line in details.splitlines() if line.strip()
+    ]
+    console.print(f"[{color}]{title}:[/{color}]")
+    for line in normalized_details:
+        console.print(f"  {line}")
+    console.print()
+
+
+def _print_ruff_issues(issues: list[dict[str, Any]]) -> None:
+    console.print("[red]Ruff issues found:[/red]")
+    for issue in issues:
+        file = issue.get("filename", "unknown")
+        line = issue.get("location", {}).get("row", "?")
+        msg = issue.get("message", "unknown issue")
+        console.print(f"  - {file}:{line} {msg}")
+    console.print()
+
+
+def _print_ty_result(ty_results: dict[str, Any]) -> None:
+    status = ty_results.get("status")
+    if status == "warning":
+        msg = str(ty_results.get("error_message", "ty not found"))
+        _print_detail_block("Ty warning", msg, "yellow")
+        return
+    if status == "success":
+        return
+
+    output = str(ty_results.get("output", ""))
+    error_msg = str(ty_results.get("error_message", ""))
+    if output:
+        _print_detail_block("Ty issues found", output, "red")
+    elif error_msg:
+        _print_detail_block("Ty error", error_msg, "red")
+    else:
+        console.print("[red]Ty failed, but no standard output was captured.[/red]")
+
+
+def _print_radon_cc_result(radon_results: dict[str, Any]) -> None:
+    status = radon_results.get("status")
+    if status == "warning":
+        err_msg = str(radon_results.get("error_message", ""))
+        _print_detail_block("Radon CC warning", err_msg, "yellow")
+        return
+    if status != "failed":
+        return
+
+    issues = radon_results.get("issues", [])
+    if issues:
+        console.print(
+            f"[red]Cyclomatic Complexity too high "
+            f"({len(issues)} functions > 15):[/red]"
+        )
+        for issue in issues:
+            console.print(
+                f"  - {issue['file']}: {issue['type']} '{issue['name']}' "
+                f"has CC {issue['complexity']}"
+            )
+        console.print()
+        return
+
+    err_msg = str(radon_results.get("error_message", ""))
+    if err_msg:
+        _print_detail_block("Radon CC error", err_msg, "red")
+        return
+    console.print("[red]Radon CC failed but no specific issues were parsed.[/red]")
+    console.print()
+
+
+def _print_hard_failure_details(hard_results: dict[str, Any]) -> None:
+    console.print("[bold red]Hard Evaluation Failed![/bold red]")
+    console.print()
+
+    ruff_issues = hard_results.get("ruff", {}).get("issues", [])
+    if hard_results.get("ruff", {}).get("status") != "success":
+        _print_ruff_issues(ruff_issues)
+
+    if hard_results.get("mypy", {}).get("status") != "success":
+        output = str(hard_results.get("mypy", {}).get("output", ""))
+        _print_detail_block("Mypy issues found", output, "red")
+
+    _print_ty_result(hard_results.get("ty", {}))
+    _print_radon_cc_result(hard_results.get("radon_cc", {}))
+
+    if hard_results.get("pytest", {}).get("status") == "failed":
+        error_msg = str(hard_results.get("pytest", {}).get("error_message", ""))
+        _print_detail_block("Pytest/Coverage issues found", error_msg, "red")
+
+    console.print(
+        "[yellow]Continuing to soft evaluation to generate "
+        "suggestions despite hard failures...[/yellow]"
+    )
+
+
+def _print_hard_evaluation_summary(hard_results: dict[str, Any]) -> None:
+    if hard_results["all_passed"]:
+        console.print("[bold green]Hard Evaluation Passed![/bold green]")
+        return
+    _print_hard_failure_details(hard_results)
+
+
+def _print_mi_scorecard(hard_results: dict[str, Any]) -> None:
+    mi_scores = hard_results.get("radon_mi", {}).get("mi_scores", {})
+    if not mi_scores:
+        return
+
+    avg_mi = sum(mi_scores.values()) / len(mi_scores)
+    color = "green" if avg_mi > 50 else "yellow" if avg_mi > 20 else "red"
+    console.print(f"[{color}]Average Maintainability Index: {avg_mi:.1f}/100[/{color}]")
+
+
+def _print_qc_summary(qc_results: dict[str, Any]) -> None:
+    console.print()
+    console.print("[bold blue]Running Governance QC (Second Fence)...[/bold blue]")
+
+    if qc_results["all_passed"]:
+        console.print(
+            "[bold green]Governance QC Passed! (Change is admissible)[/bold green]"
+        )
+        console.print()
+        return
+
+    console.print("[bold red]Governance QC Failed![/bold red]")
+    console.print()
+    console.print(
+        "[red]The proposed changes violate governance constraints "
+        "or lack sufficient evidence.[/red]"
+    )
+    for failure in qc_results["failures"]:
+        console.print(f"[red]- {failure}[/red]")
+    console.print()
+    console.print(
+        "[yellow]Continuing to soft evaluation to generate "
+        "suggestions despite QC failures...[/yellow]"
+    )
+    console.print()
+
+
+def _print_soft_evaluation_start() -> None:
+    console.print(
+        "[bold blue]Running Soft Evaluation "
+        "(Readability & Understandability)...[/bold blue]"
+    )
+
+
+def _print_soft_summary(soft_results: dict[str, Any]) -> None:
+    pkg_summary = soft_results["package_summary"]
+    console.print(
+        f"[green]Analyzed {pkg_summary['total_files']} files with a total of "
+        f"{pkg_summary['total_tokens']} tokens.[/green]"
+    )
+    console.print(
+        f"[magenta]Agent's Understanding of the Package:[/magenta]\n"
+        f"{pkg_summary['package_understanding']}"
+    )
+
+    console.print()
+    console.print(
+        f"[cyan]Overall Understandability Score:[/cyan] "
+        f"{soft_results['understandability_score']:.1f}/100"
+    )
+
+    qa_results = soft_results.get("qa_results", {}).get("sampled_entities", [])
+    if qa_results:
+        console.print()
+        console.print("[bold yellow]Blind QA Sampling Results:[/bold yellow]")
+        for qa in qa_results:
+            color = "green" if qa["score"] >= 80 else "red"
+            console.print(f"  - [{color}]{qa['entity']}: Score {qa['score']}[/{color}]")
+            console.print(f"    [dim]Feedback: {qa['feedback']}[/dim]")
+
+    console.print()
+    console.print("[yellow]Evaluation completed. Generating report...[/yellow]")
+    console.print()
+
+
+def _print_final_report(final_report: dict[str, Any]) -> None:
+    verdict = str(final_report.get("verdict", "Unknown"))
+    verdict_color = "bold green" if "Pass" in verdict else "bold red"
+
+    console.print(
+        f"[{verdict_color}]=== FINAL VERDICT: {verdict} ===[/{verdict_color}]"
+    )
+    console.print(f"[bold]Summary:[/bold] {final_report.get('summary', '')}")
+    console.print()
+
+    suggestions = final_report.get("suggestions", [])
+    if suggestions:
+        console.print("[bold cyan]Top 3 Improvement Suggestions:[/bold cyan]")
+        for i, sug in enumerate(suggestions, 1):
+            console.print(
+                f"  {i}. [bold]{sug.get('title', 'Suggestion')}[/bold] "
+                f"(Target: [yellow]{sug.get('target_file', 'unknown')}[/yellow])"
+            )
+            console.print(f"     [dim]{sug.get('description', '')}[/dim]")
+
+
+@app.command()
+def refine(
+    path: str = typer.Argument(".", help="The path to evaluate and evolve"),
+    steps: int = typer.Option(1, help="Number of evolution steps to perform"),
+    max_retries: int = typer.Option(3, help="Maximum retries per variant if tests fail")
+) -> None:
+    """
+    Refine the codebase through an agentic Edit-Test-Improve loop.
+    Generates variants based on suggestions, tests them, and picks the best.
+    """
+    console.print(
+        f"[bold magenta]Starting evolution loop for path:[/bold magenta] {path} "
+        f"[dim](steps={steps}, max_retries={max_retries})[/dim]"
+    )
+    
+    # 1. First, run a baseline evaluation to get suggestions
+    evaluator = Evaluator(path)
+    console.print("[cyan]Running baseline evaluation...[/cyan]")
+    hard_results = evaluator.hard_evaluator.evaluate()
+    soft_results = evaluator.soft_evaluator.evaluate()
+    baseline_report = evaluator.soft_evaluator.generate_final_report(
+        hard_results, {"all_passed": True, "failures": []}, soft_results
+    )
+    
+    suggestions = baseline_report.get("suggestions", [])
+    if not suggestions:
+        console.print("[yellow]No suggestions found to evolve. Exiting.[/yellow]")
+        return
+        
+    console.print(
+        f"[green]Found {len(suggestions)} suggestions. "
+        f"Starting evolution branches...[/green]"
+    )
+    
+    # TODO: Implement the Git branching and Agent modification logic here.
+    # The loop will be:
+    # for step in range(steps):
+    #   for suggestion in suggestions:
+    #     checkout new branch variant-X
+    #     for retry in range(max_retries):
+    #       ask LLM to apply suggestion to code
+    #       run pytest
+    #       if pytest passes:
+    #         run harness . to get new score
+    #         break
+    #       else:
+    #         feed error back to LLM for retry
+    #   compare all variants and checkout the best one
+    
+    console.print(
+        "[yellow]Evolution engine skeleton ready. "
+        "Actual git mutation logic pending.[/yellow]"
+    )
+@app.command()
+def measure(path: str = typer.Argument(".", help="The path to evaluate")) -> None:
+    """
+    Measure the codebase against hard, soft, and governance constraints.
+    Outputs a final report with scores and actionable improvement suggestions.
+    """
+    console.print(
+        f"[bold green]Starting harness measurement for path:[/bold green] {path}"
+    )
+    
+    evaluator = Evaluator(path)
+    console.print("[bold blue]Running Hard Evaluation (ruff, mypy)...[/bold blue]")
+    hard_results = evaluator.hard_evaluator.evaluate()
+    _print_hard_evaluation_summary(hard_results)
+    _print_mi_scorecard(hard_results)
+
+    qc_results = evaluator.qc_evaluator.evaluate()
+    _print_qc_summary(qc_results)
+
+    _print_soft_evaluation_start()
+    soft_results = evaluator.soft_evaluator.evaluate()
+    _print_soft_summary(soft_results)
+
+    final_report = evaluator.soft_evaluator.generate_final_report(
+        hard_results, qc_results, soft_results
+    )
+    if not final_report:
+        return
+
+    _print_final_report(final_report)
+    if "Fail" in str(final_report.get("verdict", "Unknown")):
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    app()

{python_harness-0.0.6 → python_harness-0.0.10}/python_harness/evaluator.py

@@ -30,7 +30,7 @@ class Evaluator:
 
         # Generate Final Synthesized Report with 3 Suggestions
         final_report = self.soft_evaluator.generate_final_report(
-            hard_results, soft_results
+            hard_results, qc_results, soft_results
         )
 
         return {

{python_harness-0.0.6 → python_harness-0.0.10}/python_harness/hard_evaluator.py

@@ -4,12 +4,15 @@ Core module for integrating hard evaluation tools like ruff, mypy, and pytest.
 
 import json
 import subprocess
+import sys
+import tempfile
 from pathlib import Path
 from typing import Any
 
 from rich.console import Console
 
 console = Console()
+PYTEST_TIMEOUT_SECONDS = 60
 
 class HardEvaluator:
     """
@@ -25,7 +28,15 @@ class HardEvaluator:
         """
         try:
             result = subprocess.run(
-                ["ruff", "check", str(self.target_path), "--output-format", "json"],
+                [
+                    sys.executable,
+                    "-m",
+                    "ruff",
+                    "check",
+                    str(self.target_path),
+                    "--output-format",
+                    "json",
+                ],
                 capture_output=True,
                 text=True,
                 check=False
@@ -46,7 +57,7 @@ class HardEvaluator:
         """
         try:
             result = subprocess.run(
-                ["mypy", str(self.target_path)],
+                [sys.executable, "-m", "mypy", str(self.target_path)],
                 capture_output=True,
                 text=True,
                 check=False
@@ -101,7 +112,15 @@ class HardEvaluator:
         """
         try:
             result = subprocess.run(
-                ["radon", "cc", "-j", "-a", str(self.target_path)],
+                [
+                    sys.executable,
+                    "-m",
+                    "radon",
+                    "cc",
+                    "-j",
+                    "-a",
+                    str(self.target_path),
+                ],
                 capture_output=True,
                 text=True,
                 check=False
@@ -136,7 +155,19 @@ class HardEvaluator:
                 "output": result.stdout,
                 "error_message": result.stderr if result.returncode != 0 else ""
             }
+        except FileNotFoundError:
+            return {
+                "status": "warning", 
+                "issues": [],
+                "error_message": "radon executable not found. Please install it."
+            }
         except Exception as e:
+            if "No module named radon" in str(e) or "radon" in str(e):
+                return {
+                    "status": "warning", 
+                    "issues": [],
+                    "error_message": "radon executable not found. Please install it."
+                }
             return {"status": "error", "error_message": str(e)}
 
     def run_radon_mi(self) -> dict[str, Any]:
@@ -147,7 +178,7 @@ class HardEvaluator:
         """
         try:
             result = subprocess.run(
-                ["radon", "mi", "-j", str(self.target_path)],
+                [sys.executable, "-m", "radon", "mi", "-j", str(self.target_path)],
                 capture_output=True,
                 text=True,
                 check=False
@@ -164,7 +195,19 @@ class HardEvaluator:
                 "mi_scores": mi_scores,
                 "return_code": result.returncode,
             }
+        except FileNotFoundError:
+            return {
+                "status": "warning",
+                "mi_scores": {},
+                "error_message": "radon executable not found. Please install it."
+            }
         except Exception as e:
+            if "No module named radon" in str(e) or "radon" in str(e):
+                return {
+                    "status": "warning",
+                    "mi_scores": {},
+                    "error_message": "radon executable not found. Please install it."
+                }
             return {"status": "error", "error_message": str(e)}
 
     def run_pytest(self) -> dict[str, Any]:
@@ -172,22 +215,44 @@ class HardEvaluator:
         Run Pytest test suite and return coverage results.
         """
         try:
-            # When pytest is run within pytest, it can cause issues or hang.
-            # Here we just run it as a subprocess to gather results.
-            result = subprocess.run(
-                ["pytest", str(self.target_path), "--cov", "--cov-report=json"],
-                capture_output=True,
-                text=True,
-                check=False
-            )
+            with tempfile.TemporaryDirectory() as tmp_dir:
+                coverage_report = Path(tmp_dir) / "coverage.json"
+                result = subprocess.run(
+                    [
+                        sys.executable,
+                        "-m",
+                        "pytest",
+                        str(self.target_path),
+                        "--cov",
+                        f"--cov-report=json:{coverage_report}",
+                    ],
+                    capture_output=True,
+                    text=True,
+                    check=False,
+                    timeout=PYTEST_TIMEOUT_SECONDS,
+                )
+                coverage_percentage = None
+                if coverage_report.exists():
+                    coverage_data = json.loads(coverage_report.read_text())
+                    coverage_percentage = coverage_data.get("totals", {}).get(
+                        "percent_covered"
+                    )
             status = "success" if result.returncode == 0 else "failed"
             return {
                 "status": status,
                 "output": result.stdout,
                 "return_code": result.returncode,
+                "coverage_percentage": coverage_percentage,
+            }
+        except subprocess.TimeoutExpired:
+            return {
+                "status": "failed",
+                "error_message": (
+                    f"Pytest run timed out after {PYTEST_TIMEOUT_SECONDS} seconds."
+                ),
             }
         except Exception as e:
-             return {"status": "error", "error_message": str(e)}
+            return {"status": "error", "error_message": str(e)}
 
     def evaluate(self) -> dict[str, Any]:
         """
@@ -199,13 +264,30 @@ class HardEvaluator:
         ty_res = self.run_ty()
         radon_cc_res = self.run_radon_cc()
         radon_mi_res = self.run_radon_mi()
-        # pytest_res = self.run_pytest() # Better handled as a separate stage
+        pytest_res = self.run_pytest()
         
+        # Parse pytest coverage to check if it's < 90%
+        cov_percentage = pytest_res.get("coverage_percentage")
+        if pytest_res.get("status") == "success":
+            if isinstance(cov_percentage, (int, float)):
+                if cov_percentage < 90.0:
+                    pytest_res["status"] = "failed"
+                    pytest_res["error_message"] = (
+                        f"Test coverage is {cov_percentage:.2f}%, "
+                        f"which is below the 90% threshold."
+                    )
+            else:
+                pytest_res["status"] = "failed"
+                pytest_res["error_message"] = (
+                    "Coverage report was missing or unreadable."
+                )
+
         all_passed = (
             ruff_res.get("status") == "success" and 
             mypy_res.get("status") == "success" and
             ty_res.get("status") in ("success", "warning") and
-            radon_cc_res.get("status") == "success"
+            radon_cc_res.get("status") in ("success", "warning") and
+            pytest_res.get("status") == "success"
         )
 
         return {
@@ -214,5 +296,6 @@ class HardEvaluator:
             "mypy": mypy_res,
             "ty": ty_res,
             "radon_cc": radon_cc_res,
-            "radon_mi": radon_mi_res
+            "radon_mi": radon_mi_res,
+            "pytest": pytest_res
         }