python-harness 0.0.1__py3-none-any.whl

python_harness/__init__.py

@@ -0,0 +1,5 @@
+"""
+Python Harness - An agentic evaluation tool for codebases.
+"""
+
+__version__ = "0.0.1"

python_harness/cli.py

@@ -0,0 +1,207 @@
+"""
+Command-line interface for python-harness.
+"""
+
+import os
+import sys
+
+import typer
+from dotenv import load_dotenv
+from rich.console import Console
+
+from python_harness.evaluator import Evaluator
+
+# Try to find .env file explicitly before anything else executes
+env_path = os.path.join(os.getcwd(), '.env')
+if os.path.exists(env_path):
+    load_dotenv(dotenv_path=env_path)
+else:
+    load_dotenv() # Fallback to default search
+
+app = typer.Typer(help="Agentic harness tool for universal Python codebase evaluation.")
+console = Console()
+
+
+@app.command()
+def refine(
+    path: str = typer.Argument(".", help="The path to evaluate and evolve"),
+    steps: int = typer.Option(1, help="Number of evolution steps to perform"),
+    max_retries: int = typer.Option(3, help="Maximum retries per variant if tests fail")
+) -> None:
+    """
+    Refine the codebase through an agentic Edit-Test-Improve loop.
+    Generates variants based on suggestions, tests them, and picks the best.
+    """
+    console.print(
+        f"[bold magenta]Starting evolution loop for path:[/bold magenta] {path} "
+        f"[dim](steps={steps}, max_retries={max_retries})[/dim]"
+    )
+    
+    # 1. First, run a baseline evaluation to get suggestions
+    evaluator = Evaluator(path)
+    console.print("[cyan]Running baseline evaluation...[/cyan]")
+    hard_results = evaluator.hard_evaluator.evaluate()
+    soft_results = evaluator.soft_evaluator.evaluate()
+    baseline_report = evaluator.soft_evaluator.generate_final_report(
+        hard_results, soft_results
+    )
+    
+    suggestions = baseline_report.get("suggestions", [])
+    if not suggestions:
+        console.print("[yellow]No suggestions found to evolve. Exiting.[/yellow]")
+        return
+        
+    console.print(
+        f"[green]Found {len(suggestions)} suggestions. "
+        f"Starting evolution branches...[/green]"
+    )
+    
+    # TODO: Implement the Git branching and Agent modification logic here.
+    # The loop will be:
+    # for step in range(steps):
+    #   for suggestion in suggestions:
+    #     checkout new branch variant-X
+    #     for retry in range(max_retries):
+    #       ask LLM to apply suggestion to code
+    #       run pytest
+    #       if pytest passes:
+    #         run harness . to get new score
+    #         break
+    #       else:
+    #         feed error back to LLM for retry
+    #   compare all variants and checkout the best one
+    
+    console.print(
+        "[yellow]Evolution engine skeleton ready. "
+        "Actual git mutation logic pending.[/yellow]"
+    )
+@app.command()
+def measure(path: str = typer.Argument(".", help="The path to evaluate")) -> None:
+    """
+    Measure the codebase against hard, soft, and governance constraints.
+    Outputs a final report with scores and actionable improvement suggestions.
+    """
+    console.print(
+        f"[bold green]Starting harness measurement for path:[/bold green] {path}"
+    )
+    
+    evaluator = Evaluator(path)
+    
+    # 1. Hard Evaluation Gate (First Fence)
+    console.print("[bold blue]Running Hard Evaluation (ruff, mypy)...[/bold blue]")
+    hard_results = evaluator.hard_evaluator.evaluate()
+    
+    if not hard_results["all_passed"]:
+        console.print("[bold red]Hard Evaluation Failed! Exiting.[/bold red]")
+        if hard_results["ruff"]["status"] != "success":
+            console.print("[red]Ruff issues found.[/red]")
+        if hard_results["mypy"]["status"] != "success":
+            output = hard_results["mypy"].get("output", "")
+            console.print(f"[red]Mypy issues found:[/red]\n{output}")
+        if hard_results["ty"]["status"] != "success":
+            output = hard_results["ty"].get("output", "")
+            console.print(f"[red]Ty issues found:[/red]\n{output}")
+        if hard_results["radon_cc"]["status"] != "success":
+            issues = hard_results["radon_cc"].get("issues", [])
+            console.print(
+                f"[red]Cyclomatic Complexity too high "
+                f"({len(issues)} functions > 15):[/red]"
+            )
+            for issue in issues:
+                console.print(
+                    f"  - {issue['file']}: {issue['type']} '{issue['name']}' "
+                    f"has CC {issue['complexity']}"
+                )
+        sys.exit(1)
+        
+    console.print("[bold green]Hard Evaluation Passed![/bold green]")
+    
+    # Print Maintainability Index scorecard
+    mi_scores = hard_results.get("radon_mi", {}).get("mi_scores", {})
+    if mi_scores:
+        avg_mi = sum(mi_scores.values()) / len(mi_scores)
+        color = "green" if avg_mi > 50 else "yellow" if avg_mi > 20 else "red"
+        console.print(
+            f"[{color}]Average Maintainability Index: {avg_mi:.1f}/100[/{color}]"
+        )
+
+    # 2. Governance/QC Evaluation (Second Fence)
+    console.print("\n[bold blue]Running Governance QC (Second Fence)...[/bold blue]")
+    qc_results = evaluator.qc_evaluator.evaluate()
+    
+    if not qc_results["all_passed"]:
+        console.print("[bold red]Governance QC Failed! Exiting.[/bold red]")
+        console.print(
+            "[red]The proposed changes violate governance constraints "
+            "or lack sufficient evidence.[/red]"
+        )
+        for failure in qc_results["failures"]:
+            console.print(f"[red]- {failure}[/red]")
+        sys.exit(1)
+        
+    console.print(
+        "[bold green]Governance QC Passed! (Change is admissible)[/bold green]"
+    )
+
+    # 3. Soft Evaluation/Readability (Third Fence)
+    console.print(
+        "[bold blue]Running Soft Evaluation "
+        "(Readability & Understandability)...[/bold blue]"
+    )
+    soft_results = evaluator.soft_evaluator.evaluate()
+
+    pkg_summary = soft_results["package_summary"]
+    console.print(
+        f"[green]Analyzed {pkg_summary['total_files']} files with a total of "
+        f"{pkg_summary['total_tokens']} tokens.[/green]"
+    )
+    console.print(
+        f"[magenta]Agent's Understanding of the Package:[/magenta]\n"
+        f"{pkg_summary['package_understanding']}"
+    )
+    
+    console.print(
+        f"\n[cyan]Overall Understandability Score:[/cyan] "
+        f"{soft_results['understandability_score']:.1f}/100"
+    )
+    
+    qa_results = soft_results.get("qa_results", {}).get("sampled_entities", [])
+    if qa_results:
+        console.print("\n[bold yellow]Blind QA Sampling Results:[/bold yellow]")
+        for qa in qa_results:
+            color = "green" if qa['score'] >= 80 else "red"
+            console.print(f"  - [{color}]{qa['entity']}: Score {qa['score']}[/{color}]")
+            console.print(f"    [dim]Feedback: {qa['feedback']}[/dim]")
+
+    console.print("\n[yellow]Evaluation completed. Generating report...[/yellow]\n")
+    
+    # Generate Final Report
+    final_report = evaluator.soft_evaluator.generate_final_report(
+        hard_results, soft_results
+    )
+    
+    if final_report:
+        verdict = final_report.get("verdict", "Unknown")
+        verdict_color = "bold green" if "Pass" in verdict else "bold red"
+        
+        console.print(
+            f"[{verdict_color}]=== FINAL VERDICT: {verdict} ===[/{verdict_color}]"
+        )
+        console.print(f"[bold]Summary:[/bold] {final_report.get('summary', '')}\n")
+        
+        suggestions = final_report.get("suggestions", [])
+        if suggestions:
+            console.print("[bold cyan]Top 3 Improvement Suggestions:[/bold cyan]")
+            for i, sug in enumerate(suggestions, 1):
+                console.print(
+                    f"  {i}. [bold]{sug.get('title', 'Suggestion')}[/bold] "
+                    f"(Target: [yellow]{sug.get('target_file', 'unknown')}[/yellow])"
+                )
+                console.print(f"     [dim]{sug.get('description', '')}[/dim]")
+        
+        if "Fail" in verdict:
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    app()

python_harness/evaluator.py

@@ -0,0 +1,42 @@
+"""
+Core module for integrating all evaluations and producing the final report.
+"""
+
+from typing import Any
+
+from python_harness.hard_evaluator import HardEvaluator
+from python_harness.qc_evaluator import QCEvaluator
+from python_harness.soft_evaluator import SoftEvaluator
+
+
+class Evaluator:
+    """
+    Main evaluator coordinating hard, QC, and soft assessments.
+    """
+
+    def __init__(self, target_path: str):
+        self.target_path = target_path
+        self.hard_evaluator = HardEvaluator(target_path)
+        self.qc_evaluator = QCEvaluator(target_path)
+        self.soft_evaluator = SoftEvaluator(target_path)
+
+    def run(self) -> dict[str, Any]:
+        """
+        Run the complete evaluation process.
+        """
+        hard_results = self.hard_evaluator.evaluate()
+        qc_results = self.qc_evaluator.evaluate()
+        soft_results = self.soft_evaluator.evaluate()
+
+        # Generate Final Synthesized Report with 3 Suggestions
+        final_report = self.soft_evaluator.generate_final_report(
+            hard_results, soft_results
+        )
+
+        return {
+            "hard_evaluation": hard_results,
+            "qc_evaluation": qc_results,
+            "soft_evaluation": soft_results,
+            "final_report": final_report,
+            "overall_status": "success",
+        }

python_harness/hard_evaluator.py

@@ -0,0 +1,200 @@
+"""
+Core module for integrating hard evaluation tools like ruff, mypy, and pytest.
+"""
+
+import json
+import subprocess
+from pathlib import Path
+from typing import Any
+
+from rich.console import Console
+
+console = Console()
+
+class HardEvaluator:
+    """
+    Evaluator for collecting structural code quality metrics.
+    """
+
+    def __init__(self, target_path: str):
+        self.target_path = Path(target_path).resolve()
+
+    def run_ruff(self) -> dict[str, Any]:
+        """
+        Run Ruff linter and return results.
+        """
+        try:
+            result = subprocess.run(
+                ["ruff", "check", str(self.target_path), "--output-format", "json"],
+                capture_output=True,
+                text=True,
+                check=False
+            )
+            issues = json.loads(result.stdout) if result.stdout else []
+            status = "success" if result.returncode == 0 else "failed"
+            return {
+                "status": status,
+                "issues": issues,
+                "return_code": result.returncode,
+            }
+        except Exception as e:
+            return {"status": "error", "error_message": str(e)}
+
+    def run_mypy(self) -> dict[str, Any]:
+        """
+        Run Mypy type checker and return results.
+        """
+        try:
+            result = subprocess.run(
+                ["mypy", str(self.target_path)],
+                capture_output=True,
+                text=True,
+                check=False
+            )
+            status = "success" if result.returncode == 0 else "failed"
+            return {
+                "status": status,
+                "output": result.stdout,
+                "return_code": result.returncode,
+            }
+        except Exception as e:
+            return {"status": "error", "error_message": str(e)}
+
+    def run_ty(self) -> dict[str, Any]:
+        """
+        Run ty language server checks.
+        """
+        try:
+            result = subprocess.run(
+                ["ty", "check", str(self.target_path)],
+                capture_output=True,
+                text=True,
+                check=False
+            )
+            status = "success" if result.returncode == 0 else "failed"
+            return {
+                "status": status,
+                "output": result.stdout,
+                "return_code": result.returncode,
+            }
+        except Exception as e:
+            return {"status": "error", "error_message": str(e)}
+
+    def run_radon_cc(self) -> dict[str, Any]:
+        """
+        Run Radon cyclomatic complexity check.
+        Flag any function/method with CC > 15 as a failure.
+        """
+        try:
+            result = subprocess.run(
+                ["radon", "cc", "-j", "-a", str(self.target_path)],
+                capture_output=True,
+                text=True,
+                check=False
+            )
+            
+            issues = []
+            status = "success"
+            
+            if result.stdout:
+                data = json.loads(result.stdout)
+                for file_path, blocks in data.items():
+                    if isinstance(blocks, list):
+                        for block in blocks:
+                            if block.get('complexity', 0) > 15:
+                                issues.append({
+                                    "file": file_path,
+                                    "name": block.get('name'),
+                                    "type": block.get('type'),
+                                    "complexity": block.get('complexity')
+                                })
+            
+            if issues:
+                status = "failed"
+                
+            return {
+                "status": status,
+                "issues": issues,
+                "return_code": result.returncode,
+                "output": result.stdout
+            }
+        except Exception as e:
+            return {"status": "error", "error_message": str(e)}
+
+    def run_radon_mi(self) -> dict[str, Any]:
+        """
+        Run Radon Maintainability Index (MI) check.
+        This is a diagnostic metric, so it won't fail the build,
+        but it contributes to the scorecard.
+        """
+        try:
+            result = subprocess.run(
+                ["radon", "mi", "-j", str(self.target_path)],
+                capture_output=True,
+                text=True,
+                check=False
+            )
+            
+            mi_scores = {}
+            if result.stdout:
+                data = json.loads(result.stdout)
+                for file_path, info in data.items():
+                    mi_scores[file_path] = info.get('mi', 100.0)
+                    
+            return {
+                "status": "success",
+                "mi_scores": mi_scores,
+                "return_code": result.returncode,
+            }
+        except Exception as e:
+            return {"status": "error", "error_message": str(e)}
+
+    def run_pytest(self) -> dict[str, Any]:
+        """
+        Run Pytest test suite and return coverage results.
+        """
+        try:
+            # When pytest is run within pytest, it can cause issues or hang.
+            # Here we just run it as a subprocess to gather results.
+            result = subprocess.run(
+                ["pytest", str(self.target_path), "--cov", "--cov-report=json"],
+                capture_output=True,
+                text=True,
+                check=False
+            )
+            status = "success" if result.returncode == 0 else "failed"
+            return {
+                "status": status,
+                "output": result.stdout,
+                "return_code": result.returncode,
+            }
+        except Exception as e:
+             return {"status": "error", "error_message": str(e)}
+
+    def evaluate(self) -> dict[str, Any]:
+        """
+        Execute all hard evaluation tools.
+        Returns a dictionary with results and an overall success boolean.
+        """
+        ruff_res = self.run_ruff()
+        mypy_res = self.run_mypy()
+        ty_res = self.run_ty()
+        radon_cc_res = self.run_radon_cc()
+        radon_mi_res = self.run_radon_mi()
+        # pytest_res = self.run_pytest() # Better handled as a separate stage
+        
+        all_passed = (
+            ruff_res.get("status") == "success" and 
+            mypy_res.get("status") == "success" and
+            ty_res.get("status") == "success" and
+            radon_cc_res.get("status") == "success"
+        )
+
+        return {
+            "all_passed": all_passed,
+            "ruff": ruff_res,
+            "mypy": mypy_res,
+            "ty": ty_res,
+            "radon_cc": radon_cc_res,
+            "radon_mi": radon_mi_res
+        }

python_harness/qc_evaluator.py

@@ -0,0 +1,89 @@
+"""
+Core module for evaluating self-improvement Governance and Quality Control (QC).
+Based on a simplified version of the LUCA/Sympan profile.
+"""
+
+from pathlib import Path
+from typing import Any
+
+
+class QCEvaluator:
+    """
+    Evaluator for checking Governance and QC constraints.
+    """
+
+    def __init__(self, target_path: str):
+        self.target_path = Path(target_path).resolve()
+
+    def check_hard_invariants(self) -> dict[str, Any]:
+        """
+        Verify that fundamental identity invariants are preserved.
+        - Ensure no bypassing of core evaluation logic.
+        - Check for explicit architectural violations.
+        """
+        failures: list[str] = []
+        
+        # Example Structural Check: 
+        # Has the agent modified the evaluation core directly without 
+        # going through a proper class D proposal?
+        # In a real system, we'd check git diffs or file modification times here.
+        # For now, we will simulate passing this invariant.
+        
+        return {
+            "status": "success",
+            "failures": failures
+        }
+
+    def check_obligations(self) -> dict[str, Any]:
+        """
+        Verify that necessary evidence and obligations are met for the changes made.
+        Every change MUST provide an Improvement Case (evidence).
+        """
+        failures: list[str] = []
+        
+        # In a real implementation, we would check if the proposal manifest 
+        # defines required obligations and whether corresponding reports 
+        # (benchmark, holdout) exist in the artifacts.
+        
+        return {
+            "status": "success",
+            "failures": failures
+        }
+
+    def check_self_touch(self) -> dict[str, Any]:
+        """
+        Verify if the agent modified the evaluation or governance criteria (Level 1/2).
+        If it did, flag it for external certification.
+        """
+        failures: list[str] = []
+        
+        # Example check: If the agent modifies QC rules or evaluation logic,
+        # it MUST require external certification (Human or higher-level Judge).
+        
+        return {
+            "status": "success",
+            "failures": failures
+        }
+
+    def evaluate(self) -> dict[str, Any]:
+        """
+        Run all QC checks.
+        """
+        invariants = self.check_hard_invariants()
+        obligations = self.check_obligations()
+        self_touch = self.check_self_touch()
+
+        failures: list[str] = []
+        failures.extend(invariants.get("failures", []))
+        failures.extend(obligations.get("failures", []))
+        failures.extend(self_touch.get("failures", []))
+
+        all_passed = len(failures) == 0
+
+        return {
+            "all_passed": all_passed,
+            "failures": failures,
+            "invariants": invariants,
+            "obligations": obligations,
+            "self_touch": self_touch
+        }

python_harness/soft_evaluator.py

@@ -0,0 +1,486 @@
+"""
+Core module for agentic soft evaluation and code understanding.
+"""
+
+import ast
+import contextlib
+import json
+import os
+import random
+from pathlib import Path
+from typing import Any
+
+import tiktoken
+from openai import OpenAI
+from pydantic import BaseModel
+from rich.console import Console
+
+console = Console()
+
+class FileSummary(BaseModel):
+    summary: str
+    key_entities: list[str]
+    complexity_score: int
+
+class SoftEvaluator:
+    """
+    Evaluator for agentic code understanding and reasoning.
+    """
+
+    def __init__(self, target_path: str):
+        self.target_path = Path(target_path).resolve()
+        # Initialize token counter (using cl100k_base for gpt-4/claude-3)
+        self.encoding: Any = None
+        with contextlib.suppress(Exception):
+            self.encoding = tiktoken.get_encoding("cl100k_base")
+
+        # Initialize OpenAI client only if API key is present
+        self.client = None
+        api_key = os.environ.get("LLM_API_KEY")
+        base_url = os.environ.get("LLM_BASE_URL", "https://api.deepseek.com/v1")
+        self.model_name = os.environ.get("LLM_MODEL_NAME", "deepseek-reasoner")
+        self.mini_model_name = os.environ.get("LLM_MINI_MODEL_NAME", "deepseek-chat")
+
+        if api_key:
+            self.client = OpenAI(api_key=api_key, base_url=base_url)
+        else:
+            console.print(
+                "[yellow]Warning: LLM_API_KEY not set. "
+                "Agent will run in mock mode.[/yellow]"
+            )
+            
+        # Store extracted AST entities for sampling
+        self.extracted_entities: list[dict[str, Any]] = []
+
+    def _get_python_files(self) -> list[Path]:
+        """
+        Recursively find all Python files in the target directory,
+        excluding hidden dirs and .venv.
+        """
+        python_files = []
+        for root, dirs, files in os.walk(self.target_path):
+            # Exclude hidden directories and virtual environments
+            dirs[:] = [
+                d
+                for d in dirs
+                if not d.startswith(".") and d not in (
+                    "__pycache__",
+                    "venv",
+                    "env",
+                    "vendors",
+                )
+            ]
+            for file in files:
+                if file.endswith(".py"):
+                    python_files.append(Path(root) / file)
+        return python_files
+
+    def calculate_token_complexity(self, file_path: Path) -> int:
+        """
+        Calculate the token count for a given file as a proxy
+        for cognitive complexity.
+        """
+        if not self.encoding:
+            return 0
+
+        try:
+            content = file_path.read_text(encoding="utf-8")
+            return len(self.encoding.encode(content))
+        except Exception as e:
+            console.print(
+                f"[yellow]Warning: Could not read {file_path} for token counting: "
+                f"{e}[/yellow]"
+            )
+            return 0
+
+    def _extract_ast_entities(self, file_path: Path, content: str) -> None:
+        """
+        Parse the AST of a file to extract:
+        1. Classes and functions for later QA sampling.
+        2. Fan-out (number of imported external modules) to measure coupling.
+        """
+        try:
+            tree = ast.parse(content)
+            
+            # Calculate Fan-out (number of unique imported top-level modules)
+            imported_modules = set()
+            for node in ast.walk(tree):
+                if isinstance(node, ast.Import):
+                    for alias in node.names:
+                        imported_modules.add(alias.name.split('.')[0])
+                elif isinstance(node, ast.ImportFrom) and node.module:
+                    imported_modules.add(node.module.split('.')[0])
+            
+            fan_out = len(imported_modules)
+            
+            # Extract classes and functions
+            for node in ast.walk(tree):
+                if isinstance(
+                    node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)
+                ):
+                    try:
+                        source_segment = ast.get_source_segment(content, node)
+                        if source_segment:
+                            if isinstance(node, ast.ClassDef):
+                                entity_type = "Class"
+                            else:
+                                entity_type = "Function"
+                            self.extracted_entities.append(
+                                {
+                                    "file": file_path.name,
+                                    "type": entity_type,
+                                    "name": node.name,
+                                    "code": source_segment,
+                                    "fan_out": fan_out,  # Context
+                                }
+                            )
+                    except Exception:
+                        pass
+        except SyntaxError:
+            pass  # Skip files with syntax errors
+
+    def summarize_file(self, file_path: Path) -> dict[str, Any]:
+        """
+        Use LLM agent to summarize a single file's logic and structure.
+        """
+        tokens = self.calculate_token_complexity(file_path)
+
+        try:
+            content = file_path.read_text(encoding="utf-8")
+            self._extract_ast_entities(file_path, content)
+        except Exception:
+            content = ""
+
+        simulated_summary = f"File {file_path.name} contains {tokens} tokens."
+        key_entities: list[str] = []
+
+        # Only call LLM if file is not empty and not too large (e.g. > 100k tokens)
+        if self.client and content and 0 < tokens < 100000:
+            try:
+                sys_prompt = (
+                    "You are a senior Python architect. Analyze the provided Python "
+                    "file and provide a concise summary of its purpose, a list of "
+                    "its key entities (classes/functions/globals), and an estimated "
+                    "cognitive complexity score (1-10).\n"
+                    "Output MUST be in valid JSON matching this schema: "
+                    '{"summary": "str", "key_entities": ["str"], "complexity_score": 1}'
+                )
+                completion = self.client.chat.completions.create(
+                    model=self.mini_model_name,
+                    messages=[
+                        {
+                            "role": "system",
+                            "content": sys_prompt,
+                        },
+                        {
+                            "role": "user",
+                            "content": (
+                                f"File name: {file_path.name}\n\nContent:\n"
+                                f"```python\n{content}\n```"
+                            ),
+                        },
+                    ],
+                    response_format={"type": "json_object"},
+                )
+
+                content_str = completion.choices[0].message.content
+                if content_str:
+                    result = json.loads(content_str)
+                    simulated_summary = result.get("summary", simulated_summary)
+                    key_entities = result.get("key_entities", key_entities)
+            except Exception as e:
+                console.print(
+                    f"[yellow]  Agent failed to read {file_path.name}: {e}[/yellow]"
+                )
+
+        try:
+            rel_path = str(file_path.relative_to(self.target_path))
+        except ValueError:
+            rel_path = str(file_path)
+
+        return {
+            "file": rel_path,
+            "tokens": tokens,
+            "summary": simulated_summary,
+            "key_entities": key_entities,
+        }
+
+    def summarize_package(self) -> dict[str, Any]:
+        """
+        Aggregate file summaries into a package-level understanding.
+        """
+        files = self._get_python_files()
+        file_summaries = []
+        total_tokens = 0
+
+        console.print(
+            f"[cyan]Agent is analyzing {len(files)} Python files "
+            f"for cognitive load and architecture...[/cyan]"
+        )
+
+        for file in files:
+            summary_data = self.summarize_file(file)
+            file_summaries.append(summary_data)
+            total_tokens += summary_data["tokens"]
+
+        # Synthesize package architecture
+        package_understanding = (
+            f"The package contains {len(files)} files with a total cognitive load "
+            f"of {total_tokens} tokens."
+        )
+        
+        if self.client and file_summaries:
+            try:
+                console.print(
+                    "[cyan]Agent is synthesizing global package architecture...[/cyan]"
+                )
+                manifest_lines = [
+                    f"- {s['file']}: {s['summary']} "
+                    f"(Entities: {', '.join(s['key_entities'])})"
+                    for s in file_summaries
+                ]
+                manifest = "\n".join(manifest_lines)
+
+                sys_prompt = (
+                    "You are a senior software architect. Based on the following "
+                    "summaries of individual files in a Python package, write a "
+                    "coherent, high-level explanation of how this entire package "
+                    "works and what its primary responsibilities are. Be concise "
+                    "but comprehensive."
+                )
+
+                completion = self.client.chat.completions.create(
+                    model=self.model_name,
+                    messages=[
+                        {
+                            "role": "system",
+                            "content": sys_prompt,
+                        },
+                        {
+                            "role": "user",
+                            "content": f"Package files and summaries:\n{manifest}",
+                        },
+                    ],
+                )
+
+                package_understanding = (
+                    completion.choices[0].message.content or package_understanding
+                )
+            except Exception as e:
+                console.print(
+                    f"[yellow]Agent failed to synthesize package: {e}[/yellow]"
+                )
+
+        return {
+            "total_files": len(files),
+            "total_tokens": total_tokens,
+            "file_level_summaries": file_summaries,
+            "package_understanding": package_understanding,
+        }
+
+    def run_sampling_qa(self) -> dict[str, Any]:
+        """
+        Randomly sample modules/variables and ask the Agent questions
+        to measure understandability.
+        """
+        if not self.extracted_entities:
+            return {
+                "qa_score": 100.0,
+                "sampled_entities": [],
+                "note": "No entities found for sampling.",
+            }
+
+        # Randomly sample up to 3 entities
+        sample_size = min(3, len(self.extracted_entities))
+        sampled = random.sample(self.extracted_entities, sample_size)
+
+        console.print(
+            f"\n[cyan]Agent is running Blind QA on {sample_size} "
+            f"sampled entities...[/cyan]"
+        )
+
+        qa_results = []
+        total_score = 0.0
+
+        for entity in sampled:
+            entity_name = entity["name"]
+            entity_type = entity["type"]
+            entity_code = entity["code"]
+            fan_out = entity.get("fan_out", 0)
+
+            if not self.client:
+                # Mock evaluation
+                score = 100.0
+                feedback = "Mock evaluation: Code is perfectly readable."
+            else:
+                try:
+                    sys_prompt = (
+                        "You are an expert Code Reviewer and Software Architect. "
+                        "You will be given a snippet of Python code (a class or "
+                        "function) along with its module's Fan-out metric (number "
+                        "of external dependencies). Your task is to evaluate its "
+                        "readability and structural cohesion.\n"
+                        "Output MUST be in valid JSON matching this schema: "
+                        '{"explanation": "str", "readability_score": 1, '
+                        '"feedback": "str"}\n'
+                        "- `explanation`: Briefly explain what this code does.\n"
+                        "- `readability_score`: A score from 0 to 100.\n"
+                        "- `feedback`: What makes it easy/hard to understand? "
+                        "Does a high Fan-out indicate bad cohesion here?"
+                    )
+
+                    user_content = (
+                        f"Module Fan-out (Dependencies): {fan_out}\n\n"
+                        f"Code Snippet:\n```python\n{entity_code}\n```"
+                    )
+
+                    completion = self.client.chat.completions.create(
+                        model=self.mini_model_name,
+                        messages=[
+                            {"role": "system", "content": sys_prompt},
+                            {"role": "user", "content": user_content},
+                        ],
+                        response_format={"type": "json_object"},
+                    )
+
+                    content_str = completion.choices[0].message.content
+                    if content_str:
+                        result = json.loads(content_str)
+                        score = float(result.get("readability_score", 100))
+                        feedback = result.get("feedback", "")
+                    else:
+                        score = 80.0
+                        feedback = "Failed to parse Agent response."
+                except Exception as e:
+                    score = 0.0
+                    feedback = f"Error during Agent evaluation: {e}"
+
+            total_score += score
+            qa_results.append(
+                {
+                    "entity": f"{entity_type} {entity_name} (from {entity['file']})",
+                    "score": score,
+                    "feedback": feedback,
+                }
+            )
+
+        final_average_score = total_score / sample_size if sample_size > 0 else 100.0
+
+        return {
+            "qa_score": final_average_score,
+            "sampled_entities": qa_results,
+            "note": "Sampling QA completed.",
+        }
+
+    def generate_final_report(
+        self, hard_results: dict[str, Any], soft_results: dict[str, Any]
+    ) -> dict[str, Any]:
+        """
+        Synthesize all evaluation results into a final verdict and exactly 
+        3 actionable suggestions.
+        """
+        if not self.client:
+            return {
+                "verdict": "Pass (Mock)",
+                "summary": "Mock evaluation completed without LLM.",
+                "suggestions": [
+                    {
+                        "title": "Mock Suggestion 1",
+                        "description": "Add more docstrings.",
+                        "target_file": "all"
+                    },
+                    {
+                        "title": "Mock Suggestion 2",
+                        "description": "Refactor large functions.",
+                        "target_file": "all"
+                    },
+                    {
+                        "title": "Mock Suggestion 3",
+                        "description": "Improve test coverage.",
+                        "target_file": "tests/"
+                    }
+                ]
+            }
+
+        try:
+            # Extract key metrics for the prompt
+            cc_issues = hard_results.get("radon_cc", {}).get("issues", [])
+            mi_scores = hard_results.get("radon_mi", {}).get("mi_scores", {})
+            avg_mi = sum(mi_scores.values()) / len(mi_scores) if mi_scores else 100.0
+            
+            qa_score = soft_results.get("understandability_score", 100.0)
+            qa_entities = soft_results.get("qa_results", {}).get("sampled_entities", [])
+            
+            sys_prompt = (
+                "You are an elite Python Codebase Evaluator. You have just analyzed "
+                "a repository. Your task is to provide a final judgment and EXACTLY "
+                "3 concrete, actionable improvement suggestions. These suggestions "
+                "MUST NOT change the external functionality (they are refactoring/"
+                "quality improvements).\n\n"
+                "Output MUST be in valid JSON matching this schema:\n"
+                "{\n"
+                '  "verdict": "Pass" or "Fail",\n'
+                '  "summary": "One paragraph summary of codebase health",\n'
+                '  "suggestions": [\n'
+                '    {"title": "str", "description": "str", "target_file": "str"}\n'
+                "  ]\n"
+                "}\n"
+                "Rule for Verdict: Pass if Average Maintainability > 50 and "
+                "QA Score > 75 and no Critical CC issues (>15). Otherwise Fail."
+            )
+
+            user_content = (
+                f"Metrics:\n"
+                f"- Average Maintainability Index (MI): {avg_mi:.1f}/100\n"
+                f"- Number of functions with Cyclomatic Complexity > 15: "
+                f"{len(cc_issues)}\n"
+                f"- Agent QA Readability Score: {qa_score:.1f}/100\n\n"
+                f"QA Feedback Snippets:\n"
+                + "\n".join(
+                    [f"  * {q['entity']}: {q['feedback']}" for q in qa_entities]
+                )
+            )
+
+            completion = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=[
+                    {"role": "system", "content": sys_prompt},
+                    {"role": "user", "content": user_content},
+                ],
+                response_format={"type": "json_object"},
+            )
+
+            content_str = completion.choices[0].message.content
+            if content_str:
+                parsed_json = json.loads(content_str)
+                if isinstance(parsed_json, dict):
+                    return parsed_json
+                else:
+                    raise ValueError("JSON response is not a dictionary.")
+            else:
+                raise ValueError("Empty response from Agent.")
+        except Exception as e:
+            console.print(f"[yellow]Failed to generate final report: {e}[/yellow]")
+            return {
+                "verdict": "Error",
+                "summary": f"Failed to synthesize report: {e}",
+                "suggestions": []
+            }
+    def evaluate(self) -> dict[str, Any]:
+        """
+        Execute soft evaluation workflows including summarization and Q&A.
+        """
+        package_summary = self.summarize_package()
+        qa_results = self.run_sampling_qa()
+
+        # Calculate a mock understandability score based on token density
+        # (just an example heuristic)
+        # In reality, this will be based on the QA results and LLM judge
+        understandability_score = qa_results["qa_score"]
+
+        return {
+            "status": "success",
+            "understandability_score": understandability_score,
+            "package_summary": package_summary,
+            "qa_results": qa_results,
+        }

python_harness-0.0.1.dist-info/METADATA

@@ -0,0 +1,88 @@
+Metadata-Version: 2.4
+Name: python-harness
+Version: 0.0.1
+Summary: An agentic codebase evaluation and evolution tool for Python projects.
+Author-email: Mingli Yuan <mingli.yuan@gmail.com>
+License: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer>=0.9.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: anthropic>=0.18.0
+Requires-Dist: tenacity>=8.2.0
+Requires-Dist: tiktoken>=0.6.0
+Requires-Dist: python-dotenv>=1.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: ruff>=0.3.0; extra == "dev"
+Requires-Dist: mypy>=1.9.0; extra == "dev"
+Requires-Dist: ty>=0.0.1; extra == "dev"
+Requires-Dist: radon>=6.0.1; extra == "dev"
+Dynamic: license-file
+
+# Python Harness
+
+An agentic codebase evaluation and evolution tool for Python projects.
+
+`python-harness` is designed to be a universal standard tool—just like `pytest` or `ruff`—but instead of just checking syntax or running tests, it evaluates the **architecture, readability, and governance** of your codebase using both static analysis and LLMs (DeepSeek/OpenAI).
+
+## Features
+
+1. **Hard Evaluation (First Fence)**: Enforces strict rules using `ruff`, `mypy`, and `ty`. Evaluates Cyclomatic Complexity (CC) and Maintainability Index (MI) via `radon`.
+2. **Governance QC (Second Fence)**: Checks if the changes violate core project governance or attempt to bypass the evaluation rules themselves.
+3. **Soft Evaluation (Third Fence)**:
+   - Calculates architecture metrics like Fan-out (coupling).
+   - Generates a holistic package understanding using LLMs.
+   - Performs "Blind QA": Randomly samples functions/classes and tests the LLM's ability to understand them without context.
+4. **Actionable Output**: Synthesizes the evaluation into a final `Pass/Fail` verdict with exactly 3 concrete, actionable refactoring suggestions.
+
+## Installation
+
+You can install `python-harness` using `uv` or `pip`:
+
+```bash
+uv pip install python-harness
+```
+
+## Configuration
+
+`python-harness` requires an LLM to perform its soft evaluation. Create a `.env` file in the root of your project:
+
+```env
+LLM_API_KEY=your_api_key_here
+LLM_BASE_URL=https://api.deepseek.com/v1
+LLM_MODEL_NAME=deepseek-reasoner
+LLM_MINI_MODEL_NAME=deepseek-chat
+```
+
+*(Note: If you don't provide an API key, the harness will safely run in Mock mode).*
+
+## Usage
+
+### 1. Measure
+
+To evaluate your codebase, run the `measure` command in your project directory:
+
+```bash
+harness measure .
+```
+
+This will run the full 3-fence evaluation and output a report with a final verdict and top 3 improvement suggestions.
+
+### 2. Refine (Evolution Loop - WIP)
+
+The `refine` command is an Agentic Edit-Test-Improve loop. It takes the suggestions generated by `measure`, automatically creates branches (variants), applies the changes, runs the tests (`pytest`), and picks the best variant.
+
+```bash
+harness refine . --steps 1 --max-retries 3
+```
+
+## License
+
+MIT License. See [LICENSE](LICENSE) for more details.
+
+A harness toolkit for Python projects

python_harness-0.0.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+python_harness/__init__.py,sha256=i1W_gUvOQUO7yp0Vu_jQbOYiqR5Nf651N5A36E118X8,90
+python_harness/cli.py,sha256=zyhC5gqo3CaieRKLkcaw_zDkOZr0PnEdzorG2AfU4bA,7815
+python_harness/evaluator.py,sha256=Mfyg5vvL0GMKxnKLqokd5dxn0J1ob8yBd7-316zsaGw,1307
+python_harness/hard_evaluator.py,sha256=AHcS2jn1GyePv8UK0vvpa5rH7bwXChpg5A_5m5WHGxk,6669
+python_harness/qc_evaluator.py,sha256=Mw_nxu253ERwV4lzWEhTTL9iN3_qBsMmi72Fz9cA4Fw,2883
+python_harness/soft_evaluator.py,sha256=OfTaRT2h14_VzNlMx1qlj2RzF_dcD-TFhjZaCri-TUg,19117
+python_harness-0.0.1.dist-info/licenses/LICENSE,sha256=rMiBapfK7KDDmBOyspVvaqy1OFWHUe-0DoiRE9A3dL0,1068
+python_harness-0.0.1.dist-info/METADATA,sha256=knKHBMROxKTsbd3_SSJFDr80HVjPIa5bdBEHSJ0Sknc,3149
+python_harness-0.0.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+python_harness-0.0.1.dist-info/entry_points.txt,sha256=6OPLethPEEz4MlRoxgUx7SYmN22P2aogi1jaorcrVVM,51
+python_harness-0.0.1.dist-info/top_level.txt,sha256=PxPMOpwPhfTaZxV4tX2LuRS5Sb6MEGutOm62DYMXXCQ,15
+python_harness-0.0.1.dist-info/RECORD,,

python_harness-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

python_harness-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+harness = python_harness.cli:app

python_harness-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mingli Yuan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

python_harness-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+python_harness