genlayer-test 0.4.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

gltest/{glchain/contract.py → contracts/contract_factory.py}

@@ -1,145 +1,26 @@
+from dataclasses import dataclass
+from typing import Type, Union, Optional, List, Any
+from pathlib import Path
 from eth_typing import (
     Address,
     ChecksumAddress,
 )
 from eth_account.signers.local import LocalAccount
-from typing import Union
-from pathlib import Path
-from dataclasses import dataclass
 from gltest.artifacts import (
     find_contract_definition_from_name,
     find_contract_definition_from_path,
 )
+from gltest.clients import (
+    get_gl_client,
+    get_gl_hosted_studio_client,
+    get_local_client,
+)
+from .contract import Contract
+from gltest.logging import logger
+from gltest.types import TransactionStatus
 from gltest.assertions import tx_execution_failed
 from gltest.exceptions import DeploymentError
-from .client import get_gl_client, get_gl_hosted_studio_client, get_local_client
-from gltest.types import CalldataEncodable, GenLayerTransaction, TransactionStatus
-from typing import List, Any, Type, Optional, Dict, Callable
-import types
 from gltest_cli.config.general import get_general_config
-from gltest.logging import logger
-
-
-@dataclass
-class Contract:
-    """
-    Class to interact with a contract, its methods
-    are implemented dynamically at build time.
-    """
-
-    address: str
-    account: Optional[LocalAccount] = None
-    _schema: Optional[Dict[str, Any]] = None
-
-    @classmethod
-    def new(
-        cls,
-        address: str,
-        schema: Dict[str, Any],
-        account: Optional[LocalAccount] = None,
-    ) -> "Contract":
-        """
-        Build the methods from the schema.
-        """
-        if not isinstance(schema, dict) or "methods" not in schema:
-            raise ValueError("Invalid schema: must contain 'methods' field")
-        instance = cls(address=address, _schema=schema, account=account)
-        instance._build_methods_from_schema()
-        return instance
-
-    def _build_methods_from_schema(self):
-        if self._schema is None:
-            raise ValueError("No schema provided")
-        for method_name, method_info in self._schema["methods"].items():
-            if not isinstance(method_info, dict) or "readonly" not in method_info:
-                raise ValueError(
-                    f"Invalid method info for '{method_name}': must contain 'readonly' field"
-                )
-            method_func = self.contract_method_factory(
-                method_name, method_info["readonly"]
-            )
-            bound_method = types.MethodType(method_func, self)
-            setattr(self, method_name, bound_method)
-
-    def connect(self, account: LocalAccount) -> "Contract":
-        """
-        Create a new instance of the contract with the same methods and a different account.
-        """
-        new_contract = self.__class__(
-            address=self.address, account=account, _schema=self._schema
-        )
-        new_contract._build_methods_from_schema()
-        return new_contract
-
-    @staticmethod
-    def contract_method_factory(method_name: str, read_only: bool) -> Callable:
-        """
-        Create a function that interacts with a specific contract method.
-        """
-
-        def read_contract_wrapper(
-            self,
-            args: Optional[List[CalldataEncodable]] = None,
-        ) -> Any:
-            """
-            Wrapper to the contract read method.
-            """
-            client = get_gl_client()
-            return client.read_contract(
-                address=self.address,
-                function_name=method_name,
-                account=self.account,
-                args=args,
-            )
-
-        def write_contract_wrapper(
-            self,
-            args: Optional[List[CalldataEncodable]] = None,
-            value: int = 0,
-            consensus_max_rotations: Optional[int] = None,
-            leader_only: bool = False,
-            wait_transaction_status: TransactionStatus = TransactionStatus.FINALIZED,
-            wait_interval: Optional[int] = None,
-            wait_retries: Optional[int] = None,
-            wait_triggered_transactions: bool = True,
-            wait_triggered_transactions_status: TransactionStatus = TransactionStatus.FINALIZED,
-        ) -> GenLayerTransaction:
-            """
-            Wrapper to the contract write method.
-            """
-            general_config = get_general_config()
-            if wait_interval is None:
-                wait_interval = general_config.get_default_wait_interval()
-            if wait_retries is None:
-                wait_retries = general_config.get_default_wait_retries()
-            client = get_gl_client()
-            tx_hash = client.write_contract(
-                address=self.address,
-                function_name=method_name,
-                account=self.account,
-                value=value,
-                consensus_max_rotations=consensus_max_rotations,
-                leader_only=leader_only,
-                args=args,
-            )
-            receipt = client.wait_for_transaction_receipt(
-                transaction_hash=tx_hash,
-                status=wait_transaction_status,
-                interval=wait_interval,
-                retries=wait_retries,
-            )
-            if wait_triggered_transactions:
-                triggered_transactions = receipt["triggered_transactions"]
-                for triggered_transaction in triggered_transactions:
-                    client.wait_for_transaction_receipt(
-                        transaction_hash=triggered_transaction,
-                        status=wait_triggered_transactions_status,
-                        interval=wait_interval,
-                        retries=wait_retries,
-                    )
-            return receipt
-
-        return read_contract_wrapper if read_only else write_contract_wrapper
 
 
 @dataclass
@@ -184,17 +65,17 @@ class ContractFactory:
         """Attempts to get the contract schema using multiple clients in a fallback pattern.
 
         This method tries to get the contract schema in the following order:
-        1. Hosted studio client
-        2. Local client
-        3. Regular client
+        1. Default client
+        2. Hosted studio client
+        3. Local client
 
         Returns:
             Optional[Dict[str, Any]]: The contract schema if successful, None if all attempts fail.
         """
         clients = (
+            ("default", get_gl_client()),
             ("hosted studio", get_gl_hosted_studio_client()),
             ("local", get_local_client()),
-            ("default", get_gl_client()),
         )
         for label, client in clients:
             try:
@@ -216,7 +97,7 @@ class ContractFactory:
         schema = self._get_schema_with_fallback()
         if schema is None:
             raise ValueError(
-                "Failed to get schema from all clients (hosted studio, local, and regular)"
+                "Failed to get schema from all clients (default, hosted studio, and local)"
             )
 
         return Contract.new(address=contract_address, schema=schema, account=account)
@@ -226,7 +107,6 @@ class ContractFactory:
         args: List[Any] = [],
         account: Optional[LocalAccount] = None,
         consensus_max_rotations: Optional[int] = None,
-        leader_only: bool = False,
         wait_interval: Optional[int] = None,
         wait_retries: Optional[int] = None,
         wait_transaction_status: TransactionStatus = TransactionStatus.FINALIZED,
@@ -239,6 +119,11 @@ class ContractFactory:
             wait_interval = general_config.get_default_wait_interval()
         if wait_retries is None:
             wait_retries = general_config.get_default_wait_retries()
+        leader_only = (
+            general_config.get_leader_only()
+            if general_config.check_studio_based_rpc()
+            else False
+        )
 
         client = get_gl_client()
         try:
@@ -273,7 +158,7 @@ class ContractFactory:
             schema = self._get_schema_with_fallback()
             if schema is None:
                 raise ValueError(
-                    "Failed to get schema from all clients (hosted studio, local, and regular)"
+                    "Failed to get schema from all clients (default, hosted studio, and local)"
                 )
 
             return Contract.new(

gltest/contracts/contract_functions.py

@@ -0,0 +1,59 @@
+from dataclasses import dataclass
+from typing import Callable, Optional, Dict, Any
+from gltest.types import TransactionStatus
+
+
+@dataclass
+class ContractFunction:
+    method_name: str
+    read_only: bool
+    call_method: Optional[Callable] = None
+    analyze_method: Optional[Callable] = None
+    transact_method: Optional[Callable] = None
+
+    def call(self):
+        if not self.read_only:
+            raise ValueError("call() not implemented for non-readonly method")
+        return self.call_method()
+
+    def transact(
+        self,
+        value: int = 0,
+        consensus_max_rotations: Optional[int] = None,
+        wait_transaction_status: TransactionStatus = TransactionStatus.FINALIZED,
+        wait_interval: Optional[int] = None,
+        wait_retries: Optional[int] = None,
+        wait_triggered_transactions: bool = True,
+        wait_triggered_transactions_status: TransactionStatus = TransactionStatus.FINALIZED,
+    ):
+        if self.read_only:
+            raise ValueError("Cannot transact read-only method")
+        return self.transact_method(
+            value=value,
+            consensus_max_rotations=consensus_max_rotations,
+            wait_transaction_status=wait_transaction_status,
+            wait_interval=wait_interval,
+            wait_retries=wait_retries,
+            wait_triggered_transactions=wait_triggered_transactions,
+            wait_triggered_transactions_status=wait_triggered_transactions_status,
+        )
+
+    def analyze(
+        self,
+        provider: str,
+        model: str,
+        config: Optional[Dict[str, Any]] = None,
+        plugin: Optional[str] = None,
+        plugin_config: Optional[Dict[str, Any]] = None,
+        runs: int = 100,
+    ):
+        if self.read_only:
+            raise ValueError("Cannot analyze read-only method")
+        return self.analyze_method(
+            provider=provider,
+            model=model,
+            config=config,
+            plugin=plugin,
+            plugin_config=plugin_config,
+            runs=runs,
+        )

gltest/contracts/method_stats.py

@@ -0,0 +1,163 @@
+from dataclasses import dataclass
+from typing import List, Any, Optional, Dict
+import json
+from pathlib import Path
+from .utils import safe_filename
+
+
+@dataclass
+class MethodStatsSummary:
+    """Statistical analysis results for a contract method execution."""
+
+    method: str
+    args: List[Any]
+    total_runs: int
+    executed_runs: int
+    server_error_runs: int
+    failed_runs: int
+    successful_runs: int
+    unique_states: int
+    most_common_state_count: int
+    reliability_score: float
+    execution_time: float
+    provider: str
+    model: str
+
+    @property
+    def success_rate(self) -> float:
+        """Calculate success rate as percentage."""
+        if self.total_runs == 0:
+            return 0.0
+        return (self.successful_runs / self.total_runs) * 100
+
+    def __str__(self) -> str:
+        """Format the statistical analysis results."""
+        return f"""Method analysis summary
+---------------------------
+Method: {self.method}
+Args: {self.args}
+Provider: {self.provider}
+Model: {self.model}
+Total runs: {self.total_runs}
+Server error runs: {self.server_error_runs}
+Method executed runs: {self.executed_runs}
+Method successful runs: {self.successful_runs}
+Method failed runs: {self.failed_runs}
+Unique states: {self.unique_states}
+Reliability score: {self.reliability_score:.2f}% ({self.most_common_state_count}/{self.executed_runs} consistent)
+Execution time: {self.execution_time:.1f}s"""
+
+
+@dataclass
+class StateGroup:
+    """Represents a group of runs with the same contract state."""
+
+    count: int
+    state_hash: str
+
+
+@dataclass
+class FailedRun:
+    """Represents a failed run with error details."""
+
+    run: int
+    error: str
+    error_type: str  # "server" or "simulation"
+    genvm_result: Optional[Dict[str, str]]
+
+
+@dataclass
+class MethodStatsDetailed:
+    """Detailed statistical analysis results for a contract method execution."""
+
+    method: str
+    params: List[Any]
+    timestamp: str
+    configuration: Dict[str, Any]
+    execution_time: float
+    executed_runs: int
+    failed_runs: int
+    successful_runs: int
+    server_error_runs: int
+    most_common_state_count: int
+    reliability_score: float
+    sim_results: List[Any]
+    unique_states: int
+    state_groups: List[StateGroup]
+    failed_runs_results: List[FailedRun]
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary format."""
+        return {
+            "method": self.method,
+            "params": self.params,
+            "timestamp": self.timestamp,
+            "configuration": self.configuration,
+            "execution_time": self.execution_time,
+            "method_executed_runs": self.executed_runs,
+            "method_failed_runs": self.failed_runs,
+            "method_successful_runs": self.successful_runs,
+            "server_error_runs": self.server_error_runs,
+            "most_common_state_count": self.most_common_state_count,
+            "reliability_score": self.reliability_score,
+            "unique_states": self.unique_states,
+            "state_groups": [
+                {"count": sg.count, "state_hash": sg.state_hash}
+                for sg in self.state_groups
+            ],
+            "failed_runs": [
+                {
+                    "run": fr.run,
+                    "error": fr.error,
+                    "error_type": fr.error_type,
+                    "genvm_result": fr.genvm_result,
+                }
+                for fr in self.failed_runs_results
+            ],
+            "simulation_results": self.filter_sim_results(),
+        }
+
+    def filter_sim_results(self) -> List[Any]:
+        """Filter the simulation results to only include specific fields."""
+        filtered_results = []
+        allowed_fields = [
+            "calldata",
+            "contract_state",
+            "eq_outputs",
+            "execution_result",
+            "genvm_result",
+            "node_config",
+            "pending_transactions",
+            "result",
+        ]
+
+        for result in self.sim_results:
+            filtered_result = {
+                key: result.get(key) for key in allowed_fields if key in result
+            }
+            filtered_results.append(filtered_result)
+
+        return filtered_results
+
+    def save_to_directory(self, directory: str, filename: Optional[str] = None) -> str:
+        """
+        Save the detailed stats to a JSON file in the specified directory.
+
+        Raises:
+            OSError: If directory creation or file writing fails.
+        """
+        directory_path = Path(directory)
+        directory_path.mkdir(parents=True, exist_ok=True)
+        if filename is None:
+            safe_method = safe_filename(self.method)
+            safe_timestamp = safe_filename(self.timestamp)
+            filename = f"{safe_method}_{safe_timestamp}.json"
+        if not filename.endswith(".json"):
+            filename += ".json"
+        filepath = directory_path / filename
+        try:
+            with open(filepath, "w") as f:
+                json.dump(self.to_dict(), f, indent=2)
+        except (OSError, TypeError) as e:
+            raise OSError(f"Failed to save stats to {filepath}: {e}") from e
+        return str(filepath)

gltest/contracts/stats_collector.py

@@ -0,0 +1,259 @@
+"""
+Stats collector module for contract method analysis.
+
+This module contains classes and functions to collect and analyze statistics
+from contract method executions, simplifying the analyze_method implementation.
+"""
+
+import time
+import json
+import hashlib
+from datetime import datetime, timezone
+from typing import List, Dict, Any, Optional
+from dataclasses import dataclass
+
+from gltest.clients import get_gl_client
+from gltest.types import CalldataEncodable
+from .method_stats import StateGroup, FailedRun, MethodStatsDetailed, MethodStatsSummary
+from gltest_cli.config.general import get_general_config
+from gltest_cli.config.pytest_context import get_current_test_nodeid
+from .utils import safe_filename
+
+
+@dataclass
+class SimulationConfig:
+    """Configuration for simulation runs."""
+
+    provider: str
+    model: str
+    config: Optional[Dict[str, Any]] = None
+    plugin: Optional[str] = None
+    plugin_config: Optional[Dict[str, Any]] = None
+
+
+@dataclass
+class SimulationResults:
+    """Results from simulation runs."""
+
+    sim_results: List[Dict[str, Any]]
+    failed_runs: List[FailedRun]
+    server_errors: int
+    execution_time: float
+    timestamp: str
+
+
+class StatsCollector:
+    """Collects and analyzes statistics for contract method executions."""
+
+    def __init__(
+        self,
+        contract_address: str,
+        method_name: str,
+        account: Any,
+        args: Optional[List[CalldataEncodable]] = None,
+    ):
+        self.contract_address = contract_address
+        self.method_name = method_name
+        self.account = account
+        self.args = args or []
+        self.client = get_gl_client()
+
+    def run_simulations(
+        self, sim_config: SimulationConfig, runs: int
+    ) -> SimulationResults:
+        """Execute multiple simulation runs and collect results."""
+        start_time = time.time()
+        timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+        sim_results = []
+        failed_runs_list = []
+        server_errors = 0
+
+        for run_idx in range(runs):
+            try:
+                sim_result = self._execute_single_simulation(sim_config)
+                sim_results.append(sim_result)
+
+                if sim_result.get("execution_result") != "SUCCESS":
+                    failed_runs_list.append(
+                        self._create_failed_run(run_idx, sim_result)
+                    )
+            except Exception as e:
+                server_errors += 1
+                failed_runs_list.append(
+                    FailedRun(
+                        run=run_idx,
+                        error=str(e),
+                        error_type="server",
+                        genvm_result=None,
+                    )
+                )
+
+        execution_time = time.time() - start_time
+
+        return SimulationResults(
+            sim_results=sim_results,
+            failed_runs=failed_runs_list,
+            server_errors=server_errors,
+            execution_time=execution_time,
+            timestamp=timestamp,
+        )
+
+    def _execute_single_simulation(
+        self, sim_config: SimulationConfig
+    ) -> Dict[str, Any]:
+        """Execute a single simulation."""
+        config_dict = {
+            "provider": sim_config.provider,
+            "model": sim_config.model,
+        }
+
+        if (
+            sim_config.config is not None
+            and sim_config.plugin is not None
+            and sim_config.plugin_config is not None
+        ):
+            config_dict["config"] = sim_config.config
+            config_dict["plugin"] = sim_config.plugin
+            config_dict["plugin_config"] = sim_config.plugin_config
+
+        return self.client.simulate_write_contract(
+            address=self.contract_address,
+            function_name=self.method_name,
+            account=self.account,
+            args=self.args,
+            sim_config=config_dict,
+        )
+
+    def _create_failed_run(self, run_idx: int, sim_result: Dict[str, Any]) -> FailedRun:
+        """Create a FailedRun object from a failed simulation result."""
+        return FailedRun(
+            run=run_idx,
+            error=sim_result.get("error", "unknown error"),
+            error_type="simulation",
+            genvm_result={
+                "stderr": sim_result.get("genvm_result", {}).get("stderr", ""),
+                "stdout": sim_result.get("genvm_result", {}).get("stdout", ""),
+            },
+        )
+
+    def analyze_results(
+        self,
+        sim_results: SimulationResults,
+        runs: int,
+        sim_config: SimulationConfig,
+    ) -> MethodStatsSummary:
+        """Analyze simulation results and generate statistics."""
+        state_groups = self._analyze_states(sim_results.sim_results)
+
+        executed_runs = runs - sim_results.server_errors
+        successful_runs = sum(
+            1
+            for sim_receipt in sim_results.sim_results
+            if sim_receipt.get("execution_result") == "SUCCESS"
+        )
+
+        most_common_count = max((group.count for group in state_groups), default=0)
+        reliability_score = (
+            (most_common_count / executed_runs) if executed_runs > 0 else 0.0
+        )
+
+        # Save detailed stats
+        detailed_stats = self._create_detailed_stats(
+            sim_results=sim_results,
+            state_groups=state_groups,
+            runs=runs,
+            executed_runs=executed_runs,
+            successful_runs=successful_runs,
+            most_common_count=most_common_count,
+            reliability_score=reliability_score,
+            sim_config=sim_config,
+        )
+        self._save_detailed_stats(detailed_stats)
+
+        # Return summary
+        return MethodStatsSummary(
+            method=self.method_name,
+            args=self.args,
+            total_runs=runs,
+            server_error_runs=sim_results.server_errors,
+            executed_runs=executed_runs,
+            failed_runs=len(sim_results.failed_runs),
+            successful_runs=successful_runs,
+            unique_states=len(state_groups),
+            most_common_state_count=most_common_count,
+            reliability_score=reliability_score,
+            execution_time=sim_results.execution_time,
+            provider=sim_config.provider,
+            model=sim_config.model,
+        )
+
+    def _analyze_states(self, sim_results: List[Dict[str, Any]]) -> List[StateGroup]:
+        """Analyze contract states from simulation results."""
+        state_counts = {}
+        state_to_hash_str = {}
+
+        for sim_receipt in sim_results:
+            contract_state = sim_receipt.get("contract_state", {})
+            state_json = json.dumps(contract_state, sort_keys=True)
+            state_hash = hashlib.sha256(state_json.encode()).hexdigest()
+            state_hash_str = f"0x{state_hash}"
+            state_to_hash_str[state_hash] = state_hash_str
+            state_counts[state_hash] = state_counts.get(state_hash, 0) + 1
+
+        return [
+            StateGroup(count=count, state_hash=state_to_hash_str[state_hash])
+            for state_hash, count in sorted(
+                state_counts.items(), key=lambda x: x[1], reverse=True
+            )
+        ]
+
+    def _create_detailed_stats(
+        self,
+        sim_results: SimulationResults,
+        state_groups: List[StateGroup],
+        runs: int,
+        executed_runs: int,
+        successful_runs: int,
+        most_common_count: int,
+        reliability_score: float,
+        sim_config: SimulationConfig,
+    ) -> MethodStatsDetailed:
+        """Create detailed statistics object."""
+        configuration = {
+            "runs": runs,
+            "provider": sim_config.provider,
+            "model": sim_config.model,
+            "config": sim_config.config,
+            "plugin": sim_config.plugin,
+            "plugin_config": sim_config.plugin_config,
+        }
+
+        return MethodStatsDetailed(
+            method=self.method_name,
+            params=self.args,
+            timestamp=sim_results.timestamp,
+            configuration=configuration,
+            execution_time=sim_results.execution_time,
+            executed_runs=executed_runs,
+            failed_runs=len(sim_results.failed_runs),
+            successful_runs=successful_runs,
+            server_error_runs=sim_results.server_errors,
+            unique_states=len(state_groups),
+            most_common_state_count=most_common_count,
+            reliability_score=reliability_score,
+            state_groups=state_groups,
+            failed_runs_results=sim_results.failed_runs,
+            sim_results=sim_results.sim_results,
+        )
+
+    def _save_detailed_stats(self, detailed_stats: MethodStatsDetailed) -> None:
+        """Save detailed statistics to the configured directory."""
+        general_config = get_general_config()
+        current_nodeid = get_current_test_nodeid()
+        if current_nodeid is None:
+            safe_name = "no_test"
+        else:
+            safe_name = safe_filename(current_nodeid)
+        stats_dir = general_config.get_analysis_dir() / safe_name
+        detailed_stats.save_to_directory(stats_dir)