genlayer-test 0.4.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

gltest/contracts/contract_functions.py

@@ -0,0 +1,61 @@
+from dataclasses import dataclass
+from typing import Callable, Optional, Dict, Any
+from gltest.types import TransactionStatus
+
+
+@dataclass
+class ContractFunction:
+    method_name: str
+    read_only: bool
+    call_method: Optional[Callable] = None
+    analyze_method: Optional[Callable] = None
+    transact_method: Optional[Callable] = None
+
+    def call(self):
+        if not self.read_only:
+            raise ValueError("call() not implemented for non-readonly method")
+        return self.call_method()
+
+    def transact(
+        self,
+        value: int = 0,
+        consensus_max_rotations: Optional[int] = None,
+        leader_only: bool = False,
+        wait_transaction_status: TransactionStatus = TransactionStatus.FINALIZED,
+        wait_interval: Optional[int] = None,
+        wait_retries: Optional[int] = None,
+        wait_triggered_transactions: bool = True,
+        wait_triggered_transactions_status: TransactionStatus = TransactionStatus.FINALIZED,
+    ):
+        if self.read_only:
+            raise ValueError("Cannot transact read-only method")
+        return self.transact_method(
+            value=value,
+            consensus_max_rotations=consensus_max_rotations,
+            leader_only=leader_only,
+            wait_transaction_status=wait_transaction_status,
+            wait_interval=wait_interval,
+            wait_retries=wait_retries,
+            wait_triggered_transactions=wait_triggered_transactions,
+            wait_triggered_transactions_status=wait_triggered_transactions_status,
+        )
+
+    def analyze(
+        self,
+        provider: str,
+        model: str,
+        config: Optional[Dict[str, Any]] = None,
+        plugin: Optional[str] = None,
+        plugin_config: Optional[Dict[str, Any]] = None,
+        runs: int = 100,
+    ):
+        if self.read_only:
+            raise ValueError("Cannot analyze read-only method")
+        return self.analyze_method(
+            provider=provider,
+            model=model,
+            config=config,
+            plugin=plugin,
+            plugin_config=plugin_config,
+            runs=runs,
+        )

gltest/contracts/method_stats.py

@@ -0,0 +1,163 @@
+from dataclasses import dataclass
+from typing import List, Any, Optional, Dict
+import json
+from pathlib import Path
+from .utils import safe_filename
+
+
+@dataclass
+class MethodStatsSummary:
+    """Statistical analysis results for a contract method execution."""
+
+    method: str
+    args: List[Any]
+    total_runs: int
+    executed_runs: int
+    server_error_runs: int
+    failed_runs: int
+    successful_runs: int
+    unique_states: int
+    most_common_state_count: int
+    reliability_score: float
+    execution_time: float
+    provider: str
+    model: str
+
+    @property
+    def success_rate(self) -> float:
+        """Calculate success rate as percentage."""
+        if self.total_runs == 0:
+            return 0.0
+        return (self.successful_runs / self.total_runs) * 100
+
+    def __str__(self) -> str:
+        """Format the statistical analysis results."""
+        return f"""Method analysis summary
+---------------------------
+Method: {self.method}
+Args: {self.args}
+Provider: {self.provider}
+Model: {self.model}
+Total runs: {self.total_runs}
+Server error runs: {self.server_error_runs}
+Method executed runs: {self.executed_runs}
+Method successful runs: {self.successful_runs}
+Method failed runs: {self.failed_runs}
+Unique states: {self.unique_states}
+Reliability score: {self.reliability_score:.2f}% ({self.most_common_state_count}/{self.executed_runs} consistent)
+Execution time: {self.execution_time:.1f}s"""
+
+
+@dataclass
+class StateGroup:
+    """Represents a group of runs with the same contract state."""
+
+    count: int
+    state_hash: str
+
+
+@dataclass
+class FailedRun:
+    """Represents a failed run with error details."""
+
+    run: int
+    error: str
+    error_type: str  # "server" or "simulation"
+    genvm_result: Optional[Dict[str, str]]
+
+
+@dataclass
+class MethodStatsDetailed:
+    """Detailed statistical analysis results for a contract method execution."""
+
+    method: str
+    params: List[Any]
+    timestamp: str
+    configuration: Dict[str, Any]
+    execution_time: float
+    executed_runs: int
+    failed_runs: int
+    successful_runs: int
+    server_error_runs: int
+    most_common_state_count: int
+    reliability_score: float
+    sim_results: List[Any]
+    unique_states: int
+    state_groups: List[StateGroup]
+    failed_runs_results: List[FailedRun]
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary format."""
+        return {
+            "method": self.method,
+            "params": self.params,
+            "timestamp": self.timestamp,
+            "configuration": self.configuration,
+            "execution_time": self.execution_time,
+            "method_executed_runs": self.executed_runs,
+            "method_failed_runs": self.failed_runs,
+            "method_successful_runs": self.successful_runs,
+            "server_error_runs": self.server_error_runs,
+            "most_common_state_count": self.most_common_state_count,
+            "reliability_score": self.reliability_score,
+            "unique_states": self.unique_states,
+            "state_groups": [
+                {"count": sg.count, "state_hash": sg.state_hash}
+                for sg in self.state_groups
+            ],
+            "failed_runs": [
+                {
+                    "run": fr.run,
+                    "error": fr.error,
+                    "error_type": fr.error_type,
+                    "genvm_result": fr.genvm_result,
+                }
+                for fr in self.failed_runs_results
+            ],
+            "simulation_results": self.filter_sim_results(),
+        }
+
+    def filter_sim_results(self) -> List[Any]:
+        """Filter the simulation results to only include specific fields."""
+        filtered_results = []
+        allowed_fields = [
+            "calldata",
+            "contract_state",
+            "eq_outputs",
+            "execution_result",
+            "genvm_result",
+            "node_config",
+            "pending_transactions",
+            "result",
+        ]
+
+        for result in self.sim_results:
+            filtered_result = {
+                key: result.get(key) for key in allowed_fields if key in result
+            }
+            filtered_results.append(filtered_result)
+
+        return filtered_results
+
+    def save_to_directory(self, directory: str, filename: Optional[str] = None) -> str:
+        """
+        Save the detailed stats to a JSON file in the specified directory.
+
+        Raises:
+            OSError: If directory creation or file writing fails.
+        """
+        directory_path = Path(directory)
+        directory_path.mkdir(parents=True, exist_ok=True)
+        if filename is None:
+            safe_method = safe_filename(self.method)
+            safe_timestamp = safe_filename(self.timestamp)
+            filename = f"{safe_method}_{safe_timestamp}.json"
+        if not filename.endswith(".json"):
+            filename += ".json"
+        filepath = directory_path / filename
+        try:
+            with open(filepath, "w") as f:
+                json.dump(self.to_dict(), f, indent=2)
+        except (OSError, TypeError) as e:
+            raise OSError(f"Failed to save stats to {filepath}: {e}") from e
+        return str(filepath)

gltest/contracts/stats_collector.py

@@ -0,0 +1,259 @@
+"""
+Stats collector module for contract method analysis.
+
+This module contains classes and functions to collect and analyze statistics
+from contract method executions, simplifying the analyze_method implementation.
+"""
+
+import time
+import json
+import hashlib
+from datetime import datetime, timezone
+from typing import List, Dict, Any, Optional
+from dataclasses import dataclass
+
+from gltest.clients import get_gl_client
+from gltest.types import CalldataEncodable
+from .method_stats import StateGroup, FailedRun, MethodStatsDetailed, MethodStatsSummary
+from gltest_cli.config.general import get_general_config
+from gltest_cli.config.pytest_context import get_current_test_nodeid
+from .utils import safe_filename
+
+
+@dataclass
+class SimulationConfig:
+    """Configuration for simulation runs."""
+
+    provider: str
+    model: str
+    config: Optional[Dict[str, Any]] = None
+    plugin: Optional[str] = None
+    plugin_config: Optional[Dict[str, Any]] = None
+
+
+@dataclass
+class SimulationResults:
+    """Results from simulation runs."""
+
+    sim_results: List[Dict[str, Any]]
+    failed_runs: List[FailedRun]
+    server_errors: int
+    execution_time: float
+    timestamp: str
+
+
+class StatsCollector:
+    """Collects and analyzes statistics for contract method executions."""
+
+    def __init__(
+        self,
+        contract_address: str,
+        method_name: str,
+        account: Any,
+        args: Optional[List[CalldataEncodable]] = None,
+    ):
+        self.contract_address = contract_address
+        self.method_name = method_name
+        self.account = account
+        self.args = args or []
+        self.client = get_gl_client()
+
+    def run_simulations(
+        self, sim_config: SimulationConfig, runs: int
+    ) -> SimulationResults:
+        """Execute multiple simulation runs and collect results."""
+        start_time = time.time()
+        timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+        sim_results = []
+        failed_runs_list = []
+        server_errors = 0
+
+        for run_idx in range(runs):
+            try:
+                sim_result = self._execute_single_simulation(sim_config)
+                sim_results.append(sim_result)
+
+                if sim_result.get("execution_result") != "SUCCESS":
+                    failed_runs_list.append(
+                        self._create_failed_run(run_idx, sim_result)
+                    )
+            except Exception as e:
+                server_errors += 1
+                failed_runs_list.append(
+                    FailedRun(
+                        run=run_idx,
+                        error=str(e),
+                        error_type="server",
+                        genvm_result=None,
+                    )
+                )
+
+        execution_time = time.time() - start_time
+
+        return SimulationResults(
+            sim_results=sim_results,
+            failed_runs=failed_runs_list,
+            server_errors=server_errors,
+            execution_time=execution_time,
+            timestamp=timestamp,
+        )
+
+    def _execute_single_simulation(
+        self, sim_config: SimulationConfig
+    ) -> Dict[str, Any]:
+        """Execute a single simulation."""
+        config_dict = {
+            "provider": sim_config.provider,
+            "model": sim_config.model,
+        }
+
+        if (
+            sim_config.config is not None
+            and sim_config.plugin is not None
+            and sim_config.plugin_config is not None
+        ):
+            config_dict["config"] = sim_config.config
+            config_dict["plugin"] = sim_config.plugin
+            config_dict["plugin_config"] = sim_config.plugin_config
+
+        return self.client.simulate_write_contract(
+            address=self.contract_address,
+            function_name=self.method_name,
+            account=self.account,
+            args=self.args,
+            sim_config=config_dict,
+        )
+
+    def _create_failed_run(self, run_idx: int, sim_result: Dict[str, Any]) -> FailedRun:
+        """Create a FailedRun object from a failed simulation result."""
+        return FailedRun(
+            run=run_idx,
+            error=sim_result.get("error", "unknown error"),
+            error_type="simulation",
+            genvm_result={
+                "stderr": sim_result.get("genvm_result", {}).get("stderr", ""),
+                "stdout": sim_result.get("genvm_result", {}).get("stdout", ""),
+            },
+        )
+
+    def analyze_results(
+        self,
+        sim_results: SimulationResults,
+        runs: int,
+        sim_config: SimulationConfig,
+    ) -> MethodStatsSummary:
+        """Analyze simulation results and generate statistics."""
+        state_groups = self._analyze_states(sim_results.sim_results)
+
+        executed_runs = runs - sim_results.server_errors
+        successful_runs = sum(
+            1
+            for sim_receipt in sim_results.sim_results
+            if sim_receipt.get("execution_result") == "SUCCESS"
+        )
+
+        most_common_count = max((group.count for group in state_groups), default=0)
+        reliability_score = (
+            (most_common_count / executed_runs) if executed_runs > 0 else 0.0
+        )
+
+        # Save detailed stats
+        detailed_stats = self._create_detailed_stats(
+            sim_results=sim_results,
+            state_groups=state_groups,
+            runs=runs,
+            executed_runs=executed_runs,
+            successful_runs=successful_runs,
+            most_common_count=most_common_count,
+            reliability_score=reliability_score,
+            sim_config=sim_config,
+        )
+        self._save_detailed_stats(detailed_stats)
+
+        # Return summary
+        return MethodStatsSummary(
+            method=self.method_name,
+            args=self.args,
+            total_runs=runs,
+            server_error_runs=sim_results.server_errors,
+            executed_runs=executed_runs,
+            failed_runs=len(sim_results.failed_runs),
+            successful_runs=successful_runs,
+            unique_states=len(state_groups),
+            most_common_state_count=most_common_count,
+            reliability_score=reliability_score,
+            execution_time=sim_results.execution_time,
+            provider=sim_config.provider,
+            model=sim_config.model,
+        )
+
+    def _analyze_states(self, sim_results: List[Dict[str, Any]]) -> List[StateGroup]:
+        """Analyze contract states from simulation results."""
+        state_counts = {}
+        state_to_hash_str = {}
+
+        for sim_receipt in sim_results:
+            contract_state = sim_receipt.get("contract_state", {})
+            state_json = json.dumps(contract_state, sort_keys=True)
+            state_hash = hashlib.sha256(state_json.encode()).hexdigest()
+            state_hash_str = f"0x{state_hash}"
+            state_to_hash_str[state_hash] = state_hash_str
+            state_counts[state_hash] = state_counts.get(state_hash, 0) + 1
+
+        return [
+            StateGroup(count=count, state_hash=state_to_hash_str[state_hash])
+            for state_hash, count in sorted(
+                state_counts.items(), key=lambda x: x[1], reverse=True
+            )
+        ]
+
+    def _create_detailed_stats(
+        self,
+        sim_results: SimulationResults,
+        state_groups: List[StateGroup],
+        runs: int,
+        executed_runs: int,
+        successful_runs: int,
+        most_common_count: int,
+        reliability_score: float,
+        sim_config: SimulationConfig,
+    ) -> MethodStatsDetailed:
+        """Create detailed statistics object."""
+        configuration = {
+            "runs": runs,
+            "provider": sim_config.provider,
+            "model": sim_config.model,
+            "config": sim_config.config,
+            "plugin": sim_config.plugin,
+            "plugin_config": sim_config.plugin_config,
+        }
+
+        return MethodStatsDetailed(
+            method=self.method_name,
+            params=self.args,
+            timestamp=sim_results.timestamp,
+            configuration=configuration,
+            execution_time=sim_results.execution_time,
+            executed_runs=executed_runs,
+            failed_runs=len(sim_results.failed_runs),
+            successful_runs=successful_runs,
+            server_error_runs=sim_results.server_errors,
+            unique_states=len(state_groups),
+            most_common_state_count=most_common_count,
+            reliability_score=reliability_score,
+            state_groups=state_groups,
+            failed_runs_results=sim_results.failed_runs,
+            sim_results=sim_results.sim_results,
+        )
+
+    def _save_detailed_stats(self, detailed_stats: MethodStatsDetailed) -> None:
+        """Save detailed statistics to the configured directory."""
+        general_config = get_general_config()
+        current_nodeid = get_current_test_nodeid()
+        if current_nodeid is None:
+            safe_name = "no_test"
+        else:
+            safe_name = safe_filename(current_nodeid)
+        stats_dir = general_config.get_analysis_dir() / safe_name
+        detailed_stats.save_to_directory(stats_dir)

gltest/contracts/utils.py

@@ -0,0 +1,12 @@
+def safe_filename(filename: str) -> str:
+    """
+    Replace problematic characters in filename.
+    """
+    return (
+        filename.replace("/", "_")
+        .replace("\\", "_")
+        .replace(":", "_")
+        .replace("-", "_")
+        .replace(" ", "_")
+        .replace(".py", "")
+    )

gltest/fixtures.py

@@ -4,12 +4,8 @@ These fixtures can be imported and used in test files.
 """
 
 import pytest
-from gltest.glchain import (
-    get_gl_client,
-    get_accounts,
-    get_default_account,
-    get_gl_provider,
-)
+from gltest.clients import get_gl_client, get_gl_provider
+from gltest.accounts import get_accounts, get_default_account
 from gltest_cli.config.general import get_general_config
 
 

gltest/helpers/take_snapshot.py

@@ -1,4 +1,4 @@
-from gltest.glchain import get_gl_provider
+from gltest.clients import get_gl_provider
 from dataclasses import dataclass
 from typing import Callable
 from gltest.exceptions import HelperError, InvalidSnapshotError

gltest_cli/config/constants.py

@@ -7,4 +7,5 @@ DEFAULT_NETWORK = "localnet"
 DEFAULT_RPC_URL = SIMULATOR_JSON_RPC_URL
 DEFAULT_ENVIRONMENT = ".env"
 DEFAULT_CONTRACTS_DIR = Path("contracts")
+DEFAULT_ARTIFACTS_DIR = Path("artifacts")
 DEFAULT_NETWORK_ID = 61999

gltest_cli/config/plugin.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+import shutil
 from gltest_cli.logging import logger
 from gltest_cli.config.user import (
     user_config_exists,
@@ -9,6 +10,7 @@ from gltest_cli.config.general import (
     get_general_config,
 )
 from gltest_cli.config.types import PluginConfig
+from gltest_cli.config.pytest_context import _pytest_context
 
 
 def pytest_addoption(parser):
@@ -20,6 +22,13 @@ def pytest_addoption(parser):
         help="Path to directory containing contract files",
     )
 
+    group.addoption(
+        "--artifacts-dir",
+        action="store",
+        default=None,
+        help="Path to directory for storing contract artifacts",
+    )
+
     group.addoption(
         "--default-wait-interval",
         action="store",
@@ -76,6 +85,7 @@ def pytest_configure(config):
 
     # Handle plugin config from command line
     contracts_dir = config.getoption("--contracts-dir")
+    artifacts_dir = config.getoption("--artifacts-dir")
     default_wait_interval = config.getoption("--default-wait-interval")
     default_wait_retries = config.getoption("--default-wait-retries")
     rpc_url = config.getoption("--rpc-url")
@@ -86,6 +96,9 @@ def pytest_configure(config):
     plugin_config.contracts_dir = (
         Path(contracts_dir) if contracts_dir is not None else None
     )
+    plugin_config.artifacts_dir = (
+        Path(artifacts_dir) if artifacts_dir is not None else None
+    )
     plugin_config.default_wait_interval = int(default_wait_interval)
     plugin_config.default_wait_retries = int(default_wait_retries)
     plugin_config.rpc_url = rpc_url
@@ -97,6 +110,18 @@ def pytest_configure(config):
 
 def pytest_sessionstart(session):
     general_config = get_general_config()
+
+    artifacts_dir = general_config.get_artifacts_dir()
+    if artifacts_dir and artifacts_dir.exists():
+        logger.info(f"Clearing artifacts directory: {artifacts_dir}")
+        try:
+            shutil.rmtree(artifacts_dir)
+            artifacts_dir.mkdir(parents=True, exist_ok=True)
+        except Exception as e:
+            logger.warning(f"Failed to clear artifacts directory: {e}")
+    elif artifacts_dir:
+        artifacts_dir.mkdir(parents=True, exist_ok=True)
+
     logger.info("Using the following configuration:")
     logger.info(f"  RPC URL: {general_config.get_rpc_url()}")
     logger.info(f"  Selected Network: {general_config.get_network_name()}")
@@ -104,6 +129,7 @@ def pytest_sessionstart(session):
         f"  Available networks: {list(general_config.user_config.networks.keys())}"
     )
     logger.info(f"  Contracts directory: {general_config.get_contracts_dir()}")
+    logger.info(f"  Artifacts directory: {general_config.get_artifacts_dir()}")
     logger.info(f"  Environment: {general_config.user_config.environment}")
     logger.info(
         f"  Default wait interval: {general_config.get_default_wait_interval()} ms"
@@ -112,4 +138,15 @@ def pytest_sessionstart(session):
     logger.info(f"  Test with mocks: {general_config.get_test_with_mocks()}")
 
 
+def pytest_runtest_setup(item):
+    _pytest_context.current_item = item
+
+
+def pytest_runtest_teardown(item):
+    try:
+        del _pytest_context.current_item
+    except AttributeError:
+        pass
+
+
 pytest_plugins = ["gltest.fixtures"]

gltest_cli/config/pytest_context.py

@@ -0,0 +1,9 @@
+from typing import Optional
+import threading
+
+_pytest_context = threading.local()
+
+
+def get_current_test_nodeid() -> Optional[str]:
+    item = getattr(_pytest_context, "current_item", None)
+    return item.nodeid if item is not None else None

gltest_cli/config/types.py

@@ -15,6 +15,7 @@ class NetworkConfig(str, Enum):
 @dataclass
 class PluginConfig:
     contracts_dir: Optional[Path] = None
+    artifacts_dir: Optional[Path] = None
     rpc_url: Optional[str] = None
     default_wait_interval: Optional[int] = None
     default_wait_retries: Optional[int] = None
@@ -46,10 +47,13 @@ class NetworkConfigData:
 @dataclass
 class PathConfig:
     contracts: Optional[Path] = None
+    artifacts: Optional[Path] = None
 
     def __post_init__(self):
         if self.contracts is not None and not isinstance(self.contracts, (str, Path)):
             raise ValueError("contracts must be a string or Path")
+        if self.artifacts is not None and not isinstance(self.artifacts, (str, Path)):
+            raise ValueError("artifacts must be a string or Path")
 
 
 @dataclass
@@ -93,6 +97,18 @@ class GeneralConfig:
     def set_contracts_dir(self, contracts_dir: Path):
         self.plugin_config.contracts_dir = contracts_dir
 
+    def get_artifacts_dir(self) -> Path:
+        if self.plugin_config.artifacts_dir is not None:
+            return self.plugin_config.artifacts_dir
+        return self.user_config.paths.artifacts
+
+    def set_artifacts_dir(self, artifacts_dir: Path):
+        self.plugin_config.artifacts_dir = artifacts_dir
+
+    def get_analysis_dir(self) -> Path:
+        artifacts_dir = self.get_artifacts_dir()
+        return artifacts_dir / "analysis"
+
     def get_rpc_url(self) -> str:
         if self.plugin_config.rpc_url is not None:
             return self.plugin_config.rpc_url