certora-cli-beta-mirror 7.29.1__py3-none-macosx_10_9_universal2.whl → 7.30.0__py3-none-macosx_10_9_universal2.whl

certora_cli/CertoraProver/certoraBuild.py

@@ -25,8 +25,11 @@ from collections import OrderedDict, defaultdict
 from enum import Enum
 from functools import lru_cache
 from pathlib import Path
-from typing import Any, Dict, List, Tuple, Optional, Set, Iterator, NoReturn
 from Crypto.Hash import keccak
+import tempfile
+
+from typing import Any, Dict, List, Tuple, Optional, Set, Iterator, NoReturn
+
 
 from CertoraProver.certoraBuildCacheManager import CertoraBuildCacheManager, CachedFiles
 from CertoraProver.certoraBuildDataClasses import CONTRACTS, ImmutableReference, ContractExtension, ContractInSDC, SDC, \
@@ -35,6 +38,7 @@ from CertoraProver.certoraCompilerParameters import SolcParameters
 from CertoraProver.certoraSourceFinders import add_source_finders
 from CertoraProver.certoraVerifyGenerator import CertoraVerifyGenerator
 from CertoraProver.certoraContractFuncs import Func, InternalFunc, STATEMUT, SourceBytes
+
 from Shared.certoraUtils import is_relative_to
 
 scripts_dir_path = Path(__file__).parent.parent.resolve()  # containing directory
@@ -54,7 +58,11 @@ from Shared import certoraValidateFuncs as Vf
 from CertoraProver import certoraContextValidator as Cv
 from Shared import certoraUtils as Util
 import CertoraProver.certoraContext as Ctx
-
+from CertoraProver import storageExtension
+from CertoraProver.storageExtension import (
+    NameSpacedStorage,
+    NewStorageInfo,
+)
 
 BUILD_IS_LIBRARY = False
 AUTO_FINDER_PREFIX = "autoFinder_"
@@ -72,6 +80,7 @@ MUTANTS_LOCATION = "mutants_location"
 
 FunctionSig = Tuple[str, List[str], List[str], str]
 
+
 # logger for building the abstract syntax tree
 ast_logger = logging.getLogger("ast")
 # logger for issues calling/shelling out to external functions
@@ -84,12 +93,10 @@ build_logger = logging.getLogger("build_conf")
 # logger of the build cache
 build_cache_logger = logging.getLogger("build_cache")
 
-
 def fatal_error(logger: logging.Logger, msg: str) -> NoReturn:
     logger.fatal(msg)
     raise Exception(msg)
 
-
 class InputConfig:
     def __init__(self, context: CertoraContext) -> None:
         """
@@ -1396,7 +1403,7 @@ class CertoraBuildGenerator:
             if not self.context.strict_solc_optimizer and self.context.solc_via_ir:
                 # The default optimizer steps (taken from libsolidity/interface/OptimiserSettings.h) but with the
                 # full inliner step removed
-                solc0_8_26_to_0_8_29 = ("dhfoDgvulfnTUtnIfxa[r]EscLMVcul[j]Trpeulxa[r]cLCTUca[r]LSsTFOtfDnca[r]" +
+                solc0_8_26_to_0_8_30 = ("dhfoDgvulfnTUtnIfxa[r]EscLMVcul[j]Trpeulxa[r]cLCTUca[r]LSsTFOtfDnca[r]" +
                                         "IulcscCTUtx[scCTUt]TOntnfDIuljmul[jul]VcTOculjmul")
                 solc0_8_13_to_0_8_25 = "dhfoDgvulfnTUtnIf[xa[r]EscLMcCTUtTOntnfDIulLculVcul[j]T" + \
                                        "peulxa[rul]xa[r]cLgvifCTUca[r]LSsTFOtfDnca[r]Iulc]jmul[jul]VcTOculjmul"
@@ -1440,8 +1447,8 @@ class CertoraBuildGenerator:
                     yul_optimizer_steps = solc0_8_12
                 elif minor == 8 and 13 <= patch <= 25:
                     yul_optimizer_steps = solc0_8_13_to_0_8_25
-                elif minor == 8 and 26 <= patch <= 29:
-                    yul_optimizer_steps = solc0_8_26_to_0_8_29
+                elif minor == 8 and 26 <= patch <= 30:
+                    yul_optimizer_steps = solc0_8_26_to_0_8_30
                 assert yul_optimizer_steps is not None, \
                     'Yul Optimizer steps missing for requested Solidity version. Please contact Certora team.'
 
@@ -2539,7 +2546,6 @@ class CertoraBuildGenerator:
             Util.print_progress_message(f"Compiling {orig_file_name}...")
             sdc_pre_finders = self.collect_for_file(build_arg_contract_file, i, compiler_lang, Path(os.getcwd()),
                                                     path_for_compiler_collector_file, original_sdc=None)
-
             # Build sources tree
             build_logger.debug("Building source tree")
             sources_from_pre_finder_SDCs = set()
@@ -2672,8 +2678,154 @@ class CertoraBuildGenerator:
         self.handle_links()
         self.handle_struct_links()
         self.handle_contract_extensions()
+        if self.context.storage_extension_annotation:
+            self.handle_erc7201_annotations()
         self.handle_storage_extension_harnesses()
 
+    def extract_slayout(self, original_file: str, ns_storage: Set[NameSpacedStorage]) -> NewStorageInfo:
+        """
+        Given a file containing a contract with namespaced storage, extract the storage information
+        corresponding to the namespaced types.
+
+        Args:
+            original_file: Path to the Solidity file containing namespaced storage declarations
+            ns_storage: Set of tuples (type_name, namespace) for each namespaced storage
+
+        Returns:
+            NewStorageInfo: A tuple (fields, types) where:
+                - fields: List of new fields added by the namespaced storage
+                - types: Dictionary of types referenced by the fields
+        """
+        file_dir = Path(original_file).parent
+
+        # Generate a unique name for the harness contract based on the contract names in the file
+        harness_name = storageExtension.generate_harness_name(original_file)
+
+        with tempfile.NamedTemporaryFile(mode="w+t", suffix=".sol", dir=file_dir, delete=True) as tmp_file:
+            # Import the original file.
+            # Note we import and don't inline the file's contents since that's how the
+            # original file is accessed also in the actual code, and compiling it
+            # directly can cause issues.
+            tmp_file.write(f"import \"{original_file}\";\n\n")
+
+            # Write the harness contract with dummy fields for each namespaced storage
+            var_to_slot = storageExtension.write_harness_contract(tmp_file, harness_name, ns_storage)
+            tmp_file.flush()
+
+            if self.context.extract_storage_extension_annotation:
+                # If the flag is set, save the storage extension contract
+                build_dir = Util.get_build_dir()
+                shutil.copyfile(
+                    Path(tmp_file.name),
+                    build_dir / f"{Path(original_file).stem}_storage_extension.sol"
+                )
+
+            # Add harness to compiler map
+            storageExtension.add_harness_to_compiler_map(
+                original_file,
+                tmp_file,
+                self.context
+            )
+
+            # normalize the path exactly the way collect_for_file expects it:
+            abs_path = Util.abs_posix_path(tmp_file.name)
+            self.context.file_to_contract[abs_path] = {harness_name}
+
+            try:
+                # Compile & fetch the raw storage_layout
+                compile_idx = storageExtension.get_next_file_index(self.file_to_sdc_name)
+                sdcs = self.collect_for_file(tmp_file.name, compile_idx, CompilerLangSol(), Path.cwd(), Util.abs_posix_path(tmp_file.name), None)
+                if not sdcs:
+                    raise RuntimeError(f"Failed to compile harness contract for {tmp_file}")
+                layout = storageExtension.extract_harness_contract_layout(sdcs, harness_name)
+
+                # Remap each slot according to the ERC-7201 namespace
+                remapped_fields = storageExtension.remapped_fields_from_layout(layout, var_to_slot)
+
+                return (remapped_fields, layout.get('types', {}))
+
+            except Exception as e:
+                build_logger.error(f"Error extracting storage layout for {original_file}: {str(e)}")
+                raise
+            finally:
+                # Delete the key from the context
+                self.context.file_to_contract.pop(abs_path, None)
+
+    def handle_erc7201_annotations(self) -> None:
+        """
+        Look for contracts that use erc-7201 namespaced storage layout
+        (see https://eips.ethereum.org/EIPS/eip-7201).
+
+        Find contracts A s.t. A contain a type declaration with such an annotation, e.g.
+          /** @custom:storage-location erc-7201:some.name.space */
+          struct T { ... }
+
+        Then, for any contract C that has A as a base contract, _extend_ C's storage layout
+        information such that it contains the information for a `T` at the slot
+        erc-7201(some.name.space) as defined in the EIP.
+        """
+        # Find all erc7201-like contracts, generate+compile a harness & extract layout information
+        # maps (path,contract) -> new storage info added by (path,contract)
+        slayouts: Dict[Tuple[str, str], NewStorageInfo] = {}
+
+        # Scan all of the contracts (including dependencies of targets) for namespaced storage
+        # layout information
+        for target_file in self.context.file_paths:
+            if target_file not in self.asts:
+                # No AST for this file, so we can't do anything
+                continue
+            for (imported_file, imported_file_ast) in self.asts[target_file].items():
+                for def_node in imported_file_ast.values():
+                    if def_node.get("nodeType") != "ContractDefinition":
+                        continue
+
+                    # Construct a key for the contract definition node
+                    contract_name = def_node.get("name")
+                    key = (imported_file, contract_name)
+                    if key in slayouts:
+                        # We already have this contract's storage layout information
+                        continue
+
+                    # Collect any @custom:storage-location annotations
+                    ns_storage = storageExtension.get_namespace_storage_from_ast(def_node)
+
+                    if not ns_storage:
+                        # No namespaced storage found in this contract
+                        continue
+
+                    # Now that we have all the storage layout information, extract it once
+                    slayouts[key] = self.extract_slayout(imported_file, ns_storage)
+
+        if self.context.test == str(Util.TestValue.STORAGE_EXTENSION_LAYOUT):
+            raise Util.TestResultsReady(slayouts)
+
+        if not slayouts:
+            # No contracts with namespaced storage found
+            return
+
+        # Finally, extend each target contract with the storage layout info from
+        # all of its base contracts
+        for target in self.get_primary_contracts_from_sdcs():
+            if target.name not in self.context.contract_to_file:
+                # This is a contract that was not compiled, so we don't have a file for it
+                continue
+            target_file = self.context.contract_to_file[target.name]
+            base_contracts = self.retrieve_base_contracts_list(
+                target_file,
+                Util.abs_posix_path(target_file),
+                target.name
+            )
+            extensions: Set[str] = set()
+            harnesses: Dict[str, NewStorageInfo] = {}
+            for base in base_contracts:
+                layout = slayouts.get((base[0], base[1]))
+                if layout is not None:
+                    extensions.add(base[1])
+                    harnesses[base[1]] = layout
+                else:
+                    build_logger.warning(f"Could not find storage layout for {base[1]} in {base[0]}")
+            storageExtension.apply_extensions(target, extensions, harnesses)
+
     def handle_storage_extension_harnesses(self) -> None:
         def new_field_of_node(ext_instance: Any, node: Dict[str, Any]) -> Optional[Dict[str, Any]]:
             """
@@ -2760,52 +2912,6 @@ class CertoraBuildGenerator:
                     new_fields.append(new_field)
 
             return (ext_instance, extension_sdc_name, new_fields)
-
-        def apply_extensions(target_contract: Any, extensions: Set[str], to_add: Dict[str, Tuple[List[Dict[str, Any]], Dict[str, Any]]]) -> None:
-            """
-            Apply the fields from each extension to the target contract,
-            """
-            if target_contract.storage_layout.get("storage") is None:
-                target_contract.storage_layout["storage"] = []
-            if target_contract.storage_layout.get("types") is None:
-                target_contract.storage_layout["types"] = {}
-            target_slots = {storage["slot"] for storage in target_contract.storage_layout["storage"]}
-            target_vars = {storage["label"] for storage in target_contract.storage_layout["storage"]}
-            # Keep track of slots we've added, and error if we
-            # find two extensions extending the same slot
-            added_slots: Dict[str, str] = {}
-            added_vars: Dict[str, str] = {}
-            for ext in extensions:
-                (new_fields, new_types) = to_add[ext]
-
-                for f in new_fields:
-                    # See if any of the new fields is a slot or variable name we've already added
-                    slot = f["slot"]
-                    var = f["label"]
-                    if slot in added_slots:
-                        seen = added_slots[slot]
-                        raise Util.CertoraUserInputError(f"Slot {slot} added to {target_contract.name} by {ext} was already added by {seen}")
-
-                    if var in added_vars:
-                        seen = added_vars[var]
-                        raise Util.CertoraUserInputError(f"Var '{var}' added to {target_contract.name} by {ext} was already added by {seen}")
-
-                    if slot in target_slots:
-                        raise Util.CertoraUserInputError(f"Slot {slot} added to {target_contract.name} by {ext} is already mapped by {target_contract.name}")
-
-                    if var in target_vars:
-                        raise Util.CertoraUserInputError(f"Var '{var}' added to {target_contract.name} by {ext} is already declared by {target_contract.name}")
-
-                    added_slots[slot] = ext
-                    added_vars[var] = ext
-
-                target_contract.storage_layout["storage"].extend(new_fields)
-
-                for (new_id, new_ty) in new_types.items():
-                    if new_id in target_contract.storage_layout["types"]:
-                        continue
-                    target_contract.storage_layout["types"][new_id] = new_ty
-
         extension_contracts: Set[str] = set()
         storage_extensions: Dict[str, Set[str]] = defaultdict(set)
         storage_ext = self.context.storage_extension_harnesses
@@ -2834,7 +2940,7 @@ class CertoraBuildGenerator:
             sdc = self.SDCs[target_sdc]
             target_contract = sdc.find_contract(target)
             assert target_contract is not None, f"could not find contract for {target}"
-            apply_extensions(target_contract, extensions, extension_to_fields_and_types)
+            storageExtension.apply_extensions(target_contract, extensions, extension_to_fields_and_types)
 
     def finders_compilation_round(self,
                                   build_arg_contract_file: str,
@@ -3436,8 +3542,9 @@ class CertoraBuildGenerator:
                 raise Util.CertoraUserInputError(f"collect_sources: {path_to_file} does not exist cwd - {Path.cwd()}."
                                                  f"abs - {os.path.normpath(Path.cwd() / path_to_file)}")
 
-        sources = set()
-        sources |= sources_from_SDCs
+        sources = set(sources_from_SDCs)
+        if context.files:
+            sources.update(Path(p) for p in context.files)  # all files in "files" attribute are uploaded
         sources |= certora_verify_generator.get_spec_files()
         if Util.PACKAGE_FILE.exists():
             add_to_sources(Util.PACKAGE_FILE)
@@ -3546,7 +3653,8 @@ def build_source_tree(sources: Set[Path], context: CertoraContext, overwrite: bo
         build_logger.debug("Couldn't copy repro conf to certora sources.", exc_info=e)
         raise
 
-def build_from_scratch(certora_build_generator: CertoraBuildGenerator,
+def build_from_scratch(context: CertoraContext,
+                       certora_build_generator: CertoraBuildGenerator,
                        certora_verify_generator: CertoraVerifyGenerator,
                        build_cache_enabled: bool) -> CachedFiles:
     """
@@ -3572,9 +3680,15 @@ def build_from_scratch(certora_build_generator: CertoraBuildGenerator,
     may_store_in_build_cache = True
     absolute_sources_dir = Util.get_certora_sources_dir().absolute()
     for sdc in certora_build_generator.SDCs.values():
+
+        # add to cache also source files that were found in the SDCs (e.g., storage extensions)
+        paths_set = sdc.all_contract_files
+        for p in context.files:
+            paths_set.add(Path(p).absolute())
+
         # the contract files in SDCs are relative to .certora_sources. Which isn't good for us here.
         # Need to be relative to original paths
-        for f in sdc.all_contract_files:
+        for f in paths_set:
             if is_relative_to(f, absolute_sources_dir):
                 rel_f = f.relative_to(absolute_sources_dir)
             else:
@@ -3623,7 +3737,7 @@ def build_from_cache_or_scratch(context: CertoraContext,
     cached_files: Optional[CachedFiles] = None
 
     if not context.build_cache:
-        cached_files = build_from_scratch(certora_build_generator,
+        cached_files = build_from_scratch(context, certora_build_generator,
                                           certora_verify_generator,
                                           False)
         return cache_hit, False, cached_files
@@ -3634,7 +3748,7 @@ def build_from_cache_or_scratch(context: CertoraContext,
         build_cache_disabling_options = certora_build_cache_manager.cache_disabling_options(context)
         build_logger.warning("Requested to enable the build cache, but the build cache is not applicable "
                              f"to this run because of the given options: {build_cache_disabling_options}")
-        cached_files = build_from_scratch(certora_build_generator,
+        cached_files = build_from_scratch(context, certora_build_generator,
                                           certora_verify_generator,
                                           False)
         return cache_hit, False, cached_files
@@ -3660,7 +3774,7 @@ def build_from_cache_or_scratch(context: CertoraContext,
         cache_hit = True
     else:
         # rebuild
-        cached_files = build_from_scratch(certora_build_generator,
+        cached_files = build_from_scratch(context, certora_build_generator,
                                           certora_verify_generator,
                                           True)
 

certora_cli/CertoraProver/certoraContextAttributes.py

@@ -998,6 +998,24 @@ class EvmAttributes(AttrUtil.Attributes):
         disables_build_cache=False
     )
 
+    STORAGE_EXTENSION_ANNOTATION = AttrUtil.AttributeDefinition(
+        arg_type=AttrUtil.AttrArgType.BOOLEAN,
+        argparse_args={
+            'action': AttrUtil.STORE_TRUE
+        },
+        affects_build_cache_key=True,
+        disables_build_cache=False
+    )
+
+    EXTRACT_STORAGE_EXTENSION_ANNOTATION = AttrUtil.AttributeDefinition(
+        arg_type=AttrUtil.AttrArgType.BOOLEAN,
+        argparse_args={
+            'action': AttrUtil.STORE_TRUE
+        },
+        affects_build_cache_key=False,
+        disables_build_cache=False
+    )
+
     OPTIMISTIC_HASHING = AttrUtil.AttributeDefinition(
         arg_type=AttrUtil.AttrArgType.BOOLEAN,
         help_msg="Bound the length of data (with potentially unbounded length) to the value given in "
@@ -1128,6 +1146,21 @@ class EvmAttributes(AttrUtil.Attributes):
         disables_build_cache=False,
     )
 
+    MAX_CONCURRENT_RULES = AttrUtil.AttributeDefinition(
+        attr_validation_func=Vf.validate_non_negative_integer,
+        argparse_args={
+            'action': AttrUtil.UniqueStore
+        },
+        help_msg="Set the maximum number of parallel rule evaluations. "
+                 "Lower values (e.g., 1, 2, or 4) may reduce memory usage in large runs. "
+                 "This can sometimes help to mitigate out of memory problems.",
+        default_desc="Number of available CPU cores.",
+        temporary_jar_invocation_allowed=True,
+        jar_flag="-maxConcurrentRules",
+        affects_build_cache_key=False,
+        disables_build_cache=False,
+    )
+
     @classmethod
     def hide_attributes(cls) -> List[str]:
         # do not show these attributes in the help message

certora_cli/CertoraProver/erc7201.py

@@ -0,0 +1,45 @@
+#     The Certora Prover
+#     Copyright (C) 2025  Certora Ltd.
+#
+#     This program is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, version 3 of the License.
+#
+#     This program is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from Crypto.Hash import keccak
+
+
+def calculate_keccak_hash(x: bytes) -> int:
+    """
+    Calculates the Keccak-256 hash of the input bytes and returns it as an integer.
+
+    Args:
+        x (bytes): The input bytes to be hashed.
+    Returns:
+        int: The Keccak-256 hash value as an integer.
+    """
+    k = keccak.new(digest_bits=256)
+    k.update(x)
+    return int(k.hexdigest(), base=16)
+
+def erc7201(x: bytes) -> int:
+    """
+    Hashes the input bytes using the Keccak-256 algorithm and returns
+    the result as an integer. The input is first hashed, then decremented
+    by 1, and the resulting hash is used to compute the final hash.
+    The final hash is masked to 256 bits and the last byte is cleared
+    (set to zero).
+
+    Args:
+        x (bytes): The input bytes to be hashed.
+    Returns:
+        int: The final hash value as an integer.
+    """
+    return calculate_keccak_hash((calculate_keccak_hash(x) - 1).to_bytes(32, byteorder='big')) & (~0xff)