itp-interface 1.0.0__py3-none-any.whl

itp_interface/lean_server/lean3_search_tool.py

@@ -0,0 +1,358 @@
+#!/usr/bin/env python3
+
+import sys
+
+root_dir = f"{__file__.split('itp_interface')[0]}"
+if root_dir not in sys.path:
+    sys.path.append(root_dir)
+import os
+import typing
+import re
+from itp_interface.lean_server.lean_utils import Lean3Utils
+
+class Constants(object):
+    mathlib_useful_imports = [
+        'algebra.algebra.basic',
+        'algebra.order.floor',
+        'algebra.associated',
+        'algebra.big_operators.basic',
+        'algebra.big_operators.enat',
+        'algebra.big_operators.order',
+        'algebra.big_operators.pi',
+        'algebra.geom_sum',
+        'algebra.group.pi',
+        'algebra.group.commute',
+        'algebra.group_power.basic',
+        'algebra.group_power.identities',
+        'algebra.order.floor',
+        'algebra.quadratic_discriminant',
+        'algebra.ring.basic',
+        'analysis.asymptotics.asymptotic_equivalent',
+        'analysis.mean_inequalities',
+        'analysis.normed_space.basic',
+        'analysis.inner_product_space.basic',
+        'analysis.inner_product_space.euclidean_dist',
+        'analysis.normed_space.pi_Lp',
+        'analysis.special_functions.exp',
+        'analysis.special_functions.exp_deriv',
+        'analysis.special_functions.log',
+        'analysis.special_functions.logb',
+        'analysis.special_functions.log_deriv',
+        'analysis.special_functions.pow',
+        'analysis.special_functions.sqrt',
+        'analysis.special_functions.trigonometric.basic',
+        'analysis.special_functions.trigonometric.complex',
+        'combinatorics.simple_graph.basic',
+        'data.complex.basic',
+        'data.complex.exponential',
+        'data.finset.basic',
+        'data.fintype.card',
+        'data.int.basic',
+        'data.int.gcd',
+        'data.int.modeq',
+        'data.int.parity',
+        'data.list.intervals',
+        'data.list.palindrome',
+        'data.multiset.basic',
+        'data.nat.basic',
+        'data.nat.choose.basic',
+        'data.nat.digits',
+        'data.nat.factorial.basic',
+        'data.nat.fib',
+        'data.nat.modeq',
+        'data.nat.multiplicity',
+        'data.nat.parity',
+        'data.nat.prime',
+        'data.pnat.basic',
+        'data.pnat.prime',
+        'data.polynomial.algebra_map',
+        'data.polynomial.field_division',
+        'data.polynomial.derivative',
+        'data.polynomial.identities',
+        'data.polynomial.integral_normalization',
+        'data.polynomial.basic',
+        'data.polynomial.eval',
+        'data.rat.basic',
+        'data.real.basic',
+        'data.real.ennreal',
+        'data.real.irrational',
+        'data.real.nnreal',
+        'data.real.sqrt',
+        'data.real.golden_ratio',
+        'data.set.finite',
+        'data.sym.sym2',
+        'data.zmod.basic',
+        'dynamics.fixed_points.basic',
+        'field_theory.finite.basic',
+        'geometry.euclidean.basic',
+        'geometry.euclidean.circumcenter',
+        'geometry.euclidean.monge_point',
+        'geometry.euclidean.sphere',
+        'linear_algebra.affine_space.affine_map',
+        'linear_algebra.affine_space.independent',
+        'linear_algebra.affine_space.ordered',
+        'linear_algebra.finite_dimensional',
+        'logic.equiv.basic',
+        'measure_theory.integral.interval_integral',
+        'number_theory.arithmetic_function',
+        'number_theory.legendre_symbol.quadratic_reciprocity',
+        'number_theory.primes_congruent_one',
+        'order.bounds',
+        'order.filter.basic',
+        'order.well_founded',
+        'topology.basic',
+        'topology.instances.nnreal'
+    ]
+
+    lean_useful_imports = [
+        'init.util',
+        'init.function',
+        'init.coe',
+        'init.ite_simp',
+        'init.propext',
+        'init.core',
+        'init.logic',
+        'init.wf',
+        'init.cc_lemmas',
+        'init.version',
+        'init.funext',
+        'init.classical',
+        'init.default',
+        'init.meta.derive',
+        'init.meta.well_founded_tactics',
+        'init.meta.interactive_base',
+        'init.meta.mk_has_reflect_instance',
+        'init.meta.decl_cmds',
+        'init.meta.hole_command',
+        'init.meta.injection_tactic',
+        'init.meta.exceptional',
+        'init.meta.match_tactic',
+        'init.meta.interactive',
+        'init.meta.mk_inhabited_instance',
+        'init.meta.has_reflect',
+        'init.meta.format',
+        'init.meta.expr_address',
+        'init.meta.type_context',
+        'init.meta.task',
+        'init.meta.simp_tactic',
+        'init.meta.congr_lemma',
+        'init.meta.ref',
+        'init.meta.feature_search',
+        'init.meta.congr_tactic',
+        'init.meta.attribute',
+        'init.meta.async_tactic',
+        'init.meta.rb_map',
+        'init.meta.tagged_format',
+        'init.meta.expr',
+        'init.meta.options',
+        'init.meta.instance_cache',
+        'init.meta.rewrite_tactic',
+        'init.meta.set_get_option_tactics',
+        'init.meta.contradiction_tactic',
+        'init.meta.declaration',
+        'init.meta.backward',
+        'init.meta.comp_value_tactics',
+        'init.meta.occurrences',
+        'init.meta.fun_info',
+        'init.meta.case_tag',
+        'init.meta.name',
+        'init.meta.json',
+        'init.meta.constructor_tactic',
+        'init.meta.float',
+        'init.meta.mk_dec_eq_instance',
+        'init.meta.module_info',
+        'init.meta.vm',
+        'init.meta.relation_tactics',
+        'init.meta.level',
+        'init.meta.pexpr',
+        'init.meta.local_context',
+        'init.meta.rec_util',
+        'init.meta.environment',
+        'init.meta.default',
+        'init.meta.ac_tactics',
+        'init.meta.mk_has_sizeof_instance',
+        'init.meta.tactic',
+        'init.meta.interaction_monad',
+        'init.meta.parser',
+        'init.meta.smt.ematch',
+        'init.meta.smt.interactive',
+        'init.meta.smt.smt_tactic',
+        'init.meta.smt.rsimp',
+        'init.meta.smt.default',
+        'init.meta.smt.congruence_closure',
+        'init.meta.converter.interactive',
+        'init.meta.converter.conv',
+        'init.meta.converter.default',
+        'init.meta.widget.basic',
+        'init.meta.widget.html_cmd',
+        'init.meta.widget.replace_save_info',
+        'init.meta.widget.tactic_component',
+        'init.meta.widget.interactive_expr',
+        'init.meta.widget.default',
+        'init.algebra.classes',
+        'init.algebra.order',
+        'init.algebra.functions',
+        'init.algebra.default',
+        'init.control.functor',
+        'init.control.lift',
+        'init.control.lawful',
+        'init.control.applicative',
+        'init.control.monad',
+        'init.control.except',
+        'init.control.state',
+        'init.control.alternative',
+        'init.control.option',
+        'init.control.combinators',
+        'init.control.reader',
+        'init.control.id',
+        'init.control.default',
+        'init.control.monad_fail',
+        'init.data.basic',
+        'init.data.to_string',
+        'init.data.prod',
+        'init.data.repr',
+        'init.data.set',
+        'init.data.punit',
+        'init.data.setoid',
+        'init.data.default',
+        'init.data.quot',
+        'init.data.char.basic',
+        'init.data.char.classes',
+        'init.data.char.lemmas',
+        'init.data.char.default',
+        'init.data.nat.gcd',
+        'init.data.nat.basic',
+        'init.data.nat.div',
+        'init.data.nat.lemmas',
+        'init.data.nat.default',
+        'init.data.nat.bitwise',
+        'init.data.option.basic',
+        'init.data.option.instances',
+        'init.data.list.basic',
+        'init.data.list.instances',
+        'init.data.list.qsort',
+        'init.data.list.lemmas',
+        'init.data.list.default',
+        'init.data.string.basic',
+        'init.data.string.ops',
+        'init.data.string.default',
+        'init.data.fin.basic',
+        'init.data.fin.ops',
+        'init.data.fin.default',
+        'init.data.ordering.basic',
+        'init.data.ordering.lemmas',
+        'init.data.ordering.default',
+        'init.data.array.slice',
+        'init.data.array.basic',
+        'init.data.array.default',
+        'init.data.subtype.basic',
+        'init.data.subtype.instances',
+        'init.data.subtype.default',
+        'init.data.unsigned.basic',
+        'init.data.unsigned.ops',
+        'init.data.unsigned.default',
+        'init.data.bool.basic',
+        'init.data.bool.lemmas',
+        'init.data.bool.default',
+        'init.data.sum.basic',
+        'init.data.sum.instances',
+        'init.data.sum.default',
+        'init.data.sigma.basic',
+        'init.data.sigma.lex',
+        'init.data.sigma.default',
+        'init.data.int.comp_lemmas',
+        'init.data.int.basic',
+        'init.data.int.order',
+        'init.data.int.default',
+        'init.data.int.bitwise',
+        'smt.prove',
+        'smt.array',
+        'smt.arith',
+        'smt.default',
+        'system.random',
+        'system.io_interface',
+        'system.io',
+        'data.buffer',
+        'data.dlist',
+        'data.vector',
+        'data.buffer.parser',
+        'tools.debugger.util',
+        'tools.debugger.cli',
+        'tools.debugger.default'
+    ]
+
+class Lean3Lemma(object):
+    def __init__(self, namespace: str, name: str, dfn: str) -> None:
+        self.namespace = namespace
+        self.name = name
+        self.dfn = dfn
+    
+    def __str__(self) -> str:
+        return f"{self.namespace}.{self.name}: {self.dfn}"
+
+class Lean3SearchTool(object):
+    theorem_lemma_search_regex = r"(theorem|lemma) ([\w+|\d+]*) ([\S|\s]*?):="
+    def __init__(self, mathlib_path = None, imports = None):
+        assert mathlib_path is None or os.path.isdir(mathlib_path)
+        self.mathlib_path = mathlib_path
+        self.lean_lib_path = Lean3Utils.get_lean_lib_path()
+        self._mathlib_src_path = os.path.join(self.mathlib_path, "src") if self.mathlib_path is not None else None
+        self._imports = imports if imports is not None else Constants.mathlib_useful_imports + Constants.lean_useful_imports
+        self._import_paths = []
+        # Map the imports to the files they are imported from
+        for i in range(len(self._imports)):
+            possible_relative_path = self._imports[i].replace(".", "/") + ".lean"
+            possible_lean_lib_path = os.path.join(self.lean_lib_path, possible_relative_path) if self.lean_lib_path is not None else None
+            possible_mathlib_src_path = os.path.join(self._mathlib_src_path, possible_relative_path) if self._mathlib_src_path is not None else None
+            if self.lean_lib_path is not None and os.path.isfile(possible_lean_lib_path):
+                self._import_paths.append(possible_lean_lib_path)
+            if self._mathlib_src_path is not None and os.path.isfile(possible_mathlib_src_path):
+                self._import_paths.append(possible_mathlib_src_path)
+        self._namespaces_to_theorems : typing.Dict[str, typing.List[Lean3Lemma]] = {}
+        self._lemmas : typing.List[Lean3Lemma] = []
+    
+    def initialize(self) -> None:
+        for import_path in self._import_paths:
+            self._add(import_path)
+
+    @property
+    def lemmas(self) -> typing.List[Lean3Lemma]:
+        return self._lemmas
+    
+    @property
+    def namespaces(self) -> typing.List[str]:
+        return list(self._namespaces_to_theorems.keys())
+    
+    def _add(self, lean_file_path: str) -> None:
+        assert os.path.isfile(lean_file_path)
+        assert lean_file_path.endswith(".lean")
+        with open(lean_file_path, "r") as f:
+            text = f.read()
+            # Remove comments
+            text = Lean3Utils.remove_comments(text)
+            theorems_with_namespaces = Lean3Utils.find_theorems_with_namespaces(text)
+            for namespace, name, dfn in theorems_with_namespaces:
+                lemma = Lean3Lemma(namespace, name, dfn)
+                if namespace not in self._namespaces_to_theorems:
+                    self._namespaces_to_theorems[namespace] = []
+                self._namespaces_to_theorems[namespace].append(lemma)
+                self._lemmas.append(lemma)
+
+if __name__ == "__main__":
+    mathlib_path = "data/benchmarks/miniF2F/_target/deps/mathlib"
+    lean3_search_tool = Lean3SearchTool(mathlib_path=mathlib_path)
+    lean3_search_tool.initialize()
+    print("Lemmas:", len(lean3_search_tool.lemmas))
+    print("Namespaces:", len(lean3_search_tool.namespaces))
+    # print all theorems
+    cnt_nat = 0
+    cnt_real = 0
+    for theorem in lean3_search_tool.lemmas:
+        if theorem.namespace.startswith("nat"):
+            print(theorem)
+            cnt_nat += 1
+        if theorem.namespace.startswith("real"):
+            print(theorem)
+            cnt_real += 1
+        if cnt_nat > 100 and cnt_real > 100:
+            break

itp_interface/lean_server/lean4_repl_interface.py

@@ -0,0 +1,151 @@
+import os
+import pty
+import subprocess
+import json
+import select
+import time
+import logging
+
+class ProcessInterface:
+    buffer_size = 1024
+    def __init__(self, command, cwd, logger: logging.Logger = None, log_level=logging.INFO, reboot_every_n_commands = 50):
+        """
+        Note: This class is not thread-safe. It is intended to be used in a single-threaded environment.
+        """
+        assert reboot_every_n_commands > 0
+        self.reboot_every_n_commands = reboot_every_n_commands
+        self.command = command
+        self.cwd = cwd
+        self._last_reboot = 0
+        self._start()
+        self.logger = logger if logger else logging.getLogger(__name__)
+        self.logger.setLevel(log_level)
+    
+    def _start(self):
+        master, slave = pty.openpty()
+        self.process = subprocess.Popen(
+            self.command.split(),
+            cwd=self.cwd,
+            stdin=slave,
+            stdout=slave,
+            stderr=subprocess.STDOUT,
+            text=True
+        )
+        os.close(slave)
+        self.master = master
+        self.buffer = ''  # Buffer to accumulate data from stdout
+        self.sent_commands = ''  # Buffer to track sent commands
+        pass
+
+    def is_rebooted(self):
+        return self._last_reboot == 0
+
+    def send_command(self, command_dict):
+        # Check if the process is still running
+        if not self.process_is_running():
+            raise Exception("Process is not running.")
+        json_command = json.dumps(command_dict, ensure_ascii=False) + '\n\n'
+        normalized_command = json_command.replace('\r\n', '\n')  # Normalize newlines
+        os.write(self.master, normalized_command.encode('utf-8'))  # Send command to process
+        self.logger.debug(f"Sent:\n{normalized_command}")
+        self.sent_commands += normalized_command  # Keep track of normalized sent commands
+        time.sleep(0.3) # Wait for the process to process the command
+    
+    def process_is_running(self):
+        return self.process.poll() is None
+
+    def read_response(self, timeout=10):
+        try:
+            response = self._read_response(timeout)
+            return response
+        except:
+            self.buffer = ''
+            self.sent_commands = ''
+            raise
+
+    def _read_response(self, timeout=10):
+        end_time = time.time() + timeout
+        wait_time = 0.2
+        response_read = False
+        input_was_read = False
+        output_was_read = False
+        while time.time() < end_time and not response_read:
+            readable, _, _ = select.select([self.master], [], [], wait_time)
+            # print("readable: ", readable)
+            while len(readable) > 0 and not response_read:
+                try:
+                    data = os.read(self.master, ProcessInterface.buffer_size).decode('utf-8', 'ignore')  # Read data from stdout
+                    if input_was_read:
+                        output_was_read = True
+                except OSError as e:
+                    # Ignore the os error
+                    if e.errno == 5:
+                        response_read = True
+                        break
+                    else:
+                        raise
+                # print("data: ", data)
+                normalized_data = data.replace('\r\n', '\n')  # Normalize received data
+                self.buffer += normalized_data
+                # Clean buffer by removing echoed commands before parsing
+                if self.sent_commands != '' and self.buffer.startswith(self.sent_commands):
+                    self.buffer = self.buffer[len(self.sent_commands):]  # Remove echoed commands
+                    self.sent_commands = ''  # Clear sent commands buffer after removing
+                    input_was_read = True
+                if input_was_read and self.buffer.find('{') != -1 and self.buffer.rfind('}') != -1:
+                    output_was_read = True
+                readable, _, _ = select.select([self.master], [], [], wait_time)
+                response_read = len(readable) == 0 and output_was_read
+                if time.time() >= end_time:
+                    response_read = False
+                    break
+        self.buffer = self.buffer.strip()  # Remove leading/trailing whitespace
+        if not response_read or len(self.buffer) == 0 or self.buffer.find('{') == -1:
+            self.logger.debug("Could not read response. Waiting for more data.")
+            raise TimeoutError(f"Could not read response. Buffer: {self.buffer}")
+        self.buffer = self.buffer[self.buffer.index('{'):]  # Remove any leading garbage
+        try:
+            # Attempt to parse the clean buffer as JSON
+            response = json.loads(self.buffer)
+            self.logger.debug(f"Received:\n{response}")
+            self.buffer = ''  # Clear buffer after successful parse
+            self._last_reboot += 1
+            if self._last_reboot >= self.reboot_every_n_commands:
+                self.close()
+                self._start()
+                self._last_reboot = 0
+            return response
+        except json.JSONDecodeError as e:
+            self.logger.debug("Failed to parse JSON. Waiting for more data.")
+            self.logger.debug(f"Buffer: {self.buffer}")
+            raise TimeoutError(f"Failed to parse JSON. Buffer: {self.buffer}", e)
+
+    def close(self):
+        try:
+            os.close(self.master)
+            self.process.terminate()
+            self.process.wait()
+        except:
+            pass
+
+# Process interface test
+if __name__ == "__main__":
+    #.lake/bin/repl
+    repl_path = "./src/itp_interface/tools/repl/.lake/build/bin/repl"
+    lean4_proj_path = "./src/data/test/lean4_proj"
+    file_path = "./src/data/test/lean4_proj/Lean4Proj/Basic.lean"
+    # lean4_proj_path = './src/data/test/Mathlib'
+    # file_path = './src/data/test/Mathlib/.lake/packages/mathlib/Mathlib/Data/Nat/Bits.lean'
+    file_path = os.path.abspath(file_path)
+    abs_repl_path = os.path.abspath(repl_path)
+    interface = ProcessInterface(f"lake env {abs_repl_path}", lean4_proj_path, log_level=logging.DEBUG)
+    try:
+        interface.send_command({"path": file_path, "allTactics": True})
+        response = interface.read_response(1000)
+        print(response)
+        print("====================================")
+        interface.send_command({"path": file_path, "allTactics": True})
+        response = interface.read_response(1000)
+        print(response)
+    finally:
+        interface.close()