itp-interface 1.0.0__py3-none-any.whl

itp_interface/lean_server/lean4_utils.py

@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+
+import sys
+root_dir = f"{__file__.split('itp_interface')[0]}"
+if root_dir not in sys.path:
+    sys.path.append(root_dir)
+import os
+import re
+import typing
+from itp_interface.lean_server.lean_context import Obligation, ProofContext
+
+class Lean4Utils:
+    lean_internal_lib_cmd = "elan which lean"
+    theorem_lemma_search_regex = re.compile(r"(theorem|lemma) ([\w+|\d+]*) ([\S|\s]*?):=")
+    proof_context_separator = "⊢"
+    proof_context_regex = r"((\d+) goals)*((case [\S]+(\n|\n\n))*[\s|\S]*?)\n\n"
+    proof_context_regex_cmp = re.compile(proof_context_regex)
+    proof_context_regex2 = r"((\d+) goals)*((case [\S]+(\n|\n\n))*[\s|\S]*?)\ncase [\S]+"
+    proof_context_regex2_cmp = re.compile(proof_context_regex2)
+    goal_regex = rf"([\s|\S]*?){proof_context_separator}([\s|\S]*)"
+
+    def remove_comments(text: str) -> str:
+        # Remove comments
+        #1. First remove all nested comments
+        #2. Then remove all single line comments
+        # Comments are of the form:
+        # 1. /- ... -/
+        # 2. -- ...
+        # Let's do 1
+        # First, let's find all the comments
+        new_text = []
+        idx = 0
+        while idx < len(text):
+            if idx < len(text) - 1 and text[idx] == '/' and text[idx+1] == '-':
+                # We found a comment
+                # Find the end of the comment
+                end_of_comment_idx = idx + 2
+                while end_of_comment_idx < len(text) and \
+                    not (text[end_of_comment_idx] == '-' and \
+                    end_of_comment_idx + 1 < len(text) and \
+                    text[end_of_comment_idx + 1] == '/'):
+                    end_of_comment_idx += 1
+                if end_of_comment_idx >= len(text):
+                    # Unfinished comment
+                    new_text.extend(text[idx:end_of_comment_idx])
+                    idx = end_of_comment_idx
+                else:
+                    # Remove the comment
+                    idx = end_of_comment_idx + 2
+            if idx < len(text):
+                new_text.append(text[idx])
+                idx += 1
+        text = "".join(new_text)
+        new_text = []
+        # Now let's do 2
+        idx = 0
+        while idx < len(text):
+            if idx < len(text) - 1 and text[idx] == '-' and text[idx+1] == '-':
+                # We found a comment
+                # Find the end of the comment
+                end_of_comment_idx = idx + 2
+                while end_of_comment_idx < len(text) and text[end_of_comment_idx] != '\n':
+                    end_of_comment_idx += 1
+                if end_of_comment_idx >= len(text):
+                    # Unfinished comment
+                    new_text.extend(text[idx:end_of_comment_idx])
+                # Remove the comment
+                idx = end_of_comment_idx
+            if idx < len(text):
+                new_text.append(text[idx])
+                idx += 1
+        text = "".join(new_text)
+        return text
+    
+    def get_lean_root_path() -> str:
+        lean_exe = os.popen(Lean4Utils.lean_internal_lib_cmd).read().strip()
+        lean_bin_path = os.path.dirname(lean_exe)
+        lean_root_path = os.path.dirname(lean_bin_path)
+        return lean_root_path
+
+    def get_lean_lib_path() -> str:
+        lean_root_path = Lean4Utils.get_lean_root_path()
+        lean_lib_path = os.path.join(lean_root_path, "lib", "lean", "library")
+        return lean_lib_path
+    
+    def find_theorems_with_namespaces(text: str) -> typing.List[typing.Tuple[str, str, str]]:
+        idx = 0
+        theorems = []
+        lines = text.split('\n')
+        current_namespace = None
+        while idx < len(lines):
+            line = lines[idx]
+            if line.startswith("namespace"):
+                current_namespace = line[len("namespace"):].strip()
+                # Find the end of the namespace
+                end_of_namespace_idx = idx + 1
+                end_line = lines[end_of_namespace_idx]
+                while end_line is not None and not end_line.startswith("end") and not end_line.endswith(current_namespace):
+                    end_of_namespace_idx += 1
+                    end_line = lines[end_of_namespace_idx] if end_of_namespace_idx < len(lines) else None
+                if end_line is not None:
+                    namespace_content = " ".join(lines[idx:end_of_namespace_idx+1])
+                    for name, dfn in Lean4Utils.find_theorems(namespace_content):
+                        theorems.append((current_namespace, name, dfn))
+                idx = end_of_namespace_idx + 1
+            else:
+                idx += 1
+        return theorems
+    
+    def find_theorems(text: str) -> typing.List[typing.Tuple[str, str]]:
+        matches = Lean4Utils.theorem_lemma_search_regex.findall(text)
+        theorems = []
+        for match in matches:
+            name = str(match[1]).strip()
+            dfn = str(match[2]).strip()
+            name = name.strip(':')
+            dfn = dfn.strip(':')
+            theorems.append((name, dfn))
+        return theorems
+
+    def parse_proof_context_human_readable(proof_context_str: str) -> ProofContext:
+        if len(proof_context_str) == 0 and Lean4Utils.proof_context_separator not in proof_context_str:
+            return None
+        if proof_context_str == "no goals":
+            return ProofContext.empty()
+        proof_context_str = proof_context_str.strip()
+        proof_context_str += "\n\n"
+        all_matches = re.findall(Lean4Utils.proof_context_regex, proof_context_str, re.MULTILINE)
+        goal_strs = []
+        for match in all_matches:
+            goal_str = match[2]
+            goal_str = goal_str.strip()
+            goal_strs.append(goal_str)
+        goals = []
+        # try:
+        for goal_str in goal_strs:
+            goal = Lean4Utils.parse_goal(goal_str)
+            goals.append(goal)
+        # except Exception as e:
+        #     print(f"proof_context_str:\n {proof_context_str}")
+        #     print(f"goal_strs:\n {goal_strs}")
+        #     raise
+        return ProofContext(goals, [], [], [])
+
+    def parse_proof_context_human_readable_as_goals(proof_context_str: str) -> typing.List[Obligation]:
+        if len(proof_context_str) == 0 and Lean4Utils.proof_context_separator not in proof_context_str:
+            return None
+        if proof_context_str == "no goals":
+            return ProofContext.empty()
+        proof_context_str = proof_context_str.strip()
+        proof_context_str += "\n\n"
+        all_matches = Lean4Utils.proof_context_regex_cmp.findall(proof_context_str)  # re.findall(Lean4Utils.proof_context_regex, proof_context_str, re.MULTILINE)
+        goal_strs = []
+        for match in all_matches:
+            goal_str = match[2]
+            goal_str = goal_str.strip()
+            goal_has_multiple_goals = True
+            while goal_has_multiple_goals:
+                match_goal = Lean4Utils.proof_context_regex2_cmp.match(goal_str)
+                if match_goal is not None:
+                    goal_sub_str = match_goal.group(3)
+                    goal_strs.append(goal_sub_str)
+                    _, span_end = match_goal.span(3)
+                    goal_str = goal_str[span_end:]
+                    goal_str = goal_str.strip()
+                    if len(goal_str) == 0:
+                        goal_has_multiple_goals = False
+                else:
+                    goal_strs.append(goal_str)
+                    goal_has_multiple_goals = False
+            # goal_strs.append(goal_str)
+        goals = []
+        # try:
+        for goal_str in goal_strs:
+            goal = Lean4Utils.parse_goal(goal_str)
+            goals.append(goal)
+        # except Exception as e:
+        #     print(f"proof_context_str:\n {proof_context_str}")
+        #     print(f"goal_strs:\n {goal_strs}")
+        #     raise
+        return goals
+
+    def parse_goal(goal_str: str):
+        goal_str = goal_str.strip()
+        goal = ""
+        hyps_goals = re.findall(Lean4Utils.goal_regex, goal_str, re.MULTILINE)
+        assert len(hyps_goals) <= 1, f"Found more than one goal in the goal string: {goal_str}"
+        if len(hyps_goals) == 1:
+            hypotheses_str, goal = hyps_goals[0]
+            hypotheses_str = hypotheses_str.strip()
+            goal = goal.strip()
+            hypotheses = [hyp.rstrip(',') for hyp in hypotheses_str.split("\n")]
+            goal = Obligation(hypotheses, goal)
+            return goal
+        else:
+            # Everyline except the last one is a hypothesis
+            hypotheses = goal_str.split("\n")
+            goal = hypotheses[-1]
+            hypotheses = hypotheses[:-1]
+            goal = Obligation(hypotheses, goal)
+            return goal
+
+if __name__ == '__main__':
+    goals = "case intro.inl\n\u03b1 : Type u_1\n\u03b2 : Type u_2\n\u03b3 : Type u_3\nr r' : \u03b1 \u2192 \u03b1 \u2192 Prop\ninst\u271d\u00b9 : LinearOrder \u03b2\nh : WellFounded fun x x_1 => x < x_1\ninst\u271d : PartialOrder \u03b3\nf g : \u03b2 \u2192 \u03b3\nhf : StrictMono f\nhg : StrictMono g\nhfg : range f = range g\nb : \u03b2\nH : \u2200 a < b, f a = g a\nc : \u03b2\nhc : f c = g b\nhcb : c < b\n\u22a2 f b \u2264 g b\n\ncase intro.inr\n\u03b1 : Type u_1\n\u03b2 : Type u_2\n\u03b3 : Type u_3\nr r' : \u03b1 \u2192 \u03b1 \u2192 Prop\ninst\u271d\u00b9 : LinearOrder \u03b2\nh : WellFounded fun x x_1 => x < x_1\ninst\u271d : PartialOrder \u03b3\nf g : \u03b2 \u2192 \u03b3\nhf : StrictMono f\nhg : StrictMono g\nhfg : range f = range g\nb : \u03b2\nH : \u2200 a < b, f a = g a\nc : \u03b2\nhc : f c = g b\nhbc : b \u2264 c\n\u22a2 f b \u2264 g b"
+    goals = """case inl
+
+⊢ Λ 1 ≠ 0 ↔ IsPrimePow 1
+
+case inr
+n : ℕ
+hn : n ≠ 1
+⊢ Λ n ≠ 0 ↔ IsPrimePow n
+"""
+    print("Testing Lean4Utils.parse_proof_context_human_readable")
+    print(f"Goals:\n{goals}")
+    proof_context = Lean4Utils.parse_proof_context_human_readable(goals)
+    print(proof_context)
+    text = """
+    -- This is a comment
+    /- This is a nested comment -/
+    theorem foo : 1 = 1 := rfl
+    lemma bar : 2 = 2 := rfl
+    """
+    print("Testing Lean4Utils")
+    print("-"*20)
+    print("Testing Lean4Utils.remove_comments")
+    print("Before:")
+    print(text)
+    print("After:")
+    print(Lean4Utils.remove_comments(text))
+    print("-"*20)
+    print("Testing Lean3Utils.get_lean_root_path")
+    print(Lean4Utils.get_lean_root_path())
+    print("-"*20)
+    print("Testing Lean3Utils.get_lean_lib_path")
+    print(Lean4Utils.get_lean_lib_path())
+    print("-"*20)
+    print("Testing Lean3Utils.find_theorems")
+    lib_path = Lean4Utils.get_lean_lib_path()
+    init_data_name = "init/data/array/basic.lean"
+    init_data_path = os.path.join(lib_path, init_data_name)
+    with open(init_data_path, 'r') as f:
+        text = f.read()
+    print(f"Testing Lean3Utils.find_theorems on {init_data_name}")
+    for name, dfn in Lean4Utils.find_theorems(text):
+        print(f"{name}: {dfn}")
+    print("-"*20)
+    print("Testing Lean3Utils.find_theorems_with_namespaces")
+    for namespace, name, dfn in Lean4Utils.find_theorems_with_namespaces(text):
+        print(f"[{namespace}]::{name}: {dfn}")
+    print("-"*20)
+    # print(text)
+
+    # print(Lean3Utils.find_theorems(text))
+    # print(Lean3Utils.find_theorems_with_namespaces(text))

itp_interface/lean_server/lean_cmd_server.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+import sys
+root_dir = f"{__file__.split('itp_interface')[0]}"
+if root_dir not in sys.path:
+    sys.path.append(root_dir)
+import os
+import typing
+from subprocess import Popen, PIPE, STDOUT
+from dataclasses import dataclass, field
+from dataclasses_json import dataclass_json
+
+@dataclass_json
+@dataclass
+class Message:
+    level: str
+    file_name: str
+    line_num: int
+    column_num: int
+    text: str
+
+@dataclass_json
+@dataclass
+class LeanCmdServerResponse:
+    state: typing.Optional[str] = None
+    messages: typing.List[Message] = field(default_factory=list)
+
+EmptyResponse = LeanCmdServerResponse()
+
+class LeanCmdServer:
+    has_state_message = 'tactic failed, there are unsolved goals\nstate:'
+    has_focused_message = 'solve1 tactic failed, focused goal has not been solved\nstate:'
+    def __init__(self, memory_in_mibs = 40000, cwd = '.', debug=False):
+        assert cwd is not None, "cwd must be provided"
+        assert os.path.isdir(cwd), "cwd must be a valid directory"
+        self.memory_in_mibs = memory_in_mibs
+        self.debug = debug
+        self.cwd = cwd
+        self.process = None
+
+    def run(self, filepath: str, timeout_in_secs: float = 120.0):
+        full_path = os.path.join(self.cwd, filepath)
+        assert os.path.isfile(full_path), f"filepath must be a valid file: {filepath}"
+        lean_cmd = f'lean --memory={self.memory_in_mibs} {filepath}'
+        self.process = Popen(
+            lean_cmd, 
+            shell = True, 
+            stdin = PIPE, 
+            stdout = PIPE, 
+            stderr = STDOUT,
+            cwd = self.cwd, 
+            bufsize = 1, 
+            universal_newlines = True)
+        # Start the process, and wait for it to finish
+        self.process.wait(timeout=timeout_in_secs)
+        # Get the output
+        output = self.process.stdout.read()
+        # Kill the process if it is still running
+        self.process.kill()
+        # Return the output
+        if len(output) == 0:
+            return EmptyResponse
+        else:
+            return self.parse_output(full_path, output) 
+    
+    def parse_output(self, full_path: str, output: str):
+        # AbsFilePath:Line:Column: [waring|error]: Message
+        # First get absolute path from full path
+        abs_path = os.path.abspath(full_path) + ':'
+        messages = output.split(abs_path)
+        messages = [msg for msg in messages if len(msg) > 0] # Remove empty strings
+        final_messages : typing.List[Message] = []
+        state = None
+        msg_unparsed = []
+        for msg in messages:
+            # Get rid of line number and column number
+            try:
+                line_num_str, col_num_str, level_str, text = msg.split(':', 3)
+                line_num_str = line_num_str.strip()
+                col_num_str = col_num_str.strip()
+                level_str = level_str.strip()
+                text = text.strip()
+                line_num = int(line_num_str)
+                col_num = int(col_num_str)
+                level = level_str.lower()
+                if level == 'error' and \
+                    (text.startswith(LeanCmdServer.has_state_message) or text.startswith(LeanCmdServer.has_focused_message)):
+                    state = text[text.find('state:') + len('state:'):].strip()
+                else:
+                    final_messages.append(Message(level, full_path, line_num, col_num, text))
+            except:
+                msg_unparsed.append(msg)
+                pass
+        if len(final_messages) > 0:
+            # Sort messages by line number
+            final_messages.sort(key=lambda msg: msg.line_num)
+        last_line_num = 0 if len(final_messages) == 0 else final_messages[-1].line_num
+        # Now add the unparsed messages
+        for msg in msg_unparsed:
+            final_messages.append(Message('info', full_path, last_line_num, 0, msg))
+        # re-sort
+        final_messages.sort(key=lambda msg: msg.line_num)
+        return LeanCmdServerResponse(state, final_messages)
+
+if __name__ == "__main__":
+    os.chdir(root_dir)
+    cwd = 'data/test/lean_proj'
+    path = 'src/simple.lean'
+    server = LeanCmdServer(cwd=cwd)
+    output = server.run(path)
+    print(output)

itp_interface/lean_server/lean_context.py

@@ -0,0 +1,60 @@
+import typing
+
+class Obligation(typing.NamedTuple):
+    hypotheses: typing.List[str]
+    goal: str
+
+    @classmethod
+    def from_dict(cls, data):
+        return cls(**data)
+
+    def to_dict(self) -> typing.Dict[str, typing.Any]:
+        return {"hypotheses": self.hypotheses,
+                "goal": self.goal}
+
+
+class ProofContext(typing.NamedTuple):
+    fg_goals: typing.List[Obligation]
+    bg_goals: typing.List[Obligation]
+    shelved_goals: typing.List[Obligation]
+    given_up_goals: typing.List[Obligation]
+
+    @classmethod
+    def empty(cls: typing.Type['ProofContext']):
+        return ProofContext([], [], [], [])
+
+    @classmethod
+    def from_dict(cls, data):
+        fg_goals = list(map(Obligation.from_dict, data["fg_goals"]))
+        bg_goals = list(map(Obligation.from_dict, data["bg_goals"]))
+        shelved_goals = list(map(Obligation.from_dict, data["shelved_goals"]))
+        given_up_goals = list(map(Obligation.from_dict,
+                                  data["given_up_goals"]))
+        return cls(fg_goals, bg_goals, shelved_goals, given_up_goals)
+
+    def to_dict(self) -> typing.Dict[str, typing.Any]:
+        return {"fg_goals": list(map(Obligation.to_dict, self.fg_goals)),
+                "bg_goals": list(map(Obligation.to_dict, self.bg_goals)),
+                "shelved_goals": list(map(Obligation.to_dict,
+                                          self.shelved_goals)),
+                "given_up_goals": list(map(Obligation.to_dict,
+                                           self.given_up_goals))}
+
+    @property
+    def all_goals(self) -> typing.List[Obligation]:
+        return self.fg_goals + self.bg_goals + \
+            self.shelved_goals + self.given_up_goals
+
+    @property
+    def focused_goal(self) -> str:
+        if self.fg_goals:
+            return self.fg_goals[0].goal
+        else:
+            return ""
+
+    @property
+    def focused_hyps(self) -> typing.List[str]:
+        if self.fg_goals:
+            return self.fg_goals[0].hypotheses
+        else:
+            return []

itp_interface/lean_server/lean_sync_server.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+
+import sys
+root_dir = f"{__file__.split('itp_interface')[0]}"
+if root_dir not in sys.path:
+    sys.path.append(root_dir)
+import os
+from subprocess import Popen, PIPE, STDOUT
+from threading import Event
+import threading
+import time
+from typing import Optional, List
+
+from itp_interface.lean_server.commands import (SyncRequest, InfoRequest,
+                                  Request, CommandResponse, Message, Task,
+                                  InfoResponse, AllMessagesResponse, CurrentTasksResponse, ErrorResponse,
+                                  OkResponse, SyncResponse)
+
+class SyncLeanServer:
+    def __init__(self, lean_cmd='lean --server --memory=40000', cwd = '.', debug=False, debug_bytes=False):
+        assert lean_cmd is not None, "lean_cmd must be provided"
+        assert cwd is not None, "cwd must be provided"
+        assert os.path.isdir(cwd), "cwd must be a valid directory"
+        self.lean_cmd = lean_cmd
+        self.seq_num = 0
+        self.debug = debug
+        self.debug_bytes = debug_bytes
+        self.response_events = {}  # Key: seq_num, Value: threading.Event
+        self.responses = {}  # Key: seq_num, Value: response
+        self.is_fully_ready = Event()
+        self.process = None
+        self.messages : List[Message] = []
+        self.cwd = cwd
+        self.current_tasks : List[Task] = []
+        self._exit_receiver = False
+        self._lock = threading.Lock() # This is needed for safety of dictionary access   
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.kill()
+
+    def start(self):
+        self.process = Popen(
+            self.lean_cmd, 
+            shell = True, 
+            stdin = PIPE, 
+            stdout = PIPE, 
+            stderr = STDOUT,
+            cwd = self.cwd, 
+            bufsize = 1, 
+            universal_newlines = True)
+        self.thread = threading.Thread(target=self.receiver)
+        self.thread.start()
+
+    def send(self, request: Request) -> Optional[CommandResponse]:
+        self.seq_num += 1
+        request.seq_num = self.seq_num
+
+        if self.debug:
+            print(f'Sending {request}')
+
+        json_request = request.to_json() + '\n'
+        if self.debug_bytes:
+            print(f'Sending {json_request.encode()}')
+
+        # First create the event
+        response_event = Event()
+        with self._lock:
+            self.response_events[self.seq_num] = response_event
+
+        # Then send the request
+        self.process.stdin.write(json_request)
+        self.process.stdin.flush()
+
+        # Wait for the response
+        response_event.wait()
+
+        # Get the response
+        with self._lock:
+            self.response_events.pop(self.seq_num)
+            response = self.responses.pop(self.seq_num)
+
+        if isinstance(response, OkResponse):
+            return response.to_command_response(request.command)
+        elif isinstance(response, ErrorResponse):
+            raise ChildProcessError(f'Lean server error while executing "{request.command}":\n{response}')
+
+    def receiver(self):
+        while not self._exit_receiver:
+            line = self.process.stdout.readline().strip()
+            if line:
+                if self.debug_bytes:
+                    print(f'Received {line}')
+                response = CommandResponse.parse_response(line)
+                if self.debug:
+                    print(f'Received {response}')
+
+                if isinstance(response, CurrentTasksResponse):
+                    self.current_tasks = response.tasks
+                    if not response.is_running:
+                        self.is_fully_ready.set()
+                elif isinstance(response, AllMessagesResponse):
+                    self.messages = response.msgs  # Storing messages
+                elif isinstance(response, OkResponse) or isinstance(response, ErrorResponse):
+                    seq_num = response.seq_num
+                    with self._lock:
+                        # seq_num will be in the dictionary because we created the event before sending the request
+                        self.responses[seq_num] = response
+                        self.response_events[seq_num].set()
+            else:
+                # sleep for a bit to avoid busy waiting
+                time.sleep(0.02)
+
+    def full_sync(self, filename, content=None):
+        request = SyncRequest(filename, content)
+        response = self.send(request)
+
+        if isinstance(response, SyncResponse) and response.message == "file invalidated":
+            self.is_fully_ready.clear()
+            self.is_fully_ready.wait()
+
+    def state(self, filename, line, col) -> str:
+        request = InfoRequest(filename, line, col)
+        response = self.send(request)
+
+        if isinstance(response, InfoResponse) and response.record:
+            return response.record.state or ''
+        else:
+            return ''
+
+    def kill(self):
+        # Ensure that the process is killed
+        self.process.kill()
+        try:
+            self.process.stdin.close()
+        except:
+            pass
+        self._exit_receiver = True
+        self.thread.join(1.0)
+
+if __name__ == "__main__":
+    from pathlib import Path
+    os.chdir(root_dir)
+    # cwd = 'data/benchmarks/miniF2F'
+    cwd = 'data/test/lean_proj'
+    # path = f'{cwd}/lean/src/test.lean'
+    path = f'{cwd}/src/simple.lean'
+    lines = Path(path).read_text().split('\n')
+    with SyncLeanServer(cwd=cwd) as server:
+        path = 'temp124321432.lean'
+        server.full_sync(path, content='')
+
+        theorem_started = False
+        for i, line in enumerate(lines):
+            content = '\n'.join(lines[:i+1])
+            if 'theorem' in line:
+                theorem_started = True
+            if theorem_started and 'end' in line:
+                theorem_started = False
+            if theorem_started:
+                content += '\nend'
+            server.full_sync(path, content)
+            if len(server.messages) > 0:
+                for idx, message in enumerate(server.messages):
+                    print(f"Message [{idx + 1}]: {message.text}")
+            before = server.state(path, i+1, 0)
+            after = server.state(path, i+1, len(line))
+            print(f'Line {i+1}: {line}')
+            if before or after:
+                print(f'State before:\n{before}\n')
+                print(f'State after:\n{after}\n')