itp-interface 1.0.0__py3-none-any.whl

itp_interface/tools/coq_executor.py

@@ -0,0 +1,508 @@
+#!/usr/bin/env python3
+
+import sys
+root_dir = f"{__file__.split('itp_interface')[0]}"
+if root_dir not in sys.path:
+    sys.path.append(root_dir)
+import os
+import logging
+import typing
+import functools
+import random
+import re
+import subprocess
+from itp_interface.coq_ser_api import SerapiInstance, ProofContext, GetCoqAgent, CoqAgent
+from itp_interface.tools.coq_parse_utils import CoqLineByLineReader, CoqStepByStepStdInReader
+logger = logging.getLogger()
+
+class CoqExecutor:
+    keywords = {
+        "Theorem", "Lemma", "Fact", "Remark", "Corollary", "Proposition", "Example", "Proof", "Qed", "Defined", "Admitted", "Abort",
+        "Fixpoint", "CoFixpoint", "Function", "Program Fixpoint", "Program CoFixpoint", "Program Function", "Let", "Let Fixpoint", 
+        "Let CoFixpoint", "Let Function", "Let Program Fixpoint", "Let Program CoFixpoint", "Let Program Function",
+        "forall", "exists", "fun", "match", "if", "then", "else", "with", "as", "in", "end", "return", "Type", "Set", "Prop",
+        "Require", "Import", "Export", "From", "Module", "Section", "End", "Variable", "Axiom", "Parameter", "Hypothesis", "Context",
+        "Notation", "Reserved Notation", "Infix", "Notation", "Reserved Notation", "Infix", "Reserved Infix", "Notation", "Reserved Notation", "Definition",
+        "intros", "intro", "apply", "assumption", "exact", "reflexivity", "symmetry", "transitivity", "rewrite", "simpl", "unfold", "cbn", "cbv", "compute",
+        "destruct", "induction", "inversion", "injection", "split", "exists", "left", "right", "constructor", "auto", "eauto", "tauto", "omega", "lia", "ring",
+        "repeat", "try", "assert", "cut", "cutrewrite", "pose", "pose proof", "remember", "set", "setoid_rewrite", "generalize", "generalize dependent",
+        "move", "move =>", "move ->", "move => ->", 
+        ":", ".", "=>", "{", "}"
+    }
+    def __init__(self, project_root: str = None, main_file: str = None, use_hammer: bool = False, timeout_in_sec: int = 60, use_human_readable_proof_context: bool = False, proof_step_iter: typing.Iterator[str] = None, suppress_error_log: bool = False, setup_cmds: typing.List[str] = []):
+        assert proof_step_iter is None or isinstance(proof_step_iter, typing.Iterator), \
+            "proof_step_iter must be an iterator"
+        assert main_file is not None or proof_step_iter is not None, \
+            "Either main_file or proof_step_iter must be provided"
+        assert main_file is None or proof_step_iter is None, \
+            "Only one of main_file or proof_step_iter must be provided"
+        assert main_file is None or (os.path.exists(main_file) and main_file.endswith(".v")), \
+            "main_file must be a valid path to a '.v' file"
+        assert project_root is None or (os.path.exists(project_root) and os.path.isdir(project_root)), \
+            "project_root must be a valid path to a directory"
+        self.use_human_readable_proof_context = use_human_readable_proof_context
+        self.project_root = project_root if project_root is not None else "."
+        self.main_file = main_file
+        self.use_hammer = use_hammer
+        self.timeout_in_sec = min(timeout_in_sec, 120) # Maximum 120s timeout
+        self.current_stmt = None
+        self.line_num = 0
+        self.main_file_iter = proof_step_iter
+        self.suppress_error_log = suppress_error_log
+        self.coq : CoqAgent = None
+        self.execution_complete = False
+        self.setup_cmds = setup_cmds
+    
+    def __enter__(self):
+        self._all_dep_handles = []
+        all_setup_cmds = '\n'.join(self.setup_cmds)
+        if len(all_setup_cmds) > 0:
+            try:
+                env_settings = subprocess.run(all_setup_cmds, shell=True, stdout=subprocess.PIPE, check=True, text=True).stdout
+            except:
+                env_settings = None
+                logger.error(f"Got an exception while running setup commands:\n {all_setup_cmds}")
+                logger.exception(f"Exception Log")
+                logger.info(f"Continuing without setting up the environment")
+        else:
+            env_settings = None
+        self.coq : CoqAgent = GetCoqAgent(prelude=self.project_root, use_human_readable_str=self.use_human_readable_proof_context, env_string=env_settings, timeout=self.timeout_in_sec)
+        if hasattr(self.coq, "quiet"):
+            self.coq.quiet = self.suppress_error_log
+        if self.main_file_iter is None:
+            self.main_file_iter = CoqLineByLineReader(self.main_file).instruction_step_generator()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        try:
+            self.main_file_iter.close() # Close the file handle
+        except:
+            pass
+        self.coq.kill() # Kill the coq instance after use
+
+    @property
+    def token_separator_set(self):
+        return set(" ()[]{}.,;:+-*/=<>!~?@#$%^&|`\"\\")
+
+    @property
+    def token_separator(self):
+        return " ()[]{}.,;:+-*/=<>!~?@#$%^&|`\"\\"
+    
+    @property
+    def token_separator_regex(self):
+        return "\s+|\(|\)|\[|\]|\{|\}|\.|,|;|:|\?|@|#|\$|%|\^|&|\||`|\"|\\\\" +\
+            ""#"|\+|-|\*|/|=|<|>|!|~"
+
+    @staticmethod
+    def get_token_separator_set():
+        return set(" ()[]{}.,;:+-*/=<>!~?@#$%^&|`\"\\")
+
+    @staticmethod
+    def get_token_separators():
+        return " ()[]{}.,;:+-*/=<>!~?@#$%^&|`\"\\"
+    
+    @staticmethod
+    def get_token_separator_regex():
+        return "\s+|\(|\)|\[|\]|\{|\}|\.|,|;|:|\?|@|#|\$|%|\^|&|\||`|\"|\\\\" +\
+            ""#"|\+|-|\*|/|=|<|>|!|~"
+    
+    def is_in_proof_mode(self):
+        return True if self.coq.proof_context else False
+    
+    def needs_qed(self):
+        return self.coq.proof_context is not None and len(self.coq.proof_context.all_goals) == 0
+    
+    def needs_cut_close(self):
+        return self.coq.proof_context is not None and len(self.coq.proof_context.fg_goals) == 0 and len(self.coq.proof_context.all_goals) > 0
+
+    def run_next(self) -> bool:
+        try:
+            stmt = next(self.main_file_iter)
+        except StopIteration:
+            self.execution_complete = True
+            return False
+        self.current_stmt = stmt
+        self.line_num += 1
+        try:
+            self.coq.run_stmt(stmt, timeout=self.timeout_in_sec)
+        except:
+            if not self.suppress_error_log:
+                logger.error(f"Got an exception while running '{stmt}' on coq. File name: {self.main_file}")
+                logger.exception(f"Exception Log")
+            raise
+        return True
+    
+    def get_tokens_in_given_stmt(self, stmt: str, ignore_first_token: bool = False) -> typing.Generator[str, None, None]:
+        idx = -1
+        for tok in re.split(self.token_separator_regex, stmt):
+            idx += 1
+            # skip the first token as it is usually a keyword
+            if idx == 0 and ignore_first_token:
+                continue
+            tok1 = tok.strip()
+            if len(tok1) > 0:
+                yield tok1
+    
+    def tokenize(stmt: str) -> typing.Generator[str, None, None]:
+        for tok in re.split(CoqExecutor.get_token_separator_regex(), stmt):
+            tok1 = tok.strip()
+            if len(tok1) > 0:
+                yield tok1
+
+    @functools.lru_cache(maxsize=10000)
+    def print_dfns(self, name: str) -> str:
+        if name in CoqExecutor.keywords:
+            return ""
+        return self.coq.print_symbols(name)
+
+    # Make this chacheable
+    @functools.lru_cache(maxsize=10000)
+    def search_type_matching_defns(self, name: str) -> typing.List[str]:
+        if name in CoqExecutor.keywords:
+            return []
+        return self.coq.search_about(name)
+    
+    def get_all_type_matching_defns(self, name: str, should_print_symbol: bool = False) -> typing.Generator[typing.Tuple[str, str], None, None]:
+        all_defns = self.search_type_matching_defns(name)
+        # Try for an exact match
+        for defn in all_defns:
+            defn = defn.split(":")
+            defn_name = defn[0].strip()
+            if len(defn) > 1:
+                if should_print_symbol:
+                    defn_val = self.print_dfns(defn_name).strip()
+                else:
+                    defn_val = ("".join(defn[1:])).strip()
+            else:
+                defn_val = ""
+            yield defn_name, defn_val
+
+    def search_exact(self, name: str, should_print_symbol: bool = False) -> typing.List[typing.Tuple[str, str]]:
+        symb_defn = self.search_type_matching_defns(name)
+        main_matches = []
+        match_until = set([name])
+        # Try for an exact match
+        for defn in symb_defn:
+            defn = defn.split(":")
+            defn_name = defn[0].strip()
+            if len(defn) > 1:
+                if should_print_symbol:
+                    defn_val = self.print_dfns(defn_name).strip()
+                else:                    
+                    defn_val = ("".join(defn[1:])).strip()
+                # print(f"should_print_symbol: {should_print_symbol}")
+                # print(defn_name)
+                # print(defn_val)
+            else:
+                defn_val = ""
+            if defn_name in match_until:
+                main_matches.append((defn_name, defn_val))
+                break
+        return main_matches
+
+    def search_defn(self, name: str, match_until: typing.Tuple[str], max_search_res: typing.Optional[int] = None, should_print_symbol: bool = False) -> typing.List[typing.Tuple[str, str, bool]]:
+        symb_defn = self.search_type_matching_defns(name)
+        match_defns = []
+        main_matches = []
+        match_until = set(match_until)
+        # Try for an exact match
+        for defn in symb_defn:
+            defn = defn.split(":")
+            defn_name = defn[0].strip()
+            if len(defn) > 1:
+                if should_print_symbol:
+                    defn_val = self.print_dfns(defn_name).strip()
+                else:
+                    defn_val = ("".join(defn[1:])).strip()
+            else:
+                defn_val = ""
+            if defn_name in match_until:
+                main_matches.append((defn_name, defn_val, True))
+            else:
+                match_defns.append((defn_name, defn_val, False))
+        if max_search_res is not None:
+            match_defns = random.sample(match_defns, max(0, min(max_search_res - 1, len(match_defns))))
+        match_defns.extend(main_matches)
+        return match_defns
+    
+    def run_without_executing(self, stmt: str):
+        while True:
+            try:
+                stmt = next(self.main_file_iter)
+            except StopIteration:
+                return
+            self.current_stmt = stmt
+            self.line_num += 1
+
+    def run_lemma_without_executing(self):
+        while True:
+            try:
+                stmt = next(self.main_file_iter)
+                self.current_stmt = stmt
+                self.line_num += 1
+                if "Qed." in stmt or "Defined." in stmt or "Admitted." in stmt:
+                    return True
+            except StopIteration:
+                return False
+    
+    def rewind_proof_steps(self) -> str:
+        # rewind the proof steps until the last lemma is found
+        current_lemma = None
+        while self.is_in_proof_mode():
+            if current_lemma is None:
+                current_lemma = self.coq.cur_lemma
+            # If we are already in proof mode, then we have already found a lemma
+            # should call run_to_finish_lemma instead
+            self.coq.cancel_last(force_update_nonfg_goals=True)
+            self.line_num -= 1
+        return "Theorem " + current_lemma if current_lemma is not None else None
+
+    def run_till_next_lemma(self) -> typing.Tuple[bool, typing.Optional[str]]:
+        # Run the coq file until the next lemma is found
+        next_stmt = None
+        in_proof_mode = self.is_in_proof_mode()
+        if in_proof_mode or self.execution_complete:
+            # If we are already in proof mode, then we have already found a lemma
+            return False, next_stmt
+        prev_stmt = self.current_stmt
+        ran_last_cmd = self.run_next()
+        next_stmt = self.current_stmt
+        if not ran_last_cmd:
+            return False, None
+        assigned = False
+        while ran_last_cmd and not in_proof_mode:
+            if not assigned:
+                prev_stmt = next_stmt
+            ran_last_cmd = self.run_next()
+            in_proof_mode = self.is_in_proof_mode()
+            if not assigned:
+                next_stmt = self.current_stmt
+                if in_proof_mode:
+                    assigned = True
+        lemma_name = next_stmt if next_stmt.startswith("Theorem") or next_stmt.startswith("Lemma") else prev_stmt
+        return in_proof_mode, lemma_name
+
+    def run_till_next_lemma_return_exec_stmt(self) -> typing.Generator[str, None, None]:
+        # Run the coq file until the next lemma is found
+        next_stmt = None
+        in_proof_mode = self.is_in_proof_mode()
+        if in_proof_mode or self.execution_complete:
+            # If we are already in proof mode, then we have already found a lemma
+            yield from []
+        else:
+            ran_last_cmd = self.run_next()
+            next_stmt = self.current_stmt
+            if not ran_last_cmd:
+                yield from []
+            else:
+                yield next_stmt
+            while ran_last_cmd and not in_proof_mode:
+                ran_last_cmd = self.run_next()
+                next_stmt = self.current_stmt
+                if ran_last_cmd:
+                    yield next_stmt
+                in_proof_mode = self.is_in_proof_mode()
+
+    def run_to_finish_lemma_return_exec(self) -> typing.Generator[str, None, None]:
+        # Run the coq file until the next lemma is found
+        next_stmt = None
+        in_proof_mode = self.is_in_proof_mode()
+        if not in_proof_mode or self.execution_complete:
+            # If we are already in proof mode, then we have already found a lemma
+            yield from []
+        else:
+            ran_last_cmd = self.run_next()
+            next_stmt = self.current_stmt
+            if not ran_last_cmd:
+                yield from []
+            else:
+                yield next_stmt
+            while ran_last_cmd and in_proof_mode:
+                ran_last_cmd = self.run_next()
+                next_stmt = self.current_stmt
+                if ran_last_cmd:
+                    yield next_stmt
+                in_proof_mode = self.is_in_proof_mode()
+
+    def run_to_finish_lemma(self) -> bool:
+        # Run the coq file and finish the current lemma
+        in_proof_mode = self.is_in_proof_mode()
+        if not in_proof_mode or self.execution_complete:
+            # If we are not in proof mode, then we are not finishing a lemma
+            return False
+        ran_last_cmd = self.run_next()
+        if not ran_last_cmd:
+            return False
+        while ran_last_cmd and in_proof_mode:
+            ran_last_cmd = self.run_next()
+            in_proof_mode = self.is_in_proof_mode()
+        return not in_proof_mode
+
+    def run_till_line_num(self, line_num: int):
+        assert line_num >= self.line_num
+        ran_last_cmd = True
+        while ran_last_cmd and self.line_num < line_num:
+            ran_last_cmd = self.run_next()
+        return self.line_num
+    
+    def run_to_finish(self):
+        ran_last_cmd = True
+        while ran_last_cmd:
+            ran_last_cmd = self.run_next()
+        
+    def get_lemma_name_if_running(self) -> typing.Optional[str]:
+        if not self.is_in_proof_mode():
+            return None
+        else:
+            try:
+                return self.coq.cur_lemma_name
+            except:
+                return None
+    
+    def get_lemma_stmt_if_running(self) -> typing.Optional[str]:
+        if not self.is_in_proof_mode():
+            return None
+        else:
+            try:
+                return self.coq.cur_lemma
+            except:
+                return None
+    
+    def get_current_lemma_name(self) -> typing.Optional[str]:
+        if not self.is_in_proof_mode():
+            return None
+        try:
+            return self.coq.cur_lemma_name
+        except:
+            return None
+    
+def get_all_lemmas_in_file(coq_executor: CoqExecutor, logger: logging.Logger) -> typing.List[str]:
+    lemmas_to_prove = []
+    while not coq_executor.execution_complete:
+        assert not coq_executor.is_in_proof_mode(), "main_executor must not be in proof mode"
+        _ = list(coq_executor.run_till_next_lemma_return_exec_stmt())
+        if coq_executor.execution_complete:
+            break
+        lemma_name = coq_executor.get_lemma_name_if_running()
+        if lemma_name is None:
+            _ = list(coq_executor.run_to_finish_lemma_return_exec())
+            if coq_executor.execution_complete:
+                break
+        else:
+            logger.info(f"Discovered lemma: {lemma_name}")
+            lemmas_to_prove.append(lemma_name)
+            coq_executor.run_to_finish_lemma()
+    return lemmas_to_prove
+
+class CoqStdInOutExecutor:
+    def __init__(self):
+        self.coq_stdin_reader = CoqStepByStepStdInReader()
+        self.coq_exec : CoqExecutor = CoqExecutor(
+            use_human_readable_proof_context=True, 
+            proof_step_iter=self.coq_stdin_reader.instruction_step_generator())
+    
+    def __enter__(self):
+        self.coq_exec.__enter__()
+        return self
+    
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.coq_exec.__exit__(exc_type, exc_value, traceback)
+    
+    def run_in_loop(self):
+        print("In> ", end="")
+        while True:
+            try:
+                cmd_ran = self.coq_exec.run_next()
+                if not cmd_ran:
+                    break
+                print(f"Coq> {self.coq_exec.current_stmt}")
+                print(f"{self.coq_exec.coq.proof_context}")
+                print("In> ", end="")
+            except:
+                pass
+            pass
+
+class CoqCustomFileExec:
+    def __init__(self, project_path: str, file_path: str, setup_cmds: typing.List[str] = []):
+        self.coq_stdin_reader = CoqLineByLineReader(file_path)
+        self.project_path = project_path
+        self.coq_exec : CoqExecutor = CoqExecutor(
+            project_root=self.project_path,
+            use_human_readable_proof_context=True, 
+            proof_step_iter=self.coq_stdin_reader.instruction_step_generator(),
+            setup_cmds=setup_cmds)
+    
+    def __enter__(self):
+        self.coq_exec.__enter__()
+        return self
+    
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.coq_exec.__exit__(exc_type, exc_value, traceback)
+    
+    def print_goals(self):
+        proof_context: ProofContext = self.coq_exec.coq.proof_context
+        if proof_context is None:
+            print("No goals to print")
+        else:
+            for idx, goal in enumerate(proof_context.all_goals):
+                print(f"Goal {idx + 1}:")
+                print(goal.goal)
+                if len(goal.hypotheses) > 0:
+                    print(f"Hypotheses: {idx + 1}")
+                    for hyp in goal.hypotheses:
+                        print(hyp)
+                print("-" * 20)
+        print('=' * 20)
+    
+    def run_in_loop(self):
+        print("In> Press 'Enter' for running next line and 'c' + 'Enter' to cancel the last command and 're-run'.", end="")
+        last_stmt = None
+        while True:
+            try:
+                opt = input()
+                if opt == "c" and last_stmt is not None:
+                    if self.coq_exec.is_in_proof_mode():
+                        print(f"Goals before cancelling")
+                        self.print_goals()
+                    else:
+                        print("No goals before cancelling")
+                    self.coq_exec.coq.cancel_last()
+                    if self.coq_exec.is_in_proof_mode():
+                        print(f"Goals after cancelling")
+                        self.print_goals()
+                    else:
+                        print("No goals after cancelling")
+                    print(f"Canceled last statement: {last_stmt}")
+                    print(f"Re-running: {last_stmt}")
+                    self.coq_exec.coq.run_stmt(last_stmt)
+                    print(f"Coq> Ran {last_stmt} again")
+                    continue
+                cmd_ran = self.coq_exec.run_next()
+                last_stmt = self.coq_exec.current_stmt
+                if self.coq_exec.is_in_proof_mode():
+                    print(f"Goals after running {last_stmt}")
+                    self.print_goals()
+                if not cmd_ran:
+                    break
+                print(f"Coq> {self.coq_exec.current_stmt}")
+                self.print_goals()
+                print("In> ", end="")
+            except Exception as e:
+                print(f"Got an exception: {e}")
+                pass
+            pass    
+
+if __name__ == "__main__":
+    logging.basicConfig(filename='coq_executor.log', filemode='w', level=logging.INFO)
+    # with CoqStdInOutExecutor() as coq_exec:
+    #     coq_exec.run_in_loop()
+    os.chdir(root_dir)
+    # with CoqCustomFileExec(".", "data/test/SimpleAlgebra.v") as coq_exec:
+    #     coq_exec.run_in_loop()
+    project_path = "<root>/data/repos/math-comp/"
+    # Get <root> from os.environ
+    project_path = project_path.replace("<root>/", os.environ.get("ROOT", "").trim('/') + "/")
+    file_path = os.path.join(project_path, "mathcomp/solvable/primitive_action.v")
+    setup_cmds = ['opam env --switch=MathComp --set-switch']
+    with CoqCustomFileExec(project_path, file_path, setup_cmds) as coq_exec:
+        coq_exec.run_in_loop()

itp_interface/tools/coq_local_data_generation_transform.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+
+import sys
+root_dir = f"{__file__.split('itp_interface')[0]}"
+if root_dir not in sys.path:
+    sys.path.append(root_dir)
+import typing
+import uuid
+import os
+from itp_interface.tools.coq_context_helper import CoqContextHelper
+from itp_interface.tools.coq_training_data_generator import GenericTrainingDataGenerationTransform, TrainingDataGenerationType
+from itp_interface.tools.training_data_format import Goal, MergableCollection, TrainingDataMetadataFormat, TrainingDataCollection, TrainingDataFormat
+from itp_interface.tools.coq_executor import CoqExecutor
+from itp_interface.tools.training_data import TrainingData
+
+class LocalDataGenerationTransform(GenericTrainingDataGenerationTransform):
+    def __init__(self,
+                depth = None,
+                max_search_results = None,
+                buffer_size : int = 10000,
+                logger = None,
+                max_parallelism : int = 4,
+                no_dfns : bool = False,
+                no_thms : bool = False):
+        super().__init__(TrainingDataGenerationType.LOCAL, buffer_size, logger)
+        self.depth = depth
+        self.max_search_results = max_search_results
+        self.max_parallelism = max_parallelism
+        self.no_dfns = no_dfns
+        self.no_thms = no_thms
+
+    def get_meta_object(self) -> MergableCollection:
+        return TrainingDataMetadataFormat(training_data_buffer_size=self.buffer_size)
+
+    def get_data_collection_object(self) -> MergableCollection:
+        return TrainingDataCollection()
+    
+    def load_meta_from_file(self, file_path) -> MergableCollection:
+        return TrainingDataMetadataFormat.load_from_file(file_path)
+    
+    def load_data_from_file(self, file_path) -> MergableCollection:
+        return TrainingDataCollection.load_from_file(file_path, self.logger)
+
+    def __call__(self, training_data: TrainingData, project_id : str, coq_executor: CoqExecutor, print_coq_executor_callback: typing.Callable[[], CoqExecutor], theorems: typing.List[str] = None, other_args: dict = {}) -> TrainingData:
+        print_coq_executor = print_coq_executor_callback()
+        coq_context_helper = CoqContextHelper(print_coq_executor, self.depth, self.logger)
+        coq_context_helper.__enter__()
+        file_namespace = coq_executor.main_file.replace('/', '.')
+        self.logger.info(f"=========================Processing {file_namespace}=========================")
+        proof_running = False
+        cmd_ran = coq_executor.run_next()
+        cmd_exec = coq_executor.current_stmt
+        prev_goal : typing.List[Goal] = coq_context_helper.get_focussed_goals(coq_executor) if coq_executor.is_in_proof_mode() else []
+        line_number = coq_executor.line_num
+        lemma_name = coq_executor.get_lemma_name_if_running()
+        if lemma_name is None:
+            lemma_name = "__NONE__"
+        theorem_id = str(uuid.uuid4())
+        proof_id = theorem_id # self.get_proof_id(theorem_id, file_namespace, line_number, lemma_name)
+        local_lemma_refs_cnt = 0
+        external_lemma_refs_cnt = 0
+        theorems = set(theorems) if theorems is not None else None
+        while cmd_ran:
+            if coq_executor.is_in_proof_mode() and lemma_name != "__NONE__" and (theorems is None or lemma_name in theorems):
+                proof_running = True
+                prev_goal : typing.List[Goal] = [Goal(goal.hypotheses, goal.goal) for goal in prev_goal]
+                next_goal : typing.List[Goal] = coq_context_helper.get_focussed_goals(coq_executor)
+                if len(prev_goal) > 0 and cmd_exec != "Proof.":
+                    training_data_format = TrainingDataFormat(
+                        proof_id=proof_id,
+                        all_useful_defns_theorems=[],
+                        start_goals=prev_goal,
+                        end_goals=next_goal,
+                        proof_steps=[cmd_exec],
+                        simplified_goals=[], 
+                        addition_state_info={},
+                        file_path=coq_executor.main_file,
+                        theorem_name=lemma_name,
+                        project_id=project_id)
+                    try:
+                        if not self.no_dfns:
+                            coq_context_helper.set_relevant_defns_in_training_data_point(training_data_format, print_coq_executor, self.logger)
+                        if not self.no_thms:
+                            coq_context_helper.set_local_thms_dfns(training_data_format, coq_executor, self.logger)
+                    except Exception:
+                        self.logger.warning(f"Ignoring error in getting useful defns for cmd: \"{cmd_exec}\"")
+                        self.logger.warning(f"Killing print coq executor")
+                        self.logger.exception("Exception occurred!!")
+                        try:
+                            coq_context_helper.__exit__(None, None, None)
+                        except:
+                            pass
+                        print_coq_executor = print_coq_executor_callback()
+                        coq_context_helper = CoqContextHelper(print_coq_executor, self.depth, self.logger)
+                        coq_context_helper.__enter__()
+                        self.logger.warning(f"Re-initialized print coq executor")
+                    assert len(training_data_format.proof_steps) > 0, f"Proof steps cannot be empty for {proof_id}"
+                    for goal in training_data_format.start_goals:
+                        lemma_cnt = len(training_data_format.all_useful_defns_theorems)
+                        assert all([0 <= lemma_ref.lemma_idx < lemma_cnt for lemma_ref in goal.used_theorems_local]), f"Invalid lemma idx in {proof_id}"
+                        assert all([0 <= lemma_ref.lemma_idx < lemma_cnt for lemma_ref in goal.used_theorems_external]), f"Invalid lemma idx in {proof_id}"
+                        assert all([0 <= lemma_ref.lemma_idx < lemma_cnt for lemma_ref in goal.possible_useful_theorems_external]), f"Invalid lemma idx in {proof_id}"
+                        assert all([0 <= lemma_ref.lemma_idx < lemma_cnt for lemma_ref in goal.possible_useful_theorems_local]), f"Invalid lemma idx in {proof_id}"
+                    training_data.merge(training_data_format)
+                    local_lemma_refs_cnt += sum([len(goal.used_theorems_local) for goal in training_data_format.start_goals])
+                    external_lemma_refs_cnt += sum([len(goal.used_theorems_external) for goal in training_data_format.start_goals])
+                prev_goal = next_goal
+            else:
+                prev_goal = []
+            cmd_ran = coq_executor.run_next()
+            cmd_exec = coq_executor.current_stmt
+            line_number = coq_executor.line_num
+            if proof_running and not coq_executor.is_in_proof_mode():
+                proof_running = False
+                self.logger.info(f"Finished processing lemma [{theorem_id}] {lemma_name}")
+                theorem_id = str(uuid.uuid4())
+            lemma_name = coq_executor.get_lemma_name_if_running()
+            if lemma_name is None:
+                lemma_name = "__NONE__"
+            proof_id = theorem_id # self.get_proof_id(theorem_id, file_namespace, line_number, lemma_name)
+            
+        self.logger.info(f"===============Finished processing {file_namespace}=====================")
+        try:
+            coq_context_helper.__exit__(None, None, None)
+        except:
+            pass
+
+
+if __name__ == "__main__":
+    import os
+    import logging
+    import time
+    os.chdir(root_dir)
+    project_dir = "data/test/coq/custom_group_theory/theories"
+    file_name = "data/test/coq/custom_group_theory/theories/grpthm.v"
+    output_path = ".log/local_data_generation_transform/data/"
+    log_path = ".log/local_data_generation_transform/log/"
+    project_id = project_dir.replace('/', '.')
+    os.makedirs(output_path, exist_ok=True)
+    os.makedirs(log_path, exist_ok=True)
+    time_str = time.strftime("%Y%m%d-%H%M%S")
+    log_file = f"{log_path}/local_data_generation_transform-{time_str}.log"
+    logging.basicConfig(filename=log_file, level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
+    logger = logging.getLogger(__name__)
+    def _print_coq_executor_callback():
+        search_coq_exec = CoqExecutor(project_dir, file_name, use_human_readable_proof_context=True, suppress_error_log=True)
+        search_coq_exec.__enter__()
+        return search_coq_exec
+    transform = LocalDataGenerationTransform(0, buffer_size=1000)
+    training_data = TrainingData(
+        output_path, 
+        "training_metadata.json",
+        training_meta=transform.get_meta_object(), 
+        logger=logger)
+    with CoqExecutor(project_dir, file_name, use_human_readable_proof_context=True, suppress_error_log=True) as coq_exec:
+        transform(training_data, project_id, coq_exec, _print_coq_executor_callback)
+    save_info = training_data.save()
+    logger.info(f"Saved training data to {save_info}")