itp-interface 1.0.0__py3-none-any.whl

itp_interface/tools/coq_parse_utils.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+root_dir = f"{__file__.split('itp_interface')[0]}"
+if root_dir not in sys.path:
+    sys.path.append(root_dir)
+import typing
+
+class CoqLineByLineReader:
+    def __init__(self, file_name: str = None, file_content: str = None):
+        assert file_name is not None or file_content is not None, "Either file_name or file_content must be provided"
+        assert file_name is None or file_content is None, "Only one of file_name or file_content must be provided"
+        self.file_name : str = file_name
+        self.file_content : str = file_content
+        if self.file_name is not None:
+            with open(file_name, 'r') as fd:
+                self.file_content : str = fd.read()
+        self.remove_comments()
+    
+    def remove_comments(self):
+        CODE_MODE = 0
+        COMMENT_MODE = 1
+        idx = 0
+        mode = CODE_MODE
+        code_without_comments = []
+        comment_stack = []
+        while idx < len(self.file_content):
+            ch = self.file_content[idx]
+            if mode == CODE_MODE and ch == '(' and idx + 1 < len(self.file_content) and self.file_content[idx + 1] == '*':
+                mode = COMMENT_MODE
+                idx += 2
+                comment_stack.append("(*")
+            elif mode == COMMENT_MODE and ch == '(' and idx + 1 < len(self.file_content) and self.file_content[idx + 1] == '*':
+                mode = COMMENT_MODE
+                idx += 2
+                comment_stack.append("(*") # Start of nested comment
+            elif mode == COMMENT_MODE and ch == '*' and idx + 1 < len(self.file_content) and self.file_content[idx + 1] == ')':
+                comment_stack.pop()
+                if len(comment_stack) == 0:
+                    mode = CODE_MODE
+                else:
+                    assert comment_stack[-1] == "(*", "Comment stack should only contain '(*'"
+                    mode = COMMENT_MODE # There is some nested comment still open
+                idx += 2
+            elif mode == COMMENT_MODE:
+                mode = COMMENT_MODE
+                idx += 1
+            elif mode == CODE_MODE:
+                code_without_comments.append(ch)
+                idx += 1
+            else:
+                raise Exception("Unknown mode")
+        self.file_content = "".join(code_without_comments)
+
+    def instruction_step_generator(self) -> typing.Iterator[str]:
+        CODE_MODE = 0
+        CODE_QT_MODE = 1
+        all_code = self.file_content
+        idx = 0
+        mode = CODE_MODE
+        instr_start = 0
+        special_context_starting_chars = ["+", "-", "*", "{", "}"]
+        while idx < len(all_code):
+            ch = all_code[idx]
+            if mode == CODE_MODE and ch == '.' and idx + 1 < len(all_code) and len(all_code[idx + 1].strip()) == 0:
+                code_to_yield = all_code[instr_start: idx + 1].strip()
+                contexts = []
+                context = ""
+                while any([code_to_yield.startswith(ctx) for ctx in special_context_starting_chars]):
+                    context += code_to_yield[0]
+                    if len(context) >= 2 and context[-2] != context[-1]:
+                        # Split the context
+                        contexts.append(context[:-1])
+                        context = context[-1]
+                    code_to_yield = code_to_yield[1:].strip()
+                    if len(code_to_yield) == 0:
+                        break
+                if len(context) > 0:
+                    contexts.append(context) # Add the last context if it was missed
+                for ctx in contexts:
+                    yield ctx.strip()
+                code_to_yield = code_to_yield.strip()
+                if len(code_to_yield) > 0:
+                    yield code_to_yield
+                idx += 1
+                mode = CODE_MODE
+                instr_start = idx
+            elif mode == CODE_MODE and ch == '.' and idx + 1 == len(all_code):
+                code_to_yield = all_code[instr_start: idx + 1].strip()
+                contexts = []
+                context = ""
+                while any([code_to_yield.startswith(ctx) for ctx in special_context_starting_chars]):
+                    context += code_to_yield[0]
+                    if len(context) >= 2 and context[-2] != context[-1]:
+                        # Split the context
+                        contexts.append(context[:-1])
+                        context = context[-1]
+                    code_to_yield = code_to_yield[1:].strip()
+                    if len(code_to_yield) == 0:
+                        break
+                code_to_yield = code_to_yield.strip()
+                if len(context) > 0:
+                    contexts.append(context) # Add the last context if it was missed
+                for ctx in contexts:
+                    yield ctx.strip()
+                if len(code_to_yield) > 0:
+                    yield code_to_yield
+                idx += 1
+                mode = CODE_MODE
+                instr_start = idx
+            elif mode == CODE_MODE and ch == "\"":
+                idx += 1
+                mode = CODE_QT_MODE
+            elif mode == CODE_QT_MODE and ch == "\"" and idx + 1 < len(all_code) and all_code[idx + 1] == "\"":
+                idx += 1
+                mode = CODE_QT_MODE
+            elif mode == CODE_QT_MODE and ch == "\"":
+                idx += 1
+                mode = CODE_MODE
+            elif mode == CODE_MODE:
+                mode = CODE_MODE
+                idx += 1
+            elif mode == CODE_QT_MODE:
+                mode = CODE_QT_MODE
+                idx += 1
+                # instruction start doesn't change
+            else:
+                raise Exception("This case is not possible")
+        if instr_start < len(all_code):
+            yield all_code[instr_start:].strip()
+
+class CoqStepByStepStdInReader:
+    def __init__(self):
+        pass
+
+    def instruction_step_generator(self) -> typing.Iterator[str]:
+        while True:
+            try:
+                line = input()
+                if line == "(* exit *)":
+                    break
+                yield line.strip() # Keep going line by line
+            except EOFError:
+                break
+
+if __name__ == "__main__":
+    file_content = """
+{reflexivity. }
+"""
+    coq_reader = CoqLineByLineReader(file_content=file_content)
+    assert coq_reader.file_content is not None
+    for idx, instruction in enumerate(coq_reader.instruction_step_generator()):
+        print(f"[{idx}]: {instruction}")

itp_interface/tools/coq_raw_proofs.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+
+import sys
+
+root_dir = f"{__file__.split('itp_interface')[0]}"
+if root_dir not in sys.path:
+    sys.path.append(root_dir)
+import logging
+import os
+import typing
+import ray
+from itp_interface.tools.ray_utils import RayUtils
+from itp_interface.tools.coq_parse_utils import CoqLineByLineReader
+from itp_interface.tools.training_data_format import Goal, TrainingDataFormat, TrainingDataMetadataFormat
+from itp_interface.tools.training_data import TrainingData
+from itp_interface.tools.proof_exec_callback import ProofExecutorCallback
+from itp_interface.tools.coq_build_tool import CoqRepoBuilder
+
+def extract_raw_proof(file_content: str, lemma_name: str) -> TrainingDataFormat:
+    # Find the first occurence of the lemma_name and then find the first occurence of "Qed."
+    # The proof is the text between the two
+    start_index = file_content.find(lemma_name)
+    assert start_index != -1, f"Lemma {lemma_name} not found in file"
+    end_index = file_content.find("Qed.", start_index)
+    assert end_index != -1, f"Lemma {lemma_name} does not have a proof"
+    lemma_proof = file_content[start_index:end_index + len("Qed.")]
+    line_reader = CoqLineByLineReader(file_content=lemma_proof)
+    # Just read the first line
+    statement = next(line_reader.instruction_step_generator())
+    # Find the next occurence of '.' after the statement which is not in a comment or quotation
+    next_dot_index = lemma_proof.find(".", len(statement) - 1) # This may not always work but it is good enough for now
+    assert next_dot_index != -1, f"Lemma {lemma_name} does not have a statement"
+    proof = lemma_proof[next_dot_index + 1:]
+    goal = Goal(goal=statement)
+    training_data = TrainingDataFormat(start_goals=[goal], end_goals=[], proof_steps=[proof])
+    return training_data
+
+@ray.remote
+def get_proofs_in_file(coq_proof_exec_callback: ProofExecutorCallback, file_content: str, file_path: str) -> typing.List[TrainingDataFormat]:
+    training_data_collection : typing.List[TrainingDataFormat] = []
+    failures = []
+    with coq_proof_exec_callback.get_proof_executor() as main_executor:
+        while not main_executor.execution_complete:
+            assert not main_executor.is_in_proof_mode(), "main_executor must not be in proof mode"
+            _ = list(main_executor.run_till_next_lemma_return_exec_stmt())
+            if main_executor.execution_complete:
+                break
+            lemma_name = main_executor.get_lemma_name_if_running()
+            if lemma_name is None:
+                _ = list(main_executor.run_to_finish_lemma_return_exec())
+
+                if main_executor.execution_complete:
+                    break
+            else:
+                try:
+                    tdf = extract_raw_proof(file_content, lemma_name)
+                    training_data_collection.append(tdf)
+                    main_executor.run_to_finish_lemma()
+                except Exception:
+                    print(f"Failed to extract proof for lemma {lemma_name} in {file_path}")
+                    _ = list(main_executor.run_to_finish_lemma_return_exec())
+                    failures.append(lemma_name)
+                    continue
+    return file_path, training_data_collection, failures
+
+def dump_raw_proofs(repo_builder: CoqRepoBuilder, output_dir: str, logger: logging.Logger):
+    assert os.path.exists(output_dir), f"output_dir {output_dir} does not exist"
+    RayUtils.init_ray(num_of_cpus=20, object_store_memory_in_gb=100, memory_in_gb=1)
+    projects = repo_builder.compilable_projects
+
+    def _log_proof_cnt(file_type: str, proof_cnt_in_file: int, file_path: str, failures: typing.List[str]):
+        logger.info(f"==============================>[{file_type}] Added proofs {proof_cnt_in_file} in {file_path} <==============================")
+        if len(failures) > 0:
+            logger.info(f"==============================>[{file_type}] Failed to extract proofs for {len(failures)} lemmas in {file_path} <==============================")
+            for lemma_name in failures:
+                logger.info(f"==============================>[{file_type}] Failed to extract proof for lemma {lemma_name} in {file_path} <==============================")
+    
+    def _get_next_files_callback(files):
+        _idx = 0
+        sorted_files = sorted(files)
+        def _get_files(cnt: int):
+            nonlocal _idx
+            file_input = []
+            for file_path in sorted_files[_idx:_idx + cnt]:
+                if file_path.startswith(project_path):
+                    with open(file_path, 'r') as fd:
+                        file_content = fd.read()
+                    coq_proof_exec_callback = ProofExecutorCallback(
+                        project_folder=project_path,
+                        file_path=file_path,
+                        use_hammer=False,
+                        timeout_in_secs=60,
+                        use_human_readable_proof_context=True,
+                        suppress_error_log=True,
+                        logger=logger)
+                    file_input.append((coq_proof_exec_callback, file_content, file_path))
+            _idx += cnt
+            return file_input
+        return _get_files
+    
+    def _get_file_op_remotes_callback(file_inputs):
+        return [get_proofs_in_file.remote(coq_proof_exec_callback, file_content, file_path) for coq_proof_exec_callback, file_content, file_path in file_inputs]
+    
+    def _transform_outputs_callback(training_data: TrainingData, file_type: str):
+        def _transform_outputs(results: typing.List[typing.List[TrainingDataFormat]]):
+            for file_path, training_data_collection, failures in results:
+                for tdf in training_data_collection:
+                    training_data.merge(tdf)
+                _log_proof_cnt(file_type, len(training_data_collection), file_path, failures)
+        return _transform_outputs
+
+    for project in sorted(projects):
+        project_proof_cnt = 0
+        # Create temporary directory for each project
+        logger.info(f"==============================> Discovering files in project {project}<==============================")
+        project_path = os.path.join(repo_builder.root, project)
+        assert os.path.exists(project_path), f"project_path {project_path} does not exist"
+        files_type_map = {
+            "train": repo_builder.train_compilable_files,
+            "test": repo_builder.test_compilable_files,
+        }
+        for file_type, files in files_type_map.items():
+            output_path = os.path.join(output_dir, project, file_type)
+            os.makedirs(output_path, exist_ok=True)
+            training_metadata = TrainingDataMetadataFormat(training_data_buffer_size=2500, data_filename_prefix="single_data_", lemma_ref_filename_prefix="single_lemma_ref_")
+            training_data = TrainingData(output_path, "single.meta.json", training_metadata)
+            RayUtils.ray_run_within_parallel_limits(
+                10,
+                len(files),
+                _transform_outputs_callback(training_data, file_type),
+                _get_next_files_callback(files),
+                _get_file_op_remotes_callback,
+                logger=logger,
+                turn_off_logging=True)
+            logger.info(f"[{file_type}] Saving training data to {output_path}")
+            save_folder = training_data.save()
+            logger.info(f"[{file_type}] Saved training data to {save_folder}")
+            project_proof_cnt += len(training_data)
+            logger.info(f"==============================>[{file_type}] Added all  {len(training_data)} proofs in project {project}<==============================")
+        logger.info(f"==============================> Added {project_proof_cnt} proofs in project {project}<==============================")
+    logger.info(f"==============================> Added proofs in all projects<==============================")
+
+def count_proofs(repo_builder: CoqRepoBuilder, logger: logging.Logger):
+    proof_cnt = 0
+    projects = repo_builder.compilable_projects
+    for project in sorted(projects):
+        project_proof_cnt = 0
+        # Create temporary directory for each project
+        logger.info(f"==============================> Discovering files in project {project}<==============================")
+        project_path = os.path.join(repo_builder.root, project)
+        assert os.path.exists(project_path), f"project_path {project_path} does not exist"
+        files_type_map = {
+            "train": repo_builder.train_compilable_files,
+            "test": repo_builder.test_compilable_files,
+        }
+        for file_type, files in files_type_map.items():
+            file_type_cnt = 0
+            for file_path in sorted(files):
+                current_proof_cnt = 0
+                if file_path.startswith(project_path):
+                    with open(file_path, 'r') as fd:
+                        file_content = fd.read()
+                    current_proof_cnt = file_content.count("Qed.")
+                logger.info(f"==============================>[{file_type}] Found {current_proof_cnt} proofs in file {file_path}<==============================")
+                proof_cnt += current_proof_cnt
+                file_type_cnt += current_proof_cnt
+                project_proof_cnt += current_proof_cnt
+            logger.info(f"==============================>[{file_type}] Found {file_type_cnt} proofs in project {project}<==============================")
+        logger.info(f"==============================> Found {project_proof_cnt} proofs in project {project}<==============================")
+    logger.info(f"==============================> Found {proof_cnt} proofs in all projects<==============================")
+
+if __name__ == "__main__":
+    import time
+    import argparse
+    os.chdir(root_dir)
+    current_time = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
+    logging_dir = f".log/tools/coq_raw_proofs/{current_time}"
+    os.makedirs(logging_dir, exist_ok=True)
+    log_file = f"{os.path.join(logging_dir, f'coq_raw_proofs.log')}"
+    logging.basicConfig(filename=log_file, filemode='w', level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    logger = logging.getLogger("ProofCounter")
+    logger.info(f"Process ID: {os.getpid()}")
+    repo_builder = CoqRepoBuilder(root_dir)
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--info_file", type=str, default="data/test/coq/custom_group_theory/compilable_projects_info.log.json", help="The file which will have the info about the compilable projects")
+    parser.add_argument("--output_dir", type=str, default=".log/tools/coq_raw_proofs/data/test/coq/custom_group_theory", help="The directory where the raw proofs will be dumped")
+    args = parser.parse_args()
+    info_file = args.info_file
+    repo_builder : CoqRepoBuilder = CoqRepoBuilder.load_from_file(info_file)
+    output_dir = os.path.join(args.output_dir, current_time)
+    os.makedirs(output_dir, exist_ok=True)
+    # count_proofs(repo_builder, logger)
+    dump_raw_proofs(repo_builder, output_dir, logger)

itp_interface/tools/coq_theorem_proof_pair_generation_transform.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+
+import sys
+
+root_dir = f"{__file__.split('itp_interface')[0]}"
+if root_dir not in sys.path:
+    sys.path.append(root_dir)
+
+import typing
+from itp_interface.tools.coq_context_helper import CoqContextHelper
+from itp_interface.tools.coq_training_data_generator import GenericTrainingDataGenerationTransform, TrainingDataGenerationType
+from itp_interface.tools.training_data_format import Goal, MergableCollection, TrainingDataMetadataFormat, TrainingDataCollection, TrainingDataFormat
+from itp_interface.tools.coq_executor import CoqExecutor
+from itp_interface.tools.training_data import TrainingData
+
+class LocalTheoremProofPairGenerationTransform(GenericTrainingDataGenerationTransform):
+    def __init__(self,
+                depth = None,
+                max_search_results = None,
+                buffer_size : int = 10000,
+                logger = None,
+                max_parallelism : int = 4):
+        super().__init__(TrainingDataGenerationType.LOCAL, buffer_size, logger)
+        self.depth = depth
+        self.max_search_results = max_search_results
+        self.max_parallelism = max_parallelism
+
+    def get_meta_object(self) -> MergableCollection:
+        return TrainingDataMetadataFormat(training_data_buffer_size=self.buffer_size)
+
+    def get_data_collection_object(self) -> MergableCollection:
+        return TrainingDataCollection()
+    
+    def load_meta_from_file(self, file_path) -> MergableCollection:
+        return TrainingDataMetadataFormat.load_from_file(file_path)
+    
+    def load_data_from_file(self, file_path) -> MergableCollection:
+        return TrainingDataCollection.load_from_file(file_path, self.logger)
+
+    def __call__(self, training_data: TrainingData, project_id : str, coq_executor: CoqExecutor, print_coq_executor_callback: typing.Callable[[], CoqExecutor]) -> TrainingData:
+        print_coq_executor = print_coq_executor_callback()
+        coq_context_helper = CoqContextHelper(print_coq_executor, self.depth, self.logger)
+        coq_context_helper.__enter__()
+        file_namespace = coq_executor.main_file.replace('/', '.')
+        self.logger.info(f"=========================Processing {file_namespace}=========================")
+        proof_running = False
+        cmd_ran = coq_executor.run_next()
+        cmd_exec = coq_executor.current_stmt
+        prev_goal : typing.List[Goal] = coq_context_helper.get_focussed_goals(coq_executor) if coq_executor.is_in_proof_mode() else []
+        line_number = coq_executor.line_num
+        lemma_name = coq_executor.get_lemma_name_if_running()
+        if lemma_name is None:
+            lemma_name = "__NONE__"
+        proof_id = self.get_proof_id(project_id, file_namespace, line_number, lemma_name)
+        local_lemma_refs_cnt = 0
+        external_lemma_refs_cnt = 0
+        while cmd_ran:
+            if coq_executor.is_in_proof_mode() and lemma_name != "__NONE__":
+                proof_running = True
+                prev_goal : typing.List[Goal] = [Goal(goal.hypotheses, goal.goal) for goal in prev_goal]
+                next_goal : typing.List[Goal] = coq_context_helper.get_focussed_goals(coq_executor)
+                if len(prev_goal) > 0 and cmd_exec != "Proof.":
+                    training_data_format = TrainingDataFormat(
+                        proof_id=proof_id,
+                        all_useful_defns_theorems=[],
+                        start_goals=prev_goal,
+                        end_goals=next_goal,
+                        proof_steps=[cmd_exec],
+                        simplified_goals=[], 
+                        addition_state_info={})
+                    try:
+                        coq_context_helper.set_relevant_defns_in_training_data_point(training_data_format, print_coq_executor, self.logger)
+                        coq_context_helper.set_local_thms_dfns(training_data_format, coq_executor, self.logger)
+                    except Exception:
+                        self.logger.warning(f"Ignoring error in getting useful defns for cmd: \"{cmd_exec}\"")
+                        self.logger.warning(f"Killing print coq executor")
+                        self.logger.exception("Exception occurred!!")
+                        try:
+                            coq_context_helper.__exit__(None, None, None)
+                        except:
+                            pass
+                        print_coq_executor = print_coq_executor_callback()
+                        coq_context_helper = CoqContextHelper(print_coq_executor, self.depth, self.logger)
+                        coq_context_helper.__enter__()
+                        self.logger.warning(f"Re-initialized print coq executor")
+                    assert len(training_data_format.proof_steps) > 0, f"Proof steps cannot be empty for {proof_id}"
+                    for goal in training_data_format.start_goals:
+                        lemma_cnt = len(training_data_format.all_useful_defns_theorems)
+                        assert all([0 <= lemma_ref.lemma_idx < lemma_cnt for lemma_ref in goal.used_theorems_local]), f"Invalid lemma idx in {proof_id}"
+                        assert all([0 <= lemma_ref.lemma_idx < lemma_cnt for lemma_ref in goal.used_theorems_external]), f"Invalid lemma idx in {proof_id}"
+                        assert all([0 <= lemma_ref.lemma_idx < lemma_cnt for lemma_ref in goal.possible_useful_theorems_external]), f"Invalid lemma idx in {proof_id}"
+                        assert all([0 <= lemma_ref.lemma_idx < lemma_cnt for lemma_ref in goal.possible_useful_theorems_local]), f"Invalid lemma idx in {proof_id}"
+                    training_data.merge(training_data_format)
+                    local_lemma_refs_cnt += sum([len(goal.used_theorems_local) for goal in training_data_format.start_goals])
+                    external_lemma_refs_cnt += sum([len(goal.used_theorems_external) for goal in training_data_format.start_goals])
+                prev_goal = next_goal
+            else:
+                prev_goal = []
+            cmd_ran = coq_executor.run_next()
+            cmd_exec = coq_executor.current_stmt
+            line_number = coq_executor.line_num
+            if proof_running and not coq_executor.is_in_proof_mode():
+                proof_running = False
+                self.logger.info(f"Finished processing lemma {lemma_name}")
+            lemma_name = coq_executor.get_lemma_name_if_running()
+            if lemma_name is None:
+                lemma_name = "__NONE__"
+            proof_id = self.get_proof_id(project_id, file_namespace, line_number, lemma_name)
+            
+        self.logger.info(f"===============Finished processing {file_namespace}=====================")
+        try:
+            coq_context_helper.__exit__(None, None, None)
+        except:
+            pass
+
+
+if __name__ == "__main__":
+    import os
+    import logging
+    import time
+    os.chdir(root_dir)
+    project_dir = "data/test/coq/custom_group_theory/theories"
+    file_name = "data/test/coq/custom_group_theory/theories/grpthm.v"
+    output_path = ".log/local_data_generation_transform/data/"
+    log_path = ".log/local_data_generation_transform/log/"
+    project_id = project_dir.replace('/', '.')
+    os.makedirs(output_path, exist_ok=True)
+    os.makedirs(log_path, exist_ok=True)
+    time_str = time.strftime("%Y%m%d-%H%M%S")
+    log_file = f"{log_path}/local_data_generation_transform-{time_str}.log"
+    logging.basicConfig(filename=log_file, level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
+    logger = logging.getLogger(__name__)
+    def _print_coq_executor_callback():
+        search_coq_exec = CoqExecutor(project_dir, file_name, use_human_readable_proof_context=True, suppress_error_log=True)
+        search_coq_exec.__enter__()
+        return search_coq_exec
+    transform = LocalTheoremProofPairGenerationTransform(0, buffer_size=1000)
+    training_data = TrainingData(
+        output_path, 
+        "training_metadata.json",
+        training_meta=transform.get_meta_object(), 
+        logger=logger)
+    with CoqExecutor(project_dir, file_name, use_human_readable_proof_context=True, suppress_error_log=True) as coq_exec:
+        transform(training_data, project_id, coq_exec, _print_coq_executor_callback)
+    save_info = training_data.save()
+    logger.info(f"Saved training data to {save_info}")

itp_interface/tools/coq_training_data_generator.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+import sys
+root_dir = f"{__file__.split('itp_interface')[0]}"
+if root_dir not in sys.path:
+    sys.path.append(root_dir)
+
+import typing
+import logging
+import enum
+from itp_interface.tools.training_data_format import MergableCollection, TrainingDataFormat
+from torch.multiprocessing import set_start_method
+try:
+     set_start_method('spawn')
+except RuntimeError:
+    pass
+logger = logging.getLogger("CoqTrainingGenerator")
+
+class TrainingDataGenerationType(enum.Enum):
+    FULL = 0
+    RETRIEVER = 1
+    GENERATION_BM25 = 2
+    GENERATION_DPR = 3
+    TEXT = 4
+    LOCAL = 5
+
+class GenericTheoremRetriever:
+    def __init__(self):
+        self.logger = None
+        pass
+
+    def filter_best_context(self, partial_data: TrainingDataFormat) -> TrainingDataFormat:
+        raise NotImplementedError("retrieve_best_context must be implemented")
+
+    def set_logger(self, logger: logging.Logger):
+        self.logger = logger
+
+class GenericTrainingDataGenerationTransform(object):
+
+    """Class to generate training data for for coq based automatic theorem provers.
+
+    This class is responsible for generating training data for coq based automatic theorem provers.
+    NOTE: This class is not thread safe!!
+    """
+    def __init__(self,
+                 training_data_type: TrainingDataGenerationType,
+                 buffer_size : int = 10000,
+                 logger = None):
+        """Initialize the training data generator.
+        """
+        assert buffer_size > 0, "Buffer size must be greater than 0"
+        self.buffer_size = buffer_size
+        self.logger : logging.Logger = logger if logger is not None else logging.getLogger()
+        self.training_data_generation_type : TrainingDataGenerationType = training_data_type
+
+    def get_meta_object(self) -> MergableCollection:
+        raise NotImplementedError("get_meta_object must be implemented")
+
+    def get_data_collection_object(self) -> MergableCollection:
+        raise NotImplementedError("get_data_collection_object must be implemented")
+    
+    def load_meta_from_file(self, file_path) -> MergableCollection:
+        raise NotImplementedError("load_meta_from_file must be implemented")
+    
+    def load_data_from_file(self, file_path) -> MergableCollection:
+        raise NotImplementedError("load_data_from_file must be implemented")
+    
+    def get_proof_id(self, project_id: str, file_namespace: str, line_number: int, lemma_name: str) -> str:
+        return f"<{project_id}>.{file_namespace}.{line_number:06d}.{lemma_name}"
+    
+    def __call__(self, *args: typing.Any, **kwds: typing.Any) -> typing.Any:
+        raise NotImplementedError("__call__ must be implemented")
+    
+    @property
+    def name(self):
+        return self.training_data_generation_type.name.lower()