itp-interface 1.0.0__py3-none-any.whl

itp_interface/main/filter_dataset.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+import sys
+root_dir = f"{__file__.split('itp_interface')[0]}"
+if root_dir not in sys.path:
+    sys.path.append(root_dir)
+import argparse
+import logging
+import time
+import os
+import typing
+import copy
+from itp_interface.tools.log_utils import setup_logger
+from itp_interface.tools.training_data import TrainingData, TrainingDataFormat
+
+def filter_keyword(tdf: TrainingDataFormat, keywords: typing.List[str]) -> bool:
+    """
+    Filter out the training data format if it contains any of the keywords
+    """
+    for keyword in keywords:
+        for proof_step in tdf.proof_steps:
+            if keyword in proof_step:
+                return True
+    return False
+
+def filter_dataset(dataset, metafilename, output, keywords, max_parallelism=8, logger=None):
+    """
+    filter dataset
+    """
+    assert os.path.exists(output), f"Output folder {output} does not exist"
+    assert max_parallelism > 0, "Max parallelism must be greater than 0"
+    logger = logger or logging.getLogger("dataset_filter")
+    training_data = TrainingData(
+        folder=dataset,
+        training_meta_filename=metafilename,
+        max_parallelism=max_parallelism
+    )
+    training_data.load_meta()
+    logger.info(f"Inited training data for {dataset}")
+    metadata = copy.deepcopy(training_data.meta)
+    logger.info(f"Start filtering datasets")
+    cloned_metadata = copy.deepcopy(metadata)
+    cloned_metadata.total_proof_step_cnt = 0
+    cloned_metadata.last_training_data = 0
+    logger.info(f"Cloned metadata:\n {cloned_metadata}")
+    filtered_td = TrainingData(
+        folder=output,
+        training_meta_filename="local.meta.json",
+        training_meta=cloned_metadata,
+        max_parallelism=max_parallelism,
+        logger=logger
+    )
+    # Load the training data and each proof step
+    logger.info(f"Start loading {dataset} ...")
+    training_data.load()
+    logger.info(f"Loaded {dataset}.")
+    logger.info(f"Start filtering {dataset} ...")
+    skipped_cnt = 0
+    total_data_points = 0
+    for tdf in training_data:
+        if total_data_points % 100 == 0 and total_data_points != 0:
+            logger.info(f"Skipped {skipped_cnt} data points.")
+        if filter_keyword(tdf, keywords):
+            skipped_cnt += 1
+        else:
+            filtered_td.merge(tdf)
+        total_data_points += 1
+    logger.info(f"In total, skipped {skipped_cnt} data points.")
+    logger.info(f"Total data points: {total_data_points}")
+    logger.info(f"Total proof steps: {metadata.total_proof_step_cnt}")
+    logger.info(f"Lenght of filtered dataset: {len(filtered_td)}")
+    logger.info(f"Length of training data: {len(training_data)}")
+    logger.info("Finished filtering dataset.")
+    logger.info(f"Start saving filtered dataset to {output} ...")
+    filtered_td.save()
+    logger.info(f"Saved filtered dataset to {output}.")
+    logger.info("Finished filtered datasets.")
+    logger.info("Verifying the filtered dataset ...")
+    new_merged_td = TrainingData(
+        folder=output,
+        training_meta_filename="local.meta.json",
+        max_parallelism=max_parallelism,
+        logger=logger
+    )
+    new_merged_td.load()
+    assert len(new_merged_td) == metadata.total_proof_step_cnt - skipped_cnt, "Filtered dataset is not correct"
+    logger.info("Verified the filtered dataset.")
+    pass
+
+
+if __name__ == "__main__":
+    args = argparse.ArgumentParser()
+    args.add_argument("--dataset", type=str, help="Dataset to filter")
+    args.add_argument("--output", type=str, help="Output file")
+    args.add_argument("--metafilename", type=str, help="metafilename for dataset", default="local.meta.json")
+    args.add_argument("--max_parallelism", type=int, help="Max parallelism", default=8)
+    args.add_argument("--filter_keywords", type=str, nargs="+", help="List of keywords to filter", default=None)
+    args = args.parse_args()
+    # Add root dir to python path
+    os.environ["PYTHONPATH"] = f"{root_dir}:{os.environ.get('PYTHONPATH', '')}"
+    os.makedirs(args.output, exist_ok=True)
+    time_str = time.strftime("%Y-%m-%d_%H-%M-%S")
+    log_folder = os.path.join(".log", "filter_dataset", time_str)
+    os.makedirs(log_folder, exist_ok=True)
+    logger = setup_logger("dataset_filter", os.path.join(log_folder, "filter.log"))
+    filter_dataset(args.dataset, args.metafilename, args.output, args.filter_keywords, args.max_parallelism, logger)
+    print(args)

itp_interface/main/install.py

@@ -0,0 +1,92 @@
+import os
+
+file_path = os.path.abspath(__file__)
+def install_itp_interface():
+    print("Installing itp_interface")
+    itp_dir = os.path.dirname(os.path.dirname(file_path))
+    tools_dir = os.path.join(itp_dir, "tools")
+    repl_dir = os.path.join(tools_dir, "repl")
+    assert os.path.exists(repl_dir), f"repl_dir: {repl_dir} does not exist"
+    assert os.path.exists(os.path.join(repl_dir, "lean-toolchain")), f"lean-toolchain does not exist in {repl_dir}, build has failed"
+    print("repl_dir: ", repl_dir)
+    print("Building itp_interface")
+    os.system(f"cd {repl_dir} && lake build repl")
+
+def install_lean_repl():
+    print("Updating Lean")
+    itp_dir = os.path.dirname(os.path.dirname(file_path))
+    tools_dir = os.path.join(itp_dir, "tools")
+    repl_dir = os.path.join(tools_dir, "repl")
+    assert os.path.exists(repl_dir), f"repl_dir: {repl_dir} does not exist"
+    assert os.path.exists(os.path.join(repl_dir, "lean-toolchain")), f"lean-toolchain does not exist in {repl_dir}, build has failed"
+    print("repl_dir: ", repl_dir)
+    assert os.system("git --version") == 0, "git is not installed"
+    print("[OK] git is installed")
+    print("Checking if Lean version is set in environment variables as LEAN_VERSION")
+    print("If not, defaulting to 4.7.0-rc2")
+    lean_version = os.environ.get("LEAN_VERSION", "4.7.0-rc2")
+    github_repo = "https://github.com/amit9oct/repl.git"
+    if lean_version.strip() == "4.7.0-rc2":
+        print("Lean version is set to 4.7.0-rc2, not cloning the REPL")
+    else:
+        # Clone the repl fresh
+        print("Cloning the REPL fresh")
+        os.system(f"rm -rf {repl_dir}")
+        os.system(f"git clone {github_repo} {repl_dir}")
+        # escape the version number
+        lean_version_esc = lean_version.replace(".", "\.")
+        print("Switching to the right REPL version", lean_version_esc)
+        cmd_to_run = f"cd {repl_dir} && git log --grep \"v{lean_version_esc}\" --pretty=\"%h %s\""
+        print("Running: ", cmd_to_run)
+        output = os.popen(cmd_to_run).read()
+        print("Output: ", output)
+        if output == "":
+            print(f"Could not find a commit with message containing {lean_version}")
+            print("Probably this version does not exist in the git history of the REPL")
+            lean_version = "4.7.0-rc2"
+            print("Switching to v4.7.0-rc2 on commit 97182f0")
+            os.system(f"cd {repl_dir} && git checkout 97182f0")
+        else:
+            # Split on first space
+            for line in output.split("\n"):
+                if line:
+                    commit, message = line.split(" ", 1)
+                    if lean_version in message:
+                        print(f"Switching to commit {commit}")
+                        os.system(f"cd {repl_dir} && git checkout {commit}")
+                        break
+    # Make sure that .elan is installed
+    print("Checking if .elan is installed")
+    if os.path.exists(os.path.expanduser("~/.elan")):
+        print("[OK] .elan is installed")
+    else:
+        print("Installing .elan")
+        elan_url = "https://raw.githubusercontent.com/leanprover/elan/master/elan-init.sh"
+        os.system(f"curl -sSL {elan_url} | sh")
+        print("[OK] .elan installed")
+        lean_repo = "leanprover-community/lean"
+        # Create a temporary script to run
+        shell_code = f"""
+source $HOME/.elan/env
+echo "Installing Lean 4 ({lean_repo}:{lean_version})..."
+elan toolchain install {lean_repo}:{lean_version}
+elan override set {lean_repo}:{lean_version}
+echo "Installed Lean 4 ({lean_repo}:{lean_version}) successfully!"
+export PATH=$PATH:$HOME/.elan/bin
+"""
+        random_string = os.urandom(8).hex()
+        with open(f"/tmp/{random_string}.sh", "w") as f:
+            f.write(shell_code)
+        os.system(f"chmod +x /tmp/{random_string}.sh")
+        print("Running the script")
+        os.system(f"bash /tmp/{random_string}.sh")
+        print("Removing the script")
+        os.system(f"rm /tmp/{random_string}.sh")
+        assert os.system("lean --version") == 0, "Lean 4 is not installed aborting"
+        print("[OK] Lean 4 installed successfully")
+    print("NOTE: Please run `install-itp-interface` to finish the installation")
+
+
+            
+    
+    

itp_interface/main/merge_dataset.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+import sys
+root_dir = f"{__file__.split('itp_interface')[0]}"
+if root_dir not in sys.path:
+    sys.path.append(root_dir)
+import argparse
+import logging
+import time
+import os
+import typing
+import copy
+from itp_interface.tools.log_utils import setup_logger
+from itp_interface.tools.training_data import TrainingData
+
+
+def merge_datasets(datasets, metafilenames, output, max_parallelism=8, logger=None):
+    """
+    Merge datasets
+    """
+    assert len(datasets) == len(metafilenames), "Length of datasets and metafilenames must be the same"
+    assert len(datasets) > 1, "At least 2 datasets are required to merge"
+    assert os.path.exists(output), f"Output folder {output} does not exist"
+    assert max_parallelism > 0, "Max parallelism must be greater than 0"
+    logger = logger or logging.getLogger("dataset_merge")
+    tds : typing.List[TrainingData] = []
+    for dataset, metafilename in zip(datasets, metafilenames):
+        training_data = TrainingData(
+            folder=dataset,
+            training_meta_filename=metafilename,
+            max_parallelism=max_parallelism
+        )
+        training_data.load_meta()
+        tds.append(training_data)
+        logger.info(f"Inited training data for {dataset}")
+    metadata = copy.deepcopy(tds[-1].meta)
+    for td in tds[:-1]:
+        metadata.total_proof_step_cnt += td.meta.total_proof_step_cnt
+        metadata.last_training_data += td.meta.last_training_data
+    logger.info(f"Merged metadata:\n {metadata}")
+    logger.info(f"Start merging datasets")
+    cloned_metadata = copy.deepcopy(metadata)
+    cloned_metadata.total_proof_step_cnt = 0
+    cloned_metadata.last_training_data = 0
+    merged_td = TrainingData(
+        folder=output,
+        training_meta_filename="local.meta.json",
+        training_meta=cloned_metadata,
+        max_parallelism=max_parallelism,
+        logger=logger
+    )
+    for td in tds:
+        logger.info(f"Start loading {td.folder} ...")
+        td.load()
+        logger.info(f"Loaded {td.folder}.")
+        logger.info(f"Start merging {td.folder} ...")
+        merged_td.merge(td)
+        logger.info(f"Merged {td.folder}")
+    logger.info("Finished merging datasets.")
+    logger.info(f"Start saving merged dataset to {output} ...")
+    merged_td.save()
+    logger.info(f"Saved merged dataset to {output}.")
+    logger.info("Finished merging datasets.")
+    logger.info("Verify the merged dataset ...")
+    new_merged_td = TrainingData(
+        folder=output,
+        training_meta_filename="local.meta.json",
+        max_parallelism=max_parallelism,
+        logger=logger
+    )
+    new_merged_td.load()
+    assert len(new_merged_td) == metadata.total_proof_step_cnt, "Merged dataset is not correct"
+    assert new_merged_td.meta.last_training_data == metadata.last_training_data, "Merged dataset is not correct"
+    assert new_merged_td.meta.last_proof_id == metadata.last_proof_id, "Merged dataset is not correct"
+    logger.info("Merged dataset is correct.")
+    pass
+
+
+if __name__ == "__main__":
+    args = argparse.ArgumentParser()
+    args.add_argument("--datasets", type=str, nargs="+", help="List of datasets to merge")
+    args.add_argument("--output", type=str, help="Output file")
+    args.add_argument("--metafilenames", type=str, nargs="+", help="List of metafilenames", default=None)
+    args.add_argument("--max_parallelism", type=int, help="Max parallelism", default=8)
+    args = args.parse_args()
+    # Add root dir to python path
+    os.environ["PYTHONPATH"] = f"{root_dir}:{os.environ.get('PYTHONPATH', '')}"
+    os.makedirs(args.output, exist_ok=True)
+    time_str = time.strftime("%Y-%m-%d_%H-%M-%S")
+    log_folder = os.path.join(".log", "merge_dataset", time_str)
+    os.makedirs(log_folder, exist_ok=True)
+    logger = setup_logger("dataset_merge", os.path.join(log_folder, "merge.log"))
+    if args.metafilenames is None:
+        args.metafilenames = ['local.meta.json' for _ in range(len(args.datasets))]
+    merge_datasets(args.datasets, args.metafilenames, args.output, args.max_parallelism, logger)
+    print(args)