itp-interface 1.0.0__py3-none-any.whl

itp_interface/pisa/command_generation/conjecturer_command_generator.py

@@ -0,0 +1,36 @@
+number_of_processes = input("Enter the number of processes you want to run at the same time:\n").strip()
+number_of_processes = int(number_of_processes)
+import glob
+import os
+script = 'echo "y" | sbt "runMain pisa.server.ConjecturerParser {} {} {} {}"'
+
+total_cmds = list()
+total_files = 0
+for file_name in glob.glob("/home/ywu/afp-2021-02-11/thys/**/*.thy", recursive=True):
+    project_single_name = file_name.split("/")[5]
+    if not os.path.isdir(os.path.join("conjecturer_extractions", project_single_name)):
+        os.makedirs(os.path.join("conjecturer_extractions", project_single_name))
+
+    theory_single_name = file_name.split("/")[-1].split(".thy")[0]
+    if os.path.isfile(os.path.join("conjecturer_extractions", project_single_name, theory_single_name+".src")):
+        continue
+    thys_index = file_name.split("/").index("thys")
+    working_directory = "/".join(file_name.split("/")[:thys_index+2])
+    total_cmds.append(script.format(file_name,
+                                    working_directory,
+                                    "/home/ywu/Isabelle2020",
+                                    "conjecturer_extractions/"+project_single_name
+                                    ))
+    total_files += 1
+
+process_number_to_cmds = {i: [] for i in range(number_of_processes)}
+print("A total of {} files are due to be generated".format(total_files))
+for i, cmd in enumerate(total_cmds):
+    process_number_to_cmds[i%number_of_processes].append(cmd)
+
+for process_number, process_cmds in process_number_to_cmds.items():
+    with open("conjecturer_parser_{}.sh".format(process_number), "w") as f:
+        for process_cmd in process_cmds:
+            f.write(process_cmd+"\n")
+            f.write("PIDmain=$!\n")
+            f.write("wait $PIDmain\n")

itp_interface/pisa/command_generation/create_dirs.py

@@ -0,0 +1,11 @@
+import os
+
+saving_directory = input("Enter the directory to save extractions at:\n").strip()
+if not os.path.isdir(saving_directory):
+    os.makedirs(saving_directory)
+
+theory_directory = input("Enter the directory to find extractions at:\n").strip()
+
+for file_name in os.listdir(theory_directory):
+    create_name = os.path.join(saving_directory, file_name)
+    os.mkdir(create_name)

itp_interface/pisa/command_generation/find_std.py

@@ -0,0 +1,67 @@
+import glob
+import os
+import shutil
+
+if __name__ == "__main__":
+    number_of_processes = input("Enter the number of processes you want to run at the same time:\n").strip()
+    number_of_processes = int(number_of_processes)
+    use_proof = input("Want to condition on the proof? (true/false)\n").strip()
+    use_conj = input("Want to conjecture? (true/false)\n").strip()
+    use_quick_ones = input("Want to use quick ones? (true/false)\n").strip()
+    use_state_first = input("Want to use the state-only agent for the first step? (true/false) (false recommended)\n").strip()
+
+    use_default = input("What to use default hyperparameters? (true/false)\n").strip()
+    if use_default.startswith("true"):
+        search_width = 32
+        maximum_queue_length = 8
+        temperature = 0.8
+        max_tokens = 128
+        max_trials = 200
+        timeout = 240000
+    else:
+        search_width = int(input("search_width:\n"))
+        maximum_queue_length = int(input("maximum_queue_length:\n"))
+        temperature = float(input("temperature:\n"))
+        max_tokens = int(input("max_tokens:\n"))
+        max_trials = int(input("max_trials:\n"))
+        timeout = int(input("timeout:\n"))
+
+    script = 'echo "y" | sbt "runMain pisa.agent.HPSearch {} {} {} {} {} {} {} {} {} {} {}"'
+
+    for repeat in range(8):
+        total_cmds = list()
+        if use_quick_ones.startswith("t") or use_quick_ones.startswith("T"):
+            pattern = "universal_test_theorems/quick_test_name_*.json"
+        else:
+            pattern = "universal_test_theorems/test_name_*.json"
+
+        results_dir = ""
+        if use_conj.startswith("t"):
+            results_dir = "results/search_eval_conj_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}_{}".format(
+                search_width, maximum_queue_length, temperature, max_tokens, max_trials, timeout, repeat)
+        elif use_proof.startswith("t"):
+            results_dir = "results/search_eval_proof_and_state_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}_{}".format(
+                search_width, maximum_queue_length, temperature, max_tokens, max_trials, timeout, repeat)
+        else:
+            results_dir = "results/search_eval_state_only_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}_{}".format(
+                search_width, maximum_queue_length, temperature, max_tokens, max_trials, timeout, repeat)
+
+        if os.path.isdir(results_dir):
+            shutil.rmtree(results_dir)
+        os.makedirs(results_dir)
+
+        for file_name in glob.glob(pattern, recursive=True):
+            total_cmds.append(script.format(file_name, use_proof, use_conj, use_state_first, results_dir,
+                                            search_width, maximum_queue_length, temperature, max_tokens, max_trials, timeout))
+
+        process_number_to_cmds = {i: [] for i in range(number_of_processes)}
+        for i, cmd in enumerate(total_cmds):
+            process_number_to_cmds[i%number_of_processes].append(cmd)
+
+        for process_number, process_cmds in process_number_to_cmds.items():
+            with open("scripts/eval_search_conj_{}_use_proof_{}_use_state_first_{}_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}_{}_{}.sh".format(
+                    use_conj, use_proof, use_state_first, search_width, maximum_queue_length, temperature, max_tokens, max_trials, timeout, process_number, repeat), "w") as f:
+                for process_cmd in process_cmds:
+                    f.write(process_cmd+"\n")
+                    f.write("PIDmain=$!\n")
+                    f.write("wait $PIDmain\n")

itp_interface/pisa/command_generation/generate_build_commands_afp.py

@@ -0,0 +1,15 @@
+import os
+
+basic_command = "Isabelle2020/bin/isabelle build -b -d $AFP {}\n"
+
+strings_to_write = []
+thy_dir = "$AFP/thys"
+for project in os.listdir(thy_dir):
+    if os.path.isdir(os.path.join(thy_dir, project)):
+        strings_to_write.append(basic_command.format(project))
+
+print("{} projects in total".format(len(strings_to_write)))
+
+with open("PISA/scripts/build_isabelle_afp.sh", "w") as fout:
+    for string_to_write in strings_to_write:
+        fout.write(string_to_write)

itp_interface/pisa/command_generation/generate_build_commands_std.py

@@ -0,0 +1,15 @@
+import os
+
+basic_command = "/home/ywu/Isabelle2020/bin/isabelle build -b -d /home/ywu/Isabelle2020/src/{} \n"
+
+strings_to_write = []
+thy_dir = "/home/ywu/Isabelle2020/src"
+for project in os.listdir(thy_dir):
+    if os.path.isdir(os.path.join(thy_dir, project)):
+        strings_to_write.append(basic_command.format(project))
+
+print("{} projects in total".format(len(strings_to_write)))
+
+with open("build_isabelle_std.sh", "w") as fout:
+    for string_to_write in strings_to_write:
+        fout.write(string_to_write)

itp_interface/pisa/command_generation/generate_commands_afp.py

@@ -0,0 +1,103 @@
+ports = str(input("Enter the ports, separated by commas:\n")).strip()
+ports = ports.split(",")
+ports = [int(port.strip()) for port in ports]
+ports = [port for port in ports if port]
+use_hammers = str(input("Use hammers (T/F):\n")).strip()
+if use_hammers == "T" or use_hammers == "t":
+    use_hammers = True
+elif use_hammers == "F" or use_hammers == "f":
+    use_hammers = False
+else:
+    raise AssertionError
+
+import glob
+import os
+import shutil
+
+home_directory = "/private/home/aqj"
+
+script = f"unset http_proxy; unset https_proxy; python3 src/main/python/one_stage_extraction.py  --isa-path {home_directory}/Isabelle2021 -wd {home_directory}" + "/afp-2021-10-22/thys/{} --saving-directory afp_extractions/{} -tfp {}"
+if use_hammers:
+    script = script + " -us"
+
+n_threads = 1
+
+cmds = []
+
+IGNORED_THEORIES = {}
+IGNORED_ENTRIES = {}
+
+total_files = 0
+for project_name in glob.glob("{}/afp-2021-10-22/thys/**/*.thy".format(home_directory), recursive=True):
+    project_single_name = project_name.split("/")[6]
+    file_single_name = project_name.split("/")[-1]
+
+    # Ignore already extracted files
+    if os.path.isfile("afp_extractions/{}/{}_ground_truth.json".format(
+            project_single_name, file_single_name.split(".thy")[0])):
+        continue
+
+    if project_single_name not in IGNORED_ENTRIES and \
+            project_single_name + "/" + file_single_name not in IGNORED_THEORIES:
+        cmds.append(
+            script.format(project_single_name, project_single_name, project_name))
+        total_files += 1
+print("A total of {} files are due to be generated".format(total_files))
+# num_cmd_per_thread = len(cmds) // n_threads + 1
+# cmds_splits = [cmds[i*num_cmd_per_thread:(i+1)*num_cmd_per_thread] for i in range(n_threads)]
+
+how_many_ports = len(ports)
+indices_to_ports = {i: port for i, port in enumerate(ports)}
+cmds_for_port = {port: [] for port in ports}
+for i, cmd in enumerate(cmds):
+    port = indices_to_ports[i%how_many_ports]
+    cmds_for_port[port].append(cmd + " -p {} ".format(port))
+
+if os.path.isdir("scripts/extraction"):
+    shutil.rmtree("scripts/extraction")
+
+os.makedirs("scripts/extraction")
+
+counter = 0
+for j, port in enumerate(ports):
+    port_cmds = cmds_for_port[port]
+    for i, cmd in enumerate(port_cmds):
+        with open(f"scripts/extraction/script_{j}.sh", "a") as f:
+            f.write('unset http_proxy; unset https_proxy; sbt "runMain pisa.server.PisaOneStageServer{}"&\n'.format(port))
+            f.write("PIDmain=$!\n")
+            f.write("sleep 12\n")
+            f.write(cmd + "&\n")
+            f.write("PID=$!\n")
+            f.write("wait $PID\n")
+            f.write("rm -rf target/bg-jobs \n")
+            f.write("kill $PIDmain\n")
+        counter += 1
+
+
+    #
+    # with open("afp_extract_script_{}.sh".format(port), "w") as f:
+    #     f.write('sbt "runMain pisa.server.PisaOneStageServer{}"&\n'.format(port))
+    #     f.write("PIDmain=$!\n")
+    #     f.write("sleep 12\n")
+    #     wait_cmds = []
+    #     for i, cmd in enumerate(port_cmds):
+    #         if (i + 1) % n_threads == 0:
+    #             f.write(cmd + "&\n")
+    #             f.write("PID{}=$!\n".format(i % n_threads))
+    #             wait_cmds.append("wait $PID{}\n".format(i % n_threads))
+    #             for wait in wait_cmds:
+    #                 f.write(wait)
+    #             wait_cmds = []
+    #             # f.write("ps aux | grep scala | awk '{print $2}' | xargs kill\n")
+    #             # f.write("ps aux | grep java | awk '{print $2}' | xargs kill\n")
+    #             # f.write("ps aux | grep poly | awk '{print $2}' | xargs kill\n")
+    #             f.write("kill $PIDmain\n")
+    #             if (i+1) % 20 == 0:
+    #                 f.write("rm -rf target/bg-jobs/* \n")
+    #             f.write('sbt "runMain pisa.server.PisaOneStageServer{}" &\n'.format(port))
+    #             f.write("PIDmain=$!\n")
+    #             f.write("sleep 12\n")
+    #         else:
+    #             f.write(cmd + "&\n")
+    #             f.write("PID{}=$!\n".format(i % n_threads))
+    #             wait_cmds.append("wait $PID{}\n".format(i % n_threads))

itp_interface/pisa/command_generation/generate_commands_mini.py

@@ -0,0 +1,73 @@
+ports = str(input("Enter the ports, separated by commas:\n")).strip()
+ports = ports.split(",")
+ports = [int(port.strip()) for port in ports]
+ports = [port for port in ports if port]
+
+import glob
+import os
+
+home_directory = "/home/qj213"
+
+script = f"python3 src/main/python/one_stage_extraction.py  --isa-path {home_directory}/Isabelle2021 " \
+         f"-wd {home_directory}" + "/afp-2021-10-22/thys/Symmetric_Polynomials --saving-directory mini_extractions/{} -tfp {}"
+n_threads = 1
+
+cmds = []
+
+IGNORED_THEORIES = {}
+IGNORED_ENTRIES = {}
+
+total_files = 0
+for project_name in glob.glob("{}/miniF2F/isabelle/**/*.thy".format(home_directory), recursive=True):
+    project_single_name = project_name.split("/")[5]
+    # file_single_name = project_name.split("/")[-1]
+    file_single_name = project_name.replace("/", "_")
+    # Ignore already extracted files
+    print("mini_extractions/{}/{}_ground_truth.json".format(project_single_name, file_single_name.split(".thy")[0]))
+    if os.path.isfile("mini_extractions/{}/{}_ground_truth.json".format(
+            project_single_name, file_single_name.split(".thy")[0])):
+        continue
+
+    if project_single_name not in IGNORED_ENTRIES and \
+            project_single_name + "/" + file_single_name not in IGNORED_THEORIES:
+        cmds.append(
+            script.format(project_single_name, project_name))
+        total_files += 1
+print("A total of {} files are due to be generated".format(total_files))
+# num_cmd_per_thread = len(cmds) // n_threads + 1
+# cmds_splits = [cmds[i*num_cmd_per_thread:(i+1)*num_cmd_per_thread] for i in range(n_threads)]
+
+how_many_ports = len(ports)
+indices_to_ports = {i: port for i, port in enumerate(ports)}
+cmds_for_port = {port: [] for port in ports}
+for i, cmd in enumerate(cmds):
+    port = indices_to_ports[i%how_many_ports]
+    cmds_for_port[port].append(cmd + " -p {} ".format(port))
+
+for port, port_cmds in cmds_for_port.items():
+    with open("mini_extract_script_{}.sh".format(port), "w") as f:
+        f.write('sbt "runMain pisa.server.PisaOneStageServer{}"&\n'.format(port))
+        f.write("PIDmain=$!\n")
+        f.write("sleep 12\n")
+        wait_cmds = []
+        for i, cmd in enumerate(port_cmds):
+            if (i + 1) % n_threads == 0:
+                f.write(cmd + "&\n")
+                f.write("PID{}=$!\n".format(i % n_threads))
+                wait_cmds.append("wait $PID{}\n".format(i % n_threads))
+                for wait in wait_cmds:
+                    f.write(wait)
+                wait_cmds = []
+                # f.write("ps aux | grep scala | awk '{print $2}' | xargs kill\n")
+                # f.write("ps aux | grep java | awk '{print $2}' | xargs kill\n")
+                # f.write("ps aux | grep poly | awk '{print $2}' | xargs kill\n")
+                f.write("kill $PIDmain\n")
+                if (i+1) % 20 == 0:
+                    f.write("rm -rf target/bg-jobs/* \n")
+                f.write('sbt "runMain pisa.server.PisaOneStageServer{}" &\n'.format(port))
+                f.write("PIDmain=$!\n")
+                f.write("sleep 12\n")
+            else:
+                f.write(cmd + "&\n")
+                f.write("PID{}=$!\n".format(i % n_threads))
+                wait_cmds.append("wait $PID{}\n".format(i % n_threads))

itp_interface/pisa/command_generation/generate_commands_std.py

@@ -0,0 +1,69 @@
+ports = input("Enter the ports, separated by commas:\n").strip()
+ports = ports.split(",")
+ports = [int(port.strip()) for port in ports]
+ports = [port for port in ports if port]
+
+import glob
+import os
+
+script = "python3 src/main/python/one_stage_extraction.py -wd {} --saving-directory std_extractions/{} --isa-path /home/qj213/Isabelle2021 -tfp {}"
+n_threads = 1
+
+cmds = []
+
+IGNORED_THEORIES = {}
+IGNORED_ENTRIES = {}
+
+total_files = 0
+for project_name in glob.glob("/home/qj213/Isabelle2021/src/**/*.thy", recursive=True):
+    project_single_name = project_name.split("/")[5]
+    file_single_name = project_name.split("/")[-1]
+
+    # Ignore already extracted files
+    if os.path.isfile("/home/qj213/Portal-to-ISAbelle/std_extractions/{}/{}_ground_truth.json".format(
+            project_single_name, file_single_name.split(".thy")[0])):
+        continue
+
+    if project_single_name not in IGNORED_ENTRIES and \
+            project_single_name + "/" + file_single_name not in IGNORED_THEORIES:
+        cmds.append(
+            script.format("/".join(project_name.split("/")[:-1]), project_single_name, project_name))
+        total_files += 1
+print("A total of {} files are due to be generated".format(total_files))
+# num_cmd_per_thread = len(cmds) // n_threads + 1
+# cmds_splits = [cmds[i*num_cmd_per_thread:(i+1)*num_cmd_per_thread] for i in range(n_threads)]
+
+how_many_ports = len(ports)
+indices_to_ports = {i: port for i, port in enumerate(ports)}
+cmds_for_port = {port: [] for port in ports}
+for i, cmd in enumerate(cmds):
+    port = indices_to_ports[i%how_many_ports]
+    cmds_for_port[port].append(cmd + " -p {} ".format(port))
+
+for port, port_cmds in cmds_for_port.items():
+    with open("std_extract_script_{}.sh".format(port), "w") as f:
+        f.write('sbt "runMain pisa.server.PisaOneStageServer{}"&\n'.format(port))
+        f.write("PIDmain=$!\n")
+        f.write("sleep 12\n")
+        wait_cmds = []
+        for i, cmd in enumerate(port_cmds):
+            if (i + 1) % n_threads == 0:
+                f.write(cmd + "&\n")
+                f.write("PID{}=$!\n".format(i % n_threads))
+                wait_cmds.append("wait $PID{}\n".format(i % n_threads))
+                for wait in wait_cmds:
+                    f.write(wait)
+                wait_cmds = []
+                # f.write("ps aux | grep scala | awk '{print $2}' | xargs kill\n")
+                # f.write("ps aux | grep java | awk '{print $2}' | xargs kill\n")
+                # f.write("ps aux | grep poly | awk '{print $2}' | xargs kill\n")
+                f.write("kill $PIDmain\n")
+                if (i+1) % 20 == 0:
+                    f.write("rm -rf target/bg-jobs/* \n")
+                f.write('sbt "runMain pisa.server.PisaOneStageServer{}" &\n'.format(port))
+                f.write("PIDmain=$!\n")
+                f.write("sleep 12\n")
+            else:
+                f.write(cmd + "&\n")
+                f.write("PID{}=$!\n".format(i % n_threads))
+                wait_cmds.append("wait $PID{}\n".format(i % n_threads))

itp_interface/pisa/command_generation/generate_hammer_extraction_text.py

@@ -0,0 +1,5 @@
+import glob
+import os
+for i, file_path in enumerate(glob.glob("/home/qj213/afp-2021-10-22/thys/**/*.thy", recursive=True)):
+    with open(f"data/hammer_extraction_{i}.txt", "w") as fout:
+        fout.write(f"{file_path}\n")

itp_interface/pisa/command_generation/hammer_command_generator.py

@@ -0,0 +1,40 @@
+number_of_processes = input("Enter the number of processes you want to run at the same time:\n").strip()
+number_of_processes = int(number_of_processes)
+mini = input("Are we doing MiniF2F? (Y/N)\n").strip()
+mini = True if mini.strip().startswith("Y") else False
+heuristic = input("Are we doing heuristics? (Y/N)\n").strip()
+heuristic = True if heuristic.strip().startswith("Y") else False
+import glob
+import os
+
+if heuristic:
+    script = 'echo "y" | sbt "runMain pisa.agent.PisaHammerTest {} true"'
+else:
+    script = 'echo "y" | sbt "runMain pisa.agent.PisaHammerTest {} false"'
+
+theorem_names_path = "/home/qj213/mini_names" if mini else "/home/qj213/Portal-to-ISAbelle/universal_test_theorems"
+
+total_cmds = list()
+total_files = 0
+for file_name in glob.glob(f"{theorem_names_path}/test_name_*.json", recursive=True):
+    number = int(file_name.split("/")[-1].rstrip(".json").split("_")[-1])
+    if number <= 1000:
+        total_cmds.append(script.format(file_name))
+        total_files += 1
+
+process_number_to_cmds = {i: [] for i in range(number_of_processes)}
+print("A total of {} files are due to be generated".format(total_files))
+for i, cmd in enumerate(total_cmds):
+    process_number_to_cmds[i%number_of_processes].append(cmd)
+
+for process_number, process_cmds in process_number_to_cmds.items():
+    with open("scripts/eval_hammer_{}.sh".format(process_number), "w") as f:
+        for process_cmd in process_cmds:
+            f.write(process_cmd+"\n")
+            f.write("PIDmain=$!\n")
+            f.write("wait $PIDmain\n")
+            # f.write("ps -ef | grep z3 | awk '{print $2}' | xargs kill -9\n")
+            # f.write("ps -ef | grep veriT | awk '{print $2}' | xargs kill -9\n")
+            # f.write("ps -ef | grep cvc4 | awk '{print $2}' | xargs kill -9\n")
+            # f.write("ps -ef | grep eprover | awk '{print $2}' | xargs kill -9\n")
+            # f.write("ps -ef | grep SPASS | awk '{print $2}' | xargs kill -9\n")

itp_interface/pisa/command_generation/hp_search_command_generator.py

@@ -0,0 +1,63 @@
+import glob
+import os
+import shutil
+
+if __name__ == "__main__":
+    number_of_processes = input("Enter the number of processes you want to run at the same time:\n").strip()
+    number_of_processes = int(number_of_processes)
+    use_proof = input("Want to condition on the proof? (true/false)\n").strip()
+    use_conj = input("Want to conjecture? (true/false)\n").strip()
+    use_quick_ones = input("Want to use quick ones? (true/false)\n").strip()
+    use_state_first = input("Want to use the state-only agent for the first step? (true/false) (false recommended)\n").strip()
+    engine_id = input("What engine id to use? (starting https: and ending with completions) (blank and will use small model by default)\n").strip()
+    if (len(engine_id) == 0):
+        engine_id = ""
+
+    script = 'echo "y" | sbt "runMain pisa.agent.HPSearch {} {} {} {} {} {} {} {} {} {} {} {}"'
+
+    max_tokens = 128
+    max_trials = 200
+    timeout = 240000
+    search_width = 32
+    total_cmds = list()
+    mql_sweep = [8, 16, 32]
+    for temperature in [0.7, 0.8, 1.0, 1.2]:
+        for maximum_queue_length in mql_sweep:
+            if use_quick_ones.startswith("t") or use_quick_ones.startswith("T"):
+                pattern = "universal_test_theorems/quick_test_name_*.json"
+            else:
+                pattern = "universal_test_theorems/test_name_*.json"
+
+            results_dir = ""
+            if use_conj.startswith("t"):
+                results_dir = "results/search_eval_conj_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}".format(
+                    search_width, maximum_queue_length, temperature, max_tokens, max_trials, timeout)
+            elif use_proof.startswith("t"):
+                results_dir = "results/search_eval_proof_and_state_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}".format(
+                    search_width, maximum_queue_length, temperature, max_tokens, max_trials, timeout)
+            else:
+                results_dir = "results/search_eval_state_only_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}".format(
+                    search_width, maximum_queue_length, temperature, max_tokens, max_trials, timeout)
+
+            if os.path.isdir(results_dir):
+                shutil.rmtree(results_dir)
+            os.makedirs(results_dir)
+
+            for file_name in glob.glob(pattern, recursive=True):
+                total_cmds.append(script.format(file_name, use_proof, use_conj, use_state_first, results_dir,
+                                                search_width, maximum_queue_length, temperature, max_tokens, max_trials, timeout, engine_id))
+
+    process_number_to_cmds = {i: [] for i in range(number_of_processes)}
+    for i, cmd in enumerate(total_cmds):
+        process_number_to_cmds[i%number_of_processes].append(cmd)
+
+    for process_number, process_cmds in process_number_to_cmds.items():
+        with open("scripts/eval_search_conj_{}_use_proof_{}_use_state_first_{}_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}_{}.sh".format(
+                use_conj, use_proof, use_state_first, search_width, "sweep_{}".format("_".join([str(element) for element in mql_sweep])), temperature, max_tokens, max_trials, timeout, process_number), "w") as f:
+            for process_cmd in process_cmds:
+                f.write(process_cmd+"\n")
+                f.write("PIDmain=$!\n")
+                f.write("wait $PIDmain\n")
+
+
+deduplication

itp_interface/pisa/command_generation/oracle_command_generator.py

@@ -0,0 +1,56 @@
+import glob
+import os
+import shutil
+
+if __name__ == "__main__":
+    number_of_processes = input("Enter the number of processes you want to run at the same time:\n").strip()
+    number_of_processes = int(number_of_processes)
+    use_proof = input("Want to condition on the proof? (true/false)\n").strip()
+    use_conj = input("Want to conjecture? (true/false)\n").strip()
+    use_quick_ones = input("Want to use quick ones? (true/false)\n").strip()
+    use_state_first = input("Want to use the state-only agent for the first step? (true/false) (false recommended)\n").strip()
+
+    use_default = input("What to use default hyperparameters? (true/false)\n").strip()
+    if use_default.startswith("true"):
+        search_width = 32
+        maximum_queue_length = 32
+        temperature = 0.8
+        max_tokens = 128
+        max_trials = 200
+        timeout = 240000
+    else:
+        search_width = int(input("search_width:\n"))
+        maximum_queue_length = int(input("maximum_queue_length:\n"))
+        temperature = float(input("temperature:\n"))
+        max_tokens = int(input("max_tokens:\n"))
+        max_trials = int(input("max_trials:\n"))
+        timeout = int(input("timeout:\n"))
+
+    script = 'echo "y" | sbt "runMain pisa.agent.OracleAgent {} {} {} {} {} {} {} {} {} {} {}"'
+
+    total_cmds = list()
+    if use_quick_ones.startswith("t") or use_quick_ones.startswith("T"):
+        pattern = "1000_train_theorems/train_theorem_*.json"
+    else:
+        pattern = "1000_train_theorems/train_theorem_*.json"
+
+    results_dir = "oracle_results"
+
+    if os.path.isdir(results_dir):
+        shutil.rmtree(results_dir)
+    os.makedirs(results_dir)
+
+    for file_name in glob.glob(pattern, recursive=True):
+        total_cmds.append(script.format(file_name, use_proof, use_conj, use_state_first, results_dir,
+                                        search_width, maximum_queue_length, temperature, max_tokens, max_trials, timeout))
+
+    process_number_to_cmds = {i: [] for i in range(number_of_processes)}
+    for i, cmd in enumerate(total_cmds):
+        process_number_to_cmds[i%number_of_processes].append(cmd)
+
+    for process_number, process_cmds in process_number_to_cmds.items():
+        with open("scripts/oracle_{}.sh".format(process_number), "w") as f:
+            for process_cmd in process_cmds:
+                f.write(process_cmd+"\n")
+                f.write("PIDmain=$!\n")
+                f.write("wait $PIDmain\n")

itp_interface/pisa/command_generation/search_command_generator.py

@@ -0,0 +1,69 @@
+import glob
+import os
+import shutil
+
+if __name__ == "__main__":
+    number_of_processes = input("Enter the number of processes you want to run at the same time:\n").strip()
+    number_of_processes = int(number_of_processes)
+    use_proof = input("Want to condition on the proof? (true/false)\n").strip()
+    use_conj = input("Want to conjecture? (true/false)\n").strip()
+    use_quick_ones = input("Want to use quick ones? (true/false)\n").strip()
+    use_state_first = input("Want to use the state-only agent for the first step? (true/false) (false recommended)\n").strip()
+
+    use_default = input("What to use default hyperparameters? (true/false)\n").strip()
+    if use_default.startswith("true"):
+        search_width = 32
+        maximum_queue_length = 32,
+        temperature = 0.8
+        max_tokens = 128
+        max_trials = 200
+        timeout = 240000
+    else:
+        search_width = int(input("search_width:\n"))
+        maximum_queue_length = int(input("maximum_queue_length:\n"))
+        temperature = float(input("temperature:\n"))
+        max_tokens = int(input("max_tokens:\n"))
+        max_trials = int(input("max_trials:\n"))
+        timeout = int(input("timeout:\n"))
+
+    script = 'echo "y" | sbt "runMain pisa.agent.HPSearch {} {} {} {} {} {} {} {} {} {} {}"'
+
+
+    total_cmds = list()
+    if use_quick_ones.startswith("t") or use_quick_ones.startswith("T"):
+        pattern = "universal_test_theorems/quick_test_name_*.json"
+    else:
+        pattern = "universal_test_theorems/test_name_*.json"
+
+    results_dir = ""
+    if use_conj.startswith("t"):
+        results_dir = "results/search_eval_conj_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}".format(
+            search_width, maximum_queue_length, temperature, max_tokens, max_trials, timeout)
+    elif use_proof.startswith("t"):
+        results_dir = "results/search_eval_proof_and_state_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}".format(
+            search_width, maximum_queue_length, temperature, max_tokens, max_trials, timeout)
+    else:
+        results_dir = "results/search_eval_state_only_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}".format(
+            search_width, maximum_queue_length, temperature, max_tokens, max_trials, timeout)
+
+
+    if os.path.isdir(results_dir):
+        shutil.rmtree(results_dir)
+    os.makedirs(results_dir)
+
+    for file_name in glob.glob(pattern, recursive=True):
+        total_cmds.append(script.format(file_name, use_proof, use_conj, use_state_first, results_dir,
+                                        search_width, maximum_queue_length, temperature, max_tokens, max_trials, timeout))
+
+    process_number_to_cmds = {i: [] for i in range(number_of_processes)}
+    for i, cmd in enumerate(total_cmds):
+        process_number_to_cmds[i%number_of_processes].append(cmd)
+
+    for process_number, process_cmds in process_number_to_cmds.items():
+        with open("scripts/eval_search_conj_{}_use_proof_{}_use_state_first_{}_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}_{}.sh".format(
+                use_conj, use_proof, use_state_first, search_width, maximum_queue_length, temperature, max_tokens, max_trials, timeout, process_number), "w") as f:
+
+            for process_cmd in process_cmds:
+                f.write(process_cmd+"\n")
+                f.write("PIDmain=$!\n")
+                f.write("wait $PIDmain\n")