itp-interface 1.0.0__py3-none-any.whl

itp_interface/pisa/command_generation/summarise_problem_names.py

@@ -0,0 +1,45 @@
+import argparse
+import os
+import json
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Summarise the names and files from extractions.')
+    parser.add_argument('--extraction-path', '-ep', help='The path to extraction files.')
+    parser.add_argument('--dataset-name', '-dn', help='Dataset name after dumping.')
+    parser.add_argument('--dump-path', '-dp', help='Path to dump the dataset.')
+    args = parser.parse_args()
+
+    dump_path = os.path.join(args.dump_path, args.dataset_name)
+    if os.path.isdir(dump_path):
+        os.rmdir(dump_path)
+
+    os.mkdir(dump_path)
+
+    total = 0
+    for split in os.listdir(args.extraction_path):
+        split_path = os.path.join(args.extraction_path, split)
+        for entry in os.listdir(split_path):
+            entry_path = os.path.join(split_path, entry)
+
+            if "ground_truth" in entry_path:
+                ground_truth = json.load(open(entry_path))
+                file_name = ground_truth["file_name"]
+                problem_names = ground_truth["problem_names"]
+                # if not len(problem_names) == 1:
+                #     print(f"{len(problem_names)}, {file_name}")
+                #     continue
+
+                for theorem_name in problem_names:
+                    theorem_name = theorem_name.replace("\n", " ")
+                    theorem_name = " ".join(theorem_name.split())
+                    saving_name = file_name.split("/")[-1].strip(".thy")
+                    total += 1
+                    json.dump(
+                        [
+                            [file_name, theorem_name]
+                        ],
+                        open(
+                            os.path.join(dump_path, f"test_name_{total}.json"), "w"
+                        )
+                    )

itp_interface/pisa/command_generation/tpu_hp_search.py

@@ -0,0 +1,75 @@
+import glob
+import os
+import shutil
+
+if __name__ == "__main__":
+    number_of_processes = input("Enter the number of processes you want to run at the same time:\n").strip()
+    number_of_processes = int(number_of_processes)
+    use_proof = input("Want to condition on the proof? (true/false)\n").strip()
+    use_conj = input("Want to conjecture? (true/false)\n").strip()
+    use_quick_ones = input("Want to use quick ones? (true/false)\n").strip()
+    use_state_first = input("Want to use the state-only agent for the first step? (true/false) (false recommended)\n").strip()
+    using_t5 = input("Are you using a T5 server? (true/false)?\n").strip()
+    greedy = input("Are you using greedy search? (true/false)?\n").strip()
+    last_k = input("How many last steps to incldue as context (>=0, 0 to disable)?\n").strip()
+    needed = input("Are you using needed steps as context (true/false)?\n").strip()
+
+
+    # script = 'echo "y" | sbt "runMain pisa.agent.TPUHPSearch {} {} {} {} {} {} {} {} {} {} {} {}"'
+
+    max_tokens = 64
+    max_trials = 100
+    timeout = 6000000
+    search_width = 8
+    total_cmds = list()
+    mql_sweep = 16
+    temperature = 1.0
+    # if using_t5.startswith("t"):
+    #     using_t5 = "true"
+    # else:
+    #     using_t5 = "false
+    
+
+    results_dir = ""
+    if use_conj.startswith("t"):
+        results_dir = "results/search_eval_conj_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}_usingT5_{}_greedy_{}_last_{}_needed_{}".format(
+            search_width, mql_sweep, temperature, max_tokens, max_trials, timeout, using_t5, greedy, last_k, needed)
+    elif use_proof.startswith("t"):
+        results_dir = "results/search_eval_proof_and_state_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}_usingT5_{}_greedy_{}_last_{}_needed_{}".format(
+            search_width, mql_sweep, temperature, max_tokens, max_trials, timeout, using_t5, greedy, last_k, needed)
+    else:
+        results_dir = "results/search_eval_state_only_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}_usingT5_{}_greedy_{}_last_{}_needed_{}".format(
+            search_width, mql_sweep, temperature, max_tokens, max_trials, timeout, using_t5, greedy, last_k, needed)
+
+    if os.path.isdir(results_dir):
+        shutil.rmtree(results_dir)
+    os.makedirs(results_dir)
+
+    if use_quick_ones.startswith("t") or use_quick_ones.startswith("T"):
+        pattern = "universal_test_theorems/quick_test_name_{}.json"
+        for i in range (1, 301):
+            total_cmds.append(
+                f'echo "y" | sbt "runMain pisa.agent.TPUHPSearch {pattern.format(i)} {use_proof} {use_conj} {use_state_first} {results_dir} ' 
+                f'{search_width} {mql_sweep} {temperature} {max_tokens} {max_trials} {timeout} {using_t5} '
+                f'{greedy} {last_k} {needed}"'
+            )
+    else:
+        pattern = "universal_test_theorems/test_name_{}.json"
+        for i in range (1, 3001):
+            total_cmds.append(
+                f'echo "y" | sbt "runMain pisa.agent.TPUHPSearch {pattern.format(i)} {use_proof} {use_conj} {use_state_first} {results_dir} ' 
+                f'{search_width} {mql_sweep} {temperature} {max_tokens} {max_trials} {timeout} {using_t5} '
+                f'{greedy} {last_k} {needed}"'
+            )
+    
+    process_number_to_cmds = {i: [] for i in range(number_of_processes)}
+    for i, cmd in enumerate(total_cmds):
+        process_number_to_cmds[i%number_of_processes].append(cmd)
+
+    for process_number, process_cmds in process_number_to_cmds.items():
+        with open("scripts/eval_search_conj_{}_use_proof_{}_use_state_first_{}_search_width_{}_maximum_queue_length_{}_temperature_{}_max_tokens_{}_max_trials_{}_timeout_{}_usingT5_{}_greedy_{}_last_{}_needed_{}_{}.sh".format(
+                use_conj, use_proof, use_state_first, search_width, "sweep_{}".format(mql_sweep), temperature, max_tokens, max_trials, timeout, using_t5, greedy, last_k, needed, process_number), "w") as f:
+            for process_cmd in process_cmds:
+                f.write(process_cmd+"\n")
+                f.write("PIDmain=$!\n")
+                f.write("wait $PIDmain\n")

itp_interface/pisa/docker/Dockerfile

@@ -0,0 +1,34 @@
+FROM eclipse-temurin:11
+
+RUN apt-get update && \
+    apt-get install -y python3-pip unzip zip vim tmux
+
+RUN python3 -m pip install grpcio func_timeout protobuf==3.20.1
+
+RUN curl -s "https://get.sdkman.io" | bash
+SHELL ["/bin/bash", "-c"]
+
+# Setting a work directory for PISA
+WORKDIR /pisa
+
+RUN apt-get install -y git && \
+    git clone https://github.com/albertqjiang/Portal-to-ISAbelle.git . && \
+    source "$HOME/.sdkman/bin/sdkman-init.sh" && \
+    sdk install java 11.0.11-open && \
+    sdk install sbt && \
+    sbt compile && \
+    sbt assembly
+
+RUN wget https://isabelle.in.tum.de/dist/Isabelle2022_linux.tar.gz && \
+    tar -xzf Isabelle2022_linux.tar.gz
+
+RUN echo 'alias isabelle=/pisa/Isabelle2022/bin/isabelle' >> ~/.bashrc
+
+RUN /pisa/Isabelle2022/bin/isabelle build -b -D /pisa/Isabelle2022/src/HOL/ -j 20
+
+# Comment out the next two lines if you want to build PISA without AFP
+RUN wget https://archive.org/download/isabelle2022_afp20221206_heaps/isabelle2022heaps.tar.gz
+
+RUN tar -xzf isabelle2022heaps.tar.gz -C /root/.isabelle
+
+EXPOSE 8000-8200

itp_interface/pisa/docker/docker_tutorial.md

@@ -0,0 +1,64 @@
+# How to install PISA with Docker and start using it
+
+This guide is designed to help you easily install and start using PISA via Docker.
+Docker simplifies the deployment process by encapsulating the software and its dependencies into a single object called a Docker image. With Docker, you don't have to worry about installing all the necessary software and libraries manually.
+
+## 1. Prerequisites
+Before getting started, please ensure that you have Docker installed on your machine.
+
+You should also make sure that you have sufficient disk space. The PISA image is available in two versions:
+- the first one is PISA with Isabelle standard library (you will need at least 10GB of disk space).
+- the second one is PISA with Isabelle standard libary and the Archive of Formal Proofs (you will need at least 66GB of disk space).
+
+## 2. Pulling the PISA image from Docker and running the container
+Open a terminal and pull the Docker image you want to use. If you want to use PISA with the standard library, type:
+```
+$ docker pull dsantosmarco/pisa:pisa-sl
+```
+If you prefer to use PISA with the standard library and the Archive of Formal Proofs, you should type:
+```
+$ docker pull dsantosmarco/pisa:pisa-sl-afp
+```
+Once the image is pulled, you can run a Docker container.
+Depending on what PISA image you chose, type either:
+```
+$ docker run -it --entrypoint=/bin/sh dsantosmarco/pisa:pisa-sl
+```
+or:
+```
+$ docker run -it --entrypoint=/bin/sh dsantosmarco/pisa:pisa-sl-afp
+```
+Note that `--entrypoint=/bin/sh` is used to run the container in bash environment instead of jshell.
+
+## 3. Launching a PISA server and running test_client.py
+Once the container is running, you can go to `/pisa/target/scala-2.13` and use `tmux` to open a new terminal window:
+```
+$ cd /pisa/target/scala-2.13
+$ tmux
+```
+You can then run the following command to launch a PISA server in this new window:
+```
+$ java -cp PISA-assembly-0.1.jar pisa.server.PisaOneStageServer8000
+```
+You can then create an other window with `tmux` (using `Ctrl+B C`).
+In this new window, you can go to `/pisa/src/main/python` and run the `test_client.py` file:
+```
+$ cd /pisa/src/main/python
+$ python3 test_client.py
+```
+If everything worked, you should see the following printed in your terminal:
+```
+----------Path to Isabelle source----------
+/pisa/Isabelle2022
+----------Path to Isabelle working directory----------
+/pisa/Isabelle2022/src/HOL/Computational_Algebra
+----------Path to Isabelle theory file----------
+/pisa/Isabelle2022/src/HOL/Computational_Algebra/Primes.thy
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Premise name: Primes.prime_int_iff'
+Premise defn:  prime_int_iff': fixes p :: "int" shows "prime p = (1 < p \<and> (\<forall>n\<in>{2..<p}. \<not> n dvd p))"
+```
+
+## Additional info: Building the PISA images yourself
+If you prefer to build the PISA images yourself, you can use Docker to build them from the Dockerfile we added to the current folder "docker".
+We used the same Dockerfile for both pisa-sl and pisa-sl-afp. If you want to build pisa-sl (without afp) you should comment out two lines of the Dockerfile (these are indicated in the Dockerfile).

itp_interface/pisa/eval_setup/copy_isabelle.py

@@ -0,0 +1,42 @@
+import argparse
+import shutil
+import os
+from tqdm import tqdm
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--isabelle", type=str, required=True)
+    parser.add_argument("--isabelle-user", type=str, required=True)
+    parser.add_argument("--number-of-copies", type=int, required=True)
+    parser.add_argument("--output-path", type=str, required=True)
+    args = parser.parse_args()
+
+    isabelle_identifier = args.isabelle.split("/")
+    isabelle_identifier = isabelle_identifier[-1] if isabelle_identifier[-1] else isabelle_identifier[-2]
+
+    for index in tqdm(range(args.number_of_copies)):
+        index_path = os.path.join(args.output_path, f"isabelle_copy_{index}")
+        if not os.path.exists(index_path):
+            os.makedirs(index_path)
+
+        main_isa_path = os.path.join(index_path, "main_isa")
+        if not os.path.exists(main_isa_path):
+            os.makedirs(main_isa_path)
+        shutil.copytree(args.isabelle, os.path.join(main_isa_path, isabelle_identifier), symlinks=True)
+
+        user_isa_path = os.path.join(index_path, "user_isa")
+        if os.path.exists(user_isa_path):
+            raise AssertionError
+        shutil.copytree(args.isabelle_user, user_isa_path, symlinks=True)
+
+        # Edit the settings file such that the user home points to the right directory
+        original_isabelle_home_user_string = "$USER_HOME/.isabelle"
+        isabelle_home_user_string = str(user_isa_path)
+        
+        isabelle_settings_path = os.path.join(main_isa_path, isabelle_identifier, "etc/settings")
+        with open(isabelle_settings_path, "r") as f:
+            settings = f.read()
+        settings = settings.replace(original_isabelle_home_user_string, isabelle_home_user_string)
+        with open(isabelle_settings_path, "w") as f:
+            f.write(settings)

itp_interface/pisa/eval_setup/copy_pisa_jars.py

@@ -0,0 +1,18 @@
+import argparse
+import shutil
+import os
+from tqdm import tqdm
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--pisa-jar-path", type=str, required=True)
+    parser.add_argument("--number-of-jars", type=int, required=True)
+    parser.add_argument("--output-path", type=str, required=True)
+    args = parser.parse_args()
+
+    for index in tqdm(range(args.number_of_jars)):
+        shutil.copy2(
+            args.pisa_jar_path,
+            os.path.join(args.output_path, f"pisa_copy{index}.jar")
+        )

itp_interface/pisa/mesh_transformer_utils/tokenization.py

@@ -0,0 +1,86 @@
+from transformers import GPT2TokenizerFast
+
+from abc import ABC, abstractmethod
+from typing import List
+from tokenizers import Tokenizer
+
+
+class TokenizerWrapper(ABC):
+    @abstractmethod
+    def encode(self, sequence) -> List[int]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def decode(self, ids) -> str:
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def eos_token_id(self):
+        raise NotImplementedError
+
+    @property
+    def eos_token_str(self):
+        return '<|endoftext|>'
+
+    @property
+    def sep_token_str(self):
+        return ' Cambridge'
+
+    @property
+    def pad_token_str(self):
+        return ' Oxford'
+
+    @property
+    def sep_token_id(self):
+        tokens = self.encode(self.sep_token_str)
+        assert len(tokens) == 1
+        return tokens[0]
+
+    @property
+    def pad_token_id(self):
+        tokens = self.encode(self.pad_token_str)
+        assert len(tokens) == 1
+        return tokens[0]
+
+    @classmethod
+    def from_file_or_gpt(cls, tokenizer_path):
+        if tokenizer_path:
+            print(f'Using BPE tokenizer from: {tokenizer_path}')
+            return BPETokenizerWrapper(Tokenizer.from_file(tokenizer_path))
+        else:
+            print('Tokenizer not provided - using default gpt2')
+            GPT2TokenizerFast.max_model_input_sizes[
+                'gpt2'] = 1e20  # disables a misleading warning
+            tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
+            return GPTTokenizerWrapper(tokenizer)
+
+
+class GPTTokenizerWrapper(TokenizerWrapper):
+    def __init__(self, tokenizer: GPT2TokenizerFast):
+        self.tokenizer = tokenizer
+
+    def encode(self, sequence):
+        return self.tokenizer.encode(sequence)
+
+    def decode(self, ids):
+        return self.tokenizer.decode(ids)
+
+    @property
+    def eos_token_id(self):
+        return self.tokenizer.eos_token_id
+
+
+class BPETokenizerWrapper(TokenizerWrapper):
+    def __init__(self, tokenizer: Tokenizer):
+        self.tokenizer = tokenizer
+
+    def encode(self, sequence):
+        return self.tokenizer.encode(sequence).ids
+
+    def decode(self, ids):
+        return self.tokenizer.decode(ids)
+
+    @property
+    def eos_token_id(self):
+        return self.tokenizer.token_to_id(self.eos_token_str)

itp_interface/pisa/project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.4.3

itp_interface/pisa/project/plugins.sbt

@@ -0,0 +1,5 @@
+val zioGrpcVersion = "0.4.2"
+
+addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.0-RC4")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0")
+libraryDependencies += "com.thesamet.scalapb.zio-grpc" %% "zio-grpc-codegen" % zioGrpcVersion

itp_interface/pisa/requirements.txt

@@ -0,0 +1,4 @@
+protobuf==3.20.3
+grpcio==1.51.1
+grpcio-tools==1.44.0
+func_timeout

itp_interface/pisa/scripts/extract_last_k_steps.py

@@ -0,0 +1,28 @@
+import argparse
+import os
+from tqdm import tqdm
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Extract the last k steps of the proof to be used in automated proof search.")
+    parser.add_argument("--path-to-proof-and-state-file", "-ptpasf", type=str, help="Path to the file of the proof steps and states.")
+    parser.add_argument("--k", type=int, help="The number of steps you want to extract.")
+    parser.add_argument("--dump-path", "-dp", type=str, help="Path to dump the resulting files.")
+    args = parser.parse_args()
+
+    file_name = args.path_to_proof_and_state_file.split("/")[-1]
+    k_step = args.k
+
+    with open(os.path.join(args.dump_path, f"last_{k_step}_step_{file_name}"), "w") as f_out, \
+            open(args.path_to_proof_and_state_file) as f_in:
+        for line in tqdm(f_in.readlines()):
+            line = line.strip()
+            proof = line.split("<PS_SEP>")[0].strip().lstrip("Proof: ")
+            state = line.split("State:")[1].strip()
+            
+            proof_steps = proof.split("\\n")
+            last_k_proof_steps = [element.strip() for element in proof_steps[-k_step:]]
+            last_k_proof_steps_string = " \\n ".join(last_k_proof_steps)
+            new_line = f"Proof: {last_k_proof_steps_string} State: {state}"
+            f_out.write(new_line)
+            f_out.write("\n")

itp_interface/pisa/scripts/extract_proof_corpus.py

@@ -0,0 +1,26 @@
+import os
+import sys
+from tqdm import tqdm
+
+file_path = sys.argv[1]
+tgt_file_path = file_path.replace("src", "tgt")
+output_file_path = sys.argv[2]
+
+valid_proofs = list()
+
+lines = open(file_path).readlines()
+tgt_lines = open(tgt_file_path).readlines()
+
+theorem_declaration = lines[0].strip()
+for i, line in tqdm(enumerate(lines)):
+    line = line.strip()
+    if theorem_declaration in line:
+        pass
+    else:
+        theorem_declaration = line
+        valid_proofs.append(lines[i-1].strip() + " \\n " + tgt_lines[i-1].strip())
+
+with open(output_file_path, "w") as fout:
+    for proof in valid_proofs:
+        fout.write(proof)
+        fout.write("\n")

itp_interface/pisa/scripts/gather_hammer_results.py

@@ -0,0 +1,27 @@
+import os
+
+proof_dict = {}
+for file_name in os.listdir("results/hammer_eval"):
+    if file_name.endswith("out"):
+        theorem_number = file_name.split("_")[0]
+        info_file_name = file_name.replace("out", "info")
+        info_file_path = os.path.join("results/hammer_eval", info_file_name)
+        if os.path.isfile(info_file_path):
+            info_lines = open(info_file_path).readlines()
+            proved = open(os.path.join("results/hammer_eval", file_name)).read().strip()
+            proved = True if (proved.startswith("t") or proved.startswith("T")) else False
+            proof_dict[theorem_number] = (proved, info_lines)
+
+with open("results/hammer_eval/success", "w") as fhand:
+    for theorem_number in proof_dict:
+        info_tuple = proof_dict[theorem_number]
+        if info_tuple[0]:
+            fhand.writelines(info_tuple[1])
+            fhand.write("\n\n")
+
+with open("results/hammer_eval/failure", "w") as fhand:
+    for theorem_number in proof_dict:
+        info_tuple = proof_dict[theorem_number]
+        if not info_tuple[0]:
+            fhand.writelines(info_tuple[1])
+            fhand.write("\n\n")

itp_interface/pisa/scripts/length_in_char_stats.py

@@ -0,0 +1,20 @@
+import argparse
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Get the ")
+    parser.add_argument("--path-to-proof-and-state-file", "-ptpasf", type=str, help="Path to the file of the proof steps and states.")
+    args = parser.parse_args()
+
+    lengths = list()
+    with open(args.path_to_proof_and_state_file) as fin:
+        for line in fin.readlines():
+            lengths.append(len(line.strip()))
+            
+    maximum_sequence_length_in_chars = 256 * 3
+
+    print(f"Minimum: {min(lengths)}")
+    print(f"Maximum: {max(lengths)}")
+    print(f"Average: {sum(lengths)/len(lengths)}")
+    print(f"Median: {sorted(lengths)[int(len(lengths)/2)]}")
+    print(f"Retainment rate: {sorted(lengths + [maximum_sequence_length_in_chars]).index(maximum_sequence_length_in_chars) / (len(lengths)+1)}")

itp_interface/pisa/scripts/mix.py

@@ -0,0 +1,127 @@
+from tqdm import tqdm
+
+
+def fill_dict_state_only(file1, file2, dictionary):
+    with open(file1) as fsrc:
+        with open(file2) as ftgt:
+            src_lines = fsrc.readlines()
+            tgt_lines = ftgt.readlines()
+
+            print(src_lines[0])
+            print(tgt_lines[0])
+
+            for i in tqdm(range(len(src_lines))):
+                src_line = src_lines[i].strip()
+                tgt_line = tgt_lines[i].strip()
+
+                src_line = src_line.replace("State:", "[IS] GOAL") + " PROOFSTEP"
+
+                if src_line in dictionary:
+                    if tgt_line in dictionary[src_line]:
+                        pass
+                    else:
+                        dictionary[src_line].append(tgt_line)
+                else:
+                    dictionary[src_line] = [tgt_line]
+    return dictionary
+
+
+def fill_dict_proof_and_state(file1, file2, dictionary):
+    with open(file1) as fsrc:
+        with open(file2) as ftgt:
+            src_lines = fsrc.readlines()
+            tgt_lines = ftgt.readlines()
+
+            print(src_lines[0])
+            print(tgt_lines[0])
+
+            for i in tqdm(range(len(src_lines))):
+                src_line = src_lines[i].strip()
+                tgt_line = tgt_lines[i].strip()
+
+                src_line = src_line.replace("Proof:", "[IS] PROOF").replace("State", "[IS] GOAL") + " PROOFSTEP"
+
+                if src_line in dictionary:
+                    if tgt_line in dictionary[src_line]:
+                        pass
+                    else:
+                        dictionary[src_line].append(tgt_line)
+                else:
+                    dictionary[src_line] = [tgt_line]
+    return dictionary
+
+
+def fill_dict_conjecture(file1, file2, dictionary):
+    with open(file1) as fsrc:
+        with open(file2) as ftgt:
+            src_lines = fsrc.readlines()
+            tgt_lines = ftgt.readlines()
+
+            print(src_lines[0])
+            print(tgt_lines[0])
+
+            for i in tqdm(range(len(src_lines))):
+                src_line = src_lines[i].strip()
+                tgt_line = tgt_lines[i].strip()
+
+                src_line = "[IS] GOAL " + src_line + " PROOFSTEP"
+
+                if src_line in dictionary:
+                    if tgt_line in dictionary[src_line]:
+                        pass
+                    else:
+                        dictionary[src_line].append(tgt_line)
+                else:
+                    dictionary[src_line] = [tgt_line]
+    return dictionary
+
+srcs_to_tgts = dict()
+srcs_to_tgts = fill_dict_state_only("data/seq2seq/seq2seq_with_state/train.src", "data/seq2seq/seq2seq_with_state/train.tgt",
+                         srcs_to_tgts)
+srcs_to_tgts = fill_dict_proof_and_state("data/seq2seq/seq2seq_with_proof_and_state/train.src", "data/seq2seq/seq2seq_with_proof_and_state/train.tgt",
+                         srcs_to_tgts)
+srcs_to_tgts = fill_dict_conjecture("data/conjecturer_seq2seq/train.src", "data/conjecturer_seq2seq/train.tgt",
+                         srcs_to_tgts)
+
+with open("data/mixture/train.src", "w") as fsrc_out:
+    with open("data/mixture/train.tgt", "w") as ftgt_out:
+        for src_line, tgt_lines in srcs_to_tgts.items():
+            for tgt_line in tgt_lines:
+                fsrc_out.write(src_line)
+                fsrc_out.write("\n")
+                ftgt_out.write(tgt_line)
+                ftgt_out.write("\n")
+
+srcs_to_tgts = dict()
+srcs_to_tgts = fill_dict_state_only("data/seq2seq/seq2seq_with_state/val.src", "data/seq2seq/seq2seq_with_state/val.tgt",
+                         srcs_to_tgts)
+srcs_to_tgts = fill_dict_proof_and_state("data/seq2seq/seq2seq_with_proof_and_state/val.src", "data/seq2seq/seq2seq_with_proof_and_state/val.tgt",
+                         srcs_to_tgts)
+srcs_to_tgts = fill_dict_conjecture("data/conjecturer_seq2seq/val.src", "data/conjecturer_seq2seq/val.tgt",
+                         srcs_to_tgts)
+
+with open("data/mixture/val.src", "w") as fsrc_out:
+    with open("data/mixture/val.tgt", "w") as ftgt_out:
+        for src_line, tgt_lines in srcs_to_tgts.items():
+            for tgt_line in tgt_lines:
+                fsrc_out.write(src_line)
+                fsrc_out.write("\n")
+                ftgt_out.write(tgt_line)
+                ftgt_out.write("\n")
+
+srcs_to_tgts = dict()
+srcs_to_tgts = fill_dict_state_only("data/seq2seq/seq2seq_with_state/test.src", "data/seq2seq/seq2seq_with_state/test.tgt",
+                         srcs_to_tgts)
+srcs_to_tgts = fill_dict_proof_and_state("data/seq2seq/seq2seq_with_proof_and_state/test.src", "data/seq2seq/seq2seq_with_proof_and_state/test.tgt",
+                         srcs_to_tgts)
+srcs_to_tgts = fill_dict_conjecture("data/conjecturer_seq2seq/test.src", "data/conjecturer_seq2seq/test.tgt",
+                         srcs_to_tgts)
+
+with open("data/mixture/test.src", "w") as fsrc_out:
+    with open("data/mixture/test.tgt", "w") as ftgt_out:
+        for src_line, tgt_lines in srcs_to_tgts.items():
+            for tgt_line in tgt_lines:
+                fsrc_out.write(src_line)
+                fsrc_out.write("\n")
+                ftgt_out.write(tgt_line)
+                ftgt_out.write("\n")

itp_interface/pisa/scripts/results_stat.py

@@ -0,0 +1,52 @@
+import os
+import sys
+
+def stats_from_an_exp(exp_folder):
+    total = 0
+    correct = 0
+    for file_name in os.listdir(exp_folder):
+        if file_name.startswith("test_result"):
+            theorem_number = int(file_name.split("_")[-1])
+            total += 1
+            correct += float(open(os.path.join(exp_folder, file_name)).read())
+
+    print("Total: ", total)
+    success = 0
+    timeout = 0
+    fuelout = 0
+    queueout = 0
+    total = 0
+    for file_name in os.listdir(exp_folder):
+        if file_name.startswith("test_cause"):
+            total += 1
+            test_cause = open(os.path.join(exp_folder, file_name)).read()
+            if test_cause.startswith("Proved"):
+                success += 1
+            elif test_cause.startswith("Overall"):
+                timeout += 1
+            elif test_cause.startswith("Out"):
+                fuelout += 1
+            elif test_cause.startswith("Queue"):
+                queueout += 1
+            else:
+                raise NotImplementedError
+    print("Success | timeout | out of fuel | queue empty proportion:\n {} | {} | {} | {}".format(
+        success/total, timeout/total, fuelout/total, queueout/total
+    ))
+
+
+all_folders = True
+for file_name in os.listdir(sys.argv[1]):
+    if not file_name.startswith("."):
+        if not os.path.isdir(os.path.join(sys.argv[1], file_name)):
+            all_folders = False
+
+if not all_folders:
+    print(sys.argv[1])
+    stats_from_an_exp(sys.argv[1])
+else:
+    for file_name in os.listdir(sys.argv[1]):
+        try:
+            stats_from_an_exp(os.path.join(sys.argv[1], file_name))
+        except:
+            pass