itp-interface 1.0.0__py3-none-any.whl

itp_interface/pisa/src/main/scala/pisa/utils/TheoryManager.scala

@@ -0,0 +1,95 @@
+/*
+This is adapted from contents in Dominique Unruh's package scala-isabelle
+The package can be found on github at https://github.com/dominique-unruh/scala-isabelle
+This particular file is adapted from https://github.com/dominique-unruh/scala-isabelle/blob/master/src/test/scala/de/unruh/isabelle/experiments/TheoryManager.scala
+ */
+
+package pisa.utils
+
+import java.nio.file.{Path, Paths}
+// import scala.concurrent.ExecutionContext
+import de.unruh.isabelle.control.{Isabelle, OperationCollection}
+import de.unruh.isabelle.mlvalue.MLValue.{compileFunction, compileFunction0}
+import de.unruh.isabelle.pure.{Position, Theory, TheoryHeader, ToplevelState}
+import de.unruh.isabelle.mlvalue.{MLFunction, MLFunction0, MLFunction2, MLFunction3}
+import pisa.utils.TheoryManager.{Heap, Source, Text}
+import pisa.server.Transition
+import TheoryManager.Ops
+
+// Implicits
+import de.unruh.isabelle.mlvalue.Implicits._
+import de.unruh.isabelle.pure.Implicits._
+import scala.concurrent.ExecutionContext.Implicits.global
+
+class TheoryManager(var path_to_isa_bin: String, var wd : String) {
+  val setup: Isabelle.Setup = Isabelle.Setup(isabelleHome = Path.of(path_to_isa_bin),
+    sessionRoots = Nil,
+    userDir = None,
+    logic = "HOL",
+    workingDirectory = Path.of(wd),
+    build=false
+  )
+  implicit val isabelle: Isabelle = new Isabelle(setup)
+  
+  val command_exception: MLFunction3[Boolean, Transition.T, ToplevelState, ToplevelState] =
+    compileFunction[Boolean, Transition.T, ToplevelState, ToplevelState](
+    "fn (int, tr, st) => Toplevel.command_exception int tr st")
+  val init_toplevel: MLFunction0[ToplevelState] = compileFunction0[ToplevelState]("Toplevel.init_toplevel")
+  val parse_text: MLFunction2[Theory, String, List[(Transition.T, String)]] =
+    compileFunction[Theory, String, List[(Transition.T, String)]](
+    """fn (thy, text) => let
+      |  val transitions = Outer_Syntax.parse_text thy (K thy) Position.start text
+      |  fun addtext symbols [tr] =
+      |        [(tr, implode symbols)]
+      |    | addtext _ [] = []
+      |    | addtext symbols (tr::nextTr::trs) = let
+      |        val (this,rest) = Library.chop (Position.distance_of (Toplevel.pos_of tr, Toplevel.pos_of nextTr) |> Option.valOf) symbols
+      |        in (tr, implode this) :: addtext rest (nextTr::trs) end
+      |  in addtext (Symbol.explode text) transitions end""".stripMargin)
+  val toplevel_end_theory: MLFunction[ToplevelState, Theory] = compileFunction[ToplevelState, Theory]("Toplevel.end_theory Position.none")
+
+  def getTheorySource(name: String): Source = Heap(name)
+  def getTheory(source: Source)(implicit isabelle: Isabelle): Theory = source match {
+    case Heap(name) => Theory(name)
+    case Text(text, path, position) =>
+      var toplevel = init_toplevel().force.retrieveNow
+      var thy0 = beginTheory(source)
+      for ((transition, text) <- parse_text(thy0, text).force.retrieveNow) {
+        toplevel = command_exception(true, transition, toplevel).retrieveNow.force
+      }
+      toplevel_end_theory(toplevel).retrieveNow.force
+  }
+
+  def beginTheory(source: Source)(implicit isabelle: Isabelle): Theory = {
+    val header = getHeader(source)
+    val masterDir = source.path.getParent
+    // println(masterDir, header, header.imports.map(getTheorySource).map(getTheory))
+    Ops.begin_theory(masterDir, header, header.imports.map(getTheorySource).map(getTheory)).retrieveNow
+  }
+  def getHeader(source: Source)(implicit isabelle: Isabelle): TheoryHeader = source match {
+    case Text(text, path, position) => Ops.header_read(text, position).retrieveNow
+  }
+}
+
+object TheoryManager extends OperationCollection {
+  trait Source { def path : Path }
+  case class Heap(name: String) extends Source {
+    override def path: Path = Paths.get("INVALID")
+  }
+  case class File(path: Path) extends Source
+  case class Text(text: String, path: Path, position: Position) extends Source
+  object Text {
+    def apply(text: String, path: Path)(implicit isabelle: Isabelle): Text = new Text(text, path, Position.none)
+  }
+
+  //noinspection TypeAnnotation
+  protected final class Ops(implicit isabelle: Isabelle) {
+    val header_read = compileFunction[String, Position, TheoryHeader]("fn (text,pos) => Thy_Header.read pos text")
+    val begin_theory = compileFunction[Path, TheoryHeader, List[Theory], Theory](
+      "fn (path, header, parents) => Resources.begin_theory path header parents")
+  }
+
+  override protected def newOps(implicit isabelle: Isabelle, ec: scala.concurrent.ExecutionContext) = {
+    new this.Ops
+  }
+}

itp_interface/pisa/src/test/python/analyse_debug.py

@@ -0,0 +1,33 @@
+import os
+import sys
+
+state_only_total_ground_truth_length = 0
+proof_and_state_total_ground_truth_length = 0
+state_only_proved = 0
+state_only_matched = 0
+proof_state_proved = 0
+proof_state_matched = 0
+
+total_one_liners = 0
+for file_name in os.listdir("debug"):
+    if file_name.startswith("debug"):
+        file_content = open(os.path.join("debug", file_name)).readlines()
+        ground_truth_length = int(file_content[0].strip("\n").split(" ")[-2])
+        if ground_truth_length == 1:
+            total_one_liners += 1
+            if file_content[1].strip("\n").split(" ")[-2].startswith("t"):
+                state_only_proved += 1
+                state_only_matched += int(file_content[2].strip("\n").split(" ")[-2])
+                state_only_total_ground_truth_length += ground_truth_length
+            if file_content[3].strip("\n").split(" ")[-2].startswith("t"):
+                proof_state_proved += 1
+                proof_state_matched += int(file_content[4].strip("\n").split(" ")[-2])
+                proof_and_state_total_ground_truth_length += ground_truth_length
+
+print("State only proved: {}".format(state_only_proved))
+print("State only matched: {}".format(state_only_matched))
+print("State only ground truth length: {}".format(state_only_total_ground_truth_length))
+print("Proof and state proved: {}".format(proof_state_proved))
+print("Proof and state matched: {}".format(proof_state_matched))
+print("Proof and state ground truth length: {}".format(proof_and_state_total_ground_truth_length))
+print("Total one liners: {}".format(total_one_liners))

itp_interface/pisa/src/test/python/extract_test_seq2seq.py

@@ -0,0 +1,53 @@
+import os
+import sys
+import json
+import random
+from tqdm import tqdm
+
+
+random.seed(0)
+
+if sys.argv[1] == "state":
+    state_only = True
+elif sys.argv[1] == "proof_state":
+    state_only = False
+else:
+    raise NotImplementedError
+
+if state_only:
+    path_to_dir = "data/seq2seq/seq2seq_with_state"
+    output_dir = "debug/with_state"
+else:
+    path_to_dir = "data/seq2seq/seq2seq_with_proof_and_state"
+    output_dir = "debug/with_proof_and_state"
+
+line_index_to_pair = dict()
+with open(os.path.join(path_to_dir, "test.src")) as f1, open(os.path.join(path_to_dir, "test.tgt")) as f2:
+    src_lines = list(f1.readlines())
+    tgt_lines = list(f2.readlines())
+    for i in range(len(src_lines)):
+        src = src_lines[i].rstrip("\n")
+        tgt = tgt_lines[i].rstrip("\n")
+        line_index_to_pair[i] = (src, tgt)
+
+problem_index_to_line_indices = dict()
+with open("debug/good_indices.txt") as fin:
+    for line in fin.readlines():
+        line = line.rstrip("\n")
+        problem_index, line_index = line.split("|")
+        problem_index = int(problem_index.strip())
+        line_index = int(line_index.strip())
+
+        if problem_index not in problem_index_to_line_indices:
+            problem_index_to_line_indices[problem_index] = [line_index]
+        else:
+            problem_index_to_line_indices[problem_index].append(line_index)
+
+for problem_index in range(300):
+    line_indices = problem_index_to_line_indices[problem_index]
+    line_indices = sorted(line_indices)
+    with open(os.path.join(output_dir, "{}_pairs.txt".format(problem_index)), "w") as fout:
+        for line_index in line_indices:
+            pair = line_index_to_pair[line_index]
+            fout.write("{} <SEP> {}".format(pair[0], pair[1]))
+            fout.write("\n")

itp_interface/pisa/src/test/python/extract_test_theorem_ground_truth_indices.py

@@ -0,0 +1,31 @@
+import json
+import os
+import sys
+
+
+path_to_dir = "data/seq2seq/seq2seq_with_proof_and_state"
+
+ground_truth_dictionary = {}
+with open(os.path.join(path_to_dir, "test.src")) as f1, open(os.path.join(path_to_dir, "test.tgt")) as f2:
+    src_lines = list(f1.readlines())
+    tgt_lines = list(f2.readlines())
+    for i in range(len(src_lines)):
+        src = src_lines[i].rstrip("\n")
+        tgt = tgt_lines[i].rstrip("\n")
+        ground_truth_dictionary[src] = i
+
+good_ones = set()
+for file_name in os.listdir("universal_test_theorems"):
+    problem_index = int(file_name.split("_")[-1].split(".")[0])
+    if file_name.startswith("quick"):
+        file_content = json.load(open(os.path.join("universal_test_theorems", file_name)))
+        theorem_declaration = file_content[0][1]
+        for key in ground_truth_dictionary:
+            if theorem_declaration in key:
+                good_ones.add((problem_index, ground_truth_dictionary[key]))
+
+with open("debug/good_indices.txt", "w") as fout:
+    for good_index in good_ones:
+        # Problem index, line index
+        fout.write("{} | {}".format(good_index[0], good_index[1]))
+        fout.write("\n")

itp_interface/pisa/src/test/python/proof_originality.py

@@ -0,0 +1,24 @@
+import sys
+import os
+target_directory = sys.argv[1]
+
+test_sentences = []
+with open("data/seq2seq/seq2seq_with_state/test.tgt") as fhand:
+    for line in fhand.readlines():
+        test_sentences.append(line.strip().rstrip("\n"))
+test_sentences = set(test_sentences)
+
+succ_proofs = 0
+originality = 0
+for file_name in os.listdir(target_directory):
+    if file_name.startswith("succ"):
+        succ_proofs += 1
+        with open(os.path.join(target_directory, file_name)) as fhand:
+            ground_truth_lines = 0
+            total_lines = fhand.readlines()
+            for line in total_lines:
+                line = line.strip().rstrip("\n")
+                if line in test_sentences:
+                    ground_truth_lines += 1
+            originality += ground_truth_lines / len(total_lines)
+print("Originality: {}".format(1-originality/succ_proofs))

itp_interface/pisa/src/test/python/test_command_generator.py

@@ -0,0 +1,25 @@
+import glob
+import os
+import shutil
+
+number_of_processes = input("Enter the number of processes you want to run at the same time:\n").strip()
+number_of_processes = int(number_of_processes)
+
+script = 'echo "y" | sbt "test:runMain pisa.TransparentComparison {}"'
+
+total_cmds = list()
+pattern = "/home/ywu/PISA/universal_test_theorems/quick_test_name_*.json"
+
+for file_name in glob.glob(pattern, recursive=True):
+    total_cmds.append(script.format(file_name))
+
+process_number_to_cmds = {i: [] for i in range(number_of_processes)}
+for i, cmd in enumerate(total_cmds):
+    process_number_to_cmds[i%number_of_processes].append(cmd)
+
+for process_number, process_cmds in process_number_to_cmds.items():
+    with open("scripts/test_search_{}.sh".format(process_number), "w") as f:
+        for process_cmd in process_cmds:
+            f.write(process_cmd+"\n")
+            f.write("PIDmain=$!\n")
+            f.write("wait $PIDmain\n")

itp_interface/pisa/src/test/python/test_model_sequence_accuracy.py

@@ -0,0 +1,70 @@
+import os
+import sys
+import json
+import random
+from tqdm import tqdm
+
+random.seed(0)
+
+if sys.argv[1] == "state":
+    state_only = True
+elif sys.argv[1] == "proof_state":
+    state_only = False
+else:
+    raise NotImplementedError
+
+final_line = " PROOFSTEP\", \"best_of\": 1, \"temperature\": 0, \"max_tokens\": 128, \"n\": 1}'"
+if state_only:
+    path_to_dir = "pisa_seq2seq/state_only"
+    preceding_line = """curl https://api.openai.com/v1/engines/formal-small-isabelle-v6-c4/completions -H 'Content-Type: application/json' -H 'Authorization: Bearer <OPENAI_TOKEN>' -d '{\"prompt\": \""""
+
+else:
+    path_to_dir = "pisa_seq2seq/proof_and_state"
+    preceding_line = """curl https://api.openai.com/v1/engines/formal-small-isabelle-wproof-v1-c4/completions -H 'Content-Type: application/json' -H 'Authorization: Bearer <OPENAI_TOKEN>' -d '{\"prompt\": \""""
+
+ground_truth_dictionary = {}
+
+with open(os.path.join(path_to_dir, "val.src")) as f1, open(os.path.join(path_to_dir, "val.tgt")) as f2:
+    src_lines = list(f1.readlines())
+    tgt_lines = list(f2.readlines())
+    for i in range(len(src_lines)):
+        src = src_lines[i].rstrip("\n")
+        tgt = tgt_lines[i].rstrip("\n")
+        ground_truth_dictionary[src] = tgt
+        # print(src)
+        # print(tgt)
+        # break
+
+
+def preprocess(cmd, state_only):
+    if state_only:
+        returned_cmd = cmd.replace("State:", "[IS] GOAL").replace("\\", "\\\\").replace("\n", "\\n")\
+            .replace("\"", "\\\"").replace("\'", "\\u0027")
+    else:
+        returned_cmd = cmd.replace("Proof:", "[IS] PROOF").replace("State:", "[IS] GOAL").replace("\\", "\\\\")\
+            .replace("\n", "\\n") \
+            .replace("\"", "\\\"").replace("\'", "\\u0027")
+    return returned_cmd
+
+
+random_keys = list(ground_truth_dictionary.keys())
+random.shuffle(random_keys)
+correct = 0
+for src_line in tqdm(random_keys[:1000]):
+    cmd = preprocess(src_line, state_only)
+    cmd = preceding_line + cmd + final_line
+    # print(cmd)
+    stream = os.popen(cmd)
+    output = stream.read().rstrip("\n")
+    try:
+        text = json.loads(output)["choices"][0]["text"].strip()
+        # print(text)
+        # print(text)
+        # print(ground_truth_dictionary[src_line])
+        # print(text == ground_truth_dictionary[src_line])
+        if text == ground_truth_dictionary[src_line]:
+            correct += 1
+    except Exception as e:
+        print(e)
+
+print(correct/1000)

itp_interface/pisa/src/test/scala/pisa/Easy.scala

@@ -0,0 +1,26 @@
+package pisa
+
+import de.unruh.isabelle.control.Isabelle
+import de.unruh.isabelle.pure.Theory
+
+import java.nio.file.Path
+import scala.concurrent.ExecutionContext
+
+object Easy {
+  implicit val ec: ExecutionContext = ExecutionContext.global
+  val path_to_isa_bin: String = "/Applications/Isabelle2020.app/Isabelle"
+  val setup: Isabelle.Setup = Isabelle.Setup(
+    isabelleHome = Path.of(path_to_isa_bin),
+    sessionRoots = Seq(Path.of("/Users/qj213/Projects/afp-2021-02-11/thys")),
+    userDir = None,
+    logic = "Functional-Automata",
+    workingDirectory = Path.of("/Users/qj213/Projects/afp-2021-02-11/thys"),
+    build = false
+  )
+  implicit val isabelle: Isabelle = new Isabelle(setup)
+
+  def main(args: Array[String]): Unit = {
+    //    Theory("Functional-Automata.RegExp2NA").force.await
+    Theory("Abstract-Rewriting.Seq").force.await
+  }
+}

itp_interface/pisa/src/test/scala/pisa/TestCurl.scala

@@ -0,0 +1,82 @@
+package pisa
+
+import net.liftweb.json._
+
+import scala.collection.mutable.ListBuffer
+import scala.sys.process._
+import scala.io.Source
+
+case class GetText(text: String)
+
+object TestCurl {
+  implicit val formats : DefaultFormats = DefaultFormats
+  def preprocess(cmd : String, state_only : Boolean): String = {
+    if (state_only) {
+      cmd.trim.replace("State:", "[IS] GOAL").replace("\\", "\\\\").replace("\n", "\\n")
+        .replace("\"", "\\\"").replace("\'", "\\u0027")
+    } else {
+      cmd.replace("Proof:", "[IS] PROOF").replace("State:", "[IS] GOAL").replace("\\", "\\\\")
+      .replace("\n", "\\n").replace("\"", "\\\"").replace("\'", "\\u0027")
+    }
+  }
+
+  def main(args: Array[String]): Unit = {
+    val state_only : Boolean = {
+      if (args(0).startsWith("state")) true
+      else false
+    }
+
+    val path_to_dir : String = {
+      if (state_only) "pisa_seq2seq/state_only"
+      else "pisa_seq2seq/proof_and_state"
+    }
+    val preceding_line : String = {
+      if (state_only) "curl https://api.openai.com/v1/engines/formal-small-isabelle-v6-c4/completions -H 'Content-Type: application/json' -H 'Authorization: Bearer <OPENAI_TOKEN>' -d '{\"prompt\": \""
+      else "curl https://api.openai.com/v1/engines/formal-small-isabelle-wproof-v1-c4/completions -H 'Content-Type: application/json' -H 'Authorization: Bearer <OPENAI_TOKEN>' -d '{\"prompt\": \""
+    }
+    val final_line = " PROOFSTEP\", \"best_of\": 1, \"temperature\": 0.1, \"max_tokens\": 128, \"n\": 1}'"
+
+    val src_name : String = path_to_dir + "/" + "val.src"
+    val tgt_name : String = path_to_dir + "/" + "val.tgt"
+
+    val src_open = Source.fromFile(src_name)
+    val src_list : List[String] = src_open.getLines.toList
+    src_open.close
+    val tgt_open = Source.fromFile(tgt_name)
+    val tgt_list : List[String] = tgt_open.getLines.toList
+    tgt_open.close
+
+    var src2tgt : Map[String, ListBuffer[String]] = Map()
+    for (i <- List.range(0, src_list.length)) {
+      val src : String = src_list(i)
+      if (src2tgt.contains(src)) {
+        var tgt_old_list : ListBuffer[String] = src2tgt(src)
+        tgt_old_list += tgt_list(i)
+        src2tgt += (src -> tgt_old_list)
+      } else {
+        src2tgt += (src_list(i) -> ListBuffer(tgt_list(i)))
+      }
+    }
+
+    val src_keys = src2tgt.keys.toList.take(1000)
+
+    var correct = 0
+    for (key <- src_keys) {
+      val test_value = src2tgt(key)
+      val test_str = preceding_line + preprocess(key, state_only) + final_line
+//      println(test_str)
+
+      val returned_str = test_str.!!
+      if (returned_str.contains("error")) {}
+      else {
+        val parsed_value = (parse(returned_str) \ "choices")(0).extract[GetText].text.trim
+
+        if (test_value.contains(parsed_value)) {
+          correct += 1
+        }
+      }
+    }
+    println(correct/1000.0)
+
+  }
+}

itp_interface/pisa/src/test/scala/pisa/TestIsa.scala

@@ -0,0 +1,27 @@
+package pisa
+
+import de.unruh.isabelle.control.Isabelle
+
+import java.nio.file.Path
+
+object TestIsa {
+  val setup: Isabelle.Setup = Isabelle.Setup(isabelleHome = Path.of("/Applications/Isabelle2020.app/Isabelle"),
+    sessionRoots = Nil,
+    userDir = None,
+    logic = "HOL",
+    workingDirectory = Path.of("./"),
+    build = false
+  )
+
+  implicit val isabelle: Isabelle = new Isabelle(setup)
+
+  println("Initialised isabelle")
+
+
+  def main(args: Array[String]): Unit = {
+    Thread.sleep(10000)
+    isabelle.destroy()
+    println("Destroyed isabelle")
+    Thread.sleep(100000)
+  }
+}

itp_interface/pisa/test.sh

@@ -0,0 +1,19 @@
+sbt "runMain pisa.server.PisaServer" &
+PIDmain=$!
+sleep 12
+python src/main/python/extract_and_load_problems.py --json-path first_stage_output/Functional-Automata/Automata.json --isa-path /home/ywu/Isabelle2020 --afp /home/ywu/afp-2021-02-11 --saving-directory second_stage_output &
+PID0=$!
+python src/main/python/extract_and_load_problems.py --json-path first_stage_output/Functional-Automata/MaxChop.json --isa-path /home/ywu/Isabelle2020 --afp /home/ywu/afp-2021-02-11 --saving-directory second_stage_output &
+PID1=$!
+python src/main/python/extract_and_load_problems.py --json-path first_stage_output/Functional-Automata/DA.json --isa-path /home/ywu/Isabelle2020 --afp /home/ywu/afp-2021-02-11 --saving-directory second_stage_output &
+PID2=$!
+python src/main/python/extract_and_load_problems.py --json-path first_stage_output/Functional-Automata/RegExp2NA.json --isa-path /home/ywu/Isabelle2020 --afp /home/ywu/afp-2021-02-11 --saving-directory second_stage_output &
+PID3=$!
+wait $PID0
+wait $PID1
+wait $PID2
+wait $PID3
+ps aux |grep scala |awk '{print $2}' |xargs kill
+ps aux |grep java |awk '{print $2}' |xargs kill
+ps aux |grep poly |awk '{print $2}' |xargs kill
+ps aux | grep "bash sbt" | awk '{print $2}' | xargs kill

itp_interface/repo/build.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+
+import sys
+root_dir = f"{__file__.split('itp_interface')[0]}"
+if root_dir not in sys.path:
+    sys.path.append(root_dir)
+import os
+import subprocess
+import yaml
+import random
+from multiprocessing import Pool
+from itp_interface.repo.clone import RepoCloner
+
+class RepoBuilder:
+    def __init__(self, folder, disable_build=False):
+        self.folder = folder
+        self.folder = os.path.abspath(self.folder)
+        self.disable_build = disable_build
+        self._check_git_installed()
+        self.cloner = RepoCloner(folder)
+
+    def _check_git_installed(self):
+        try:
+            subprocess.run(['git', '--version'], check=True, stdout=subprocess.PIPE)
+        except subprocess.CalledProcessError:
+            raise RuntimeError("Git is not installed on this machine.")
+
+    def build_repos(self, yaml_file):
+        with open(yaml_file, 'r') as stream:
+            data = yaml.safe_load(stream)
+            repos = data.get('repos', [])
+        # Using a multiprocessing Pool for parallel building
+        with Pool(processes=len(repos)) as pool:
+            pool.map(self.clone_repo, repos)
+        
+        # Build cannot be parallelized because of opam locks
+        for repo in repos:
+            self.build_repo(repo)
+    
+    def clone_repo(self, repo):
+        repo_name = next(iter(repo))
+        repo_data = repo[repo_name]
+        self.cloner.clone_repo(repo_data)
+    
+    def build_repo(self, repo):
+        print(f"Building repo: {repo}")
+        repo_name = next(iter(repo))
+        repo_data = repo[repo_name]
+        build_cmds = repo_data.get('build_cmds', [])
+        build_path = os.path.join(self.folder, repo_name)
+        self._build_repo(build_path, build_cmds, self.disable_build)
+        print(f"Finished building repo: {repo}")
+
+    def _build_repo(self, build_path, build_cmds, build_disabled):
+        random_id = random.randint(0, 100000)
+        os.chdir(build_path)
+        # Make a temporary shell script to run all build commands
+        with open(f"temptodelbuild{random_id}.sh", 'w') as f:
+            f.write("#!/bin/bash\n")
+            for cmd in build_cmds:
+                f.write(f"{cmd}\n")
+        os.system(f"chmod 777 ./temptodelbuild{random_id}.sh")
+        if not build_disabled:
+            # Run the shell script
+            os.system(f"./temptodelbuild{random_id}.sh")
+            # Clean up the shell script
+            os.remove(f"./temptodelbuild{random_id}.sh")
+
+# Example usage
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--repo_yaml", help="Path to the yaml file containing repo data")
+    parser.add_argument("--disable_build", action="store_true", help="Disable building the repos")
+    parser.add_argument("--repo_folder", default=".repo", help="Folder to clone and build the repos in")
+    args = parser.parse_args()
+    builder = RepoBuilder(args.repo_folder, args.disable_build)
+    builder.build_repos(args.repo_yaml)

itp_interface/repo/clone.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+import sys
+root_dir = f"{__file__.split('itp_interface')[0]}"
+if root_dir not in sys.path:
+    sys.path.append(root_dir)
+import os
+import subprocess
+import yaml
+
+class RepoCloner:
+    def __init__(self, folder):
+        self.folder = folder
+        self._check_git_installed()
+
+    def _check_git_installed(self):
+        try:
+            subprocess.run(['git', '--version'], check=True, stdout=subprocess.PIPE)
+        except subprocess.CalledProcessError:
+            raise RuntimeError("Git is not installed on this machine.")
+
+    def clone_repos(self, yaml_file):
+        with open(yaml_file, 'r') as stream:
+            data = yaml.safe_load(stream)
+            repos = data.get('repos', [])
+            for repo in repos:
+                assert len(repo) == 1, "Each repo entry should have exactly one key-value pair"
+                repo_name = list(repo.keys())[0]
+                print(f"Cloning {repo_name}")
+                repo = list(repo.values())[0]
+                self.clone_repo(repo)
+    
+    def clone_repo(self, repo_dict):
+        url = repo_dict.get('url')
+        commit = repo_dict.get('commit', None)
+        branch = repo_dict.get('branch', None)
+        if not branch:
+            branch = self._get_default_branch(url)
+        if not commit:
+            commit = self._get_latest_commit(url, branch)
+        self._clone_repo(url, commit, branch)        
+
+    def _clone_repo(self, url, commit, branch):
+        repo_name = url.split('/')[-1].replace('.git', '')
+        repo_path = os.path.join(self.folder, repo_name)
+        if not os.path.exists(repo_path):
+            os.makedirs(repo_path)
+        os.chdir(repo_path)
+        print(f"Cloning {url} at commit {commit} to branch {branch}")
+        subprocess.run(['git', 'clone', '--single-branch', '--branch', branch, url, '.'])
+        subprocess.run(['git', 'checkout', commit])
+        print(f"Cloned {url} at commit {commit} to branch {branch}")
+
+    def _get_default_branch(self, url):
+        try:
+            output = subprocess.run(['git', 'ls-remote', '--symref', url, 'HEAD'], stdout=subprocess.PIPE)
+            default_branch = None
+            for line in output.stdout.decode().split('\n'):
+                if line.startswith('ref: refs/heads/'):
+                    default_branch = line.split('/')[-1].split('\t')[0].strip()
+                    break
+            if not default_branch:
+                raise RuntimeError("Unable to determine default branch for repository: {}".format(url))
+            return default_branch
+        except subprocess.CalledProcessError:
+            raise RuntimeError("Unable to determine default branch for repository: {}".format(url))
+
+    def _get_latest_commit(self, url, branch):
+        try:
+            output = subprocess.run(['git', 'ls-remote', url, 'refs/heads/' + branch], stdout=subprocess.PIPE)
+            latest_commit = output.stdout.decode().split()[0]
+            return latest_commit
+        except subprocess.CalledProcessError:
+            raise RuntimeError("Unable to fetch latest commit for repository: {}".format(url))
+
+# Example usage
+if __name__ == "__main__":
+    cloner = RepoCloner(".repo")
+    cloner.clone_repos("src/itp_interface/main/config/repo/coq_repos.yaml")