itp-interface 1.0.0__py3-none-any.whl

itp_interface/tools/repl/README.md

@@ -0,0 +1,174 @@
+# A read-eval-print-loop for Lean 4
+
+Run using `lake exe repl`.
+Communicates via JSON on stdin and stdout.
+Commands should be separated by blank lines.
+
+The REPL works both in "command" mode and "tactic" mode.
+
+## Command mode
+
+In command mode, you send complete commands (e.g. declarations) to the REPL.
+
+Commands may be of the form
+
+```json
+{ "cmd" : "def f := 2" }
+```
+
+```json
+{ "cmd" : "example : f = 2 := rfl", "env" : 1 }
+```
+
+The `env` field, if present,
+must contain a number received in the `env` field of a previous response,
+and causes the command to be run in the existing environment.
+
+If there is no `env` field, a new environment is created.
+
+You can only use `import` commands when you do not specify the `env` field.
+
+You can backtrack simply by using earlier values for `env`.
+
+The response includes:
+* A numeric label for the `Environment` after your command,
+  which you can use as the starting point for subsequent commands.
+* Any messages generated while processing your command.
+* A list of the `sorry`s in your command, including
+  * their expected type, and
+  * a numeric label for the proof state at the `sorry`, which you can then use in tactic mode.
+
+Example output:
+
+```json
+{"sorries":
+ [{"pos": {"line": 1, "column": 18},
+   "endPos": {"line": 1, "column": 23},
+   "goal": "⊢ Nat",
+   "proofState": 0}],
+ "messages":
+ [{"severity": "error",
+   "pos": {"line": 1, "column": 23},
+   "endPos": {"line": 1, "column": 26},
+   "data":
+   "type mismatch\n  rfl\nhas type\n  f = f : Prop\nbut is expected to have type\n  f = 2 : Prop"}],
+ "env": 6}
+```
+
+showing any messages generated, and sorries with their goal states.
+
+## File mode
+
+There is a simple wrapper around command mode that allows reading in an entire file.
+
+If `test/file.lean` contains
+```lean
+def f : Nat := 37
+
+def g := 2
+
+theorem h : f + g = 39 := by exact rfl
+```
+
+then
+```
+echo '{"path": "test/file.lean", "allTactics": true}' | lake exe repl
+```
+results in output
+```json
+{"tactics":
+ [{"tactic": "exact rfl",
+   "proofState": 0,
+   "pos": {"line": 5, "column": 29},
+   "goals": "⊢ f + g = 39",
+   "endPos": {"line": 5, "column": 38}}],
+ "env": 0}
+ ```
+
+## Tactic mode (experimental)
+
+To enter tactic mode issue a command containing a `sorry`,
+and then use the `proofState` index returned for each `sorry`.
+
+Example usage:
+```json
+{"cmd" : "def f (x : Unit) : Nat := by sorry"}
+
+{"sorries":
+ [{"proofState": 0,
+   "pos": {"line": 1, "column": 29},
+   "goal": "x : Unit\n⊢ Nat",
+   "endPos": {"line": 1, "column": 34}}],
+ "messages":
+ [{"severity": "warning",
+   "pos": {"line": 1, "column": 4},
+   "endPos": {"line": 1, "column": 5},
+   "data": "declaration uses 'sorry'"}],
+ "env": 0}
+
+{"tactic": "apply Int.natAbs", "proofState": 0}
+
+{"proofState": 1, "goals": ["x : Unit\n⊢ Int"]}
+
+{"tactic": "exact -37", "proofState": 1}
+
+{"proofState": 2, "goals": []}
+```
+
+You can use `sorry` in tactic mode.
+The result will contain additional `proofState` identifiers for the goal at each sorry.
+
+At present there is nothing you can do with a completed proof state:
+we would like to extend this so that you can replace the original `sorry` with your tactic script,
+and obtain the resulting `Environment`
+
+## Pickling
+
+The REPL supports pickling environments and proof states to disk as `.olean` files.
+As long as the same imports are available, it should be possible to move such an `.olean` file
+to another machine and unpickle into a new REPL session.
+
+The commands are
+
+```json
+{"pickleTo": "path/to/file.olean", "env": 7}
+
+{"pickleTo": "path/to/file.olean", "proofState": 17}
+
+{"unpickleEnvFrom": "path/to/file.olean"}
+
+{"unpickleProofStateFrom": "path/to/file.olean"}
+```
+
+The unpickling commands will report the new "env" or "proofState" identifier that
+you can use in subsequent commands.
+
+Pickling is quite efficient:
+* we don't record full `Environment`s, only the changes relative to imports
+* unpickling uses memory mapping
+* file sizes are generally small, but see https://github.com/digama0/leangz if compression is
+  desirable
+
+## Using the REPL from another project
+
+Set up your project as usual using `lake new` or `lake init`
+(or the interactive setup GUI available via the VSCode extension under the `∀` menu).
+
+In that project, add `require` statements in the `lakefile.lean` for any dependencies you need
+(e.g. Mathlib). (You probably should verify that `lake build` works as expected in that project.)
+
+Now you can run the REPL as:
+```shell
+lake env ../path/to/repl/.lake/build/bin/repl < commands.in
+```
+(Here `../path/to/repl/` represents the path to your checkout of this repository,
+in which you've already run `lake build`.)
+
+The `lake env` prefix sets up the environment associated to your local project, so that the REPL
+can find needed imports.
+
+## Future work
+
+* Replay tactic scripts from tactic mode back into the original `sorry`.
+* Currently if you create scoped environment extensions (e.g. scoped notations) in a session
+  these are not correctly pickled and unpickled in later sessions.

itp_interface/tools/repl/REPL/Frontend.lean

@@ -0,0 +1,47 @@
+/-
+Copyright (c) 2023 Scott Morrison. All rights reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Authors: Scott Morrison
+-/
+import Lean.Elab.Frontend
+
+open Lean Elab
+
+namespace Lean.Elab.IO
+
+/--
+Wrapper for `IO.processCommands` that enables info states, and returns
+* the new command state
+* messages
+* info trees
+-/
+def processCommandsWithInfoTrees
+    (inputCtx : Parser.InputContext) (parserState : Parser.ModuleParserState)
+    (commandState : Command.State) : IO (Command.State × List Message × List InfoTree) := do
+  let commandState := { commandState with infoState.enabled := true }
+  let s ← IO.processCommands inputCtx parserState commandState <&> Frontend.State.commandState
+  pure (s, s.messages.msgs.toList, s.infoState.trees.toList)
+
+/--
+Process some text input, with or without an existing command state.
+If there is no existing environment, we parse the input for headers (e.g. import statements),
+and create a new environment.
+Otherwise, we add to the existing environment.
+
+Returns the resulting command state, along with a list of messages and info trees.
+-/
+def processInput (input : String) (cmdState? : Option Command.State)
+    (opts : Options := {}) (fileName : Option String := none) :
+    IO (Command.State × List Message × List InfoTree) := unsafe do
+  Lean.initSearchPath (← Lean.findSysroot)
+  enableInitializersExecution
+  let fileName   := fileName.getD "<input>"
+  let inputCtx   := Parser.mkInputContext input fileName
+  let (parserState, commandState) ← match cmdState? with
+  | none => do
+    let (header, parserState, messages) ← Parser.parseHeader inputCtx
+    let (env, messages) ← processHeader header opts messages inputCtx
+    pure (parserState, (Command.mkState env messages opts))
+  | some cmdState => do
+    pure ({ : Parser.ModuleParserState }, cmdState)
+  processCommandsWithInfoTrees inputCtx parserState commandState

itp_interface/tools/repl/REPL/JSON.lean

@@ -0,0 +1,186 @@
+/-
+Copyright (c) 2023 Scott Morrison. All rights reserved.
+Released under Apache 2.0 license as described in the file LICENSE.
+Authors: Scott Morrison
+-/
+import Lean.Data.Json
+import Lean.Message
+import Lean.Elab.InfoTree.Main
+
+open Lean Elab InfoTree
+
+namespace REPL
+
+structure CommandOptions where
+  allTactics : Option Bool := none
+  /--
+  Should be "full", "tactics", "original", or "substantive".
+  Anything else is ignored.
+  -/
+  infotree : Option String
+
+/-- Run Lean commands.
+If `env = none`, starts a new session (in which you can use `import`).
+If `env = some n`, builds on the existing environment `n`.
+-/
+structure Command extends CommandOptions where
+  env : Option Nat
+  cmd : String
+deriving ToJson, FromJson
+
+/-- Process a Lean file in a fresh environment. -/
+structure File extends CommandOptions where
+  path : System.FilePath
+deriving FromJson
+
+/--
+Run a tactic in a proof state.
+-/
+structure ProofStep where
+  proofState : Nat
+  tactic : String
+deriving ToJson, FromJson
+
+/-- Line and column information for error messages and sorries. -/
+structure Pos where
+  line : Nat
+  column : Nat
+deriving ToJson, FromJson
+
+/-- Severity of a message. -/
+inductive Severity
+  | trace | info | warning | error
+deriving ToJson, FromJson
+
+/-- A Lean message. -/
+structure Message where
+  pos : Pos
+  endPos : Option Pos
+  severity : Severity
+  data : String
+deriving ToJson, FromJson
+
+/-- Construct the JSON representation of a Lean message. -/
+def Message.of (m : Lean.Message) : IO Message := do pure <|
+  { pos := ⟨m.pos.line, m.pos.column⟩,
+    endPos := m.endPos.map fun p => ⟨p.line, p.column⟩,
+    severity := match m.severity with
+    | .information => .info
+    | .warning => .warning
+    | .error => .error,
+    data := (← m.data.toString).trim }
+
+/-- A Lean `sorry`. -/
+structure Sorry where
+  pos : Pos
+  endPos : Pos
+  goal : String
+  /--
+  The index of the proof state at the sorry.
+  You can use the `ProofStep` instruction to run a tactic at this state.
+  -/
+  proofState : Option Nat
+deriving FromJson
+
+instance : ToJson Sorry where
+  toJson r := Json.mkObj <| .join [
+    [("goal", r.goal)],
+    [("proofState", toJson r.proofState)],
+    if r.pos.line ≠ 0 then [("pos", toJson r.pos)] else [],
+    if r.endPos.line ≠ 0 then [("endPos", toJson r.endPos)] else [],
+  ]
+
+/-- Construct the JSON representation of a Lean sorry. -/
+def Sorry.of (goal : String) (pos endPos : Lean.Position) (proofState : Option Nat) : Sorry :=
+  { pos := ⟨pos.line, pos.column⟩,
+    endPos := ⟨endPos.line, endPos.column⟩,
+    goal,
+    proofState }
+
+structure Tactic where
+  pos : Pos
+  endPos : Pos
+  goals : String
+  tactic : String
+  proofState : Option Nat
+deriving ToJson, FromJson
+
+/-- Construct the JSON representation of a Lean tactic. -/
+def Tactic.of (goals tactic : String) (pos endPos : Lean.Position) (proofState : Option Nat) : Tactic :=
+  { pos := ⟨pos.line, pos.column⟩,
+    endPos := ⟨endPos.line, endPos.column⟩,
+    goals,
+    tactic,
+    proofState }
+
+/--
+A response to a Lean command.
+`env` can be used in later calls, to build on the stored environment.
+-/
+structure CommandResponse where
+  env : Nat
+  messages : List Message := []
+  sorries : List Sorry := []
+  tactics : List Tactic := []
+  infotree : Option Json := none
+deriving FromJson
+
+def Json.nonemptyList [ToJson α] (k : String) : List α → List (String × Json)
+  | [] => []
+  | l  => [⟨k, toJson l⟩]
+
+instance : ToJson CommandResponse where
+  toJson r := Json.mkObj <| .join [
+    [("env", r.env)],
+    Json.nonemptyList "messages" r.messages,
+    Json.nonemptyList "sorries" r.sorries,
+    Json.nonemptyList "tactics" r.tactics,
+    match r.infotree with | some j => [("infotree", j)] | none => []
+  ]
+
+/--
+A response to a Lean tactic.
+`proofState` can be used in later calls, to run further tactics.
+-/
+structure ProofStepResponse where
+  proofState : Nat
+  goals : List String
+  messages : List Message := []
+  sorries : List Sorry := []
+  traces : List String
+deriving ToJson, FromJson
+
+instance : ToJson ProofStepResponse where
+  toJson r := Json.mkObj <| .join [
+    [("proofState", r.proofState)],
+    [("goals", toJson r.goals)],
+    Json.nonemptyList "messages" r.messages,
+    Json.nonemptyList "sorries" r.sorries,
+    Json.nonemptyList "traces" r.traces
+  ]
+
+/-- Json wrapper for an error. -/
+structure Error where
+  message : String
+deriving ToJson, FromJson
+
+structure PickleEnvironment where
+  env : Nat
+  pickleTo : System.FilePath
+deriving ToJson, FromJson
+
+structure UnpickleEnvironment where
+  unpickleEnvFrom : System.FilePath
+deriving ToJson, FromJson
+
+structure PickleProofState where
+  proofState : Nat
+  pickleTo : System.FilePath
+deriving ToJson, FromJson
+
+structure UnpickleProofState where
+  unpickleProofStateFrom : System.FilePath
+  env : Option Nat
+deriving ToJson, FromJson
+
+end REPL

itp_interface/tools/repl/REPL/Lean/ContextInfo.lean

@@ -0,0 +1,9 @@
+import Lean
+
+namespace Lean.Elab.ContextInfo
+
+/-- Pretty print an expression in the given `ContextInfo` with the given `LocalContext`. -/
+def ppExpr (ctx : ContextInfo) (lctx : LocalContext) (e : Expr) : IO Format :=
+  ctx.runMetaM lctx (do Meta.ppExpr (← instantiateMVars e))
+
+end Lean.Elab.ContextInfo

itp_interface/tools/repl/REPL/Lean/Environment.lean

@@ -0,0 +1,31 @@
+import REPL.Util.Pickle
+import Lean.Replay
+
+open System (FilePath)
+
+namespace Lean.Environment
+
+/--
+Pickle an `Environment` to disk.
+
+We only store:
+* the list of imports
+* the new constants from `Environment.constants`
+and when unpickling, we build a fresh `Environment` from the imports,
+and then add the new constants.
+-/
+def pickle (env : Environment) (path : FilePath) : IO Unit :=
+  _root_.pickle path (env.header.imports, env.constants.map₂)
+
+/--
+Unpickle an `Environment` from disk.
+
+We construct a fresh `Environment` with the relevant imports,
+and then replace the new constants.
+-/
+def unpickle (path : FilePath) : IO (Environment × CompactedRegion) := unsafe do
+  let ((imports, map₂), region) ← _root_.unpickle (Array Import × PHashMap Name ConstantInfo) path
+  let env ← importModules imports {} 0
+  return (← env.replay (HashMap.ofList map₂.toList), region)
+
+end Lean.Environment

itp_interface/tools/repl/REPL/Lean/InfoTree/ToJson.lean

@@ -0,0 +1,114 @@
+import REPL.Lean.InfoTree
+import REPL.Lean.ContextInfo
+
+/-!
+# Exporting an `InfoTree` as Json
+
+-/
+
+namespace Lean.Elab
+
+structure InfoTreeNode (α : Type) where
+  kind : String
+  node : Option α
+  children : List Json
+deriving ToJson
+
+deriving instance ToJson for Lean.Position
+
+structure Syntax.Range where
+  synthetic : Bool
+  start : Lean.Position
+  finish : Lean.Position
+deriving ToJson
+
+structure Syntax.Json where
+  pp : Option String
+  -- raw : String
+  range : Range
+deriving ToJson
+
+def _root_.Lean.Syntax.toRange (stx : Syntax) (ctx : ContextInfo) : Syntax.Range :=
+  let pos    := stx.getPos?.getD 0
+  let endPos := stx.getTailPos?.getD pos
+  { start := ctx.fileMap.toPosition pos
+    finish := ctx.fileMap.toPosition endPos
+    synthetic := match stx.getHeadInfo with
+    | .original .. => false
+    | _ => true }
+
+def _root_.Lean.Syntax.toJson (stx : Syntax) (ctx : ContextInfo) (lctx : LocalContext) : IO Syntax.Json := do
+  return {
+    pp := match (← ctx.ppSyntax lctx stx).pretty with
+      | "failed to pretty print term (use 'set_option pp.rawOnError true' for raw representation)" => none
+      | pp => some pp
+    -- raw := toString stx
+    range := stx.toRange ctx }
+
+structure TacticInfo.Json where
+  name : Option Name
+  stx : Syntax.Json
+  goalsBefore : List String
+  goalsAfter : List String
+deriving ToJson
+
+-- Note: this is not responsible for converting the children to Json.
+def TacticInfo.toJson (i : TacticInfo) (ctx : ContextInfo) : IO TacticInfo.Json := do
+  return {
+    name := i.name?
+    stx :=
+    { pp := Format.pretty (← i.pp ctx),
+      -- raw := toString i.info.stx,
+      range := i.stx.toRange ctx },
+    goalsBefore := (← i.goalState ctx).map Format.pretty,
+    goalsAfter := (← i.goalStateAfter ctx).map Format.pretty }
+
+structure CommandInfo.Json where
+  elaborator : Option Name
+  stx : Syntax.Json
+deriving ToJson
+
+def CommandInfo.toJson (info : CommandInfo) (ctx : ContextInfo) : IO CommandInfo.Json := do
+  return {
+    elaborator := match info.elaborator with | .anonymous => none | n => some n,
+    stx := ← info.stx.toJson ctx {} }
+
+structure TermInfo.Json where
+  elaborator : Option Name
+  stx : Syntax.Json
+  expectedType? : Option String
+  expr : String
+  isBinder : Bool
+deriving ToJson
+
+def TermInfo.toJson (info : TermInfo) (ctx : ContextInfo) : IO TermInfo.Json := do
+  return {
+    elaborator := match info.elaborator with | .anonymous => none | n => some n,
+    stx := ← info.stx.toJson ctx info.lctx,
+    expectedType? := ← info.expectedType?.mapM fun ty => do
+      pure (← ctx.ppExpr info.lctx ty).pretty
+    expr := (← ctx.ppExpr info.lctx info.expr).pretty
+    isBinder := info.isBinder }
+
+structure InfoTree.HoleJson where
+  goalState : String
+deriving ToJson
+
+partial def InfoTree.toJson (t : InfoTree) (ctx? : Option ContextInfo) : IO Json := do
+  match t with
+  | .context ctx t => t.toJson (ctx.mergeIntoOuter? ctx?)
+  | .node info children =>
+    if let some ctx := ctx? then
+      let node : Option Json ← match info with
+      | .ofTermInfo    info => some <$> (do pure <| Lean.toJson (← info.toJson ctx))
+      | .ofCommandInfo info => some <$> (do pure <| Lean.toJson (← info.toJson ctx))
+      | .ofTacticInfo  info => some <$> (do pure <| Lean.toJson (← info.toJson ctx))
+      | _                   => pure none
+      return Lean.toJson (InfoTreeNode.mk info.kind node (← children.toList.mapM fun t' => t'.toJson ctx))
+    else throw <| IO.userError "No `ContextInfo` available."
+  | .hole mvarId =>
+    if let some ctx := ctx? then
+     return Lean.toJson (InfoTree.HoleJson.mk (← ctx.runMetaM {} (do Meta.ppGoal mvarId)).pretty)
+    else throw <| IO.userError "No `ContextInfo` available."
+
+end Lean.Elab