janus-llm 4.2.0__py3-none-any.whl → 4.3.5__py3-none-any.whl

janus/parsers/partition_parser.py

@@ -9,7 +9,7 @@ from langchain_core.messages import BaseMessage
 from langchain_core.pydantic_v1 import BaseModel, Field
 
 from janus.language.block import CodeBlock
-from janus.parsers.parser import JanusParser
+from janus.parsers.parser import JanusParser, JanusParserException
 from janus.utils.logger import create_logger
 
 log = create_logger(__name__)
@@ -36,6 +36,29 @@ class PartitionList(BaseModel):
     )
 
 
+# The following IDs appear in the prompt example. If the LLM produces them,
+#  they should be ignored
+EXAMPLE_IDS = {
+    "0d2f4f8d",
+    "def2a953",
+    "75315253",
+    "e7f928da",
+    "1781b2a9",
+    "2fe21e27",
+    "9aef6179",
+    "6061bd82",
+    "22bd0c30",
+    "5d85e19e",
+    "06027969",
+    "91b722fb",
+    "4b3f79be",
+    "k57w964a",
+    "51638s96",
+    "065o6q32",
+    "j5q6p852",
+}
+
+
 class PartitionParser(JanusParser, PydanticOutputParser):
     token_limit: int
     model: BaseLanguageModel
@@ -59,7 +82,10 @@ class PartitionParser(JanusParser, PydanticOutputParser):
         # Generate a unique ID for each line (ensure they are unique)
         line_ids = set()
         while len(line_ids) < len(self.lines):
-            line_ids.add(str(uuid.UUID(int=RNG.getrandbits(128), version=4))[:8])
+            line_id = str(uuid.UUID(int=RNG.getrandbits(128), version=4))[:8]
+            if line_id in EXAMPLE_IDS:
+                continue
+            line_ids.add(line_id)
 
         # Prepend each line with the corresponding ID, save the mapping
         self.line_id_to_index = {lid: i for i, lid in enumerate(line_ids)}
@@ -71,6 +97,11 @@ class PartitionParser(JanusParser, PydanticOutputParser):
     def parse(self, text: str | BaseMessage) -> str:
         if isinstance(text, BaseMessage):
             text = str(text.content)
+        original_text = text
+
+        # Strip everything outside the JSON object
+        begin, end = text.find("["), text.rfind("]")
+        text = text[begin : end + 1]
 
         try:
             out: PartitionList = super().parse(text)
@@ -78,26 +109,28 @@ class PartitionParser(JanusParser, PydanticOutputParser):
             log.debug(f"Invalid JSON object. Output:\n{text}")
             raise
 
+        # Get partition locations, discard reasoning
+        partition_locations = {partition.location for partition in out.__root__}
+
+        # Ignore IDs from the example input
+        partition_locations.difference_update(EXAMPLE_IDS)
+
         # Locate any invalid line IDs, raise exception if any found
-        invalid_splits = [
-            partition.location
-            for partition in out.__root__
-            if partition.location not in self.line_id_to_index
-        ]
+        invalid_splits = partition_locations.difference(self.line_id_to_index)
         if invalid_splits:
             err_msg = (
                 f"{len(invalid_splits)} line ID(s) not found in input: "
                 + ", ".join(invalid_splits)
             )
             log.warning(err_msg)
-            raise OutputParserException(err_msg)
+            raise JanusParserException(original_text, err_msg)
 
         # Map line IDs to indices (so they can be sorted and lines indexed)
         index_to_line_id = {0: "START", None: "END"}
         split_points = {0}
-        for partition in out.__root__:
-            index = self.line_id_to_index[partition.location]
-            index_to_line_id[index] = partition.location
+        for partition in partition_locations:
+            index = self.line_id_to_index[partition]
+            index_to_line_id[index] = partition
             split_points.add(index)
 
         # Get partition start/ends, chunks, chunk lengths
@@ -128,9 +161,10 @@ class PartitionParser(JanusParser, PydanticOutputParser):
                 "Oversized chunks:\n"
                 + "\n#############\n".join(chunk for _, chunk, _ in data)
             )
-            raise OutputParserException(
+            raise JanusParserException(
+                original_text,
                 f"The following segments are too long and must be "
-                f"further subdivided:\n{problem_points}"
+                f"further subdivided:\n{problem_points}",
             )
 
         return "\n<JANUS_PARTITION>\n".join(chunks)

janus/parsers/reqs_parser.py

@@ -2,10 +2,9 @@ import json
 import re
 
 from langchain.output_parsers.json import parse_json_markdown
-from langchain_core.exceptions import OutputParserException
 from langchain_core.messages import BaseMessage
 
-from janus.parsers.parser import JanusParser
+from janus.parsers.parser import JanusParser, JanusParserException
 from janus.utils.logger import create_logger
 
 log = create_logger(__name__)
@@ -20,6 +19,7 @@ class RequirementsParser(JanusParser):
     def parse(self, text: str | BaseMessage) -> str:
         if isinstance(text, BaseMessage):
             text = str(text.content)
+        original_text = text
 
         # TODO: This is an incorrect implementation (lstrip and rstrip take character
         #       lists and strip any instances of those characters, not the full str)
@@ -30,11 +30,14 @@ class RequirementsParser(JanusParser):
             obj = parse_json_markdown(text)
         except json.JSONDecodeError as e:
             log.debug(f"Invalid JSON object. Output:\n{text}")
-            raise OutputParserException(f"Got invalid JSON object. Error: {e}")
+            raise JanusParserException(
+                original_text, f"Got invalid JSON object. Error: {e}"
+            )
 
         if not isinstance(obj, dict):
-            raise OutputParserException(
-                f"Got invalid return object. Expected a dictionary, but got {type(obj)}"
+            raise JanusParserException(
+                original_text,
+                f"Got invalid return object. Expected a dictionary, but got {type(obj)}",
             )
         return json.dumps(obj)
 

janus/parsers/uml.py

@@ -3,10 +3,10 @@ import subprocess  # nosec
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 
-from langchain_core.exceptions import OutputParserException
 from langchain_core.messages import BaseMessage
 
 from janus.parsers.code_parser import CodeParser
+from janus.parsers.parser import JanusParserException
 from janus.utils.logger import create_logger
 
 log = create_logger(__name__)
@@ -14,6 +14,7 @@ log = create_logger(__name__)
 
 class UMLSyntaxParser(CodeParser):
     def _check_plantuml(self, text: str) -> None:
+        original_text = text
         # Leading newlines can break the parser, remove them
         text = text.replace("\\n", "\n").strip()
 
@@ -43,7 +44,7 @@ class UMLSyntaxParser(CodeParser):
             log.error(err_txt)
             raise Exception(err_txt)
 
-        # Check for bad outputs, raise OutputParserExceptions if so
+        # Check for bad outputs, raise JanusParserExceptions if so
         if "Error" in stderr or "Error" in stdout:
             err_txt = "Recieved UML parsing error(s)."
 
@@ -64,7 +65,7 @@ class UMLSyntaxParser(CodeParser):
                 err_txt += f"\nError located at line {i} must be fixed:\n"
                 err_txt += "\n".join(err_lines)
             log.warning(err_txt)
-            raise OutputParserException(err_txt)
+            raise JanusParserException(original_text, err_txt)
 
         if "Warning" in stdout or "Warning" in stderr:
             err_txt = "Recieved UML parsing warning (often due to missing PLANTUML)."
@@ -74,7 +75,7 @@ class UMLSyntaxParser(CodeParser):
                 err_txt += f"\nSTDOUT:\n```\n{stdout.strip()}\n```\n"
 
             log.warning(err_txt)
-            raise OutputParserException(err_txt)
+            raise JanusParserException(original_text, err_txt)
 
     def _get_error_lines(self, s: str) -> list[int]:
         return [int(x.group(1)) for x in re.finditer(r"Error line (\d+) in file:", s)]

janus/prompts/prompt.py

@@ -23,7 +23,7 @@ TEXT_OUTPUT = []
 # same language as the input, regardless of the `output-lang` argument.
 SAME_OUTPUT = ["document_inline"]
 
-JSON_OUTPUT = ["evaluate", "document", "document_madlibs", "requirements"]
+JSON_OUTPUT = ["evaluate", "document", "document_cloze", "requirements"]
 
 # Directory containing Janus prompt template directories and files
 JANUS_PROMPT_TEMPLATES_DIR = Path(__file__).parent / "templates"
@@ -109,7 +109,7 @@ class PromptEngine(ABC):
         source_language = source_language.lower()
         self.variables = dict(
             SOURCE_LANGUAGE=source_language,
-            FILE_SUFFIX=LANGUAGES[source_language]["suffix"],
+            FILE_SUFFIX=LANGUAGES[source_language]["suffixes"],
             SOURCE_CODE_EXAMPLE=LANGUAGES[source_language]["example"],
         )
         if target_language is not None:

janus/prompts/templates/README.md

@@ -0,0 +1,30 @@
+# Prompt Template Files
+
+Janus supports defining custom prompts in text files.
+
+```
+directory_name/
+    system.txt
+    human.txt
+    variables.json (optional)
+```
+
+## Prompt templates
+- `system.txt` contains text representing the system prompt template,
+  + Ex. "Your purpose is to understand {SOURCE_LANGUAGE} code."
+- `human.txt` contains text representing the human prompt template.
+  + Ex. "Summarize the contents of the following {SOURCE_LANGUAGE} code in {written_language} sentences.
+
+Both prompt templates can make use of f-string-style arguments, i.e. `{VARIABLE}`. Multiple lines are supported.
+
+To reuse a prompt, say for the same system directive with differing output styles, create a symbolic link to the original file.  For example:
+`ln -s ../document/system.txt system.txt`
+
+## Variables
+- (Optional) `variables.json` contains a JSON object representing additional variables and their values used in the templates above, beyond what is provided to Janus via command-line arguments.
+  + Ex.
+    ```
+    {
+        "written_language": "Spanish"
+    }
+    ```

janus/prompts/templates/basic_aggregation/human.txt

@@ -0,0 +1,6 @@
+Combine the following documentation for a program in {SOURCE_LANGUAGE} code into a single description
+Make sure to put the resultant description within triple backticks.
+Here are the representations:
+```
+{SOURCE_CODE}
+```

janus/prompts/templates/basic_aggregation/system.txt

@@ -0,0 +1 @@
+You are a senior software engineer named John and tasked with combining representations of code into a single representation.

janus/prompts/templates/basic_refinement/human.txt

@@ -0,0 +1,14 @@
+Please fix the following output generated by a large language model.
+Provide your corrected output in the same format as the original.
+The large language model was given the following prompt in triple backticks:
+```
+{ORIGINAL_PROMPT}
+```
+and produced the following output:
+```
+{OUTPUT}
+```
+but received the following errors:
+```
+{ERRORS}
+```

janus/prompts/templates/basic_refinement/system.txt

@@ -0,0 +1 @@
+You are a senior software engineer named John and tasked with fixing the output created by a large language model.

janus/prompts/templates/diagram/human.txt

@@ -0,0 +1,9 @@
+Generate a UML {DIAGRAM_TYPE} diagram using PLANTUML syntax that improves the readability of the following {SOURCE_LANGUAGE} code for a programmer.
+In your output, make sure to reformat any {SOURCE_LANGUAGE} code that would break PLANTUML syntax rules.
+Do not output any {SOURCE_LANGUAGE} code in the diagram.
+Make sure to capture all relevant syntax, functions and branching in the source code.
+Make sure to document all functions in the code
+Here is the source code:
+```
+{SOURCE_CODE}
+```

janus/prompts/templates/diagram/system.txt

@@ -0,0 +1 @@
+You are a senior software engineer named John and tasked with creating PLANTUML documentation of {SOURCE_LANGUAGE} code.

janus/prompts/templates/diagram_with_documentation/human.txt

@@ -0,0 +1,15 @@
+Generate a UML {DIAGRAM_TYPE} diagram using PLANTUML syntax that improves the readability of the following {SOURCE_LANGUAGE} code for a programmer.
+You are also provided with documentation for this code.
+In your output, make sure to reformat any {SOURCE_LANGUAGE} code that would break PLANTUML syntax rules.
+Do not output any {SOURCE_LANGUAGE} code in the diagram.
+Make sure to capture all relevant syntax, functions and branching in the source code.
+Make sure to document all functions in the code
+Make sure to put the resultant PLANTUML code within triple backticks.
+Here is the code documentation:
+```
+{DOCUMENTATION}
+```
+Here is the source code:
+```
+{SOURCE_CODE}
+```

janus/prompts/templates/diagram_with_documentation/system.txt

@@ -0,0 +1 @@
+You are a senior software engineer named John and tasked with creating PLANTUML documentation of {SOURCE_LANGUAGE} code.

janus/prompts/templates/document/human.txt

@@ -0,0 +1,10 @@
+Please explain the {SOURCE_LANGUAGE} code section below. Your response should be in plain text with no delimiters. It should contain a natural language description of the code's intended functionality; do not describe the execution step-by-step, simply explain the overall purpose. This description should be roughly one paragraph in length; multiple paragraphs may be used if and only if the code is particularly complex or has multiple independent functions.
+After this description, describe the expected initial state and/or inputs, the expected terminal state and/or outputs, and any potential exceptions that might arise in the code's execution.
+
+It is vital that you do not include any other context, questions, or text of any kind, other than the documentation for this piece of code. You should include all of the fields described above, and those fields only.
+
+Here is the code:
+
+```
+{SOURCE_CODE}
+```

janus/prompts/templates/document/system.txt

@@ -0,0 +1 @@
+You are a senior software engineer tasked with documenting {SOURCE_LANGUAGE} code.

janus/prompts/templates/document_cloze/human.txt

@@ -0,0 +1,11 @@
+The {SOURCE_LANGUAGE} code provided below has had its comments replaced by either `<INLINE_COMMENT #>` (for single-line comments) or `<BLOCK_COMMENT #>` (for multiple consecutive lines of comments), where `#` takes the place of an 8-character alphanumeric ID. You are to write replacement comments based on the source code.
+
+Return a JSON-formatted string where the keys are the alphanumeric IDs and the values are the comments that should be inserted in the code. Be sure to include comments for all placeholders present in the input. Do not provide any other commentary, do not write any code or additional comments.
+
+Example input: ```{EXAMPLE_INPUT}```
+Example output: ```{EXAMPLE_OUTPUT}```
+
+Please provide comments for the following code:
+```
+{SOURCE_CODE}
+```

janus/prompts/templates/document_cloze/system.txt

@@ -0,0 +1 @@
+You are a senior software engineer tasked with documenting {SOURCE_LANGUAGE} code.

janus/prompts/templates/document_cloze/variables.json

@@ -0,0 +1,4 @@
+{
+  "EXAMPLE_INPUT": " I '$D(CATLIST) Q\n ; <INLINE_COMMENT 14c59f05>\n S CAT=\"\"\n F  S CAT=$O(CATLIST(CAT)) Q:CAT=\"\"  D\n . S LDATE=$O(CATLIST(CAT,\"\"),-1)\n .; <INLINE_COMMENT 97a39adf>\n . S DATE=\"\"\n . F  S DATE=$O(CATLIST(CAT,DATE)) Q:DATE=LDATE  D\n .. S WCR=\"\"\n .. F  S WCR=$O(CATLIST(CAT,DATE,WCR)) Q:WCR=\"\"  D\n ... S FI=\"\"\n ... F  S FI=$O(CATLIST(CAT,DATE,WCR,FI)) Q:FI=\"\"  D\n .... S FIEVAL(FI)=0\n ....; <INLINE_COMMENT 4fd34837>\n .... S IND=0\n .... F  S IND=+$O(FIEVAL(FI,IND)) Q:IND=0  S FIEVAL(FI,IND)=0\n .; <BLOCK_COMMENT 997ec49a>\n . S (NTRUE,WCR)=0\n . F  S WCR=$O(CATLIST(CAT,LDATE,WCR)) Q:WCR=\"\"  D\n .. S FI=\"\"\n .. F  S FI=$O(CATLIST(CAT,LDATE,WCR,FI)) Q:FI=\"\"  D\n ... I NTRUE=0 D  Q\n ....; <INLINE_COMMENT 3ac32fb5>\n .... S (IND,NTRUE)=1\n .... F  S IND=+$O(FIEVAL(FI,IND)) Q:IND=0  S FIEVAL(FI,IND)=0\n ... S FIEVAL(FI)=0\n ...; <INLINE_COMMENT d04a8fdf>\n ... S IND=0\n ... F  S IND=+$O(FIEVAL(FI,IND)) Q:IND=0  S FIEVAL(FI,IND)=0\n Q\n",
+  "EXAMPLE_OUTPUT": "{\n  \"14c59f05\": \";Only the most recent HF in a category can be true.\",\n  \"97a39adf\": \";For each category set all but the most recent HF false.\",\n  \"4fd34837\": \";If there are multiple occurrences set them all false.\",\n  \"997ec49a\": \" .;\\n .;If there is more than on HF on the most recent date then only the\\n .;one with the highest WCR can be true. The highest possible WCR is 1.\\n .;Set all with lower WCRs false.\\n .;If the most recent health factor has multiple occurrences only\\n .;the first occurrence can be true.\",\n  \"3ac32fb5\": \";If there are multiple sub-occurrences set them all false.\"\n}"
+}

janus/prompts/templates/document_cloze/variables_asm.json

@@ -0,0 +1,4 @@
+{
+  "EXAMPLE_INPUT": "***********************************************************************\n* <BLOCK_COMMENT d8453f99>\n***********************************************************************\nZUIDSTCK AMODE 31\nZUIDSTCK RMODE 31\nZUIDSTCK CSECT\n         STM   R14,R12,12(R13)         <INLINE_COMMENT b2cc1643>\n         L     R1,0(R1)                <INLINE_COMMENT a315e5ca>\n         USING DSA,R1                  <INLINE_COMMENT 3155f463>\n         STCKE EISTOD                  <INLINE_COMMENT d84b5ebf>\n\n         LM    R14,R12,12(R13)         <INLINE_COMMENT a79a8e65>\n         XR    R15,R15                 <INLINE_COMMENT 47ad1b4d>\n         BR    R14                     <INLINE_COMMENT eb971719>\n",
+  "EXAMPLE_OUTPUT": "{\n  \"d8453f99\": \"Control Section\",\n  \"b2cc1643\": \"Save registers\",\n  \"a315e5ca\": \"Load parameter address\",\n  \"3155f463\": \"... tell assembler\",\n  \"d84b5ebf\": \"Save STCKE TOD\",\n  \"a79a8e65\": \"Load Registers\",\n  \"47ad1b4d\": \"Clear R15 (RC)\",\n  \"eb971719\": \"Return to calling program\"\n}"
+}

janus/prompts/templates/document_inline/human.txt

@@ -0,0 +1,13 @@
+Please add inline comments to the {SOURCE_LANGUAGE} code
+provided below in triple backticks.
+
+```
+{SOURCE_CODE}
+```
+
+Keep all source code in the output.
+
+Please add a comment at the top of the file which summarizes
+the purpose of the code.
+
+Please add comments to functions which summarize their functionality.

janus/prompts/templates/eval_prompts/incose/human.txt

@@ -0,0 +1,32 @@
+Below is a snippet of {SOURCE_LANGUAGE} code to use as reference for the following task:
+```
+{SOURCE_CODE}
+```
+
+Given the above code and the list of requirements I will soon supply, please evaluate each requirement individually based on the following criteria:
+
+C1 - Necessary: The need or requirement statement defines an essential capability, characteristic, constraint, or quality factor needed to satisfy a lifecycle concept, need, source, or parent requirement.
+C2 - Appropriate: The specific intent and amount of detail of the need or requirement statement is appropriate to the level (the level of abstraction, organization, or system architecture) of the entity to which it refers.
+C3 - Unambiguous: Need statements must be written such that the stakeholder intent is clear. Requirement statements must be stated such that the requirement can be interpreted in only one way by all the intended stakeholders.
+C4 - Complete: The requirement statement sufficiently describes the necessary capability, characteristic, constraint, or quality factor to meet the need, source, or parent requirement from which it was transformed without needing other information to understand the requirement.
+C5 - Singular: The stakeholder need or requirement statement should state a single capability, characteristic, constraint, or quality factor.
+C6 - Feasible: The need or requirement can be realized within entity constraints (for example: cost, schedule, technical, legal, ethical, safety) with acceptable risk.
+C7 - Verifiable: The requirement statement is structured and worded such that its realization can be verified to the approving authority’s satisfaction.
+C8 - Correct: The need statement must be an accurate representation of the lifecycle concept or source from which it was transformed. The requirement statement must be an accurate representation of the need, source, or parent requirement from which it was transformed.
+C9 - Conforming: Individual needs and requirements should conform to an approved standard pattern and style guide or standard for writing and managing needs and requirements.
+
+For each and every requirement below, you must indicate whether they "pass" or "fail" each of the above criteria. Briefly explain your reasoning before providing each pass/fail.
+
+Your response should be formatted as a list of JSON objects, with each object corresponding to one requirement. Each object should include 10 keys: `requirement_id`, `C1`, `C2`, ..., `C9`. `requirement_id` should have a string value that holds the 8-character UUID associated with the requirement. The other four values should each be a JSON object with two keys: `reasoning` (a clear explanation of why the criterion is passed or failed) and a `score` (the literal string "pass" or "fail").
+
+Be discerning in your evaluation; only very high-quality requirements should pass all criteria. Be a hard grader. If a requirement fails a criterion, be thorough and detailed in your explanation of why.
+
+Below is an example output for a snippet of code with three labeled requirements:
+```
+{EXAMPLE_OUTPUT}
+```
+
+Here are the requirements that you are to evaluate:
+{REQUIREMENTS}
+
+Don't forget to include your final scores in JSON format!

janus/prompts/templates/eval_prompts/incose/system.txt

@@ -0,0 +1 @@
+You are a software quality engineer, your job is to evaluate requirments according to a rubric.

janus/prompts/templates/eval_prompts/incose/variables.json

@@ -0,0 +1,3 @@
+{
+  "EXAMPLE_OUTPUT": "[\n  {\n    \"requirement_id\": \"c3caa172\",\n    \"requirement\": \"The UserID field must be followed by a comma (,) as a field separator.\",\n    \"C1\": {\n      \"reasoning\": \"This defines an essential characteristic of the data structure.\",\n      \"score\": \"pass\"\n    },\n    \"C2\": {\n      \"reasoning\": \"The detail provided is appropriate for the software's data structure level.\",\n      \"score\": \"pass\"\n    },\n    \"C3\": {\n      \"reasoning\": \"The statement is clear and unambiguous.\",\n      \"score\": \"pass\"\n    },\n    \"C4\": {\n      \"reasoning\": \"The requirement sufficiently describes the separator without needing additional information.\",\n      \"score\": \"pass\"\n    },\n    \"C5\": {\n      \"reasoning\": \"The requirement states a single characteristic.\",\n      \"score\": \"pass\"\n    },\n    \"C6\": {\n      \"reasoning\": \"Including a field separator is feasible.\",\n      \"score\": \"pass\"\n    },\n    \"C7\": {\n      \"reasoning\": \"The requirement can be verified by checking the data structure.\",\n      \"score\": \"pass\"\n    },\n    \"C8\": {\n      \"reasoning\": \"The requirement accurately represents the need for a separator.\",\n      \"score\": \"pass\"\n    },\n    \"C9\": {\n      \"reasoning\": \"The requirement conforms to standard pattern and style.\",\n      \"score\": \"pass\"\n    }\n  },\n  {\n    \"requirement_id\": \"fab48ab9\",\n    \"requirement\": \"The software must handle web communication parameters, including buffer addresses and lengths for web receive operations, query string management, and basic and query mode service program identifiers.\",\n    \"C1\": {\n      \"reasoning\": \"Defines essential capabilities for handling web communication parameters.\",\n      \"score\": \"pass\"\n    },\n    \"C2\": {\n      \"reasoning\": \"Appropriate detail for software handling web communication.\",\n      \"score\": \"pass\"\n    },\n    \"C3\": {\n      \"reasoning\": \"Clear and unambiguous about what the software must handle.\",\n      \"score\": \"pass\"\n    },\n    \"C4\": {\n      \"reasoning\": \"Sufficiently describes the necessary capabilities without needing additional information.\",\n      \"score\": \"pass\"\n    },\n    \"C5\": {\n      \"reasoning\": \"States multiple capabilities related to web communication parameters.\",\n      \"score\": \"fail\"\n    },\n    \"C6\": {\n      \"reasoning\": \"Feasible to implement within typical software constraints.\",\n      \"score\": \"pass\"\n    },\n    \"C7\": {\n      \"reasoning\": \"Verification can be done through testing the software's handling of these parameters.\",\n      \"score\": \"pass\"\n    },\n    \"C8\": {\n      \"reasoning\": \"Accurately represents the need for handling web communication parameters.\",\n      \"score\": \"pass\"\n    },\n    \"C9\": {\n      \"reasoning\": \"Conforms to standard requirement style and structure.\",\n      \"score\": \"pass\"\n    }\n  }\n]"
+}

janus/prompts/templates/eval_prompts/inline_comments/human.txt

@@ -0,0 +1,49 @@
+Please evaluate each comment in the provided {SOURCE_LANGUAGE} code based on the following criteria:
+
+Completeness - Does the comment address all capabilities of the relevant source code?
+
+4 - All essential functionality is documented.
+3 - Most essential functionality is documented.
+2 - Little essential functionality is documented.
+1 - No essential functionality is documented.
+
+
+Hallucination - Does the comment provide true information?
+
+4 - The comment provides only true information.
+3 - The comment provides mostly true information.
+2 - The comment provides mostly untrue information.
+1 - The comment is completely untrue.
+
+
+Readability - Is the comment clear to read?
+
+4 - The comment is well-written.
+3 - The comment has few problems.
+2 - The comment has many problems.
+1 - The comment is unreadable.
+
+Usefulness - Is the comment useful?
+
+4 - The comment helps an expert programmer understand the code better.
+3 - The comment helps an average programmer understand the code better.
+2 - The comment documents only trivial functionality.
+1 - The comment is not useful at any level.
+
+
+Look through the code and find each individual comment, they will be deliniated by <BLOCK_COMMENT id> or <INLINE_COMMENT id> where "id" is an 8-character UUID for the comment that follows.
+
+Each comment should be evaluated independently based on the above criteria. Your response should be formatted as a list of JSON objects, with each object corresponding to one comment. Each object should include five keys: `comment_id`, `completeness`, `hallucination`, `readability`, and `usefulness`. `comment_id` should have a string value that holds the 8-character UUID associated with the comment. The other four values should each be a JSON object with two keys: `reasoning` (a clear explanation of why the criteria is rated the way it is) and `score` (an integer rating from 1 to 4).
+
+Be discerning in your evaluation; only very high-quality comments should get top marks. Be a hard grader. If a comment is rated low, be thorough and detailed in your explanation of your score.
+
+Below is an example output for a snippet of code with three labeled comments:
+```{EXAMPLE_OUTPUT}```
+
+
+Evaluate the following code:
+```
+{SOURCE_CODE}
+```
+
+Don't forget to include your final scores in JSON format!

janus/prompts/templates/eval_prompts/inline_comments/system.txt

@@ -0,0 +1 @@
+You are a software quality engineer, your job is to evaluate comments in code according to a rubric.

janus/prompts/templates/eval_prompts/inline_comments/variables.json

@@ -0,0 +1,3 @@
+{
+  "EXAMPLE_OUTPUT": "[\n  {\n      \"comment_id\": \"abcd1234\",\n      \"completeness\": {\n          \"reasoning\": \"The comment completely describes the functionality of the block\",\n          \"score\": 4\n      },\n      \"hallucination\": {\n          \"reasoning\": \"The comment misrepresents the behavior of for-loops in this language, and as a result is entirely wrong about what this code does\",\n          \"score\": 1\n      },\n      \"readability\": {\n          \"reasoning\": \"The comment is easy to read and understand\",\n          \"score\": 4\n      },\n      \"usefulness\": {\n          \"reasoning\": \"The comment would be necessary to understand this block (if the comment were correct)\",\n          \"score\": 3\n      }\n  },\n  {\n      \"comment_id\": \"5678efgh\",\n      \"completeness\": {\n          \"reasoning\": \"There is no explanation of the GR_lby variable\",\n          \"score\": 2\n      },\n      \"hallucination\": {\n          \"reasoning\": \"The comment is accurate\",\n          \"score\": 4\n      },\n      \"readability\": {\n          \"reasoning\": \"The comment is wordy but generally understandable\",\n          \"score\": 3\n      },\n      \"usefulness\": {\n          \"reasoning\": \"The comment didn't explain anything that isn't already obvious from the code\",\n          \"score\": 1\n      }\n  },\n  {\n      \"comment_id\": \"00aa22bb\",\n      \"completeness\": {\n          \"reasoning\": \"The comment describes the functionality of the line, but not an explanation/origin of the hard-coded integer\",\n          \"score\": 3\n      },\n      \"hallucination\": {\n          \"reasoning\": \"The comment mentions a function that does not appear to exist in the file\",\n          \"score\": 2\n      },\n      \"readability\": {\n          \"reasoning\": \"The comment was difficult to follow, and formatted poorly\",\n          \"score\": 1\n      },\n      \"usefulness\": {\n          \"reasoning\": \"This line would be completely incomprehensible without this comment\",\n          \"score\": 4\n      }\n  }\n]"
+}

janus/prompts/templates/micromanaged_mumps_v1.0/human.txt

@@ -0,0 +1,23 @@
+Adhere to the following rules for translating MUMPS to Python:
+
+1. Routines from other files
+When a function from another file is invoked, treat the file like a module. Keep all imports at the beginning of the returned code.
+
+2. Naming Conventions
+Adhere to PEP8 for variable and function names. Improve readability when possible, making use of context and documentation. For example, a MUMPS variable like `RXQTY` might be translated to `prescription_quantity`.
+
+3. Ignore K(ill) Commands
+Memory allocation and garbage collection is generally handled automatically in Python, so any MUMPS K(ill) commands should be ignored.
+
+4. Arrays
+MUMPS arrays should generally be treated as nested dictionaries.
+
+5. Global Variables
+When globals (prepended by a circumflex) are used in a routine, treat them as coming from a mysql database. Assume that database credentials are stored in environment variables ('SQL_HOST`, `SQL_USER`, `SQL_PWD`, `SQL_DB`).
+
+
+Please convert the following MUMPS .m code found in between triple backticks into {TARGET_LANGUAGE} code. The returned code should also be delimited with triple backticks.
+
+```
+{SOURCE_CODE}
+```

janus/prompts/templates/micromanaged_mumps_v1.0/system.txt

@@ -0,0 +1,3 @@
+Your purpose is to convert MUMPS .m code
+into runnable {TARGET_LANGUAGE} code ({TARGET_LANGUAGE} version
+{TARGET_LANGUAGE_VERSION}).

janus/prompts/templates/micromanaged_mumps_v2.0/human.txt

@@ -0,0 +1,28 @@
+Adhere to the following rules for translating MUMPS to Python:
+
+1. Routines from other files
+When a function from another file is invoked (e.g. `D FUNC^ABC`), treat the file like a module (e.g. `from abc import func`, `func()` ). Keep all imports at the beginning of the returned code.
+
+2. Naming Conventions
+Adhere to PEP8 for variable and function names. Improve readability when possible, making use of context and documentation. For example, a MUMPS variable like `RXQTY` might be translated to `prescription_quantity`.
+
+3. Ignore K(ill) Commands
+Memory allocation and garbage collection is generally handled automatically in Python, so any MUMPS K(ill) commands should be ignored.
+
+4. Arrays
+MUMPS arrays should generally be treated as nested dictionaries.
+
+5. Global Variables
+When globals (prepended by a circumflex) are used in a routine, treat them as coming from a mysql database. Assume that database credentials are stored in environment variables ('SQL_HOST`, `SQL_USER`, `SQL_PWD`, `SQL_DB`).
+
+6. Local Variables
+In MUMPS, even "local" variables are accessible from any subroutine. If a variable would not be defined, declare it as global.
+
+7. Translate Everything
+Translate ALL the given code to the best of your ability.
+
+Please convert the following MUMPS .m code found in between triple backticks into {TARGET_LANGUAGE} code. The returned code should also be delimited with triple backticks.
+
+```
+{SOURCE_CODE}
+```

janus/prompts/templates/micromanaged_mumps_v2.0/system.txt

@@ -0,0 +1,3 @@
+Your purpose is to convert MUMPS .m code
+into runnable {TARGET_LANGUAGE} code ({TARGET_LANGUAGE} version
+{TARGET_LANGUAGE_VERSION}).

janus/prompts/templates/micromanaged_mumps_v2.1/human.txt

@@ -0,0 +1,29 @@
+Adhere to the following rules for translating MUMPS to Python:
+
+1. Routines from other files
+When a function from another file is invoked (e.g. `D FUNC^ABC`), treat the file like a module (e.g. `from abc import func`, `func()` ). Keep all imports at the beginning of the returned code.
+
+2. Naming Conventions
+Adhere to PEP8 for variable and function names. Improve readability when possible, making use of context and documentation. For example, a MUMPS variable like `RXQTY` might be translated to `prescription_quantity`.
+
+3. Ignore K(ill) Commands
+Memory allocation and garbage collection is generally handled automatically in Python, so any MUMPS K(ill) commands should be ignored.
+
+4. Arrays
+MUMPS arrays should be treated as nested dictionaries.
+
+5. Global Variables
+When globals (prepended by a circumflex) are used in a routine, treat them as coming from a mysql database. Assume that database credentials are stored in environment variables ('SQL_HOST`, `SQL_USER`, `SQL_PWD`, `SQL_DB`).
+
+6. Local Variables
+In MUMPS, even "local" variables are accessible from any subroutine. Declare all variables used in any function as global at the beginning of the function.
+
+7. Translate Everything
+Translate ALL the given code to the best of your ability. DO NOT use pseudocode. DO NOT leave functions empty. DO NOT give up on translation.
+
+
+Please convert the following MUMPS .m code found in between triple backticks into {TARGET_LANGUAGE} code. The returned code should also be delimited with triple backticks.
+
+```
+{SOURCE_CODE}
+```

janus/prompts/templates/micromanaged_mumps_v2.1/system.txt

@@ -0,0 +1,3 @@
+Your purpose is to convert MUMPS .m code
+into runnable {TARGET_LANGUAGE} code ({TARGET_LANGUAGE} version
+{TARGET_LANGUAGE_VERSION}).

janus/prompts/templates/multidocument/human.txt

@@ -0,0 +1,15 @@
+Please document the {SOURCE_LANGUAGE} function or module below. Your response should be in JSON format, and include three string fields:
+
+docstring: A Sphinx-style docstring for the code, including a summary of its functionality; the name, type, and description of any parameters or returns; and any potential exceptions that might arise in its execution. This should be a string value, NOT a nested JSON object.
+example_usage: A well-commented minimal example in {SOURCE_LANGUAGE} utilizing the given code's functionality.
+pseudocode: A Python-stype pseudocode implementation of the module or function's behavior.
+
+If no executable code is provided (for example, if the input is a simple label with no logic attached), return an empty string for each of the above fields.
+
+It is vital that you do not include any other context, questions, or text of any kind, other than the documentation for this piece of code. You should include all of the fields described above, and those fields only.
+
+Here is the code:
+
+```
+{SOURCE_CODE}
+```

janus/prompts/templates/multidocument/system.txt

@@ -0,0 +1 @@
+You are a senior software engineer tasked with documenting {SOURCE_LANGUAGE} code.

janus/prompts/templates/partition/human.txt

@@ -0,0 +1,22 @@
+Partition the {SOURCE_LANGUAGE} code into logical blocks. Each block should be relatively self-contained and ideally constitute a complete "subroutine", including any associated comments. These breakpoints should usually be inserted between labeled blocks, but perhaps not between *every* labeled block (depending on things like fallthrough).
+
+INPUT FORMAT:
+Each line of code has been prepended with an 8-character unique ID. a Python example would look like this:
+```
+{EXAMPLE_INPUT}
+```
+
+And your output might look like this:
+```
+{EXAMPLE_OUTPUT}
+```
+
+You are to output a JSON object containing a subset of these IDs, corresponding to the lines that should start a new block. Each partition should be paired with an explanation (please output the explanation first, before giving the line ID). DO NOT include any additional commentary before or after the JSON object, your response should be the JSON object ONLY.
+
+Format instructions:
+{format_instructions}
+
+Input:
+```
+{SOURCE_CODE}
+```

janus/prompts/templates/partition/system.txt

@@ -0,0 +1 @@
+Your purpose is to partition {SOURCE_LANGUAGE} code into self-contained logical blocks.