janus-llm 4.3.1__py3-none-any.whl → 4.3.5__py3-none-any.whl

janus/llm/models_info.py

@@ -50,6 +50,7 @@ except ImportError:
 ModelType = TypeVar(
     "ModelType",
     AzureChatOpenAI,
+    ChatOpenAI,
     HuggingFaceTextGenInference,
     Bedrock,
     BedrockChat,
@@ -247,6 +248,7 @@ def load_model(model_id) -> JanusModel:
         token_limit = model_config["token_limit"]
         input_token_cost = model_config["model_cost"]["input"]
         output_token_cost = model_config["model_cost"]["output"]
+        input_token_proportion = model_config["input_token_proportion"]
 
     elif model_id in DEFAULT_MODELS:
         model_id = model_id
@@ -257,6 +259,7 @@ def load_model(model_id) -> JanusModel:
         token_limit = 0
         input_token_cost = 0.0
         output_token_cost = 0.0
+        input_token_proportion = 0.4
         if model_long_id in TOKEN_LIMITS:
             token_limit = TOKEN_LIMITS[model_long_id]
         if model_long_id in COST_PER_1K_TOKENS:
@@ -286,7 +289,6 @@ def load_model(model_id) -> JanusModel:
     elif model_type_name == "OpenAI":
         model_args.update(
             openai_api_key=str(os.getenv("OPENAI_API_KEY")),
-            openai_organization=str(os.getenv("OPENAI_ORG_ID")),
         )
         # log.warning("Do NOT use this model in sensitive environments!")
         # log.warning("If you would like to cancel, please press Ctrl+C.")
@@ -310,15 +312,20 @@ def load_model(model_id) -> JanusModel:
 
     class JanusModel(model_type):
         model_id: str
+        # model_name is for LangChain compatibility
+        # It searches for `self.model_name` when counting tokens
+        model_name: str
         short_model_id: str
         model_type_name: str
         token_limit: int
+        input_token_proportion: float
         input_token_cost: float
         output_token_cost: float
         prompt_engine: type[PromptEngine]
 
     model_args.update(
         model_id=MODEL_ID_TO_LONG_ID[model_id],
+        model_name=model_id,  # This is for LangChain compatibility
         short_model_id=model_id,
     )
 
@@ -327,6 +334,7 @@ def load_model(model_id) -> JanusModel:
         token_limit=token_limit,
         input_token_cost=input_token_cost,
         output_token_cost=output_token_cost,
+        input_token_proportion=input_token_proportion,
         prompt_engine=prompt_engine,
         **model_args,
     )

janus/metrics/_tests/asm_test_file.asm

@@ -0,0 +1,10 @@
+NAME     OPA   OPSA,OPSB
+         OPB   OPSC,OPSC REMARK
+NAME     OPC   OPSA,OPSB
+         OPD   OPSA,OPSB REMARK2
+         B     OPSA
+         OPD
+         B     OPSB
+NAME     OPC   OPSA,OPSB
+         OPC
+         OPC   OPSA,OPSB

janus/metrics/_tests/mumps_test_file.m

@@ -0,0 +1,6 @@
+FUNC(a, b) ; apples
+  set apples=8
+  write a,!
+  write a,!
+  if abc=70 set f=1
+  quit 0

janus/metrics/_tests/test_treesitter_metrics.py

@@ -3,7 +3,7 @@ from pathlib import Path
 
 from typer.testing import CliRunner
 
-from janus.cli import app
+from janus.cli.cli import app
 from janus.metrics.complexity_metrics import (
     TreeSitterMetric,
     cyclomatic_complexity,

janus/metrics/prompts/clarity.txt

@@ -0,0 +1,8 @@
+Based on the following target written in the {language} programming language, how would you rate the code clarity of the target on a scale of integers from 1 to 10? Higher is better.
+
+Think through your answer before selecting a rating with the following format:
+
+Target: the target code
+{format_instructions}
+
+Target: {target}

janus/metrics/prompts/completeness.txt

@@ -0,0 +1,16 @@
+Use the following rubric to evaluate the target written in the {language} programming language:
+
+Rubric:
+Does the comment address all capabilities of the relevant source code?
+
+10 - All essential functionality is documented.
+6-9 - Most essential functionality is documented.
+2-5 - Little essential functionality is documented.
+1 - No essential functionality is documented.
+
+Think through your answer before selecting a rating with the following format:
+
+Target: the target code
+{format_instructions}
+
+Target: {target}

janus/metrics/prompts/faithfulness.txt

@@ -0,0 +1,10 @@
+Based on the following target and reference written in the {language} programming language, how would you rate the faithfulness of the target to the original reference on a scale of integers from 1 to 10? Higher is better.
+
+Think through your answer before selecting a rating with the following format:
+
+Target: the target code
+Reference: the reference code that we are judging the target against
+{format_instructions}
+
+Target: {target}
+Reference: {reference}

janus/metrics/prompts/hallucination.txt

@@ -0,0 +1,16 @@
+Use the following rubric to evaluate the target written in the {language} programming language:
+
+Rubric:
+Does the comment provide true information?
+
+10 - The comment provides only true information.
+6-9 - The comment provides mostly true information.
+2-5 - The comment provides mostly untrue information.
+1 - The comment is completely untrue.
+
+Think through your answer before selecting a rating with the following format:
+
+Target: the target code
+{format_instructions}
+
+Target: {target}

janus/metrics/prompts/quality.txt

@@ -0,0 +1,8 @@
+Based on the following target written in the {language} programming language, how would you rate the code quality of the target on a scale of integers from 1 to 10? Higher is better.
+
+Think through your answer before selecting a rating with the following format:
+
+Target: the target code
+{format_instructions}
+
+Target: {target}

janus/metrics/prompts/readability.txt

@@ -0,0 +1,16 @@
+Use the following rubric to evaluate the target written in the {language} programming language:
+
+Rubric:
+Is the comment clear to read?
+
+10 - The comment is well-written.
+6-9 - The comment has few problems.
+2-5 - The comment has many problems.
+1 - The comment is unreadable.
+
+Think through your answer before selecting a rating with the following format:
+
+Target: the target code
+{format_instructions}
+
+Target: {target}

janus/metrics/prompts/usefulness.txt

@@ -0,0 +1,16 @@
+Use the following rubric to evaluate the target written in the {language} programming language:
+
+Rubric:
+Is the comment useful?
+
+10 - The comment helps an expert programmer understand the code better.
+6-9 - The comment helps an average programmer understand the code better.
+2-5 - The comment documents only trivial functionality.
+1 - The comment is not useful at any level.
+
+Think through your answer before selecting a rating with the following format:
+
+Target: the target code
+{format_instructions}
+
+Target: {target}

janus/parsers/code_parser.py

@@ -1,9 +1,8 @@
 import re
 
-from langchain_core.exceptions import OutputParserException
 from langchain_core.messages import BaseMessage
 
-from janus.parsers.parser import JanusParser
+from janus.parsers.parser import JanusParser, JanusParserException
 from janus.utils.logger import create_logger
 
 log = create_logger(__name__)
@@ -18,8 +17,9 @@ class CodeParser(JanusParser):
         pattern = rf"```[^\S\r\n]*(?:{self.language}[^\S\r\n]*)?\n?(.*?)\n*```"
         code = re.search(pattern, text, re.DOTALL)
         if code is None:
-            raise OutputParserException(
-                "Code not find code between triple square brackets"
+            raise JanusParserException(
+                text,
+                "Code not find code between triple square brackets",
             )
         return str(code.group(1))
 

janus/parsers/doc_parser.py

@@ -8,7 +8,7 @@ from langchain_core.messages import BaseMessage
 from langchain_core.pydantic_v1 import BaseModel, Field
 
 from janus.language.block import CodeBlock
-from janus.parsers.parser import JanusParser
+from janus.parsers.parser import JanusParser, JanusParserException
 from janus.utils.logger import create_logger
 
 log = create_logger(__name__)
@@ -86,7 +86,7 @@ class MultiDocumentationParser(JanusParser, PydanticOutputParser):
         return str(self.__class__.name)
 
 
-class MadlibsDocumentationParser(JanusParser):
+class ClozeDocumentationParser(JanusParser):
     expected_keys: set[str]
 
     def __init__(self):
@@ -107,11 +107,12 @@ class MadlibsDocumentationParser(JanusParser):
             obj = parse_json_markdown(text)
         except json.JSONDecodeError as e:
             log.debug(f"Invalid JSON object. Output:\n{text}")
-            raise OutputParserException(f"Got invalid JSON object. Error: {e}")
+            raise JanusParserException(text, f"Got invalid JSON object. Error: {e}")
 
         if not isinstance(obj, dict):
-            raise OutputParserException(
-                f"Got invalid return object. Expected a dictionary, but got {type(obj)}"
+            raise JanusParserException(
+                text,
+                f"Got invalid return object. Expected a dictionary, but got {type(obj)}",
             )
 
         seen_keys = set(obj.keys())
@@ -122,9 +123,10 @@ class MadlibsDocumentationParser(JanusParser):
             if invalid_keys:
                 log.debug(f"Invalid keys: {invalid_keys}")
             log.debug(f"Missing keys: {missing_keys}")
-            raise OutputParserException(
+            raise JanusParserException(
+                text,
                 f"Got invalid return object. Missing the following expected "
-                f"keys: {missing_keys}"
+                f"keys: {missing_keys}",
             )
 
         for key in invalid_keys:
@@ -132,9 +134,10 @@ class MadlibsDocumentationParser(JanusParser):
 
         for value in obj.values():
             if not isinstance(value, str):
-                raise OutputParserException(
+                raise JanusParserException(
+                    text,
                     f"Got invalid return object. Expected all string values,"
-                    f' but got type "{type(value)}"'
+                    f' but got type "{type(value)}"',
                 )
 
         return json.dumps(obj)

janus/parsers/parser.py

@@ -1,4 +1,5 @@
 from langchain.schema.output_parser import BaseOutputParser
+from langchain_core.exceptions import OutputParserException
 from langchain_core.messages import BaseMessage
 from langchain_core.output_parsers import StrOutputParser
 
@@ -49,3 +50,9 @@ class GenericParser(JanusParser, StrOutputParser):
 
     def get_format_instructions(self) -> str:
         return "Output should be a string"
+
+
+class JanusParserException(OutputParserException):
+    def __init__(self, unparsed_output, *args, **kwargs):
+        self.unparsed_output = unparsed_output
+        super().__init__(*args, **kwargs)

janus/parsers/partition_parser.py

@@ -9,7 +9,7 @@ from langchain_core.messages import BaseMessage
 from langchain_core.pydantic_v1 import BaseModel, Field
 
 from janus.language.block import CodeBlock
-from janus.parsers.parser import JanusParser
+from janus.parsers.parser import JanusParser, JanusParserException
 from janus.utils.logger import create_logger
 
 log = create_logger(__name__)
@@ -97,6 +97,7 @@ class PartitionParser(JanusParser, PydanticOutputParser):
     def parse(self, text: str | BaseMessage) -> str:
         if isinstance(text, BaseMessage):
             text = str(text.content)
+        original_text = text
 
         # Strip everything outside the JSON object
         begin, end = text.find("["), text.rfind("]")
@@ -122,7 +123,7 @@ class PartitionParser(JanusParser, PydanticOutputParser):
                 + ", ".join(invalid_splits)
             )
             log.warning(err_msg)
-            raise OutputParserException(err_msg)
+            raise JanusParserException(original_text, err_msg)
 
         # Map line IDs to indices (so they can be sorted and lines indexed)
         index_to_line_id = {0: "START", None: "END"}
@@ -160,9 +161,10 @@ class PartitionParser(JanusParser, PydanticOutputParser):
                 "Oversized chunks:\n"
                 + "\n#############\n".join(chunk for _, chunk, _ in data)
             )
-            raise OutputParserException(
+            raise JanusParserException(
+                original_text,
                 f"The following segments are too long and must be "
-                f"further subdivided:\n{problem_points}"
+                f"further subdivided:\n{problem_points}",
             )
 
         return "\n<JANUS_PARTITION>\n".join(chunks)

janus/parsers/reqs_parser.py

@@ -2,10 +2,9 @@ import json
 import re
 
 from langchain.output_parsers.json import parse_json_markdown
-from langchain_core.exceptions import OutputParserException
 from langchain_core.messages import BaseMessage
 
-from janus.parsers.parser import JanusParser
+from janus.parsers.parser import JanusParser, JanusParserException
 from janus.utils.logger import create_logger
 
 log = create_logger(__name__)
@@ -20,6 +19,7 @@ class RequirementsParser(JanusParser):
     def parse(self, text: str | BaseMessage) -> str:
         if isinstance(text, BaseMessage):
             text = str(text.content)
+        original_text = text
 
         # TODO: This is an incorrect implementation (lstrip and rstrip take character
         #       lists and strip any instances of those characters, not the full str)
@@ -30,11 +30,14 @@ class RequirementsParser(JanusParser):
             obj = parse_json_markdown(text)
         except json.JSONDecodeError as e:
             log.debug(f"Invalid JSON object. Output:\n{text}")
-            raise OutputParserException(f"Got invalid JSON object. Error: {e}")
+            raise JanusParserException(
+                original_text, f"Got invalid JSON object. Error: {e}"
+            )
 
         if not isinstance(obj, dict):
-            raise OutputParserException(
-                f"Got invalid return object. Expected a dictionary, but got {type(obj)}"
+            raise JanusParserException(
+                original_text,
+                f"Got invalid return object. Expected a dictionary, but got {type(obj)}",
             )
         return json.dumps(obj)
 

janus/parsers/uml.py

@@ -3,10 +3,10 @@ import subprocess  # nosec
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 
-from langchain_core.exceptions import OutputParserException
 from langchain_core.messages import BaseMessage
 
 from janus.parsers.code_parser import CodeParser
+from janus.parsers.parser import JanusParserException
 from janus.utils.logger import create_logger
 
 log = create_logger(__name__)
@@ -14,6 +14,7 @@ log = create_logger(__name__)
 
 class UMLSyntaxParser(CodeParser):
     def _check_plantuml(self, text: str) -> None:
+        original_text = text
         # Leading newlines can break the parser, remove them
         text = text.replace("\\n", "\n").strip()
 
@@ -43,7 +44,7 @@ class UMLSyntaxParser(CodeParser):
             log.error(err_txt)
             raise Exception(err_txt)
 
-        # Check for bad outputs, raise OutputParserExceptions if so
+        # Check for bad outputs, raise JanusParserExceptions if so
         if "Error" in stderr or "Error" in stdout:
             err_txt = "Recieved UML parsing error(s)."
 
@@ -64,7 +65,7 @@ class UMLSyntaxParser(CodeParser):
                 err_txt += f"\nError located at line {i} must be fixed:\n"
                 err_txt += "\n".join(err_lines)
             log.warning(err_txt)
-            raise OutputParserException(err_txt)
+            raise JanusParserException(original_text, err_txt)
 
         if "Warning" in stdout or "Warning" in stderr:
             err_txt = "Recieved UML parsing warning (often due to missing PLANTUML)."
@@ -74,7 +75,7 @@ class UMLSyntaxParser(CodeParser):
                 err_txt += f"\nSTDOUT:\n```\n{stdout.strip()}\n```\n"
 
             log.warning(err_txt)
-            raise OutputParserException(err_txt)
+            raise JanusParserException(original_text, err_txt)
 
     def _get_error_lines(self, s: str) -> list[int]:
         return [int(x.group(1)) for x in re.finditer(r"Error line (\d+) in file:", s)]

janus/prompts/prompt.py

@@ -23,7 +23,7 @@ TEXT_OUTPUT = []
 # same language as the input, regardless of the `output-lang` argument.
 SAME_OUTPUT = ["document_inline"]
 
-JSON_OUTPUT = ["evaluate", "document", "document_madlibs", "requirements"]
+JSON_OUTPUT = ["evaluate", "document", "document_cloze", "requirements"]
 
 # Directory containing Janus prompt template directories and files
 JANUS_PROMPT_TEMPLATES_DIR = Path(__file__).parent / "templates"
@@ -109,7 +109,7 @@ class PromptEngine(ABC):
         source_language = source_language.lower()
         self.variables = dict(
             SOURCE_LANGUAGE=source_language,
-            FILE_SUFFIX=LANGUAGES[source_language]["suffix"],
+            FILE_SUFFIX=LANGUAGES[source_language]["suffixes"],
             SOURCE_CODE_EXAMPLE=LANGUAGES[source_language]["example"],
         )
         if target_language is not None:

janus/prompts/templates/README.md

@@ -0,0 +1,30 @@
+# Prompt Template Files
+
+Janus supports defining custom prompts in text files.
+
+```
+directory_name/
+    system.txt
+    human.txt
+    variables.json (optional)
+```
+
+## Prompt templates
+- `system.txt` contains text representing the system prompt template,
+  + Ex. "Your purpose is to understand {SOURCE_LANGUAGE} code."
+- `human.txt` contains text representing the human prompt template.
+  + Ex. "Summarize the contents of the following {SOURCE_LANGUAGE} code in {written_language} sentences.
+
+Both prompt templates can make use of f-string-style arguments, i.e. `{VARIABLE}`. Multiple lines are supported.
+
+To reuse a prompt, say for the same system directive with differing output styles, create a symbolic link to the original file.  For example:
+`ln -s ../document/system.txt system.txt`
+
+## Variables
+- (Optional) `variables.json` contains a JSON object representing additional variables and their values used in the templates above, beyond what is provided to Janus via command-line arguments.
+  + Ex.
+    ```
+    {
+        "written_language": "Spanish"
+    }
+    ```

janus/prompts/templates/basic_aggregation/human.txt

@@ -0,0 +1,6 @@
+Combine the following documentation for a program in {SOURCE_LANGUAGE} code into a single description
+Make sure to put the resultant description within triple backticks.
+Here are the representations:
+```
+{SOURCE_CODE}
+```

janus/prompts/templates/basic_aggregation/system.txt

@@ -0,0 +1 @@
+You are a senior software engineer named John and tasked with combining representations of code into a single representation.

janus/prompts/templates/basic_refinement/human.txt

@@ -0,0 +1,14 @@
+Please fix the following output generated by a large language model.
+Provide your corrected output in the same format as the original.
+The large language model was given the following prompt in triple backticks:
+```
+{ORIGINAL_PROMPT}
+```
+and produced the following output:
+```
+{OUTPUT}
+```
+but received the following errors:
+```
+{ERRORS}
+```

janus/prompts/templates/basic_refinement/system.txt

@@ -0,0 +1 @@
+You are a senior software engineer named John and tasked with fixing the output created by a large language model.

janus/prompts/templates/diagram/human.txt

@@ -0,0 +1,9 @@
+Generate a UML {DIAGRAM_TYPE} diagram using PLANTUML syntax that improves the readability of the following {SOURCE_LANGUAGE} code for a programmer.
+In your output, make sure to reformat any {SOURCE_LANGUAGE} code that would break PLANTUML syntax rules.
+Do not output any {SOURCE_LANGUAGE} code in the diagram.
+Make sure to capture all relevant syntax, functions and branching in the source code.
+Make sure to document all functions in the code
+Here is the source code:
+```
+{SOURCE_CODE}
+```

janus/prompts/templates/diagram/system.txt

@@ -0,0 +1 @@
+You are a senior software engineer named John and tasked with creating PLANTUML documentation of {SOURCE_LANGUAGE} code.

janus/prompts/templates/diagram_with_documentation/human.txt

@@ -0,0 +1,15 @@
+Generate a UML {DIAGRAM_TYPE} diagram using PLANTUML syntax that improves the readability of the following {SOURCE_LANGUAGE} code for a programmer.
+You are also provided with documentation for this code.
+In your output, make sure to reformat any {SOURCE_LANGUAGE} code that would break PLANTUML syntax rules.
+Do not output any {SOURCE_LANGUAGE} code in the diagram.
+Make sure to capture all relevant syntax, functions and branching in the source code.
+Make sure to document all functions in the code
+Make sure to put the resultant PLANTUML code within triple backticks.
+Here is the code documentation:
+```
+{DOCUMENTATION}
+```
+Here is the source code:
+```
+{SOURCE_CODE}
+```

janus/prompts/templates/diagram_with_documentation/system.txt

@@ -0,0 +1 @@
+You are a senior software engineer named John and tasked with creating PLANTUML documentation of {SOURCE_LANGUAGE} code.

janus/prompts/templates/document/human.txt

@@ -0,0 +1,10 @@
+Please explain the {SOURCE_LANGUAGE} code section below. Your response should be in plain text with no delimiters. It should contain a natural language description of the code's intended functionality; do not describe the execution step-by-step, simply explain the overall purpose. This description should be roughly one paragraph in length; multiple paragraphs may be used if and only if the code is particularly complex or has multiple independent functions.
+After this description, describe the expected initial state and/or inputs, the expected terminal state and/or outputs, and any potential exceptions that might arise in the code's execution.
+
+It is vital that you do not include any other context, questions, or text of any kind, other than the documentation for this piece of code. You should include all of the fields described above, and those fields only.
+
+Here is the code:
+
+```
+{SOURCE_CODE}
+```

janus/prompts/templates/document/system.txt

@@ -0,0 +1 @@
+You are a senior software engineer tasked with documenting {SOURCE_LANGUAGE} code.

janus/prompts/templates/document_cloze/human.txt

@@ -0,0 +1,11 @@
+The {SOURCE_LANGUAGE} code provided below has had its comments replaced by either `<INLINE_COMMENT #>` (for single-line comments) or `<BLOCK_COMMENT #>` (for multiple consecutive lines of comments), where `#` takes the place of an 8-character alphanumeric ID. You are to write replacement comments based on the source code.
+
+Return a JSON-formatted string where the keys are the alphanumeric IDs and the values are the comments that should be inserted in the code. Be sure to include comments for all placeholders present in the input. Do not provide any other commentary, do not write any code or additional comments.
+
+Example input: ```{EXAMPLE_INPUT}```
+Example output: ```{EXAMPLE_OUTPUT}```
+
+Please provide comments for the following code:
+```
+{SOURCE_CODE}
+```

janus/prompts/templates/document_cloze/system.txt

@@ -0,0 +1 @@
+You are a senior software engineer tasked with documenting {SOURCE_LANGUAGE} code.

janus/prompts/templates/document_cloze/variables.json

@@ -0,0 +1,4 @@
+{
+  "EXAMPLE_INPUT": " I '$D(CATLIST) Q\n ; <INLINE_COMMENT 14c59f05>\n S CAT=\"\"\n F  S CAT=$O(CATLIST(CAT)) Q:CAT=\"\"  D\n . S LDATE=$O(CATLIST(CAT,\"\"),-1)\n .; <INLINE_COMMENT 97a39adf>\n . S DATE=\"\"\n . F  S DATE=$O(CATLIST(CAT,DATE)) Q:DATE=LDATE  D\n .. S WCR=\"\"\n .. F  S WCR=$O(CATLIST(CAT,DATE,WCR)) Q:WCR=\"\"  D\n ... S FI=\"\"\n ... F  S FI=$O(CATLIST(CAT,DATE,WCR,FI)) Q:FI=\"\"  D\n .... S FIEVAL(FI)=0\n ....; <INLINE_COMMENT 4fd34837>\n .... S IND=0\n .... F  S IND=+$O(FIEVAL(FI,IND)) Q:IND=0  S FIEVAL(FI,IND)=0\n .; <BLOCK_COMMENT 997ec49a>\n . S (NTRUE,WCR)=0\n . F  S WCR=$O(CATLIST(CAT,LDATE,WCR)) Q:WCR=\"\"  D\n .. S FI=\"\"\n .. F  S FI=$O(CATLIST(CAT,LDATE,WCR,FI)) Q:FI=\"\"  D\n ... I NTRUE=0 D  Q\n ....; <INLINE_COMMENT 3ac32fb5>\n .... S (IND,NTRUE)=1\n .... F  S IND=+$O(FIEVAL(FI,IND)) Q:IND=0  S FIEVAL(FI,IND)=0\n ... S FIEVAL(FI)=0\n ...; <INLINE_COMMENT d04a8fdf>\n ... S IND=0\n ... F  S IND=+$O(FIEVAL(FI,IND)) Q:IND=0  S FIEVAL(FI,IND)=0\n Q\n",
+  "EXAMPLE_OUTPUT": "{\n  \"14c59f05\": \";Only the most recent HF in a category can be true.\",\n  \"97a39adf\": \";For each category set all but the most recent HF false.\",\n  \"4fd34837\": \";If there are multiple occurrences set them all false.\",\n  \"997ec49a\": \" .;\\n .;If there is more than on HF on the most recent date then only the\\n .;one with the highest WCR can be true. The highest possible WCR is 1.\\n .;Set all with lower WCRs false.\\n .;If the most recent health factor has multiple occurrences only\\n .;the first occurrence can be true.\",\n  \"3ac32fb5\": \";If there are multiple sub-occurrences set them all false.\"\n}"
+}

janus/prompts/templates/document_cloze/variables_asm.json

@@ -0,0 +1,4 @@
+{
+  "EXAMPLE_INPUT": "***********************************************************************\n* <BLOCK_COMMENT d8453f99>\n***********************************************************************\nZUIDSTCK AMODE 31\nZUIDSTCK RMODE 31\nZUIDSTCK CSECT\n         STM   R14,R12,12(R13)         <INLINE_COMMENT b2cc1643>\n         L     R1,0(R1)                <INLINE_COMMENT a315e5ca>\n         USING DSA,R1                  <INLINE_COMMENT 3155f463>\n         STCKE EISTOD                  <INLINE_COMMENT d84b5ebf>\n\n         LM    R14,R12,12(R13)         <INLINE_COMMENT a79a8e65>\n         XR    R15,R15                 <INLINE_COMMENT 47ad1b4d>\n         BR    R14                     <INLINE_COMMENT eb971719>\n",
+  "EXAMPLE_OUTPUT": "{\n  \"d8453f99\": \"Control Section\",\n  \"b2cc1643\": \"Save registers\",\n  \"a315e5ca\": \"Load parameter address\",\n  \"3155f463\": \"... tell assembler\",\n  \"d84b5ebf\": \"Save STCKE TOD\",\n  \"a79a8e65\": \"Load Registers\",\n  \"47ad1b4d\": \"Clear R15 (RC)\",\n  \"eb971719\": \"Return to calling program\"\n}"
+}

janus/prompts/templates/document_inline/human.txt

@@ -0,0 +1,13 @@
+Please add inline comments to the {SOURCE_LANGUAGE} code
+provided below in triple backticks.
+
+```
+{SOURCE_CODE}
+```
+
+Keep all source code in the output.
+
+Please add a comment at the top of the file which summarizes
+the purpose of the code.
+
+Please add comments to functions which summarize their functionality.

janus/prompts/templates/eval_prompts/incose/human.txt

@@ -0,0 +1,32 @@
+Below is a snippet of {SOURCE_LANGUAGE} code to use as reference for the following task:
+```
+{SOURCE_CODE}
+```
+
+Given the above code and the list of requirements I will soon supply, please evaluate each requirement individually based on the following criteria:
+
+C1 - Necessary: The need or requirement statement defines an essential capability, characteristic, constraint, or quality factor needed to satisfy a lifecycle concept, need, source, or parent requirement.
+C2 - Appropriate: The specific intent and amount of detail of the need or requirement statement is appropriate to the level (the level of abstraction, organization, or system architecture) of the entity to which it refers.
+C3 - Unambiguous: Need statements must be written such that the stakeholder intent is clear. Requirement statements must be stated such that the requirement can be interpreted in only one way by all the intended stakeholders.
+C4 - Complete: The requirement statement sufficiently describes the necessary capability, characteristic, constraint, or quality factor to meet the need, source, or parent requirement from which it was transformed without needing other information to understand the requirement.
+C5 - Singular: The stakeholder need or requirement statement should state a single capability, characteristic, constraint, or quality factor.
+C6 - Feasible: The need or requirement can be realized within entity constraints (for example: cost, schedule, technical, legal, ethical, safety) with acceptable risk.
+C7 - Verifiable: The requirement statement is structured and worded such that its realization can be verified to the approving authority’s satisfaction.
+C8 - Correct: The need statement must be an accurate representation of the lifecycle concept or source from which it was transformed. The requirement statement must be an accurate representation of the need, source, or parent requirement from which it was transformed.
+C9 - Conforming: Individual needs and requirements should conform to an approved standard pattern and style guide or standard for writing and managing needs and requirements.
+
+For each and every requirement below, you must indicate whether they "pass" or "fail" each of the above criteria. Briefly explain your reasoning before providing each pass/fail.
+
+Your response should be formatted as a list of JSON objects, with each object corresponding to one requirement. Each object should include 10 keys: `requirement_id`, `C1`, `C2`, ..., `C9`. `requirement_id` should have a string value that holds the 8-character UUID associated with the requirement. The other four values should each be a JSON object with two keys: `reasoning` (a clear explanation of why the criterion is passed or failed) and a `score` (the literal string "pass" or "fail").
+
+Be discerning in your evaluation; only very high-quality requirements should pass all criteria. Be a hard grader. If a requirement fails a criterion, be thorough and detailed in your explanation of why.
+
+Below is an example output for a snippet of code with three labeled requirements:
+```
+{EXAMPLE_OUTPUT}
+```
+
+Here are the requirements that you are to evaluate:
+{REQUIREMENTS}
+
+Don't forget to include your final scores in JSON format!

janus/prompts/templates/eval_prompts/incose/system.txt

@@ -0,0 +1 @@
+You are a software quality engineer, your job is to evaluate requirments according to a rubric.

janus/prompts/templates/eval_prompts/incose/variables.json

@@ -0,0 +1,3 @@
+{
+  "EXAMPLE_OUTPUT": "[\n  {\n    \"requirement_id\": \"c3caa172\",\n    \"requirement\": \"The UserID field must be followed by a comma (,) as a field separator.\",\n    \"C1\": {\n      \"reasoning\": \"This defines an essential characteristic of the data structure.\",\n      \"score\": \"pass\"\n    },\n    \"C2\": {\n      \"reasoning\": \"The detail provided is appropriate for the software's data structure level.\",\n      \"score\": \"pass\"\n    },\n    \"C3\": {\n      \"reasoning\": \"The statement is clear and unambiguous.\",\n      \"score\": \"pass\"\n    },\n    \"C4\": {\n      \"reasoning\": \"The requirement sufficiently describes the separator without needing additional information.\",\n      \"score\": \"pass\"\n    },\n    \"C5\": {\n      \"reasoning\": \"The requirement states a single characteristic.\",\n      \"score\": \"pass\"\n    },\n    \"C6\": {\n      \"reasoning\": \"Including a field separator is feasible.\",\n      \"score\": \"pass\"\n    },\n    \"C7\": {\n      \"reasoning\": \"The requirement can be verified by checking the data structure.\",\n      \"score\": \"pass\"\n    },\n    \"C8\": {\n      \"reasoning\": \"The requirement accurately represents the need for a separator.\",\n      \"score\": \"pass\"\n    },\n    \"C9\": {\n      \"reasoning\": \"The requirement conforms to standard pattern and style.\",\n      \"score\": \"pass\"\n    }\n  },\n  {\n    \"requirement_id\": \"fab48ab9\",\n    \"requirement\": \"The software must handle web communication parameters, including buffer addresses and lengths for web receive operations, query string management, and basic and query mode service program identifiers.\",\n    \"C1\": {\n      \"reasoning\": \"Defines essential capabilities for handling web communication parameters.\",\n      \"score\": \"pass\"\n    },\n    \"C2\": {\n      \"reasoning\": \"Appropriate detail for software handling web communication.\",\n      \"score\": \"pass\"\n    },\n    \"C3\": {\n      \"reasoning\": \"Clear and unambiguous about what the software must handle.\",\n      \"score\": \"pass\"\n    },\n    \"C4\": {\n      \"reasoning\": \"Sufficiently describes the necessary capabilities without needing additional information.\",\n      \"score\": \"pass\"\n    },\n    \"C5\": {\n      \"reasoning\": \"States multiple capabilities related to web communication parameters.\",\n      \"score\": \"fail\"\n    },\n    \"C6\": {\n      \"reasoning\": \"Feasible to implement within typical software constraints.\",\n      \"score\": \"pass\"\n    },\n    \"C7\": {\n      \"reasoning\": \"Verification can be done through testing the software's handling of these parameters.\",\n      \"score\": \"pass\"\n    },\n    \"C8\": {\n      \"reasoning\": \"Accurately represents the need for handling web communication parameters.\",\n      \"score\": \"pass\"\n    },\n    \"C9\": {\n      \"reasoning\": \"Conforms to standard requirement style and structure.\",\n      \"score\": \"pass\"\n    }\n  }\n]"
+}