janus-llm 4.1.0__py3-none-any.whl → 4.3.1__py3-none-any.whl

janus/__init__.py

@@ -5,7 +5,7 @@ from langchain_core._api.deprecation import LangChainDeprecationWarning
 from janus.converter.translate import Translator
 from janus.metrics import *  # noqa: F403
 
-__version__ = "4.1.0"
+__version__ = "4.3.1"
 
 # Ignoring a deprecation warning from langchain_core that I can't seem to hunt down
 warnings.filterwarnings("ignore", category=LangChainDeprecationWarning)

janus/cli.py

@@ -13,10 +13,14 @@ from rich.console import Console
 from rich.prompt import Confirm
 from typing_extensions import Annotated
 
+import janus.refiners.refiner
+import janus.refiners.uml
 from janus.converter.aggregator import Aggregator
 from janus.converter.converter import Converter
 from janus.converter.diagram import DiagramGenerator
 from janus.converter.document import Documenter, MadLibsDocumenter, MultiDocumenter
+from janus.converter.evaluate import InlineCommentEvaluator, RequirementEvaluator
+from janus.converter.partition import Partitioner
 from janus.converter.requirements import RequirementsDocumenter
 from janus.converter.translate import Translator
 from janus.embedding.collections import Collections
@@ -44,7 +48,6 @@ from janus.llm.models_info import (
     openai_models,
 )
 from janus.metrics.cli import evaluate
-from janus.refiners.refiner import REFINERS
 from janus.utils.enums import LANGUAGES
 from janus.utils.logger import create_logger
 
@@ -69,6 +72,18 @@ with open(db_file, "r") as f:
 collections_config_file = Path(db_loc) / "collections.json"
 
 
+def get_subclasses(cls):
+    return set(cls.__subclasses__()).union(
+        set(s for c in cls.__subclasses__() for s in get_subclasses(c))
+    )
+
+
+REFINER_TYPES = get_subclasses(janus.refiners.refiner.JanusRefiner).union(
+    {janus.refiners.refiner.JanusRefiner}
+)
+REFINERS = {r.__name__: r for r in REFINER_TYPES}
+
+
 def get_collections_config():
     if collections_config_file.exists():
         with open(collections_config_file, "r") as f:
@@ -113,7 +128,7 @@ embedding = typer.Typer(
 
 def version_callback(value: bool) -> None:
     if value:
-        from janus import __version__ as version
+        from . import __version__ as version
 
         print(f"Janus CLI [blue]v{version}[/blue]")
         raise typer.Exit()
@@ -244,22 +259,23 @@ def translate(
             click_type=click.Choice(list(CUSTOM_SPLITTERS.keys())),
         ),
     ] = "file",
-    refiner_type: Annotated[
-        str,
+    refiner_types: Annotated[
+        list[str],
         typer.Option(
             "-r",
             "--refiner",
-            help="Name of custom refiner to use",
+            help="List of refiner types to use. Add -r for each refiner to use in\
+                refinement chain",
             click_type=click.Choice(list(REFINERS.keys())),
         ),
-    ] = "none",
+    ] = ["JanusRefiner"],
     retriever_type: Annotated[
         str,
         typer.Option(
             "-R",
             "--retriever",
             help="Name of custom retriever to use",
-            click_type=click.Choice(["active_usings"]),
+            click_type=click.Choice(["active_usings", "language_docs"]),
         ),
     ] = None,
     max_tokens: Annotated[
@@ -272,6 +288,7 @@ def translate(
         ),
     ] = None,
 ):
+    refiner_types = [REFINERS[r] for r in refiner_types]
     try:
         target_language, target_version = target_lang.split("-")
     except ValueError:
@@ -296,7 +313,7 @@ def translate(
         db_path=db_loc,
         db_config=collections_config,
         splitter_type=splitter_type,
-        refiner_type=refiner_type,
+        refiner_types=refiner_types,
         retriever_type=retriever_type,
     )
     translator.translate(input_dir, output_dir, overwrite, collection)
@@ -402,22 +419,23 @@ def document(
             click_type=click.Choice(list(CUSTOM_SPLITTERS.keys())),
         ),
     ] = "file",
-    refiner_type: Annotated[
-        str,
+    refiner_types: Annotated[
+        list[str],
         typer.Option(
             "-r",
             "--refiner",
-            help="Name of custom refiner to use",
+            help="List of refiner types to use. Add -r for each refiner to use in\
+                refinement chain",
             click_type=click.Choice(list(REFINERS.keys())),
         ),
-    ] = "none",
+    ] = ["JanusRefiner"],
     retriever_type: Annotated[
         str,
         typer.Option(
             "-R",
             "--retriever",
             help="Name of custom retriever to use",
-            click_type=click.Choice(["active_usings"]),
+            click_type=click.Choice(["active_usings", "language_docs"]),
         ),
     ] = None,
     max_tokens: Annotated[
@@ -430,6 +448,7 @@ def document(
         ),
     ] = None,
 ):
+    refiner_types = [REFINERS[r] for r in refiner_types]
     model_arguments = dict(temperature=temperature)
     collections_config = get_collections_config()
     kwargs = dict(
@@ -441,7 +460,7 @@ def document(
         db_path=db_loc,
         db_config=collections_config,
         splitter_type=splitter_type,
-        refiner_type=refiner_type,
+        refiner_types=refiner_types,
         retriever_type=retriever_type,
     )
     if doc_mode == "madlibs":
@@ -458,12 +477,6 @@ def document(
     documenter.translate(input_dir, output_dir, overwrite, collection)
 
 
-def get_subclasses(cls):
-    return set(cls.__subclasses__()).union(
-        set(s for c in cls.__subclasses__() for s in get_subclasses(c))
-    )
-
-
 @app.command()
 def aggregate(
     input_dir: Annotated[
@@ -578,6 +591,115 @@ def aggregate(
     aggregator.translate(input_dir, output_dir, overwrite, collection)
 
 
+@app.command(
+    help="Partition input code using an LLM.",
+    no_args_is_help=True,
+)
+def partition(
+    input_dir: Annotated[
+        Path,
+        typer.Option(
+            "--input",
+            "-i",
+            help="The directory containing the source code to be partitioned. ",
+        ),
+    ],
+    language: Annotated[
+        str,
+        typer.Option(
+            "--language",
+            "-l",
+            help="The language of the source code.",
+            click_type=click.Choice(sorted(LANGUAGES)),
+        ),
+    ],
+    output_dir: Annotated[
+        Path,
+        typer.Option(
+            "--output-dir", "-o", help="The directory to store the partitioned code in."
+        ),
+    ],
+    llm_name: Annotated[
+        str,
+        typer.Option(
+            "--llm",
+            "-L",
+            help="The custom name of the model set with 'janus llm add'.",
+        ),
+    ] = "gpt-4o",
+    max_prompts: Annotated[
+        int,
+        typer.Option(
+            "--max-prompts",
+            "-m",
+            help="The maximum number of times to prompt a model on one functional block "
+            "before exiting the application. This is to prevent wasting too much money.",
+        ),
+    ] = 10,
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite/--preserve",
+            help="Whether to overwrite existing files in the output directory",
+        ),
+    ] = False,
+    temperature: Annotated[
+        float,
+        typer.Option("--temperature", "-t", help="Sampling temperature.", min=0, max=2),
+    ] = 0.7,
+    splitter_type: Annotated[
+        str,
+        typer.Option(
+            "-S",
+            "--splitter",
+            help="Name of custom splitter to use",
+            click_type=click.Choice(list(CUSTOM_SPLITTERS.keys())),
+        ),
+    ] = "file",
+    refiner_types: Annotated[
+        list[str],
+        typer.Option(
+            "-r",
+            "--refiner",
+            help="List of refiner types to use. Add -r for each refiner to use in\
+                refinement chain",
+            click_type=click.Choice(list(REFINERS.keys())),
+        ),
+    ] = ["JanusRefiner"],
+    max_tokens: Annotated[
+        int,
+        typer.Option(
+            "--max-tokens",
+            "-M",
+            help="The maximum number of tokens the model will take in. "
+            "If unspecificed, model's default max will be used.",
+        ),
+    ] = None,
+    partition_token_limit: Annotated[
+        int,
+        typer.Option(
+            "--partition-tokens",
+            "-pt",
+            help="The limit on the number of tokens per partition.",
+        ),
+    ] = 8192,
+):
+    refiner_types = [REFINERS[r] for r in refiner_types]
+    model_arguments = dict(temperature=temperature)
+    kwargs = dict(
+        model=llm_name,
+        model_arguments=model_arguments,
+        source_language=language,
+        max_prompts=max_prompts,
+        max_tokens=max_tokens,
+        splitter_type=splitter_type,
+        refiner_types=refiner_types,
+        partition_token_limit=partition_token_limit,
+    )
+    partitioner = Partitioner(**kwargs)
+    partitioner.translate(input_dir, output_dir, overwrite)
+
+
 @app.command(
     help="Diagram input code using an LLM.",
     no_args_is_help=True,
@@ -667,25 +789,27 @@ def diagram(
             click_type=click.Choice(list(CUSTOM_SPLITTERS.keys())),
         ),
     ] = "file",
-    refiner_type: Annotated[
-        str,
+    refiner_types: Annotated[
+        list[str],
         typer.Option(
             "-r",
             "--refiner",
-            help="Name of custom refiner to use",
+            help="List of refiner types to use. Add -r for each refiner to use in\
+                refinement chain",
             click_type=click.Choice(list(REFINERS.keys())),
         ),
-    ] = "none",
+    ] = ["JanusRefiner"],
     retriever_type: Annotated[
         str,
         typer.Option(
             "-R",
             "--retriever",
             help="Name of custom retriever to use",
-            click_type=click.Choice(["active_usings"]),
+            click_type=click.Choice(["active_usings", "language_docs"]),
         ),
     ] = None,
 ):
+    refiner_types = [REFINERS[r] for r in refiner_types]
     model_arguments = dict(temperature=temperature)
     collections_config = get_collections_config()
     diagram_generator = DiagramGenerator(
@@ -696,7 +820,7 @@ def diagram(
         db_path=db_loc,
         db_config=collections_config,
         splitter_type=splitter_type,
-        refiner_type=refiner_type,
+        refiner_types=refiner_types,
         retriever_type=retriever_type,
         diagram_type=diagram_type,
         add_documentation=add_documentation,
@@ -704,6 +828,139 @@ def diagram(
     diagram_generator.translate(input_dir, output_dir, overwrite, collection)
 
 
+@app.command(
+    help="LLM self evaluation",
+    no_args_is_help=True,
+)
+def llm_self_eval(
+    input_dir: Annotated[
+        Path,
+        typer.Option(
+            "--input",
+            "-i",
+            help="The directory containing the source code to be evaluated. "
+            "The files should all be in one flat directory.",
+        ),
+    ],
+    language: Annotated[
+        str,
+        typer.Option(
+            "--language",
+            "-l",
+            help="The language of the source code.",
+            click_type=click.Choice(sorted(LANGUAGES)),
+        ),
+    ],
+    output_dir: Annotated[
+        Path,
+        typer.Option(
+            "--output-dir", "-o", help="The directory to store the evaluations in."
+        ),
+    ],
+    llm_name: Annotated[
+        str,
+        typer.Option(
+            "--llm",
+            "-L",
+            help="The custom name of the model set with 'janus llm add'.",
+        ),
+    ] = "gpt-4o",
+    evaluation_type: Annotated[
+        str,
+        typer.Option(
+            "--evaluation-type",
+            "-e",
+            help="Type of output to evaluate.",
+            click_type=click.Choice(["incose", "comments"]),
+        ),
+    ] = "incose",
+    max_prompts: Annotated[
+        int,
+        typer.Option(
+            "--max-prompts",
+            "-m",
+            help="The maximum number of times to prompt a model on one functional block "
+            "before exiting the application. This is to prevent wasting too much money.",
+        ),
+    ] = 10,
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite/--preserve",
+            help="Whether to overwrite existing files in the output directory",
+        ),
+    ] = False,
+    temperature: Annotated[
+        float,
+        typer.Option("--temperature", "-t", help="Sampling temperature.", min=0, max=2),
+    ] = 0.7,
+    collection: Annotated[
+        str,
+        typer.Option(
+            "--collection",
+            "-c",
+            help="If set, will put the translated result into a Chroma DB "
+            "collection with the name provided.",
+        ),
+    ] = None,
+    splitter_type: Annotated[
+        str,
+        typer.Option(
+            "-S",
+            "--splitter",
+            help="Name of custom splitter to use",
+            click_type=click.Choice(list(CUSTOM_SPLITTERS.keys())),
+        ),
+    ] = "file",
+    refiner_types: Annotated[
+        list[str],
+        typer.Option(
+            "-r",
+            "--refiner",
+            help="List of refiner types to use. Add -r for each refiner to use in\
+                refinement chain",
+            click_type=click.Choice(list(REFINERS.keys())),
+        ),
+    ] = ["JanusRefiner"],
+    eval_items_per_request: Annotated[
+        int,
+        typer.Option(
+            "--eval-items-per-request",
+            "-rc",
+            help="The maximum number of evaluation items per request",
+        ),
+    ] = None,
+    max_tokens: Annotated[
+        int,
+        typer.Option(
+            "--max-tokens",
+            "-M",
+            help="The maximum number of tokens the model will take in. "
+            "If unspecificed, model's default max will be used.",
+        ),
+    ] = None,
+):
+    model_arguments = dict(temperature=temperature)
+    refiner_types = [REFINERS[r] for r in refiner_types]
+    kwargs = dict(
+        eval_items_per_request=eval_items_per_request,
+        model=llm_name,
+        model_arguments=model_arguments,
+        source_language=language,
+        max_prompts=max_prompts,
+        max_tokens=max_tokens,
+        splitter_type=splitter_type,
+        refiner_types=refiner_types,
+    )
+    # Setting parser type here
+    if evaluation_type == "incose":
+        evaluator = RequirementEvaluator(**kwargs)
+    elif evaluation_type == "comments":
+        evaluator = InlineCommentEvaluator(**kwargs)
+
+    evaluator.translate(input_dir, output_dir, overwrite, collection)
+
+
 @db.command("init", help="Connect to or create a database.")
 def db_init(
     path: Annotated[
@@ -1005,13 +1262,12 @@ def llm_add(
             show_choices=False,
         )
         params = dict(
-            # OpenAI uses the "model_name" key for what we're calling "long_model_id"
-            model_name=MODEL_ID_TO_LONG_ID[model_id],
+            model_name=model_name,
             temperature=0.7,
             n=1,
         )
-        max_tokens = TOKEN_LIMITS[MODEL_ID_TO_LONG_ID[model_id]]
-        model_cost = COST_PER_1K_TOKENS[MODEL_ID_TO_LONG_ID[model_id]]
+        max_tokens = TOKEN_LIMITS[model_name]
+        model_cost = COST_PER_1K_TOKENS[model_name]
         cfg = {
             "model_type": model_type,
             "model_id": model_id,

janus/converter/__init__.py

@@ -2,5 +2,6 @@ from janus.converter.converter import Converter
 from janus.converter.diagram import DiagramGenerator
 from janus.converter.document import Documenter, MadLibsDocumenter, MultiDocumenter
 from janus.converter.evaluate import Evaluator
+from janus.converter.partition import Partitioner
 from janus.converter.requirements import RequirementsDocumenter
 from janus.converter.translate import Translator

janus/converter/converter.py

@@ -6,7 +6,12 @@ from typing import Any
 
 from langchain_core.exceptions import OutputParserException
 from langchain_core.prompts import ChatPromptTemplate
-from langchain_core.runnables import Runnable, RunnableParallel, RunnablePassthrough
+from langchain_core.runnables import (
+    Runnable,
+    RunnableLambda,
+    RunnableParallel,
+    RunnablePassthrough,
+)
 from openai import BadRequestError, RateLimitError
 from pydantic import ValidationError
 
@@ -23,15 +28,14 @@ from janus.language.splitter import (
 from janus.llm.model_callbacks import get_model_callback
 from janus.llm.models_info import MODEL_PROMPT_ENGINES, JanusModel, load_model
 from janus.parsers.parser import GenericParser, JanusParser
-from janus.refiners.refiner import (
-    FixParserExceptions,
-    HallucinationRefiner,
-    JanusRefiner,
-    ReflectionRefiner,
-)
+from janus.refiners.refiner import JanusRefiner
 
 # from janus.refiners.refiner import BasicRefiner, Refiner
-from janus.retrievers.retriever import ActiveUsingsRetriever, JanusRetriever
+from janus.retrievers.retriever import (
+    ActiveUsingsRetriever,
+    JanusRetriever,
+    LanguageDocsRetriever,
+)
 from janus.utils.enums import LANGUAGES
 from janus.utils.logger import create_logger
 
@@ -78,7 +82,7 @@ class Converter:
         protected_node_types: tuple[str, ...] = (),
         prune_node_types: tuple[str, ...] = (),
         splitter_type: str = "file",
-        refiner_type: str | None = None,
+        refiner_types: list[type[JanusRefiner]] = [JanusRefiner],
         retriever_type: str | None = None,
     ) -> None:
         """Initialize a Converter instance.
@@ -105,6 +109,7 @@ class Converter:
                 - None
             retriever_type: The type of retriever to use. Valid values:
                 - "active_usings"
+                - "language_docs"
                 - None
         """
         self._changed_attrs: set = set()
@@ -133,10 +138,11 @@ class Converter:
         self._prompt: ChatPromptTemplate
 
         self._parser: JanusParser = GenericParser()
+        self._base_parser: JanusParser = GenericParser()
         self._combiner: Combiner = Combiner()
 
         self._splitter_type: str
-        self._refiner_type: str | None
+        self._refiner_types: list[type[JanusRefiner]]
         self._retriever_type: str | None
 
         self._splitter: Splitter
@@ -144,7 +150,7 @@ class Converter:
         self._retriever: JanusRetriever
 
         self.set_splitter(splitter_type=splitter_type)
-        self.set_refiner(refiner_type=refiner_type)
+        self.set_refiner_types(refiner_types=refiner_types)
         self.set_retriever(retriever_type=retriever_type)
         self.set_model(model_name=model, **model_arguments)
         self.set_prompt(prompt_template=prompt_template)
@@ -170,7 +176,7 @@ class Converter:
         self._load_model()
         self._load_prompt()
         self._load_retriever()
-        self._load_refiner()
+        self._load_refiner_chain()
         self._load_splitter()
         self._load_vectorizer()
         self._load_chain()
@@ -210,13 +216,13 @@ class Converter:
 
         self._splitter_type = splitter_type
 
-    def set_refiner(self, refiner_type: str | None) -> None:
+    def set_refiner_types(self, refiner_types: list[type[JanusRefiner]]) -> None:
         """Validate and set the refiner type
 
         Arguments:
             refiner_type: the type of refiner to use
         """
-        self._refiner_type = refiner_type
+        self._refiner_types = refiner_types
 
     def set_retriever(self, retriever_type: str | None) -> None:
         """Validate and set the retriever type
@@ -355,48 +361,40 @@ class Converter:
     def _load_retriever(self):
         if self._retriever_type == "active_usings":
             self._retriever = ActiveUsingsRetriever()
+        elif self._retriever_type == "language_docs":
+            self._retriever = LanguageDocsRetriever(self._llm, self._source_language)
         else:
             self._retriever = JanusRetriever()
 
-    @run_if_changed("_refiner_type", "_model_name", "max_prompts", "_parser", "_llm")
-    def _load_refiner(self) -> None:
-        """Load the refiner according to this instance's attributes.
-
-        If the relevant fields have not been changed since the last time this method was
-        called, nothing happens.
-        """
-        if self._refiner_type == "parser":
-            self._refiner = FixParserExceptions(
-                llm=self._llm,
-                parser=self._parser,
-                max_retries=self.max_prompts,
-            )
-        elif self._refiner_type == "reflection":
-            self._refiner = ReflectionRefiner(
-                llm=self._llm,
-                parser=self._parser,
-                max_retries=self.max_prompts,
+    @run_if_changed("_refiner_types", "_model_name", "max_prompts", "_parser")
+    def _load_refiner_chain(self) -> None:
+        self._refiner_chain = RunnableParallel(
+            completion=self._llm,
+            prompt_value=RunnablePassthrough(),
+        )
+        for refiner_type in self._refiner_types[:-1]:
+            # NOTE: Do NOT remove refiner_type=refiner_type from lambda.
+            # Due to lambda capture, must be present or chain will not
+            # be correctly constructed.
+            self._refiner_chain = self._refiner_chain | RunnableParallel(
+                completion=lambda x, refiner_type=refiner_type: refiner_type(
+                    llm=self._llm,
+                    parser=self._base_parser,
+                    max_retries=self.max_prompts,
+                ).parse_completion(**x),
+                prompt_value=lambda x: x["prompt_value"],
             )
-        elif self._refiner_type == "hallucination":
-            self._refiner = HallucinationRefiner(
+        self._refiner_chain = self._refiner_chain | RunnableLambda(
+            lambda x: self._refiner_types[-1](
                 llm=self._llm,
                 parser=self._parser,
                 max_retries=self.max_prompts,
-            )
-        else:
-            self._refiner = JanusRefiner(parser=self._parser)
+            ).parse_completion(**x)
+        )
 
-    @run_if_changed("_parser", "_retriever", "_prompt", "_llm", "_refiner")
+    @run_if_changed("_parser", "_retriever", "_prompt", "_llm", "_refiner_chain")
     def _load_chain(self):
-        self.chain = (
-            self._input_runnable()
-            | self._prompt
-            | RunnableParallel(
-                completion=self._llm,
-                prompt_value=RunnablePassthrough(),
-            )
-            | self._refiner.parse_runnable
-        )
+        self.chain = self._input_runnable() | self._prompt | self._refiner_chain
 
     def _input_runnable(self) -> Runnable:
         return RunnableParallel(
@@ -466,6 +464,7 @@ class Converter:
         for in_path, out_path in in_out_pairs:
             # Translate the file, skip it if there's a rate limit error
             try:
+                log.info(f"Processing {in_path.relative_to(input_directory)}")
                 out_block = self.translate_file(in_path)
                 total_cost += out_block.total_cost
             except RateLimitError: