janus-llm 3.1.1__tar.gz → 3.2.0__tar.gz

{janus_llm-3.1.1 → janus_llm-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: janus-llm
-Version: 3.1.1
+Version: 3.2.0
 Summary: A transcoding library using LLMs.
 Home-page: https://github.com/janus-llm/janus-llm
 License: Apache 2.0

{janus_llm-3.1.1 → janus_llm-3.2.0}/janus/__init__.py

@@ -5,7 +5,7 @@ from langchain_core._api.deprecation import LangChainDeprecationWarning
 from .converter.translate import Translator
 from .metrics import *  # noqa: F403
 
-__version__ = "3.1.1"
+__version__ = "3.2.0"
 
 # Ignoring a deprecation warning from langchain_core that I can't seem to hunt down
 warnings.filterwarnings("ignore", category=LangChainDeprecationWarning)

{janus_llm-3.1.1 → janus_llm-3.2.0}/janus/cli.py

@@ -32,8 +32,12 @@ from janus.language.treesitter import TreeSitterSplitter
 from janus.llm.model_callbacks import COST_PER_1K_TOKENS
 from janus.llm.models_info import (
     MODEL_CONFIG_DIR,
+    MODEL_ID_TO_LONG_ID,
     MODEL_TYPE_CONSTRUCTORS,
+    MODEL_TYPES,
     TOKEN_LIMITS,
+    bedrock_models,
+    openai_models,
 )
 from janus.metrics.cli import evaluate
 from janus.utils.enums import LANGUAGES
@@ -179,7 +183,7 @@ def translate(
             "-L",
             help="The custom name of the model set with 'janus llm add'.",
         ),
-    ] = "gpt-3.5-turbo-0125",
+    ] = "gpt-4o",
     max_prompts: Annotated[
         int,
         typer.Option(
@@ -301,7 +305,7 @@ def document(
             "-L",
             help="The custom name of the model set with 'janus llm add'.",
         ),
-    ] = "gpt-3.5-turbo-0125",
+    ] = "gpt-4o",
     max_prompts: Annotated[
         int,
         typer.Option(
@@ -437,7 +441,7 @@ def diagram(
             "-L",
             help="The custom name of the model set with 'janus llm add'.",
         ),
-    ] = "gpt-3.5-turbo-0125",
+    ] = "gpt-4o",
     max_prompts: Annotated[
         int,
         typer.Option(
@@ -800,16 +804,44 @@ def llm_add(
             "model_cost": {"input": in_cost, "output": out_cost},
         }
     elif model_type == "OpenAI":
-        model_name = typer.prompt("Enter the model name", default="gpt-3.5-turbo-0125")
+        model_id = typer.prompt(
+            "Enter the model ID (list model IDs with `janus llm ls -a`)",
+            default="gpt-4o",
+            type=click.Choice(openai_models),
+            show_choices=False,
+        )
         params = dict(
-            model_name=model_name,
+            # OpenAI uses the "model_name" key for what we're calling "long_model_id"
+            model_name=MODEL_ID_TO_LONG_ID[model_id],
             temperature=0.7,
             n=1,
         )
-        max_tokens = TOKEN_LIMITS[model_name]
-        model_cost = COST_PER_1K_TOKENS[model_name]
+        max_tokens = TOKEN_LIMITS[MODEL_ID_TO_LONG_ID[model_id]]
+        model_cost = COST_PER_1K_TOKENS[MODEL_ID_TO_LONG_ID[model_id]]
+        cfg = {
+            "model_type": model_type,
+            "model_id": model_id,
+            "model_args": params,
+            "token_limit": max_tokens,
+            "model_cost": model_cost,
+        }
+    elif model_type == "BedrockChat" or model_type == "Bedrock":
+        model_id = typer.prompt(
+            "Enter the model ID (list model IDs with `janus llm ls -a`)",
+            default="bedrock-claude-sonnet",
+            type=click.Choice(bedrock_models),
+            show_choices=False,
+        )
+        params = dict(
+            # Bedrock uses the "model_id" key for what we're calling "long_model_id"
+            model_id=MODEL_ID_TO_LONG_ID[model_id],
+            model_kwargs={"temperature": 0.7},
+        )
+        max_tokens = TOKEN_LIMITS[MODEL_ID_TO_LONG_ID[model_id]]
+        model_cost = COST_PER_1K_TOKENS[MODEL_ID_TO_LONG_ID[model_id]]
         cfg = {
             "model_type": model_type,
+            "model_id": model_id,
             "model_args": params,
             "token_limit": max_tokens,
             "model_cost": model_cost,
@@ -821,6 +853,31 @@ def llm_add(
     print(f"Model config written to {model_cfg}")
 
 
+@llm.command("ls", help="List all of the user-configured models")
+def llm_ls(
+    all: Annotated[
+        bool,
+        typer.Option(
+            "--all",
+            "-a",
+            is_flag=True,
+            help="List all models, including the default model IDs.",
+            click_type=click.Choice(sorted(list(MODEL_TYPE_CONSTRUCTORS.keys()))),
+        ),
+    ] = False,
+):
+    print("\n[green]User-configured models[/green]:")
+    for model_cfg in MODEL_CONFIG_DIR.glob("*.json"):
+        with open(model_cfg, "r") as f:
+            cfg = json.load(f)
+        print(f"\t[blue]{model_cfg.stem}[/blue]: [purple]{cfg['model_type']}[/purple]")
+
+    if all:
+        print("\n[green]Available model IDs[/green]:")
+        for model_id, model_type in MODEL_TYPES.items():
+            print(f"\t[blue]{model_id}[/blue]: [purple]{model_type}[/purple]")
+
+
 @embedding.command("add", help="Add an embedding model config to janus")
 def embedding_add(
     model_name: Annotated[

{janus_llm-3.1.1 → janus_llm-3.2.0}/janus/converter/_tests/test_translate.py

@@ -79,7 +79,7 @@ class TestTranslator(unittest.TestCase):
     def setUp(self):
         """Set up the tests."""
         self.translator = Translator(
-            model="gpt-3.5-turbo-0125",
+            model="gpt-4o",
             source_language="fortran",
             target_language="python",
             target_version="3.10",
@@ -88,7 +88,7 @@ class TestTranslator(unittest.TestCase):
         self.TEST_FILE_EMBEDDING_COUNT = 14
 
         self.req_translator = RequirementsDocumenter(
-            model="gpt-3.5-turbo-0125",
+            model="gpt-4o",
             source_language="fortran",
             prompt_template="requirements",
         )
@@ -317,14 +317,14 @@ class TestDiagramGenerator(unittest.TestCase):
     def setUp(self):
         """Set up the tests."""
         self.diagram_generator = DiagramGenerator(
-            model="gpt-3.5-turbo-0125",
+            model="gpt-4o",
             source_language="fortran",
             diagram_type="Activity",
         )
 
     def test_init(self):
         """Test __init__ method."""
-        self.assertEqual(self.diagram_generator._model_name, "gpt-3.5-turbo-0125")
+        self.assertEqual(self.diagram_generator._model_name, "gpt-4o")
         self.assertEqual(self.diagram_generator._source_language, "fortran")
         self.assertEqual(self.diagram_generator._diagram_type, "Activity")
 
@@ -370,8 +370,8 @@ def test_language_combinations(
     """Tests that translator target language settings are consistent
     with prompt template expectations.
     """
-    translator = Translator(model="gpt-3.5-turbo-0125")
-    translator.set_model("gpt-3.5-turbo-0125")
+    translator = Translator(model="gpt-4o")
+    translator.set_model("gpt-4o")
     translator.set_source_language(source_language)
     translator.set_target_language(expected_target_language, expected_target_version)
     translator.set_prompt(prompt_template)
@@ -379,5 +379,5 @@ def test_language_combinations(
     assert translator._target_language == expected_target_language  # nosec
     assert translator._target_version == expected_target_version  # nosec
     assert translator._splitter.language == source_language  # nosec
-    assert translator._splitter.model.model_name == "gpt-3.5-turbo-0125"  # nosec
+    assert translator._splitter.model.model_name == "gpt-4o"  # nosec
     assert translator._prompt_template_name == prompt_template  # nosec

{janus_llm-3.1.1 → janus_llm-3.2.0}/janus/converter/converter.py

@@ -64,7 +64,7 @@ class Converter:
 
     def __init__(
         self,
-        model: str = "gpt-3.5-turbo-0125",
+        model: str = "gpt-4o",
         model_arguments: dict[str, Any] = {},
         source_language: str = "fortran",
         max_prompts: int = 10,
@@ -92,6 +92,7 @@ class Converter:
         self.override_token_limit: bool = max_tokens is not None
 
         self._model_name: str
+        self._model_id: str
         self._custom_model_arguments: dict[str, Any]
 
         self._source_language: str
@@ -265,7 +266,9 @@ class Converter:
         # model_arguments.update(self._custom_model_arguments)
 
         # Load the model
-        self._llm, token_limit, self.model_cost = load_model(self._model_name)
+        self._llm, self._model_id, token_limit, self.model_cost = load_model(
+            self._model_name
+        )
         # Set the max_tokens to less than half the model's limit to allow for enough
         # tokens at output
         # Only modify max_tokens if it is not specified by user
@@ -283,7 +286,7 @@ class Converter:
         If the relevant fields have not been changed since the last time this
         method was called, nothing happens.
         """
-        prompt_engine = MODEL_PROMPT_ENGINES[self._model_name](
+        prompt_engine = MODEL_PROMPT_ENGINES[self._model_id](
             source_language=self._source_language,
             prompt_template=self._prompt_template_name,
         )

{janus_llm-3.1.1 → janus_llm-3.2.0}/janus/converter/translate.py

@@ -90,7 +90,7 @@ class Translator(Converter):
                     f"({self._source_language} != {self._target_language})"
                 )
 
-        prompt_engine = MODEL_PROMPT_ENGINES[self._model_name](
+        prompt_engine = MODEL_PROMPT_ENGINES[self._model_id](
             source_language=self._source_language,
             target_language=self._target_language,
             target_version=self._target_version,

{janus_llm-3.1.1 → janus_llm-3.2.0}/janus/language/alc/_tests/test_alc.py

@@ -11,8 +11,8 @@ class TestAlcSplitter(unittest.TestCase):
 
     def setUp(self):
         """Set up the tests."""
-        model_name = "gpt-3.5-turbo-0125"
-        llm, _, _ = load_model(model_name)
+        model_name = "gpt-4o"
+        llm, _, _, _ = load_model(model_name)
         self.splitter = AlcSplitter(model=llm)
         self.combiner = Combiner(language="ibmhlasm")
         self.test_file = Path("janus/language/alc/_tests/alc.asm")
@@ -20,7 +20,7 @@ class TestAlcSplitter(unittest.TestCase):
     def test_split(self):
         """Test the split method."""
         tree_root = self.splitter.split(self.test_file)
-        self.assertEqual(tree_root.n_descendents, 34)
+        self.assertAlmostEqual(tree_root.n_descendents, 32, delta=5)
         self.assertLessEqual(tree_root.max_tokens, self.splitter.max_tokens)
         self.assertFalse(tree_root.complete)
         self.combiner.combine_children(tree_root)

{janus_llm-3.1.1 → janus_llm-3.2.0}/janus/language/binary/_tests/test_binary.py

@@ -13,9 +13,9 @@ class TestBinarySplitter(unittest.TestCase):
     """Tests for the BinarySplitter class."""
 
     def setUp(self):
-        model_name = "gpt-3.5-turbo-0125"
+        model_name = "gpt-4o"
         self.binary_file = Path("janus/language/binary/_tests/hello")
-        self.llm, _, _ = load_model(model_name)
+        self.llm, _, _, _ = load_model(model_name)
         self.splitter = BinarySplitter(model=self.llm)
         os.environ["GHIDRA_INSTALL_PATH"] = "~/programs/ghidra_10.4_PUBLIC"
 

{janus_llm-3.1.1 → janus_llm-3.2.0}/janus/language/mumps/_tests/test_mumps.py

@@ -11,8 +11,8 @@ class TestMumpsSplitter(unittest.TestCase):
 
     def setUp(self):
         """Set up the tests."""
-        model_name = "gpt-3.5-turbo-0125"
-        llm, _, _ = load_model(model_name)
+        model_name = "gpt-4o"
+        llm, _, _, _ = load_model(model_name)
         self.splitter = MumpsSplitter(model=llm)
         self.combiner = Combiner(language="mumps")
         self.test_file = Path("janus/language/mumps/_tests/mumps.m")

{janus_llm-3.1.1 → janus_llm-3.2.0}/janus/language/treesitter/_tests/test_treesitter.py

@@ -11,9 +11,9 @@ class TestTreeSitterSplitter(unittest.TestCase):
 
     def setUp(self):
         """Set up the tests."""
-        model_name = "gpt-3.5-turbo-0125"
+        model_name = "gpt-4o"
         self.maxDiff = None
-        self.llm, _, _ = load_model(model_name)
+        self.llm, _, _, _ = load_model(model_name)
 
     def _split(self):
         """Split the test file."""

{janus_llm-3.1.1 → janus_llm-3.2.0}/janus/llm/models_info.py

@@ -55,11 +55,12 @@ openai_model_reroutes = {
 }
 
 openai_models = [
-    "gpt-4-0613",
-    "gpt-4-1106-preview",
-    "gpt-4-0125-preview",
-    "gpt-4o-2024-05-13",
-    "gpt-3.5-turbo-0125",
+    "gpt-4o",
+    "gpt-4",
+    "gpt-4-turbo",
+    "gpt-4-turbo-preview",
+    "gpt-3.5-turbo",
+    "gpt-3.5-turbo-16k",
 ]
 claude_models = [
     "bedrock-claude-v2",
@@ -133,8 +134,8 @@ _open_ai_defaults: dict[str, str] = {
     "openai_organization": os.getenv("OPENAI_ORG_ID"),
 }
 
-model_identifiers = {
-    **{m: m for m in openai_models},
+MODEL_ID_TO_LONG_ID = {
+    **{m: mr for m, mr in openai_model_reroutes.items()},
     "bedrock-claude-v2": "anthropic.claude-v2",
     "bedrock-claude-instant-v1": "anthropic.claude-instant-v1",
     "bedrock-claude-haiku": "anthropic.claude-3-haiku-20240307-v1:0",
@@ -157,7 +158,7 @@ model_identifiers = {
 
 MODEL_DEFAULT_ARGUMENTS: dict[str, dict[str, str]] = {
     k: (dict(model_name=k) if k in openai_models else dict(model_id=v))
-    for k, v in model_identifiers.items()
+    for k, v in MODEL_ID_TO_LONG_ID.items()
 }
 
 DEFAULT_MODELS = list(MODEL_DEFAULT_ARGUMENTS.keys())
@@ -199,22 +200,38 @@ TOKEN_LIMITS: dict[str, int] = {
 }
 
 
-def load_model(model_name: str) -> tuple[BaseLanguageModel, int, dict[str, float]]:
+def get_available_model_names() -> list[str]:
+    avaialable_models = []
+    for file in MODEL_CONFIG_DIR.iterdir():
+        if file.is_file():
+            avaialable_models.append(MODEL_CONFIG_DIR.stem)
+    return avaialable_models
+
+
+def load_model(user_model_name: str) -> tuple[BaseLanguageModel, int, dict[str, float]]:
     if not MODEL_CONFIG_DIR.exists():
         MODEL_CONFIG_DIR.mkdir(parents=True)
-    model_config_file = MODEL_CONFIG_DIR / f"{model_name}.json"
+    model_config_file = MODEL_CONFIG_DIR / f"{user_model_name}.json"
     if not model_config_file.exists():
-        if model_name not in DEFAULT_MODELS:
-            if model_name in openai_model_reroutes:
-                model_name = openai_model_reroutes[model_name]
-            else:
-                raise ValueError(f"Error: could not find model {model_name}")
+        log.warning(
+            f"Model {user_model_name} not found in user-defined models, searching "
+            f"default models for {user_model_name}."
+        )
+        model_id = user_model_name
+        if user_model_name not in DEFAULT_MODELS:
+            message = (
+                f"Model {user_model_name} not found in default models. Make sure to run "
+                "`janus llm add` first."
+            )
+            log.error(message)
+            raise ValueError(message)
         model_config = {
-            "model_type": MODEL_TYPES[model_name],
-            "model_args": MODEL_DEFAULT_ARGUMENTS[model_name],
-            "token_limit": TOKEN_LIMITS.get(model_identifiers[model_name], 4096),
+            "model_type": MODEL_TYPES[model_id],
+            "model_id": model_id,
+            "model_args": MODEL_DEFAULT_ARGUMENTS[model_id],
+            "token_limit": TOKEN_LIMITS.get(MODEL_ID_TO_LONG_ID[model_id], 4096),
             "model_cost": COST_PER_1K_TOKENS.get(
-                model_identifiers[model_name], {"input": 0, "output": 0}
+                MODEL_ID_TO_LONG_ID[model_id], {"input": 0, "output": 0}
             ),
         }
         with open(model_config_file, "w") as f:
@@ -227,4 +244,9 @@ def load_model(model_name: str) -> tuple[BaseLanguageModel, int, dict[str, float
     if model_config["model_type"] == "OpenAI":
         model_args.update(_open_ai_defaults)
     model = model_constructor(**model_args)
-    return model, model_config["token_limit"], model_config["model_cost"]
+    return (
+        model,
+        model_config["model_id"],
+        model_config["token_limit"],
+        model_config["model_cost"],
+    )

{janus_llm-3.1.1 → janus_llm-3.2.0}/janus/metrics/_tests/test_llm.py

@@ -53,7 +53,7 @@ class TestLLMMetrics(unittest.TestCase):
             self.bad_code,
             metric="quality",
             language="python",
-            llm=load_model("gpt-3.5-turbo-0125")[0],
+            llm=load_model("gpt-4o")[0],
         )
         self.assertLess(bad_code_quality, 5)
 
@@ -63,7 +63,7 @@ class TestLLMMetrics(unittest.TestCase):
             self.impressive_code,
             metric="quality",
             language="python",
-            llm=load_model("gpt-3.5-turbo-0125")[0],
+            llm=load_model("gpt-4o")[0],
         )
         self.assertGreater(impressive_code_quality, 5)
 
@@ -81,7 +81,7 @@ class TestLLMMetrics(unittest.TestCase):
             self.impressive_code_reference,
             metric="faithfulness",
             language="python",
-            llm=load_model("gpt-3.5-turbo-0125")[0],
+            llm=load_model("gpt-4o")[0],
         )
         self.assertGreater(faithfulness, 8)
 

{janus_llm-3.1.1 → janus_llm-3.2.0}/janus/metrics/metric.py

@@ -112,7 +112,7 @@ def metric(
                         "-L",
                         help="The custom name of the model set with 'janus llm add'.",
                     ),
-                ] = "gpt-3.5-turbo-0125",
+                ] = "gpt-4o",
                 progress: Annotated[
                     bool,
                     typer.Option(
@@ -135,7 +135,7 @@ def metric(
                 **kwargs,
             ):
                 out = []
-                llm, token_limit, model_cost = load_model(llm_name)
+                llm, _, token_limit, model_cost = load_model(llm_name)
                 if json_file_name is not None:
                     with open(json_file_name, "r") as f:
                         json_obj = json.load(f)
@@ -274,7 +274,7 @@ def metric(
                         "-L",
                         help="The custom name of the model set with 'janus llm add'.",
                     ),
-                ] = "gpt-3.5-turbo-0125",
+                ] = "gpt-4o",
                 progress: Annotated[
                     bool,
                     typer.Option(
@@ -296,7 +296,7 @@ def metric(
                 *args,
                 **kwargs,
             ):
-                llm, token_limit, model_cost = load_model(llm_name)
+                llm, _, token_limit, model_cost = load_model(llm_name)
                 if json_file_name is not None:
                     with open(json_file_name, "r") as f:
                         json_obj = json.load(f)

{janus_llm-3.1.1 → janus_llm-3.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "janus-llm"
-version = "3.1.1"
+version = "3.2.0"
 description = "A transcoding library using LLMs."
 authors = ["Michael Doyle <mdoyle@mitre.org>", "Chris Glasz <cglasz@mitre.org>",
            "Chris Tohline <ctohline@mitre.org>", "William Macke <wmacke@mitre.org>",