janus-llm 2.0.2__py3-none-any.whl → 3.0.0__py3-none-any.whl

janus/__init__.py

@@ -2,10 +2,10 @@ import warnings
 
 from langchain_core._api.deprecation import LangChainDeprecationWarning
 
+from .converter.translate import Translator
 from .metrics import *  # noqa: F403
-from .translate import Translator
 
-__version__ = "2.0.2"
+__version__ = "3.0.0"
 
 # Ignoring a deprecation warning from langchain_core that I can't seem to hunt down
 warnings.filterwarnings("ignore", category=LangChainDeprecationWarning)

janus/__main__.py

@@ -1,4 +1,4 @@
-from .cli import app
+from janus.cli import app
 
 if __name__ == "__main__":
     app()

janus/_tests/test_cli.py

@@ -104,7 +104,7 @@ class TestCli(unittest.TestCase):
         result = self.runner.invoke(app, ["db", "rm", "test-db-name", "-y"])
         self.assertEqual(result.exit_code, 0)
 
-    @patch("janus.translate.Translator.translate", autospec=True)
+    @patch("janus.converter.translate.Translator.translate", autospec=True)
     def test_translate(self, mock_translate):
         # Arrange
         mock_instance = mock_translate.return_value
@@ -119,7 +119,6 @@ class TestCli(unittest.TestCase):
             overwrite=True,
             temp=0.7,
             prompt_template="simple",
-            parser_type="code",
             collection=None,
         )
 

janus/cli.py

@@ -12,34 +12,32 @@ from rich.console import Console
 from rich.prompt import Confirm
 from typing_extensions import Annotated
 
-from janus.language.naive.registry import CUSTOM_SPLITTERS
-
-from .embedding.collections import Collections
-from .embedding.database import ChromaEmbeddingDatabase
-from .embedding.embedding_models_info import (
+from janus.converter.diagram import DiagramGenerator
+from janus.converter.document import Documenter, MadLibsDocumenter, MultiDocumenter
+from janus.converter.requirements import RequirementsDocumenter
+from janus.converter.translate import Translator
+from janus.embedding.collections import Collections
+from janus.embedding.database import ChromaEmbeddingDatabase
+from janus.embedding.embedding_models_info import (
     EMBEDDING_COST_PER_MODEL,
     EMBEDDING_MODEL_CONFIG_DIR,
     EMBEDDING_TOKEN_LIMITS,
     EmbeddingModelType,
 )
-from .embedding.vectorize import ChromaDBVectorizer
-from .language.binary import BinarySplitter
-from .language.mumps import MumpsSplitter
-from .language.treesitter import TreeSitterSplitter
-from .llm.model_callbacks import COST_PER_1K_TOKENS
-from .llm.models_info import MODEL_CONFIG_DIR, MODEL_TYPE_CONSTRUCTORS, TOKEN_LIMITS
-from .metrics.cli import evaluate
-from .translate import (
-    PARSER_TYPES,
-    DiagramGenerator,
-    Documenter,
-    MadLibsDocumenter,
-    MultiDocumenter,
-    RequirementsDocumenter,
-    Translator,
+from janus.embedding.vectorize import ChromaDBVectorizer
+from janus.language.binary import BinarySplitter
+from janus.language.mumps import MumpsSplitter
+from janus.language.naive.registry import CUSTOM_SPLITTERS
+from janus.language.treesitter import TreeSitterSplitter
+from janus.llm.model_callbacks import COST_PER_1K_TOKENS
+from janus.llm.models_info import (
+    MODEL_CONFIG_DIR,
+    MODEL_TYPE_CONSTRUCTORS,
+    TOKEN_LIMITS,
 )
-from .utils.enums import LANGUAGES
-from .utils.logger import create_logger
+from janus.metrics.cli import evaluate
+from janus.utils.enums import LANGUAGES
+from janus.utils.logger import create_logger
 
 httpx_logger = logging.getLogger("httpx")
 httpx_logger.setLevel(logging.WARNING)
@@ -72,10 +70,14 @@ def get_collections_config():
 
 
 app = typer.Typer(
-    help="Choose a command",
+    help=(
+        "[bold][dark_orange]Janus[/dark_orange] is a CLI for translating, "
+        "documenting, and diagramming code using large language models.[/bold]"
+    ),
     add_completion=False,
     no_args_is_help=True,
     context_settings={"help_option_names": ["-h", "--help"]},
+    rich_markup_mode="rich",
 )
 
 
@@ -207,15 +209,6 @@ def translate(
             "path to a directory containing those template files.",
         ),
     ] = "simple",
-    parser_type: Annotated[
-        str,
-        typer.Option(
-            "--parser",
-            "-P",
-            click_type=click.Choice(sorted(PARSER_TYPES)),
-            help="The type of parser to use.",
-        ),
-    ] = "code",
     collection: Annotated[
         str,
         typer.Option(
@@ -225,15 +218,15 @@ def translate(
             "collection with the name provided.",
         ),
     ] = None,
-    custom_splitter: Annotated[
-        Optional[str],
+    splitter_type: Annotated[
+        str,
         typer.Option(
-            "-cs",
-            "--custom-splitter",
+            "-S",
+            "--splitter",
             help="Name of custom splitter to use",
             click_type=click.Choice(list(CUSTOM_SPLITTERS.keys())),
         ),
-    ] = None,
+    ] = "file",
     max_tokens: Annotated[
         int,
         typer.Option(
@@ -265,10 +258,9 @@ def translate(
         max_prompts=max_prompts,
         max_tokens=max_tokens,
         prompt_template=prompt_template,
-        parser_type=parser_type,
         db_path=db_loc,
         db_config=collections_config,
-        custom_splitter=custom_splitter,
+        splitter_type=splitter_type,
     )
     translator.translate(input_dir, output_dir, overwrite, collection)
 
@@ -364,15 +356,15 @@ def document(
             "collection with the name provided.",
         ),
     ] = None,
-    custom_splitter: Annotated[
-        Optional[str],
+    splitter_type: Annotated[
+        str,
         typer.Option(
-            "-cs",
-            "--custom-splitter",
+            "-S",
+            "--splitter",
             help="Name of custom splitter to use",
             click_type=click.Choice(list(CUSTOM_SPLITTERS.keys())),
         ),
-    ] = None,
+    ] = "file",
     max_tokens: Annotated[
         int,
         typer.Option(
@@ -393,7 +385,7 @@ def document(
         max_tokens=max_tokens,
         db_path=db_loc,
         db_config=collections_config,
-        custom_splitter=custom_splitter,
+        splitter_type=splitter_type,
     )
     if doc_mode == "madlibs":
         documenter = MadLibsDocumenter(
@@ -489,15 +481,15 @@ def diagram(
             help="Whether to use documentation in generation",
         ),
     ] = False,
-    custom_splitter: Annotated[
-        Optional[str],
+    splitter_type: Annotated[
+        str,
         typer.Option(
-            "-cs",
-            "--custom-splitter",
+            "-S",
+            "--splitter",
             help="Name of custom splitter to use",
             click_type=click.Choice(list(CUSTOM_SPLITTERS.keys())),
         ),
-    ] = None,
+    ] = "file",
 ):
     model_arguments = dict(temperature=temperature)
     collections_config = get_collections_config()
@@ -510,7 +502,7 @@ def diagram(
         db_config=collections_config,
         diagram_type=diagram_type,
         add_documentation=add_documentation,
-        custom_splitter=custom_splitter,
+        splitter_type=splitter_type,
     )
     diagram_generator.translate(input_dir, output_dir, overwrite, collection)
 

janus/converter/__init__.py

@@ -0,0 +1,6 @@
+from .converter import Converter
+from .diagram import DiagramGenerator
+from .document import Documenter, MadLibsDocumenter, MultiDocumenter
+from .evaluate import Evaluator
+from .requirements import RequirementsDocumenter
+from .translate import Translator

janus/{_tests → converter/_tests}/test_translate.py

@@ -7,8 +7,11 @@ from langchain.schema import Document
 from langchain.schema.embeddings import Embeddings
 from langchain.schema.vectorstore import VST, VectorStore
 
-from ..language.block import CodeBlock, TranslatedCodeBlock
-from ..translate import DiagramGenerator, Translator
+from janus.language.block import CodeBlock, TranslatedCodeBlock
+
+from ..diagram import DiagramGenerator
+from ..requirements import RequirementsDocumenter
+from ..translate import Translator
 
 # from langchain.vectorstores import Chroma
 
@@ -84,14 +87,10 @@ class TestTranslator(unittest.TestCase):
         self.test_file = Path("janus/language/treesitter/_tests/languages/fortran.f90")
         self.TEST_FILE_EMBEDDING_COUNT = 14
 
-        self.req_translator = Translator(
+        self.req_translator = RequirementsDocumenter(
             model="gpt-3.5-turbo-0125",
-            # embeddings_override=MockEmbeddingsFactory(),
             source_language="fortran",
-            target_language="text",
-            target_version="3.10",
             prompt_template="requirements",
-            parser_type="text",
         )
 
     @pytest.mark.translate
@@ -273,11 +272,6 @@ class TestTranslator(unittest.TestCase):
         # print_query_results(QUERY_STRING, n_results)
         # self.assertTrue(len(n_results) == 1, "Was splitting changed?")
 
-    def test_output_as_requirements(self):
-        """Is output type requirements?"""
-        self.assertFalse(self.translator.outputting_requirements())
-        self.assertTrue(self.req_translator.outputting_requirements())
-
     # @pytest.mark.slow
     # def test_document_embeddings_added_by_translate(self):
     #     vector_store = self.req_translator.embeddings(EmbeddingType.REQUIREMENT)
@@ -310,7 +304,6 @@ class TestTranslator(unittest.TestCase):
         self.assertRaises(
             ValueError, self.translator.set_target_language, "gobbledy", "goobledy"
         )
-        self.assertRaises(ValueError, self.translator.set_parser_type, "blah")
         self.assertRaises(
             ValueError, self.translator.set_source_language, "scribbledy-doop"
         )
@@ -360,13 +353,12 @@ class TestDiagramGenerator(unittest.TestCase):
 
 
 @pytest.mark.parametrize(
-    "source_language,prompt_template,expected_target_language,expected_target_version,"
-    "parser_type",
+    "source_language,prompt_template,expected_target_language,expected_target_version,",
     [
-        ("python", "document_inline", "python", "3.10", "code"),
-        ("fortran", "document", "text", None, "text"),
-        ("mumps", "requirements", "text", None, "text"),
-        ("python", "simple", "javascript", "es6", "code"),
+        ("python", "document_inline", "python", "3.10"),
+        ("fortran", "document", "text", None),
+        ("mumps", "requirements", "text", None),
+        ("python", "simple", "javascript", "es6"),
     ],
 )
 def test_language_combinations(
@@ -374,7 +366,6 @@ def test_language_combinations(
     prompt_template: str,
     expected_target_language: str,
     expected_target_version: str,
-    parser_type: str,
 ):
     """Tests that translator target language settings are consistent
     with prompt template expectations.
@@ -383,12 +374,10 @@ def test_language_combinations(
     translator.set_model("gpt-3.5-turbo-0125")
     translator.set_source_language(source_language)
     translator.set_target_language(expected_target_language, expected_target_version)
-    translator.set_parser_type(parser_type)
     translator.set_prompt(prompt_template)
     translator._load_parameters()
     assert translator._target_language == expected_target_language  # nosec
     assert translator._target_version == expected_target_version  # nosec
-    assert translator._parser_type == parser_type  # nosec
     assert translator._splitter.language == source_language  # nosec
     assert translator._splitter.model.model_name == "gpt-3.5-turbo-0125"  # nosec
     assert translator._prompt_template_name == prompt_template  # nosec