janus-llm 1.0.0__tar.gz → 2.0.1__tar.gz

{janus_llm-1.0.0 → janus_llm-2.0.1}/PKG-INFO

@@ -1,30 +1,41 @@
 Metadata-Version: 2.1
 Name: janus-llm
-Version: 1.0.0
+Version: 2.0.1
 Summary: A transcoding library using LLMs.
 Home-page: https://github.com/janus-llm/janus-llm
 License: Apache 2.0
 Author: Michael Doyle
 Author-email: mdoyle@mitre.org
-Requires-Python: >=3.10,<4.0
+Requires-Python: >=3.11,<3.12
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: chromadb (>=0.4.14,<0.5.0)
+Provides-Extra: all
+Provides-Extra: bedrock
+Provides-Extra: hf-local
+Requires-Dist: aenum (>=3.1.15,<4.0.0)
+Requires-Dist: boto3 (>=1.34.142,<2.0.0) ; extra == "bedrock" or extra == "all"
+Requires-Dist: chromadb (>=0.5.0,<0.6.0)
 Requires-Dist: gitpython (>=3.1.32,<4.0.0)
 Requires-Dist: gpt4all (>=2.0.2,<3.0.0)
-Requires-Dist: langchain (>=0.0.306,<0.0.307)
-Requires-Dist: langchain-community (>=0.0.16,<0.0.17)
+Requires-Dist: langchain (>=0.2.0,<0.3.0)
+Requires-Dist: langchain-anthropic (>=0.1.15,<0.2.0)
+Requires-Dist: langchain-community (>=0.2.0,<0.3.0)
+Requires-Dist: langchain-core (>=0.2.0,<0.3.0)
+Requires-Dist: langchain-openai (>=0.1.8,<0.2.0)
+Requires-Dist: nltk (>=3.8.1,<4.0.0)
 Requires-Dist: numpy (>=1.24.3,<2.0.0)
-Requires-Dist: openai (>=0.27.6,<0.28.0)
+Requires-Dist: openai (>=1.14.0,<2.0.0)
+Requires-Dist: py-readability-metrics (>=1.4.5,<2.0.0)
+Requires-Dist: py-rouge (>=1.1,<2.0)
 Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
-Requires-Dist: rich (>=13.3.5,<14.0.0)
+Requires-Dist: rich (>=13.7.1,<14.0.0)
+Requires-Dist: sacrebleu (>=2.4.1,<3.0.0)
+Requires-Dist: sentence-transformers (>=2.6.1,<3.0.0) ; extra == "hf-local" or extra == "all"
 Requires-Dist: text-generation (>=0.6.0,<0.7.0)
-Requires-Dist: tiktoken (>=0.4.0,<0.5.0)
+Requires-Dist: tiktoken (>=0.7.0,<0.8.0)
 Requires-Dist: transformers (>=4.31.0,<5.0.0)
-Requires-Dist: tree-sitter (>=0.20.1,<0.21.0)
+Requires-Dist: tree-sitter (>=0.21.0,<0.22.0)
 Requires-Dist: typer (>=0.9.0,<0.10.0)
 Project-URL: Documentation, https://janus-llm.github.io/janus-llm
 Project-URL: Repository, https://github.com/janus-llm/janus-llm
@@ -43,8 +54,10 @@ Description-Content-Type: text/markdown
 </a>
 <a href="https://github.com/psf/black" target="_blank">
     <img src="https://img.shields.io/badge/code%20style-black-000000.svg" alt="Code Style: black">
+</a>
 <a href="https://pypi.org/project/janus-llm" target="_blank">
     <img src="https://img.shields.io/pypi/pyversions/janus-llm" alt="Python versions">
+</a>
 <a href="https://pypi.org/project/janus-llm" target="_blank">
     <img src="https://img.shields.io/pypi/v/janus-llm?color=%2334D058&label=pypi%20package" alt="Package version">
 </a>
@@ -94,3 +107,6 @@ poetry install
 
 See our [contributing pages](https://janus-llm.github.io/janus-llm/contributing.html)
 
+### Copyright
+Copyright ©2024 The MITRE Corporation. ALL RIGHTS RESERVED. Approved for Public Release; Distribution Unlimited. Public Release Case Number 23-4084.
+

{janus_llm-1.0.0 → janus_llm-2.0.1}/README.md

@@ -11,8 +11,10 @@
 </a>
 <a href="https://github.com/psf/black" target="_blank">
     <img src="https://img.shields.io/badge/code%20style-black-000000.svg" alt="Code Style: black">
+</a>
 <a href="https://pypi.org/project/janus-llm" target="_blank">
     <img src="https://img.shields.io/pypi/pyversions/janus-llm" alt="Python versions">
+</a>
 <a href="https://pypi.org/project/janus-llm" target="_blank">
     <img src="https://img.shields.io/pypi/v/janus-llm?color=%2334D058&label=pypi%20package" alt="Package version">
 </a>
@@ -61,3 +63,6 @@ poetry install
 ### Contributing
 
 See our [contributing pages](https://janus-llm.github.io/janus-llm/contributing.html)
+
+### Copyright
+Copyright ©2024 The MITRE Corporation. ALL RIGHTS RESERVED. Approved for Public Release; Distribution Unlimited. Public Release Case Number 23-4084.

janus_llm-2.0.1/janus/__init__.py

@@ -0,0 +1,11 @@
+import warnings
+
+from langchain_core._api.deprecation import LangChainDeprecationWarning
+
+from .metrics import *  # noqa: F403
+from .translate import Translator
+
+__version__ = "2.0.1"
+
+# Ignoring a deprecation warning from langchain_core that I can't seem to hunt down
+warnings.filterwarnings("ignore", category=LangChainDeprecationWarning)

janus_llm-2.0.1/janus/__main__.py

@@ -0,0 +1,4 @@
+from .cli import app
+
+if __name__ == "__main__":
+    app()

janus_llm-2.0.1/janus/_tests/test_cli.py

@@ -0,0 +1,128 @@
+import traceback
+import unittest
+from unittest.mock import ANY, patch
+
+from typer.testing import CliRunner
+
+from ..cli import app, translate
+from ..embedding.embedding_models_info import EMBEDDING_MODEL_CONFIG_DIR
+from ..llm.models_info import MODEL_CONFIG_DIR
+
+
+class TestCli(unittest.TestCase):
+    def setUp(self):
+        self.runner = CliRunner()
+
+    def test_help(self):
+        result = self.runner.invoke(app, ["--help"])
+        self.assertEqual(result.exit_code, 0)
+
+    def test_version(self):
+        result = self.runner.invoke(app, ["--version"])
+        self.assertEqual(result.exit_code, 0)
+
+    def test_invalid_command(self):
+        result = self.runner.invoke(app, ["invalid_command"])
+        self.assertNotEqual(result.exit_code, 0)
+
+    def test_no_arguments(self):
+        result = self.runner.invoke(app)
+        self.assertEqual(result.exit_code, 0)
+
+    def test_translate_help(self):
+        result = self.runner.invoke(app, ["translate", "-h"])
+        self.assertEqual(result.exit_code, 0)
+
+    def test_document_help(self):
+        result = self.runner.invoke(app, ["document", "-h"])
+        self.assertEqual(result.exit_code, 0)
+
+    def test_db_help(self):
+        result = self.runner.invoke(app, ["db", "-h"])
+        self.assertEqual(result.exit_code, 0)
+
+    def test_llm_help(self):
+        result = self.runner.invoke(app, ["llm", "-h"])
+        self.assertEqual(result.exit_code, 0)
+
+    def test_llm_add(self):
+        llm_model_path = MODEL_CONFIG_DIR / "test-model-name.json"
+        if llm_model_path.exists():
+            llm_model_path.unlink()
+        result = self.runner.invoke(app, ["llm", "add", "test-model-name"])
+        self.assertEqual(result.exit_code, 0)
+        self.assertTrue(llm_model_path.exists())
+        llm_model_path.unlink()
+
+    def test_db_ls(self):
+        result = self.runner.invoke(app, ["db", "ls"])
+        self.assertEqual(result.exit_code, 0)
+
+    def test_db_status(self):
+        result = self.runner.invoke(app, ["db", "status"])
+        self.assertEqual(result.exit_code, 0)
+
+    def test_embedding_add(self):
+        embedding_model_path = (
+            EMBEDDING_MODEL_CONFIG_DIR / "test-embedding-model-name.json"
+        )
+        if embedding_model_path.exists():
+            embedding_model_path.unlink()
+        result = self.runner.invoke(
+            app, ["embedding", "add", "test-embedding-model-name"]
+        )
+        self.assertEqual(result.exit_code, 0)
+        self.assertTrue(embedding_model_path.exists())
+        embedding_model_path.unlink()
+
+    def test_db_add_and_rm(self):
+        embedding_model_path = (
+            EMBEDDING_MODEL_CONFIG_DIR / "test-embedding-model-name.json"
+        )
+        if embedding_model_path.exists():
+            embedding_model_path.unlink()
+        result = self.runner.invoke(
+            app,
+            ["embedding", "add", "test-embedding-model-name", "-t", "HuggingFaceLocal"],
+        )
+        self.assertEqual(result.exit_code, 0)
+        result = self.runner.invoke(
+            app,
+            [
+                "db",
+                "add",
+                "test-db-name",
+                "test-embedding-model-name",
+                "-i",
+                "janus/language/mumps",
+            ],
+        )
+        traceback.print_exception(result.exception)
+        embedding_model_path.unlink()
+        self.assertEqual(result.exit_code, 0)
+
+        result = self.runner.invoke(app, ["db", "rm", "test-db-name", "-y"])
+        self.assertEqual(result.exit_code, 0)
+
+    @patch("janus.translate.Translator.translate", autospec=True)
+    def test_translate(self, mock_translate):
+        # Arrange
+        mock_instance = mock_translate.return_value
+        mock_instance.translate.return_value = None  # or whatever you expect
+
+        # Act
+        translate(
+            source_lang="matlab",
+            target_lang="python",
+            input_dir="janus/",
+            output_dir="janus/",
+            overwrite=True,
+            temp=0.7,
+            prompt_template="simple",
+            parser_type="code",
+            collection=None,
+        )
+
+        # Assert
+        mock_translate.assert_called_once()
+        mock_translate.assert_called_once_with(ANY, "janus/", "janus/", True, None)

{janus_llm-1.0.0 → janus_llm-2.0.1}/janus/_tests/test_translate.py

@@ -7,7 +7,8 @@ from langchain.schema import Document
 from langchain.schema.embeddings import Embeddings
 from langchain.schema.vectorstore import VST, VectorStore
 
-from ..translate import Translator
+from ..language.block import CodeBlock, TranslatedCodeBlock
+from ..translate import DiagramGenerator, Translator
 
 # from langchain.vectorstores import Chroma
 
@@ -75,7 +76,7 @@ class TestTranslator(unittest.TestCase):
     def setUp(self):
         """Set up the tests."""
         self.translator = Translator(
-            model="gpt-3.5-turbo",
+            model="gpt-3.5-turbo-0125",
             source_language="fortran",
             target_language="python",
             target_version="3.10",
@@ -84,7 +85,7 @@ class TestTranslator(unittest.TestCase):
         self.TEST_FILE_EMBEDDING_COUNT = 14
 
         self.req_translator = Translator(
-            model="gpt-3.5-turbo",
+            model="gpt-3.5-turbo-0125",
             # embeddings_override=MockEmbeddingsFactory(),
             source_language="fortran",
             target_language="text",
@@ -317,6 +318,47 @@ class TestTranslator(unittest.TestCase):
         self.assertRaises(ValueError, self.translator._load_parameters)
 
 
+class TestDiagramGenerator(unittest.TestCase):
+    """Tests for the DiagramGenerator class."""
+
+    def setUp(self):
+        """Set up the tests."""
+        self.diagram_generator = DiagramGenerator(
+            model="gpt-3.5-turbo-0125",
+            source_language="fortran",
+            diagram_type="Activity",
+        )
+
+    def test_init(self):
+        """Test __init__ method."""
+        self.assertEqual(self.diagram_generator._model_name, "gpt-3.5-turbo-0125")
+        self.assertEqual(self.diagram_generator._source_language, "fortran")
+        self.assertEqual(self.diagram_generator._diagram_type, "Activity")
+
+    def test_add_translation(self):
+        """Test _add_translation method."""
+        block = TranslatedCodeBlock(
+            original=CodeBlock(
+                id="test",
+                name="Test Block",
+                node_type="function",
+                language="python",
+                text="print('Hello, World!')",
+                start_point=(0, 0),
+                end_point=(1, 0),
+                start_byte=0,
+                end_byte=1,
+                tokens=5,
+                children=[],
+            ),
+            language="python",
+        )
+        self.diagram_generator._add_translation(block)
+        self.assertTrue(block.translated)
+        self.assertIsNotNone(block.text)
+        self.assertIsNotNone(block.tokens)
+
+
 @pytest.mark.parametrize(
     "source_language,prompt_template,expected_target_language,expected_target_version,"
     "parser_type",
@@ -337,8 +379,8 @@ def test_language_combinations(
     """Tests that translator target language settings are consistent
     with prompt template expectations.
     """
-    translator = Translator(model="gpt-3.5-turbo")
-    translator.set_model("gpt-3.5-turbo-16k")
+    translator = Translator(model="gpt-3.5-turbo-0125")
+    translator.set_model("gpt-3.5-turbo-0125")
     translator.set_source_language(source_language)
     translator.set_target_language(expected_target_language, expected_target_version)
     translator.set_parser_type(parser_type)
@@ -348,5 +390,5 @@ def test_language_combinations(
     assert translator._target_version == expected_target_version  # nosec
     assert translator._parser_type == parser_type  # nosec
     assert translator._splitter.language == source_language  # nosec
-    assert translator._splitter.model.model_name == "gpt-3.5-turbo-16k"  # nosec
-    assert translator._prompt_engine._template_name == prompt_template  # nosec
+    assert translator._splitter.model.model_name == "gpt-3.5-turbo-0125"  # nosec
+    assert translator._prompt_template_name == prompt_template  # nosec