janus-llm 4.4.5__tar.gz → 4.5.4__tar.gz

{janus_llm-4.4.5 → janus_llm-4.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: janus-llm
-Version: 4.4.5
+Version: 4.5.4
 Summary: A transcoding library using LLMs.
 License: Apache 2.0
 Author: Michael Doyle
@@ -24,22 +24,21 @@ Requires-Dist: langchain-community (>=0.2.0,<0.3.0)
 Requires-Dist: langchain-core (>=0.2.0,<0.3.0)
 Requires-Dist: langchain-openai (>=0.1.8,<0.2.0)
 Requires-Dist: langchain-unstructured (>=0.1.2,<0.2.0)
-Requires-Dist: nltk (>=3.8.1,<4.0.0)
 Requires-Dist: numpy (>=1.24.3,<2.0.0)
 Requires-Dist: openai (>=1.14.0,<2.0.0)
 Requires-Dist: pi-heif (>=0.20.0,<0.21.0)
-Requires-Dist: py-readability-metrics (>=1.4.5,<2.0.0)
-Requires-Dist: py-rouge (>=1.1,<2.0)
 Requires-Dist: pybind11 (>=2.13.6,<3.0.0)
 Requires-Dist: pytesseract (>=0.3.13,<0.4.0)
 Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
 Requires-Dist: rich (>=13.7.1,<14.0.0)
+Requires-Dist: rouge-score (>=0.1.2,<0.2.0)
 Requires-Dist: sacrebleu (>=2.4.1,<3.0.0)
 Requires-Dist: scikit-learn (>=1.5.2,<2.0.0)
 Requires-Dist: sentence-transformers (>=2.6.1,<3.0.0) ; extra == "hf-local" or extra == "all"
 Requires-Dist: setuptools (>=75.6.0,<76.0.0)
 Requires-Dist: tesseract (>=0.1.3,<0.2.0)
 Requires-Dist: text-generation (>=0.6.0,<0.7.0)
+Requires-Dist: textstat (>=0.7.5,<0.8.0)
 Requires-Dist: tiktoken (>=0.7.0,<0.8.0)
 Requires-Dist: transformers (>=4.31.0,<5.0.0)
 Requires-Dist: tree-sitter (>=0.21.0,<0.22.0)

{janus_llm-4.4.5 → janus_llm-4.5.4}/janus/__init__.py

@@ -5,7 +5,7 @@ from langchain_core._api.deprecation import LangChainDeprecationWarning
 from janus.converter.translate import Translator
 from janus.metrics import *  # noqa: F403
 
-__version__ = "4.4.5"
+__version__ = "4.5.4"
 
 # Ignoring a deprecation warning from langchain_core that I can't seem to hunt down
 warnings.filterwarnings("ignore", category=LangChainDeprecationWarning)

{janus_llm-4.4.5 → janus_llm-4.5.4}/janus/cli/pipeline.py

@@ -8,6 +8,7 @@ from typing_extensions import Annotated
 
 from janus.cli.constants import CONVERTERS
 from janus.converter.chain import ConverterChain
+from janus.converter.pool import ConverterPool
 from janus.utils.enums import LANGUAGES
 
 
@@ -41,11 +42,13 @@ def instiantiate_pipeline(
         pipeline[0]["kwargs"] = {}
     pipeline[0]["kwargs"].update(source_language=language, model=model)
     if use_janus_inputs is not None:
-        pipeline[0]["kwargs"].update(janus_inputs=use_janus_inputs)
-    print(pipeline[0])
+        pipeline[0]["kwargs"].update(use_janus_inputs=use_janus_inputs)
     converters = [instiantiate(pipeline[0])]
     for p in pipeline[1:]:
-        p["kwargs"].update(source_language=converters[-1].target_language, model=model)
+        if not isinstance(converters[-1], ConverterPool) and p["type"] != "ConverterPool":
+            p["kwargs"].update(
+                source_language=converters[-1].target_language, model=model
+            )
         converters.append(instiantiate(p))
     return ConverterChain(*converters)
 

{janus_llm-4.4.5 → janus_llm-4.5.4}/janus/cli/self_eval.py

@@ -123,6 +123,14 @@ def llm_self_eval(
             "If unspecificed, model's default max will be used.",
         ),
     ] = None,
+    use_janus_inputs: Annotated[
+        bool,
+        typer.Option(
+            "-j",
+            "--use-janus-inputs",
+            help="Prsent if translator should use janus files as inputs",
+        ),
+    ] = False,
 ):
     from janus.converter.evaluate import InlineCommentEvaluator, RequirementEvaluator
 
@@ -137,6 +145,7 @@ def llm_self_eval(
         max_tokens=max_tokens,
         splitter_type=splitter_type,
         refiner_types=refiner_types,
+        use_janus_inputs=use_janus_inputs,
     )
     # Setting parser type here
     if evaluation_type == "incose":

{janus_llm-4.4.5 → janus_llm-4.5.4}/janus/converter/__init__.py

@@ -3,5 +3,7 @@ from janus.converter.diagram import DiagramGenerator
 from janus.converter.document import ClozeDocumenter, Documenter, MultiDocumenter
 from janus.converter.evaluate import Evaluator
 from janus.converter.partition import Partitioner
+from janus.converter.passthrough import ConverterPassthrough
+from janus.converter.pool import ConverterPool
 from janus.converter.requirements import RequirementsDocumenter
 from janus.converter.translate import Translator

{janus_llm-4.4.5 → janus_llm-4.5.4}/janus/converter/_tests/test_translate.py

@@ -120,6 +120,7 @@ class TestDiagramGenerator(unittest.TestCase):
                 children=[],
             ),
             language="python",
+            converter=self.diagram_generator,
         )
         self.diagram_generator._add_translation(block)
         self.assertTrue(block.translated)

janus_llm-4.5.4/janus/converter/chain.py

@@ -0,0 +1,100 @@
+from pathlib import Path
+
+from janus.converter.converter import Converter
+from janus.language.block import BlockCollection, CodeBlock, TranslatedCodeBlock
+from janus.utils.logger import create_logger
+
+log = create_logger(__name__)
+
+
+class ConverterChain(Converter):
+    """
+    Class for representing multiple converters chained together
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        if len(args) == 0:
+            raise ValueError("Error: Converter chain must be passed at least 1 converter")
+        for converter in args:
+            if not isinstance(converter, Converter):
+                raise ValueError(f"Error: unrecognized type: {type(converter)}")
+        self._converters = args
+        kwargs.update(
+            source_language=self._converters[0].source_language,
+            target_language=self._converters[-1]._target_language,
+            target_version=self._converters[-1]._target_version,
+            use_janus_inputs=self._converters[0]._use_janus_inputs,
+            input_types=self._converters[0]._input_types,
+            input_labels=self._converters[0]._input_labels,
+            output_type=self._converters[-1]._output_type,
+            output_label=self._converters[-1]._output_label,
+        )
+        super().__init__(**kwargs)
+
+    def translate_blocks(
+        self, input_blocks: CodeBlock | list[CodeBlock], failure_path: Path | None = None
+    ):
+        failed = False
+        for i, converter in enumerate(self._converters):
+            translated_code_blocks = converter.translate_blocks(input_blocks)
+            if not translated_code_blocks.translation_completed:
+                log.info(
+                    f"Error: chain failed to translate at step {i}:"
+                    f"{self._converters[i].__class__.__name__}"
+                )
+                failed = True
+                break
+            input_blocks = translated_code_blocks.to_codeblock()
+        if not failed and not translated_code_blocks.translation_completed:
+            log.info(
+                f"Error: chain failed to translate at step {len(self._converters)-1}: "
+                f"{self._converters[-1].__class__.__name__}"
+            )
+        return translated_code_blocks
+
+    def _combine_metadata(self, metadatas: list[dict]):
+        metadata = super()._combine_metadata(metadatas)
+        if isinstance(metadata["type"], list):
+            metadata["type"] = metadata["type"][-1]
+        if isinstance(metadata["label"], list):
+            metadata["label"] = metadata["label"][-1]
+        metadata["type"] = metadatas[-1]["type"]
+        metadata["label"] = metadatas[-1]["label"]
+        return metadata
+
+    def _get_output_obj(
+        self,
+        block: TranslatedCodeBlock | BlockCollection,
+        combine_children: bool = True,
+        include_previous_outputs: bool = True,
+    ) -> dict[str, int | float | str | dict[str, str] | dict[str, float]]:
+        intermediate_outputs = []
+        c_index = 0  # current converter index
+        start_index = 0  # start index of newly generated intermediate outputs
+        for g in block.previous_generations:
+            if isinstance(g, dict):
+                intermediate_outputs.append(g)
+                # Find the first index where we generated code
+                start_index += 1
+            else:
+                intermediate_outputs.append(
+                    self._converters[c_index]._get_output_obj(
+                        g, self._converters[c_index]._combine_output, False
+                    )
+                )
+                c_index += 1
+        intermediate_outputs.append(
+            self._converters[-1]._get_output_obj(
+                block, self._converters[-1]._combine_output, False
+            )
+        )
+        out = dict(
+            input=intermediate_outputs[start_index]["input"],
+            metadata=self._combine_metadata(
+                [i["metadata"] for i in intermediate_outputs]
+            ),
+            outputs=intermediate_outputs[-1]["outputs"],
+        )
+        if include_previous_outputs:
+            out["intermediate_outputs"] = intermediate_outputs
+        return out