janus-llm 4.4.5__py3-none-any.whl → 4.5.4__py3-none-any.whl

janus/__init__.py

@@ -5,7 +5,7 @@ from langchain_core._api.deprecation import LangChainDeprecationWarning
 from janus.converter.translate import Translator
 from janus.metrics import *  # noqa: F403
 
-__version__ = "4.4.5"
+__version__ = "4.5.4"
 
 # Ignoring a deprecation warning from langchain_core that I can't seem to hunt down
 warnings.filterwarnings("ignore", category=LangChainDeprecationWarning)

janus/cli/pipeline.py

@@ -8,6 +8,7 @@ from typing_extensions import Annotated
 
 from janus.cli.constants import CONVERTERS
 from janus.converter.chain import ConverterChain
+from janus.converter.pool import ConverterPool
 from janus.utils.enums import LANGUAGES
 
 
@@ -41,11 +42,13 @@ def instiantiate_pipeline(
         pipeline[0]["kwargs"] = {}
     pipeline[0]["kwargs"].update(source_language=language, model=model)
     if use_janus_inputs is not None:
-        pipeline[0]["kwargs"].update(janus_inputs=use_janus_inputs)
-    print(pipeline[0])
+        pipeline[0]["kwargs"].update(use_janus_inputs=use_janus_inputs)
     converters = [instiantiate(pipeline[0])]
     for p in pipeline[1:]:
-        p["kwargs"].update(source_language=converters[-1].target_language, model=model)
+        if not isinstance(converters[-1], ConverterPool) and p["type"] != "ConverterPool":
+            p["kwargs"].update(
+                source_language=converters[-1].target_language, model=model
+            )
         converters.append(instiantiate(p))
     return ConverterChain(*converters)
 

janus/cli/self_eval.py

@@ -123,6 +123,14 @@ def llm_self_eval(
             "If unspecificed, model's default max will be used.",
         ),
     ] = None,
+    use_janus_inputs: Annotated[
+        bool,
+        typer.Option(
+            "-j",
+            "--use-janus-inputs",
+            help="Prsent if translator should use janus files as inputs",
+        ),
+    ] = False,
 ):
     from janus.converter.evaluate import InlineCommentEvaluator, RequirementEvaluator
 
@@ -137,6 +145,7 @@ def llm_self_eval(
         max_tokens=max_tokens,
         splitter_type=splitter_type,
         refiner_types=refiner_types,
+        use_janus_inputs=use_janus_inputs,
     )
     # Setting parser type here
     if evaluation_type == "incose":

janus/converter/__init__.py

@@ -3,5 +3,7 @@ from janus.converter.diagram import DiagramGenerator
 from janus.converter.document import ClozeDocumenter, Documenter, MultiDocumenter
 from janus.converter.evaluate import Evaluator
 from janus.converter.partition import Partitioner
+from janus.converter.passthrough import ConverterPassthrough
+from janus.converter.pool import ConverterPool
 from janus.converter.requirements import RequirementsDocumenter
 from janus.converter.translate import Translator

janus/converter/_tests/test_translate.py

@@ -120,6 +120,7 @@ class TestDiagramGenerator(unittest.TestCase):
                 children=[],
             ),
             language="python",
+            converter=self.diagram_generator,
         )
         self.diagram_generator._add_translation(block)
         self.assertTrue(block.translated)

janus/converter/chain.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
 from janus.converter.converter import Converter
-from janus.language.block import CodeBlock, TranslatedCodeBlock
+from janus.language.block import BlockCollection, CodeBlock, TranslatedCodeBlock
 from janus.utils.logger import create_logger
 
 log = create_logger(__name__)
@@ -24,157 +24,77 @@ class ConverterChain(Converter):
             target_language=self._converters[-1]._target_language,
             target_version=self._converters[-1]._target_version,
             use_janus_inputs=self._converters[0]._use_janus_inputs,
+            input_types=self._converters[0]._input_types,
+            input_labels=self._converters[0]._input_labels,
+            output_type=self._converters[-1]._output_type,
+            output_label=self._converters[-1]._output_label,
         )
         super().__init__(**kwargs)
 
-    def _run_converters(
-        self, translated_code_block, name: str, failure_path: Path | None = None
+    def translate_blocks(
+        self, input_blocks: CodeBlock | list[CodeBlock], failure_path: Path | None = None
     ):
-        for i, converter in enumerate(self._converters[1:]):
-            if not translated_code_block.translated:
+        failed = False
+        for i, converter in enumerate(self._converters):
+            translated_code_blocks = converter.translate_blocks(input_blocks)
+            if not translated_code_blocks.translation_completed:
                 log.info(
                     f"Error: chain failed to translate at step {i}:"
                     f"{self._converters[i].__class__.__name__}"
                 )
+                failed = True
                 break
-            if converter._use_janus_inputs:
-                janus_obj = self._converters[i]._get_output_obj(translated_code_block)
-                translated_code_block = converter.translate_janus_obj(
-                    janus_obj, name, failure_path
-                )
-            else:
-                translated_code_block = converter.translate_block(
-                    translated_code_block.to_codeblock(), name, failure_path
-                )
-        if not translated_code_block.translated:
+            input_blocks = translated_code_blocks.to_codeblock()
+        if not failed and not translated_code_blocks.translation_completed:
             log.info(
                 f"Error: chain failed to translate at step {len(self._converters)-1}: "
                 f"{self._converters[-1].__class__.__name__}"
             )
-
-        return translated_code_block
-
-    def translate_file(
-        self, file: Path, failure_path: Path | None = None
-    ) -> TranslatedCodeBlock:
-        """Translate a file using the chain of converters
-
-        Arguments:
-            file: The file to translate
-            failure_path: The path to write the failure file to
-
-        Returns:
-            The translated code block
-        """
-        filename = file.name
-        translated_code_block = self._converters[0].translate_file(file, failure_path)
-        translated_code_block = self._run_converters(
-            translated_code_block, filename, failure_path
-        )
-        return translated_code_block
-
-    def translate_text(
-        self, text: str, name: str, failure_path: Path | None = None
-    ) -> TranslatedCodeBlock:
-        """Translate a text using the chain of converters
-
-        Arguments:
-            text: The text to translate
-            name: The name of the file
-            failure_path: The path to write the failure file to
-
-        Returns:
-            The translated code block
-        """
-        translated_code_block = self._converters[0].translate_text(
-            text, name, failure_path
-        )
-        translated_code_block = self._run_converters(
-            translated_code_block, name, failure_path
-        )
-        return translated_code_block
-
-    def translate_block(
-        self,
-        input_block: CodeBlock | list[CodeBlock],
-        name: str,
-        failure_path: Path | None = None,
-    ) -> TranslatedCodeBlock:
-        """Translate a block of code using the chain of converters
-
-        Arguments:
-            input_block: The block of code to translate
-            name: The name of the file
-            failure_path: The path to write the failure file to
-
-        Returns:
-            The translated code block
-        """
-        translated_code_block = self._converters[0].translate_block(
-            input_block, name, failure_path
-        )
-        translated_code_block = self._run_converters(
-            translated_code_block, name, failure_path
-        )
-        return translated_code_block
+        return translated_code_blocks
+
+    def _combine_metadata(self, metadatas: list[dict]):
+        metadata = super()._combine_metadata(metadatas)
+        if isinstance(metadata["type"], list):
+            metadata["type"] = metadata["type"][-1]
+        if isinstance(metadata["label"], list):
+            metadata["label"] = metadata["label"][-1]
+        metadata["type"] = metadatas[-1]["type"]
+        metadata["label"] = metadatas[-1]["label"]
+        return metadata
 
     def _get_output_obj(
-        self, block: TranslatedCodeBlock | list, combine_children: bool = True
+        self,
+        block: TranslatedCodeBlock | BlockCollection,
+        combine_children: bool = True,
+        include_previous_outputs: bool = True,
     ) -> dict[str, int | float | str | dict[str, str] | dict[str, float]]:
-        output_obj = super()._get_output_obj(block, combine_children)
         intermediate_outputs = []
-        for i, intermediate_out in enumerate(block.previous_generations):
-            if isinstance(intermediate_out, TranslatedCodeBlock):
+        c_index = 0  # current converter index
+        start_index = 0  # start index of newly generated intermediate outputs
+        for g in block.previous_generations:
+            if isinstance(g, dict):
+                intermediate_outputs.append(g)
+                # Find the first index where we generated code
+                start_index += 1
+            else:
                 intermediate_outputs.append(
-                    self._converters[i]._get_output_obj(intermediate_out)
+                    self._converters[c_index]._get_output_obj(
+                        g, self._converters[c_index]._combine_output, False
+                    )
                 )
-            else:
-                intermediate_outputs.append(intermediate_out)
-        intermediate_outputs.append(self._converters[-1]._get_output_obj(block))
-        output_obj["intermediate_outputs"] = intermediate_outputs
-        metadata = output_obj["metadata"]
-        metadata["cost"] += sum(
-            b.cost if isinstance(b, TranslatedCodeBlock) else b["metadata"]["cost"]
-            for b in block.previous_generations
-        )
-        metadata["processing_time"] += sum(
-            (
-                b.processing_time
-                if isinstance(b, TranslatedCodeBlock)
-                else b["metadata"]["processing_time"]
-            )
-            for b in block.previous_generations
-        )
-        metadata["num_requests"] += sum(
-            (
-                b.total_num_requests
-                if isinstance(b, TranslatedCodeBlock)
-                else b["metadata"]["num_requests"]
+                c_index += 1
+        intermediate_outputs.append(
+            self._converters[-1]._get_output_obj(
+                block, self._converters[-1]._combine_output, False
             )
-            for b in block.previous_generations
         )
-        metadata["input_tokens"] += sum(
-            (
-                b.total_request_input_tokens
-                if isinstance(b, TranslatedCodeBlock)
-                else b["metadata"]["input_tokens"]
-            )
-            for b in block.previous_generations
+        out = dict(
+            input=intermediate_outputs[start_index]["input"],
+            metadata=self._combine_metadata(
+                [i["metadata"] for i in intermediate_outputs]
+            ),
+            outputs=intermediate_outputs[-1]["outputs"],
         )
-        metadata["output_tokens"] += sum(
-            (
-                b.total_request_output_tokens
-                if isinstance(b, TranslatedCodeBlock)
-                else b["metadata"]["output_tokens"]
-            )
-            for b in block.previous_generations
-        )
-        output_obj["metadata"] = metadata
-        if len(block.previous_generations) > 0:
-            b = block.previous_generations[0]
-            output_obj["input"] = (
-                (b.original.text or "")
-                if isinstance(b, TranslatedCodeBlock)
-                else b["input"]
-            )
-        return output_obj
+        if include_previous_outputs:
+            out["intermediate_outputs"] = intermediate_outputs
+        return out