janus-llm 3.4.3__py3-none-any.whl → 3.5.1__py3-none-any.whl

janus/__init__.py

@@ -5,7 +5,7 @@ from langchain_core._api.deprecation import LangChainDeprecationWarning
 from janus.converter.translate import Translator
 from janus.metrics import *  # noqa: F403
 
-__version__ = "3.4.3"
+__version__ = "3.5.1"
 
 # Ignoring a deprecation warning from langchain_core that I can't seem to hunt down
 warnings.filterwarnings("ignore", category=LangChainDeprecationWarning)

janus/cli.py

@@ -2,7 +2,7 @@ import json
 import logging
 import os
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional
 
 import click
 import typer
@@ -12,6 +12,8 @@ from rich.console import Console
 from rich.prompt import Confirm
 from typing_extensions import Annotated
 
+from janus.converter.aggregator import Aggregator
+from janus.converter.converter import Converter
 from janus.converter.diagram import DiagramGenerator
 from janus.converter.document import Documenter, MadLibsDocumenter, MultiDocumenter
 from janus.converter.requirements import RequirementsDocumenter
@@ -439,6 +441,126 @@ def document(
     documenter.translate(input_dir, output_dir, overwrite, collection)
 
 
+def get_subclasses(cls):
+    return set(cls.__subclasses__()).union(
+        set(s for c in cls.__subclasses__() for s in get_subclasses(c))
+    )
+
+
+@app.command()
+def aggregate(
+    input_dir: Annotated[
+        Path,
+        typer.Option(
+            "--input",
+            "-i",
+            help="The directory containing the source code to be translated. "
+            "The files should all be in one flat directory.",
+        ),
+    ],
+    language: Annotated[
+        str,
+        typer.Option(
+            "--language",
+            "-l",
+            help="The language of the source code.",
+            click_type=click.Choice(sorted(LANGUAGES)),
+        ),
+    ],
+    output_dir: Annotated[
+        Path,
+        typer.Option(
+            "--output-dir", "-o", help="The directory to store the translated code in."
+        ),
+    ],
+    llm_name: Annotated[
+        str,
+        typer.Option(
+            "--llm",
+            "-L",
+            help="The custom name of the model set with 'janus llm add'.",
+        ),
+    ] = "gpt-4o",
+    max_prompts: Annotated[
+        int,
+        typer.Option(
+            "--max-prompts",
+            "-m",
+            help="The maximum number of times to prompt a model on one functional block "
+            "before exiting the application. This is to prevent wasting too much money.",
+        ),
+    ] = 10,
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite/--preserve",
+            help="Whether to overwrite existing files in the output directory",
+        ),
+    ] = False,
+    temperature: Annotated[
+        float,
+        typer.Option("--temperature", "-t", help="Sampling temperature.", min=0, max=2),
+    ] = 0.7,
+    collection: Annotated[
+        str,
+        typer.Option(
+            "--collection",
+            "-c",
+            help="If set, will put the translated result into a Chroma DB "
+            "collection with the name provided.",
+        ),
+    ] = None,
+    splitter_type: Annotated[
+        str,
+        typer.Option(
+            "-S",
+            "--splitter",
+            help="Name of custom splitter to use",
+            click_type=click.Choice(list(CUSTOM_SPLITTERS.keys())),
+        ),
+    ] = "file",
+    intermediate_converters: Annotated[
+        List[str],
+        typer.Option(
+            "-C",
+            "--converter",
+            help="Name of an intermediate converter to use",
+            click_type=click.Choice([c.__name__ for c in get_subclasses(Converter)]),
+        ),
+    ] = ["Documenter"],
+):
+    converter_subclasses = get_subclasses(Converter)
+    converter_subclasses_map = {c.__name__: c for c in converter_subclasses}
+    model_arguments = dict(temperature=temperature)
+    collections_config = get_collections_config()
+    converters = []
+    for ic in intermediate_converters:
+        converters.append(
+            converter_subclasses_map[ic](
+                model=llm_name,
+                model_arguments=model_arguments,
+                source_language=language,
+                max_prompts=max_prompts,
+                db_path=db_loc,
+                db_config=collections_config,
+                splitter_type=splitter_type,
+            )
+        )
+
+    aggregator = Aggregator(
+        intermediate_converters=converters,
+        model=llm_name,
+        model_arguments=model_arguments,
+        source_language=language,
+        max_prompts=max_prompts,
+        db_path=db_loc,
+        db_config=collections_config,
+        splitter_type=splitter_type,
+        prompt_template="basic_aggregation",
+    )
+    aggregator.translate(input_dir, output_dir, overwrite, collection)
+
+
 @app.command(
     help="Diagram input code using an LLM.",
     no_args_is_help=True,

janus/converter/aggregator.py

@@ -0,0 +1,52 @@
+from copy import deepcopy
+from typing import List
+
+from janus.converter.converter import Converter
+from janus.language.block import CodeBlock, TranslatedCodeBlock
+
+
+class Aggregator(Converter):
+    def __init__(
+        self,
+        intermediate_converters: List[Converter],
+        separator: str = "\n==============\n",
+        **kwargs,
+    ):
+        """
+        Initialization Method
+        Arguments:
+            intermediate_converters - list of converters to use
+            separator: separator string to partition different outputs before combination
+        """
+        self._intermediate_converters = intermediate_converters
+        self._separator = separator
+        super().__init__(**kwargs)
+        self._load_parameters()
+
+    def _iterative_translate(self, root: CodeBlock) -> TranslatedCodeBlock:
+        res = TranslatedCodeBlock(root, language=self._target_language)
+        return self._recursive_translate(res)
+
+    def _recursive_translate(self, root: TranslatedCodeBlock) -> None:
+        """
+        Recursively translates code blocks from a bottom up approach
+        """
+        original_text = root.original.text
+        if len(root.children) > 0:
+            for c in root.children:
+                self._recursive_translate(c)
+            root.original.text = self._combine_blocks(root.children, self._separator)
+        else:
+            int_reps = [deepcopy(root) for ic in self._intermediate_converters]
+            for ic, r in zip(self._intermediate_converters, int_reps):
+                ic._add_translation(r)
+            root.original.text = self._combine_blocks(int_reps, self._separator)
+        self._add_translation(root)
+        root.original.text = original_text
+        return root
+
+    def _combine_blocks(self, blocks: List[TranslatedCodeBlock], separator: str) -> str:
+        """
+        Combines code blocks into a single piece of text
+        """
+        return separator.join([block.text for block in blocks])

janus/converter/diagram.py

@@ -130,7 +130,7 @@ class DiagramGenerator(Documenter):
         If the relevant fields have not been changed since the last time this method was
         called, nothing happens.
         """
-        self._diagram_prompt_engine = MODEL_PROMPT_ENGINES[self._model_name](
+        self._diagram_prompt_engine = MODEL_PROMPT_ENGINES[self._model_id](
             source_language=self._source_language,
             target_language="text",
             target_version=None,

janus/converter/document.py

@@ -125,7 +125,7 @@ class MadLibsDocumenter(Documenter):
             out_text = self._parser.parse(working_block.text)
             obj.update(json.loads(out_text))
 
-        self._parser.set_reference(block.original)
+        self._parser.parse_input(block.original)
         block.text = self._parser.parse(json.dumps(obj))
         block.tokens = self._llm.get_num_tokens(block.text)
         block.translated = True

janus/parsers/doc_parser.py

@@ -92,11 +92,12 @@ class MadlibsDocumentationParser(JanusParser):
     def __init__(self):
         super().__init__(expected_keys=[])
 
-    def parse_input(self, block: CodeBlock):
+    def parse_input(self, block: CodeBlock) -> str:
         # TODO: Perform comment stripping/placeholding here rather than in script
         text = super().parse_input(block)
         comment_ids = re.findall(r"<(?:BLOCK|INLINE)_COMMENT (\w{8})>", text)
         self.expected_keys = set(comment_ids)
+        return text
 
     def parse(self, text: str | BaseMessage) -> str:
         if isinstance(text, BaseMessage):

{janus_llm-3.4.3.dist-info → janus_llm-3.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: janus-llm
-Version: 3.4.3
+Version: 3.5.1
 Summary: A transcoding library using LLMs.
 Home-page: https://github.com/janus-llm/janus-llm
 License: Apache 2.0

{janus_llm-3.4.3.dist-info → janus_llm-3.5.1.dist-info}/RECORD

@@ -1,15 +1,16 @@
-janus/__init__.py,sha256=HX3kYLkqXTi8C9sXb0P1mGki4DMJJzxHSQSxoilv7sA,361
+janus/__init__.py,sha256=nMslKA6BZu1FW0pfi-tziluM8r6gETz2o0YNYzKC6Lo,361
 janus/__main__.py,sha256=lEkpNtLVPtFo8ySDZeXJ_NXDHb0GVdZFPWB4gD4RPS8,64
 janus/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/_tests/conftest.py,sha256=V7uW-oq3YbFiRPvrq15YoVVrA1n_83pjgiyTZ-IUGW8,963
 janus/_tests/test_cli.py,sha256=oYJsUGWfpBJWEGRG5NGxdJedU5DU_m6fwJ7xEbJVYl0,4244
-janus/cli.py,sha256=_92FvDV4qza0nSmyiXqacYxyo1gY6IPwD4gCm6kZfqI,33213
+janus/cli.py,sha256=84deHkbRWxT2ThHa9b08HBIIThy-mw1b5WCfjWOcuME,36950
 janus/converter/__init__.py,sha256=U2EOMcCykiC0ZqhorNefOP_04hOF18qhYoPKrVp1Vrk,345
 janus/converter/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/converter/_tests/test_translate.py,sha256=yzcFEGc_z8QmBBBmC9dZnfL9tT8CD1rkpc8Hz44Jp4c,5631
+janus/converter/aggregator.py,sha256=MuAXMKmq6PuUo_w6ljyiuDn81Gk2dN-Ci7FVeLc6vhs,1966
 janus/converter/converter.py,sha256=GL4moCOi5vvvsEjAf_7Vrn5GHq12nptH47AAP4soeQc,27956
-janus/converter/diagram.py,sha256=AyxkoyfMoQ7jpuOqyJ3bIvYqHJSEhmR-gYrIsq-d8tk,5285
-janus/converter/document.py,sha256=hsW512veNjFWbdl5WriuUdNmMEqZy8ktRvqn9rRmA6E,4566
+janus/converter/diagram.py,sha256=sqztG_8T_gQ4njk5V7tg1QCgT6gFbL3CUy7t2l0X5i4,5283
+janus/converter/document.py,sha256=qNt2UncMheUBadXCFHGq74tqCrvZub5DCgZpd3Qa54o,4564
 janus/converter/evaluate.py,sha256=APWQUY3gjAXqkJkPzvj0UA4wPK3Cv9QSJLM-YK9t-ng,476
 janus/converter/requirements.py,sha256=9tvQ40FZJtG8niIFn45gPQCgKKHVPPoFLinBv6RAqO4,2027
 janus/converter/translate.py,sha256=0brQTlSfBYmXtoM8QYIOiyr0LrTr0S1n68Du-BR7_WQ,4236
@@ -85,7 +86,7 @@ janus/parsers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/parsers/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/parsers/_tests/test_code_parser.py,sha256=3ay5QpUPcynX_EJ-YLl3PR28poutUkT7qvyWUSQ7Too,928
 janus/parsers/code_parser.py,sha256=3l0HfzgrvJuiwk779s9ZsgUl3xbp1nE1qZxh8aDYRBI,873
-janus/parsers/doc_parser.py,sha256=G7kDdJ3davHhEFXqwNYw7hOB26hKH_d_0XdpyyBHq_U,5835
+janus/parsers/doc_parser.py,sha256=0pUsNZ9hKQLjIi8L8BgkOBHQZ_EGoFLHrBQ4hoDkjSw,5862
 janus/parsers/eval_parser.py,sha256=Gjh6aTZgpYd2ASJUEPMo4LpCL00cBmbOqc4KM3hy8x8,2922
 janus/parsers/parser.py,sha256=y6VV64bgVidf-oEFla3I--_28tnJsPBc6QUD_SkbfSE,1614
 janus/parsers/refiner_parser.py,sha256=W7ecDX7j6N-uWtRBYFD1EiEzDpDQws16nrVFzyArzf8,1632
@@ -101,8 +102,8 @@ janus/utils/_tests/test_progress.py,sha256=Rs_u5PiGjP-L-o6C1fhwfE1ig8jYu9Xo9s4p8
 janus/utils/enums.py,sha256=AoilbdiYyMvY2Mp0AM4xlbLSELfut2XMwhIM1S_msP4,27610
 janus/utils/logger.py,sha256=KZeuaMAnlSZCsj4yL0P6N-JzZwpxXygzACWfdZFeuek,2337
 janus/utils/progress.py,sha256=PIpcQec7SrhsfqB25LHj2CDDkfm9umZx90d9LZnAx6k,1469
-janus_llm-3.4.3.dist-info/LICENSE,sha256=_j0st0a-HB6MRbP3_BW3PUqpS16v54luyy-1zVyl8NU,10789
-janus_llm-3.4.3.dist-info/METADATA,sha256=rFGRSJIXyld0d04oeG6s7iC9HfQlyuN6I7b1UaCXXWo,4184
-janus_llm-3.4.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-janus_llm-3.4.3.dist-info/entry_points.txt,sha256=OGhQwzj6pvXp79B0SaBD5apGekCu7Dwe9fZZT_TZ544,39
-janus_llm-3.4.3.dist-info/RECORD,,
+janus_llm-3.5.1.dist-info/LICENSE,sha256=_j0st0a-HB6MRbP3_BW3PUqpS16v54luyy-1zVyl8NU,10789
+janus_llm-3.5.1.dist-info/METADATA,sha256=kIZdLtkybJ-S_dyglL8kPRwnt2wND8wS2nEyM76FL-U,4184
+janus_llm-3.5.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+janus_llm-3.5.1.dist-info/entry_points.txt,sha256=OGhQwzj6pvXp79B0SaBD5apGekCu7Dwe9fZZT_TZ544,39
+janus_llm-3.5.1.dist-info/RECORD,,