janus-llm 4.3.5__py3-none-any.whl → 4.4.5__py3-none-any.whl

janus/metrics/metric.py

@@ -1,12 +1,13 @@
 import inspect
 import json
-from pathlib import Path
 from typing import Callable, Optional
 
 import click
 import typer
 from typing_extensions import Annotated
 
+from janus.cli.constants import CONVERTERS
+from janus.converter.converter import Converter
 from janus.llm import load_model
 from janus.llm.model_callbacks import COST_PER_1K_TOKENS
 from janus.metrics.cli import evaluate
@@ -70,31 +71,6 @@ def metric(
                         help="Reference file or string to use as reference/baseline.",
                     ),
                 ] = None,
-                json_file_name: Annotated[
-                    Optional[str],
-                    typer.Option(
-                        "--json",
-                        "-j",
-                        help="Json file to extract pairs from \
-                            (if set ignores --target and --reference)",
-                    ),
-                ] = None,
-                target_key: Annotated[
-                    str,
-                    typer.Option(
-                        "--target-key",
-                        "-tk",
-                        help="json key to extract list of target strings",
-                    ),
-                ] = "target",
-                reference_key: Annotated[
-                    str,
-                    typer.Option(
-                        "--reference-key",
-                        "-rk",
-                        help="json key to extract list of reference strings",
-                    ),
-                ] = "reference",
                 file_pairing_method: Annotated[
                     str,
                     typer.Option(
@@ -123,6 +99,14 @@ def metric(
                         is_flag=True,
                     ),
                 ] = False,
+                use_janus_inputs: Annotated[
+                    bool,
+                    typer.Option(
+                        "-j",
+                        "--use-janus-inputs",
+                        help="present if janus output files should be evaluated",
+                    ),
+                ] = False,
                 use_strings: Annotated[
                     bool,
                     typer.Option(
@@ -137,25 +121,23 @@ def metric(
             ):
                 out = []
                 llm = load_model(llm_name)
-                if json_file_name is not None:
-                    with open(json_file_name, "r") as f:
-                        json_obj = json.load(f)
-                    pairs = {}
-                    for key in json_obj:
-                        doc = json_obj[key]
-                        ref = doc[reference_key]
-                        experiments = doc["experiments"]
-                        for model_key in experiments:
-                            model_dict = experiments[model_key]
-                            if not isinstance(model_dict, dict):
-                                continue
-                            if target_key not in model_dict:
-                                continue
-                            if model_key not in pairs:
-                                pairs[model_key] = {}
-                            for k in model_dict[target_key]:
-                                pairs[model_key][k] = (model_dict[target_key][k], ref[k])
-                elif target is not None and reference is not None:
+                if use_janus_inputs:
+                    with open(target, "r") as f:
+                        target_obj = json.load(f)
+                    with open(reference, "r") as f:
+                        reference_obj = json.load(f)
+                    converter_cls = CONVERTERS.get(
+                        target_obj["metadata"].get("converter_name", "Converter"),
+                        Converter,
+                    )
+                    out = converter_cls.eval_obj_reference(
+                        target=target_obj,
+                        reference=reference_obj,
+                        metric_func=function,
+                        *args,
+                        **kwargs,
+                    )
+                else:
                     if use_strings:
                         target_contents = target
                         reference_contents = reference
@@ -175,25 +157,6 @@ def metric(
                         token_limit=llm.token_limit,
                         model_cost=COST_PER_1K_TOKENS[llm.model_id],
                     )
-                else:
-                    raise ValueError(
-                        "Error, specify json or target and reference files/strings"
-                    )
-                if isinstance(pairs, dict):
-                    out = {}
-                    for k in pairs:
-                        out[k] = apply_function_pairs(
-                            pairs[k],
-                            function,
-                            progress,
-                            language,
-                            llm,
-                            llm.token_limit,
-                            COST_PER_1K_TOKENS[llm.model_id],
-                            *args,
-                            **kwargs,
-                        )
-                else:
                     out = apply_function_pairs(
                         pairs,
                         function,
@@ -205,17 +168,15 @@ def metric(
                         *args,
                         **kwargs,
                     )
-                out_file = Path(out_file)
-                out_file.parent.mkdir(parents=True, exist_ok=True)
                 with open(out_file, "w") as f:
+                    log.info(f"Writing output to {out_file}")
                     json.dump(out, f)
-                    log.info(f"Saved results to file: {out_file}")
 
             sig1 = inspect.signature(function)
             sig2 = inspect.signature(func)
             func.__signature__ = sig2.replace(
                 parameters=tuple(
-                    list(sig2.parameters.values())[:11]
+                    list(sig2.parameters.values())[:9]
                     + list(sig1.parameters.values())[2:-1]
                 )
             )
@@ -241,23 +202,14 @@ def metric(
                         "--target", "-t", help="Target file or string to evaluate."
                     ),
                 ] = None,
-                json_file_name: Annotated[
-                    Optional[str],
+                use_janus_inputs: Annotated[
+                    bool,
                     typer.Option(
-                        "--json",
                         "-j",
-                        help="Json file to extract pairs from \
-                            (if set ignores --target)",
+                        "--use-janus-inputs",
+                        help="whether to use a janus output file as input",
                     ),
-                ] = None,
-                target_key: Annotated[
-                    str,
-                    typer.Option(
-                        "--target-key",
-                        "-tk",
-                        help="json key to extract list of target strings",
-                    ),
-                ] = "target",
+                ] = False,
                 splitting_method: Annotated[
                     str,
                     typer.Option(
@@ -298,25 +250,17 @@ def metric(
                 **kwargs,
             ):
                 llm = load_model(llm_name)
-                if json_file_name is not None:
-                    with open(json_file_name, "r") as f:
-                        json_obj = json.load(f)
-                    strings = {}
-                    for key in json_obj:
-                        doc = json_obj[key]
-                        experiments = doc["experiments"]
-                        for model_key in experiments:
-                            model_dict = experiments[model_key]
-                            if not isinstance(model_dict, dict):
-                                continue
-                            if target_key not in model_dict:
-                                continue
-                            if model_key not in strings:
-                                strings[model_key] = {}
-                            for k in model_dict[target_key]:
-                                strings[model_key][k] = model_dict[target_key][k]
-                        # strings += list(json_obj[key][target_key].values())
-                elif target is not None:
+                if use_janus_inputs:
+                    with open(target, "r") as f:
+                        target_obj = json.load(f)
+                    converter_cls = CONVERTERS.get(
+                        target_obj["metadata"].get("converter_name", "Converter"),
+                        Converter,
+                    )
+                    out = converter_cls.eval_obj(
+                        target=target_obj, metric_func=function, *args, **kwargs
+                    )
+                else:
                     if use_strings:
                         target_contents = target
                     else:
@@ -332,25 +276,6 @@ def metric(
                         token_limit=llm.token_limit,
                         model_cost=COST_PER_1K_TOKENS[llm.model_id],
                     )
-                else:
-                    raise ValueError(
-                        "Error: must specify either json file or target file/string"
-                    )
-                if isinstance(strings, dict):
-                    out = {}
-                    for k in strings:
-                        out[k] = apply_function_strings(
-                            strings[k],
-                            function,
-                            progress,
-                            language,
-                            llm,
-                            llm.token_limit,
-                            COST_PER_1K_TOKENS[llm.model_id],
-                            *args,
-                            **kwargs,
-                        )
-                else:
                     out = apply_function_strings(
                         strings,
                         function,
@@ -362,17 +287,15 @@ def metric(
                         *args,
                         **kwargs,
                     )
-                out_file = Path(out_file)
-                out_file.parent.mkdir(parents=True, exist_ok=True)
                 with open(out_file, "w") as f:
+                    log.info(f"Writing output to {out_file}")
                     json.dump(out, f)
-                    log.info(f"Saved results to file: {out_file}")
 
             sig1 = inspect.signature(function)
             sig2 = inspect.signature(func)
             func.__signature__ = sig2.replace(
                 parameters=tuple(
-                    list(sig2.parameters.values())[:9]
+                    list(sig2.parameters.values())[:7]
                     + list(sig1.parameters.values())[1:-1]
                 )
             )

janus/parsers/reqs_parser.py

@@ -41,7 +41,7 @@ class RequirementsParser(JanusParser):
             )
         return json.dumps(obj)
 
-    def parse_combined_output(self, text: str):
+    def parse_combined_output(self, text: str) -> str:
         """Parse the output text from the LLM when multiple inputs are combined.
 
         Arguments:
@@ -52,10 +52,10 @@ class RequirementsParser(JanusParser):
         """
         json_strings = re.findall(r"\{.*?\}", text)
         output_list = list()
-        for i, json_string in enumerate(json_strings, 1):
+        for _, json_string in enumerate(json_strings, 1):
             json_dict = json.loads(json_string)
             output_list.append(json_dict["requirements"])
-        return output_list
+        return json.dumps(output_list)
 
     def get_format_instructions(self) -> str:
         """Get the format instructions for the parser.

{janus_llm-4.3.5.dist-info → janus_llm-4.4.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: janus-llm
-Version: 4.3.5
+Version: 4.4.5
 Summary: A transcoding library using LLMs.
 License: Apache 2.0
 Author: Michael Doyle
@@ -53,7 +53,7 @@ Description-Content-Type: text/markdown
 
 
 <p align="center">
-    <img src="assets/icons/logo_horizontal.png">
+    <img src="https://raw.githubusercontent.com/janus-llm/janus-llm/public/assets/icons/logo_horizontal.png">
 </p>
 <p align="center">
 <a href="https://github.com/janus-llm/janus-llm/actions/workflows/pages.yml" target="_blank">
@@ -78,16 +78,12 @@ Description-Content-Type: text/markdown
 Janus (`janus-llm`) uses LLMs to aid in the modernization of legacy IT systems. The repository can currently do the following:
 
 1. Chunk code of over 100 programming languages to fit within different model context windows and add to a [Chroma](https://trychroma.com) vector database.
-2. Translate from one programming language to another on a file-by-file basis using an LLM with varying results (with the `translate.py` script).
-3. Translate from a binary file to a programming language using Ghidra decompilation.
-4. Do 1-3 with a CLI tool (`janus`).
+2. Translate from one programming language to another on a file-by-file basis using an LLM.
+3. Translate from a binary file to a programming language using [Ghidra](https://github.com/NationalSecurityAgency/ghidra) decompilation.
+4. Generate requirements, UML diagrams, code comments, and summaries from source code.
+5. Evaluate the products that you generate.
+6. Do 1-5 with a CLI tool (`janus`).
 
-## Roadmap
-
-### Priorities
-
-1. Scripts interacting with Chroma Vector DB for RAG translation and understanding.
-2. Evaluation of outputs in CLI using LLM self-evaluation or static analysis.
 
 ## Installation
 
@@ -111,10 +107,14 @@ export PATH=$PATH:$HOME/.local/bin
 poetry install
 ```
 
+### Documentation
+
+See [the documentation](https://janus-llm.github.io/janus-llm) for more information on how to use the package.
+
 ### Contributing
 
 See our [contributing pages](https://janus-llm.github.io/janus-llm/contributing.html)
 
 ### Copyright
-Copyright ©2024 The MITRE Corporation. ALL RIGHTS RESERVED. Approved for Public Release; Distribution Unlimited. Public Release Case Number 23-4084.
+Copyright ©2025 The MITRE Corporation. ALL RIGHTS RESERVED. Approved for Public Release; Distribution Unlimited. Public Release Case Number 23-4084.
 

{janus_llm-4.3.5.dist-info → janus_llm-4.4.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-janus/__init__.py,sha256=2-o8OirYdtEMQhfO5X6nIzP0AsdZHGY4PtRjBg33AFo,361
+janus/__init__.py,sha256=GnzTgemQclNcCJL0IUvvlWD0XnSmMjzf9cBI55HYKv8,361
 janus/__main__.py,sha256=Z1-R163-Ryac2kY1sKYfflqBl40nm-o54tlPjGiOP98,68
 janus/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/_tests/conftest.py,sha256=V7uW-oq3YbFiRPvrq15YoVVrA1n_83pjgiyTZ-IUGW8,963
@@ -7,28 +7,30 @@ janus/_tests/evaluator_tests/incose_tests/incose_large_test.json,sha256=_tiBnF_D
 janus/_tests/evaluator_tests/incose_tests/incose_small_test.json,sha256=KHizvtrEvRLrBw25q34I-qonDyFUSUpHpOpFhCmKjAo,2557
 janus/_tests/evaluator_tests/inline_comment_tests/mumps_inline_comment_test.m,sha256=i2U_YymEFxpX9w9_5wN_bV4ZHzk8THyVdvhH-2Xildk,3490
 janus/_tests/test_cli.py,sha256=Y1Gx17fBNTQWJGmjPff8bZgg5AnUY0fMWAHzQ0qwi08,4316
-janus/cli/aggregate.py,sha256=VLv32jMTNruCMOdnzMeNpSilu3h95HosrfGrh3YACPk,4142
-janus/cli/cli.py,sha256=-wBRRtjReFGyTUBZnXwGINFiWpi0X6prPQJWXVFt6bE,2684
-janus/cli/constants.py,sha256=hqQY-laRMArNCN0eXrQjpQbVn_-RfPoaBn9mTnWrfvA,1029
+janus/cli/aggregate.py,sha256=MlK1ZHrCLcwp8Pl4sronWZaB5Pp6AkWiCtHMriOkLgQ,4139
+janus/cli/cli.py,sha256=8n7Eo9-2-joVnCTDzrGu4RgemNtyuDaasaVGlhEiQXc,2820
+janus/cli/constants.py,sha256=4mb3YW18Q8bWAfKa05IJbXUxzQrmN81ELL5pR37PtCw,1206
 janus/cli/database.py,sha256=EkLv_fJNlG6oKvCcEzezdHIChaJlPUl3AgeejJU4rW0,9540
-janus/cli/diagram.py,sha256=qEJYGp66gs5v5BaC53eoqbRmxc5wecbqf0exS9JSDK4,5257
-janus/cli/document.py,sha256=jA7FVGQNmktG-qmTrAdixocEX9LhHTTOdyc9YCEolgQ,5425
+janus/cli/diagram.py,sha256=gOOneADH2klUzUx8CEM6qRtq2C5zDukzkF74R46DgWs,6254
+janus/cli/document.py,sha256=3tp85dIZ6sF59v-fEE1BLzk9Hkv4tL5CuFtNWvVzhBQ,5685
 janus/cli/embedding.py,sha256=WBNtcWPrXB4VfzKjeB_fOUm945gzgGjF2XbZkGltgYY,4215
-janus/cli/llm.py,sha256=LdtL0VmEMXS7usYRtFfsIe7q_OKyGziJZUykYtq8mSU,6343
-janus/cli/partition.py,sha256=D3w7s1w9GogXMOeIHzZZJ1MHMmTJupOqdPB80zmXAi0,3647
-janus/cli/self_eval.py,sha256=pZ7kDshISCfqOAqba5v56NhHnReXwxIsx0taRjAQIB4,4445
-janus/cli/translate.py,sha256=UodCvTv2_x-7XT7hqKq4inF5NLEI3zfbVLNAVJzVYxo,5683
+janus/cli/llm.py,sha256=JfSPUfkGAQyg-chc18ZoBu8K8w0Q9OUbktHsG90wnsg,6537
+janus/cli/partition.py,sha256=AHpB0mfhoQzcD0lAQFObbbD2Pw3KE5Hk0r_2XapT6Fg,3898
+janus/cli/pipeline.py,sha256=8LB3_qqd284Sfs_hFu-Ih-9G_ZUcM00LtlaCsM6xzMk,3667
+janus/cli/self_eval.py,sha256=fyshuaj_L36fIyBEpeBaVaSq5lPCLJzpeecI6ILV6ac,4419
+janus/cli/translate.py,sha256=Etob8z-SU0V8oEimUtYfPvYXHgLUP6pkf4IRDr4dbU0,5948
 janus/converter/__init__.py,sha256=7oDRGUyVGhxyiZQDYoY7z6Vyokmu-06hm2RMMfo9adg,393
 janus/converter/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-janus/converter/_tests/test_translate.py,sha256=1NxlQdWoKeMxmNCR4ka-asO6jcVbenJIv1VEpIDiSzU,5740
+janus/converter/_tests/test_translate.py,sha256=be1g0R0CUfIUI1U1TkqYLIVKRQMOe-rzS3J0xq_7aIE,5750
 janus/converter/aggregator.py,sha256=MuAXMKmq6PuUo_w6ljyiuDn81Gk2dN-Ci7FVeLc6vhs,1966
-janus/converter/converter.py,sha256=e7xWx2eXnv8TMRSm9jvhXOCggJMVjrAAQkKkquztLI8,27871
-janus/converter/diagram.py,sha256=-wktVBPrSBgNIQfHIfa2bJNg6L9CYJQgrr9-xU8DFPw,1646
-janus/converter/document.py,sha256=2eFWTrQvo9bhA9INLIwx7KJSgic1GtqO83IPREm5b_U,5156
-janus/converter/evaluate.py,sha256=EjZyqvuUfoDsmOv95Mv6As3BUiD5jUzuRSi_aX1LuqI,9514
-janus/converter/partition.py,sha256=ASvv4hAue44qHobO4kqr_tKr-eJsXCPPdD3NtNd9V-E,993
-janus/converter/requirements.py,sha256=9tvQ40FZJtG8niIFn45gPQCgKKHVPPoFLinBv6RAqO4,2027
-janus/converter/translate.py,sha256=64PouZ6nO95E-5YDRgZa5wA3TSgXkmHcND1qoSi1yzU,4232
+janus/converter/chain.py,sha256=PSE3AyzJFldiHs6nT4HwiDEcdlpAz3T42Rik3DuIbIw,6712
+janus/converter/converter.py,sha256=_uuZk1E275Vv1EVznUeDCuej4PTLrDSjXmcabgQDw6o,38980
+janus/converter/diagram.py,sha256=4FCeaEkjn5jh44_uYPrEP70rJ7plfIlpgjtExJBTQwo,1784
+janus/converter/document.py,sha256=9qYdN0eDaacKi2wrT3qYiyjyhTSu4ZBg1VrJ9JVyH_o,5225
+janus/converter/evaluate.py,sha256=w43xEyCGju901Ov9hFGO9EttLbi5hxzQ7nUmPSds33w,10625
+janus/converter/partition.py,sha256=ASpo7cwLGSZenOvUQ04LN9uI_RbvpQvf9j868rtoddA,644
+janus/converter/requirements.py,sha256=3HnRu7Z_h6OgsYxaKj3p_KAYizJYia2qvUnOHvjFMko,685
+janus/converter/translate.py,sha256=jwuhSWIzhjuMJEbHnZS9Y2D8FXhfs6W8PLfpiAeRE6M,1973
 janus/embedding/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/embedding/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/embedding/_tests/test_collections.py,sha256=llg-JSuRRFhKkHFiWWSHEWV3iaT6Lwue0lp2tEml9io,2668
@@ -53,7 +55,7 @@ janus/language/binary/_tests/hello.bin,sha256=SRcqx55Vm9GhstlNSA1jDm0AJCCPhP0Xuq
 janus/language/binary/_tests/test_binary.py,sha256=cIKIxjj6kIY3rcxLwqUPESP9bxWrHqMHx9TNuICgfeQ,1724
 janus/language/binary/binary.py,sha256=PHsVa8jcM7sW9aTboGRWXj6ewQznz0kVPNWtP4B9YPU,6555
 janus/language/binary/reveng/decompile_script.py,sha256=veW51oJzuO-4UD3Er062jXZ_FYtTFo9OCkl82Z2xr6A,2182
-janus/language/block.py,sha256=ob2sblvzn06jKodkQ2LvPR9fXVyuXSR1nhmbmDWn6TA,10543
+janus/language/block.py,sha256=cLWQLVCx-8joXTxGCBNmTTe85H80HP3Awoit61Dlf40,11679
 janus/language/combine.py,sha256=egZRl1xZXAFXa2ZjjfqnNckc9uxuo6e1MJgkRrCgvd8,3650
 janus/language/file.py,sha256=dh2S3owuDOAKaV-GopLmdezmZ4U_Fyp2pLyGmEf0-QU,581
 janus/language/mumps/__init__.py,sha256=-Ou_wJ-JgHezfp1dub2_qCYNiK9wO-zo2MlqxM9qiwE,48
@@ -89,7 +91,6 @@ janus/metrics/_tests/target.py,sha256=hiaJPP9CXkvFBV_wL-gOe_BzELTw0nvB6uCxhxtIiE
 janus/metrics/_tests/test_bleu.py,sha256=k9hSSdGMITsTiXs-Aj4Q5t1_p2v2eD0O9m-vcRi2GKo,1652
 janus/metrics/_tests/test_chrf.py,sha256=I6AB323mSNj9_0dRIgQdslN0OcQH2fzYxdep68d1LZU,2214
 janus/metrics/_tests/test_file_pairing.py,sha256=8qB9xaZlaWmc6oAqQGYreRZLrZmrlVALcgtCz0Zp0uA,1892
-janus/metrics/_tests/test_llm.py,sha256=foKsYM12E4hq4cWjrUDN3WCl9-ViiREgLhudcqRol7g,2977
 janus/metrics/_tests/test_reading.py,sha256=iXu5SIo2-Opu7hgKcqCwfV-m6ZUSnEmHuXCXddOj1Vg,852
 janus/metrics/_tests/test_rouge_score.py,sha256=dnP99nry-U5wyE-CiC0eQwm78IyScnmGQH3BeIEdmLY,2032
 janus/metrics/_tests/test_similarity_score.py,sha256=tdzH_8hYb2h7fKxpd_a75di-GPnU_frZ0zn2aeYzkso,811
@@ -99,8 +100,7 @@ janus/metrics/chrf.py,sha256=c8AJLVUtzOezH0R7cJ9hwkLQCIHonm56TTYRTmYuBK4,1485
 janus/metrics/cli.py,sha256=Duuw2RF47Z-t1pal0cg3L_-N_91rx29krirqtIwjYLY,157
 janus/metrics/complexity_metrics.py,sha256=9sHxa9tY9IgdFC4grCmDSuu8Nkphm_FWbTecxgW78go,6574
 janus/metrics/file_pairing.py,sha256=b0848gbDTqKVge-uGiujtHdsgcbkpJ04nWF2ul_gbCw,3749
-janus/metrics/llm_metrics.py,sha256=nzzJMSnoZfgtzzltjoCu1l4-Qwuycd-V3z17AtStYdY,5700
-janus/metrics/metric.py,sha256=ZQXz2LLXA3_WW6BluF8Q2LdJKYVWZLYmhO4Ai116ZPY,17168
+janus/metrics/metric.py,sha256=CXnj6gM1LzMJ2QUPWc1N7bg3V36EdHlLo3Idu31o7E0,13709
 janus/metrics/prompts/clarity.txt,sha256=UTo7a_E28a9JIto6wskQSUANPGdO6oPxf_3B2sqhgZw,328
 janus/metrics/prompts/completeness.txt,sha256=5z4a_9a2ruToQnQiW6S02A_IYBIOgwg4KIWHXChA2Co,518
 janus/metrics/prompts/faithfulness.txt,sha256=UdvfuW99_OsT7D7GhVQpG1StwnC3PFZwNbENS_Gev_c,460
@@ -122,7 +122,7 @@ janus/parsers/eval_parsers/incose_parser.py,sha256=udyK-24ocfrB1SzmggcERm73dBynr
 janus/parsers/eval_parsers/inline_comment_parser.py,sha256=QzKgzeWPhyIEkLxJBpeutSocSJjjXEcWRRS635bXEO8,3973
 janus/parsers/parser.py,sha256=YwkiFthesUaxHngrt4v2lb_HYgKpusnlVgTMwi3WttE,1874
 janus/parsers/partition_parser.py,sha256=2l4fERb7lf-RTKBTi-TZBys0kJXlzRJWeB8RHNzyTDs,5834
-janus/parsers/reqs_parser.py,sha256=NHoHdJ4F50YDWhYmBQgypP05u6aCPl8NyWTuMX94M7s,2504
+janus/parsers/reqs_parser.py,sha256=5kS3gtiofZLfnV4GaPEiSwxx3bAG4k-DBDc5wuGF6MQ,2523
 janus/parsers/uml.py,sha256=iB9j0gtaT4Zo4tYFCGQNRzXa6--wDCuw-sLeQk19kfc,3429
 janus/prompts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/prompts/prompt.py,sha256=JPr6wvIu1ZpUtTa93WJxa3fbosswtIJqqOY5H8nE2XA,10586
@@ -203,8 +203,8 @@ janus/utils/enums.py,sha256=o26C6xIqYSiC2areAGSAB4alP_uHfpKxNn5TnlZzlTI,29656
 janus/utils/logger.py,sha256=vKofNjSvnXduvfkAWf1kkEES32tI7iI-bojfjIa3XoQ,2466
 janus/utils/pdf_docs_reader.py,sha256=beMKHdYrFwg0m_i7n0OTJrut3sf4rEWFd7P_80A76WY,5140
 janus/utils/progress.py,sha256=PIpcQec7SrhsfqB25LHj2CDDkfm9umZx90d9LZnAx6k,1469
-janus_llm-4.3.5.dist-info/LICENSE,sha256=_j0st0a-HB6MRbP3_BW3PUqpS16v54luyy-1zVyl8NU,10789
-janus_llm-4.3.5.dist-info/METADATA,sha256=IRLmrJ7U-Xb0Pota4K-u4esm6o6RQL8fSIE367sZdxA,4608
-janus_llm-4.3.5.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-janus_llm-4.3.5.dist-info/entry_points.txt,sha256=rzOrXUSS0SNN8I-fKV0-IBoYLTCMXnZfMtu4HetlCkU,43
-janus_llm-4.3.5.dist-info/RECORD,,
+janus_llm-4.4.5.dist-info/LICENSE,sha256=_j0st0a-HB6MRbP3_BW3PUqpS16v54luyy-1zVyl8NU,10789
+janus_llm-4.4.5.dist-info/METADATA,sha256=EXj2wwef6SuUSXttNoB4oWXxD2eyqTKvsgXDqD0MQEo,4739
+janus_llm-4.4.5.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+janus_llm-4.4.5.dist-info/entry_points.txt,sha256=rzOrXUSS0SNN8I-fKV0-IBoYLTCMXnZfMtu4HetlCkU,43
+janus_llm-4.4.5.dist-info/RECORD,,

janus/metrics/_tests/test_llm.py

@@ -1,90 +0,0 @@
-import unittest
-from unittest.mock import patch
-
-import pytest
-
-from janus.llm.models_info import load_model
-from janus.metrics.llm_metrics import llm_evaluate_option, llm_evaluate_ref_option
-
-
-class TestLLMMetrics(unittest.TestCase):
-    def setUp(self):
-        self.bad_code = """
-        if __name__ == "__main__":
-            a1, a2, b3, b4 = 0, [1, 2000, "a"], 2, (1, 2)
-        for a in a2:
-            if b3:
-            elif not b3:
-                try:
-                    pass
-                except:
-                    raise ValueError
-            elif 1:
-                print(1)
-            else:
-                print(b4[0])
-                for (x, y) in range(a1, b3):
-                    for i in range(003300):
-                        for z in a2:
-                            printf(b4[2])
-        """
-        self.impressive_code = """
-        # This program prints out Hello, world!
-
-        print('Hello, world!')
-        """
-        self.impressive_code_reference = """
-        # An implementation of python Hello, world!
-
-        print("'Hello, world!")
-        """
-
-    @patch(".llm.models_info.load_model")
-    @patch("janus.metrics.llm_metrics.llm_evaluate")
-    @pytest.mark.llm_eval
-    def test_llm_self_eval_quality(self, mock_llm_evaluate, mock_load_model):
-        """Test that the quality llm self eval recognizes bad_code as bad code
-        (<5 on a scale of 1-10)"""
-        mock_llm_evaluate.return_value = 4  # return a value less than 5
-        mock_load_model.return_value = [None]  # return a dummy model
-
-        bad_code_quality = llm_evaluate_option(
-            self.bad_code,
-            self.bad_code,
-            metric="quality",
-            language="python",
-            llm=load_model("gpt-4o")[0],
-        )
-        self.assertLess(bad_code_quality, 5)
-
-        mock_llm_evaluate.return_value = 6  # return a value greater than 5
-        impressive_code_quality = llm_evaluate_option(
-            self.impressive_code,
-            self.impressive_code,
-            metric="quality",
-            language="python",
-            llm=load_model("gpt-4o")[0],
-        )
-        self.assertGreater(impressive_code_quality, 5)
-
-    @patch("janus.llm.models_info.load_model")
-    @patch("janus.metrics.llm_metrics.llm_evaluate")
-    @pytest.mark.llm_eval
-    def test_llm_self_eval_faithfulness(self, mock_llm_evaluate, mock_load_model):
-        """The two Hello, world! samples are more or less the same,
-        so the faithfulness score should be high"""
-        mock_llm_evaluate.return_value = 9  # return a high value
-        mock_load_model.return_value = [None]  # return a dummy model
-
-        faithfulness = llm_evaluate_ref_option(
-            self.impressive_code,
-            self.impressive_code_reference,
-            metric="faithfulness",
-            language="python",
-            llm=load_model("gpt-4o")[0],
-        )
-        self.assertGreater(faithfulness, 8)
-
-
-if __name__ == "__main__":
-    unittest.main()