janus-llm 1.0.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

janus/metrics/_tests/test_llm.py

@@ -0,0 +1,91 @@
+import unittest
+from unittest.mock import patch
+
+import pytest
+
+from janus.llm.models_info import load_model
+
+from ..llm_metrics import llm_evaluate_option, llm_evaluate_ref_option
+
+
+class TestLLMMetrics(unittest.TestCase):
+    def setUp(self):
+        self.bad_code = """
+        if __name__ == "__main__":
+            a1, a2, b3, b4 = 0, [1, 2000, "a"], 2, (1, 2)
+        for a in a2:
+            if b3:
+            elif not b3:
+                try:
+                    pass
+                except:
+                    raise ValueError
+            elif 1:
+                print(1)
+            else:
+                print(b4[0])
+                for (x, y) in range(a1, b3):
+                    for i in range(003300):
+                        for z in a2:
+                            printf(b4[2])
+        """
+        self.impressive_code = """
+        # This program prints out Hello, world!
+
+        print('Hello, world!')
+        """
+        self.impressive_code_reference = """
+        # An implementation of python Hello, world!
+
+        print("'Hello, world!")
+        """
+
+    @patch("janus.llm.models_info.load_model")
+    @patch("janus.metrics.llm_metrics.llm_evaluate")
+    @pytest.mark.llm_eval
+    def test_llm_self_eval_quality(self, mock_llm_evaluate, mock_load_model):
+        """Test that the quality llm self eval recognizes bad_code as bad code
+        (<5 on a scale of 1-10)"""
+        mock_llm_evaluate.return_value = 4  # return a value less than 5
+        mock_load_model.return_value = [None]  # return a dummy model
+
+        bad_code_quality = llm_evaluate_option(
+            self.bad_code,
+            self.bad_code,
+            metric="quality",
+            language="python",
+            llm=load_model("gpt-3.5-turbo-0125")[0],
+        )
+        self.assertLess(bad_code_quality, 5)
+
+        mock_llm_evaluate.return_value = 6  # return a value greater than 5
+        impressive_code_quality = llm_evaluate_option(
+            self.impressive_code,
+            self.impressive_code,
+            metric="quality",
+            language="python",
+            llm=load_model("gpt-3.5-turbo-0125")[0],
+        )
+        self.assertGreater(impressive_code_quality, 5)
+
+    @patch("janus.llm.models_info.load_model")
+    @patch("janus.metrics.llm_metrics.llm_evaluate")
+    @pytest.mark.llm_eval
+    def test_llm_self_eval_faithfulness(self, mock_llm_evaluate, mock_load_model):
+        """The two Hello, world! samples are more or less the same,
+        so the faithfulness score should be high"""
+        mock_llm_evaluate.return_value = 9  # return a high value
+        mock_load_model.return_value = [None]  # return a dummy model
+
+        faithfulness = llm_evaluate_ref_option(
+            self.impressive_code,
+            self.impressive_code_reference,
+            metric="faithfulness",
+            language="python",
+            llm=load_model("gpt-3.5-turbo-0125")[0],
+        )
+        self.assertGreater(faithfulness, 8)
+
+
+if __name__ == "__main__":
+    unittest.main()

janus/metrics/_tests/test_reading.py

@@ -0,0 +1,28 @@
+import unittest
+
+from ..reading import _repeat_text, flesch, gunning_fog
+
+
+class TestReading(unittest.TestCase):
+    def setUp(self):
+        self.text = "This is a sample text for testing readability metrics"
+
+    def test_repeat_text(self):
+        """Test the _repeat_text function."""
+        repeated_text = _repeat_text(self.text)
+        self.assertIsInstance(repeated_text, str)
+        self.assertTrue(len(repeated_text.split()) >= 100)
+
+    def test_flesch(self):
+        """Test the Flesch readability score."""
+        score = flesch(self.text)
+        self.assertAlmostEqual(score, 47.3, places=2)
+
+    def test_gunning_fog(self):
+        """Test the Gunning-Fog readability score."""
+        score = gunning_fog(self.text)
+        self.assertAlmostEqual(score, 8.04, places=2)
+
+
+if __name__ == "__main__":
+    unittest.main()

janus/metrics/_tests/test_rouge_score.py

@@ -0,0 +1,65 @@
+import unittest
+
+from janus.metrics.rouge_score import rouge
+
+
+class TestRouge(unittest.TestCase):
+    def setUp(self):
+        self.target = "This is a test sentence."
+        self.reference = "This is a reference sentence."
+
+    def test_rouge_with_granularity_n(self):
+        score = rouge(
+            self.target, self.reference, granularity="n", n_gram=2, score_type="f"
+        )
+        self.assertIsInstance(score, float)
+
+    def test_rouge_with_granularity_l(self):
+        score = rouge(
+            self.target, self.reference, granularity="l", n_gram=2, score_type="f"
+        )
+        self.assertIsInstance(score, float)
+
+    def test_rouge_with_granularity_w(self):
+        score = rouge(
+            self.target, self.reference, granularity="w", n_gram=2, score_type="f"
+        )
+        self.assertIsInstance(score, float)
+
+    def test_rouge_with_invalid_granularity(self):
+        with self.assertRaises(ValueError):
+            rouge(
+                self.target,
+                self.reference,
+                granularity="invalid",
+                n_gram=2,
+                score_type="f",
+            )
+
+    def test_rouge_with_score_type_f(self):
+        score = rouge(
+            self.target, self.reference, granularity="n", n_gram=2, score_type="f"
+        )
+        self.assertIsInstance(score, float)
+
+    def test_rouge_with_score_type_p(self):
+        score = rouge(
+            self.target, self.reference, granularity="n", n_gram=2, score_type="p"
+        )
+        self.assertIsInstance(score, float)
+
+    def test_rouge_with_score_type_r(self):
+        score = rouge(
+            self.target, self.reference, granularity="n", n_gram=2, score_type="r"
+        )
+        self.assertIsInstance(score, float)
+
+    def test_rouge_with_invalid_score_type(self):
+        with self.assertRaises(ValueError):
+            rouge(
+                self.target,
+                self.reference,
+                granularity="n",
+                n_gram=2,
+                score_type="invalid",
+            )

janus/metrics/_tests/test_similarity_score.py

@@ -0,0 +1,23 @@
+import unittest
+
+from janus.metrics.similarity import similarity_score
+
+
+class TestSimilarityScore(unittest.TestCase):
+    def setUp(self):
+        self.target = "This is a test sentence."
+        self.reference = "This is a reference sentence."
+
+    def test_similarity_score(self):
+        score = similarity_score(self.target, self.reference)
+        self.assertIsInstance(score, float)
+
+    def test_similarity_score_with_different_model(self):
+        score = similarity_score(
+            self.target, self.reference, model_name="text-embedding-ada-002"
+        )
+        self.assertIsInstance(score, float)
+
+    def test_similarity_score_with_different_distance(self):
+        score = similarity_score(self.target, self.reference, distance_metric="euclidean")
+        self.assertIsInstance(score, float)

janus/metrics/_tests/test_treesitter_metrics.py

@@ -0,0 +1,110 @@
+import unittest
+from pathlib import Path
+
+from typer.testing import CliRunner
+
+from ...cli import app
+from ..complexity_metrics import (
+    TreeSitterMetric,
+    cyclomatic_complexity,
+    difficulty,
+    effort,
+    maintainability,
+    volume,
+)
+
+
+class TestTreesitterMetrics(unittest.TestCase):
+    def setUp(self):
+        self.runner = CliRunner()
+        asm_file = Path(__file__).parent.resolve() / "asm_test_file.asm"
+        self.asm_target_text = asm_file.read_text()
+        mumps_file = Path(__file__).parent.resolve() / "mumps_test_file.m"
+        self.mumps_target_text = mumps_file.read_text()
+
+    def test_cyclomatic_complexity(self):
+        """Test the cyclomatic complexity function."""
+        function_score = cyclomatic_complexity(self.asm_target_text, language="ibmhlasm")
+        expected_score = 3
+        self.assertEqual(function_score, expected_score)
+        function_score = cyclomatic_complexity(self.mumps_target_text, language="mumps")
+        expected_score = 2
+        self.assertEqual(function_score, expected_score)
+
+    def test_length(self):
+        """Test the get_program_vocabulary function."""
+        tsm_asm = TreeSitterMetric(code=self.asm_target_text, language="ibmhlasm")
+        function_score = tsm_asm.get_program_length()
+        expected_score = 18
+        self.assertEqual(function_score, expected_score)
+        tsm_mumps = TreeSitterMetric(code=self.mumps_target_text, language="mumps")
+        function_score = tsm_mumps.get_program_length()
+        expected_score = 11
+        self.assertEqual(function_score, expected_score)
+
+    def test_vocabulary(self):
+        """Test the get_program_vocabulary function."""
+        tsm_asm = TreeSitterMetric(code=self.asm_target_text, language="ibmhlasm")
+        function_score = tsm_asm.get_program_vocabulary()
+        expected_score = 9
+        self.assertEqual(function_score, expected_score)
+        tsm_mumps = TreeSitterMetric(code=self.mumps_target_text, language="mumps")
+        function_score = tsm_mumps.get_program_vocabulary()
+        expected_score = 7
+        self.assertEqual(function_score, expected_score)
+
+    def test_difficulty(self):
+        """Test the get_program_vocabulary function."""
+        function_score = difficulty(self.asm_target_text, language="ibmhlasm")
+        expected_score = 5
+        self.assertEqual(function_score, expected_score)
+        function_score = difficulty(self.mumps_target_text, language="mumps")
+        expected_score = 2.625
+        self.assertAlmostEqual(function_score, expected_score, places=2)
+
+    def test_effort(self):
+        """Test the halstead effort."""
+        function_score = effort(self.asm_target_text, language="ibmhlasm")
+        self.assertAlmostEqual(function_score, 285.29, places=2)
+        function_score = effort(self.mumps_target_text, language="mumps")
+        self.assertAlmostEqual(function_score, 81.06, places=2)
+
+    def test_volume(self):
+        """Test the halstead volume."""
+        function_score = volume(self.asm_target_text, language="ibmhlasm")
+        self.assertAlmostEqual(function_score, 57.06, places=2)
+        function_score = volume(self.mumps_target_text, language="mumps")
+        self.assertAlmostEqual(function_score, 30.88, places=2)
+
+    def test_maintainability(self):
+        """Test the halstead volume."""
+        function_score = maintainability(self.asm_target_text, language="ibmhlasm")
+        self.assertAlmostEqual(function_score, 65.48, places=2)
+        function_score = maintainability(self.mumps_target_text, language="mumps")
+        self.assertAlmostEqual(function_score, 72.326, places=2)
+
+    def test_in_cli(self):
+        """Test the function in the CLI."""
+        output_path = Path("test.json")
+        if output_path.exists():
+            output_path.unlink()
+        result = self.runner.invoke(
+            app,
+            [
+                "evaluate",
+                "cyclomatic-complexity",
+                "-l",
+                "ibmhlasm",
+                "-t",
+                "janus/language/treesitter/_tests/languages/ibmhlasm.asm",
+                "-o",
+                f"{output_path}",
+            ],
+        )
+        self.assertEqual(result.exit_code, 0)
+        self.assertTrue(output_path.exists())
+        output_path.unlink()
+
+
+if __name__ == "__main__":
+    unittest.main()

janus/metrics/bleu.py

@@ -0,0 +1,66 @@
+from typing import Annotated, Optional
+
+import click
+import typer
+from sacrebleu import sentence_bleu
+
+from .metric import metric
+
+# from sacrebleu import sentence_chrf
+
+
+@metric(help="BLEU score using sacrebleu")
+def bleu(
+    target: str,
+    reference: str,
+    smooth_method: Annotated[
+        str,
+        typer.Option(
+            click_type=click.Choice(["exp", "floor", "add-k", "none"]),
+            help="Smoothing method to use.",
+        ),
+    ] = "exp",
+    smooth_value: Annotated[
+        Optional[float],
+        typer.Option(
+            help="Smoothing value (only for 'floor' and 'add-k').",
+        ),
+    ] = None,
+    lowercase: Annotated[
+        bool,
+        typer.Option(
+            help="Whether to lowercase the data.",
+        ),
+    ] = False,
+    use_effective_order: Annotated[
+        bool,
+        typer.Option(
+            help="Whether to use n-gram orders without matches.",
+        ),
+    ] = True,
+    **kwargs,
+) -> float:
+    """Computes BLEU score using sacrebleu
+
+    Arguments:
+        target: The target text.
+        reference: The reference text.
+        smooth_method: smoothing method to use.
+        smooth_value: smoothing value (only for floor and add-k).
+        lowercase: whether to lowercase the data.
+        use_effective_order: Don't use n-gram orders without matches.
+
+    Returns:
+        The BLEU score float.
+    """
+    score = sentence_bleu(
+        target,
+        [reference],
+        smooth_method=smooth_method,
+        smooth_value=smooth_value,
+        lowercase=lowercase,
+        use_effective_order=use_effective_order,
+    )
+    # Dividing by 100 to get the score in the range [0, 1]
+    # sacrebleu gives the score in percentage
+    return float(score.score) / 100.0

janus/metrics/chrf.py

@@ -0,0 +1,55 @@
+import typer
+from sacrebleu import sentence_chrf
+
+from .metric import metric
+
+
+@metric(help="chrF score using sacrebleu")
+def chrf(
+    target: str,
+    reference: str,
+    n_char_order: int = typer.Option(
+        default=6,
+        help=(
+            "A character n-gram order. If n_char_order=6, the metrics refers to the "
+            "official chrF/chrF++."
+        ),
+    ),
+    n_word_order: int = typer.Option(
+        default=2,
+        help=(
+            "A word n-gram order. If n_word_order=2, the metric refers to the official "
+            "chrF++. If n_word_order=0, the metric is equivalent to the original ChrF."
+        ),
+    ),
+    beta: float = typer.Option(
+        default=2.0,
+        help=(
+            "Determines importance of recall w.r.t. precision. If beta=1, their "
+            "importance is equal."
+        ),
+    ),
+    **kwargs,
+) -> float:
+    """Calculate the chrF Score using Torchmetrics.
+
+    Arguments:
+        target: The target text.
+        reference: The reference text.
+        n_char_order: The character order.
+        n_word_order: The word order.
+        beta: The beta value.
+
+    Returns:
+        The chrF score.
+    """
+    score = sentence_chrf(
+        target,
+        [reference],
+        char_order=n_char_order,
+        word_order=n_word_order,
+        beta=beta,
+    )
+    # Dividing by 100 to get the score in the range [0, 1]
+    # sacrebleu gives the score in percentage
+    return float(score.score) / 100.0

janus/metrics/cli.py

@@ -0,0 +1,7 @@
+import typer
+
+evaluate = typer.Typer(
+    help="Evaluation of generated source code or documentation",
+    add_completion=False,
+    no_args_is_help=True,
+)

janus/metrics/complexity_metrics.py

@@ -0,0 +1,208 @@
+import math
+from typing import List, Optional
+
+from janus.language.block import CodeBlock
+from janus.language.treesitter.treesitter import TreeSitterSplitter
+from janus.utils.enums import LANGUAGES
+
+from .metric import metric
+
+
+class NodeException(Exception):
+    pass
+
+
+class TreeSitterMetric:
+    """A class for calculating node-based complexity metrics of code."""
+
+    def __init__(
+        self,
+        code: str,
+        language: Optional[str],
+    ):
+        """
+        Arguments:
+            code: The code to get metrics on
+            language: The language the code is written in
+        """
+        if language is None:
+            raise ValueError("Error: must provide language for tree-sitter metrics")
+        self.branch_nodes: List[str] = LANGUAGES[language].get("branch_node_types")
+        self.operation_nodes: List[str] = LANGUAGES[language].get("operation_node_types")
+        self.operand_nodes: List[str] = LANGUAGES[language].get("operand_node_types")
+        self.code = code
+        self.language = language
+        self.splitter = TreeSitterSplitter(
+            language=language,
+        )
+        self.ast = self.splitter._get_ast(code)
+
+    def get_cyclomatic_complexity(self) -> int:
+        if not self.branch_nodes:
+            raise NodeException(f"No branch nodes are set for {self.language}")
+        else:
+            return self._count_nodes_of_type(self.ast, self.branch_nodes) + 1
+
+    def get_lines_of_code(self) -> int:
+        return self.code.count("\n")
+
+    """
+    The following metrics are based on Halstead complexity measures:
+    https://en.wikipedia.org/wiki/Halstead_complexity_measures
+    """
+
+    def get_program_vocabulary(self) -> int:
+        if not self.operation_nodes:
+            raise NodeException(f"No operation nodes are set for {self.language}")
+        else:
+            return self._count_nodes_of_type(
+                self.ast, self.operand_nodes, distinct=True
+            ) + self._count_nodes_of_type(self.ast, self.operation_nodes, distinct=True)
+
+    def get_program_length(self) -> int:
+        if not self.operation_nodes:
+            raise NodeException(f"No operation nodes are set for {self.language}")
+        else:
+            return self._count_nodes_of_type(
+                self.ast, self.operation_nodes
+            ) + self._count_nodes_of_type(self.ast, self.operand_nodes)
+
+    def get_volume(self) -> float:
+        vocabulary = self.get_program_vocabulary()
+        if not vocabulary:
+            raise ValueError(
+                "Volume cannot be calculated because program vocabulary is 0. \
+                Confirm that your code is parsing properly."
+            )
+        return self.get_program_length() * math.log2(vocabulary)
+
+    def get_difficulty(self) -> float:
+        return (
+            self._count_nodes_of_type(self.ast, self.operation_nodes, distinct=True)
+            / 2
+            * self._count_nodes_of_type(self.ast, self.operand_nodes, distinct=False)
+            / self._count_nodes_of_type(self.ast, self.operand_nodes, distinct=True)
+        )
+
+    def get_effort(self) -> float:
+        return self.get_volume() * self.get_difficulty()
+
+    def get_time_to_program(self) -> float:
+        return self.get_effort() / 18
+
+    def get_num_bugs(self) -> float:
+        return self.get_effort() ** (2 / 3) / 3000
+
+    def get_maintainability(self) -> float:
+        volume = self.get_volume()
+        cyclomatic_complexity = self.get_cyclomatic_complexity()
+        lines_of_code = self.get_lines_of_code()
+        if not (volume and lines_of_code):
+            raise ValueError(
+                "Maintainability cannot be calculated because volume or lines of code\
+                is 0. Confirm that your code is parsing properly."
+            )
+        return max(
+            0,
+            (
+                171
+                - (5.2 * math.log(volume))
+                - (0.23 * cyclomatic_complexity)
+                - (16.2 * math.log(lines_of_code))
+            )
+            * 100
+            / 171,
+        )
+
+    def _count_nodes_of_type(
+        self, code_block: CodeBlock, nodes: List[str], distinct=False
+    ) -> int:
+        """Recurse through all nodes of a CodeBlock,
+        take count of the number of nodes of a specified type"""
+        seen_nodes = set()
+        count = 0
+        nodes_left = [code_block]
+        while nodes_left:
+            node = nodes_left.pop()
+            if str(node.node_type) in nodes:
+                if distinct:
+                    seen_nodes.add(node.text.strip())
+                else:
+                    count += 1
+            nodes_left.extend(node.children)
+        return len(seen_nodes) if distinct else count
+
+
+@metric(use_reference=False, help="Cyclomatic complexity score")
+def cyclomatic_complexity(target: str, **kwargs) -> float:
+    """Calculate the cyclomatic complexity score.
+
+    Arguments:
+        target: The target text.
+
+    Returns:
+        The cyclomatic complexity.
+    """
+    language = kwargs["language"]
+    score = TreeSitterMetric(target, language).get_cyclomatic_complexity()
+    return score
+
+
+@metric(use_reference=False, help="Halstead effort score")
+def effort(target: str, **kwargs) -> float:
+    """Calculate the Halstead effort.
+
+    Arguments:
+        target: The target text.
+
+    Returns:
+        The Halstead effort.
+    """
+    language = kwargs["language"]
+    score = TreeSitterMetric(target, language).get_effort()
+    return score
+
+
+@metric(use_reference=False, help="Halstead volume score")
+def volume(target: str, **kwargs) -> float:
+    """Calculate the Halstead volume.
+
+    Arguments:
+        target: The target text.
+
+    Returns:
+        The Halstead volume.
+    """
+    language = kwargs["language"]
+    score = TreeSitterMetric(target, language).get_volume()
+    return score
+
+
+@metric(use_reference=False, help="Halstead difficulty score")
+def difficulty(target: str, **kwargs) -> float:
+    """Calculate the Halstead difficulty.
+
+    Arguments:
+        target: The target text.
+
+    Returns:
+        The Halstead difficulty.
+    """
+    language = kwargs["language"]
+    score = TreeSitterMetric(target, language).get_difficulty()
+    return score
+
+
+@metric(use_reference=False, help="Maintainability score")
+def maintainability(target: str, **kwargs) -> float:
+    """Calculate the maintainability score.
+
+    Arguments:
+        target: The target text.
+
+    Returns:
+        The maintainability score.
+    """
+    language = kwargs["language"]
+    score = TreeSitterMetric(target, language).get_maintainability()
+    return score