janus-llm 2.1.0__py3-none-any.whl → 3.0.1__py3-none-any.whl

janus/metrics/reading.py

@@ -1,9 +1,30 @@
+import re
+
 import nltk
 import readability
+from nltk.tokenize import TweetTokenizer
 
 from .metric import metric
 
 
+def word_count(text):
+    """Calculates word count exactly how readability package does
+
+    Arguments:
+        text: The input string.
+
+    Returns:
+        Word Count
+    """
+    tokenizer = TweetTokenizer()
+    word_count = 0
+    tokens = tokenizer.tokenize(text)
+    for t in tokens:
+        if not re.match(r"^[.,\/#!$%'\^&\*;:{}=\-_`~()]+$", t):
+            word_count += 1
+    return word_count
+
+
 def _repeat_text(text):
     """Repeats a string until its length is over 100 words.
 
@@ -20,11 +41,10 @@ def _repeat_text(text):
     if not text.endswith("."):
         text += "."  # Add a period if missing
 
-    # Check if repeated text is long enough, repeat more if needed
     repeated_text = text
-    while len(repeated_text.split()) < 100:
-        repeated_text += " " + text
 
+    while word_count(repeated_text) < 100:
+        repeated_text += " " + text
     return repeated_text
 
 
@@ -52,7 +72,8 @@ def flesch(target: str, **kwargs) -> float:
     Returns:
         The Flesch score.
     """
-
+    if not target.strip():  # Check if the target text is blank
+        return None
     return get_readability(target).flesch().score
 
 
@@ -66,5 +87,6 @@ def gunning_fog(target: str, **kwargs) -> float:
     Returns:
         The Gunning-Fog score.
     """
-
+    if not target.strip():  # Check if the target text is blank
+        return None
     return get_readability(target).gunning_fog().score

janus/prompts/prompt.py

@@ -2,12 +2,12 @@ import json
 from abc import ABC, abstractmethod
 from pathlib import Path
 
-from langchain import PromptTemplate
 from langchain.prompts import ChatPromptTemplate
 from langchain.prompts.chat import (
     HumanMessagePromptTemplate,
     SystemMessagePromptTemplate,
 )
+from langchain_core.prompts import PromptTemplate
 
 from ..utils.enums import LANGUAGES
 from ..utils.logger import create_logger
@@ -40,7 +40,7 @@ retry_with_output_prompt_text = """Instructions:
 --------------
 Completion:
 --------------
-{input}
+{completion}
 --------------
 
 Above, the Completion did not satisfy the constraints given in the Instructions.
@@ -54,13 +54,23 @@ constraints laid out in the Instructions:"""
 
 
 retry_with_error_and_output_prompt_text = """Prompt:
+--------------
 {prompt}
+--------------
 Completion:
-{input}
+--------------
+{completion}
+--------------
 
 Above, the Completion did not satisfy the constraints given in the Prompt.
-Details: {error}
-Please try again:"""
+Error:
+--------------
+{error}
+--------------
+
+Please try again. Please only respond with an answer that satisfies the
+constraints laid out in the Prompt:"""
+
 
 retry_with_output_prompt = PromptTemplate.from_template(retry_with_output_prompt_text)
 retry_with_error_and_output_prompt = PromptTemplate.from_template(
@@ -74,9 +84,9 @@ class PromptEngine(ABC):
     def __init__(
         self,
         source_language: str,
-        target_language: str,
-        target_version: str,
         prompt_template: str,
+        target_language: str | None = None,
+        target_version: str | None = None,
     ) -> None:
         """Initialize a PromptEngine instance.
 
@@ -97,15 +107,18 @@ class PromptEngine(ABC):
 
         # Define variables to be passed in to the prompt formatter
         source_language = source_language.lower()
-        target_language = target_language.lower()
         self.variables = dict(
             SOURCE_LANGUAGE=source_language,
-            TARGET_LANGUAGE=target_language,
-            TARGET_LANGUAGE_VERSION=str(target_version),
             FILE_SUFFIX=LANGUAGES[source_language]["suffix"],
             SOURCE_CODE_EXAMPLE=LANGUAGES[source_language]["example"],
-            TARGET_CODE_EXAMPLE=LANGUAGES[target_language]["example"],
         )
+        if target_language is not None:
+            target_language = target_language.lower()
+            self.variables.update(
+                TARGET_LANGUAGE=target_language,
+                TARGET_CODE_EXAMPLE=LANGUAGES[target_language]["example"],
+            )
+            self.variables.update(TARGET_LANGUAGE_VERSION=str(target_version))
         variables_path = template_path / PROMPT_VARIABLES_FILENAME
         if variables_path.exists():
             self.variables.update(json.loads(variables_path.read_text()))
@@ -253,3 +266,16 @@ class CoherePromptEngine(PromptEngine):
             f"{human_prompt}"
             f"<|END_OF_TURN_TOKEN|>"
         )
+
+
+class MistralPromptEngine(PromptEngine):
+    def load_prompt_template(self, template_path: Path) -> ChatPromptTemplate:
+        system_prompt_path = template_path / SYSTEM_PROMPT_TEMPLATE_FILENAME
+        system_prompt = system_prompt_path.read_text()
+
+        human_prompt_path = template_path / HUMAN_PROMPT_TEMPLATE_FILENAME
+        human_prompt = human_prompt_path.read_text()
+
+        return PromptTemplate.from_template(
+            f"<s>[INST] {system_prompt} [/INST] </s>[INST] {human_prompt}  [/INST]"
+        )

{janus_llm-2.1.0.dist-info → janus_llm-3.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: janus-llm
-Version: 2.1.0
+Version: 3.0.1
 Summary: A transcoding library using LLMs.
 Home-page: https://github.com/janus-llm/janus-llm
 License: Apache 2.0

{janus_llm-2.1.0.dist-info → janus_llm-3.0.1.dist-info}/RECORD

@@ -1,11 +1,18 @@
-janus/__init__.py,sha256=BDely1z7CW0_cUz6BOjQbE3ZTFvMNAfp2WD5ZkVDF4U,341
-janus/__main__.py,sha256=Qd-f8z2Q2vpiEP2x6PBFsJrpACWDVxFKQk820MhFmHo,59
+janus/__init__.py,sha256=OkY9msOgEgU_jai5YdGjfv6hQG6UopI7S2J_M9EKs-U,351
+janus/__main__.py,sha256=lEkpNtLVPtFo8ySDZeXJ_NXDHb0GVdZFPWB4gD4RPS8,64
 janus/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/_tests/conftest.py,sha256=V7uW-oq3YbFiRPvrq15YoVVrA1n_83pjgiyTZ-IUGW8,963
-janus/_tests/test_cli.py,sha256=oP-WOM-ai4jZLDGqjLzI6kCtfXWpoeUR8TynP5p6cVg,4254
-janus/_tests/test_translate.py,sha256=71oRTTfdSVFOBvUhuOavgl3TuimTz1K6eG-04pUGpfE,16439
-janus/cli.py,sha256=ASM9OtdQ2S-KZa-23prrympPbMzKBjSNIclv_w81MSs,29543
-janus/converter.py,sha256=Kxd8eq03XNHbZZI-fU8NUNJR6N2yDxPHTc6QENC8Bqo,6092
+janus/_tests/test_cli.py,sha256=mi7wAWV07ZFli5nQdExRGIGA3AMFD9s39-HcmDV4B6Y,4232
+janus/cli.py,sha256=-aeg8R6CobK2EG_BPoZgBy_x1d6G9gp-KKKhnLMepo4,29541
+janus/converter/__init__.py,sha256=kzVmWOPXRDayqqBZ8ZDaFQzA_q8PEdv407dc-DefPxY,255
+janus/converter/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+janus/converter/_tests/test_translate.py,sha256=eiLbmouokZrAeAYmdoJgnlx5-k4QiO6i0N6e6ZvZsvM,15885
+janus/converter/converter.py,sha256=Bq07_9N_3Dv9NBqVACvb7LC2HxdQmfVZ1b0BlWrxjgo,23521
+janus/converter/diagram.py,sha256=v-3ZZ4t1q74lDOjF2N6NRPkC3IK-sjLDn5_VChZTEGA,4608
+janus/converter/document.py,sha256=hsW512veNjFWbdl5WriuUdNmMEqZy8ktRvqn9rRmA6E,4566
+janus/converter/evaluate.py,sha256=APWQUY3gjAXqkJkPzvj0UA4wPK3Cv9QSJLM-YK9t-ng,476
+janus/converter/requirements.py,sha256=6YvrJRVH9BuPCOPxnXmaJQFYmoLYYvCu3zTntDLHeNg,1832
+janus/converter/translate.py,sha256=kMlGUiBYGQBXSxwX5in3CUyUifPM95wynCaRMxSDxMw,4238
 janus/embedding/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/embedding/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/embedding/_tests/test_collections.py,sha256=eT0cYv-qmPrHJRjDZqWPFTkqVzFDRoPrRKR__FPiz58,2651
@@ -28,8 +35,8 @@ janus/language/binary/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
 janus/language/binary/_tests/test_binary.py,sha256=a-8RSfKA23UrJC9c1xPQK792XZCz8npCHI7isN2dAP8,1727
 janus/language/binary/binary.py,sha256=CS1RAieN8klSsCeXQEFYKUWioatUX-sOPXKQr5S6NzE,6534
 janus/language/binary/reveng/decompile_script.py,sha256=veW51oJzuO-4UD3Er062jXZ_FYtTFo9OCkl82Z2xr6A,2182
-janus/language/block.py,sha256=DVbnnthFWLQ2JjvS09kFqNmeCx7SGtYkBnFjtPdW8mM,9278
-janus/language/combine.py,sha256=hSEc1dHLcOELks-ZGsRKHOgYBXAronKus6BTmb1u42k,2940
+janus/language/block.py,sha256=57hfOY-KSVMioKhkCvfDtovQt4h8lCg9cJbRF7ddV1s,9280
+janus/language/combine.py,sha256=e7j8zQO_D3_LElaVCsGgtnzia7aFFK56m-mhArQBlR0,2908
 janus/language/file.py,sha256=X2MYcAMlCABK77uhMdI_J2foXLrqEdinapYRfLPyKB8,563
 janus/language/mumps/__init__.py,sha256=-Ou_wJ-JgHezfp1dub2_qCYNiK9wO-zo2MlqxM9qiwE,48
 janus/language/mumps/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -47,10 +54,10 @@ janus/language/splitter.py,sha256=4XAe0hXka7njS30UHGCngJzDgHxn3lygUjikSHuV7Xo,16
 janus/language/treesitter/__init__.py,sha256=mUliw7ZJLZ8NkJKyUQMSoUV82hYXE0HvLHrEdGPJF4Q,43
 janus/language/treesitter/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/language/treesitter/_tests/test_treesitter.py,sha256=nsavUV0aI6cpT9FkQve58eTTehLyQG6qJJBGlNa_bIw,2170
-janus/language/treesitter/treesitter.py,sha256=bWimG5yNxweb-W6xGnl4Bbpsd9lJn9WGuYZla_lWdxQ,6863
+janus/language/treesitter/treesitter.py,sha256=UiV4OuWTt6IwMohHSw4FHsVNA_zxr9lNk4_Du09APdo,7509
 janus/llm/__init__.py,sha256=8Pzn3Jdx867PzDc4xmwm8wvJDGzWSIhpN0NCEYFe0LQ,36
-janus/llm/model_callbacks.py,sha256=cQb0Gmy__Mh27b1UdTZkdCN3747bEi3DN_Wxb6gnI7Q,6762
-janus/llm/models_info.py,sha256=_roSyd9oQX6QMshkyW880qgWt2pEm5Jby-U4XhkMjSg,7137
+janus/llm/model_callbacks.py,sha256=h_xlBAHRx-gxQBBjVKRpGXxdxYf6d9L6kBoXjbEAEdI,7106
+janus/llm/models_info.py,sha256=B9Dn5mHc43OeZe5mHFj5wuhO194XHCTwShAa2ybnPyY,7688
 janus/metrics/__init__.py,sha256=AsxtZJUzZiXJPr2ehPPltuYP-ddechjg6X85WZUO7mA,241
 janus/metrics/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/metrics/_tests/reference.py,sha256=hiaJPP9CXkvFBV_wL-gOe_BzELTw0nvB6uCxhxtIiE8,13
@@ -70,7 +77,7 @@ janus/metrics/complexity_metrics.py,sha256=1Z9n0o_CrILqayk40wRkjR1f7yvHIsJG38DxA
 janus/metrics/file_pairing.py,sha256=WNHRV1D8GOJMq8Pla5SPkTDAT7yVaS4-UU0XIGKvEVs,3729
 janus/metrics/llm_metrics.py,sha256=3677S6GYcoVcokpmAN-fwvNu-lYWAKd7M5mebiE6RZc,5687
 janus/metrics/metric.py,sha256=Lgdtq87oJ-kWC_6jdPQ6-d1MqoeTnhkRszo6IZJV6c0,16974
-janus/metrics/reading.py,sha256=KYuWjKnk0ALRU5S7mCNNZtaNgK02l0fdIGsaNvxLMO4,1690
+janus/metrics/reading.py,sha256=srLb2MO-vZL5ccRjaHz-dA4MwAvXVNyIKnOrvJXg77E,2244
 janus/metrics/rouge_score.py,sha256=HfUJwUWI-yq5pOjML2ee4QTOMl0NQahnqEY2Mt8Dtnw,2865
 janus/metrics/similarity.py,sha256=9pjWWpLKCsk0QfFfSgQNdPXiisqi7WJYOOHaiT8S0iY,1613
 janus/metrics/splitting.py,sha256=610ScHRvALwdkqA6YyGI-tr3a18_cUofldBxGYX0SwE,968
@@ -82,8 +89,7 @@ janus/parsers/doc_parser.py,sha256=X8eCb1QXbL6sVWLEFGjsPyxrpJ9XnOPg7G4KZSo9A9E,5
 janus/parsers/eval_parser.py,sha256=HB5-zY_Jpmkj6FDbuNCCVCRxwmzhViSAjPKbyyC0Ebc,2723
 janus/parsers/reqs_parser.py,sha256=MFBvtR3otpyPZlkZxu0dVH1YeEJhvhNzhaGKGHaQVHA,2359
 janus/prompts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-janus/prompts/prompt.py,sha256=FPjqCtWla9BW4AdgrcuY8JXo5rYBSJ4b0PDDw6i1AKs,9701
-janus/translate.py,sha256=bIrvyFBXUH1Cf8M-h-qSybFe0NQwuCA38heiV2toP8w,38958
+janus/prompts/prompt.py,sha256=vd7UbitF0VFCi21RsggDebD51xcuyls_lQLGKkphfI8,10578
 janus/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/utils/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/utils/_tests/test_logger.py,sha256=4jZFm8LX828Dt9lOjiFHZIPbxYy_hHaswyrMPkscgdM,2199
@@ -91,8 +97,8 @@ janus/utils/_tests/test_progress.py,sha256=Yh5NDNq-24n2nhHHbJm39pENAH70PYnh9ymwd
 janus/utils/enums.py,sha256=AoilbdiYyMvY2Mp0AM4xlbLSELfut2XMwhIM1S_msP4,27610
 janus/utils/logger.py,sha256=KZeuaMAnlSZCsj4yL0P6N-JzZwpxXygzACWfdZFeuek,2337
 janus/utils/progress.py,sha256=pKcCzO9JOU9fSD7qTmLWcqY5smc8mujqQMXoPgqNysE,1458
-janus_llm-2.1.0.dist-info/LICENSE,sha256=_j0st0a-HB6MRbP3_BW3PUqpS16v54luyy-1zVyl8NU,10789
-janus_llm-2.1.0.dist-info/METADATA,sha256=a7PmWYKoHGa6ynFGQJvOVLMtBYkVa8foj6zpJgN1Sfs,4184
-janus_llm-2.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-janus_llm-2.1.0.dist-info/entry_points.txt,sha256=OGhQwzj6pvXp79B0SaBD5apGekCu7Dwe9fZZT_TZ544,39
-janus_llm-2.1.0.dist-info/RECORD,,
+janus_llm-3.0.1.dist-info/LICENSE,sha256=_j0st0a-HB6MRbP3_BW3PUqpS16v54luyy-1zVyl8NU,10789
+janus_llm-3.0.1.dist-info/METADATA,sha256=tVkX6eswouFez-ASfy-iWJk_iXptEeylGETEZF4MeeI,4184
+janus_llm-3.0.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+janus_llm-3.0.1.dist-info/entry_points.txt,sha256=OGhQwzj6pvXp79B0SaBD5apGekCu7Dwe9fZZT_TZ544,39
+janus_llm-3.0.1.dist-info/RECORD,,

janus/converter.py

@@ -1,161 +0,0 @@
-import functools
-from typing import Any
-
-from langchain.schema.language_model import BaseLanguageModel
-
-from .language.alc.alc import AlcSplitter
-from .language.binary import BinarySplitter
-from .language.mumps import MumpsSplitter
-from .language.splitter import Splitter
-from .language.treesitter import TreeSitterSplitter
-from .utils.enums import CUSTOM_SPLITTERS, LANGUAGES
-from .utils.logger import create_logger
-
-log = create_logger(__name__)
-
-
-def run_if_changed(*tracked_vars):
-    """Wrapper to skip function calls if the given instance attributes haven't
-    been updated. Requires the _changed_attrs set to exist, and the __setattr__
-    method to be overridden to track parameter updates in _changed_attrs.
-    """
-
-    def wrapper(func):
-        @functools.wraps(func)
-        def wrapped(self, *args, **kwargs):
-            # If there is overlap between the tracked variables and the changed
-            #  ones, then call the function as normal
-            if self._changed_attrs.intersection(tracked_vars):
-                func(self, *args, **kwargs)
-
-        return wrapped
-
-    return wrapper
-
-
-class Converter:
-    """Parent class that converts code into something else.
-
-    Children will determine what the code gets converted into. Whether that's translated
-    into another language, into pseudocode, requirements, documentation, etc., or
-    converted into embeddings
-    """
-
-    def __init__(
-        self,
-        source_language: str = "fortran",
-        max_tokens: None | int = None,
-        protected_node_types: set[str] | list[str] | tuple[str] = (),
-        prune_node_types: set[str] | list[str] | tuple[str] = (),
-    ) -> None:
-        """Initialize a Converter instance.
-
-        Arguments:
-            source_language: The source programming language.
-            parser_type: The type of parser to use for parsing the LLM output. Valid
-                values are `"code"`, `"text"`, `"eval"`, and `None` (default). If `None`,
-                the `Converter` assumes you won't be parsing an output (i.e., adding to an
-                embedding DB).
-        """
-        self._changed_attrs: set = set()
-
-        self._source_language: None | str
-        self._source_glob: None | str
-        self._protected_node_types: tuple[str] = ()
-        self._prune_node_types: tuple[str] = ()
-        self._splitter: None | Splitter
-        self._llm: None | BaseLanguageModel = None
-        self._max_tokens: None | int = max_tokens
-
-        self.set_source_language(source_language)
-        self.set_protected_node_types(protected_node_types)
-        self.set_prune_node_types(prune_node_types)
-
-        # Child class must call this. Should we enforce somehow?
-        # self._load_parameters()
-
-    def __setattr__(self, key: Any, value: Any) -> None:
-        if hasattr(self, "_changed_attrs"):
-            if not hasattr(self, key) or getattr(self, key) != value:
-                self._changed_attrs.add(key)
-        # Avoid infinite recursion
-        elif key != "_changed_attrs":
-            self._changed_attrs = set()
-        super().__setattr__(key, value)
-
-    def _load_parameters(self) -> None:
-        self._load_splitter()
-        self._changed_attrs.clear()
-
-    def set_source_language(self, source_language: str) -> None:
-        """Validate and set the source language.
-
-        The affected objects will not be updated until _load_parameters() is called.
-
-        Arguments:
-            source_language: The source programming language.
-        """
-        source_language = source_language.lower()
-        if source_language not in LANGUAGES:
-            raise ValueError(
-                f"Invalid source language: {source_language}. "
-                "Valid source languages are found in `janus.utils.enums.LANGUAGES`."
-            )
-
-        self._source_glob = f"**/*.{LANGUAGES[source_language]['suffix']}"
-        self._source_language = source_language
-
-    def set_protected_node_types(
-        self, protected_node_types: set[str] | list[str] | tuple[str]
-    ) -> None:
-        """Set the protected (non-mergeable) node types. This will often be structures
-        like functions, classes, or modules which you might want to keep separate
-
-        The affected objects will not be updated until _load_parameters() is called.
-
-        Arguments:
-            protected_node_types: A set of node types that aren't to be merged
-        """
-        self._protected_node_types = tuple(set(protected_node_types or []))
-
-    def set_prune_node_types(
-        self, prune_node_types: set[str] | list[str] | tuple[str]
-    ) -> None:
-        """Set the node types to prune. This will often be structures
-        like comments or whitespace which you might want to keep out of the LLM
-
-        The affected objects will not be updated until _load_parameters() is called.
-
-        Arguments:
-            prune_node_types: A set of node types which should be pruned
-        """
-        self._prune_node_types = tuple(set(prune_node_types or []))
-
-    @run_if_changed(
-        "_source_language",
-        "_max_tokens",
-        "_llm",
-        "_protected_node_types",
-        "_prune_node_types",
-    )
-    def _load_splitter(self) -> None:
-        """Load the splitter according to this instance's attributes.
-
-        If the relevant fields have not been changed since the last time this method was
-        called, nothing happens.
-        """
-        kwargs = dict(
-            max_tokens=self._max_tokens,
-            model=self._llm,
-            protected_node_types=self._protected_node_types,
-            prune_node_types=self._prune_node_types,
-        )
-        if self._source_language in CUSTOM_SPLITTERS:
-            if self._source_language == "mumps":
-                self._splitter = MumpsSplitter(**kwargs)
-            elif self._source_language == "ibmhlasm":
-                self._splitter = AlcSplitter(**kwargs)
-            elif self._source_language == "binary":
-                self._splitter = BinarySplitter(**kwargs)
-        else:
-            self._splitter = TreeSitterSplitter(language=self._source_language, **kwargs)