janus-llm 3.4.2__py3-none-any.whl → 3.4.3__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- janus/__init__.py +1 -1
- janus/converter/converter.py +19 -9
- janus/converter/diagram.py +5 -5
- janus/llm/models_info.py +3 -1
- janus/parsers/_tests/test_code_parser.py +3 -2
- janus/parsers/code_parser.py +4 -36
- janus/parsers/doc_parser.py +22 -17
- janus/parsers/eval_parser.py +11 -8
- janus/parsers/parser.py +51 -0
- janus/parsers/refiner_parser.py +5 -10
- janus/parsers/reqs_parser.py +11 -12
- janus/parsers/uml.py +3 -2
- {janus_llm-3.4.2.dist-info → janus_llm-3.4.3.dist-info}/METADATA +1 -1
- {janus_llm-3.4.2.dist-info → janus_llm-3.4.3.dist-info}/RECORD +17 -16
- {janus_llm-3.4.2.dist-info → janus_llm-3.4.3.dist-info}/LICENSE +0 -0
- {janus_llm-3.4.2.dist-info → janus_llm-3.4.3.dist-info}/WHEEL +0 -0
- {janus_llm-3.4.2.dist-info → janus_llm-3.4.3.dist-info}/entry_points.txt +0 -0
janus/__init__.py
CHANGED
@@ -5,7 +5,7 @@ from langchain_core._api.deprecation import LangChainDeprecationWarning
|
|
5
5
|
from janus.converter.translate import Translator
|
6
6
|
from janus.metrics import * # noqa: F403
|
7
7
|
|
8
|
-
__version__ = "3.4.
|
8
|
+
__version__ = "3.4.3"
|
9
9
|
|
10
10
|
# Ignoring a deprecation warning from langchain_core that I can't seem to hunt down
|
11
11
|
warnings.filterwarnings("ignore", category=LangChainDeprecationWarning)
|
janus/converter/converter.py
CHANGED
@@ -7,7 +7,6 @@ from typing import Any, List, Optional, Tuple
|
|
7
7
|
from langchain.output_parsers import RetryWithErrorOutputParser
|
8
8
|
from langchain_core.exceptions import OutputParserException
|
9
9
|
from langchain_core.language_models import BaseLanguageModel
|
10
|
-
from langchain_core.output_parsers import BaseOutputParser
|
11
10
|
from langchain_core.prompts import ChatPromptTemplate, SystemMessagePromptTemplate
|
12
11
|
from langchain_core.runnables import RunnableLambda, RunnableParallel
|
13
12
|
from openai import BadRequestError, RateLimitError
|
@@ -26,7 +25,7 @@ from janus.language.splitter import (
|
|
26
25
|
from janus.llm import load_model
|
27
26
|
from janus.llm.model_callbacks import get_model_callback
|
28
27
|
from janus.llm.models_info import MODEL_PROMPT_ENGINES
|
29
|
-
from janus.parsers.
|
28
|
+
from janus.parsers.parser import GenericParser, JanusParser
|
30
29
|
from janus.parsers.refiner_parser import RefinerParser
|
31
30
|
from janus.refiners.refiner import BasicRefiner, Refiner
|
32
31
|
from janus.utils.enums import LANGUAGES
|
@@ -129,7 +128,7 @@ class Converter:
|
|
129
128
|
self._llm: BaseLanguageModel
|
130
129
|
self._prompt: ChatPromptTemplate
|
131
130
|
|
132
|
-
self._parser:
|
131
|
+
self._parser: JanusParser = GenericParser()
|
133
132
|
self._combiner: Combiner = Combiner()
|
134
133
|
|
135
134
|
self._refiner_type: str
|
@@ -328,6 +327,7 @@ class Converter:
|
|
328
327
|
"_prompt_template_name",
|
329
328
|
"_source_language",
|
330
329
|
"_model_name",
|
330
|
+
"_parser",
|
331
331
|
)
|
332
332
|
def _load_prompt(self) -> None:
|
333
333
|
"""Load the prompt according to this instance's attributes.
|
@@ -340,6 +340,9 @@ class Converter:
|
|
340
340
|
prompt_template=self._prompt_template_name,
|
341
341
|
)
|
342
342
|
self._prompt = prompt_engine.prompt
|
343
|
+
self._prompt = self._prompt.partial(
|
344
|
+
format_instructions=self._parser.get_format_instructions()
|
345
|
+
)
|
343
346
|
|
344
347
|
@run_if_changed("_db_path", "_db_config")
|
345
348
|
def _load_vectorizer(self) -> None:
|
@@ -445,6 +448,15 @@ class Converter:
|
|
445
448
|
except FileSizeError:
|
446
449
|
log.warning("Current tile is too large for basic splitter, skipping")
|
447
450
|
continue
|
451
|
+
except ValueError as e:
|
452
|
+
if str(e).startswith(
|
453
|
+
"Error raised by bedrock service"
|
454
|
+
) and "maximum context length" in str(e):
|
455
|
+
log.warning(
|
456
|
+
"Input is too large for this model's context length, skipping"
|
457
|
+
)
|
458
|
+
continue
|
459
|
+
raise e
|
448
460
|
|
449
461
|
# Don't attempt to write files for which translation failed
|
450
462
|
if not out_block.translated:
|
@@ -599,7 +611,7 @@ class Converter:
|
|
599
611
|
to the cube root of self.max_retries, so the total calls to the
|
600
612
|
LLM will be roughly as expected (up to sqrt(self.max_retries) over)
|
601
613
|
"""
|
602
|
-
self._parser.
|
614
|
+
input = self._parser.parse_input(block.original)
|
603
615
|
|
604
616
|
# Retries with just the output and the error
|
605
617
|
n1 = round(self.max_prompts ** (1 / 2))
|
@@ -607,15 +619,12 @@ class Converter:
|
|
607
619
|
# Retries with the input, output, and error
|
608
620
|
n2 = round(self.max_prompts // n1)
|
609
621
|
|
610
|
-
# Retries with just the input
|
611
622
|
if not self.skip_context:
|
612
623
|
self._make_prompt_additions(block)
|
613
624
|
if not self.skip_refiner: # Make replacements in the prompt
|
614
625
|
refine_output = RefinerParser(
|
615
626
|
parser=self._parser,
|
616
|
-
initial_prompt=self._prompt.format(
|
617
|
-
**{"SOURCE_CODE": block.original.text}
|
618
|
-
),
|
627
|
+
initial_prompt=self._prompt.format(**{"SOURCE_CODE": input}),
|
619
628
|
refiner=self._refiner,
|
620
629
|
max_retries=n1,
|
621
630
|
llm=self._llm,
|
@@ -626,13 +635,14 @@ class Converter:
|
|
626
635
|
parser=self._parser,
|
627
636
|
max_retries=n1,
|
628
637
|
)
|
638
|
+
|
629
639
|
completion_chain = self._prompt | self._llm
|
630
640
|
chain = RunnableParallel(
|
631
641
|
completion=completion_chain, prompt_value=self._prompt
|
632
642
|
) | RunnableLambda(lambda x: refine_output.parse_with_prompt(**x))
|
633
643
|
for _ in range(n2):
|
634
644
|
try:
|
635
|
-
return chain.invoke({"SOURCE_CODE":
|
645
|
+
return chain.invoke({"SOURCE_CODE": input})
|
636
646
|
except OutputParserException:
|
637
647
|
pass
|
638
648
|
|
janus/converter/diagram.py
CHANGED
@@ -52,7 +52,7 @@ class DiagramGenerator(Documenter):
|
|
52
52
|
self._load_diagram_prompt_engine()
|
53
53
|
|
54
54
|
def _run_chain(self, block: TranslatedCodeBlock) -> str:
|
55
|
-
self._parser.
|
55
|
+
input = self._parser.parse_input(block.original)
|
56
56
|
n1 = round(self.max_prompts ** (1 / 3))
|
57
57
|
|
58
58
|
# Retries with the input, output, and error
|
@@ -67,7 +67,7 @@ class DiagramGenerator(Documenter):
|
|
67
67
|
parser=self._diagram_parser,
|
68
68
|
initial_prompt=self._diagram_prompt.format(
|
69
69
|
**{
|
70
|
-
"SOURCE_CODE":
|
70
|
+
"SOURCE_CODE": input,
|
71
71
|
"DOCUMENTATION": documentation_text,
|
72
72
|
"DIAGRAM_TYPE": self._diagram_type,
|
73
73
|
}
|
@@ -81,7 +81,7 @@ class DiagramGenerator(Documenter):
|
|
81
81
|
parser=self._diagram_parser,
|
82
82
|
initial_prompt=self._diagram_prompt.format(
|
83
83
|
**{
|
84
|
-
"SOURCE_CODE":
|
84
|
+
"SOURCE_CODE": input,
|
85
85
|
"DIAGRAM_TYPE": self._diagram_type,
|
86
86
|
}
|
87
87
|
),
|
@@ -103,7 +103,7 @@ class DiagramGenerator(Documenter):
|
|
103
103
|
if self._add_documentation:
|
104
104
|
return chain.invoke(
|
105
105
|
{
|
106
|
-
"SOURCE_CODE":
|
106
|
+
"SOURCE_CODE": input,
|
107
107
|
"DOCUMENTATION": documentation_text,
|
108
108
|
"DIAGRAM_TYPE": self._diagram_type,
|
109
109
|
}
|
@@ -111,7 +111,7 @@ class DiagramGenerator(Documenter):
|
|
111
111
|
else:
|
112
112
|
return chain.invoke(
|
113
113
|
{
|
114
|
-
"SOURCE_CODE":
|
114
|
+
"SOURCE_CODE": input,
|
115
115
|
"DIAGRAM_TYPE": self._diagram_type,
|
116
116
|
}
|
117
117
|
)
|
janus/llm/models_info.py
CHANGED
@@ -210,7 +210,9 @@ def get_available_model_names() -> list[str]:
|
|
210
210
|
return avaialable_models
|
211
211
|
|
212
212
|
|
213
|
-
def load_model(
|
213
|
+
def load_model(
|
214
|
+
user_model_name: str,
|
215
|
+
) -> tuple[BaseLanguageModel, str, int, dict[str, float]]:
|
214
216
|
if not MODEL_CONFIG_DIR.exists():
|
215
217
|
MODEL_CONFIG_DIR.mkdir(parents=True)
|
216
218
|
model_config_file = MODEL_CONFIG_DIR / f"{user_model_name}.json"
|
@@ -1,11 +1,12 @@
|
|
1
1
|
import unittest
|
2
2
|
|
3
|
-
from janus.parsers.code_parser import CodeParser
|
3
|
+
from janus.parsers.code_parser import CodeParser
|
4
|
+
from janus.parsers.parser import GenericParser
|
4
5
|
|
5
6
|
|
6
7
|
class TestJanusParser(unittest.TestCase):
|
7
8
|
def setUp(self):
|
8
|
-
self.parser =
|
9
|
+
self.parser = GenericParser()
|
9
10
|
|
10
11
|
def test_parse_combined_output(self):
|
11
12
|
text = "test text"
|
janus/parsers/code_parser.py
CHANGED
@@ -1,52 +1,20 @@
|
|
1
1
|
import re
|
2
2
|
|
3
|
-
from langchain.schema.output_parser import BaseOutputParser
|
4
3
|
from langchain_core.exceptions import OutputParserException
|
5
4
|
from langchain_core.messages import BaseMessage
|
6
|
-
from langchain_core.output_parsers import StrOutputParser
|
7
5
|
|
8
|
-
from janus.
|
6
|
+
from janus.parsers.parser import JanusParser
|
9
7
|
from janus.utils.logger import create_logger
|
10
8
|
|
11
9
|
log = create_logger(__name__)
|
12
10
|
|
13
11
|
|
14
|
-
class JanusParser:
|
15
|
-
def parse_combined_output(self, text: str) -> str:
|
16
|
-
"""Parse the output text from the LLM when multiple inputs are combined
|
17
|
-
|
18
|
-
Arguments:
|
19
|
-
text: The output text from the LLM
|
20
|
-
|
21
|
-
Returns:
|
22
|
-
A parsed version of the text
|
23
|
-
"""
|
24
|
-
if isinstance(text, BaseMessage):
|
25
|
-
text = text.content
|
26
|
-
return text
|
27
|
-
|
28
|
-
def parse_into_block(self, text: str, block: CodeBlock):
|
29
|
-
if isinstance(text, BaseMessage):
|
30
|
-
text = text.content
|
31
|
-
block.text = text
|
32
|
-
|
33
|
-
def set_reference(self, block: CodeBlock):
|
34
|
-
pass
|
35
|
-
|
36
|
-
|
37
|
-
class GenericParser(StrOutputParser, JanusParser):
|
38
|
-
def parse(self, text: str) -> str:
|
39
|
-
if isinstance(text, BaseMessage):
|
40
|
-
return text.content
|
41
|
-
return text
|
42
|
-
|
43
|
-
|
44
|
-
class CodeParser(BaseOutputParser[str], JanusParser):
|
12
|
+
class CodeParser(JanusParser):
|
45
13
|
language: str
|
46
14
|
|
47
|
-
def parse(self, text: str) -> str:
|
15
|
+
def parse(self, text: str | BaseMessage) -> str:
|
48
16
|
if isinstance(text, BaseMessage):
|
49
|
-
text = text.content
|
17
|
+
text = str(text.content)
|
50
18
|
pattern = rf"```[^\S\r\n]*(?:{self.language}[^\S\r\n]*)?\n?(.*?)\n*```"
|
51
19
|
code = re.search(pattern, text, re.DOTALL)
|
52
20
|
if code is None:
|
janus/parsers/doc_parser.py
CHANGED
@@ -3,13 +3,12 @@ import re
|
|
3
3
|
|
4
4
|
from langchain.output_parsers import PydanticOutputParser
|
5
5
|
from langchain.output_parsers.json import parse_json_markdown
|
6
|
-
from langchain.schema.output_parser import BaseOutputParser
|
7
6
|
from langchain_core.exceptions import OutputParserException
|
8
|
-
from langchain_core.messages import
|
7
|
+
from langchain_core.messages import BaseMessage
|
9
8
|
from langchain_core.pydantic_v1 import BaseModel, Field
|
10
9
|
|
11
10
|
from janus.language.block import CodeBlock
|
12
|
-
from janus.parsers.
|
11
|
+
from janus.parsers.parser import JanusParser
|
13
12
|
from janus.utils.logger import create_logger
|
14
13
|
|
15
14
|
log = create_logger(__name__)
|
@@ -32,18 +31,21 @@ class MultiDoc(BaseModel):
|
|
32
31
|
)
|
33
32
|
|
34
33
|
|
35
|
-
class MultiDocumentationParser(
|
34
|
+
class MultiDocumentationParser(JanusParser, PydanticOutputParser):
|
36
35
|
block_name: str = ""
|
37
36
|
|
38
37
|
def __init__(self):
|
39
38
|
PydanticOutputParser.__init__(self, pydantic_object=MultiDoc)
|
40
39
|
|
41
|
-
def
|
42
|
-
|
40
|
+
def parse_input(self, block: CodeBlock) -> str:
|
41
|
+
text = super().parse_input(block)
|
42
|
+
self.block_name = str(block.name)
|
43
|
+
return text
|
44
|
+
|
45
|
+
def parse(self, text: str | BaseMessage) -> str:
|
46
|
+
if isinstance(text, BaseMessage):
|
47
|
+
text = str(text.content)
|
43
48
|
|
44
|
-
def parse(self, text: str) -> str:
|
45
|
-
if isinstance(text, AIMessage):
|
46
|
-
text = text.content
|
47
49
|
try:
|
48
50
|
docs = json.loads(super().parse(text).json())
|
49
51
|
except (OutputParserException, json.JSONDecodeError):
|
@@ -81,22 +83,25 @@ class MultiDocumentationParser(PydanticOutputParser, JanusParser):
|
|
81
83
|
|
82
84
|
@property
|
83
85
|
def _type(self) -> str:
|
84
|
-
return self.__class__.name
|
86
|
+
return str(self.__class__.name)
|
85
87
|
|
86
88
|
|
87
|
-
class MadlibsDocumentationParser(
|
89
|
+
class MadlibsDocumentationParser(JanusParser):
|
88
90
|
expected_keys: set[str]
|
89
91
|
|
90
92
|
def __init__(self):
|
91
93
|
super().__init__(expected_keys=[])
|
92
94
|
|
93
|
-
def
|
94
|
-
|
95
|
+
def parse_input(self, block: CodeBlock):
|
96
|
+
# TODO: Perform comment stripping/placeholding here rather than in script
|
97
|
+
text = super().parse_input(block)
|
98
|
+
comment_ids = re.findall(r"<(?:BLOCK|INLINE)_COMMENT (\w{8})>", text)
|
95
99
|
self.expected_keys = set(comment_ids)
|
96
100
|
|
97
|
-
def parse(self, text: str) -> str:
|
98
|
-
if isinstance(text,
|
99
|
-
text = text.content
|
101
|
+
def parse(self, text: str | BaseMessage) -> str:
|
102
|
+
if isinstance(text, BaseMessage):
|
103
|
+
text = str(text.content)
|
104
|
+
|
100
105
|
try:
|
101
106
|
obj = parse_json_markdown(text)
|
102
107
|
except json.JSONDecodeError as e:
|
@@ -166,4 +171,4 @@ class MadlibsDocumentationParser(BaseOutputParser[str], JanusParser):
|
|
166
171
|
|
167
172
|
@property
|
168
173
|
def _type(self) -> str:
|
169
|
-
return self.__class__.name
|
174
|
+
return str(self.__class__.name)
|
janus/parsers/eval_parser.py
CHANGED
@@ -1,9 +1,10 @@
|
|
1
1
|
import json
|
2
2
|
|
3
3
|
from langchain.output_parsers import PydanticOutputParser
|
4
|
+
from langchain_core.messages import BaseMessage
|
4
5
|
from langchain_core.pydantic_v1 import BaseModel, Field, validator
|
5
6
|
|
6
|
-
from janus.parsers.
|
7
|
+
from janus.parsers.parser import JanusParser
|
7
8
|
from janus.utils.logger import create_logger
|
8
9
|
|
9
10
|
log = create_logger(__name__)
|
@@ -28,7 +29,7 @@ class Eval(BaseModel):
|
|
28
29
|
|
29
30
|
return v
|
30
31
|
|
31
|
-
def __add__(self, other):
|
32
|
+
def __add__(self, other: "Eval"):
|
32
33
|
if isinstance(other, int) and other == 0:
|
33
34
|
return self.copy()
|
34
35
|
return Eval.construct(
|
@@ -38,10 +39,10 @@ class Eval(BaseModel):
|
|
38
39
|
completeness=self.completeness + other.completeness,
|
39
40
|
)
|
40
41
|
|
41
|
-
def __radd__(self, other):
|
42
|
+
def __radd__(self, other) -> "Eval":
|
42
43
|
return self.__add__(other)
|
43
44
|
|
44
|
-
def __truediv__(self, other):
|
45
|
+
def __truediv__(self, other) -> "Eval":
|
45
46
|
if isinstance(other, int):
|
46
47
|
return Eval.construct(
|
47
48
|
syntax=self.syntax / other,
|
@@ -57,11 +58,13 @@ class Eval(BaseModel):
|
|
57
58
|
)
|
58
59
|
|
59
60
|
|
60
|
-
class EvaluationParser(
|
61
|
+
class EvaluationParser(JanusParser, PydanticOutputParser):
|
61
62
|
def __init__(self):
|
62
63
|
PydanticOutputParser.__init__(self, pydantic_object=Eval)
|
63
64
|
|
64
|
-
def parse(self, text: str) -> str:
|
65
|
+
def parse(self, text: str | BaseMessage) -> str:
|
66
|
+
if isinstance(text, BaseMessage):
|
67
|
+
text = str(text.content)
|
65
68
|
eval = super().parse(text)
|
66
69
|
return json.dumps(eval.json())
|
67
70
|
|
@@ -75,6 +78,6 @@ class EvaluationParser(PydanticOutputParser, JanusParser):
|
|
75
78
|
Returns:
|
76
79
|
A parsed version of the text.
|
77
80
|
"""
|
78
|
-
objs = [super().parse(line.strip()) for line in text.split("\n")]
|
79
|
-
avg_obj = sum(objs) / len(objs)
|
81
|
+
objs: list[Eval] = [super().parse(line.strip()) for line in text.split("\n")]
|
82
|
+
avg_obj: Eval = sum(objs) / len(objs)
|
80
83
|
return json.dumps(avg_obj.json())
|
janus/parsers/parser.py
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
from langchain.schema.output_parser import BaseOutputParser
|
2
|
+
from langchain_core.messages import BaseMessage
|
3
|
+
from langchain_core.output_parsers import StrOutputParser
|
4
|
+
|
5
|
+
from janus.language.block import CodeBlock
|
6
|
+
from janus.language.splitter import EmptyTreeError
|
7
|
+
from janus.utils.logger import create_logger
|
8
|
+
|
9
|
+
log = create_logger(__name__)
|
10
|
+
|
11
|
+
|
12
|
+
class JanusParser(BaseOutputParser[str]):
|
13
|
+
def parse_input(self, block: CodeBlock) -> str:
|
14
|
+
"""Parse the input block into raw string input ready to be passed to
|
15
|
+
an LLM. Also perform any processing or saving of metadata.
|
16
|
+
|
17
|
+
Arguments:
|
18
|
+
block: The CodeBlock to be processed
|
19
|
+
|
20
|
+
Returns:
|
21
|
+
A parsed version of the input text
|
22
|
+
"""
|
23
|
+
if block.text is None:
|
24
|
+
raise EmptyTreeError("No text in input CodeBlock!")
|
25
|
+
return block.text
|
26
|
+
|
27
|
+
def parse_combined_output(self, text: str) -> str:
|
28
|
+
"""Parse the output text from the LLM when multiple inputs are combined
|
29
|
+
|
30
|
+
Arguments:
|
31
|
+
text: The output text from the LLM
|
32
|
+
|
33
|
+
Returns:
|
34
|
+
A parsed version of the text
|
35
|
+
"""
|
36
|
+
return text
|
37
|
+
|
38
|
+
def parse_into_block(self, text: str | BaseMessage, block: CodeBlock):
|
39
|
+
if isinstance(text, BaseMessage):
|
40
|
+
text = str(text.content)
|
41
|
+
block.text = text
|
42
|
+
|
43
|
+
|
44
|
+
class GenericParser(JanusParser, StrOutputParser):
|
45
|
+
def parse(self, text: str | BaseMessage) -> str:
|
46
|
+
if isinstance(text, BaseMessage):
|
47
|
+
text = str(text.content)
|
48
|
+
return text
|
49
|
+
|
50
|
+
def get_format_instructions(self) -> str:
|
51
|
+
return "Output should be a string"
|
janus/parsers/refiner_parser.py
CHANGED
@@ -1,11 +1,13 @@
|
|
1
1
|
from langchain_core.exceptions import OutputParserException
|
2
2
|
from langchain_core.language_models import BaseLanguageModel
|
3
|
+
from langchain_core.messages import BaseMessage
|
3
4
|
from langchain_core.output_parsers import BaseOutputParser
|
4
5
|
|
6
|
+
from janus.parsers.parser import JanusParser
|
5
7
|
from janus.refiners.refiner import Refiner
|
6
8
|
|
7
9
|
|
8
|
-
class RefinerParser(
|
10
|
+
class RefinerParser(JanusParser):
|
9
11
|
"""Parser for performing refinement with a refiner
|
10
12
|
|
11
13
|
Properties:
|
@@ -25,15 +27,7 @@ class RefinerParser(BaseOutputParser):
|
|
25
27
|
refiner: Refiner
|
26
28
|
max_retries: int
|
27
29
|
|
28
|
-
def parse(self, text: str) -> str:
|
29
|
-
"""Parses the text using the refiner
|
30
|
-
|
31
|
-
Arguments:
|
32
|
-
text: text to parse
|
33
|
-
|
34
|
-
Returns:
|
35
|
-
Parsed text
|
36
|
-
"""
|
30
|
+
def parse(self, text: str | BaseMessage) -> str:
|
37
31
|
last_prompt = self.initial_prompt
|
38
32
|
for _ in range(self.max_retries):
|
39
33
|
try:
|
@@ -46,6 +40,7 @@ class RefinerParser(BaseOutputParser):
|
|
46
40
|
new_chain = new_prompt | self.llm
|
47
41
|
text = new_chain.invoke(prompt_arguments)
|
48
42
|
last_prompt = new_prompt.format(**prompt_arguments)
|
43
|
+
|
49
44
|
raise OutputParserException(
|
50
45
|
f"Error: unable to correct output after {self.max_retries} attempts"
|
51
46
|
)
|
janus/parsers/reqs_parser.py
CHANGED
@@ -2,29 +2,28 @@ import json
|
|
2
2
|
import re
|
3
3
|
|
4
4
|
from langchain.output_parsers.json import parse_json_markdown
|
5
|
-
from langchain.schema.output_parser import BaseOutputParser
|
6
5
|
from langchain_core.exceptions import OutputParserException
|
7
|
-
from langchain_core.messages import
|
6
|
+
from langchain_core.messages import BaseMessage
|
8
7
|
|
9
|
-
from janus.
|
10
|
-
from janus.parsers.code_parser import JanusParser
|
8
|
+
from janus.parsers.parser import JanusParser
|
11
9
|
from janus.utils.logger import create_logger
|
12
10
|
|
13
11
|
log = create_logger(__name__)
|
14
12
|
|
15
13
|
|
16
|
-
class RequirementsParser(
|
17
|
-
|
14
|
+
class RequirementsParser(JanusParser):
|
15
|
+
expected_keys: set[str]
|
18
16
|
|
19
17
|
def __init__(self):
|
20
18
|
super().__init__(expected_keys=[])
|
21
19
|
|
22
|
-
def
|
23
|
-
|
20
|
+
def parse(self, text: str | BaseMessage) -> str:
|
21
|
+
if isinstance(text, BaseMessage):
|
22
|
+
text = str(text.content)
|
24
23
|
|
25
|
-
|
26
|
-
|
27
|
-
|
24
|
+
# TODO: This is an incorrect implementation (lstrip and rstrip take character
|
25
|
+
# lists and strip any instances of those characters, not the full str)
|
26
|
+
# Should be replaced with a regex search, see CodeParser
|
28
27
|
text = text.lstrip("```json")
|
29
28
|
text = text.rstrip("```")
|
30
29
|
try:
|
@@ -70,4 +69,4 @@ class RequirementsParser(BaseOutputParser[str], JanusParser):
|
|
70
69
|
|
71
70
|
@property
|
72
71
|
def _type(self) -> str:
|
73
|
-
return self.__class__.name
|
72
|
+
return str(self.__class__.name)
|
janus/parsers/uml.py
CHANGED
@@ -4,6 +4,7 @@ from pathlib import Path
|
|
4
4
|
from typing import List, Tuple
|
5
5
|
|
6
6
|
from langchain_core.exceptions import OutputParserException
|
7
|
+
from langchain_core.messages import BaseMessage
|
7
8
|
|
8
9
|
from janus.parsers.code_parser import CodeParser
|
9
10
|
from janus.utils.logger import create_logger
|
@@ -12,7 +13,7 @@ log = create_logger(__name__)
|
|
12
13
|
|
13
14
|
|
14
15
|
class UMLSyntaxParser(CodeParser):
|
15
|
-
def _get_uml_output(self, file:
|
16
|
+
def _get_uml_output(self, file: Path) -> Tuple[str, str]:
|
16
17
|
# NOTE: running subprocess with shell=False, added nosec to label that we know
|
17
18
|
# risk exists
|
18
19
|
try:
|
@@ -33,7 +34,7 @@ class UMLSyntaxParser(CodeParser):
|
|
33
34
|
def _get_errs(self, s: str) -> List[str]:
|
34
35
|
return [x.group() for x in re.finditer(r"Error (.*)\n", s)]
|
35
36
|
|
36
|
-
def parse(self, text: str) -> str:
|
37
|
+
def parse(self, text: str | BaseMessage) -> str:
|
37
38
|
text = super().parse(text)
|
38
39
|
janus_path = Path.home().expanduser() / Path(".janus")
|
39
40
|
if not janus_path.exists():
|
@@ -1,4 +1,4 @@
|
|
1
|
-
janus/__init__.py,sha256=
|
1
|
+
janus/__init__.py,sha256=HX3kYLkqXTi8C9sXb0P1mGki4DMJJzxHSQSxoilv7sA,361
|
2
2
|
janus/__main__.py,sha256=lEkpNtLVPtFo8ySDZeXJ_NXDHb0GVdZFPWB4gD4RPS8,64
|
3
3
|
janus/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
4
4
|
janus/_tests/conftest.py,sha256=V7uW-oq3YbFiRPvrq15YoVVrA1n_83pjgiyTZ-IUGW8,963
|
@@ -7,8 +7,8 @@ janus/cli.py,sha256=_92FvDV4qza0nSmyiXqacYxyo1gY6IPwD4gCm6kZfqI,33213
|
|
7
7
|
janus/converter/__init__.py,sha256=U2EOMcCykiC0ZqhorNefOP_04hOF18qhYoPKrVp1Vrk,345
|
8
8
|
janus/converter/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
9
9
|
janus/converter/_tests/test_translate.py,sha256=yzcFEGc_z8QmBBBmC9dZnfL9tT8CD1rkpc8Hz44Jp4c,5631
|
10
|
-
janus/converter/converter.py,sha256=
|
11
|
-
janus/converter/diagram.py,sha256=
|
10
|
+
janus/converter/converter.py,sha256=GL4moCOi5vvvsEjAf_7Vrn5GHq12nptH47AAP4soeQc,27956
|
11
|
+
janus/converter/diagram.py,sha256=AyxkoyfMoQ7jpuOqyJ3bIvYqHJSEhmR-gYrIsq-d8tk,5285
|
12
12
|
janus/converter/document.py,sha256=hsW512veNjFWbdl5WriuUdNmMEqZy8ktRvqn9rRmA6E,4566
|
13
13
|
janus/converter/evaluate.py,sha256=APWQUY3gjAXqkJkPzvj0UA4wPK3Cv9QSJLM-YK9t-ng,476
|
14
14
|
janus/converter/requirements.py,sha256=9tvQ40FZJtG8niIFn45gPQCgKKHVPPoFLinBv6RAqO4,2027
|
@@ -57,7 +57,7 @@ janus/language/treesitter/_tests/test_treesitter.py,sha256=4S_UdH6AfJ0j6hyInZ2CF
|
|
57
57
|
janus/language/treesitter/treesitter.py,sha256=FU86H8j2cfRLtwyNeEnf9A6gzZEvDwmnFCIrQymUJso,7541
|
58
58
|
janus/llm/__init__.py,sha256=TKLYvnsWKWfxMucy-lCLQ-4bkN9ENotJZDywDEQmrKg,45
|
59
59
|
janus/llm/model_callbacks.py,sha256=K7P5NY-rf7IYRAFHnZ3kzhrQWE6g_najx8uxlaSiz3E,7110
|
60
|
-
janus/llm/models_info.py,sha256=
|
60
|
+
janus/llm/models_info.py,sha256=26lY_w-LTiPvSFRlKdpqEQjVnsmd7ns7XBUl0wuxe00,8370
|
61
61
|
janus/metrics/__init__.py,sha256=AsxtZJUzZiXJPr2ehPPltuYP-ddechjg6X85WZUO7mA,241
|
62
62
|
janus/metrics/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
63
63
|
janus/metrics/_tests/reference.py,sha256=hiaJPP9CXkvFBV_wL-gOe_BzELTw0nvB6uCxhxtIiE8,13
|
@@ -83,13 +83,14 @@ janus/metrics/similarity.py,sha256=NKd-m8hMB27i1_1CQkG5o7HrAK2wTr89B21Y36kqHiE,1
|
|
83
83
|
janus/metrics/splitting.py,sha256=610ScHRvALwdkqA6YyGI-tr3a18_cUofldBxGYX0SwE,968
|
84
84
|
janus/parsers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
85
85
|
janus/parsers/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
86
|
-
janus/parsers/_tests/test_code_parser.py,sha256=
|
87
|
-
janus/parsers/code_parser.py,sha256=
|
88
|
-
janus/parsers/doc_parser.py,sha256=
|
89
|
-
janus/parsers/eval_parser.py,sha256=
|
90
|
-
janus/parsers/
|
91
|
-
janus/parsers/
|
92
|
-
janus/parsers/
|
86
|
+
janus/parsers/_tests/test_code_parser.py,sha256=3ay5QpUPcynX_EJ-YLl3PR28poutUkT7qvyWUSQ7Too,928
|
87
|
+
janus/parsers/code_parser.py,sha256=3l0HfzgrvJuiwk779s9ZsgUl3xbp1nE1qZxh8aDYRBI,873
|
88
|
+
janus/parsers/doc_parser.py,sha256=G7kDdJ3davHhEFXqwNYw7hOB26hKH_d_0XdpyyBHq_U,5835
|
89
|
+
janus/parsers/eval_parser.py,sha256=Gjh6aTZgpYd2ASJUEPMo4LpCL00cBmbOqc4KM3hy8x8,2922
|
90
|
+
janus/parsers/parser.py,sha256=y6VV64bgVidf-oEFla3I--_28tnJsPBc6QUD_SkbfSE,1614
|
91
|
+
janus/parsers/refiner_parser.py,sha256=W7ecDX7j6N-uWtRBYFD1EiEzDpDQws16nrVFzyArzf8,1632
|
92
|
+
janus/parsers/reqs_parser.py,sha256=uRQC41Iqp22GjIvakb5UKv70UWHkcOTbOVl_RDnipYw,2438
|
93
|
+
janus/parsers/uml.py,sha256=T9MzYbXz_gjlfgJlIMGSzgACcXLhoN3RnV9Y-5vYJ9A,1888
|
93
94
|
janus/prompts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
94
95
|
janus/prompts/prompt.py,sha256=3796YXIzzIec9b0iUzd8VZlq-AdQbzq8qUGXLy4KH-0,10586
|
95
96
|
janus/refiners/refiner.py,sha256=GkV4oUSCrLAhyDJY2aY_Jt8PRF3sC6-bv58YbL2PaNk,2227
|
@@ -100,8 +101,8 @@ janus/utils/_tests/test_progress.py,sha256=Rs_u5PiGjP-L-o6C1fhwfE1ig8jYu9Xo9s4p8
|
|
100
101
|
janus/utils/enums.py,sha256=AoilbdiYyMvY2Mp0AM4xlbLSELfut2XMwhIM1S_msP4,27610
|
101
102
|
janus/utils/logger.py,sha256=KZeuaMAnlSZCsj4yL0P6N-JzZwpxXygzACWfdZFeuek,2337
|
102
103
|
janus/utils/progress.py,sha256=PIpcQec7SrhsfqB25LHj2CDDkfm9umZx90d9LZnAx6k,1469
|
103
|
-
janus_llm-3.4.
|
104
|
-
janus_llm-3.4.
|
105
|
-
janus_llm-3.4.
|
106
|
-
janus_llm-3.4.
|
107
|
-
janus_llm-3.4.
|
104
|
+
janus_llm-3.4.3.dist-info/LICENSE,sha256=_j0st0a-HB6MRbP3_BW3PUqpS16v54luyy-1zVyl8NU,10789
|
105
|
+
janus_llm-3.4.3.dist-info/METADATA,sha256=rFGRSJIXyld0d04oeG6s7iC9HfQlyuN6I7b1UaCXXWo,4184
|
106
|
+
janus_llm-3.4.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
107
|
+
janus_llm-3.4.3.dist-info/entry_points.txt,sha256=OGhQwzj6pvXp79B0SaBD5apGekCu7Dwe9fZZT_TZ544,39
|
108
|
+
janus_llm-3.4.3.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|