janus-llm 3.4.2__py3-none-any.whl → 3.4.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- janus/__init__.py +1 -1
- janus/converter/converter.py +19 -9
- janus/converter/diagram.py +5 -5
- janus/llm/models_info.py +3 -1
- janus/parsers/_tests/test_code_parser.py +3 -2
- janus/parsers/code_parser.py +4 -36
- janus/parsers/doc_parser.py +22 -17
- janus/parsers/eval_parser.py +11 -8
- janus/parsers/parser.py +51 -0
- janus/parsers/refiner_parser.py +5 -10
- janus/parsers/reqs_parser.py +11 -12
- janus/parsers/uml.py +3 -2
- {janus_llm-3.4.2.dist-info → janus_llm-3.4.3.dist-info}/METADATA +1 -1
- {janus_llm-3.4.2.dist-info → janus_llm-3.4.3.dist-info}/RECORD +17 -16
- {janus_llm-3.4.2.dist-info → janus_llm-3.4.3.dist-info}/LICENSE +0 -0
- {janus_llm-3.4.2.dist-info → janus_llm-3.4.3.dist-info}/WHEEL +0 -0
- {janus_llm-3.4.2.dist-info → janus_llm-3.4.3.dist-info}/entry_points.txt +0 -0
janus/__init__.py
CHANGED
@@ -5,7 +5,7 @@ from langchain_core._api.deprecation import LangChainDeprecationWarning
|
|
5
5
|
from janus.converter.translate import Translator
|
6
6
|
from janus.metrics import * # noqa: F403
|
7
7
|
|
8
|
-
__version__ = "3.4.
|
8
|
+
__version__ = "3.4.3"
|
9
9
|
|
10
10
|
# Ignoring a deprecation warning from langchain_core that I can't seem to hunt down
|
11
11
|
warnings.filterwarnings("ignore", category=LangChainDeprecationWarning)
|
janus/converter/converter.py
CHANGED
@@ -7,7 +7,6 @@ from typing import Any, List, Optional, Tuple
|
|
7
7
|
from langchain.output_parsers import RetryWithErrorOutputParser
|
8
8
|
from langchain_core.exceptions import OutputParserException
|
9
9
|
from langchain_core.language_models import BaseLanguageModel
|
10
|
-
from langchain_core.output_parsers import BaseOutputParser
|
11
10
|
from langchain_core.prompts import ChatPromptTemplate, SystemMessagePromptTemplate
|
12
11
|
from langchain_core.runnables import RunnableLambda, RunnableParallel
|
13
12
|
from openai import BadRequestError, RateLimitError
|
@@ -26,7 +25,7 @@ from janus.language.splitter import (
|
|
26
25
|
from janus.llm import load_model
|
27
26
|
from janus.llm.model_callbacks import get_model_callback
|
28
27
|
from janus.llm.models_info import MODEL_PROMPT_ENGINES
|
29
|
-
from janus.parsers.
|
28
|
+
from janus.parsers.parser import GenericParser, JanusParser
|
30
29
|
from janus.parsers.refiner_parser import RefinerParser
|
31
30
|
from janus.refiners.refiner import BasicRefiner, Refiner
|
32
31
|
from janus.utils.enums import LANGUAGES
|
@@ -129,7 +128,7 @@ class Converter:
|
|
129
128
|
self._llm: BaseLanguageModel
|
130
129
|
self._prompt: ChatPromptTemplate
|
131
130
|
|
132
|
-
self._parser:
|
131
|
+
self._parser: JanusParser = GenericParser()
|
133
132
|
self._combiner: Combiner = Combiner()
|
134
133
|
|
135
134
|
self._refiner_type: str
|
@@ -328,6 +327,7 @@ class Converter:
|
|
328
327
|
"_prompt_template_name",
|
329
328
|
"_source_language",
|
330
329
|
"_model_name",
|
330
|
+
"_parser",
|
331
331
|
)
|
332
332
|
def _load_prompt(self) -> None:
|
333
333
|
"""Load the prompt according to this instance's attributes.
|
@@ -340,6 +340,9 @@ class Converter:
|
|
340
340
|
prompt_template=self._prompt_template_name,
|
341
341
|
)
|
342
342
|
self._prompt = prompt_engine.prompt
|
343
|
+
self._prompt = self._prompt.partial(
|
344
|
+
format_instructions=self._parser.get_format_instructions()
|
345
|
+
)
|
343
346
|
|
344
347
|
@run_if_changed("_db_path", "_db_config")
|
345
348
|
def _load_vectorizer(self) -> None:
|
@@ -445,6 +448,15 @@ class Converter:
|
|
445
448
|
except FileSizeError:
|
446
449
|
log.warning("Current tile is too large for basic splitter, skipping")
|
447
450
|
continue
|
451
|
+
except ValueError as e:
|
452
|
+
if str(e).startswith(
|
453
|
+
"Error raised by bedrock service"
|
454
|
+
) and "maximum context length" in str(e):
|
455
|
+
log.warning(
|
456
|
+
"Input is too large for this model's context length, skipping"
|
457
|
+
)
|
458
|
+
continue
|
459
|
+
raise e
|
448
460
|
|
449
461
|
# Don't attempt to write files for which translation failed
|
450
462
|
if not out_block.translated:
|
@@ -599,7 +611,7 @@ class Converter:
|
|
599
611
|
to the cube root of self.max_retries, so the total calls to the
|
600
612
|
LLM will be roughly as expected (up to sqrt(self.max_retries) over)
|
601
613
|
"""
|
602
|
-
self._parser.
|
614
|
+
input = self._parser.parse_input(block.original)
|
603
615
|
|
604
616
|
# Retries with just the output and the error
|
605
617
|
n1 = round(self.max_prompts ** (1 / 2))
|
@@ -607,15 +619,12 @@ class Converter:
|
|
607
619
|
# Retries with the input, output, and error
|
608
620
|
n2 = round(self.max_prompts // n1)
|
609
621
|
|
610
|
-
# Retries with just the input
|
611
622
|
if not self.skip_context:
|
612
623
|
self._make_prompt_additions(block)
|
613
624
|
if not self.skip_refiner: # Make replacements in the prompt
|
614
625
|
refine_output = RefinerParser(
|
615
626
|
parser=self._parser,
|
616
|
-
initial_prompt=self._prompt.format(
|
617
|
-
**{"SOURCE_CODE": block.original.text}
|
618
|
-
),
|
627
|
+
initial_prompt=self._prompt.format(**{"SOURCE_CODE": input}),
|
619
628
|
refiner=self._refiner,
|
620
629
|
max_retries=n1,
|
621
630
|
llm=self._llm,
|
@@ -626,13 +635,14 @@ class Converter:
|
|
626
635
|
parser=self._parser,
|
627
636
|
max_retries=n1,
|
628
637
|
)
|
638
|
+
|
629
639
|
completion_chain = self._prompt | self._llm
|
630
640
|
chain = RunnableParallel(
|
631
641
|
completion=completion_chain, prompt_value=self._prompt
|
632
642
|
) | RunnableLambda(lambda x: refine_output.parse_with_prompt(**x))
|
633
643
|
for _ in range(n2):
|
634
644
|
try:
|
635
|
-
return chain.invoke({"SOURCE_CODE":
|
645
|
+
return chain.invoke({"SOURCE_CODE": input})
|
636
646
|
except OutputParserException:
|
637
647
|
pass
|
638
648
|
|
janus/converter/diagram.py
CHANGED
@@ -52,7 +52,7 @@ class DiagramGenerator(Documenter):
|
|
52
52
|
self._load_diagram_prompt_engine()
|
53
53
|
|
54
54
|
def _run_chain(self, block: TranslatedCodeBlock) -> str:
|
55
|
-
self._parser.
|
55
|
+
input = self._parser.parse_input(block.original)
|
56
56
|
n1 = round(self.max_prompts ** (1 / 3))
|
57
57
|
|
58
58
|
# Retries with the input, output, and error
|
@@ -67,7 +67,7 @@ class DiagramGenerator(Documenter):
|
|
67
67
|
parser=self._diagram_parser,
|
68
68
|
initial_prompt=self._diagram_prompt.format(
|
69
69
|
**{
|
70
|
-
"SOURCE_CODE":
|
70
|
+
"SOURCE_CODE": input,
|
71
71
|
"DOCUMENTATION": documentation_text,
|
72
72
|
"DIAGRAM_TYPE": self._diagram_type,
|
73
73
|
}
|
@@ -81,7 +81,7 @@ class DiagramGenerator(Documenter):
|
|
81
81
|
parser=self._diagram_parser,
|
82
82
|
initial_prompt=self._diagram_prompt.format(
|
83
83
|
**{
|
84
|
-
"SOURCE_CODE":
|
84
|
+
"SOURCE_CODE": input,
|
85
85
|
"DIAGRAM_TYPE": self._diagram_type,
|
86
86
|
}
|
87
87
|
),
|
@@ -103,7 +103,7 @@ class DiagramGenerator(Documenter):
|
|
103
103
|
if self._add_documentation:
|
104
104
|
return chain.invoke(
|
105
105
|
{
|
106
|
-
"SOURCE_CODE":
|
106
|
+
"SOURCE_CODE": input,
|
107
107
|
"DOCUMENTATION": documentation_text,
|
108
108
|
"DIAGRAM_TYPE": self._diagram_type,
|
109
109
|
}
|
@@ -111,7 +111,7 @@ class DiagramGenerator(Documenter):
|
|
111
111
|
else:
|
112
112
|
return chain.invoke(
|
113
113
|
{
|
114
|
-
"SOURCE_CODE":
|
114
|
+
"SOURCE_CODE": input,
|
115
115
|
"DIAGRAM_TYPE": self._diagram_type,
|
116
116
|
}
|
117
117
|
)
|
janus/llm/models_info.py
CHANGED
@@ -210,7 +210,9 @@ def get_available_model_names() -> list[str]:
|
|
210
210
|
return avaialable_models
|
211
211
|
|
212
212
|
|
213
|
-
def load_model(
|
213
|
+
def load_model(
|
214
|
+
user_model_name: str,
|
215
|
+
) -> tuple[BaseLanguageModel, str, int, dict[str, float]]:
|
214
216
|
if not MODEL_CONFIG_DIR.exists():
|
215
217
|
MODEL_CONFIG_DIR.mkdir(parents=True)
|
216
218
|
model_config_file = MODEL_CONFIG_DIR / f"{user_model_name}.json"
|
@@ -1,11 +1,12 @@
|
|
1
1
|
import unittest
|
2
2
|
|
3
|
-
from janus.parsers.code_parser import CodeParser
|
3
|
+
from janus.parsers.code_parser import CodeParser
|
4
|
+
from janus.parsers.parser import GenericParser
|
4
5
|
|
5
6
|
|
6
7
|
class TestJanusParser(unittest.TestCase):
|
7
8
|
def setUp(self):
|
8
|
-
self.parser =
|
9
|
+
self.parser = GenericParser()
|
9
10
|
|
10
11
|
def test_parse_combined_output(self):
|
11
12
|
text = "test text"
|
janus/parsers/code_parser.py
CHANGED
@@ -1,52 +1,20 @@
|
|
1
1
|
import re
|
2
2
|
|
3
|
-
from langchain.schema.output_parser import BaseOutputParser
|
4
3
|
from langchain_core.exceptions import OutputParserException
|
5
4
|
from langchain_core.messages import BaseMessage
|
6
|
-
from langchain_core.output_parsers import StrOutputParser
|
7
5
|
|
8
|
-
from janus.
|
6
|
+
from janus.parsers.parser import JanusParser
|
9
7
|
from janus.utils.logger import create_logger
|
10
8
|
|
11
9
|
log = create_logger(__name__)
|
12
10
|
|
13
11
|
|
14
|
-
class JanusParser:
|
15
|
-
def parse_combined_output(self, text: str) -> str:
|
16
|
-
"""Parse the output text from the LLM when multiple inputs are combined
|
17
|
-
|
18
|
-
Arguments:
|
19
|
-
text: The output text from the LLM
|
20
|
-
|
21
|
-
Returns:
|
22
|
-
A parsed version of the text
|
23
|
-
"""
|
24
|
-
if isinstance(text, BaseMessage):
|
25
|
-
text = text.content
|
26
|
-
return text
|
27
|
-
|
28
|
-
def parse_into_block(self, text: str, block: CodeBlock):
|
29
|
-
if isinstance(text, BaseMessage):
|
30
|
-
text = text.content
|
31
|
-
block.text = text
|
32
|
-
|
33
|
-
def set_reference(self, block: CodeBlock):
|
34
|
-
pass
|
35
|
-
|
36
|
-
|
37
|
-
class GenericParser(StrOutputParser, JanusParser):
|
38
|
-
def parse(self, text: str) -> str:
|
39
|
-
if isinstance(text, BaseMessage):
|
40
|
-
return text.content
|
41
|
-
return text
|
42
|
-
|
43
|
-
|
44
|
-
class CodeParser(BaseOutputParser[str], JanusParser):
|
12
|
+
class CodeParser(JanusParser):
|
45
13
|
language: str
|
46
14
|
|
47
|
-
def parse(self, text: str) -> str:
|
15
|
+
def parse(self, text: str | BaseMessage) -> str:
|
48
16
|
if isinstance(text, BaseMessage):
|
49
|
-
text = text.content
|
17
|
+
text = str(text.content)
|
50
18
|
pattern = rf"```[^\S\r\n]*(?:{self.language}[^\S\r\n]*)?\n?(.*?)\n*```"
|
51
19
|
code = re.search(pattern, text, re.DOTALL)
|
52
20
|
if code is None:
|
janus/parsers/doc_parser.py
CHANGED
@@ -3,13 +3,12 @@ import re
|
|
3
3
|
|
4
4
|
from langchain.output_parsers import PydanticOutputParser
|
5
5
|
from langchain.output_parsers.json import parse_json_markdown
|
6
|
-
from langchain.schema.output_parser import BaseOutputParser
|
7
6
|
from langchain_core.exceptions import OutputParserException
|
8
|
-
from langchain_core.messages import
|
7
|
+
from langchain_core.messages import BaseMessage
|
9
8
|
from langchain_core.pydantic_v1 import BaseModel, Field
|
10
9
|
|
11
10
|
from janus.language.block import CodeBlock
|
12
|
-
from janus.parsers.
|
11
|
+
from janus.parsers.parser import JanusParser
|
13
12
|
from janus.utils.logger import create_logger
|
14
13
|
|
15
14
|
log = create_logger(__name__)
|
@@ -32,18 +31,21 @@ class MultiDoc(BaseModel):
|
|
32
31
|
)
|
33
32
|
|
34
33
|
|
35
|
-
class MultiDocumentationParser(
|
34
|
+
class MultiDocumentationParser(JanusParser, PydanticOutputParser):
|
36
35
|
block_name: str = ""
|
37
36
|
|
38
37
|
def __init__(self):
|
39
38
|
PydanticOutputParser.__init__(self, pydantic_object=MultiDoc)
|
40
39
|
|
41
|
-
def
|
42
|
-
|
40
|
+
def parse_input(self, block: CodeBlock) -> str:
|
41
|
+
text = super().parse_input(block)
|
42
|
+
self.block_name = str(block.name)
|
43
|
+
return text
|
44
|
+
|
45
|
+
def parse(self, text: str | BaseMessage) -> str:
|
46
|
+
if isinstance(text, BaseMessage):
|
47
|
+
text = str(text.content)
|
43
48
|
|
44
|
-
def parse(self, text: str) -> str:
|
45
|
-
if isinstance(text, AIMessage):
|
46
|
-
text = text.content
|
47
49
|
try:
|
48
50
|
docs = json.loads(super().parse(text).json())
|
49
51
|
except (OutputParserException, json.JSONDecodeError):
|
@@ -81,22 +83,25 @@ class MultiDocumentationParser(PydanticOutputParser, JanusParser):
|
|
81
83
|
|
82
84
|
@property
|
83
85
|
def _type(self) -> str:
|
84
|
-
return self.__class__.name
|
86
|
+
return str(self.__class__.name)
|
85
87
|
|
86
88
|
|
87
|
-
class MadlibsDocumentationParser(
|
89
|
+
class MadlibsDocumentationParser(JanusParser):
|
88
90
|
expected_keys: set[str]
|
89
91
|
|
90
92
|
def __init__(self):
|
91
93
|
super().__init__(expected_keys=[])
|
92
94
|
|
93
|
-
def
|
94
|
-
|
95
|
+
def parse_input(self, block: CodeBlock):
|
96
|
+
# TODO: Perform comment stripping/placeholding here rather than in script
|
97
|
+
text = super().parse_input(block)
|
98
|
+
comment_ids = re.findall(r"<(?:BLOCK|INLINE)_COMMENT (\w{8})>", text)
|
95
99
|
self.expected_keys = set(comment_ids)
|
96
100
|
|
97
|
-
def parse(self, text: str) -> str:
|
98
|
-
if isinstance(text,
|
99
|
-
text = text.content
|
101
|
+
def parse(self, text: str | BaseMessage) -> str:
|
102
|
+
if isinstance(text, BaseMessage):
|
103
|
+
text = str(text.content)
|
104
|
+
|
100
105
|
try:
|
101
106
|
obj = parse_json_markdown(text)
|
102
107
|
except json.JSONDecodeError as e:
|
@@ -166,4 +171,4 @@ class MadlibsDocumentationParser(BaseOutputParser[str], JanusParser):
|
|
166
171
|
|
167
172
|
@property
|
168
173
|
def _type(self) -> str:
|
169
|
-
return self.__class__.name
|
174
|
+
return str(self.__class__.name)
|
janus/parsers/eval_parser.py
CHANGED
@@ -1,9 +1,10 @@
|
|
1
1
|
import json
|
2
2
|
|
3
3
|
from langchain.output_parsers import PydanticOutputParser
|
4
|
+
from langchain_core.messages import BaseMessage
|
4
5
|
from langchain_core.pydantic_v1 import BaseModel, Field, validator
|
5
6
|
|
6
|
-
from janus.parsers.
|
7
|
+
from janus.parsers.parser import JanusParser
|
7
8
|
from janus.utils.logger import create_logger
|
8
9
|
|
9
10
|
log = create_logger(__name__)
|
@@ -28,7 +29,7 @@ class Eval(BaseModel):
|
|
28
29
|
|
29
30
|
return v
|
30
31
|
|
31
|
-
def __add__(self, other):
|
32
|
+
def __add__(self, other: "Eval"):
|
32
33
|
if isinstance(other, int) and other == 0:
|
33
34
|
return self.copy()
|
34
35
|
return Eval.construct(
|
@@ -38,10 +39,10 @@ class Eval(BaseModel):
|
|
38
39
|
completeness=self.completeness + other.completeness,
|
39
40
|
)
|
40
41
|
|
41
|
-
def __radd__(self, other):
|
42
|
+
def __radd__(self, other) -> "Eval":
|
42
43
|
return self.__add__(other)
|
43
44
|
|
44
|
-
def __truediv__(self, other):
|
45
|
+
def __truediv__(self, other) -> "Eval":
|
45
46
|
if isinstance(other, int):
|
46
47
|
return Eval.construct(
|
47
48
|
syntax=self.syntax / other,
|
@@ -57,11 +58,13 @@ class Eval(BaseModel):
|
|
57
58
|
)
|
58
59
|
|
59
60
|
|
60
|
-
class EvaluationParser(
|
61
|
+
class EvaluationParser(JanusParser, PydanticOutputParser):
|
61
62
|
def __init__(self):
|
62
63
|
PydanticOutputParser.__init__(self, pydantic_object=Eval)
|
63
64
|
|
64
|
-
def parse(self, text: str) -> str:
|
65
|
+
def parse(self, text: str | BaseMessage) -> str:
|
66
|
+
if isinstance(text, BaseMessage):
|
67
|
+
text = str(text.content)
|
65
68
|
eval = super().parse(text)
|
66
69
|
return json.dumps(eval.json())
|
67
70
|
|
@@ -75,6 +78,6 @@ class EvaluationParser(PydanticOutputParser, JanusParser):
|
|
75
78
|
Returns:
|
76
79
|
A parsed version of the text.
|
77
80
|
"""
|
78
|
-
objs = [super().parse(line.strip()) for line in text.split("\n")]
|
79
|
-
avg_obj = sum(objs) / len(objs)
|
81
|
+
objs: list[Eval] = [super().parse(line.strip()) for line in text.split("\n")]
|
82
|
+
avg_obj: Eval = sum(objs) / len(objs)
|
80
83
|
return json.dumps(avg_obj.json())
|
janus/parsers/parser.py
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
from langchain.schema.output_parser import BaseOutputParser
|
2
|
+
from langchain_core.messages import BaseMessage
|
3
|
+
from langchain_core.output_parsers import StrOutputParser
|
4
|
+
|
5
|
+
from janus.language.block import CodeBlock
|
6
|
+
from janus.language.splitter import EmptyTreeError
|
7
|
+
from janus.utils.logger import create_logger
|
8
|
+
|
9
|
+
log = create_logger(__name__)
|
10
|
+
|
11
|
+
|
12
|
+
class JanusParser(BaseOutputParser[str]):
|
13
|
+
def parse_input(self, block: CodeBlock) -> str:
|
14
|
+
"""Parse the input block into raw string input ready to be passed to
|
15
|
+
an LLM. Also perform any processing or saving of metadata.
|
16
|
+
|
17
|
+
Arguments:
|
18
|
+
block: The CodeBlock to be processed
|
19
|
+
|
20
|
+
Returns:
|
21
|
+
A parsed version of the input text
|
22
|
+
"""
|
23
|
+
if block.text is None:
|
24
|
+
raise EmptyTreeError("No text in input CodeBlock!")
|
25
|
+
return block.text
|
26
|
+
|
27
|
+
def parse_combined_output(self, text: str) -> str:
|
28
|
+
"""Parse the output text from the LLM when multiple inputs are combined
|
29
|
+
|
30
|
+
Arguments:
|
31
|
+
text: The output text from the LLM
|
32
|
+
|
33
|
+
Returns:
|
34
|
+
A parsed version of the text
|
35
|
+
"""
|
36
|
+
return text
|
37
|
+
|
38
|
+
def parse_into_block(self, text: str | BaseMessage, block: CodeBlock):
|
39
|
+
if isinstance(text, BaseMessage):
|
40
|
+
text = str(text.content)
|
41
|
+
block.text = text
|
42
|
+
|
43
|
+
|
44
|
+
class GenericParser(JanusParser, StrOutputParser):
|
45
|
+
def parse(self, text: str | BaseMessage) -> str:
|
46
|
+
if isinstance(text, BaseMessage):
|
47
|
+
text = str(text.content)
|
48
|
+
return text
|
49
|
+
|
50
|
+
def get_format_instructions(self) -> str:
|
51
|
+
return "Output should be a string"
|
janus/parsers/refiner_parser.py
CHANGED
@@ -1,11 +1,13 @@
|
|
1
1
|
from langchain_core.exceptions import OutputParserException
|
2
2
|
from langchain_core.language_models import BaseLanguageModel
|
3
|
+
from langchain_core.messages import BaseMessage
|
3
4
|
from langchain_core.output_parsers import BaseOutputParser
|
4
5
|
|
6
|
+
from janus.parsers.parser import JanusParser
|
5
7
|
from janus.refiners.refiner import Refiner
|
6
8
|
|
7
9
|
|
8
|
-
class RefinerParser(
|
10
|
+
class RefinerParser(JanusParser):
|
9
11
|
"""Parser for performing refinement with a refiner
|
10
12
|
|
11
13
|
Properties:
|
@@ -25,15 +27,7 @@ class RefinerParser(BaseOutputParser):
|
|
25
27
|
refiner: Refiner
|
26
28
|
max_retries: int
|
27
29
|
|
28
|
-
def parse(self, text: str) -> str:
|
29
|
-
"""Parses the text using the refiner
|
30
|
-
|
31
|
-
Arguments:
|
32
|
-
text: text to parse
|
33
|
-
|
34
|
-
Returns:
|
35
|
-
Parsed text
|
36
|
-
"""
|
30
|
+
def parse(self, text: str | BaseMessage) -> str:
|
37
31
|
last_prompt = self.initial_prompt
|
38
32
|
for _ in range(self.max_retries):
|
39
33
|
try:
|
@@ -46,6 +40,7 @@ class RefinerParser(BaseOutputParser):
|
|
46
40
|
new_chain = new_prompt | self.llm
|
47
41
|
text = new_chain.invoke(prompt_arguments)
|
48
42
|
last_prompt = new_prompt.format(**prompt_arguments)
|
43
|
+
|
49
44
|
raise OutputParserException(
|
50
45
|
f"Error: unable to correct output after {self.max_retries} attempts"
|
51
46
|
)
|
janus/parsers/reqs_parser.py
CHANGED
@@ -2,29 +2,28 @@ import json
|
|
2
2
|
import re
|
3
3
|
|
4
4
|
from langchain.output_parsers.json import parse_json_markdown
|
5
|
-
from langchain.schema.output_parser import BaseOutputParser
|
6
5
|
from langchain_core.exceptions import OutputParserException
|
7
|
-
from langchain_core.messages import
|
6
|
+
from langchain_core.messages import BaseMessage
|
8
7
|
|
9
|
-
from janus.
|
10
|
-
from janus.parsers.code_parser import JanusParser
|
8
|
+
from janus.parsers.parser import JanusParser
|
11
9
|
from janus.utils.logger import create_logger
|
12
10
|
|
13
11
|
log = create_logger(__name__)
|
14
12
|
|
15
13
|
|
16
|
-
class RequirementsParser(
|
17
|
-
|
14
|
+
class RequirementsParser(JanusParser):
|
15
|
+
expected_keys: set[str]
|
18
16
|
|
19
17
|
def __init__(self):
|
20
18
|
super().__init__(expected_keys=[])
|
21
19
|
|
22
|
-
def
|
23
|
-
|
20
|
+
def parse(self, text: str | BaseMessage) -> str:
|
21
|
+
if isinstance(text, BaseMessage):
|
22
|
+
text = str(text.content)
|
24
23
|
|
25
|
-
|
26
|
-
|
27
|
-
|
24
|
+
# TODO: This is an incorrect implementation (lstrip and rstrip take character
|
25
|
+
# lists and strip any instances of those characters, not the full str)
|
26
|
+
# Should be replaced with a regex search, see CodeParser
|
28
27
|
text = text.lstrip("```json")
|
29
28
|
text = text.rstrip("```")
|
30
29
|
try:
|
@@ -70,4 +69,4 @@ class RequirementsParser(BaseOutputParser[str], JanusParser):
|
|
70
69
|
|
71
70
|
@property
|
72
71
|
def _type(self) -> str:
|
73
|
-
return self.__class__.name
|
72
|
+
return str(self.__class__.name)
|
janus/parsers/uml.py
CHANGED
@@ -4,6 +4,7 @@ from pathlib import Path
|
|
4
4
|
from typing import List, Tuple
|
5
5
|
|
6
6
|
from langchain_core.exceptions import OutputParserException
|
7
|
+
from langchain_core.messages import BaseMessage
|
7
8
|
|
8
9
|
from janus.parsers.code_parser import CodeParser
|
9
10
|
from janus.utils.logger import create_logger
|
@@ -12,7 +13,7 @@ log = create_logger(__name__)
|
|
12
13
|
|
13
14
|
|
14
15
|
class UMLSyntaxParser(CodeParser):
|
15
|
-
def _get_uml_output(self, file:
|
16
|
+
def _get_uml_output(self, file: Path) -> Tuple[str, str]:
|
16
17
|
# NOTE: running subprocess with shell=False, added nosec to label that we know
|
17
18
|
# risk exists
|
18
19
|
try:
|
@@ -33,7 +34,7 @@ class UMLSyntaxParser(CodeParser):
|
|
33
34
|
def _get_errs(self, s: str) -> List[str]:
|
34
35
|
return [x.group() for x in re.finditer(r"Error (.*)\n", s)]
|
35
36
|
|
36
|
-
def parse(self, text: str) -> str:
|
37
|
+
def parse(self, text: str | BaseMessage) -> str:
|
37
38
|
text = super().parse(text)
|
38
39
|
janus_path = Path.home().expanduser() / Path(".janus")
|
39
40
|
if not janus_path.exists():
|
@@ -1,4 +1,4 @@
|
|
1
|
-
janus/__init__.py,sha256=
|
1
|
+
janus/__init__.py,sha256=HX3kYLkqXTi8C9sXb0P1mGki4DMJJzxHSQSxoilv7sA,361
|
2
2
|
janus/__main__.py,sha256=lEkpNtLVPtFo8ySDZeXJ_NXDHb0GVdZFPWB4gD4RPS8,64
|
3
3
|
janus/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
4
4
|
janus/_tests/conftest.py,sha256=V7uW-oq3YbFiRPvrq15YoVVrA1n_83pjgiyTZ-IUGW8,963
|
@@ -7,8 +7,8 @@ janus/cli.py,sha256=_92FvDV4qza0nSmyiXqacYxyo1gY6IPwD4gCm6kZfqI,33213
|
|
7
7
|
janus/converter/__init__.py,sha256=U2EOMcCykiC0ZqhorNefOP_04hOF18qhYoPKrVp1Vrk,345
|
8
8
|
janus/converter/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
9
9
|
janus/converter/_tests/test_translate.py,sha256=yzcFEGc_z8QmBBBmC9dZnfL9tT8CD1rkpc8Hz44Jp4c,5631
|
10
|
-
janus/converter/converter.py,sha256=
|
11
|
-
janus/converter/diagram.py,sha256=
|
10
|
+
janus/converter/converter.py,sha256=GL4moCOi5vvvsEjAf_7Vrn5GHq12nptH47AAP4soeQc,27956
|
11
|
+
janus/converter/diagram.py,sha256=AyxkoyfMoQ7jpuOqyJ3bIvYqHJSEhmR-gYrIsq-d8tk,5285
|
12
12
|
janus/converter/document.py,sha256=hsW512veNjFWbdl5WriuUdNmMEqZy8ktRvqn9rRmA6E,4566
|
13
13
|
janus/converter/evaluate.py,sha256=APWQUY3gjAXqkJkPzvj0UA4wPK3Cv9QSJLM-YK9t-ng,476
|
14
14
|
janus/converter/requirements.py,sha256=9tvQ40FZJtG8niIFn45gPQCgKKHVPPoFLinBv6RAqO4,2027
|
@@ -57,7 +57,7 @@ janus/language/treesitter/_tests/test_treesitter.py,sha256=4S_UdH6AfJ0j6hyInZ2CF
|
|
57
57
|
janus/language/treesitter/treesitter.py,sha256=FU86H8j2cfRLtwyNeEnf9A6gzZEvDwmnFCIrQymUJso,7541
|
58
58
|
janus/llm/__init__.py,sha256=TKLYvnsWKWfxMucy-lCLQ-4bkN9ENotJZDywDEQmrKg,45
|
59
59
|
janus/llm/model_callbacks.py,sha256=K7P5NY-rf7IYRAFHnZ3kzhrQWE6g_najx8uxlaSiz3E,7110
|
60
|
-
janus/llm/models_info.py,sha256=
|
60
|
+
janus/llm/models_info.py,sha256=26lY_w-LTiPvSFRlKdpqEQjVnsmd7ns7XBUl0wuxe00,8370
|
61
61
|
janus/metrics/__init__.py,sha256=AsxtZJUzZiXJPr2ehPPltuYP-ddechjg6X85WZUO7mA,241
|
62
62
|
janus/metrics/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
63
63
|
janus/metrics/_tests/reference.py,sha256=hiaJPP9CXkvFBV_wL-gOe_BzELTw0nvB6uCxhxtIiE8,13
|
@@ -83,13 +83,14 @@ janus/metrics/similarity.py,sha256=NKd-m8hMB27i1_1CQkG5o7HrAK2wTr89B21Y36kqHiE,1
|
|
83
83
|
janus/metrics/splitting.py,sha256=610ScHRvALwdkqA6YyGI-tr3a18_cUofldBxGYX0SwE,968
|
84
84
|
janus/parsers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
85
85
|
janus/parsers/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
86
|
-
janus/parsers/_tests/test_code_parser.py,sha256=
|
87
|
-
janus/parsers/code_parser.py,sha256=
|
88
|
-
janus/parsers/doc_parser.py,sha256=
|
89
|
-
janus/parsers/eval_parser.py,sha256=
|
90
|
-
janus/parsers/
|
91
|
-
janus/parsers/
|
92
|
-
janus/parsers/
|
86
|
+
janus/parsers/_tests/test_code_parser.py,sha256=3ay5QpUPcynX_EJ-YLl3PR28poutUkT7qvyWUSQ7Too,928
|
87
|
+
janus/parsers/code_parser.py,sha256=3l0HfzgrvJuiwk779s9ZsgUl3xbp1nE1qZxh8aDYRBI,873
|
88
|
+
janus/parsers/doc_parser.py,sha256=G7kDdJ3davHhEFXqwNYw7hOB26hKH_d_0XdpyyBHq_U,5835
|
89
|
+
janus/parsers/eval_parser.py,sha256=Gjh6aTZgpYd2ASJUEPMo4LpCL00cBmbOqc4KM3hy8x8,2922
|
90
|
+
janus/parsers/parser.py,sha256=y6VV64bgVidf-oEFla3I--_28tnJsPBc6QUD_SkbfSE,1614
|
91
|
+
janus/parsers/refiner_parser.py,sha256=W7ecDX7j6N-uWtRBYFD1EiEzDpDQws16nrVFzyArzf8,1632
|
92
|
+
janus/parsers/reqs_parser.py,sha256=uRQC41Iqp22GjIvakb5UKv70UWHkcOTbOVl_RDnipYw,2438
|
93
|
+
janus/parsers/uml.py,sha256=T9MzYbXz_gjlfgJlIMGSzgACcXLhoN3RnV9Y-5vYJ9A,1888
|
93
94
|
janus/prompts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
94
95
|
janus/prompts/prompt.py,sha256=3796YXIzzIec9b0iUzd8VZlq-AdQbzq8qUGXLy4KH-0,10586
|
95
96
|
janus/refiners/refiner.py,sha256=GkV4oUSCrLAhyDJY2aY_Jt8PRF3sC6-bv58YbL2PaNk,2227
|
@@ -100,8 +101,8 @@ janus/utils/_tests/test_progress.py,sha256=Rs_u5PiGjP-L-o6C1fhwfE1ig8jYu9Xo9s4p8
|
|
100
101
|
janus/utils/enums.py,sha256=AoilbdiYyMvY2Mp0AM4xlbLSELfut2XMwhIM1S_msP4,27610
|
101
102
|
janus/utils/logger.py,sha256=KZeuaMAnlSZCsj4yL0P6N-JzZwpxXygzACWfdZFeuek,2337
|
102
103
|
janus/utils/progress.py,sha256=PIpcQec7SrhsfqB25LHj2CDDkfm9umZx90d9LZnAx6k,1469
|
103
|
-
janus_llm-3.4.
|
104
|
-
janus_llm-3.4.
|
105
|
-
janus_llm-3.4.
|
106
|
-
janus_llm-3.4.
|
107
|
-
janus_llm-3.4.
|
104
|
+
janus_llm-3.4.3.dist-info/LICENSE,sha256=_j0st0a-HB6MRbP3_BW3PUqpS16v54luyy-1zVyl8NU,10789
|
105
|
+
janus_llm-3.4.3.dist-info/METADATA,sha256=rFGRSJIXyld0d04oeG6s7iC9HfQlyuN6I7b1UaCXXWo,4184
|
106
|
+
janus_llm-3.4.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
107
|
+
janus_llm-3.4.3.dist-info/entry_points.txt,sha256=OGhQwzj6pvXp79B0SaBD5apGekCu7Dwe9fZZT_TZ544,39
|
108
|
+
janus_llm-3.4.3.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|