janus-llm 3.5.1__tar.gz → 3.5.3__tar.gz

Sign up to get free protection for your applications and to get access to all the features.
Files changed (108) hide show
  1. {janus_llm-3.5.1 → janus_llm-3.5.3}/PKG-INFO +1 -1
  2. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/__init__.py +1 -1
  3. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/_tests/test_cli.py +1 -0
  4. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/cli.py +33 -4
  5. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/alc/alc.py +1 -1
  6. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/naive/simple_ast.py +3 -2
  7. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/splitter.py +5 -2
  8. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/llm/models_info.py +7 -0
  9. {janus_llm-3.5.1 → janus_llm-3.5.3}/pyproject.toml +1 -1
  10. {janus_llm-3.5.1 → janus_llm-3.5.3}/LICENSE +0 -0
  11. {janus_llm-3.5.1 → janus_llm-3.5.3}/README.md +0 -0
  12. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/__main__.py +0 -0
  13. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/_tests/__init__.py +0 -0
  14. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/_tests/conftest.py +0 -0
  15. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/converter/__init__.py +0 -0
  16. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/converter/_tests/__init__.py +0 -0
  17. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/converter/_tests/test_translate.py +0 -0
  18. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/converter/aggregator.py +0 -0
  19. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/converter/converter.py +0 -0
  20. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/converter/diagram.py +0 -0
  21. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/converter/document.py +0 -0
  22. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/converter/evaluate.py +0 -0
  23. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/converter/requirements.py +0 -0
  24. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/converter/translate.py +0 -0
  25. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/embedding/__init__.py +0 -0
  26. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/embedding/_tests/__init__.py +0 -0
  27. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/embedding/_tests/test_collections.py +0 -0
  28. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/embedding/_tests/test_database.py +0 -0
  29. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/embedding/_tests/test_vectorize.py +0 -0
  30. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/embedding/collections.py +0 -0
  31. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/embedding/database.py +0 -0
  32. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/embedding/embedding_models_info.py +0 -0
  33. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/embedding/vectorize.py +0 -0
  34. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/__init__.py +0 -0
  35. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/_tests/__init__.py +0 -0
  36. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/_tests/test_combine.py +0 -0
  37. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/_tests/test_splitter.py +0 -0
  38. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/alc/__init__.py +0 -0
  39. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/alc/_tests/__init__.py +0 -0
  40. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/alc/_tests/test_alc.py +0 -0
  41. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/binary/__init__.py +0 -0
  42. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/binary/_tests/__init__.py +0 -0
  43. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/binary/_tests/test_binary.py +0 -0
  44. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/binary/binary.py +0 -0
  45. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/binary/reveng/decompile_script.py +0 -0
  46. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/block.py +0 -0
  47. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/combine.py +0 -0
  48. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/file.py +0 -0
  49. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/mumps/__init__.py +0 -0
  50. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/mumps/_tests/__init__.py +0 -0
  51. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/mumps/_tests/test_mumps.py +0 -0
  52. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/mumps/mumps.py +0 -0
  53. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/mumps/patterns.py +0 -0
  54. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/naive/__init__.py +0 -0
  55. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/naive/basic_splitter.py +0 -0
  56. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/naive/chunk_splitter.py +0 -0
  57. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/naive/registry.py +0 -0
  58. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/naive/tag_splitter.py +0 -0
  59. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/node.py +0 -0
  60. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/treesitter/__init__.py +0 -0
  61. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/treesitter/_tests/__init__.py +0 -0
  62. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/treesitter/_tests/test_treesitter.py +0 -0
  63. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/language/treesitter/treesitter.py +0 -0
  64. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/llm/__init__.py +0 -0
  65. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/llm/model_callbacks.py +0 -0
  66. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/__init__.py +0 -0
  67. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/_tests/__init__.py +0 -0
  68. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/_tests/reference.py +0 -0
  69. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/_tests/target.py +0 -0
  70. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/_tests/test_bleu.py +0 -0
  71. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/_tests/test_chrf.py +0 -0
  72. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/_tests/test_file_pairing.py +0 -0
  73. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/_tests/test_llm.py +0 -0
  74. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/_tests/test_reading.py +0 -0
  75. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/_tests/test_rouge_score.py +0 -0
  76. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/_tests/test_similarity_score.py +0 -0
  77. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/_tests/test_treesitter_metrics.py +0 -0
  78. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/bleu.py +0 -0
  79. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/chrf.py +0 -0
  80. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/cli.py +0 -0
  81. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/complexity_metrics.py +0 -0
  82. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/file_pairing.py +0 -0
  83. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/llm_metrics.py +0 -0
  84. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/metric.py +0 -0
  85. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/reading.py +0 -0
  86. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/rouge_score.py +0 -0
  87. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/similarity.py +0 -0
  88. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/metrics/splitting.py +0 -0
  89. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/parsers/__init__.py +0 -0
  90. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/parsers/_tests/__init__.py +0 -0
  91. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/parsers/_tests/test_code_parser.py +0 -0
  92. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/parsers/code_parser.py +0 -0
  93. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/parsers/doc_parser.py +0 -0
  94. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/parsers/eval_parser.py +0 -0
  95. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/parsers/parser.py +0 -0
  96. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/parsers/refiner_parser.py +0 -0
  97. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/parsers/reqs_parser.py +0 -0
  98. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/parsers/uml.py +0 -0
  99. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/prompts/__init__.py +0 -0
  100. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/prompts/prompt.py +0 -0
  101. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/refiners/refiner.py +0 -0
  102. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/utils/__init__.py +0 -0
  103. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/utils/_tests/__init__.py +0 -0
  104. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/utils/_tests/test_logger.py +0 -0
  105. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/utils/_tests/test_progress.py +0 -0
  106. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/utils/enums.py +0 -0
  107. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/utils/logger.py +0 -0
  108. {janus_llm-3.5.1 → janus_llm-3.5.3}/janus/utils/progress.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: janus-llm
3
- Version: 3.5.1
3
+ Version: 3.5.3
4
4
  Summary: A transcoding library using LLMs.
5
5
  Home-page: https://github.com/janus-llm/janus-llm
6
6
  License: Apache 2.0
@@ -5,7 +5,7 @@ from langchain_core._api.deprecation import LangChainDeprecationWarning
5
5
  from janus.converter.translate import Translator
6
6
  from janus.metrics import * # noqa: F403
7
7
 
8
- __version__ = "3.5.1"
8
+ __version__ = "3.5.3"
9
9
 
10
10
  # Ignoring a deprecation warning from langchain_core that I can't seem to hunt down
11
11
  warnings.filterwarnings("ignore", category=LangChainDeprecationWarning)
@@ -120,6 +120,7 @@ class TestCli(unittest.TestCase):
120
120
  temp=0.7,
121
121
  prompt_template="simple",
122
122
  collection=None,
123
+ llm_name="gpt-4o",
123
124
  )
124
125
 
125
126
  # Assert
@@ -1,6 +1,7 @@
1
1
  import json
2
2
  import logging
3
3
  import os
4
+ import subprocess # nosec
4
5
  from pathlib import Path
5
6
  from typing import List, Optional
6
7
 
@@ -185,7 +186,7 @@ def translate(
185
186
  "-L",
186
187
  help="The custom name of the model set with 'janus llm add'.",
187
188
  ),
188
- ] = "gpt-4o",
189
+ ],
189
190
  max_prompts: Annotated[
190
191
  int,
191
192
  typer.Option(
@@ -324,7 +325,7 @@ def document(
324
325
  "-L",
325
326
  help="The custom name of the model set with 'janus llm add'.",
326
327
  ),
327
- ] = "gpt-4o",
328
+ ],
328
329
  max_prompts: Annotated[
329
330
  int,
330
331
  typer.Option(
@@ -480,7 +481,7 @@ def aggregate(
480
481
  "-L",
481
482
  help="The custom name of the model set with 'janus llm add'.",
482
483
  ),
483
- ] = "gpt-4o",
484
+ ],
484
485
  max_prompts: Annotated[
485
486
  int,
486
487
  typer.Option(
@@ -597,7 +598,7 @@ def diagram(
597
598
  "-L",
598
599
  help="The custom name of the model set with 'janus llm add'.",
599
600
  ),
600
- ] = "gpt-4o",
601
+ ],
601
602
  max_prompts: Annotated[
602
603
  int,
603
604
  typer.Option(
@@ -1156,5 +1157,33 @@ app.add_typer(evaluate, name="evaluate")
1156
1157
  app.add_typer(embedding, name="embedding")
1157
1158
 
1158
1159
 
1160
+ @app.command()
1161
+ def render(
1162
+ input_dir: Annotated[
1163
+ str,
1164
+ typer.Option(
1165
+ "--input",
1166
+ "-i",
1167
+ ),
1168
+ ],
1169
+ output_dir: Annotated[str, typer.Option("--output", "-o")],
1170
+ ):
1171
+ input_dir = Path(input_dir)
1172
+ output_dir = Path(output_dir)
1173
+ for input_file in input_dir.rglob("*.json"):
1174
+ with open(input_file, "r") as f:
1175
+ data = json.load(f)
1176
+ input_tail = input_file.relative_to(input_dir)
1177
+ output_file = output_dir / input_tail
1178
+ output_file = output_file.with_suffix(".txt")
1179
+ if not output_file.parent.exists():
1180
+ output_file.parent.mkdir()
1181
+ with open(output_file, "w") as f:
1182
+ f.write(data["output"])
1183
+ jar_path = homedir / ".janus/lib/plantuml.jar"
1184
+ subprocess.run(["java", "-jar", jar_path, output_file]) # nosec
1185
+ output_file.unlink()
1186
+
1187
+
1159
1188
  if __name__ == "__main__":
1160
1189
  app()
@@ -63,7 +63,7 @@ class AlcSplitter(TreeSitterSplitter):
63
63
  # instruction and containing all the subsequent nodes up until the
64
64
  # next csect or dsect instruction
65
65
  sects: list[list[CodeBlock]] = [[]]
66
- for c in block.children:
66
+ for c in sorted(block.children):
67
67
  if c.node_type == "csect_instruction":
68
68
  c.context_tags["alc_section"] = "CSECT"
69
69
  sects.append([c])
@@ -19,6 +19,7 @@ def get_flexible_ast(language: str, **kwargs) -> Splitter:
19
19
  Returns:
20
20
  A flexible AST splitter for the given language.
21
21
  """
22
+ kwargs.update(protected_node_types=())
22
23
  if language == "ibmhlasm":
23
24
  return AlcSplitter(**kwargs)
24
25
  elif language == "mumps":
@@ -28,7 +29,7 @@ def get_flexible_ast(language: str, **kwargs) -> Splitter:
28
29
 
29
30
 
30
31
  @register_splitter("ast-strict")
31
- def get_strict_ast(language: str, **kwargs) -> Splitter:
32
+ def get_strict_ast(language: str, prune_unprotected=True, **kwargs) -> Splitter:
32
33
  """Get a strict AST splitter for the given language.
33
34
 
34
35
  The strict splitter will only return nodes that are of a functional type.
@@ -41,7 +42,7 @@ def get_strict_ast(language: str, **kwargs) -> Splitter:
41
42
  """
42
43
  kwargs.update(
43
44
  protected_node_types=LANGUAGES[language]["functional_node_types"],
44
- prune_unprotected=True,
45
+ prune_unprotected=prune_unprotected,
45
46
  )
46
47
  if language == "ibmhlasm":
47
48
  return AlcSplitter(**kwargs)
@@ -387,7 +387,10 @@ class Splitter(FileManager):
387
387
  return
388
388
 
389
389
  if self._is_protected(node):
390
- raise TokenLimitError(r"Irreducible node too large for context!")
390
+ log.error(
391
+ "Protected node too large for context!"
392
+ f" ({node.tokens} > {self.max_tokens})"
393
+ )
391
394
 
392
395
  if node.children:
393
396
  for child in node.children:
@@ -423,7 +426,7 @@ class Splitter(FileManager):
423
426
  name = f"{node.name}-L#{node_line}"
424
427
  tokens = self._count_tokens(line)
425
428
  if tokens > self.max_tokens:
426
- raise TokenLimitError(
429
+ log.error(
427
430
  "Irreducible node too large for context!"
428
431
  f" ({tokens} > {self.max_tokens})"
429
432
  )
@@ -1,5 +1,6 @@
1
1
  import json
2
2
  import os
3
+ import time
3
4
  from pathlib import Path
4
5
  from typing import Any, Callable
5
6
 
@@ -247,6 +248,12 @@ def load_model(
247
248
  model_args = model_config["model_args"]
248
249
  if model_config["model_type"] == "OpenAI":
249
250
  model_args.update(_open_ai_defaults)
251
+ log.warning("Do NOT use this model in sensitive environments!")
252
+ log.warning("If you would like to cancel, please press Ctrl+C.")
253
+ log.warning("Waiting 10 seconds...")
254
+ # Give enough time for the user to read the warnings and cancel
255
+ time.sleep(10)
256
+
250
257
  model = model_constructor(**model_args)
251
258
  return (
252
259
  model,
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "janus-llm"
3
- version = "3.5.1"
3
+ version = "3.5.3"
4
4
  description = "A transcoding library using LLMs."
5
5
  authors = ["Michael Doyle <mdoyle@mitre.org>", "Chris Glasz <cglasz@mitre.org>",
6
6
  "Chris Tohline <ctohline@mitre.org>", "William Macke <wmacke@mitre.org>",
File without changes
File without changes
File without changes