parsethon 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parsethon/__init__.py +82 -0
- parsethon/cli.py +131 -0
- parsethon/importer.py +215 -0
- parsethon/js_syntax.py +499 -0
- parsethon/lexicon.py +328 -0
- parsethon/runtime.py +240 -0
- parsethon/sourcemap.py +155 -0
- parsethon/tokenizer.py +352 -0
- parsethon/tongue.py +370 -0
- parsethon/translator.py +414 -0
- parsethon-0.3.0.dist-info/METADATA +219 -0
- parsethon-0.3.0.dist-info/RECORD +14 -0
- parsethon-0.3.0.dist-info/WHEEL +4 -0
- parsethon-0.3.0.dist-info/entry_points.txt +3 -0
parsethon/__init__.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Parsethon: Extensible Python lexical parser, tokenizer, and Tongue translation system."""
|
|
2
|
+
|
|
3
|
+
from parsethon.cli import app as cli_app, main
|
|
4
|
+
from parsethon.importer import (
|
|
5
|
+
disable,
|
|
6
|
+
enable,
|
|
7
|
+
install_codec,
|
|
8
|
+
install_importer,
|
|
9
|
+
uninstall_codec,
|
|
10
|
+
uninstall_importer,
|
|
11
|
+
)
|
|
12
|
+
from parsethon.lexicon import (
|
|
13
|
+
LexicalToken,
|
|
14
|
+
SourcePosition,
|
|
15
|
+
SourceSpan,
|
|
16
|
+
TokenCategory,
|
|
17
|
+
TokenSubelement,
|
|
18
|
+
classify_identifier_or_keyword,
|
|
19
|
+
classify_symbol,
|
|
20
|
+
)
|
|
21
|
+
from parsethon.js_syntax import (
|
|
22
|
+
desugar_js_syntax,
|
|
23
|
+
has_js_style_syntax,
|
|
24
|
+
to_js_syntax,
|
|
25
|
+
)
|
|
26
|
+
from parsethon.runtime import (
|
|
27
|
+
ParsethonRuntime,
|
|
28
|
+
TongueExecutionError,
|
|
29
|
+
execute,
|
|
30
|
+
execute_file,
|
|
31
|
+
runtime,
|
|
32
|
+
)
|
|
33
|
+
from parsethon.sourcemap import SourceMap, SourceMapEntry
|
|
34
|
+
from parsethon.tokenizer import ParsethonTokenizer, scan_string_literal
|
|
35
|
+
from parsethon.tongue import (
|
|
36
|
+
Tongue,
|
|
37
|
+
TongueRegistry,
|
|
38
|
+
TongueValidationError,
|
|
39
|
+
registry,
|
|
40
|
+
)
|
|
41
|
+
from parsethon.translator import (
|
|
42
|
+
ParsethonTranslator,
|
|
43
|
+
translate_from_python,
|
|
44
|
+
translate_to_python,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
__version__ = "0.3.0"
|
|
48
|
+
__all__ = [
|
|
49
|
+
"LexicalToken",
|
|
50
|
+
"SourcePosition",
|
|
51
|
+
"SourceSpan",
|
|
52
|
+
"TokenCategory",
|
|
53
|
+
"TokenSubelement",
|
|
54
|
+
"classify_identifier_or_keyword",
|
|
55
|
+
"classify_symbol",
|
|
56
|
+
"desugar_js_syntax",
|
|
57
|
+
"has_js_style_syntax",
|
|
58
|
+
"to_js_syntax",
|
|
59
|
+
"ParsethonTokenizer",
|
|
60
|
+
"SourceMap",
|
|
61
|
+
"SourceMapEntry",
|
|
62
|
+
"Tongue",
|
|
63
|
+
"TongueRegistry",
|
|
64
|
+
"TongueValidationError",
|
|
65
|
+
"registry",
|
|
66
|
+
"ParsethonTranslator",
|
|
67
|
+
"translate_to_python",
|
|
68
|
+
"translate_from_python",
|
|
69
|
+
"ParsethonRuntime",
|
|
70
|
+
"TongueExecutionError",
|
|
71
|
+
"execute",
|
|
72
|
+
"execute_file",
|
|
73
|
+
"runtime",
|
|
74
|
+
"install_importer",
|
|
75
|
+
"uninstall_importer",
|
|
76
|
+
"install_codec",
|
|
77
|
+
"uninstall_codec",
|
|
78
|
+
"enable",
|
|
79
|
+
"disable",
|
|
80
|
+
"cli_app",
|
|
81
|
+
"main",
|
|
82
|
+
]
|
parsethon/cli.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Command-line interface for Parsethon powered by expose-interface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ast
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import List, Optional
|
|
10
|
+
|
|
11
|
+
import expose_interface as ei
|
|
12
|
+
from parsethon.lexicon import TokenCategory
|
|
13
|
+
from parsethon.runtime import execute_file
|
|
14
|
+
from parsethon.tokenizer import ParsethonTokenizer
|
|
15
|
+
from parsethon.tongue import registry
|
|
16
|
+
from parsethon.translator import translate_from_python, translate_to_python
|
|
17
|
+
|
|
18
|
+
app = ei.Interface("parsethon", description="Parsethon: Extensible Python Lexical Parser and Tongue System")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@app.command
|
|
22
|
+
def run(path: str, tongue: str = "") -> None:
|
|
23
|
+
"""Execute a Tongue source file under CPython."""
|
|
24
|
+
try:
|
|
25
|
+
execute_file(path, tongue=tongue if tongue else None)
|
|
26
|
+
except Exception as e:
|
|
27
|
+
sys.stderr.write(f"{e}\n")
|
|
28
|
+
sys.exit(1)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@app.command
|
|
32
|
+
def translate(
|
|
33
|
+
path: str,
|
|
34
|
+
tongue: str = "",
|
|
35
|
+
to_python: bool = True,
|
|
36
|
+
output: str = "",
|
|
37
|
+
) -> None:
|
|
38
|
+
"""Translate source code between Tongue and canonical Python syntax."""
|
|
39
|
+
file_path = Path(path)
|
|
40
|
+
if not file_path.exists():
|
|
41
|
+
sys.stderr.write(f"File not found: {path}\n")
|
|
42
|
+
sys.exit(1)
|
|
43
|
+
|
|
44
|
+
source = file_path.read_text(encoding="utf-8")
|
|
45
|
+
t_name = tongue if tongue else None
|
|
46
|
+
|
|
47
|
+
# Infer tongue if to_python
|
|
48
|
+
if not t_name:
|
|
49
|
+
from parsethon.runtime import detect_tongue_from_filename, detect_tongue_from_source
|
|
50
|
+
t_name = detect_tongue_from_source(source) or detect_tongue_from_filename(path) or "es"
|
|
51
|
+
|
|
52
|
+
if to_python:
|
|
53
|
+
result = translate_to_python(source, t_name)
|
|
54
|
+
else:
|
|
55
|
+
result = translate_from_python(source, t_name)
|
|
56
|
+
|
|
57
|
+
if output:
|
|
58
|
+
Path(output).write_text(result, encoding="utf-8")
|
|
59
|
+
print(f"Wrote translated code to {output}")
|
|
60
|
+
else:
|
|
61
|
+
sys.stdout.write(result)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@app.command
|
|
65
|
+
def tokenize(path: str, verbose: bool = False) -> None:
|
|
66
|
+
"""Tokenize a source file and display lexical classification and subelements."""
|
|
67
|
+
file_path = Path(path)
|
|
68
|
+
if not file_path.exists():
|
|
69
|
+
sys.stderr.write(f"File not found: {path}\n")
|
|
70
|
+
sys.exit(1)
|
|
71
|
+
|
|
72
|
+
source = file_path.read_text(encoding="utf-8")
|
|
73
|
+
tokenizer = ParsethonTokenizer(source)
|
|
74
|
+
tokens = tokenizer.tokenize()
|
|
75
|
+
|
|
76
|
+
print(f"Total tokens emitted: {len(tokens)}")
|
|
77
|
+
print(f"{'SPAN':<16} {'CATEGORY':<14} {'SUBELEMENT':<20} {'VALUE'}")
|
|
78
|
+
print("-" * 65)
|
|
79
|
+
|
|
80
|
+
for tok in tokens:
|
|
81
|
+
if not verbose and tok.category in (TokenCategory.WHITESPACE, TokenCategory.NEWLINE):
|
|
82
|
+
continue
|
|
83
|
+
span_str = f"{tok.span.start.line}:{tok.span.start.column}-{tok.span.end.line}:{tok.span.end.column}"
|
|
84
|
+
repr_val = repr(tok.value)
|
|
85
|
+
if len(repr_val) > 25:
|
|
86
|
+
repr_val = repr_val[:22] + "..."
|
|
87
|
+
print(f"{span_str:<16} {tok.category.value:<14} {tok.subelement.value:<20} {repr_val}")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@app.command
|
|
91
|
+
def list_tongues() -> None:
|
|
92
|
+
"""Enumerate all available registered Tongues."""
|
|
93
|
+
print("Available Tongues:")
|
|
94
|
+
for name in registry.list_tongues():
|
|
95
|
+
t = registry.get(name)
|
|
96
|
+
desc = t.description or "No description"
|
|
97
|
+
sample_kw = list(t.keywords.items())[:3]
|
|
98
|
+
samples = ", ".join(f"{k}->{v}" for k, v in sample_kw)
|
|
99
|
+
print(f" * {name:<10} (v{t.version}): {desc} [{samples}...]")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@app.command
|
|
103
|
+
def check(path: str, tongue: str = "") -> None:
|
|
104
|
+
"""Validate syntax of a Tongue file without executing."""
|
|
105
|
+
file_path = Path(path)
|
|
106
|
+
if not file_path.exists():
|
|
107
|
+
sys.stderr.write(f"File not found: {path}\n")
|
|
108
|
+
sys.exit(1)
|
|
109
|
+
|
|
110
|
+
source = file_path.read_text(encoding="utf-8")
|
|
111
|
+
t_name = tongue if tongue else None
|
|
112
|
+
if not t_name:
|
|
113
|
+
from parsethon.runtime import detect_tongue_from_filename, detect_tongue_from_source
|
|
114
|
+
t_name = detect_tongue_from_source(source) or detect_tongue_from_filename(path) or "es"
|
|
115
|
+
|
|
116
|
+
try:
|
|
117
|
+
py_source = translate_to_python(source, t_name)
|
|
118
|
+
ast.parse(py_source, filename=path)
|
|
119
|
+
print(f"Syntax valid: {path} (Tongue: {t_name})")
|
|
120
|
+
except SyntaxError as se:
|
|
121
|
+
sys.stderr.write(f"Syntax error at line {se.lineno}: {se.msg}\n")
|
|
122
|
+
sys.exit(1)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def main(argv: Optional[List[str]] = None) -> None:
|
|
126
|
+
"""Main CLI entry point."""
|
|
127
|
+
app.main(argv, prog_name="parsethon")
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
if __name__ == "__main__":
|
|
131
|
+
main()
|
parsethon/importer.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""Custom import hook and codec registration for Tongue modules."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import codecs
|
|
6
|
+
import importlib.abc
|
|
7
|
+
import importlib.machinery
|
|
8
|
+
import importlib.util
|
|
9
|
+
import os
|
|
10
|
+
import sys
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, Callable, List, Optional, Sequence, Tuple
|
|
13
|
+
|
|
14
|
+
from parsethon.runtime import detect_tongue_from_source
|
|
15
|
+
from parsethon.tongue import registry
|
|
16
|
+
from parsethon.translator import translate_to_python
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class TongueSourceLoader(importlib.abc.SourceLoader):
|
|
20
|
+
"""Loader that compiles Tongue source into standard Python bytecode."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, fullname: str, path: str, tongue_name: str) -> None:
|
|
23
|
+
self.fullname = fullname
|
|
24
|
+
self.path = path
|
|
25
|
+
self.tongue_name = tongue_name
|
|
26
|
+
|
|
27
|
+
def get_filename(self, fullname: str) -> str:
|
|
28
|
+
return self.path
|
|
29
|
+
|
|
30
|
+
def get_data(self, path: str) -> bytes:
|
|
31
|
+
with open(path, "rb") as f:
|
|
32
|
+
return f.read()
|
|
33
|
+
|
|
34
|
+
def source_to_code(self, data: bytes, path: str, *, _optimize: int = -1) -> Any:
|
|
35
|
+
source_text = data.decode("utf-8")
|
|
36
|
+
py_source = translate_to_python(source_text, self.tongue_name)
|
|
37
|
+
return compile(py_source, path, "exec", optimize=_optimize)
|
|
38
|
+
|
|
39
|
+
def exec_module(self, module: Any) -> None:
|
|
40
|
+
if self.tongue_name == "js":
|
|
41
|
+
if not hasattr(module, "console"):
|
|
42
|
+
class _Console:
|
|
43
|
+
log = staticmethod(print)
|
|
44
|
+
error = staticmethod(print)
|
|
45
|
+
warn = staticmethod(print)
|
|
46
|
+
info = staticmethod(print)
|
|
47
|
+
module.__dict__["console"] = _Console()
|
|
48
|
+
super().exec_module(module)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class TongueMetaPathFinder(importlib.abc.MetaPathFinder):
|
|
52
|
+
"""Finder that resolves `<module>.<tongue>.py`, `<module>.<tongue>`, and `<module>.pyt` files."""
|
|
53
|
+
|
|
54
|
+
def find_spec(
|
|
55
|
+
self,
|
|
56
|
+
fullname: str,
|
|
57
|
+
path: Optional[Sequence[str]] = None,
|
|
58
|
+
target: Optional[Any] = None,
|
|
59
|
+
) -> Optional[importlib.machinery.ModuleSpec]:
|
|
60
|
+
subname = fullname.rpartition(".")[-1]
|
|
61
|
+
search_dirs = list(path) if path is not None else sys.path
|
|
62
|
+
|
|
63
|
+
known_tongues = registry.list_tongues()
|
|
64
|
+
|
|
65
|
+
for directory in search_dirs:
|
|
66
|
+
if not directory or not os.path.isdir(directory):
|
|
67
|
+
continue
|
|
68
|
+
|
|
69
|
+
# 1. Search for `<subname>.<tongue>.py`
|
|
70
|
+
for tongue_name in known_tongues:
|
|
71
|
+
candidate = os.path.join(directory, f"{subname}.{tongue_name}.py")
|
|
72
|
+
if os.path.isfile(candidate):
|
|
73
|
+
loader = TongueSourceLoader(fullname, candidate, tongue_name)
|
|
74
|
+
return importlib.util.spec_from_loader(
|
|
75
|
+
fullname, loader, origin=candidate
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
# 2. Search for direct extension `<subname>.<tongue>` (e.g. `<subname>.js`)
|
|
79
|
+
for tongue_name in known_tongues:
|
|
80
|
+
candidate = os.path.join(directory, f"{subname}.{tongue_name}")
|
|
81
|
+
if os.path.isfile(candidate):
|
|
82
|
+
loader = TongueSourceLoader(fullname, candidate, tongue_name)
|
|
83
|
+
return importlib.util.spec_from_loader(
|
|
84
|
+
fullname, loader, origin=candidate
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
# 3. Search for package directory `<subname>/`
|
|
88
|
+
pkg_dir = os.path.join(directory, subname)
|
|
89
|
+
if os.path.isdir(pkg_dir):
|
|
90
|
+
for tongue_name in known_tongues:
|
|
91
|
+
for init_cand in [
|
|
92
|
+
os.path.join(pkg_dir, f"__init__.{tongue_name}.py"),
|
|
93
|
+
os.path.join(pkg_dir, f"__init__.{tongue_name}"),
|
|
94
|
+
]:
|
|
95
|
+
if os.path.isfile(init_cand):
|
|
96
|
+
loader = TongueSourceLoader(fullname, init_cand, tongue_name)
|
|
97
|
+
return importlib.util.spec_from_loader(
|
|
98
|
+
fullname, loader, origin=init_cand, is_package=True
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
# 4. Search for `<subname>.pyt` with header detection
|
|
102
|
+
candidate_pyt = os.path.join(directory, f"{subname}.pyt")
|
|
103
|
+
if os.path.isfile(candidate_pyt):
|
|
104
|
+
with open(candidate_pyt, "r", encoding="utf-8") as f:
|
|
105
|
+
content = f.read(500)
|
|
106
|
+
detected = detect_tongue_from_source(content)
|
|
107
|
+
if detected:
|
|
108
|
+
loader = TongueSourceLoader(fullname, candidate_pyt, detected)
|
|
109
|
+
return importlib.util.spec_from_loader(
|
|
110
|
+
fullname, loader, origin=candidate_pyt
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
return None
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# --- Codec Support for `# coding: parsethon-<tongue>` ---
|
|
117
|
+
|
|
118
|
+
def _tongue_decode(input_bytes: bytes, errors: str = "strict", tongue_name: str = "es") -> Tuple[str, int]:
|
|
119
|
+
raw_str, length = codecs.utf_8_decode(input_bytes, errors)
|
|
120
|
+
translated = translate_to_python(raw_str, tongue_name)
|
|
121
|
+
return translated, length
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class TongueIncrementalDecoder(codecs.BufferedIncrementalDecoder):
|
|
125
|
+
def __init__(self, errors: str = "strict", tongue_name: str = "es") -> None:
|
|
126
|
+
super().__init__(errors)
|
|
127
|
+
self.tongue_name = tongue_name
|
|
128
|
+
|
|
129
|
+
def _buffer_decode(self, input_bytes: bytes, errors: str, final: bool) -> Tuple[str, int]:
|
|
130
|
+
if final:
|
|
131
|
+
return _tongue_decode(input_bytes, errors, self.tongue_name)
|
|
132
|
+
return "", 0
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class TongueStreamReader(codecs.StreamReader):
|
|
136
|
+
def __init__(self, stream: Any, errors: str = "strict", tongue_name: str = "es") -> None:
|
|
137
|
+
super().__init__(stream, errors)
|
|
138
|
+
self.tongue_name = tongue_name
|
|
139
|
+
|
|
140
|
+
def decode(self, input_bytes: bytes, errors: str = "strict") -> Tuple[str, int]:
|
|
141
|
+
return _tongue_decode(input_bytes, errors, self.tongue_name)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def search_tongue_codec(encoding: str) -> Optional[codecs.CodecInfo]:
|
|
145
|
+
"""Finds codecs matching pattern `parsethon-<tongue>` or `parsethon_<tongue>`."""
|
|
146
|
+
if not _codec_installed:
|
|
147
|
+
return None
|
|
148
|
+
|
|
149
|
+
normalized = encoding.lower().replace("-", "_")
|
|
150
|
+
if not normalized.startswith("parsethon_"):
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
tongue_name = normalized.partition("parsethon_")[2]
|
|
154
|
+
if tongue_name not in registry.list_tongues():
|
|
155
|
+
return None
|
|
156
|
+
|
|
157
|
+
return codecs.CodecInfo(
|
|
158
|
+
name=f"parsethon-{tongue_name}",
|
|
159
|
+
encode=codecs.utf_8_encode,
|
|
160
|
+
decode=lambda b, err="strict": _tongue_decode(b, err, tongue_name),
|
|
161
|
+
incrementalencoder=codecs.getincrementalencoder("utf-8"),
|
|
162
|
+
incrementaldecoder=lambda err="strict": TongueIncrementalDecoder(err, tongue_name),
|
|
163
|
+
streamreader=lambda stream, err="strict": TongueStreamReader(stream, err, tongue_name),
|
|
164
|
+
streamwriter=codecs.getwriter("utf-8"),
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
_finder_installed = False
|
|
169
|
+
_codec_installed = False
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def install_importer() -> None:
|
|
173
|
+
"""Installs the TongueMetaPathFinder in sys.meta_path."""
|
|
174
|
+
global _finder_installed
|
|
175
|
+
if not _finder_installed:
|
|
176
|
+
sys.meta_path.insert(0, TongueMetaPathFinder())
|
|
177
|
+
_finder_installed = True
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def uninstall_importer() -> None:
|
|
181
|
+
"""Removes the TongueMetaPathFinder from sys.meta_path."""
|
|
182
|
+
global _finder_installed
|
|
183
|
+
sys.meta_path = [f for f in sys.meta_path if not isinstance(f, TongueMetaPathFinder)]
|
|
184
|
+
_finder_installed = False
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def install_codec() -> None:
|
|
188
|
+
"""Registers the parsethon codec search function."""
|
|
189
|
+
global _codec_installed
|
|
190
|
+
if not _codec_installed:
|
|
191
|
+
codecs.register(search_tongue_codec)
|
|
192
|
+
_codec_installed = True
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def uninstall_codec() -> None:
|
|
196
|
+
"""Unregisters the parsethon codec search function if supported."""
|
|
197
|
+
global _codec_installed
|
|
198
|
+
_codec_installed = False
|
|
199
|
+
if hasattr(codecs, "unregister"):
|
|
200
|
+
try:
|
|
201
|
+
codecs.unregister(search_tongue_codec)
|
|
202
|
+
except Exception:
|
|
203
|
+
pass
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def enable() -> None:
|
|
207
|
+
"""Enables both the meta_path import hook and codec support."""
|
|
208
|
+
install_importer()
|
|
209
|
+
install_codec()
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def disable() -> None:
|
|
213
|
+
"""Disables the meta_path import hook."""
|
|
214
|
+
uninstall_importer()
|
|
215
|
+
uninstall_codec()
|