parsethon 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
parsethon/__init__.py ADDED
@@ -0,0 +1,82 @@
1
+ """Parsethon: Extensible Python lexical parser, tokenizer, and Tongue translation system."""
2
+
3
+ from parsethon.cli import app as cli_app, main
4
+ from parsethon.importer import (
5
+ disable,
6
+ enable,
7
+ install_codec,
8
+ install_importer,
9
+ uninstall_codec,
10
+ uninstall_importer,
11
+ )
12
+ from parsethon.lexicon import (
13
+ LexicalToken,
14
+ SourcePosition,
15
+ SourceSpan,
16
+ TokenCategory,
17
+ TokenSubelement,
18
+ classify_identifier_or_keyword,
19
+ classify_symbol,
20
+ )
21
+ from parsethon.js_syntax import (
22
+ desugar_js_syntax,
23
+ has_js_style_syntax,
24
+ to_js_syntax,
25
+ )
26
+ from parsethon.runtime import (
27
+ ParsethonRuntime,
28
+ TongueExecutionError,
29
+ execute,
30
+ execute_file,
31
+ runtime,
32
+ )
33
+ from parsethon.sourcemap import SourceMap, SourceMapEntry
34
+ from parsethon.tokenizer import ParsethonTokenizer, scan_string_literal
35
+ from parsethon.tongue import (
36
+ Tongue,
37
+ TongueRegistry,
38
+ TongueValidationError,
39
+ registry,
40
+ )
41
+ from parsethon.translator import (
42
+ ParsethonTranslator,
43
+ translate_from_python,
44
+ translate_to_python,
45
+ )
46
+
47
+ __version__ = "0.3.0"
48
+ __all__ = [
49
+ "LexicalToken",
50
+ "SourcePosition",
51
+ "SourceSpan",
52
+ "TokenCategory",
53
+ "TokenSubelement",
54
+ "classify_identifier_or_keyword",
55
+ "classify_symbol",
56
+ "desugar_js_syntax",
57
+ "has_js_style_syntax",
58
+ "to_js_syntax",
59
+ "ParsethonTokenizer",
60
+ "SourceMap",
61
+ "SourceMapEntry",
62
+ "Tongue",
63
+ "TongueRegistry",
64
+ "TongueValidationError",
65
+ "registry",
66
+ "ParsethonTranslator",
67
+ "translate_to_python",
68
+ "translate_from_python",
69
+ "ParsethonRuntime",
70
+ "TongueExecutionError",
71
+ "execute",
72
+ "execute_file",
73
+ "runtime",
74
+ "install_importer",
75
+ "uninstall_importer",
76
+ "install_codec",
77
+ "uninstall_codec",
78
+ "enable",
79
+ "disable",
80
+ "cli_app",
81
+ "main",
82
+ ]
parsethon/cli.py ADDED
@@ -0,0 +1,131 @@
1
+ """Command-line interface for Parsethon powered by expose-interface."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ast
6
+ import os
7
+ import sys
8
+ from pathlib import Path
9
+ from typing import List, Optional
10
+
11
+ import expose_interface as ei
12
+ from parsethon.lexicon import TokenCategory
13
+ from parsethon.runtime import execute_file
14
+ from parsethon.tokenizer import ParsethonTokenizer
15
+ from parsethon.tongue import registry
16
+ from parsethon.translator import translate_from_python, translate_to_python
17
+
18
+ app = ei.Interface("parsethon", description="Parsethon: Extensible Python Lexical Parser and Tongue System")
19
+
20
+
21
+ @app.command
22
+ def run(path: str, tongue: str = "") -> None:
23
+ """Execute a Tongue source file under CPython."""
24
+ try:
25
+ execute_file(path, tongue=tongue if tongue else None)
26
+ except Exception as e:
27
+ sys.stderr.write(f"{e}\n")
28
+ sys.exit(1)
29
+
30
+
31
+ @app.command
32
+ def translate(
33
+ path: str,
34
+ tongue: str = "",
35
+ to_python: bool = True,
36
+ output: str = "",
37
+ ) -> None:
38
+ """Translate source code between Tongue and canonical Python syntax."""
39
+ file_path = Path(path)
40
+ if not file_path.exists():
41
+ sys.stderr.write(f"File not found: {path}\n")
42
+ sys.exit(1)
43
+
44
+ source = file_path.read_text(encoding="utf-8")
45
+ t_name = tongue if tongue else None
46
+
47
+ # Infer tongue if to_python
48
+ if not t_name:
49
+ from parsethon.runtime import detect_tongue_from_filename, detect_tongue_from_source
50
+ t_name = detect_tongue_from_source(source) or detect_tongue_from_filename(path) or "es"
51
+
52
+ if to_python:
53
+ result = translate_to_python(source, t_name)
54
+ else:
55
+ result = translate_from_python(source, t_name)
56
+
57
+ if output:
58
+ Path(output).write_text(result, encoding="utf-8")
59
+ print(f"Wrote translated code to {output}")
60
+ else:
61
+ sys.stdout.write(result)
62
+
63
+
64
+ @app.command
65
+ def tokenize(path: str, verbose: bool = False) -> None:
66
+ """Tokenize a source file and display lexical classification and subelements."""
67
+ file_path = Path(path)
68
+ if not file_path.exists():
69
+ sys.stderr.write(f"File not found: {path}\n")
70
+ sys.exit(1)
71
+
72
+ source = file_path.read_text(encoding="utf-8")
73
+ tokenizer = ParsethonTokenizer(source)
74
+ tokens = tokenizer.tokenize()
75
+
76
+ print(f"Total tokens emitted: {len(tokens)}")
77
+ print(f"{'SPAN':<16} {'CATEGORY':<14} {'SUBELEMENT':<20} {'VALUE'}")
78
+ print("-" * 65)
79
+
80
+ for tok in tokens:
81
+ if not verbose and tok.category in (TokenCategory.WHITESPACE, TokenCategory.NEWLINE):
82
+ continue
83
+ span_str = f"{tok.span.start.line}:{tok.span.start.column}-{tok.span.end.line}:{tok.span.end.column}"
84
+ repr_val = repr(tok.value)
85
+ if len(repr_val) > 25:
86
+ repr_val = repr_val[:22] + "..."
87
+ print(f"{span_str:<16} {tok.category.value:<14} {tok.subelement.value:<20} {repr_val}")
88
+
89
+
90
+ @app.command
91
+ def list_tongues() -> None:
92
+ """Enumerate all available registered Tongues."""
93
+ print("Available Tongues:")
94
+ for name in registry.list_tongues():
95
+ t = registry.get(name)
96
+ desc = t.description or "No description"
97
+ sample_kw = list(t.keywords.items())[:3]
98
+ samples = ", ".join(f"{k}->{v}" for k, v in sample_kw)
99
+ print(f" * {name:<10} (v{t.version}): {desc} [{samples}...]")
100
+
101
+
102
+ @app.command
103
+ def check(path: str, tongue: str = "") -> None:
104
+ """Validate syntax of a Tongue file without executing."""
105
+ file_path = Path(path)
106
+ if not file_path.exists():
107
+ sys.stderr.write(f"File not found: {path}\n")
108
+ sys.exit(1)
109
+
110
+ source = file_path.read_text(encoding="utf-8")
111
+ t_name = tongue if tongue else None
112
+ if not t_name:
113
+ from parsethon.runtime import detect_tongue_from_filename, detect_tongue_from_source
114
+ t_name = detect_tongue_from_source(source) or detect_tongue_from_filename(path) or "es"
115
+
116
+ try:
117
+ py_source = translate_to_python(source, t_name)
118
+ ast.parse(py_source, filename=path)
119
+ print(f"Syntax valid: {path} (Tongue: {t_name})")
120
+ except SyntaxError as se:
121
+ sys.stderr.write(f"Syntax error at line {se.lineno}: {se.msg}\n")
122
+ sys.exit(1)
123
+
124
+
125
+ def main(argv: Optional[List[str]] = None) -> None:
126
+ """Main CLI entry point."""
127
+ app.main(argv, prog_name="parsethon")
128
+
129
+
130
+ if __name__ == "__main__":
131
+ main()
parsethon/importer.py ADDED
@@ -0,0 +1,215 @@
1
+ """Custom import hook and codec registration for Tongue modules."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import codecs
6
+ import importlib.abc
7
+ import importlib.machinery
8
+ import importlib.util
9
+ import os
10
+ import sys
11
+ from pathlib import Path
12
+ from typing import Any, Callable, List, Optional, Sequence, Tuple
13
+
14
+ from parsethon.runtime import detect_tongue_from_source
15
+ from parsethon.tongue import registry
16
+ from parsethon.translator import translate_to_python
17
+
18
+
19
+ class TongueSourceLoader(importlib.abc.SourceLoader):
20
+ """Loader that compiles Tongue source into standard Python bytecode."""
21
+
22
+ def __init__(self, fullname: str, path: str, tongue_name: str) -> None:
23
+ self.fullname = fullname
24
+ self.path = path
25
+ self.tongue_name = tongue_name
26
+
27
+ def get_filename(self, fullname: str) -> str:
28
+ return self.path
29
+
30
+ def get_data(self, path: str) -> bytes:
31
+ with open(path, "rb") as f:
32
+ return f.read()
33
+
34
+ def source_to_code(self, data: bytes, path: str, *, _optimize: int = -1) -> Any:
35
+ source_text = data.decode("utf-8")
36
+ py_source = translate_to_python(source_text, self.tongue_name)
37
+ return compile(py_source, path, "exec", optimize=_optimize)
38
+
39
+ def exec_module(self, module: Any) -> None:
40
+ if self.tongue_name == "js":
41
+ if not hasattr(module, "console"):
42
+ class _Console:
43
+ log = staticmethod(print)
44
+ error = staticmethod(print)
45
+ warn = staticmethod(print)
46
+ info = staticmethod(print)
47
+ module.__dict__["console"] = _Console()
48
+ super().exec_module(module)
49
+
50
+
51
+ class TongueMetaPathFinder(importlib.abc.MetaPathFinder):
52
+ """Finder that resolves `<module>.<tongue>.py`, `<module>.<tongue>`, and `<module>.pyt` files."""
53
+
54
+ def find_spec(
55
+ self,
56
+ fullname: str,
57
+ path: Optional[Sequence[str]] = None,
58
+ target: Optional[Any] = None,
59
+ ) -> Optional[importlib.machinery.ModuleSpec]:
60
+ subname = fullname.rpartition(".")[-1]
61
+ search_dirs = list(path) if path is not None else sys.path
62
+
63
+ known_tongues = registry.list_tongues()
64
+
65
+ for directory in search_dirs:
66
+ if not directory or not os.path.isdir(directory):
67
+ continue
68
+
69
+ # 1. Search for `<subname>.<tongue>.py`
70
+ for tongue_name in known_tongues:
71
+ candidate = os.path.join(directory, f"{subname}.{tongue_name}.py")
72
+ if os.path.isfile(candidate):
73
+ loader = TongueSourceLoader(fullname, candidate, tongue_name)
74
+ return importlib.util.spec_from_loader(
75
+ fullname, loader, origin=candidate
76
+ )
77
+
78
+ # 2. Search for direct extension `<subname>.<tongue>` (e.g. `<subname>.js`)
79
+ for tongue_name in known_tongues:
80
+ candidate = os.path.join(directory, f"{subname}.{tongue_name}")
81
+ if os.path.isfile(candidate):
82
+ loader = TongueSourceLoader(fullname, candidate, tongue_name)
83
+ return importlib.util.spec_from_loader(
84
+ fullname, loader, origin=candidate
85
+ )
86
+
87
+ # 3. Search for package directory `<subname>/`
88
+ pkg_dir = os.path.join(directory, subname)
89
+ if os.path.isdir(pkg_dir):
90
+ for tongue_name in known_tongues:
91
+ for init_cand in [
92
+ os.path.join(pkg_dir, f"__init__.{tongue_name}.py"),
93
+ os.path.join(pkg_dir, f"__init__.{tongue_name}"),
94
+ ]:
95
+ if os.path.isfile(init_cand):
96
+ loader = TongueSourceLoader(fullname, init_cand, tongue_name)
97
+ return importlib.util.spec_from_loader(
98
+ fullname, loader, origin=init_cand, is_package=True
99
+ )
100
+
101
+ # 4. Search for `<subname>.pyt` with header detection
102
+ candidate_pyt = os.path.join(directory, f"{subname}.pyt")
103
+ if os.path.isfile(candidate_pyt):
104
+ with open(candidate_pyt, "r", encoding="utf-8") as f:
105
+ content = f.read(500)
106
+ detected = detect_tongue_from_source(content)
107
+ if detected:
108
+ loader = TongueSourceLoader(fullname, candidate_pyt, detected)
109
+ return importlib.util.spec_from_loader(
110
+ fullname, loader, origin=candidate_pyt
111
+ )
112
+
113
+ return None
114
+
115
+
116
+ # --- Codec Support for `# coding: parsethon-<tongue>` ---
117
+
118
+ def _tongue_decode(input_bytes: bytes, errors: str = "strict", tongue_name: str = "es") -> Tuple[str, int]:
119
+ raw_str, length = codecs.utf_8_decode(input_bytes, errors)
120
+ translated = translate_to_python(raw_str, tongue_name)
121
+ return translated, length
122
+
123
+
124
+ class TongueIncrementalDecoder(codecs.BufferedIncrementalDecoder):
125
+ def __init__(self, errors: str = "strict", tongue_name: str = "es") -> None:
126
+ super().__init__(errors)
127
+ self.tongue_name = tongue_name
128
+
129
+ def _buffer_decode(self, input_bytes: bytes, errors: str, final: bool) -> Tuple[str, int]:
130
+ if final:
131
+ return _tongue_decode(input_bytes, errors, self.tongue_name)
132
+ return "", 0
133
+
134
+
135
+ class TongueStreamReader(codecs.StreamReader):
136
+ def __init__(self, stream: Any, errors: str = "strict", tongue_name: str = "es") -> None:
137
+ super().__init__(stream, errors)
138
+ self.tongue_name = tongue_name
139
+
140
+ def decode(self, input_bytes: bytes, errors: str = "strict") -> Tuple[str, int]:
141
+ return _tongue_decode(input_bytes, errors, self.tongue_name)
142
+
143
+
144
+ def search_tongue_codec(encoding: str) -> Optional[codecs.CodecInfo]:
145
+ """Finds codecs matching pattern `parsethon-<tongue>` or `parsethon_<tongue>`."""
146
+ if not _codec_installed:
147
+ return None
148
+
149
+ normalized = encoding.lower().replace("-", "_")
150
+ if not normalized.startswith("parsethon_"):
151
+ return None
152
+
153
+ tongue_name = normalized.partition("parsethon_")[2]
154
+ if tongue_name not in registry.list_tongues():
155
+ return None
156
+
157
+ return codecs.CodecInfo(
158
+ name=f"parsethon-{tongue_name}",
159
+ encode=codecs.utf_8_encode,
160
+ decode=lambda b, err="strict": _tongue_decode(b, err, tongue_name),
161
+ incrementalencoder=codecs.getincrementalencoder("utf-8"),
162
+ incrementaldecoder=lambda err="strict": TongueIncrementalDecoder(err, tongue_name),
163
+ streamreader=lambda stream, err="strict": TongueStreamReader(stream, err, tongue_name),
164
+ streamwriter=codecs.getwriter("utf-8"),
165
+ )
166
+
167
+
168
+ _finder_installed = False
169
+ _codec_installed = False
170
+
171
+
172
+ def install_importer() -> None:
173
+ """Installs the TongueMetaPathFinder in sys.meta_path."""
174
+ global _finder_installed
175
+ if not _finder_installed:
176
+ sys.meta_path.insert(0, TongueMetaPathFinder())
177
+ _finder_installed = True
178
+
179
+
180
+ def uninstall_importer() -> None:
181
+ """Removes the TongueMetaPathFinder from sys.meta_path."""
182
+ global _finder_installed
183
+ sys.meta_path = [f for f in sys.meta_path if not isinstance(f, TongueMetaPathFinder)]
184
+ _finder_installed = False
185
+
186
+
187
+ def install_codec() -> None:
188
+ """Registers the parsethon codec search function."""
189
+ global _codec_installed
190
+ if not _codec_installed:
191
+ codecs.register(search_tongue_codec)
192
+ _codec_installed = True
193
+
194
+
195
+ def uninstall_codec() -> None:
196
+ """Unregisters the parsethon codec search function if supported."""
197
+ global _codec_installed
198
+ _codec_installed = False
199
+ if hasattr(codecs, "unregister"):
200
+ try:
201
+ codecs.unregister(search_tongue_codec)
202
+ except Exception:
203
+ pass
204
+
205
+
206
+ def enable() -> None:
207
+ """Enables both the meta_path import hook and codec support."""
208
+ install_importer()
209
+ install_codec()
210
+
211
+
212
+ def disable() -> None:
213
+ """Disables the meta_path import hook."""
214
+ uninstall_importer()
215
+ uninstall_codec()