parsethon 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parsethon-0.3.0/PKG-INFO +219 -0
- parsethon-0.3.0/README.md +204 -0
- parsethon-0.3.0/pyproject.toml +31 -0
- parsethon-0.3.0/pyproject.toml.orig +32 -0
- parsethon-0.3.0/src/parsethon/__init__.py +82 -0
- parsethon-0.3.0/src/parsethon/cli.py +131 -0
- parsethon-0.3.0/src/parsethon/importer.py +215 -0
- parsethon-0.3.0/src/parsethon/js_syntax.py +499 -0
- parsethon-0.3.0/src/parsethon/lexicon.py +328 -0
- parsethon-0.3.0/src/parsethon/runtime.py +240 -0
- parsethon-0.3.0/src/parsethon/sourcemap.py +155 -0
- parsethon-0.3.0/src/parsethon/tokenizer.py +352 -0
- parsethon-0.3.0/src/parsethon/tongue.py +370 -0
- parsethon-0.3.0/src/parsethon/translator.py +414 -0
parsethon-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: parsethon
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Extensible Python Lexical Parser and Tongue Translation System
|
|
5
|
+
Author: ellainng
|
|
6
|
+
Author-email: ellainng <299958882+ellainng@users.noreply.github.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Requires-Dist: expose-interface>=0.1.0
|
|
9
|
+
Requires-Python: >=3.14
|
|
10
|
+
Project-URL: Homepage, https://github.com/wispsy/parsethon
|
|
11
|
+
Project-URL: Repository, https://github.com/wispsy/parsethon
|
|
12
|
+
Project-URL: Issues, https://github.com/wispsy/parsethon/issues
|
|
13
|
+
Project-URL: Documentation, https://github.com/wispsy/parsethon/tree/main/docs
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# Parsethon 🐍🗣️
|
|
17
|
+
|
|
18
|
+
**Parsethon** is an extensible Python lexical parser, tokenization engine, and language translation system. Rather than creating a new programming language implementation, parsethon is a mechanism for expressing Python in multiple syntactic vocabularies while retaining Python's ecosystem and runtime.
|
|
19
|
+
|
|
20
|
+
It enables developers to:
|
|
21
|
+
1. **Tokenize** Python source code with fine-grained lexical classification and subelement taxonomy (keywords, operators, delimiters, literals, trivia, and spans).
|
|
22
|
+
2. **Define a "Tongue"**: A declarative mapping of replacement keywords, operators, literals, and built-ins.
|
|
23
|
+
3. **Semantically Correct Bidirectional Translation**: Convert source code written in a Tongue back to standard canonical Python syntax (and vice-versa) with demonstrably safe, reversible transformations and AST equivalence.
|
|
24
|
+
4. **Execute under CPython**: Run Tongue source files directly with standard CPython runtime semantics and mapped tracebacks via explicit bidirectional source maps.
|
|
25
|
+
5. **Import Seamlessly**: Import `.tongue.py` modules directly via Python's `sys.meta_path` loader or codec hook (`# coding: parsethon-<tongue>`).
|
|
26
|
+
|
|
27
|
+
Built with **zero third-party dependencies** beyond the in-house CLI interface library [`expose-interface`](https://pypi.org/project/expose-interface/).
|
|
28
|
+
|
|
29
|
+
---
|
|
30
|
+
|
|
31
|
+
## Installation
|
|
32
|
+
|
|
33
|
+
Using `uv`:
|
|
34
|
+
```bash
|
|
35
|
+
uv add parsethon
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Using `pip`:
|
|
39
|
+
```bash
|
|
40
|
+
pip install parsethon
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## Quickstart
|
|
46
|
+
|
|
47
|
+
### 1. Command Line Interface (powered by `expose-interface`)
|
|
48
|
+
|
|
49
|
+
Parsethon exports a unified CLI tool:
|
|
50
|
+
|
|
51
|
+
#### List available Tongues
|
|
52
|
+
```bash
|
|
53
|
+
parsethon list-tongues
|
|
54
|
+
```
|
|
55
|
+
Output:
|
|
56
|
+
```
|
|
57
|
+
Available Tongues:
|
|
58
|
+
* emoji (v1.0.0): Iconic Emoji Python Tongue [def->🔨, return->📦, if->❓...]
|
|
59
|
+
* es (v1.0.0): Spanish language Python Tongue [def->definir, return->retornar, if->si...]
|
|
60
|
+
* js (v1.0.0): JavaScript-style Python Tongue with braces, semicolons, and JS vocabulary
|
|
61
|
+
* pirate (v1.0.0): Swashbuckling Pirate Python Tongue [def->chart, return->booty, if->perchance...]
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
#### Run a Tongue file
|
|
65
|
+
```bash
|
|
66
|
+
parsethon run examples/fibonacci.es.py
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
#### Translate Tongue code to Python
|
|
70
|
+
```bash
|
|
71
|
+
parsethon translate examples/fibonacci.es.py --output fibonacci.py
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
#### Inspect Token Breakdown
|
|
75
|
+
```bash
|
|
76
|
+
parsethon tokenize examples/counter.emoji.py
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
#### Validate Syntax
|
|
80
|
+
```bash
|
|
81
|
+
parsethon check examples/fibonacci.es.py
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
## 2. Python API
|
|
87
|
+
|
|
88
|
+
### Tokenizing Code with Subelement Taxonomy
|
|
89
|
+
```python
|
|
90
|
+
from parsethon import ParsethonTokenizer
|
|
91
|
+
|
|
92
|
+
code = "def add(x: int, y: int) -> int:\n return x + y\n"
|
|
93
|
+
tokenizer = ParsethonTokenizer(code)
|
|
94
|
+
tokens = tokenizer.tokenize()
|
|
95
|
+
|
|
96
|
+
for token in tokens:
|
|
97
|
+
if token.category.value not in ("whitespace", "newline"):
|
|
98
|
+
print(f"{token.span}: {token.category.value} ({token.subelement.value}) -> {token.value}")
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
### Defining a Custom Tongue
|
|
102
|
+
```python
|
|
103
|
+
from parsethon import Tongue, registry
|
|
104
|
+
|
|
105
|
+
# Create a Latin Tongue
|
|
106
|
+
latin = Tongue(
|
|
107
|
+
name="la",
|
|
108
|
+
version="1.0.0",
|
|
109
|
+
description="Latin Python Tongue",
|
|
110
|
+
keywords={
|
|
111
|
+
"def": "definire",
|
|
112
|
+
"return": "reddere",
|
|
113
|
+
"if": "si",
|
|
114
|
+
"else": "aliter",
|
|
115
|
+
"while": "dum",
|
|
116
|
+
"for": "pro",
|
|
117
|
+
"in": "in",
|
|
118
|
+
},
|
|
119
|
+
constants={
|
|
120
|
+
"True": "Verum",
|
|
121
|
+
"False": "Falsum",
|
|
122
|
+
"None": "Nihil",
|
|
123
|
+
},
|
|
124
|
+
builtins={
|
|
125
|
+
"print": "scribere",
|
|
126
|
+
},
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
# Register it
|
|
130
|
+
registry.register(latin)
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
### Translating Between Tongue and Python
|
|
134
|
+
```python
|
|
135
|
+
from parsethon import translate_to_python, translate_from_python
|
|
136
|
+
|
|
137
|
+
latin_code = """
|
|
138
|
+
definire addere(a, b):
|
|
139
|
+
si a > 0:
|
|
140
|
+
reddere a + b
|
|
141
|
+
aliter:
|
|
142
|
+
reddere b
|
|
143
|
+
"""
|
|
144
|
+
|
|
145
|
+
py_code = translate_to_python(latin_code, "la")
|
|
146
|
+
print(py_code)
|
|
147
|
+
# Output:
|
|
148
|
+
# def addere(a, b):
|
|
149
|
+
# if a > 0:
|
|
150
|
+
# return a + b
|
|
151
|
+
# else:
|
|
152
|
+
# return b
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
### Direct CPython Execution
|
|
156
|
+
```python
|
|
157
|
+
from parsethon import execute
|
|
158
|
+
|
|
159
|
+
scope = execute(latin_code, tongue="la")
|
|
160
|
+
# Functions and variables are ready in scope!
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
### Seamless Module Imports
|
|
164
|
+
```python
|
|
165
|
+
import parsethon
|
|
166
|
+
|
|
167
|
+
# Enable Tongue import hooks
|
|
168
|
+
parsethon.enable()
|
|
169
|
+
|
|
170
|
+
# Directly import foo.es.py or foo.emoji.py!
|
|
171
|
+
import my_spanish_module
|
|
172
|
+
my_spanish_module.saludar("Mundo")
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
### JavaScript-Style Syntax (Braces & Semicolons)
|
|
176
|
+
Parsethon supports full JavaScript-style block syntax, semicolons, and complete whitespace indifference:
|
|
177
|
+
|
|
178
|
+
```javascript
|
|
179
|
+
// calculator.js
|
|
180
|
+
function factorial(n) {
|
|
181
|
+
if (n <= 1) {
|
|
182
|
+
return 1;
|
|
183
|
+
} else {
|
|
184
|
+
return n * factorial(n - 1);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
let result = factorial(5);
|
|
189
|
+
console.log("Factorial:", result);
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
You can execute `.js` scripts directly:
|
|
193
|
+
```bash
|
|
194
|
+
parsethon run calculator.js
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Or import them transparently as Python modules:
|
|
198
|
+
```python
|
|
199
|
+
import parsethon
|
|
200
|
+
parsethon.enable()
|
|
201
|
+
|
|
202
|
+
import calculator
|
|
203
|
+
print(calculator.factorial(5)) # 120
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
---
|
|
207
|
+
|
|
208
|
+
## Architecture & Documentation
|
|
209
|
+
|
|
210
|
+
- [Implementation Plan](docs/IMPLEMENTATION_PLAN.md)
|
|
211
|
+
- [Tongue Authoring Guide](docs/TONGUE_AUTHORING_GUIDE.md)
|
|
212
|
+
- [Architecture Deep Dive](docs/ARCHITECTURE.md)
|
|
213
|
+
- [CLI Reference](docs/CLI_REFERENCE.md)
|
|
214
|
+
|
|
215
|
+
---
|
|
216
|
+
|
|
217
|
+
## License
|
|
218
|
+
|
|
219
|
+
MIT
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
# Parsethon 🐍🗣️
|
|
2
|
+
|
|
3
|
+
**Parsethon** is an extensible Python lexical parser, tokenization engine, and language translation system. Rather than creating a new programming language implementation, parsethon is a mechanism for expressing Python in multiple syntactic vocabularies while retaining Python's ecosystem and runtime.
|
|
4
|
+
|
|
5
|
+
It enables developers to:
|
|
6
|
+
1. **Tokenize** Python source code with fine-grained lexical classification and subelement taxonomy (keywords, operators, delimiters, literals, trivia, and spans).
|
|
7
|
+
2. **Define a "Tongue"**: A declarative mapping of replacement keywords, operators, literals, and built-ins.
|
|
8
|
+
3. **Semantically Correct Bidirectional Translation**: Convert source code written in a Tongue back to standard canonical Python syntax (and vice-versa) with demonstrably safe, reversible transformations and AST equivalence.
|
|
9
|
+
4. **Execute under CPython**: Run Tongue source files directly with standard CPython runtime semantics and mapped tracebacks via explicit bidirectional source maps.
|
|
10
|
+
5. **Import Seamlessly**: Import `.tongue.py` modules directly via Python's `sys.meta_path` loader or codec hook (`# coding: parsethon-<tongue>`).
|
|
11
|
+
|
|
12
|
+
Built with **zero third-party dependencies** beyond the in-house CLI interface library [`expose-interface`](https://pypi.org/project/expose-interface/).
|
|
13
|
+
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
## Installation
|
|
17
|
+
|
|
18
|
+
Using `uv`:
|
|
19
|
+
```bash
|
|
20
|
+
uv add parsethon
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Using `pip`:
|
|
24
|
+
```bash
|
|
25
|
+
pip install parsethon
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
## Quickstart
|
|
31
|
+
|
|
32
|
+
### 1. Command Line Interface (powered by `expose-interface`)
|
|
33
|
+
|
|
34
|
+
Parsethon exports a unified CLI tool:
|
|
35
|
+
|
|
36
|
+
#### List available Tongues
|
|
37
|
+
```bash
|
|
38
|
+
parsethon list-tongues
|
|
39
|
+
```
|
|
40
|
+
Output:
|
|
41
|
+
```
|
|
42
|
+
Available Tongues:
|
|
43
|
+
* emoji (v1.0.0): Iconic Emoji Python Tongue [def->🔨, return->📦, if->❓...]
|
|
44
|
+
* es (v1.0.0): Spanish language Python Tongue [def->definir, return->retornar, if->si...]
|
|
45
|
+
* js (v1.0.0): JavaScript-style Python Tongue with braces, semicolons, and JS vocabulary
|
|
46
|
+
* pirate (v1.0.0): Swashbuckling Pirate Python Tongue [def->chart, return->booty, if->perchance...]
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
#### Run a Tongue file
|
|
50
|
+
```bash
|
|
51
|
+
parsethon run examples/fibonacci.es.py
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
#### Translate Tongue code to Python
|
|
55
|
+
```bash
|
|
56
|
+
parsethon translate examples/fibonacci.es.py --output fibonacci.py
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
#### Inspect Token Breakdown
|
|
60
|
+
```bash
|
|
61
|
+
parsethon tokenize examples/counter.emoji.py
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
#### Validate Syntax
|
|
65
|
+
```bash
|
|
66
|
+
parsethon check examples/fibonacci.es.py
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## 2. Python API
|
|
72
|
+
|
|
73
|
+
### Tokenizing Code with Subelement Taxonomy
|
|
74
|
+
```python
|
|
75
|
+
from parsethon import ParsethonTokenizer
|
|
76
|
+
|
|
77
|
+
code = "def add(x: int, y: int) -> int:\n return x + y\n"
|
|
78
|
+
tokenizer = ParsethonTokenizer(code)
|
|
79
|
+
tokens = tokenizer.tokenize()
|
|
80
|
+
|
|
81
|
+
for token in tokens:
|
|
82
|
+
if token.category.value not in ("whitespace", "newline"):
|
|
83
|
+
print(f"{token.span}: {token.category.value} ({token.subelement.value}) -> {token.value}")
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
### Defining a Custom Tongue
|
|
87
|
+
```python
|
|
88
|
+
from parsethon import Tongue, registry
|
|
89
|
+
|
|
90
|
+
# Create a Latin Tongue
|
|
91
|
+
latin = Tongue(
|
|
92
|
+
name="la",
|
|
93
|
+
version="1.0.0",
|
|
94
|
+
description="Latin Python Tongue",
|
|
95
|
+
keywords={
|
|
96
|
+
"def": "definire",
|
|
97
|
+
"return": "reddere",
|
|
98
|
+
"if": "si",
|
|
99
|
+
"else": "aliter",
|
|
100
|
+
"while": "dum",
|
|
101
|
+
"for": "pro",
|
|
102
|
+
"in": "in",
|
|
103
|
+
},
|
|
104
|
+
constants={
|
|
105
|
+
"True": "Verum",
|
|
106
|
+
"False": "Falsum",
|
|
107
|
+
"None": "Nihil",
|
|
108
|
+
},
|
|
109
|
+
builtins={
|
|
110
|
+
"print": "scribere",
|
|
111
|
+
},
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
# Register it
|
|
115
|
+
registry.register(latin)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
### Translating Between Tongue and Python
|
|
119
|
+
```python
|
|
120
|
+
from parsethon import translate_to_python, translate_from_python
|
|
121
|
+
|
|
122
|
+
latin_code = """
|
|
123
|
+
definire addere(a, b):
|
|
124
|
+
si a > 0:
|
|
125
|
+
reddere a + b
|
|
126
|
+
aliter:
|
|
127
|
+
reddere b
|
|
128
|
+
"""
|
|
129
|
+
|
|
130
|
+
py_code = translate_to_python(latin_code, "la")
|
|
131
|
+
print(py_code)
|
|
132
|
+
# Output:
|
|
133
|
+
# def addere(a, b):
|
|
134
|
+
# if a > 0:
|
|
135
|
+
# return a + b
|
|
136
|
+
# else:
|
|
137
|
+
# return b
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
### Direct CPython Execution
|
|
141
|
+
```python
|
|
142
|
+
from parsethon import execute
|
|
143
|
+
|
|
144
|
+
scope = execute(latin_code, tongue="la")
|
|
145
|
+
# Functions and variables are ready in scope!
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
### Seamless Module Imports
|
|
149
|
+
```python
|
|
150
|
+
import parsethon
|
|
151
|
+
|
|
152
|
+
# Enable Tongue import hooks
|
|
153
|
+
parsethon.enable()
|
|
154
|
+
|
|
155
|
+
# Directly import foo.es.py or foo.emoji.py!
|
|
156
|
+
import my_spanish_module
|
|
157
|
+
my_spanish_module.saludar("Mundo")
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
### JavaScript-Style Syntax (Braces & Semicolons)
|
|
161
|
+
Parsethon supports full JavaScript-style block syntax, semicolons, and complete whitespace indifference:
|
|
162
|
+
|
|
163
|
+
```javascript
|
|
164
|
+
// calculator.js
|
|
165
|
+
function factorial(n) {
|
|
166
|
+
if (n <= 1) {
|
|
167
|
+
return 1;
|
|
168
|
+
} else {
|
|
169
|
+
return n * factorial(n - 1);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
let result = factorial(5);
|
|
174
|
+
console.log("Factorial:", result);
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
You can execute `.js` scripts directly:
|
|
178
|
+
```bash
|
|
179
|
+
parsethon run calculator.js
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
Or import them transparently as Python modules:
|
|
183
|
+
```python
|
|
184
|
+
import parsethon
|
|
185
|
+
parsethon.enable()
|
|
186
|
+
|
|
187
|
+
import calculator
|
|
188
|
+
print(calculator.factorial(5)) # 120
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
---
|
|
192
|
+
|
|
193
|
+
## Architecture & Documentation
|
|
194
|
+
|
|
195
|
+
- [Implementation Plan](docs/IMPLEMENTATION_PLAN.md)
|
|
196
|
+
- [Tongue Authoring Guide](docs/TONGUE_AUTHORING_GUIDE.md)
|
|
197
|
+
- [Architecture Deep Dive](docs/ARCHITECTURE.md)
|
|
198
|
+
- [CLI Reference](docs/CLI_REFERENCE.md)
|
|
199
|
+
|
|
200
|
+
---
|
|
201
|
+
|
|
202
|
+
## License
|
|
203
|
+
|
|
204
|
+
MIT
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "parsethon"
|
|
3
|
+
version = "0.3.0"
|
|
4
|
+
description = "Extensible Python Lexical Parser and Tongue Translation System"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
requires-python = ">=3.14"
|
|
8
|
+
dependencies = ["expose-interface>=0.1.0"]
|
|
9
|
+
|
|
10
|
+
[[project.authors]]
|
|
11
|
+
name = "ellainng"
|
|
12
|
+
email = "299958882+ellainng@users.noreply.github.com"
|
|
13
|
+
|
|
14
|
+
[project.urls]
|
|
15
|
+
Homepage = "https://github.com/wispsy/parsethon"
|
|
16
|
+
Repository = "https://github.com/wispsy/parsethon"
|
|
17
|
+
Issues = "https://github.com/wispsy/parsethon/issues"
|
|
18
|
+
Documentation = "https://github.com/wispsy/parsethon/tree/main/docs"
|
|
19
|
+
|
|
20
|
+
[project.scripts]
|
|
21
|
+
parsethon = "parsethon.cli:main"
|
|
22
|
+
|
|
23
|
+
[build-system]
|
|
24
|
+
requires = ["uv_build>=0.12.19,<0.14.0"]
|
|
25
|
+
build-backend = "uv_build"
|
|
26
|
+
|
|
27
|
+
[dependency-groups]
|
|
28
|
+
dev = [
|
|
29
|
+
"hypothesis>=6.170.14",
|
|
30
|
+
"pytest>=9.2.0",
|
|
31
|
+
]
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "parsethon"
|
|
3
|
+
version = "0.3.0"
|
|
4
|
+
description = "Extensible Python Lexical Parser and Tongue Translation System"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
authors = [
|
|
8
|
+
{ name = "ellainng", email = "299958882+ellainng@users.noreply.github.com" }
|
|
9
|
+
]
|
|
10
|
+
requires-python = ">=3.14"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"expose-interface>=0.1.0",
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
[project.urls]
|
|
16
|
+
Homepage = "https://github.com/wispsy/parsethon"
|
|
17
|
+
Repository = "https://github.com/wispsy/parsethon"
|
|
18
|
+
Issues = "https://github.com/wispsy/parsethon/issues"
|
|
19
|
+
Documentation = "https://github.com/wispsy/parsethon/tree/main/docs"
|
|
20
|
+
|
|
21
|
+
[project.scripts]
|
|
22
|
+
parsethon = "parsethon.cli:main"
|
|
23
|
+
|
|
24
|
+
[build-system]
|
|
25
|
+
requires = ["uv_build>=0.12.19,<0.14.0"]
|
|
26
|
+
build-backend = "uv_build"
|
|
27
|
+
|
|
28
|
+
[dependency-groups]
|
|
29
|
+
dev = [
|
|
30
|
+
"hypothesis>=6.170.14",
|
|
31
|
+
"pytest>=9.2.0",
|
|
32
|
+
]
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Parsethon: Extensible Python lexical parser, tokenizer, and Tongue translation system."""
|
|
2
|
+
|
|
3
|
+
from parsethon.cli import app as cli_app, main
|
|
4
|
+
from parsethon.importer import (
|
|
5
|
+
disable,
|
|
6
|
+
enable,
|
|
7
|
+
install_codec,
|
|
8
|
+
install_importer,
|
|
9
|
+
uninstall_codec,
|
|
10
|
+
uninstall_importer,
|
|
11
|
+
)
|
|
12
|
+
from parsethon.lexicon import (
|
|
13
|
+
LexicalToken,
|
|
14
|
+
SourcePosition,
|
|
15
|
+
SourceSpan,
|
|
16
|
+
TokenCategory,
|
|
17
|
+
TokenSubelement,
|
|
18
|
+
classify_identifier_or_keyword,
|
|
19
|
+
classify_symbol,
|
|
20
|
+
)
|
|
21
|
+
from parsethon.js_syntax import (
|
|
22
|
+
desugar_js_syntax,
|
|
23
|
+
has_js_style_syntax,
|
|
24
|
+
to_js_syntax,
|
|
25
|
+
)
|
|
26
|
+
from parsethon.runtime import (
|
|
27
|
+
ParsethonRuntime,
|
|
28
|
+
TongueExecutionError,
|
|
29
|
+
execute,
|
|
30
|
+
execute_file,
|
|
31
|
+
runtime,
|
|
32
|
+
)
|
|
33
|
+
from parsethon.sourcemap import SourceMap, SourceMapEntry
|
|
34
|
+
from parsethon.tokenizer import ParsethonTokenizer, scan_string_literal
|
|
35
|
+
from parsethon.tongue import (
|
|
36
|
+
Tongue,
|
|
37
|
+
TongueRegistry,
|
|
38
|
+
TongueValidationError,
|
|
39
|
+
registry,
|
|
40
|
+
)
|
|
41
|
+
from parsethon.translator import (
|
|
42
|
+
ParsethonTranslator,
|
|
43
|
+
translate_from_python,
|
|
44
|
+
translate_to_python,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
__version__ = "0.3.0"
|
|
48
|
+
__all__ = [
|
|
49
|
+
"LexicalToken",
|
|
50
|
+
"SourcePosition",
|
|
51
|
+
"SourceSpan",
|
|
52
|
+
"TokenCategory",
|
|
53
|
+
"TokenSubelement",
|
|
54
|
+
"classify_identifier_or_keyword",
|
|
55
|
+
"classify_symbol",
|
|
56
|
+
"desugar_js_syntax",
|
|
57
|
+
"has_js_style_syntax",
|
|
58
|
+
"to_js_syntax",
|
|
59
|
+
"ParsethonTokenizer",
|
|
60
|
+
"SourceMap",
|
|
61
|
+
"SourceMapEntry",
|
|
62
|
+
"Tongue",
|
|
63
|
+
"TongueRegistry",
|
|
64
|
+
"TongueValidationError",
|
|
65
|
+
"registry",
|
|
66
|
+
"ParsethonTranslator",
|
|
67
|
+
"translate_to_python",
|
|
68
|
+
"translate_from_python",
|
|
69
|
+
"ParsethonRuntime",
|
|
70
|
+
"TongueExecutionError",
|
|
71
|
+
"execute",
|
|
72
|
+
"execute_file",
|
|
73
|
+
"runtime",
|
|
74
|
+
"install_importer",
|
|
75
|
+
"uninstall_importer",
|
|
76
|
+
"install_codec",
|
|
77
|
+
"uninstall_codec",
|
|
78
|
+
"enable",
|
|
79
|
+
"disable",
|
|
80
|
+
"cli_app",
|
|
81
|
+
"main",
|
|
82
|
+
]
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Command-line interface for Parsethon powered by expose-interface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ast
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import List, Optional
|
|
10
|
+
|
|
11
|
+
import expose_interface as ei
|
|
12
|
+
from parsethon.lexicon import TokenCategory
|
|
13
|
+
from parsethon.runtime import execute_file
|
|
14
|
+
from parsethon.tokenizer import ParsethonTokenizer
|
|
15
|
+
from parsethon.tongue import registry
|
|
16
|
+
from parsethon.translator import translate_from_python, translate_to_python
|
|
17
|
+
|
|
18
|
+
app = ei.Interface("parsethon", description="Parsethon: Extensible Python Lexical Parser and Tongue System")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@app.command
|
|
22
|
+
def run(path: str, tongue: str = "") -> None:
|
|
23
|
+
"""Execute a Tongue source file under CPython."""
|
|
24
|
+
try:
|
|
25
|
+
execute_file(path, tongue=tongue if tongue else None)
|
|
26
|
+
except Exception as e:
|
|
27
|
+
sys.stderr.write(f"{e}\n")
|
|
28
|
+
sys.exit(1)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@app.command
|
|
32
|
+
def translate(
|
|
33
|
+
path: str,
|
|
34
|
+
tongue: str = "",
|
|
35
|
+
to_python: bool = True,
|
|
36
|
+
output: str = "",
|
|
37
|
+
) -> None:
|
|
38
|
+
"""Translate source code between Tongue and canonical Python syntax."""
|
|
39
|
+
file_path = Path(path)
|
|
40
|
+
if not file_path.exists():
|
|
41
|
+
sys.stderr.write(f"File not found: {path}\n")
|
|
42
|
+
sys.exit(1)
|
|
43
|
+
|
|
44
|
+
source = file_path.read_text(encoding="utf-8")
|
|
45
|
+
t_name = tongue if tongue else None
|
|
46
|
+
|
|
47
|
+
# Infer tongue if to_python
|
|
48
|
+
if not t_name:
|
|
49
|
+
from parsethon.runtime import detect_tongue_from_filename, detect_tongue_from_source
|
|
50
|
+
t_name = detect_tongue_from_source(source) or detect_tongue_from_filename(path) or "es"
|
|
51
|
+
|
|
52
|
+
if to_python:
|
|
53
|
+
result = translate_to_python(source, t_name)
|
|
54
|
+
else:
|
|
55
|
+
result = translate_from_python(source, t_name)
|
|
56
|
+
|
|
57
|
+
if output:
|
|
58
|
+
Path(output).write_text(result, encoding="utf-8")
|
|
59
|
+
print(f"Wrote translated code to {output}")
|
|
60
|
+
else:
|
|
61
|
+
sys.stdout.write(result)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@app.command
|
|
65
|
+
def tokenize(path: str, verbose: bool = False) -> None:
|
|
66
|
+
"""Tokenize a source file and display lexical classification and subelements."""
|
|
67
|
+
file_path = Path(path)
|
|
68
|
+
if not file_path.exists():
|
|
69
|
+
sys.stderr.write(f"File not found: {path}\n")
|
|
70
|
+
sys.exit(1)
|
|
71
|
+
|
|
72
|
+
source = file_path.read_text(encoding="utf-8")
|
|
73
|
+
tokenizer = ParsethonTokenizer(source)
|
|
74
|
+
tokens = tokenizer.tokenize()
|
|
75
|
+
|
|
76
|
+
print(f"Total tokens emitted: {len(tokens)}")
|
|
77
|
+
print(f"{'SPAN':<16} {'CATEGORY':<14} {'SUBELEMENT':<20} {'VALUE'}")
|
|
78
|
+
print("-" * 65)
|
|
79
|
+
|
|
80
|
+
for tok in tokens:
|
|
81
|
+
if not verbose and tok.category in (TokenCategory.WHITESPACE, TokenCategory.NEWLINE):
|
|
82
|
+
continue
|
|
83
|
+
span_str = f"{tok.span.start.line}:{tok.span.start.column}-{tok.span.end.line}:{tok.span.end.column}"
|
|
84
|
+
repr_val = repr(tok.value)
|
|
85
|
+
if len(repr_val) > 25:
|
|
86
|
+
repr_val = repr_val[:22] + "..."
|
|
87
|
+
print(f"{span_str:<16} {tok.category.value:<14} {tok.subelement.value:<20} {repr_val}")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@app.command
|
|
91
|
+
def list_tongues() -> None:
|
|
92
|
+
"""Enumerate all available registered Tongues."""
|
|
93
|
+
print("Available Tongues:")
|
|
94
|
+
for name in registry.list_tongues():
|
|
95
|
+
t = registry.get(name)
|
|
96
|
+
desc = t.description or "No description"
|
|
97
|
+
sample_kw = list(t.keywords.items())[:3]
|
|
98
|
+
samples = ", ".join(f"{k}->{v}" for k, v in sample_kw)
|
|
99
|
+
print(f" * {name:<10} (v{t.version}): {desc} [{samples}...]")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@app.command
|
|
103
|
+
def check(path: str, tongue: str = "") -> None:
|
|
104
|
+
"""Validate syntax of a Tongue file without executing."""
|
|
105
|
+
file_path = Path(path)
|
|
106
|
+
if not file_path.exists():
|
|
107
|
+
sys.stderr.write(f"File not found: {path}\n")
|
|
108
|
+
sys.exit(1)
|
|
109
|
+
|
|
110
|
+
source = file_path.read_text(encoding="utf-8")
|
|
111
|
+
t_name = tongue if tongue else None
|
|
112
|
+
if not t_name:
|
|
113
|
+
from parsethon.runtime import detect_tongue_from_filename, detect_tongue_from_source
|
|
114
|
+
t_name = detect_tongue_from_source(source) or detect_tongue_from_filename(path) or "es"
|
|
115
|
+
|
|
116
|
+
try:
|
|
117
|
+
py_source = translate_to_python(source, t_name)
|
|
118
|
+
ast.parse(py_source, filename=path)
|
|
119
|
+
print(f"Syntax valid: {path} (Tongue: {t_name})")
|
|
120
|
+
except SyntaxError as se:
|
|
121
|
+
sys.stderr.write(f"Syntax error at line {se.lineno}: {se.msg}\n")
|
|
122
|
+
sys.exit(1)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def main(argv: Optional[List[str]] = None) -> None:
|
|
126
|
+
"""Main CLI entry point."""
|
|
127
|
+
app.main(argv, prog_name="parsethon")
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
if __name__ == "__main__":
|
|
131
|
+
main()
|