ssc_codegen 0.37.2__tar.gz → 0.39.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/.gitignore +6 -4
  2. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/PKG-INFO +26 -15
  3. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/README.md +27 -16
  4. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/pyproject.toml +15 -1
  5. ssc_codegen-0.39.0/ssc_codegen/__init__.py +51 -0
  6. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/ast/__init__.py +32 -6
  7. ssc_codegen-0.39.0/ssc_codegen/ast/array.py +124 -0
  8. ssc_codegen-0.39.0/ssc_codegen/ast/base.py +85 -0
  9. ssc_codegen-0.39.0/ssc_codegen/ast/cast.py +138 -0
  10. ssc_codegen-0.39.0/ssc_codegen/ast/control.py +165 -0
  11. ssc_codegen-0.39.0/ssc_codegen/ast/extension.py +168 -0
  12. ssc_codegen-0.39.0/ssc_codegen/ast/extract.py +86 -0
  13. ssc_codegen-0.39.0/ssc_codegen/ast/function.py +33 -0
  14. ssc_codegen-0.39.0/ssc_codegen/ast/jsondef.py +170 -0
  15. ssc_codegen-0.39.0/ssc_codegen/ast/module.py +128 -0
  16. ssc_codegen-0.39.0/ssc_codegen/ast/predicate_containers.py +82 -0
  17. ssc_codegen-0.39.0/ssc_codegen/ast/predicate_ops.py +387 -0
  18. ssc_codegen-0.39.0/ssc_codegen/ast/regex.py +108 -0
  19. ssc_codegen-0.39.0/ssc_codegen/ast/rest.py +167 -0
  20. ssc_codegen-0.39.0/ssc_codegen/ast/selectors.py +198 -0
  21. ssc_codegen-0.39.0/ssc_codegen/ast/string.py +349 -0
  22. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/ast/struct.py +382 -147
  23. ssc_codegen-0.39.0/ssc_codegen/ast/typedef.py +86 -0
  24. ssc_codegen-0.39.0/ssc_codegen/ast/types.py +91 -0
  25. ssc_codegen-0.39.0/ssc_codegen/core/contexts.py +306 -0
  26. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/core/expressions.py +206 -97
  27. ssc_codegen-0.39.0/ssc_codegen/core/extensions.py +180 -0
  28. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/core/format.py +43 -3
  29. ssc_codegen-0.39.0/ssc_codegen/core/imports.py +577 -0
  30. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/core/linter.py +1345 -240
  31. ssc_codegen-0.39.0/ssc_codegen/core/module_handler.py +228 -0
  32. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/core/predicates.py +32 -0
  33. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/core/reader.py +51 -8
  34. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/core/rest_artifacts.py +18 -12
  35. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/core/struct_parser.py +296 -11
  36. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/core/type_checking.py +56 -8
  37. ssc_codegen-0.39.0/ssc_codegen/exceptions.py +44 -0
  38. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/explore.py +117 -19
  39. ssc_codegen-0.39.0/ssc_codegen/generation/builder.py +162 -0
  40. ssc_codegen-0.39.0/ssc_codegen/generation/runtime.py +502 -0
  41. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/health.py +523 -444
  42. ssc_codegen-0.39.0/ssc_codegen/json_to_kdl.py +175 -0
  43. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/main.py +224 -10
  44. ssc_codegen-0.39.0/ssc_codegen/symbols.py +204 -0
  45. ssc_codegen-0.39.0/ssc_codegen/targets/golang/http_libs/base.py +31 -0
  46. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/golang/http_libs/nethttp.py +4 -2
  47. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/golang/visitor.py +441 -42
  48. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/javascript/http_libs/axios.py +4 -1
  49. ssc_codegen-0.39.0/ssc_codegen/targets/javascript/http_libs/base.py +25 -0
  50. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/javascript/http_libs/fetch.py +4 -1
  51. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/javascript/rest.py +111 -3
  52. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/javascript/visitor.py +420 -39
  53. ssc_codegen-0.39.0/ssc_codegen/targets/profile.py +52 -0
  54. ssc_codegen-0.39.0/ssc_codegen/targets/python/html_libs/base.py +381 -0
  55. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/python/html_libs/bs4.py +16 -1
  56. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/python/html_libs/lxml.py +16 -1
  57. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/python/html_libs/parsel.py +15 -1
  58. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/python/html_libs/slax.py +16 -1
  59. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/python/http_libs/aiohttp.py +11 -1
  60. ssc_codegen-0.39.0/ssc_codegen/targets/python/http_libs/base.py +92 -0
  61. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/python/http_libs/httpx.py +10 -1
  62. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/python/http_libs/requests.py +9 -3
  63. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/python/rest.py +118 -4
  64. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/python/visitor.py +397 -50
  65. ssc_codegen-0.39.0/ssc_codegen/targets/resolver.py +227 -0
  66. ssc_codegen-0.39.0/ssc_codegen/targets/rust/__init__.py +7 -0
  67. ssc_codegen-0.39.0/ssc_codegen/targets/rust/regex.py +64 -0
  68. ssc_codegen-0.39.0/ssc_codegen/targets/rust/rest.py +799 -0
  69. ssc_codegen-0.39.0/ssc_codegen/targets/rust/runtime.py +279 -0
  70. ssc_codegen-0.39.0/ssc_codegen/targets/rust/visitor.py +1927 -0
  71. ssc_codegen-0.39.0/ssc_codegen/targets/spec.py +61 -0
  72. ssc_codegen-0.39.0/ssc_codegen/traversal/context.py +100 -0
  73. ssc_codegen-0.39.0/ssc_codegen/traversal/utils.py +396 -0
  74. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/traversal/walker.py +71 -17
  75. ssc_codegen-0.37.2/ssc_codegen/__init__.py +0 -1
  76. ssc_codegen-0.37.2/ssc_codegen/ast/array.py +0 -76
  77. ssc_codegen-0.37.2/ssc_codegen/ast/base.py +0 -52
  78. ssc_codegen-0.37.2/ssc_codegen/ast/cast.py +0 -92
  79. ssc_codegen-0.37.2/ssc_codegen/ast/control.py +0 -94
  80. ssc_codegen-0.37.2/ssc_codegen/ast/extract.py +0 -67
  81. ssc_codegen-0.37.2/ssc_codegen/ast/function.py +0 -28
  82. ssc_codegen-0.37.2/ssc_codegen/ast/helpers.py +0 -4
  83. ssc_codegen-0.37.2/ssc_codegen/ast/jsondef.py +0 -31
  84. ssc_codegen-0.37.2/ssc_codegen/ast/module.py +0 -114
  85. ssc_codegen-0.37.2/ssc_codegen/ast/predicate_containers.py +0 -65
  86. ssc_codegen-0.37.2/ssc_codegen/ast/predicate_ops.py +0 -266
  87. ssc_codegen-0.37.2/ssc_codegen/ast/regex.py +0 -72
  88. ssc_codegen-0.37.2/ssc_codegen/ast/rest.py +0 -82
  89. ssc_codegen-0.37.2/ssc_codegen/ast/selectors.py +0 -97
  90. ssc_codegen-0.37.2/ssc_codegen/ast/string.py +0 -221
  91. ssc_codegen-0.37.2/ssc_codegen/ast/typedef.py +0 -33
  92. ssc_codegen-0.37.2/ssc_codegen/ast/types.py +0 -52
  93. ssc_codegen-0.37.2/ssc_codegen/core/contexts.py +0 -165
  94. ssc_codegen-0.37.2/ssc_codegen/core/module_handler.py +0 -281
  95. ssc_codegen-0.37.2/ssc_codegen/exceptions.py +0 -6
  96. ssc_codegen-0.37.2/ssc_codegen/generation/builder.py +0 -73
  97. ssc_codegen-0.37.2/ssc_codegen/generation/runtime.py +0 -161
  98. ssc_codegen-0.37.2/ssc_codegen/targets/golang/http_libs/base.py +0 -27
  99. ssc_codegen-0.37.2/ssc_codegen/targets/javascript/http_libs/base.py +0 -19
  100. ssc_codegen-0.37.2/ssc_codegen/targets/profile.py +0 -18
  101. ssc_codegen-0.37.2/ssc_codegen/targets/python/html_libs/base.py +0 -146
  102. ssc_codegen-0.37.2/ssc_codegen/targets/python/http_libs/base.py +0 -82
  103. ssc_codegen-0.37.2/ssc_codegen/targets/resolver.py +0 -121
  104. ssc_codegen-0.37.2/ssc_codegen/targets/spec.py +0 -15
  105. ssc_codegen-0.37.2/ssc_codegen/traversal/context.py +0 -47
  106. ssc_codegen-0.37.2/ssc_codegen/traversal/utils.py +0 -122
  107. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/LICENSE +0 -0
  108. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/_logging.py +0 -0
  109. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/core/__init__.py +0 -0
  110. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/generation/__init__.py +0 -0
  111. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/naming.py +0 -0
  112. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/parsers/__init__.py +0 -0
  113. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/parsers/curl.py +0 -0
  114. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/parsers/http.py +0 -0
  115. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/regex_utils.py +0 -0
  116. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/request_spec.py +0 -0
  117. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/__init__.py +0 -0
  118. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/golang/__init__.py +0 -0
  119. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/golang/http_libs/__init__.py +0 -0
  120. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/golang/literals.py +0 -0
  121. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/golang/regex.py +0 -0
  122. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/golang/rest.py +0 -0
  123. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/golang/runtime.py +0 -0
  124. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/javascript/__init__.py +0 -0
  125. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/javascript/http_libs/__init__.py +0 -0
  126. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/python/__init__.py +0 -0
  127. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/python/html_libs/__init__.py +0 -0
  128. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/targets/python/http_libs/__init__.py +0 -0
  129. {ssc_codegen-0.37.2 → ssc_codegen-0.39.0}/ssc_codegen/traversal/__init__.py +0 -0
@@ -133,9 +133,10 @@ dmypy.json
133
133
  # Pyre type checker
134
134
  .pyre/
135
135
 
136
- # project
137
- test_schemas/
138
- .idea/.gitignore
136
+ # project
137
+ test_schemas/
138
+ graphify-out/
139
+ .idea/.gitignore
139
140
  .idea/.name
140
141
  .idea/misc.xml
141
142
  .idea/modules.xml
@@ -156,4 +157,5 @@ go.work.sum
156
157
  /tests/go/scratch/
157
158
 
158
159
  # slopmachines
159
- .claude/*
160
+ .claude/*
161
+ .scratch/*
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: ssc_codegen
3
- Version: 0.37.2
3
+ Version: 0.39.0
4
4
  Summary: Python-dsl code converter to html parser for web scraping
5
5
  Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
6
6
  Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
@@ -42,7 +42,8 @@ Code generator for web scraping parsers. Describe HTML extraction rules in a dec
42
42
 
43
43
  - Declarative DSL based on KDL 2.0 syntax
44
44
  - Static type checking and linting before code generation
45
- - Multiple output targets: Python (bs4, lxml, parsel, selectolax), JavaScript (DOM API)
45
+ - Multiple output targets: Python (bs4, lxml, parsel, selectolax), JavaScript (DOM API), Go (goquery)
46
+ - Typed user extensions with lazy target imports and runtime helpers
46
47
  - Struct types: `item`, `list`, `dict`, `table`, `flat`
47
48
  - LLM-friendly: system prompt + linter loop for AI-assisted schema generation
48
49
 
@@ -154,37 +155,47 @@ curl https://books.toscrape.com/ | ssc-gen health examples/booksToScrape.kdl:Mai
154
155
  LLM agents can generate and validate `.kdl` schemas automatically using the
155
156
  linter feedback loop.
156
157
 
157
- ### In chats (ChatGPT, Claude, etc.)
158
-
159
- Use [SYSTEM_PROMPT.md](SYSTEM_PROMPT.md) as system prompt. After generation,
160
- run `ssc-gen check -f json` and send errors back to the LLM for correction.
161
-
162
158
  ### In AI-powered IDEs (Claude Code, Cursor, opencode, etc.)
163
159
 
164
- Use the skills under [.agents/skills/](.agents/skills/):
160
+ Project works through `agents + skills`. Use skills under [.agents/skills/](.agents/skills/):
165
161
 
166
162
  - **`sscgen-dsl`** — HTML scraping schemas (`css`, `text`, `(item)struct`,
167
163
  `(list)struct`, `(table)struct`, predicates, `jsonify`).
168
164
  - **`sscgen-rest`** — REST/JSON API clients (`(rest)struct`, `@request`,
169
165
  `@error`, typed placeholders).
170
- - **`sscgen-openapi`** — convert OpenAPI/Swagger specs into `.kdl`.
171
166
 
172
167
  ## Development
173
168
 
174
169
  ```bash
175
170
  uv sync # install dependencies
176
171
  uv build --wheel # build wheel
177
- uv run pytest # run tests
178
172
  uv run ruff check ssc_codegen/
173
+ uv run mypy ssc_codegen/
179
174
  ```
180
175
 
181
- ### Test dependencies
182
-
183
- Python tests require only `uv sync`. JS integration tests additionally need:
176
+ ### Running tests
184
177
 
185
178
  ```bash
186
- npm install # installs jsdom (dev dependency in package.json)
179
+ # Run full test suite
180
+ uv run pytest
181
+
182
+ # Run full test suite in parallel (faster, recommended)
183
+ uv run pytest -n auto
184
+
185
+ # Fast test run: skip external toolchains (Rust cargo, Go, Node.js)
186
+ uv run pytest -m "not toolchain"
187
+
188
+ # Run only toolchain tests (Go, Rust, JavaScript integration)
189
+ uv run pytest -m toolchain
187
190
  ```
188
191
 
189
- Node.js must be installed and available as `node` in PATH. JS tests are automatically skipped if Node.js is not found.
192
+ ### Test dependencies
193
+
194
+ Python and parser tests require only `uv sync`.
195
+
196
+ Integration smoke tests for external compilers are skipped automatically if their toolchains are missing:
197
+
198
+ - **JavaScript**: Requires Node.js in PATH and `npm install` (`jsdom` in `package.json`).
199
+ - **Go**: Requires `go` in PATH.
200
+ - **Rust**: Requires `cargo` in PATH.
190
201
  ```
@@ -10,7 +10,8 @@ Code generator for web scraping parsers. Describe HTML extraction rules in a dec
10
10
 
11
11
  - Declarative DSL based on KDL 2.0 syntax
12
12
  - Static type checking and linting before code generation
13
- - Multiple output targets: Python (bs4, lxml, parsel, selectolax), JavaScript (DOM API)
13
+ - Multiple output targets: Python (bs4, lxml, parsel, selectolax), JavaScript (DOM API), Go (goquery)
14
+ - Typed user extensions with lazy target imports and runtime helpers
14
15
  - Struct types: `item`, `list`, `dict`, `table`, `flat`
15
16
  - LLM-friendly: system prompt + linter loop for AI-assisted schema generation
16
17
 
@@ -122,37 +123,47 @@ curl https://books.toscrape.com/ | ssc-gen health examples/booksToScrape.kdl:Mai
122
123
  LLM agents can generate and validate `.kdl` schemas automatically using the
123
124
  linter feedback loop.
124
125
 
125
- ### In chats (ChatGPT, Claude, etc.)
126
-
127
- Use [SYSTEM_PROMPT.md](SYSTEM_PROMPT.md) as system prompt. After generation,
128
- run `ssc-gen check -f json` and send errors back to the LLM for correction.
129
-
130
- ### In AI-powered IDEs (Claude Code, Cursor, opencode, etc.)
131
-
132
- Use the skills under [.agents/skills/](.agents/skills/):
126
+ ### In AI-powered IDEs (Claude Code, Cursor, opencode, etc.)
127
+
128
+ Project works through `agents + skills`. Use skills under [.agents/skills/](.agents/skills/):
133
129
 
134
130
  - **`sscgen-dsl`** — HTML scraping schemas (`css`, `text`, `(item)struct`,
135
131
  `(list)struct`, `(table)struct`, predicates, `jsonify`).
136
132
  - **`sscgen-rest`** — REST/JSON API clients (`(rest)struct`, `@request`,
137
133
  `@error`, typed placeholders).
138
- - **`sscgen-openapi`** — convert OpenAPI/Swagger specs into `.kdl`.
139
134
 
140
135
  ## Development
141
136
 
142
137
  ```bash
143
138
  uv sync # install dependencies
144
139
  uv build --wheel # build wheel
145
- uv run pytest # run tests
146
140
  uv run ruff check ssc_codegen/
141
+ uv run mypy ssc_codegen/
147
142
  ```
148
143
 
149
- ### Test dependencies
150
-
151
- Python tests require only `uv sync`. JS integration tests additionally need:
144
+ ### Running tests
152
145
 
153
146
  ```bash
154
- npm install # installs jsdom (dev dependency in package.json)
147
+ # Run full test suite
148
+ uv run pytest
149
+
150
+ # Run full test suite in parallel (faster, recommended)
151
+ uv run pytest -n auto
152
+
153
+ # Fast test run: skip external toolchains (Rust cargo, Go, Node.js)
154
+ uv run pytest -m "not toolchain"
155
+
156
+ # Run only toolchain tests (Go, Rust, JavaScript integration)
157
+ uv run pytest -m toolchain
155
158
  ```
156
159
 
157
- Node.js must be installed and available as `node` in PATH. JS tests are automatically skipped if Node.js is not found.
160
+ ### Test dependencies
161
+
162
+ Python and parser tests require only `uv sync`.
163
+
164
+ Integration smoke tests for external compilers are skipped automatically if their toolchains are missing:
165
+
166
+ - **JavaScript**: Requires Node.js in PATH and `npm install` (`jsdom` in `package.json`).
167
+ - **Go**: Requires `go` in PATH.
168
+ - **Rust**: Requires `cargo` in PATH.
158
169
  ```
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "ssc_codegen"
3
- version = "0.37.2"
3
+ version = "0.39.0"
4
4
  description = "Python-dsl code converter to html parser for web scraping "
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.10"
@@ -64,11 +64,18 @@ dev = [
64
64
  "mypy>=1.14.1",
65
65
  "parsel>=1.9.1",
66
66
  "pytest>=8.3.4",
67
+ "pytest-xdist>=3.6.0",
67
68
  "respx>=0.23.1",
68
69
  "responses>=0.25",
69
70
  "ruff>=0.9.3",
70
71
  "selectolax>=0.3.27",
71
72
  ]
73
+ docs = [
74
+ "mkdocs>=1.6.0",
75
+ "mkdocs-material>=9.5.0",
76
+ "mkdocstrings[python]>=0.27.0",
77
+ "mkdocs-click>=0.8.1",
78
+ ]
72
79
 
73
80
  [tool.ruff]
74
81
  target-version = "py310"
@@ -108,3 +115,10 @@ omit = [
108
115
  "*/scripts/*",
109
116
  "*/tests/*"
110
117
  ]
118
+
119
+ [tool.pytest.ini_options]
120
+ testpaths = ["tests"]
121
+ markers = [
122
+ "toolchain: Tests requiring external toolchains (Rust cargo, Go compiler, Node.js)",
123
+ ]
124
+
@@ -0,0 +1,51 @@
1
+ """KDL-based schema code generator for web scraping parsers.
2
+
3
+ This package provides tools for parsing KDL v2.1 schemas into an intermediate
4
+ AST, validating selectors and pipelines, and compiling ASTs into native parser
5
+ code for Python (bs4, lxml, parsel, selectolax), JavaScript (DOM API), and Go
6
+ (goquery + net/http).
7
+
8
+ Key API symbols:
9
+ parse_module: Parse KDL schema source into a Module AST and diagnostics.
10
+ format_diagnostics: Format diagnostics into text or JSON output.
11
+ resolve: Resolve a TargetSpec into a concrete TargetProfile.
12
+ TargetSpec: User-supplied target backend specification.
13
+ TargetProfile: Validated target backend profile with converter factory.
14
+ ResolutionError: Raised when target resolution or configuration fails.
15
+ ParseError: Raised on DSL syntax or schema parsing errors.
16
+ BuildTimeError: Raised on type mismatches or unresolved references at build time.
17
+ check_struct_health: Verify selector match rates against real HTML.
18
+ HealthResult: Container holding selector health statistics.
19
+ run_scout: Perform regex and CSS reconnaissance on raw HTML.
20
+ run_discover: High-level page structure analysis for selector discovery.
21
+ ScoutResult: Detailed results from an HTML scout run.
22
+ DiscoverResult: Aggregated structural overview of an HTML document.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ from ssc_codegen.core.format import format_diagnostics
28
+ from ssc_codegen.core.reader import parse_module
29
+ from ssc_codegen.exceptions import BuildTimeError, ParseError
30
+ from ssc_codegen.explore import DiscoverResult, ScoutResult, run_discover, run_scout
31
+ from ssc_codegen.health import HealthResult, check_struct_health
32
+ from ssc_codegen.targets.profile import TargetProfile
33
+ from ssc_codegen.targets.resolver import ResolutionError, resolve
34
+ from ssc_codegen.targets.spec import TargetSpec
35
+
36
+ __all__ = [
37
+ "BuildTimeError",
38
+ "DiscoverResult",
39
+ "HealthResult",
40
+ "ParseError",
41
+ "ResolutionError",
42
+ "ScoutResult",
43
+ "TargetProfile",
44
+ "TargetSpec",
45
+ "check_struct_health",
46
+ "format_diagnostics",
47
+ "parse_module",
48
+ "resolve",
49
+ "run_discover",
50
+ "run_scout",
51
+ ]
@@ -1,9 +1,19 @@
1
1
  # mypy: ignore-errors
2
- """
3
- AST nodes for the KDL Schema DSL.
2
+ """Intermediate representation (AST) node definitions for the KDL Schema DSL.
3
+
4
+ Provides strongly-typed dataclass nodes representing all language constructs,
5
+ pipeline operations, predicates, REST endpoint definitions, and type models.
6
+
7
+ Examples:
8
+ ```python
9
+ from ssc_codegen.ast import CssSelect, Field, Module, Struct, Text
4
10
 
5
- Import everything from here:
6
- from kdl_ast import Module, Field, CssSelect, ...
11
+ module = Module(source_file="schema.kdl")
12
+ struct = Struct(name="Product")
13
+ field = Field(name="title", body=[CssSelect(queries=["h1"]), Text()])
14
+ struct.body.append(field)
15
+ module.body.append(struct)
16
+ ```
7
17
  """
8
18
 
9
19
  from .types import VariableType, StructType, TypeInfo, VT
@@ -20,7 +30,16 @@ from .module import (
20
30
  Utilities,
21
31
  )
22
32
 
23
- from .function import FunctionDef
33
+ from .function import FunctionDef
34
+
35
+ from .extension import (
36
+ ExtensionCall,
37
+ ExtensionDef,
38
+ ExtensionHelper,
39
+ ExtensionImport,
40
+ ExtensionTarget,
41
+ ExtensionType,
42
+ )
24
43
 
25
44
  from .typedef import TypeDef, TypeDefField
26
45
 
@@ -150,7 +169,14 @@ __all__ = [
150
169
  "Docstring",
151
170
  "Utilities",
152
171
  # function
153
- "FunctionDef",
172
+ "FunctionDef",
173
+ # extensions
174
+ "ExtensionCall",
175
+ "ExtensionDef",
176
+ "ExtensionHelper",
177
+ "ExtensionImport",
178
+ "ExtensionTarget",
179
+ "ExtensionType",
154
180
  # typedef
155
181
  "TypeDef",
156
182
  "TypeDefField",
@@ -0,0 +1,124 @@
1
+ """AST nodes for array and list collection operations.
2
+
3
+ This module defines nodes operating on lists of elements (`is_array=True`):
4
+ - `Index`: Extracts a single element at a 0-based offset (`first`, `last`, `index N`).
5
+ - `Slice`: Extracts a sublist slice `[start:end]`.
6
+ - `Len`: Returns the integer count of elements in a list.
7
+ - `Unique`: Deduplicates elements in a list with optional order preservation (`keep_order`).
8
+ """
9
+
10
+ from __future__ import annotations
11
+ from dataclasses import dataclass, field
12
+
13
+ from .base import Node
14
+ from .types import TypeInfo, VariableType
15
+
16
+ # Index, First, Last, Slice accept LIST_AUTO / return AUTO or LIST_AUTO.
17
+ # Concrete types are resolved by the builder from the cursor type via
18
+ # VariableType.scalar / VariableType.as_list helpers.
19
+
20
+
21
+ @dataclass
22
+ class Index(Node):
23
+ """Extracts a single element from a list by its index position.
24
+
25
+ Index is 0-based; negative indices count backwards from the end of the list (`-1` is the last item).
26
+ Target-specific backends without native negative or 0-based indexing must adapt expressions
27
+ accordingly during code generation (e.g. converting negative index `-k` to `len(arr) - k`,
28
+ or adding `+1` for 1-based index targets).
29
+
30
+ Attributes:
31
+ i: Zero-based integer index (negative values index backwards from the end).
32
+
33
+ Examples:
34
+ - KDL: `index 0`, `first`, `last`
35
+ - Python: `v1 = v[0]`, `v1 = v[-1]`
36
+ - JavaScript: `const v1 = v[0];`, `const v1 = v[v.length - 1];`
37
+ - Go: `v1 := v[0]`, `v1 := v[len(v)-1]`
38
+ """
39
+
40
+ i: int = 0
41
+ accept_type_info: TypeInfo = field(
42
+ default_factory=lambda: TypeInfo(base=VariableType.AUTO)
43
+ )
44
+ ret_type_info: TypeInfo = field(
45
+ default_factory=lambda: TypeInfo(base=VariableType.AUTO)
46
+ )
47
+
48
+
49
+ @dataclass
50
+ class Slice(Node):
51
+ """Extracts a sublist slice from start to end index `[start:end]`.
52
+
53
+ Indices are 0-based with half-open interval semantics `[start, end)`. Negative values
54
+ count backwards from the end of the collection. Backend codegen must adapt boundary
55
+ calculations to target language slice conventions.
56
+
57
+ Attributes:
58
+ start: Starting index of the slice (0-based, inclusive).
59
+ end: Ending index of the slice (0-based, exclusive).
60
+
61
+ Examples:
62
+ - KDL: `slice 0 5`
63
+ - Python: `v1 = v[0:5]`
64
+ - JavaScript: `const v1 = v.slice(0, 5);`
65
+ - Go: `v1 := v[0:5]`
66
+ """
67
+
68
+ start: int = 0
69
+ end: int = 0
70
+ accept_type_info: TypeInfo = field(
71
+ default_factory=lambda: TypeInfo(base=VariableType.AUTO)
72
+ )
73
+ ret_type_info: TypeInfo = field(
74
+ default_factory=lambda: TypeInfo(base=VariableType.AUTO, is_array=True)
75
+ )
76
+
77
+
78
+ @dataclass
79
+ class Len(Node):
80
+ """Calculates the number of elements in a list.
81
+
82
+ Examples:
83
+ - KDL: `len`
84
+ - Python: `v1 = len(v)`
85
+ - JavaScript: `const v1 = v.length;`
86
+ - Go: `v1 := len(v)`
87
+ """
88
+
89
+ accept_type_info: TypeInfo = field(
90
+ default_factory=lambda: TypeInfo(base=VariableType.AUTO)
91
+ )
92
+ ret_type_info: TypeInfo = field(
93
+ default_factory=lambda: TypeInfo(base=VariableType.INT)
94
+ )
95
+
96
+
97
+ @dataclass
98
+ class Unique(Node):
99
+ """Deduplicates string elements in a list.
100
+
101
+ When `keep_order=True`, the generator preserves the first-seen item order (e.g. `list(dict.fromkeys(v))`
102
+ in Python or helper in Go). When `keep_order=False`, backends may use faster unordered deduplication
103
+ (e.g. `list(set(v))` if target language optimizes set conversion).
104
+
105
+ Attributes:
106
+ keep_order: Flag indicating whether to preserve the original element order.
107
+
108
+ Examples:
109
+ - KDL: `unique`
110
+ - Python (keep_order=True): `v1 = list(dict.fromkeys(v))`
111
+ - Python (keep_order=False): `v1 = list(set(v))`
112
+ - JavaScript: `const v1 = Array.from(new Set(v));`
113
+ - Go: `v1 := stdUnique(v)`
114
+ """
115
+
116
+ keep_order: bool = False
117
+ accept_type_info: TypeInfo = field(
118
+ default_factory=lambda: TypeInfo(base=VariableType.STRING)
119
+ )
120
+ ret_type_info: TypeInfo = field(
121
+ default_factory=lambda: TypeInfo(
122
+ base=VariableType.STRING, is_array=True
123
+ )
124
+ )
@@ -0,0 +1,85 @@
1
+ from __future__ import annotations
2
+ import warnings
3
+ from dataclasses import dataclass, field
4
+
5
+ from kdlquery.parser import Span
6
+
7
+ from .types import TypeInfo, VariableType
8
+
9
+
10
+ @dataclass
11
+ class Node:
12
+ """Base abstract syntax tree node for intermediate representations.
13
+
14
+ All AST nodes inherit from this class to participate in traversal,
15
+ type propagation, and code generation passes.
16
+
17
+ Attributes:
18
+ ret_type_info: Resolved type metadata of the value returned by this node.
19
+ accept_type_info: Resolved type metadata of the input value accepted by this node.
20
+ parent: Reference to the parent AST node (excluded from representation and comparison).
21
+ body: Ordered list of child AST nodes contained within this node.
22
+ span: Optional source code location tracking (line, column) from the KDL source.
23
+ """
24
+
25
+ ret_type_info: TypeInfo = field(
26
+ default_factory=lambda: TypeInfo(base=VariableType.AUTO)
27
+ )
28
+ accept_type_info: TypeInfo = field(
29
+ default_factory=lambda: TypeInfo(base=VariableType.AUTO)
30
+ )
31
+ parent: Node | None = field(default=None, repr=False)
32
+ body: list[Node] = field(default_factory=list)
33
+ span: Span | None = field(default=None, repr=False, compare=False)
34
+
35
+ # ── backport properties (deprecated: prefer the TypeInfo fields) ───────────
36
+ @property
37
+ def ret(self) -> VariableType:
38
+ """Scalar return type shortcut.
39
+
40
+ Warning:
41
+ Deprecated: Use `ret_type_info.base` instead.
42
+
43
+ Returns:
44
+ The base `VariableType` of `ret_type_info`.
45
+ """
46
+ warnings.warn(
47
+ "Node.ret is deprecated; use Node.ret_type_info.base instead.",
48
+ DeprecationWarning,
49
+ stacklevel=2,
50
+ )
51
+ return self.ret_type_info.base
52
+
53
+ @property
54
+ def accept(self) -> VariableType:
55
+ """Scalar accepted input type shortcut.
56
+
57
+ Warning:
58
+ Deprecated: Use `accept_type_info.base` instead.
59
+
60
+ Returns:
61
+ The base `VariableType` of `accept_type_info`.
62
+ """
63
+ warnings.warn(
64
+ "Node.accept is deprecated; use Node.accept_type_info.base instead.",
65
+ DeprecationWarning,
66
+ stacklevel=2,
67
+ )
68
+ return self.accept_type_info.base
69
+
70
+ @property
71
+ def type_info(self) -> TypeInfo:
72
+ """TypeInfo alias shortcut.
73
+
74
+ Warning:
75
+ Deprecated: Use `ret_type_info` instead.
76
+
77
+ Returns:
78
+ The `ret_type_info` instance of this node.
79
+ """
80
+ warnings.warn(
81
+ "Node.type_info is deprecated; use Node.ret_type_info instead.",
82
+ DeprecationWarning,
83
+ stacklevel=2,
84
+ )
85
+ return self.ret_type_info
@@ -0,0 +1,138 @@
1
+ """AST nodes for scalar type casting, JSON deserialization, and struct nesting.
2
+
3
+ This module defines data conversion operations within field pipelines:
4
+ - `ToInt`: Casts string/numeric values to 64-bit integer (`int` / `int64`).
5
+ - `ToFloat`: Casts string/numeric values to 64-bit float (`float` / `float64`).
6
+ - `ToBool`: Converts input to a boolean truth value.
7
+ - `Jsonify`: Deserializes JSON string into a strongly-typed `JsonDef` model with key remapping.
8
+ - `Nested`: Instantiates another parser struct passing current DOM element and invokes `.parse()`.
9
+ """
10
+
11
+ from __future__ import annotations
12
+ from dataclasses import dataclass, field
13
+
14
+ from .base import Node
15
+ from .types import TypeInfo, VariableType
16
+
17
+
18
+ @dataclass
19
+ class ToInt(Node):
20
+ """Casts string or numeric values directly to integers.
21
+
22
+ Validation of the input format is not performed prior to casting; the operation directly
23
+ attempts conversion and is allowed to fail with a runtime error (e.g. `ValueError` in Python,
24
+ `NaN` / `parseInt` error in JS, `strconv.Atoi` error in Go) unless protected by a `Fallback` node.
25
+
26
+ Examples:
27
+ - KDL: `to-int`
28
+ - Python: `v1 = int(v)`
29
+ - JavaScript: `const v1 = parseInt(v, 10);`
30
+ - Go: `v1, _ := strconv.Atoi(v)`
31
+ """
32
+
33
+ accept_type_info: TypeInfo = field(
34
+ default_factory=lambda: TypeInfo(base=VariableType.STRING)
35
+ )
36
+ ret_type_info: TypeInfo = field(
37
+ default_factory=lambda: TypeInfo(base=VariableType.INT)
38
+ )
39
+
40
+
41
+ @dataclass
42
+ class ToFloat(Node):
43
+ """Casts string or numeric values directly to floating-point numbers.
44
+
45
+ Validation of the input format is not performed prior to casting; the operation directly
46
+ invokes standard target language float parsing and may raise a runtime error unless
47
+ handled by `Fallback`.
48
+
49
+ Examples:
50
+ - KDL: `to-float`
51
+ - Python: `v1 = float(v)`
52
+ - JavaScript: `const v1 = parseFloat(v);`
53
+ - Go: `v1, _ := strconv.ParseFloat(v, 64)`
54
+ """
55
+
56
+ accept_type_info: TypeInfo = field(
57
+ default_factory=lambda: TypeInfo(base=VariableType.STRING)
58
+ )
59
+ ret_type_info: TypeInfo = field(
60
+ default_factory=lambda: TypeInfo(base=VariableType.FLOAT)
61
+ )
62
+
63
+
64
+ @dataclass
65
+ class ToBool(Node):
66
+ """Casts input values to boolean truth values.
67
+
68
+ Scalar values are cast using standard truthiness rules of the target language.
69
+
70
+ Examples:
71
+ - KDL: `to-bool`
72
+ - Python: `v1 = bool(v)`
73
+ - JavaScript: `const v1 = Boolean(v);`
74
+ - Go: `v1 := bool(len(v) > 0)`
75
+ """
76
+
77
+ accept_type_info: TypeInfo = field(
78
+ default_factory=lambda: TypeInfo(base=VariableType.AUTO)
79
+ )
80
+ ret_type_info: TypeInfo = field(
81
+ default_factory=lambda: TypeInfo(base=VariableType.BOOL)
82
+ )
83
+
84
+
85
+ @dataclass
86
+ class Jsonify(Node):
87
+ """Deserializes a JSON string into a structured model per a `JsonDef` schema.
88
+
89
+ Parses JSON text into an object or array. If `path` is specified, extracts the nested
90
+ sub-tree first. If the referenced `JsonDef` contains field aliases (`from="..."`),
91
+ remapping logic is executed to project keys into canonical schema properties.
92
+ Allowed to fail with a JSON parsing error if input is malformed.
93
+
94
+ Attributes:
95
+ schema_name: Identifier of the target `JsonDef` mapping schema.
96
+ path: Optional dot-notation path to extract from the parsed JSON before schema projection.
97
+
98
+ Examples:
99
+ - KDL: `jsonify User`, `jsonify Article path="data.item"`
100
+ - Python: `v1 = json.loads(v)` + `ssc_remap_json_keys(v1, ...)`
101
+ - JavaScript: `const v1 = sscRemapJsonKeys(JSON.parse(v), ...);`
102
+ - Go: `var v1 UserJson; json.Unmarshal([]byte(v), &v1)`
103
+ """
104
+
105
+ schema_name: str = ""
106
+ path: str | None = None
107
+ accept_type_info: TypeInfo = field(
108
+ default_factory=lambda: TypeInfo(base=VariableType.STRING)
109
+ )
110
+ ret_type_info: TypeInfo = field(
111
+ default_factory=lambda: TypeInfo(base=VariableType.JSON)
112
+ )
113
+
114
+
115
+ @dataclass
116
+ class Nested(Node):
117
+ """Passes the current document to another struct parser and returns its result.
118
+
119
+ Instantiates the target parser struct passing the current DOM element as the root document,
120
+ then executes its `.parse()` entry point.
121
+
122
+ Attributes:
123
+ struct_name: Identifier of the target parser struct.
124
+
125
+ Examples:
126
+ - KDL: `nested AuthorInfo`
127
+ - Python: `v1 = AuthorInfo(v).parse()`
128
+ - JavaScript: `const v1 = new AuthorInfo(v).parse();`
129
+ - Go: `v1 := NewAuthorInfo(v).Parse()`
130
+ """
131
+
132
+ struct_name: str = ""
133
+ accept_type_info: TypeInfo = field(
134
+ default_factory=lambda: TypeInfo(base=VariableType.DOCUMENT)
135
+ )
136
+ ret_type_info: TypeInfo = field(
137
+ default_factory=lambda: TypeInfo(base=VariableType.NESTED)
138
+ )