ssc_codegen 0.9.9__tar.gz → 0.10.1__tar.gz

ssc_codegen-0.10.1/PKG-INFO

@@ -0,0 +1,253 @@
+Metadata-Version: 2.4
+Name: ssc_codegen
+Version: 0.10.1
+Summary: Python-dsl code converter to html parser for web scraping 
+Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
+Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
+Project-URL: Source, https://github.com/vypivshiy/selector_schema_codegen
+Project-URL: Examples, https://github.com/vypivshiy/selector_schema_codegen/examples
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Code Generators
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Requires-Dist: ast-grep-cli>=0.38.1
+Requires-Dist: bs4>=0.0.2
+Requires-Dist: click<8.2.0
+Requires-Dist: colorama>=0.4.6; sys_platform == 'win32'
+Requires-Dist: cssselect>=1.2.0
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: ichrome>=4.0.4
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: lxml>=5.3.0
+Requires-Dist: parsel>=1.10.0
+Requires-Dist: soupsieve>=2.6
+Requires-Dist: tinycss2>=1.4.0
+Requires-Dist: typer>=0.15.1
+Requires-Dist: typing-extensions; python_version < '3.11'
+Description-Content-Type: text/markdown
+
+# Selector Schema codegen
+
+## Introduction
+
+ssc-gen - a python-based DSL to describe parsers for html documents, which is translated into a standalone parsing module
+
+### For a better experience using this library, you should know:
+
+- HTML CSS selectors (CSS3 standard min), Xpath
+- regular expressions (PCRE)
+
+### Project solving next problems:
+
+- designed for SSR (server-side-render) html pages parsers, **NOT FOR REST-API, GRAPHQL ENDPOINTS**
+- decrease boilerplate code
+- generates independent modules from the project that can be reused.
+- generates docstring documentation and the signature of the parser output.
+- for a better IDE experience, generates a typedefs, type annotations (if the target programming language supports it).
+- support annotation and parsing of JSON-like strings from a document
+- AST API codegen for developing a converter for parsing
+
+## Support converters
+
+Current support converters
+
+| Language                                 | HTML parser lib + dependencies         | XPath | CSS3 | CSS4 | Generated annotations, types, structs | formatter dependency |
+| ---------------------------------------- | -------------------------------------- | ----- | ---- | ---- | ------------------------------------- | -------------------- |
+| Python (3.8+)                            | bs4, lxml                              | N     | Y    | Y    | TypedDict`1`, list, dict              | ruff                 |
+| ...                                      | parsel                                 | Y     | Y    | N    | ...                                   | ...                  |
+| ...                                      | selectolax (lexbor)                    | N     | Y    | N    | ...                                   | ...                  |
+| ...                                      | lxml                                   | Y     | Y    | N    | ...                                   | ...                  |
+| js (ES6)`2`                              | pure (firefox/chrome extension/nodejs) | Y     | Y    | Y    | Array, Map`3`                         | prettier             |
+| go (1.10+) **(UNSTABLE)**                | goquery, gjson (`4`)                   | N     | Y    | N    | struct(+json anchors), array, map     | gofmt                |
+| lua (5.2+), luajit(2+) **(UNSTABLE)**`5` | lua-htmlparser, lrexlib(opt), dkjson   | N     | Y    | N    | EmmyLua                               | LuaFormatter         |
+
+- **CSS3** means support next selectors:
+  - basic: (`tag`, `.class`, `#id`, `tag1,tag2`)
+  - combined: (`div p`, `ul > li`, `h2 +p`, `title ~head`)
+  - attribute: (`a[href]`, `input[type='text']`, `a[href*='...']`, ...)
+  - CSS3 pseudo classes: (`:nth-child(n)`, `:first-child`, `:last-child`)
+- **CSS4** means support next selectors:
+  - `:nth-of-type()`, `:where()`, `:is()`, `:not()` etc
+- `1`this annotation type was deliberately chosen as a compromise reasons:
+  Python has many ways of serialization: `namedtuple, dataclass, attrs, pydantic, msgspec, etc`
+  - TypedDict is like a build-in dict, but with IDE and linter hint support, and you can easily implement an adapter for the required structure.
+- `2`ES8 standart required if needed use PCRE `re.S | re.DOTALL` flag
+- `3`js exclude build-in serialization methods, used standard Array and Map types. Focus on the singanutur documentation!
+- `4`golang has not been tested much, there may be issues
+- **formatter dependency** - optional dependency for prettify and fix codestyle
+
+- `5`lua
+  - Experimental Research PoC, performance and stability are not guaranteed
+  - Priority on generation to pure lua without C-libs dependencies. using [mva/htmlparser](https://luarocks.org/modules/mva/htmlparser) and [dhkolf/dkjson](https://luarocks.org/modules/dhkolf/dkjson)
+  - Translates unsupported CSS3 selectors into the equivalent in the form of function calls:
+    - for example, `div +p` is equivalent to `CssExt.combine_plus(root:select("div"), "p")`
+  - Translates PCRE regex to [string pattern matching](https://www.lua.org/manual/5.4/manual.html#6.4.1) (with restrictions) for more information in [lua_re_compat.py ](ssc_codegen/converters/templates/lua_re_compat.py)
+
+### Limitations
+
+For maximum portability of the configuration to the target language:
+
+- If possible, use CSS selectors: they are guaranteed to be converted to XPATH
+- Unlike javascript, most html parse libs implement [CSS3 selectors standard](https://www.w3.org/TR/selectors-3/). They may not fully implement the functionality!
+  Check the html parser lib documentation aboud CSS selectors before implement code. Examples:
+  1. Several libs not support `+` operations (eg: [selectolax(modest)](https://github.com/rushter/selectolax), [dart.universal_html](https://pub.dev/packages/universal_html))
+  2. For research purpose, lua_htmlparser include converter for unsupported CSS3 query syntax
+2. HTML parser libs maybe not supports attribute selectors: `*=`, `~=`, `|=`, `^=`, `$=`
+3. Several libs not support pseudo classes (eg: standard [dart.html](https://dart.dev/libraries/dart-html) lib miss this feature).
+
+## Getting started
+
+ssc_gen required python 3.10 version or higher
+
+### Install
+
+pip:
+
+```shell
+pip install ssc_codegen
+```
+
+uv:
+
+```shell
+uv pip install ssc_codegen
+```
+
+## Example
+
+### Create a file `schema.py` with:
+
+```python
+from ssc_codegen import ItemSchema, D
+
+class HelloWorld(ItemSchema):
+    title = D().css('title').text()
+    a_hrefs = D().css_all('a').attr('href')
+```
+
+### try it in cli
+
+> [!note]
+> this tools developed for testing purposes, not for web-scraping tasks
+
+### eval from file
+
+Download any html file and pass as argument:
+
+```shell
+ssc-gen parse-from-file index.html -t schema.py:HelloWorld
+```
+
+Short options descriptions:
+
+- `-t --target` - config schema file and class from where to start the parser
+
+![out1](docs/assets/parse_from_file.gif)
+
+### send GET request to url and parse response
+
+```shell
+ssc-gen parse-from-url https://example.com -t schema.py:HelloWorld
+```
+
+![out1](docs/assets/parse_from_url.gif)
+
+### send request via Chromium browser (CDP protocol)
+
+```shell
+ssc-gen parse-from-chrome https://example.com -t schema.py:HelloWorld
+```
+
+> [!note]
+> if script cannot found chrome executable - provide it manually:
+
+```shell
+ssc-gen parse-from-chrome https://example.com -t schema.py:HelloWorld -sc /usr/bin/chromium
+```
+
+### Convert to code
+
+Convert to code for use in projects:
+
+> [!note]
+> for example, used js: it can be fast test in developer console
+
+```shell
+ssc-gen js schema.py -o .
+```
+
+Code output looks like this:
+
+```javascript
+// autogenerated by ssc-gen DO NOT_EDIT
+/***
+ *
+ * {
+ *     "title": "String",
+ *     "a_hrefs": "Array<String>"
+ * }*/
+class HelloWorld {
+  constructor(doc) {
+    if (typeof doc === "string") {
+      this._doc = new DOMParser().parseFromString(doc, "text/html");
+    } else if (doc instanceof Document || doc instanceof Element) {
+      this._doc = doc;
+    } else {
+      throw new Error("Invalid input: Expected a Document, Element, or string");
+    }
+  }
+
+  _parseTitle(v) {
+    let v0 = v.querySelector("title");
+    return typeof v0.textContent === "undefined"
+      ? v0.documentElement.textContent
+      : v0.textContent;
+  }
+
+  _parseAHrefs(v) {
+    let v0 = Array.from(v.querySelectorAll("a"));
+    return v0.map((e) => e.getAttribute("href"));
+  }
+
+  parse() {
+    return {
+      title: this._parseTitle(this._doc),
+      a_hrefs: this._parseAHrefs(this._doc),
+    };
+  }
+}
+```
+
+### Copy code output and past to developer console:
+
+Print output:
+
+```javascript
+alert(JSON.stringify(new HelloWorld(document).parse()));
+```
+
+![example](docs/assets/example.png)
+
+You can use any html source:
+
+- parse from html files
+- parse from http responses
+- parse from browsers: playwright, selenium, chrome-cdp, etc.
+- call curl in shell and parse STDIN
+- use in STDIN pipelines with third-party tools like [projectdiscovery/httpx](https://github.com/projectdiscovery/httpx)
+
+## See also
+
+- [Brief](docs/briefing.md) about css selectors and regular expressions.
+- [Quickstart](docs/quickstart.md) about css selectors and regular expressions.
+- [Tutorial](docs/tutorial.md) basic usage ssc-gen
+- [AST reference](docs/ast_reference.md) about generation code from AST

ssc_codegen-0.10.1/README.md

@@ -0,0 +1,216 @@
+# Selector Schema codegen
+
+## Introduction
+
+ssc-gen - a python-based DSL to describe parsers for html documents, which is translated into a standalone parsing module
+
+### For a better experience using this library, you should know:
+
+- HTML CSS selectors (CSS3 standard min), Xpath
+- regular expressions (PCRE)
+
+### Project solving next problems:
+
+- designed for SSR (server-side-render) html pages parsers, **NOT FOR REST-API, GRAPHQL ENDPOINTS**
+- decrease boilerplate code
+- generates independent modules from the project that can be reused.
+- generates docstring documentation and the signature of the parser output.
+- for a better IDE experience, generates a typedefs, type annotations (if the target programming language supports it).
+- support annotation and parsing of JSON-like strings from a document
+- AST API codegen for developing a converter for parsing
+
+## Support converters
+
+Current support converters
+
+| Language                                 | HTML parser lib + dependencies         | XPath | CSS3 | CSS4 | Generated annotations, types, structs | formatter dependency |
+| ---------------------------------------- | -------------------------------------- | ----- | ---- | ---- | ------------------------------------- | -------------------- |
+| Python (3.8+)                            | bs4, lxml                              | N     | Y    | Y    | TypedDict`1`, list, dict              | ruff                 |
+| ...                                      | parsel                                 | Y     | Y    | N    | ...                                   | ...                  |
+| ...                                      | selectolax (lexbor)                    | N     | Y    | N    | ...                                   | ...                  |
+| ...                                      | lxml                                   | Y     | Y    | N    | ...                                   | ...                  |
+| js (ES6)`2`                              | pure (firefox/chrome extension/nodejs) | Y     | Y    | Y    | Array, Map`3`                         | prettier             |
+| go (1.10+) **(UNSTABLE)**                | goquery, gjson (`4`)                   | N     | Y    | N    | struct(+json anchors), array, map     | gofmt                |
+| lua (5.2+), luajit(2+) **(UNSTABLE)**`5` | lua-htmlparser, lrexlib(opt), dkjson   | N     | Y    | N    | EmmyLua                               | LuaFormatter         |
+
+- **CSS3** means support next selectors:
+  - basic: (`tag`, `.class`, `#id`, `tag1,tag2`)
+  - combined: (`div p`, `ul > li`, `h2 +p`, `title ~head`)
+  - attribute: (`a[href]`, `input[type='text']`, `a[href*='...']`, ...)
+  - CSS3 pseudo classes: (`:nth-child(n)`, `:first-child`, `:last-child`)
+- **CSS4** means support next selectors:
+  - `:nth-of-type()`, `:where()`, `:is()`, `:not()` etc
+- `1`this annotation type was deliberately chosen as a compromise reasons:
+  Python has many ways of serialization: `namedtuple, dataclass, attrs, pydantic, msgspec, etc`
+  - TypedDict is like a build-in dict, but with IDE and linter hint support, and you can easily implement an adapter for the required structure.
+- `2`ES8 standart required if needed use PCRE `re.S | re.DOTALL` flag
+- `3`js exclude build-in serialization methods, used standard Array and Map types. Focus on the singanutur documentation!
+- `4`golang has not been tested much, there may be issues
+- **formatter dependency** - optional dependency for prettify and fix codestyle
+
+- `5`lua
+  - Experimental Research PoC, performance and stability are not guaranteed
+  - Priority on generation to pure lua without C-libs dependencies. using [mva/htmlparser](https://luarocks.org/modules/mva/htmlparser) and [dhkolf/dkjson](https://luarocks.org/modules/dhkolf/dkjson)
+  - Translates unsupported CSS3 selectors into the equivalent in the form of function calls:
+    - for example, `div +p` is equivalent to `CssExt.combine_plus(root:select("div"), "p")`
+  - Translates PCRE regex to [string pattern matching](https://www.lua.org/manual/5.4/manual.html#6.4.1) (with restrictions) for more information in [lua_re_compat.py ](ssc_codegen/converters/templates/lua_re_compat.py)
+
+### Limitations
+
+For maximum portability of the configuration to the target language:
+
+- If possible, use CSS selectors: they are guaranteed to be converted to XPATH
+- Unlike javascript, most html parse libs implement [CSS3 selectors standard](https://www.w3.org/TR/selectors-3/). They may not fully implement the functionality!
+  Check the html parser lib documentation aboud CSS selectors before implement code. Examples:
+  1. Several libs not support `+` operations (eg: [selectolax(modest)](https://github.com/rushter/selectolax), [dart.universal_html](https://pub.dev/packages/universal_html))
+  2. For research purpose, lua_htmlparser include converter for unsupported CSS3 query syntax
+2. HTML parser libs maybe not supports attribute selectors: `*=`, `~=`, `|=`, `^=`, `$=`
+3. Several libs not support pseudo classes (eg: standard [dart.html](https://dart.dev/libraries/dart-html) lib miss this feature).
+
+## Getting started
+
+ssc_gen required python 3.10 version or higher
+
+### Install
+
+pip:
+
+```shell
+pip install ssc_codegen
+```
+
+uv:
+
+```shell
+uv pip install ssc_codegen
+```
+
+## Example
+
+### Create a file `schema.py` with:
+
+```python
+from ssc_codegen import ItemSchema, D
+
+class HelloWorld(ItemSchema):
+    title = D().css('title').text()
+    a_hrefs = D().css_all('a').attr('href')
+```
+
+### try it in cli
+
+> [!note]
+> this tools developed for testing purposes, not for web-scraping tasks
+
+### eval from file
+
+Download any html file and pass as argument:
+
+```shell
+ssc-gen parse-from-file index.html -t schema.py:HelloWorld
+```
+
+Short options descriptions:
+
+- `-t --target` - config schema file and class from where to start the parser
+
+![out1](docs/assets/parse_from_file.gif)
+
+### send GET request to url and parse response
+
+```shell
+ssc-gen parse-from-url https://example.com -t schema.py:HelloWorld
+```
+
+![out1](docs/assets/parse_from_url.gif)
+
+### send request via Chromium browser (CDP protocol)
+
+```shell
+ssc-gen parse-from-chrome https://example.com -t schema.py:HelloWorld
+```
+
+> [!note]
+> if script cannot found chrome executable - provide it manually:
+
+```shell
+ssc-gen parse-from-chrome https://example.com -t schema.py:HelloWorld -sc /usr/bin/chromium
+```
+
+### Convert to code
+
+Convert to code for use in projects:
+
+> [!note]
+> for example, used js: it can be fast test in developer console
+
+```shell
+ssc-gen js schema.py -o .
+```
+
+Code output looks like this:
+
+```javascript
+// autogenerated by ssc-gen DO NOT_EDIT
+/***
+ *
+ * {
+ *     "title": "String",
+ *     "a_hrefs": "Array<String>"
+ * }*/
+class HelloWorld {
+  constructor(doc) {
+    if (typeof doc === "string") {
+      this._doc = new DOMParser().parseFromString(doc, "text/html");
+    } else if (doc instanceof Document || doc instanceof Element) {
+      this._doc = doc;
+    } else {
+      throw new Error("Invalid input: Expected a Document, Element, or string");
+    }
+  }
+
+  _parseTitle(v) {
+    let v0 = v.querySelector("title");
+    return typeof v0.textContent === "undefined"
+      ? v0.documentElement.textContent
+      : v0.textContent;
+  }
+
+  _parseAHrefs(v) {
+    let v0 = Array.from(v.querySelectorAll("a"));
+    return v0.map((e) => e.getAttribute("href"));
+  }
+
+  parse() {
+    return {
+      title: this._parseTitle(this._doc),
+      a_hrefs: this._parseAHrefs(this._doc),
+    };
+  }
+}
+```
+
+### Copy code output and past to developer console:
+
+Print output:
+
+```javascript
+alert(JSON.stringify(new HelloWorld(document).parse()));
+```
+
+![example](docs/assets/example.png)
+
+You can use any html source:
+
+- parse from html files
+- parse from http responses
+- parse from browsers: playwright, selenium, chrome-cdp, etc.
+- call curl in shell and parse STDIN
+- use in STDIN pipelines with third-party tools like [projectdiscovery/httpx](https://github.com/projectdiscovery/httpx)
+
+## See also
+
+- [Brief](docs/briefing.md) about css selectors and regular expressions.
+- [Quickstart](docs/quickstart.md) about css selectors and regular expressions.
+- [Tutorial](docs/tutorial.md) basic usage ssc-gen
+- [AST reference](docs/ast_reference.md) about generation code from AST

{ssc_codegen-0.9.9 → ssc_codegen-0.10.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.9.9"
+version = "0.10.1"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"
@@ -20,6 +20,7 @@ dependencies = [
     # https://github.com/fastapi/typer/pull/1145
     "click<8.2.0",
     "ast-grep-cli>=0.38.1",
+    "tinycss2>=1.4.0",
 ]
 
 classifiers = [

{ssc_codegen-0.9.9 → ssc_codegen-0.10.1}/ssc_codegen/__init__.py

@@ -1,4 +1,4 @@
-from ssc_codegen.document import HTMLDocument, StringDocument, ArrayDocument, AssertDocument, NestedDocument, \
+from ssc_codegen.document import HTMLDocument, ClassVarDocument, StringDocument, ArrayDocument, AssertDocument, NestedDocument, \
     DefaultDocument, \
     NumericDocument, JsonDocument, BooleanDocument, DocumentFilter
 from ssc_codegen.json_struct import Json
@@ -7,7 +7,7 @@ from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchem
 
 setup_logger()
 
-VERSION = "0.9.9"
+VERSION = "0.10.1"
 
 
 class __MISSING(object):
@@ -63,4 +63,30 @@ def R(default: None | str | int | float | list | __MISSING = _NO_DEFAULT) -> Doc
 
 def F() -> DocumentFilter:
     """Shortcut as a DocumentFilter() object"""
-    return DocumentFilter()
+    return DocumentFilter()
+
+
+def CV(value, self_cls_path: str | None = None, *, returns: bool = False) -> ClassVarDocument:
+    """Shortcut as ClassVarDocument(value, self_cls, parse_returns) object
+
+    pass `self_cls_path` argument if need use this var inner schema context:
+
+    self_cls_path use case example:
+
+```
+class B(ItemSchema):
+    # dont need pass class and attr names
+    C = CV("test {{}}")
+
+class A(ItemShema):
+    # current impl cannot be extract class and field names
+    # need pass it manually
+    C1 = CV("title", "A.C1")
+
+    title = D().css(C1).text().fmt(C)
+
+```
+
+    """
+    
+    return ClassVarDocument(value, self_cls_path, parse_returns=returns)

{ssc_codegen-0.9.9 → ssc_codegen-0.10.1}/ssc_codegen/ast_/__init__.py

@@ -1,6 +1,6 @@
 from .base import BaseAstNode
 from .nodes_array import ExprIndex, ExprToListLength, ExprListStringJoin, ExprListUnique
-from .nodes_cast import ExprNested, ExprJsonify, ExprToInt, ExprToFloat, ExprToListInt, ExprToListFloat, ExprToBool
+from .nodes_cast import ExprNested, ExprJsonify, ExprToInt, ExprToFloat, ExprToListInt, ExprToListFloat, ExprToBool, ExprJsonifyDynamic
 from .nodes_core import (ExprReturn,
                          ExprNoReturn,
                          ExprDefaultValueEnd,
@@ -19,13 +19,17 @@ from .nodes_core import (ExprReturn,
                          JsonStructField,
                          TypeDef,
                          TypeDefField,
-                         ExprCallStructMethod)
+                         ExprCallStructMethod, 
+                         ExprClassVar,
+                         ExprCallStructClassVar,
+                         CodeStart,
+                         CodeEnd)
 from .nodes_filter import FilterOr, FilterAnd, FilterNot, FilterNotEqual, FilterEqual, FilterStrIn, \
     FilterStrStarts, FilterStrEnds, FilterStrRe, ExprFilter, FilterStrLenEq, FilterStrLenNe, FilterStrLenLt, \
     FilterStrLenLe, FilterStrLenGt, FilterStrLenGe
 
 from .nodes_selectors import ExprCss, ExprCssAll, ExprXpathAll, ExprXpath, ExprGetHtmlText, ExprGetHtmlRaw, \
-    ExprGetHtmlAttr, ExprGetHtmlAttrAll, ExprGetHtmlRawAll, ExprGetHtmlTextAll, ExprMapAttrs, ExprMapAttrsAll
+    ExprGetHtmlAttr, ExprGetHtmlAttrAll, ExprGetHtmlRawAll, ExprGetHtmlTextAll, ExprMapAttrs, ExprMapAttrsAll, ExprCssElementRemove, ExprXpathElementRemove
 from .nodes_string import ExprStringTrim, ExprStringRegex, ExprStringSplit, ExprStringReplace, ExprStringLeftTrim, \
     ExprStringFormat, ExprListStringTrim, ExprStringRegexAll, ExprStringRegexSub, ExprStringRightTrim, \
     ExprListStringFormat, ExprListStringReplace, ExprListStringRightTrim, ExprListStringLeftTrim, \

{ssc_codegen-0.9.9 → ssc_codegen-0.10.1}/ssc_codegen/ast_/base.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass, field
 from typing import (
+    Callable,
     ClassVar,
     Mapping,
     TypeVar,
@@ -9,10 +10,14 @@ from typing import (
     Optional,
     MutableSequence,
     Sequence,
+    TYPE_CHECKING,
 )
 
 from ssc_codegen.tokens import TokenType, VariableType
 
+if TYPE_CHECKING:
+    from ssc_codegen.ast_.nodes_core import ExprClassVar
+
 T_ARGUMENTS = TypeVar("T_ARGUMENTS")
 T_MAPPING_FIELD = TypeVar("T_MAPPING_FIELD", bound=Mapping[str, Any])
 T_EMPTY_KWARGS = TypedDict("T_EMPTY_KWARGS", {})
@@ -31,7 +36,10 @@ class BaseAstNode(Generic[T_MAPPING_FIELD, T_ARGUMENTS]):
     # used for type check expressions
     accept_type: VariableType = VariableType.ANY
     ret_type: VariableType = VariableType.ANY
-    exclude_types: Sequence[VariableType] = field(default_factory=tuple)
+    exclude_types: Sequence[VariableType] = field(
+        default_factory=tuple, repr=False
+    )
+    classvar_hooks: dict[str, "ExprClassVar"] = field(default_factory=dict)
 
     def unpack_args(self) -> T_ARGUMENTS:
         """extract all values from kwargs field wout keys"""
@@ -67,6 +75,39 @@ class BaseAstNode(Generic[T_MAPPING_FIELD, T_ARGUMENTS]):
                 return self.kwargs[name]
             raise e
 
+    def find_node(
+        self, func: Callable[["BaseAstNode"], bool]
+    ) -> Optional["BaseAstNode"]:
+        """find nodes in body by func filter
+
+        return None if node not founded or node.body is empty
+        """
+        if len(self.body) == 0:
+            return None
+
+        for n in self.body:
+            if func(n):
+                return n
+        return None
+
+    def find_node_all(
+        self, func: Callable[["BaseAstNode"], bool]
+    ) -> "BaseAstNode":
+        """find nodes in body by func filter"""
+        if len(self.body) == 0:
+            return []
+        return [i for i in self.body if i.kind == func(i)]
+
+    def find_node_by_token(
+        self, token_type: TokenType
+    ) -> Optional["BaseAstNode"]:
+        """find node by TokenType"""
+        return self.find_node(lambda n: n.kind == token_type)
+
+    def find_nodes_by_token(self, token_type: TokenType) -> list["BaseAstNode"]:
+        """find nodes by TokenType"""
+        return self.find_nodes_by_token(lambda n: n.kind == token_type)
+
 
 @dataclass(kw_only=True)
 class _DisableReprBody(BaseAstNode):

{ssc_codegen-0.9.9 → ssc_codegen-0.10.1}/ssc_codegen/ast_/nodes_array.py

@@ -33,8 +33,12 @@ class ExprToListLength(BaseAstNode[T_EMPTY_KWARGS, tuple]):
     ret_type: VariableType = VariableType.INT
 
 
+KW_UNIQUE = TypedDict("KW_UNIQUE", {"keep_order": bool})
+ARGS_UNIQUE = tuple[bool]
+
+
 @dataclass(kw_only=True)
-class ExprListUnique(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+class ExprListUnique(BaseAstNode[KW_UNIQUE, ARGS_UNIQUE]):
     # TODO: support LIST_INT, LIST_FLOAT
     kind: ClassVar[TokenType] = TokenType.EXPR_LIST_UNIQUE
     accept_type: VariableType = VariableType.LIST_STRING

{ssc_codegen-0.9.9 → ssc_codegen-0.10.1}/ssc_codegen/ast_/nodes_cast.py

@@ -58,9 +58,9 @@ class ExprNested(BaseAstNode[KW_EXPR_NESTED, EXPR_NESTED_ARGS]):
 
 
 KW_EXPR_JSONIFY = TypedDict(
-    "KW_EXPR_JSONIFY", {"json_struct_name": str, "is_array": bool}
+    "KW_EXPR_JSONIFY", {"json_struct_name": str, "is_array": bool, "query": str}
 )
-ARGS_EXPR_JSONIFY = tuple[str, bool]
+ARGS_EXPR_JSONIFY = tuple[str, bool, str]
 
 
 @dataclass(kw_only=True)
@@ -68,3 +68,16 @@ class ExprJsonify(BaseAstNode[KW_EXPR_JSONIFY, ARGS_EXPR_JSONIFY]):
     kind: ClassVar[TokenType] = TokenType.TO_JSON
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.JSON
+
+
+KW_EXPR_JSONIFY_DYNAMIC = TypedDict("KW_EXPR_JSONIFY_DYNAMIC", {"query": str})
+ARGS_EXPR_JSONIFY_DYNAMIC = tuple[str]
+
+
+@dataclass(kw_only=True)
+class ExprJsonifyDynamic(
+    BaseAstNode[KW_EXPR_JSONIFY_DYNAMIC, ARGS_EXPR_JSONIFY_DYNAMIC]
+):
+    kind: ClassVar[TokenType] = TokenType.TO_JSON_DYNAMIC
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.ANY  # STUBBED