ssc_codegen 0.7.2__tar.gz → 0.7.4__tar.gz

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.7.2
+Version: 0.7.4
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
@@ -230,9 +230,12 @@ You can use any html source:
 - get from browser (playwright, selenium, chrome-cdp)
 - paste code to developer console (js)
 - or call curl in shell and parse stdin
+- use in third-party tools in stdin pipeline: 
+(from [ProjectDiscovery](https://github.com/projectdiscovery), for example)
 
 
 ## See also
 - [Brief](docs/brief.md) about css selectors and regular expressions.
 - [Tutorial](docs/tutorial.md) how to use ssc-gen
-- [Reference](docs/reference.md) about high-level API
+- [Reference](docs/reference.md) about high-level API
+- [AST reference](docs/ast_reference.md) about generation code from AST

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/README.md

@@ -200,9 +200,12 @@ You can use any html source:
 - get from browser (playwright, selenium, chrome-cdp)
 - paste code to developer console (js)
 - or call curl in shell and parse stdin
+- use in third-party tools in stdin pipeline: 
+(from [ProjectDiscovery](https://github.com/projectdiscovery), for example)
 
 
 ## See also
 - [Brief](docs/brief.md) about css selectors and regular expressions.
 - [Tutorial](docs/tutorial.md) how to use ssc-gen
-- [Reference](docs/reference.md) about high-level API
+- [Reference](docs/reference.md) about high-level API
+- [AST reference](docs/ast_reference.md) about generation code from AST

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.7.2"
+version = "0.7.4"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"
@@ -13,6 +13,7 @@ dependencies = [
     "typer>=0.15.1",
     "typing_extensions;python_version<'3.11'",
 ]
+
 classifiers = [
     "Development Status :: 3 - Alpha",
     "Environment :: Console",

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/ssc_codegen/__init__.py

@@ -1,16 +1,25 @@
 from .document import HTMLDocument, StringDocument, ArrayDocument, AssertDocument, NestedDocument, DefaultDocument, \
-    NumericDocument, JsonDocument
+    NumericDocument, JsonDocument, BooleanDocument
 from .json_struct import Json
 from .schema import ItemSchema, DictSchema, ListSchema, FlatListSchema
 
-VERSION = "0.7.2"
+VERSION = "0.7.4"
 
 class __MISSING(object):
     pass
 
 _NO_DEFAULT = __MISSING()
 
-class Document(HTMLDocument, StringDocument, ArrayDocument, AssertDocument, DefaultDocument, NumericDocument, JsonDocument):
+class Document(
+    HTMLDocument,
+    StringDocument,
+    ArrayDocument,
+    AssertDocument,
+    DefaultDocument,
+    NumericDocument,
+    JsonDocument,
+    BooleanDocument
+):
     pass
 
 

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/ssc_codegen/ast_builder.py

@@ -207,6 +207,7 @@ def build_ast_struct(
         assert_schema_dict_key_is_string(field, name, schema)
         assert_split_doc_is_list_document(field, name, schema)
         assert_ret_type_not_document(field, name, schema)
+        # TODO: compare default placeholder value type and last expr ret_type
 
     ast_struct_parser = StructParser(
         type=schema.__SCHEMA_TYPE__,

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/ssc_codegen/ast_ssc.py

@@ -69,6 +69,11 @@ class Variable(BaseAstNode):
         """
         return self.num + 1
 
+    def __repr__(self) -> str:
+        return (
+            f"{self.kind.name!r}({self.num!r} of {self.count!r}) {self.type!r}"
+        )
+
 
 @dataclass(kw_only=True)
 class Docstring(BaseAstNode):
@@ -80,8 +85,8 @@ class Docstring(BaseAstNode):
 
     kind: ClassVar[TokenType] = TokenType.DOCSTRING
     value: str
-    parent: Optional[Union["StructParser", "ModuleProgram"]] = (
-        None  # LATE INIT in builder
+    parent: Optional[Union["StructParser", "ModuleProgram"]] = field(
+        default=None, repr=False
     )
 
 
@@ -104,7 +109,7 @@ class TypeDefField(BaseAstNode):
     kind: ClassVar[TokenType] = TokenType.TYPEDEF_FIELD
     name: str
     ret_type: VariableType
-    parent: Optional["TypeDef"] = None
+    parent: Optional["TypeDef"] = field(default=None, repr=False)
 
     @property
     def nested_class(self) -> str | None:
@@ -146,8 +151,8 @@ class TypeDef(BaseAstNode):
     kind: ClassVar[TokenType] = TokenType.TYPEDEF
     name: str
     body: list[TypeDefField]
-    struct_ref: Optional["StructParser"] = None
-    parent: Optional[ModuleProgram] = None
+    struct_ref: Optional["StructParser"] = field(default=None, repr=False)
+    parent: Optional[ModuleProgram] = field(default=None, repr=False)
 
     def __post_init__(self) -> None:
         for f in self.body:
@@ -174,9 +179,13 @@ class BaseExpression(BaseAstNode):
             "PreValidateFunction",
             "PartDocFunction",
         ]
-    ] = None  # LATE INIT
-    prev: Optional["BaseExpression"] = None  # LATE INIT
-    next: Optional["BaseExpression"] = None  # LATE INIT
+    ] = field(init=False, repr=False)  # LATE INIT
+    prev: Optional["BaseExpression"] = field(
+        default=None, repr=False
+    )  # LATE INIT
+    next: Optional["BaseExpression"] = field(
+        default=None, repr=False
+    )  # LATE INIT
 
     accept_type: VariableType
     ret_type: VariableType
@@ -218,7 +227,7 @@ class DefaultValueWrapper(BaseExpression):
 
     accept_type: VariableType = VariableType.DOCUMENT
     ret_type: VariableType = VariableType.DOCUMENT
-    value: str | int | float | None
+    value: str | int | float | bool | None
 
 
 @dataclass(kw_only=True)
@@ -233,7 +242,7 @@ class DefaultStart(BaseExpression):
     kind: ClassVar[TokenType] = TokenType.EXPR_DEFAULT_START
     accept_type: VariableType = VariableType.ANY
     ret_type: VariableType = VariableType.ANY
-    value: str | int | float | None
+    value: str | int | float | bool | None
 
 
 @dataclass(kw_only=True)
@@ -249,7 +258,7 @@ class DefaultEnd(BaseExpression):
     kind: ClassVar[TokenType] = TokenType.EXPR_DEFAULT_END
     accept_type: VariableType = VariableType.ANY
     ret_type: VariableType = VariableType.ANY
-    value: str | int | float | None
+    value: str | int | float | bool | None
 
 
 # DOCUMENT
@@ -415,6 +424,18 @@ class JoinExpression(BaseExpression):
     ret_type: VariableType = VariableType.STRING
 
 
+@dataclass(kw_only=True)
+class ArrayLengthExpression(BaseExpression):
+    kind: ClassVar[TokenType] = TokenType.EXPR_LIST_LEN
+    accept_type: VariableType = (
+        VariableType.LIST_STRING
+        | VariableType.LIST_DOCUMENT
+        | VariableType.LIST_INT
+        | VariableType.LIST_FLOAT
+    )
+    ret_type: VariableType = VariableType.INT
+
+
 # STRING
 @dataclass(kw_only=True)
 class __TrimNode(BaseExpression):
@@ -482,9 +503,9 @@ class RTrimExpression(__TrimNode):
 
 @dataclass(kw_only=True)
 class __MapTrimNode(BaseExpression):
-    value: str
     accept_type: VariableType = VariableType.LIST_STRING
     ret_type: VariableType = VariableType.LIST_STRING
+    value: str
 
 
 @dataclass(kw_only=True)
@@ -552,9 +573,9 @@ class SplitExpression(BaseExpression):
     """
 
     kind: ClassVar[TokenType] = TokenType.EXPR_STRING_SPLIT
-    sep: str
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.LIST_STRING
+    sep: str
 
 
 @dataclass(kw_only=True)
@@ -570,9 +591,9 @@ class FormatExpression(BaseExpression):
     """
 
     kind: ClassVar[TokenType] = TokenType.EXPR_STRING_FORMAT
-    fmt: str
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.STRING
+    fmt: str
 
 
 @dataclass(kw_only=True)
@@ -588,9 +609,9 @@ class MapFormatExpression(BaseExpression):
     """
 
     kind: ClassVar[TokenType] = TokenType.EXPR_LIST_STRING_FORMAT
-    fmt: str
     accept_type: VariableType = VariableType.LIST_STRING
     ret_type: VariableType = VariableType.LIST_STRING
+    fmt: str
 
 
 @dataclass(kw_only=True)
@@ -603,10 +624,10 @@ class ReplaceExpression(BaseExpression):
     """
 
     kind: ClassVar[TokenType] = TokenType.EXPR_STRING_REPLACE
-    old: str
-    new: str
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.STRING
+    old: str
+    new: str
 
 
 @dataclass(kw_only=True)
@@ -619,10 +640,10 @@ class MapReplaceExpression(BaseExpression):
     """
 
     kind: ClassVar[TokenType] = TokenType.EXPR_LIST_STRING_REPLACE
-    old: str
-    new: str
     accept_type: VariableType = VariableType.LIST_STRING
     ret_type: VariableType = VariableType.LIST_STRING
+    old: str
+    new: str
 
 
 # REGEX
@@ -639,11 +660,10 @@ class RegexExpression(BaseExpression):
     """
 
     kind: ClassVar[TokenType] = TokenType.EXPR_REGEX
-
-    pattern: str
-    group: int
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.STRING
+    pattern: str
+    group: int
 
 
 @dataclass(kw_only=True)
@@ -660,10 +680,9 @@ class RegexAllExpression(BaseExpression):
     """
 
     kind: ClassVar[TokenType] = TokenType.EXPR_REGEX_ALL
-
-    pattern: str
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.LIST_STRING
+    pattern: str
 
 
 @dataclass(kw_only=True)
@@ -677,11 +696,10 @@ class RegexSubExpression(BaseExpression):
     """
 
     kind: ClassVar[TokenType] = TokenType.EXPR_REGEX_SUB
-
-    pattern: str
-    repl: str
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.STRING
+    pattern: str
+    repl: str
 
 
 @dataclass(kw_only=True)
@@ -695,11 +713,10 @@ class MapRegexSubExpression(BaseExpression):
     """
 
     kind: ClassVar[TokenType] = TokenType.EXPR_LIST_REGEX_SUB
-
-    pattern: str
-    repl: str
     accept_type: VariableType = VariableType.LIST_STRING
     ret_type: VariableType = VariableType.LIST_STRING
+    pattern: str
+    repl: str
 
 
 # asserts - validators - not modify variables
@@ -717,11 +734,10 @@ class IsEqualExpression(BaseExpression):
     """
 
     kind: ClassVar[TokenType] = TokenType.IS_EQUAL
-
-    value: str | int | float
-    msg: str
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.STRING
+    value: str | int | float
+    msg: str
 
 
 @dataclass(kw_only=True)
@@ -738,11 +754,10 @@ class IsNotEqualExpression(BaseExpression):
     """
 
     kind: ClassVar[TokenType] = TokenType.IS_NOT_EQUAL
-
-    value: str | int | float
-    msg: str
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.STRING
+    value: str | int | float
+    msg: str
 
 
 @dataclass(kw_only=True)
@@ -762,10 +777,10 @@ class IsCssExpression(BaseExpression):
     """
 
     kind: ClassVar[TokenType] = TokenType.IS_CSS
-    query: str
-    msg: str
     accept_type: VariableType = VariableType.DOCUMENT
     ret_type: VariableType = VariableType.DOCUMENT
+    query: str
+    msg: str
 
 
 @dataclass(kw_only=True)
@@ -785,10 +800,10 @@ class IsXPathExpression(BaseExpression):
     """
 
     kind: ClassVar[TokenType] = TokenType.IS_XPATH
-    query: str
-    msg: str
     accept_type: VariableType = VariableType.DOCUMENT
     ret_type: VariableType = VariableType.DOCUMENT
+    query: str
+    msg: str
 
 
 @dataclass(kw_only=True)
@@ -808,10 +823,10 @@ class IsContainsExpression(BaseExpression):
     """
 
     kind: ClassVar[TokenType] = TokenType.IS_CONTAINS
-    item: str | int | float
-    msg: str
     accept_type: VariableType = VariableType.LIST_STRING
     ret_type: VariableType = VariableType.LIST_STRING
+    item: str | int | float
+    msg: str
 
 
 @dataclass(kw_only=True)
@@ -830,10 +845,10 @@ class IsRegexMatchExpression(BaseExpression):
     """
 
     kind: ClassVar[TokenType] = TokenType.IS_REGEX_MATCH
-    pattern: str
-    msg: str
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.STRING
+    pattern: str
+    msg: str
 
 
 @dataclass(kw_only=True)
@@ -917,7 +932,7 @@ class CallStructFunctionExpression(BaseExpression):
     accept_type: VariableType = VariableType.DOCUMENT
     fn_ref: Union[
         "StructFieldFunction", "PreValidateFunction", "PartDocFunction", None
-    ]
+    ] = field(default=None, repr=False)
     nested_cls_name_ref: str | None = None
     name: str
     ret_type: VariableType
@@ -1029,7 +1044,9 @@ class StructFieldFunction(__StructNode):
 
     name: M_ITEM | M_KEY | M_VALUE | str
     kind: ClassVar[TokenType] = TokenType.STRUCT_FIELD
-    parent: Optional["StructParser"] = None  # LATE INIT
+    parent: Optional["StructParser"] = field(
+        default=None, repr=False
+    )  # LATE INIT
 
     def find_associated_typedef(self) -> TypeDef | None:
         """try get associated typedef Node
@@ -1093,7 +1110,9 @@ class PreValidateFunction(__StructNode):
 
     kind: ClassVar[TokenType] = TokenType.STRUCT_PRE_VALIDATE
     name: M_PRE_VALIDATE = "__PRE_VALIDATE__"
-    parent: Optional["StructParser"] = None  # LATE INIT
+    parent: Optional["StructParser"] = field(
+        default=None, init=False, repr=False
+    )  # LATE INIT
 
 
 @dataclass(kw_only=True)
@@ -1126,7 +1145,9 @@ class StartParseFunction(__StructNode):
     kind: ClassVar[TokenType] = TokenType.STRUCT_PARSE_START
     body: list[CallStructFunctionExpression]
     type: StructType
-    parent: Optional["StructParser"] = None  # LATE INIT
+    parent: Optional["StructParser"] = field(
+        default=None, repr=False
+    )  # LATE INIT
 
     def have_assert_expr(self) -> bool:
         for expr in self.body:
@@ -1155,7 +1176,9 @@ class StructInit(BaseAstNode):
     """
 
     kind: ClassVar[TokenType] = TokenType.STRUCT_INIT
-    parent: Optional["StructParser"] = None  # LATE INIT
+    parent: Optional["StructParser"] = field(
+        default=None, repr=False
+    )  # LATE INIT
     name: str
 
 
@@ -1185,8 +1208,10 @@ class StructParser(BaseAstNode):
         | PreValidateFunction
         | PartDocFunction
     ]
-    typedef: Optional["TypeDef"] = field(init=False)
-    parent: Optional[ModuleProgram] = None  # LATE INIT
+    typedef: Optional["TypeDef"] = field(default=None, repr=False)
+    parent: Optional[ModuleProgram] = field(
+        default=None, repr=False
+    )  # LATE INIT
 
     def _build_typedef(self) -> None:
         ast_typedef = TypeDef(
@@ -1241,7 +1266,7 @@ class JsonStruct(BaseAstNode):
     name: str
     body: list["JsonStructField"]
 
-    parent: Optional[ModuleProgram] = None
+    parent: Optional[ModuleProgram] = field(default=None, repr=False)
 
     def __post_init__(self) -> None:
         for f in self.body:
@@ -1254,7 +1279,7 @@ class JsonStructField(BaseAstNode):
     name: str
     value: BaseJsonType
 
-    parent: Optional[JsonStruct] = None  # LATE INIT
+    parent: Optional[JsonStruct] = field(default=None, repr=False)  # LATE INIT
 
     @property
     def ret_type(self) -> BaseJsonType | dict[str, BaseJsonType]:
@@ -1270,6 +1295,8 @@ class JsonStructField(BaseAstNode):
 
 @dataclass(kw_only=True)
 class ToJson(BaseExpression):
+    """jsonify string to structure"""
+
     kind: ClassVar[TokenType] = TokenType.TO_JSON
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.JSON
@@ -1282,3 +1309,19 @@ class ToJson(BaseExpression):
     @property
     def is_array_entrypoint(self) -> bool:
         return self.value.__IS_ARRAY__
+
+
+# BOOLEAN
+@dataclass(kw_only=True)
+class ToBool(BaseExpression):
+    """convert variable to boolean
+
+    boolean convert rules
+
+    None, nil, empty array, empty string - False
+    other - True
+    """
+
+    kind: ClassVar[TokenType] = TokenType.TO_BOOL
+    accept_type: VariableType = VariableType.ANY
+    ret_type: VariableType = VariableType.BOOL

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/ssc_codegen/converters/dart_universal_html.py

@@ -1,6 +1,7 @@
 # TODO: required enchant, not tested
 import warnings
 from functools import partial
+from typing_extensions import assert_never
 
 from ssc_codegen.ast_ssc import (
     DefaultValueWrapper,
@@ -55,8 +56,10 @@ from ssc_codegen.ast_ssc import (
     TypeDef,
     ToJson,
     JsonStruct,
+    ToBool,
+    ArrayLengthExpression,
 )
-from ssc_codegen.tokens import StructType, TokenType
+from ssc_codegen.tokens import StructType, TokenType, VariableType
 from .base import BaseCodeConverter, left_right_var_names
 from .templates import dart
 from ssc_codegen.str_utils import (
@@ -520,9 +523,41 @@ def tt_to_json(node: ToJson) -> str:
 
 
 @converter.pre(TokenType.JSON_STRUCT)
-def tt_json_struct(node: JsonStruct) -> str:
+def tt_json_struct(_: JsonStruct) -> str:
     warnings.warn(
         "Serialization current not available, generate dynamic struct",
         category=FutureWarning,
     )
     return ""
+
+
+@converter.pre(TokenType.TO_BOOL)
+def to_bool(node: ToBool) -> str:
+    prv, nxt = lr_var_names(variable=node.variable)
+    match node.ret_type:
+        case VariableType.LIST_DOCUMENT:
+            code = f"var {nxt} = ({prv} != null) && !{prv}.isEmpty; "
+        case VariableType.LIST_INT:
+            code = f"var {nxt} = ({prv} != null) && !{prv}.isEmpty; "
+        case VariableType.LIST_FLOAT:
+            code = f"var {nxt} = ({prv} != null) && !{prv}.isEmpty; "
+        case VariableType.LIST_STRING:
+            code = f"var {nxt} = ({prv} != null) && !{prv}.isEmpty; "
+        case VariableType.STRING:
+            code = f"var {nxt} = ({prv} != null) && !{prv}.isEmpty; "
+
+        case VariableType.INT:
+            code = f"var {nxt} = {prv} != null; "
+        case VariableType.FLOAT:
+            code = f"var {nxt} = {prv} != null; "
+        case VariableType.DOCUMENT:
+            code = f"var {nxt} = {prv} != null; "
+        case _:
+            assert_never(node.ret_type)
+    return code
+
+
+@converter.pre(TokenType.EXPR_LIST_LEN)
+def to_len(node: ArrayLengthExpression) -> str:
+    prv, nxt = lr_var_names(variable=node.variable)
+    return f"var {nxt} = {prv}.length; "

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/ssc_codegen/converters/go_goquery.py

@@ -51,6 +51,8 @@ from ssc_codegen.ast_ssc import (
     ToJson,
     JsonStruct,
     TypeDefField,
+    ArrayLengthExpression,
+    ToBool,
 )
 from ssc_codegen.tokens import (
     StructType,
@@ -304,6 +306,8 @@ def tt_default(node: DefaultStart) -> str:
     # defer func() wrapper, set default value if code throw error
     if isinstance(node.value, str):
         value = wrap_double_quotes(node.value)
+    elif isinstance(node.value, bool):
+        value = "true" if node.value else "false"
     elif isinstance(node.value, (int, float)):
         value = node.value
     else:
@@ -753,3 +757,47 @@ def tt_json_struct(node: JsonStruct) -> str:
             case _:
                 assert_never(field.value.kind)
     return code + go.BRACKET_END
+
+
+@converter.pre(TokenType.EXPR_LIST_LEN)
+def tt_to_len(node: ArrayLengthExpression) -> str:
+    prv, nxt = left_right_var_names("value", node.variable)
+    match node.ret_type:
+        # https://pkg.go.dev/gopkg.in/goquery.v1#Selection.Length
+        case VariableType.LIST_DOCUMENT:
+            code = f"{nxt} := {prv}.Length();"
+        # build-in array used
+        case _:
+            code = f"{nxt} := len({prv});"
+    return code
+
+
+@converter.pre(TokenType.TO_BOOL)
+def tt_to_bool(node: ToBool) -> str:
+    prv, nxt = left_right_var_names("value", node.variable)
+    match node.ret_type:
+        # https://pkg.go.dev/gopkg.in/goquery.v1#Selection.Length
+        case VariableType.DOCUMENT:
+            code = f"{nxt} := {prv} != nil && {prv}.Length() > 0; "
+        case VariableType.LIST_DOCUMENT:
+            code = f"{nxt} := {prv} != nil && {prv}.Length() > 0; "
+
+        case VariableType.STRING:
+            code = f'{nxt} := {prv} != nil && {prv} != ""; '
+
+        # `0` is true
+        case VariableType.INT:
+            code = f"{nxt} := {prv} != nil; "
+        case VariableType.FLOAT:
+            code = f"{nxt} := {prv} != nil; "
+
+        # build-in array
+        case VariableType.LIST_STRING:
+            code = f"{nxt} := {prv} != nil && len({prv}) > 0; "
+        case VariableType.LIST_INT:
+            code = f"{nxt} := {prv} != nil && len({prv}) > 0; "
+        case VariableType.LIST_FLOAT:
+            code = f"{nxt} := {prv} != nil && len({prv}) > 0; "
+        case _:
+            assert_never(node.prev.ret_type)
+    return code

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/ssc_codegen/converters/js_pure.py

@@ -50,6 +50,8 @@ from ssc_codegen.ast_ssc import (
     ToFloat,
     ToListFloat,
     ToJson,
+    ToBool,
+    ArrayLengthExpression,
 )
 from ssc_codegen.tokens import StructType, TokenType
 from .base import BaseCodeConverter, left_right_var_names
@@ -473,3 +475,15 @@ def tt_to_list_float(node: ToListFloat) -> str:
 def tt_to_json(node: ToJson) -> str:
     prv, nxt = left_right_var_names("value", node.variable)
     return f"let {nxt} = JSON.parse({prv}); "
+
+
+@converter.pre(TokenType.TO_BOOL)
+def tt_to_bool(node: ToBool) -> str:
+    prv, nxt = left_right_var_names("value", node.variable)
+    return f"let {nxt} = {prv} || {prv} === 0 ? true : false;"
+
+
+@converter.pre(TokenType.EXPR_LIST_LEN)
+def tt_to_len(node: ArrayLengthExpression) -> str:
+    prv, nxt = left_right_var_names("value", node.variable)
+    return f"let {nxt} = {prv}.length; "

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/ssc_codegen/converters/py_base.py

@@ -47,6 +47,8 @@ from ssc_codegen.ast_ssc import (
     ToJson,
     JsonStruct,
     JsonStructField,
+    ArrayLengthExpression,
+    ToBool,
 )
 from ssc_codegen.tokens import (
     StructType,
@@ -133,6 +135,9 @@ class BasePyCodeConverter(BaseCodeConverter):
         self.post_definitions[TokenType.JSON_STRUCT] = tt_json_struct_post
         self.pre_definitions[TokenType.JSON_FIELD] = tt_json_field
 
+        self.pre_definitions[TokenType.EXPR_LIST_LEN] = tt_to_len
+        self.pre_definitions[TokenType.TO_BOOL] = tt_to_bool
+
 
 def tt_imports(node: ModuleImports) -> str:
     return py.BINDINGS[node.kind]
@@ -287,7 +292,7 @@ def tt_default_start(node: DefaultStart) -> str:
 
 
 def tt_default_end(node: DefaultEnd) -> str:
-    # prv, nxt = lr_var_names(variable=node.variable)
+    # bool, int, float, None represent as same
     val = repr(node.value) if isinstance(node.value, str) else node.value
     return py.INDENT_METHOD_BODY + py.BINDINGS[node.kind, val]
 
@@ -562,3 +567,17 @@ def tt_to_json(node: ToJson) -> str:
     prv, nxt = lr_var_names(variable=node.variable)
     code = f"{nxt} = json.loads({prv})"
     return indent + code
+
+
+def tt_to_len(node: ArrayLengthExpression) -> str:
+    indent = py.suggest_indent(node)
+    prv, nxt = lr_var_names(variable=node.variable)
+    code = f"{nxt} = len({prv})"
+    return indent + code
+
+
+def tt_to_bool(node: ToBool) -> str:
+    indent = py.suggest_indent(node)
+    prv, nxt = lr_var_names(variable=node.variable)
+    code = f"{nxt} = bool({prv} or {prv} == 0)"
+    return indent + code

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/ssc_codegen/converters/templates/dart.py

@@ -20,6 +20,7 @@ TYPES = {
     VariableType.OPTIONAL_FLOAT: "double?",
     VariableType.LIST_FLOAT: "List<double>",
     VariableType.OPTIONAL_LIST_FLOAT: "List<double>?",
+    VariableType.BOOL: "bool",
 }
 
 

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/ssc_codegen/converters/templates/go.py

@@ -39,6 +39,7 @@ TYPES = {
     VariableType.OPTIONAL_FLOAT: "*float64",
     VariableType.LIST_FLOAT: "[]float64",
     VariableType.OPTIONAL_LIST_FLOAT: "*[]float64",
+    VariableType.BOOL: "bool",
 }
 
 JSON_TYPES = {

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/ssc_codegen/converters/templates/py.py

@@ -31,6 +31,7 @@ TYPES = {
     VariableType.FLOAT: "float",
     VariableType.LIST_INT: "List[int]",
     VariableType.LIST_FLOAT: "List[float]",
+    VariableType.BOOL: "bool",
 }
 
 JSON_TYPES = {

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/ssc_codegen/document.py

@@ -1,6 +1,6 @@
 """high-level AST builder interface"""
 
-from typing import Type
+from typing import Type, Pattern
 
 from typing_extensions import Self
 
@@ -48,8 +48,10 @@ from .ast_ssc import (
     ToListInteger,
     TrimExpression,
     ToJson,
+    ToBool,
+    ArrayLengthExpression,
 )
-from .document_utlis import assert_re_expression
+from .document_utlis import assert_re_expression, unverbosify_regex
 from .schema import BaseSchema
 from .selector_utils import validate_css_query, validate_xpath_query
 from .tokens import VariableType
@@ -269,9 +271,15 @@ class ArrayDocument(BaseDocument):
 
         match self.stack_last_ret:
             case VariableType.LIST_DOCUMENT:
-                self._add(IndexDocumentExpression(value=i, ret_type=VariableType.DOCUMENT))
+                self._add(
+                    IndexDocumentExpression(
+                        value=i, ret_type=VariableType.DOCUMENT
+                    )
+                )
             case VariableType.LIST_STRING:
-                self._add(IndexStringExpression(value=i, ret_type=VariableType.STRING))
+                self._add(
+                    IndexStringExpression(value=i, ret_type=VariableType.STRING)
+                )
             case VariableType.LIST_INT:
                 self._add(
                     IndexStringExpression(value=i, ret_type=VariableType.INT)
@@ -315,6 +323,32 @@ class ArrayDocument(BaseDocument):
                 )
         return self
 
+    def to_len(self) -> Self:
+        """Get length of items in array object
+
+        - accept LIST_STRING | LIST_DOCUMENT | LIST_INT | LIST_FLOAT, return INT
+
+        """
+        match self.stack_last_ret:
+            case VariableType.LIST_STRING:
+                pass
+            case VariableType.LIST_DOCUMENT:
+                pass
+            case VariableType.LIST_INT:
+                pass
+            case VariableType.LIST_FLOAT:
+                pass
+            case _:
+                self._raise_wrong_type_error(
+                    self.stack_last_ret,
+                    VariableType.LIST_STRING,
+                    VariableType.LIST_DOCUMENT,
+                    VariableType.LIST_INT,
+                    VariableType.LIST_FLOAT,
+                )
+        self._add(ArrayLengthExpression())
+        return self
+
 
 class StringDocument(BaseDocument):
     def trim(self, substr: str = " ") -> Self:
@@ -424,7 +458,7 @@ class StringDocument(BaseDocument):
                 )
         return self
 
-    def re(self, pattern: str, group: int = 1) -> Self:
+    def re(self, pattern: str | Pattern, group: int = 1) -> Self:
         """extract first regex result.
 
         NOTE:
@@ -432,6 +466,7 @@ class StringDocument(BaseDocument):
 
         - accept STRING, return STRING
         """
+        pattern = unverbosify_regex(pattern)
         assert_re_expression(pattern)
 
         if self.stack_last_ret != VariableType.STRING:
@@ -441,11 +476,12 @@ class StringDocument(BaseDocument):
         self._add(RegexExpression(pattern=pattern, group=group))
         return self
 
-    def re_all(self, pattern: str) -> Self:
+    def re_all(self, pattern: str | Pattern) -> Self:
         """extract all regex results.
 
         - accept STRING, return LIST_STRING
         """
+        pattern = unverbosify_regex(pattern)
         assert_re_expression(pattern, max_groups=1)
         if self.stack_last_ret != VariableType.STRING:
             self._raise_wrong_type_error(
@@ -454,12 +490,13 @@ class StringDocument(BaseDocument):
         self._add(RegexAllExpression(pattern=pattern))
         return self
 
-    def re_sub(self, pattern: str, repl: str = "") -> Self:
+    def re_sub(self, pattern: str | Pattern, repl: str = "") -> Self:
         """Replace substring by `pattern` to `repl`.
 
         - accept STRING, return STRING
         - accept LIST_STRING, return LIST_STRING
         """
+        pattern = unverbosify_regex(pattern)
         assert_re_expression(pattern, allow_empty_groups=True)
 
         match self.stack_last_ret:
@@ -710,6 +747,23 @@ class NumericDocument(BaseDocument):
         return self
 
 
+class BooleanDocument(BaseDocument):
+    def to_bool(self) -> Self:
+        """convert current value to bool. Accept any type
+
+        value returns false if:
+
+        - None
+        - empty sequence
+        - empty string
+
+        other - true
+
+        """
+        self._add(ToBool())
+        return self
+
+
 class JsonDocument(BaseDocument):
     def jsonify(self, struct: Type[Json]) -> Self:
         """marshal json string to object

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/ssc_codegen/document_utlis.py

@@ -1,5 +1,5 @@
 import re
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Pattern
 
 from ssc_codegen.ast_ssc import (
     BaseExpression,
@@ -16,6 +16,40 @@ if TYPE_CHECKING:
     from .document import BaseDocument
 
 
+# https://stackoverflow.com/a/14919203
+CM1_RX = r"(?m)(?<!\\)((\\{2})*)#.*$"
+CM2_RX = r"(\\)?((\\{2})*)(#)"
+WS_RX = r"(\\)?((\\{2})*)(\s)\s*"
+
+
+def unverbosify_regex(pattern: str | Pattern) -> str:
+    if isinstance(pattern, str):
+        return pattern
+
+    def strip_escapes(match):  # type: ignore
+        ## if even slashes: delete space and retain slashes
+        if match.group(1) is None:
+            return match.group(2)
+        ## if number of slashes is odd: delete slash and keep space (or 'comment')
+        elif match.group(1) == "\\":
+            return match.group(2) + match.group(4)
+        ## error
+        else:
+            raise Exception
+
+    if pattern.flags & re.X:
+        not_verbose_regex = re.sub(
+            WS_RX,
+            strip_escapes,
+            re.sub(
+                CM2_RX, strip_escapes, re.sub(CM1_RX, "\\1", pattern.pattern)
+            ),
+        )
+
+        return not_verbose_regex
+    return pattern.pattern
+
+
 def assert_re_expression(
     pattern: str, allow_empty_groups: bool = False, max_groups: int = -1
 ) -> None:

{ssc_codegen-0.7.2 → ssc_codegen-0.7.4}/ssc_codegen/tokens.py

@@ -28,6 +28,8 @@ class VariableType(IntEnum):
     JSON = auto()
     OPTIONAL_JSON = auto()
 
+    BOOL = auto()
+
 
 class JsonVariableType(IntEnum):
     # https://json-schema.org/understanding-json-schema/reference/type
@@ -135,6 +137,7 @@ class TokenType(IntEnum):
     EXPR_LIST_STRING_INDEX = auto()
     EXPR_LIST_DOCUMENT_INDEX = auto()
     EXPR_LIST_JOIN = auto()
+    EXPR_LIST_LEN = auto()
 
     # ASSERT
     IS_EQUAL = auto()
@@ -155,3 +158,6 @@ class TokenType(IntEnum):
     # STRUCTS
     JSON_STRUCT = auto()
     JSON_FIELD = auto()
+
+    # bool
+    TO_BOOL = auto()