ssc_codegen 0.8.16__tar.gz → 0.8.18__tar.gz

{ssc_codegen-0.8.16 → ssc_codegen-0.8.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.8.16
+Version: 0.8.18
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
@@ -19,6 +19,7 @@ Classifier: Topic :: Software Development :: Code Generators
 Classifier: Topic :: Text Processing :: Markup :: HTML
 Classifier: Topic :: Utilities
 Requires-Python: >=3.10
+Requires-Dist: bs4>=0.0.2
 Requires-Dist: colorama>=0.4.6; sys_platform == 'win32'
 Requires-Dist: cssselect>=1.2.0
 Requires-Dist: httpx>=0.28.1

{ssc_codegen-0.8.16 → ssc_codegen-0.8.18}/pyproject.toml

@@ -1,10 +1,11 @@
 [project]
 name = "ssc_codegen"
-version = "0.8.16"
+version = "0.8.18"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
+    "bs4>=0.0.2",
     "colorama>=0.4.6 ; sys_platform == 'win32'",
     "cssselect>=1.2.0",
     "httpx>=0.28.1",

{ssc_codegen-0.8.16 → ssc_codegen-0.8.18}/ssc_codegen/__init__.py

@@ -1,13 +1,13 @@
 from ssc_codegen.document import HTMLDocument, StringDocument, ArrayDocument, AssertDocument, NestedDocument, \
     DefaultDocument, \
-    NumericDocument, JsonDocument, BooleanDocument
+    NumericDocument, JsonDocument, BooleanDocument, DocumentFilter
 from ssc_codegen.json_struct import Json
 from ssc_codegen.logs import setup_logger
 from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchema
 
 setup_logger()
 
-VERSION = "0.8.16"
+VERSION = "0.8.18"
 
 
 class __MISSING(object):
@@ -59,3 +59,8 @@ def R(default: None | str | int | float | list | __MISSING = _NO_DEFAULT) -> Doc
     if default == _NO_DEFAULT:
         return Document().raw()
     return Document().default(default).raw()  # type: ignore
+
+
+def F() -> DocumentFilter:
+    """Shortcut as a DocumentFilter() object"""
+    return DocumentFilter()

{ssc_codegen-0.8.16 → ssc_codegen-0.8.18}/ssc_codegen/ast_/__init__.py

@@ -20,9 +20,12 @@ from .nodes_core import (ExprReturn,
                          TypeDef,
                          TypeDefField,
                          ExprCallStructMethod)
+from .nodes_filter import FilterOr, FilterAnd, FilterNot, FilterNotEqual, FilterEqual, FilterStrIn, \
+    FilterStrStarts, FilterStrEnds, FilterStrRe, ExprFilter, FilterStrLenEq, FilterStrLenNe, FilterStrLenLt, \
+    FilterStrLenLe, FilterStrLenGt, FilterStrLenGe
+
 from .nodes_selectors import ExprCss, ExprCssAll, ExprXpathAll, ExprXpath, ExprGetHtmlText, ExprGetHtmlRaw, \
     ExprGetHtmlAttr, ExprGetHtmlAttrAll, ExprGetHtmlRawAll, ExprGetHtmlTextAll
-
 from .nodes_string import ExprStringTrim, ExprStringRegex, ExprStringSplit, ExprStringReplace, ExprStringLeftTrim, \
     ExprStringFormat, ExprListStringTrim, ExprStringRegexAll, ExprStringRegexSub, ExprStringRightTrim, \
     ExprListStringFormat, ExprListStringReplace, ExprListStringRightTrim, ExprListStringLeftTrim, \

ssc_codegen-0.8.18/ssc_codegen/ast_/nodes_filter.py

@@ -0,0 +1,151 @@
+from dataclasses import dataclass
+from typing import TypedDict, ClassVar
+
+from ssc_codegen.ast_.base import BaseAstNode, T_EMPTY_KWARGS
+from ssc_codegen.tokens import TokenType, VariableType
+
+
+# TODO: static analyzer for filter operations
+# TODO: implement operators for DOCUMENT, INT, FLOAT
+
+
+@dataclass(kw_only=True)
+class FilterOr(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_OR
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+    # TODO: typing body accept Filter-like nodes
+
+
+@dataclass(kw_only=True)
+class FilterAnd(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_AND
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+    # TODO: typing body accept Filter-like nodes
+
+
+@dataclass(kw_only=True)
+class FilterNot(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_NOT
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+    # TODO: typing body accept Filter-like nodes
+
+
+KW_STR_IN = TypedDict("KW_STR_IN", {"substr": tuple[str, ...]})
+ARGS_STR_IN = tuple[tuple[str, ...]]
+
+KW_STR_STARTS_OR_ENDS = TypedDict(
+    "KW_STR_STARTS_OR_ENDS", {"substr": tuple[str, ...]}
+)
+ARGS_STR_STARTS_OR_ENDS = tuple[tuple[str, ...]]
+
+
+@dataclass(kw_only=True)
+class FilterStrIn(BaseAstNode[KW_STR_IN, ARGS_STR_IN]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_STR_IN
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+@dataclass(kw_only=True)
+class FilterStrStarts(
+    BaseAstNode[KW_STR_STARTS_OR_ENDS, ARGS_STR_STARTS_OR_ENDS]
+):
+    kind: ClassVar[TokenType] = TokenType.FILTER_STR_STARTS
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+@dataclass(kw_only=True)
+class FilterStrEnds(
+    BaseAstNode[KW_STR_STARTS_OR_ENDS, ARGS_STR_STARTS_OR_ENDS]
+):
+    kind: ClassVar[TokenType] = TokenType.FILTER_STR_ENDS
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+KW_STR_RE = TypedDict("KW_STR_RE", {"pattern": str, "ignore_case": bool})
+ARGS_STR_RE = tuple[str, bool]
+
+
+@dataclass(kw_only=True)
+class FilterStrRe(BaseAstNode[KW_STR_RE, ARGS_STR_RE]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_STR_RE
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+KW_STR_LEN = TypedDict("KW_STR_LEN", {"length": int})
+ARGS_STR_LEN = tuple[int]
+
+
+@dataclass(kw_only=True)
+class FilterStrLenEq(BaseAstNode[KW_STR_LEN, ARGS_STR_LEN]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_STR_LEN_EQ
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+@dataclass(kw_only=True)
+class FilterStrLenNe(BaseAstNode[KW_STR_LEN, ARGS_STR_LEN]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_STR_LEN_NE
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+@dataclass(kw_only=True)
+class FilterStrLenLt(BaseAstNode[KW_STR_LEN, ARGS_STR_LEN]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_STR_LEN_LT
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+@dataclass(kw_only=True)
+class FilterStrLenLe(BaseAstNode[KW_STR_LEN, ARGS_STR_LEN]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_STR_LEN_LE
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+@dataclass(kw_only=True)
+class FilterStrLenGt(BaseAstNode[KW_STR_LEN, ARGS_STR_LEN]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_STR_LEN_GT
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+@dataclass(kw_only=True)
+class FilterStrLenGe(BaseAstNode[KW_STR_LEN, ARGS_STR_LEN]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_STR_LEN_GE
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+# TODO: provide API for int, float, etc
+KW_STR_EQ_OR_NE = TypedDict("KW_STR_EQ_OR_NE", {"values": tuple[str, ...]})
+ARGS_STR_EQ_OR_NE = tuple[tuple[str, ...]]
+
+
+@dataclass(kw_only=True)
+class FilterEqual(BaseAstNode[KW_STR_EQ_OR_NE, ARGS_STR_EQ_OR_NE]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_EQ
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+@dataclass(kw_only=True)
+class FilterNotEqual(BaseAstNode[KW_STR_EQ_OR_NE, ARGS_STR_EQ_OR_NE]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_NE
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+@dataclass(kw_only=True)
+class ExprFilter(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    kind: ClassVar[TokenType] = TokenType.EXPR_FILTER
+    accept_type: VariableType = VariableType.LIST_STRING
+    ret_type: VariableType = VariableType.LIST_STRING
+    # TODO: typing body accept Filter-like nodes

{ssc_codegen-0.8.16 → ssc_codegen-0.8.18}/ssc_codegen/ast_build/main.py

@@ -1,6 +1,6 @@
 import warnings
 from pathlib import Path
-from typing import Type, cast
+from typing import Type, cast, MutableSequence
 
 from ssc_codegen import Json
 from ssc_codegen.ast_ import (
@@ -230,6 +230,15 @@ def build_ast_struct_parser(
     return st
 
 
+def set_parent_for_expr_filter(
+    parent: BaseAstNode, parent_body: MutableSequence[BaseAstNode]
+) -> None:
+    for j in parent_body:
+        j.parent = parent
+        if j.body:
+            set_parent_for_expr_filter(j, j.body)
+
+
 def _fetch_field_nodes(
     body: list[BaseAstNode],
     body_parse_method_expr: list[BaseAstNode],
@@ -263,8 +272,12 @@ def _fetch_field_nodes(
             )
 
         _unwrap_default_node(document, ret_type)
+
         for i in document.stack:
             i.parent = method
+            if i.kind == TokenType.EXPR_FILTER:
+                set_parent_for_expr_filter(i, i.body)
+
         method.body = document.stack
 
         body.append(method)
@@ -361,6 +374,9 @@ def _try_fetch_split_doc_node(
             )
         for i in split_doc.stack:
             i.parent = method
+            if i.kind == TokenType.EXPR_FILTER:
+                set_parent_for_expr_filter(i, i.body)
+
         method.body = split_doc.stack
         body.append(method)
 
@@ -388,6 +404,8 @@ def _try_fetch_pre_validate_node(
         validate_field.stack.append(ExprNoReturn())
         for i in validate_field.stack:
             i.parent = method
+            if i.kind == TokenType.EXPR_FILTER:
+                set_parent_for_expr_filter(i, i.body)
 
         body.append(method)
         call_methods_expr.append(

{ssc_codegen-0.8.16 → ssc_codegen-0.8.18}/ssc_codegen/converters/base.py

@@ -14,6 +14,9 @@ from ssc_codegen.ast_ import (
     StructInitMethod,
     StructPartDocMethod,
     StructPreValidateMethod,
+    FilterOr,
+    FilterAnd,
+    FilterNot,
 )
 from ssc_codegen.tokens import TokenType
 
@@ -168,12 +171,22 @@ class BaseCodeConverter:
             case TokenType.JSON_FIELD:
                 ast_entry = cast(JsonStructField, ast_entry)
                 self._convert_json_struct_field(ast_entry, acc)
-            case _ if ast_entry.kind in {
-                TokenType.STRUCT_PART_DOCUMENT,
-                TokenType.STRUCT_PRE_VALIDATE,
-                TokenType.STRUCT_PARSE_START,
-                TokenType.STRUCT_FIELD,
-            }:
+
+            case (
+                TokenType.EXPR_FILTER
+                | TokenType.FILTER_OR
+                | TokenType.FILTER_AND
+                | TokenType.FILTER_NOT
+            ):
+                ast_entry = cast(FilterOr | FilterAnd | FilterNot, ast_entry)
+                self._convert_logic_filter(ast_entry, acc)
+
+            case (
+                TokenType.STRUCT_PART_DOCUMENT
+                | TokenType.STRUCT_PRE_VALIDATE
+                | TokenType.STRUCT_PARSE_START
+                | TokenType.STRUCT_FIELD
+            ):
                 ast_entry = cast(
                     StructFieldMethod
                     | StructInitMethod
@@ -186,6 +199,18 @@ class BaseCodeConverter:
                 self._convert_default(ast_entry, acc)
         return acc
 
+    def _convert_logic_filter(
+        self, ast_entry: FilterOr | FilterAnd | FilterNot, acc: list[str]
+    ) -> None:
+        # AND(body=[F1, F2]) ->
+        # AND ( (OPEN)
+        #       F1 and F2
+        # ) (CLOSE)
+        acc.append(self._pre_convert_node(ast_entry))
+        for node in ast_entry.body:
+            self.convert(node, acc)
+        acc.append(self._post_convert_node(ast_entry))
+
     def _convert_module(self, ast_entry: ModuleProgram, acc: list[str]) -> None:
         """Handle module conversion."""
         for node in ast_entry.body:

{ssc_codegen-0.8.16 → ssc_codegen-0.8.18}/ssc_codegen/converters/go_goquery.py

@@ -1262,3 +1262,4 @@ def pre_list_str_rm_prefix_and_suffix(
 
 
 # TODO impl ExprHasAttr, ExprListHasAttr
+# TODO: impl Filter Expr

{ssc_codegen-0.8.16 → ssc_codegen-0.8.18}/ssc_codegen/converters/helpers.py

@@ -47,3 +47,12 @@ def get_struct_field_method_by_name(
     ][0]
     result = cast(StructFieldMethod, result)
     return result
+
+
+def is_prev_node_atomic_cond(node: BaseAstNode) -> bool:
+    """return true if node is atomic condition (exclude body), NOT FIRST"""
+    return node.index != 0 and not node.parent.body[node.index_prev].body
+
+
+def is_first_node_cond(node: BaseAstNode) -> bool:
+    return node.index == 0 and node.parent.kind == TokenType.EXPR_FILTER

{ssc_codegen-0.8.16 → ssc_codegen-0.8.18}/ssc_codegen/converters/js_pure.py

@@ -85,12 +85,30 @@ from ssc_codegen.ast_ import (
     ExprListStringAllRegex,
     ExprListHasAttr,
     ExprHasAttr,
+    FilterEqual,
+    FilterNotEqual,
+    FilterStrRe,
+    FilterStrEnds,
+    FilterStrStarts,
+    FilterStrIn,
+    FilterNot,
+    FilterAnd,
+    FilterOr,
+    ExprFilter,
+    FilterStrLenEq,
+    FilterStrLenNe,
+    FilterStrLenLt,
+    FilterStrLenLe,
+    FilterStrLenGt,
+    FilterStrLenGe,
 )
 from ssc_codegen.converters.base import BaseCodeConverter
 from ssc_codegen.converters.helpers import (
     prev_next_var,
     is_last_var_no_ret,
     have_pre_validate_call,
+    is_first_node_cond,
+    is_prev_node_atomic_cond,
 )
 from ssc_codegen.converters.templates.js_pure import (
     J2_STRUCT_INIT,
@@ -130,11 +148,15 @@ CONVERTER = BaseCodeConverter(debug_comment_prefix="// ")
 
 
 # TODO: move to string_utils
-def to_js_regexp(pattern: str, ignore_case: bool = False) -> str:
+def to_js_regexp(
+    pattern: str, ignore_case: bool = False, is_global: bool = True
+) -> str:
     """helper function for convert string pattern to js"""
     pattern = pattern.replace("<\\/", "</")
     pattern = pattern.replace("/", "\\/")
-    pattern = f"/{pattern}/g"
+    pattern = f"/{pattern}/"
+    if is_global:
+        pattern += "g"
     if ignore_case:
         pattern += "i"
     return pattern
@@ -368,8 +390,8 @@ def pre_list_str_replace(node: ExprListStringReplace) -> str:
 def pre_str_regex(node: ExprStringRegex) -> str:
     prv, nxt = prev_next_var(node)
     pattern, group, ignore_case = node.unpack_args()
-    pattern = to_js_regexp(pattern, ignore_case)
-    return f"let {nxt} = (new RegExp({pattern})).exec({prv})[{group}];"
+    pattern = to_js_regexp(pattern, ignore_case, is_global=False)
+    return f"let {nxt} = {prv}.match({pattern})[{group}];"
 
 
 @CONVERTER(ExprStringRegexAll.kind)
@@ -377,8 +399,7 @@ def pre_str_regex_all(node: ExprStringRegexAll) -> str:
     prv, nxt = prev_next_var(node)
     pattern, ignore_case = node.unpack_args()
     pattern = to_js_regexp(pattern, ignore_case)
-
-    return f"let {nxt} = (new RegExp({pattern})).exec({prv});"
+    return f"let {nxt} = Array.from({prv}.match({pattern}));"
 
 
 @CONVERTER(ExprStringRegexSub.kind)
@@ -714,3 +735,182 @@ def pre_list_has_attr(node: ExprListHasAttr) -> str:
     if is_last_var_no_ret(node):
         return expr
     return expr + f"let {nxt} = {prv};"
+
+
+@CONVERTER(ExprFilter.kind)
+def pre_expr_filter(node: ExprFilter) -> str:
+    prv, nxt = prev_next_var(node)
+    return f"let {nxt} = {prv}.filter(i => "
+
+
+@CONVERTER.post(ExprFilter.kind)
+def post_expr_filter(_node: ExprFilter) -> str:
+    return ");"
+
+
+@CONVERTER(FilterOr.kind)
+def pre_filter_or(_node: FilterOr) -> str:
+    return " || ("
+
+
+@CONVERTER.post(FilterOr.kind)
+def post_filter_or(_node: FilterOr) -> str:
+    return ")"
+
+
+@CONVERTER(FilterAnd.kind)
+def pre_filter_and(_node: FilterAnd) -> str:
+    return " && ("
+
+
+@CONVERTER.post(FilterAnd.kind)
+def post_filter_and(_node: FilterAnd) -> str:
+    return ")"
+
+
+@CONVERTER(FilterNot.kind)
+def pre_filter_not(_node: FilterNot) -> str:  # type: ignore
+    return "!("
+
+
+@CONVERTER.post(FilterNot.kind)
+def post_filter_not(_node: FilterNot) -> str:
+    return ")"
+
+
+@CONVERTER(FilterStrIn.kind)
+def pre_filter_in(node: FilterStrIn) -> str:
+    values, *_ = node.unpack_args()
+    if len(values) == 1:
+        expr = f"i.includes({values[0]!r})"
+    else:
+        # to js array
+        val_arr = str(values)
+        val_arr = "[" + val_arr[1:-1] + "]"
+        expr = f"{val_arr}.some(e => i.includes(e))"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterStrStarts.kind)
+def pre_filter_starts_with(node: FilterStrStarts) -> str:
+    values, *_ = node.unpack_args()
+    if len(values) == 1:
+        expr = f"i.startsWith({values[0]!r})"
+    else:
+        # to js array
+        val_arr = str(values)
+        val_arr = "[" + val_arr[1:-1] + "]"
+        expr = f"{val_arr}.some(e => i.startsWith(e))"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterStrEnds.kind)
+def pre_filter_ends_with(node: FilterStrEnds) -> str:
+    values, *_ = node.unpack_args()
+    if len(values) == 1:
+        expr = f"i.endsWith({values[0]!r})"
+    else:
+        # to js array
+        val_arr = str(values)
+        val_arr = "[" + val_arr[1:-1] + "]"
+        expr = f"{val_arr}.some(e => i.endsWith(e))"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterStrRe.kind)
+def pre_filter_re(node: FilterStrRe) -> str:
+    pattern, ignore_case, *_ = node.unpack_args()
+    pattern = to_js_regexp(pattern, ignore_case, is_global=False)
+
+    expr = f"(new RegExp({pattern})).test(i)"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterEqual.kind)
+def pre_filter_eq(node: FilterEqual) -> str:
+    values, *_ = node.unpack_args()
+    if len(values) == 1:
+        expr = f"i === {values[0]!r}"
+    else:
+        val_arr = str(values)
+        val_arr = "[" + val_arr[1:-1] + "]"
+        expr = f"{val_arr}.every(e => i === e)"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterNotEqual.kind)
+def pre_filter_ne(node: FilterNotEqual) -> str:
+    values, *_ = node.unpack_args()
+    if len(values) == 1:
+        expr = f"i !== {values[0]!r}"
+    else:
+        val_arr = str(values)
+        val_arr = "[" + val_arr[1:-1] + "]"
+        expr = f"{val_arr}.every(e => i !== e)"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterStrLenEq.kind)
+def pre_filter_str_len_eq(node: FilterStrLenEq) -> str:
+    length, *_ = node.unpack_args()
+    expr = f"i.length == {length}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterStrLenNe.kind)
+def pre_filter_str_len_ne(node: FilterStrLenNe) -> str:
+    length, *_ = node.unpack_args()
+    expr = f"i.length != {length}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterStrLenLt.kind)
+def pre_filter_str_len_lt(node: FilterStrLenLt) -> str:
+    length, *_ = node.unpack_args()
+    expr = f"i.length < {length}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterStrLenLe.kind)
+def pre_filter_str_len_le(node: FilterStrLenLe) -> str:
+    length, *_ = node.unpack_args()
+    expr = f"i.length <= {length}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterStrLenGt.kind)
+def pre_filter_str_len_gt(node: FilterStrLenGt) -> str:
+    length, *_ = node.unpack_args()
+    expr = f"i.length > {length}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterStrLenGe.kind)
+def pre_filter_str_len_ge(node: FilterStrLenGe) -> str:
+    length, *_ = node.unpack_args()
+    expr = f"i.length >= {length}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr

{ssc_codegen-0.8.16 → ssc_codegen-0.8.18}/ssc_codegen/converters/py_base.py

@@ -87,6 +87,22 @@ from ssc_codegen.ast_ import (
     ExprListStringRmPrefixAndSuffix,
     ExprListStringAnyRegex,
     ExprListStringAllRegex,
+    FilterOr,
+    FilterAnd,
+    FilterNot,
+    ExprFilter,
+    FilterStrIn,
+    FilterStrStarts,
+    FilterStrEnds,
+    FilterStrRe,
+    FilterEqual,
+    FilterNotEqual,
+    FilterStrLenEq,
+    FilterStrLenNe,
+    FilterStrLenLt,
+    FilterStrLenLe,
+    FilterStrLenGt,
+    FilterStrLenGe,
 )
 from ssc_codegen.converters.base import (
     BaseCodeConverter,
@@ -98,6 +114,8 @@ from ssc_codegen.converters.helpers import (
     have_pre_validate_call,
     prev_next_var,
     is_last_var_no_ret,
+    is_prev_node_atomic_cond,
+    is_first_node_cond,
 )
 from ssc_codegen.tokens import StructType, VariableType, JsonVariableType
 
@@ -181,6 +199,8 @@ class BasePyCodeConverter(BaseCodeConverter):
         )
         from ssc_codegen.ast_ import ExprListStringRmPrefix
 
+        from ssc_codegen.ast_ import FilterNotEqual
+
         self.pre_definitions = {
             Docstring.kind: pre_docstring,
             ModuleImports.kind: pre_imports,
@@ -234,12 +254,33 @@ class BasePyCodeConverter(BaseCodeConverter):
             ExprToListLength.kind: pre_to_len,
             ExprToBool.kind: pre_to_bool,
             ExprJsonify.kind: pre_jsonify,
+            # FILTER,
+            ExprFilter.kind: pre_expr_filter,
+            FilterAnd.kind: pre_filter_and,
+            FilterOr.kind: pre_filter_or,
+            FilterNot.kind: pre_filter_not,
+            FilterStrStarts.kind: pre_filter_starts_with,
+            FilterStrEnds.kind: pre_filter_ends_with,
+            FilterStrIn.kind: pre_filter_in,
+            FilterStrRe.kind: pre_filter_re,
+            FilterEqual.kind: pre_filter_eq,
+            FilterNotEqual.kind: pre_filter_ne,
+            FilterStrLenEq.kind: pre_filter_str_len_eq,
+            FilterStrLenNe.kind: pre_filter_str_len_ne,
+            FilterStrLenLt.kind: pre_filter_str_len_lt,
+            FilterStrLenLe.kind: pre_filter_str_len_le,
+            FilterStrLenGe.kind: pre_filter_str_len_ge,
+            FilterStrLenGt.kind: pre_filter_str_len_gt,
         }
 
         self.post_definitions = {
             TypeDef.kind: post_typedef,
             JsonStruct.kind: post_json_struct,
             StartParseMethod.kind: post_start_parse,
+            ExprFilter.kind: post_expr_filter,
+            FilterAnd.kind: post_filter_and,
+            FilterOr.kind: post_filter_or,
+            FilterNot.kind: post_filter_not,
         }
 
 
@@ -876,3 +917,155 @@ def pre_jsonify(node: ExprJsonify) -> str:
     prv, nxt = prev_next_var(node)
     _name, _is_array = node.unpack_args()
     return indent + f"{nxt} = json.loads({prv})"
+
+
+# FILTERS
+
+
+def pre_expr_filter(node: ExprFilter) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    return indent + f"{nxt} = [i for i in {prv} if "
+
+
+def post_expr_filter(_node: ExprFilter) -> str:
+    return "]"
+
+
+def pre_filter_or(_node: FilterOr) -> str:
+    return " or ("
+
+
+def post_filter_or(_node: FilterOr) -> str:
+    return ")"
+
+
+def pre_filter_and(_node: FilterAnd) -> str:
+    return " and ("
+
+
+def post_filter_and(_node: FilterAnd) -> str:
+    return ")"
+
+
+def pre_filter_not(_node: FilterNot) -> str:
+    return "not ("
+
+
+def post_filter_not(_node: FilterNot) -> str:
+    return ")"
+
+
+def pre_filter_in(node: FilterStrIn) -> str:
+    values, *_ = node.unpack_args()
+
+    if len(values) == 1:
+        expr = f"{values[0]!r} in i"
+    else:
+        expr = f"any(s in i for s in {values})"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_starts_with(node: FilterStrStarts) -> str:
+    start_, *_ = node.unpack_args()
+    # build-in python startswith accept tuple[str, ...]
+    expr = f"i.startswith({start_})"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_ends_with(node: FilterStrEnds) -> str:
+    suffix_, *_ = node.unpack_args()
+    # build-in python endswith accept tuple[str, ...]
+    expr = f"i.endswith({suffix_})"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_re(node: FilterStrRe) -> str:
+    pattern, ignore_case, *_ = node.unpack_args()
+    if ignore_case:
+        expr = f"re.search({pattern!r}, i, re.IGNORECASE)"
+    else:
+        expr = f"re.search({pattern!r}, i)"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_eq(node: FilterEqual) -> str:
+    values, *_ = node.unpack_args()
+    # currently support only str
+    if len(values) == 1:
+        expr = f"i == {values[0]!r}"
+    else:
+        expr = f"all(s == i for s in {values})"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_ne(node: FilterNotEqual) -> str:
+    values, *_ = node.unpack_args()
+    # currently support only str
+    if len(values) == 1:
+        expr = f"i != {values[0]!r}"
+    else:
+        expr = f"all(s != i for s in {values})"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_str_len_eq(node: FilterStrLenEq) -> str:
+    length, *_ = node.unpack_args()
+    expr = f"len(i) == {length}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_str_len_ne(node: FilterStrLenNe) -> str:
+    length, *_ = node.unpack_args()
+    expr = f"len(i) != {length}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_str_len_lt(node: FilterStrLenLt) -> str:
+    length, *_ = node.unpack_args()
+    expr = f"len(i) < {length}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_str_len_le(node: FilterStrLenLe) -> str:
+    length, *_ = node.unpack_args()
+    expr = f"len(i) <= {length}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_str_len_gt(node: FilterStrLenGt) -> str:
+    length, *_ = node.unpack_args()
+    expr = f"len(i) > {length}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_str_len_ge(node: FilterStrLenGe) -> str:
+    length, *_ = node.unpack_args()
+    expr = f"len(i) >= {length}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr

{ssc_codegen-0.8.16 → ssc_codegen-0.8.18}/ssc_codegen/document.py

@@ -1,7 +1,7 @@
 """high-level AST builder interface"""
 
 import logging
-from typing import Type, Pattern
+from typing import Type, Pattern, Sequence
 
 from cssselect import SelectorSyntaxError
 from typing_extensions import Self, assert_never
@@ -59,18 +59,34 @@ from ssc_codegen.ast_ import (
     ExprListStringAnyRegex,
     ExprHasAttr,
     ExprListHasAttr,
+    ExprFilter,
+    FilterEqual,
+    FilterNotEqual,
+    FilterStrStarts,
+    FilterStrEnds,
+    FilterStrIn,
+    FilterStrRe,
+    FilterAnd,
+    FilterOr,
+    FilterNot,
+    FilterStrLenEq,
+    FilterStrLenNe,
+    FilterStrLenLt,
+    FilterStrLenLe,
+    FilterStrLenGt,
+    FilterStrLenGe,
 )
 from ssc_codegen.document_utlis import (
     analyze_re_expression,
     unverbosify_regex,
     is_ignore_case_regex,
 )
+from ssc_codegen.json_struct import Json
 from ssc_codegen.pseudo_selectors import (
     parse_pseudo_xpath_query,
     parse_pseudo_css_query,
     PseudoAction,
 )
-from ssc_codegen.json_struct import Json
 from ssc_codegen.schema import BaseSchema
 from ssc_codegen.selector_utils import validate_css_query, validate_xpath_query
 from ssc_codegen.tokens import VariableType
@@ -457,6 +473,20 @@ class ArrayDocument(BaseDocument):
         self._add(ExprToListLength(accept_type=expected_type))
         return self
 
+    def filter(self, expr: "DocumentFilter") -> Self:
+        """filter array of resuts by expr
+
+        - accept LIST_STRING, return LIST_STRING
+        """
+        if self.stack_last_ret != VariableType.LIST_STRING:
+            LOGGER.warning(
+                "to_len(): Expected type(s) %s got %s",
+                (VariableType.LIST_STRING.name,),
+                self.stack_last_ret.name,
+            )
+        self._add(ExprFilter(body=expr.stack))
+        return self
+
 
 class StringDocument(BaseDocument):
     def rm_prefix(self, substr: str) -> Self:
@@ -1236,3 +1266,181 @@ class JsonDocument(BaseDocument):
         # HACK: store to class instance for generate docstring signature
         BaseSchema.__JSON_SCHEMAS__[struct.__name__] = struct  # type: ignore
         return self
+
+
+class DocumentFilter(BaseDocument):
+    def eq(self, *values: str) -> Self:
+        """check if value equal
+
+        multiple values converts to logical AND
+
+        pseudocode example:
+
+            F().eq("foo") -> value == "foo"
+
+            F().eq("bar", "foo") -> (value == "bar" && value == "foo")
+        """
+        self._add(FilterEqual(kwargs={"values": values}))
+        return self
+
+    def ne(self, *values: str) -> Self:
+        """check if value not equal
+
+        multiple values converts to logical AND
+
+        pseudocode example:
+
+            F().eq("foo") -> value == "foo"
+
+            F().eq("bar", "foo") -> (value != "bar" && value != "foo")
+        """
+        self._add(FilterNotEqual(kwargs={"values": values}))
+        return self
+
+    def starts(self, *values: str) -> Self:
+        """check if value starts by substring
+
+        multiple values converts to logical OR
+
+        pseudocode example:
+
+            F().eq("foo") -> value == "foo"
+
+            F().eq("foo", "bar") -> (value.starts_with("bar") || value.starts_with("foo"))
+        """
+        self._add(FilterStrStarts(kwargs={"substr": values}))
+        return self
+
+    def ends(self, *values: str) -> Self:
+        """check if value starts by substring
+
+        multiple values converts to logical OR
+
+        pseudocode example:
+
+            F().eq("foo") -> value == "foo"
+
+            F().eq("foo", "bar") -> (value.ends_with("bar") || value.ends_with("foo"))
+        """
+        self._add(FilterStrEnds(kwargs={"substr": values}))
+        return self
+
+    def contains(self, *values: str) -> Self:
+        """check if value contains by substring
+
+        multiple values converts to logical OR
+
+        pseudocode example:
+
+            F().eq("foo") -> value == "foo"
+
+            F().eq("foo", "bar") -> (value.include("bar") || value.include("foo"))
+        """
+        self._add(FilterStrIn(kwargs={"substr": values}))
+        return self
+
+    def re(
+        self, pattern: str | Pattern[str], ignore_case: bool = False
+    ) -> Self:
+        """check if pattern matched result in value"""
+        if not isinstance(pattern, str):
+            ignore_case = is_ignore_case_regex(pattern)
+
+        pattern = unverbosify_regex(pattern)
+        result = analyze_re_expression(pattern, allow_empty_groups=True)
+        if not result:
+            LOGGER.warning(result.msg)
+        self._add(
+            FilterStrRe(kwargs={"pattern": pattern, "ignore_case": ignore_case})
+        )
+        return self
+
+    def and_(self, filter_expr: "DocumentFilter") -> Self:
+        if self.stack[0].kind in (FilterAnd.kind, FilterOr.kind):
+            LOGGER.warning(
+                "logic AND: first node is `%s`", self.stack[0].kind.name
+            )
+        tmp_stack = filter_expr.stack.copy()
+        self._add(FilterAnd(body=tmp_stack))
+        return self
+
+    def or_(self, filter_expr: "DocumentFilter") -> Self:
+        if self.stack[0].kind in (FilterAnd.kind, FilterOr.kind):
+            LOGGER.warning(
+                "logic OR: first node is `%s`", self.stack[0].kind.name
+            )
+        tmp_stack = filter_expr.stack.copy()
+        self._add(FilterOr(body=tmp_stack))
+        return self
+
+    def len_eq(self, value: int) -> Self:
+        self._add(FilterStrLenEq(kwargs={"length": value}))
+        return self
+
+    def len_ne(self, value: int) -> Self:
+        self._add(FilterStrLenNe(kwargs={"length": value}))
+        return self
+
+    def len_lt(self, value: int) -> Self:
+        self._add(FilterStrLenLt(kwargs={"length": value}))
+        return self
+
+    def len_le(self, value: int) -> Self:
+        self._add(FilterStrLenLe(kwargs={"length": value}))
+        return self
+
+    def len_gt(self, value: int) -> Self:
+        self._add(FilterStrLenGt(kwargs={"length": value}))
+        return self
+
+    def len_ge(self, value: int) -> Self:
+        self._add(FilterStrLenGe(kwargs={"length": value}))
+        return self
+
+    def not_(self, filter_expr: "DocumentFilter") -> Self:
+        tmp_stack = filter_expr.stack.copy()
+        self._add(FilterNot(body=tmp_stack))
+        return self
+
+    def __or__(self, other: "DocumentFilter") -> "DocumentFilter":
+        """syntax suger F().or_(...)"""
+
+        return self.or_(other)
+
+    def __and__(self, other: "DocumentFilter") -> "DocumentFilter":
+        """syntax sugar F().and_(...)"""
+        return self.and_(other)
+
+    def __invert__(self) -> "DocumentFilter":
+        """syntax sugar F().not_(...)"""
+        return DocumentFilter().not_(self)
+
+    def __eq__(self, other: int | str | Sequence[str]) -> Self:
+        """syntax sugar F().eq(...)"""
+        if isinstance(other, int):
+            return self.len_eq(other)
+
+        if isinstance(other, str):
+            other = (other,)
+        return self.eq(*other)
+
+    def __ne__(self, other: int | str | Sequence[str]) -> Self:
+        """syntax sugar F().ne(...)"""
+        if isinstance(other, int):
+            return self.len_ne(other)
+
+        if isinstance(other, str):
+            other = (other,)
+        return self.eq(*other)
+
+    def __lt__(self, other: int) -> Self:
+        return self.len_lt(other)
+
+    def __le__(self, other: int) -> Self:
+        return self.len_le(other)
+
+    def __gt__(self, other: int) -> Self:
+        return self.len_gt(other)
+
+    def __ge__(self, other: int) -> Self:
+        return self.len_ge(other)

{ssc_codegen-0.8.16 → ssc_codegen-0.8.18}/ssc_codegen/str_utils.py

@@ -103,7 +103,7 @@ RE_PY_METHOD_BLOCK = re.compile(
     r"""
 def\s_(?:parse|split)\w+\(.*\:\s*   # SPLIT_DOC STRUCT_PARSE HEAD
 (?:\n|.)*?                          # BLOCK OF CODE
-return\s(?P<ret_var>\w+)    # RET STMT 
+return\s(?P<ret_var>\w+)            # RET STMT 
 """,
     re.X,
 )
@@ -148,6 +148,9 @@ def py_optimize_return_naive(py_code: str) -> str:
         re_expr = f"{ret_var}\\s*=\\s*(?P<expr>.*)"
         if match := re.search(re_expr, method_block[0]):
             expr = match["expr"]  # noqa
+            # NAIVE apologize, its .filter(...) expr, skip optimization
+            if expr.strip().endswith(" if"):
+                continue
 
             # optimization function convert double backslash to single (\\ -> \) in regex patterns
             # add `r` prefix for avoid SyntaxWarning then compile generated code to cpython bytecode

{ssc_codegen-0.8.16 → ssc_codegen-0.8.18}/ssc_codegen/tokens.py

@@ -174,6 +174,28 @@ class TokenType(IntEnum):
     # bool
     TO_BOOL = auto()
 
+    # FILTER
+    EXPR_FILTER = auto()
+
+    FILTER_OR = auto()
+    FILTER_AND = auto()
+    FILTER_NOT = auto()
+    # FILTER EXPR (STR)
+    FILTER_STR_IN = auto()
+    FILTER_STR_STARTS = auto()
+    FILTER_STR_ENDS = auto()
+    FILTER_STR_RE = auto()
+
+    FILTER_STR_LEN_EQ = auto()
+    FILTER_STR_LEN_NE = auto()
+    FILTER_STR_LEN_LT = auto()
+    FILTER_STR_LEN_LE = auto()
+    FILTER_STR_LEN_GT = auto()
+    FILTER_STR_LEN_GE = auto()
+
+    FILTER_EQ = auto()
+    FILTER_NE = auto()
+
     @classmethod
     def default_tokens(cls) -> tuple["TokenType", ...]:
         return (