ssc_codegen 0.8.15__tar.gz → 0.8.17__tar.gz

{ssc_codegen-0.8.15 → ssc_codegen-0.8.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.8.15
+Version: 0.8.17
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues

{ssc_codegen-0.8.15 → ssc_codegen-0.8.17}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.8.15"
+version = "0.8.17"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.8.15 → ssc_codegen-0.8.17}/ssc_codegen/__init__.py

@@ -1,13 +1,13 @@
 from ssc_codegen.document import HTMLDocument, StringDocument, ArrayDocument, AssertDocument, NestedDocument, \
     DefaultDocument, \
-    NumericDocument, JsonDocument, BooleanDocument
+    NumericDocument, JsonDocument, BooleanDocument, DocumentFilter
 from ssc_codegen.json_struct import Json
 from ssc_codegen.logs import setup_logger
 from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchema
 
 setup_logger()
 
-VERSION = "0.8.15"
+VERSION = "0.8.17"
 
 
 class __MISSING(object):
@@ -59,3 +59,8 @@ def R(default: None | str | int | float | list | __MISSING = _NO_DEFAULT) -> Doc
     if default == _NO_DEFAULT:
         return Document().raw()
     return Document().default(default).raw()  # type: ignore
+
+
+def F() -> DocumentFilter:
+    """Shortcut as a DocumentFilter() object"""
+    return DocumentFilter()

{ssc_codegen-0.8.15 → ssc_codegen-0.8.17}/ssc_codegen/ast_/__init__.py

@@ -20,9 +20,10 @@ from .nodes_core import (ExprReturn,
                          TypeDef,
                          TypeDefField,
                          ExprCallStructMethod)
+from .nodes_filter import Filter, FilterOr, FilterAnd, FilterNot, FilterNotEqual, FilterEqual, FilterStrIn, \
+    FilterStrStarts, FilterStrEnds, FilterStrRe, ExprFilter
 from .nodes_selectors import ExprCss, ExprCssAll, ExprXpathAll, ExprXpath, ExprGetHtmlText, ExprGetHtmlRaw, \
     ExprGetHtmlAttr, ExprGetHtmlAttrAll, ExprGetHtmlRawAll, ExprGetHtmlTextAll
-
 from .nodes_string import ExprStringTrim, ExprStringRegex, ExprStringSplit, ExprStringReplace, ExprStringLeftTrim, \
     ExprStringFormat, ExprListStringTrim, ExprStringRegexAll, ExprStringRegexSub, ExprStringRightTrim, \
     ExprListStringFormat, ExprListStringReplace, ExprListStringRightTrim, ExprListStringLeftTrim, \

ssc_codegen-0.8.17/ssc_codegen/ast_/nodes_filter.py

@@ -0,0 +1,110 @@
+from dataclasses import dataclass
+from typing import TypedDict, ClassVar
+
+from ssc_codegen.ast_.base import BaseAstNode, T_EMPTY_KWARGS
+from ssc_codegen.tokens import TokenType, VariableType
+
+
+# TODO: static analyzer for filter operations
+# TODO: implement operators for DOCUMENT, INT, FLOAT
+@dataclass(kw_only=True)
+class Filter(BaseAstNode):
+    kind: ClassVar[TokenType] = TokenType.FILTER
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+@dataclass(kw_only=True)
+class FilterOr(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_OR
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+    # TODO: typing body accept Filter-like nodes
+
+
+@dataclass(kw_only=True)
+class FilterAnd(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_AND
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+    # TODO: typing body accept Filter-like nodes
+
+
+@dataclass(kw_only=True)
+class FilterNot(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_NOT
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+    # TODO: typing body accept Filter-like nodes
+
+
+KW_STR_IN = TypedDict("KW_STR_IN", {"substr": str})
+ARGS_STR_IN = tuple[str]
+
+KW_STR_STARTS_OR_ENDS = TypedDict(
+    "KW_STR_STARTS_OR_ENDS", {"substr": tuple[str, ...]}
+)
+ARGS_STR_STARTS_OR_ENDS = tuple[tuple[str, ...]]
+
+
+@dataclass(kw_only=True)
+class FilterStrIn(BaseAstNode[KW_STR_IN, ARGS_STR_IN]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_STR_IN
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+@dataclass(kw_only=True)
+class FilterStrStarts(
+    BaseAstNode[KW_STR_STARTS_OR_ENDS, ARGS_STR_STARTS_OR_ENDS]
+):
+    kind: ClassVar[TokenType] = TokenType.FILTER_STR_STARTS
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+@dataclass(kw_only=True)
+class FilterStrEnds(
+    BaseAstNode[KW_STR_STARTS_OR_ENDS, ARGS_STR_STARTS_OR_ENDS]
+):
+    kind: ClassVar[TokenType] = TokenType.FILTER_STR_ENDS
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+KW_STR_RE = TypedDict("KW_STR_RE", {"pattern": str, "ignore_case": bool})
+ARGS_STR_RE = tuple[str, bool]
+
+
+@dataclass(kw_only=True)
+class FilterStrRe(BaseAstNode[KW_STR_RE, ARGS_STR_RE]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_STR_RE
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+# TODO: provide API for int, float, etc
+KW_STR_EQ_OR_NE = TypedDict("KW_STR_EQ_OR_NE", {"value": str})
+ARGS_STR_EQ_OR_NE = tuple[str]
+
+
+@dataclass(kw_only=True)
+class FilterEqual(BaseAstNode[KW_STR_EQ_OR_NE, ARGS_STR_EQ_OR_NE]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_EQ
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+@dataclass(kw_only=True)
+class FilterNotEqual(BaseAstNode[KW_STR_EQ_OR_NE, ARGS_STR_EQ_OR_NE]):
+    kind: ClassVar[TokenType] = TokenType.FILTER_EQ
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+@dataclass(kw_only=True)
+class ExprFilter(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    kind: ClassVar[TokenType] = TokenType.EXPR_FILTER
+    accept_type: VariableType = VariableType.LIST_STRING
+    ret_type: VariableType = VariableType.LIST_STRING
+    # TODO: typing body accept Filter-like nodes

{ssc_codegen-0.8.15 → ssc_codegen-0.8.17}/ssc_codegen/ast_build/main.py

@@ -1,6 +1,6 @@
 import warnings
 from pathlib import Path
-from typing import Type, cast
+from typing import Type, cast, MutableSequence
 
 from ssc_codegen import Json
 from ssc_codegen.ast_ import (
@@ -230,6 +230,15 @@ def build_ast_struct_parser(
     return st
 
 
+def set_parent_for_expr_filter(
+    parent: BaseAstNode, parent_body: MutableSequence[BaseAstNode]
+) -> None:
+    for j in parent_body:
+        j.parent = parent
+        if j.body:
+            set_parent_for_expr_filter(j, j.body)
+
+
 def _fetch_field_nodes(
     body: list[BaseAstNode],
     body_parse_method_expr: list[BaseAstNode],
@@ -263,8 +272,12 @@ def _fetch_field_nodes(
             )
 
         _unwrap_default_node(document, ret_type)
+
         for i in document.stack:
             i.parent = method
+            if i.kind == TokenType.EXPR_FILTER:
+                set_parent_for_expr_filter(i, i.body)
+
         method.body = document.stack
 
         body.append(method)
@@ -361,6 +374,9 @@ def _try_fetch_split_doc_node(
             )
         for i in split_doc.stack:
             i.parent = method
+            if i.kind == TokenType.EXPR_FILTER:
+                set_parent_for_expr_filter(i, i.body)
+
         method.body = split_doc.stack
         body.append(method)
 
@@ -388,6 +404,8 @@ def _try_fetch_pre_validate_node(
         validate_field.stack.append(ExprNoReturn())
         for i in validate_field.stack:
             i.parent = method
+            if i.kind == TokenType.EXPR_FILTER:
+                set_parent_for_expr_filter(i, i.body)
 
         body.append(method)
         call_methods_expr.append(

{ssc_codegen-0.8.15 → ssc_codegen-0.8.17}/ssc_codegen/converters/base.py

@@ -14,6 +14,9 @@ from ssc_codegen.ast_ import (
     StructInitMethod,
     StructPartDocMethod,
     StructPreValidateMethod,
+    FilterOr,
+    FilterAnd,
+    FilterNot,
 )
 from ssc_codegen.tokens import TokenType
 
@@ -168,12 +171,22 @@ class BaseCodeConverter:
             case TokenType.JSON_FIELD:
                 ast_entry = cast(JsonStructField, ast_entry)
                 self._convert_json_struct_field(ast_entry, acc)
-            case _ if ast_entry.kind in {
-                TokenType.STRUCT_PART_DOCUMENT,
-                TokenType.STRUCT_PRE_VALIDATE,
-                TokenType.STRUCT_PARSE_START,
-                TokenType.STRUCT_FIELD,
-            }:
+
+            case (
+                TokenType.EXPR_FILTER
+                | TokenType.FILTER_OR
+                | TokenType.FILTER_AND
+                | TokenType.FILTER_NOT
+            ):
+                ast_entry = cast(FilterOr | FilterAnd | FilterNot, ast_entry)
+                self._convert_logic_filter(ast_entry, acc)
+
+            case (
+                TokenType.STRUCT_PART_DOCUMENT
+                | TokenType.STRUCT_PRE_VALIDATE
+                | TokenType.STRUCT_PARSE_START
+                | TokenType.STRUCT_FIELD
+            ):
                 ast_entry = cast(
                     StructFieldMethod
                     | StructInitMethod
@@ -186,6 +199,18 @@ class BaseCodeConverter:
                 self._convert_default(ast_entry, acc)
         return acc
 
+    def _convert_logic_filter(
+        self, ast_entry: FilterOr | FilterAnd | FilterNot, acc: list[str]
+    ) -> None:
+        # AND(body=[F1, F2]) ->
+        # AND ( (OPEN)
+        #       F1 and F2
+        # ) (CLOSE)
+        acc.append(self._pre_convert_node(ast_entry))
+        for node in ast_entry.body:
+            self.convert(node, acc)
+        acc.append(self._post_convert_node(ast_entry))
+
     def _convert_module(self, ast_entry: ModuleProgram, acc: list[str]) -> None:
         """Handle module conversion."""
         for node in ast_entry.body:

{ssc_codegen-0.8.15 → ssc_codegen-0.8.17}/ssc_codegen/converters/go_goquery.py

@@ -1262,3 +1262,4 @@ def pre_list_str_rm_prefix_and_suffix(
 
 
 # TODO impl ExprHasAttr, ExprListHasAttr
+# TODO: impl Filter Expr

{ssc_codegen-0.8.15 → ssc_codegen-0.8.17}/ssc_codegen/converters/helpers.py

@@ -47,3 +47,12 @@ def get_struct_field_method_by_name(
     ][0]
     result = cast(StructFieldMethod, result)
     return result
+
+
+def is_prev_node_atomic_cond(node: BaseAstNode) -> bool:
+    """return true if node is atomic condition (exclude body), NOT FIRST"""
+    return node.index != 0 and not node.parent.body[node.index_prev].body
+
+
+def is_first_node_cond(node: BaseAstNode) -> bool:
+    return node.index == 0 and node.parent.kind == TokenType.EXPR_FILTER

{ssc_codegen-0.8.15 → ssc_codegen-0.8.17}/ssc_codegen/converters/js_pure.py

@@ -85,12 +85,24 @@ from ssc_codegen.ast_ import (
     ExprListStringAllRegex,
     ExprListHasAttr,
     ExprHasAttr,
+    FilterEqual,
+    FilterNotEqual,
+    FilterStrRe,
+    FilterStrEnds,
+    FilterStrStarts,
+    FilterStrIn,
+    FilterNot,
+    FilterAnd,
+    FilterOr,
+    ExprFilter,
 )
 from ssc_codegen.converters.base import BaseCodeConverter
 from ssc_codegen.converters.helpers import (
     prev_next_var,
     is_last_var_no_ret,
     have_pre_validate_call,
+    is_first_node_cond,
+    is_prev_node_atomic_cond,
 )
 from ssc_codegen.converters.templates.js_pure import (
     J2_STRUCT_INIT,
@@ -129,6 +141,21 @@ DOCSTR_SEP = "* "
 CONVERTER = BaseCodeConverter(debug_comment_prefix="// ")
 
 
+# TODO: move to string_utils
+def to_js_regexp(
+    pattern: str, ignore_case: bool = False, is_global: bool = True
+) -> str:
+    """helper function for convert string pattern to js"""
+    pattern = pattern.replace("<\\/", "</")
+    pattern = pattern.replace("/", "\\/")
+    pattern = f"/{pattern}/"
+    if is_global:
+        pattern += "g"
+    if ignore_case:
+        pattern += "i"
+    return pattern
+
+
 def make_js_docstring(value: str) -> str:
     if not value:
         return ""
@@ -357,30 +384,23 @@ def pre_list_str_replace(node: ExprListStringReplace) -> str:
 def pre_str_regex(node: ExprStringRegex) -> str:
     prv, nxt = prev_next_var(node)
     pattern, group, ignore_case = node.unpack_args()
-    pattern = pattern.replace("/", "\\/")
-    pattern = f"/{pattern}/g"
-    if ignore_case:
-        pattern += "i"
-    return f"let {nxt} = (new RegExp({pattern})).exec({prv})[{group}];"
+    pattern = to_js_regexp(pattern, ignore_case, is_global=False)
+    return f"let {nxt} = {prv}.match({pattern})[{group}];"
 
 
 @CONVERTER(ExprStringRegexAll.kind)
 def pre_str_regex_all(node: ExprStringRegexAll) -> str:
     prv, nxt = prev_next_var(node)
     pattern, ignore_case = node.unpack_args()
-    pattern = pattern.replace("/", "\\/")
-    pattern = f"/{pattern}/g"
-    if ignore_case:
-        pattern += "i"
-    return f"let {nxt} = (new RegExp({pattern})).exec({prv});"
+    pattern = to_js_regexp(pattern, ignore_case)
+    return f"let {nxt} = Array.from({prv}.match({pattern}));"
 
 
 @CONVERTER(ExprStringRegexSub.kind)
 def pre_str_regex_sub(node: ExprStringRegexSub) -> str:
     prv, nxt = prev_next_var(node)
     pattern, repl = node.unpack_args()
-    pattern = pattern.replace("/", "\\/")
-    pattern = f"/{pattern}/g"
+    pattern = to_js_regexp(pattern)
 
     return f"let {nxt} = {prv}.replace({pattern}, {repl!r});"
 
@@ -389,8 +409,8 @@ def pre_str_regex_sub(node: ExprStringRegexSub) -> str:
 def pre_list_str_regex_sub(node: ExprListStringRegexSub) -> str:
     prv, nxt = prev_next_var(node)
     pattern, repl = node.unpack_args()
-    pattern = pattern.replace("/", "\\/")
-    pattern = f"/{pattern}/g"
+    pattern = to_js_regexp(pattern)
+
     return f"let {nxt} = {prv}.map(e => e.replace({pattern}, {repl!r}));"
 
 
@@ -456,10 +476,7 @@ def pre_is_contains(node: ExprIsContains) -> str:
 def pre_is_regex(node: ExprStringIsRegex) -> str:
     prv, nxt = prev_next_var(node)
     pattern, ignore_case, msg = node.unpack_args()
-    pattern = pattern.replace("/", "\\/")
-    pattern = f"/{pattern}/g"
-    if ignore_case:
-        pattern += "i"
+    pattern = to_js_regexp(pattern, ignore_case)
 
     expr = f"if ({prv}.match({pattern}) === null) throw new Error({msg!r});"
     if is_last_var_no_ret(node):
@@ -712,3 +729,106 @@ def pre_list_has_attr(node: ExprListHasAttr) -> str:
     if is_last_var_no_ret(node):
         return expr
     return expr + f"let {nxt} = {prv};"
+
+
+@CONVERTER(ExprFilter.kind)
+def pre_expr_filter(node: ExprFilter) -> str:
+    prv, nxt = prev_next_var(node)
+    return f"let {nxt} = {prv}.filter(i => "
+
+
+@CONVERTER.post(ExprFilter.kind)
+def post_expr_filter(_node: ExprFilter) -> str:
+    return ");"
+
+
+@CONVERTER(FilterOr.kind)
+def pre_filter_or(_node: FilterOr) -> str:
+    return " || ("
+
+
+@CONVERTER.post(FilterOr.kind)
+def post_filter_or(_node: FilterOr) -> str:
+    return ")"
+
+
+@CONVERTER(FilterAnd.kind)
+def pre_filter_and(_node: FilterAnd) -> str:
+    return " && ("
+
+
+@CONVERTER.post(FilterAnd.kind)
+def post_filter_and(_node: FilterAnd) -> str:
+    return ")"
+
+
+@CONVERTER(FilterNot.kind)
+def pre_filter_not(_node: FilterNot) -> str:  # type: ignore
+    return "!("
+
+
+@CONVERTER.post(FilterNot.kind)
+def post_filter_not(_node: FilterNot) -> str:
+    return ")"
+
+
+@CONVERTER(FilterStrIn.kind)
+def pre_filter_in(node: FilterStrIn) -> str:
+    substr, *_ = node.unpack_args()
+    expr = f"i.includes({substr!r})"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterStrStarts.kind)
+def pre_filter_starts_with(node: FilterStrStarts) -> str:
+    start_, *_ = node.unpack_args()
+    if len(start_) == 1:
+        expr = f"i.startsWith({start_[0]!r})"
+    else:
+        expr = "(" + " || ".join(f"i.startsWith({s!r})" for s in start_) + ")"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterStrEnds.kind)
+def pre_filter_ends_with(node: FilterStrEnds) -> str:
+    suffix_, *_ = node.unpack_args()
+    if len(suffix_) == 1:
+        expr = f"i.endsWith({suffix_[0]!r})"
+    else:
+        expr = "(" + " || ".join(f"i.endsWith({s!r})" for s in suffix_) + ")"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterStrRe.kind)
+def pre_filter_re(node: FilterStrRe) -> str:
+    pattern, ignore_case, *_ = node.unpack_args()
+    pattern = to_js_regexp(pattern, ignore_case, is_global=False)
+
+    expr = f"(new RegExp({pattern})).test(i)"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterEqual.kind)
+def pre_filter_eq(node: FilterEqual) -> str:
+    value, *_ = node.unpack_args()
+    expr = f"i === {value!r}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr
+
+
+@CONVERTER(FilterNotEqual.kind)
+def pre_filter_ne(node: FilterNotEqual) -> str:
+    value, *_ = node.unpack_args()
+    expr = f"i !== {value!r}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" && {expr}"
+    return expr

{ssc_codegen-0.8.15 → ssc_codegen-0.8.17}/ssc_codegen/converters/py_base.py

@@ -87,6 +87,16 @@ from ssc_codegen.ast_ import (
     ExprListStringRmPrefixAndSuffix,
     ExprListStringAnyRegex,
     ExprListStringAllRegex,
+    FilterOr,
+    FilterAnd,
+    FilterNot,
+    ExprFilter,
+    FilterStrIn,
+    FilterStrStarts,
+    FilterStrEnds,
+    FilterStrRe,
+    FilterEqual,
+    FilterNotEqual,
 )
 from ssc_codegen.converters.base import (
     BaseCodeConverter,
@@ -98,6 +108,8 @@ from ssc_codegen.converters.helpers import (
     have_pre_validate_call,
     prev_next_var,
     is_last_var_no_ret,
+    is_prev_node_atomic_cond,
+    is_first_node_cond,
 )
 from ssc_codegen.tokens import StructType, VariableType, JsonVariableType
 
@@ -181,6 +193,8 @@ class BasePyCodeConverter(BaseCodeConverter):
         )
         from ssc_codegen.ast_ import ExprListStringRmPrefix
 
+        from ssc_codegen.ast_ import FilterNotEqual
+
         self.pre_definitions = {
             Docstring.kind: pre_docstring,
             ModuleImports.kind: pre_imports,
@@ -234,12 +248,27 @@ class BasePyCodeConverter(BaseCodeConverter):
             ExprToListLength.kind: pre_to_len,
             ExprToBool.kind: pre_to_bool,
             ExprJsonify.kind: pre_jsonify,
+            # FILTER,
+            ExprFilter.kind: pre_expr_filter,
+            FilterAnd.kind: pre_filter_and,
+            FilterOr.kind: pre_filter_or,
+            FilterNot.kind: pre_filter_not,
+            FilterStrStarts.kind: pre_filter_starts_with,
+            FilterStrEnds.kind: pre_filter_ends_with,
+            FilterStrIn.kind: pre_filter_in,
+            FilterStrRe.kind: pre_filter_re,
+            FilterEqual.kind: pre_filter_eq,
+            FilterNotEqual.kind: pre_filter_ne,
         }
 
         self.post_definitions = {
             TypeDef.kind: post_typedef,
             JsonStruct.kind: post_json_struct,
             StartParseMethod.kind: post_start_parse,
+            ExprFilter.kind: post_expr_filter,
+            FilterAnd.kind: post_filter_and,
+            FilterOr.kind: post_filter_or,
+            FilterNot.kind: post_filter_not,
         }
 
 
@@ -876,3 +905,95 @@ def pre_jsonify(node: ExprJsonify) -> str:
     prv, nxt = prev_next_var(node)
     _name, _is_array = node.unpack_args()
     return indent + f"{nxt} = json.loads({prv})"
+
+
+# FILTERS
+
+
+def pre_expr_filter(node: ExprFilter) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    return indent + f"{nxt} = [i for i in {prv} if "
+
+
+def post_expr_filter(_node: ExprFilter) -> str:
+    return "]"
+
+
+def pre_filter_or(_node: FilterOr) -> str:
+    return " or ("
+
+
+def post_filter_or(_node: FilterOr) -> str:
+    return ")"
+
+
+def pre_filter_and(_node: FilterAnd) -> str:
+    return " and ("
+
+
+def post_filter_and(_node: FilterAnd) -> str:
+    return ")"
+
+
+def pre_filter_not(_node: FilterNot) -> str:
+    return "not ("
+
+
+def post_filter_not(_node: FilterNot) -> str:
+    return ")"
+
+
+def pre_filter_in(node: FilterStrIn) -> str:
+    substr, *_ = node.unpack_args()
+    expr = f"{substr!r} in i"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_starts_with(node: FilterStrStarts) -> str:
+    start_, *_ = node.unpack_args()
+    expr = f"i.startswith({start_})"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_ends_with(node: FilterStrEnds) -> str:
+    suffix_, *_ = node.unpack_args()
+    expr = f"i.endswith({suffix_})"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_re(node: FilterStrRe) -> str:
+    pattern, ignore_case, *_ = node.unpack_args()
+    if ignore_case:
+        expr = f"re.search({pattern!r}, i, re.IGNORECASE)"
+    else:
+        expr = f"re.search({pattern!r}, i)"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_eq(node: FilterEqual) -> str:
+    value, *_ = node.unpack_args()
+    # currently support only str
+    expr = f"i == {value!r}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr
+
+
+def pre_filter_ne(node: FilterNotEqual) -> str:
+    value, *_ = node.unpack_args()
+    # currently support only str
+    expr = f"i != {value!r}"
+    if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
+        return f" and {expr}"
+    return expr

{ssc_codegen-0.8.15 → ssc_codegen-0.8.17}/ssc_codegen/document.py

@@ -59,6 +59,16 @@ from ssc_codegen.ast_ import (
     ExprListStringAnyRegex,
     ExprHasAttr,
     ExprListHasAttr,
+    ExprFilter,
+    FilterEqual,
+    FilterNotEqual,
+    FilterStrStarts,
+    FilterStrEnds,
+    FilterStrIn,
+    FilterStrRe,
+    FilterAnd,
+    FilterOr,
+    FilterNot,
 )
 from ssc_codegen.document_utlis import (
     analyze_re_expression,
@@ -457,6 +467,20 @@ class ArrayDocument(BaseDocument):
         self._add(ExprToListLength(accept_type=expected_type))
         return self
 
+    def filter(self, expr: "DocumentFilter") -> Self:
+        """filter array of resuts by expr
+
+        - accept LIST_STRING, return LIST_STRING
+        """
+        if self.stack_last_ret != VariableType.LIST_STRING:
+            LOGGER.warning(
+                "to_len(): Expected type(s) %s got %s",
+                (VariableType.LIST_STRING.name,),
+                self.stack_last_ret.name,
+            )
+        self._add(ExprFilter(body=expr.stack))
+        return self
+
 
 class StringDocument(BaseDocument):
     def rm_prefix(self, substr: str) -> Self:
@@ -1236,3 +1260,78 @@ class JsonDocument(BaseDocument):
         # HACK: store to class instance for generate docstring signature
         BaseSchema.__JSON_SCHEMAS__[struct.__name__] = struct  # type: ignore
         return self
+
+
+class DocumentFilter(BaseDocument):
+    def eq(self, value: str) -> Self:
+        self._add(FilterEqual(kwargs={"value": value}))
+        return self
+
+    def ne(self, value: str) -> Self:
+        self._add(FilterNotEqual(kwargs={"value": value}))
+        return self
+
+    def starts(self, *values: str) -> Self:
+        self._add(FilterStrStarts(kwargs={"substr": values}))
+        return self
+
+    def ends(self, *values: str) -> Self:
+        self._add(FilterStrEnds(kwargs={"substr": values}))
+        return self
+
+    def contains(self, value: str) -> Self:
+        self._add(FilterStrIn(kwargs={"substr": value}))
+        return self
+
+    def re(
+        self, pattern: str | Pattern[str], ignore_case: bool = False
+    ) -> Self:
+        if not isinstance(pattern, str):
+            ignore_case = is_ignore_case_regex(pattern)
+
+        pattern = unverbosify_regex(pattern)
+        result = analyze_re_expression(pattern, allow_empty_groups=True)
+        if not result:
+            LOGGER.warning(result.msg)
+        self._add(
+            FilterStrRe(kwargs={"pattern": pattern, "ignore_case": ignore_case})
+        )
+        return self
+
+    def and_(self, filter_expr: "DocumentFilter") -> Self:
+        if self.stack[0].kind in (FilterAnd.kind, FilterOr.kind):
+            LOGGER.warning(
+                "logic AND: first node is `%s`", self.stack[0].kind.name
+            )
+        tmp_stack = filter_expr.stack.copy()
+        self._add(FilterAnd(body=tmp_stack))
+        return self
+
+    def or_(self, filter_expr: "DocumentFilter") -> Self:
+        if self.stack[0].kind in (FilterAnd.kind, FilterOr.kind):
+            LOGGER.warning(
+                "logic OR: first node is `%s`", self.stack[0].kind.name
+            )
+        tmp_stack = filter_expr.stack.copy()
+        self._add(FilterOr(body=tmp_stack))
+        return self
+
+    def not_(self, filter_expr: "DocumentFilter") -> Self:
+        tmp_stack = filter_expr.stack.copy()
+        self._add(FilterNot(body=tmp_stack))
+        return self
+
+    def __or__(self, other: "DocumentFilter") -> "DocumentFilter":
+        return self.or_(other)
+
+    def __and__(self, other: "DocumentFilter") -> "DocumentFilter":
+        return self.and_(other)
+
+    def __invert__(self) -> "DocumentFilter":
+        return DocumentFilter().not_(self)
+
+    def __eq__(self, other: str) -> Self:
+        return self.eq(other)
+
+    def __ne__(self, other: str) -> Self:
+        return self.ne(other)

{ssc_codegen-0.8.15 → ssc_codegen-0.8.17}/ssc_codegen/str_utils.py

@@ -103,7 +103,7 @@ RE_PY_METHOD_BLOCK = re.compile(
     r"""
 def\s_(?:parse|split)\w+\(.*\:\s*   # SPLIT_DOC STRUCT_PARSE HEAD
 (?:\n|.)*?                          # BLOCK OF CODE
-return\s(?P<ret_var>\w+)    # RET STMT 
+return\s(?P<ret_var>\w+)            # RET STMT 
 """,
     re.X,
 )
@@ -148,6 +148,9 @@ def py_optimize_return_naive(py_code: str) -> str:
         re_expr = f"{ret_var}\\s*=\\s*(?P<expr>.*)"
         if match := re.search(re_expr, method_block[0]):
             expr = match["expr"]  # noqa
+            # NAIVE apologize, its .filter(...) expr, skip optimization
+            if expr.strip().endswith(" if"):
+                continue
 
             # optimization function convert double backslash to single (\\ -> \) in regex patterns
             # add `r` prefix for avoid SyntaxWarning then compile generated code to cpython bytecode

{ssc_codegen-0.8.15 → ssc_codegen-0.8.17}/ssc_codegen/tokens.py

@@ -174,6 +174,21 @@ class TokenType(IntEnum):
     # bool
     TO_BOOL = auto()
 
+    # FILTER
+    EXPR_FILTER = auto()
+
+    FILTER = auto()  # TODO: REMOVE?
+    FILTER_OR = auto()
+    FILTER_AND = auto()
+    FILTER_NOT = auto()
+    # FILTER EXPR
+    FILTER_STR_IN = auto()
+    FILTER_STR_STARTS = auto()
+    FILTER_STR_ENDS = auto()
+    FILTER_STR_RE = auto()
+    FILTER_EQ = auto()
+    FILTER_NE = auto()
+
     @classmethod
     def default_tokens(cls) -> tuple["TokenType", ...]:
         return (