ssc_codegen 0.8.14__tar.gz → 0.8.16__tar.gz

{ssc_codegen-0.8.14 → ssc_codegen-0.8.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.8.14
+Version: 0.8.16
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
@@ -52,7 +52,7 @@ Current support converters
 |---------------|-------------------------------|---------------|-------------|------------------------------------------|----------------|
 | Python (3.8+) | bs4                           | N             | Y           | TypedDict*, list, dict                   | ruff           |
 | ...           | parsel                        | Y             | Y           | ...                                      | -              |
-| ...           | selectolax (modest)           | N             | Y           | ...                                      | -              |
+| ...           | selectolax (lexbor)           | N             | Y           | ...                                      | -              |
 | js (ES6)      | pure (firefox/chrome/nodejs)  | Y             | Y           | Array, Map**                             | -              |
 | go (1.10+)    | goquery                       | N             | Y           | struct(json anchors include), array, map | gofmt          |
 

{ssc_codegen-0.8.14 → ssc_codegen-0.8.16}/README.md

@@ -19,7 +19,7 @@ Current support converters
 |---------------|-------------------------------|---------------|-------------|------------------------------------------|----------------|
 | Python (3.8+) | bs4                           | N             | Y           | TypedDict*, list, dict                   | ruff           |
 | ...           | parsel                        | Y             | Y           | ...                                      | -              |
-| ...           | selectolax (modest)           | N             | Y           | ...                                      | -              |
+| ...           | selectolax (lexbor)           | N             | Y           | ...                                      | -              |
 | js (ES6)      | pure (firefox/chrome/nodejs)  | Y             | Y           | Array, Map**                             | -              |
 | go (1.10+)    | goquery                       | N             | Y           | struct(json anchors include), array, map | gofmt          |
 

{ssc_codegen-0.8.14 → ssc_codegen-0.8.16}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.8.14"
+version = "0.8.16"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.8.14 → ssc_codegen-0.8.16}/ssc_codegen/__init__.py

@@ -7,7 +7,7 @@ from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchem
 
 setup_logger()
 
-VERSION = "0.8.14"
+VERSION = "0.8.16"
 
 
 class __MISSING(object):

{ssc_codegen-0.8.14 → ssc_codegen-0.8.16}/ssc_codegen/ast_/__init__.py

@@ -30,4 +30,4 @@ from .nodes_string import ExprStringTrim, ExprStringRegex, ExprStringSplit, Expr
     ExprStringRmPrefix, ExprStringRmSuffix, ExprStringRmPrefixAndSuffix, ExprListStringRmPrefixAndSuffix, \
     ExprListStringRmPrefix, ExprListStringRmSuffix
 from .nodes_validate import ExprIsCss, ExprIsEqual, ExprStringIsRegex, ExprIsXpath, ExprIsNotEqual, ExprIsContains, \
-    ExprListStringAnyRegex, ExprListStringAllRegex
+    ExprListStringAnyRegex, ExprListStringAllRegex, ExprHasAttr, ExprListHasAttr

{ssc_codegen-0.8.14 → ssc_codegen-0.8.16}/ssc_codegen/ast_/nodes_validate.py

@@ -97,6 +97,24 @@ class ExprIsXpath(BaseAstNode[KW_IS_SELECT, ARGS_IS_SELECT]):
     ret_type: VariableType = VariableType.DOCUMENT
 
 
+KW_HAS_ATTR = TypedDict("KW_HAS_ATTR", {"key": str, "msg": str})
+ARGS_HAS_ATTR = tuple[str, str]
+
+
+@dataclass(kw_only=True)
+class ExprHasAttr(BaseAstNode[KW_HAS_ATTR, ARGS_HAS_ATTR]):
+    kind: ClassVar[TokenType] = TokenType.HAS_ATTR
+    accept_type: VariableType = VariableType.DOCUMENT
+    ret_type: VariableType = VariableType.DOCUMENT
+
+
+@dataclass(kw_only=True)
+class ExprListHasAttr(BaseAstNode[KW_HAS_ATTR, ARGS_HAS_ATTR]):
+    kind: ClassVar[TokenType] = TokenType.HAS_LIST_ATTR
+    accept_type: VariableType = VariableType.DOCUMENT
+    ret_type: VariableType = VariableType.DOCUMENT
+
+
 # TODO:
 # ExprIsBiggerThan
 # ExprIsBiggerOrEq

{ssc_codegen-0.8.14 → ssc_codegen-0.8.16}/ssc_codegen/converters/go_goquery.py

@@ -1259,3 +1259,6 @@ def pre_list_str_rm_prefix_and_suffix(
     return J2_PRE_LIST_STR_RM_PREFIX_AND_SUFFIX.render(
         prv=prv, nxt=nxt, substr=substr, tmp_var=tmp_var
     )
+
+
+# TODO impl ExprHasAttr, ExprListHasAttr

{ssc_codegen-0.8.14 → ssc_codegen-0.8.16}/ssc_codegen/converters/js_pure.py

@@ -83,6 +83,8 @@ from ssc_codegen.ast_ import (
     ExprListStringRmPrefixAndSuffix,
     ExprListStringAnyRegex,
     ExprListStringAllRegex,
+    ExprListHasAttr,
+    ExprHasAttr,
 )
 from ssc_codegen.converters.base import BaseCodeConverter
 from ssc_codegen.converters.helpers import (
@@ -127,6 +129,17 @@ DOCSTR_SEP = "* "
 CONVERTER = BaseCodeConverter(debug_comment_prefix="// ")
 
 
+# TODO: move to string_utils
+def to_js_regexp(pattern: str, ignore_case: bool = False) -> str:
+    """helper function for convert string pattern to js"""
+    pattern = pattern.replace("<\\/", "</")
+    pattern = pattern.replace("/", "\\/")
+    pattern = f"/{pattern}/g"
+    if ignore_case:
+        pattern += "i"
+    return pattern
+
+
 def make_js_docstring(value: str) -> str:
     if not value:
         return ""
@@ -355,10 +368,7 @@ def pre_list_str_replace(node: ExprListStringReplace) -> str:
 def pre_str_regex(node: ExprStringRegex) -> str:
     prv, nxt = prev_next_var(node)
     pattern, group, ignore_case = node.unpack_args()
-    pattern = pattern.replace("/", "\\/")
-    pattern = f"/{pattern}/g"
-    if ignore_case:
-        pattern += "i"
+    pattern = to_js_regexp(pattern, ignore_case)
     return f"let {nxt} = (new RegExp({pattern})).exec({prv})[{group}];"
 
 
@@ -366,10 +376,8 @@ def pre_str_regex(node: ExprStringRegex) -> str:
 def pre_str_regex_all(node: ExprStringRegexAll) -> str:
     prv, nxt = prev_next_var(node)
     pattern, ignore_case = node.unpack_args()
-    pattern = pattern.replace("/", "\\/")
-    pattern = f"/{pattern}/g"
-    if ignore_case:
-        pattern += "i"
+    pattern = to_js_regexp(pattern, ignore_case)
+
     return f"let {nxt} = (new RegExp({pattern})).exec({prv});"
 
 
@@ -377,8 +385,7 @@ def pre_str_regex_all(node: ExprStringRegexAll) -> str:
 def pre_str_regex_sub(node: ExprStringRegexSub) -> str:
     prv, nxt = prev_next_var(node)
     pattern, repl = node.unpack_args()
-    pattern = pattern.replace("/", "\\/")
-    pattern = f"/{pattern}/g"
+    pattern = to_js_regexp(pattern)
 
     return f"let {nxt} = {prv}.replace({pattern}, {repl!r});"
 
@@ -387,8 +394,8 @@ def pre_str_regex_sub(node: ExprStringRegexSub) -> str:
 def pre_list_str_regex_sub(node: ExprListStringRegexSub) -> str:
     prv, nxt = prev_next_var(node)
     pattern, repl = node.unpack_args()
-    pattern = pattern.replace("/", "\\/")
-    pattern = f"/{pattern}/g"
+    pattern = to_js_regexp(pattern)
+
     return f"let {nxt} = {prv}.map(e => e.replace({pattern}, {repl!r}));"
 
 
@@ -454,10 +461,7 @@ def pre_is_contains(node: ExprIsContains) -> str:
 def pre_is_regex(node: ExprStringIsRegex) -> str:
     prv, nxt = prev_next_var(node)
     pattern, ignore_case, msg = node.unpack_args()
-    pattern = pattern.replace("/", "\\/")
-    pattern = f"/{pattern}/g"
-    if ignore_case:
-        pattern += "i"
+    pattern = to_js_regexp(pattern, ignore_case)
 
     expr = f"if ({prv}.match({pattern}) === null) throw new Error({msg!r});"
     if is_last_var_no_ret(node):
@@ -690,3 +694,23 @@ def pre_list_str_rm_prefix_and_suffix(
         f"let {nxt} = {prv}.map((e) => e.startsWith({substr}) ? e.slice({substr}.length) : e);"
         + f"{nxt} = {nxt}.map((e) => e.endsWith({substr}) ? e.slice(0, -{substr}.length) : e);"
     )
+
+
+@CONVERTER(ExprHasAttr.kind)
+def pre_has_attr(node: ExprHasAttr) -> str:
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = f"if (!{prv}?.hasAttribute({key!r}) throw new Error({msg!r});"
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + f"let {nxt} = {prv};"
+
+
+@CONVERTER(ExprListHasAttr.kind)
+def pre_list_has_attr(node: ExprListHasAttr) -> str:
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = f"if (!{prv}.every(e => e?.hasAttribute({key!r}))) throw new Error({msg!r});"
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + f"let {nxt} = {prv};"

{ssc_codegen-0.8.14 → ssc_codegen-0.8.16}/ssc_codegen/converters/py_bs4.py

@@ -16,6 +16,8 @@ from ssc_codegen.ast_ import (
     StructFieldMethod,
     StructPreValidateMethod,
     StructPartDocMethod,
+    ExprHasAttr,
+    ExprListHasAttr,
 )
 from ssc_codegen.converters.helpers import (
     prev_next_var,
@@ -173,6 +175,32 @@ def pre_is_css(node: ExprIsCss) -> str:
     return expr + "\n" + indent + f"{nxt} = {prv}"
 
 
+@CONVERTER(ExprHasAttr.kind)
+def pre_has_attr(node: ExprHasAttr) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = indent + f"assert {prv}.get({key!r}, None), {msg!r}"
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + "\n" + indent + f"{nxt} = {prv}"
+
+
+@CONVERTER(ExprListHasAttr.kind)
+def pre_list_has_attr(node: ExprListHasAttr) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = indent + f"assert all(i.get({key!r}, None) for i in {prv}), {msg!r}"
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + "\n" + indent + f"{nxt} = {prv}"
+
+
 # NOT SUPPORT
 @CONVERTER(ExprXpath.kind)
 def pre_xpath(_: ExprXpath) -> str:

{ssc_codegen-0.8.14 → ssc_codegen-0.8.16}/ssc_codegen/converters/py_parsel.py

@@ -16,6 +16,8 @@ from ssc_codegen.ast_ import (
     StructFieldMethod,
     StructPreValidateMethod,
     StructPartDocMethod,
+    ExprListHasAttr,
+    ExprHasAttr,
 )
 from ssc_codegen.converters.helpers import (
     prev_next_var,
@@ -196,3 +198,32 @@ def pre_is_xpath(node: ExprIsXpath) -> str:
     if is_last_var_no_ret(node):
         return expr
     return expr + "\n" + indent + f"{nxt} = {prv}"
+
+
+@CONVERTER(ExprHasAttr.kind)
+def pre_has_attr(node: ExprHasAttr) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = indent + f"assert {prv}.attrib.get({key!r}, None), {msg!r}"
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + "\n" + indent + f"{nxt} = {prv}"
+
+
+@CONVERTER(ExprListHasAttr.kind)
+def pre_list_has_attr(node: ExprListHasAttr) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = (
+        indent
+        + f"assert all(i.attrib.get({key!r}, None) for i in {prv}), {msg!r}"
+    )
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + "\n" + indent + f"{nxt} = {prv}"

{ssc_codegen-0.8.14 → ssc_codegen-0.8.16}/ssc_codegen/converters/py_selectolax.py

@@ -16,6 +16,8 @@ from ssc_codegen.ast_ import (
     StructFieldMethod,
     StructPreValidateMethod,
     StructPartDocMethod,
+    ExprListHasAttr,
+    ExprHasAttr,
 )
 from ssc_codegen.converters.helpers import (
     prev_next_var,
@@ -171,6 +173,35 @@ def pre_is_css(node: ExprIsCss) -> str:
     return expr + "\n" + indent + f"{nxt} = {prv}"
 
 
+@CONVERTER(ExprHasAttr.kind)
+def pre_has_attr(node: ExprHasAttr) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = indent + f"assert {prv}.attributes.get({key!r}, None), {msg!r}"
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + "\n" + indent + f"{nxt} = {prv}"
+
+
+@CONVERTER(ExprListHasAttr.kind)
+def pre_list_has_attr(node: ExprListHasAttr) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = (
+        indent
+        + f"assert all(i.attributes.get({key!r}, None) for i in {prv}), {msg!r}"
+    )
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + "\n" + indent + f"{nxt} = {prv}"
+
+
 # NOT SUPPORT
 @CONVERTER(ExprXpath.kind)
 def pre_xpath(_: ExprXpath) -> str:

{ssc_codegen-0.8.14 → ssc_codegen-0.8.16}/ssc_codegen/converters/templates/js_pure.py

@@ -100,3 +100,12 @@ let {{ nxt }} = Array.from({ length: {{ snapshot_var }}.snapshotLength }, (_, i)
 J2_IS_XPATH = Template("""
 if (document.evaluate("{{ query }}", {{ prv }}, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue === null) throw new Error({{ msg }});
 """)
+# TODO: use in covverter
+JS_HAS_ATTR = Template("""
+if (!{{prv}}?.hasAttribute({{key}}) throw new Error({{ msg }});
+""")
+
+
+JS_HAS_ATTR_ALL = Template("""
+if (!{{prv}}.every(e => e?.hasAttribute({{key}}));}) throw new Error({{ msg }});
+""")

{ssc_codegen-0.8.14 → ssc_codegen-0.8.16}/ssc_codegen/document.py

@@ -57,6 +57,8 @@ from ssc_codegen.ast_ import (
     ExprListStringRmPrefixAndSuffix,
     ExprListStringAllRegex,
     ExprListStringAnyRegex,
+    ExprHasAttr,
+    ExprListHasAttr,
 )
 from ssc_codegen.document_utlis import (
     analyze_re_expression,
@@ -1086,6 +1088,33 @@ class AssertDocument(BaseDocument):
         )
         return self
 
+    def has_attr(self, key: str, msg: str = "") -> Self:
+        """assert document has attribute key. If in generated code check failed - throw exception with passed msg
+
+        EXPR DO NOT MODIFY variable
+
+        - accept DOCUMENT, return DOCUMENT
+        - accept LIST_DOCUMENT, return LIST_DOCUMENT
+        """
+        if self.stack_last_ret == VariableType.DOCUMENT:
+            self._add(ExprHasAttr(kwargs={"key": key, "msg": msg}))
+        elif self.stack_last_ret == VariableType.LIST_DOCUMENT:
+            self._add(ExprListHasAttr(kwargs={"key": key, "msg": msg}))
+        else:
+            LOGGER.warning(
+                "has_attr(%s): Expected type(s) %s got %s",
+                repr(key),
+                (VariableType.DOCUMENT.name, VariableType.LIST_DOCUMENT.name),
+                self.stack_last_ret.name,
+            )
+            self._add(
+                ExprHasAttr(
+                    kwargs={"key": key, "msg": msg},
+                    ret_type=self.stack_last_ret,
+                )
+            )
+        return self
+
 
 class NestedDocument(BaseDocument):
     def sub_parser(self, schema: Type["BaseSchema"]) -> Self:

{ssc_codegen-0.8.14 → ssc_codegen-0.8.16}/ssc_codegen/tokens.py

@@ -156,6 +156,8 @@ class TokenType(IntEnum):
     IS_STRING_REGEX_MATCH = auto()
     ANY_LIST_STRING_REGEX_MATCH = auto()
     ALL_LIST_STRING_REGEX_MATCH = auto()
+    HAS_ATTR = auto()
+    HAS_LIST_ATTR = auto()
 
     # NUMERIC
     TO_INT = auto()