ssc_codegen 0.8.13__tar.gz → 0.8.15__tar.gz

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.8.13
+Version: 0.8.15
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
@@ -52,7 +52,7 @@ Current support converters
 |---------------|-------------------------------|---------------|-------------|------------------------------------------|----------------|
 | Python (3.8+) | bs4                           | N             | Y           | TypedDict*, list, dict                   | ruff           |
 | ...           | parsel                        | Y             | Y           | ...                                      | -              |
-| ...           | selectolax (modest)           | N             | Y           | ...                                      | -              |
+| ...           | selectolax (lexbor)           | N             | Y           | ...                                      | -              |
 | js (ES6)      | pure (firefox/chrome/nodejs)  | Y             | Y           | Array, Map**                             | -              |
 | go (1.10+)    | goquery                       | N             | Y           | struct(json anchors include), array, map | gofmt          |
 

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/README.md

@@ -19,7 +19,7 @@ Current support converters
 |---------------|-------------------------------|---------------|-------------|------------------------------------------|----------------|
 | Python (3.8+) | bs4                           | N             | Y           | TypedDict*, list, dict                   | ruff           |
 | ...           | parsel                        | Y             | Y           | ...                                      | -              |
-| ...           | selectolax (modest)           | N             | Y           | ...                                      | -              |
+| ...           | selectolax (lexbor)           | N             | Y           | ...                                      | -              |
 | js (ES6)      | pure (firefox/chrome/nodejs)  | Y             | Y           | Array, Map**                             | -              |
 | go (1.10+)    | goquery                       | N             | Y           | struct(json anchors include), array, map | gofmt          |
 

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.8.13"
+version = "0.8.15"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/ssc_codegen/__init__.py

@@ -7,7 +7,7 @@ from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchem
 
 setup_logger()
 
-VERSION = "0.8.13"
+VERSION = "0.8.15"
 
 
 class __MISSING(object):

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/ssc_codegen/ast_/__init__.py

@@ -30,4 +30,4 @@ from .nodes_string import ExprStringTrim, ExprStringRegex, ExprStringSplit, Expr
     ExprStringRmPrefix, ExprStringRmSuffix, ExprStringRmPrefixAndSuffix, ExprListStringRmPrefixAndSuffix, \
     ExprListStringRmPrefix, ExprListStringRmSuffix
 from .nodes_validate import ExprIsCss, ExprIsEqual, ExprStringIsRegex, ExprIsXpath, ExprIsNotEqual, ExprIsContains, \
-    ExprListStringAnyRegex, ExprListStringAllRegex
+    ExprListStringAnyRegex, ExprListStringAllRegex, ExprHasAttr, ExprListHasAttr

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/ssc_codegen/ast_/nodes_validate.py

@@ -97,6 +97,24 @@ class ExprIsXpath(BaseAstNode[KW_IS_SELECT, ARGS_IS_SELECT]):
     ret_type: VariableType = VariableType.DOCUMENT
 
 
+KW_HAS_ATTR = TypedDict("KW_HAS_ATTR", {"key": str, "msg": str})
+ARGS_HAS_ATTR = tuple[str, str]
+
+
+@dataclass(kw_only=True)
+class ExprHasAttr(BaseAstNode[KW_HAS_ATTR, ARGS_HAS_ATTR]):
+    kind: ClassVar[TokenType] = TokenType.HAS_ATTR
+    accept_type: VariableType = VariableType.DOCUMENT
+    ret_type: VariableType = VariableType.DOCUMENT
+
+
+@dataclass(kw_only=True)
+class ExprListHasAttr(BaseAstNode[KW_HAS_ATTR, ARGS_HAS_ATTR]):
+    kind: ClassVar[TokenType] = TokenType.HAS_LIST_ATTR
+    accept_type: VariableType = VariableType.DOCUMENT
+    ret_type: VariableType = VariableType.DOCUMENT
+
+
 # TODO:
 # ExprIsBiggerThan
 # ExprIsBiggerOrEq

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/ssc_codegen/ast_build/main.py

@@ -77,15 +77,32 @@ def build_ast_module_parser(
     json_structs = extract_json_structs_from_module(py_module)
     # TODO: API for build single json
     ast_jsons = build_ast_json(ast_module, *json_structs)
-    ast_schemas = [
-        build_ast_struct_parser(
-            sc,
-            ast_module,
-            css_to_xpath=css_to_xpath,
-            xpath_to_css=xpath_to_css,
-        )
-        for sc in schemas
-    ]
+    # TODO: implement analyze-only AST function
+    try:
+        ast_schemas = [
+            build_ast_struct_parser(
+                sc,
+                ast_module,
+                css_to_xpath=css_to_xpath,
+                xpath_to_css=xpath_to_css,
+            )
+            for sc in schemas
+        ]
+    except SyntaxError:
+        # get all schemas exceptions and throw to logs:
+        errors = []
+        for sc in schemas:
+            try:
+                build_ast_struct_parser(
+                    sc,
+                    ast_module,
+                    css_to_xpath=css_to_xpath,
+                    xpath_to_css=xpath_to_css,
+                )
+            except SyntaxError as e:
+                errors.append(e)
+        raise SyntaxError(f"{path}: Founded errors: {len(errors)}")
+
     # TODO: API for build single typedef
     ast_typedefs = build_ast_typedef(ast_module, *ast_schemas)
     module_body.extend(ast_jsons)
@@ -173,7 +190,6 @@ def build_ast_struct_parser(
 ) -> StructParser:
     errors_count = run_analyze_schema(schema)
     if errors_count > 0:
-        # todo: exc
         msg = f"{schema.__name__} founded errors: {errors_count}"
         raise SyntaxError(msg)
     docstring = (
@@ -237,8 +253,7 @@ def _fetch_field_nodes(
         )
         # TODO: add ast tests
         # in inheritance schemas, child classes use same fields are used as in the parent class
-        # avoid duplicate ExprReturn node
-        # second case used in ExprDefaultValueWrapper cases
+        # avoid duplicate ExprReturn or ExprDefaultValueWrapper node
         if (
             document.stack[-1].kind != ExprReturn.kind
             and document.stack[-1].kind != TokenType.EXPR_DEFAULT_END
@@ -335,7 +350,7 @@ def _try_fetch_split_doc_node(
         method = StructPartDocMethod(parent=st_ref)
 
         # in inheritance schemas, child classes use same fields are used as in the parent class
-        # second case used in ExprDefaultValueWrapper cases
+        # avoid duplicate ExprReturn or ExprDefaultValueWrapper node
         if (
             split_doc.stack[-1].kind != ExprReturn.kind
             and split_doc.stack[-1].kind != TokenType.EXPR_DEFAULT_END

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/ssc_codegen/converters/go_goquery.py

@@ -1259,3 +1259,6 @@ def pre_list_str_rm_prefix_and_suffix(
     return J2_PRE_LIST_STR_RM_PREFIX_AND_SUFFIX.render(
         prv=prv, nxt=nxt, substr=substr, tmp_var=tmp_var
     )
+
+
+# TODO impl ExprHasAttr, ExprListHasAttr

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/ssc_codegen/converters/js_pure.py

@@ -83,6 +83,8 @@ from ssc_codegen.ast_ import (
     ExprListStringRmPrefixAndSuffix,
     ExprListStringAnyRegex,
     ExprListStringAllRegex,
+    ExprListHasAttr,
+    ExprHasAttr,
 )
 from ssc_codegen.converters.base import BaseCodeConverter
 from ssc_codegen.converters.helpers import (
@@ -690,3 +692,23 @@ def pre_list_str_rm_prefix_and_suffix(
         f"let {nxt} = {prv}.map((e) => e.startsWith({substr}) ? e.slice({substr}.length) : e);"
         + f"{nxt} = {nxt}.map((e) => e.endsWith({substr}) ? e.slice(0, -{substr}.length) : e);"
     )
+
+
+@CONVERTER(ExprHasAttr.kind)
+def pre_has_attr(node: ExprHasAttr) -> str:
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = f"if (!{prv}?.hasAttribute({key!r}) throw new Error({msg!r});"
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + f"let {nxt} = {prv};"
+
+
+@CONVERTER(ExprListHasAttr.kind)
+def pre_list_has_attr(node: ExprListHasAttr) -> str:
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = f"if (!{prv}.every(e => e?.hasAttribute({key!r}))) throw new Error({msg!r});"
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + f"let {nxt} = {prv};"

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/ssc_codegen/converters/py_bs4.py

@@ -16,6 +16,8 @@ from ssc_codegen.ast_ import (
     StructFieldMethod,
     StructPreValidateMethod,
     StructPartDocMethod,
+    ExprHasAttr,
+    ExprListHasAttr,
 )
 from ssc_codegen.converters.helpers import (
     prev_next_var,
@@ -173,6 +175,32 @@ def pre_is_css(node: ExprIsCss) -> str:
     return expr + "\n" + indent + f"{nxt} = {prv}"
 
 
+@CONVERTER(ExprHasAttr.kind)
+def pre_has_attr(node: ExprHasAttr) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = indent + f"assert {prv}.get({key!r}, None), {msg!r}"
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + "\n" + indent + f"{nxt} = {prv}"
+
+
+@CONVERTER(ExprListHasAttr.kind)
+def pre_list_has_attr(node: ExprListHasAttr) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = indent + f"assert all(i.get({key!r}, None) for i in {prv}), {msg!r}"
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + "\n" + indent + f"{nxt} = {prv}"
+
+
 # NOT SUPPORT
 @CONVERTER(ExprXpath.kind)
 def pre_xpath(_: ExprXpath) -> str:

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/ssc_codegen/converters/py_parsel.py

@@ -16,6 +16,8 @@ from ssc_codegen.ast_ import (
     StructFieldMethod,
     StructPreValidateMethod,
     StructPartDocMethod,
+    ExprListHasAttr,
+    ExprHasAttr,
 )
 from ssc_codegen.converters.helpers import (
     prev_next_var,
@@ -196,3 +198,32 @@ def pre_is_xpath(node: ExprIsXpath) -> str:
     if is_last_var_no_ret(node):
         return expr
     return expr + "\n" + indent + f"{nxt} = {prv}"
+
+
+@CONVERTER(ExprHasAttr.kind)
+def pre_has_attr(node: ExprHasAttr) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = indent + f"assert {prv}.attrib.get({key!r}, None), {msg!r}"
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + "\n" + indent + f"{nxt} = {prv}"
+
+
+@CONVERTER(ExprListHasAttr.kind)
+def pre_list_has_attr(node: ExprListHasAttr) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = (
+        indent
+        + f"assert all(i.attrib.get({key!r}, None) for i in {prv}), {msg!r}"
+    )
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + "\n" + indent + f"{nxt} = {prv}"

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/ssc_codegen/converters/py_selectolax.py

@@ -16,6 +16,8 @@ from ssc_codegen.ast_ import (
     StructFieldMethod,
     StructPreValidateMethod,
     StructPartDocMethod,
+    ExprListHasAttr,
+    ExprHasAttr,
 )
 from ssc_codegen.converters.helpers import (
     prev_next_var,
@@ -171,6 +173,35 @@ def pre_is_css(node: ExprIsCss) -> str:
     return expr + "\n" + indent + f"{nxt} = {prv}"
 
 
+@CONVERTER(ExprHasAttr.kind)
+def pre_has_attr(node: ExprHasAttr) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = indent + f"assert {prv}.attributes.get({key!r}, None), {msg!r}"
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + "\n" + indent + f"{nxt} = {prv}"
+
+
+@CONVERTER(ExprListHasAttr.kind)
+def pre_list_has_attr(node: ExprListHasAttr) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    key, msg = node.unpack_args()
+    expr = (
+        indent
+        + f"assert all(i.attributes.get({key!r}, None) for i in {prv}), {msg!r}"
+    )
+    if is_last_var_no_ret(node):
+        return expr
+    return expr + "\n" + indent + f"{nxt} = {prv}"
+
+
 # NOT SUPPORT
 @CONVERTER(ExprXpath.kind)
 def pre_xpath(_: ExprXpath) -> str:

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/ssc_codegen/converters/templates/js_pure.py

@@ -100,3 +100,12 @@ let {{ nxt }} = Array.from({ length: {{ snapshot_var }}.snapshotLength }, (_, i)
 J2_IS_XPATH = Template("""
 if (document.evaluate("{{ query }}", {{ prv }}, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue === null) throw new Error({{ msg }});
 """)
+# TODO: use in covverter
+JS_HAS_ATTR = Template("""
+if (!{{prv}}?.hasAttribute({{key}}) throw new Error({{ msg }});
+""")
+
+
+JS_HAS_ATTR_ALL = Template("""
+if (!{{prv}}.every(e => e?.hasAttribute({{key}}));}) throw new Error({{ msg }});
+""")

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/ssc_codegen/document.py

@@ -57,6 +57,8 @@ from ssc_codegen.ast_ import (
     ExprListStringRmPrefixAndSuffix,
     ExprListStringAllRegex,
     ExprListStringAnyRegex,
+    ExprHasAttr,
+    ExprListHasAttr,
 )
 from ssc_codegen.document_utlis import (
     analyze_re_expression,
@@ -1086,6 +1088,33 @@ class AssertDocument(BaseDocument):
         )
         return self
 
+    def has_attr(self, key: str, msg: str = "") -> Self:
+        """assert document has attribute key. If in generated code check failed - throw exception with passed msg
+
+        EXPR DO NOT MODIFY variable
+
+        - accept DOCUMENT, return DOCUMENT
+        - accept LIST_DOCUMENT, return LIST_DOCUMENT
+        """
+        if self.stack_last_ret == VariableType.DOCUMENT:
+            self._add(ExprHasAttr(kwargs={"key": key, "msg": msg}))
+        elif self.stack_last_ret == VariableType.LIST_DOCUMENT:
+            self._add(ExprListHasAttr(kwargs={"key": key, "msg": msg}))
+        else:
+            LOGGER.warning(
+                "has_attr(%s): Expected type(s) %s got %s",
+                repr(key),
+                (VariableType.DOCUMENT.name, VariableType.LIST_DOCUMENT.name),
+                self.stack_last_ret.name,
+            )
+            self._add(
+                ExprHasAttr(
+                    kwargs={"key": key, "msg": msg},
+                    ret_type=self.stack_last_ret,
+                )
+            )
+        return self
+
 
 class NestedDocument(BaseDocument):
     def sub_parser(self, schema: Type["BaseSchema"]) -> Self:

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/ssc_codegen/document_utlis.py

@@ -31,6 +31,8 @@ CM1_RX = r"(?m)(?<!\\)((\\{2})*)#.*$"
 CM2_RX = r"(\\)?((\\{2})*)(#)"
 WS_RX = r"(\\)?((\\{2})*)(\s)\s*"
 
+RE_CAPTURED_GROUPS = re.compile(r"(?<!\()\((?!\?:)[^)]+\)")
+
 
 def is_ignore_case_regex(pattern: str | Pattern) -> bool:
     if isinstance(pattern, str):
@@ -76,10 +78,14 @@ def analyze_re_expression(
     try:
         re_pattern = re.compile(pattern)
         if not allow_empty_groups and re_pattern.groups == 0:
-            msg = f"`{re_pattern.pattern}` pattern groups is empty"
+            msg = f"`{re_pattern.pattern}` pattern groups is empty."
+            msg += "\nTIP: maybe you remember wrap pattern to brackets `()`?"
             return AnalyzeResult.error(msg)
         elif max_groups != -1 and re_pattern.groups > max_groups:
-            msg = f"`{re_pattern.pattern}` too many groups in pattern, expected {max_groups}"
+            captured_groups = RE_CAPTURED_GROUPS.findall(re_pattern.pattern)
+            msg = f"`{re_pattern.pattern}` too many groups in pattern, expected groups count: {max_groups}."
+            msg += f"\nTIP: fix regular expression for extract {max_groups}:"
+            msg += f"\nGroups founded: {captured_groups}"
             return AnalyzeResult.error(msg)
     except re.error as e:
         msg = f"`{pattern}` wrong regex pattern syntax: {e!r}"

{ssc_codegen-0.8.13 → ssc_codegen-0.8.15}/ssc_codegen/tokens.py

@@ -156,6 +156,8 @@ class TokenType(IntEnum):
     IS_STRING_REGEX_MATCH = auto()
     ANY_LIST_STRING_REGEX_MATCH = auto()
     ALL_LIST_STRING_REGEX_MATCH = auto()
+    HAS_ATTR = auto()
+    HAS_LIST_ATTR = auto()
 
     # NUMERIC
     TO_INT = auto()