ssc_codegen 0.9.6__tar.gz → 0.9.8__tar.gz

{ssc_codegen-0.9.6 → ssc_codegen-0.9.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.9.6
+Version: 0.9.8
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues

{ssc_codegen-0.9.6 → ssc_codegen-0.9.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.9.6"
+version = "0.9.8"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.9.6 → ssc_codegen-0.9.8}/ssc_codegen/__init__.py

@@ -7,7 +7,7 @@ from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchem
 
 setup_logger()
 
-VERSION = "0.9.6"
+VERSION = "0.9.8"
 
 
 class __MISSING(object):

{ssc_codegen-0.9.6 → ssc_codegen-0.9.8}/ssc_codegen/ast_/__init__.py

@@ -25,7 +25,7 @@ from .nodes_filter import FilterOr, FilterAnd, FilterNot, FilterNotEqual, Filter
     FilterStrLenLe, FilterStrLenGt, FilterStrLenGe
 
 from .nodes_selectors import ExprCss, ExprCssAll, ExprXpathAll, ExprXpath, ExprGetHtmlText, ExprGetHtmlRaw, \
-    ExprGetHtmlAttr, ExprGetHtmlAttrAll, ExprGetHtmlRawAll, ExprGetHtmlTextAll
+    ExprGetHtmlAttr, ExprGetHtmlAttrAll, ExprGetHtmlRawAll, ExprGetHtmlTextAll, ExprMapAttrs, ExprMapAttrsAll
 from .nodes_string import ExprStringTrim, ExprStringRegex, ExprStringSplit, ExprStringReplace, ExprStringLeftTrim, \
     ExprStringFormat, ExprListStringTrim, ExprStringRegexAll, ExprStringRegexSub, ExprStringRightTrim, \
     ExprListStringFormat, ExprListStringReplace, ExprListStringRightTrim, ExprListStringLeftTrim, \

{ssc_codegen-0.9.6 → ssc_codegen-0.9.8}/ssc_codegen/ast_/nodes_selectors.py

@@ -80,3 +80,17 @@ class ExprGetHtmlRawAll(BaseAstNode[T_EMPTY_KWARGS, tuple]):
     kind: ClassVar[TokenType] = TokenType.EXPR_RAW_ALL
     accept_type: VariableType = VariableType.LIST_DOCUMENT
     ret_type: VariableType = VariableType.LIST_STRING
+
+
+@dataclass(kw_only=True)
+class ExprMapAttrs(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    kind: ClassVar[TokenType] = TokenType.EXPR_MAP_ATTRS
+    accept_type: VariableType = VariableType.DOCUMENT
+    ret_type: VariableType = VariableType.LIST_STRING
+
+
+@dataclass(kw_only=True)
+class ExprMapAttrsAll(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    kind: ClassVar[TokenType] = TokenType.EXPR_MAP_ATTRS_ALL
+    accept_type: VariableType = VariableType.LIST_DOCUMENT
+    ret_type: VariableType = VariableType.LIST_STRING

{ssc_codegen-0.9.6 → ssc_codegen-0.9.8}/ssc_codegen/converters/go_goquery.py

@@ -1267,3 +1267,4 @@ def pre_list_str_rm_prefix_and_suffix(
 # TODO: impl ExprStringMapReplace, ExprListStringMapReplace
 # TODO: impl ExprAttr, ExprListAttr for multiple extract keys
 # TODO impl unescape
+# TODO: impl ExprHasAttr, ExprHasAttrAll

{ssc_codegen-0.9.6 → ssc_codegen-0.9.8}/ssc_codegen/converters/js_pure.py

@@ -106,6 +106,7 @@ from ssc_codegen.ast_ import (
     ExprListStringMapReplace,
 )
 from ssc_codegen.ast_.nodes_core import ModuleImports
+from ssc_codegen.ast_.nodes_selectors import ExprMapAttrs, ExprMapAttrsAll
 from ssc_codegen.ast_.nodes_string import (
     ExprListStringUnescape,
     ExprStringUnescape,
@@ -161,8 +162,9 @@ def to_js_regexp(
     pattern: str, ignore_case: bool = False, is_global: bool = True
 ) -> str:
     """helper function for convert string pattern to js"""
-    pattern = pattern.replace("<\\/", "</")
-    pattern = pattern.replace("/", "\\/")
+    # fix backslashes translate
+    pattern = pattern.replace("/", r"\/").replace(r"\\/", r"\/")
+
     pattern = f"/{pattern}/"
     if is_global:
         pattern += "g"
@@ -186,12 +188,6 @@ def py_sequence_to_js_array(values: tuple[str, ...] | list[str]) -> str:
     return "[" + val_arr[1:-1] + "]"
 
 
-@CONVERTER(ModuleImports.kind)
-def pre_imports(_node: ModuleImports) -> str:
-    # HACK: used imports node for provide helpers scripts
-    pass
-
-
 @CONVERTER(Docstring.kind)
 def pre_docstring(node: Docstring) -> str:
     value = node.kwargs["value"]
@@ -1012,3 +1008,15 @@ def pre_list_str_unescape(node: ExprListStringUnescape) -> str:
         + r".replace(/\\([bfnrt])/g, (_, ch) => ({b:'\b',f:'\f',n:'\n',r:'\r',t:'\\t'})[ch])"
         + ");"
     )
+
+
+@CONVERTER(ExprMapAttrs.kind)
+def pre_map_attrs(node: ExprMapAttrs) -> str:
+    prv, nxt = prev_next_var(node)
+    return f"let {nxt} = Array.from({prv}.attributes).map(a => a.value); "
+
+
+@CONVERTER(ExprMapAttrsAll.kind)
+def pre_map_attrs_all(node: ExprMapAttrsAll) -> str:
+    prv, nxt = prev_next_var(node)
+    return f"let {nxt} = [].concat(...{prv}.map(e => Array.from(e.attributes).map(a => a.value))); "

{ssc_codegen-0.9.6 → ssc_codegen-0.9.8}/ssc_codegen/converters/py_bs4.py

@@ -19,6 +19,7 @@ from ssc_codegen.ast_ import (
     ExprHasAttr,
     ExprListHasAttr,
 )
+from ssc_codegen.ast_.nodes_selectors import ExprMapAttrs, ExprMapAttrsAll
 from ssc_codegen.converters.helpers import (
     prev_next_var,
     have_default_expr,
@@ -211,6 +212,30 @@ def pre_list_has_attr(node: ExprListHasAttr) -> str:
     return expr + "\n" + indent + f"{nxt} = {prv}"
 
 
+@CONVERTER(ExprMapAttrs.kind)
+def pre_map_attrs(node: ExprMapAttrs) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    return (
+        indent
+        + f"{nxt} = [' '.join(i) if isinstance(i, list) else i for i in {prv}.attrs.values()]"
+    )
+
+
+@CONVERTER(ExprMapAttrsAll.kind)
+def pre_map_attrs_all(node: ExprMapAttrsAll) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    return (
+        indent
+        + f"{nxt} = [' '.join(i) if isinstance(i, list) else i for e in {prv} for i in e.attrs.values()]"
+    )
+
+
 # NOT SUPPORT
 @CONVERTER(ExprXpath.kind)
 def pre_xpath(_: ExprXpath) -> str:

{ssc_codegen-0.9.6 → ssc_codegen-0.9.8}/ssc_codegen/converters/py_parsel.py

@@ -19,6 +19,7 @@ from ssc_codegen.ast_ import (
     ExprListHasAttr,
     ExprHasAttr,
 )
+from ssc_codegen.ast_.nodes_selectors import ExprMapAttrs, ExprMapAttrsAll
 from ssc_codegen.converters.helpers import (
     prev_next_var,
     have_default_expr,
@@ -239,3 +240,21 @@ def pre_list_has_attr(node: ExprListHasAttr) -> str:
     if is_last_var_no_ret(node):
         return expr
     return expr + "\n" + indent + f"{nxt} = {prv}"
+
+
+@CONVERTER(ExprMapAttrs.kind)
+def pre_map_attrs(node: ExprMapAttrs) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    return indent + f"{nxt} = list({prv}.attrib.values())"
+
+
+@CONVERTER(ExprMapAttrsAll.kind)
+def pre_map_attrs_all(node: ExprMapAttrsAll) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    return indent + f"{nxt} = [i for e in {prv} for i in e.attrib.values()]"

{ssc_codegen-0.9.6 → ssc_codegen-0.9.8}/ssc_codegen/converters/py_selectolax.py

@@ -19,6 +19,7 @@ from ssc_codegen.ast_ import (
     ExprListHasAttr,
     ExprHasAttr,
 )
+from ssc_codegen.ast_.nodes_selectors import ExprMapAttrs, ExprMapAttrsAll
 from ssc_codegen.converters.helpers import (
     prev_next_var,
     have_default_expr,
@@ -214,6 +215,24 @@ def pre_list_has_attr(node: ExprListHasAttr) -> str:
     return expr + "\n" + indent + f"{nxt} = {prv}"
 
 
+@CONVERTER(ExprMapAttrs.kind)
+def pre_map_attrs(node: ExprMapAttrs) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    return indent + f"{nxt} = list({prv}.attributes.values())"
+
+
+@CONVERTER(ExprMapAttrsAll.kind)
+def pre_map_attrs_all(node: ExprMapAttrsAll) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    return indent + f"{nxt} = [i for e in {prv} for i in e.attributes.values()]"
+
+
 # NOT SUPPORT
 @CONVERTER(ExprXpath.kind)
 def pre_xpath(_: ExprXpath) -> str:

{ssc_codegen-0.9.6 → ssc_codegen-0.9.8}/ssc_codegen/document.py

@@ -79,6 +79,7 @@ from ssc_codegen.ast_ import (
     ExprListStringMapReplace,
     ExprStringMapReplace,
 )
+from ssc_codegen.ast_.nodes_selectors import ExprMapAttrs, ExprMapAttrsAll
 from ssc_codegen.ast_.nodes_string import (
     ExprListStringUnescape,
     ExprStringUnescape,
@@ -338,6 +339,28 @@ class HTMLDocument(BaseDocument):
                 self._add(ExprGetHtmlRaw())
         return self
 
+    def attrs_map(self) -> Self:
+        """extract all attributes from current tag or list of tags
+        - accept DOCUMENT, return LIST_STRING
+        - accept LIST_DOCUMENT, return LIST_STRING
+        """
+        match self.stack_last_ret:
+            case VariableType.DOCUMENT:
+                self._add(ExprMapAttrs())
+            case VariableType.LIST_DOCUMENT:
+                self._add(ExprMapAttrsAll())
+            case _:
+                LOGGER.warning(
+                    "attrs_list(): Expected type(s) %s got %s",
+                    (
+                        VariableType.DOCUMENT.name,
+                        VariableType.LIST_DOCUMENT.name,
+                    ),
+                    self.stack_last_ret.name,
+                )
+                self._add(ExprMapAttrs())
+        return self
+
 
 class ArrayDocument(BaseDocument):
     def first(self) -> Self:

{ssc_codegen-0.9.6 → ssc_codegen-0.9.8}/ssc_codegen/tokens.py

@@ -111,6 +111,7 @@ class TokenType(IntEnum):
     EXPR_ATTR = auto()
     EXPR_TEXT = auto()
     EXPR_RAW = auto()
+    EXPR_MAP_ATTRS = auto()
 
     # LIST_DOCUMENT
     EXPR_CSS_ALL = auto()
@@ -118,6 +119,7 @@ class TokenType(IntEnum):
     EXPR_ATTR_ALL = auto()
     EXPR_TEXT_ALL = auto()
     EXPR_RAW_ALL = auto()
+    EXPR_MAP_ATTRS_ALL = auto()
 
     # STRING
     EXPR_REGEX = auto()