ssc_codegen 0.9.1__tar.gz → 0.9.3__tar.gz

{ssc_codegen-0.9.1 → ssc_codegen-0.9.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.9.1
+Version: 0.9.3
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
@@ -45,6 +45,9 @@ Project solving next problems:
 - generate types or type annotations if possible and documentation 
 - simplify code support
 - portability to other languages and HTML parser backends
+- designed for SSR (server-side-render) pages parsers. 
+For REST-API, GRAPHQL endpoints use tools like [mitmproxy2swagger](https://github.com/alufers/mitmproxy2swagger)
+
 
 ## Support converters
 
@@ -56,7 +59,7 @@ Current support converters
 | ...           | parsel                       | Y     | Y    | N    | ...                               | ...                  |
 | ...           | selectolax (lexbor)          | N     | Y    | N    | ...                               | ...                  |
 | js (ES6)      | pure (firefox/chrome/nodejs) | Y     | Y    | Y    | Array, Map**                      | prettier             |
-| go (1.10+)    | goquery                      | N     | Y    | N    | struct(+json anchors), array, map | gofmt                |
+| go (1.10+)    | goquery (WIP)                | N     | Y    | N    | struct(+json anchors), array, map | gofmt                |
 
 - **CSS3** means support next selectors:
   - basic: (`tag`, `.class`, `#id`, `tag1,tag2`)

{ssc_codegen-0.9.1 → ssc_codegen-0.9.3}/README.md

@@ -11,6 +11,9 @@ Project solving next problems:
 - generate types or type annotations if possible and documentation 
 - simplify code support
 - portability to other languages and HTML parser backends
+- designed for SSR (server-side-render) pages parsers. 
+For REST-API, GRAPHQL endpoints use tools like [mitmproxy2swagger](https://github.com/alufers/mitmproxy2swagger)
+
 
 ## Support converters
 
@@ -22,7 +25,7 @@ Current support converters
 | ...           | parsel                       | Y     | Y    | N    | ...                               | ...                  |
 | ...           | selectolax (lexbor)          | N     | Y    | N    | ...                               | ...                  |
 | js (ES6)      | pure (firefox/chrome/nodejs) | Y     | Y    | Y    | Array, Map**                      | prettier             |
-| go (1.10+)    | goquery                      | N     | Y    | N    | struct(+json anchors), array, map | gofmt                |
+| go (1.10+)    | goquery (WIP)                | N     | Y    | N    | struct(+json anchors), array, map | gofmt                |
 
 - **CSS3** means support next selectors:
   - basic: (`tag`, `.class`, `#id`, `tag1,tag2`)

{ssc_codegen-0.9.1 → ssc_codegen-0.9.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.9.1"
+version = "0.9.3"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.9.1 → ssc_codegen-0.9.3}/ssc_codegen/__init__.py

@@ -7,7 +7,7 @@ from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchem
 
 setup_logger()
 
-VERSION = "0.9.1"
+VERSION = "0.9.3"
 
 
 class __MISSING(object):

{ssc_codegen-0.9.1 → ssc_codegen-0.9.3}/ssc_codegen/ast_/nodes_selectors.py

@@ -36,8 +36,8 @@ class ExprXpathAll(BaseAstNode[KW_EXPR_SELECTOR_LIKE, ARGS_EXPR_SELECTOR_LIKE]):
     ret_type: VariableType = VariableType.LIST_DOCUMENT
 
 
-KW_EXPR_ATTR = TypedDict("KW_EXPR_ATTR", {"key": str})
-ARGS_EXPR_ATTR = tuple[str]
+KW_EXPR_ATTR = TypedDict("KW_EXPR_ATTR", {"key": tuple[str, ...]})
+ARGS_EXPR_ATTR = tuple[tuple[str, ...]]
 
 
 @dataclass(kw_only=True)

{ssc_codegen-0.9.1 → ssc_codegen-0.9.3}/ssc_codegen/cli/main.py

@@ -216,8 +216,8 @@ def gen_js(
     converter = import_converter(f"js_{lib.value}")
     if fmt:
         commands: list[str] = [
-            'prettier --write --tab-width 4 --single-quote --trailing-comma all --semi {}',
-            ]
+            "prettier --write --tab-width 4 --single-quote --trailing-comma all --semi {}",
+        ]
         fmt_cmd = create_fmt_cmd(ssc_files, prefix, suffix, out, commands)
     else:
         fmt_cmd = []

{ssc_codegen-0.9.1 → ssc_codegen-0.9.3}/ssc_codegen/converters/go_goquery.py

@@ -1264,4 +1264,5 @@ def pre_list_str_rm_prefix_and_suffix(
 # TODO impl ExprHasAttr, ExprListHasAttr
 # TODO: impl Filter Expr
 # TODO: impl ExprListUnique
-# TODO: inpt ExprStringMapReplace, ExprListStringMapReplace
+# TODO: impl ExprStringMapReplace, ExprListStringMapReplace
+# TODO: impl ExprAttr, ExprListAttr for multiple extract keys

{ssc_codegen-0.9.1 → ssc_codegen-0.9.3}/ssc_codegen/converters/js_pure.py

@@ -175,6 +175,12 @@ def make_js_docstring(value: str) -> str:
     return docstr_start + docstr_parts + docstr_end
 
 
+def py_sequence_to_js_array(values: tuple[str, ...] | list[str]) -> str:
+    """note: value should be wrapper to"""
+    val_arr = str(values)
+    return "[" + val_arr[1:-1] + "]"
+
+
 @CONVERTER(Docstring.kind)
 def pre_docstring(node: Docstring) -> str:
     value = node.kwargs["value"]
@@ -631,15 +637,23 @@ def pre_xpath_all(node: ExprXpathAll) -> str:
 @CONVERTER(ExprGetHtmlAttr.kind)
 def pre_html_attr(node: ExprGetHtmlAttr) -> str:
     prv, nxt = prev_next_var(node)
-    key = node.kwargs["key"]
-    return f"let {nxt} = {prv}.getAttribute({key!r});"
+    keys = node.kwargs["key"]
+    if len(keys) == 1:
+        key = keys[0]
+        return f"let {nxt} = {prv}.getAttribute({key!r});"
+    js_keys = py_sequence_to_js_array(keys)
+    return f"let {nxt} = {js_keys}.map(k => {prv}?.getAttribute(k)).filter(Boolean); "
 
 
 @CONVERTER(ExprGetHtmlAttrAll.kind)
 def pre_html_attr_all(node: ExprGetHtmlAttrAll) -> str:
     prv, nxt = prev_next_var(node)
-    key = node.kwargs["key"]
-    return f"let {nxt} = {prv}.map(e => e.getAttribute({key!r}));"
+    keys = node.kwargs["key"]
+    if len(keys) == 1:
+        key = keys[0]
+        return f"let {nxt} = {prv}.map(e => e.getAttribute({key!r})); "
+    js_keys = py_sequence_to_js_array(keys)
+    return f"let {nxt} = Array.from({prv}).flatMap(e => {js_keys}.map(k => e.getAttribute(k)).filter(Boolean)); "
 
 
 @CONVERTER(ExprGetHtmlText.kind)

{ssc_codegen-0.9.1 → ssc_codegen-0.9.3}/ssc_codegen/converters/py_base.py

@@ -491,7 +491,7 @@ def post_start_parse(node: StartParseMethod) -> str:
                 code += f"{name!r}: self._parse_{name}(self._document),"
             code += "}"
         case StructType.ACC_LIST:
-            code += INDENT_METHOD_BODY + "return list(dict.fromkeys("
+            code += INDENT_METHOD_BODY + "return list(set("
             methods = []
             for expr in node.body:
                 name = expr.kwargs["name"]
@@ -1100,8 +1100,7 @@ def pre_list_unique(node: ExprListUnique) -> str:
         INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
     )
     prv, nxt = prev_next_var(node)
-    # use `dict.fromkeys()` instead `set()` for guarantees insertion order (py 3.7+) reason
-    return f"{indent}{nxt} = list(dict.fromkeys({prv}))"
+    return f"{indent}{nxt} = list(set({prv}))"
 
 
 def pre_str_map_replace(node: ExprStringMapReplace) -> str:

{ssc_codegen-0.9.1 → ssc_codegen-0.9.3}/ssc_codegen/converters/py_bs4.py

@@ -104,11 +104,14 @@ def pre_html_attr(node: ExprGetHtmlAttr) -> str:
         INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
     )
     prv, nxt = prev_next_var(node)
-    key = node.kwargs["key"]
-    # as default, bs4 return list of attrs, set string for API consistence
+    keys = node.kwargs["key"]
+    # bs4 maybe return list of attrs, set string for API consistence
+    if len(keys) == 1:
+        key = keys[0]
+        return indent + f"{nxt} = ' '.join({prv}.get_attribute_list({key!r}))"
     return (
         indent
-        + f"{nxt} = {prv}[{key!r}] if isinstance({prv}[{key!r}], str) else ' '.join({prv}[{key!r}])"
+        + f"{nxt} = [' '.join({prv}.get_attribute_list(k)) for k in {keys} if {prv}.get(k)]"
     )
 
 
@@ -118,11 +121,18 @@ def pre_html_attr_all(node: ExprGetHtmlAttrAll) -> str:
         INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
     )
     prv, nxt = prev_next_var(node)
-    key = node.kwargs["key"]
-    # as default, bs4 return list of attrs, set string for API consistence
+    keys = node.kwargs["key"]
+    # as default, bs4 maybe return list of attrs, set string for API consistence
+    if len(keys) == 1:
+        key = keys[0]
+        return (
+            indent
+            + f"{nxt} = [' '.join(e.get_attribute_list({key!r})) for e in {prv} if e.get({key!r})]"
+        )
+    # [tag.get(attr) for tag in tags for attr in ('src', 'href') if tag.has_attr(attr)]
     return (
         indent
-        + f"{nxt} = [e[{key!r}] if isinstance(e[{key!r}], str) else ' '.join(e[{key!r}]) for e in {prv}]"
+        + f"{nxt} = [' '.join(e.get_attribute_list(k)) for e in {prv} for k in {keys} if e.get(k)]"
     )
 
 

{ssc_codegen-0.9.1 → ssc_codegen-0.9.3}/ssc_codegen/converters/py_parsel.py

@@ -124,8 +124,14 @@ def pre_html_attr(node: ExprGetHtmlAttr) -> str:
         INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
     )
     prv, nxt = prev_next_var(node)
-    key = node.kwargs["key"]
-    return indent + f"{nxt} = {prv}.attrib[{key!r}]"
+    keys = node.kwargs["key"]
+    if len(keys) == 1:
+        key = keys[0]
+        return indent + f"{nxt} = {prv}.attrib[{key!r}]"
+    return (
+        indent
+        + f"{nxt} = [{prv}.attrib[k] for k in {keys} if {prv}.attrib.get(k)]"
+    )
 
 
 @CONVERTER(ExprGetHtmlAttrAll.kind)
@@ -134,8 +140,14 @@ def pre_html_attr_all(node: ExprGetHtmlAttrAll) -> str:
         INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
     )
     prv, nxt = prev_next_var(node)
-    key = node.kwargs["key"]
-    return indent + f"{nxt} = [e.attrib[{key!r}] for e in {prv}]"
+    keys = node.kwargs["key"]
+    if len(keys) == 1:
+        key = keys[0]
+        return indent + f"{nxt} = [e.attrib[{key!r}] for e in {prv}]"
+    return (
+        indent
+        + f"{nxt} = [e.attrib[k] for e in {prv} for k in {keys} if e.attrib.get(k)]"
+    )
 
 
 @CONVERTER(ExprGetHtmlText.kind)

{ssc_codegen-0.9.1 → ssc_codegen-0.9.3}/ssc_codegen/converters/py_selectolax.py

@@ -110,8 +110,14 @@ def pre_html_attr(node: ExprGetHtmlAttr) -> str:
         INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
     )
     prv, nxt = prev_next_var(node)
-    key = node.kwargs["key"]
-    return indent + f"{nxt} = {prv}.attributes[{key!r}]"
+    keys = node.kwargs["key"]
+    if len(keys) == 1:
+        key = keys[0]
+        return indent + f"{nxt} = {prv}.attributes[{key!r}]"
+    return (
+        indent
+        + f"{nxt} = [{prv}.attributes[k] for k in {keys} if {prv}.attributes.get(k)]"
+    )
 
 
 @CONVERTER(ExprGetHtmlAttrAll.kind)
@@ -120,8 +126,14 @@ def pre_html_attr_all(node: ExprGetHtmlAttrAll) -> str:
         INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
     )
     prv, nxt = prev_next_var(node)
-    key = node.kwargs["key"]
-    return indent + f"{nxt} = [e.attributes[{key!r}] for e in {prv}]"
+    keys = node.kwargs["key"]
+    if len(keys) == 1:
+        key = keys[0]
+        return indent + f"{nxt} = [e.attributes[{key!r}] for e in {prv}]"
+    return (
+        indent
+        + f"{nxt} = [e.attributes[k] for e in {prv} for k in {keys} if e.attributes.get(k)]"
+    )
 
 
 @CONVERTER(ExprGetHtmlText.kind)

{ssc_codegen-0.9.1 → ssc_codegen-0.9.3}/ssc_codegen/document.py

@@ -187,7 +187,7 @@ class HTMLDocument(BaseDocument):
                 case "raw":
                     self.raw()
                 case "attr":
-                    self.attr(action[1])  # type: ignore
+                    self.attr(*action[1])  # type: ignore
                 case _:
                     assert_never(action[0])
 
@@ -252,7 +252,7 @@ class HTMLDocument(BaseDocument):
         self._pseudo_query_action_to_expr(action)
         return self
 
-    def attr(self, key: str) -> Self:
+    def attr(self, *key: str) -> Self:
         """Extract attribute value by name
 
         - accept DOCUMENT, return STRING
@@ -266,7 +266,15 @@ class HTMLDocument(BaseDocument):
             case VariableType.LIST_DOCUMENT:
                 self._add(ExprGetHtmlAttrAll(kwargs={"key": key}))
             case VariableType.DOCUMENT:
-                self._add(ExprGetHtmlAttr(kwargs={"key": key}))
+                if len(key) == 1:
+                    self._add(ExprGetHtmlAttr(kwargs={"key": key}))
+                else:
+                    self._add(
+                        ExprGetHtmlAttr(
+                            kwargs={"key": key},
+                            ret_type=VariableType.LIST_STRING,
+                        )
+                    )
             case _:
                 LOGGER.warning(
                     "attr(%s): Expected type(s) %s got %s",

{ssc_codegen-0.9.1 → ssc_codegen-0.9.3}/ssc_codegen/pseudo_selectors.py

@@ -6,7 +6,7 @@ PseudoAction = Literal["text", "raw", "attr"]
 
 def parse_pseudo_css_query(
     query: str,
-) -> tuple[str, tuple[PseudoAction | None, str | None]]:
+) -> tuple[str, tuple[PseudoAction | None, str | tuple[str, ...] | None]]:
     """parse css selector and extract pseudo class action"""
     query = query.strip()
 
@@ -17,14 +17,17 @@ def parse_pseudo_css_query(
         return query[: -len("::raw")], ("raw", None)
 
     if match := re.search(r"::attr\(([^)]+)\)$", query):
-        return query[: match.start()], ("attr", match.group(1))
+        return query[: match.start()], (
+            "attr",
+            tuple([i.strip() for i in match.group(1).split(",")]),
+        )
 
     return query, (None, None)
 
 
 def parse_pseudo_xpath_query(
     query: str,
-) -> tuple[str, tuple[PseudoAction | None, str | None]]:
+) -> tuple[str, tuple[PseudoAction | None, str | tuple[str, ...] | None]]:
     """parse xpath selector and extract pseudo class action"""
     query = query.strip()
 
@@ -35,7 +38,10 @@ def parse_pseudo_xpath_query(
         return query[: -len("/raw()")], ("raw", None)
 
     if match := re.search(r"/@([\w:-]+)$", query):
-        return query[: match.start()], ("attr", match.group(1))
+        return query[: match.start()], (
+            "attr",
+            tuple([i.strip() for i in match.group(1).split(",")]),
+        )
 
     return query, (None, None)