ssc_codegen 0.8.6__tar.gz → 0.8.8__tar.gz

{ssc_codegen-0.8.6 → ssc_codegen-0.8.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.8.6
+Version: 0.8.8
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues

{ssc_codegen-0.8.6 → ssc_codegen-0.8.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.8.6"
+version = "0.8.8"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.8.6 → ssc_codegen-0.8.8}/ssc_codegen/__init__.py

@@ -7,7 +7,7 @@ from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchem
 
 setup_logger()
 
-VERSION = "0.8.6"
+VERSION = "0.8.8"
 
 
 class __MISSING(object):

{ssc_codegen-0.8.6 → ssc_codegen-0.8.8}/ssc_codegen/ast_build/main.py

@@ -288,6 +288,13 @@ def _unwrap_default_node(
                     default_type = VariableType.OPTIONAL_INT
                 case VariableType.FLOAT:
                     default_type = VariableType.OPTIONAL_FLOAT
+                case VariableType.LIST_STRING:
+                    default_type = VariableType.OPTIONAL_LIST_STRING
+                case VariableType.LIST_INT:
+                    default_type = VariableType.OPTIONAL_LIST_INT
+                case VariableType.LIST_FLOAT:
+                    default_type = VariableType.OPTIONAL_LIST_FLOAT
+                # TODO: warning for BOOL type
                 case _:
                     assert_never(ret_type)  # type: ignore
         expr_default_start = ExprDefaultValueStart(kwargs={"value": value})

{ssc_codegen-0.8.6 → ssc_codegen-0.8.8}/ssc_codegen/document.py

@@ -4,7 +4,7 @@ import logging
 from typing import Type, Pattern
 
 from cssselect import SelectorSyntaxError
-from typing_extensions import Self
+from typing_extensions import Self, assert_never
 
 from ssc_codegen.ast_ import (
     BaseAstNode,
@@ -63,6 +63,11 @@ from ssc_codegen.document_utlis import (
     unverbosify_regex,
     is_ignore_case_regex,
 )
+from ssc_codegen.pseudo_selectors import (
+    parse_pseudo_xpath_query,
+    parse_pseudo_css_query,
+    PseudoAction,
+)
 from ssc_codegen.json_struct import Json
 from ssc_codegen.schema import BaseSchema
 from ssc_codegen.selector_utils import validate_css_query, validate_xpath_query
@@ -72,6 +77,8 @@ LOGGER = logging.getLogger("ssc_gen")
 
 
 class BaseDocument:
+    LOGGER = LOGGER
+
     def __init__(self) -> None:
         self._stack: list[BaseAstNode] = []
 
@@ -98,14 +105,6 @@ class BaseDocument:
             return VariableType.DOCUMENT
         return self.stack[-1].ret_type
 
-    @staticmethod
-    def _raise_wrong_type_error(
-        type_: VariableType, *expected: VariableType
-    ) -> None:
-        fmt_types = "(" + ",".join(i.name for i in expected) + ")"
-        msg = f"Expected type(s): {fmt_types}, got {type_.name}"
-        raise SyntaxError(msg)
-
     def _add(self, expr: BaseAstNode) -> None:
         self._stack.append(expr)
 
@@ -140,6 +139,7 @@ class HTMLDocument(BaseDocument):
         - accept: DOCUMENT, return DOCUMENT
         """
         query = " ".join(query.splitlines())
+        query, action = parse_pseudo_css_query(query)
         try:
             validate_css_query(query)
         except SelectorSyntaxError:
@@ -153,13 +153,29 @@ class HTMLDocument(BaseDocument):
             )
 
         self._add(ExprCss(kwargs={"query": query}))
+        self._pseudo_query_action_to_expr(action)
         return self
 
+    def _pseudo_query_action_to_expr(
+        self, action: tuple[PseudoAction | None, str | None]
+    ) -> None:
+        if action[0]:
+            match action[0]:
+                case "text":
+                    self.text()
+                case "raw":
+                    self.raw()
+                case "attr":
+                    self.attr(action[1])  # type: ignore
+                case _:
+                    assert_never(action[0])
+
     def xpath(self, query: str) -> Self:
         """Xpath query. returns first founded element
         - accept: DOCUMENT, return DOCUMENT
         """
         query = " ".join(query.splitlines())
+        query, action = parse_pseudo_xpath_query(query)
         try:
             validate_xpath_query(query)
         except SelectorSyntaxError:
@@ -172,22 +188,15 @@ class HTMLDocument(BaseDocument):
                 self.stack_last_ret.name,
             )
         self._add(ExprXpath(kwargs={"query": query}))
+        self._pseudo_query_action_to_expr(action)
         return self
-        # match self.stack_last_ret:
-        #     case VariableType.DOCUMENT:
-        #         self._add(HtmlXpathExpression(query=query))
-        #     case VariableType.ANY:
-        #         self._add(HtmlXpathExpression(query=query))
-        #     case _:
-        #         self._raise_wrong_type_error(
-        #             self.stack_last_ret, VariableType.DOCUMENT
-        #         )
 
     def css_all(self, query: str) -> Self:
         """Css query. returns all founded elements
         - accept: DOCUMENT, return: LIST_DOCUMENT
         """
         query = " ".join(query.splitlines())
+        query, action = parse_pseudo_css_query(query)
         try:
             validate_css_query(query)
         except SelectorSyntaxError:
@@ -199,6 +208,7 @@ class HTMLDocument(BaseDocument):
                 self.stack_last_ret.name,
             )
         self._add(ExprCssAll(kwargs={"query": query}))
+        self._pseudo_query_action_to_expr(action)
         return self
 
     def xpath_all(self, query: str) -> Self:
@@ -206,6 +216,7 @@ class HTMLDocument(BaseDocument):
         - accept: DOCUMENT, return: LIST_DOCUMENT
         """
         query = " ".join(query.splitlines())
+        query, action = parse_pseudo_xpath_query(query)
         try:
             validate_xpath_query(query)
         except SelectorSyntaxError as e:
@@ -217,6 +228,7 @@ class HTMLDocument(BaseDocument):
                 self.stack_last_ret.name,
             )
         self._add(ExprXpathAll(kwargs={"query": query}))
+        self._pseudo_query_action_to_expr(action)
         return self
 
     def attr(self, key: str) -> Self:
@@ -768,6 +780,13 @@ class AssertDocument(BaseDocument):
         - accept DOCUMENT, return DOCUMENT
         """
         query = " ".join(query.splitlines())
+        new_query, action = parse_pseudo_css_query(query)
+        if action[0]:
+            LOGGER.warning(
+                "is_css(%s) not support pseudo parse classes, skip", repr(query)
+            )
+            query = new_query
+
         try:
             validate_css_query(query)
         except SelectorSyntaxError:
@@ -790,6 +809,13 @@ class AssertDocument(BaseDocument):
         - accept DOCUMENT, return DOCUMENT
         """
         query = " ".join(query.splitlines())
+        new_query, action = parse_pseudo_xpath_query(query)
+        if action[0]:
+            LOGGER.warning(
+                "is_xpath(%s) not support pseudo parse classes, skip",
+                repr(query),
+            )
+            query = new_query
         try:
             validate_xpath_query(query)
         except SelectorSyntaxError:

{ssc_codegen-0.8.6 → ssc_codegen-0.8.8}/ssc_codegen/document_utlis.py

@@ -1,6 +1,8 @@
+import logging
 import re
 from typing import TYPE_CHECKING, Pattern
-import logging
+from typing_extensions import assert_never
+
 from ssc_codegen.ast_ import (
     BaseAstNode,
     ExprIsCss,
@@ -10,6 +12,12 @@ from ssc_codegen.ast_ import (
     ExprXpath,
     ExprCssAll,
 )
+from ssc_codegen.pseudo_selectors import (
+    parse_pseudo_xpath_query,
+    pseudo_action_to_pseudo_xpath,
+    parse_pseudo_css_query,
+    pseudo_action_to_pseudo_css,
+)
 from ssc_codegen.selector_utils import css_to_xpath, xpath_to_css
 from ssc_codegen.static_checker.base import AnalyzeResult
 
@@ -18,7 +26,6 @@ LOGGER = logging.getLogger("ssc_gen")
 if TYPE_CHECKING:
     from .document import BaseDocument
 
-
 # https://stackoverflow.com/a/14919203
 CM1_RX = r"(?m)(?<!\\)((\\{2})*)#.*$"
 CM2_RX = r"(\\)?((\\{2})*)(#)"
@@ -88,35 +95,34 @@ def convert_css_to_xpath(
     new_stack: list[BaseAstNode] = []
 
     for expr in old_stack:
+        if expr.kind not in (ExprCss.kind, ExprCssAll.kind, ExprIsCss.kind):
+            new_stack.append(expr)
+            continue
+        query = expr.kwargs["query"]
+        query, pseudo_action = parse_pseudo_css_query(query)
+        new_query = css_to_xpath(query, prefix=prefix)
+        if pseudo_action[0]:
+            new_query += pseudo_action_to_pseudo_css(*pseudo_action)
         match expr.kind:
             case ExprCss.kind:
-                new_expr = ExprXpath(  # type: ignore[assignment]
-                    kwargs={
-                        "query": css_to_xpath(
-                            expr.kwargs["query"], prefix=prefix
-                        )
-                    }
-                )
+                new_stack.append(ExprXpath(kwargs={"query": new_query}))
             case ExprCssAll.kind:
-                new_expr = ExprXpathAll(  # type: ignore[assignment]
-                    kwargs={
-                        "query": css_to_xpath(
-                            expr.kwargs["query"], prefix=prefix
-                        )
-                    }
+                new_stack.append(
+                    ExprXpathAll(  # type: ignore[assignment]
+                        kwargs={"query": new_query}
+                    )
                 )
             case ExprIsCss.kind:
-                new_expr = ExprIsXpath(  # type: ignore[assignment]
-                    kwargs={
-                        "query": css_to_xpath(
-                            expr.kwargs["query"], prefix=prefix
-                        ),
-                        "msg": expr.kwargs["msg"],
-                    }
+                new_stack.append(
+                    ExprIsXpath(  # type: ignore[assignment]
+                        kwargs={
+                            "query": new_query,
+                            "msg": expr.kwargs["msg"],
+                        }
+                    )
                 )
             case _:
-                new_expr = expr  # type: ignore[assignment]
-        new_stack.append(new_expr)
+                assert_never(expr.kind)
     doc._stack = new_stack
     return doc
 
@@ -126,24 +132,33 @@ def convert_xpath_to_css(doc: "BaseDocument") -> "BaseDocument":
     old_stack = doc.stack.copy()
     new_stack: list[BaseAstNode] = []
     for expr in old_stack:
+        if expr.kind not in (
+            ExprXpath.kind,
+            ExprXpathAll.kind,
+            ExprIsXpath.kind,
+        ):
+            new_stack.append(expr)
+            continue
+        query = expr.kwargs["query"]
+        query, pseudo_action = parse_pseudo_xpath_query(query)
+        new_query = xpath_to_css(query)
+        if pseudo_action[0]:
+            new_query += pseudo_action_to_pseudo_xpath(*pseudo_action)
         match expr.kind:
             case ExprXpath.kind:
-                new_expr = ExprCss(
-                    kwargs={"query": xpath_to_css(expr.kwargs["query"])}
-                )
+                new_stack.append(ExprCss(kwargs={"query": new_query}))
             case ExprXpathAll.kind:
-                new_expr = ExprCssAll(  # type: ignore[assignment]
-                    kwargs={"query": xpath_to_css(expr.kwargs["query"])}
-                )
+                new_stack.append(ExprCssAll(kwargs={"query": new_query}))
             case ExprIsXpath.kind:
-                new_expr = ExprIsCss(  # type: ignore[assignment]
-                    kwargs={
-                        "query": xpath_to_css(expr.kwargs["query"]),
-                        "msg": expr.kwargs["msg"],
-                    }
+                new_stack.append(
+                    ExprIsCss(
+                        kwargs={
+                            "query": new_query,
+                            "msg": expr.kwargs["msg"],
+                        }
+                    )
                 )
             case _:
-                new_expr = expr  # type: ignore[assignment]
-        new_stack.append(new_expr)
+                assert_never(expr.kind)
     doc._stack = new_stack
     return doc

ssc_codegen-0.8.8/ssc_codegen/pseudo_selectors.py

@@ -0,0 +1,64 @@
+import re
+from typing import Literal
+
+PseudoAction = Literal["text", "raw", "attr"]
+
+
+def parse_pseudo_css_query(
+    query: str,
+) -> tuple[str, tuple[PseudoAction | None, str | None]]:
+    """parse css selector and extract pseudo class action"""
+    query = query.strip()
+
+    if query.endswith("::text"):
+        return query[: -len("::text")], ("text", None)
+
+    if query.endswith("::raw"):
+        return query[: -len("::raw")], ("raw", None)
+
+    if match := re.search(r"::attr\(([^)]+)\)$", query):
+        return query[: match.start()], ("attr", match.group(1))
+
+    return query, (None, None)
+
+
+def parse_pseudo_xpath_query(
+    query: str,
+) -> tuple[str, tuple[PseudoAction | None, str | None]]:
+    """parse xpath selector and extract pseudo class action"""
+    query = query.strip()
+
+    if query.endswith("/text()"):
+        return query[: -len("/text()")], ("text", None)
+
+    if query.endswith("/raw()"):
+        return query[: -len("/raw()")], ("raw", None)
+
+    if match := re.search(r"/@([\w:-]+)$", query):
+        return query[: match.start()], ("attr", match.group(1))
+
+    return query, (None, None)
+
+
+def pseudo_action_to_pseudo_css(action: PseudoAction, arg: str | None) -> str:
+    match action:
+        case "text":
+            return "::text"
+        case "raw":
+            return "::raw"
+        case "attr":
+            return f"::attr({arg})"
+        case _:
+            return ""
+
+
+def pseudo_action_to_pseudo_xpath(action: PseudoAction, arg: str | None) -> str:
+    match action:
+        case "text":
+            return "/text()"
+        case "raw":
+            return "/raw()"
+        case "attr":
+            return f"/@{arg}"
+        case _:
+            return ""