ssc_codegen 0.12.0__tar.gz → 0.13.0__tar.gz

{ssc_codegen-0.12.0 → ssc_codegen-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.12.0
+Version: 0.13.0
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
@@ -26,7 +26,6 @@ Requires-Dist: colorama>=0.4.6; sys_platform == 'win32'
 Requires-Dist: cssselect>=1.2.0
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: ichrome>=4.0.4
-Requires-Dist: jinja2>=3.1.6
 Requires-Dist: lxml>=5.3.0
 Requires-Dist: parsel>=1.10.0
 Requires-Dist: soupsieve>=2.6
@@ -66,7 +65,7 @@ Current support converters
 | ...                                      | parsel                                 | Y     | Y    | N    | ...                                   | ...                  |
 | ...                                      | selectolax (lexbor)                    | N     | Y    | N    | ...                                   | ...                  |
 | ...                                      | lxml                                   | Y     | Y    | N    | ...                                   | ...                  |
-| js (ES6)`2`                              | pure (firefox/chrome extension/nodejs) | Y     | Y    | Y    | Array, Map`3`                         | prettier             |
+| js (ES6)`2`                              | pure (firefox/chrome extension/nodejs) | Y     | Y    | Y    | JSDoc                                 | prettier             |
 | go (1.10+) **(UNSTABLE)**                | goquery, gjson (`4`)                   | N     | Y    | N    | struct(+json anchors), array, map     | gofmt                |
 | lua (5.2+), luajit(2+) **(UNSTABLE)**`5` | lua-htmlparser, lrexlib(opt), dkjson   | N     | Y    | N    | EmmyLua                               | LuaFormatter         |
 

{ssc_codegen-0.12.0 → ssc_codegen-0.13.0}/README.md

@@ -29,7 +29,7 @@ Current support converters
 | ...                                      | parsel                                 | Y     | Y    | N    | ...                                   | ...                  |
 | ...                                      | selectolax (lexbor)                    | N     | Y    | N    | ...                                   | ...                  |
 | ...                                      | lxml                                   | Y     | Y    | N    | ...                                   | ...                  |
-| js (ES6)`2`                              | pure (firefox/chrome extension/nodejs) | Y     | Y    | Y    | Array, Map`3`                         | prettier             |
+| js (ES6)`2`                              | pure (firefox/chrome extension/nodejs) | Y     | Y    | Y    | JSDoc                                 | prettier             |
 | go (1.10+) **(UNSTABLE)**                | goquery, gjson (`4`)                   | N     | Y    | N    | struct(+json anchors), array, map     | gofmt                |
 | lua (5.2+), luajit(2+) **(UNSTABLE)**`5` | lua-htmlparser, lrexlib(opt), dkjson   | N     | Y    | N    | EmmyLua                               | LuaFormatter         |
 

{ssc_codegen-0.12.0 → ssc_codegen-0.13.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.12.0"
+version = "0.13.0"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"
@@ -10,7 +10,6 @@ dependencies = [
     "cssselect>=1.2.0",
     "httpx>=0.28.1",
     "ichrome>=4.0.4",
-    "jinja2>=3.1.6",
     "lxml>=5.3.0",
     "parsel>=1.10.0",
     "soupsieve>=2.6",
@@ -61,8 +60,10 @@ dev = [
     "coverage>=7.6.12",
     "httpx>=0.28.1",
     "mypy>=1.14.1",
+    "parsel>=1.10.0",
     "pytest>=8.3.4",
     "ruff>=0.9.3",
+    "selectolax>=0.3.27",
 ]
 # python generated code tests
 tests = [

{ssc_codegen-0.12.0 → ssc_codegen-0.13.0}/ssc_codegen/__init__.py

@@ -7,7 +7,7 @@ from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchem
 
 setup_logger()
 
-VERSION = "0.12.0"
+VERSION = "0.13.0"
 
 
 class __MISSING(object):

ssc_codegen-0.13.0/ssc_codegen/ast_/base.py

@@ -0,0 +1,242 @@
+from dataclasses import dataclass, field
+from typing import (
+    Callable,
+    ClassVar,
+    Mapping,
+    TypeVar,
+    Generic,
+    Any,
+    TypedDict,
+    Optional,
+    MutableSequence,
+    Sequence,
+    TYPE_CHECKING,
+)
+
+from ssc_codegen.tokens import TokenType, VariableType
+
+if TYPE_CHECKING:
+    from ssc_codegen.ast_.nodes_core import ExprClassVar
+
+T_ARGUMENTS = TypeVar("T_ARGUMENTS")
+T_MAPPING_FIELD = TypeVar("T_MAPPING_FIELD", bound=Mapping[str, Any])
+T_EMPTY_KWARGS = TypedDict("T_EMPTY_KWARGS", {})
+
+
+@dataclass(kw_only=True)
+class BaseAstNode(Generic[T_MAPPING_FIELD, T_ARGUMENTS]):
+    """Base abstract syntax tree (AST) node.
+
+    This is the base class for all AST nodes in the code generation system.
+    It provides common functionality and properties for AST node manipulation,
+    including parent-child relationships, type checking, and dynamic attribute access.
+
+    Attributes:
+        kind: The token type that identifies the kind of AST node.
+        kwargs: A mapping containing arguments or properties for this node.
+        body: A mutable sequence of child AST nodes.
+        parent: A reference to the parent AST node, if any.
+        accept_type: The variable type that this node accepts as input.
+        ret_type: The variable type that this node type returns.
+        exclude_types: A sequence of variable types that should be excluded from processing.
+        classvar_hooks: Dictionary of class variable hooks for expression evaluation.
+    """
+
+    kind: ClassVar[TokenType]
+    kwargs: T_MAPPING_FIELD = field(default_factory=dict)  # type: ignore[assignment]
+
+    body: MutableSequence["BaseAstNode"] = field(default_factory=list)
+    parent: Optional["BaseAstNode"] = field(default=None, repr=False)
+
+    # used for type check expressions
+    accept_type: VariableType = VariableType.ANY
+    ret_type: VariableType = VariableType.ANY
+    exclude_types: Sequence[VariableType] = field(
+        default_factory=tuple, repr=False
+    )
+    classvar_hooks: dict[str, "ExprClassVar"] = field(default_factory=dict)
+
+    def unpack_args(self) -> T_ARGUMENTS:
+        """Extract all values from the kwargs field without their keys.
+
+        Returns:
+            A tuple containing all values from the kwargs mapping, preserving their order.
+        """
+        return tuple(self.kwargs.values())  # type: ignore
+
+    @property
+    def index(self) -> int:
+        """Get the index of this node in its parent's body.
+
+        Returns:
+            The index of this node in the parent's body sequence.
+            Returns -1 if this node has no parent.
+        """
+        if self.parent:
+            # self.parent.body.index(self) add side effect if expr has same operations:
+            #      0      1      1       1
+            # D().raw().trim().trim().trim()
+            # use compare by id
+            for i, child in enumerate(self.parent.body):
+                if id(child) == id(self):
+                    return i
+            return self.parent.body.index(self)
+        return -1
+
+    @property
+    def index_next(self) -> int:
+        """Get the index of the next node in the parent's body.
+
+        Returns:
+            The index of the next node in the parent's body sequence.
+            Returns -1 if there is no next node or no parent.
+        """
+        if self.parent and len(self.parent.body) > self.index:
+            return self.index + 1
+        return -1
+
+    @property
+    def index_prev(self) -> int:
+        """Get the index of the previous node in the parent's body.
+
+        Returns:
+            The index of the previous node in the parent's body sequence.
+            Returns -1 if there is no previous node, no parent, or current index is -1.
+        """
+        if self.parent and self.index == 0:
+            return -1
+        elif self.index == -1:
+            return -1
+        return self.index - 1
+
+    def __getattr__(self, name: str) -> Any:
+        """Provides dynamic attribute access from the kwargs container.
+
+        This method allows accessing values in the kwargs dictionary as if they
+        were instance attributes, enabling convenient property-like access.
+
+        Args:
+            name: The name of the attribute to access.
+
+        Returns:
+            The value from kwargs if it exists, otherwise raises AttributeError.
+
+        Raises:
+            AttributeError: If the attribute is not found in kwargs or regular attributes.
+        """
+        try:
+            return super().__getattribute__(name)
+        except AttributeError as e:
+            if self.kwargs.get(name):
+                return self.kwargs[name]
+            raise e
+
+    def find_node(
+        self, predicate: Callable[["BaseAstNode"], bool]
+    ) -> Optional["BaseAstNode"]:
+        """Finds the first node in the body that matches the given filter function.
+
+        Args:
+            func: A callable that takes a BaseAstNode and returns True if it matches the criteria.
+
+        Returns:
+            The first matching node, or None if no node matches or the body is empty.
+        """
+        if len(self.body) == 0:
+            return None
+
+        for n in self.body:
+            if predicate(n):
+                return n
+        return None
+
+    def find_node_all(
+        self, predicate: Callable[["BaseAstNode"], bool]
+    ) -> list["BaseAstNode"]:
+        """Finds all nodes in the body that match the given filter function.
+
+        Args:
+            func: A callable that takes a BaseAstNode and returns True if it matches the criteria.
+
+        Returns:
+            A list of all matching nodes, or an empty list if no nodes match or the body is empty.
+        """
+        if len(self.body) == 0:
+            return []  # type: ignore
+        return [i for i in self.body if predicate(i)]
+
+    def find_node_by_token(
+        self, token_type: TokenType
+    ) -> Optional["BaseAstNode"]:
+        """Finds the first node in the body with the specified token type.
+
+        Args:
+            token_type: The TokenType to search for.
+
+        Returns:
+            The first matching node, or None if no node matches or the body is empty.
+        """
+        return self.find_node(lambda n: n.kind == token_type)
+
+    def find_nodes_by_token(self, token_type: TokenType) -> list["BaseAstNode"]:
+        """Finds all nodes in the body with the specified token type.
+
+        Args:
+            token_type: The TokenType to search for.
+
+        Returns:
+            A list of all matching nodes, or an empty list if no nodes match or the body is empty.
+        """
+        return self.find_node_all(lambda n: n.kind == token_type)
+
+
+@dataclass(kw_only=True)
+class _DisableReprBody(BaseAstNode):
+    """Special class to disable representation of unnecessary node fields.
+
+    This class is used to prevent the 'body' field from being included in
+    string representations, which can be useful for preventing excessive
+    output during debugging or logging.
+
+    Attributes:
+        body: A mutable sequence of child AST nodes (not included in repr).
+    """
+
+    body: MutableSequence[BaseAstNode] = field(default_factory=list, repr=False)
+
+
+@dataclass(kw_only=True)
+class _DisableReprAcceptAndRetType(BaseAstNode):
+    """Special class to disable representation of type fields.
+
+    This class is used to prevent the 'accept_type' and 'ret_type' fields
+    from being included in string representations, which can help reduce
+    verbosity in debugging or logging output.
+
+    Attributes:
+        accept_type: The variable type that this node accepts as input (not included in repr).
+        ret_type: The variable type that this node returns (not included in repr).
+    """
+
+    accept_type: VariableType = field(default=VariableType.ANY, repr=False)
+    ret_type: VariableType = field(default=VariableType.ANY, repr=False)
+
+
+@dataclass(kw_only=True)
+class _DisableRepr(_DisableReprBody, _DisableReprAcceptAndRetType):
+    """Special class to disable representation of multiple node fields.
+
+    This class combines the functionality of _DisableReprBody and
+    _DisableReprAcceptAndRetType to prevent the 'body', 'accept_type',
+    and 'ret_type' fields from being included in string representations.
+    This provides a more concise output for debugging and logging.
+
+    Attributes:
+        body: A mutable sequence of child AST nodes (not included in repr).
+        accept_type: The variable type that this node accepts as input (not included in repr).
+        ret_type: The variable type that this node returns (not included in repr).
+    """
+
+    body: MutableSequence[BaseAstNode] = field(default_factory=list, repr=False)
+    accept_type: VariableType = field(default=VariableType.ANY, repr=False)
+    ret_type: VariableType = field(default=VariableType.ANY, repr=False)

{ssc_codegen-0.12.0 → ssc_codegen-0.13.0}/ssc_codegen/ast_/nodes_array.py

@@ -10,6 +10,23 @@ ARGS_EXPR_INDEX = tuple[int]
 
 @dataclass(kw_only=True)
 class ExprIndex(BaseAstNode[KW_EXPR_INDEX, ARGS_EXPR_INDEX]):
+    """AST node representing an index access expression.
+
+    This node represents an operation that accesses an element at a specific index
+    in a list of any type.
+
+    Rules:
+        - first index starts from `0`. if target language index starts by `1`,
+         codegen should be automatically convert
+        - allow use negative indexes: last index starts by `-1`.
+            codegen should be automatically convert by (len(curr_value)-{index})
+    Note:
+        - if target html parser backend support pseudo selectors - recommended use it instead extract by index
+
+    Kwargs:
+        "index": int - index where need extract variable
+    """
+
     kind: ClassVar[TokenType] = TokenType.EXPR_LIST_ANY_INDEX
     accept_type: VariableType = VariableType.LIST_ANY
     ret_type: VariableType = VariableType.ANY
@@ -21,6 +38,15 @@ ARGS_EXPR_JOIN = tuple[str]
 
 @dataclass(kw_only=True)
 class ExprListStringJoin(BaseAstNode[KW_EXPR_JOIN, ARGS_EXPR_JOIN]):
+    """AST node representing a string join operation on a list of strings.
+
+    This node represents an operation that joins a list of strings using a
+    specified separator.
+
+    Kwargs:
+        "sep": str - separator for array of strings
+    """
+
     kind: ClassVar[TokenType] = TokenType.EXPR_LIST_JOIN
     accept_type: VariableType = VariableType.LIST_STRING
     ret_type: VariableType = VariableType.STRING
@@ -28,6 +54,11 @@ class ExprListStringJoin(BaseAstNode[KW_EXPR_JOIN, ARGS_EXPR_JOIN]):
 
 @dataclass(kw_only=True)
 class ExprToListLength(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    """AST node representing a list length operation.
+
+    This node represents an operation that returns the length of a list.
+    """
+
     kind: ClassVar[TokenType] = TokenType.EXPR_LIST_LEN
     accept_type: VariableType = VariableType.LIST_ANY
     ret_type: VariableType = VariableType.INT
@@ -39,6 +70,17 @@ ARGS_UNIQUE = tuple[bool]
 
 @dataclass(kw_only=True)
 class ExprListUnique(BaseAstNode[KW_UNIQUE, ARGS_UNIQUE]):
+    """AST node representing a unique elements operation on a list.
+
+    This node represents an operation that removes duplicate elements from a list
+    of strings, optionally preserving the order of elements.
+    Note: Currently only supports LIST_STRING, but TODO indicates support for
+    LIST_INT and LIST_FLOAT should be added.
+
+    Kwargs:
+        - keep_order: bool - guarante save order elements
+    """
+
     # TODO: support LIST_INT, LIST_FLOAT
     kind: ClassVar[TokenType] = TokenType.EXPR_LIST_UNIQUE
     accept_type: VariableType = VariableType.LIST_STRING

{ssc_codegen-0.12.0 → ssc_codegen-0.13.0}/ssc_codegen/ast_/nodes_cast.py

@@ -10,6 +10,11 @@ if TYPE_CHECKING:
 
 @dataclass(kw_only=True)
 class ExprToInt(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    """AST node representing an integer conversion expression.
+
+    This node represents an operation that converts a string value to an integer.
+    """
+
     kind: ClassVar[TokenType] = TokenType.TO_INT
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.INT
@@ -17,6 +22,11 @@ class ExprToInt(BaseAstNode[T_EMPTY_KWARGS, tuple]):
 
 @dataclass(kw_only=True)
 class ExprToListInt(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    """AST node representing a list of integers conversion expression.
+
+    This node represents an operation that converts a list of string values to a list of integers.
+    """
+
     kind: ClassVar[TokenType] = TokenType.TO_INT_LIST
     accept_type: VariableType = VariableType.LIST_STRING
     ret_type: VariableType = VariableType.LIST_INT
@@ -24,6 +34,11 @@ class ExprToListInt(BaseAstNode[T_EMPTY_KWARGS, tuple]):
 
 @dataclass(kw_only=True)
 class ExprToFloat(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    """AST node representing a float conversion expression.
+
+    This node represents an operation that converts a string value to a float.
+    """
+
     kind: ClassVar[TokenType] = TokenType.TO_FLOAT
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.FLOAT
@@ -31,6 +46,11 @@ class ExprToFloat(BaseAstNode[T_EMPTY_KWARGS, tuple]):
 
 @dataclass(kw_only=True)
 class ExprToListFloat(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    """AST node representing a list of floats conversion expression.
+
+    This node represents an operation that converts a list of string values to a list of floats.
+    """
+
     kind: ClassVar[TokenType] = TokenType.TO_FLOAT_LIST
     accept_type: VariableType = VariableType.LIST_STRING
     ret_type: VariableType = VariableType.LIST_FLOAT
@@ -38,6 +58,19 @@ class ExprToListFloat(BaseAstNode[T_EMPTY_KWARGS, tuple]):
 
 @dataclass(kw_only=True)
 class ExprToBool(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    """AST node representing a boolean conversion expression.
+
+    This node represents an operation that converts any value to a boolean.
+    Currently cast to bool logic:
+
+    returns `false` if value is:
+        - None
+        - empty sequence
+        - empty string
+
+    Other returns `true`
+    """
+
     kind: ClassVar[TokenType] = TokenType.TO_BOOL
     accept_type: VariableType = VariableType.ANY
     ret_type: VariableType = VariableType.BOOL
@@ -52,6 +85,17 @@ EXPR_NESTED_ARGS = tuple[str, StructType]
 
 @dataclass(kw_only=True)
 class ExprNested(BaseAstNode[KW_EXPR_NESTED, EXPR_NESTED_ARGS]):
+    """AST node representing a nested structure expression.
+
+    This node represents an operation that processes nested structures,
+    using a schema name and structure type to determine how to handle
+    the nested data.
+
+    Kwargs:
+        "schema_name": str - called nested schema name
+        "schema_type": StructType - called nested schema StructType
+    """
+
     kind: ClassVar[TokenType] = TokenType.EXPR_NESTED
     accept_type: VariableType = VariableType.DOCUMENT
     ret_type: VariableType = VariableType.NESTED
@@ -65,6 +109,19 @@ ARGS_EXPR_JSONIFY = tuple[str, bool, str]
 
 @dataclass(kw_only=True)
 class ExprJsonify(BaseAstNode[KW_EXPR_JSONIFY, ARGS_EXPR_JSONIFY]):
+    """AST node representing a JSON conversion expression.
+
+    This node represents an operation that converts a string to JSON format,
+    using a JSON structure name, array flag, and query to determine how to
+    structure the resulting JSON.
+
+    Kwargs:
+        "json_struct_name": str - json struct name
+        "is_array": bool - true if entrypoint array of structs
+        "query": str - optional query syntax where need start parse and serialize raw json data
+            maybe useful for skip non-nessesary data (eg: NEST.js states)
+    """
+
     kind: ClassVar[TokenType] = TokenType.TO_JSON
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.JSON
@@ -78,6 +135,20 @@ ARGS_EXPR_JSONIFY_DYNAMIC = tuple[str]
 class ExprJsonifyDynamic(
     BaseAstNode[KW_EXPR_JSONIFY_DYNAMIC, ARGS_EXPR_JSONIFY_DYNAMIC]
 ):
+    """AST node representing a dynamic JSON conversion expression.
+
+    This node represents an operation that dynamically converts a string to JSON format,
+    using a query to determine how to structure the resulting JSON. The return type
+    is currently stubbed to ANY.
+
+    NOTE:
+        maybe not supports in static-typed languages
+
+    Kwargs:
+        "query": str - optional query syntax where need start parse and serialize raw json data
+            maybe useful for skip non-nessesary data (eg: NEST.js states)
+    """
+
     kind: ClassVar[TokenType] = TokenType.TO_JSON_DYNAMIC
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.ANY  # STUBBED