ssc_codegen 0.14.0__tar.gz → 0.15.0__tar.gz

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.14.0
+Version: 0.15.0
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.14.0"
+version = "0.15.0"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/ssc_codegen/__init__.py

@@ -1,13 +1,13 @@
 from ssc_codegen.document import DocumentElementsFilter, HTMLDocument, ClassVarDocument, StringDocument, ArrayDocument, AssertDocument, NestedDocument, \
     DefaultDocument, \
-    NumericDocument, JsonDocument, BooleanDocument, DocumentFilter
+    NumericDocument, JsonDocument, BooleanDocument, DocumentFilter, TransformDocument
 from ssc_codegen.json_struct import Json
 from ssc_codegen.logs import setup_logger
 from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchema, AccUniqueListSchema
 
 setup_logger()
 
-VERSION = "0.14.0"
+VERSION = "0.15.0"
 
 
 class __MISSING(object):
@@ -26,7 +26,8 @@ class Document(
     DefaultDocument,
     NumericDocument,
     JsonDocument,
-    BooleanDocument
+    BooleanDocument,
+    TransformDocument
 ):
     """Special Common Document or Element marker manipulations"""
     pass

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/ssc_codegen/ast_/__init__.py

@@ -38,7 +38,10 @@ from .nodes_core import (
     ExprClassVar,
     ExprCallStructClassVar,
     CodeStart,
-    CodeEnd
+    CodeEnd,
+    ExprTransform,
+    ModuleTransformImports,
+    ModuleUtilities
 )
 from .nodes_filter import (
     FilterOr,

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/ssc_codegen/ast_/nodes_core.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass, field
-from typing import Sequence, TypedDict, ClassVar, TypeVar
+from typing import Sequence, TypedDict, ClassVar, TypeVar, TYPE_CHECKING
 
 from ssc_codegen.ast_.base import T_EMPTY_KWARGS, BaseAstNode
 from ssc_codegen.ast_.base import (
@@ -14,6 +14,9 @@ from ssc_codegen.tokens import (
     VariableType,
 )
 
+if TYPE_CHECKING:
+    from ssc_codegen.transform import BaseTransform
+
 KW_DOCSTRING = TypedDict("KW_DOCSTRING", {"value": str})
 
 
@@ -34,11 +37,32 @@ class Docstring(_DisableRepr, BaseAstNode[KW_DOCSTRING, tuple[str]]):
     """
 
     kind: ClassVar[TokenType] = TokenType.DOCSTRING
-    kwargs: KW_DOCSTRING
+
+
+# push transform for resolve imports (dependencies)
+KW_MODULE_IMPORTS = TypedDict(
+    "KW_MODULE_IMPORTS", {"transforms": list["BaseTransform"]}
+)
+ARGS_MODULE_IMPORTS = tuple[list["BaseTransform"]]
+
+
+@dataclass(kw_only=True)
+class ModuleTransformImports(
+    BaseAstNode[KW_MODULE_IMPORTS, ARGS_MODULE_IMPORTS]
+):
+    """AST node representing module import transfrom dependencies.
+
+    This node contains import statements that will be added to the
+    generated module to ensure required dependencies are available.
+    """
+
+    kind: ClassVar[TokenType] = TokenType.TRANSFORM_IMPORTS
 
 
 @dataclass(kw_only=True)
-class ModuleImports(_DisableRepr, BaseAstNode[T_EMPTY_KWARGS, tuple]):
+class ModuleImports(
+    _DisableRepr, BaseAstNode[KW_MODULE_IMPORTS, ARGS_MODULE_IMPORTS]
+):
     """AST node representing module import statements.
 
     This node contains import statements that will be added to the
@@ -551,3 +575,17 @@ class CodeEnd(_DisableRepr, BaseAstNode):
     """
 
     kind: ClassVar[TokenType] = TokenType.CODE_END
+
+
+KW_TRANSFORM = TypedDict("KW_TRANSFORM", {"transform": "BaseTransform"})
+ARGS_TRANSFORM = tuple["BaseTransform"]
+
+
+@dataclass(kw_only=True)
+class ExprTransform(BaseAstNode[KW_TRANSFORM, ARGS_TRANSFORM]):
+    kind: ClassVar[TokenType] = TokenType.TRANSFORM
+
+
+@dataclass(kw_only=True)
+class ModuleUtilities(_DisableRepr, BaseAstNode):
+    kind: ClassVar[TokenType] = TokenType.UTILITIES

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/ssc_codegen/ast_build/builder.py

@@ -3,6 +3,7 @@ from typing import MutableSequence, Type, cast
 import warnings
 from typing_extensions import Self
 
+from ssc_codegen.ast_ import ModuleTransformImports, ModuleUtilities
 from ssc_codegen.ast_.base import BaseAstNode
 from ssc_codegen.ast_.nodes_cast import ExprJsonify, ExprNested
 from ssc_codegen.ast_.nodes_core import (
@@ -113,7 +114,16 @@ class AstBuilder:
                 Docstring(kwargs={"value": module_doc}, parent=self.module)
             )
         # import node
-        self.module.body.append(ModuleImports(parent=self.module))
+        # fallback add ModuleImports.kwargs["transform"] if target language not allowed use multiple import directives
+        self.module.body.append(
+            ModuleImports(parent=self.module, kwargs={"transforms": []})
+        )
+        self.module.body.append(
+            ModuleTransformImports(
+                parent=self.module, kwargs={"transforms": []}
+            )
+        )
+        self.module.body.append(ModuleUtilities(parent=self.module))
         # CodeStart hook node
         self.module.body.append(CodeStart(parent=self.module))
         return self
@@ -398,10 +408,23 @@ class AstBuilder:
         # generate typedef as annotation node
         return st
 
+    @staticmethod
+    def is_collected_type_transform(transforms: list, target: object) -> bool:
+        for t in transforms:
+            if isinstance(t, type(target)):
+                return True
+        return False
+
     def add_struct_parsers(self, *struct_parsers: Type[BaseSchema]) -> Self:
         ssc_structs = []
         ssc_typedefs = []
+        ssc_transforms = []
         for schema in struct_parsers:
+            if schema.__SSC_TRANSFORMS__:
+                # check current used transforms for avoid generate duplicate imports
+                for t in schema.__SSC_TRANSFORMS__:
+                    if not any(isinstance(i, type(t)) for i in ssc_transforms):
+                        ssc_transforms.append(t)
             if is_literals_only_schema(schema):
                 if schema.__SCHEMA_TYPE__ != StructType.ITEM:
                     raise TypeError(
@@ -420,6 +443,21 @@ class AstBuilder:
                 continue
             ssc_typedefs.append(typedef)
         # assembly
+
+        # insert transforms imports (in high-level API provide insert dependencies)
+
+        node_transform_import = self.module.find_node_by_token(
+            ModuleTransformImports.kind
+        )
+        node_transform_import = cast(
+            ModuleTransformImports, node_transform_import
+        )
+        node_transform_import.kwargs["transforms"] = ssc_transforms.copy()
+
+        node_import = self.module.find_node_by_token(ModuleImports.kind)
+        node_import = cast(ModuleImports, node_import)
+        node_import.kwargs["transform"] = ssc_transforms.copy()
+
         self.module.body.extend(ssc_typedefs)
         self.module.body.extend(ssc_structs)
         self.module.body.append(CodeEnd(parent=self.module))

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/ssc_codegen/converters/go_goquery.py

@@ -51,6 +51,9 @@ from typing_extensions import assert_never
 
 from ssc_codegen.ast_ import (
     Docstring,
+    ExprTransform,
+    ModuleTransformImports,
+    ModuleUtilities,
     StructParser,
     ExprReturn,
     ExprNoReturn,
@@ -393,21 +396,37 @@ def pre_docstring(node: Docstring) -> str:
 
 @CONVERTER(ModuleImports.kind)
 def pre_module_imports(_node: ModuleImports) -> str:
+    # TODO: resolve imports
+    return IMPORTS.replace("$PACKAGE$", CONVERTER.PACKAGE)
+
+
+@CONVERTER(ModuleTransformImports.kind)
+def pre_transform_imports(node: ModuleTransformImports) -> str:
+    transforms, *_ = node.unpack_args()
+    if transforms:
+        for t in transforms:
+            if t.collect_dependencies("go_goquery"):
+                raise NotImplementedError(
+                    "Current converter implementation not support pass extra import dependencies"
+                )
+    return ""
+
+
+@CONVERTER(ExprTransform.kind)
+def pre_transform(node: ExprTransform) -> str:
+    prv, nxt = prev_next_var(node)
+    transform, *_ = node.unpack_args()
+    parts = transform.emit("go_goquery", prv, nxt)
+    return "\n" + "\n".join(parts)
+
+
+@CONVERTER(ModuleUtilities.kind)
+def pre_module_utilities(_: ModuleUtilities) -> str:
     # HACK:
-    # golang not allowed override functions, check loaded helper functions by flag
-    helper_functions = (
-        HELPER_FUNCTIONS if not CONVERTER.HELPER_FUNCS_IMPORTED else ""
-    )
-    code = (
-        # magic constant
-        IMPORTS.replace("$PACKAGE$", CONVERTER.PACKAGE)
-        # HACK: push helper functions to ModuleImports token:
-        #       its generated second after module docstring
-        + helper_functions
-    )
+    # golang not allowed override or duplicate functions, check loaded helper functions by flag
     if not CONVERTER.HELPER_FUNCS_IMPORTED:
         CONVERTER.HELPER_FUNCS_IMPORTED = True
-    return code
+    return HELPER_FUNCTIONS if not CONVERTER.HELPER_FUNCS_IMPORTED else ""
 
 
 def get_typedef_field_by_name(node: TypeDef, field_name: str) -> str:

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/ssc_codegen/converters/js_pure.py

@@ -23,6 +23,9 @@ from typing import cast
 
 from ssc_codegen.ast_ import (
     Docstring,
+    ExprTransform,
+    ModuleTransformImports,
+    ModuleUtilities,
     StructParser,
     ExprReturn,
     ExprNoReturn,
@@ -173,7 +176,11 @@ DOCSTR_START = "/**"
 DOCSTR_END = "*/"
 DOCSTR_SEP = "* "
 CONVERTER = BaseCodeConverter(debug_comment_prefix="// ")
-# javascript not support typing/annotations
+CONVERTER.TEST_EXCLUDE_NODES.extend(
+    [
+        ModuleImports.kind,  # not use imports
+    ]
+)
 
 
 # TODO: move to string_utils
@@ -213,8 +220,30 @@ def py_sequence_to_js_array(values: tuple[str, ...] | list[str]) -> str:
     return "[" + val_arr[1:-1] + "]"
 
 
-@CONVERTER(ModuleImports.kind)
-def pre_imports(_: ModuleImports) -> str:
+@CONVERTER(ModuleTransformImports.kind)
+def pre_transform_imports(node: ModuleTransformImports) -> str:
+    transforms, *_ = node.unpack_args()
+
+    if not transforms:
+        return ""
+
+    code_imports = []
+    for t in transforms:
+        if deps := t.collect_dependencies("js_pure"):
+            code_imports.append("\n".join(deps))
+    return "\n".join(code_imports)
+
+
+@CONVERTER(ExprTransform.kind)
+def pre_transform(node: ExprTransform) -> str:
+    # extend transform search provide py_bs4 module
+    prv, nxt = prev_next_var(node)
+    transform, *_ = node.unpack_args()
+    return "\n".join(f"{i};" for i in transform.emit("js_pure", prv, nxt))
+
+
+@CONVERTER(ModuleUtilities.kind)
+def pre_imports(_: ModuleUtilities) -> str:
     return HELPER_FUNCTIONS
 
 

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/ssc_codegen/converters/lua_htmlparser.py

@@ -28,6 +28,9 @@ from typing import Any, cast
 
 from ssc_codegen.ast_ import (
     Docstring,
+    ExprTransform,
+    ModuleTransformImports,
+    ModuleUtilities,
     StructParser,
     ExprReturn,
     ExprNoReturn,
@@ -187,6 +190,7 @@ CONVERTER.TEST_EXCLUDE_NODES.extend(
         FilterDocHasText.kind,
         FilterDocIsRegexText.kind,
         FilterDocIsRegexRaw.kind,
+        # NOT IMPLEMENTED
     ]
 )
 
@@ -291,7 +295,34 @@ def py_regex_to_lua_rex_pcre(
 
 @CONVERTER(ModuleImports.kind)
 def pre_imports(_: ModuleImports) -> str:
-    return IMPORTS + "\n" + HELPER_FUNCTIONS
+    return IMPORTS
+
+
+@CONVERTER(ModuleTransformImports.kind)
+def pre_transform_imports(node: ModuleTransformImports) -> str:
+    transforms, *_ = node.unpack_args()
+
+    if not transforms:
+        return ""
+
+    code_imports = []
+    for t in transforms:
+        if deps := t.collect_dependencies("lua_htmlparser"):
+            code_imports.append("\n".join(deps))
+    return "\n".join(code_imports)
+
+
+@CONVERTER(ModuleUtilities.kind)
+def pre_utilities(_: ModuleUtilities) -> str:
+    return HELPER_FUNCTIONS
+
+
+@CONVERTER(ExprTransform.kind)
+def pre_transform(node: ExprTransform) -> str:
+    prv, nxt = prev_next_var(node)
+    transform, *_ = node.unpack_args()
+    parts = transform.emit("lua_htmlparser", prv, nxt)
+    return "\n" + "\n".join(parts)
 
 
 @CONVERTER(Docstring.kind)

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/ssc_codegen/converters/py_base.py

@@ -35,7 +35,10 @@ from typing import cast
 
 from ssc_codegen.ast_ import (
     Docstring,
+    ExprTransform,
     ModuleImports,
+    ModuleTransformImports,
+    ModuleUtilities,
     TypeDef,
     TypeDefField,
     JsonStruct,
@@ -214,6 +217,9 @@ class BasePyCodeConverter(BaseCodeConverter):
         self.pre_definitions = {
             Docstring.kind: pre_docstring,
             ModuleImports.kind: pre_imports,
+            ModuleTransformImports.kind: pre_transform_imports,
+            ExprTransform.kind: pre_transform,
+            ModuleUtilities.kind: pre_utilities,
             TypeDefField.kind: pre_typedef_field,
             JsonStruct.kind: pre_json_struct,
             JsonStructField.kind: pre_json_field,
@@ -295,7 +301,6 @@ class BasePyCodeConverter(BaseCodeConverter):
         }
 
         self.post_definitions = {
-            ModuleImports.kind: post_imports,
             JsonStruct.kind: post_json_struct,
             ExprFilter.kind: post_expr_filter,
             FilterAnd.kind: post_filter_and,
@@ -321,6 +326,16 @@ class BasePyCodeConverter(BaseCodeConverter):
         }
 
 
+def pre_transform(node: ExprTransform) -> str:
+    prv, nxt = prev_next_var(node)
+    transform, *_ = node.unpack_args()
+    parts = transform.emit("py_base", prv, nxt)
+    if have_default_expr(node):
+        code = INDENT_DEFAULT_BODY + INDENT_DEFAULT_BODY.join(parts)
+        return code
+    return INDENT_METHOD_BODY + INDENT_METHOD_BODY.join(parts)
+
+
 def get_typedef_field_by_name(node: TypeDef, field_name: str) -> str:
     value = [i for i in node.body if i.kwargs["name"] == field_name][0]
     value = cast(TypeDefField, value)
@@ -366,12 +381,25 @@ def pre_docstring(node: Docstring) -> str:
     return ""
 
 
-def pre_imports(_node: ModuleImports) -> str:
+def pre_utilities(_: ModuleUtilities) -> str:
+    return HELPER_FUNCTIONS
+
+
+def pre_imports(_: ModuleImports) -> str:
     return IMPORTS_MIN
 
 
-def post_imports(_: ModuleImports) -> str:
-    return HELPER_FUNCTIONS
+def pre_transform_imports(node: ModuleTransformImports) -> str:
+    transforms, *_ = node.unpack_args()
+
+    if not transforms:
+        return ""
+
+    code_imports = []
+    for t in transforms:
+        if deps := t.collect_dependencies("py_base"):
+            code_imports.append("\n".join(deps))
+    return "\n".join(code_imports)
 
 
 # TYPEDEF

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/ssc_codegen/converters/py_bs4.py

@@ -1,6 +1,7 @@
 from ssc_codegen.ast_ import (
     ExprCss,
     ExprCssAll,
+    ExprTransform,
     ModuleImports,
     ExprXpathAll,
     ExprXpath,
@@ -12,6 +13,7 @@ from ssc_codegen.ast_ import (
     ExprGetHtmlRaw,
     ExprGetHtmlRawAll,
     ExprIsCss,
+    ModuleTransformImports,
     StructInitMethod,
     StructFieldMethod,
     StructPreValidateMethod,
@@ -64,6 +66,32 @@ from ssc_codegen.converters.templates.py_base import (
 CONVERTER = BasePyCodeConverter()
 
 
+@CONVERTER.post(ModuleTransformImports.kind)
+def post_transform_imports(node: ModuleTransformImports) -> str:
+    transforms, *_ = node.unpack_args()
+
+    if not transforms:
+        return ""
+
+    code_imports = []
+    for t in transforms:
+        if deps := t.collect_dependencies("py_bs4"):
+            code_imports.append("\n".join(deps))
+    return "\n".join(code_imports)
+
+
+@CONVERTER.post(ExprTransform.kind)
+def post_transform(node: ExprTransform) -> str:
+    # extend transform search provide py_bs4 module
+    prv, nxt = prev_next_var(node)
+    transform, *_ = node.unpack_args()
+    parts = transform.emit("py_bs4", prv, nxt)
+    if have_default_expr(node):
+        code = INDENT_DEFAULT_BODY + INDENT_DEFAULT_BODY.join(parts)
+        return code
+    return INDENT_METHOD_BODY + INDENT_METHOD_BODY.join(parts)
+
+
 @CONVERTER(StructInitMethod.kind)
 def pre_init(_node: StructInitMethod) -> str:
     return (
@@ -102,10 +130,10 @@ def pre_struct_field_method(node: StructFieldMethod) -> str:
 
 
 @CONVERTER.pre(ModuleImports.kind)
-def pre_imports(_: ModuleImports) -> str:
+def pre_imports(node: ModuleImports) -> str:
     return (
         IMPORTS_MIN
-        + "from bs4 import BeautifulSoup, ResultSet, Tag  # noqa (for typing)"
+        + "from bs4 import BeautifulSoup, ResultSet, Tag  # noqa (for typing)\n"
     )
 
 

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/ssc_codegen/converters/py_lxml.py

@@ -1,6 +1,7 @@
 from ssc_codegen.ast_ import (
     ExprCss,
     ExprCssAll,
+    ExprTransform,
     ModuleImports,
     ExprXpathAll,
     ExprXpath,
@@ -12,6 +13,8 @@ from ssc_codegen.ast_ import (
     ExprGetHtmlRaw,
     ExprGetHtmlRawAll,
     ExprIsCss,
+    ModuleTransformImports,
+    ModuleUtilities,
     StructInitMethod,
     StructFieldMethod,
     StructPreValidateMethod,
@@ -62,24 +65,6 @@ from ssc_codegen.converters.templates.py_base import IMPORTS_MIN
 
 CONVERTER = BasePyCodeConverter()
 
-"""
-# HtmlElement → сразу принимаем
-if isinstance(document, html.HtmlElement):
-    self._document = document
-    return
-
-# Строка → пробуем парсить
-if isinstance(document, str):
-    try:
-        self._document = html.fromstring(document.strip() or self.FALLBACK_HTML)
-        return
-    except (ParserError, ValueError):
-        pass
-
-# Всё остальное или ошибка → заглушка
-self._document = html.fromstring(self.FALLBACK_HTML)
-"""
-
 
 @CONVERTER(StructInitMethod.kind)
 def pre_init(_node: StructInitMethod) -> str:
@@ -123,15 +108,42 @@ def pre_struct_field_method(node: StructFieldMethod) -> str:
 
 
 @CONVERTER(ModuleImports.kind)
-def pre_imports(_: ModuleImports) -> str:
+def pre_imports(node: ModuleImports) -> str:
     # lxml throw parse error if passed empty string
     # most mainsteram html parser libs use `FALLBACK_HTML_STR` stub value
-    return (
-        IMPORTS_MIN
-        + """from lxml import html
-FALLBACK_HTML_STR = "<html><body></body></html>"
-    """
-    )
+
+    return IMPORTS_MIN + "from lxml import html"
+
+
+@CONVERTER.post(ModuleTransformImports.kind)
+def post_transform_imports(node: ModuleTransformImports) -> str:
+    transforms, *_ = node.unpack_args()
+
+    if not transforms:
+        return ""
+
+    code_imports = []
+    for t in transforms:
+        if deps := t.collect_dependencies("py_lxml"):
+            code_imports.append("\n".join(deps))
+    return "\n".join(code_imports)
+
+
+@CONVERTER.post(ExprTransform.kind)
+def post_transform(node: ExprTransform) -> str:
+    # extend transform search provide py_bs4 module
+    prv, nxt = prev_next_var(node)
+    transform, *_ = node.unpack_args()
+    parts = transform.emit("py_lxml", prv, nxt)
+    if have_default_expr(node):
+        code = INDENT_DEFAULT_BODY + INDENT_DEFAULT_BODY.join(parts)
+        return code
+    return INDENT_METHOD_BODY + INDENT_METHOD_BODY.join(parts)
+
+
+@CONVERTER.post(ModuleUtilities.kind)
+def post_utilities(_: ModuleUtilities) -> str:
+    return 'FALLBACK_HTML_STR = "<html><body></body></html>"'
 
 
 @CONVERTER(ExprCss.kind)

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/ssc_codegen/converters/py_parsel.py

@@ -1,6 +1,7 @@
 from ssc_codegen.ast_ import (
     ExprCss,
     ExprCssAll,
+    ExprTransform,
     ModuleImports,
     ExprXpathAll,
     ExprXpath,
@@ -12,6 +13,7 @@ from ssc_codegen.ast_ import (
     ExprGetHtmlRaw,
     ExprGetHtmlRawAll,
     ExprIsCss,
+    ModuleTransformImports,
     StructInitMethod,
     StructFieldMethod,
     StructPreValidateMethod,
@@ -62,6 +64,32 @@ from ssc_codegen.converters.templates.py_base import IMPORTS_MIN
 CONVERTER = BasePyCodeConverter()
 
 
+@CONVERTER.post(ModuleTransformImports.kind)
+def post_transform_imports(node: ModuleTransformImports) -> str:
+    transforms, *_ = node.unpack_args()
+
+    if not transforms:
+        return ""
+
+    code_imports = []
+    for t in transforms:
+        if deps := t.collect_dependencies("py_parsel"):
+            code_imports.append("\n".join(deps))
+    return "\n".join(code_imports)
+
+
+@CONVERTER.post(ExprTransform.kind)
+def post_transform(node: ExprTransform) -> str:
+    # extend transform search provide py_bs4 module
+    prv, nxt = prev_next_var(node)
+    transform, *_ = node.unpack_args()
+    parts = transform.emit("py_parsel", prv, nxt)
+    if have_default_expr(node):
+        code = INDENT_DEFAULT_BODY + INDENT_DEFAULT_BODY.join(parts)
+        return code
+    return INDENT_METHOD_BODY + INDENT_METHOD_BODY.join(parts)
+
+
 @CONVERTER(StructInitMethod.kind)
 def pre_init(_node: StructInitMethod) -> str:
     return (

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/ssc_codegen/converters/py_selectolax.py

@@ -1,6 +1,7 @@
 from ssc_codegen.ast_ import (
     ExprCss,
     ExprCssAll,
+    ExprTransform,
     ModuleImports,
     ExprXpathAll,
     ExprXpath,
@@ -12,6 +13,7 @@ from ssc_codegen.ast_ import (
     ExprGetHtmlRaw,
     ExprGetHtmlRawAll,
     ExprIsCss,
+    ModuleTransformImports,
     StructInitMethod,
     StructFieldMethod,
     StructPreValidateMethod,
@@ -62,6 +64,32 @@ from ssc_codegen.converters.templates.py_base import IMPORTS_MIN
 CONVERTER = BasePyCodeConverter()
 
 
+@CONVERTER.post(ModuleTransformImports.kind)
+def post_transform_imports(node: ModuleTransformImports) -> str:
+    transforms, *_ = node.unpack_args()
+
+    if not transforms:
+        return ""
+
+    code_imports = []
+    for t in transforms:
+        if deps := t.collect_dependencies("py_selectolax"):
+            code_imports.append("\n".join(deps))
+    return "\n".join(code_imports)
+
+
+@CONVERTER.post(ExprTransform.kind)
+def post_transform(node: ExprTransform) -> str:
+    # extend transform search provide py_bs4 module
+    prv, nxt = prev_next_var(node)
+    transform, *_ = node.unpack_args()
+    parts = transform.emit("py_selectolax", prv, nxt)
+    if have_default_expr(node):
+        code = INDENT_DEFAULT_BODY + INDENT_DEFAULT_BODY.join(parts)
+        return code
+    return INDENT_METHOD_BODY + INDENT_METHOD_BODY.join(parts)
+
+
 @CONVERTER(StructInitMethod.kind)
 def pre_init(_node: StructInitMethod) -> str:
     return (

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/ssc_codegen/document.py

@@ -2,7 +2,16 @@
 
 from functools import wraps
 import logging
-from typing import Any, Callable, Type, Pattern, Sequence, TypeVar, Union
+from typing import (
+    Any,
+    Callable,
+    Type,
+    Pattern,
+    Sequence,
+    TypeVar,
+    Union,
+    TYPE_CHECKING,
+)
 from re import Pattern as RePattern
 
 from cssselect import SelectorSyntaxError
@@ -13,6 +22,7 @@ from ssc_codegen.ast_ import (
     ExprDefaultValueWrapper,
     ExprCss,
     ExprCssAll,
+    ExprTransform,
     ExprXpathAll,
     ExprGetHtmlAttr,
     ExprGetHtmlText,
@@ -125,6 +135,9 @@ from ssc_codegen.schema import BaseSchema
 from ssc_codegen.selector_utils import validate_css_query, validate_xpath_query
 from ssc_codegen.tokens import TokenType, VariableType
 
+if TYPE_CHECKING:
+    from .transform import BaseTransform
+
 LOGGER = logging.getLogger("ssc_gen")
 
 T = TypeVar("T", bound="BaseDocument")
@@ -2626,3 +2639,23 @@ class DocumentElementsFilter(BaseDocument):
         new_filter = DocumentElementsFilter()
         new_filter.stack.extend(self.stack)
         return DocumentElementsFilter().not_(new_filter)
+
+
+class TransformDocument(BaseDocument):
+    def transform(self, transformer: "BaseTransform") -> Self:
+        # manual test types from transformer
+        if self.stack_last_ret != transformer.accept_type:
+            LOGGER.warning(
+                "transform() [%s]: Expected type(s) %s, got %s",
+                transformer.__class__.__name__,
+                transformer.accept_type.name,
+                self.stack_last_ret.name,
+            )
+        self._add(
+            ExprTransform(
+                kwargs={"transform": transformer},
+                accept_type=transformer.accept_type,
+                ret_type=transformer.return_type,
+            )
+        )
+        return self

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/ssc_codegen/schema.py

@@ -9,6 +9,7 @@ from .tokens import StructType, TokenType, VariableType
 if TYPE_CHECKING:
     from .document import BaseDocument, ClassVarDocument
     from .json_struct import Json
+    from .transform import BaseTransform
 
 
 class MISSING_FIELD(object):  # noqa
@@ -155,6 +156,16 @@ class SchemaMeta(type):
                                     tmp_cvars.pop(cvar_name)
                                     break
 
+    @staticmethod
+    def __resolve_self_transforms(cls: Type["BaseSchema"]) -> None:
+        # called after __fill_mro_fields(cls)
+        result: list["BaseTransform"] = []
+        for field in cls.__SSC_MRO_FIELDS__.values():
+            for node in field.stack:
+                if node.kind == TokenType.TRANSFORM:
+                    result.append(node.kwargs["transform"])
+        cls.__SSC_TRANSFORMS__ = result
+
     def __new__(mcs, name, bases, namespace, **kwargs):  # type: ignore
         cls = super().__new__(mcs, name, bases, namespace)
         cls = cast(Type["BaseSchema"], cls)
@@ -169,7 +180,7 @@ class SchemaMeta(type):
         mcs.__fill_mro_fields(cls)
         mcs.__fill_literals(cls, name)
         mcs.__resolve_self_classvars(cls)
-
+        mcs.__resolve_self_transforms(cls)
         return cls
 
 
@@ -213,6 +224,9 @@ class BaseSchema(metaclass=SchemaMeta):
     # old name - literals
     __SSC_MRO_CLASSVARS__: dict[str, "ClassVarDocument"]
 
+    # it opmimize collect extra imports
+    __SSC_TRANSFORMS__: list["BaseTransform"]
+
     # retained for API backward compatibility
     @classmethod
     def __class_signature__(cls) -> dict[str, Any] | list[str | Any] | None:

{ssc_codegen-0.14.0 → ssc_codegen-0.15.0}/ssc_codegen/tokens.py

@@ -87,6 +87,9 @@ class TokenType(IntEnum):
     due to the risk of race-condition or other side effects.
     """
 
+    UTILITIES = auto()
+    """used for part import logic and helper functions/constants"""
+
     # utils nodes
     CODE_START = auto()
     """insert to ast after DOCSTRING and IMPORTS tokens
@@ -100,6 +103,9 @@ class TokenType(IntEnum):
     maybe usef for inject custom code or logic
     """
 
+    TRANSFORM = auto()
+    TRANSFORM_IMPORTS = auto()
+
     # STRUCTS
     STRUCT = auto()
     STRUCT_INIT = auto()

ssc_codegen-0.15.0/ssc_codegen/transform.py

@@ -0,0 +1,95 @@
+from typing import Callable, ClassVar, TypedDict
+from .tokens import VariableType
+
+
+class EmitSpec(TypedDict):
+    method_name: str
+    dependencies: list[str]
+    func: Callable[..., list[str]]
+
+
+def target(backend: str, dependencies: list[str] | None = None):
+    """decorator for provide transformation
+
+    Args:
+        backend - target lib (py_base, js_pure, py_bs4 etc)
+        dependencies - optional required imports (push to ModuleImports() AST nodes)
+    """
+    dependencies = dependencies or []
+
+    def decorator(fn: Callable[..., list[str]]) -> Callable[..., list[str]]:
+        setattr(fn, "_emit_backend", backend)
+        setattr(fn, "_emit_deps", dependencies)
+        return fn
+
+    return decorator
+
+
+class BaseTransformMeta(type):
+    def __new__(mcs, name, bases, namespace):
+        cls = super().__new__(mcs, name, bases, namespace)
+
+        impls: dict[str, EmitSpec] = {}
+
+        for base in reversed(cls.__mro__[1:]):
+            base_impls = getattr(base, "_emit_impls", None)
+            if isinstance(base_impls, dict):
+                for backend, spec in base_impls.items():
+                    impls.setdefault(
+                        backend,
+                        {
+                            "method_name": spec["method_name"],
+                            "dependencies": list(spec["dependencies"]),
+                            "func": spec["func"],
+                        },
+                    )
+
+        for attr_name, attr_value in namespace.items():
+            if callable(attr_value) and hasattr(attr_value, "_emit_backend"):
+                backend = getattr(attr_value, "_emit_backend")
+                deps = getattr(attr_value, "_emit_deps", []) or []
+
+                if (
+                    backend in impls
+                    and impls[backend]["method_name"] in namespace
+                ):
+                    raise ValueError(
+                        f"Duplicate emit implementations for backend '{backend}' in class {name}"
+                    )
+
+                impls[backend] = EmitSpec(
+                    method_name=attr_name,
+                    dependencies=list(deps),
+                    func=attr_value,  # type: ignore
+                )
+
+        setattr(cls, "_emit_impls", impls)
+        return cls
+
+
+class BaseTransform(metaclass=BaseTransformMeta):
+    # change this variables for provide static analyze
+    accept_type: ClassVar[VariableType]
+    return_type: ClassVar[VariableType]
+
+    _emit_impls: ClassVar[dict[str, EmitSpec]] = {}  # autofill from metaclass
+
+    @classmethod
+    def has_backend(cls, backend: str) -> bool:
+        return backend in getattr(cls, "_emit_impls", {})
+
+    @classmethod
+    def get_emit_spec(cls, backend: str) -> EmitSpec | None:
+        return getattr(cls, "_emit_impls", {}).get(backend)
+
+    def collect_dependencies(self, backend: str) -> list[str]:
+        spec = self.get_emit_spec(backend)
+        return spec["dependencies"][:] if spec else []
+
+    # Instance method to call emitter (wraps class method to bind self)
+    def emit(self, backend: str, prv: str, nxt: str) -> list[str]:
+        spec = self.get_emit_spec(backend)
+        if not spec:
+            return []
+        fn = getattr(self, spec["method_name"])
+        return fn(prv, nxt)