ssc_codegen 0.13.4__tar.gz → 0.15.0__tar.gz

{ssc_codegen-0.13.4 → ssc_codegen-0.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.13.4
+Version: 0.15.0
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
@@ -58,15 +58,15 @@ ssc-gen - a python-based DSL to describe parsers for html documents, which is tr
 
 Current support converters
 
-| Language                                 | HTML parser lib + dependencies         | XPath | CSS3 | CSS4 | Generated annotations, types, structs | formatter dependency |
-| ---------------------------------------- | -------------------------------------- | ----- | ---- | ---- | ------------------------------------- | -------------------- |
-| Python (3.8+)                            | bs4, lxml                              | N     | Y    | Y    | TypedDict`1`, list, dict              | ruff                 |
-| ...                                      | parsel                                 | Y     | Y    | N    | ...                                   | ...                  |
-| ...                                      | selectolax (lexbor)                    | N     | Y    | N    | ...                                   | ...                  |
-| ...                                      | lxml                                   | Y     | Y    | N    | ...                                   | ...                  |
-| js (ES6)`2`                              | pure (firefox/chrome extension/nodejs) | Y     | Y    | Y    | JSDoc                                 | prettier             |
-| go (1.10+) **(UNSTABLE)**                | goquery, gjson (`4`)                   | N     | Y    | N    | struct(+json anchors), array, map     | gofmt                |
-| lua (5.2+), luajit(2+) **(UNSTABLE)**`5` | lua-htmlparser, lrexlib(opt), dkjson   | N     | Y    | N    | EmmyLua                               | LuaFormatter         |
+| Language                                 | HTML parser lib + dependencies                         | XPath | CSS3 | CSS4 | Generated annotations, types, structs | formatter dependency |
+| ---------------------------------------- | ------------------------------------------------------ | ----- | ---- | ---- | ------------------------------------- | -------------------- |
+| Python (3.8-3.13)                        | bs4, lxml ( typing_extensions if py < 3.10 )           | N     | Y    | Y    | TypedDict`1`, list, dict              | ruff                 |
+| ...                                      | parsel ( typing_extensions if py < 3.10 )              | Y     | Y    | N    | ...                                   | ...                  |
+| ...                                      | selectolax (lexbor) ( typing_extensions if py < 3.10 ) | N     | Y    | N    | ...                                   | ...                  |
+| ...                                      | lxml ( typing_extensions if py < 3.10 )                | Y     | Y    | N    | ...                                   | ...                  |
+| js (ES6)`2`                              | pure (firefox/chrome extension/nodejs)                 | Y     | Y    | Y    | JSDoc                                 | prettier             |
+| go (1.10+) **(UNSTABLE)**                | goquery, gjson (`4`)                                   | N     | Y    | N    | struct(+json anchors), array, map     | gofmt                |
+| lua (5.2+), luajit(2+) **(UNSTABLE)**`5` | lua-htmlparser, lrexlib(opt), dkjson                   | N     | Y    | N    | EmmyLua                               | LuaFormatter         |
 
 - **CSS3** means support next selectors:
   - basic: (`tag`, `.class`, `#id`, `tag1,tag2`)
@@ -99,6 +99,7 @@ For maximum portability of the configuration to the target language:
   Check the html parser lib documentation aboud CSS selectors before implement code. Examples:
   1. Several libs not support `+` operations (eg: [selectolax(modest)](https://github.com/rushter/selectolax), [dart.universal_html](https://pub.dev/packages/universal_html))
   2. For research purpose, lua_htmlparser include converter for unsupported CSS3 query syntax
+
 2. HTML parser libs maybe not supports attribute selectors: `*=`, `~=`, `|=`, `^=`, `$=`
 3. Several libs not support pseudo classes (eg: standard [dart.html](https://dart.dev/libraries/dart-html) lib miss this feature).
 
@@ -245,7 +246,6 @@ You can use any html source:
 
 ## See also
 
-
 - [Brief](docs/briefing.md) about css selectors and regular expressions
 - [Explain](docs/explain.md) short document on how to understand DSL syntax
 - [LLM](llm.md) experimental prompt for generate code

{ssc_codegen-0.13.4 → ssc_codegen-0.15.0}/README.md

@@ -23,15 +23,15 @@ ssc-gen - a python-based DSL to describe parsers for html documents, which is tr
 
 Current support converters
 
-| Language                                 | HTML parser lib + dependencies         | XPath | CSS3 | CSS4 | Generated annotations, types, structs | formatter dependency |
-| ---------------------------------------- | -------------------------------------- | ----- | ---- | ---- | ------------------------------------- | -------------------- |
-| Python (3.8+)                            | bs4, lxml                              | N     | Y    | Y    | TypedDict`1`, list, dict              | ruff                 |
-| ...                                      | parsel                                 | Y     | Y    | N    | ...                                   | ...                  |
-| ...                                      | selectolax (lexbor)                    | N     | Y    | N    | ...                                   | ...                  |
-| ...                                      | lxml                                   | Y     | Y    | N    | ...                                   | ...                  |
-| js (ES6)`2`                              | pure (firefox/chrome extension/nodejs) | Y     | Y    | Y    | JSDoc                                 | prettier             |
-| go (1.10+) **(UNSTABLE)**                | goquery, gjson (`4`)                   | N     | Y    | N    | struct(+json anchors), array, map     | gofmt                |
-| lua (5.2+), luajit(2+) **(UNSTABLE)**`5` | lua-htmlparser, lrexlib(opt), dkjson   | N     | Y    | N    | EmmyLua                               | LuaFormatter         |
+| Language                                 | HTML parser lib + dependencies                         | XPath | CSS3 | CSS4 | Generated annotations, types, structs | formatter dependency |
+| ---------------------------------------- | ------------------------------------------------------ | ----- | ---- | ---- | ------------------------------------- | -------------------- |
+| Python (3.8-3.13)                        | bs4, lxml ( typing_extensions if py < 3.10 )           | N     | Y    | Y    | TypedDict`1`, list, dict              | ruff                 |
+| ...                                      | parsel ( typing_extensions if py < 3.10 )              | Y     | Y    | N    | ...                                   | ...                  |
+| ...                                      | selectolax (lexbor) ( typing_extensions if py < 3.10 ) | N     | Y    | N    | ...                                   | ...                  |
+| ...                                      | lxml ( typing_extensions if py < 3.10 )                | Y     | Y    | N    | ...                                   | ...                  |
+| js (ES6)`2`                              | pure (firefox/chrome extension/nodejs)                 | Y     | Y    | Y    | JSDoc                                 | prettier             |
+| go (1.10+) **(UNSTABLE)**                | goquery, gjson (`4`)                                   | N     | Y    | N    | struct(+json anchors), array, map     | gofmt                |
+| lua (5.2+), luajit(2+) **(UNSTABLE)**`5` | lua-htmlparser, lrexlib(opt), dkjson                   | N     | Y    | N    | EmmyLua                               | LuaFormatter         |
 
 - **CSS3** means support next selectors:
   - basic: (`tag`, `.class`, `#id`, `tag1,tag2`)
@@ -64,6 +64,7 @@ For maximum portability of the configuration to the target language:
   Check the html parser lib documentation aboud CSS selectors before implement code. Examples:
   1. Several libs not support `+` operations (eg: [selectolax(modest)](https://github.com/rushter/selectolax), [dart.universal_html](https://pub.dev/packages/universal_html))
   2. For research purpose, lua_htmlparser include converter for unsupported CSS3 query syntax
+
 2. HTML parser libs maybe not supports attribute selectors: `*=`, `~=`, `|=`, `^=`, `$=`
 3. Several libs not support pseudo classes (eg: standard [dart.html](https://dart.dev/libraries/dart-html) lib miss this feature).
 
@@ -210,7 +211,6 @@ You can use any html source:
 
 ## See also
 
-
 - [Brief](docs/briefing.md) about css selectors and regular expressions
 - [Explain](docs/explain.md) short document on how to understand DSL syntax
 - [LLM](llm.md) experimental prompt for generate code

{ssc_codegen-0.13.4 → ssc_codegen-0.15.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.13.4"
+version = "0.15.0"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.13.4 → ssc_codegen-0.15.0}/ssc_codegen/__init__.py

@@ -1,13 +1,13 @@
 from ssc_codegen.document import DocumentElementsFilter, HTMLDocument, ClassVarDocument, StringDocument, ArrayDocument, AssertDocument, NestedDocument, \
     DefaultDocument, \
-    NumericDocument, JsonDocument, BooleanDocument, DocumentFilter
+    NumericDocument, JsonDocument, BooleanDocument, DocumentFilter, TransformDocument
 from ssc_codegen.json_struct import Json
 from ssc_codegen.logs import setup_logger
 from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchema, AccUniqueListSchema
 
 setup_logger()
 
-VERSION = "0.13.4"
+VERSION = "0.15.0"
 
 
 class __MISSING(object):
@@ -26,7 +26,8 @@ class Document(
     DefaultDocument,
     NumericDocument,
     JsonDocument,
-    BooleanDocument
+    BooleanDocument,
+    TransformDocument
 ):
     """Special Common Document or Element marker manipulations"""
     pass

{ssc_codegen-0.13.4 → ssc_codegen-0.15.0}/ssc_codegen/ast_/__init__.py

@@ -38,7 +38,10 @@ from .nodes_core import (
     ExprClassVar,
     ExprCallStructClassVar,
     CodeStart,
-    CodeEnd
+    CodeEnd,
+    ExprTransform,
+    ModuleTransformImports,
+    ModuleUtilities
 )
 from .nodes_filter import (
     FilterOr,

{ssc_codegen-0.13.4 → ssc_codegen-0.15.0}/ssc_codegen/ast_/nodes_core.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass, field
-from typing import Sequence, TypedDict, ClassVar, TypeVar
+from typing import Sequence, TypedDict, ClassVar, TypeVar, TYPE_CHECKING
 
 from ssc_codegen.ast_.base import T_EMPTY_KWARGS, BaseAstNode
 from ssc_codegen.ast_.base import (
@@ -14,6 +14,9 @@ from ssc_codegen.tokens import (
     VariableType,
 )
 
+if TYPE_CHECKING:
+    from ssc_codegen.transform import BaseTransform
+
 KW_DOCSTRING = TypedDict("KW_DOCSTRING", {"value": str})
 
 
@@ -34,11 +37,32 @@ class Docstring(_DisableRepr, BaseAstNode[KW_DOCSTRING, tuple[str]]):
     """
 
     kind: ClassVar[TokenType] = TokenType.DOCSTRING
-    kwargs: KW_DOCSTRING
+
+
+# push transform for resolve imports (dependencies)
+KW_MODULE_IMPORTS = TypedDict(
+    "KW_MODULE_IMPORTS", {"transforms": list["BaseTransform"]}
+)
+ARGS_MODULE_IMPORTS = tuple[list["BaseTransform"]]
+
+
+@dataclass(kw_only=True)
+class ModuleTransformImports(
+    BaseAstNode[KW_MODULE_IMPORTS, ARGS_MODULE_IMPORTS]
+):
+    """AST node representing module import transfrom dependencies.
+
+    This node contains import statements that will be added to the
+    generated module to ensure required dependencies are available.
+    """
+
+    kind: ClassVar[TokenType] = TokenType.TRANSFORM_IMPORTS
 
 
 @dataclass(kw_only=True)
-class ModuleImports(_DisableRepr, BaseAstNode[T_EMPTY_KWARGS, tuple]):
+class ModuleImports(
+    _DisableRepr, BaseAstNode[KW_MODULE_IMPORTS, ARGS_MODULE_IMPORTS]
+):
     """AST node representing module import statements.
 
     This node contains import statements that will be added to the
@@ -551,3 +575,17 @@ class CodeEnd(_DisableRepr, BaseAstNode):
     """
 
     kind: ClassVar[TokenType] = TokenType.CODE_END
+
+
+KW_TRANSFORM = TypedDict("KW_TRANSFORM", {"transform": "BaseTransform"})
+ARGS_TRANSFORM = tuple["BaseTransform"]
+
+
+@dataclass(kw_only=True)
+class ExprTransform(BaseAstNode[KW_TRANSFORM, ARGS_TRANSFORM]):
+    kind: ClassVar[TokenType] = TokenType.TRANSFORM
+
+
+@dataclass(kw_only=True)
+class ModuleUtilities(_DisableRepr, BaseAstNode):
+    kind: ClassVar[TokenType] = TokenType.UTILITIES

{ssc_codegen-0.13.4 → ssc_codegen-0.15.0}/ssc_codegen/ast_build/builder.py

@@ -3,6 +3,7 @@ from typing import MutableSequence, Type, cast
 import warnings
 from typing_extensions import Self
 
+from ssc_codegen.ast_ import ModuleTransformImports, ModuleUtilities
 from ssc_codegen.ast_.base import BaseAstNode
 from ssc_codegen.ast_.nodes_cast import ExprJsonify, ExprNested
 from ssc_codegen.ast_.nodes_core import (
@@ -113,7 +114,16 @@ class AstBuilder:
                 Docstring(kwargs={"value": module_doc}, parent=self.module)
             )
         # import node
-        self.module.body.append(ModuleImports(parent=self.module))
+        # fallback add ModuleImports.kwargs["transform"] if target language not allowed use multiple import directives
+        self.module.body.append(
+            ModuleImports(parent=self.module, kwargs={"transforms": []})
+        )
+        self.module.body.append(
+            ModuleTransformImports(
+                parent=self.module, kwargs={"transforms": []}
+            )
+        )
+        self.module.body.append(ModuleUtilities(parent=self.module))
         # CodeStart hook node
         self.module.body.append(CodeStart(parent=self.module))
         return self
@@ -398,10 +408,23 @@ class AstBuilder:
         # generate typedef as annotation node
         return st
 
+    @staticmethod
+    def is_collected_type_transform(transforms: list, target: object) -> bool:
+        for t in transforms:
+            if isinstance(t, type(target)):
+                return True
+        return False
+
     def add_struct_parsers(self, *struct_parsers: Type[BaseSchema]) -> Self:
         ssc_structs = []
         ssc_typedefs = []
+        ssc_transforms = []
         for schema in struct_parsers:
+            if schema.__SSC_TRANSFORMS__:
+                # check current used transforms for avoid generate duplicate imports
+                for t in schema.__SSC_TRANSFORMS__:
+                    if not any(isinstance(i, type(t)) for i in ssc_transforms):
+                        ssc_transforms.append(t)
             if is_literals_only_schema(schema):
                 if schema.__SCHEMA_TYPE__ != StructType.ITEM:
                     raise TypeError(
@@ -420,6 +443,21 @@ class AstBuilder:
                 continue
             ssc_typedefs.append(typedef)
         # assembly
+
+        # insert transforms imports (in high-level API provide insert dependencies)
+
+        node_transform_import = self.module.find_node_by_token(
+            ModuleTransformImports.kind
+        )
+        node_transform_import = cast(
+            ModuleTransformImports, node_transform_import
+        )
+        node_transform_import.kwargs["transforms"] = ssc_transforms.copy()
+
+        node_import = self.module.find_node_by_token(ModuleImports.kind)
+        node_import = cast(ModuleImports, node_import)
+        node_import.kwargs["transform"] = ssc_transforms.copy()
+
         self.module.body.extend(ssc_typedefs)
         self.module.body.extend(ssc_structs)
         self.module.body.append(CodeEnd(parent=self.module))

{ssc_codegen-0.13.4 → ssc_codegen-0.15.0}/ssc_codegen/cli/code_callbacks.py

@@ -3,7 +3,6 @@
 from typing import Callable
 from ssc_codegen.str_utils import (
     go_unimport_naive,
-    py_remove_unused_none_type_import,
 )
 
 
@@ -28,7 +27,6 @@ class BaseCodeCallback:
         return code
 
 
-CB_PY_CODE = BaseCodeCallback(py_remove_unused_none_type_import)
 CB_GO_CODE = BaseCodeCallback(go_unimport_naive, remove_empty_lines=True)
 CB_DART_CODE = BaseCodeCallback(remove_empty_lines=True)
 CB_JS_CODE = BaseCodeCallback(remove_empty_lines=True)

{ssc_codegen-0.13.4 → ssc_codegen-0.15.0}/ssc_codegen/cli/main.py

@@ -25,7 +25,6 @@ from ssc_codegen.cli.cli_utils import (
     suggest_class_name,
 )
 from ssc_codegen.cli.code_callbacks import (
-    CB_PY_CODE,
     CB_GO_CODE,
     CB_JS_CODE,
 )
@@ -208,7 +207,6 @@ def gen_py(
         xpath_to_css=to_css,
         css_to_xpath=to_xpath,
         debug_instructions=debug,
-        code_cb=CB_PY_CODE,
         gen_docstring=docstring,
     )
 

{ssc_codegen-0.13.4 → ssc_codegen-0.15.0}/ssc_codegen/converters/go_goquery.py

@@ -51,6 +51,9 @@ from typing_extensions import assert_never
 
 from ssc_codegen.ast_ import (
     Docstring,
+    ExprTransform,
+    ModuleTransformImports,
+    ModuleUtilities,
     StructParser,
     ExprReturn,
     ExprNoReturn,
@@ -393,21 +396,37 @@ def pre_docstring(node: Docstring) -> str:
 
 @CONVERTER(ModuleImports.kind)
 def pre_module_imports(_node: ModuleImports) -> str:
+    # TODO: resolve imports
+    return IMPORTS.replace("$PACKAGE$", CONVERTER.PACKAGE)
+
+
+@CONVERTER(ModuleTransformImports.kind)
+def pre_transform_imports(node: ModuleTransformImports) -> str:
+    transforms, *_ = node.unpack_args()
+    if transforms:
+        for t in transforms:
+            if t.collect_dependencies("go_goquery"):
+                raise NotImplementedError(
+                    "Current converter implementation not support pass extra import dependencies"
+                )
+    return ""
+
+
+@CONVERTER(ExprTransform.kind)
+def pre_transform(node: ExprTransform) -> str:
+    prv, nxt = prev_next_var(node)
+    transform, *_ = node.unpack_args()
+    parts = transform.emit("go_goquery", prv, nxt)
+    return "\n" + "\n".join(parts)
+
+
+@CONVERTER(ModuleUtilities.kind)
+def pre_module_utilities(_: ModuleUtilities) -> str:
     # HACK:
-    # golang not allowed override functions, check loaded helper functions by flag
-    helper_functions = (
-        HELPER_FUNCTIONS if not CONVERTER.HELPER_FUNCS_IMPORTED else ""
-    )
-    code = (
-        # magic constant
-        IMPORTS.replace("$PACKAGE$", CONVERTER.PACKAGE)
-        # HACK: push helper functions to ModuleImports token:
-        #       its generated second after module docstring
-        + helper_functions
-    )
+    # golang not allowed override or duplicate functions, check loaded helper functions by flag
     if not CONVERTER.HELPER_FUNCS_IMPORTED:
         CONVERTER.HELPER_FUNCS_IMPORTED = True
-    return code
+    return HELPER_FUNCTIONS if not CONVERTER.HELPER_FUNCS_IMPORTED else ""
 
 
 def get_typedef_field_by_name(node: TypeDef, field_name: str) -> str:

{ssc_codegen-0.13.4 → ssc_codegen-0.15.0}/ssc_codegen/converters/js_pure.py

@@ -23,6 +23,9 @@ from typing import cast
 
 from ssc_codegen.ast_ import (
     Docstring,
+    ExprTransform,
+    ModuleTransformImports,
+    ModuleUtilities,
     StructParser,
     ExprReturn,
     ExprNoReturn,
@@ -173,7 +176,11 @@ DOCSTR_START = "/**"
 DOCSTR_END = "*/"
 DOCSTR_SEP = "* "
 CONVERTER = BaseCodeConverter(debug_comment_prefix="// ")
-# javascript not support typing/annotations
+CONVERTER.TEST_EXCLUDE_NODES.extend(
+    [
+        ModuleImports.kind,  # not use imports
+    ]
+)
 
 
 # TODO: move to string_utils
@@ -213,8 +220,30 @@ def py_sequence_to_js_array(values: tuple[str, ...] | list[str]) -> str:
     return "[" + val_arr[1:-1] + "]"
 
 
-@CONVERTER(ModuleImports.kind)
-def pre_imports(_: ModuleImports) -> str:
+@CONVERTER(ModuleTransformImports.kind)
+def pre_transform_imports(node: ModuleTransformImports) -> str:
+    transforms, *_ = node.unpack_args()
+
+    if not transforms:
+        return ""
+
+    code_imports = []
+    for t in transforms:
+        if deps := t.collect_dependencies("js_pure"):
+            code_imports.append("\n".join(deps))
+    return "\n".join(code_imports)
+
+
+@CONVERTER(ExprTransform.kind)
+def pre_transform(node: ExprTransform) -> str:
+    # extend transform search provide py_bs4 module
+    prv, nxt = prev_next_var(node)
+    transform, *_ = node.unpack_args()
+    return "\n".join(f"{i};" for i in transform.emit("js_pure", prv, nxt))
+
+
+@CONVERTER(ModuleUtilities.kind)
+def pre_imports(_: ModuleUtilities) -> str:
     return HELPER_FUNCTIONS
 
 

{ssc_codegen-0.13.4 → ssc_codegen-0.15.0}/ssc_codegen/converters/lua_htmlparser.py

@@ -28,6 +28,9 @@ from typing import Any, cast
 
 from ssc_codegen.ast_ import (
     Docstring,
+    ExprTransform,
+    ModuleTransformImports,
+    ModuleUtilities,
     StructParser,
     ExprReturn,
     ExprNoReturn,
@@ -187,6 +190,7 @@ CONVERTER.TEST_EXCLUDE_NODES.extend(
         FilterDocHasText.kind,
         FilterDocIsRegexText.kind,
         FilterDocIsRegexRaw.kind,
+        # NOT IMPLEMENTED
     ]
 )
 
@@ -291,7 +295,34 @@ def py_regex_to_lua_rex_pcre(
 
 @CONVERTER(ModuleImports.kind)
 def pre_imports(_: ModuleImports) -> str:
-    return IMPORTS + "\n" + HELPER_FUNCTIONS
+    return IMPORTS
+
+
+@CONVERTER(ModuleTransformImports.kind)
+def pre_transform_imports(node: ModuleTransformImports) -> str:
+    transforms, *_ = node.unpack_args()
+
+    if not transforms:
+        return ""
+
+    code_imports = []
+    for t in transforms:
+        if deps := t.collect_dependencies("lua_htmlparser"):
+            code_imports.append("\n".join(deps))
+    return "\n".join(code_imports)
+
+
+@CONVERTER(ModuleUtilities.kind)
+def pre_utilities(_: ModuleUtilities) -> str:
+    return HELPER_FUNCTIONS
+
+
+@CONVERTER(ExprTransform.kind)
+def pre_transform(node: ExprTransform) -> str:
+    prv, nxt = prev_next_var(node)
+    transform, *_ = node.unpack_args()
+    parts = transform.emit("lua_htmlparser", prv, nxt)
+    return "\n" + "\n".join(parts)
 
 
 @CONVERTER(Docstring.kind)