ssc_codegen 0.8.12__tar.gz → 0.8.14__tar.gz

{ssc_codegen-0.8.12 → ssc_codegen-0.8.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.8.12
+Version: 0.8.14
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues

{ssc_codegen-0.8.12 → ssc_codegen-0.8.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.8.12"
+version = "0.8.14"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.8.12 → ssc_codegen-0.8.14}/ssc_codegen/__init__.py

@@ -7,10 +7,11 @@ from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchem
 
 setup_logger()
 
-VERSION = "0.8.12"
+VERSION = "0.8.14"
 
 
 class __MISSING(object):
+    """special marker for mark is not passed default value"""
     pass
 
 
@@ -34,7 +35,7 @@ class Nested(HTMLDocument, NestedDocument, ArrayDocument, AssertDocument):
     pass
 
 
-def D(default: None | str | int | float | __MISSING = _NO_DEFAULT) -> Document:  # noqa
+def D(default: None | str | int | float | list | __MISSING = _NO_DEFAULT) -> Document:  # noqa
     """Shortcut as a Document() object
 
     :param default: .default() operator shortcut
@@ -49,7 +50,7 @@ def N() -> Nested:  # noqa
     return Nested()
 
 
-def R(default: None | str | int | float | __MISSING = _NO_DEFAULT) -> Document:  # noqa
+def R(default: None | str | int | float | list | __MISSING = _NO_DEFAULT) -> Document:  # noqa
     """Shortcut as a Document().raw() object.
     For regex and format string operations
 

{ssc_codegen-0.8.12 → ssc_codegen-0.8.14}/ssc_codegen/ast_/nodes_core.py

@@ -194,9 +194,9 @@ class ModuleProgram(
 
 
 KW_AST_DEFAULT = TypedDict(
-    "KW_AST_DEFAULT", {"value": str | int | float | bool | None}
+    "KW_AST_DEFAULT", {"value": str | int | float | bool | list | None}
 )
-ARGS_AST_DEFAULT = tuple[str | int | float | bool | None]
+ARGS_AST_DEFAULT = tuple[str | int | float | bool | list | None]
 
 
 @dataclass(kw_only=True)

{ssc_codegen-0.8.12 → ssc_codegen-0.8.14}/ssc_codegen/ast_build/main.py

@@ -1,6 +1,6 @@
+import warnings
 from pathlib import Path
 from typing import Type, cast
-from typing_extensions import assert_never
 
 from ssc_codegen import Json
 from ssc_codegen.ast_ import (
@@ -45,7 +45,6 @@ from ssc_codegen.tokens import TokenType, VariableType, StructType
 def build_ast_module_parser(
     path: str | Path,
     *,
-    docstring_class_top: bool = False,
     css_to_xpath: bool = False,
     xpath_to_css: bool = False,
 ) -> ModuleProgram:
@@ -78,15 +77,32 @@ def build_ast_module_parser(
     json_structs = extract_json_structs_from_module(py_module)
     # TODO: API for build single json
     ast_jsons = build_ast_json(ast_module, *json_structs)
-    ast_schemas = [
-        build_ast_struct_parser(
-            sc,
-            ast_module,
-            css_to_xpath=css_to_xpath,
-            xpath_to_css=xpath_to_css,
-        )
-        for sc in schemas
-    ]
+    # TODO: implement analyze-only AST function
+    try:
+        ast_schemas = [
+            build_ast_struct_parser(
+                sc,
+                ast_module,
+                css_to_xpath=css_to_xpath,
+                xpath_to_css=xpath_to_css,
+            )
+            for sc in schemas
+        ]
+    except SyntaxError:
+        # get all schemas exceptions and throw to logs:
+        errors = []
+        for sc in schemas:
+            try:
+                build_ast_struct_parser(
+                    sc,
+                    ast_module,
+                    css_to_xpath=css_to_xpath,
+                    xpath_to_css=xpath_to_css,
+                )
+            except SyntaxError as e:
+                errors.append(e)
+        raise SyntaxError(f"{path}: Founded errors: {len(errors)}")
+
     # TODO: API for build single typedef
     ast_typedefs = build_ast_typedef(ast_module, *ast_schemas)
     module_body.extend(ast_jsons)
@@ -174,7 +190,6 @@ def build_ast_struct_parser(
 ) -> StructParser:
     errors_count = run_analyze_schema(schema)
     if errors_count > 0:
-        # todo: exc
         msg = f"{schema.__name__} founded errors: {errors_count}"
         raise SyntaxError(msg)
     docstring = (
@@ -238,8 +253,7 @@ def _fetch_field_nodes(
         )
         # TODO: add ast tests
         # in inheritance schemas, child classes use same fields are used as in the parent class
-        # avoid duplicate ExprReturn node
-        # second case used in ExprDefaultValueWrapper cases
+        # avoid duplicate ExprReturn or ExprDefaultValueWrapper node
         if (
             document.stack[-1].kind != ExprReturn.kind
             and document.stack[-1].kind != TokenType.EXPR_DEFAULT_END
@@ -296,7 +310,27 @@ def _unwrap_default_node(
                     default_type = VariableType.OPTIONAL_LIST_FLOAT
                 # TODO: warning for BOOL type
                 case _:
-                    assert_never(ret_type)  # type: ignore
+                    warnings.warn(
+                        f"'None' default value not allowed return type '{ret_type.name}'. ",
+                        category=SyntaxWarning,
+                    )
+                    default_type = VariableType.ANY
+        elif isinstance(value, list):
+            # todo: check if empty list passed
+            match ret_type:
+                case VariableType.LIST_STRING:
+                    default_type = VariableType.LIST_STRING
+                case VariableType.LIST_INT:
+                    default_type = VariableType.LIST_INT
+                case VariableType.LIST_FLOAT:
+                    default_type = VariableType.LIST_FLOAT
+                case _:
+                    warnings.warn(
+                        f"`empty list` default value not allowed return type `{ret_type.name}`. "
+                        f"Expected types `{(VariableType.LIST_STRING.name, VariableType.LIST_INT, VariableType.LIST_FLOAT)}`",
+                        category=SyntaxWarning,
+                    )
+                    default_type = VariableType.ANY
         expr_default_start = ExprDefaultValueStart(kwargs={"value": value})
         expr_default_end = ExprDefaultValueEnd(
             kwargs={"value": value}, ret_type=default_type
@@ -316,7 +350,7 @@ def _try_fetch_split_doc_node(
         method = StructPartDocMethod(parent=st_ref)
 
         # in inheritance schemas, child classes use same fields are used as in the parent class
-        # second case used in ExprDefaultValueWrapper cases
+        # avoid duplicate ExprReturn or ExprDefaultValueWrapper node
         if (
             split_doc.stack[-1].kind != ExprReturn.kind
             and split_doc.stack[-1].kind != TokenType.EXPR_DEFAULT_END

{ssc_codegen-0.8.12 → ssc_codegen-0.8.14}/ssc_codegen/cli/main.py

@@ -80,7 +80,6 @@ def generate_code(
     comment_str: str,
     fmt_cmd: list[str],
     code_cb: Callable[[list[str]], str] = lambda c: "\n".join(c),
-    docstring_class_top: bool = False,
     variables_patches: dict[str, str] | None = None,
     css_to_xpath: bool = False,
     xpath_to_css: bool = False,
@@ -112,7 +111,6 @@ def generate_code(
         LOGGER.info("Make AST %s...", file_cfg.name)
         ast_module = build_ast_module_parser(
             file_cfg,
-            docstring_class_top=docstring_class_top,
             css_to_xpath=css_to_xpath,
             xpath_to_css=xpath_to_css,
         )
@@ -220,7 +218,6 @@ def gen_js(
         suffix=suffix,
         comment_str=f"// {COMMENT_STRING}",
         fmt_cmd=fmt_cmd,
-        docstring_class_top=True,
         xpath_to_css=to_css,
         css_to_xpath=to_xpath,
         debug_instructions=debug,
@@ -272,7 +269,6 @@ def gen_go(
         fmt_cmd=fmt_cmd,
         # todo: better API for code callbacks
         code_cb=CB_GO_CODE,
-        docstring_class_top=True,
         variables_patches={"PACKAGE": package or out.name},
         xpath_to_css=to_css,
         css_to_xpath=to_xpath,
@@ -478,7 +474,7 @@ def parse_from_chrome_(
     if not url.startswith("http"):
         raise typer.BadParameter(f"`{url}` not a http url")
 
-    ast = build_ast_module_parser(schema_config, docstring_class_top=True)  # type: ignore
+    ast = build_ast_module_parser(schema_config)  # type: ignore
     code_parts = JS_CONVERTER.convert_program(ast)
     code = CB_JS_CODE(code_parts)
     code += f"; JSON.stringify((new {cls_target}(document).parse()))"

{ssc_codegen-0.8.12 → ssc_codegen-0.8.14}/ssc_codegen/converters/go_goquery.py

@@ -244,13 +244,20 @@ DOCSTR = "// "
 CONVERTER = BaseCodeConverter(debug_comment_prefix="// ")
 
 
-def py_var_to_go_var(item: None | str | int | float) -> str | int | float:
+def py_var_to_go_var(
+    item: None | str | int | float | list, ret_type: VariableType
+) -> str | int | float:
     if item is None:
         item = "nil"
     elif isinstance(item, str):
         item = wrap_double_quotes(item)
     elif isinstance(item, bool):
         item = "true" if item else "false"
+    elif isinstance(item, list):
+        # in AST static check step check if return type is
+        # LIST_STRING, LIST_INT, LIST_FLOAT
+        type_ = TYPES[ret_type]
+        item = f"make({type_}, 0)"
     return item
 
 
@@ -576,7 +583,8 @@ def post_start_parse_method(node: StartParseMethod) -> str:
 def pre_default_start(node: ExprDefaultValueStart) -> str:
     prv, nxt = prev_next_var(node)
     value = node.kwargs["value"]
-    value = py_var_to_go_var(value)
+    ret_type = get_last_ret_type(node)
+    value = py_var_to_go_var(value, ret_type)
     return J2_PRE_DEFAULT_START.render(prv=prv, nxt=nxt, value=value)
 
 
@@ -797,7 +805,8 @@ def pre_is_equal(node: ExprIsEqual) -> str:
     item, msg = node.unpack_args()
 
     msg = wrap_double_quotes(msg)
-    item = py_var_to_go_var(item)
+    ret_type = get_last_ret_type(node)
+    item = py_var_to_go_var(item, ret_type)
 
     context = {
         "prv": prv,
@@ -819,7 +828,8 @@ def pre_is_not_equal(node: ExprIsNotEqual) -> str:
     item, msg = node.unpack_args()
 
     msg = wrap_double_quotes(msg) if msg else '""'
-    item = py_var_to_go_var(item)
+    ret_type = get_last_ret_type(node)
+    item = py_var_to_go_var(item, ret_type)
 
     context = {
         "prv": prv,
@@ -841,7 +851,8 @@ def pre_is_contains(node: ExprIsContains) -> str:
     item, msg = node.unpack_args()
 
     msg = wrap_double_quotes(msg) if msg else '""'
-    item = py_var_to_go_var(item)
+    ret_type = get_last_ret_type(node)
+    item = py_var_to_go_var(item, ret_type)
 
     context = {
         "prv": prv,

{ssc_codegen-0.8.12 → ssc_codegen-0.8.14}/ssc_codegen/converters/js_pure.py

@@ -265,6 +265,8 @@ def pre_default_end(node: ExprDefaultValueEnd) -> str:
         value = repr(value)
     elif isinstance(value, bool):
         value = "true" if value else "false"
+    elif isinstance(value, list):
+        value = "[]"
     return f"}}catch(Error) {{ return {value}; }}"
 
 

{ssc_codegen-0.8.12 → ssc_codegen-0.8.14}/ssc_codegen/converters/py_base.py

@@ -483,6 +483,8 @@ def pre_default_end(node: ExprDefaultValueEnd) -> str:
     value = node.kwargs["value"]
     if isinstance(value, str):
         value = repr(value)
+    elif isinstance(value, list):
+        value = "[]"
     return INDENT_METHOD_BODY + f"return {value}"
 
 

{ssc_codegen-0.8.12 → ssc_codegen-0.8.14}/ssc_codegen/document.py

@@ -115,7 +115,7 @@ class BaseDocument:
 
 
 class DefaultDocument(BaseDocument):
-    def default(self, value: str | int | float | None) -> Self:
+    def default(self, value: str | int | float | list | None) -> Self:
         """Set default value. Accept string, int, float or None.
         Should be a first else raise SyntaxError
 

{ssc_codegen-0.8.12 → ssc_codegen-0.8.14}/ssc_codegen/document_utlis.py

@@ -31,6 +31,8 @@ CM1_RX = r"(?m)(?<!\\)((\\{2})*)#.*$"
 CM2_RX = r"(\\)?((\\{2})*)(#)"
 WS_RX = r"(\\)?((\\{2})*)(\s)\s*"
 
+RE_CAPTURED_GROUPS = re.compile(r"(?<!\()\((?!\?:)[^)]+\)")
+
 
 def is_ignore_case_regex(pattern: str | Pattern) -> bool:
     if isinstance(pattern, str):
@@ -76,10 +78,14 @@ def analyze_re_expression(
     try:
         re_pattern = re.compile(pattern)
         if not allow_empty_groups and re_pattern.groups == 0:
-            msg = f"`{re_pattern.pattern}` pattern groups is empty"
+            msg = f"`{re_pattern.pattern}` pattern groups is empty."
+            msg += "\nTIP: maybe you remember wrap pattern to brackets `()`?"
             return AnalyzeResult.error(msg)
         elif max_groups != -1 and re_pattern.groups > max_groups:
-            msg = f"`{re_pattern.pattern}` too many groups in pattern, expected {max_groups}"
+            captured_groups = RE_CAPTURED_GROUPS.findall(re_pattern.pattern)
+            msg = f"`{re_pattern.pattern}` too many groups in pattern, expected groups count: {max_groups}."
+            msg += f"\nTIP: fix regular expression for extract {max_groups}:"
+            msg += f"\nGroups founded: {captured_groups}"
             return AnalyzeResult.error(msg)
     except re.error as e:
         msg = f"`{pattern}` wrong regex pattern syntax: {e!r}"

{ssc_codegen-0.8.12 → ssc_codegen-0.8.14}/ssc_codegen/static_checker/callbacks.py

@@ -1,3 +1,4 @@
+import warnings
 from typing import Callable, Type, TYPE_CHECKING
 
 from cssselect import SelectorSyntaxError
@@ -171,7 +172,7 @@ def analyze_field_default_value(
 
     elif (index := default_expr_pos[0][0]) and index != 0:
         return AnalyzeResult.error(
-            f"{sc.__name__}.{name}  # default expr should be a first, not {index}"
+            f"{sc.__name__}.{name}  # default expr should be a first, not {index} position"
         )
 
     elif name in ("__PRE_VALIDATE__", "__SPLIT_DOC__"):
@@ -203,12 +204,30 @@ def analyze_field_default_value(
         default_ast_type = VariableType.INT
     elif isinstance(default_value, float):
         default_ast_type = VariableType.FLOAT
+    elif isinstance(default_value, list):
+        if len(default_value) != 0:
+            warnings.warn(
+                f"{sc.__name__}.{name} # not supported pass values to list",
+                category=SyntaxWarning,
+            )
+        default_ast_type = ret_type
     else:
         return AnalyzeResult.error(
             f"{sc.__name__}.{name} # Unsupported default value: `{default_value!r}`<{type(default_value).__name__}>"
         )
 
-    if default_ast_type != ret_type:
+    if isinstance(default_value, list) and default_ast_type not in (
+        VariableType.LIST_STRING,
+        VariableType.LIST_INT,
+        VariableType.LIST_FLOAT,
+    ):
+        return AnalyzeResult.error(
+            f"{sc.__name__}.{name} # default({default_value!r}) wrong last list return type expr"
+            f" (expected type(s) "
+            f"`{(VariableType.LIST_STRING.name, VariableType.LIST_FLOAT.name, VariableType.LIST_INT.name)}` "
+            f"got `{ret_type.name}`)"
+        )
+    elif default_ast_type != ret_type:
         return AnalyzeResult.error(
             f"{sc.__name__}.{name} # default({default_value!r}) wrong last return type expr"
             f" (expected type `{default_ast_type.name}` got `{ret_type.name}`)"