ssc_codegen 0.8.0__tar.gz → 0.8.1__tar.gz

{ssc_codegen-0.8.0 → ssc_codegen-0.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.8.0
+Version: 0.8.1
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues

{ssc_codegen-0.8.0 → ssc_codegen-0.8.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.8.0"
+version = "0.8.1"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.8.0 → ssc_codegen-0.8.1}/ssc_codegen/__init__.py

@@ -7,7 +7,7 @@ from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchem
 
 setup_logger()
 
-VERSION = "0.8.0"
+VERSION = "0.8.1"
 
 
 class __MISSING(object):

{ssc_codegen-0.8.0 → ssc_codegen-0.8.1}/ssc_codegen/ast_build/main.py

@@ -236,9 +236,18 @@ def _fetch_field_nodes(
             parent=st_ref,
             ret_type=ret_type,
         )
-        document.stack.append(
-            ExprReturn(accept_type=ret_type, ret_type=ret_type)
-        )
+        # TODO: add ast tests
+        # in inheritance schemas, child classes use same fields are used as in the parent class
+        # avoid duplicate ExprReturn node
+        # second case used in ExprDefaultValueWrapper cases
+        if (
+            document.stack[-1].kind != ExprReturn.kind
+            and document.stack[-1].kind != TokenType.EXPR_DEFAULT_END
+        ):
+            document.stack.append(
+                ExprReturn(accept_type=ret_type, ret_type=ret_type)
+            )
+
         _unwrap_default_node(document, ret_type)
         for i in document.stack:
             i.parent = method
@@ -255,7 +264,6 @@ def _fetch_field_nodes(
             cls_name = None
         body_parse_method_expr.append(
             ExprCallStructMethod(
-                # parent=method,
                 kwargs={
                     "name": field_name,
                     "type": ret_type,
@@ -298,9 +306,18 @@ def _try_fetch_split_doc_node(
 ) -> None:
     if fields.get("__SPLIT_DOC__"):
         split_doc = fields.pop("__SPLIT_DOC__")
-        # always sequence of elements
         method = StructPartDocMethod(parent=st_ref)
-        split_doc.stack.append(ExprReturn(ret_type=VariableType.LIST_DOCUMENT))
+
+        # in inheritance schemas, child classes use same fields are used as in the parent class
+        # second case used in ExprDefaultValueWrapper cases
+        if (
+            split_doc.stack[-1].kind != ExprReturn.kind
+            and split_doc.stack[-1].kind != TokenType.EXPR_DEFAULT_END
+        ):
+            # always returns sequence of elements
+            split_doc.stack.append(
+                ExprReturn(ret_type=VariableType.LIST_DOCUMENT)
+            )
         for i in split_doc.stack:
             i.parent = method
         method.body = split_doc.stack

{ssc_codegen-0.8.0 → ssc_codegen-0.8.1}/ssc_codegen/cli/code_callbacks.py

@@ -4,8 +4,8 @@ from typing import Callable
 from ssc_codegen.str_utils import (
     go_unimport_naive,
     py_optimize_return_naive,
-    py_str_format_to_fstring,
     js_pure_optimize_return,
+    py_remove_unused_none_type_import,
 )
 
 
@@ -31,7 +31,7 @@ class BaseCodeCallback:
 
 
 CB_PY_CODE = BaseCodeCallback(
-    py_optimize_return_naive, py_str_format_to_fstring
+    py_optimize_return_naive, py_remove_unused_none_type_import
 )
 CB_GO_CODE = BaseCodeCallback(go_unimport_naive, remove_empty_lines=True)
 CB_DART_CODE = BaseCodeCallback(remove_empty_lines=True)

{ssc_codegen-0.8.0 → ssc_codegen-0.8.1}/ssc_codegen/converters/py_base.py

@@ -150,7 +150,6 @@ MAGIC_METHODS = {
 
 # used old style typing for support old python versions (3.8)
 IMPORTS_MIN = """
-from __future__ import annotations
 import re
 import sys
 import json
@@ -281,7 +280,7 @@ def get_field_method_ret_type(node: StructFieldMethod) -> str:
 def pre_docstring(node: Docstring) -> str:
     value, *_ = node.unpack_args()
     if value:
-        return '"""' + value + '""""'
+        return '"""' + value + '"""'
     return ""
 
 
@@ -368,7 +367,8 @@ def pre_struct_parser(node: StructParser) -> str:
 
 
 def pre_ret(node: ExprReturn) -> str:
-    expr = f"return v{node.index_prev}"
+    prv, _ = prev_next_var(node)
+    expr = f"return {prv}"
     if have_default_expr(node):
         return INDENT_DEFAULT_BODY + expr
     return INDENT_METHOD_BODY + expr
@@ -670,13 +670,12 @@ def pre_str_regex(node: ExprStringRegex) -> str:
     )
     prv, nxt = prev_next_var(node)
     pattern, group, ignore_case = node.unpack_args()
-    pattern = repr(pattern)
     if ignore_case:
         return (
             indent
-            + f"{nxt} = re.search({pattern}, {prv}, re.IGNORECASE)[{group}]"
+            + f"{nxt} = re.search(r{pattern!r}, {prv}, re.IGNORECASE)[{group}]"
         )
-    return indent + f"{nxt} = re.search({pattern}, {prv})[{group}]"
+    return indent + f"{nxt} = re.search(r{pattern!r}, {prv})[{group}]"
 
 
 def pre_str_regex_all(node: ExprStringRegexAll) -> str:
@@ -686,8 +685,10 @@ def pre_str_regex_all(node: ExprStringRegexAll) -> str:
     prv, nxt = prev_next_var(node)
     pattern, ignore_case = node.unpack_args()
     if ignore_case:
-        return indent + f"{nxt} = re.findall({pattern!r}, {prv}, re.IGNORECASE)"
-    return indent + f"{nxt} = re.findall({pattern!r}, {prv})"
+        return (
+            indent + f"{nxt} = re.findall(r{pattern!r}, {prv}, re.IGNORECASE)"
+        )
+    return indent + f"{nxt} = re.findall(r{pattern!r}, {prv})"
 
 
 def pre_str_regex_sub(node: ExprStringRegexSub) -> str:
@@ -696,7 +697,7 @@ def pre_str_regex_sub(node: ExprStringRegexSub) -> str:
     )
     prv, nxt = prev_next_var(node)
     pattern, repl = node.unpack_args()
-    return indent + f"{nxt} = re.sub({pattern!r}, {repl!r}, {prv})"
+    return indent + f"{nxt} = re.sub(r{pattern!r}, {repl!r}, {prv})"
 
 
 def pre_list_str_regex_sub(node: ExprListStringRegexSub) -> str:
@@ -705,7 +706,9 @@ def pre_list_str_regex_sub(node: ExprListStringRegexSub) -> str:
     )
     prv, nxt = prev_next_var(node)
     pattern, repl = node.unpack_args()
-    return indent + f"{nxt} = [re.sub({pattern!r}, {repl!r}, i) for i in {prv}]"
+    return (
+        indent + f"{nxt} = [re.sub(r{pattern!r}, {repl!r}, i) for i in {prv}]"
+    )
 
 
 def pre_index(node: ExprIndex) -> str:
@@ -777,7 +780,7 @@ def pre_is_regex(node: ExprIsRegex) -> str:
     )
     prv, nxt = prev_next_var(node)
     pattern, ignore_case, msg = node.unpack_args()
-    expr = indent + f"assert re.search({pattern!r}, {prv}), {msg!r}"
+    expr = indent + f"assert re.search(r{pattern!r}, {prv}), {msg!r}"
     if is_last_var_no_ret(node):
         return expr
     # HACK: avoid recalc variables

{ssc_codegen-0.8.0 → ssc_codegen-0.8.1}/ssc_codegen/converters/py_parsel.py

@@ -38,7 +38,7 @@ CONVERTER = BasePyCodeConverter()
 def pre_init(_node: StructInitMethod) -> str:
     return (
         INDENT_METHOD
-        + "def __init__(self, document: Union[str, _SelectorType]) -> None:\n"
+        + "def __init__(self, document: Union[str, Selector]) -> None:\n"
         + INDENT_METHOD_BODY
         + "self._document = Selector(document) if isinstance(document, str) else document"
     )
@@ -48,7 +48,7 @@ def pre_init(_node: StructInitMethod) -> str:
 def pre_struct_pre_validate(_node: StructPreValidateMethod) -> str:
     return (
         INDENT_METHOD
-        + "def _pre_validate(self, v: Union[Selector, _SelectorType]) -> None:"
+        + "def _pre_validate(self, v: Union[Selector, SelectorList]) -> None:"
     )
 
 
@@ -56,7 +56,7 @@ def pre_struct_pre_validate(_node: StructPreValidateMethod) -> str:
 def pre_struct_part_doc_method(_node: StructPartDocMethod) -> str:
     return (
         INDENT_METHOD
-        + "def _split_doc(self, v: Union[Selector, _SelectorType]) -> SelectorList:"
+        + "def _split_doc(self, v: Union[Selector, SelectorList]) -> SelectorList:"
     )
 
 
@@ -67,15 +67,13 @@ def pre_struct_field_method(node: StructFieldMethod) -> str:
     name = MAGIC_METHODS.get(name, name)
     return (
         INDENT_METHOD
-        + f"def _parse_{name}(self, v: Union[Selector, _SelectorType]) -> {type_}:"
+        + f"def _parse_{name}(self, v: Union[Selector, SelectorList]) -> {type_}:"
     )
 
 
 @CONVERTER.post(ModuleImports.kind)
 def post_imports(_: ModuleImports) -> str:
-    return """from parsel import Selector, SelectorList
-from parsel.selector import _SelectorType  # noqa
-"""
+    return """from parsel import Selector, SelectorList"""
 
 
 @CONVERTER(ExprCss.kind)

{ssc_codegen-0.8.0 → ssc_codegen-0.8.1}/ssc_codegen/schema.py

@@ -86,9 +86,8 @@ class BaseSchema:
             ].kwargs["schema_name"]
             signature[key] = cls.__NESTED_SCHEMAS__[name].__class_signature__()
         elif field.stack_last_ret == VariableType.JSON:
-            json_cls = cls.__JSON_SCHEMAS__[
-                field.stack[-1].kwargs["json_struct_name"]
-            ]
+            name = [e for e in field.stack if e.kind == TokenType.TO_JSON][0].kwargs["json_struct_name"]
+            json_cls = cls.__JSON_SCHEMAS__[name]
             signature[key] = json_struct_to_signature(json_cls)
         else:
             signature[key] = SIGNATURE_MAP[field.stack_last_ret]

{ssc_codegen-0.8.0 → ssc_codegen-0.8.1}/ssc_codegen/str_utils.py

@@ -16,6 +16,7 @@ __all__ = [
     "py_str_format_to_fstring",
     "py_optimize_return_naive",
     "js_pure_optimize_return",
+    "py_remove_unused_none_type_import",
 ]
 
 
@@ -178,6 +179,17 @@ RE_JS_METHOD_BLOCK = re.compile(
 )
 
 
+RE_PY_REMOVE_NONE_TYPE_IMPORT = re.compile(
+    r"if sys\.version_info >= \(3, 10\):[^)]*\(None\)", flags=re.MULTILINE
+)
+
+
+def py_remove_unused_none_type_import(py_code: str) -> str:
+    if len(re.findall(r"\bNoneType\b", py_code)) == 2:
+        return RE_PY_REMOVE_NONE_TYPE_IMPORT.sub("", py_code)
+    return py_code
+
+
 def js_pure_optimize_return(js_code: str) -> str:
     tmp_code = js_code
     for method_code in RE_JS_METHOD_BLOCK.finditer(tmp_code):