ssc_codegen 0.9.4__tar.gz → 0.9.6__tar.gz

{ssc_codegen-0.9.4 → ssc_codegen-0.9.6}/.gitignore

@@ -130,4 +130,14 @@ dmypy.json
 .pyre/
 
 # project
-test_schemas/
+test_schemas/
+.idea/.gitignore
+.idea/.name
+.idea/misc.xml
+.idea/modules.xml
+.idea/selector_schema_codegen.iml
+.idea/vcs.xml
+.idea/yh_parser.iml
+.idea/inspectionProfiles/profiles_settings.xml
+.idea/inspectionProfiles/Project_Default.xml
+.idea/libraries/Dart_SDK.xml

{ssc_codegen-0.9.4 → ssc_codegen-0.9.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.9.4
+Version: 0.9.6
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
@@ -19,7 +19,9 @@ Classifier: Topic :: Software Development :: Code Generators
 Classifier: Topic :: Text Processing :: Markup :: HTML
 Classifier: Topic :: Utilities
 Requires-Python: >=3.10
+Requires-Dist: ast-grep-cli>=0.38.1
 Requires-Dist: bs4>=0.0.2
+Requires-Dist: click<8.2.0
 Requires-Dist: colorama>=0.4.6; sys_platform == 'win32'
 Requires-Dist: cssselect>=1.2.0
 Requires-Dist: httpx>=0.28.1

{ssc_codegen-0.9.4 → ssc_codegen-0.9.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.9.4"
+version = "0.9.6"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"
@@ -16,6 +16,10 @@ dependencies = [
     "soupsieve>=2.6",
     "typer>=0.15.1",
     "typing_extensions;python_version<'3.11'",
+    # https://github.com/fastapi/typer/discussions/1215
+    # https://github.com/fastapi/typer/pull/1145
+    "click<8.2.0",
+    "ast-grep-cli>=0.38.1",
 ]
 
 classifiers = [

{ssc_codegen-0.9.4 → ssc_codegen-0.9.6}/ssc_codegen/__init__.py

@@ -7,7 +7,7 @@ from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchem
 
 setup_logger()
 
-VERSION = "0.9.4"
+VERSION = "0.9.6"
 
 
 class __MISSING(object):

ssc_codegen-0.9.6/ssc_codegen/_compat.py

@@ -0,0 +1,28 @@
+import sys
+import warnings
+import functools
+
+
+if sys.version_info >= (3, 11):
+    from enum import StrEnum
+else:
+    from enum import Enum
+
+    class StrEnum(str, Enum):  # noqa
+        pass
+
+
+def deprecated(reason=""):
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            warnings.warn(
+                f"Function {func.__name__} is deprecated. {reason}",
+                category=DeprecationWarning,
+                stacklevel=2,
+            )
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator

{ssc_codegen-0.9.4 → ssc_codegen-0.9.6}/ssc_codegen/ast_/__init__.py

@@ -31,6 +31,6 @@ from .nodes_string import ExprStringTrim, ExprStringRegex, ExprStringSplit, Expr
     ExprListStringFormat, ExprListStringReplace, ExprListStringRightTrim, ExprListStringLeftTrim, \
     ExprListStringRegexSub, \
     ExprStringRmPrefix, ExprStringRmSuffix, ExprStringRmPrefixAndSuffix, ExprListStringRmPrefixAndSuffix, \
-    ExprListStringRmPrefix, ExprListStringRmSuffix, ExprStringMapReplace, ExprListStringMapReplace
+    ExprListStringRmPrefix, ExprListStringRmSuffix, ExprStringMapReplace, ExprListStringMapReplace, ExprStringUnescape, ExprListStringUnescape
 from .nodes_validate import ExprIsCss, ExprIsEqual, ExprStringIsRegex, ExprIsXpath, ExprIsNotEqual, ExprIsContains, \
     ExprListStringAnyRegex, ExprListStringAllRegex, ExprHasAttr, ExprListHasAttr

{ssc_codegen-0.9.4 → ssc_codegen-0.9.6}/ssc_codegen/ast_/nodes_string.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 from typing import TypedDict, ClassVar
 
-from ssc_codegen.ast_.base import BaseAstNode
+from ssc_codegen.ast_.base import BaseAstNode, T_EMPTY_KWARGS
 from ssc_codegen.tokens import TokenType, VariableType
 
 KW_STR_TRIM = TypedDict("KW_STR_TRIM", {"substr": str})
@@ -212,3 +212,17 @@ class ExprListStringRmPrefixAndSuffix(
     kind: ClassVar[TokenType] = TokenType.EXPR_LIST_STRING_RM_PREFIX_AND_SUFFIX
     accept_type: VariableType = VariableType.LIST_STRING
     ret_type: VariableType = VariableType.LIST_STRING
+
+
+@dataclass(kw_only=True)
+class ExprStringUnescape(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    kind: ClassVar[TokenType] = TokenType.EXPR_STRING_UNESCAPE
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.STRING
+
+
+@dataclass(kw_only=True)
+class ExprListStringUnescape(BaseAstNode[T_EMPTY_KWARGS, tuple]):
+    kind: ClassVar[TokenType] = TokenType.EXPR_LIST_STRING_UNESCAPE
+    accept_type: VariableType = VariableType.LIST_STRING
+    ret_type: VariableType = VariableType.LIST_STRING

ssc_codegen-0.9.6/ssc_codegen/ast_grep_rules/js_rules.yml

@@ -0,0 +1,20 @@
+id: replace-last-decl
+language: js
+rule:
+  follows:
+    pattern:
+      context: let $VAR = $$$EXPR
+      selector: lexical_declaration
+  pattern: return $VAR
+fix: |-
+  return $$$EXPR
+---
+id: remove-last-decl
+language: js
+rule:
+  pattern:
+      context: let $VAR = $$$EXPR
+      selector: lexical_declaration
+  precedes:
+    pattern: "return $VAR"
+fix: ''

ssc_codegen-0.9.6/ssc_codegen/ast_grep_rules/py_rules.yml

@@ -0,0 +1,20 @@
+id: replace-last-decl
+language: python
+rule:
+  follows:
+    pattern:
+      context: $VAR = $$$EXPR
+      selector: expression_statement
+  pattern: return $VAR
+fix: |-
+  return $$$EXPR
+---
+id: remove-last-decl
+language: python
+rule:
+  pattern:
+    context: $VAR = $$$EXPR
+    selector: expression_statement
+  precedes:
+    pattern: "return $VAR"
+fix: ''

ssc_codegen-0.9.6/ssc_codegen/cli/ast_grep.py

@@ -0,0 +1,7 @@
+from importlib import resources
+
+
+RULES_PATH = resources.files("ssc_codegen.ast_grep_rules")
+
+PY_RULES = str(RULES_PATH.joinpath("py_rules.yml"))
+JS_RULES = str(RULES_PATH.joinpath("js_rules.yml"))

{ssc_codegen-0.9.4 → ssc_codegen-0.9.6}/ssc_codegen/cli/code_callbacks.py

@@ -3,8 +3,6 @@
 from typing import Callable
 from ssc_codegen.str_utils import (
     go_unimport_naive,
-    py_optimize_return_naive,
-    js_pure_optimize_return,
     py_remove_unused_none_type_import,
 )
 
@@ -30,9 +28,7 @@ class BaseCodeCallback:
         return code
 
 
-CB_PY_CODE = BaseCodeCallback(
-    py_optimize_return_naive, py_remove_unused_none_type_import
-)
+CB_PY_CODE = BaseCodeCallback(py_remove_unused_none_type_import)
 CB_GO_CODE = BaseCodeCallback(go_unimport_naive, remove_empty_lines=True)
 CB_DART_CODE = BaseCodeCallback(remove_empty_lines=True)
-CB_JS_CODE = BaseCodeCallback(js_pure_optimize_return)
+CB_JS_CODE = BaseCodeCallback(remove_empty_lines=True)

{ssc_codegen-0.9.4 → ssc_codegen-0.9.6}/ssc_codegen/cli/main.py

@@ -11,6 +11,7 @@ from ichrome.exceptions import ChromeRuntimeError
 from typer import Argument, BadParameter, Option, Typer
 
 from ssc_codegen.ast_build import build_ast_module_parser
+from ssc_codegen.cli.ast_grep import JS_RULES, PY_RULES
 from ssc_codegen.cli.cli_callbacks import cb_check_ssc_files, cb_folder_out
 from ssc_codegen.cli.cli_utils import (
     ConverterLike,
@@ -164,10 +165,15 @@ def gen_py(
 ) -> None:
     converter = import_converter(f"py_{lib.value}")
     if fmt:
-        commands = ["ruff format {}", "ruff check {} --fix"]
+        commands = [
+            "ruff format {}",
+            "ruff check {} --fix",
+            f"ast-grep scan --rule {PY_RULES} -U " + "{}",
+        ]
         fmt_cmd = create_fmt_cmd(ssc_files, prefix, suffix, out, commands)
     else:
         fmt_cmd = []
+
     generate_code(
         converter=converter,
         out=out,
@@ -216,6 +222,7 @@ def gen_js(
     converter = import_converter(f"js_{lib.value}")
     if fmt:
         commands: list[str] = [
+            f"ast-grep scan --rule {JS_RULES} -U " + "{}",
             "prettier --write --tab-width 4 --single-quote --trailing-comma all --semi {}",
         ]
         fmt_cmd = create_fmt_cmd(ssc_files, prefix, suffix, out, commands)

{ssc_codegen-0.9.4 → ssc_codegen-0.9.6}/ssc_codegen/converters/go_goquery.py

@@ -1266,3 +1266,4 @@ def pre_list_str_rm_prefix_and_suffix(
 # TODO: impl ExprListUnique
 # TODO: impl ExprStringMapReplace, ExprListStringMapReplace
 # TODO: impl ExprAttr, ExprListAttr for multiple extract keys
+# TODO impl unescape

{ssc_codegen-0.9.4 → ssc_codegen-0.9.6}/ssc_codegen/converters/js_pure.py

@@ -105,6 +105,11 @@ from ssc_codegen.ast_ import (
     ExprStringMapReplace,
     ExprListStringMapReplace,
 )
+from ssc_codegen.ast_.nodes_core import ModuleImports
+from ssc_codegen.ast_.nodes_string import (
+    ExprListStringUnescape,
+    ExprStringUnescape,
+)
 from ssc_codegen.converters.base import BaseCodeConverter
 from ssc_codegen.converters.helpers import (
     prev_next_var,
@@ -181,6 +186,12 @@ def py_sequence_to_js_array(values: tuple[str, ...] | list[str]) -> str:
     return "[" + val_arr[1:-1] + "]"
 
 
+@CONVERTER(ModuleImports.kind)
+def pre_imports(_node: ModuleImports) -> str:
+    # HACK: used imports node for provide helpers scripts
+    pass
+
+
 @CONVERTER(Docstring.kind)
 def pre_docstring(node: Docstring) -> str:
     value = node.kwargs["value"]
@@ -961,3 +972,43 @@ def pre_list_str_map_repl(node: ExprListStringMapReplace) -> str:
     new_arr = list(new_arr)  # type: ignore
     prv, nxt = prev_next_var(node)
     return f"let {nxt} = {prv}.map(s => {old_arr}.reduce((s, v, i) => s.replaceAll(v, {new_arr}[i] ?? ''), s));"
+
+
+@CONVERTER(ExprStringUnescape.kind)
+def pre_str_unescape(node: ExprStringUnescape) -> str:
+    prv, nxt = prev_next_var(node)
+    return (
+        f"let {nxt} = {prv}"
+        + ".replace(/&/g, '&')"
+        + ".replace(/&lt;/g, '<')"
+        + ".replace(/&gt;/g, '>')"
+        + ".replace(/&quot;/g, '\"')"
+        + '.replace(/&#039;/g, "\'")'
+        + ".replace(/&#x2F;/g, '/')"
+        + ".replace(/&nbsp;/g, ' ')"
+        + ".replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))"
+        + r".replace(/\\u([0-9a-fA-F]{4})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))"
+        + r".replace(/\\x([0-9a-fA-F]{2})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))"
+        + r".replace(/\\([bfnrt])/g, (_, ch) => ({b:'\\b',f:'\\f',n:'\\n',r:'\\r',t:'\\t'})[ch])"
+        + ";"
+    )
+
+
+@CONVERTER(ExprListStringUnescape.kind)
+def pre_list_str_unescape(node: ExprListStringUnescape) -> str:
+    prv, nxt = prev_next_var(node)
+    return (
+        f"let {nxt} = {prv}.map(s => s"
+        + ".replace(/&amp;/g, '&')"
+        + ".replace(/&lt;/g, '<')"
+        + ".replace(/&gt;/g, '>')"
+        + ".replace(/&quot;/g, '\"')"
+        + '.replace(/&#039;/g, "\'")'
+        + ".replace(/&#x2F;/g, '/')"
+        + ".replace(/&nbsp;/g, ' ')"
+        + ".replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))"
+        + r".replace(/\\u([0-9a-fA-F]{4})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))"
+        + r".replace(/\\x([0-9a-fA-F]{2})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))"
+        + r".replace(/\\([bfnrt])/g, (_, ch) => ({b:'\b',f:'\f',n:'\n',r:'\r',t:'\\t'})[ch])"
+        + ");"
+    )

{ssc_codegen-0.9.4 → ssc_codegen-0.9.6}/ssc_codegen/converters/py_base.py

@@ -107,6 +107,10 @@ from ssc_codegen.ast_ import (
     ExprStringMapReplace,
     ExprListStringMapReplace,
 )
+from ssc_codegen.ast_.nodes_string import (
+    ExprListStringUnescape,
+    ExprStringUnescape,
+)
 from ssc_codegen.converters.base import (
     BaseCodeConverter,
     CB_FMT_DEBUG_COMMENT,
@@ -176,6 +180,7 @@ IMPORTS_MIN = """
 import re
 import sys
 import json
+from html import unescape as _html_unescape
 from typing import List, Dict, TypedDict, Union, Optional
 from contextlib import suppress
 from functools import reduce
@@ -278,6 +283,8 @@ class BasePyCodeConverter(BaseCodeConverter):
             ExprListUnique.kind: pre_list_unique,
             ExprStringMapReplace.kind: pre_str_map_replace,
             ExprListStringMapReplace.kind: pre_list_str_map_replace,
+            ExprStringUnescape.kind: pre_str_unescape,
+            ExprListStringUnescape.kind: pre_list_str_unescape,
         }
 
         self.post_definitions = {
@@ -1125,3 +1132,47 @@ def pre_list_str_map_replace(node: ExprListStringMapReplace) -> str:
     prv, nxt = prev_next_var(node)
     expr = f"[reduce(lambda acc, kv: acc.replace(kv[0], kv[1]), {replacements}.items(), i) for i in {prv}]"
     return f"{indent}{nxt} = {expr}"
+
+
+def pre_str_unescape(node: ExprStringUnescape) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    return (
+        f"{indent}{nxt} = _html_unescape({prv})\n"
+        # hex unescape
+        + f"{indent}{nxt} = re.sub(r'&#x([0-9a-fA-F]+);', lambda m: chr(int(m.group(1), 16)), {nxt})\n"
+        + f"{indent}{nxt} = re.sub(r'\\\\u([0-9a-fA-F]{{4}})', lambda m: chr(int(m.group(1), 16)), {nxt})\n"
+        + f"{indent}{nxt} = re.sub(r'\\\\x([0-9a-fA-F]{{2}})', lambda m: chr(int(m.group(1), 16)), {nxt})\n"
+        # chars
+        + f"{indent}{nxt} = {nxt}"
+        + r".replace('\\b', '\b')"
+        + r".replace('\\f', '\f')"
+        + r".replace('\\n', '\n')"
+        + r".replace('\\r', '\r')"
+        + r".replace('\\t', '\t')"
+    )
+
+
+def pre_list_str_unescape(node: ExprListStringUnescape) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    return (
+        f"{indent}{nxt} = [_html_unescape(i) for i in {prv}]\n"
+        # hex unescape
+        + f"{indent}{nxt} = [re.sub(r'&#x([0-9a-fA-F]+);', lambda m: chr(int(m.group(1), 16)), i) for i in {nxt}]\n"
+        + f"{indent}{nxt} = [re.sub(r'\\\\u([0-9a-fA-F]{{4}})', lambda m: chr(int(m.group(1), 16)), i) for i in {nxt}]\n"
+        + f"{indent}{nxt} = [re.sub(r'\\\\x([0-9a-fA-F]{{2}})', lambda m: chr(int(m.group(1), 16)), i) for i in {nxt}]\n"
+        + f"{indent}{nxt} = "
+        # chars
+        + "[i"
+        + r".replace('\\b', '\b')"
+        + r".replace('\\f', '\f')"
+        + r".replace('\\n', '\n')"
+        + r".replace('\\r', '\r')"
+        + r".replace('\\t', '\t')"
+        + f" for i in {nxt}]"
+    )

{ssc_codegen-0.9.4 → ssc_codegen-0.9.6}/ssc_codegen/document.py

@@ -79,6 +79,10 @@ from ssc_codegen.ast_ import (
     ExprListStringMapReplace,
     ExprStringMapReplace,
 )
+from ssc_codegen.ast_.nodes_string import (
+    ExprListStringUnescape,
+    ExprStringUnescape,
+)
 from ssc_codegen.document_utlis import (
     analyze_re_expression,
     unverbosify_regex,
@@ -852,6 +856,26 @@ class StringDocument(BaseDocument):
         self.re_sub(f"{pattern}$")
         return self
 
+    def unescape(self) -> Self:
+        """unescape string output
+
+        - accept STRING, return STRING
+        - accept LIST_STRING, return LIST_STRING
+        """
+        match self.stack_last_ret:
+            case VariableType.STRING:
+                self._add(ExprStringUnescape())
+            case VariableType.LIST_STRING:
+                self._add(ExprListStringUnescape())
+            case _:
+                LOGGER.warning(
+                    "unescape(): Expected type(s) %s got %s",
+                    (VariableType.STRING.name, VariableType.LIST_STRING.name),
+                    self.stack_last_ret.name,
+                )
+                self._add(ExprStringUnescape())
+        return self
+
 
 class AssertDocument(BaseDocument):
     def is_css(self, query: str, msg: str = "") -> Self:

{ssc_codegen-0.9.4 → ssc_codegen-0.9.6}/ssc_codegen/str_utils.py

@@ -1,6 +1,7 @@
 """post-processing functions for fix code output and utils operations with strings"""
 
 import re
+from ssc_codegen._compat import deprecated
 
 RE_GO_IMPORTS_BLOCK = re.compile(r"import\s*\(\s*([\s\S]*?)\s*\)")
 
@@ -109,6 +110,7 @@ return\s(?P<ret_var>\w+)            # RET STMT
 )
 
 
+@deprecated("use ast-grep rule")
 def py_optimize_return_naive(py_code: str) -> str:
     """optimize _parse_[a-zA-Z_0-9] and _split_doc return statements
 
@@ -151,6 +153,9 @@ def py_optimize_return_naive(py_code: str) -> str:
             # NAIVE apologize, its .filter(...) expr, skip optimization
             if expr.strip().endswith(" if"):
                 continue
+            # unescape expr, skip
+            if "_html_unescape" in expr.strip():
+                continue
 
             # optimization function convert double backslash to single (\\ -> \) in regex patterns
             # add `r` prefix for avoid SyntaxWarning then compile generated code to cpython bytecode
@@ -205,6 +210,7 @@ def py_remove_unused_none_type_import(py_code: str) -> str:
     return py_code
 
 
+@deprecated("use ast-grep rule")
 def js_pure_optimize_return(js_code: str) -> str:
     tmp_code = js_code
     for method_code in RE_JS_METHOD_BLOCK.finditer(tmp_code):

{ssc_codegen-0.9.4 → ssc_codegen-0.9.6}/ssc_codegen/tokens.py

@@ -133,6 +133,7 @@ class TokenType(IntEnum):
     EXPR_STRING_RM_SUFFIX = auto()
     EXPR_STRING_RM_PREFIX_AND_SUFFIX = auto()
     EXPR_STRING_MAP_REPLACE = auto()
+    EXPR_STRING_UNESCAPE = auto()
 
     # LIST_STRING
     EXPR_LIST_REGEX_SUB = auto()
@@ -145,6 +146,7 @@ class TokenType(IntEnum):
     EXPR_LIST_STRING_RM_SUFFIX = auto()
     EXPR_LIST_STRING_RM_PREFIX_AND_SUFFIX = auto()
     EXPR_LIST_STRING_MAP_REPLACE = auto()
+    EXPR_LIST_STRING_UNESCAPE = auto()
 
     # ARRAY
     EXPR_LIST_ANY_INDEX = auto()

ssc_codegen-0.9.4/ssc_codegen/_compat.py

@@ -1,9 +0,0 @@
-import sys
-
-if sys.version_info >= (3, 11):
-    from enum import StrEnum
-else:
-    from enum import Enum
-
-    class StrEnum(str, Enum):  # noqa
-        pass