ssc_codegen 0.13.3__tar.gz → 0.14.0__tar.gz

{ssc_codegen-0.13.3 → ssc_codegen-0.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.13.3
+Version: 0.14.0
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
@@ -27,7 +27,6 @@ Requires-Dist: cssselect>=1.2.0
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: ichrome>=4.0.4
 Requires-Dist: lxml>=5.3.0
-Requires-Dist: parsel>=1.10.0
 Requires-Dist: soupsieve>=2.6
 Requires-Dist: tinycss2>=1.4.0
 Requires-Dist: typer>=0.15.1
@@ -59,15 +58,15 @@ ssc-gen - a python-based DSL to describe parsers for html documents, which is tr
 
 Current support converters
 
-| Language                                 | HTML parser lib + dependencies         | XPath | CSS3 | CSS4 | Generated annotations, types, structs | formatter dependency |
-| ---------------------------------------- | -------------------------------------- | ----- | ---- | ---- | ------------------------------------- | -------------------- |
-| Python (3.8+)                            | bs4, lxml                              | N     | Y    | Y    | TypedDict`1`, list, dict              | ruff                 |
-| ...                                      | parsel                                 | Y     | Y    | N    | ...                                   | ...                  |
-| ...                                      | selectolax (lexbor)                    | N     | Y    | N    | ...                                   | ...                  |
-| ...                                      | lxml                                   | Y     | Y    | N    | ...                                   | ...                  |
-| js (ES6)`2`                              | pure (firefox/chrome extension/nodejs) | Y     | Y    | Y    | JSDoc                                 | prettier             |
-| go (1.10+) **(UNSTABLE)**                | goquery, gjson (`4`)                   | N     | Y    | N    | struct(+json anchors), array, map     | gofmt                |
-| lua (5.2+), luajit(2+) **(UNSTABLE)**`5` | lua-htmlparser, lrexlib(opt), dkjson   | N     | Y    | N    | EmmyLua                               | LuaFormatter         |
+| Language                                 | HTML parser lib + dependencies                         | XPath | CSS3 | CSS4 | Generated annotations, types, structs | formatter dependency |
+| ---------------------------------------- | ------------------------------------------------------ | ----- | ---- | ---- | ------------------------------------- | -------------------- |
+| Python (3.8-3.13)                        | bs4, lxml ( typing_extensions if py < 3.10 )           | N     | Y    | Y    | TypedDict`1`, list, dict              | ruff                 |
+| ...                                      | parsel ( typing_extensions if py < 3.10 )              | Y     | Y    | N    | ...                                   | ...                  |
+| ...                                      | selectolax (lexbor) ( typing_extensions if py < 3.10 ) | N     | Y    | N    | ...                                   | ...                  |
+| ...                                      | lxml ( typing_extensions if py < 3.10 )                | Y     | Y    | N    | ...                                   | ...                  |
+| js (ES6)`2`                              | pure (firefox/chrome extension/nodejs)                 | Y     | Y    | Y    | JSDoc                                 | prettier             |
+| go (1.10+) **(UNSTABLE)**                | goquery, gjson (`4`)                                   | N     | Y    | N    | struct(+json anchors), array, map     | gofmt                |
+| lua (5.2+), luajit(2+) **(UNSTABLE)**`5` | lua-htmlparser, lrexlib(opt), dkjson                   | N     | Y    | N    | EmmyLua                               | LuaFormatter         |
 
 - **CSS3** means support next selectors:
   - basic: (`tag`, `.class`, `#id`, `tag1,tag2`)
@@ -100,6 +99,7 @@ For maximum portability of the configuration to the target language:
   Check the html parser lib documentation aboud CSS selectors before implement code. Examples:
   1. Several libs not support `+` operations (eg: [selectolax(modest)](https://github.com/rushter/selectolax), [dart.universal_html](https://pub.dev/packages/universal_html))
   2. For research purpose, lua_htmlparser include converter for unsupported CSS3 query syntax
+
 2. HTML parser libs maybe not supports attribute selectors: `*=`, `~=`, `|=`, `^=`, `$=`
 3. Several libs not support pseudo classes (eg: standard [dart.html](https://dart.dev/libraries/dart-html) lib miss this feature).
 
@@ -246,7 +246,6 @@ You can use any html source:
 
 ## See also
 
-
 - [Brief](docs/briefing.md) about css selectors and regular expressions
 - [Explain](docs/explain.md) short document on how to understand DSL syntax
 - [LLM](llm.md) experimental prompt for generate code

{ssc_codegen-0.13.3 → ssc_codegen-0.14.0}/README.md

@@ -23,15 +23,15 @@ ssc-gen - a python-based DSL to describe parsers for html documents, which is tr
 
 Current support converters
 
-| Language                                 | HTML parser lib + dependencies         | XPath | CSS3 | CSS4 | Generated annotations, types, structs | formatter dependency |
-| ---------------------------------------- | -------------------------------------- | ----- | ---- | ---- | ------------------------------------- | -------------------- |
-| Python (3.8+)                            | bs4, lxml                              | N     | Y    | Y    | TypedDict`1`, list, dict              | ruff                 |
-| ...                                      | parsel                                 | Y     | Y    | N    | ...                                   | ...                  |
-| ...                                      | selectolax (lexbor)                    | N     | Y    | N    | ...                                   | ...                  |
-| ...                                      | lxml                                   | Y     | Y    | N    | ...                                   | ...                  |
-| js (ES6)`2`                              | pure (firefox/chrome extension/nodejs) | Y     | Y    | Y    | JSDoc                                 | prettier             |
-| go (1.10+) **(UNSTABLE)**                | goquery, gjson (`4`)                   | N     | Y    | N    | struct(+json anchors), array, map     | gofmt                |
-| lua (5.2+), luajit(2+) **(UNSTABLE)**`5` | lua-htmlparser, lrexlib(opt), dkjson   | N     | Y    | N    | EmmyLua                               | LuaFormatter         |
+| Language                                 | HTML parser lib + dependencies                         | XPath | CSS3 | CSS4 | Generated annotations, types, structs | formatter dependency |
+| ---------------------------------------- | ------------------------------------------------------ | ----- | ---- | ---- | ------------------------------------- | -------------------- |
+| Python (3.8-3.13)                        | bs4, lxml ( typing_extensions if py < 3.10 )           | N     | Y    | Y    | TypedDict`1`, list, dict              | ruff                 |
+| ...                                      | parsel ( typing_extensions if py < 3.10 )              | Y     | Y    | N    | ...                                   | ...                  |
+| ...                                      | selectolax (lexbor) ( typing_extensions if py < 3.10 ) | N     | Y    | N    | ...                                   | ...                  |
+| ...                                      | lxml ( typing_extensions if py < 3.10 )                | Y     | Y    | N    | ...                                   | ...                  |
+| js (ES6)`2`                              | pure (firefox/chrome extension/nodejs)                 | Y     | Y    | Y    | JSDoc                                 | prettier             |
+| go (1.10+) **(UNSTABLE)**                | goquery, gjson (`4`)                                   | N     | Y    | N    | struct(+json anchors), array, map     | gofmt                |
+| lua (5.2+), luajit(2+) **(UNSTABLE)**`5` | lua-htmlparser, lrexlib(opt), dkjson                   | N     | Y    | N    | EmmyLua                               | LuaFormatter         |
 
 - **CSS3** means support next selectors:
   - basic: (`tag`, `.class`, `#id`, `tag1,tag2`)
@@ -64,6 +64,7 @@ For maximum portability of the configuration to the target language:
   Check the html parser lib documentation aboud CSS selectors before implement code. Examples:
   1. Several libs not support `+` operations (eg: [selectolax(modest)](https://github.com/rushter/selectolax), [dart.universal_html](https://pub.dev/packages/universal_html))
   2. For research purpose, lua_htmlparser include converter for unsupported CSS3 query syntax
+
 2. HTML parser libs maybe not supports attribute selectors: `*=`, `~=`, `|=`, `^=`, `$=`
 3. Several libs not support pseudo classes (eg: standard [dart.html](https://dart.dev/libraries/dart-html) lib miss this feature).
 
@@ -210,7 +211,6 @@ You can use any html source:
 
 ## See also
 
-
 - [Brief](docs/briefing.md) about css selectors and regular expressions
 - [Explain](docs/explain.md) short document on how to understand DSL syntax
 - [LLM](llm.md) experimental prompt for generate code

{ssc_codegen-0.13.3 → ssc_codegen-0.14.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.13.3"
+version = "0.14.0"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"
@@ -11,7 +11,6 @@ dependencies = [
     "httpx>=0.28.1",
     "ichrome>=4.0.4",
     "lxml>=5.3.0",
-    "parsel>=1.10.0",
     "soupsieve>=2.6",
     "typer>=0.15.1",
     "typing_extensions;python_version<'3.11'",

{ssc_codegen-0.13.3 → ssc_codegen-0.14.0}/ssc_codegen/__init__.py

@@ -7,7 +7,7 @@ from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchem
 
 setup_logger()
 
-VERSION = "0.13.3"
+VERSION = "0.14.0"
 
 
 class __MISSING(object):

{ssc_codegen-0.13.3 → ssc_codegen-0.14.0}/ssc_codegen/cli/code_callbacks.py

@@ -3,7 +3,6 @@
 from typing import Callable
 from ssc_codegen.str_utils import (
     go_unimport_naive,
-    py_remove_unused_none_type_import,
 )
 
 
@@ -28,7 +27,6 @@ class BaseCodeCallback:
         return code
 
 
-CB_PY_CODE = BaseCodeCallback(py_remove_unused_none_type_import)
 CB_GO_CODE = BaseCodeCallback(go_unimport_naive, remove_empty_lines=True)
 CB_DART_CODE = BaseCodeCallback(remove_empty_lines=True)
 CB_JS_CODE = BaseCodeCallback(remove_empty_lines=True)

{ssc_codegen-0.13.3 → ssc_codegen-0.14.0}/ssc_codegen/cli/main.py

@@ -25,7 +25,6 @@ from ssc_codegen.cli.cli_utils import (
     suggest_class_name,
 )
 from ssc_codegen.cli.code_callbacks import (
-    CB_PY_CODE,
     CB_GO_CODE,
     CB_JS_CODE,
 )
@@ -208,7 +207,6 @@ def gen_py(
         xpath_to_css=to_css,
         css_to_xpath=to_xpath,
         debug_instructions=debug,
-        code_cb=CB_PY_CODE,
         gen_docstring=docstring,
     )
 
@@ -439,7 +437,7 @@ def parse_from_file(
     ] = None,
 ) -> None:
     from ssc_codegen.compiler import Compiler
-    from ssc_codegen.converters.py_parsel import CONVERTER
+    from ssc_codegen.converters.py_lxml import CONVERTER
 
     cls_target, schema_config = _validate_parser_target(cls_target)
 
@@ -563,7 +561,7 @@ def parse_from_chrome_(
     from ssc_codegen.converters.js_pure import CONVERTER as JS_CONVERTER
     import asyncio
     from ssc_codegen.compiler import Compiler
-    from ssc_codegen.converters.py_parsel import CONVERTER
+    from ssc_codegen.converters.py_lxml import CONVERTER
 
     cls_target, schema_config = _validate_parser_target(cls_target)
 

{ssc_codegen-0.13.3 → ssc_codegen-0.14.0}/ssc_codegen/converters/py_base.py

@@ -143,24 +143,24 @@ INDENT_METHOD = INDENT_CH * 4
 INDENT_METHOD_BODY = INDENT_CH * (4 * 2)
 INDENT_DEFAULT_BODY = INDENT_CH * (4 * 3)
 TYPE_PREFIX = "T_{}"
-TYPE_DICT = "Dict[str, {}]"
-TYPE_LIST = "List[{}]"
+TYPE_DICT = "dict[str, {}]"
+TYPE_LIST = "list[{}]"
 TYPE_ITEM = "TypedDict({}, {})"
 
 TYPES = {
     VariableType.ANY: "Any",
     VariableType.STRING: "str",
-    VariableType.LIST_STRING: "List[str]",
+    VariableType.LIST_STRING: "list[str]",
     VariableType.OPTIONAL_STRING: "Optional[str]",
-    VariableType.OPTIONAL_LIST_STRING: "Optional[List[str]]",
+    VariableType.OPTIONAL_LIST_STRING: "Optional[list[str]]",
     VariableType.OPTIONAL_INT: "Optional[int]",
-    VariableType.OPTIONAL_LIST_INT: "Optional[List[int]]",
+    VariableType.OPTIONAL_LIST_INT: "Optional[list[int]]",
     VariableType.OPTIONAL_FLOAT: "Optional[float]",
-    VariableType.OPTIONAL_LIST_FLOAT: "Optional[List[float]]",
+    VariableType.OPTIONAL_LIST_FLOAT: "Optional[list[float]]",
     VariableType.INT: "int",
     VariableType.FLOAT: "float",
-    VariableType.LIST_INT: "List[int]",
-    VariableType.LIST_FLOAT: "List[float]",
+    VariableType.LIST_INT: "list[int]",
+    VariableType.LIST_FLOAT: "list[float]",
     VariableType.BOOL: "bool",
 }
 
@@ -174,11 +174,11 @@ JSON_TYPES = {
     JsonVariableType.OPTIONAL_NUMBER: "Optional[int]",
     JsonVariableType.OPTIONAL_FLOAT: "Optional[float]",
     JsonVariableType.OPTIONAL_BOOLEAN: "Optional[bool]",
-    JsonVariableType.ARRAY: "List",
-    JsonVariableType.ARRAY_FLOAT: "List[float]",
-    JsonVariableType.ARRAY_NUMBER: "List[int]",
-    JsonVariableType.ARRAY_STRING: "List[str]",
-    JsonVariableType.ARRAY_BOOLEAN: "List[bool]",
+    JsonVariableType.ARRAY: "list",
+    JsonVariableType.ARRAY_FLOAT: "list[float]",
+    JsonVariableType.ARRAY_NUMBER: "list[int]",
+    JsonVariableType.ARRAY_STRING: "list[str]",
+    JsonVariableType.ARRAY_BOOLEAN: "list[bool]",
 }
 
 LITERAL_TYPES = {
@@ -327,11 +327,11 @@ def get_typedef_field_by_name(node: TypeDef, field_name: str) -> str:
     if value.kwargs["type"] == VariableType.NESTED:
         type_ = f"T_{value.kwargs['cls_nested']}"
         if value.kwargs["cls_nested_type"] == StructType.LIST:
-            type_ = f"List[{type_}]"
+            type_ = f"list[{type_}]"
     elif value.kwargs["type"] == VariableType.JSON:
         type_ = f"J_{value.kwargs['cls_nested']}"
         if value.kwargs["cls_nested_type"] == StructType.LIST:
-            type_ = f"List[{type_}]"
+            type_ = f"list[{type_}]"
     else:
         type_ = TYPES[value.kwargs["type"]]
     return type_
@@ -346,14 +346,14 @@ def get_field_method_ret_type(node: StructFieldMethod) -> str:
             schema_name, schema_type = last_expr.unpack_args()
             type_ = f"T_{schema_name}"
             if schema_type == StructType.LIST:
-                type_ = f"List[{type_}]"
+                type_ = f"list[{type_}]"
         case VariableType.JSON:
             last_expr = node.body[-2]
             last_expr = cast(ExprJsonify, last_expr)
             json_struct_name, is_array, *_ = last_expr.unpack_args()
             type_ = f"J_{json_struct_name}"
             if is_array:
-                type_ = f"List[{type_}]"
+                type_ = f"list[{type_}]"
         case _:
             type_ = TYPES[last_expr.ret_type]
     return type_
@@ -396,7 +396,7 @@ def post_typedef_list(_: TypeDef) -> str:
 def pre_typedef_dict(node: TypeDef) -> str:
     name, _ = node.unpack_args()
     type_ = get_typedef_field_by_name(node, "__VALUE__")
-    return f"T_{name} = Dict[str, {type_}]"
+    return f"T_{name}: TypeAlias = dict[str, {type_}]"
 
 
 def post_typedef_dict(_: TypeDef) -> str:
@@ -406,7 +406,7 @@ def post_typedef_dict(_: TypeDef) -> str:
 def pre_typedef_flat_list(node: TypeDef) -> str:
     name, _ = node.unpack_args()
     type_ = get_typedef_field_by_name(node, "__VALUE__")
-    return f"T_{name} = List[{type_}]"
+    return f"T_{name}: TypeAlias = list[{type_}]"
 
 
 def post_typedef_flat_list(_: TypeDef) -> str:
@@ -415,7 +415,7 @@ def post_typedef_flat_list(_: TypeDef) -> str:
 
 def pre_typedef_acc_list(node: TypeDef) -> str:
     name, _ = node.unpack_args()
-    return f"T_{name} = List[str]"
+    return f"T_{name}: TypeAlias = list[str]"
 
 
 def post_typedef_acc_list(_: TypeDef) -> str:
@@ -438,11 +438,11 @@ def pre_typedef_field(node: TypeDefField) -> str:
     elif var_type == VariableType.NESTED:
         type_ = f"T_{cls_nested}"
         if cls_nested_type == StructType.LIST:
-            type_ = f"List[{type_}]"
+            type_ = f"list[{type_}]"
     elif var_type == VariableType.JSON:
         type_ = f"J_{cls_nested}"
         if cls_nested_type == StructType.LIST:
-            type_ = f"List[{type_}]"
+            type_ = f"list[{type_}]"
     else:
         type_ = TYPES[var_type]
     return f"{name!r}: {type_},"
@@ -466,7 +466,7 @@ def pre_json_field(node: JsonStructField) -> str:
     if json_type.type == JsonVariableType.OBJECT:
         type_ = f"J_{json_type.name}"
     elif json_type.type == JsonVariableType.ARRAY_OBJECTS:
-        type_ = f"List[J_{json_type.name}]"
+        type_ = f"list[J_{json_type.name}]"
     else:
         type_ = JSON_TYPES[json_type.type]
 
@@ -529,7 +529,7 @@ def pre_start_parse(node: StartParseMethod) -> str:
     st_type = node.parent.kwargs["struct_type"]
     type_ = f"T_{name}"
     if st_type == StructType.LIST:
-        type_ = f"List[{type_}]"
+        type_ = f"list[{type_}]"
     return INDENT_METHOD + f"def parse(self) -> {type_}:"
 
 
@@ -1010,7 +1010,7 @@ def pre_list_str_any_is_regex(node: ExprListStringAnyRegex) -> str:
         INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
     )
     prv, nxt = prev_next_var(node)
-    pattern, ignore_case, msg = node.unpack_args()
+    pattern, ignore_case, msg, _invert = node.unpack_args()
 
     if node.classvar_hooks.get("pattern"):
         pattern = py_get_classvar_hook_or_value(node, "pattern")
@@ -1033,7 +1033,7 @@ def pre_list_str_all_is_regex(node: ExprListStringAllRegex) -> str:
         INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
     )
     prv, nxt = prev_next_var(node)
-    pattern, ignore_case, msg = node.unpack_args()
+    pattern, ignore_case, msg, _invert = node.unpack_args()
 
     if node.classvar_hooks.get("pattern"):
         pattern = py_get_classvar_hook_or_value(node, "pattern")
@@ -1181,7 +1181,7 @@ def pre_filter_ends_with(node: FilterStrEnds) -> str:
 def pre_filter_re(node: FilterStrRe) -> str:
     pattern, ignore_case, *_ = node.unpack_args()
     if ignore_case:
-        expr = f"re.search({pattern!r}, i, re.IGNORECASE)"
+        expr = f"re.search({pattern!r}, i, re.I)"
     else:
         expr = f"re.search({pattern!r}, i)"
     if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):

{ssc_codegen-0.13.3 → ssc_codegen-0.14.0}/ssc_codegen/converters/py_lxml.py

@@ -107,7 +107,7 @@ def pre_struct_pre_validate(_node: StructPreValidateMethod) -> str:
 def pre_struct_part_doc_method(_node: StructPartDocMethod) -> str:
     return (
         INDENT_METHOD
-        + "def _split_doc(self, v: html.HtmlElement) -> List[html.HtmlElement]:"
+        + "def _split_doc(self, v: html.HtmlElement) -> list[html.HtmlElement]:"
     )
 
 
@@ -128,8 +128,7 @@ def pre_imports(_: ModuleImports) -> str:
     # most mainsteram html parser libs use `FALLBACK_HTML_STR` stub value
     return (
         IMPORTS_MIN
-        + """from lxml import html, etree
-    
+        + """from lxml import html
 FALLBACK_HTML_STR = "<html><body></body></html>"
     """
     )

{ssc_codegen-0.13.3 → ssc_codegen-0.14.0}/ssc_codegen/converters/py_parsel.py

@@ -272,7 +272,7 @@ def pre_has_attr(node: ExprHasAttr) -> str:
     expr = indent + f"assert {expr}, {msg!r}"
     if is_last_var_no_ret(node):
         return expr
-    return "\n".join(expr, indent + f"{nxt} = {prv}")
+    return "\n".join([expr, indent + f"{nxt} = {prv}"])
 
 
 @CONVERTER(ExprListHasAttr.kind)
@@ -292,7 +292,7 @@ def pre_list_has_attr(node: ExprListHasAttr) -> str:
 
     if is_last_var_no_ret(node):
         return expr
-    return "\n".join(expr, indent + f"{nxt} = {prv}")
+    return "\n".join([expr, indent + f"{nxt} = {prv}"])
 
 
 @CONVERTER(ExprMapAttrs.kind)

{ssc_codegen-0.13.3 → ssc_codegen-0.14.0}/ssc_codegen/converters/py_selectolax.py

@@ -84,7 +84,7 @@ def pre_struct_pre_validate(_node: StructPreValidateMethod) -> str:
 def pre_struct_part_doc_method(_node: StructPartDocMethod) -> str:
     return (
         INDENT_METHOD
-        + "def _split_doc(self, v: Union[HTMLParser, Node]) -> List[Node]:"
+        + "def _split_doc(self, v: Union[HTMLParser, Node]) -> list[Node]:"
     )
 
 

{ssc_codegen-0.13.3 → ssc_codegen-0.14.0}/ssc_codegen/converters/templates/py_base.py

@@ -1,19 +1,26 @@
-# used old style typing for support old python versions (3.8)
-
-
+# from __future__ import annotations for support old python versions (3.8)
 IMPORTS_MIN = """
+from __future__ import annotations
+
 import re
 import sys
 import json
 from html import unescape as _html_unescape
-from typing import List, Dict, TypedDict, Union, Optional, ClassVar
+from typing import TypedDict, Union, Optional, ClassVar
 from contextlib import suppress
 from functools import reduce
 
 if sys.version_info >= (3, 10):
     from types import NoneType
+    from typing import TypeAlias
 else:
     NoneType = type(None)
+
+    try:
+        from typing_extensions import TypeAlias  # noqa
+    except ImportError:
+        msg = "python < 3.10 required 'typing_extensions' dependency"
+        raise ImportError(msg)
 """
 
 
@@ -36,7 +43,7 @@ def ssc_unescape(s: str) -> str:
     return s
 
     
-def ssc_map_replace(s: str, replacements: Dict[str, str]) -> str:
+def ssc_map_replace(s: str, replacements: dict[str, str]) -> str:
     return reduce(lambda acc, kv: acc.replace(kv[0], kv[1]), replacements.items(), s)