ssc_codegen 0.40.0__tar.gz → 0.41.0__tar.gz

{ssc_codegen-0.40.0 → ssc_codegen-0.41.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ssc_codegen
-Version: 0.40.0
+Version: 0.41.0
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues

{ssc_codegen-0.40.0 → ssc_codegen-0.41.0}/pyproject.toml

@@ -1,124 +1,124 @@
-[project]
-name = "ssc_codegen"
-version = "0.40.0"
-description = "Python-dsl code converter to html parser for web scraping "
-readme = "README.md"
-requires-python = ">=3.10"
-dependencies = [
-    "bs4>=0.0.2",
-    "colorama>=0.4.6 ; sys_platform == 'win32'",
-    "cssselect>=1.2.0",
-    "lxml>=5.3.0",
-    "soupsieve>=2.6",
-    "typer>0.26",
-    "typing_extensions;python_version<'3.11'",
-    "kdlquery>=1.3.0",
-    "httpx>=0.28.1",
-]
-
-classifiers = [
-    "Development Status :: 3 - Alpha",
-    "Environment :: Console",
-    "Intended Audience :: Developers",
-    "License :: OSI Approved :: MIT License",
-    "Topic :: Software Development :: Code Generators",
-    "Topic :: Text Processing :: Markup :: HTML",
-    "Topic :: Utilities",
-    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.12",
-    "Programming Language :: Python :: 3.13",
-]
-
-[project.urls]
-Documentation = "https://github.com/vypivshiy/selector_schema_codegen#readme"
-Issues = "https://github.com/vypivshiy/selector_schema_codegen/issues"
-Source = "https://github.com/vypivshiy/selector_schema_codegen"
-Examples = "https://github.com/vypivshiy/selector_schema_codegen/examples"
-
-
-[project.scripts]
-ssc-gen = 'ssc_codegen.main:main'
-
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
-[tool.hatch.build.targets.sdist]
-include = [
-  "ssc_codegen/**",
-]
-
-[tool.hatch.build.targets.wheel]
-packages = ["ssc_codegen"]
-
-[dependency-groups]
-dev = [
-    "aiohttp>=3.9,<3.12",  # aioresponses 0.7.x breaks on 3.12+ (stream_writer)
-    "aioresponses>=0.7",
-    "bs4>=0.0.2",
-    "coverage>=7.6.12",
-    "httpx>=0.28.1",
-    "hypothesis>=6.151.9",
-    "lxml>=5.3.0",
-    "mypy>=1.14.1",
-    "parsel>=1.9.1",
-    "pytest>=8.3.4",
-    "pytest-xdist>=3.6.0",
-    "respx>=0.23.1",
-    "responses>=0.25",
-    "ruff>=0.9.3",
-    "selectolax>=0.3.27",
-]
-docs = [
-    "mkdocs>=1.6.0",
-    "mkdocs-material>=9.5.0",
-    "mkdocstrings[python]>=0.27.0",
-    "mkdocs-click>=0.8.1",
-]
-
-[tool.ruff]
-target-version = "py310"
-line-length = 80
-exclude = ["__init__.py"]
-
-[tool.mypy]
-python_version = "3.10"
-show_error_codes = true
-[[tool.mypy.overrides]]
-module="tests.*"
-disallow_untyped_defs = false
-
-
-[[tool.mypy.overrides]]
-module = ["colorama", "lxml", "lxml.*"]
-ignore_missing_imports = true
-
-[tool.coverage.report]
-exclude_also = [
-    'def __repr__',
-    'if self.debug:',
-    'if settings.DEBUG',
-    'raise AssertionError',
-    'raise NotImplementedError',
-    'if 0:',
-    'if __name__ == .__main__.:',
-    'if TYPE_CHECKING:',
-    'class .*\bProtocol\):',
-    '@(abc\.)?abstractmethod',
-]
-
-[tool.coverage.run]
-source = ["ssc_codegen"]
-omit = [
-    "*/.venv/*",
-    "*/scripts/*",
-    "*/tests/*"
-]
-
-[tool.pytest.ini_options]
-testpaths = ["tests"]
-markers = [
-    "toolchain: Tests requiring external toolchains (Rust cargo, Go compiler, Node.js)",
-]
-
+[project]
+name = "ssc_codegen"
+version = "0.41.0"
+description = "Python-dsl code converter to html parser for web scraping "
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "bs4>=0.0.2",
+    "colorama>=0.4.6 ; sys_platform == 'win32'",
+    "cssselect>=1.2.0",
+    "lxml>=5.3.0",
+    "soupsieve>=2.6",
+    "typer>0.26",
+    "typing_extensions;python_version<'3.11'",
+    "kdlquery>=1.3.0",
+    "httpx>=0.28.1",
+]
+
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Topic :: Software Development :: Code Generators",
+    "Topic :: Text Processing :: Markup :: HTML",
+    "Topic :: Utilities",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+
+[project.urls]
+Documentation = "https://github.com/vypivshiy/selector_schema_codegen#readme"
+Issues = "https://github.com/vypivshiy/selector_schema_codegen/issues"
+Source = "https://github.com/vypivshiy/selector_schema_codegen"
+Examples = "https://github.com/vypivshiy/selector_schema_codegen/examples"
+
+
+[project.scripts]
+ssc-gen = 'ssc_codegen.main:main'
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "ssc_codegen/**",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["ssc_codegen"]
+
+[dependency-groups]
+dev = [
+    "aiohttp>=3.9,<3.12",  # aioresponses 0.7.x breaks on 3.12+ (stream_writer)
+    "aioresponses>=0.7",
+    "bs4>=0.0.2",
+    "coverage>=7.6.12",
+    "httpx>=0.28.1",
+    "hypothesis>=6.151.9",
+    "lxml>=5.3.0",
+    "mypy>=1.14.1",
+    "parsel>=1.9.1",
+    "pytest>=8.3.4",
+    "pytest-xdist>=3.6.0",
+    "respx>=0.23.1",
+    "responses>=0.25",
+    "ruff>=0.9.3",
+    "selectolax>=0.3.27",
+]
+docs = [
+    "mkdocs>=1.6.0",
+    "mkdocs-material>=9.5.0",
+    "mkdocstrings[python]>=0.27.0",
+    "mkdocs-click>=0.8.1",
+]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 80
+exclude = ["__init__.py"]
+
+[tool.mypy]
+python_version = "3.10"
+show_error_codes = true
+[[tool.mypy.overrides]]
+module="tests.*"
+disallow_untyped_defs = false
+
+
+[[tool.mypy.overrides]]
+module = ["colorama", "lxml", "lxml.*"]
+ignore_missing_imports = true
+
+[tool.coverage.report]
+exclude_also = [
+    'def __repr__',
+    'if self.debug:',
+    'if settings.DEBUG',
+    'raise AssertionError',
+    'raise NotImplementedError',
+    'if 0:',
+    'if __name__ == .__main__.:',
+    'if TYPE_CHECKING:',
+    'class .*\bProtocol\):',
+    '@(abc\.)?abstractmethod',
+]
+
+[tool.coverage.run]
+source = ["ssc_codegen"]
+omit = [
+    "*/.venv/*",
+    "*/scripts/*",
+    "*/tests/*"
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+    "toolchain: Tests requiring external toolchains (Rust cargo, Go compiler, Node.js)",
+]
+

{ssc_codegen-0.40.0 → ssc_codegen-0.41.0}/ssc_codegen/generation/builder.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import inspect
+
 
 class ModuleBuilder:
     """Language-agnostic accumulator for imports and helper definitions.
@@ -50,12 +52,17 @@ class ModuleBuilder:
     def require_import(self, line: str) -> None:
         """Register a top-level import line.
 
-        Deduplicated and order-preserving. The line string is stored as-is.
+        Deduplicated and order-preserving. Multi-line blocks are normalized
+        with `inspect.cleandoc` to remove uniform indentation.
 
         Args:
             line: The verbatim import statement line for the target language.
         """
-        self._imports.setdefault(line, None)
+        if "\n" in line:
+            cleaned = inspect.cleandoc(line)
+        else:
+            cleaned = line
+        self._imports.setdefault(cleaned, None)
 
     def require_std(
         self,

{ssc_codegen-0.40.0 → ssc_codegen-0.41.0}/ssc_codegen/generation/runtime.py

@@ -6,7 +6,6 @@ from collections.abc import Callable
 from typing import Any
 
 import ssc_codegen.ast as a
-
 from ssc_codegen.targets.python.http_libs.base import HttpLibStrategy
 from ssc_codegen.targets.python.http_libs.httpx import HttpxStrategy
 from ssc_codegen.traversal.utils import module_has_rest
@@ -439,7 +438,7 @@ def runtime_module_content(
     lines.extend(_BASE_UTILITY_LINES)
     lines.append("")
     if has_rest:
-        lines.extend(strategy.rest_runtime_lines())
+        lines.extend(strategy.rest_runtime_lines(http_io="both"))
     if extension_defs:
         for _imports, code in extension_defs.values():
             lines.append("")
@@ -485,10 +484,17 @@ def register_runtime_file(
 
     @converter.file(f"{runtime_name}.py")
     def _runtime_provider(module_ast: a.Module, meta):
+        strat = http_strategy
+        if (
+            meta
+            and meta.get("http_client")
+            and hasattr(converter, "http_strategy_for")
+        ):
+            strat = converter.http_strategy_for(meta.get("http_client"))
         return _apply_fallback(
             runtime_module_content(
                 module_ast,
-                http_strategy=http_strategy,
+                http_strategy=strat,
                 extension_defs=_extension_runtime_defs([module_ast]),
             )
         )

{ssc_codegen-0.40.0 → ssc_codegen-0.41.0}/ssc_codegen/main.py

@@ -152,6 +152,8 @@ def _run_generate(
     skip_lint: bool = False,
     verbose: bool = False,
     fmt: FmtType = FmtType.TEXT,
+    bs4_parser: Optional[str] = None,
+    http_io: Optional[str] = None,
 ) -> None:
     """Execute the shared compilation pipeline for all code generator subcommands.
 
@@ -216,6 +218,10 @@ def _run_generate(
     meta: dict = {"package": package or default_package}
     if http_client:
         meta["http_client"] = http_client
+    if bs4_parser:
+        meta["bs4_parser"] = bs4_parser
+    if http_io:
+        meta["http_io"] = http_io
 
     if separate_runtime:
         from ssc_codegen.generation.runtime import register_runtime_file
@@ -385,6 +391,16 @@ def generate_python(
             help="HTML parsing library. Default: bs4.",
         ),
     ] = None,
+    bs4_parser: Annotated[
+        Optional[str],
+        typer.Option(
+            "--bs4-parser",
+            help=(
+                "Underlying HTML parser engine for bs4: "
+                "lxml (default) | html.parser | html5lib."
+            ),
+        ),
+    ] = None,
     verbose: Annotated[
         bool,
         typer.Option(
@@ -413,10 +429,17 @@ def generate_python(
             "--http-client",
             help=(
                 "HTTP client for @request codegen: "
-                "httpx (default) | aiohttp | requests."
+                "httpx (default) | httpx2 | aiohttp | requests."
             ),
         ),
     ] = None,
+    http_io: Annotated[
+        Optional[str],
+        typer.Option(
+            "--http-io",
+            help="HTTP I/O mode for generated methods: both (default) | sync | async.",
+        ),
+    ] = None,
     separate_runtime: Annotated[
         bool,
         typer.Option(
@@ -452,6 +475,8 @@ def generate_python(
                 lib=lib.value if lib else None,
                 http_client=http_client,
                 separate_runtime=separate_runtime,
+                bs4_parser=bs4_parser,
+                http_io=http_io,
             )
         )
     except ResolutionError as exc:
@@ -474,6 +499,8 @@ def generate_python(
         skip_lint=skip_lint,
         verbose=verbose,
         fmt=fmt,
+        bs4_parser=bs4_parser,
+        http_io=http_io,
     )
 
 
@@ -886,6 +913,16 @@ def run(
             help="HTML parsing library. Default: bs4.",
         ),
     ] = None,
+    bs4_parser: Annotated[
+        Optional[str],
+        typer.Option(
+            "--bs4-parser",
+            help=(
+                "Underlying HTML parser engine for bs4: "
+                "lxml (default) | html.parser | html5lib."
+            ),
+        ),
+    ] = None,
     input_file: Annotated[
         Path | None,
         typer.Option(
@@ -984,6 +1021,7 @@ def run(
             TargetSpec(
                 lang="python",
                 lib=lib.value if lib else None,
+                bs4_parser=bs4_parser,
             )
         )
     except ResolutionError as exc:

{ssc_codegen-0.40.0 → ssc_codegen-0.41.0}/ssc_codegen/targets/python/html_libs/base.py

@@ -29,8 +29,8 @@ from ssc_codegen.ast.selectors import (
     XpathSelect,
     XpathSelectAll,
 )
-from ssc_codegen.traversal.context import WalkContext as ConverterContext
 from ssc_codegen.generation.builder import ModuleBuilder
+from ssc_codegen.traversal.context import WalkContext as ConverterContext
 
 
 class DomSpelling(ABC):
@@ -54,6 +54,8 @@ class DomSpelling(ABC):
         document_type: Type annotation string for a single parsed HTML element/document.
         document_array_type: Type annotation string for a collection of elements.
         init_arg_type: Type annotation string accepted by `__init__` (e.g. `Union[str, ...]`).
+        init_extra_params: Optional keyword parameter signature fragment for `__init__`.
+        fn_extra_params: Optional keyword parameter signature fragment for function definitions.
         init_from_str_expr: Python expression parsing a string `document` into a DOM tree.
         extra_utilities: Module-level helper constants or definitions.
         supports_xpath: `True` if this DOM library supports XPath expressions natively.
@@ -78,6 +80,8 @@ class DomSpelling(ABC):
     document_type: str = "Any"
     document_array_type: str = "List[Any]"
     init_arg_type: str = "Any"
+    init_extra_params: str = ""
+    fn_extra_params: str = ""
     init_from_str_expr: str = "document"
     extra_utilities: tuple[str, ...] = ()
     supports_xpath: bool = False

{ssc_codegen-0.40.0 → ssc_codegen-0.41.0}/ssc_codegen/targets/python/html_libs/bs4.py

@@ -27,8 +27,9 @@ from ssc_codegen.ast.selectors import (
     XpathSelect,
     XpathSelectAll,
 )
-from ssc_codegen.traversal.context import WalkContext as ConverterContext
+from ssc_codegen.generation.builder import ModuleBuilder
 from ssc_codegen.targets.python.html_libs.base import DomSpelling
+from ssc_codegen.traversal.context import WalkContext as ConverterContext
 
 
 class Bs4DomSpelling(DomSpelling):
@@ -38,24 +39,71 @@ class Bs4DomSpelling(DomSpelling):
     (such as ``select_one``, ``select``, ``decompose``, ``get_text``, and tag attributes).
 
     Attributes:
-        parser_imports: Imports ``from bs4 import BeautifulSoup, ResultSet, Tag``.
+        parser_imports: Imports ``from bs4 import BeautifulSoup, ResultSet, Tag`` and ``Literal``.
         document_type: Type annotation ``Union[Tag, BeautifulSoup]``.
         document_array_type: Type annotation ``ResultSet[Tag]``.
         init_arg_type: Accepted input type ``Union[str, BeautifulSoup, Tag]``.
-        init_from_str_expr: Expression ``BeautifulSoup(document, features=BS4_FEATURES)``.
-        extra_utilities: Defines ``BS4_FEATURES = 'lxml'``.
+        init_extra_params: Keyword argument signature with default ``features: Literal[...] = BS4_FEATURES``.
+        fn_extra_params: Function keyword argument signature with default ``features: Literal[...] = BS4_FEATURES``.
+        init_from_str_expr: Expression ``BeautifulSoup(document, features=features)``.
+        extra_utilities: Defines ``BS4_FEATURES: Literal["html.parser", "lxml", "html5lib"] = ...``.
         supports_xpath: `False` (BeautifulSoup4 does not support XPath natively).
     """
 
     # === DATA ===
-    parser_imports = ("from bs4 import BeautifulSoup, ResultSet, Tag",)
+    parser_imports = (
+        "from bs4 import BeautifulSoup, ResultSet, Tag",
+        "from typing import Literal",
+    )
     document_type = "Union[Tag, BeautifulSoup]"
     document_array_type = "ResultSet[Tag]"
     init_arg_type = "Union[str, BeautifulSoup, Tag]"
-    init_from_str_expr = "BeautifulSoup(document, features=BS4_FEATURES)"
-    extra_utilities = ("BS4_FEATURES = 'lxml'", "")
+    init_extra_params = ', *, features: Literal["html.parser", "lxml", "html5lib"] = BS4_FEATURES'
+    fn_extra_params = ', *, features: Literal["html.parser", "lxml", "html5lib"] = BS4_FEATURES'
+    init_from_str_expr = "BeautifulSoup(document, features=features)"
+    extra_utilities: tuple[str, ...] = (
+        'BS4_FEATURES: Literal["html.parser", "lxml", "html5lib"] = \'lxml\'',
+        "",
+    )
     supports_xpath = False
 
+    def __init__(self, builder: ModuleBuilder, parser: str = "lxml") -> None:
+        """Initialize the BeautifulSoup4 DOM spelling instance.
+
+        Args:
+            builder: Module builder used for registering imports and runtime helpers.
+            parser: Underlying HTML parser feature engine ("lxml", "html.parser",
+                or "html5lib"). Defaults to "lxml".
+        """
+        super().__init__(builder)
+        self._parser = parser
+        self.extra_utilities = (
+            f'BS4_FEATURES: Literal["html.parser", "lxml", "html5lib"] = {parser!r}',
+            "",
+        )
+
+    @property
+    def parser(self) -> str:
+        """Get the underlying HTML parser feature engine.
+
+        Returns:
+            Configured HTML parser engine name ("lxml", "html.parser", or "html5lib").
+        """
+        return self._parser
+
+    @parser.setter
+    def parser(self, value: str) -> None:
+        """Set the underlying HTML parser feature engine and update module constants.
+
+        Args:
+            value: HTML parser engine name ("lxml", "html.parser", or "html5lib").
+        """
+        self._parser = value
+        self.extra_utilities = (
+            f'BS4_FEATURES: Literal["html.parser", "lxml", "html5lib"] = {value!r}',
+            "",
+        )
+
     # === EXPRESSIONS ===
 
     def css_select(self, ctx: ConverterContext, node: CssSelect) -> list[str]:
@@ -167,8 +215,10 @@ class Bs4DomSpelling(DomSpelling):
         if node.accept_type_info.is_array:
             return [f"{ctx.indent}{ctx.nxt} = len({ctx.prv}) > 0"]
         return [
-            f"{ctx.indent}{ctx.nxt} = not ({ctx.prv} is None or {ctx.prv} == '' "
-            f"or (type({ctx.prv}) is int and {ctx.prv} == 0))"
+            (
+                f"{ctx.indent}{ctx.nxt} = not ({ctx.prv} is None or {ctx.prv} == '' "
+                f"or (type({ctx.prv}) is int and {ctx.prv} == 0))"
+            )
         ]
 
     # === PREDICATES ===