ssc_codegen 0.40.0__tar.gz → 0.42.0__tar.gz

{ssc_codegen-0.40.0 → ssc_codegen-0.42.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ssc_codegen
-Version: 0.40.0
+Version: 0.42.0
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues

{ssc_codegen-0.40.0 → ssc_codegen-0.42.0}/pyproject.toml

@@ -1,124 +1,124 @@
-[project]
-name = "ssc_codegen"
-version = "0.40.0"
-description = "Python-dsl code converter to html parser for web scraping "
-readme = "README.md"
-requires-python = ">=3.10"
-dependencies = [
-    "bs4>=0.0.2",
-    "colorama>=0.4.6 ; sys_platform == 'win32'",
-    "cssselect>=1.2.0",
-    "lxml>=5.3.0",
-    "soupsieve>=2.6",
-    "typer>0.26",
-    "typing_extensions;python_version<'3.11'",
-    "kdlquery>=1.3.0",
-    "httpx>=0.28.1",
-]
-
-classifiers = [
-    "Development Status :: 3 - Alpha",
-    "Environment :: Console",
-    "Intended Audience :: Developers",
-    "License :: OSI Approved :: MIT License",
-    "Topic :: Software Development :: Code Generators",
-    "Topic :: Text Processing :: Markup :: HTML",
-    "Topic :: Utilities",
-    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.12",
-    "Programming Language :: Python :: 3.13",
-]
-
-[project.urls]
-Documentation = "https://github.com/vypivshiy/selector_schema_codegen#readme"
-Issues = "https://github.com/vypivshiy/selector_schema_codegen/issues"
-Source = "https://github.com/vypivshiy/selector_schema_codegen"
-Examples = "https://github.com/vypivshiy/selector_schema_codegen/examples"
-
-
-[project.scripts]
-ssc-gen = 'ssc_codegen.main:main'
-
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
-[tool.hatch.build.targets.sdist]
-include = [
-  "ssc_codegen/**",
-]
-
-[tool.hatch.build.targets.wheel]
-packages = ["ssc_codegen"]
-
-[dependency-groups]
-dev = [
-    "aiohttp>=3.9,<3.12",  # aioresponses 0.7.x breaks on 3.12+ (stream_writer)
-    "aioresponses>=0.7",
-    "bs4>=0.0.2",
-    "coverage>=7.6.12",
-    "httpx>=0.28.1",
-    "hypothesis>=6.151.9",
-    "lxml>=5.3.0",
-    "mypy>=1.14.1",
-    "parsel>=1.9.1",
-    "pytest>=8.3.4",
-    "pytest-xdist>=3.6.0",
-    "respx>=0.23.1",
-    "responses>=0.25",
-    "ruff>=0.9.3",
-    "selectolax>=0.3.27",
-]
-docs = [
-    "mkdocs>=1.6.0",
-    "mkdocs-material>=9.5.0",
-    "mkdocstrings[python]>=0.27.0",
-    "mkdocs-click>=0.8.1",
-]
-
-[tool.ruff]
-target-version = "py310"
-line-length = 80
-exclude = ["__init__.py"]
-
-[tool.mypy]
-python_version = "3.10"
-show_error_codes = true
-[[tool.mypy.overrides]]
-module="tests.*"
-disallow_untyped_defs = false
-
-
-[[tool.mypy.overrides]]
-module = ["colorama", "lxml", "lxml.*"]
-ignore_missing_imports = true
-
-[tool.coverage.report]
-exclude_also = [
-    'def __repr__',
-    'if self.debug:',
-    'if settings.DEBUG',
-    'raise AssertionError',
-    'raise NotImplementedError',
-    'if 0:',
-    'if __name__ == .__main__.:',
-    'if TYPE_CHECKING:',
-    'class .*\bProtocol\):',
-    '@(abc\.)?abstractmethod',
-]
-
-[tool.coverage.run]
-source = ["ssc_codegen"]
-omit = [
-    "*/.venv/*",
-    "*/scripts/*",
-    "*/tests/*"
-]
-
-[tool.pytest.ini_options]
-testpaths = ["tests"]
-markers = [
-    "toolchain: Tests requiring external toolchains (Rust cargo, Go compiler, Node.js)",
-]
-
+[project]
+name = "ssc_codegen"
+version = "0.42.0"
+description = "Python-dsl code converter to html parser for web scraping "
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "bs4>=0.0.2",
+    "colorama>=0.4.6 ; sys_platform == 'win32'",
+    "cssselect>=1.2.0",
+    "lxml>=5.3.0",
+    "soupsieve>=2.6",
+    "typer>0.26",
+    "typing_extensions;python_version<'3.11'",
+    "kdlquery>=1.3.0",
+    "httpx>=0.28.1",
+]
+
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Topic :: Software Development :: Code Generators",
+    "Topic :: Text Processing :: Markup :: HTML",
+    "Topic :: Utilities",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+
+[project.urls]
+Documentation = "https://github.com/vypivshiy/selector_schema_codegen#readme"
+Issues = "https://github.com/vypivshiy/selector_schema_codegen/issues"
+Source = "https://github.com/vypivshiy/selector_schema_codegen"
+Examples = "https://github.com/vypivshiy/selector_schema_codegen/examples"
+
+
+[project.scripts]
+ssc-gen = 'ssc_codegen.main:main'
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "ssc_codegen/**",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["ssc_codegen"]
+
+[dependency-groups]
+dev = [
+    "aiohttp>=3.9,<3.12",  # aioresponses 0.7.x breaks on 3.12+ (stream_writer)
+    "aioresponses>=0.7",
+    "bs4>=0.0.2",
+    "coverage>=7.6.12",
+    "httpx>=0.28.1",
+    "hypothesis>=6.151.9",
+    "lxml>=5.3.0",
+    "mypy>=1.14.1",
+    "parsel>=1.9.1",
+    "pytest>=8.3.4",
+    "pytest-xdist>=3.6.0",
+    "respx>=0.23.1",
+    "responses>=0.25",
+    "ruff>=0.9.3",
+    "selectolax>=0.3.27",
+]
+docs = [
+    "mkdocs>=1.6.0",
+    "mkdocs-material>=9.5.0",
+    "mkdocstrings[python]>=0.27.0",
+    "mkdocs-click>=0.8.1",
+]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 80
+exclude = ["__init__.py"]
+
+[tool.mypy]
+python_version = "3.10"
+show_error_codes = true
+[[tool.mypy.overrides]]
+module="tests.*"
+disallow_untyped_defs = false
+
+
+[[tool.mypy.overrides]]
+module = ["colorama", "lxml", "lxml.*"]
+ignore_missing_imports = true
+
+[tool.coverage.report]
+exclude_also = [
+    'def __repr__',
+    'if self.debug:',
+    'if settings.DEBUG',
+    'raise AssertionError',
+    'raise NotImplementedError',
+    'if 0:',
+    'if __name__ == .__main__.:',
+    'if TYPE_CHECKING:',
+    'class .*\bProtocol\):',
+    '@(abc\.)?abstractmethod',
+]
+
+[tool.coverage.run]
+source = ["ssc_codegen"]
+omit = [
+    "*/.venv/*",
+    "*/scripts/*",
+    "*/tests/*"
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+    "toolchain: Tests requiring external toolchains (Rust cargo, Go compiler, Node.js)",
+]
+

{ssc_codegen-0.40.0 → ssc_codegen-0.42.0}/ssc_codegen/generation/builder.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import inspect
+
 
 class ModuleBuilder:
     """Language-agnostic accumulator for imports and helper definitions.
@@ -50,12 +52,17 @@ class ModuleBuilder:
     def require_import(self, line: str) -> None:
         """Register a top-level import line.
 
-        Deduplicated and order-preserving. The line string is stored as-is.
+        Deduplicated and order-preserving. Multi-line blocks are normalized
+        with `inspect.cleandoc` to remove uniform indentation.
 
         Args:
             line: The verbatim import statement line for the target language.
         """
-        self._imports.setdefault(line, None)
+        if "\n" in line:
+            cleaned = inspect.cleandoc(line)
+        else:
+            cleaned = line
+        self._imports.setdefault(cleaned, None)
 
     def require_std(
         self,

{ssc_codegen-0.40.0 → ssc_codegen-0.42.0}/ssc_codegen/generation/runtime.py

@@ -6,7 +6,6 @@ from collections.abc import Callable
 from typing import Any
 
 import ssc_codegen.ast as a
-
 from ssc_codegen.targets.python.http_libs.base import HttpLibStrategy
 from ssc_codegen.targets.python.http_libs.httpx import HttpxStrategy
 from ssc_codegen.traversal.utils import module_has_rest
@@ -439,7 +438,7 @@ def runtime_module_content(
     lines.extend(_BASE_UTILITY_LINES)
     lines.append("")
     if has_rest:
-        lines.extend(strategy.rest_runtime_lines())
+        lines.extend(strategy.rest_runtime_lines(http_io="both"))
     if extension_defs:
         for _imports, code in extension_defs.values():
             lines.append("")
@@ -485,10 +484,17 @@ def register_runtime_file(
 
     @converter.file(f"{runtime_name}.py")
     def _runtime_provider(module_ast: a.Module, meta):
+        strat = http_strategy
+        if (
+            meta
+            and meta.get("http_client")
+            and hasattr(converter, "http_strategy_for")
+        ):
+            strat = converter.http_strategy_for(meta.get("http_client"))
         return _apply_fallback(
             runtime_module_content(
                 module_ast,
-                http_strategy=http_strategy,
+                http_strategy=strat,
                 extension_defs=_extension_runtime_defs([module_ast]),
             )
         )

{ssc_codegen-0.40.0 → ssc_codegen-0.42.0}/ssc_codegen/main.py

@@ -152,6 +152,8 @@ def _run_generate(
     skip_lint: bool = False,
     verbose: bool = False,
     fmt: FmtType = FmtType.TEXT,
+    bs4_parser: Optional[str] = None,
+    http_io: Optional[str] = None,
 ) -> None:
     """Execute the shared compilation pipeline for all code generator subcommands.
 
@@ -216,6 +218,10 @@ def _run_generate(
     meta: dict = {"package": package or default_package}
     if http_client:
         meta["http_client"] = http_client
+    if bs4_parser:
+        meta["bs4_parser"] = bs4_parser
+    if http_io:
+        meta["http_io"] = http_io
 
     if separate_runtime:
         from ssc_codegen.generation.runtime import register_runtime_file
@@ -385,6 +391,16 @@ def generate_python(
             help="HTML parsing library. Default: bs4.",
         ),
     ] = None,
+    bs4_parser: Annotated[
+        Optional[str],
+        typer.Option(
+            "--bs4-parser",
+            help=(
+                "Underlying HTML parser engine for bs4: "
+                "lxml (default) | html.parser | html5lib."
+            ),
+        ),
+    ] = None,
     verbose: Annotated[
         bool,
         typer.Option(
@@ -413,10 +429,17 @@ def generate_python(
             "--http-client",
             help=(
                 "HTTP client for @request codegen: "
-                "httpx (default) | aiohttp | requests."
+                "httpx (default) | httpx2 | aiohttp | requests."
             ),
         ),
     ] = None,
+    http_io: Annotated[
+        Optional[str],
+        typer.Option(
+            "--http-io",
+            help="HTTP I/O mode for generated methods: both (default) | sync | async.",
+        ),
+    ] = None,
     separate_runtime: Annotated[
         bool,
         typer.Option(
@@ -452,6 +475,8 @@ def generate_python(
                 lib=lib.value if lib else None,
                 http_client=http_client,
                 separate_runtime=separate_runtime,
+                bs4_parser=bs4_parser,
+                http_io=http_io,
             )
         )
     except ResolutionError as exc:
@@ -474,6 +499,8 @@ def generate_python(
         skip_lint=skip_lint,
         verbose=verbose,
         fmt=fmt,
+        bs4_parser=bs4_parser,
+        http_io=http_io,
     )
 
 
@@ -886,6 +913,16 @@ def run(
             help="HTML parsing library. Default: bs4.",
         ),
     ] = None,
+    bs4_parser: Annotated[
+        Optional[str],
+        typer.Option(
+            "--bs4-parser",
+            help=(
+                "Underlying HTML parser engine for bs4: "
+                "lxml (default) | html.parser | html5lib."
+            ),
+        ),
+    ] = None,
     input_file: Annotated[
         Path | None,
         typer.Option(
@@ -984,6 +1021,7 @@ def run(
             TargetSpec(
                 lang="python",
                 lib=lib.value if lib else None,
+                bs4_parser=bs4_parser,
             )
         )
     except ResolutionError as exc:

{ssc_codegen-0.40.0 → ssc_codegen-0.42.0}/ssc_codegen/naming.py

@@ -22,6 +22,23 @@ def to_snake_case(s: str) -> str:
     return s.lower()
 
 
+def json_descriptor_var_name(name: str) -> str:
+    """Format canonical module-level JSON descriptor constant identifier.
+
+    Args:
+        name: Struct, type, or response schema name to format.
+
+    Returns:
+        Canonical uppercase identifier prefixed with ``JSON_DESCRIPTOR_``.
+
+    Examples:
+        'Episode'                 -> 'JSON_DESCRIPTOR_EPISODE'
+        'CatalogReleasesResponse' -> 'JSON_DESCRIPTOR_CATALOG_RELEASES_RESPONSE'
+    """
+    clean_name = to_snake_case(name).lstrip("_").upper()
+    return f"JSON_DESCRIPTOR_{clean_name}"
+
+
 def to_pascal_case(s: str) -> str:
     """
     'my_field_name' -> 'MyFieldName'

{ssc_codegen-0.40.0 → ssc_codegen-0.42.0}/ssc_codegen/targets/javascript/rest.py

@@ -8,14 +8,12 @@ Called by JsVisitor via thin delegate methods.  The HTTP library strategy
 from __future__ import annotations
 
 import json
-from typing import Mapping
 
 from ssc_codegen.ast import (
     JsonDef,
     MatcherListDef,
     MethodFetch,
     MethodRest,
-    Module,
     PlaceholderSpec,
     PlaceholderTemplate,
     ResultAliasDef,
@@ -23,18 +21,21 @@ from ssc_codegen.ast import (
     StructBase,
 )
 from ssc_codegen.ast.struct import RequestHttp
-from ssc_codegen.naming import to_camel_case, to_pascal_case, to_snake_case
+from ssc_codegen.naming import (
+    json_descriptor_var_name,
+    to_camel_case,
+    to_pascal_case,
+    to_snake_case,
+)
 from ssc_codegen.request_spec import parse_json_template
 from ssc_codegen.targets.javascript.http_libs.base import JsHttpLibStrategy
 from ssc_codegen.traversal.context import WalkContext
 from ssc_codegen.traversal.utils import (
     dict_needs_builder,
     err_subclass_name,
-    json_def_descriptors,
-    json_def_mapping,
+    find_enclosing_module,
 )
 
-
 # ===========================================================================
 # REST shared runtime source (Ok/Err/sscDispatchErr/etc.)
 # ===========================================================================
@@ -287,8 +288,7 @@ def js_name(name: str) -> str:
 def ok_payload_type(node: MethodRest) -> str:
     if not node.response_schema:
         return "null"
-    struct = node.parent
-    module = struct.parent if struct is not None else None
+    module = find_enclosing_module(node)
     schema_type = f"{to_pascal_case(node.response_schema)}Json"
     if module is not None:
         for n in module.body:
@@ -376,21 +376,28 @@ def emit_method_rest(
     )
 
     fn_name = http.fn_name
-    response_desc = _response_descriptors(node)
+    module = find_enclosing_module(node)
+    has_response_schema = False
+    if node.response_schema and module is not None:
+        has_response_schema = any(
+            isinstance(n, JsonDef) and n.name == node.response_schema
+            for n in module.body
+        )
+
     if node.response_path:
         accessor = "".join(
             f"[{json.dumps(p)}]" for p in node.response_path.split(".")
         )
-        if response_desc is None:
-            value_fn = f"(_b) => _b{accessor}"
+        if has_response_schema:
+            desc_var = json_descriptor_var_name(node.response_schema)
+            value_fn = f"(_b) => sscJsonProject(_b{accessor}, {desc_var})"
         else:
-            value_fn = f"(_b) => sscJsonProject(_b{accessor}, {_render_descriptors(response_desc)})"
+            value_fn = f"(_b) => _b{accessor}"
     elif not node.response_schema:
         value_fn = "(_b) => null"
-    elif response_desc is not None:
-        value_fn = (
-            f"(_b) => sscJsonProject(_b, {_render_descriptors(response_desc)})"
-        )
+    elif has_response_schema:
+        desc_var = json_descriptor_var_name(node.response_schema)
+        value_fn = f"(_b) => sscJsonProject(_b, {desc_var})"
     else:
         value_fn = "null"
 
@@ -466,82 +473,6 @@ def emit_method_rest(
     return lines
 
 
-def _response_descriptors(
-    node: MethodRest,
-) -> dict[str, tuple[str, bool, bool, object]] | None:
-    if not node.response_schema:
-        return None
-    module = node.parent.parent if node.parent is not None else None
-    if not isinstance(module, Module):
-        return None
-    definitions = {n.name: n for n in module.body if isinstance(n, JsonDef)}
-    definition = definitions.get(node.response_schema)
-    if definition is None:
-        return None
-    return json_def_descriptors(definition, definitions)
-
-
-def _render_descriptors(descriptors: Mapping[str, object]) -> str:
-    def render(value: object) -> str:
-        if value is None:
-            return "null"
-        if isinstance(value, bool):
-            return "true" if value else "false"
-        if isinstance(value, str):
-            return json.dumps(value)
-        if isinstance(value, tuple):
-            return "[" + ", ".join(render(x) for x in value) + "]"
-        if isinstance(value, list):
-            return "[" + ", ".join(render(item) for item in value) + "]"
-        if isinstance(value, dict):
-            return (
-                "{"
-                + ", ".join(
-                    f"{json.dumps(key)}: {render(item)}"
-                    for key, item in value.items()
-                )
-                + "}"
-            )
-        raise TypeError(f"unsupported JSON descriptor value: {value!r}")
-
-    return render(descriptors)
-
-
-def _response_mapping(node: MethodRest) -> dict[str, object] | None:
-    if not node.response_schema:
-        return None
-    module = node.parent.parent if node.parent is not None else None
-    if not isinstance(module, Module):
-        return None
-    definitions = {n.name: n for n in module.body if isinstance(n, JsonDef)}
-    definition = definitions.get(node.response_schema)
-    if definition is None or not definition.has_alias_key:
-        return None
-    return json_def_mapping(definition, definitions)
-
-
-def _render_mapping(mapping: dict[str, object]) -> str:
-    def render(value: object) -> str:
-        if isinstance(value, str):
-            return json.dumps(value)
-        if isinstance(value, tuple):
-            return "[" + json.dumps(value[0]) + ", " + render(value[1]) + "]"
-        if isinstance(value, list):
-            return "[" + ", ".join(render(item) for item in value) + "]"
-        if isinstance(value, dict):
-            return (
-                "{"
-                + ", ".join(
-                    f"{json.dumps(key)}: {render(item)}"
-                    for key, item in value.items()
-                )
-                + "}"
-            )
-        raise TypeError(f"unsupported JSON mapping value: {value!r}")
-
-    return render(mapping)
-
-
 def emit_method_fetch(
     node: MethodFetch, ctx: WalkContext, http: JsHttpLibStrategy
 ) -> list[str]:
@@ -679,26 +610,21 @@ def emit_result_alias_def(node: ResultAliasDef) -> list[str]:
 def emit_matcher_list_def(node: MatcherListDef) -> list[str]:
     var = f"_{to_snake_case(node.struct_name)}Matchers"
     lines = [f"const {var} = ["]
-    module = (
-        node.parent
-        if isinstance(node.parent, Module)
-        else (node.parent.parent if node.parent is not None else None)
-    )
+    module = find_enclosing_module(node)
     definitions = (
         {n.name: n for n in module.body if isinstance(n, JsonDef)}
-        if isinstance(module, Module)
+        if module is not None
         else {}
     )
     for e in node.entries:
         check = render_js_condition_check(e.required_keys, e.conditions)
         check_arg = check if check else "null"
-        desc = None
-        definition = definitions.get(e.error_schema)
-        if definition is not None:
-            desc = _render_descriptors(
-                json_def_descriptors(definition, definitions)
-            )
-        value_expr = f"sscJsonProject(_b, {desc})" if desc is not None else "_b"
+        desc_var = None
+        if e.error_schema and e.error_schema in definitions:
+            desc_var = json_descriptor_var_name(e.error_schema)
+        value_expr = (
+            f"sscJsonProject(_b, {desc_var})" if desc_var is not None else "_b"
+        )
         lines.append(
             f"    {{ status: {e.status}, check: {check_arg}, "
             f"factory: (_s, _h, _b) => ({{ isOk: false, status: _s, "