ssc_codegen 0.33.0__tar.gz → 0.35.0__tar.gz

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.33.0
+Version: 0.35.0
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.33.0"
+version = "0.35.0"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/ssc_codegen/ast/__init__.py

@@ -20,6 +20,8 @@ from .module import (
     Utilities,
 )
 
+from .function import FunctionDef
+
 from .typedef import TypeDef, TypeDefField
 
 from .jsondef import JsonDef, JsonDefField
@@ -147,6 +149,8 @@ __all__ = [
     "CodeEndHook",
     "Docstring",
     "Utilities",
+    # function
+    "FunctionDef",
     # typedef
     "TypeDef",
     "TypeDefField",

ssc_codegen-0.35.0/ssc_codegen/ast/function.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from .base import Node
+from .types import TypeInfo, VariableType
+
+
+@dataclass
+class FunctionDef(Node):
+    """Module-level single-value extraction function.
+
+    DSL: ``fn <name> { pipeline }`` or ``(raw)fn <name> { pipeline }``.
+
+    Generates a standalone function instead of a class.
+    ``is_raw`` selects the document type: HTML (DOCUMENT) or plain text (STRING).
+    ``ret_type_info`` is resolved after pipeline build (AUTO → concrete type).
+    """
+
+    name: str = ""
+    is_raw: bool = False
+    doc: str = ""
+    accept_type_info: TypeInfo = field(
+        default_factory=lambda: TypeInfo(base=VariableType.DOCUMENT)
+    )
+    ret_type_info: TypeInfo = field(
+        default_factory=lambda: TypeInfo(base=VariableType.AUTO)
+    )

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/ssc_codegen/core/expressions.py

@@ -19,6 +19,7 @@ from ssc_codegen.ast import (
     Field,
     Filter,
     Fmt,
+    FunctionDef,
     Index,
     InitField,
     JsonDef,
@@ -104,6 +105,7 @@ FieldLikeNode: TypeAlias = (
     | Value
     | Field
     | InitField
+    | FunctionDef
 )
 
 
@@ -480,8 +482,8 @@ def parse_expressions(
         )
         parent.ret_type_info = last_ti
 
-        # Populate type_info on Field/Value/InitField
-        if isinstance(parent, (Field, Value, InitField)):
+        # Populate type_info on Field/Value/InitField/FunctionDef
+        if isinstance(parent, (Field, Value, InitField, FunctionDef)):
             last_ret = last_ti.base
             is_arr = (
                 parent.body[-2].is_array if len(parent.body) >= 2 else False

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/ssc_codegen/core/linter.py

@@ -168,6 +168,7 @@ def lint_module(
     _lint_defines(doc, source_path, diags, children_defines)
     _lint_json_defs(doc, source_path, diags, children_defines)
     _lint_structs(doc, source_path, diags, children_defines)
+    _lint_fns(doc, source_path, diags)
     _lint_nested_topdown(doc, source_path, diags)
     _lint_json_refs_topdown(doc, source_path, diags, children_defines)
     return diags
@@ -178,7 +179,7 @@ def _lint_top_level(
     source_path: str,
     diags: list[ReadDiagnostic],
 ) -> None:
-    for node in doc.select(":root:not(@doc, json, struct, define, import)"):
+    for node in doc.select(":root:not(@doc, json, struct, fn, define, import)"):
         diags.append(
             _error(
                 node,
@@ -378,6 +379,69 @@ def _lint_structs(
         _lint_single_struct(node, source_path, diags, children_defines)
 
 
+# Directives allowed inside ``fn`` / ``(raw)fn`` body.
+_FN_ALLOWED_RESERVED: frozenset[str] = frozenset({"@doc"})
+
+
+def _lint_fns(
+    doc: KdlDocument,
+    source_path: str,
+    diags: list[ReadDiagnostic],
+) -> None:
+    """Structural lint for ``fn`` / ``(raw)fn`` directives."""
+    for node in doc.select("fn:root"):
+        name = _node_arg(node, 0)
+        if not name:
+            diags.append(
+                _error(
+                    node,
+                    "'fn' requires a name",
+                    source_path,
+                    code="E001",
+                    hint='example: fn page_title { css "h1" { text } }',
+                )
+            )
+            continue
+
+        annotation = node.type_annotation
+        if annotation and annotation != "(raw)":
+            diags.append(
+                _error(
+                    node,
+                    f"unsupported fn annotation '{annotation}' — only '(raw)' is recognized",
+                    source_path,
+                    code="E400",
+                    hint="use 'fn { ... }' (HTML) or '(raw)fn { ... }' (plain text)",
+                )
+            )
+
+        ops = list(node.children)
+        if not ops:
+            diags.append(
+                _error(
+                    node,
+                    f"fn '{name}' must contain at least one pipeline operation",
+                    source_path,
+                    code="E001",
+                    hint='example: fn title { css "h1" { text } }',
+                )
+            )
+
+        for child in ops:
+            if (
+                child.name.startswith("@")
+                and child.name not in _FN_ALLOWED_RESERVED
+            ):
+                diags.append(
+                    _error(
+                        child,
+                        f"'{child.name}' is not allowed inside fn — use a struct for init/check/request/etc.",
+                        source_path,
+                        code="E203",
+                    )
+                )
+
+
 def _lint_nested_topdown(
     doc: KdlDocument,
     source_path: str,

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/ssc_codegen/core/module_handler.py

@@ -5,12 +5,15 @@ from __future__ import annotations
 from pathlib import Path
 
 from ssc_codegen.ast import (
+    FunctionDef,
     JsonDef,
     Module,
     StructBase,
     Struct,
     StructRest,
     StructType,
+    VariableType,
+    TypeInfo,
 )
 from ssc_codegen.exceptions import BuildTimeError
 from kdlquery import KdlNode, parse as kdl_parse
@@ -23,7 +26,11 @@ from ssc_codegen.core.contexts import (
     ParseContext,
 )
 from ssc_codegen.core.expressions import resolve_define_references
-from ssc_codegen.core.struct_parser import parse_json_fields, parse_struct
+from ssc_codegen.core.struct_parser import (
+    parse_function,
+    parse_json_fields,
+    parse_struct,
+)
 
 _KDL_TEXT_ENCODING = "utf-8-sig"
 
@@ -65,6 +72,28 @@ def handle_struct(
     return struct
 
 
+def handle_function(
+    node: KdlNode, module: Module, ctx: ParseContext, lint: LintContext
+) -> FunctionDef:
+    raw_annotation = node.type_annotation
+    is_raw = raw_annotation == "(raw)"
+    if not node.args:
+        raise BuildTimeError("'fn' requires a name")
+    name = str(node.args[0].value)
+    fn = FunctionDef(
+        parent=module,
+        name=name,
+        is_raw=is_raw,
+        accept_type_info=(
+            TypeInfo(base=VariableType.STRING)
+            if is_raw
+            else TypeInfo(base=VariableType.DOCUMENT)
+        ),
+    )
+    parse_function(node.children, fn, ctx, lint)
+    return fn
+
+
 def handle_json(
     node: KdlNode, module: Module, ctx: ParseContext, lint: LintContext
 ) -> JsonDef:

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/ssc_codegen/core/reader.py

@@ -16,6 +16,7 @@ from ssc_codegen.core.expressions import typedef_from_struct
 from ssc_codegen.core.rest_artifacts import rest_artifacts_from_struct
 from ssc_codegen.core.module_handler import (
     handle_define,
+    handle_function,
     handle_json,
     handle_struct,
     resolve_imports,
@@ -68,6 +69,7 @@ def parse_module(
         module.source_file = source_path.name if source_path else ""
         structs: list = []
         typedefs: list = []
+        functions: list = []
 
         for node in top_nodes:
             lint.push(node.name)
@@ -79,6 +81,9 @@ def parse_module(
                 struct = handle_struct(node, module, ctx, lint)
                 typedefs.append(typedef_from_struct(struct, module))
                 structs.append(struct)
+            elif node.name == "fn":
+                fn = handle_function(node, module, ctx, lint)
+                functions.append(fn)
             elif node.name in ("define", "import"):
                 pass  # already handled
             else:
@@ -95,6 +100,7 @@ def parse_module(
             if isinstance(struct, StructRest):
                 module.body.extend(rest_artifacts_from_struct(struct, module))
             module.body.append(struct)
+        module.body.extend(functions)
 
         # merge lint diagnostics into walker diagnostics
         diagnostics.extend(lint.diagnostics)

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/ssc_codegen/core/struct_parser.py

@@ -13,6 +13,7 @@ from ssc_codegen.ast import (
     CssSelectAll,
     ErrorResponse,
     Field,
+    FunctionDef,
     InitField,
     InitFieldCall,
     JsonDef,
@@ -313,6 +314,47 @@ def parse_struct(
     lint.walk_context = prev_ctx
 
 
+def parse_function(
+    kdl_nodes: Sequence[KdlNode],
+    fn: FunctionDef,
+    ctx: ParseContext,
+    lint: LintContext,
+) -> None:
+    """Parse the body of a ``fn`` / ``(raw)fn`` directive.
+
+    The body is a flat pipeline (same ops as a Field), optionally preceded
+    by ``@doc``. No struct-level directives (@init, @check, @split-doc, etc.)
+    are accepted — those are reported by the structural linter.
+    """
+    prev_ctx = lint.walk_context
+    lint.walk_context = WalkCtx.PIPELINE
+
+    pipeline_nodes: list[KdlNode] = []
+    for node in kdl_nodes:
+        if node.name == "@doc":
+            fn.doc = str(node.args[0].value)
+        elif node.name.startswith("@"):
+            continue  # structural linter already reported E203
+        else:
+            pipeline_nodes.append(node)
+
+    ops = pipeline_nodes
+    parse_expressions(ops, fn, ctx, lint)
+    if ops:
+        check_pipeline_types(
+            ops,
+            ctx,
+            lint,
+            start_type=VariableType.STRING
+            if fn.is_raw
+            else VariableType.DOCUMENT,
+        )
+    if fn.is_raw:
+        _lint_raw_forbidden_ops(fn, lint)
+
+    lint.walk_context = prev_ctx
+
+
 def parse_json_fields(
     nodes: Sequence[KdlNode], parent: JsonDef, ctx: ParseContext
 ) -> None:

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/ssc_codegen/targets/golang/rest.py

@@ -206,7 +206,7 @@ def emit_method_fetch(
     i2 = i1 + ctx.indent_char
 
     lines: list[str] = [
-        f"{i1}func {method_name}(ctx context.Context, client {http.client_type}{ph_params}) (*{name}, error) {{",
+        f"{i1}func {method_name}(ctx context.Context, client {http.client_type}{ph_params}, opts ...sscReqOpt) (*{name}, error) {{",
         f"{i2}req, err := http.NewRequestWithContext(ctx, {_go_str(spec.method)}, {url_expr}, nil)",
         f"{i2}if err != nil {{",
         f"{i2}\treturn nil, err",
@@ -233,6 +233,15 @@ def emit_method_fetch(
             f"{i2}req.Body = io.NopCloser(strings.NewReader({body_expr}))"
         )
 
+    # Apply per-call opts (user headers are additive).
+    lines.append(f"{i2}if len(opts) > 0 {{")
+    lines.append(f"{i2}\t_opts := &sscReqOpts{{}}")
+    lines.append(f"{i2}\tfor _, o := range opts {{ o(_opts) }}")
+    lines.append(f"{i2}\tfor _k, _vs := range _opts.Headers {{")
+    lines.append(f"{i2}\t\tfor _, _v := range _vs {{ req.Header.Add(_k, _v) }}")
+    lines.append(f"{i2}\t}}")
+    lines.append(f"{i2}}}")
+
     # Execute.
     lines.extend(
         [
@@ -325,29 +334,28 @@ def emit_method_rest(
 
     # Function signature.
     lines.append(
-        f"{i1}func {method_name}(client {http.client_type}{ph_params}) ({ret_type}, error) {{"
+        f"{i1}func {method_name}(client {http.client_type}{ph_params}, opts ...sscReqOpt) ({ret_type}, error) {{"
     )
 
-    # Build opts.
-    opts_lines = _build_opts(spec, i2)
-
-    # Call sscRestCall — inline if nil, multi-line if struct literal.
-    # For void responses, use `_` since the body is discarded.
+    # Build _opts from DSL, apply per-call user opts, then call sscRestCall.
+    opts_lines = _build_opts(spec, i2 + "\t")
     body_var = "_" if not node.response_schema else "body"
-    if len(opts_lines) == 1:
-        lines.append(
-            f"{i2}{body_var}, err := sscRestCall(client, {matchers_var}, "
-            f"{_go_str(spec.method)}, {url_expr}, {opts_lines[0].strip()})"
-        )
+
+    # Assign DSL opts to _opts variable.
+    if len(opts_lines) == 1 and opts_lines[0].strip() == "nil":
+        lines.append(f"{i2}_opts := &sscReqOpts{{}}")
     else:
-        lines.append(
-            f"{i2}{body_var}, err := sscRestCall(client, {matchers_var}, "
-            f"{_go_str(spec.method)}, {url_expr},"
-        )
-        lines.append(f"{opts_lines[0].strip()}")
-        for ol in opts_lines[1:-1]:
+        lines.append(f"{i2}_opts := {opts_lines[0].strip()}")
+        for ol in opts_lines[1:]:
             lines.append(ol)
-        lines.append(f"{opts_lines[-1].strip()})")
+
+    # Apply per-call opts (functional options).
+    lines.append(f"{i2}for _, o := range opts {{ o(_opts) }}")
+
+    lines.append(
+        f"{i2}{body_var}, err := sscRestCall(client, {matchers_var}, "
+        f"{_go_str(spec.method)}, {url_expr}, _opts)"
+    )
 
     # Error propagation.
     lines.append(f"{i2}if err != nil {{")

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/ssc_codegen/targets/golang/runtime.py

@@ -493,6 +493,20 @@ BASE_REST_RUNTIME: list[str] = [
     "\tBody    string",
     "}",
     "",
+    "// sscReqOpt is a functional option for per-call request overrides.",
+    "// Applied after DSL-specified opts; user headers are additive (shallow-merge).",
+    "type sscReqOpt func(*sscReqOpts)",
+    "",
+    "// WithHeader adds a per-call header value (additive — does not replace DSL headers).",
+    "func WithHeader(key, val string) sscReqOpt {",
+    "\treturn func(o *sscReqOpts) {",
+    "\t\tif o.Headers == nil {",
+    "\t\t\to.Headers = sscHeaders{}",
+    "\t\t}",
+    "\t\to.Headers[key] = append(o.Headers[key], val)",
+    "\t}",
+    "}",
+    "",
     "type sscErrMatcher struct {",
     "\tStatus  int",
     "\tCheck   func([]byte) bool",

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/ssc_codegen/targets/golang/visitor.py

@@ -35,6 +35,7 @@ from ssc_codegen.ast import (
     Field,
     Filter,
     Fmt,
+    FunctionDef,
     Init,
     InitField,
     InitFieldCall,
@@ -136,8 +137,8 @@ from ssc_codegen.naming import (
 )
 from ssc_codegen.traversal.utils import (
     find_predicate_container,
+    module_has_html_struct,
     module_has_rest,
-    module_is_rest_only,
     module_uses_http,
 )
 from ssc_codegen.generation.builder import ModuleBuilder
@@ -439,10 +440,9 @@ class GoVisitor(BaseWalker):
                 lines.append(f"// {doc_line}" if doc_line else "//")
             lines.append("")
 
-        is_rest_only = module_is_rest_only(node)
         uses_http = module_uses_http(node)
 
-        if not is_rest_only:
+        if module_has_html_struct(node):
             self._builder.require_import('"github.com/PuerkitoBio/goquery"')
             self._builder.require_import('"strings"')
         if uses_http:
@@ -842,6 +842,33 @@ class GoVisitor(BaseWalker):
         lines.append("")
         return lines
 
+    def visit_function_def(
+        self, node: FunctionDef, ctx: WalkContext
+    ) -> list[str]:
+        name = to_pascal_case(node.name)
+        t_ret = self._resolve_type(node.ret_type_info)
+        inner = ctx.deeper()
+        lines: list[str] = []
+        if node.doc:
+            for doc_line in node.doc.splitlines():
+                lines.append(f"// {doc_line}")
+        lines.append(f"func {name}(document string) {t_ret} {{")
+        if node.is_raw:
+            lines.append(f"{inner.indent}{inner.var_name} := document")
+        else:
+            self._builder.require_import('"strings"')
+            lines.append(
+                f"{inner.indent}doc, err := goquery.NewDocumentFromReader(strings.NewReader(document))"
+            )
+            lines.append(f"{inner.indent}if err != nil {{")
+            lines.append(f"{inner.indent}\tpanic(err)")
+            lines.append(f"{inner.indent}}}")
+            lines.append(f"{inner.indent}{inner.var_name} := doc.Selection")
+        lines.extend(self.walk_children(node, ctx))
+        lines.append("}")
+        lines.append("")
+        return lines
+
     def visit_split_doc(self, node: SplitDoc, ctx: WalkContext) -> list[str]:
         struct = self._enclosing_struct_name(node)
         rcv = _receiver(struct)

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/ssc_codegen/targets/javascript/rest.py

@@ -390,13 +390,22 @@ def emit_method_rest(
             t = f"{t}[]"
         bracket = f"[params.{p.name}]" if p.is_optional else f"params.{p.name}"
         lines.append(f"{i1} * @param {{{t}}} {bracket}")
+    lines.append(f"{i1} * @param {{Object}} [opts] per-call request options (headers, etc.)")
     lines.append(f"{i1} * @returns {{Promise<{return_union}>}}")
     lines.append(f"{i1} */")
-    lines.append(f"{i1}static async {method_name}(client{ph_param}) {{")
+    lines.append(f"{i1}static async {method_name}(client{ph_param}, opts = {{}}) {{")
     lines.extend(pre_lines)
+    lines.append(f"{i2}const _kw = {opts_obj};")
+    lines.append(f"{i2}for (const [_k, _v] of Object.entries(opts)) {{")
+    lines.append(f"{i2}    if (typeof _kw[_k] === 'object' && _kw[_k] !== null && typeof _v === 'object' && _v !== null) {{")
+    lines.append(f"{i2}        _kw[_k] = {{..._kw[_k], ..._v}};")
+    lines.append(f"{i2}    }} else {{")
+    lines.append(f"{i2}        _kw[_k] = _v;")
+    lines.append(f"{i2}    }}")
+    lines.append(f"{i2}}}")
     lines.append(
         f"{i2}return {fn_name}(client, {matchers_var}, "
-        f"{spec.method!r}, {url_expr}, {value_fn}, {opts_obj});"
+        f"{spec.method!r}, {url_expr}, {value_fn}, _kw);"
     )
     lines.append(f"{i1}}}")
     return lines
@@ -434,9 +443,11 @@ def emit_method_fetch(
         rl.append(f"{i2}return new {struct_name}(_body);")
         return rl
 
-    lines: list[str] = [f"{i1}static async {method_name}(client{ph_param}) {{"]
+    lines: list[str] = [f"{i1}static async {method_name}(client{ph_param}, opts = {{}}) {{"]
     lines.extend(pre_lines)
 
+    # Build _kw from DSL-specified request options (method/url stay out).
+    kw_parts: list[str] = []
     if http_client == "fetch":
         if params_expr:
             url_inner = spec.url.map(
@@ -446,21 +457,30 @@ def emit_method_fetch(
             url_expr = f"`{url_inner}?${{new URLSearchParams({params_expr})}}`"
         else:
             url_expr = render_value(spec.url)
-        options: list[str] = [f"{i3}method: {spec.method!r},"]
         if headers_expr:
-            options.append(f"{i3}headers: {headers_expr},")
+            kw_parts.append(f"headers: {headers_expr}")
         if spec.cookies:
             cookie_str = "; ".join(
                 f"{k}={v.source}" for k, v in spec.cookies.items()
             )
-            options.append(
-                f"{i3}// cookies: {cookie_str!r}  /* set via headers or credentials */"
+            kw_parts.append(
+                f"// cookies: {cookie_str!r}  /* set via headers or credentials */"
             )
         if body_expr:
-            options.append(f"{i3}body: {body_expr},")
-        lines.append(f"{i2}const _resp = await client({url_expr}, {{")
-        lines.extend(options)
-        lines.append(f"{i2}}});")
+            kw_parts.append(f"body: {body_expr}")
+        kw_obj = "{" + ", ".join(kw_parts) + "}" if kw_parts else "{}"
+        lines.append(f"{i2}const _kw = {kw_obj};")
+        lines.append(f"{i2}for (const [_k, _v] of Object.entries(opts)) {{")
+        lines.append(f"{i2}    if (typeof _kw[_k] === 'object' && _kw[_k] !== null && typeof _v === 'object' && _v !== null) {{")
+        lines.append(f"{i2}        _kw[_k] = {{..._kw[_k], ..._v}};")
+        lines.append(f"{i2}    }} else {{")
+        lines.append(f"{i2}        _kw[_k] = _v;")
+        lines.append(f"{i2}    }}")
+        lines.append(f"{i2}}}")
+        lines.append(
+            f"{i2}const _resp = await client({url_expr},"
+            f" {{ method: {spec.method!r}, ..._kw }});"
+        )
         lines.append(
             f"{i2}if (!_resp.ok) throw new Error(`HTTP ${{_resp.status}}`);"
         )
@@ -469,23 +489,30 @@ def emit_method_fetch(
         else:
             lines.extend(_response_lines("await _resp.text()"))
     else:
-        req_props: list[str] = [
-            f"{i3}method: {spec.method!r},",
-            f"{i3}url: {render_value(spec.url)},",
-        ]
+        url_expr = render_value(spec.url)
         if params_expr:
-            req_props.append(f"{i3}params: {params_expr},")
+            kw_parts.append(f"params: {params_expr}")
         if headers_expr:
-            req_props.append(f"{i3}headers: {headers_expr},")
+            kw_parts.append(f"headers: {headers_expr}")
         if spec.cookies:
-            req_props.append(
-                f"{i3}// cookies: {render_obj(spec.cookies)},",
+            kw_parts.append(
+                f"// cookies: {render_obj(spec.cookies)},"
             )
         if body_expr:
-            req_props.append(f"{i3}data: {body_expr},")
-        lines.append(f"{i2}const _resp = await client.request({{")
-        lines.extend(req_props)
-        lines.append(f"{i2}}});")
+            kw_parts.append(f"data: {body_expr}")
+        kw_obj = "{" + ", ".join(kw_parts) + "}" if kw_parts else "{}"
+        lines.append(f"{i2}const _kw = {kw_obj};")
+        lines.append(f"{i2}for (const [_k, _v] of Object.entries(opts)) {{")
+        lines.append(f"{i2}    if (typeof _kw[_k] === 'object' && _kw[_k] !== null && typeof _v === 'object' && _v !== null) {{")
+        lines.append(f"{i2}        _kw[_k] = {{..._kw[_k], ..._v}};")
+        lines.append(f"{i2}    }} else {{")
+        lines.append(f"{i2}        _kw[_k] = _v;")
+        lines.append(f"{i2}    }}")
+        lines.append(f"{i2}}}")
+        lines.append(
+            f"{i2}const _resp = await client.request({{"
+            f" method: {spec.method!r}, url: {url_expr}, ..._kw }});"
+        )
         lines.extend(_response_lines("_resp.data"))
 
     lines.append(f"{i1}}}")

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/ssc_codegen/targets/javascript/visitor.py

@@ -31,6 +31,7 @@ from ssc_codegen.ast import (
     Field,
     Filter,
     Fmt,
+    FunctionDef,
     Init,
     InitField,
     InitFieldCall,
@@ -528,6 +529,31 @@ class JsVisitor(BaseWalker):
         lines.append("}")
         return lines
 
+    def visit_function_def(
+        self, node: FunctionDef, ctx: WalkContext
+    ) -> list[str]:
+        name = to_camel_case(node.name)
+        inner = ctx.deeper()
+        lines: list[str] = []
+        if node.doc:
+            lines.extend(_js_docblock(node.doc.splitlines()))
+        lines.append(f"function {name}(document) {{")
+        if node.is_raw:
+            lines.append(f"{inner.indent}let {inner.var_name} = document;")
+        else:
+            lines.append(f"{inner.indent}let {inner.var_name};")
+            lines.append(f"{inner.indent}if (typeof document === 'string') {{")
+            lines.append(
+                f"{inner.deeper().indent}{inner.var_name} = (new DOMParser()).parseFromString(document, 'text/html');"
+            )
+            lines.append(f"{inner.indent}}} else {{")
+            lines.append(f"{inner.deeper().indent}{inner.var_name} = document;")
+            lines.append(f"{inner.indent}}}")
+        lines.extend(self.walk_children(node, ctx))
+        lines.append("}")
+        lines.append("")
+        return lines
+
     def visit_split_doc(self, node: SplitDoc, ctx: WalkContext) -> list[str]:
         lines = [f"{ctx.indent}_splitDoc(v) {{"]
         lines.extend(self.walk_children(node, ctx))

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/ssc_codegen/targets/python/rest.py

@@ -296,32 +296,44 @@ def placeholder_args(http: RequestHttp) -> str:
 # ===========================================================================
 
 
-def _build_kwargs(
-    spec: RequestHttp, i2: str, i3: str, include_method_url: bool
+def _build_kw_dict(
+    spec: RequestHttp, i2: str, i3: str
 ) -> tuple[list[str], list[str]]:
-    """Build (pre_lines, kwargs_lines) for request call.
+    """Build (pre_lines, kw_assign_lines) for ``_kw`` dict construction.
 
-    include_method_url=True for fetch (client.request(method, url, ...)).
-    include_method_url=False for rest (ssc_rest_call(client, matchers, method, url, ...)).
+    Produces ``_kw["headers"] = {...}`` style assignments so caller kwargs
+    can be shallow-merged into the same dict before passing ``**_kw`` to
+    ``client.request()`` / ``ssc_rest_call()``.
+
+    pre_lines:   builder helper code (complex dicts needing intermediate vars).
+    kw_lines:    ``_kw["key"] = value`` assignment lines (at i2 indent).
     """
     pre_lines: list[str] = []
-    kwargs_lines: list[str] = []
-    if include_method_url:
-        kwargs_lines.append(f"{i3}{spec.method!r},")
-        kwargs_lines.append(f"{i3}{render_value(spec.url)},")
+    kw_lines: list[str] = []
     for attr, varname in _DICT_KWARGS:
         d = getattr(spec, attr)
         if not d:
             continue
         if dict_needs_builder(d):
             pre_lines.extend(emit_dict_builder(varname, d, i2))
-            kwargs_lines.append(f"{i3}{attr}={varname},")
+            kw_lines.append(f'{i2}_kw["{attr}"] = {varname}')
         else:
-            kwargs_lines.append(f"{i3}{attr}={render_dict(d)},")
+            kw_lines.append(f'{i2}_kw["{attr}"] = {render_dict(d)}')
     body_result = render_body(spec)
     if body_result:
-        kwargs_lines.append(f"{i3}{body_result[0]}={body_result[1]},")
-    return pre_lines, kwargs_lines
+        kw_lines.append(f'{i2}_kw["{body_result[0]}"] = {body_result[1]}')
+    return pre_lines, kw_lines
+
+
+def _merge_loop_lines(i2: str) -> list[str]:
+    """Shallow-merge user kwargs into _kw (dict keys merged key-by-key)."""
+    return [
+        f"{i2}for _k, _v in kwargs.items():",
+        f"{i2}    if isinstance(_kw.get(_k), dict) and isinstance(_v, dict):",
+        f"{i2}        _kw[_k] = {{**_kw[_k], **_v}}",
+        f"{i2}    else:",
+        f"{i2}        _kw[_k] = _v",
+    ]
 
 
 # ===========================================================================
@@ -343,9 +355,22 @@ def emit_method_fetch(
     i2 = i1 + ctx.indent_char
     i3 = i2 + ctx.indent_char
 
-    pre_lines, kwargs_lines = _build_kwargs(
-        spec, i2, i3, include_method_url=True
-    )
+    pre_lines, kw_lines = _build_kw_dict(spec, i2, i3)
+
+    # kwargs_lines for fetch_body_lines: method, url, **_kw
+    fetch_kwargs = [
+        f"{i3}{spec.method!r},",
+        f"{i3}{render_value(spec.url)},",
+        f"{i3}**_kw,",
+    ]
+
+    def _kw_setup() -> list[str]:
+        """Lines to build _kw dict + merge user kwargs before the request."""
+        setup: list[str] = [f"{i2}_kw: Dict[str, Any] = {{}}"]
+        setup.extend(pre_lines)
+        setup.extend(kw_lines)
+        setup.extend(_merge_loop_lines(i2))
+        return setup
 
     lines: list[str] = []
 
@@ -353,14 +378,14 @@ def emit_method_fetch(
     if http.supports_sync_fetch:
         lines.append(f"{i1}@classmethod")
         lines.append(
-            f'{i1}def fetch{suffix}(cls, client: {http.sync_client_type}{ph_params}) -> "{struct_name}":'
+            f'{i1}def fetch{suffix}(cls, client: {http.sync_client_type}{ph_params}, **kwargs: Any) -> "{struct_name}":'
         )
-        lines.extend(pre_lines)
+        lines.extend(_kw_setup())
         lines.extend(
             http.fetch_body_lines(
                 is_async=False,
                 request_call=f"{i2}_resp = client.request(",
-                kwargs_lines=kwargs_lines,
+                kwargs_lines=fetch_kwargs,
                 response_path=node.response_path,
                 response_join=node.response_join,
                 i2=i2,
@@ -372,24 +397,27 @@ def emit_method_fetch(
     # --- async_fetch ------------------------------------------------------
     lines.append(f"{i1}@classmethod")
     lines.append(
-        f'{i1}async def async_fetch{suffix}(cls, client: {http.async_client_type}{ph_params}) -> "{struct_name}":'
+        f'{i1}async def async_fetch{suffix}(cls, client: {http.async_client_type}{ph_params}, **kwargs: Any) -> "{struct_name}":'
     )
     if http.async_fetch_delegates_to_sync:
         # requests: no native async — run the sync fetch in a worker thread
         # via asyncio.to_thread (non-blocking, yields control to the loop).
-        to_thread_args = f"client, {ph_kwargs}" if ph_kwargs else "client"
+        if ph_kwargs:
+            to_thread_args = f"client, {ph_kwargs}, **kwargs"
+        else:
+            to_thread_args = "client, **kwargs"
         lines.append(f"{i2}import asyncio")
         lines.append(
             f"{i2}return await asyncio.to_thread("
             f"cls.fetch{suffix}, {to_thread_args})"
         )
     else:
-        lines.extend(pre_lines)
+        lines.extend(_kw_setup())
         lines.extend(
             http.fetch_body_lines(
                 is_async=True,
                 request_call=f"{i2}_resp = await client.request(",
-                kwargs_lines=kwargs_lines,
+                kwargs_lines=fetch_kwargs,
                 response_path=node.response_path,
                 response_join=node.response_join,
                 i2=i2,
@@ -416,9 +444,7 @@ def emit_method_rest(
 
     doc_line = f'{i2}"""{node.doc}"""' if node.doc else None
 
-    pre_lines, kwargs_lines = _build_kwargs(
-        spec, i2, i3, include_method_url=False
-    )
+    pre_lines, kw_lines = _build_kw_dict(spec, i2, i3)
 
     value_kwarg: list[str] = []
     if node.response_path:
@@ -431,36 +457,31 @@ def emit_method_rest(
         body: list[str] = []
         if doc_line:
             body.append(doc_line)
+        body.append(f"{i2}_kw: Dict[str, Any] = {{}}")
         body.extend(pre_lines)
-        # Wrap in cast(): ssc_rest_call returns Union[Ok[_T], Err] (Err
-        # base — it cannot know the specific Err subclasses that the
-        # heterogeneous matchers list will produce at runtime). The
-        # parser-declared Result alias narrows to the precise union
-        # (Err400 | UnknownErr | TransportErr | ...). cast() documents
-        # this intent at the call site without suppressing mypy via
-        # ``# type: ignore``; the wrapping method's return type is the
-        # stable monad consumers see.
+        body.extend(kw_lines)
+        body.extend(_merge_loop_lines(i2))
         body.append(f"{i2}return cast({ret_type}, {await_kw}{fn_name}(")
         body.append(
             f"{i3}client, {matchers_var}, {spec.method!r},"
             f" {render_value(spec.url)},"
         )
         body.extend(value_kwarg)
-        body.extend(kwargs_lines)
+        body.append(f"{i3}**_kw,")
         body.append(f"{i2}))")
         return body
 
     lines: list[str] = []
     lines.append(f"{i1}@classmethod")
     lines.append(
-        f"{i1}def {method_name}(cls, client: {http.sync_client_type}{ph_params}) -> {ret_type}:"
+        f"{i1}def {method_name}(cls, client: {http.sync_client_type}{ph_params}, **kwargs: Any) -> {ret_type}:"
     )
     lines.extend(_body("ssc_rest_call", ""))
     lines.append("")
 
     lines.append(f"{i1}@classmethod")
     lines.append(
-        f"{i1}async def async_{method_name}(cls, client: {http.async_client_type}{ph_params}) -> {ret_type}:"
+        f"{i1}async def async_{method_name}(cls, client: {http.async_client_type}{ph_params}, **kwargs: Any) -> {ret_type}:"
     )
     lines.extend(_body("ssc_rest_call_async", "await "))
     return lines

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/ssc_codegen/targets/python/visitor.py

@@ -28,6 +28,7 @@ from ssc_codegen.ast import (
     InitFieldCall,
     InitField,
     Field,
+    FunctionDef,
     PreValidate,
     CheckMethod,
     SplitDoc,
@@ -661,6 +662,31 @@ class PythonVisitor(BaseWalker):
         lines.extend(self.walk_children(node, ctx))
         return lines
 
+    def visit_function_def(
+        self, node: FunctionDef, ctx: WalkContext
+    ) -> list[str]:
+        name = to_snake_case(node.name)
+        t_ret = self._resolve_type(node.ret_type_info)
+        inner = ctx.deeper()
+        lines = [f"{ctx.indent}def {name}(document: str) -> {t_ret}:"]
+        if node.doc:
+            lines.append(f'{inner.indent}"""')
+            for doc_line in node.doc.splitlines():
+                lines.append(inner.indent + doc_line)
+            lines.append(f'{inner.indent}"""')
+        if node.is_raw:
+            lines.append(f"{inner.indent}{inner.var_name} = document")
+        else:
+            lines.append(f"{inner.indent}if isinstance(document, str):")
+            lines.append(
+                f"{inner.deeper().indent}{inner.var_name} = {self._dom.init_from_str_expr}"
+            )
+            lines.append(f"{inner.indent}else:")
+            lines.append(f"{inner.deeper().indent}{inner.var_name} = document")
+        lines.extend(self.walk_children(node, ctx))
+        lines.append("")
+        return lines
+
     def visit_split_doc(self, node: SplitDoc, ctx: WalkContext) -> list[str]:
         t_arg = self._resolve_type(node.accept_type_info)
         t_ret = self._resolve_type(node.ret_type_info)

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/ssc_codegen/traversal/utils.py

@@ -6,6 +6,7 @@ from ssc_codegen.ast import (
     Assert,
     ErrorResponse,
     Filter,
+    FunctionDef,
     Match,
     MethodBase,
     Module,
@@ -49,13 +50,20 @@ def module_is_rest_only(module: Module) -> bool:
 
 
 def module_has_html_struct(module: Module) -> bool:
-    """True if the module has at least one HTML-parsing struct (not REST, not RAW)."""
+    """True if the module has at least one HTML-parsing struct or function.
+
+    RAW structs, (raw)fn, and REST structs are excluded — they don't need
+    an HTML parser backend (bs4/lxml/goquery/DOMParser).
+    """
     for n in module.body:
         if isinstance(n, StructRest):
             continue
         if isinstance(n, Struct):
             if n.type != StructType.RAW:
                 return True
+        if isinstance(n, FunctionDef):
+            if not n.is_raw:
+                return True
     return False
 
 

{ssc_codegen-0.33.0 → ssc_codegen-0.35.0}/ssc_codegen/traversal/walker.py

@@ -16,6 +16,7 @@ from ssc_codegen.ast import (
     InitFieldCall,
     InitField,
     Field,
+    FunctionDef,
     PreValidate,
     CheckMethod,
     SplitDoc,
@@ -115,6 +116,7 @@ _CONTAINER_NODES = (JsonDef, TypeDef, StructBase, Init)
 # Pipeline: index advances after each node (delegated to walk_pipeline).
 _PIPELINE_NODES = (
     Field,
+    FunctionDef,
     InitField,
     PreValidate,
     CheckMethod,
@@ -179,6 +181,8 @@ class BaseWalker:
         MethodFetch: "visit_method_fetch",
         MethodRest: "visit_method_rest",
         ErrorResponse: "visit_error_response",
+        # function
+        FunctionDef: "visit_function_def",
         # REST result artifacts
         ResultVariantDef: "visit_result_variant_def",
         ResultAliasDef: "visit_result_alias_def",