ssc_codegen 0.33.0__tar.gz → 0.34.0__tar.gz

{ssc_codegen-0.33.0 → ssc_codegen-0.34.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.33.0
+Version: 0.34.0
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues

{ssc_codegen-0.33.0 → ssc_codegen-0.34.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.33.0"
+version = "0.34.0"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.33.0 → ssc_codegen-0.34.0}/ssc_codegen/ast/__init__.py

@@ -20,6 +20,8 @@ from .module import (
     Utilities,
 )
 
+from .function import FunctionDef
+
 from .typedef import TypeDef, TypeDefField
 
 from .jsondef import JsonDef, JsonDefField
@@ -147,6 +149,8 @@ __all__ = [
     "CodeEndHook",
     "Docstring",
     "Utilities",
+    # function
+    "FunctionDef",
     # typedef
     "TypeDef",
     "TypeDefField",

ssc_codegen-0.34.0/ssc_codegen/ast/function.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from .base import Node
+from .types import TypeInfo, VariableType
+
+
+@dataclass
+class FunctionDef(Node):
+    """Module-level single-value extraction function.
+
+    DSL: ``fn <name> { pipeline }`` or ``(raw)fn <name> { pipeline }``.
+
+    Generates a standalone function instead of a class.
+    ``is_raw`` selects the document type: HTML (DOCUMENT) or plain text (STRING).
+    ``ret_type_info`` is resolved after pipeline build (AUTO → concrete type).
+    """
+
+    name: str = ""
+    is_raw: bool = False
+    doc: str = ""
+    accept_type_info: TypeInfo = field(
+        default_factory=lambda: TypeInfo(base=VariableType.DOCUMENT)
+    )
+    ret_type_info: TypeInfo = field(
+        default_factory=lambda: TypeInfo(base=VariableType.AUTO)
+    )

{ssc_codegen-0.33.0 → ssc_codegen-0.34.0}/ssc_codegen/core/expressions.py

@@ -19,6 +19,7 @@ from ssc_codegen.ast import (
     Field,
     Filter,
     Fmt,
+    FunctionDef,
     Index,
     InitField,
     JsonDef,
@@ -104,6 +105,7 @@ FieldLikeNode: TypeAlias = (
     | Value
     | Field
     | InitField
+    | FunctionDef
 )
 
 
@@ -480,8 +482,8 @@ def parse_expressions(
         )
         parent.ret_type_info = last_ti
 
-        # Populate type_info on Field/Value/InitField
-        if isinstance(parent, (Field, Value, InitField)):
+        # Populate type_info on Field/Value/InitField/FunctionDef
+        if isinstance(parent, (Field, Value, InitField, FunctionDef)):
             last_ret = last_ti.base
             is_arr = (
                 parent.body[-2].is_array if len(parent.body) >= 2 else False

{ssc_codegen-0.33.0 → ssc_codegen-0.34.0}/ssc_codegen/core/linter.py

@@ -168,6 +168,7 @@ def lint_module(
     _lint_defines(doc, source_path, diags, children_defines)
     _lint_json_defs(doc, source_path, diags, children_defines)
     _lint_structs(doc, source_path, diags, children_defines)
+    _lint_fns(doc, source_path, diags)
     _lint_nested_topdown(doc, source_path, diags)
     _lint_json_refs_topdown(doc, source_path, diags, children_defines)
     return diags
@@ -178,7 +179,7 @@ def _lint_top_level(
     source_path: str,
     diags: list[ReadDiagnostic],
 ) -> None:
-    for node in doc.select(":root:not(@doc, json, struct, define, import)"):
+    for node in doc.select(":root:not(@doc, json, struct, fn, define, import)"):
         diags.append(
             _error(
                 node,
@@ -378,6 +379,69 @@ def _lint_structs(
         _lint_single_struct(node, source_path, diags, children_defines)
 
 
+# Directives allowed inside ``fn`` / ``(raw)fn`` body.
+_FN_ALLOWED_RESERVED: frozenset[str] = frozenset({"@doc"})
+
+
+def _lint_fns(
+    doc: KdlDocument,
+    source_path: str,
+    diags: list[ReadDiagnostic],
+) -> None:
+    """Structural lint for ``fn`` / ``(raw)fn`` directives."""
+    for node in doc.select("fn:root"):
+        name = _node_arg(node, 0)
+        if not name:
+            diags.append(
+                _error(
+                    node,
+                    "'fn' requires a name",
+                    source_path,
+                    code="E001",
+                    hint='example: fn page_title { css "h1" { text } }',
+                )
+            )
+            continue
+
+        annotation = node.type_annotation
+        if annotation and annotation != "(raw)":
+            diags.append(
+                _error(
+                    node,
+                    f"unsupported fn annotation '{annotation}' — only '(raw)' is recognized",
+                    source_path,
+                    code="E400",
+                    hint="use 'fn { ... }' (HTML) or '(raw)fn { ... }' (plain text)",
+                )
+            )
+
+        ops = list(node.children)
+        if not ops:
+            diags.append(
+                _error(
+                    node,
+                    f"fn '{name}' must contain at least one pipeline operation",
+                    source_path,
+                    code="E001",
+                    hint='example: fn title { css "h1" { text } }',
+                )
+            )
+
+        for child in ops:
+            if (
+                child.name.startswith("@")
+                and child.name not in _FN_ALLOWED_RESERVED
+            ):
+                diags.append(
+                    _error(
+                        child,
+                        f"'{child.name}' is not allowed inside fn — use a struct for init/check/request/etc.",
+                        source_path,
+                        code="E203",
+                    )
+                )
+
+
 def _lint_nested_topdown(
     doc: KdlDocument,
     source_path: str,

{ssc_codegen-0.33.0 → ssc_codegen-0.34.0}/ssc_codegen/core/module_handler.py

@@ -5,12 +5,15 @@ from __future__ import annotations
 from pathlib import Path
 
 from ssc_codegen.ast import (
+    FunctionDef,
     JsonDef,
     Module,
     StructBase,
     Struct,
     StructRest,
     StructType,
+    VariableType,
+    TypeInfo,
 )
 from ssc_codegen.exceptions import BuildTimeError
 from kdlquery import KdlNode, parse as kdl_parse
@@ -23,7 +26,11 @@ from ssc_codegen.core.contexts import (
     ParseContext,
 )
 from ssc_codegen.core.expressions import resolve_define_references
-from ssc_codegen.core.struct_parser import parse_json_fields, parse_struct
+from ssc_codegen.core.struct_parser import (
+    parse_function,
+    parse_json_fields,
+    parse_struct,
+)
 
 _KDL_TEXT_ENCODING = "utf-8-sig"
 
@@ -65,6 +72,28 @@ def handle_struct(
     return struct
 
 
+def handle_function(
+    node: KdlNode, module: Module, ctx: ParseContext, lint: LintContext
+) -> FunctionDef:
+    raw_annotation = node.type_annotation
+    is_raw = raw_annotation == "(raw)"
+    if not node.args:
+        raise BuildTimeError("'fn' requires a name")
+    name = str(node.args[0].value)
+    fn = FunctionDef(
+        parent=module,
+        name=name,
+        is_raw=is_raw,
+        accept_type_info=(
+            TypeInfo(base=VariableType.STRING)
+            if is_raw
+            else TypeInfo(base=VariableType.DOCUMENT)
+        ),
+    )
+    parse_function(node.children, fn, ctx, lint)
+    return fn
+
+
 def handle_json(
     node: KdlNode, module: Module, ctx: ParseContext, lint: LintContext
 ) -> JsonDef:

{ssc_codegen-0.33.0 → ssc_codegen-0.34.0}/ssc_codegen/core/reader.py

@@ -16,6 +16,7 @@ from ssc_codegen.core.expressions import typedef_from_struct
 from ssc_codegen.core.rest_artifacts import rest_artifacts_from_struct
 from ssc_codegen.core.module_handler import (
     handle_define,
+    handle_function,
     handle_json,
     handle_struct,
     resolve_imports,
@@ -68,6 +69,7 @@ def parse_module(
         module.source_file = source_path.name if source_path else ""
         structs: list = []
         typedefs: list = []
+        functions: list = []
 
         for node in top_nodes:
             lint.push(node.name)
@@ -79,6 +81,9 @@ def parse_module(
                 struct = handle_struct(node, module, ctx, lint)
                 typedefs.append(typedef_from_struct(struct, module))
                 structs.append(struct)
+            elif node.name == "fn":
+                fn = handle_function(node, module, ctx, lint)
+                functions.append(fn)
             elif node.name in ("define", "import"):
                 pass  # already handled
             else:
@@ -95,6 +100,7 @@ def parse_module(
             if isinstance(struct, StructRest):
                 module.body.extend(rest_artifacts_from_struct(struct, module))
             module.body.append(struct)
+        module.body.extend(functions)
 
         # merge lint diagnostics into walker diagnostics
         diagnostics.extend(lint.diagnostics)

{ssc_codegen-0.33.0 → ssc_codegen-0.34.0}/ssc_codegen/core/struct_parser.py

@@ -13,6 +13,7 @@ from ssc_codegen.ast import (
     CssSelectAll,
     ErrorResponse,
     Field,
+    FunctionDef,
     InitField,
     InitFieldCall,
     JsonDef,
@@ -313,6 +314,47 @@ def parse_struct(
     lint.walk_context = prev_ctx
 
 
+def parse_function(
+    kdl_nodes: Sequence[KdlNode],
+    fn: FunctionDef,
+    ctx: ParseContext,
+    lint: LintContext,
+) -> None:
+    """Parse the body of a ``fn`` / ``(raw)fn`` directive.
+
+    The body is a flat pipeline (same ops as a Field), optionally preceded
+    by ``@doc``. No struct-level directives (@init, @check, @split-doc, etc.)
+    are accepted — those are reported by the structural linter.
+    """
+    prev_ctx = lint.walk_context
+    lint.walk_context = WalkCtx.PIPELINE
+
+    pipeline_nodes: list[KdlNode] = []
+    for node in kdl_nodes:
+        if node.name == "@doc":
+            fn.doc = str(node.args[0].value)
+        elif node.name.startswith("@"):
+            continue  # structural linter already reported E203
+        else:
+            pipeline_nodes.append(node)
+
+    ops = pipeline_nodes
+    parse_expressions(ops, fn, ctx, lint)
+    if ops:
+        check_pipeline_types(
+            ops,
+            ctx,
+            lint,
+            start_type=VariableType.STRING
+            if fn.is_raw
+            else VariableType.DOCUMENT,
+        )
+    if fn.is_raw:
+        _lint_raw_forbidden_ops(fn, lint)
+
+    lint.walk_context = prev_ctx
+
+
 def parse_json_fields(
     nodes: Sequence[KdlNode], parent: JsonDef, ctx: ParseContext
 ) -> None:

{ssc_codegen-0.33.0 → ssc_codegen-0.34.0}/ssc_codegen/targets/golang/visitor.py

@@ -35,6 +35,7 @@ from ssc_codegen.ast import (
     Field,
     Filter,
     Fmt,
+    FunctionDef,
     Init,
     InitField,
     InitFieldCall,
@@ -136,8 +137,8 @@ from ssc_codegen.naming import (
 )
 from ssc_codegen.traversal.utils import (
     find_predicate_container,
+    module_has_html_struct,
     module_has_rest,
-    module_is_rest_only,
     module_uses_http,
 )
 from ssc_codegen.generation.builder import ModuleBuilder
@@ -439,10 +440,9 @@ class GoVisitor(BaseWalker):
                 lines.append(f"// {doc_line}" if doc_line else "//")
             lines.append("")
 
-        is_rest_only = module_is_rest_only(node)
         uses_http = module_uses_http(node)
 
-        if not is_rest_only:
+        if module_has_html_struct(node):
             self._builder.require_import('"github.com/PuerkitoBio/goquery"')
             self._builder.require_import('"strings"')
         if uses_http:
@@ -842,6 +842,33 @@ class GoVisitor(BaseWalker):
         lines.append("")
         return lines
 
+    def visit_function_def(
+        self, node: FunctionDef, ctx: WalkContext
+    ) -> list[str]:
+        name = to_pascal_case(node.name)
+        t_ret = self._resolve_type(node.ret_type_info)
+        inner = ctx.deeper()
+        lines: list[str] = []
+        if node.doc:
+            for doc_line in node.doc.splitlines():
+                lines.append(f"// {doc_line}")
+        lines.append(f"func {name}(document string) {t_ret} {{")
+        if node.is_raw:
+            lines.append(f"{inner.indent}{inner.var_name} := document")
+        else:
+            self._builder.require_import('"strings"')
+            lines.append(
+                f"{inner.indent}doc, err := goquery.NewDocumentFromReader(strings.NewReader(document))"
+            )
+            lines.append(f"{inner.indent}if err != nil {{")
+            lines.append(f"{inner.indent}\tpanic(err)")
+            lines.append(f"{inner.indent}}}")
+            lines.append(f"{inner.indent}{inner.var_name} := doc.Selection")
+        lines.extend(self.walk_children(node, ctx))
+        lines.append("}")
+        lines.append("")
+        return lines
+
     def visit_split_doc(self, node: SplitDoc, ctx: WalkContext) -> list[str]:
         struct = self._enclosing_struct_name(node)
         rcv = _receiver(struct)

{ssc_codegen-0.33.0 → ssc_codegen-0.34.0}/ssc_codegen/targets/javascript/visitor.py

@@ -31,6 +31,7 @@ from ssc_codegen.ast import (
     Field,
     Filter,
     Fmt,
+    FunctionDef,
     Init,
     InitField,
     InitFieldCall,
@@ -528,6 +529,31 @@ class JsVisitor(BaseWalker):
         lines.append("}")
         return lines
 
+    def visit_function_def(
+        self, node: FunctionDef, ctx: WalkContext
+    ) -> list[str]:
+        name = to_camel_case(node.name)
+        inner = ctx.deeper()
+        lines: list[str] = []
+        if node.doc:
+            lines.extend(_js_docblock(node.doc.splitlines()))
+        lines.append(f"function {name}(document) {{")
+        if node.is_raw:
+            lines.append(f"{inner.indent}let {inner.var_name} = document;")
+        else:
+            lines.append(f"{inner.indent}let {inner.var_name};")
+            lines.append(f"{inner.indent}if (typeof document === 'string') {{")
+            lines.append(
+                f"{inner.deeper().indent}{inner.var_name} = (new DOMParser()).parseFromString(document, 'text/html');"
+            )
+            lines.append(f"{inner.indent}}} else {{")
+            lines.append(f"{inner.deeper().indent}{inner.var_name} = document;")
+            lines.append(f"{inner.indent}}}")
+        lines.extend(self.walk_children(node, ctx))
+        lines.append("}")
+        lines.append("")
+        return lines
+
     def visit_split_doc(self, node: SplitDoc, ctx: WalkContext) -> list[str]:
         lines = [f"{ctx.indent}_splitDoc(v) {{"]
         lines.extend(self.walk_children(node, ctx))

{ssc_codegen-0.33.0 → ssc_codegen-0.34.0}/ssc_codegen/targets/python/visitor.py

@@ -28,6 +28,7 @@ from ssc_codegen.ast import (
     InitFieldCall,
     InitField,
     Field,
+    FunctionDef,
     PreValidate,
     CheckMethod,
     SplitDoc,
@@ -661,6 +662,31 @@ class PythonVisitor(BaseWalker):
         lines.extend(self.walk_children(node, ctx))
         return lines
 
+    def visit_function_def(
+        self, node: FunctionDef, ctx: WalkContext
+    ) -> list[str]:
+        name = to_snake_case(node.name)
+        t_ret = self._resolve_type(node.ret_type_info)
+        inner = ctx.deeper()
+        lines = [f"{ctx.indent}def {name}(document: str) -> {t_ret}:"]
+        if node.doc:
+            lines.append(f'{inner.indent}"""')
+            for doc_line in node.doc.splitlines():
+                lines.append(inner.indent + doc_line)
+            lines.append(f'{inner.indent}"""')
+        if node.is_raw:
+            lines.append(f"{inner.indent}{inner.var_name} = document")
+        else:
+            lines.append(f"{inner.indent}if isinstance(document, str):")
+            lines.append(
+                f"{inner.deeper().indent}{inner.var_name} = {self._dom.init_from_str_expr}"
+            )
+            lines.append(f"{inner.indent}else:")
+            lines.append(f"{inner.deeper().indent}{inner.var_name} = document")
+        lines.extend(self.walk_children(node, ctx))
+        lines.append("")
+        return lines
+
     def visit_split_doc(self, node: SplitDoc, ctx: WalkContext) -> list[str]:
         t_arg = self._resolve_type(node.accept_type_info)
         t_ret = self._resolve_type(node.ret_type_info)

{ssc_codegen-0.33.0 → ssc_codegen-0.34.0}/ssc_codegen/traversal/utils.py

@@ -6,6 +6,7 @@ from ssc_codegen.ast import (
     Assert,
     ErrorResponse,
     Filter,
+    FunctionDef,
     Match,
     MethodBase,
     Module,
@@ -49,13 +50,20 @@ def module_is_rest_only(module: Module) -> bool:
 
 
 def module_has_html_struct(module: Module) -> bool:
-    """True if the module has at least one HTML-parsing struct (not REST, not RAW)."""
+    """True if the module has at least one HTML-parsing struct or function.
+
+    RAW structs, (raw)fn, and REST structs are excluded — they don't need
+    an HTML parser backend (bs4/lxml/goquery/DOMParser).
+    """
     for n in module.body:
         if isinstance(n, StructRest):
             continue
         if isinstance(n, Struct):
             if n.type != StructType.RAW:
                 return True
+        if isinstance(n, FunctionDef):
+            if not n.is_raw:
+                return True
     return False
 
 

{ssc_codegen-0.33.0 → ssc_codegen-0.34.0}/ssc_codegen/traversal/walker.py

@@ -16,6 +16,7 @@ from ssc_codegen.ast import (
     InitFieldCall,
     InitField,
     Field,
+    FunctionDef,
     PreValidate,
     CheckMethod,
     SplitDoc,
@@ -115,6 +116,7 @@ _CONTAINER_NODES = (JsonDef, TypeDef, StructBase, Init)
 # Pipeline: index advances after each node (delegated to walk_pipeline).
 _PIPELINE_NODES = (
     Field,
+    FunctionDef,
     InitField,
     PreValidate,
     CheckMethod,
@@ -179,6 +181,8 @@ class BaseWalker:
         MethodFetch: "visit_method_fetch",
         MethodRest: "visit_method_rest",
         ErrorResponse: "visit_error_response",
+        # function
+        FunctionDef: "visit_function_def",
         # REST result artifacts
         ResultVariantDef: "visit_result_variant_def",
         ResultAliasDef: "visit_result_alias_def",