ssc_codegen 0.37.2__tar.gz → 0.38.0__tar.gz

{ssc_codegen-0.37.2 → ssc_codegen-0.38.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ssc_codegen
-Version: 0.37.2
+Version: 0.38.0
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
@@ -42,7 +42,8 @@ Code generator for web scraping parsers. Describe HTML extraction rules in a dec
 
 - Declarative DSL based on KDL 2.0 syntax
 - Static type checking and linting before code generation
-- Multiple output targets: Python (bs4, lxml, parsel, selectolax), JavaScript (DOM API)
+- Multiple output targets: Python (bs4, lxml, parsel, selectolax), JavaScript (DOM API), Go (goquery)
+- Typed user extensions with lazy target imports and runtime helpers
 - Struct types: `item`, `list`, `dict`, `table`, `flat`
 - LLM-friendly: system prompt + linter loop for AI-assisted schema generation
 

{ssc_codegen-0.37.2 → ssc_codegen-0.38.0}/README.md

@@ -10,7 +10,8 @@ Code generator for web scraping parsers. Describe HTML extraction rules in a dec
 
 - Declarative DSL based on KDL 2.0 syntax
 - Static type checking and linting before code generation
-- Multiple output targets: Python (bs4, lxml, parsel, selectolax), JavaScript (DOM API)
+- Multiple output targets: Python (bs4, lxml, parsel, selectolax), JavaScript (DOM API), Go (goquery)
+- Typed user extensions with lazy target imports and runtime helpers
 - Struct types: `item`, `list`, `dict`, `table`, `flat`
 - LLM-friendly: system prompt + linter loop for AI-assisted schema generation
 

{ssc_codegen-0.37.2 → ssc_codegen-0.38.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.37.2"
+version = "0.38.0"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.37.2 → ssc_codegen-0.38.0}/ssc_codegen/ast/__init__.py

@@ -20,7 +20,16 @@ from .module import (
     Utilities,
 )
 
-from .function import FunctionDef
+from .function import FunctionDef
+
+from .extension import (
+    ExtensionCall,
+    ExtensionDef,
+    ExtensionHelper,
+    ExtensionImport,
+    ExtensionTarget,
+    ExtensionType,
+)
 
 from .typedef import TypeDef, TypeDefField
 
@@ -150,7 +159,14 @@ __all__ = [
     "Docstring",
     "Utilities",
     # function
-    "FunctionDef",
+    "FunctionDef",
+    # extensions
+    "ExtensionCall",
+    "ExtensionDef",
+    "ExtensionHelper",
+    "ExtensionImport",
+    "ExtensionTarget",
+    "ExtensionType",
     # typedef
     "TypeDef",
     "TypeDefField",

ssc_codegen-0.38.0/ssc_codegen/ast/extension.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from .base import Node
+from .types import TypeInfo, VariableType
+
+
+@dataclass(frozen=True)
+class ExtensionType:
+    """Type pattern used by an extension operation signature."""
+
+    base: VariableType | None = None
+    generic: str | None = None
+    is_array: bool = False
+    is_optional: bool = False
+
+    def resolve(self, binding: TypeInfo | None = None) -> TypeInfo:
+        if self.generic:
+            if binding is None:
+                return TypeInfo(base=VariableType.AUTO)
+            return binding
+        return TypeInfo(
+            base=self.base or VariableType.AUTO,
+            is_array=self.is_array,
+            is_optional=self.is_optional,
+        )
+
+    def accepts(self, value: TypeInfo) -> bool:
+        if self.generic:
+            return True
+        return (
+            value.base in (self.base, VariableType.AUTO)
+            and value.is_array == self.is_array
+            and value.is_optional == self.is_optional
+        )
+
+
+@dataclass(frozen=True)
+class ExtensionImport:
+    value: str
+    alias: str = ""
+
+
+@dataclass(frozen=True)
+class ExtensionHelper:
+    name: str
+    source: str
+    imports: tuple[ExtensionImport, ...] = ()
+
+
+@dataclass(frozen=True)
+class ExtensionTarget:
+    language: str
+    emit: str
+    imports: tuple[ExtensionImport, ...] = ()
+    helpers: tuple[ExtensionHelper, ...] = ()
+
+
+@dataclass
+class ExtensionDef:
+    namespace: str
+    name: str
+    accept: ExtensionType
+    ret: ExtensionType
+    targets: dict[str, ExtensionTarget] = field(default_factory=dict)
+
+    @property
+    def qualified_name(self) -> str:
+        return f"{self.namespace}.{self.name}"
+
+    def resolve_return(self, input_type: TypeInfo) -> TypeInfo:
+        binding = input_type if self.accept.generic else None
+        return self.ret.resolve(binding)
+
+
+@dataclass
+class ExtensionCall(Node):
+    qualified_name: str = ""
+    definition: ExtensionDef | None = field(default=None, repr=False)

{ssc_codegen-0.37.2 → ssc_codegen-0.38.0}/ssc_codegen/ast/module.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 import warnings
 from dataclasses import dataclass
+from dataclasses import field
 
 from .base import Node
 
@@ -28,6 +29,7 @@ class Module(Node):
 
     doc: str = ""
     source_file: str = ""
+    extensions: dict[str, "ExtensionDef"] = field(default_factory=dict)
 
     def __post_init__(self):
         self.body.extend(
@@ -112,3 +114,6 @@ class Utilities(Node):
     """
 
     pass
+
+
+from .extension import ExtensionDef  # noqa: E402

{ssc_codegen-0.37.2 → ssc_codegen-0.38.0}/ssc_codegen/core/contexts.py

@@ -6,7 +6,7 @@ from dataclasses import dataclass, field
 from enum import Enum, auto
 from pathlib import Path
 
-from ssc_codegen.ast import StructBase, VariableType
+from ssc_codegen.ast import ExtensionDef, StructBase, VariableType
 from kdlquery import KdlNode, ReadDiagnostic, Severity
 
 
@@ -49,6 +49,7 @@ class ParseContext:
     children_defines: dict[str, list[KdlNode]] = field(default_factory=dict)
     structs: dict[str, StructBase] = field(default_factory=dict)
     json_defs: dict[str, JsonDef] = field(default_factory=dict)
+    extensions: dict[str, ExtensionDef] = field(default_factory=dict)
     source_path: Path | None = None
     node_source_paths: dict[int, Path] = field(default_factory=dict)
 
@@ -58,6 +59,7 @@ class ParseContext:
             | set(self.children_defines)
             | set(self.structs)
             | set(self.json_defs)
+            | set(self.extensions)
         )
 
 

{ssc_codegen-0.37.2 → ssc_codegen-0.38.0}/ssc_codegen/core/expressions.py

@@ -16,6 +16,7 @@ from ssc_codegen.ast import (
     CssSelect,
     CssSelectAll,
     Fallback,
+    ExtensionCall,
     Field,
     Filter,
     Fmt,
@@ -80,6 +81,7 @@ from ssc_codegen.regex_utils import normalize_regex_pattern
 from typing import cast
 
 from ssc_codegen.core.contexts import LintContext, ParseContext, WalkCtx
+from ssc_codegen.core.extensions import resolve_extension_call
 from ssc_codegen.core.linter import (
     lint_pipeline_op,
     lint_validate_css,
@@ -106,6 +108,7 @@ FieldLikeNode: TypeAlias = (
     | Field
     | InitField
     | FunctionDef
+    | CheckMethod
 )
 
 
@@ -343,6 +346,9 @@ def _build_expression(
             f"'self {ref_name}' syntax is no longer supported; use '@{ref_name}' instead"
         )
 
+    if name.startswith("!"):
+        return _expr_extension(node, parent, ctx, lint)
+
     # dispatch
     handler = _EXPRESSION_HANDLERS.get(name)
     if handler is None:
@@ -417,6 +423,13 @@ def parse_expressions(
             raise BuildTimeError(
                 f"'self {ref_name}' syntax is no longer supported; use '@{ref_name}' instead"
             )
+        if node.name.startswith("!"):
+            lint.push(node.name)
+            lint_pipeline_op(node, lint)
+            extension_expr = _expr_extension(node, parent, ctx, lint)
+            parent.body.append(extension_expr)
+            lint.pop()
+            continue
         handler = _EXPRESSION_HANDLERS.get(node.name)
         if handler is None:
             lint_wildcard_op(node, ctx, lint)
@@ -501,7 +514,10 @@ def parse_expressions(
                         break
             parent.is_array = is_arr
             parent.ret_type_info = TypeInfo(
-                base=last_ret, is_array=is_arr, ref=ref
+                base=last_ret,
+                is_array=is_arr,
+                is_optional=last_ti.is_optional,
+                ref=ref or last_ti.ref,
             )
 
     lint.walk_context = prev_ctx
@@ -537,6 +553,59 @@ def _prev_type_info(
     )
 
 
+def _expr_extension(
+    node: KdlNode,
+    parent: FieldLikeNode,
+    ctx: ParseContext,
+    lint: LintContext,
+) -> ExtensionCall:
+    ref = node.name[1:]
+    if node.args or node.properties or node.children:
+        lint.error(
+            node,
+            message=f"custom operation '!{ref}' does not accept entries",
+            code="E001",
+        )
+    definition = ctx.extensions.get(ref)
+    prev = _prev_type_info(parent)
+    if definition is None:
+        lint.error(
+            node,
+            message=f"unknown extension operation '{ref}'",
+            code="E300",
+            hint="import its extension namespace and check the operation name",
+        )
+        return ExtensionCall(
+            parent=parent,
+            qualified_name=ref,
+            accept_type_info=prev.ret_type_info,
+            ret_type_info=TypeInfo(base=VariableType.AUTO),
+        )
+    if not definition.accept.accepts(prev.ret_type_info):
+        expected = (
+            definition.accept.generic
+            or (definition.accept.base or VariableType.AUTO).name
+        )
+        lint.error(
+            node,
+            message=(
+                f"custom operation '{ref}' does not accept "
+                f"{prev.ret_type_info.base.name}; expected {expected}"
+            ),
+            code="E100",
+        )
+    ret_type = resolve_extension_call(definition, prev.ret_type_info)
+    return ExtensionCall(
+        parent=parent,
+        qualified_name=ref,
+        definition=definition,
+        accept_type_info=prev.ret_type_info,
+        ret_type_info=ret_type,
+        is_array=ret_type.is_array,
+        span=node.span,
+    )
+
+
 # Ops whose handlers access ``parent.body[-1]`` unconditionally and therefore
 # require a preceding node. Used by ``parse_expressions`` to emit a clean
 # E100 diagnostic instead of crashing with IndexError on an empty pipeline.

ssc_codegen-0.38.0/ssc_codegen/core/extensions.py

@@ -0,0 +1,144 @@
+"""Parsing and type resolution for user-defined pipeline extensions."""
+
+from __future__ import annotations
+
+from kdlquery import KdlNode
+
+from ssc_codegen.ast import (
+    ExtensionDef,
+    ExtensionHelper,
+    ExtensionImport,
+    ExtensionTarget,
+    ExtensionType,
+    Module,
+    TypeInfo,
+    VariableType,
+)
+from ssc_codegen.core.contexts import LintContext, ParseContext
+from ssc_codegen.exceptions import BuildTimeError
+
+_TYPE_NAMES: dict[str, VariableType] = {
+    "bool": VariableType.BOOL,
+    "doc": VariableType.DOCUMENT,
+    "float": VariableType.FLOAT,
+    "int": VariableType.INT,
+    "str": VariableType.STRING,
+}
+
+
+def parse_extension_type(value) -> ExtensionType:
+    name = str(value.value)
+    is_optional = name.endswith("?")
+    if is_optional:
+        name = name[:-1]
+    is_array = value.type_annotation == "(array)"
+    if name == "T":
+        if is_array or is_optional:
+            raise BuildTimeError("generic type T cannot have modifiers")
+        return ExtensionType(generic="T")
+    base = _TYPE_NAMES.get(name)
+    if base is None:
+        raise BuildTimeError(f"unknown extension signature type: {name}")
+    return ExtensionType(base=base, is_array=is_array, is_optional=is_optional)
+
+
+def resolve_extension_call(
+    definition: ExtensionDef, input_type: TypeInfo
+) -> TypeInfo:
+    if not definition.accept.accepts(input_type):
+        return TypeInfo(base=VariableType.AUTO)
+    return definition.resolve_return(input_type)
+
+
+def _parse_import(node: KdlNode) -> ExtensionImport:
+    return ExtensionImport(
+        value=str(node.args[0].value),
+        alias=str(node.get_prop("alias") or ""),
+    )
+
+
+def _parse_helper(node: KdlNode) -> ExtensionHelper:
+    name = str(node.args[0].value)
+    imports = tuple(
+        _parse_import(child)
+        for child in node.children
+        if child.name == "import" and child.args
+    )
+    source_node = next(
+        (child for child in node.children if child.name == "source"), None
+    )
+    source = (
+        str(source_node.args[0].value)
+        if source_node is not None and source_node.args
+        else ""
+    )
+    return ExtensionHelper(name=name, source=source, imports=imports)
+
+
+def _parse_target(node: KdlNode) -> ExtensionTarget:
+    imports = tuple(
+        _parse_import(child)
+        for child in node.children
+        if child.name == "import" and child.args
+    )
+    helpers = tuple(
+        _parse_helper(child)
+        for child in node.children
+        if child.name == "helper" and child.args
+    )
+    emit_node = next(
+        (child for child in node.children if child.name == "emit"), None
+    )
+    emit = (
+        str(emit_node.args[0].value)
+        if emit_node is not None and emit_node.args
+        else ""
+    )
+    return ExtensionTarget(
+        language=node.name,
+        emit=emit,
+        imports=imports,
+        helpers=helpers,
+    )
+
+
+def handle_extension(
+    node: KdlNode,
+    module: Module,
+    ctx: ParseContext,
+    lint: LintContext,
+) -> list[ExtensionDef]:
+    if not node.args:
+        raise BuildTimeError("'extension' requires a namespace")
+    namespace = str(node.args[0].value)
+    definitions: list[ExtensionDef] = []
+    for operation in node.children:
+        sig = next(
+            (child for child in operation.children if child.name == "sig"),
+            None,
+        )
+        if sig is None or len(sig.args) != 2:
+            continue
+        definition = ExtensionDef(
+            namespace=namespace,
+            name=operation.name,
+            accept=parse_extension_type(sig.args[0]),
+            ret=parse_extension_type(sig.args[1]),
+            targets={
+                child.name: _parse_target(child)
+                for child in operation.children
+                if child.name in ("py", "js", "go")
+            },
+        )
+        qualified_name = definition.qualified_name
+        if qualified_name in ctx.extensions:
+            lint.error(
+                operation,
+                message=f"duplicate extension operation '{qualified_name}'",
+                code="E402",
+            )
+            continue
+        ctx.extensions[qualified_name] = definition
+        module.extensions[qualified_name] = definition
+        definitions.append(definition)
+    return definitions