ssc_codegen 0.37.1__tar.gz → 0.38.0__tar.gz

{ssc_codegen-0.37.1 → ssc_codegen-0.38.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ssc_codegen
-Version: 0.37.1
+Version: 0.38.0
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
@@ -42,7 +42,8 @@ Code generator for web scraping parsers. Describe HTML extraction rules in a dec
 
 - Declarative DSL based on KDL 2.0 syntax
 - Static type checking and linting before code generation
-- Multiple output targets: Python (bs4, lxml, parsel, selectolax), JavaScript (DOM API)
+- Multiple output targets: Python (bs4, lxml, parsel, selectolax), JavaScript (DOM API), Go (goquery)
+- Typed user extensions with lazy target imports and runtime helpers
 - Struct types: `item`, `list`, `dict`, `table`, `flat`
 - LLM-friendly: system prompt + linter loop for AI-assisted schema generation
 
@@ -86,6 +87,9 @@ ssc-gen generate js examples/ -o ./output
 # Go (goquery + net/http)
 ssc-gen generate go schema.kdl -o ./output
 
+# Go package defaults to main; override explicitly when generating a library
+ssc-gen generate go schema.kdl -o ./output --package scraper
+
 # with custom package name
 ssc-gen generate python schema.kdl -L bs4 -o ./parsers --package scraper
 
@@ -116,12 +120,14 @@ ssc-gen check examples/
 
 ```bash
 # from file
-ssc-gen run examples/booksToScrape.kdl:MainCatalogue -l python -L bs4 -i page.html
+ssc-gen run examples/booksToScrape.kdl:MainCatalogue -L bs4 -i page.html
 
 # from stdin
-curl https://books.toscrape.com/ | ssc-gen run examples/booksToScrape.kdl:MainCatalogue -l python -L bs4
+curl https://books.toscrape.com/ | ssc-gen run examples/booksToScrape.kdl:MainCatalogue -L bs4
 ```
 
+`ssc-gen run` executes generated Python in-process. Run only trusted schema files.
+
 ### Health check (verify selectors match elements)
 
 ```bash

{ssc_codegen-0.37.1 → ssc_codegen-0.38.0}/README.md

@@ -10,7 +10,8 @@ Code generator for web scraping parsers. Describe HTML extraction rules in a dec
 
 - Declarative DSL based on KDL 2.0 syntax
 - Static type checking and linting before code generation
-- Multiple output targets: Python (bs4, lxml, parsel, selectolax), JavaScript (DOM API)
+- Multiple output targets: Python (bs4, lxml, parsel, selectolax), JavaScript (DOM API), Go (goquery)
+- Typed user extensions with lazy target imports and runtime helpers
 - Struct types: `item`, `list`, `dict`, `table`, `flat`
 - LLM-friendly: system prompt + linter loop for AI-assisted schema generation
 
@@ -51,8 +52,11 @@ ssc-gen generate python schema.kdl -L bs4 -o ./output
 # all .kdl files in a directory (JavaScript)
 ssc-gen generate js examples/ -o ./output
 
-# Go (goquery + net/http)
-ssc-gen generate go schema.kdl -o ./output
+# Go (goquery + net/http)
+ssc-gen generate go schema.kdl -o ./output
+
+# Go package defaults to main; override explicitly when generating a library
+ssc-gen generate go schema.kdl -o ./output --package scraper
 
 # with custom package name
 ssc-gen generate python schema.kdl -L bs4 -o ./parsers --package scraper
@@ -84,11 +88,13 @@ ssc-gen check examples/
 
 ```bash
 # from file
-ssc-gen run examples/booksToScrape.kdl:MainCatalogue -l python -L bs4 -i page.html
+ssc-gen run examples/booksToScrape.kdl:MainCatalogue -L bs4 -i page.html
 
 # from stdin
-curl https://books.toscrape.com/ | ssc-gen run examples/booksToScrape.kdl:MainCatalogue -l python -L bs4
-```
+curl https://books.toscrape.com/ | ssc-gen run examples/booksToScrape.kdl:MainCatalogue -L bs4
+```
+
+`ssc-gen run` executes generated Python in-process. Run only trusted schema files.
 
 ### Health check (verify selectors match elements)
 

{ssc_codegen-0.37.1 → ssc_codegen-0.38.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.37.1"
+version = "0.38.0"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"
@@ -78,10 +78,6 @@ exclude = ["__init__.py"]
 [tool.mypy]
 python_version = "3.10"
 show_error_codes = true
-exclude = [
-    "ssc_codegen/ast"  # ignore built-in ast and project ast
-
-]
 [[tool.mypy.overrides]]
 module="tests.*"
 disallow_untyped_defs = false
@@ -91,12 +87,6 @@ disallow_untyped_defs = false
 module = ["colorama", "lxml", "lxml.*"]
 ignore_missing_imports = true
 
-[[tool.mypy.overrides]]
-# Large visitor classes with match statements may need relaxed return checks
-module = ["ssc_codegen.targets.python.visitor", "ssc_codegen.targets.javascript.visitor"]
-disable_error_code = ["return", "return-value"]
-
-
 [tool.coverage.report]
 exclude_also = [
     'def __repr__',

{ssc_codegen-0.37.1 → ssc_codegen-0.38.0}/ssc_codegen/ast/__init__.py

@@ -20,7 +20,16 @@ from .module import (
     Utilities,
 )
 
-from .function import FunctionDef
+from .function import FunctionDef
+
+from .extension import (
+    ExtensionCall,
+    ExtensionDef,
+    ExtensionHelper,
+    ExtensionImport,
+    ExtensionTarget,
+    ExtensionType,
+)
 
 from .typedef import TypeDef, TypeDefField
 
@@ -150,7 +159,14 @@ __all__ = [
     "Docstring",
     "Utilities",
     # function
-    "FunctionDef",
+    "FunctionDef",
+    # extensions
+    "ExtensionCall",
+    "ExtensionDef",
+    "ExtensionHelper",
+    "ExtensionImport",
+    "ExtensionTarget",
+    "ExtensionType",
     # typedef
     "TypeDef",
     "TypeDefField",

ssc_codegen-0.38.0/ssc_codegen/ast/extension.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from .base import Node
+from .types import TypeInfo, VariableType
+
+
+@dataclass(frozen=True)
+class ExtensionType:
+    """Type pattern used by an extension operation signature."""
+
+    base: VariableType | None = None
+    generic: str | None = None
+    is_array: bool = False
+    is_optional: bool = False
+
+    def resolve(self, binding: TypeInfo | None = None) -> TypeInfo:
+        if self.generic:
+            if binding is None:
+                return TypeInfo(base=VariableType.AUTO)
+            return binding
+        return TypeInfo(
+            base=self.base or VariableType.AUTO,
+            is_array=self.is_array,
+            is_optional=self.is_optional,
+        )
+
+    def accepts(self, value: TypeInfo) -> bool:
+        if self.generic:
+            return True
+        return (
+            value.base in (self.base, VariableType.AUTO)
+            and value.is_array == self.is_array
+            and value.is_optional == self.is_optional
+        )
+
+
+@dataclass(frozen=True)
+class ExtensionImport:
+    value: str
+    alias: str = ""
+
+
+@dataclass(frozen=True)
+class ExtensionHelper:
+    name: str
+    source: str
+    imports: tuple[ExtensionImport, ...] = ()
+
+
+@dataclass(frozen=True)
+class ExtensionTarget:
+    language: str
+    emit: str
+    imports: tuple[ExtensionImport, ...] = ()
+    helpers: tuple[ExtensionHelper, ...] = ()
+
+
+@dataclass
+class ExtensionDef:
+    namespace: str
+    name: str
+    accept: ExtensionType
+    ret: ExtensionType
+    targets: dict[str, ExtensionTarget] = field(default_factory=dict)
+
+    @property
+    def qualified_name(self) -> str:
+        return f"{self.namespace}.{self.name}"
+
+    def resolve_return(self, input_type: TypeInfo) -> TypeInfo:
+        binding = input_type if self.accept.generic else None
+        return self.ret.resolve(binding)
+
+
+@dataclass
+class ExtensionCall(Node):
+    qualified_name: str = ""
+    definition: ExtensionDef | None = field(default=None, repr=False)

{ssc_codegen-0.37.1 → ssc_codegen-0.38.0}/ssc_codegen/ast/module.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 import warnings
 from dataclasses import dataclass
+from dataclasses import field
 
 from .base import Node
 
@@ -28,6 +29,7 @@ class Module(Node):
 
     doc: str = ""
     source_file: str = ""
+    extensions: dict[str, "ExtensionDef"] = field(default_factory=dict)
 
     def __post_init__(self):
         self.body.extend(
@@ -112,3 +114,6 @@ class Utilities(Node):
     """
 
     pass
+
+
+from .extension import ExtensionDef  # noqa: E402

{ssc_codegen-0.37.1 → ssc_codegen-0.38.0}/ssc_codegen/ast/struct.py

@@ -136,10 +136,16 @@ class PlaceholderSpec:
 def parse_placeholder(match: "_re.Match[str]") -> PlaceholderSpec:
     return PlaceholderSpec(
         name=match.group(1),
-        type_name=match.group(2) or "str",
+        type_name=cast(
+            Literal["str", "int", "float", "bool"],
+            match.group(2) or "str",
+        ),
         is_array=bool(match.group(3)),
         is_optional=bool(match.group(4)),
-        style=match.group(5) or None,
+        style=cast(
+            Literal["repeat", "csv", "bracket", "pipe", "space"] | None,
+            match.group(5) or None,
+        ),
     )
 
 
@@ -260,7 +266,7 @@ class RequestHttp(Node):
 
     Created once at parse time from ``parse_to_http(raw_payload)``;
     converters read fields directly instead of re-parsing raw_payload.
-    String fields (url, headers, cookies, params, body) are ``Template``
+    String fields (url, headers, cookies, params, payload) are ``Template``
     instances: tokenized literal/placeholder parts.
     """
 
@@ -270,7 +276,7 @@ class RequestHttp(Node):
     cookies: dict[str, PlaceholderTemplate] = field(default_factory=dict)
     params: dict[str, PlaceholderTemplate] = field(default_factory=dict)
     body_kind: str = "empty"  # "empty" | "json" | "form" | "raw"
-    body: PlaceholderTemplate | dict[str, PlaceholderTemplate] | None = None
+    payload: PlaceholderTemplate | dict[str, PlaceholderTemplate] | None = None
 
     @property
     def placeholders(self) -> list[PlaceholderSpec]:
@@ -289,10 +295,10 @@ class RequestHttp(Node):
         yield self.url
         for d in (self.headers, self.cookies, self.params):
             yield from d.values()
-        if isinstance(self.body, PlaceholderTemplate):
-            yield self.body
-        elif isinstance(self.body, dict):
-            yield from self.body.values()
+        if isinstance(self.payload, PlaceholderTemplate):
+            yield self.payload
+        elif isinstance(self.payload, dict):
+            yield from self.payload.values()
 
     def with_renamed_placeholders(
         self, transform: Callable[[str], str]
@@ -312,13 +318,13 @@ class RequestHttp(Node):
         ) -> dict[str, PlaceholderTemplate]:
             return {k: v.renamed(mapping) for k, v in d.items()}
 
-        new_body: (
+        new_payload: (
             PlaceholderTemplate | dict[str, PlaceholderTemplate] | None
-        ) = self.body
-        if isinstance(self.body, PlaceholderTemplate):
-            new_body = self.body.renamed(mapping)
-        elif isinstance(self.body, dict):
-            new_body = _rename_dict(self.body)
+        ) = self.payload
+        if isinstance(self.payload, PlaceholderTemplate):
+            new_payload = self.payload.renamed(mapping)
+        elif isinstance(self.payload, dict):
+            new_payload = _rename_dict(self.payload)
 
         return RequestHttp(
             method=self.method,
@@ -327,7 +333,7 @@ class RequestHttp(Node):
             cookies=_rename_dict(self.cookies),
             params=_rename_dict(self.params),
             body_kind=self.body_kind,
-            body=new_body,
+            payload=new_payload,
         )
 
 

{ssc_codegen-0.37.1 → ssc_codegen-0.38.0}/ssc_codegen/ast/typedef.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 from dataclasses import dataclass
+from typing import cast
 
 from .base import Node
 from .types import StructType
@@ -29,4 +30,4 @@ class TypeDef(Node):
 
     @property
     def fields(self) -> list[TypeDefField]:
-        return self.body
+        return cast(list[TypeDefField], self.body)

{ssc_codegen-0.37.1 → ssc_codegen-0.38.0}/ssc_codegen/core/contexts.py

@@ -6,7 +6,7 @@ from dataclasses import dataclass, field
 from enum import Enum, auto
 from pathlib import Path
 
-from ssc_codegen.ast import StructBase, VariableType
+from ssc_codegen.ast import ExtensionDef, StructBase, VariableType
 from kdlquery import KdlNode, ReadDiagnostic, Severity
 
 
@@ -49,7 +49,9 @@ class ParseContext:
     children_defines: dict[str, list[KdlNode]] = field(default_factory=dict)
     structs: dict[str, StructBase] = field(default_factory=dict)
     json_defs: dict[str, JsonDef] = field(default_factory=dict)
+    extensions: dict[str, ExtensionDef] = field(default_factory=dict)
     source_path: Path | None = None
+    node_source_paths: dict[int, Path] = field(default_factory=dict)
 
     def all_names(self) -> set[str]:
         return (
@@ -57,6 +59,7 @@ class ParseContext:
             | set(self.children_defines)
             | set(self.structs)
             | set(self.json_defs)
+            | set(self.extensions)
         )
 
 
@@ -77,6 +80,7 @@ class LintContext:
     )
     _predicate_depth: int = field(default=0)
     _predicate_context: str = ""
+    node_source_paths: dict[int, Path] = field(default_factory=dict)
 
     def error(
         self,
@@ -88,18 +92,22 @@ class LintContext:
         label: str | None = None,
         notes: list[str] | None = None,
     ) -> None:
-        self.diagnostics.append(
-            ReadDiagnostic(
-                message=message,
-                severity=Severity.ERROR,
-                span=node.span,
-                path=self.path,
-                hint=hint,
-                code=code,
-                label=label,
-                notes=tuple(notes) if notes else (),
-            )
+        scope = self.path
+        source_path = self.node_source_paths.get(id(node))
+        diagnostic_notes = tuple(notes) if notes else ()
+        if source_path is not None and scope:
+            diagnostic_notes = (*diagnostic_notes, f"scope: {scope}")
+        diagnostic = ReadDiagnostic(
+            message=message,
+            severity=Severity.ERROR,
+            span=node.span,
+            path=str(source_path) if source_path is not None else scope,
+            hint=hint,
+            code=code,
+            label=label,
+            notes=diagnostic_notes,
         )
+        self.diagnostics.append(diagnostic)
 
     def warning(
         self,
@@ -111,18 +119,22 @@ class LintContext:
         label: str | None = None,
         notes: list[str] | None = None,
     ) -> None:
-        self.diagnostics.append(
-            ReadDiagnostic(
-                message=message,
-                severity=Severity.WARNING,
-                span=node.span,
-                path=self.path,
-                hint=hint,
-                code=code,
-                label=label,
-                notes=tuple(notes) if notes else (),
-            )
+        scope = self.path
+        source_path = self.node_source_paths.get(id(node))
+        diagnostic_notes = tuple(notes) if notes else ()
+        if source_path is not None and scope:
+            diagnostic_notes = (*diagnostic_notes, f"scope: {scope}")
+        diagnostic = ReadDiagnostic(
+            message=message,
+            severity=Severity.WARNING,
+            span=node.span,
+            path=str(source_path) if source_path is not None else scope,
+            hint=hint,
+            code=code,
+            label=label,
+            notes=diagnostic_notes,
         )
+        self.diagnostics.append(diagnostic)
 
     @property
     def path(self) -> str:

{ssc_codegen-0.37.1 → ssc_codegen-0.38.0}/ssc_codegen/core/expressions.py

@@ -16,6 +16,7 @@ from ssc_codegen.ast import (
     CssSelect,
     CssSelectAll,
     Fallback,
+    ExtensionCall,
     Field,
     Filter,
     Fmt,
@@ -80,6 +81,7 @@ from ssc_codegen.regex_utils import normalize_regex_pattern
 from typing import cast
 
 from ssc_codegen.core.contexts import LintContext, ParseContext, WalkCtx
+from ssc_codegen.core.extensions import resolve_extension_call
 from ssc_codegen.core.linter import (
     lint_pipeline_op,
     lint_validate_css,
@@ -106,6 +108,7 @@ FieldLikeNode: TypeAlias = (
     | Field
     | InitField
     | FunctionDef
+    | CheckMethod
 )
 
 
@@ -343,6 +346,9 @@ def _build_expression(
             f"'self {ref_name}' syntax is no longer supported; use '@{ref_name}' instead"
         )
 
+    if name.startswith("!"):
+        return _expr_extension(node, parent, ctx, lint)
+
     # dispatch
     handler = _EXPRESSION_HANDLERS.get(name)
     if handler is None:
@@ -417,6 +423,13 @@ def parse_expressions(
             raise BuildTimeError(
                 f"'self {ref_name}' syntax is no longer supported; use '@{ref_name}' instead"
             )
+        if node.name.startswith("!"):
+            lint.push(node.name)
+            lint_pipeline_op(node, lint)
+            extension_expr = _expr_extension(node, parent, ctx, lint)
+            parent.body.append(extension_expr)
+            lint.pop()
+            continue
         handler = _EXPRESSION_HANDLERS.get(node.name)
         if handler is None:
             lint_wildcard_op(node, ctx, lint)
@@ -501,7 +514,10 @@ def parse_expressions(
                         break
             parent.is_array = is_arr
             parent.ret_type_info = TypeInfo(
-                base=last_ret, is_array=is_arr, ref=ref
+                base=last_ret,
+                is_array=is_arr,
+                is_optional=last_ti.is_optional,
+                ref=ref or last_ti.ref,
             )
 
     lint.walk_context = prev_ctx
@@ -537,6 +553,59 @@ def _prev_type_info(
     )
 
 
+def _expr_extension(
+    node: KdlNode,
+    parent: FieldLikeNode,
+    ctx: ParseContext,
+    lint: LintContext,
+) -> ExtensionCall:
+    ref = node.name[1:]
+    if node.args or node.properties or node.children:
+        lint.error(
+            node,
+            message=f"custom operation '!{ref}' does not accept entries",
+            code="E001",
+        )
+    definition = ctx.extensions.get(ref)
+    prev = _prev_type_info(parent)
+    if definition is None:
+        lint.error(
+            node,
+            message=f"unknown extension operation '{ref}'",
+            code="E300",
+            hint="import its extension namespace and check the operation name",
+        )
+        return ExtensionCall(
+            parent=parent,
+            qualified_name=ref,
+            accept_type_info=prev.ret_type_info,
+            ret_type_info=TypeInfo(base=VariableType.AUTO),
+        )
+    if not definition.accept.accepts(prev.ret_type_info):
+        expected = (
+            definition.accept.generic
+            or (definition.accept.base or VariableType.AUTO).name
+        )
+        lint.error(
+            node,
+            message=(
+                f"custom operation '{ref}' does not accept "
+                f"{prev.ret_type_info.base.name}; expected {expected}"
+            ),
+            code="E100",
+        )
+    ret_type = resolve_extension_call(definition, prev.ret_type_info)
+    return ExtensionCall(
+        parent=parent,
+        qualified_name=ref,
+        definition=definition,
+        accept_type_info=prev.ret_type_info,
+        ret_type_info=ret_type,
+        is_array=ret_type.is_array,
+        span=node.span,
+    )
+
+
 # Ops whose handlers access ``parent.body[-1]`` unconditionally and therefore
 # require a preceding node. Used by ``parse_expressions`` to emit a clean
 # E100 diagnostic instead of crashing with IndexError on an empty pipeline.
@@ -606,6 +675,9 @@ def _reg_expr(name: str):
 def _expr_css(
     node: KdlNode, parent: FieldLikeNode, ctx: ParseContext, lint: LintContext
 ):
+    if not node.children and not node.args:
+        lint.error(node, message="'css' requires a selector", code="E001")
+        return None
     if node.children:
         queries = [
             resolve_selector_child_name(c.name, ctx) for c in node.children
@@ -622,6 +694,9 @@ def _expr_css(
 def _expr_css_all(
     node: KdlNode, parent: FieldLikeNode, ctx: ParseContext, lint: LintContext
 ):
+    if not node.children and not node.args:
+        lint.error(node, message="'css-all' requires a selector", code="E001")
+        return None
     if node.children:
         queries = [
             resolve_selector_child_name(c.name, ctx) for c in node.children
@@ -638,6 +713,9 @@ def _expr_css_all(
 def _expr_xpath(
     node: KdlNode, parent: FieldLikeNode, ctx: ParseContext, lint: LintContext
 ):
+    if not node.children and not node.args:
+        lint.error(node, message="'xpath' requires an expression", code="E001")
+        return None
     if node.children:
         queries = [
             resolve_selector_child_name(c.name, ctx) for c in node.children
@@ -654,6 +732,11 @@ def _expr_xpath(
 def _expr_xpath_all(
     node: KdlNode, parent: FieldLikeNode, ctx: ParseContext, lint: LintContext
 ):
+    if not node.children and not node.args:
+        lint.error(
+            node, message="'xpath-all' requires an expression", code="E001"
+        )
+        return None
     if node.children:
         queries = [
             resolve_selector_child_name(c.name, ctx) for c in node.children