ssc_codegen 0.21.0__tar.gz → 0.22.0__tar.gz

{ssc_codegen-0.21.0 → ssc_codegen-0.22.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.21.0
+Version: 0.22.0
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues

{ssc_codegen-0.21.0 → ssc_codegen-0.22.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.21.0"
+version = "0.22.0"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.21.0 → ssc_codegen-0.22.0}/ssc_codegen/ast/__init__.py

@@ -25,6 +25,7 @@ from .struct import (
     Struct,
     StructDocstring,
     PreValidate,
+    CheckMethod,
     Init,
     InitField,
     SplitDoc,
@@ -134,7 +135,7 @@ __all__ = [
     # jsondef
     "JsonDef", "JsonDefField",
     # struct
-    "Struct", "StructDocstring", "PreValidate",
+    "Struct", "StructDocstring", "PreValidate", "CheckMethod",
     "Init", "InitField", "SplitDoc",
     "Key", "Value",
     "TableConfig", "TableRow", "TableMatchKey",

{ssc_codegen-0.21.0 → ssc_codegen-0.22.0}/ssc_codegen/ast/jsondef.py

@@ -10,11 +10,13 @@ class JsonDefField(Node):
     Single field in a JSON mapping definition.
 
     type_name   — primitive ("str", "int", "float", "bool") or ref name.
-    is_optional — True when field declared with ? suffix (value | null).
-    is_array    — True when field declared with {} suffix.
+    is_optional — True when field declared with ? suffix or @optional arg.
+    is_array    — True when field declared with (array) prefix.
     ref_name    — set when type_name references another JsonDef.
-    alias       — original JSON key when it differs from name
-                  (e.g. name="context", alias="@context").
+    alias       — original JSON key when it differs from name.
+    skip        — field is parsed but excluded from output TypedDict.
+    may_miss    — field key may be absent from JSON (use .get() instead of []).
+    doc         — documentation string for the field.
     """
 
     name: str = ""
@@ -23,15 +25,19 @@ class JsonDefField(Node):
     is_array: bool = False
     ref_name: str | None = None
     alias: str = ""
+    skip: bool = False
+    may_miss: bool = False
+    doc: str = ""
 
 
 @dataclass
 class JsonDef(Node):
     """
     JSON mapping definition.
-    DSL: json Name { ... } / json Name array=#true { ... }
+    DSL: json Name { ... } / json Name array=#true { ... } / json Name path="a.b" { ... }
     body: list[JsonDefField]
     """
 
     name: str = ""
     is_array: bool = False
+    path: str = ""

{ssc_codegen-0.21.0 → ssc_codegen-0.22.0}/ssc_codegen/ast/struct.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 import re as _re
 from dataclasses import dataclass, field
+from typing import Any
 from typing import cast
 
 from .base import Node
@@ -119,6 +120,20 @@ class PreValidate(Node):
     ret: VariableType = field(default=VariableType.DOCUMENT)
 
 
+@dataclass
+class CheckMethod(Node):
+    """
+    Boolean check method on the parsed class.
+    DSL: @check <name> { pipeline ... }
+    Runs a pipeline on the document and returns True on success, False on failure.
+    Called manually by the user before parse().
+    """
+
+    name: str = ""
+    accept: VariableType = field(default=VariableType.DOCUMENT)
+    ret: VariableType = field(default=VariableType.BOOL)
+
+
 @dataclass
 class Init(Node):
     """
@@ -276,18 +291,18 @@ class RequestConfig(Node):
 class ErrorResponse(Node):
     """
     Error response mapping for type=rest struct.
-    DSL: @error <status> [field="<name>"] <JsonSchema>
+    DSL: @error <status> <SchemaName> [field=value ...]
 
     status: HTTP status code [100..599].
     schema_name: json schema reference for deserialised error body.
-    discriminator_field: optional body field name. When set, the error triggers
-        on 2xx responses where <field> is present in the parsed JSON body
-        (used for APIs that return 200 + error payload).
+    conditions: field=value pairs checked against the parsed JSON body.
+        Keys are dot-paths (e.g. "response.success", "data.0.type").
+        When non-empty, the error triggers on matching status + all conditions.
     """
 
     status: int = 0
     schema_name: str = ""
-    discriminator_field: str | None = None
+    conditions: dict[str, Any] = field(default_factory=dict)
 
 
 @dataclass

{ssc_codegen-0.21.0 → ssc_codegen-0.22.0}/ssc_codegen/converters/base.py

@@ -9,6 +9,7 @@ from ssc_codegen.ast import LogicNot, LogicAnd, LogicOr
 from ssc_codegen.ast import JsonDef, TypeDef, Struct, Init
 from ssc_codegen.ast import (
     PreValidate,
+    CheckMethod,
     Field,
     InitField,
     SplitDoc,
@@ -64,6 +65,7 @@ _PIPELINE_NODES = (
     Field,
     InitField,
     PreValidate,
+    CheckMethod,
     SplitDoc,
     Key,
     Value,

{ssc_codegen-0.21.0 → ssc_codegen-0.22.0}/ssc_codegen/converters/go_goquery.py

@@ -25,6 +25,7 @@ from ssc_codegen.ast import (
     Init,
     InitField,
     PreValidate,
+    CheckMethod,
     SplitDoc,
     TableConfig,
     TableMatchKey,
@@ -1011,6 +1012,18 @@ def post_pre_validate(node: PreValidate, _):
     return ["}"]
 
 
+@GO_GOQUERY_CONVERTER(CheckMethod)
+def pre_check_method(node: CheckMethod, _):
+    struct_name = to_pascal_case(node.parent.name)
+    recv = to_camel_case(struct_name)
+    return [f"func ({recv} *{struct_name}) {to_camel_case(node.name)}() bool {{", "\tdefer func() { recover() }()"]
+
+
+@GO_GOQUERY_CONVERTER.post(CheckMethod)
+def post_check_method(node: CheckMethod, _):
+    return ["\treturn true", "}"]
+
+
 @GO_GOQUERY_CONVERTER(SplitDoc)
 def pre_split_doc(node: SplitDoc, _):
     struct_name = to_pascal_case(node.parent.name)

{ssc_codegen-0.21.0 → ssc_codegen-0.22.0}/ssc_codegen/converters/js_pure.py

@@ -51,6 +51,7 @@ from ssc_codegen.ast import (
     Init,
     InitField,
     PreValidate,
+    CheckMethod,
     SplitDoc,
     TableConfig,
     TableMatchKey,
@@ -464,11 +465,40 @@ def pre_typedef_field(node: TypeDefField, ctx: ConverterContext):
 
 def _js_err_subclass_name(struct_name: str, err) -> str:
     base = f"{to_pascal_case(struct_name)}Err{err.status}"
-    if err.discriminator_field:
-        base += to_pascal_case(err.discriminator_field)
+    if err.conditions:
+        for key in err.conditions:
+            base += to_pascal_case(key.replace(".", "_").replace("-", "_"))
     return base
 
 
+def _js_resolve_path_expr(body_var: str, path: str) -> str:
+    """Generate a JS expression navigating a dot-path into a JSON body."""
+    parts = path.split(".")
+    expr = body_var
+    for seg in parts:
+        if seg.isdigit():
+            expr += f"[{seg}]"
+        else:
+            expr += f"[{seg!r}]"
+    return expr
+
+
+def _js_condition_check_expr(err) -> str:
+    """Build a compound JS condition expression for @error field conditions."""
+    parts = []
+    for path, value in err.conditions.items():
+        lhs = _js_resolve_path_expr("_body", path)
+        if isinstance(value, bool):
+            parts.append(f"{lhs} === {str(value).lower()}")
+        elif value is None:
+            parts.append(f"{lhs} === null")
+        elif isinstance(value, (int, float)):
+            parts.append(f"{lhs} === {value}")
+        else:
+            parts.append(f"{lhs} === {value!r}")
+    return " && ".join(parts)
+
+
 def _js_err_value_type(err, struct) -> str:
     schema = err.schema_name
     if not schema:
@@ -527,8 +557,8 @@ def _emit_dispatch_err_js(node: Struct, ctx: ConverterContext) -> list[str]:
     i3 = i2 + ctx.indent_char  # nested (if ...)
 
     errors = node.errors
-    status_errors = [e for e in errors if e.discriminator_field is None]
-    field_errors = [e for e in errors if e.discriminator_field is not None]
+    status_errors = [e for e in errors if not e.conditions]
+    field_errors = [e for e in errors if e.conditions]
 
     lines: list[str] = [
         f"{i1}static _dispatchErr(_status, _headers, _body) {{",
@@ -536,10 +566,10 @@ def _emit_dispatch_err_js(node: Struct, ctx: ConverterContext) -> list[str]:
     ]
     for err in field_errors:
         if 200 <= err.status < 300:
-            field = err.discriminator_field
+            cond = _js_condition_check_expr(err)
             lines.append(
                 f"{i3}if (_status === {err.status} && _body "
-                f"&& _body[{field!r}] !== undefined) {{"
+                f"&& _body instanceof Object && {cond}) {{"
             )
             lines.append(
                 f"{i3}{ctx.indent_char}return {{ isOk: false, "
@@ -558,10 +588,10 @@ def _emit_dispatch_err_js(node: Struct, ctx: ConverterContext) -> list[str]:
         lines.append(f"{i2}}}")
     for err in field_errors:
         if not (200 <= err.status < 300):
-            field = err.discriminator_field
+            cond = _js_condition_check_expr(err)
             lines.append(
                 f"{i2}if (_status === {err.status} && _body "
-                f"&& _body[{field!r}] !== undefined) {{"
+                f"&& _body instanceof Object && {cond}) {{"
             )
             lines.append(
                 f"{i3}return {{ isOk: false, status: _status, "
@@ -632,6 +662,26 @@ def pre_struct_pre_validate(node: PreValidate, ctx: ConverterContext):
     ]
 
 
+@JS_CONVERTER(CheckMethod)
+def pre_struct_check_method(node: CheckMethod, ctx: ConverterContext):
+    method_name = to_camel_case(node.name)
+    return [
+        f"{ctx.indent}{method_name}() " + "{",
+        f"{ctx.indent}{ctx.indent_char}try " + "{",
+    ]
+
+
+@JS_CONVERTER.post(CheckMethod)
+def post_struct_check_method(node: CheckMethod, ctx: ConverterContext):
+    return [
+        f"{ctx.indent}{ctx.indent_char}{ctx.indent_char}return true;",
+        f"{ctx.indent}{ctx.indent_char}" + "} catch (e) {",
+        f"{ctx.indent}{ctx.indent_char}{ctx.indent_char}return false;",
+        f"{ctx.indent}{ctx.indent_char}" + "}",
+        f"{ctx.indent}" + "}",
+    ]
+
+
 @JS_CONVERTER(SplitDoc, post_callback=lambda _, ctx: ctx.indent + "}")
 def pre_struct_split_doc(node: SplitDoc, ctx: ConverterContext):
     return [f"{ctx.indent}_splitDoc(v) " + "{"]

{ssc_codegen-0.21.0 → ssc_codegen-0.22.0}/ssc_codegen/converters/py_bs4.py

@@ -37,6 +37,7 @@ from ssc_codegen.ast import (
     Init,
     InitField,
     PreValidate,
+    CheckMethod,
     SplitDoc,
     TableConfig,
     TableMatchKey,
@@ -537,6 +538,26 @@ def pre_struct_pre_validate(node: PreValidate, ctx: ConverterContext):
     ]
 
 
+@PY_BASE_CONVERTER(CheckMethod)
+def pre_struct_check_method(node: CheckMethod, ctx: ConverterContext):
+    from ssc_codegen.converters.helpers import to_snake_case
+
+    method_name = to_snake_case(node.name)
+    return [
+        f"    def {method_name}(self) -> bool:",
+        "        try:",
+    ]
+
+
+@PY_BASE_CONVERTER.post(CheckMethod)
+def post_struct_check_method(node: CheckMethod, ctx: ConverterContext):
+    return [
+        "            return True",
+        "        except Exception:",
+        "            return False",
+    ]
+
+
 @PY_BASE_CONVERTER(SplitDoc)
 def pre_struct_split_doc(node: SplitDoc, ctx: ConverterContext):
     return [

{ssc_codegen-0.21.0 → ssc_codegen-0.22.0}/ssc_codegen/converters/py_helpers.py

@@ -99,10 +99,11 @@ def rest_utilities(node) -> list[str]:
 
 
 def _err_subclass_name(struct_name: str, err: ErrorResponse) -> str:
-    """Naming: `<Struct>Err<Status>[<FieldPascal>]`."""
+    """Naming: `<Struct>Err<Status>[<ConditionKeys>]`."""
     base = f"{to_pascal_case(struct_name)}Err{err.status}"
-    if err.discriminator_field:
-        base += to_pascal_case(err.discriminator_field)
+    if err.conditions:
+        for key in err.conditions:
+            base += to_pascal_case(key.replace(".", "_").replace("-", "_"))
     return base
 
 
@@ -132,6 +133,37 @@ def _rest_err_union_type(struct: Struct) -> str:
     return "Union[" + ", ".join([*variants, "UnknownErr", "None"]) + "]"
 
 
+def _resolve_path_expr(body_var: str, path: str) -> str:
+    """Generate a Python expression that navigates a dot-path into a JSON body.
+
+    Segments: digits → list index, else → dict key via .get().
+    """
+    parts = path.split(".")
+    expr = body_var
+    for seg in parts:
+        if seg.isdigit():
+            expr += f"[{seg}]"
+        else:
+            expr += f".get({seg!r})"
+    return expr
+
+
+def _condition_check_expr(err: ErrorResponse) -> str:
+    """Build a compound condition expression for an @error's field conditions."""
+    parts = []
+    for path, value in err.conditions.items():
+        lhs = _resolve_path_expr("_body", path)
+        if isinstance(value, bool):
+            parts.append(f"{lhs} == {value}")
+        elif value is None:
+            parts.append(f"{lhs} is None")
+        elif isinstance(value, (int, float)):
+            parts.append(f"{lhs} == {value}")
+        else:
+            parts.append(f"{lhs} == {value!r}")
+    return " and ".join(parts)
+
+
 def _emit_dispatch_err_py(node: Struct, ctx: ConverterContext) -> list[str]:
     """Emit the `_dispatch_err` @staticmethod lines inside a REST class.
 
@@ -143,8 +175,8 @@ def _emit_dispatch_err_py(node: Struct, ctx: ConverterContext) -> list[str]:
     i4 = i3 + ctx.indent_char
 
     errors = node.errors
-    status_errors = [e for e in errors if e.discriminator_field is None]
-    field_errors = [e for e in errors if e.discriminator_field is not None]
+    status_errors = [e for e in errors if not e.conditions]
+    field_errors = [e for e in errors if e.conditions]
     union_type = _rest_err_union_type(node)
 
     lines: list[str] = [
@@ -162,9 +194,9 @@ def _emit_dispatch_err_py(node: Struct, ctx: ConverterContext) -> list[str]:
         for err in field_errors:
             if 200 <= err.status < 300:
                 cls_name = _err_subclass_name(node.name, err)
+                cond = _condition_check_expr(err)
                 lines.append(
-                    f"{i4}if _status == {err.status} "
-                    f"and {err.discriminator_field!r} in _body:"
+                    f"{i4}if _status == {err.status} and {cond}:"
                 )
                 lines.append(
                     f"{i4}{ctx.indent_char}return {cls_name}("
@@ -185,10 +217,10 @@ def _emit_dispatch_err_py(node: Struct, ctx: ConverterContext) -> list[str]:
     for err in field_errors:
         if not (200 <= err.status < 300):
             cls_name = _err_subclass_name(node.name, err)
+            cond = _condition_check_expr(err)
             lines.append(
                 f"{i2}if _status == {err.status} "
-                f"and isinstance(_body, dict) "
-                f"and {err.discriminator_field!r} in _body:"
+                f"and isinstance(_body, dict) and {cond}:"
             )
             lines.append(
                 f"{i3}return {cls_name}(headers=_headers, value=_body)"

{ssc_codegen-0.21.0 → ssc_codegen-0.22.0}/ssc_codegen/linter/rules_struct.py

@@ -45,6 +45,7 @@ _RESERVED_ALLOWED: dict[str, frozenset[str] | None] = {
     "@request": None,
     "@doc": None,
     "@pre-validate": frozenset({"item", "list", "dict", "table", "flat"}),
+    "@check": frozenset({"item", "list", "dict", "table", "flat"}),
     "@init": frozenset({"item", "list", "dict", "table", "flat"}),
     "@split-doc": frozenset({"list", "dict"}),  # dict can also use @split-doc
     "@key": frozenset({"dict"}),

{ssc_codegen-0.21.0 → ssc_codegen-0.22.0}/ssc_codegen/parser.py

@@ -56,6 +56,7 @@ from ssc_codegen.ast import (
 # stuct layer
 from ssc_codegen.ast import (
     PreValidate,
+    CheckMethod,
     SplitDoc,
     TableConfig,
     TableMatchKey,
@@ -461,6 +462,27 @@ class AstParser:
                     ret_type = VariableType.JSON
             # second argument is optional alias (original JSON key)
             alias = str(node.args[1]) if len(node.args) > 1 else ""
+            # scan remaining args for @modifiers (@optional, @missing, @skip)
+            skip = False
+            may_miss = False
+            for arg in node.args[2:]:
+                arg_s = str(arg)
+                if arg_s == "@skip":
+                    skip = True
+                elif arg_s == "@missing":
+                    may_miss = True
+                elif arg_s == "@optional":
+                    is_optional = True
+            # if alias was actually an @modifier, correct alias
+            if alias.startswith("@"):
+                if alias == "@skip":
+                    skip = True
+                elif alias == "@missing":
+                    may_miss = True
+                elif alias == "@optional":
+                    is_optional = True
+                alias = ""
+            doc = str(node.properties.get("doc", ""))
             jf = JsonDefField(
                 parent=parent,
                 ret=ret_type,
@@ -469,15 +491,20 @@ class AstParser:
                 is_array=is_array,
                 ref_name=ref_name,
                 alias=alias,
+                skip=skip,
+                may_miss=may_miss,
+                doc=doc,
             )
             logger.debug(
-                "  json field %r: ret=%s, optional=%s, array=%s%s%s",
+                "  json field %r: ret=%s, optional=%s, array=%s%s%s%s%s",
                 name,
                 ret_type,
                 is_optional,
                 is_array,
                 f", ref={ref_name!r}" if ref_name else "",
                 f", alias={alias!r}" if alias else "",
+                ", skip" if skip else "",
+                ", miss" if may_miss else "",
             )
             parent.body.append(jf)
 
@@ -777,6 +804,15 @@ class AstParser:
                 expr = PreValidate(parent=parent)
                 self._parse_expressions(node.children, expr)
                 parent.body.append(expr)
+            elif node.name == "@check":
+                if not node.args:
+                    raise ParseError(
+                        "@check requires a name: @check <name> { ... }"
+                    )
+                check_name = str(node.args[0])
+                expr = CheckMethod(parent=parent, name=check_name)
+                self._parse_expressions(node.children, expr)
+                parent.body.append(expr)
             elif node.name == "@split-doc":
                 expr = SplitDoc(parent=parent)
                 self._parse_expressions(node.children, expr)
@@ -836,7 +872,7 @@ class AstParser:
                 if not node.args:
                     raise ParseError(
                         "@error requires status and schema name: "
-                        '@error <status> [field="..."] <Schema>'
+                        "@error <status> <Schema> [field=value ...]"
                     )
                 if len(node.args) < 2:
                     raise ParseError(
@@ -852,23 +888,25 @@ class AstParser:
                 schema_name = str(
                     self.ctx.property_defines.get(node.args[1], node.args[1])
                 )
-                field_prop = node.properties.get("field", None)
-                if field_prop is not None:
-                    field_prop = str(
-                        self.ctx.property_defines.get(field_prop, field_prop)
+                conditions: dict[str, Any] = {}
+                for k, v in node.properties.items():
+                    key = str(
+                        self.ctx.property_defines.get(k, k)
                     )
+                    val = self.ctx.property_defines.get(v, v)
+                    conditions[key] = val
                 err = ErrorResponse(
                     parent=parent,
                     status=status_int,
                     schema_name=schema_name,
-                    discriminator_field=field_prop,
+                    conditions=conditions,
                 )
                 parent.body.append(err)
                 logger.debug(
-                    "  @error: status=%d schema=%r field=%r",
+                    "  @error: status=%d schema=%r conditions=%r",
                     status_int,
                     schema_name,
-                    field_prop,
+                    conditions,
                 )
             else:
                 if parent.struct_type == StructType.TABLE:
@@ -1310,6 +1348,24 @@ _FLOAT_RE = _re.compile(
 )
 
 
+_DEFINE_REF_RE = _re.compile(r"\{\{([A-Za-z][A-Za-z0-9_-]*)\}\}")
+
+
+def _resolve_define_references(value: str, ctx: ParseContext) -> str:
+    """Replace {{NAME}} in a scalar define value with previously-defined values."""
+
+    def _replacer(m: _re.Match) -> str:
+        name = m.group(1)
+        resolved = ctx.property_defines.get(name)
+        if resolved is None:
+            raise ParseError(
+                f"define references undefined name {name!r}"
+            )
+        return str(resolved)
+
+    return _DEFINE_REF_RE.sub(_replacer, value)
+
+
 # contexts
 @PARSER.register_module_context("define")
 def reg_define(node: KdlNode, ctx: ParseContext):
@@ -1322,7 +1378,10 @@ def reg_define(node: KdlNode, ctx: ParseContext):
         )
     else:
         pair = list(node.properties.items())[0]
-        ctx.property_defines[pair[0]] = pair[1]
+        value = pair[1]
+        if isinstance(value, str):
+            value = _resolve_define_references(value, ctx)
+        ctx.property_defines[pair[0]] = value
         logger.debug("define: property %r = %r", pair[0], pair[1])
 
 
@@ -1428,7 +1487,8 @@ def reg_module_struct(node: KdlNode, parent: Module, _: ParseContext):
 def reg_module_json(node: KdlNode, parent: Module, _: ParseContext):
     name = node.args[0]
     is_array = node.properties.get("array", False)
-    return JsonDef(parent=parent, name=name, is_array=is_array)
+    path = str(node.properties.get("path", ""))
+    return JsonDef(parent=parent, name=name, is_array=is_array, path=path)
 
 
 # expressions layer

{ssc_codegen-0.21.0 → ssc_codegen-0.22.0}/ssc_codegen/repl.py

@@ -39,8 +39,9 @@ def _try_prompt_toolkit():
             Completion,
             FuzzyWordCompleter,
         )
+        from prompt_toolkit.history import InMemoryHistory
 
-        return pt_prompt, Completer, Completion, FuzzyWordCompleter
+        return pt_prompt, Completer, Completion, FuzzyWordCompleter, InMemoryHistory
     except ImportError:
         return None
 
@@ -159,6 +160,12 @@ def _make_completer(repl: Repl):
 
     return _SsgCompleter()
 
+def _make_history():
+    if _PROMPT_TOOLKIT is None:
+        return
+    from prompt_toolkit.history import InMemoryHistory
+
+    return InMemoryHistory()
 
 def _get_converter(target: str):
     mod_path, attr = _CONVERTERS[target].rsplit(":", 1)
@@ -179,7 +186,7 @@ def _fetch_html(url: str) -> str:
     except ImportError:
         raise RuntimeError(
             "httpx is required for URL fetching. "
-            "Install with: pip install ssc-gen[repl]"
+            "Install with: pip install ssc-codegen[repl]"
         )
 
 
@@ -224,14 +231,16 @@ class Repl:
         self._generated_code: str = ""
         self._multiline_kdl: list[str] | None = None
         self._completer = _make_completer(self)
+        self._history = _make_history()
 
     def _read_line(self, prompt_str: str) -> str:
         if _PROMPT_TOOLKIT is not None:
-            pt_prompt, _, _, _ = _PROMPT_TOOLKIT
+            pt_prompt, _, _, _, _ = _PROMPT_TOOLKIT
             return pt_prompt(
                 prompt_str,
                 completer=self._completer,
                 complete_while_typing=True,
+                history=self._history
             )
         return input(prompt_str)