ssc_codegen 0.15.2__tar.gz → 0.17.0a0__tar.gz

ssc_codegen-0.17.0a0/MANIFEST.in

@@ -0,0 +1,3 @@
+# Включаем C-исходники tree-sitter-kdl из submodule в sdist
+recursive-include vendor/tree-sitter-kdl/src *.c *.h
+recursive-include vendor/tree-sitter-kdl/bindings/python/tree_sitter_kdl *.c

ssc_codegen-0.17.0a0/PKG-INFO

@@ -0,0 +1,111 @@
+Metadata-Version: 2.4
+Name: ssc_codegen
+Version: 0.17.0a0
+Summary: Python-dsl code converter to html parser for web scraping 
+Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
+Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
+Project-URL: Source, https://github.com/vypivshiy/selector_schema_codegen
+Project-URL: Examples, https://github.com/vypivshiy/selector_schema_codegen/examples
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Software Development :: Code Generators
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Classifier: Topic :: Utilities
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: bs4>=0.0.2
+Requires-Dist: colorama>=0.4.6; sys_platform == "win32"
+Requires-Dist: cssselect>=1.2.0
+Requires-Dist: lxml>=5.3.0
+Requires-Dist: soupsieve>=2.6
+Requires-Dist: typer>=0.15.1
+Requires-Dist: typing_extensions; python_version < "3.11"
+Requires-Dist: click<8.2.0
+Requires-Dist: tree-sitter>=0.25.2
+Dynamic: license-file
+
+# Selector Schema codegen
+
+Experimental PoC implementation of a code generator based on KDL2.0 syntax DSL.
+
+## install
+
+### From git (requires C/C++ compiler)
+
+```bash
+# Clone with submodules
+git clone --recursive https://github.com/vypivshiy/selector_schema_codegen
+cd selector_schema_codegen
+
+# Install with pip (builds tree-sitter-kdl extension)
+pip install .
+```
+
+**Requirements:**
+- Linux/macOS: `gcc` or `clang`
+- Windows: MSVC (Visual Studio Build Tools or Visual Studio)
+
+### Via uv tool
+
+```bash
+uv tool install git+https://github.com/vypivshiy/selector_schema_codegen@features-kdl
+```
+
+## usage
+
+### generate modules
+
+```
+ssc-gen generate examples/ -t js-pure -o .
+```
+
+### lint syntax
+
+```
+ssc-gen check examples/
+```
+
+### test schema by html output
+
+from file:
+```
+python main.py run .\examples\booksToScrape.kdl:MainCatalogue -t py-bs4 -i index.html
+```
+
+from stdin:
+```
+curl https://books.toscrape.com/ | python main.py run .\examples\booksToScrape.kdl:MainCatalogue -t py-bs4
+```
+
+### test selectors:
+
+from file
+```
+python main.py health .\examples\booksToScrape.kdl:MainCatalogue -i index.html
+```
+
+from stdin
+```
+curl https://books.toscrape.com/catalogue/page-2.html | python main.py health .\examples\booksToScrape.kdl:MainCatalogue
+```
+
+
+## syntax
+
+see [docs](docs) and [examples](examples) how to use syntax
+
+## LLM generate dsl config (experimental, not ready)
+
+### prompt
+
+use [SYSTEM_PROMPT](SYSTEM_PROMPT.md) for use in API pipelines or chats. before generate, call `ssc-gen check [FILES...] -f json` liner and send errors output if exists
+
+### skill
+use [kdl-schema-dsl](.agents/skills/kdl-schema-dsl) for generate config

ssc_codegen-0.17.0a0/README.md

@@ -0,0 +1,78 @@
+# Selector Schema codegen
+
+Experimental PoC implementation of a code generator based on KDL2.0 syntax DSL.
+
+## install
+
+### From git (requires C/C++ compiler)
+
+```bash
+# Clone with submodules
+git clone --recursive https://github.com/vypivshiy/selector_schema_codegen
+cd selector_schema_codegen
+
+# Install with pip (builds tree-sitter-kdl extension)
+pip install .
+```
+
+**Requirements:**
+- Linux/macOS: `gcc` or `clang`
+- Windows: MSVC (Visual Studio Build Tools or Visual Studio)
+
+### Via uv tool
+
+```bash
+uv tool install git+https://github.com/vypivshiy/selector_schema_codegen@features-kdl
+```
+
+## usage
+
+### generate modules
+
+```
+ssc-gen generate examples/ -t js-pure -o .
+```
+
+### lint syntax
+
+```
+ssc-gen check examples/
+```
+
+### test schema by html output
+
+from file:
+```
+python main.py run .\examples\booksToScrape.kdl:MainCatalogue -t py-bs4 -i index.html
+```
+
+from stdin:
+```
+curl https://books.toscrape.com/ | python main.py run .\examples\booksToScrape.kdl:MainCatalogue -t py-bs4
+```
+
+### test selectors:
+
+from file
+```
+python main.py health .\examples\booksToScrape.kdl:MainCatalogue -i index.html
+```
+
+from stdin
+```
+curl https://books.toscrape.com/catalogue/page-2.html | python main.py health .\examples\booksToScrape.kdl:MainCatalogue
+```
+
+
+## syntax
+
+see [docs](docs) and [examples](examples) how to use syntax
+
+## LLM generate dsl config (experimental, not ready)
+
+### prompt
+
+use [SYSTEM_PROMPT](SYSTEM_PROMPT.md) for use in API pipelines or chats. before generate, call `ssc-gen check [FILES...] -f json` liner and send errors output if exists
+
+### skill
+use [kdl-schema-dsl](.agents/skills/kdl-schema-dsl) for generate config

{ssc_codegen-0.15.2 → ssc_codegen-0.17.0a0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.15.2"
+version = "0.17.0a"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"
@@ -8,8 +8,6 @@ dependencies = [
     "bs4>=0.0.2",
     "colorama>=0.4.6 ; sys_platform == 'win32'",
     "cssselect>=1.2.0",
-    "httpx>=0.28.1",
-    "ichrome>=4.0.4",
     "lxml>=5.3.0",
     "soupsieve>=2.6",
     "typer>=0.15.1",
@@ -17,9 +15,10 @@ dependencies = [
     # https://github.com/fastapi/typer/discussions/1215
     # https://github.com/fastapi/typer/pull/1145
     "click<8.2.0",
-    "ast-grep-cli>=0.38.1",
-    "tinycss2>=1.4.0",
+    "tree-sitter>=0.25.2",
 ]
+# Для Windows нужен MSVC, для Linux/macOS — gcc/clang
+# Install C/C++ compiler: https://wiki.python.org/moin/CompilingPython
 
 classifiers = [
     "Development Status :: 3 - Alpha",
@@ -43,33 +42,31 @@ Examples = "https://github.com/vypivshiy/selector_schema_codegen/examples"
 
 
 [project.scripts]
-ssc-gen = 'ssc_codegen.cli.main:main'
+ssc-gen = 'ssc_codegen.main:main'
 
 [build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
+requires = ["setuptools>=62.4.0", "wheel"]
+build-backend = "setuptools.build_meta"
 
-[tool.hatch.build.targets.sdist]
-include = [
-  "ssc_codegen/*",
-]
+[tool.setuptools.packages.find]
+include = ["ssc_codegen*"]
+
+[tool.setuptools.package-data]
+ssc_codegen = ["py.typed"]
 
 [dependency-groups]
 dev = [
+    "bs4>=0.0.2",
     "coverage>=7.6.12",
     "httpx>=0.28.1",
+    "hypothesis>=6.151.9",
+    "lxml>=5.3.0",
     "mypy>=1.14.1",
-    "parsel>=1.10.0",
+    "parsel>=1.9.1",
     "pytest>=8.3.4",
     "ruff>=0.9.3",
     "selectolax>=0.3.27",
 ]
-# python generated code tests
-tests = [
-    "bs4>=0.0.2",
-    "parsel>=1.10.0",
-    "selectolax>=0.3.27",
-]
 
 [tool.ruff]
 target-version = "py310"

ssc_codegen-0.17.0a0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

ssc_codegen-0.17.0a0/setup.py

@@ -0,0 +1,68 @@
+"""
+Setup script for ssc_codegen with tree-sitter-kdl extension.
+Requires C/C++ compiler (MSVC on Windows, gcc/clang on Linux/macOS).
+
+Before installing, initialize the submodule:
+    git submodule update --init
+"""
+
+from pathlib import Path
+from setuptools import setup, Extension
+from setuptools.command.build_ext import build_ext
+
+# Relative paths from setup.py location
+VENDOR_KDL_REL = Path("vendor") / "tree-sitter-kdl"
+VENDOR_KDL_SRC_REL = VENDOR_KDL_REL / "src"
+VENDOR_BINDING_C = VENDOR_KDL_REL / "bindings" / "python" / "tree_sitter_kdl" / "binding.c"
+
+
+class KdlBuildExt(build_ext):
+    """Custom build_ext that builds tree-sitter-kdl from copied sources."""
+
+    def build_extension(self, ext: Extension):
+        # Compiler flags
+        if self.compiler.compiler_type != "msvc":
+            ext.extra_compile_args = ["-std=c11", "-fvisibility=hidden", "-O2"]
+        else:
+            ext.extra_compile_args = ["/std:c11", "/utf-8", "/O2"]
+
+        # Add scanner.c if present (relative to setup.py)
+        scanner_c = VENDOR_KDL_SRC_REL / "scanner.c"
+        if scanner_c.exists():
+            ext.sources.append(str(scanner_c))
+
+        # Py_LIMITED_API for abi3 wheels
+        if ext.py_limited_api:
+            ext.define_macros.append(("Py_LIMITED_API", "0x030A0000"))
+
+        super().build_extension(ext)
+
+
+def get_kdl_extension():
+    """Create Extension for tree-sitter-kdl."""
+    parser_c = VENDOR_KDL_SRC_REL / "parser.c"
+    if not parser_c.exists():
+        raise RuntimeError(
+            f"tree-sitter-kdl sources not found at {VENDOR_KDL_SRC_REL}.\n"
+            "Initialize the git submodule before installing:\n"
+            "  git submodule update --init"
+        )
+
+    if not VENDOR_BINDING_C.exists():
+        raise RuntimeError(f"binding.c not found at {VENDOR_BINDING_C}")
+
+    return Extension(
+        name="ssc_codegen.linter._binding",
+        sources=[str(VENDOR_BINDING_C), str(parser_c)],
+        include_dirs=[str(VENDOR_KDL_SRC_REL)],
+        define_macros=[
+            ("PY_SSIZE_T_CLEAN", None),
+            ("TREE_SITTER_HIDE_SYMBOLS", None),
+        ],
+    )
+
+
+setup(
+    cmdclass={"build_ext": KdlBuildExt},
+    ext_modules=[get_kdl_extension()],
+)

ssc_codegen-0.17.0a0/ssc_codegen/__init__.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+
+from ssc_codegen.parser import Module, PARSER
+
+_KDL_TEXT_ENCODING = "utf-8-sig"
+
+
+def parse_ast(
+    src: str | None = None,
+    path: str | None = None,
+    *,
+    css_to_xpath: bool = False,
+) -> Module:
+    if not src and not path:
+        raise AttributeError("required src or path argument")
+    source_path: Path | None = None
+    if path:
+        source_path = Path(path).resolve()
+        src = source_path.read_text(encoding=_KDL_TEXT_ENCODING)
+    if not src:
+        raise AttributeError("required src or path argument")
+    module = PARSER.parse(src, source_path=source_path)
+    if css_to_xpath:
+        from ssc_codegen.document_utils import convert_css_to_xpath_module
+
+        convert_css_to_xpath_module(module)
+    return module

ssc_codegen-0.17.0a0/ssc_codegen/_logging.py

@@ -0,0 +1,102 @@
+"""Logging configuration for ssc_codegen.
+
+Usage
+-----
+Import the logger in any submodule::
+
+    from ssc_codegen._logging import logger
+
+To enable DEBUG output from the CLI, pass ``--verbose`` / ``-v`` flag,
+or configure it manually::
+
+    import logging
+    logging.getLogger("ssc_codegen").setLevel(logging.DEBUG)
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+
+# Windows ANSI backport
+try:
+    import colorama
+
+    colorama.init(autoreset=False)
+    _COLORAMA_AVAILABLE = True
+except ImportError:
+    _COLORAMA_AVAILABLE = False
+
+# ANSI color codes (used on all platforms; colorama translates them on Windows)
+_RESET = "\033[0m"
+_BOLD = "\033[1m"
+_COLORS: dict[int, str] = {
+    logging.DEBUG: "\033[36m",  # cyan
+    logging.INFO: "\033[32m",  # green
+    logging.WARNING: "\033[33m",  # yellow
+    logging.ERROR: "\033[31m",  # red
+    logging.CRITICAL: "\033[35m",  # magenta
+}
+
+
+class _ColorFormatter(logging.Formatter):
+    """Formatter that wraps the level name in ANSI color codes."""
+
+    _FMT = "[{color}{bold}{level}{reset}] {name}: {message}"
+
+    def format(self, record: logging.LogRecord) -> str:  # noqa: A003
+        color = _COLORS.get(record.levelno, "")
+        level = record.levelname
+        name = record.name
+        # format the message part the normal way (handles exc_info etc.)
+        record.message = record.getMessage()
+        if record.exc_info and not record.exc_text:
+            record.exc_text = self.formatException(record.exc_info)
+
+        msg = self._FMT.format(
+            color=color,
+            bold=_BOLD,
+            level=level,
+            reset=_RESET,
+            name=name,
+            message=record.message,
+        )
+        if record.exc_text:
+            msg = f"{msg}\n{record.exc_text}"
+        return msg
+
+
+# Single named logger for the entire ssc_codegen package.
+# All child loggers (parser, main, …) are children of this one,
+# so a single ``logging.getLogger("ssc_codegen").setLevel(DEBUG)``
+# enables everything at once.
+logger = logging.getLogger("ssc_codegen")
+
+
+def setup_debug_logging() -> None:
+    """Enable DEBUG-level logging to stderr for the ssc_codegen logger.
+
+    Called by the CLI when ``--verbose`` is passed.
+    Idempotent: calling multiple times is safe.
+    """
+    pkg_logger = logging.getLogger("ssc_codegen")
+    if pkg_logger.level > logging.DEBUG or pkg_logger.level == logging.NOTSET:
+        pkg_logger.setLevel(logging.DEBUG)
+
+    # Avoid adding duplicate handlers if already configured
+    if not any(
+        isinstance(h, logging.StreamHandler) for h in pkg_logger.handlers
+    ):
+        # Use colors only when stderr is a real TTY or colorama is available
+        use_color = _COLORAMA_AVAILABLE or (
+            hasattr(sys.stderr, "isatty") and sys.stderr.isatty()
+        )
+        handler = logging.StreamHandler()
+        handler.setLevel(logging.DEBUG)
+        formatter: logging.Formatter = (
+            _ColorFormatter()
+            if use_color
+            else logging.Formatter("[%(levelname)s] %(name)s: %(message)s")
+        )
+        handler.setFormatter(formatter)
+        pkg_logger.addHandler(handler)

ssc_codegen-0.17.0a0/ssc_codegen/ast/__init__.py

@@ -0,0 +1,169 @@
+"""
+AST nodes for the KDL Schema DSL.
+
+Import everything from here:
+    from kdl_ast import Module, Field, CssSelect, ...
+"""
+from .types import VariableType, StructType
+
+from .base import Node
+
+from .module import (
+    Module,
+    CodeStartHook,
+    CodeEndHook,
+    Docstring,
+    Imports,
+    Utilities,
+)
+
+from .typedef import TypeDef, TypeDefField
+
+from .jsondef import JsonDef, JsonDefField
+
+from .struct import (
+    Struct,
+    StructDocstring,
+    PreValidate,
+    Init,
+    InitField,
+    SplitDoc,
+    Key,
+    Value,
+    TableConfig,
+    TableRow,
+    TableMatchKey,
+    Field,
+    StartParse
+)
+
+from .selectors import (
+    CssSelect,
+    CssSelectAll,
+    XpathSelect,
+    XpathSelectAll,
+    CssRemove,
+    XpathRemove,
+)
+
+from .extract import Text, Raw, Attr
+
+from .string import (
+    Trim,
+    Ltrim,
+    Rtrim,
+    NormalizeSpace,
+    RmPrefix,
+    RmSuffix,
+    RmPrefixSuffix,
+    Fmt,
+    Repl,
+    ReplMap,
+    Lower,
+    Upper,
+    Split,
+    Join,
+    Unescape,
+)
+
+from .regex import Re, ReAll, ReSub
+
+from .array import Index, Slice, Len, Unique
+
+from .cast import ToInt, ToFloat, ToBool, Jsonify, Nested
+
+from .control import Self, Fallback, FallbackStart, FallbackEnd, Return
+
+from .predicate_containers import Filter, Assert, Match
+
+from .predicate_ops import (
+    PredEq,
+    PredNe,
+    PredGt,
+    PredLt,
+    PredGe,
+    PredLe,
+    PredRange,
+    PredStarts,
+    PredEnds,
+    PredContains,
+    PredIn,
+    PredRe,
+    PredReAny,
+    PredReAll,
+    PredCss,
+    PredXpath,
+    PredHasAttr,
+    PredCountEq,
+    PredCountGt,
+    PredCountLt,
+    PredAttrEnds,
+    PredAttrEq,
+    PredAttrNe,
+    PredAttrRe,
+    PredAttrStarts,
+    PredAttrContains,
+    PredTextContains,
+    PredTextEnds,
+    PredTextRe,
+    PredTextStarts,
+    LogicNot,
+    LogicAnd,
+    LogicOr,
+)
+
+from .transform import TransformDef, TransformTarget, TransformCall
+
+__all__ = [
+    # types
+    "VariableType", "StructType",
+    # base
+    "Node",
+    # module
+    "Module", "CodeStartHook", "CodeEndHook",
+    "Docstring", "Imports", "Utilities",
+    # typedef
+    "TypeDef", "TypeDefField",
+    # jsondef
+    "JsonDef", "JsonDefField",
+    # struct
+    "Struct", "StructDocstring", "PreValidate",
+    "Init", "InitField", "SplitDoc",
+    "Key", "Value",
+    "TableConfig", "TableRow", "TableMatchKey",
+    "Field", "StartParse",
+    # selectors
+    "CssSelect", "CssSelectAll",
+    "XpathSelect", "XpathSelectAll",
+    "CssRemove", "XpathRemove",
+    # extract
+    "Text", "Raw", "Attr",
+    # string
+    "Trim", "Ltrim", "Rtrim", "NormalizeSpace",
+    "RmPrefix", "RmSuffix", "RmPrefixSuffix",
+    "Fmt", "Repl", "ReplMap",
+    "Lower", "Upper", "Split", "Join", "Unescape",
+    # regex
+    "Re", "ReAll", "ReSub",
+    # array
+    "Index", "Slice", "Len", "Unique",
+    # cast
+    "ToInt", "ToFloat", "ToBool", "Jsonify", "Nested",
+    # control
+    "Self", "Fallback", "FallbackStart", "FallbackEnd", "Return",
+    # predicate containers
+    "Filter", "Assert", "Match",
+    # predicate ops
+    "PredEq", "PredNe",
+    "PredGt", "PredLt", "PredGe", "PredLe", "PredRange",
+    "PredStarts", "PredEnds", "PredContains", "PredIn",
+    "PredRe", "PredReAny", "PredReAll",
+    "PredCss", "PredXpath", "PredHasAttr",
+    "PredAttrEq", "PredAttrNe",
+    "PredAttrStarts", "PredAttrEnds", "PredAttrContains", "PredAttrRe",
+    "PredTextStarts", "PredTextEnds", "PredTextContains", "PredTextRe",
+    "PredCountEq", "PredCountGt", "PredCountLt",
+    "LogicNot", "LogicAnd", "LogicOr",
+    # transform
+    "TransformDef", "TransformTarget", "TransformCall",
+]

ssc_codegen-0.17.0a0/ssc_codegen/ast/array.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+
+from .base import Node
+from .types import VariableType
+
+# Index, First, Last, Slice accept LIST_AUTO / return AUTO or LIST_AUTO.
+# Concrete types are resolved by the builder from the cursor type via
+# VariableType.scalar / VariableType.as_list helpers.
+
+
+@dataclass
+class Index(Node):
+    """
+    Returns element at position i.
+    Negative index counts from end.
+    LIST_AUTO → AUTO (resolved by builder).
+    """
+
+    i: int = 0
+    accept: VariableType = field(default=VariableType.LIST_AUTO)
+    ret: VariableType = field(default=VariableType.AUTO)
+
+
+@dataclass
+class Slice(Node):
+    """
+    Returns sublist [start:end].
+    LIST_AUTO → LIST_AUTO (resolved by builder).
+    """
+
+    start: int = 0
+    end: int = 0
+    accept: VariableType = field(default=VariableType.LIST_AUTO)
+    ret: VariableType = field(default=VariableType.LIST_AUTO)
+
+
+@dataclass
+class Len(Node):
+    """Returns list length as INT."""
+
+    accept: VariableType = field(default=VariableType.LIST_AUTO)
+    ret: VariableType = field(default=VariableType.INT)
+
+
+@dataclass
+class Unique(Node):
+    """
+    Removes duplicate strings from list.
+    keep_order=True — preserves original order (default: False).
+    LIST_STRING → LIST_STRING.
+    """
+
+    keep_order: bool = False
+    accept: VariableType = field(default=VariableType.LIST_STRING)
+    ret: VariableType = field(default=VariableType.LIST_STRING)