ssc_codegen 0.15.3__tar.gz → 0.17.1__tar.gz

{ssc_codegen-0.15.3 → ssc_codegen-0.17.1}/.gitignore

@@ -3,8 +3,12 @@ __pycache__/
 *.py[cod]
 *$py.class
 
-# C extensions
+# C extensions / native shared libs
 *.so
+*.dll
+*.dylib
+*.exp
+*.lib
 
 # Distribution / packaging
 .Python

ssc_codegen-0.17.1/PKG-INFO

@@ -0,0 +1,155 @@
+Metadata-Version: 2.4
+Name: ssc_codegen
+Version: 0.17.1
+Summary: Python-dsl code converter to html parser for web scraping 
+Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
+Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
+Project-URL: Source, https://github.com/vypivshiy/selector_schema_codegen
+Project-URL: Examples, https://github.com/vypivshiy/selector_schema_codegen/examples
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Code Generators
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Requires-Dist: bs4>=0.0.2
+Requires-Dist: click<8.2.0
+Requires-Dist: colorama>=0.4.6; sys_platform == 'win32'
+Requires-Dist: cssselect>=1.2.0
+Requires-Dist: lxml>=5.3.0
+Requires-Dist: soupsieve>=2.6
+Requires-Dist: typer>=0.15.1
+Requires-Dist: typing-extensions; python_version < '3.11'
+Description-Content-Type: text/markdown
+
+# ssc-codegen
+
+Code generator for web scraping parsers. Describe HTML extraction rules in a declarative KDL 2.0 DSL, then generate ready-to-use parser code for multiple languages and libraries.
+
+```
+.kdl schema --> [kdl parser] --> AST --> [linter] --> [converter] --> output code
+```
+
+## Features
+
+- Declarative DSL based on KDL 2.0 syntax
+- Static type checking and linting before code generation
+- Multiple output targets: Python (bs4, lxml, parsel, selectolax), JavaScript (DOM API)
+- Struct types: `item`, `list`, `dict`, `table`, `flat`
+- LLM-friendly: system prompt + linter loop for AI-assisted schema generation
+
+## Install
+
+```bash
+uv tool install ssc_codegen
+```
+
+## Quick example
+
+`books.kdl`:
+
+```kdl
+struct Book type=list {
+    @split-doc { css-all ".product-card" }
+
+    title { css ".title"; text }
+    price { css ".price"; text; re #"(\d+\.\d+)"#; to-float }
+    url   { css "a[href]"; attr "href"; fallback #null }
+}
+```
+
+Generate Python parser:
+
+```bash
+ssc-gen generate books.kdl -t py-bs4 -o ./output
+```
+
+## Usage
+
+### Generate code
+
+```bash
+# single file
+ssc-gen generate schema.kdl -t py-bs4 -o ./output
+
+# all .kdl files in a directory
+ssc-gen generate examples/ -t js-pure -o ./output
+
+# with custom package name (for Go and other targets)
+ssc-gen generate schema.kdl -t go-goquery -o ./parsers --package scraper
+```
+
+Targets: `py-bs4`, `py-lxml`, `py-parsel`, `py-slax`, `js-pure`
+
+### Lint schemas
+
+```bash
+# human-readable output
+ssc-gen check schema.kdl
+
+# JSON output (for LLM pipelines)
+ssc-gen check schema.kdl -f json
+
+# check all files in a directory
+ssc-gen check examples/
+```
+
+### Test schema against HTML
+
+```bash
+# from file
+ssc-gen run examples/booksToScrape.kdl:MainCatalogue -t py-bs4 -i page.html
+
+# from stdin
+curl https://books.toscrape.com/ | ssc-gen run examples/booksToScrape.kdl:MainCatalogue -t py-bs4
+```
+
+### Health check (verify selectors match elements)
+
+```bash
+# from file
+ssc-gen health examples/booksToScrape.kdl:MainCatalogue -i page.html
+
+# from stdin
+curl https://books.toscrape.com/ | ssc-gen health examples/booksToScrape.kdl:MainCatalogue
+```
+
+## Documentation
+
+- [Quick start](docs2/guide.md)
+- [Syntax and file structure](docs2/syntax.md)
+- [Type system](docs2/types.md)
+- [Pipeline operations](docs2/operations.md)
+- [Predicates and logic](docs2/predicates.md)
+- [JSON schemas and jsonify](docs2/json.md)
+- [Transforms and dsl blocks](docs2/transforms.md)
+- [LLM-compact reference](docs2/llm.txt) -- full DSL spec in one file for LLM context
+- [Examples](examples/)
+
+## LLM integration
+
+LLM agents can generate and validate `.kdl` schemas automatically using the linter feedback loop.
+
+### In chats (ChatGPT, Claude, etc.)
+
+Use [SYSTEM_PROMPT.md](SYSTEM_PROMPT.md) as system prompt. After generation, run `ssc-gen check -f json` and send errors back to the LLM for correction.
+
+### In AI-powered IDEs (Claude Code, Cursor, etc.)
+
+Use the [kdl-schema-dsl](.agents/skills/kdl-schema-dsl) skill for automatic generation, validation, and iteration.
+
+## Development
+
+```bash
+uv sync                  # install dependencies
+uv build --wheel         # build wheel
+uv run pytest            # run tests
+uv run ruff check ssc_codegen/
+```

ssc_codegen-0.17.1/README.md

@@ -0,0 +1,124 @@
+# ssc-codegen
+
+Code generator for web scraping parsers. Describe HTML extraction rules in a declarative KDL 2.0 DSL, then generate ready-to-use parser code for multiple languages and libraries.
+
+```
+.kdl schema --> [kdl parser] --> AST --> [linter] --> [converter] --> output code
+```
+
+## Features
+
+- Declarative DSL based on KDL 2.0 syntax
+- Static type checking and linting before code generation
+- Multiple output targets: Python (bs4, lxml, parsel, selectolax), JavaScript (DOM API)
+- Struct types: `item`, `list`, `dict`, `table`, `flat`
+- LLM-friendly: system prompt + linter loop for AI-assisted schema generation
+
+## Install
+
+```bash
+uv tool install ssc_codegen
+```
+
+## Quick example
+
+`books.kdl`:
+
+```kdl
+struct Book type=list {
+    @split-doc { css-all ".product-card" }
+
+    title { css ".title"; text }
+    price { css ".price"; text; re #"(\d+\.\d+)"#; to-float }
+    url   { css "a[href]"; attr "href"; fallback #null }
+}
+```
+
+Generate Python parser:
+
+```bash
+ssc-gen generate books.kdl -t py-bs4 -o ./output
+```
+
+## Usage
+
+### Generate code
+
+```bash
+# single file
+ssc-gen generate schema.kdl -t py-bs4 -o ./output
+
+# all .kdl files in a directory
+ssc-gen generate examples/ -t js-pure -o ./output
+
+# with custom package name (for Go and other targets)
+ssc-gen generate schema.kdl -t go-goquery -o ./parsers --package scraper
+```
+
+Targets: `py-bs4`, `py-lxml`, `py-parsel`, `py-slax`, `js-pure`
+
+### Lint schemas
+
+```bash
+# human-readable output
+ssc-gen check schema.kdl
+
+# JSON output (for LLM pipelines)
+ssc-gen check schema.kdl -f json
+
+# check all files in a directory
+ssc-gen check examples/
+```
+
+### Test schema against HTML
+
+```bash
+# from file
+ssc-gen run examples/booksToScrape.kdl:MainCatalogue -t py-bs4 -i page.html
+
+# from stdin
+curl https://books.toscrape.com/ | ssc-gen run examples/booksToScrape.kdl:MainCatalogue -t py-bs4
+```
+
+### Health check (verify selectors match elements)
+
+```bash
+# from file
+ssc-gen health examples/booksToScrape.kdl:MainCatalogue -i page.html
+
+# from stdin
+curl https://books.toscrape.com/ | ssc-gen health examples/booksToScrape.kdl:MainCatalogue
+```
+
+## Documentation
+
+- [Quick start](docs2/guide.md)
+- [Syntax and file structure](docs2/syntax.md)
+- [Type system](docs2/types.md)
+- [Pipeline operations](docs2/operations.md)
+- [Predicates and logic](docs2/predicates.md)
+- [JSON schemas and jsonify](docs2/json.md)
+- [Transforms and dsl blocks](docs2/transforms.md)
+- [LLM-compact reference](docs2/llm.txt) -- full DSL spec in one file for LLM context
+- [Examples](examples/)
+
+## LLM integration
+
+LLM agents can generate and validate `.kdl` schemas automatically using the linter feedback loop.
+
+### In chats (ChatGPT, Claude, etc.)
+
+Use [SYSTEM_PROMPT.md](SYSTEM_PROMPT.md) as system prompt. After generation, run `ssc-gen check -f json` and send errors back to the LLM for correction.
+
+### In AI-powered IDEs (Claude Code, Cursor, etc.)
+
+Use the [kdl-schema-dsl](.agents/skills/kdl-schema-dsl) skill for automatic generation, validation, and iteration.
+
+## Development
+
+```bash
+uv sync                  # install dependencies
+uv build --wheel         # build wheel
+uv run pytest            # run tests
+uv run ruff check ssc_codegen/
+```

{ssc_codegen-0.15.3 → ssc_codegen-0.17.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.15.3"
+version = "0.17.1"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"
@@ -8,8 +8,6 @@ dependencies = [
     "bs4>=0.0.2",
     "colorama>=0.4.6 ; sys_platform == 'win32'",
     "cssselect>=1.2.0",
-    "httpx>=0.28.1",
-    "ichrome>=4.0.4",
     "lxml>=5.3.0",
     "soupsieve>=2.6",
     "typer>=0.15.1",
@@ -17,8 +15,6 @@ dependencies = [
     # https://github.com/fastapi/typer/discussions/1215
     # https://github.com/fastapi/typer/pull/1145
     "click<8.2.0",
-    "ast-grep-cli>=0.38.1",
-    "tinycss2>=1.4.0",
 ]
 
 classifiers = [
@@ -43,7 +39,7 @@ Examples = "https://github.com/vypivshiy/selector_schema_codegen/examples"
 
 
 [project.scripts]
-ssc-gen = 'ssc_codegen.cli.main:main'
+ssc-gen = 'ssc_codegen.main:main'
 
 [build-system]
 requires = ["hatchling"]
@@ -51,25 +47,25 @@ build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.sdist]
 include = [
-  "ssc_codegen/*",
+  "ssc_codegen/**",
 ]
 
+[tool.hatch.build.targets.wheel]
+packages = ["ssc_codegen"]
+
 [dependency-groups]
 dev = [
+    "bs4>=0.0.2",
     "coverage>=7.6.12",
     "httpx>=0.28.1",
+    "hypothesis>=6.151.9",
+    "lxml>=5.3.0",
     "mypy>=1.14.1",
-    "parsel>=1.10.0",
+    "parsel>=1.9.1",
     "pytest>=8.3.4",
     "ruff>=0.9.3",
     "selectolax>=0.3.27",
 ]
-# python generated code tests
-tests = [
-    "bs4>=0.0.2",
-    "parsel>=1.10.0",
-    "selectolax>=0.3.27",
-]
 
 [tool.ruff]
 target-version = "py310"

ssc_codegen-0.17.1/ssc_codegen/__init__.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+
+from ssc_codegen.parser import Module, PARSER
+
+_KDL_TEXT_ENCODING = "utf-8-sig"
+
+
+def parse_ast(
+    src: str | None = None,
+    path: str | None = None,
+    *,
+    css_to_xpath: bool = False,
+) -> Module:
+    if not src and not path:
+        raise AttributeError("required src or path argument")
+    source_path: Path | None = None
+    if path:
+        source_path = Path(path).resolve()
+        src = source_path.read_text(encoding=_KDL_TEXT_ENCODING)
+    if not src:
+        raise AttributeError("required src or path argument")
+    module = PARSER.parse(src, source_path=source_path)
+    if css_to_xpath:
+        from ssc_codegen.document_utils import convert_css_to_xpath_module
+
+        convert_css_to_xpath_module(module)
+    return module

ssc_codegen-0.17.1/ssc_codegen/_logging.py

@@ -0,0 +1,102 @@
+"""Logging configuration for ssc_codegen.
+
+Usage
+-----
+Import the logger in any submodule::
+
+    from ssc_codegen._logging import logger
+
+To enable DEBUG output from the CLI, pass ``--verbose`` / ``-v`` flag,
+or configure it manually::
+
+    import logging
+    logging.getLogger("ssc_codegen").setLevel(logging.DEBUG)
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+
+# Windows ANSI backport
+try:
+    import colorama
+
+    colorama.init(autoreset=False)
+    _COLORAMA_AVAILABLE = True
+except ImportError:
+    _COLORAMA_AVAILABLE = False
+
+# ANSI color codes (used on all platforms; colorama translates them on Windows)
+_RESET = "\033[0m"
+_BOLD = "\033[1m"
+_COLORS: dict[int, str] = {
+    logging.DEBUG: "\033[36m",  # cyan
+    logging.INFO: "\033[32m",  # green
+    logging.WARNING: "\033[33m",  # yellow
+    logging.ERROR: "\033[31m",  # red
+    logging.CRITICAL: "\033[35m",  # magenta
+}
+
+
+class _ColorFormatter(logging.Formatter):
+    """Formatter that wraps the level name in ANSI color codes."""
+
+    _FMT = "[{color}{bold}{level}{reset}] {name}: {message}"
+
+    def format(self, record: logging.LogRecord) -> str:  # noqa: A003
+        color = _COLORS.get(record.levelno, "")
+        level = record.levelname
+        name = record.name
+        # format the message part the normal way (handles exc_info etc.)
+        record.message = record.getMessage()
+        if record.exc_info and not record.exc_text:
+            record.exc_text = self.formatException(record.exc_info)
+
+        msg = self._FMT.format(
+            color=color,
+            bold=_BOLD,
+            level=level,
+            reset=_RESET,
+            name=name,
+            message=record.message,
+        )
+        if record.exc_text:
+            msg = f"{msg}\n{record.exc_text}"
+        return msg
+
+
+# Single named logger for the entire ssc_codegen package.
+# All child loggers (parser, main, …) are children of this one,
+# so a single ``logging.getLogger("ssc_codegen").setLevel(DEBUG)``
+# enables everything at once.
+logger = logging.getLogger("ssc_codegen")
+
+
+def setup_debug_logging() -> None:
+    """Enable DEBUG-level logging to stderr for the ssc_codegen logger.
+
+    Called by the CLI when ``--verbose`` is passed.
+    Idempotent: calling multiple times is safe.
+    """
+    pkg_logger = logging.getLogger("ssc_codegen")
+    if pkg_logger.level > logging.DEBUG or pkg_logger.level == logging.NOTSET:
+        pkg_logger.setLevel(logging.DEBUG)
+
+    # Avoid adding duplicate handlers if already configured
+    if not any(
+        isinstance(h, logging.StreamHandler) for h in pkg_logger.handlers
+    ):
+        # Use colors only when stderr is a real TTY or colorama is available
+        use_color = _COLORAMA_AVAILABLE or (
+            hasattr(sys.stderr, "isatty") and sys.stderr.isatty()
+        )
+        handler = logging.StreamHandler()
+        handler.setLevel(logging.DEBUG)
+        formatter: logging.Formatter = (
+            _ColorFormatter()
+            if use_color
+            else logging.Formatter("[%(levelname)s] %(name)s: %(message)s")
+        )
+        handler.setFormatter(formatter)
+        pkg_logger.addHandler(handler)

ssc_codegen-0.17.1/ssc_codegen/ast/__init__.py

@@ -0,0 +1,174 @@
+"""
+AST nodes for the KDL Schema DSL.
+
+Import everything from here:
+    from kdl_ast import Module, Field, CssSelect, ...
+"""
+from .types import VariableType, StructType
+
+from .base import Node
+
+from .module import (
+    Module,
+    CodeStartHook,
+    CodeEndHook,
+    Docstring,
+    Imports,
+    Utilities,
+)
+
+from .typedef import TypeDef, TypeDefField
+
+from .jsondef import JsonDef, JsonDefField
+
+from .struct import (
+    Struct,
+    StructDocstring,
+    PreValidate,
+    Init,
+    InitField,
+    SplitDoc,
+    Key,
+    Value,
+    TableConfig,
+    TableRow,
+    TableMatchKey,
+    Field,
+    StartParse
+)
+
+from .selectors import (
+    CssSelect,
+    CssSelectAll,
+    XpathSelect,
+    XpathSelectAll,
+    CssRemove,
+    XpathRemove,
+)
+
+from .extract import Text, Raw, Attr
+
+from .string import (
+    Trim,
+    Ltrim,
+    Rtrim,
+    NormalizeSpace,
+    RmPrefix,
+    RmSuffix,
+    RmPrefixSuffix,
+    Fmt,
+    Repl,
+    ReplMap,
+    Lower,
+    Upper,
+    Split,
+    Join,
+    Unescape,
+)
+
+from .regex import Re, ReAll, ReSub
+
+from .array import Index, Slice, Len, Unique
+
+from .cast import ToInt, ToFloat, ToBool, Jsonify, Nested
+
+from .control import Self, Fallback, FallbackStart, FallbackEnd, Return
+
+from .predicate_containers import Filter, Assert, Match
+
+from .predicate_ops import (
+    PredEq,
+    PredNe,
+    PredGt,
+    PredLt,
+    PredGe,
+    PredLe,
+    PredRange,
+    PredStarts,
+    PredEnds,
+    PredContains,
+    PredIn,
+    PredRe,
+    PredReAny,
+    PredReAll,
+    PredCss,
+    PredXpath,
+    PredHasAttr,
+    PredCountEq,
+    PredCountGt,
+    PredCountLt,
+    PredCountNe,
+    PredCountGe,
+    PredCountLe,
+    PredCountRange,
+    PredAttrEnds,
+    PredAttrEq,
+    PredAttrNe,
+    PredAttrRe,
+    PredAttrStarts,
+    PredAttrContains,
+    PredTextContains,
+    PredTextEnds,
+    PredTextRe,
+    PredTextStarts,
+    LogicNot,
+    LogicAnd,
+    LogicOr,
+)
+
+from .transform import TransformDef, TransformTarget, TransformCall
+
+__all__ = [
+    # types
+    "VariableType", "StructType",
+    # base
+    "Node",
+    # module
+    "Module", "CodeStartHook", "CodeEndHook",
+    "Docstring", "Imports", "Utilities",
+    # typedef
+    "TypeDef", "TypeDefField",
+    # jsondef
+    "JsonDef", "JsonDefField",
+    # struct
+    "Struct", "StructDocstring", "PreValidate",
+    "Init", "InitField", "SplitDoc",
+    "Key", "Value",
+    "TableConfig", "TableRow", "TableMatchKey",
+    "Field", "StartParse",
+    # selectors
+    "CssSelect", "CssSelectAll",
+    "XpathSelect", "XpathSelectAll",
+    "CssRemove", "XpathRemove",
+    # extract
+    "Text", "Raw", "Attr",
+    # string
+    "Trim", "Ltrim", "Rtrim", "NormalizeSpace",
+    "RmPrefix", "RmSuffix", "RmPrefixSuffix",
+    "Fmt", "Repl", "ReplMap",
+    "Lower", "Upper", "Split", "Join", "Unescape",
+    # regex
+    "Re", "ReAll", "ReSub",
+    # array
+    "Index", "Slice", "Len", "Unique",
+    # cast
+    "ToInt", "ToFloat", "ToBool", "Jsonify", "Nested",
+    # control
+    "Self", "Fallback", "FallbackStart", "FallbackEnd", "Return",
+    # predicate containers
+    "Filter", "Assert", "Match",
+    # predicate ops
+    "PredEq", "PredNe",
+    "PredGt", "PredLt", "PredGe", "PredLe", "PredRange",
+    "PredStarts", "PredEnds", "PredContains", "PredIn",
+    "PredRe", "PredReAny", "PredReAll",
+    "PredCss", "PredXpath", "PredHasAttr",
+    "PredAttrEq", "PredAttrNe",
+    "PredAttrStarts", "PredAttrEnds", "PredAttrContains", "PredAttrRe",
+    "PredTextStarts", "PredTextEnds", "PredTextContains", "PredTextRe",
+    "PredCountEq", "PredCountGt", "PredCountLt",
+    "PredCountNe", "PredCountGe", "PredCountLe", "PredCountRange",
+    "LogicNot", "LogicAnd", "LogicOr",
+    # transform
+    "TransformDef", "TransformTarget", "TransformCall",
+]