ssc_codegen 0.6.1__tar.gz → 0.7.1__tar.gz

ssc_codegen-0.7.1/PKG-INFO

@@ -0,0 +1,238 @@
+Metadata-Version: 2.4
+Name: ssc_codegen
+Version: 0.7.1
+Summary: Python-dsl code converter to html parser for web scraping 
+Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
+Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
+Project-URL: Source, https://github.com/vypivshiy/selector_schema_codegen
+Project-URL: Examples, https://github.com/vypivshiy/selector_schema_codegen/examples
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Code Generators
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Requires-Dist: cssselect>=1.2.0
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: ichrome>=4.0.4
+Requires-Dist: lxml>=5.3.0
+Requires-Dist: parsel>=1.10.0
+Requires-Dist: typer>=0.15.1
+Requires-Dist: typing-extensions; python_version < '3.11'
+Description-Content-Type: text/markdown
+
+# Selector Schema codegen
+
+## Introduction
+
+ssc-gen - based-python DSL language for writing html parsers in dataclass style for converting to targeting language.
+
+Project solving next problems:
+
+- boilerplate code
+- create types (type annotations) and documentation 
+- simplify code support
+- portability to other languages
+
+## Support converters
+
+Current support converters
+
+
+| Language      | Library (html parser backend)                                | XPath Support | CSS Support | Generated types                          | Code formatter |
+|---------------|--------------------------------------------------------------|---------------|-------------|------------------------------------------|----------------|
+| Python (3.8+) | bs4                                                          | N             | Y           | TypedDict*, list, dict                   | ruff           |
+| ...           | parsel                                                       | Y             | Y           | ...                                      | -              |
+| ...           | selectolax (modest)                                          | N             | Y           | ...                                      | -              |
+| ...           | scrapy (possibly use parsel - pass Response.selector object) | Y             | Y           | ...                                      | -              |
+| Dart (3)      | universal_html                                               | N             | Y           | record, List, Map                        | dart format    |
+| js (ES6)      | pure (firefox/chrome)                                        | Y             | Y           | Array, Map**                             | -              |
+| go (1.10+)    | goquery                                                      | N             | Y           | struct(json anchors include), array, map | gofmt          |
+
+- *this annotation type was deliberately chosen as a compromise reasons. 
+Python has many ways of serialization: `dataclass, namedtuple, attrs, pydantic`
+  - TypedDict is like a build-in dict, but with IDE and linter hint support, and you can easily implement an adapter for the required structure.
+- **js not exists build-in serialization methods
+
+### Limitations
+
+For maximum portability of the configuration to the target language:
+
+- Use CSS selectors: they are guaranteed to be converted to XPATH
+- Unlike javascript, most html parse libs implement [CSS3 selectors standard](https://www.w3.org/TR/selectors-3/)
+  - basic selectors: (`tag`, `.class`, `#id`)
+  - combined: (`div p`, `ul > li`, `h2 +p`\[1])
+  - attribute: (`a[href]`, `input[type='text']`)\[2]
+  - pseudo classes: (`:nth-child(n)`, `:first-child`, `:last-child`)\[3]
+  - **often, not support more complex, dynamic styles**: (`:has()`, `:nth-of-type()`, `:where()`, `:is()`)
+
+1. Several libs not support `+` operations (eg: [selectolax(modest)](https://github.com/rushter/selectolax), [dart.universal_html](https://pub.dev/packages/universal_html))
+2. Often, web scraping libs not supports attribute operations like `*=`, `~=`, `|=`, `^=` and `$=`
+3. Several libs not support pseudo classes (eg: standard [dart.html](https://dart.dev/libraries/dart-html) lib miss this feature). 
+This project will not implement converters with such a cons
+
+## Getting started
+
+ssc_gen required python 3.10 version or higher
+
+### Install
+
+pip:
+
+```shell
+pip install ssc_codegen
+```
+
+uv:
+
+```shell
+uv pip install ssc_codegen
+```
+
+as cli converter tool:
+
+| package manager | command                       |
+|-----------------|-------------------------------|
+| pipx            | `pipx install ssc_codegen`    |
+| uv              | `uv tool install ssc_codegen` |
+
+## Example
+
+### Create a file `schema.py` with:
+
+```python
+from ssc_codegen import ItemSchema, D
+
+class HelloWorld(ItemSchema):
+    title = D().css('title').text()
+    a_hrefs = D().css_all('a').attr('href')
+```
+
+### try it in cli
+
+>[!note]
+> this tools developed for testing purposes, not for web-scraping
+
+### from file
+
+>[!warning]
+> DO NOT PASS CONFIGS FROM UNKNOWN SOURCES: 
+> 
+> PYTHON CODE FROM CONFIGS COMPILE IN RUNTIME WOUT SECURITY CHECKS!!!
+
+Download any html file and pass as argument:
+
+```shell
+ssc-gen parse-from-file index.html -t schema.py:HelloWorld  
+```
+
+Short options descriptions:
+
+- `-t --target` - config schema file and class from where to start the parser
+
+![out1](docs/assets/parse_from_file.gif)
+
+### from url
+
+```shell
+ssc-gen parse-from-url https://example.com -t schema.py:HelloWorld  
+```
+
+![out1](docs/assets/parse_from_url.gif)
+### from Chromium browser (CDP protocol)
+
+
+```shell
+ssc-gen parse-from-chrome https://example.com -t schema.py:HelloWorld
+```
+
+>[!note]
+> if script cannot found chrome executable - provide it manually:
+
+```shell
+ssc-gen parse-from-chrome https://example.com -t schema.py:HelloWorld -sc /usr/bin/chromium
+```
+
+
+### Convert to code
+
+Convert to code for use in projects:
+
+>![note]
+> for example, used js: it can be fast test in developer console
+
+
+```shell
+ssc-gen js schema.py -o .
+```
+
+Code output looks like this (code formatted by IDE):
+
+```javascript
+// autogenerated by ssc-gen DO NOT_EDIT
+/**
+ *
+ *
+ * {
+ *     "title": "String",
+ *     "a_hrefs": "Array<String>"
+ * }
+ */
+class HelloWorld {
+    constructor(doc) {
+        if (typeof doc === 'string') {
+            this._doc = new DOMParser().parseFromString(doc, 'text/html');
+        } else if (doc instanceof Document || doc instanceof Element) {
+            this._doc = doc;
+        } else {
+            throw new Error("Invalid input: Expected a Document, Element, or string");
+        }
+    }
+
+    _parseTitle(value) {
+        let value1 = value.querySelector('title');
+        return typeof value1.textContent === "undefined" ? value1.documentElement.textContent : value1.textContent;
+    }
+
+    _parseAHrefs(value) {
+        let value1 = Array.from(value.querySelectorAll('a'));
+        return value1.map(e => e.getAttribute('href'));
+    }
+
+    parse() {
+        return {title: this._parseTitle(this._doc), a_hrefs: this._parseAHrefs(this._doc)};
+    }
+}
+```
+
+### copy code output and past to developer console:
+
+Print output:
+
+```javascript
+alert(JSON.stringify((new HelloWorld(document).parse())))
+```
+
+![example](assets/example.png)
+
+
+You can use any html source:
+
+- read from html file
+- get from http request
+- get from browser (playwright, selenium, chrome-cdp)
+- paste code to developer console (js)
+- or call curl in shell and parse stdin
+
+
+## See also
+- [Brief](docs/brief.md) about css selectors and regular expressions.
+- [Tutorial](docs/tutorial.md) how to use ssc-gen
+- [Reference](docs/reference.md) about high-level API

ssc_codegen-0.7.1/README.md

@@ -0,0 +1,208 @@
+# Selector Schema codegen
+
+## Introduction
+
+ssc-gen - based-python DSL language for writing html parsers in dataclass style for converting to targeting language.
+
+Project solving next problems:
+
+- boilerplate code
+- create types (type annotations) and documentation 
+- simplify code support
+- portability to other languages
+
+## Support converters
+
+Current support converters
+
+
+| Language      | Library (html parser backend)                                | XPath Support | CSS Support | Generated types                          | Code formatter |
+|---------------|--------------------------------------------------------------|---------------|-------------|------------------------------------------|----------------|
+| Python (3.8+) | bs4                                                          | N             | Y           | TypedDict*, list, dict                   | ruff           |
+| ...           | parsel                                                       | Y             | Y           | ...                                      | -              |
+| ...           | selectolax (modest)                                          | N             | Y           | ...                                      | -              |
+| ...           | scrapy (possibly use parsel - pass Response.selector object) | Y             | Y           | ...                                      | -              |
+| Dart (3)      | universal_html                                               | N             | Y           | record, List, Map                        | dart format    |
+| js (ES6)      | pure (firefox/chrome)                                        | Y             | Y           | Array, Map**                             | -              |
+| go (1.10+)    | goquery                                                      | N             | Y           | struct(json anchors include), array, map | gofmt          |
+
+- *this annotation type was deliberately chosen as a compromise reasons. 
+Python has many ways of serialization: `dataclass, namedtuple, attrs, pydantic`
+  - TypedDict is like a build-in dict, but with IDE and linter hint support, and you can easily implement an adapter for the required structure.
+- **js not exists build-in serialization methods
+
+### Limitations
+
+For maximum portability of the configuration to the target language:
+
+- Use CSS selectors: they are guaranteed to be converted to XPATH
+- Unlike javascript, most html parse libs implement [CSS3 selectors standard](https://www.w3.org/TR/selectors-3/)
+  - basic selectors: (`tag`, `.class`, `#id`)
+  - combined: (`div p`, `ul > li`, `h2 +p`\[1])
+  - attribute: (`a[href]`, `input[type='text']`)\[2]
+  - pseudo classes: (`:nth-child(n)`, `:first-child`, `:last-child`)\[3]
+  - **often, not support more complex, dynamic styles**: (`:has()`, `:nth-of-type()`, `:where()`, `:is()`)
+
+1. Several libs not support `+` operations (eg: [selectolax(modest)](https://github.com/rushter/selectolax), [dart.universal_html](https://pub.dev/packages/universal_html))
+2. Often, web scraping libs not supports attribute operations like `*=`, `~=`, `|=`, `^=` and `$=`
+3. Several libs not support pseudo classes (eg: standard [dart.html](https://dart.dev/libraries/dart-html) lib miss this feature). 
+This project will not implement converters with such a cons
+
+## Getting started
+
+ssc_gen required python 3.10 version or higher
+
+### Install
+
+pip:
+
+```shell
+pip install ssc_codegen
+```
+
+uv:
+
+```shell
+uv pip install ssc_codegen
+```
+
+as cli converter tool:
+
+| package manager | command                       |
+|-----------------|-------------------------------|
+| pipx            | `pipx install ssc_codegen`    |
+| uv              | `uv tool install ssc_codegen` |
+
+## Example
+
+### Create a file `schema.py` with:
+
+```python
+from ssc_codegen import ItemSchema, D
+
+class HelloWorld(ItemSchema):
+    title = D().css('title').text()
+    a_hrefs = D().css_all('a').attr('href')
+```
+
+### try it in cli
+
+>[!note]
+> this tools developed for testing purposes, not for web-scraping
+
+### from file
+
+>[!warning]
+> DO NOT PASS CONFIGS FROM UNKNOWN SOURCES: 
+> 
+> PYTHON CODE FROM CONFIGS COMPILE IN RUNTIME WOUT SECURITY CHECKS!!!
+
+Download any html file and pass as argument:
+
+```shell
+ssc-gen parse-from-file index.html -t schema.py:HelloWorld  
+```
+
+Short options descriptions:
+
+- `-t --target` - config schema file and class from where to start the parser
+
+![out1](docs/assets/parse_from_file.gif)
+
+### from url
+
+```shell
+ssc-gen parse-from-url https://example.com -t schema.py:HelloWorld  
+```
+
+![out1](docs/assets/parse_from_url.gif)
+### from Chromium browser (CDP protocol)
+
+
+```shell
+ssc-gen parse-from-chrome https://example.com -t schema.py:HelloWorld
+```
+
+>[!note]
+> if script cannot found chrome executable - provide it manually:
+
+```shell
+ssc-gen parse-from-chrome https://example.com -t schema.py:HelloWorld -sc /usr/bin/chromium
+```
+
+
+### Convert to code
+
+Convert to code for use in projects:
+
+>![note]
+> for example, used js: it can be fast test in developer console
+
+
+```shell
+ssc-gen js schema.py -o .
+```
+
+Code output looks like this (code formatted by IDE):
+
+```javascript
+// autogenerated by ssc-gen DO NOT_EDIT
+/**
+ *
+ *
+ * {
+ *     "title": "String",
+ *     "a_hrefs": "Array<String>"
+ * }
+ */
+class HelloWorld {
+    constructor(doc) {
+        if (typeof doc === 'string') {
+            this._doc = new DOMParser().parseFromString(doc, 'text/html');
+        } else if (doc instanceof Document || doc instanceof Element) {
+            this._doc = doc;
+        } else {
+            throw new Error("Invalid input: Expected a Document, Element, or string");
+        }
+    }
+
+    _parseTitle(value) {
+        let value1 = value.querySelector('title');
+        return typeof value1.textContent === "undefined" ? value1.documentElement.textContent : value1.textContent;
+    }
+
+    _parseAHrefs(value) {
+        let value1 = Array.from(value.querySelectorAll('a'));
+        return value1.map(e => e.getAttribute('href'));
+    }
+
+    parse() {
+        return {title: this._parseTitle(this._doc), a_hrefs: this._parseAHrefs(this._doc)};
+    }
+}
+```
+
+### copy code output and past to developer console:
+
+Print output:
+
+```javascript
+alert(JSON.stringify((new HelloWorld(document).parse())))
+```
+
+![example](assets/example.png)
+
+
+You can use any html source:
+
+- read from html file
+- get from http request
+- get from browser (playwright, selenium, chrome-cdp)
+- paste code to developer console (js)
+- or call curl in shell and parse stdin
+
+
+## See also
+- [Brief](docs/brief.md) about css selectors and regular expressions.
+- [Tutorial](docs/tutorial.md) how to use ssc-gen
+- [Reference](docs/reference.md) about high-level API

{ssc_codegen-0.6.1 → ssc_codegen-0.7.1}/pyproject.toml

@@ -1,14 +1,31 @@
 [project]
 name = "ssc_codegen"
-version = "0.6.1"
-description = "Add your description here"
+version = "0.7.1"
+description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
     "cssselect>=1.2.0",
+    "httpx>=0.28.1",
+    "ichrome>=4.0.4",
     "lxml>=5.3.0",
+    "parsel>=1.10.0",
     "typer>=0.15.1",
-    "typing_extensions;python_version<'3.11'"
+    "typing_extensions;python_version<'3.11'",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Topic :: Software Development :: Code Generators",
+    "Topic :: Text Processing :: Markup :: HTML",
+    "Topic :: Utilities",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+
 ]
 
 [project.urls]
@@ -25,12 +42,14 @@ ssc-gen = 'ssc_codegen.cli.main:main'
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
-[tool.setuptools]
-license-files = []
-packages=["ssc_codegen"]
+[tool.hatch.build.targets.sdist]
+include = [
+  "ssc_codegen/*",
+]
 
 [dependency-groups]
 dev = [
+    "coverage>=7.6.12",
     "httpx>=0.28.1",
     "mypy>=1.14.1",
     "pytest>=8.3.4",
@@ -40,7 +59,6 @@ dev = [
 tests = [
     "bs4>=0.0.2",
     "parsel>=1.10.0",
-    "pytest>=8.3.4",
     "selectolax>=0.3.27",
 ]
 

{ssc_codegen-0.6.1 → ssc_codegen-0.7.1}/ssc_codegen/__init__.py

@@ -3,7 +3,7 @@ from .document import HTMLDocument, StringDocument, ArrayDocument, AssertDocumen
 from .json_struct import Json
 from .schema import ItemSchema, DictSchema, ListSchema, FlatListSchema
 
-VERSION = "0.6.1"
+VERSION = "0.7.1"
 
 class __MISSING(object):
     pass
@@ -18,14 +18,14 @@ class Nested(HTMLDocument, NestedDocument, ArrayDocument, AssertDocument):
     pass
 
 
-def D(default_value: None | str | int | float | __MISSING =_NO_DEFAULT) -> Document:  # noqa
+def D(default: None | str | int | float | __MISSING =_NO_DEFAULT) -> Document:  # noqa
     """Shortcut as a Document() object
 
-    :param default_value: .default() operator shortcut
+    :param default: .default() operator shortcut
     """
-    if default_value==_NO_DEFAULT:
+    if default==_NO_DEFAULT:
         return Document()
-    return Document().default(value=default_value)  # type: ignore
+    return Document().default(value=default)  # type: ignore
 
 
 def N() -> Nested:  # noqa
@@ -33,12 +33,12 @@ def N() -> Nested:  # noqa
     return Nested()
 
 
-def R(default_value: None | str | int | float | __MISSING =_NO_DEFAULT) -> Document: # noqa
+def R(default: None | str | int | float | __MISSING =_NO_DEFAULT) -> Document: # noqa
     """Shortcut as a Document().raw() object.
     For regex and format string operations
 
-    :param default_value: .default() operator shortcut
+    :param default: .default() operator shortcut
     """
-    if default_value==_NO_DEFAULT:
+    if default==_NO_DEFAULT:
         return Document().raw()
-    return Document().default(default_value).raw()  # type: ignore
+    return Document().default(default).raw()  # type: ignore

{ssc_codegen-0.6.1 → ssc_codegen-0.7.1}/ssc_codegen/ast_build_utils.py

@@ -52,11 +52,31 @@ def check_schema_required_fields(schema: Type[BaseSchema]) -> Type[BaseSchema]:
             _check_required_attributes(
                 schema, "__SPLIT_DOC__", "__KEY__", "__VALUE__"
             )
+            # delete non required fields
+            not_required_fields: list[str] = []
+            for k, v in schema.__get_mro_fields__().items():
+                if k not in {"__SPLIT_DOC__", "__KEY__", "__VALUE__"}:
+                    msg = f"{schema.__name__}.{k} not required, remove"
+                    warnings.warn(msg, category=SyntaxWarning)
+                    not_required_fields.append(k)
+            schema = _remove_non_required_attributes(
+                schema, *not_required_fields
+            )
         case StructType.FLAT_LIST:
             schema = _remove_non_required_attributes(
                 schema, "__KEY__", "__VALUE__"
             )
             _check_required_attributes(schema, "__SPLIT_DOC__", "__ITEM__")
+
+            not_required_fields: list[str] = []  # type: ignore
+            for k, v in schema.__get_mro_fields__().items():
+                if k not in {"__SPLIT_DOC__", "__ITEM__"}:
+                    msg = f"{schema.__name__}.{k} not required, remove"
+                    warnings.warn(msg, category=SyntaxWarning)
+                    not_required_fields.append(k)
+            schema = _remove_non_required_attributes(
+                schema, *not_required_fields
+            )
         case _:
             msg = f"{schema.__name__}: Unknown schema type"
             raise SyntaxError(msg)

{ssc_codegen-0.6.1 → ssc_codegen-0.7.1}/ssc_codegen/cli/cli_utils.py

@@ -2,7 +2,9 @@ import importlib
 import re
 import warnings
 from pathlib import Path
+from types import ModuleType
 from typing import TYPE_CHECKING, Protocol
+import difflib
 
 from typer import BadParameter
 
@@ -77,3 +79,14 @@ def raw_json_check_keys(jsn: str) -> None:
         raise BadParameter(
             f"bad json keys count: {len(results)}, keys: {all_keys}"
         )
+
+
+def suggest_class_name(module: ModuleType, target_class_name: str) -> str:
+    """try to suggest correct class name"""
+    class_names = [
+        name for name in dir(module) if isinstance(getattr(module, name), type)
+    ]
+
+    return difflib.get_close_matches(
+        target_class_name, class_names, n=3, cutoff=0.6
+    )[0]

ssc_codegen-0.7.1/ssc_codegen/cli/code_callbacks.py

@@ -0,0 +1,38 @@
+""""""
+
+from typing import Callable
+from ssc_codegen.str_utils import (
+    go_unimport_naive,
+    py_optimize_return_naive,
+    py_str_format_to_fstring,
+    js_pure_optimize_return,
+)
+
+
+class BaseCodeCallback:
+    def __init__(
+        self,
+        *code_callbacks: Callable[[str], str],
+        join_sep: str = "\n",
+        remove_empty_lines: bool = False,
+    ):
+        self.join_sep = join_sep
+        self.remove_empty_lines = remove_empty_lines
+        self.callbacks = code_callbacks
+
+    def __call__(self, lines: list[str]) -> str:
+        if self.remove_empty_lines:
+            code = self.join_sep.join([i for i in lines if i])
+        else:
+            code = self.join_sep.join(lines)
+        for cb in self.callbacks:
+            code = cb(code)
+        return code
+
+
+CB_PY_CODE = BaseCodeCallback(
+    py_optimize_return_naive, py_str_format_to_fstring
+)
+CB_GO_CODE = BaseCodeCallback(go_unimport_naive, remove_empty_lines=True)
+CB_DART_CODE = BaseCodeCallback(remove_empty_lines=True)
+CB_JS_CODE = BaseCodeCallback(js_pure_optimize_return)

{ssc_codegen-0.6.1 → ssc_codegen-0.7.1}/ssc_codegen/cli/consts.py

@@ -1,4 +1,4 @@
-from enum import StrEnum
+from ssc_codegen._compat import StrEnum
 
 CONVERTERS_PATH = "ssc_codegen.converters"
 
@@ -40,8 +40,12 @@ HELP_JSON_TO_SCHEMA = "convert json response to ssc-gen schema"
 CMD_VERSION = "Show version and exit"
 CMD_PY = "generate python modules"
 CMD_JS = "generate javascript modules"
-CMD_DART = "generate dart modules (json not implemented)"
+CMD_DART = "generate dart modules"
 CMD_GO = "generate golang modules"
 CMD_JSON_GEN = """generate schema code from json.
-entrypoint should be: stats as dict(map) or array[map] object, not contains reserved words and key starts as _[a-z][A-Z] char
+entrypoint should be: map or array object and key starts as _[a-zA-Z] char
 """
+
+DEFAULT_UA = (
+    "Mozilla/5.0 (X11; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0"
+)