ssc_codegen 0.1.13__tar.gz → 0.2.0__tar.gz

ssc_codegen-0.2.0/PKG-INFO

@@ -0,0 +1,77 @@
+Metadata-Version: 2.1
+Name: ssc_codegen
+Version: 0.2.0
+Summary: generate selector schemas classes from dsl-like language based on python
+License: MIT
+Author: vypivshiy
+Requires-Python: >=3.10,<4.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Topic :: Internet
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing
+Requires-Dist: Jinja2 (>=3.1.2,<4.0.0)
+Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
+Requires-Dist: cssselect (>=1.2.0,<2.0.0)
+Requires-Dist: lxml (>=4.9.3,<5.0.0)
+Description-Content-Type: text/markdown
+
+# Selector schema codegen
+
+ssc_codegen - generator of parsers for various programming languages (for html priority) using
+python-DSL configurations with built-in declarative language.
+
+Designed to port parsers to various programming languages and libs
+
+# Motivation
+- ~~interesting in practice write DSL-like language~~
+- decrease boilerplate code for web-parsers
+- write once - convert to other mainstream http parser libs
+- minimal operations for easy add another libs and languages in future
+  - include css, xpath, attributes operations, regex, minimal string formatting operations
+- pre validate css/xpath queries and logic before generate code
+- standardisation: generate classes with minimal dependencies and documented parsed signature 
+
+## Install
+
+### pipx (recommended for CLI usage)
+
+```shell
+pipx install ssc_codegen
+```
+
+### pip
+
+```shell
+pip install ssc_codegen
+```
+
+## Supported libs and languages
+
+| language | lib                                                          | xpath | css | formatter   |
+|----------|--------------------------------------------------------------|-------|-----|-------------|
+| python   | bs4                                                          | NO    | YES | black       |
+| -        | parsel                                                       | YES   | YES | -           |
+| -        | selectolax (modest)                                          | NO    | YES | -           |
+| -        | scrapy (based on parsel, but class init argument - Response) | YES   | YES | -           |
+| dart     | universal_html                                               | NO    | YES | dart format |
+
+### Quickstart
+
+see [example](example) and read code with comments
+
+### Recommendations
+
+- usage css selector: they can be **guaranteed** converted to xpath (if target language not support CSS selectors)
+- usage simple operations for more compatibility other libraries. 
+  - Some libraries may not fully support selector specifications
+  - for example, `#product_description+ p` selector in `parsel` works fine, but not works in `selectolax`, `dart` libs
+- there is a xpath to css converter for simple queries **without guarantees of functionality**. 
+For example, in css there is no analogue of `contains` from xpath, etc.
+

ssc_codegen-0.2.0/README.md

@@ -0,0 +1,52 @@
+# Selector schema codegen
+
+ssc_codegen - generator of parsers for various programming languages (for html priority) using
+python-DSL configurations with built-in declarative language.
+
+Designed to port parsers to various programming languages and libs
+
+# Motivation
+- ~~interesting in practice write DSL-like language~~
+- decrease boilerplate code for web-parsers
+- write once - convert to other mainstream http parser libs
+- minimal operations for easy add another libs and languages in future
+  - include css, xpath, attributes operations, regex, minimal string formatting operations
+- pre validate css/xpath queries and logic before generate code
+- standardisation: generate classes with minimal dependencies and documented parsed signature 
+
+## Install
+
+### pipx (recommended for CLI usage)
+
+```shell
+pipx install ssc_codegen
+```
+
+### pip
+
+```shell
+pip install ssc_codegen
+```
+
+## Supported libs and languages
+
+| language | lib                                                          | xpath | css | formatter   |
+|----------|--------------------------------------------------------------|-------|-----|-------------|
+| python   | bs4                                                          | NO    | YES | black       |
+| -        | parsel                                                       | YES   | YES | -           |
+| -        | selectolax (modest)                                          | NO    | YES | -           |
+| -        | scrapy (based on parsel, but class init argument - Response) | YES   | YES | -           |
+| dart     | universal_html                                               | NO    | YES | dart format |
+
+### Quickstart
+
+see [example](example) and read code with comments
+
+### Recommendations
+
+- usage css selector: they can be **guaranteed** converted to xpath (if target language not support CSS selectors)
+- usage simple operations for more compatibility other libraries. 
+  - Some libraries may not fully support selector specifications
+  - for example, `#product_description+ p` selector in `parsel` works fine, but not works in `selectolax`, `dart` libs
+- there is a xpath to css converter for simple queries **without guarantees of functionality**. 
+For example, in css there is no analogue of `contains` from xpath, etc.

{ssc_codegen-0.1.13 → ssc_codegen-0.2.0}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "ssc_codegen"
-version = "0.1.13"
-description = "generate selector schemas classes from yaml config and DSL-lang script"
+version = "0.2.0"
+description = "generate selector schemas classes from dsl-like language based on python"
 authors = ["vypivshiy"]
 readme = "README.md"
 license = "MIT"
@@ -17,6 +17,18 @@ classifiers = [
   "Topic :: Text Processing"
 ]
 
+[tool.poetry.group.dev.dependencies]
+bs4 = "^0.0.2"
+selectolax = "^0.3.20"
+parsel = "^1.8.1"
+pytest = "^8.0.0"
+mypy = "^1.6.1"
+black = "^23.10.1"
+isort = "^5.12.0"
+ruff = "^0.1.4"
+pre-commit = "^3.5.0"
+scrapy = "^2.11.0"
+
 [project.urls]
 Documentation = "https://github.com/vypivshiy/selector_schema_codegen#readme"
 Issues = "https://github.com/vypivshiy/selector_schema_codegen/issues"
@@ -30,16 +42,9 @@ Jinja2 = "^3.1.2"
 cssselect = "^1.2.0"
 lxml = "^4.9.3"
 
-[tool.poetry.dev-dependencies]
-mypy = "^1.6.1"
-black = "^23.10.1"
-isort = "^5.12.0"
-ruff = "^0.1.4"
-pre-commit = "^3.5.0"
-pytest = "^7.4.3"
 
 [tool.poetry.scripts]
-ssc-gen="ssc_codegen.cli:main"
+ssc-gen="ssc_codegen._cli:main"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

ssc_codegen-0.2.0/ssc_codegen/__init__.py

@@ -0,0 +1,4 @@
+from .template import render_code
+from .schemas import BaseSchemaStrategy, ItemSchema, DictSchema, ListSchema
+from .document import Document
+from .validate import assert_

ssc_codegen-0.2.0/ssc_codegen/_cli.py

@@ -0,0 +1,175 @@
+import subprocess
+from pathlib import Path
+from typing import NamedTuple
+import argparse
+import importlib.util
+
+from ssc_codegen import render_code
+from ssc_codegen.converters import py_scrapy, py_bs4, py_parsel, py_selectolax, dart_universal_html
+from ssc_codegen.schemas import BaseSchemaStrategy
+
+
+_CONVERTER_NAMESPACES = {
+        "py.bs4": py_bs4.converter,
+        "py.parsel": py_parsel.converter,
+        "py.selectolax": py_selectolax.converter,
+        "py.scrapy": py_scrapy.converter,
+        "dart": dart_universal_html.converter,
+    }
+
+
+_EXT_PYTONIC = {"py.bs4", "py.parsel", "py.selectolax", "py.scrapy"}
+_EXT_DART = {"dart", }
+
+# formatters
+_FMT_DART = "dart format {}"
+_FMT_PYTHON = "black {}"
+
+
+class ArgsNamespace(NamedTuple):
+    scenario: str
+    lang: str
+    out: str
+    OVERWRITE: bool
+    XPATH_TO_CSS: bool
+    CSS_TO_XPATH: bool
+    XPATH_PREFIX: str
+    NO_FORMAT: bool
+
+
+def parse_arguments() -> ArgsNamespace:
+    parser = argparse.ArgumentParser(
+        prog="Selector schema generator",
+        description="Generate selector schemas from config file",
+        usage="ssc-gen my_conf.py py.bs4 -o out_file",
+    )
+
+    parser.add_argument("scenario", help="ssc_gen python file")
+    parser.add_argument(
+        "lang",
+        choices=list(_CONVERTER_NAMESPACES.keys()),
+        help="Programming language and lib choice",
+    )
+
+    parser.add_argument(
+        "-o",
+        "--out",
+        default=f"schema.{{}}",
+        help="Output directory (default: current working directory/schema.<ext>)",
+    )
+    parser.add_argument(
+        "-y",
+        dest="OVERWRITE",
+        default=False,
+        action="store_true",
+        help="Suggest overwrite file",
+    )
+    parser.add_argument("--xpath-to-css",
+                        dest="XPATH_TO_CSS",
+                        default=False,
+                        action="store_true",
+                        help="Convert xpath selectors to css (work not guaranteed)")
+
+    parser.add_argument("--css-to-xpath",
+                        dest="CSS_TO_XPATH",
+                        default=False,
+                        action="store_true",
+                        help="convert css selectors to xpath")
+    parser.add_argument("--xpath-prefix",
+                        dest="XPATH_PREFIX",
+                        default='descendant-or-self::',
+                        type=str,
+                        help="Xpath prefix for --css-to-xpath autoconverter. Default `descendant-or-self::`")
+
+    parser.add_argument(
+        "--no-format",
+        dest="NO_FORMAT",
+        default=False,
+        action="store_true",
+        help="Disable code format by third party solutions",
+    )
+
+    namespace: ArgsNamespace = parser.parse_args()  # type: ignore
+    return namespace
+
+
+def _extract_schemas(path: str):
+    spec = importlib.util.spec_from_file_location("script_module", path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    schemas: list[BaseSchemaStrategy] = []
+    # Check if __all__ is defined in the module
+    if hasattr(module, '__all__'):
+        for obj_name in module.__all__:
+            obj = getattr(module, obj_name)
+            if issubclass(obj, BaseSchemaStrategy):
+                print("Founded", f"'{obj.__name__}'")
+                schemas.append(obj())
+    else:
+        print("Not founded objects, exit")
+        exit(1)
+
+    if len(schemas) == 0:
+        print("Not founded schemas, exit")
+        exit(1)
+
+    return schemas
+
+
+def _suggest_overwrite(path_out: str, overwrite: bool):
+    if Path(path_out).exists() and not overwrite:
+        choice = input("Overwrite? (y/n)? ")
+        if choice.lower() != "y":
+            print("Cancel operation")
+            exit(1)
+
+
+def _file_out(out: str, lang: str):
+    if out == "schema.{}":
+        if lang in _EXT_PYTONIC:
+            ext = "py"
+        elif lang in _EXT_DART:
+            ext = "dart"
+        else:
+            print("not founded extension, exit")
+            exit(1)
+        return out.format(ext)
+    return out
+
+
+def main():
+    args = parse_arguments()
+    if args.CSS_TO_XPATH and args.XPATH_TO_CSS:
+        print('Should be choice --css-to-xpath or --xpath-to-css')
+        exit(1)
+
+    converter = _CONVERTER_NAMESPACES.get(args.lang)
+    print(f"Load '{args.lang}' converter")
+    schemas = _extract_schemas(args.scenario)
+    output = _file_out(args.out, args.lang)
+    _suggest_overwrite(output, args.OVERWRITE)
+
+    print("Render code")
+    if args.CSS_TO_XPATH:
+        print(F"convert CSS to XPATH with prefix '{args.XPATH_PREFIX}'")
+    elif args.XPATH_TO_CSS:
+        print(f"convert XPATH to CSS")
+
+    code = render_code(converter,
+                       *schemas,
+                       css_to_xpath=args.CSS_TO_XPATH,
+                       xpath_to_css=args.XPATH_TO_CSS,
+                       xpath_prefix=args.XPATH_PREFIX)
+    print(f"Save to {output}")
+    Path(output).write_text(code)
+    if not args.NO_FORMAT:
+        print("Format code...")
+        if args.lang.startswith('py.'):
+            subprocess.call(_FMT_PYTHON.format(output), shell=True)
+        elif args.lang == "dart":
+            subprocess.call(_FMT_DART.format(output), shell=True)
+    print("done.")
+
+
+if __name__ == '__main__':
+    main()

ssc_codegen-0.2.0/ssc_codegen/converters/base.py

@@ -0,0 +1,65 @@
+from functools import partial
+
+from ssc_codegen.objects import TokenType, Node
+
+
+__all__ = ['CodeConverter', "VAR_L", "VAR_R"]
+
+
+def var_left(node: Node, prefix: str) -> str:
+    """get next variable name"""
+    return prefix.format(node.id or 0)
+
+
+def var_right(node: Node, prefix: str) -> str:
+    """get previous variable name"""
+    return prefix.format(node.prev_node.id or 0)
+
+
+# Shortcuts hooks variable name
+VAR_L = partial(var_left, prefix='var_{}')
+VAR_R = partial(var_right, prefix='var_{}')
+
+
+class CodeConverter:
+    """Translate Expressions into the parser code"""
+
+    def __init__(self,
+                 indent: str = " " * 4,
+                 end: str = "",
+                 intent_inner_try: str = " " * 4 * 2,
+                 templates_path: str = ""):
+
+        self.indent = indent
+        self.end = end
+        self.indent_inner_try = intent_inner_try
+        self.definitions: Dict[TokenType, Callable[[Node], str]] = {}  # type: ignore
+        self._templates_path: str = templates_path
+
+        self._in_inner_try: bool = False
+
+    @property
+    def templates_path(self) -> str:
+        return self._templates_path
+
+    def __call__(self, for_definition: TokenType):
+        def decorator(func):
+            self.definitions[for_definition] = func
+            return func
+
+        return decorator
+
+    def convert(self, node: Node) -> str:
+        if callback := self.definitions.get(node.token):
+            if node.token == TokenType.OP_DEFAULT_START:
+                self._in_inner_try = True
+                return self.indent + callback(node)
+
+            elif node.token == TokenType.OP_DEFAULT_END:
+                self._in_inner_try = False
+                return self.indent + callback(node)
+
+            if self._in_inner_try:
+                return self.indent_inner_try + callback(node) + self.end
+            return self.indent + callback(node) + self.end
+        return ""