ssc_codegen 0.9.0__tar.gz → 0.9.2__tar.gz

{ssc_codegen-0.9.0 → ssc_codegen-0.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.9.0
+Version: 0.9.2
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
@@ -41,42 +41,49 @@ ssc-gen - based-python DSL language for writing html parsers in dataclass style
 Project solving next problems:
 
 - decrease boilerplate code
-- create types (or type annotations if possible) and documentation 
+- generation DDD-like modules independent of the project
+- generate types or type annotations if possible and documentation 
 - simplify code support
-- portability to other languages
+- portability to other languages and HTML parser backends
+- designed for server-side-render pages parse. 
+For REST-API, GRAPHQL use tools like [mitmproxy2swagger](https://github.com/alufers/mitmproxy2swagger)
+for simplify convert to code
 
 ## Support converters
 
 Current support converters
 
-| Language      | Library (html parser backend) | XPath Support | CSS Support | Generated types                          | Code formatter |
-|---------------|-------------------------------|---------------|-------------|------------------------------------------|----------------|
-| Python (3.8+) | bs4                           | N             | Y           | TypedDict*, list, dict                   | ruff           |
-| ...           | parsel                        | Y             | Y           | ...                                      | -              |
-| ...           | selectolax (lexbor)           | N             | Y           | ...                                      | -              |
-| js (ES6)      | pure (firefox/chrome/nodejs)  | Y             | Y           | Array, Map**                             | -              |
-| go (1.10+)    | goquery                       | N             | Y           | struct(json anchors include), array, map | gofmt          |
-
+| Language      | HTML parser backend          | XPath | CSS3 | CSS4 | Generated types, structs          | formatter dependency |
+|---------------|------------------------------|-------|------|------|-----------------------------------|----------------------|
+| Python (3.8+) | bs4                          | N     | Y    | Y    | TypedDict*, list, dict            | ruff                 |
+| ...           | parsel                       | Y     | Y    | N    | ...                               | ...                  |
+| ...           | selectolax (lexbor)          | N     | Y    | N    | ...                               | ...                  |
+| js (ES6)      | pure (firefox/chrome/nodejs) | Y     | Y    | Y    | Array, Map**                      | prettier             |
+| go (1.10+)    | goquery                      | N     | Y    | N    | struct(+json anchors), array, map | gofmt                |
+
+- **CSS3** means support next selectors:
+  - basic: (`tag`, `.class`, `#id`, `tag1,tag2`)
+  - combined: (`div p`, `ul > li`, `h2 +p`, `title ~head`)
+  - attribute: (`a[href]`, `input[type='text']`, `a[href*='...']`, ...)
+  - basic pseudo classes: (`:nth-child(n)`, `:first-child`, `:last-child`)
+- **CSS4** means support next selectors:
+  - `:has()`, `:nth-of-type()`, `:where()`, `:is()` etc
 - *this annotation type was deliberately chosen as a compromise reasons. 
 Python has many ways of serialization: `namedtuple, dataclass, attrs, pydantic, msgspec, etc`
   - TypedDict is like a build-in dict, but with IDE and linter hint support, and you can easily implement an adapter for the required structure.
-- **js exclude build-in serialization methods
+- **js exclude build-in serialization methods, used standard Array and Map structures 
+- **formatter** - optional dependency for prettify and fix codestyle
 
 ### Limitations
 
 For maximum portability of the configuration to the target language:
 
 - Use CSS selectors: they are guaranteed to be converted to XPATH
-- Unlike javascript, most html parse libs implement [CSS3 selectors standard](https://www.w3.org/TR/selectors-3/).
-check in target lib complex selectors before implement:
-  - basic selectors: (`tag`, `.class`, `#id`, `tag1,tag2`)
-  - combined: (`div p`, `ul > li`, `h2 +p`, `title ~head`\[1])
-  - attribute: (`a[href]`, `input[type='text']`)\[2]
-  - pseudo classes: (`:nth-child(n)`, `:first-child`, `:last-child`)\[3]
-  - **often, not support more complex, dynamic styles**: (`:has()`, `:nth-of-type()`, `:where()`, `:is()`)
+- Unlike javascript, most html parse libs implement [CSS3 selectors standard](https://www.w3.org/TR/selectors-3/). 
+Check the html parser lib documentation before implement CSS selectors
 
 1. Several libs not support `+` operations (eg: [selectolax(modest)](https://github.com/rushter/selectolax), [dart.universal_html](https://pub.dev/packages/universal_html))
-2. Often, web scraping libs not supports attribute operations like `*=`, `~=`, `|=`, `^=` and `$=`
+2. HTML parser libs maybe not supports attribute operations like `*=`, `~=`, `|=`, `^=` and `$=`
 3. Several libs not support pseudo classes (eg: standard [dart.html](https://dart.dev/libraries/dart-html) lib miss this feature). 
 This project will not implement converters with such a cons
 
@@ -120,7 +127,7 @@ class HelloWorld(ItemSchema):
 ### try it in cli
 
 >[!note]
-> this tools developed for testing purposes, not for web-scraping
+> this tools developed for testing purposes, not for web-scraping tasks
 
 ### from file
 
@@ -167,7 +174,7 @@ ssc-gen parse-from-chrome https://example.com -t schema.py:HelloWorld -sc /usr/b
 
 Convert to code for use in projects:
 
->![note]
+>[!note]
 > for example, used js: it can be fast test in developer console
 
 
@@ -175,7 +182,7 @@ Convert to code for use in projects:
 ssc-gen js schema.py -o .
 ```
 
-Code output looks like this (code formatted by IDE):
+Code output looks like this:
 
 ```javascript
 // autogenerated by ssc-gen DO NOT_EDIT
@@ -216,7 +223,7 @@ class HelloWorld {
 }
 ```
 
-### copy code output and past to developer console:
+### Copy code output and past to developer console:
 
 Print output:
 
@@ -229,17 +236,14 @@ alert(JSON.stringify((new HelloWorld(document).parse())))
 
 You can use any html source:
 
-- read from html file
-- get from http request
-- get from browser (playwright, selenium, chrome-cdp)
-- paste code to developer console (js)
-- or call curl in shell and parse stdin
-- use in third-party tools in stdin pipeline: 
-(from [ProjectDiscovery](https://github.com/projectdiscovery), for example)
-
+- parse from html files
+- parse from http responses
+- parse from browsers: playwright, selenium, chrome-cdp, etc.
+- call curl in shell and parse STDIN
+- use in STDIN pipelines with third-party tools like [projectdiscovery/httpx](https://github.com/projectdiscovery/httpx)
 
 ## See also
 - [Brief](docs/brief.md) about css selectors and regular expressions.
-- [Tutorial](docs/tutorial.md) how to use ssc-gen
-- [Reference](docs/reference.md) about high-level API
+- [Tutorial](docs/tutorial.md) basic usage ssc-gen
+- [Reference](docs/reference.md) about lib API
 - [AST reference](docs/ast_reference.md) about generation code from AST

{ssc_codegen-0.9.0 → ssc_codegen-0.9.2}/README.md

@@ -7,42 +7,49 @@ ssc-gen - based-python DSL language for writing html parsers in dataclass style
 Project solving next problems:
 
 - decrease boilerplate code
-- create types (or type annotations if possible) and documentation 
+- generation DDD-like modules independent of the project
+- generate types or type annotations if possible and documentation 
 - simplify code support
-- portability to other languages
+- portability to other languages and HTML parser backends
+- designed for server-side-render pages parse. 
+For REST-API, GRAPHQL use tools like [mitmproxy2swagger](https://github.com/alufers/mitmproxy2swagger)
+for simplify convert to code
 
 ## Support converters
 
 Current support converters
 
-| Language      | Library (html parser backend) | XPath Support | CSS Support | Generated types                          | Code formatter |
-|---------------|-------------------------------|---------------|-------------|------------------------------------------|----------------|
-| Python (3.8+) | bs4                           | N             | Y           | TypedDict*, list, dict                   | ruff           |
-| ...           | parsel                        | Y             | Y           | ...                                      | -              |
-| ...           | selectolax (lexbor)           | N             | Y           | ...                                      | -              |
-| js (ES6)      | pure (firefox/chrome/nodejs)  | Y             | Y           | Array, Map**                             | -              |
-| go (1.10+)    | goquery                       | N             | Y           | struct(json anchors include), array, map | gofmt          |
-
+| Language      | HTML parser backend          | XPath | CSS3 | CSS4 | Generated types, structs          | formatter dependency |
+|---------------|------------------------------|-------|------|------|-----------------------------------|----------------------|
+| Python (3.8+) | bs4                          | N     | Y    | Y    | TypedDict*, list, dict            | ruff                 |
+| ...           | parsel                       | Y     | Y    | N    | ...                               | ...                  |
+| ...           | selectolax (lexbor)          | N     | Y    | N    | ...                               | ...                  |
+| js (ES6)      | pure (firefox/chrome/nodejs) | Y     | Y    | Y    | Array, Map**                      | prettier             |
+| go (1.10+)    | goquery                      | N     | Y    | N    | struct(+json anchors), array, map | gofmt                |
+
+- **CSS3** means support next selectors:
+  - basic: (`tag`, `.class`, `#id`, `tag1,tag2`)
+  - combined: (`div p`, `ul > li`, `h2 +p`, `title ~head`)
+  - attribute: (`a[href]`, `input[type='text']`, `a[href*='...']`, ...)
+  - basic pseudo classes: (`:nth-child(n)`, `:first-child`, `:last-child`)
+- **CSS4** means support next selectors:
+  - `:has()`, `:nth-of-type()`, `:where()`, `:is()` etc
 - *this annotation type was deliberately chosen as a compromise reasons. 
 Python has many ways of serialization: `namedtuple, dataclass, attrs, pydantic, msgspec, etc`
   - TypedDict is like a build-in dict, but with IDE and linter hint support, and you can easily implement an adapter for the required structure.
-- **js exclude build-in serialization methods
+- **js exclude build-in serialization methods, used standard Array and Map structures 
+- **formatter** - optional dependency for prettify and fix codestyle
 
 ### Limitations
 
 For maximum portability of the configuration to the target language:
 
 - Use CSS selectors: they are guaranteed to be converted to XPATH
-- Unlike javascript, most html parse libs implement [CSS3 selectors standard](https://www.w3.org/TR/selectors-3/).
-check in target lib complex selectors before implement:
-  - basic selectors: (`tag`, `.class`, `#id`, `tag1,tag2`)
-  - combined: (`div p`, `ul > li`, `h2 +p`, `title ~head`\[1])
-  - attribute: (`a[href]`, `input[type='text']`)\[2]
-  - pseudo classes: (`:nth-child(n)`, `:first-child`, `:last-child`)\[3]
-  - **often, not support more complex, dynamic styles**: (`:has()`, `:nth-of-type()`, `:where()`, `:is()`)
+- Unlike javascript, most html parse libs implement [CSS3 selectors standard](https://www.w3.org/TR/selectors-3/). 
+Check the html parser lib documentation before implement CSS selectors
 
 1. Several libs not support `+` operations (eg: [selectolax(modest)](https://github.com/rushter/selectolax), [dart.universal_html](https://pub.dev/packages/universal_html))
-2. Often, web scraping libs not supports attribute operations like `*=`, `~=`, `|=`, `^=` and `$=`
+2. HTML parser libs maybe not supports attribute operations like `*=`, `~=`, `|=`, `^=` and `$=`
 3. Several libs not support pseudo classes (eg: standard [dart.html](https://dart.dev/libraries/dart-html) lib miss this feature). 
 This project will not implement converters with such a cons
 
@@ -86,7 +93,7 @@ class HelloWorld(ItemSchema):
 ### try it in cli
 
 >[!note]
-> this tools developed for testing purposes, not for web-scraping
+> this tools developed for testing purposes, not for web-scraping tasks
 
 ### from file
 
@@ -133,7 +140,7 @@ ssc-gen parse-from-chrome https://example.com -t schema.py:HelloWorld -sc /usr/b
 
 Convert to code for use in projects:
 
->![note]
+>[!note]
 > for example, used js: it can be fast test in developer console
 
 
@@ -141,7 +148,7 @@ Convert to code for use in projects:
 ssc-gen js schema.py -o .
 ```
 
-Code output looks like this (code formatted by IDE):
+Code output looks like this:
 
 ```javascript
 // autogenerated by ssc-gen DO NOT_EDIT
@@ -182,7 +189,7 @@ class HelloWorld {
 }
 ```
 
-### copy code output and past to developer console:
+### Copy code output and past to developer console:
 
 Print output:
 
@@ -195,17 +202,14 @@ alert(JSON.stringify((new HelloWorld(document).parse())))
 
 You can use any html source:
 
-- read from html file
-- get from http request
-- get from browser (playwright, selenium, chrome-cdp)
-- paste code to developer console (js)
-- or call curl in shell and parse stdin
-- use in third-party tools in stdin pipeline: 
-(from [ProjectDiscovery](https://github.com/projectdiscovery), for example)
-
+- parse from html files
+- parse from http responses
+- parse from browsers: playwright, selenium, chrome-cdp, etc.
+- call curl in shell and parse STDIN
+- use in STDIN pipelines with third-party tools like [projectdiscovery/httpx](https://github.com/projectdiscovery/httpx)
 
 ## See also
 - [Brief](docs/brief.md) about css selectors and regular expressions.
-- [Tutorial](docs/tutorial.md) how to use ssc-gen
-- [Reference](docs/reference.md) about high-level API
+- [Tutorial](docs/tutorial.md) basic usage ssc-gen
+- [Reference](docs/reference.md) about lib API
 - [AST reference](docs/ast_reference.md) about generation code from AST

{ssc_codegen-0.9.0 → ssc_codegen-0.9.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.9.0"
+version = "0.9.2"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.9.0 → ssc_codegen-0.9.2}/ssc_codegen/__init__.py

@@ -7,7 +7,7 @@ from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchem
 
 setup_logger()
 
-VERSION = "0.9.0"
+VERSION = "0.9.2"
 
 
 class __MISSING(object):

{ssc_codegen-0.9.0 → ssc_codegen-0.9.2}/ssc_codegen/ast_build/main.py

@@ -47,6 +47,7 @@ def build_ast_module_parser(
     *,
     css_to_xpath: bool = False,
     xpath_to_css: bool = False,
+    gen_docstring: bool = True,
 ) -> ModuleProgram:
     """build ast from python sscgen config file
 
@@ -66,12 +67,18 @@ def build_ast_module_parser(
             "Should be chosen one variant (css_to_xpath OR xpath_to_css)"
         )
     py_module = exec_module_code(path)
-    docstr = py_module.__dict__.get("__doc__") or ""  # type: str
     ast_module = ModuleProgram()
-    module_body: list[BaseAstNode] = [
-        Docstring(kwargs={"value": docstr}, parent=ast_module),
-        ModuleImports(parent=ast_module),
-    ]
+
+    if gen_docstring:
+        docstr = py_module.__dict__.get("__doc__") or ""  # type: str
+        module_body: list[BaseAstNode] = [
+            Docstring(kwargs={"value": docstr}, parent=ast_module),
+            ModuleImports(parent=ast_module),
+        ]
+    else:
+        module_body: list[BaseAstNode] = [
+            ModuleImports(parent=ast_module),
+        ]
 
     schemas = extract_schemas_from_module(py_module)
     json_structs = extract_json_structs_from_module(py_module)
@@ -85,6 +92,7 @@ def build_ast_module_parser(
                 ast_module,
                 css_to_xpath=css_to_xpath,
                 xpath_to_css=xpath_to_css,
+                gen_docstring=gen_docstring,
             )
             for sc in schemas
         ]
@@ -98,6 +106,7 @@ def build_ast_module_parser(
                     ast_module,
                     css_to_xpath=css_to_xpath,
                     xpath_to_css=xpath_to_css,
+                    gen_docstring=gen_docstring,
                 )
             except SyntaxError as e:
                 errors.append(e)
@@ -187,6 +196,7 @@ def build_ast_struct_parser(
     *,
     css_to_xpath: bool = False,
     xpath_to_css: bool = False,
+    gen_docstring: bool = True,
 ) -> StructParser:
     errors_count = run_analyze_schema(schema)
     if errors_count > 0:
@@ -195,11 +205,12 @@ def build_ast_struct_parser(
     docstring = (
         (schema.__doc__ or "") + "\n\n" + generate_docstring_signature(schema)
     )
+
     st = StructParser(
         kwargs={
             "name": schema.__name__,
             "struct_type": schema.__SCHEMA_TYPE__,
-            "docstring": docstring,
+            "docstring": docstring if gen_docstring else "",
         },
         parent=module_ref,
     )

{ssc_codegen-0.9.0 → ssc_codegen-0.9.2}/ssc_codegen/cli/consts.py

@@ -32,6 +32,7 @@ HELP_TO_CSS = "convert all xpath queries to css (works not guaranteed)"
 HELP_DEBUG_COMM_TOKENS = "add debug comment for each generated instruction"
 HELP_JSON_TO_SCHEMA = "convert json response to ssc-gen schema"
 # cmd group description
+HELP_DOCSTRING = "disable docstring and fields signature generate"
 CMD_VERSION = "Show version and exit"
 CMD_PY = "generate python modules"
 CMD_JS = "generate javascript modules"

{ssc_codegen-0.9.0 → ssc_codegen-0.9.2}/ssc_codegen/cli/main.py

@@ -43,6 +43,7 @@ from ssc_codegen.cli.consts import (
     CMD_GO,
     CMD_JSON_GEN,
     DEFAULT_UA,
+    HELP_DOCSTRING,
 )
 from ssc_codegen.cli.runtime_parse_runners import (
     assert_cls_target,
@@ -84,6 +85,7 @@ def generate_code(
     css_to_xpath: bool = False,
     xpath_to_css: bool = False,
     debug_instructions: bool = False,
+    gen_docstring: bool = True,
 ) -> None:
     """universal generate code entrypoint"""
     variables_patches = variables_patches or {}
@@ -113,6 +115,7 @@ def generate_code(
             file_cfg,
             css_to_xpath=css_to_xpath,
             xpath_to_css=xpath_to_css,
+            gen_docstring=gen_docstring,
         )
         LOGGER.info("Convert to code %s...", file_cfg.name)
         code_parts = converter.convert_program(ast_module, comment=comment_str)
@@ -155,6 +158,9 @@ def gen_py(
     debug: Annotated[
         bool, Option(help=HELP_DEBUG_COMM_TOKENS, is_flag=True)
     ] = False,
+    docstring: Annotated[
+        bool, Option(help=HELP_DOCSTRING, is_flag=True)
+    ] = True,
 ) -> None:
     converter = import_converter(f"py_{lib.value}")
     if fmt:
@@ -174,6 +180,7 @@ def gen_py(
         css_to_xpath=to_xpath,
         debug_instructions=debug,
         code_cb=CB_PY_CODE,
+        gen_docstring=docstring,
     )
 
 
@@ -202,11 +209,15 @@ def gen_js(
     debug: Annotated[
         bool, Option(help=HELP_DEBUG_COMM_TOKENS, is_flag=True)
     ] = False,
+    docstring: Annotated[
+        bool, Option(help=HELP_DOCSTRING, is_flag=True)
+    ] = True,
 ) -> None:
     converter = import_converter(f"js_{lib.value}")
     if fmt:
-        # TODO: js formatters
-        commands: list[str] = []
+        commands: list[str] = [
+            'prettier --write --tab-width 4 --single-quote --trailing-comma all --semi {}',
+            ]
         fmt_cmd = create_fmt_cmd(ssc_files, prefix, suffix, out, commands)
     else:
         fmt_cmd = []
@@ -222,6 +233,7 @@ def gen_js(
         css_to_xpath=to_xpath,
         debug_instructions=debug,
         code_cb=CB_JS_CODE,
+        gen_docstring=docstring,
     )
 
 
@@ -252,6 +264,9 @@ def gen_go(
     to_xpath: Annotated[bool, Option(help=HELP_TO_XPATH, is_flag=True)] = False,
     to_css: Annotated[bool, Option(help=HELP_TO_CSS, is_flag=True)] = False,
     debug: Annotated[bool, Option(help=HELP_DEBUG_COMM_TOKENS)] = False,
+    docstring: Annotated[
+        bool, Option(help=HELP_DOCSTRING, is_flag=True)
+    ] = True,
 ) -> None:
     converter = import_converter(f"go_{lib.value}")
     if fmt:
@@ -273,6 +288,7 @@ def gen_go(
         xpath_to_css=to_css,
         css_to_xpath=to_xpath,
         debug_instructions=debug,
+        gen_docstring=docstring,
     )
 
 

{ssc_codegen-0.9.0 → ssc_codegen-0.9.2}/ssc_codegen/converters/py_base.py

@@ -491,7 +491,7 @@ def post_start_parse(node: StartParseMethod) -> str:
                 code += f"{name!r}: self._parse_{name}(self._document),"
             code += "}"
         case StructType.ACC_LIST:
-            code += INDENT_METHOD_BODY + "return list(dict.fromkeys("
+            code += INDENT_METHOD_BODY + "return list(set("
             methods = []
             for expr in node.body:
                 name = expr.kwargs["name"]
@@ -1100,8 +1100,7 @@ def pre_list_unique(node: ExprListUnique) -> str:
         INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
     )
     prv, nxt = prev_next_var(node)
-    # use `dict.fromkeys()` instead `set()` for guarantees insertion order (py 3.7+) reason
-    return f"{indent}{nxt} = list(dict.fromkeys({prv}))"
+    return f"{indent}{nxt} = list(set({prv}))"
 
 
 def pre_str_map_replace(node: ExprStringMapReplace) -> str: