ssc_codegen 0.8.23__tar.gz → 0.9.0__tar.gz

{ssc_codegen-0.8.23 → ssc_codegen-0.9.0}/.gitignore

@@ -38,7 +38,7 @@ MANIFEST
 pip-log.txt
 pip-delete-this-directory.txt
 
-# Unit test / coverage reports
+# Unit tests / coverage reports
 htmlcov/
 .tox/
 .nox/

{ssc_codegen-0.8.23 → ssc_codegen-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.8.23
+Version: 0.9.0
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues

{ssc_codegen-0.8.23 → ssc_codegen-0.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.8.23"
+version = "0.9.0"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.8.23 → ssc_codegen-0.9.0}/ssc_codegen/__init__.py

@@ -3,11 +3,11 @@ from ssc_codegen.document import HTMLDocument, StringDocument, ArrayDocument, As
     NumericDocument, JsonDocument, BooleanDocument, DocumentFilter
 from ssc_codegen.json_struct import Json
 from ssc_codegen.logs import setup_logger
-from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchema
+from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchema, AccUniqueListSchema
 
 setup_logger()
 
-VERSION = "0.8.23"
+VERSION = "0.9.0"
 
 
 class __MISSING(object):

{ssc_codegen-0.8.23 → ssc_codegen-0.9.0}/ssc_codegen/ast_build/utils.py

@@ -13,6 +13,7 @@ from ssc_codegen.schema import (
     DictSchema,
     ItemSchema,
     FlatListSchema,
+    AccUniqueListSchema,
 )
 from ssc_codegen.tokens import VariableType
 
@@ -59,6 +60,7 @@ def is_template_schema_cls(cls: object) -> bool:
         DictSchema,
         ListSchema,
         BaseSchema,
+        AccUniqueListSchema,
     )
 
 

{ssc_codegen-0.8.23 → ssc_codegen-0.9.0}/ssc_codegen/converters/js_pure.py

@@ -128,6 +128,7 @@ from ssc_codegen.converters.templates.js_pure import (
     J2_PRE_XPATH_ALL,
     J2_PRE_STR_TRIM,
     J2_IS_XPATH,
+    J2_START_PARSE_ACC_LIST,
 )
 from ssc_codegen.str_utils import (
     wrap_backtick,
@@ -283,6 +284,8 @@ def pre_start_parse(node: StartParseMethod) -> str:
             code += J2_START_PARSE_DICT
         case StructType.FLAT_LIST:
             code += J2_START_PARSE_FLAT_LIST
+        case StructType.ACC_LIST:
+            code += J2_START_PARSE_ACC_LIST.render(exprs=exprs)
         case _:
             assert_never(node.parent.struct_type)  # type: ignore
     return code

{ssc_codegen-0.8.23 → ssc_codegen-0.9.0}/ssc_codegen/converters/py_base.py

@@ -351,6 +351,9 @@ def pre_typedef(node: TypeDef) -> str:
             return f"T_{name} = List[{type_}]"
         case StructType.ITEM | StructType.LIST:
             return f'T_{name} = TypedDict("T_{name}", ' + "{"
+        case StructType.ACC_LIST:
+            # current impl support only str type
+            return f"T_{name} = List[str]"
         case _:
             assert_never(struct_type)
     raise NotImplementedError()  # noqa
@@ -359,9 +362,7 @@ def pre_typedef(node: TypeDef) -> str:
 def post_typedef(node: TypeDef) -> str:
     _, struct_type = node.unpack_args()
     match struct_type:
-        case StructType.DICT:
-            return ""
-        case StructType.FLAT_LIST:
+        case StructType.DICT | StructType.FLAT_LIST | StructType.ACC_LIST:
             return ""
         # close TypedDict type
         case StructType.ITEM | StructType.LIST:
@@ -375,7 +376,11 @@ def pre_typedef_field(node: TypeDefField) -> str:
     name, var_type, cls_nested, cls_nested_type = node.unpack_args()
     node.parent = cast(TypeDef, node.parent)
     # skip generated types
-    if node.parent.struct_type in (StructType.DICT, StructType.FLAT_LIST):
+    if node.parent.struct_type in (
+        StructType.DICT,
+        StructType.FLAT_LIST,
+        StructType.ACC_LIST,
+    ):
         return ""
     # always string
     elif name == "__KEY__":
@@ -485,6 +490,18 @@ def post_start_parse(node: StartParseMethod) -> str:
                     continue
                 code += f"{name!r}: self._parse_{name}(self._document),"
             code += "}"
+        case StructType.ACC_LIST:
+            code += INDENT_METHOD_BODY + "return list(dict.fromkeys("
+            methods = []
+            for expr in node.body:
+                name = expr.kwargs["name"]
+                if name.startswith("__"):
+                    continue
+                # acc list of str (all list are flatten)
+                methods.append(f"self._parse_{name}(self._document)")
+            code += " + ".join(methods)
+            code += "))"
+
         case StructType.LIST:
             # return [{...} for el in self._split_doc(self.document)]
             code += INDENT_METHOD_BODY + "return [{"

{ssc_codegen-0.8.23 → ssc_codegen-0.9.0}/ssc_codegen/converters/templates/js_pure.py

@@ -13,7 +13,15 @@ throw new Error("Invalid input: Expected a Document, Element, or string");}
 
 J2_START_PARSE_DICT = "return Array.from(this._splitDoc(this._doc)).reduce((item, e) => (item[this._parseKey(e)] = this._parseValue(e), item), {});"
 J2_START_PARSE_FLAT_LIST = "return Array.from(this._splitDoc(this._doc)).map((e) => this._parseItem(e));"
-
+J2_START_PARSE_ACC_LIST = Template(
+    """
+return [...new Set([
+{% for expr in exprs %}
+this._parse{{ expr.upper_name }}(this._doc),
+{% endfor %}
+].flat())];
+"""
+)
 J2_START_PARSE_ITEM = Template("""return {
     {% for expr in exprs %}
 {{ expr.name }}: this._parse{{ expr.upper_name }}(this._doc),

{ssc_codegen-0.8.23 → ssc_codegen-0.9.0}/ssc_codegen/schema.py

@@ -22,19 +22,29 @@ class BaseSchema:
     __SCHEMA_TYPE__: StructType = NotImplemented
 
     __SIGNATURE__: dict | list = NotImplemented
-    """manually write signature attribute"""
+    """manually write signature for docstring"""
 
     __PRE_VALIDATE__: _T_OPT_FIELD = MISSING_FIELD
+    """Optional method for pre validation input document
+    
+    if checks not passed - throw error
+    """
 
     # DICT, LIST, FLAT_LIST HOOK
     __SPLIT_DOC__: _T_OPT_FIELD = MISSING_FIELD
+    """ListSchema, DictSchema, FlatListSchema method for write split document to parts of elements
+    """
 
     # FLAT LIST HOOK
     __ITEM__: _T_OPT_FIELD = MISSING_FIELD
+    """FlatListSchema item method for parse items for array"""
 
     # DICT HOOKS
     __KEY__: _T_OPT_FIELD = MISSING_FIELD
+    """DictSchema method for parse key """
+
     __VALUE__: _T_OPT_FIELD = MISSING_FIELD
+    """DictSchema method for parse value"""
 
     __EXCLUDE_SIGNATURE__: _T_OPT_FIELD = MISSING_FIELD
 
@@ -61,6 +71,9 @@ class BaseSchema:
                 signature = cls._get_list_signature()
             case StructType.ITEM:
                 signature = cls._get_item_signature()
+
+            case StructType.ACC_LIST:
+                signature = [VariableType.STRING, "..."]
             case _:
                 # code unreached
                 raise TypeError("Unknown schema type")
@@ -117,8 +130,11 @@ class BaseSchema:
         signature = []
         field = cls.__ITEM__
         if field.stack_last_ret == VariableType.NESTED:
-            nested_class = cls._get_nested_signature(field)
-            signature.append(nested_class.__class_signature__())
+            name = [e for e in field.stack if e.kind == TokenType.EXPR_NESTED][
+                0
+            ].kwargs["schema_name"]
+            nested_class = cls.__NESTED_SCHEMAS__[name].__class_signature__()
+            signature.append(nested_class)
         elif field.stack_last_ret == VariableType.JSON:
             signature[k] = json_struct_to_signature(v.stack[-1].value)  # noqa
         else:
@@ -208,3 +224,7 @@ class DictSchema(BaseSchema):
 
 class FlatListSchema(BaseSchema):
     __SCHEMA_TYPE__ = StructType.FLAT_LIST
+
+
+class AccUniqueListSchema(BaseSchema):
+    __SCHEMA_TYPE__ = StructType.ACC_LIST

{ssc_codegen-0.8.23 → ssc_codegen-0.9.0}/ssc_codegen/static_checker/__init__.py

@@ -6,7 +6,7 @@ from ssc_codegen.static_checker.callbacks import analyze_schema_item_field, anal
     analyze_schema_value_field, analyze_field_default_value, analyze_field_type_static, analyze_field_html_queries, \
     analyze_schema_split_doc_field, analyze_field_split_doc_ret_type, CB_SCHEMA, CB_DOCUMENT, analyze_regex_expr, \
     analyze_field_key_ret_type, analyze_other_field_type, analyze_dict_schema_fields, analyze_flat_list_schema_fields, \
-    analyze_jsonify_expr
+    analyze_jsonify_expr, analyze_schema_acc_list
 
 LOGGER = logging.getLogger("ssc_gen")
 
@@ -16,7 +16,8 @@ _DEFAULT_CB_SCHEMAS = (
     analyze_schema_key_field,
     analyze_schema_item_field,
     analyze_dict_schema_fields,
-    analyze_flat_list_schema_fields
+    analyze_flat_list_schema_fields,
+    analyze_schema_acc_list
 )
 
 _DEFAULT_CB_DOCUMENTS = (

{ssc_codegen-0.8.23 → ssc_codegen-0.9.0}/ssc_codegen/static_checker/callbacks.py

@@ -66,6 +66,26 @@ def analyze_flat_list_schema_fields(sc: Type["BaseSchema"]) -> AnalyzeResult:
     return AnalyzeResult.ok()
 
 
+def analyze_schema_acc_list(sc: Type["BaseSchema"]) -> AnalyzeResult:
+    # current impl support only LIST_STRING expr
+    if sc.__SCHEMA_TYPE__ != StructType.ACC_LIST:
+        return AnalyzeResult.ok()
+    skip_fields = {"__SPLIT_DOC__", "__PRE_VALIDATE__"}
+    err_fields = {
+        name: field
+        for name, field in sc.__get_mro_fields__().items()
+        if name not in skip_fields
+        and field.stack_last_ret != VariableType.LIST_STRING
+    }
+    if err_fields:
+        msg = ""
+        for name, field in err_fields.items():
+            msg += f"{sc.__name__}.{name} expected type(s) {VariableType.LIST_STRING.name}, got {field.stack_last_ret}"
+            msg += "\n"
+        return AnalyzeResult.error(msg)
+    return AnalyzeResult.ok()
+
+
 # schema check segment
 def analyze_schema_split_doc_field(sc: Type["BaseSchema"]) -> AnalyzeResult:
     if sc.__SCHEMA_TYPE__ in (

{ssc_codegen-0.8.23 → ssc_codegen-0.9.0}/ssc_codegen/tokens.py

@@ -66,6 +66,7 @@ class StructType(IntEnum):
     DICT = auto()
     LIST = auto()
     FLAT_LIST = auto()
+    ACC_LIST = auto()
 
 
 class TokenType(IntEnum):