ssc_codegen 0.10.0__tar.gz → 0.10.1__tar.gz

{ssc_codegen-0.10.0 → ssc_codegen-0.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ssc_codegen
-Version: 0.10.0
+Version: 0.10.1
 Summary: Python-dsl code converter to html parser for web scraping 
 Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
 Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
@@ -247,7 +247,7 @@ You can use any html source:
 
 ## See also
 
-- [Brief](docs/brief.md) about css selectors and regular expressions.
+- [Brief](docs/briefing.md) about css selectors and regular expressions.
+- [Quickstart](docs/quickstart.md) about css selectors and regular expressions.
 - [Tutorial](docs/tutorial.md) basic usage ssc-gen
-- [Reference](docs/reference.md) about lib API
 - [AST reference](docs/ast_reference.md) about generation code from AST

{ssc_codegen-0.10.0 → ssc_codegen-0.10.1}/README.md

@@ -210,7 +210,7 @@ You can use any html source:
 
 ## See also
 
-- [Brief](docs/brief.md) about css selectors and regular expressions.
+- [Brief](docs/briefing.md) about css selectors and regular expressions.
+- [Quickstart](docs/quickstart.md) about css selectors and regular expressions.
 - [Tutorial](docs/tutorial.md) basic usage ssc-gen
-- [Reference](docs/reference.md) about lib API
 - [AST reference](docs/ast_reference.md) about generation code from AST

{ssc_codegen-0.10.0 → ssc_codegen-0.10.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ssc_codegen"
-version = "0.10.0"
+version = "0.10.1"
 description = "Python-dsl code converter to html parser for web scraping "
 readme = "README.md"
 requires-python = ">=3.10"

{ssc_codegen-0.10.0 → ssc_codegen-0.10.1}/ssc_codegen/__init__.py

@@ -7,7 +7,7 @@ from ssc_codegen.schema import ItemSchema, DictSchema, ListSchema, FlatListSchem
 
 setup_logger()
 
-VERSION = "0.10.0"
+VERSION = "0.10.1"
 
 
 class __MISSING(object):

{ssc_codegen-0.10.0 → ssc_codegen-0.10.1}/ssc_codegen/ast_/__init__.py

@@ -1,6 +1,6 @@
 from .base import BaseAstNode
 from .nodes_array import ExprIndex, ExprToListLength, ExprListStringJoin, ExprListUnique
-from .nodes_cast import ExprNested, ExprJsonify, ExprToInt, ExprToFloat, ExprToListInt, ExprToListFloat, ExprToBool
+from .nodes_cast import ExprNested, ExprJsonify, ExprToInt, ExprToFloat, ExprToListInt, ExprToListFloat, ExprToBool, ExprJsonifyDynamic
 from .nodes_core import (ExprReturn,
                          ExprNoReturn,
                          ExprDefaultValueEnd,

{ssc_codegen-0.10.0 → ssc_codegen-0.10.1}/ssc_codegen/ast_/nodes_cast.py

@@ -68,3 +68,16 @@ class ExprJsonify(BaseAstNode[KW_EXPR_JSONIFY, ARGS_EXPR_JSONIFY]):
     kind: ClassVar[TokenType] = TokenType.TO_JSON
     accept_type: VariableType = VariableType.STRING
     ret_type: VariableType = VariableType.JSON
+
+
+KW_EXPR_JSONIFY_DYNAMIC = TypedDict("KW_EXPR_JSONIFY_DYNAMIC", {"query": str})
+ARGS_EXPR_JSONIFY_DYNAMIC = tuple[str]
+
+
+@dataclass(kw_only=True)
+class ExprJsonifyDynamic(
+    BaseAstNode[KW_EXPR_JSONIFY_DYNAMIC, ARGS_EXPR_JSONIFY_DYNAMIC]
+):
+    kind: ClassVar[TokenType] = TokenType.TO_JSON_DYNAMIC
+    accept_type: VariableType = VariableType.STRING
+    ret_type: VariableType = VariableType.ANY  # STUBBED

{ssc_codegen-0.10.0 → ssc_codegen-0.10.1}/ssc_codegen/converters/go_goquery.py

@@ -267,6 +267,7 @@ CONVERTER.TEST_EXCLUDE_NODES.extend(
         TokenType.STRUCT_INIT,  # defined inside a struct
         TokenType.EXPR_DEFAULT_END,  # emulated by defer func() + rescue
         TokenType.CLASSVAR,  # emulated by type ST struct{ f1: T1, f2: T2, ...}; var ST {f1=V1, f2=V2, ...};
+        TokenType.TO_JSON_DYNAMIC,  # golang exclude dynamic json structures (DDD aka dict driven development)
     ]
 )
 

{ssc_codegen-0.10.0 → ssc_codegen-0.10.1}/ssc_codegen/converters/js_pure.py

@@ -106,6 +106,7 @@ from ssc_codegen.ast_ import (
     ExprListStringMapReplace,
     ExprClassVar,
 )
+from ssc_codegen.ast_.nodes_cast import ExprJsonifyDynamic
 from ssc_codegen.ast_.nodes_core import ModuleImports
 from ssc_codegen.ast_.nodes_selectors import (
     ExprCssElementRemove,
@@ -1064,3 +1065,13 @@ def pre_xpath_remove(node: ExprXpathElementRemove) -> str:
         f"for (let {prv}r = document.evaluate({query}, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null), {prv}i = {prv}r.snapshotLength; {prv}i--; ) {prv}r.snapshotItem({prv}i).remove(); "
         + f"let {nxt} = {prv}; "
     )
+
+
+@CONVERTER(ExprJsonifyDynamic.kind)
+def pre_jsonify_dynamic(node: ExprJsonifyDynamic) -> str:
+    prv, nxt = prev_next_var(node)
+    query = node.unpack_args()
+
+    expr = "".join(f"[{i}]" for i in jsonify_query_parse(query))
+
+    return f"let {nxt} = JSON.parse({prv}){expr};"

{ssc_codegen-0.10.0 → ssc_codegen-0.10.1}/ssc_codegen/converters/lua_htmlparser.py

@@ -110,6 +110,7 @@ from ssc_codegen.ast_ import (
     ExprStringUnescape,
     ModuleImports,
 )
+from ssc_codegen.ast_.nodes_cast import ExprJsonifyDynamic
 from ssc_codegen.ast_.nodes_core import (
     CodeEnd,
     ExprCallStructClassVar,
@@ -1380,3 +1381,16 @@ def pre_code_end(node: CodeEnd) -> str:
     ]
     names = [n.kwargs["name"] + "=" + n.kwargs["name"] for n in classes]
     return "return {" + ", ".join(names) + "}"
+
+
+@CONVERTER(ExprJsonifyDynamic.kind)
+def pre_jsonify_dynamic(node: ExprJsonifyDynamic) -> str:
+    prv, nxt = prev_next_var(node)
+    query = node.unpack_args()
+    # lua index starts by 1, fix query
+    expr = "".join(
+        f"[{int(i) + 1}]" if i.isdigit() else f"[{i}]"
+        for i in jsonify_query_parse(query)
+    )
+
+    return f"local {nxt} = json.decode({prv}){expr}; "

{ssc_codegen-0.10.0 → ssc_codegen-0.10.1}/ssc_codegen/converters/py_base.py

@@ -107,6 +107,7 @@ from ssc_codegen.ast_ import (
     ExprStringMapReplace,
     ExprListStringMapReplace,
 )
+from ssc_codegen.ast_.nodes_cast import ExprJsonifyDynamic
 from ssc_codegen.ast_.nodes_core import ExprClassVar
 from ssc_codegen.ast_.nodes_string import (
     ExprListStringUnescape,
@@ -270,6 +271,7 @@ class BasePyCodeConverter(BaseCodeConverter):
             ExprToListLength.kind: pre_to_len,
             ExprToBool.kind: pre_to_bool,
             ExprJsonify.kind: pre_jsonify,
+            ExprJsonifyDynamic.kind: pre_to_json_dynamic,
             # FILTER,
             ExprFilter.kind: pre_expr_filter,
             FilterAnd.kind: pre_filter_and,
@@ -1269,3 +1271,15 @@ def pre_list_str_unescape(node: ExprListStringUnescape) -> str:
     prv, nxt = prev_next_var(node)
     # ssc_unescape(s: str) -> str
     return f"{indent}{nxt} = [ssc_unescape(i) for i in {prv}]"
+
+
+def pre_to_json_dynamic(node: ExprJsonifyDynamic) -> str:
+    indent = (
+        INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
+    )
+    prv, nxt = prev_next_var(node)
+    query = node.unpack_args()
+
+    expr = "".join(f"[{i}]" for i in jsonify_query_parse(query))
+
+    return f"{indent}{nxt} = json.loads({prv}){expr}"

{ssc_codegen-0.10.0 → ssc_codegen-0.10.1}/ssc_codegen/document.py

@@ -80,6 +80,7 @@ from ssc_codegen.ast_ import (
     ExprListStringMapReplace,
     ExprStringMapReplace,
 )
+from ssc_codegen.ast_.nodes_cast import ExprJsonifyDynamic
 from ssc_codegen.ast_.nodes_core import ExprClassVar
 from ssc_codegen.ast_.nodes_selectors import (
     ExprCssElementRemove,
@@ -1759,6 +1760,26 @@ class BooleanDocument(BaseDocument):
 
 
 class JsonDocument(BaseDocument):
+    def jsonify_dynamic(self, start_query: str = "") -> Self:
+        """marshal json to dynamic object wout annotated type/struct
+        
+        - accept STRING, return ANY
+        """
+        if self.stack_last_ret != VariableType.STRING:
+            LOGGER.warning(
+                "jsonify_dynamic(): Expected type(s) %s got %s",
+                VariableType.STRING.name,
+                self.stack_last_ret.name,
+            )
+        self._add(
+            ExprJsonifyDynamic(
+                kwargs={
+                    "query": start_query,
+                }
+            )
+        )
+        return self
+
     def jsonify(self, struct: Type[Json], start_query: str = "") -> Self:
         """marshal json string to object
 

{ssc_codegen-0.10.0 → ssc_codegen-0.10.1}/ssc_codegen/tokens.py

@@ -215,6 +215,7 @@ class TokenType(IntEnum):
 
     # JSON OP
     TO_JSON = auto()
+    TO_JSON_DYNAMIC = auto()
     # STRUCTS
     JSON_STRUCT = auto()
     JSON_FIELD = auto()