ssc_codegen 0.8.17__tar.gz → 0.8.19__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ssc_codegen might be problematic. Click here for more details.
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/PKG-INFO +2 -1
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/pyproject.toml +2 -1
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/__init__.py +1 -1
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/ast_/__init__.py +4 -2
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/ast_/nodes_filter.py +51 -10
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/converters/js_pure.py +96 -14
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/converters/py_base.py +78 -6
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/document.py +120 -12
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/tokens.py +9 -2
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/.gitignore +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/LICENSE +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/README.md +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/_compat.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/ast_/base.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/ast_/nodes_array.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/ast_/nodes_cast.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/ast_/nodes_core.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/ast_/nodes_selectors.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/ast_/nodes_string.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/ast_/nodes_validate.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/ast_build/__init__.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/ast_build/main.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/ast_build/utils.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/cli/__init__.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/cli/cli_callbacks.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/cli/cli_utils.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/cli/code_callbacks.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/cli/consts.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/cli/main.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/cli/runtime_parse_runners.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/compiler.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/consts.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/converters/__init__.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/converters/base.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/converters/go_goquery.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/converters/helpers.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/converters/py_bs4.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/converters/py_parsel.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/converters/py_selectolax.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/converters/templates/__init__.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/converters/templates/go_goquery.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/converters/templates/js_pure.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/document_utlis.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/json_struct.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/json_to_scc.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/logs.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/pseudo_selectors.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/schema.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/selector_utils.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/static_checker/__init__.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/static_checker/base.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/static_checker/callbacks.py +0 -0
- {ssc_codegen-0.8.17 → ssc_codegen-0.8.19}/ssc_codegen/str_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ssc_codegen
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.19
|
|
4
4
|
Summary: Python-dsl code converter to html parser for web scraping
|
|
5
5
|
Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
|
|
@@ -19,6 +19,7 @@ Classifier: Topic :: Software Development :: Code Generators
|
|
|
19
19
|
Classifier: Topic :: Text Processing :: Markup :: HTML
|
|
20
20
|
Classifier: Topic :: Utilities
|
|
21
21
|
Requires-Python: >=3.10
|
|
22
|
+
Requires-Dist: bs4>=0.0.2
|
|
22
23
|
Requires-Dist: colorama>=0.4.6; sys_platform == 'win32'
|
|
23
24
|
Requires-Dist: cssselect>=1.2.0
|
|
24
25
|
Requires-Dist: httpx>=0.28.1
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "ssc_codegen"
|
|
3
|
-
version = "0.8.
|
|
3
|
+
version = "0.8.19"
|
|
4
4
|
description = "Python-dsl code converter to html parser for web scraping "
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.10"
|
|
7
7
|
dependencies = [
|
|
8
|
+
"bs4>=0.0.2",
|
|
8
9
|
"colorama>=0.4.6 ; sys_platform == 'win32'",
|
|
9
10
|
"cssselect>=1.2.0",
|
|
10
11
|
"httpx>=0.28.1",
|
|
@@ -20,8 +20,10 @@ from .nodes_core import (ExprReturn,
|
|
|
20
20
|
TypeDef,
|
|
21
21
|
TypeDefField,
|
|
22
22
|
ExprCallStructMethod)
|
|
23
|
-
from .nodes_filter import
|
|
24
|
-
FilterStrStarts, FilterStrEnds, FilterStrRe, ExprFilter
|
|
23
|
+
from .nodes_filter import FilterOr, FilterAnd, FilterNot, FilterNotEqual, FilterEqual, FilterStrIn, \
|
|
24
|
+
FilterStrStarts, FilterStrEnds, FilterStrRe, ExprFilter, FilterStrLenEq, FilterStrLenNe, FilterStrLenLt, \
|
|
25
|
+
FilterStrLenLe, FilterStrLenGt, FilterStrLenGe
|
|
26
|
+
|
|
25
27
|
from .nodes_selectors import ExprCss, ExprCssAll, ExprXpathAll, ExprXpath, ExprGetHtmlText, ExprGetHtmlRaw, \
|
|
26
28
|
ExprGetHtmlAttr, ExprGetHtmlAttrAll, ExprGetHtmlRawAll, ExprGetHtmlTextAll
|
|
27
29
|
from .nodes_string import ExprStringTrim, ExprStringRegex, ExprStringSplit, ExprStringReplace, ExprStringLeftTrim, \
|
|
@@ -7,11 +7,6 @@ from ssc_codegen.tokens import TokenType, VariableType
|
|
|
7
7
|
|
|
8
8
|
# TODO: static analyzer for filter operations
|
|
9
9
|
# TODO: implement operators for DOCUMENT, INT, FLOAT
|
|
10
|
-
@dataclass(kw_only=True)
|
|
11
|
-
class Filter(BaseAstNode):
|
|
12
|
-
kind: ClassVar[TokenType] = TokenType.FILTER
|
|
13
|
-
accept_type: VariableType = VariableType.STRING
|
|
14
|
-
ret_type: VariableType = VariableType.STRING
|
|
15
10
|
|
|
16
11
|
|
|
17
12
|
@dataclass(kw_only=True)
|
|
@@ -38,8 +33,8 @@ class FilterNot(BaseAstNode[T_EMPTY_KWARGS, tuple]):
|
|
|
38
33
|
# TODO: typing body accept Filter-like nodes
|
|
39
34
|
|
|
40
35
|
|
|
41
|
-
KW_STR_IN = TypedDict("KW_STR_IN", {"substr": str})
|
|
42
|
-
ARGS_STR_IN = tuple[str]
|
|
36
|
+
KW_STR_IN = TypedDict("KW_STR_IN", {"substr": tuple[str, ...]})
|
|
37
|
+
ARGS_STR_IN = tuple[tuple[str, ...]]
|
|
43
38
|
|
|
44
39
|
KW_STR_STARTS_OR_ENDS = TypedDict(
|
|
45
40
|
"KW_STR_STARTS_OR_ENDS", {"substr": tuple[str, ...]}
|
|
@@ -83,9 +78,55 @@ class FilterStrRe(BaseAstNode[KW_STR_RE, ARGS_STR_RE]):
|
|
|
83
78
|
ret_type: VariableType = VariableType.STRING
|
|
84
79
|
|
|
85
80
|
|
|
81
|
+
KW_STR_LEN = TypedDict("KW_STR_LEN", {"length": int})
|
|
82
|
+
ARGS_STR_LEN = tuple[int]
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass(kw_only=True)
|
|
86
|
+
class FilterStrLenEq(BaseAstNode[KW_STR_LEN, ARGS_STR_LEN]):
|
|
87
|
+
kind: ClassVar[TokenType] = TokenType.FILTER_STR_LEN_EQ
|
|
88
|
+
accept_type: VariableType = VariableType.STRING
|
|
89
|
+
ret_type: VariableType = VariableType.STRING
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclass(kw_only=True)
|
|
93
|
+
class FilterStrLenNe(BaseAstNode[KW_STR_LEN, ARGS_STR_LEN]):
|
|
94
|
+
kind: ClassVar[TokenType] = TokenType.FILTER_STR_LEN_NE
|
|
95
|
+
accept_type: VariableType = VariableType.STRING
|
|
96
|
+
ret_type: VariableType = VariableType.STRING
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@dataclass(kw_only=True)
|
|
100
|
+
class FilterStrLenLt(BaseAstNode[KW_STR_LEN, ARGS_STR_LEN]):
|
|
101
|
+
kind: ClassVar[TokenType] = TokenType.FILTER_STR_LEN_LT
|
|
102
|
+
accept_type: VariableType = VariableType.STRING
|
|
103
|
+
ret_type: VariableType = VariableType.STRING
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@dataclass(kw_only=True)
|
|
107
|
+
class FilterStrLenLe(BaseAstNode[KW_STR_LEN, ARGS_STR_LEN]):
|
|
108
|
+
kind: ClassVar[TokenType] = TokenType.FILTER_STR_LEN_LE
|
|
109
|
+
accept_type: VariableType = VariableType.STRING
|
|
110
|
+
ret_type: VariableType = VariableType.STRING
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@dataclass(kw_only=True)
|
|
114
|
+
class FilterStrLenGt(BaseAstNode[KW_STR_LEN, ARGS_STR_LEN]):
|
|
115
|
+
kind: ClassVar[TokenType] = TokenType.FILTER_STR_LEN_GT
|
|
116
|
+
accept_type: VariableType = VariableType.STRING
|
|
117
|
+
ret_type: VariableType = VariableType.STRING
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@dataclass(kw_only=True)
|
|
121
|
+
class FilterStrLenGe(BaseAstNode[KW_STR_LEN, ARGS_STR_LEN]):
|
|
122
|
+
kind: ClassVar[TokenType] = TokenType.FILTER_STR_LEN_GE
|
|
123
|
+
accept_type: VariableType = VariableType.STRING
|
|
124
|
+
ret_type: VariableType = VariableType.STRING
|
|
125
|
+
|
|
126
|
+
|
|
86
127
|
# TODO: provide API for int, float, etc
|
|
87
|
-
KW_STR_EQ_OR_NE = TypedDict("KW_STR_EQ_OR_NE", {"
|
|
88
|
-
ARGS_STR_EQ_OR_NE = tuple[str]
|
|
128
|
+
KW_STR_EQ_OR_NE = TypedDict("KW_STR_EQ_OR_NE", {"values": tuple[str, ...]})
|
|
129
|
+
ARGS_STR_EQ_OR_NE = tuple[tuple[str, ...]]
|
|
89
130
|
|
|
90
131
|
|
|
91
132
|
@dataclass(kw_only=True)
|
|
@@ -97,7 +138,7 @@ class FilterEqual(BaseAstNode[KW_STR_EQ_OR_NE, ARGS_STR_EQ_OR_NE]):
|
|
|
97
138
|
|
|
98
139
|
@dataclass(kw_only=True)
|
|
99
140
|
class FilterNotEqual(BaseAstNode[KW_STR_EQ_OR_NE, ARGS_STR_EQ_OR_NE]):
|
|
100
|
-
kind: ClassVar[TokenType] = TokenType.
|
|
141
|
+
kind: ClassVar[TokenType] = TokenType.FILTER_NE
|
|
101
142
|
accept_type: VariableType = VariableType.STRING
|
|
102
143
|
ret_type: VariableType = VariableType.STRING
|
|
103
144
|
|
|
@@ -95,6 +95,12 @@ from ssc_codegen.ast_ import (
|
|
|
95
95
|
FilterAnd,
|
|
96
96
|
FilterOr,
|
|
97
97
|
ExprFilter,
|
|
98
|
+
FilterStrLenEq,
|
|
99
|
+
FilterStrLenNe,
|
|
100
|
+
FilterStrLenLt,
|
|
101
|
+
FilterStrLenLe,
|
|
102
|
+
FilterStrLenGt,
|
|
103
|
+
FilterStrLenGe,
|
|
98
104
|
)
|
|
99
105
|
from ssc_codegen.converters.base import BaseCodeConverter
|
|
100
106
|
from ssc_codegen.converters.helpers import (
|
|
@@ -774,8 +780,14 @@ def post_filter_not(_node: FilterNot) -> str:
|
|
|
774
780
|
|
|
775
781
|
@CONVERTER(FilterStrIn.kind)
|
|
776
782
|
def pre_filter_in(node: FilterStrIn) -> str:
|
|
777
|
-
|
|
778
|
-
|
|
783
|
+
values, *_ = node.unpack_args()
|
|
784
|
+
if len(values) == 1:
|
|
785
|
+
expr = f"i.includes({values[0]!r})"
|
|
786
|
+
else:
|
|
787
|
+
# to js array
|
|
788
|
+
val_arr = str(values)
|
|
789
|
+
val_arr = "[" + val_arr[1:-1] + "]"
|
|
790
|
+
expr = f"{val_arr}.some(e => i.includes(e))"
|
|
779
791
|
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
780
792
|
return f" && {expr}"
|
|
781
793
|
return expr
|
|
@@ -783,11 +795,14 @@ def pre_filter_in(node: FilterStrIn) -> str:
|
|
|
783
795
|
|
|
784
796
|
@CONVERTER(FilterStrStarts.kind)
|
|
785
797
|
def pre_filter_starts_with(node: FilterStrStarts) -> str:
|
|
786
|
-
|
|
787
|
-
if len(
|
|
788
|
-
expr = f"i.startsWith({
|
|
798
|
+
values, *_ = node.unpack_args()
|
|
799
|
+
if len(values) == 1:
|
|
800
|
+
expr = f"i.startsWith({values[0]!r})"
|
|
789
801
|
else:
|
|
790
|
-
|
|
802
|
+
# to js array
|
|
803
|
+
val_arr = str(values)
|
|
804
|
+
val_arr = "[" + val_arr[1:-1] + "]"
|
|
805
|
+
expr = f"{val_arr}.some(e => i.startsWith(e))"
|
|
791
806
|
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
792
807
|
return f" && {expr}"
|
|
793
808
|
return expr
|
|
@@ -795,11 +810,14 @@ def pre_filter_starts_with(node: FilterStrStarts) -> str:
|
|
|
795
810
|
|
|
796
811
|
@CONVERTER(FilterStrEnds.kind)
|
|
797
812
|
def pre_filter_ends_with(node: FilterStrEnds) -> str:
|
|
798
|
-
|
|
799
|
-
if len(
|
|
800
|
-
expr = f"i.endsWith({
|
|
813
|
+
values, *_ = node.unpack_args()
|
|
814
|
+
if len(values) == 1:
|
|
815
|
+
expr = f"i.endsWith({values[0]!r})"
|
|
801
816
|
else:
|
|
802
|
-
|
|
817
|
+
# to js array
|
|
818
|
+
val_arr = str(values)
|
|
819
|
+
val_arr = "[" + val_arr[1:-1] + "]"
|
|
820
|
+
expr = f"{val_arr}.some(e => i.endsWith(e))"
|
|
803
821
|
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
804
822
|
return f" && {expr}"
|
|
805
823
|
return expr
|
|
@@ -818,8 +836,13 @@ def pre_filter_re(node: FilterStrRe) -> str:
|
|
|
818
836
|
|
|
819
837
|
@CONVERTER(FilterEqual.kind)
|
|
820
838
|
def pre_filter_eq(node: FilterEqual) -> str:
|
|
821
|
-
|
|
822
|
-
|
|
839
|
+
values, *_ = node.unpack_args()
|
|
840
|
+
if len(values) == 1:
|
|
841
|
+
expr = f"i === {values[0]!r}"
|
|
842
|
+
else:
|
|
843
|
+
val_arr = str(values)
|
|
844
|
+
val_arr = "[" + val_arr[1:-1] + "]"
|
|
845
|
+
expr = f"{val_arr}.every(e => i === e)"
|
|
823
846
|
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
824
847
|
return f" && {expr}"
|
|
825
848
|
return expr
|
|
@@ -827,8 +850,67 @@ def pre_filter_eq(node: FilterEqual) -> str:
|
|
|
827
850
|
|
|
828
851
|
@CONVERTER(FilterNotEqual.kind)
|
|
829
852
|
def pre_filter_ne(node: FilterNotEqual) -> str:
|
|
830
|
-
|
|
831
|
-
|
|
853
|
+
values, *_ = node.unpack_args()
|
|
854
|
+
if len(values) == 1:
|
|
855
|
+
expr = f"i !== {values[0]!r}"
|
|
856
|
+
else:
|
|
857
|
+
val_arr = str(values)
|
|
858
|
+
val_arr = "[" + val_arr[1:-1] + "]"
|
|
859
|
+
expr = f"{val_arr}.every(e => i !== e)"
|
|
860
|
+
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
861
|
+
return f" && {expr}"
|
|
862
|
+
return expr
|
|
863
|
+
|
|
864
|
+
|
|
865
|
+
@CONVERTER(FilterStrLenEq.kind)
|
|
866
|
+
def pre_filter_str_len_eq(node: FilterStrLenEq) -> str:
|
|
867
|
+
length, *_ = node.unpack_args()
|
|
868
|
+
expr = f"i.length == {length}"
|
|
869
|
+
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
870
|
+
return f" && {expr}"
|
|
871
|
+
return expr
|
|
872
|
+
|
|
873
|
+
|
|
874
|
+
@CONVERTER(FilterStrLenNe.kind)
|
|
875
|
+
def pre_filter_str_len_ne(node: FilterStrLenNe) -> str:
|
|
876
|
+
length, *_ = node.unpack_args()
|
|
877
|
+
expr = f"i.length != {length}"
|
|
878
|
+
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
879
|
+
return f" && {expr}"
|
|
880
|
+
return expr
|
|
881
|
+
|
|
882
|
+
|
|
883
|
+
@CONVERTER(FilterStrLenLt.kind)
|
|
884
|
+
def pre_filter_str_len_lt(node: FilterStrLenLt) -> str:
|
|
885
|
+
length, *_ = node.unpack_args()
|
|
886
|
+
expr = f"i.length < {length}"
|
|
887
|
+
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
888
|
+
return f" && {expr}"
|
|
889
|
+
return expr
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
@CONVERTER(FilterStrLenLe.kind)
|
|
893
|
+
def pre_filter_str_len_le(node: FilterStrLenLe) -> str:
|
|
894
|
+
length, *_ = node.unpack_args()
|
|
895
|
+
expr = f"i.length <= {length}"
|
|
896
|
+
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
897
|
+
return f" && {expr}"
|
|
898
|
+
return expr
|
|
899
|
+
|
|
900
|
+
|
|
901
|
+
@CONVERTER(FilterStrLenGt.kind)
|
|
902
|
+
def pre_filter_str_len_gt(node: FilterStrLenGt) -> str:
|
|
903
|
+
length, *_ = node.unpack_args()
|
|
904
|
+
expr = f"i.length > {length}"
|
|
905
|
+
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
906
|
+
return f" && {expr}"
|
|
907
|
+
return expr
|
|
908
|
+
|
|
909
|
+
|
|
910
|
+
@CONVERTER(FilterStrLenGe.kind)
|
|
911
|
+
def pre_filter_str_len_ge(node: FilterStrLenGe) -> str:
|
|
912
|
+
length, *_ = node.unpack_args()
|
|
913
|
+
expr = f"i.length >= {length}"
|
|
832
914
|
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
833
915
|
return f" && {expr}"
|
|
834
916
|
return expr
|
|
@@ -97,6 +97,12 @@ from ssc_codegen.ast_ import (
|
|
|
97
97
|
FilterStrRe,
|
|
98
98
|
FilterEqual,
|
|
99
99
|
FilterNotEqual,
|
|
100
|
+
FilterStrLenEq,
|
|
101
|
+
FilterStrLenNe,
|
|
102
|
+
FilterStrLenLt,
|
|
103
|
+
FilterStrLenLe,
|
|
104
|
+
FilterStrLenGt,
|
|
105
|
+
FilterStrLenGe,
|
|
100
106
|
)
|
|
101
107
|
from ssc_codegen.converters.base import (
|
|
102
108
|
BaseCodeConverter,
|
|
@@ -259,6 +265,12 @@ class BasePyCodeConverter(BaseCodeConverter):
|
|
|
259
265
|
FilterStrRe.kind: pre_filter_re,
|
|
260
266
|
FilterEqual.kind: pre_filter_eq,
|
|
261
267
|
FilterNotEqual.kind: pre_filter_ne,
|
|
268
|
+
FilterStrLenEq.kind: pre_filter_str_len_eq,
|
|
269
|
+
FilterStrLenNe.kind: pre_filter_str_len_ne,
|
|
270
|
+
FilterStrLenLt.kind: pre_filter_str_len_lt,
|
|
271
|
+
FilterStrLenLe.kind: pre_filter_str_len_le,
|
|
272
|
+
FilterStrLenGe.kind: pre_filter_str_len_ge,
|
|
273
|
+
FilterStrLenGt.kind: pre_filter_str_len_gt,
|
|
262
274
|
}
|
|
263
275
|
|
|
264
276
|
self.post_definitions = {
|
|
@@ -947,8 +959,12 @@ def post_filter_not(_node: FilterNot) -> str:
|
|
|
947
959
|
|
|
948
960
|
|
|
949
961
|
def pre_filter_in(node: FilterStrIn) -> str:
|
|
950
|
-
|
|
951
|
-
|
|
962
|
+
values, *_ = node.unpack_args()
|
|
963
|
+
|
|
964
|
+
if len(values) == 1:
|
|
965
|
+
expr = f"{values[0]!r} in i"
|
|
966
|
+
else:
|
|
967
|
+
expr = f"any(s in i for s in {values})"
|
|
952
968
|
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
953
969
|
return f" and {expr}"
|
|
954
970
|
return expr
|
|
@@ -956,6 +972,7 @@ def pre_filter_in(node: FilterStrIn) -> str:
|
|
|
956
972
|
|
|
957
973
|
def pre_filter_starts_with(node: FilterStrStarts) -> str:
|
|
958
974
|
start_, *_ = node.unpack_args()
|
|
975
|
+
# build-in python startswith accept tuple[str, ...]
|
|
959
976
|
expr = f"i.startswith({start_})"
|
|
960
977
|
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
961
978
|
return f" and {expr}"
|
|
@@ -964,6 +981,7 @@ def pre_filter_starts_with(node: FilterStrStarts) -> str:
|
|
|
964
981
|
|
|
965
982
|
def pre_filter_ends_with(node: FilterStrEnds) -> str:
|
|
966
983
|
suffix_, *_ = node.unpack_args()
|
|
984
|
+
# build-in python endswith accept tuple[str, ...]
|
|
967
985
|
expr = f"i.endswith({suffix_})"
|
|
968
986
|
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
969
987
|
return f" and {expr}"
|
|
@@ -982,18 +1000,72 @@ def pre_filter_re(node: FilterStrRe) -> str:
|
|
|
982
1000
|
|
|
983
1001
|
|
|
984
1002
|
def pre_filter_eq(node: FilterEqual) -> str:
|
|
985
|
-
|
|
1003
|
+
values, *_ = node.unpack_args()
|
|
986
1004
|
# currently support only str
|
|
987
|
-
|
|
1005
|
+
if len(values) == 1:
|
|
1006
|
+
expr = f"i == {values[0]!r}"
|
|
1007
|
+
else:
|
|
1008
|
+
expr = f"all(s == i for s in {values})"
|
|
988
1009
|
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
989
1010
|
return f" and {expr}"
|
|
990
1011
|
return expr
|
|
991
1012
|
|
|
992
1013
|
|
|
993
1014
|
def pre_filter_ne(node: FilterNotEqual) -> str:
|
|
994
|
-
|
|
1015
|
+
values, *_ = node.unpack_args()
|
|
995
1016
|
# currently support only str
|
|
996
|
-
|
|
1017
|
+
if len(values) == 1:
|
|
1018
|
+
expr = f"i != {values[0]!r}"
|
|
1019
|
+
else:
|
|
1020
|
+
expr = f"all(s != i for s in {values})"
|
|
1021
|
+
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
1022
|
+
return f" and {expr}"
|
|
1023
|
+
return expr
|
|
1024
|
+
|
|
1025
|
+
|
|
1026
|
+
def pre_filter_str_len_eq(node: FilterStrLenEq) -> str:
|
|
1027
|
+
length, *_ = node.unpack_args()
|
|
1028
|
+
expr = f"len(i) == {length}"
|
|
1029
|
+
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
1030
|
+
return f" and {expr}"
|
|
1031
|
+
return expr
|
|
1032
|
+
|
|
1033
|
+
|
|
1034
|
+
def pre_filter_str_len_ne(node: FilterStrLenNe) -> str:
|
|
1035
|
+
length, *_ = node.unpack_args()
|
|
1036
|
+
expr = f"len(i) != {length}"
|
|
1037
|
+
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
1038
|
+
return f" and {expr}"
|
|
1039
|
+
return expr
|
|
1040
|
+
|
|
1041
|
+
|
|
1042
|
+
def pre_filter_str_len_lt(node: FilterStrLenLt) -> str:
|
|
1043
|
+
length, *_ = node.unpack_args()
|
|
1044
|
+
expr = f"len(i) < {length}"
|
|
1045
|
+
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
1046
|
+
return f" and {expr}"
|
|
1047
|
+
return expr
|
|
1048
|
+
|
|
1049
|
+
|
|
1050
|
+
def pre_filter_str_len_le(node: FilterStrLenLe) -> str:
|
|
1051
|
+
length, *_ = node.unpack_args()
|
|
1052
|
+
expr = f"len(i) <= {length}"
|
|
1053
|
+
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
1054
|
+
return f" and {expr}"
|
|
1055
|
+
return expr
|
|
1056
|
+
|
|
1057
|
+
|
|
1058
|
+
def pre_filter_str_len_gt(node: FilterStrLenGt) -> str:
|
|
1059
|
+
length, *_ = node.unpack_args()
|
|
1060
|
+
expr = f"len(i) > {length}"
|
|
1061
|
+
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
1062
|
+
return f" and {expr}"
|
|
1063
|
+
return expr
|
|
1064
|
+
|
|
1065
|
+
|
|
1066
|
+
def pre_filter_str_len_ge(node: FilterStrLenGe) -> str:
|
|
1067
|
+
length, *_ = node.unpack_args()
|
|
1068
|
+
expr = f"len(i) >= {length}"
|
|
997
1069
|
if not is_first_node_cond(node) and is_prev_node_atomic_cond(node):
|
|
998
1070
|
return f" and {expr}"
|
|
999
1071
|
return expr
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""high-level AST builder interface"""
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
-
from typing import Type, Pattern
|
|
4
|
+
from typing import Type, Pattern, Sequence
|
|
5
5
|
|
|
6
6
|
from cssselect import SelectorSyntaxError
|
|
7
7
|
from typing_extensions import Self, assert_never
|
|
@@ -69,18 +69,24 @@ from ssc_codegen.ast_ import (
|
|
|
69
69
|
FilterAnd,
|
|
70
70
|
FilterOr,
|
|
71
71
|
FilterNot,
|
|
72
|
+
FilterStrLenEq,
|
|
73
|
+
FilterStrLenNe,
|
|
74
|
+
FilterStrLenLt,
|
|
75
|
+
FilterStrLenLe,
|
|
76
|
+
FilterStrLenGt,
|
|
77
|
+
FilterStrLenGe,
|
|
72
78
|
)
|
|
73
79
|
from ssc_codegen.document_utlis import (
|
|
74
80
|
analyze_re_expression,
|
|
75
81
|
unverbosify_regex,
|
|
76
82
|
is_ignore_case_regex,
|
|
77
83
|
)
|
|
84
|
+
from ssc_codegen.json_struct import Json
|
|
78
85
|
from ssc_codegen.pseudo_selectors import (
|
|
79
86
|
parse_pseudo_xpath_query,
|
|
80
87
|
parse_pseudo_css_query,
|
|
81
88
|
PseudoAction,
|
|
82
89
|
)
|
|
83
|
-
from ssc_codegen.json_struct import Json
|
|
84
90
|
from ssc_codegen.schema import BaseSchema
|
|
85
91
|
from ssc_codegen.selector_utils import validate_css_query, validate_xpath_query
|
|
86
92
|
from ssc_codegen.tokens import VariableType
|
|
@@ -1263,29 +1269,80 @@ class JsonDocument(BaseDocument):
|
|
|
1263
1269
|
|
|
1264
1270
|
|
|
1265
1271
|
class DocumentFilter(BaseDocument):
|
|
1266
|
-
def eq(self,
|
|
1267
|
-
|
|
1272
|
+
def eq(self, *values: str) -> Self:
|
|
1273
|
+
"""check if value equal
|
|
1274
|
+
|
|
1275
|
+
multiple values converts to logical AND
|
|
1276
|
+
|
|
1277
|
+
pseudocode example:
|
|
1278
|
+
|
|
1279
|
+
F().eq("foo") -> value == "foo"
|
|
1280
|
+
|
|
1281
|
+
F().eq("bar", "foo") -> (value == "bar" && value == "foo")
|
|
1282
|
+
"""
|
|
1283
|
+
self._add(FilterEqual(kwargs={"values": values}))
|
|
1268
1284
|
return self
|
|
1269
1285
|
|
|
1270
|
-
def ne(self,
|
|
1271
|
-
|
|
1286
|
+
def ne(self, *values: str) -> Self:
|
|
1287
|
+
"""check if value not equal
|
|
1288
|
+
|
|
1289
|
+
multiple values converts to logical AND
|
|
1290
|
+
|
|
1291
|
+
pseudocode example:
|
|
1292
|
+
|
|
1293
|
+
F().eq("foo") -> value == "foo"
|
|
1294
|
+
|
|
1295
|
+
F().eq("bar", "foo") -> (value != "bar" && value != "foo")
|
|
1296
|
+
"""
|
|
1297
|
+
self._add(FilterNotEqual(kwargs={"values": values}))
|
|
1272
1298
|
return self
|
|
1273
1299
|
|
|
1274
1300
|
def starts(self, *values: str) -> Self:
|
|
1301
|
+
"""check if value starts by substring
|
|
1302
|
+
|
|
1303
|
+
multiple values converts to logical OR
|
|
1304
|
+
|
|
1305
|
+
pseudocode example:
|
|
1306
|
+
|
|
1307
|
+
F().eq("foo") -> value == "foo"
|
|
1308
|
+
|
|
1309
|
+
F().eq("foo", "bar") -> (value.starts_with("bar") || value.starts_with("foo"))
|
|
1310
|
+
"""
|
|
1275
1311
|
self._add(FilterStrStarts(kwargs={"substr": values}))
|
|
1276
1312
|
return self
|
|
1277
1313
|
|
|
1278
1314
|
def ends(self, *values: str) -> Self:
|
|
1315
|
+
"""check if value starts by substring
|
|
1316
|
+
|
|
1317
|
+
multiple values converts to logical OR
|
|
1318
|
+
|
|
1319
|
+
pseudocode example:
|
|
1320
|
+
|
|
1321
|
+
F().eq("foo") -> value == "foo"
|
|
1322
|
+
|
|
1323
|
+
F().eq("foo", "bar") -> (value.ends_with("bar") || value.ends_with("foo"))
|
|
1324
|
+
"""
|
|
1279
1325
|
self._add(FilterStrEnds(kwargs={"substr": values}))
|
|
1280
1326
|
return self
|
|
1281
1327
|
|
|
1282
|
-
def contains(self,
|
|
1283
|
-
|
|
1328
|
+
def contains(self, *values: str) -> Self:
|
|
1329
|
+
"""check if value contains by substring
|
|
1330
|
+
|
|
1331
|
+
multiple values converts to logical OR
|
|
1332
|
+
|
|
1333
|
+
pseudocode example:
|
|
1334
|
+
|
|
1335
|
+
F().eq("foo") -> value == "foo"
|
|
1336
|
+
|
|
1337
|
+
F().eq("foo", "bar") -> (value.include("bar") || value.include("foo"))
|
|
1338
|
+
"""
|
|
1339
|
+
self._add(FilterStrIn(kwargs={"substr": values}))
|
|
1284
1340
|
return self
|
|
1285
1341
|
|
|
1286
1342
|
def re(
|
|
1287
1343
|
self, pattern: str | Pattern[str], ignore_case: bool = False
|
|
1288
1344
|
) -> Self:
|
|
1345
|
+
"""check if pattern matched result in value"""
|
|
1289
1346
|
if not isinstance(pattern, str):
|
|
1290
1347
|
ignore_case = is_ignore_case_regex(pattern)
|
|
1291
1348
|
|
|
@@ -1316,22 +1373,73 @@ class DocumentFilter(BaseDocument):
|
|
|
1316
1373
|
self._add(FilterOr(body=tmp_stack))
|
|
1317
1374
|
return self
|
|
1318
1375
|
|
|
1376
|
+
def len_eq(self, value: int) -> Self:
|
|
1377
|
+
self._add(FilterStrLenEq(kwargs={"length": value}))
|
|
1378
|
+
return self
|
|
1379
|
+
|
|
1380
|
+
def len_ne(self, value: int) -> Self:
|
|
1381
|
+
self._add(FilterStrLenNe(kwargs={"length": value}))
|
|
1382
|
+
return self
|
|
1383
|
+
|
|
1384
|
+
def len_lt(self, value: int) -> Self:
|
|
1385
|
+
self._add(FilterStrLenLt(kwargs={"length": value}))
|
|
1386
|
+
return self
|
|
1387
|
+
|
|
1388
|
+
def len_le(self, value: int) -> Self:
|
|
1389
|
+
self._add(FilterStrLenLe(kwargs={"length": value}))
|
|
1390
|
+
return self
|
|
1391
|
+
|
|
1392
|
+
def len_gt(self, value: int) -> Self:
|
|
1393
|
+
self._add(FilterStrLenGt(kwargs={"length": value}))
|
|
1394
|
+
return self
|
|
1395
|
+
|
|
1396
|
+
def len_ge(self, value: int) -> Self:
|
|
1397
|
+
self._add(FilterStrLenGe(kwargs={"length": value}))
|
|
1398
|
+
return self
|
|
1399
|
+
|
|
1319
1400
|
def not_(self, filter_expr: "DocumentFilter") -> Self:
|
|
1320
1401
|
tmp_stack = filter_expr.stack.copy()
|
|
1321
1402
|
self._add(FilterNot(body=tmp_stack))
|
|
1322
1403
|
return self
|
|
1323
1404
|
|
|
1324
1405
|
def __or__(self, other: "DocumentFilter") -> "DocumentFilter":
|
|
1406
|
+
"""syntax suger F().or_(...)"""
|
|
1325
1407
|
return self.or_(other)
|
|
1326
1408
|
|
|
1327
1409
|
def __and__(self, other: "DocumentFilter") -> "DocumentFilter":
|
|
1410
|
+
"""syntax sugar F().and_(...)"""
|
|
1328
1411
|
return self.and_(other)
|
|
1329
1412
|
|
|
1330
1413
|
def __invert__(self) -> "DocumentFilter":
|
|
1414
|
+
"""syntax sugar F().not_(...)"""
|
|
1331
1415
|
return DocumentFilter().not_(self)
|
|
1332
1416
|
|
|
1333
|
-
def __eq__(self, other: str) -> Self:
|
|
1334
|
-
|
|
1417
|
+
def __eq__(self, other: int | str | Sequence[str]) -> Self:
|
|
1418
|
+
"""syntax sugar F().eq(...)"""
|
|
1419
|
+
if isinstance(other, int):
|
|
1420
|
+
return self.len_eq(other)
|
|
1421
|
+
|
|
1422
|
+
if isinstance(other, str):
|
|
1423
|
+
other = (other,)
|
|
1424
|
+
return self.eq(*other)
|
|
1425
|
+
|
|
1426
|
+
def __ne__(self, other: int | str | Sequence[str]) -> Self:
|
|
1427
|
+
"""syntax sugar F().ne(...)"""
|
|
1428
|
+
if isinstance(other, int):
|
|
1429
|
+
return self.len_ne(other)
|
|
1430
|
+
|
|
1431
|
+
if isinstance(other, str):
|
|
1432
|
+
other = (other,)
|
|
1433
|
+
return self.ne(*other)
|
|
1434
|
+
|
|
1435
|
+
def __lt__(self, other: int) -> Self:
|
|
1436
|
+
return self.len_lt(other)
|
|
1437
|
+
|
|
1438
|
+
def __le__(self, other: int) -> Self:
|
|
1439
|
+
return self.len_le(other)
|
|
1440
|
+
|
|
1441
|
+
def __gt__(self, other: int) -> Self:
|
|
1442
|
+
return self.len_gt(other)
|
|
1335
1443
|
|
|
1336
|
-
def
|
|
1337
|
-
return self.
|
|
1444
|
+
def __ge__(self, other: int) -> Self:
|
|
1445
|
+
return self.len_ge(other)
|
|
@@ -177,15 +177,22 @@ class TokenType(IntEnum):
|
|
|
177
177
|
# FILTER
|
|
178
178
|
EXPR_FILTER = auto()
|
|
179
179
|
|
|
180
|
-
FILTER = auto() # TODO: REMOVE?
|
|
181
180
|
FILTER_OR = auto()
|
|
182
181
|
FILTER_AND = auto()
|
|
183
182
|
FILTER_NOT = auto()
|
|
184
|
-
# FILTER EXPR
|
|
183
|
+
# FILTER EXPR (STR)
|
|
185
184
|
FILTER_STR_IN = auto()
|
|
186
185
|
FILTER_STR_STARTS = auto()
|
|
187
186
|
FILTER_STR_ENDS = auto()
|
|
188
187
|
FILTER_STR_RE = auto()
|
|
188
|
+
|
|
189
|
+
FILTER_STR_LEN_EQ = auto()
|
|
190
|
+
FILTER_STR_LEN_NE = auto()
|
|
191
|
+
FILTER_STR_LEN_LT = auto()
|
|
192
|
+
FILTER_STR_LEN_LE = auto()
|
|
193
|
+
FILTER_STR_LEN_GT = auto()
|
|
194
|
+
FILTER_STR_LEN_GE = auto()
|
|
195
|
+
|
|
189
196
|
FILTER_EQ = auto()
|
|
190
197
|
FILTER_NE = auto()
|
|
191
198
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|