ssc_codegen 0.8.4__tar.gz → 0.8.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ssc_codegen might be problematic. Click here for more details.
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/PKG-INFO +2 -1
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/pyproject.toml +3 -2
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/__init__.py +1 -1
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/ast_/__init__.py +2 -1
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/ast_/nodes_validate.py +16 -2
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/ast_build/utils.py +4 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/cli/main.py +7 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/converters/go_goquery.py +49 -3
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/converters/js_pure.py +45 -3
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/converters/py_base.py +39 -3
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/converters/templates/go_goquery.py +39 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/document.py +76 -2
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/selector_utils.py +14 -5
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/static_checker/base.py +3 -1
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/static_checker/callbacks.py +1 -1
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/tokens.py +5 -3
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/.gitignore +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/LICENSE +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/README.md +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/_compat.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/ast_/base.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/ast_/nodes_array.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/ast_/nodes_cast.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/ast_/nodes_core.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/ast_/nodes_selectors.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/ast_/nodes_string.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/ast_build/__init__.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/ast_build/main.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/cli/__init__.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/cli/cli_callbacks.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/cli/cli_utils.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/cli/code_callbacks.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/cli/consts.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/cli/runtime_parse_runners.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/compiler.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/consts.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/converters/__init__.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/converters/base.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/converters/helpers.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/converters/py_bs4.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/converters/py_parsel.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/converters/py_selectolax.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/converters/templates/__init__.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/converters/templates/js_pure.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/document_utlis.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/json_struct.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/json_to_scc.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/logs.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/schema.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/static_checker/__init__.py +0 -0
- {ssc_codegen-0.8.4 → ssc_codegen-0.8.6}/ssc_codegen/str_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ssc_codegen
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.6
|
|
4
4
|
Summary: Python-dsl code converter to html parser for web scraping
|
|
5
5
|
Project-URL: Documentation, https://github.com/vypivshiy/selector_schema_codegen#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/vypivshiy/selector_schema_codegen/issues
|
|
@@ -26,6 +26,7 @@ Requires-Dist: ichrome>=4.0.4
|
|
|
26
26
|
Requires-Dist: jinja2>=3.1.6
|
|
27
27
|
Requires-Dist: lxml>=5.3.0
|
|
28
28
|
Requires-Dist: parsel>=1.10.0
|
|
29
|
+
Requires-Dist: soupsieve>=2.6
|
|
29
30
|
Requires-Dist: typer>=0.15.1
|
|
30
31
|
Requires-Dist: typing-extensions; python_version < '3.11'
|
|
31
32
|
Description-Content-Type: text/markdown
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "ssc_codegen"
|
|
3
|
-
version = "0.8.
|
|
3
|
+
version = "0.8.6"
|
|
4
4
|
description = "Python-dsl code converter to html parser for web scraping "
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.10"
|
|
@@ -12,6 +12,7 @@ dependencies = [
|
|
|
12
12
|
"jinja2>=3.1.6",
|
|
13
13
|
"lxml>=5.3.0",
|
|
14
14
|
"parsel>=1.10.0",
|
|
15
|
+
"soupsieve>=2.6",
|
|
15
16
|
"typer>=0.15.1",
|
|
16
17
|
"typing_extensions;python_version<'3.11'",
|
|
17
18
|
]
|
|
@@ -106,4 +107,4 @@ omit = [
|
|
|
106
107
|
"*/.venv/*",
|
|
107
108
|
"*/scripts/*",
|
|
108
109
|
"*/tests/*"
|
|
109
|
-
]
|
|
110
|
+
]
|
|
@@ -29,4 +29,5 @@ from .nodes_string import ExprStringTrim, ExprStringRegex, ExprStringSplit, Expr
|
|
|
29
29
|
ExprListStringRegexSub, \
|
|
30
30
|
ExprStringRmPrefix, ExprStringRmSuffix, ExprStringRmPrefixAndSuffix, ExprListStringRmPrefixAndSuffix, \
|
|
31
31
|
ExprListStringRmPrefix, ExprListStringRmSuffix
|
|
32
|
-
from .nodes_validate import ExprIsCss, ExprIsEqual,
|
|
32
|
+
from .nodes_validate import ExprIsCss, ExprIsEqual, ExprStringIsRegex, ExprIsXpath, ExprIsNotEqual, ExprIsContains, \
|
|
33
|
+
ExprListStringAnyRegex, ExprListStringAllRegex
|
|
@@ -59,12 +59,26 @@ ARGS_IS_REGEX = tuple[str, bool, str]
|
|
|
59
59
|
|
|
60
60
|
|
|
61
61
|
@dataclass(kw_only=True)
|
|
62
|
-
class
|
|
63
|
-
kind: ClassVar[TokenType] = TokenType.
|
|
62
|
+
class ExprStringIsRegex(BaseAstNode[KW_IS_REGEX, ARGS_IS_REGEX]):
|
|
63
|
+
kind: ClassVar[TokenType] = TokenType.IS_STRING_REGEX_MATCH
|
|
64
64
|
accept_type: VariableType = VariableType.STRING
|
|
65
65
|
ret_type: VariableType = VariableType.STRING
|
|
66
66
|
|
|
67
67
|
|
|
68
|
+
@dataclass(kw_only=True)
|
|
69
|
+
class ExprListStringAnyRegex(BaseAstNode[KW_IS_REGEX, ARGS_IS_REGEX]):
|
|
70
|
+
kind: ClassVar[TokenType] = TokenType.ANY_LIST_STRING_REGEX_MATCH
|
|
71
|
+
accept_type: VariableType = VariableType.LIST_STRING
|
|
72
|
+
ret_type: VariableType = VariableType.LIST_STRING
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass(kw_only=True)
|
|
76
|
+
class ExprListStringAllRegex(BaseAstNode[KW_IS_REGEX, ARGS_IS_REGEX]):
|
|
77
|
+
kind: ClassVar[TokenType] = TokenType.ALL_LIST_STRING_REGEX_MATCH
|
|
78
|
+
accept_type: VariableType = VariableType.LIST_STRING
|
|
79
|
+
ret_type: VariableType = VariableType.LIST_STRING
|
|
80
|
+
|
|
81
|
+
|
|
68
82
|
KW_IS_SELECT = TypedDict("KW_IS_SELECT", {"query": str, "msg": str})
|
|
69
83
|
ARGS_IS_SELECT = tuple[str, str]
|
|
70
84
|
|
|
@@ -95,9 +95,13 @@ def exec_module_code(path: str | Path, add_sys_path: bool = True) -> ModuleType:
|
|
|
95
95
|
module = ModuleType("_")
|
|
96
96
|
abs_path = path.resolve()
|
|
97
97
|
# required for correct imports (eg: constants)
|
|
98
|
+
# TODO: calculate configs dirs depth
|
|
98
99
|
if add_sys_path and str(abs_path.parent) not in sys.path:
|
|
99
100
|
sys.path.append(str(abs_path.parent))
|
|
100
101
|
|
|
102
|
+
if add_sys_path and str(abs_path.parent.parent) not in sys.path:
|
|
103
|
+
sys.path.append(str(abs_path.parent.parent))
|
|
104
|
+
|
|
101
105
|
code = Path(abs_path).read_text()
|
|
102
106
|
exec(code, module.__dict__)
|
|
103
107
|
return module
|
|
@@ -100,6 +100,13 @@ def generate_code(
|
|
|
100
100
|
LOGGER.info("TOGGLE debug generated tokens")
|
|
101
101
|
converter.debug_instructions = True
|
|
102
102
|
for file_cfg in ssc_files:
|
|
103
|
+
# check ssc-gen files
|
|
104
|
+
if (
|
|
105
|
+
file_cfg.suffix != ".py"
|
|
106
|
+
and "ssc_codegen" not in file_cfg.read_text()
|
|
107
|
+
):
|
|
108
|
+
LOGGER.info("Skip %s (not ssc-gen config)", file_cfg.name)
|
|
109
|
+
continue
|
|
103
110
|
name = file_cfg.name.split(".")[0]
|
|
104
111
|
out_file = f"{prefix}{name}{suffix}"
|
|
105
112
|
LOGGER.info("Make AST %s...", file_cfg.name)
|
|
@@ -90,7 +90,7 @@ from ssc_codegen.ast_ import (
|
|
|
90
90
|
ExprIsEqual,
|
|
91
91
|
ExprIsNotEqual,
|
|
92
92
|
ExprIsContains,
|
|
93
|
-
|
|
93
|
+
ExprStringIsRegex,
|
|
94
94
|
ExprIsCss,
|
|
95
95
|
ExprToInt,
|
|
96
96
|
ExprToListInt,
|
|
@@ -122,6 +122,8 @@ from ssc_codegen.ast_ import (
|
|
|
122
122
|
ExprListStringRmSuffix,
|
|
123
123
|
ExprStringRmPrefixAndSuffix,
|
|
124
124
|
ExprListStringRmPrefixAndSuffix,
|
|
125
|
+
ExprListStringAnyRegex,
|
|
126
|
+
ExprListStringAllRegex,
|
|
125
127
|
)
|
|
126
128
|
from ssc_codegen.converters.base import BaseCodeConverter
|
|
127
129
|
from ssc_codegen.converters.helpers import (
|
|
@@ -166,6 +168,8 @@ from ssc_codegen.converters.templates.go_goquery import (
|
|
|
166
168
|
J2_PRE_LIST_STR_RM_SUFFIX,
|
|
167
169
|
J2_PRE_STR_RM_PREFIX_AND_SUFFIX,
|
|
168
170
|
J2_PRE_LIST_STR_RM_PREFIX_AND_SUFFIX,
|
|
171
|
+
J2_PRE_LIST_STR_ANY_IS_RE,
|
|
172
|
+
J2_PRE_LIST_STR_ALL_IS_RE,
|
|
169
173
|
)
|
|
170
174
|
from ssc_codegen.str_utils import (
|
|
171
175
|
wrap_backtick,
|
|
@@ -853,8 +857,8 @@ def pre_is_contains(node: ExprIsContains) -> str:
|
|
|
853
857
|
return J2_PRE_IS_CONTAINS.render(**context)
|
|
854
858
|
|
|
855
859
|
|
|
856
|
-
@CONVERTER(
|
|
857
|
-
def pre_is_regex(node:
|
|
860
|
+
@CONVERTER(ExprStringIsRegex.kind)
|
|
861
|
+
def pre_is_regex(node: ExprStringIsRegex) -> str:
|
|
858
862
|
prv, nxt = prev_next_var(node)
|
|
859
863
|
pattern, ignore_case, msg = node.unpack_args()
|
|
860
864
|
|
|
@@ -880,6 +884,48 @@ def pre_is_regex(node: ExprIsRegex) -> str:
|
|
|
880
884
|
return J2_PRE_IS_REGEX.render(**context)
|
|
881
885
|
|
|
882
886
|
|
|
887
|
+
@CONVERTER(ExprListStringAnyRegex.kind)
|
|
888
|
+
def pre_list_str_any_is_regex(node: ExprListStringAnyRegex) -> str:
|
|
889
|
+
prv, nxt = prev_next_var(node)
|
|
890
|
+
pattern, ignore_case, msg = node.unpack_args()
|
|
891
|
+
if ignore_case:
|
|
892
|
+
pattern = f"(?i){pattern}"
|
|
893
|
+
pattern = wrap_backtick(pattern)
|
|
894
|
+
msg = wrap_double_quotes(msg)
|
|
895
|
+
context = {
|
|
896
|
+
"prv": prv,
|
|
897
|
+
"nxt": nxt,
|
|
898
|
+
"pattern": pattern, # Already processed with ignore_case and backticks
|
|
899
|
+
"msg": msg,
|
|
900
|
+
"have_default_expr": have_default_expr(node),
|
|
901
|
+
"is_pre_validate_parent": is_pre_validate_parent(node),
|
|
902
|
+
"is_last_var_no_ret": is_last_var_no_ret(node),
|
|
903
|
+
"return_type": RETURN_ERR_TYPES.get(get_last_ret_type(node), "nil"),
|
|
904
|
+
}
|
|
905
|
+
return J2_PRE_LIST_STR_ANY_IS_RE.render(**context)
|
|
906
|
+
|
|
907
|
+
|
|
908
|
+
@CONVERTER(ExprListStringAllRegex.kind)
|
|
909
|
+
def pre_list_str_all_is_regex(node: ExprListStringAllRegex) -> str:
|
|
910
|
+
prv, nxt = prev_next_var(node)
|
|
911
|
+
pattern, ignore_case, msg = node.unpack_args()
|
|
912
|
+
if ignore_case:
|
|
913
|
+
pattern = f"(?i){pattern}"
|
|
914
|
+
pattern = wrap_backtick(pattern)
|
|
915
|
+
msg = wrap_double_quotes(msg)
|
|
916
|
+
context = {
|
|
917
|
+
"prv": prv,
|
|
918
|
+
"nxt": nxt,
|
|
919
|
+
"pattern": pattern, # Already processed with ignore_case and backticks
|
|
920
|
+
"msg": msg,
|
|
921
|
+
"have_default_expr": have_default_expr(node),
|
|
922
|
+
"is_pre_validate_parent": is_pre_validate_parent(node),
|
|
923
|
+
"is_last_var_no_ret": is_last_var_no_ret(node),
|
|
924
|
+
"return_type": RETURN_ERR_TYPES.get(get_last_ret_type(node), "nil"),
|
|
925
|
+
}
|
|
926
|
+
return J2_PRE_LIST_STR_ALL_IS_RE.render(**context)
|
|
927
|
+
|
|
928
|
+
|
|
883
929
|
@CONVERTER(ExprIsCss.kind)
|
|
884
930
|
def pre_is_css(node: ExprIsCss) -> str:
|
|
885
931
|
prv, nxt = prev_next_var(node)
|
|
@@ -54,7 +54,7 @@ from ssc_codegen.ast_ import (
|
|
|
54
54
|
ExprIsEqual,
|
|
55
55
|
ExprIsNotEqual,
|
|
56
56
|
ExprIsContains,
|
|
57
|
-
|
|
57
|
+
ExprStringIsRegex,
|
|
58
58
|
ExprToInt,
|
|
59
59
|
ExprToListInt,
|
|
60
60
|
ExprToFloat,
|
|
@@ -81,6 +81,8 @@ from ssc_codegen.ast_ import (
|
|
|
81
81
|
ExprListStringRmSuffix,
|
|
82
82
|
ExprStringRmPrefixAndSuffix,
|
|
83
83
|
ExprListStringRmPrefixAndSuffix,
|
|
84
|
+
ExprListStringAnyRegex,
|
|
85
|
+
ExprListStringAllRegex,
|
|
84
86
|
)
|
|
85
87
|
from ssc_codegen.converters.base import BaseCodeConverter
|
|
86
88
|
from ssc_codegen.converters.helpers import (
|
|
@@ -146,6 +148,11 @@ def pre_part_doc(_node: StructPartDocMethod) -> str:
|
|
|
146
148
|
return "_splitDoc(v) " + BRACKET_START
|
|
147
149
|
|
|
148
150
|
|
|
151
|
+
@CONVERTER.post(StructPartDocMethod.kind)
|
|
152
|
+
def post_part_doc(_node: StructPartDocMethod) -> str:
|
|
153
|
+
return BRACKET_END
|
|
154
|
+
|
|
155
|
+
|
|
149
156
|
@CONVERTER(StructParser.kind)
|
|
150
157
|
def pre_struct_parser(node: StructParser) -> str:
|
|
151
158
|
name = node.kwargs["name"]
|
|
@@ -437,11 +444,14 @@ def pre_is_contains(node: ExprIsContains) -> str:
|
|
|
437
444
|
return expr + f"let {nxt} = {prv};"
|
|
438
445
|
|
|
439
446
|
|
|
440
|
-
@CONVERTER(
|
|
441
|
-
def pre_is_regex(node:
|
|
447
|
+
@CONVERTER(ExprStringIsRegex.kind)
|
|
448
|
+
def pre_is_regex(node: ExprStringIsRegex) -> str:
|
|
442
449
|
prv, nxt = prev_next_var(node)
|
|
443
450
|
pattern, ignore_case, msg = node.unpack_args()
|
|
444
451
|
pattern = f"/{node.pattern}/g"
|
|
452
|
+
if ignore_case:
|
|
453
|
+
pattern += "i"
|
|
454
|
+
|
|
445
455
|
expr = f"if ({prv}.match({pattern}) === null) throw new Error({msg!r});"
|
|
446
456
|
if is_last_var_no_ret(node):
|
|
447
457
|
return expr
|
|
@@ -449,6 +459,38 @@ def pre_is_regex(node: ExprIsRegex) -> str:
|
|
|
449
459
|
return expr + f"let {nxt} = {prv};"
|
|
450
460
|
|
|
451
461
|
|
|
462
|
+
@CONVERTER(ExprListStringAnyRegex.kind)
|
|
463
|
+
def pre_list_str_any_is_regex(node: ExprListStringAnyRegex) -> str:
|
|
464
|
+
# a.some(i => (new RegExp("foo")).test(i))
|
|
465
|
+
prv, nxt = prev_next_var(node)
|
|
466
|
+
pattern, ignore_case, msg = node.unpack_args()
|
|
467
|
+
pattern = f"/{node.pattern}/g"
|
|
468
|
+
if ignore_case:
|
|
469
|
+
pattern += "i"
|
|
470
|
+
|
|
471
|
+
expr = f"if (!{prv}.some(i => (new RegExp({pattern})).test(i))) throw new Error({msg!r});"
|
|
472
|
+
if is_last_var_no_ret(node):
|
|
473
|
+
return expr
|
|
474
|
+
# HACK: avoid recalc variables
|
|
475
|
+
return expr + f"let {nxt} = {prv};"
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
@CONVERTER(ExprListStringAllRegex.kind)
|
|
479
|
+
def pre_list_str_all_is_regex(node: ExprListStringAllRegex) -> str:
|
|
480
|
+
prv, nxt = prev_next_var(node)
|
|
481
|
+
# a.every(i => (new RegExp("foo")).test)
|
|
482
|
+
pattern, ignore_case, msg = node.unpack_args()
|
|
483
|
+
pattern = f"/{node.pattern}/g"
|
|
484
|
+
if ignore_case:
|
|
485
|
+
pattern += "i"
|
|
486
|
+
|
|
487
|
+
expr = f"if (!{prv}.every(i => (new RegExp({pattern})).test(i))) throw new Error({msg!r});"
|
|
488
|
+
if is_last_var_no_ret(node):
|
|
489
|
+
return expr
|
|
490
|
+
# HACK: avoid recalc variables
|
|
491
|
+
return expr + f"let {nxt} = {prv};"
|
|
492
|
+
|
|
493
|
+
|
|
452
494
|
@CONVERTER(ExprIsCss.kind)
|
|
453
495
|
def pre_is_css(node: ExprIsCss) -> str:
|
|
454
496
|
prv, nxt = prev_next_var(node)
|
|
@@ -72,7 +72,7 @@ from ssc_codegen.ast_ import (
|
|
|
72
72
|
ExprIsEqual,
|
|
73
73
|
ExprIsNotEqual,
|
|
74
74
|
ExprIsContains,
|
|
75
|
-
|
|
75
|
+
ExprStringIsRegex,
|
|
76
76
|
ExprToInt,
|
|
77
77
|
ExprToListInt,
|
|
78
78
|
ExprToFloat,
|
|
@@ -85,6 +85,8 @@ from ssc_codegen.ast_ import (
|
|
|
85
85
|
ExprListStringRmSuffix,
|
|
86
86
|
ExprStringRmPrefixAndSuffix,
|
|
87
87
|
ExprListStringRmPrefixAndSuffix,
|
|
88
|
+
ExprListStringAnyRegex,
|
|
89
|
+
ExprListStringAllRegex,
|
|
88
90
|
)
|
|
89
91
|
from ssc_codegen.converters.base import (
|
|
90
92
|
BaseCodeConverter,
|
|
@@ -222,7 +224,9 @@ class BasePyCodeConverter(BaseCodeConverter):
|
|
|
222
224
|
ExprIsEqual.kind: pre_is_equal,
|
|
223
225
|
ExprIsNotEqual.kind: pre_is_not_equal,
|
|
224
226
|
ExprIsContains.kind: pre_is_contains,
|
|
225
|
-
|
|
227
|
+
ExprStringIsRegex.kind: pre_is_regex,
|
|
228
|
+
ExprListStringAllRegex.kind: pre_list_str_all_is_regex,
|
|
229
|
+
ExprListStringAnyRegex.kind: pre_list_str_any_is_regex,
|
|
226
230
|
ExprToInt.kind: pre_to_int,
|
|
227
231
|
ExprToListInt.kind: pre_to_list_int,
|
|
228
232
|
ExprToFloat.kind: pre_to_float,
|
|
@@ -770,7 +774,7 @@ def pre_is_contains(node: ExprIsContains) -> str:
|
|
|
770
774
|
return expr + "\n" + indent + f"{nxt} = {prv}"
|
|
771
775
|
|
|
772
776
|
|
|
773
|
-
def pre_is_regex(node:
|
|
777
|
+
def pre_is_regex(node: ExprStringIsRegex) -> str:
|
|
774
778
|
indent = (
|
|
775
779
|
INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
|
|
776
780
|
)
|
|
@@ -783,6 +787,38 @@ def pre_is_regex(node: ExprIsRegex) -> str:
|
|
|
783
787
|
return expr + "\n" + indent + f"{nxt} = {prv}"
|
|
784
788
|
|
|
785
789
|
|
|
790
|
+
def pre_list_str_any_is_regex(node: ExprListStringAnyRegex) -> str:
|
|
791
|
+
indent = (
|
|
792
|
+
INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
|
|
793
|
+
)
|
|
794
|
+
prv, nxt = prev_next_var(node)
|
|
795
|
+
pattern, ignore_case, msg = node.unpack_args()
|
|
796
|
+
expr = (
|
|
797
|
+
indent
|
|
798
|
+
+ f"assert any(re.search({pattern!r}, i) for i in {prv}), {msg!r}"
|
|
799
|
+
)
|
|
800
|
+
if is_last_var_no_ret(node):
|
|
801
|
+
return expr
|
|
802
|
+
# HACK: avoid recalc variables
|
|
803
|
+
return expr + "\n" + indent + f"{nxt} = {prv}"
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def pre_list_str_all_is_regex(node: ExprListStringAllRegex) -> str:
|
|
807
|
+
indent = (
|
|
808
|
+
INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
|
|
809
|
+
)
|
|
810
|
+
prv, nxt = prev_next_var(node)
|
|
811
|
+
pattern, ignore_case, msg = node.unpack_args()
|
|
812
|
+
expr = (
|
|
813
|
+
indent
|
|
814
|
+
+ f"assert all(re.search({pattern!r}, i) for i in {prv}), {msg!r}"
|
|
815
|
+
)
|
|
816
|
+
if is_last_var_no_ret(node):
|
|
817
|
+
return expr
|
|
818
|
+
# HACK: avoid recalc variables
|
|
819
|
+
return expr + "\n" + indent + f"{nxt} = {prv}"
|
|
820
|
+
|
|
821
|
+
|
|
786
822
|
def pre_to_int(node: ExprToInt) -> str:
|
|
787
823
|
indent = (
|
|
788
824
|
INDENT_DEFAULT_BODY if have_default_expr(node) else INDENT_METHOD_BODY
|
|
@@ -409,3 +409,42 @@ for _, {{ tmp_var }} := range {{ prv }} {
|
|
|
409
409
|
{{nxt}} = append({{ nxt }}, {{ tmp_var }});
|
|
410
410
|
}
|
|
411
411
|
""")
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
J2_PRE_LIST_STR_ANY_IS_RE = Template("""
|
|
415
|
+
re{{ nxt }} := regexp.MustCompile({{ pattern }});
|
|
416
|
+
for _, {{ tmp_var }} := range {{ prv }} {
|
|
417
|
+
if re{{ nxt }}.MatchString({{ tmp_var }}) {
|
|
418
|
+
break;
|
|
419
|
+
}
|
|
420
|
+
{% if have_default_expr %}
|
|
421
|
+
panic(fmt.Errorf({{ msg }}));
|
|
422
|
+
{% elif is_pre_validate_parent %}
|
|
423
|
+
return nil, fmt.Errorf({{ msg }});
|
|
424
|
+
{% else %}
|
|
425
|
+
return {{ return_type }}, fmt.Errorf({{ msg }});
|
|
426
|
+
{% endif %}
|
|
427
|
+
}
|
|
428
|
+
{% if not is_last_var_no_ret %}
|
|
429
|
+
{{ nxt }} := {{ prv }};
|
|
430
|
+
{% endif %}
|
|
431
|
+
""")
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
J2_PRE_LIST_STR_ALL_IS_RE = Template("""
|
|
435
|
+
re{{ nxt }} := regexp.MustCompile({{ pattern }});
|
|
436
|
+
for _, {{ tmp_var }} := range {{ prv }} {
|
|
437
|
+
if !re{{ nxt }}.MatchString({{ tmp_var }}) {
|
|
438
|
+
{% if have_default_expr %}
|
|
439
|
+
panic(fmt.Errorf({{ msg }}));
|
|
440
|
+
{% elif is_pre_validate_parent %}
|
|
441
|
+
return nil, fmt.Errorf({{ msg }});
|
|
442
|
+
{% else %}
|
|
443
|
+
return {{ return_type }}, fmt.Errorf({{ msg }});
|
|
444
|
+
{% endif %}
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
{% if not is_last_var_no_ret %}
|
|
448
|
+
{{ nxt }} := {{ prv }};
|
|
449
|
+
{% endif %}
|
|
450
|
+
""")
|
|
@@ -41,7 +41,7 @@ from ssc_codegen.ast_ import (
|
|
|
41
41
|
ExprIsEqual,
|
|
42
42
|
ExprIsNotEqual,
|
|
43
43
|
ExprIsContains,
|
|
44
|
-
|
|
44
|
+
ExprStringIsRegex,
|
|
45
45
|
ExprNested,
|
|
46
46
|
ExprToInt,
|
|
47
47
|
ExprToListInt,
|
|
@@ -55,6 +55,8 @@ from ssc_codegen.ast_ import (
|
|
|
55
55
|
ExprListStringRmSuffix,
|
|
56
56
|
ExprStringRmPrefixAndSuffix,
|
|
57
57
|
ExprListStringRmPrefixAndSuffix,
|
|
58
|
+
ExprListStringAllRegex,
|
|
59
|
+
ExprListStringAnyRegex,
|
|
58
60
|
)
|
|
59
61
|
from ssc_codegen.document_utlis import (
|
|
60
62
|
analyze_re_expression,
|
|
@@ -953,6 +955,78 @@ class AssertDocument(BaseDocument):
|
|
|
953
955
|
)
|
|
954
956
|
return self
|
|
955
957
|
|
|
958
|
+
def any_is_re(
|
|
959
|
+
self, pattern: str | Pattern, msg: str = "", ignore_case: bool = False
|
|
960
|
+
) -> Self:
|
|
961
|
+
"""assert any value matched in array of strings by regex.
|
|
962
|
+
If in generated code check failed - throw exception with passed msg
|
|
963
|
+
|
|
964
|
+
EXPR DO NOT MODIFY variable
|
|
965
|
+
|
|
966
|
+
- accept LIST_STRING, return LIST_STRING
|
|
967
|
+
"""
|
|
968
|
+
if not isinstance(pattern, str):
|
|
969
|
+
ignore_case = is_ignore_case_regex(pattern)
|
|
970
|
+
pattern = unverbosify_regex(pattern)
|
|
971
|
+
analyze_re_expression(pattern, allow_empty_groups=True)
|
|
972
|
+
if self.stack_last_ret != VariableType.LIST_STRING:
|
|
973
|
+
LOGGER.warning(
|
|
974
|
+
"any_is_re(%s): Expected type(s) %s got %s",
|
|
975
|
+
repr(pattern),
|
|
976
|
+
VariableType.LIST_STRING.name,
|
|
977
|
+
self.stack_last_ret.name,
|
|
978
|
+
)
|
|
979
|
+
|
|
980
|
+
self._add(
|
|
981
|
+
ExprListStringAnyRegex(
|
|
982
|
+
kwargs={
|
|
983
|
+
"pattern": pattern,
|
|
984
|
+
"ignore_case": ignore_case,
|
|
985
|
+
"msg": msg,
|
|
986
|
+
}
|
|
987
|
+
)
|
|
988
|
+
)
|
|
989
|
+
return self
|
|
990
|
+
|
|
991
|
+
def all_is_re(
|
|
992
|
+
self, pattern: str | Pattern, msg: str = "", ignore_case: bool = False
|
|
993
|
+
) -> Self:
|
|
994
|
+
"""assert all value matched in array of strings by regex.
|
|
995
|
+
If in generated code check failed - throw exception with passed msg
|
|
996
|
+
|
|
997
|
+
EXPR DO NOT MODIFY variable
|
|
998
|
+
|
|
999
|
+
- accept LIST_STRING, return LIST_STRING
|
|
1000
|
+
"""
|
|
1001
|
+
if not isinstance(pattern, str):
|
|
1002
|
+
ignore_case = is_ignore_case_regex(pattern)
|
|
1003
|
+
pattern = unverbosify_regex(pattern)
|
|
1004
|
+
analyze_re_expression(pattern, allow_empty_groups=True)
|
|
1005
|
+
if self.stack_last_ret != VariableType.LIST_STRING:
|
|
1006
|
+
LOGGER.warning(
|
|
1007
|
+
"all_is_re(%s): Expected type(s) %s got %s",
|
|
1008
|
+
repr(pattern),
|
|
1009
|
+
VariableType.LIST_STRING.name,
|
|
1010
|
+
self.stack_last_ret.name,
|
|
1011
|
+
)
|
|
1012
|
+
|
|
1013
|
+
self._add(
|
|
1014
|
+
ExprListStringAllRegex(
|
|
1015
|
+
kwargs={
|
|
1016
|
+
"pattern": pattern,
|
|
1017
|
+
"ignore_case": ignore_case,
|
|
1018
|
+
"msg": msg,
|
|
1019
|
+
}
|
|
1020
|
+
)
|
|
1021
|
+
)
|
|
1022
|
+
return self
|
|
1023
|
+
|
|
1024
|
+
def is_re(
|
|
1025
|
+
self, pattern: str | Pattern, msg: str = "", ignore_case: bool = False
|
|
1026
|
+
) -> Self:
|
|
1027
|
+
"""shortcut of is_regex() method"""
|
|
1028
|
+
return self.is_regex(pattern, msg, ignore_case)
|
|
1029
|
+
|
|
956
1030
|
def is_regex(
|
|
957
1031
|
self, pattern: str | Pattern, msg: str = "", ignore_case: bool = False
|
|
958
1032
|
) -> Self:
|
|
@@ -976,7 +1050,7 @@ class AssertDocument(BaseDocument):
|
|
|
976
1050
|
)
|
|
977
1051
|
|
|
978
1052
|
self._add(
|
|
979
|
-
|
|
1053
|
+
ExprStringIsRegex(
|
|
980
1054
|
kwargs={
|
|
981
1055
|
"pattern": pattern,
|
|
982
1056
|
"ignore_case": ignore_case,
|
|
@@ -5,6 +5,12 @@ from cssselect import HTMLTranslator, SelectorSyntaxError
|
|
|
5
5
|
from lxml import etree
|
|
6
6
|
from lxml.etree import XPathSyntaxError
|
|
7
7
|
|
|
8
|
+
# cssselect not support CSS4 standard syntax
|
|
9
|
+
# used for particularly provide CSS 4 API
|
|
10
|
+
# https://facelessuser.github.io/soupsieve/selectors/
|
|
11
|
+
from soupsieve import compile as css_compile
|
|
12
|
+
from soupsieve.util import SelectorSyntaxError as SoupSieveSelectorSyntaxError
|
|
13
|
+
|
|
8
14
|
|
|
9
15
|
def css_to_xpath(query: str, prefix: str = "descendant-or-self::") -> str:
|
|
10
16
|
"""convert css to XPATH selector"""
|
|
@@ -24,6 +30,10 @@ def xpath_to_css(query: str) -> str:
|
|
|
24
30
|
EG FAIL:
|
|
25
31
|
//p[contains(@class, "star-rating")]
|
|
26
32
|
|
|
33
|
+
|
|
34
|
+
NOTE:
|
|
35
|
+
cssselect not fully support CSS4 standard
|
|
36
|
+
|
|
27
37
|
"""
|
|
28
38
|
# https://stackoverflow.com/a/18421383
|
|
29
39
|
css = re.sub(
|
|
@@ -41,8 +51,8 @@ def xpath_to_css(query: str) -> str:
|
|
|
41
51
|
|
|
42
52
|
def validate_css_query(query: str) -> None:
|
|
43
53
|
try:
|
|
44
|
-
|
|
45
|
-
except
|
|
54
|
+
css_compile(query.strip('"'))
|
|
55
|
+
except SoupSieveSelectorSyntaxError:
|
|
46
56
|
# maybe is XPATH?
|
|
47
57
|
with suppress(XPathSyntaxError):
|
|
48
58
|
etree.XPath(query) # type: ignore
|
|
@@ -57,10 +67,9 @@ def validate_xpath_query(query: str) -> None:
|
|
|
57
67
|
etree.XPath(query.strip('"'))
|
|
58
68
|
# etree.XPath accept CSS-like queries without throw exception, check it!
|
|
59
69
|
is_css = False
|
|
60
|
-
with suppress(
|
|
61
|
-
|
|
70
|
+
with suppress(SoupSieveSelectorSyntaxError):
|
|
71
|
+
css_compile(query.strip('"'))
|
|
62
72
|
is_css = True
|
|
63
|
-
|
|
64
73
|
if is_css:
|
|
65
74
|
msg = f"`{query}` looks like CSS query, not XPATH"
|
|
66
75
|
raise SelectorSyntaxError(msg)
|
|
@@ -62,7 +62,9 @@ FMT_MAPPING_METHODS = {
|
|
|
62
62
|
TokenType.IS_CONTAINS: "is_contains",
|
|
63
63
|
TokenType.IS_CSS: "is_css",
|
|
64
64
|
TokenType.IS_XPATH: "is_xpath",
|
|
65
|
-
TokenType.
|
|
65
|
+
TokenType.IS_STRING_REGEX_MATCH: "is_regex",
|
|
66
|
+
TokenType.ANY_LIST_STRING_REGEX_MATCH: "any_is_re",
|
|
67
|
+
TokenType.ALL_LIST_STRING_REGEX_MATCH: "all_is_re",
|
|
66
68
|
TokenType.TO_INT: "to_int",
|
|
67
69
|
TokenType.TO_INT_LIST: "to_int",
|
|
68
70
|
TokenType.TO_FLOAT: "to_float",
|
|
@@ -329,7 +329,7 @@ def analyze_regex_expr(
|
|
|
329
329
|
case (
|
|
330
330
|
TokenType.EXPR_REGEX_SUB
|
|
331
331
|
| TokenType.EXPR_LIST_REGEX_SUB
|
|
332
|
-
| TokenType.
|
|
332
|
+
| TokenType.IS_STRING_REGEX_MATCH
|
|
333
333
|
):
|
|
334
334
|
result = analyze_re_expression(
|
|
335
335
|
re_expr.kwargs["pattern"], allow_empty_groups=True
|
|
@@ -153,7 +153,9 @@ class TokenType(IntEnum):
|
|
|
153
153
|
IS_CONTAINS = auto()
|
|
154
154
|
IS_CSS = auto()
|
|
155
155
|
IS_XPATH = auto()
|
|
156
|
-
|
|
156
|
+
IS_STRING_REGEX_MATCH = auto()
|
|
157
|
+
ANY_LIST_STRING_REGEX_MATCH = auto()
|
|
158
|
+
ALL_LIST_STRING_REGEX_MATCH = auto()
|
|
157
159
|
|
|
158
160
|
# NUMERIC
|
|
159
161
|
TO_INT = auto()
|
|
@@ -186,7 +188,7 @@ class TokenType(IntEnum):
|
|
|
186
188
|
TokenType.IS_EQUAL,
|
|
187
189
|
TokenType.IS_NOT_EQUAL,
|
|
188
190
|
TokenType.IS_CONTAINS,
|
|
189
|
-
TokenType.
|
|
191
|
+
TokenType.IS_STRING_REGEX_MATCH,
|
|
190
192
|
)
|
|
191
193
|
|
|
192
194
|
@classmethod
|
|
@@ -196,5 +198,5 @@ class TokenType(IntEnum):
|
|
|
196
198
|
TokenType.EXPR_REGEX_ALL,
|
|
197
199
|
TokenType.EXPR_REGEX_SUB,
|
|
198
200
|
TokenType.EXPR_LIST_REGEX_SUB,
|
|
199
|
-
TokenType.
|
|
201
|
+
TokenType.IS_STRING_REGEX_MATCH,
|
|
200
202
|
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|