pyanyapi 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyanyapi/__init__.py +3 -0
- pyanyapi/decorators.py +17 -0
- pyanyapi/exceptions.py +9 -0
- pyanyapi/helpers.py +31 -0
- pyanyapi/interfaces.py +395 -0
- pyanyapi/parsers.py +185 -0
- pyanyapi/py.typed +0 -0
- pyanyapi-1.0.0.dist-info/METADATA +110 -0
- pyanyapi-1.0.0.dist-info/RECORD +11 -0
- pyanyapi-1.0.0.dist-info/WHEEL +4 -0
- pyanyapi-1.0.0.dist-info/licenses/LICENSE +21 -0
pyanyapi/__init__.py
ADDED
pyanyapi/decorators.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from typing import ParamSpec, TypeVar
|
|
3
|
+
|
|
4
|
+
P = ParamSpec("P")
|
|
5
|
+
R = TypeVar("R")
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def interface_property(method: Callable[P, R]) -> "staticmethod[P, R]":
|
|
9
|
+
"""Marks method to be included in parsing result as property."""
|
|
10
|
+
method._interface_property = True # type: ignore[attr-defined]
|
|
11
|
+
return staticmethod(method)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def interface_method(method: Callable[P, R]) -> "staticmethod[P, R]":
|
|
15
|
+
"""Marks method to be included in parsing result."""
|
|
16
|
+
method._interface_method = True # type: ignore[attr-defined]
|
|
17
|
+
return staticmethod(method)
|
pyanyapi/exceptions.py
ADDED
pyanyapi/helpers.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Functions to dynamically attach attributes to classes.
|
|
2
|
+
|
|
3
|
+
Most of parsing results are cached because of immutability of input data.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
from typing import Any, Generic, TypeVar
|
|
8
|
+
|
|
9
|
+
T = TypeVar("T")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class cached_property(Generic[T]):
|
|
13
|
+
"""Copied from Django."""
|
|
14
|
+
|
|
15
|
+
def __init__(self, func: Callable[[Any], T]) -> None:
|
|
16
|
+
self.func = func
|
|
17
|
+
|
|
18
|
+
def __get__(self, instance: Any, owner: type | None = None) -> T:
|
|
19
|
+
res = instance.__dict__[self.func.__name__] = self.func(instance)
|
|
20
|
+
return res
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def attach_attribute(target: type, name: str, attr: Any) -> None:
|
|
24
|
+
attr.__name__ = name
|
|
25
|
+
attr._attached = True
|
|
26
|
+
setattr(target, name, attr)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def attach_cached_property(target: type, name: str, prop: Callable[[Any], Any]) -> None:
|
|
30
|
+
method = cached_property(prop)
|
|
31
|
+
attach_attribute(target, name, method)
|
pyanyapi/interfaces.py
ADDED
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
"""Classes to be filled with interface declarations."""
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
import json
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Callable, Sequence
|
|
7
|
+
from functools import cache
|
|
8
|
+
from typing import TYPE_CHECKING, Any
|
|
9
|
+
|
|
10
|
+
import yaml
|
|
11
|
+
from lxml import etree, objectify
|
|
12
|
+
from lxml.etree import HTMLParser, XMLParser
|
|
13
|
+
|
|
14
|
+
from .exceptions import ResponseParseError
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from .parsers import BaseParser
|
|
18
|
+
|
|
19
|
+
DICT_LOOKUP = " > "
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def expand_results(value: Any) -> Any:
|
|
23
|
+
if isinstance(value, list):
|
|
24
|
+
return [item.parse_all() for item in value]
|
|
25
|
+
return value
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class BaseInterface:
|
|
29
|
+
"""Basic dynamically generated interface."""
|
|
30
|
+
|
|
31
|
+
content: Any = None
|
|
32
|
+
empty_result: Any = None
|
|
33
|
+
_parsed_content: Any
|
|
34
|
+
|
|
35
|
+
def __init__(self, content: Any, strip: bool = False) -> None:
|
|
36
|
+
self.content = content
|
|
37
|
+
self.strip = strip
|
|
38
|
+
self.parse = cache(self.parse) # type: ignore[method-assign]
|
|
39
|
+
|
|
40
|
+
if TYPE_CHECKING:
|
|
41
|
+
# Attributes are generated from parser settings at runtime
|
|
42
|
+
def __getattr__(self, name: str) -> Any: ...
|
|
43
|
+
|
|
44
|
+
@classmethod
|
|
45
|
+
def init_attr(cls, settings: Any) -> Callable[["BaseInterface"], Any]:
|
|
46
|
+
|
|
47
|
+
def inner(self: BaseInterface) -> Any:
|
|
48
|
+
return cls.execute_method(self, settings)
|
|
49
|
+
|
|
50
|
+
return inner
|
|
51
|
+
|
|
52
|
+
def execute_method(self, settings: Any) -> Any:
|
|
53
|
+
raise NotImplementedError
|
|
54
|
+
|
|
55
|
+
@property
|
|
56
|
+
def parsed_content(self) -> Any:
|
|
57
|
+
if not hasattr(self, "_parsed_content"):
|
|
58
|
+
self._parsed_content = self.perform_parsing()
|
|
59
|
+
return self._parsed_content
|
|
60
|
+
|
|
61
|
+
def perform_parsing(self) -> Any:
|
|
62
|
+
raise NotImplementedError
|
|
63
|
+
|
|
64
|
+
def parse(self, query: Any) -> Any:
|
|
65
|
+
raise NotImplementedError
|
|
66
|
+
|
|
67
|
+
def parse_all(self) -> dict[str, Any]:
|
|
68
|
+
"""Processes all available properties and returns results as dictionary."""
|
|
69
|
+
return {
|
|
70
|
+
key: expand_results(getattr(self, key, self.empty_result))
|
|
71
|
+
for key, attr in self.__class__.__dict__.items()
|
|
72
|
+
if hasattr(attr, "_attached") and type(attr).__name__ == "cached_property"
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
def maybe_strip(self, value: Any) -> Any:
|
|
76
|
+
if self.strip and isinstance(value, str):
|
|
77
|
+
return value.strip()
|
|
78
|
+
return value
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# Uses as fallback. None - can be obtained from JSON's null, any string also can be, so unique object is a best choice
|
|
82
|
+
EMPTY_RESULT = object()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class CombinedInterface(BaseInterface):
|
|
86
|
+
def __init__(self, parsers: Sequence["BaseParser"], content: Any, strip: bool = False) -> None:
|
|
87
|
+
self.parsers = parsers
|
|
88
|
+
super().__init__(content, strip)
|
|
89
|
+
|
|
90
|
+
def __getattribute__(self, item: str) -> Any:
|
|
91
|
+
# Catch self.parsers and dynamically attached attributes
|
|
92
|
+
try:
|
|
93
|
+
return super().__getattribute__(item)
|
|
94
|
+
except AttributeError:
|
|
95
|
+
return self.walk(item)
|
|
96
|
+
|
|
97
|
+
def walk(self, item: str) -> Any:
|
|
98
|
+
"""Recursively walks through all available parsers."""
|
|
99
|
+
for parser in self.parsers:
|
|
100
|
+
try:
|
|
101
|
+
if item not in parser.attributes:
|
|
102
|
+
continue
|
|
103
|
+
result = getattr(parser.parse(self.content), item, EMPTY_RESULT)
|
|
104
|
+
# Ignore empty results in current parser
|
|
105
|
+
if result in (EMPTY_RESULT, parser.interface_class.empty_result):
|
|
106
|
+
continue
|
|
107
|
+
return result
|
|
108
|
+
except (AttributeError, ResponseParseError):
|
|
109
|
+
pass
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
def parse_all(self) -> dict[str, Any]:
|
|
113
|
+
result = super().parse_all()
|
|
114
|
+
for parser in self.parsers:
|
|
115
|
+
result.update(parser.parse_all(self.content))
|
|
116
|
+
return result
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class XPathInterface(BaseInterface):
|
|
120
|
+
"""Uses as base class for HTML/XML-based content.
|
|
121
|
+
|
|
122
|
+
Use XPath 1.0 syntax, which is compatible with LXML.
|
|
123
|
+
Because of lack of support of XPath 2.0 some parts of settings structure
|
|
124
|
+
is not intuitive.
|
|
125
|
+
Settings example:
|
|
126
|
+
|
|
127
|
+
{
|
|
128
|
+
'errors': {
|
|
129
|
+
'base': "//ul[@class='alerts']/div",
|
|
130
|
+
'children': 'text()|*//text()'
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
'children' key usually uses for modification of result of 'base' expression
|
|
135
|
+
before concatenation.
|
|
136
|
+
"""
|
|
137
|
+
|
|
138
|
+
parser_class: type[HTMLParser] | type[XMLParser] = HTMLParser
|
|
139
|
+
empty_result = ""
|
|
140
|
+
_error_message = "HTML data can not be parsed."
|
|
141
|
+
|
|
142
|
+
def perform_parsing(self) -> Any:
|
|
143
|
+
try:
|
|
144
|
+
return etree.fromstring(self.content, self.parser_class())
|
|
145
|
+
except etree.XMLSyntaxError as exc:
|
|
146
|
+
raise ResponseParseError(self._error_message, self.content) from exc
|
|
147
|
+
|
|
148
|
+
def execute_method(self, settings: Any) -> Any:
|
|
149
|
+
if isinstance(settings, dict):
|
|
150
|
+
result = self.parse(settings["base"])
|
|
151
|
+
child_query = settings.get("children")
|
|
152
|
+
if child_query:
|
|
153
|
+
return [self.maybe_strip("".join(element.xpath(child_query))) for element in result]
|
|
154
|
+
sub_parser = settings.get("parser")
|
|
155
|
+
if sub_parser:
|
|
156
|
+
return [
|
|
157
|
+
(sub_parser() if callable(sub_parser) else sub_parser).parse(etree.tostring(element))
|
|
158
|
+
for element in result
|
|
159
|
+
]
|
|
160
|
+
return result
|
|
161
|
+
|
|
162
|
+
return self.parse(settings)
|
|
163
|
+
|
|
164
|
+
def parse(self, query: Any) -> Any:
|
|
165
|
+
return self.maybe_strip(self.parsed_content.xpath(query))
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
class XMLInterface(XPathInterface):
|
|
169
|
+
parser_class = XMLParser
|
|
170
|
+
_error_message = "XML data can not be parsed."
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
class XMLObjectifyInterface(BaseInterface):
|
|
174
|
+
"""Parse XML in the way, that its attributes can be accessed like attributes of python object.
|
|
175
|
+
|
|
176
|
+
<xml><test>123</test></xml>
|
|
177
|
+
|
|
178
|
+
From it you can get:
|
|
179
|
+
>> obj.test
|
|
180
|
+
123
|
|
181
|
+
>> obj.not_test
|
|
182
|
+
None
|
|
183
|
+
|
|
184
|
+
Also this interface does not require any settings.
|
|
185
|
+
"""
|
|
186
|
+
|
|
187
|
+
_error_message = "XML data can not be parsed."
|
|
188
|
+
|
|
189
|
+
def perform_parsing(self) -> Any:
|
|
190
|
+
try:
|
|
191
|
+
return objectify.fromstring(self.content)
|
|
192
|
+
except etree.XMLSyntaxError as exc:
|
|
193
|
+
raise ResponseParseError(self._error_message, self.content) from exc
|
|
194
|
+
|
|
195
|
+
def __getattribute__(self, item: str) -> Any:
|
|
196
|
+
try:
|
|
197
|
+
return super().__getattribute__(item)
|
|
198
|
+
except AttributeError:
|
|
199
|
+
if item == "_parsed_content":
|
|
200
|
+
raise
|
|
201
|
+
try:
|
|
202
|
+
return self.maybe_strip(self.parsed_content.__getattribute__(item))
|
|
203
|
+
except AttributeError:
|
|
204
|
+
return None
|
|
205
|
+
|
|
206
|
+
def maybe_strip(self, value: Any) -> Any:
|
|
207
|
+
if self.strip and isinstance(value, objectify.ObjectifiedElement):
|
|
208
|
+
if isinstance(value, objectify.StringElement):
|
|
209
|
+
return value.text.strip() if value.text is not None else value
|
|
210
|
+
# Not `value.__dict__`: on PyPy it is not lxml's children mapping, so nothing was stripped
|
|
211
|
+
for child in value.iterdescendants(tag=etree.Element):
|
|
212
|
+
if isinstance(child, objectify.StringElement) and child.text is not None:
|
|
213
|
+
child._setText(child.text.strip())
|
|
214
|
+
return value
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class DictInterface(BaseInterface):
|
|
218
|
+
"""Interface for python dictionaries. Based on PostgreSQL statements syntax.
|
|
219
|
+
|
|
220
|
+
{
|
|
221
|
+
'external_id': 'container > id'
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
which will get "123" from {"container":{"id":"123"}}
|
|
225
|
+
"""
|
|
226
|
+
|
|
227
|
+
def get_from_dict(self, target: Any, query: str) -> Any:
|
|
228
|
+
if not target:
|
|
229
|
+
return target
|
|
230
|
+
action_list = query.split(DICT_LOOKUP)
|
|
231
|
+
for action in action_list:
|
|
232
|
+
if target:
|
|
233
|
+
action = action.strip()
|
|
234
|
+
if isinstance(target, dict):
|
|
235
|
+
target = target.get(action, self.empty_result)
|
|
236
|
+
else:
|
|
237
|
+
try:
|
|
238
|
+
target = target[int(action)]
|
|
239
|
+
except (IndexError, TypeError, ValueError):
|
|
240
|
+
return self.empty_result
|
|
241
|
+
else:
|
|
242
|
+
return target
|
|
243
|
+
return self.maybe_strip(target)
|
|
244
|
+
|
|
245
|
+
def execute_method(self, settings: Any) -> Any:
|
|
246
|
+
if isinstance(settings, dict):
|
|
247
|
+
result = self.parse(settings["base"])
|
|
248
|
+
child_query = settings.get("children")
|
|
249
|
+
if child_query:
|
|
250
|
+
return (
|
|
251
|
+
[self.get_from_dict(r, child_query) or self.empty_result for r in result]
|
|
252
|
+
if result
|
|
253
|
+
else self.empty_result
|
|
254
|
+
)
|
|
255
|
+
return result
|
|
256
|
+
|
|
257
|
+
return self.parse(settings)
|
|
258
|
+
|
|
259
|
+
def parse(self, query: Any) -> Any:
|
|
260
|
+
return self.get_from_dict(self.parsed_content, query)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
class JSONInterface(DictInterface):
|
|
264
|
+
_error_message = "JSON data can not be parsed."
|
|
265
|
+
|
|
266
|
+
def perform_parsing(self) -> Any:
|
|
267
|
+
try:
|
|
268
|
+
return json.loads(self.content)
|
|
269
|
+
except (ValueError, TypeError) as exc:
|
|
270
|
+
raise ResponseParseError(self._error_message, self.content) from exc
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
class YAMLInterface(DictInterface):
|
|
274
|
+
_error_message = "YAML data can not be parsed."
|
|
275
|
+
|
|
276
|
+
def perform_parsing(self) -> Any:
|
|
277
|
+
try:
|
|
278
|
+
return yaml.safe_load(self.content)
|
|
279
|
+
except yaml.error.YAMLError as exc:
|
|
280
|
+
raise ResponseParseError(self._error_message, self.content) from exc
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
class AJAXInterface(JSONInterface):
|
|
284
|
+
"""Allows to execute XPath, combined with dictionary-based lookups from DictInterface.
|
|
285
|
+
|
|
286
|
+
{
|
|
287
|
+
'p': 'container > string(//p)'
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
which will get "p_content" from {"container":"<p>p_content</p>"}
|
|
291
|
+
"""
|
|
292
|
+
|
|
293
|
+
inner_interface_class = XPathInterface
|
|
294
|
+
|
|
295
|
+
def __init__(self, content: Any, strip: bool = False) -> None:
|
|
296
|
+
self._inner_cache: dict[str, XPathInterface] = {}
|
|
297
|
+
super().__init__(content, strip)
|
|
298
|
+
|
|
299
|
+
def get_inner_interface(self, text: Any, json_part: str) -> "XPathInterface":
|
|
300
|
+
if json_part not in self._inner_cache:
|
|
301
|
+
inner_content = super().get_from_dict(text, json_part)
|
|
302
|
+
self._inner_cache[json_part] = self.inner_interface_class(inner_content, self.strip)
|
|
303
|
+
return self._inner_cache[json_part]
|
|
304
|
+
|
|
305
|
+
def get_from_dict(self, target: Any, query: str) -> Any:
|
|
306
|
+
json_part, xpath_part = query.rsplit(DICT_LOOKUP, 1)
|
|
307
|
+
inner_interface = self.get_inner_interface(target, json_part)
|
|
308
|
+
try:
|
|
309
|
+
return inner_interface.parse(xpath_part)
|
|
310
|
+
except (etree.XMLSyntaxError, ValueError, TypeError):
|
|
311
|
+
return inner_interface.empty_result
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
class RegExpInterface(BaseInterface):
|
|
315
|
+
r"""Parser based on regular expressions.
|
|
316
|
+
|
|
317
|
+
It is the most powerful parser, because of its simplicity.
|
|
318
|
+
Settings example:
|
|
319
|
+
|
|
320
|
+
{
|
|
321
|
+
"result": "^ok$",
|
|
322
|
+
"errors": "^Error \d+$",
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
So, response will be like 'ok' or 'Error 100'.
|
|
326
|
+
"""
|
|
327
|
+
|
|
328
|
+
def __init__(self, content: Any, strip: bool = False, flags: int = 0) -> None:
|
|
329
|
+
self.flags = flags
|
|
330
|
+
super().__init__(content, strip)
|
|
331
|
+
|
|
332
|
+
def execute_method(self, settings: Any) -> Any:
|
|
333
|
+
matches = re.findall(settings, self.content, self.flags)
|
|
334
|
+
if matches:
|
|
335
|
+
return self.maybe_strip(matches[0])
|
|
336
|
+
return self.empty_result
|
|
337
|
+
|
|
338
|
+
def parse(self, query: Any) -> Any:
|
|
339
|
+
return self.execute_method(query)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
class CSVInterface(BaseInterface):
|
|
343
|
+
r"""Operates with CSV data with simple queries in format 'row_id:column_id'.
|
|
344
|
+
|
|
345
|
+
{
|
|
346
|
+
"value": "1:2"
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
Will get 6 from "1,2,3\r\n4,5,6"
|
|
350
|
+
"""
|
|
351
|
+
|
|
352
|
+
_error_message = "CSV data can not be parsed."
|
|
353
|
+
|
|
354
|
+
def __init__(self, content: Any, strip: bool = False, **reader_kwargs: Any) -> None:
|
|
355
|
+
self.reader_kwargs = reader_kwargs
|
|
356
|
+
super().__init__(content, strip)
|
|
357
|
+
|
|
358
|
+
def perform_parsing(self) -> Any:
|
|
359
|
+
try:
|
|
360
|
+
return list(csv.reader(self.content.split(), **self.reader_kwargs))
|
|
361
|
+
except (TypeError, AttributeError) as exc:
|
|
362
|
+
raise ResponseParseError(self._error_message, self.content) from exc
|
|
363
|
+
|
|
364
|
+
def execute_method(self, settings: Any) -> Any:
|
|
365
|
+
row, column = settings.split(":")
|
|
366
|
+
try:
|
|
367
|
+
return self.parsed_content[int(row)][int(column)]
|
|
368
|
+
except (IndexError, TypeError):
|
|
369
|
+
return self.empty_result
|
|
370
|
+
|
|
371
|
+
def parse(self, query: Any) -> Any:
|
|
372
|
+
return self.execute_method(query)
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
class IndexOfInterface(BaseInterface):
|
|
376
|
+
"""Simple interface that tries to find specified string inside another string, storing boolean values.
|
|
377
|
+
|
|
378
|
+
Settings example:
|
|
379
|
+
{
|
|
380
|
+
"has_bar": "bar",
|
|
381
|
+
"has_foo": "foo"
|
|
382
|
+
}
|
|
383
|
+
If content contains "bar" string, interface property "has_bar" will be True.
|
|
384
|
+
"""
|
|
385
|
+
|
|
386
|
+
_error_message = "Can not perform string search."
|
|
387
|
+
|
|
388
|
+
def execute_method(self, settings: Any) -> Any:
|
|
389
|
+
try:
|
|
390
|
+
return str(settings) in str(self.content)
|
|
391
|
+
except (TypeError, ValueError) as exc:
|
|
392
|
+
raise ResponseParseError(self._error_message, self.content) from exc
|
|
393
|
+
|
|
394
|
+
def parse(self, query: Any) -> Any:
|
|
395
|
+
return self.execute_method(query)
|
pyanyapi/parsers.py
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"""Classes for fabrics of interfaces.
|
|
2
|
+
|
|
3
|
+
Generates interfaces dynamically from given settings.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from .helpers import attach_attribute, attach_cached_property
|
|
10
|
+
from .interfaces import (
|
|
11
|
+
AJAXInterface,
|
|
12
|
+
BaseInterface,
|
|
13
|
+
CombinedInterface,
|
|
14
|
+
CSVInterface,
|
|
15
|
+
IndexOfInterface,
|
|
16
|
+
JSONInterface,
|
|
17
|
+
RegExpInterface,
|
|
18
|
+
XMLInterface,
|
|
19
|
+
XMLObjectifyInterface,
|
|
20
|
+
XPathInterface,
|
|
21
|
+
YAMLInterface,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class BaseParser:
|
|
26
|
+
"""Fabric for some API-like components, which supposes to provide interface to different types of content."""
|
|
27
|
+
|
|
28
|
+
interface_class: type[BaseInterface]
|
|
29
|
+
settings: dict[str, Any]
|
|
30
|
+
strip = False
|
|
31
|
+
|
|
32
|
+
def __init__(self, settings: dict[str, Any] | None = None, strip: bool | None = None) -> None:
|
|
33
|
+
if strip is not None:
|
|
34
|
+
self.strip = strip
|
|
35
|
+
parents_settings = self.get_parents_settings()
|
|
36
|
+
if settings:
|
|
37
|
+
parents_settings.update(settings)
|
|
38
|
+
self.settings = parents_settings
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def attributes(self) -> list[str]:
|
|
42
|
+
extra_attributes = []
|
|
43
|
+
for name in dir(self):
|
|
44
|
+
if name == "attributes":
|
|
45
|
+
continue
|
|
46
|
+
attr = getattr(self, name)
|
|
47
|
+
if hasattr(attr, "_interface_property") or hasattr(attr, "_interface_method"):
|
|
48
|
+
extra_attributes.append(name)
|
|
49
|
+
return list(self.settings.keys()) + extra_attributes
|
|
50
|
+
|
|
51
|
+
def get_parents_settings(self) -> dict[str, Any]:
|
|
52
|
+
"""Gather settings from parent classes. It provides some kind of settings inheritance."""
|
|
53
|
+
parents_settings: dict[str, Any] = {}
|
|
54
|
+
for klass in reversed(self.__class__.mro()):
|
|
55
|
+
parents_settings.update(getattr(klass, "settings", {}))
|
|
56
|
+
return parents_settings
|
|
57
|
+
|
|
58
|
+
def parse(self, content: Any = "") -> BaseInterface:
|
|
59
|
+
"""Generates new class instance with desired attributes."""
|
|
60
|
+
self.content = self.prepare_content(content)
|
|
61
|
+
|
|
62
|
+
Interface: type[BaseInterface] = type("Interface", (self.interface_class,), {})
|
|
63
|
+
self.setup_class(Interface)
|
|
64
|
+
|
|
65
|
+
init_kwargs = self.get_interface_kwargs()
|
|
66
|
+
|
|
67
|
+
return Interface(**init_kwargs)
|
|
68
|
+
|
|
69
|
+
def parse_all(self, content: Any = "") -> dict[str, Any]:
|
|
70
|
+
return self.parse(content).parse_all()
|
|
71
|
+
|
|
72
|
+
def get_interface_kwargs(self) -> dict[str, Any]:
|
|
73
|
+
return {"content": self.content, "strip": self.strip}
|
|
74
|
+
|
|
75
|
+
def prepare_content(self, content: Any) -> Any:
|
|
76
|
+
"""Hook to provide way to transform content."""
|
|
77
|
+
return content
|
|
78
|
+
|
|
79
|
+
def setup_class(self, cls: type[BaseInterface]) -> None:
|
|
80
|
+
"""Attaches dynamic properties & methods."""
|
|
81
|
+
self.process_settings(cls)
|
|
82
|
+
self.process_decorators(cls)
|
|
83
|
+
|
|
84
|
+
def process_settings(self, cls: type[BaseInterface]) -> None:
|
|
85
|
+
"""Generates methods, based on settings."""
|
|
86
|
+
for name, settings in self.settings.items():
|
|
87
|
+
attr = cls.init_attr(settings)
|
|
88
|
+
attach_cached_property(cls, name, attr)
|
|
89
|
+
|
|
90
|
+
def process_decorators(self, cls: type[BaseInterface]) -> None:
|
|
91
|
+
"""Re-attach all attributes decorated with `@interface_property` or `@interface_method` to the new class."""
|
|
92
|
+
for name in dir(self):
|
|
93
|
+
attr = getattr(self, name)
|
|
94
|
+
if getattr(attr, "_interface_property", False):
|
|
95
|
+
attach_cached_property(cls, name, attr)
|
|
96
|
+
elif getattr(attr, "_interface_method", False):
|
|
97
|
+
attach_attribute(cls, name, attr)
|
|
98
|
+
|
|
99
|
+
def __and__(self, other: "BaseParser") -> "CombinedParser":
|
|
100
|
+
return CombinedParser(self, other)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class CombinedParser(BaseParser):
|
|
104
|
+
"""Combines multiple parsers in one. This can also be in different types."""
|
|
105
|
+
|
|
106
|
+
interface_class = CombinedInterface
|
|
107
|
+
parsers: Sequence[BaseParser]
|
|
108
|
+
|
|
109
|
+
def __init__(self, *parsers: BaseParser, settings: dict[str, Any] | None = None, strip: bool | None = None) -> None:
|
|
110
|
+
if parsers:
|
|
111
|
+
self.parsers = parsers
|
|
112
|
+
super().__init__(settings, strip)
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def attributes(self) -> list[str]:
|
|
116
|
+
return super().attributes + [name for parser in self.parsers for name in parser.attributes]
|
|
117
|
+
|
|
118
|
+
def get_interface_kwargs(self) -> dict[str, Any]:
|
|
119
|
+
kwargs = super().get_interface_kwargs()
|
|
120
|
+
kwargs["parsers"] = self.parsers
|
|
121
|
+
return kwargs
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class HTMLParser(BaseParser):
|
|
125
|
+
interface_class = XPathInterface
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class XMLParser(BaseParser):
|
|
129
|
+
interface_class: type[BaseInterface] = XMLInterface
|
|
130
|
+
|
|
131
|
+
def prepare_content(self, content: Any) -> Any:
|
|
132
|
+
if isinstance(content, bytes):
|
|
133
|
+
return content.replace(b'encoding="UTF-8"', b"").replace(b'encoding="utf-8"', b"")
|
|
134
|
+
return content.replace('encoding="UTF-8"', "").replace('encoding="utf-8"', "")
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class XMLObjectifyParser(XMLParser):
|
|
138
|
+
interface_class = XMLObjectifyInterface
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class JSONParser(BaseParser):
|
|
142
|
+
interface_class = JSONInterface
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class YAMLParser(BaseParser):
|
|
146
|
+
interface_class = YAMLInterface
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class AJAXParser(BaseParser):
|
|
150
|
+
interface_class = AJAXInterface
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class RegExpParser(BaseParser):
|
|
154
|
+
interface_class = RegExpInterface
|
|
155
|
+
|
|
156
|
+
def __init__(self, settings: dict[str, Any] | None = None, strip: bool | None = None, flags: int = 0) -> None:
|
|
157
|
+
self.flags = flags
|
|
158
|
+
super().__init__(settings, strip)
|
|
159
|
+
|
|
160
|
+
def get_interface_kwargs(self) -> dict[str, Any]:
|
|
161
|
+
kwargs = super().get_interface_kwargs()
|
|
162
|
+
kwargs["flags"] = self.flags
|
|
163
|
+
return kwargs
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class CSVParser(BaseParser):
|
|
167
|
+
interface_class = CSVInterface
|
|
168
|
+
|
|
169
|
+
def __init__(self, settings: dict[str, Any] | None = None, strip: bool | None = None, **reader_kwargs: Any) -> None:
|
|
170
|
+
self.reader_kwargs = reader_kwargs
|
|
171
|
+
super().__init__(settings, strip)
|
|
172
|
+
|
|
173
|
+
def get_interface_kwargs(self) -> dict[str, Any]:
|
|
174
|
+
kwargs = super().get_interface_kwargs()
|
|
175
|
+
kwargs.update(self.reader_kwargs)
|
|
176
|
+
return kwargs
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class IndexOfParser(BaseParser):
|
|
180
|
+
interface_class = IndexOfInterface
|
|
181
|
+
|
|
182
|
+
def prepare_content(self, content: Any) -> Any:
|
|
183
|
+
if isinstance(content, bytes):
|
|
184
|
+
return content.decode()
|
|
185
|
+
return content
|
pyanyapi/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pyanyapi
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Tools for convenient interface creation over various types of data in a declarative way.
|
|
5
|
+
Project-URL: Documentation, https://pyanyapi.readthedocs.io/en/stable/
|
|
6
|
+
Project-URL: Changelog, https://github.com/Stranger6667/pyanyapi/blob/master/CHANGELOG.md
|
|
7
|
+
Project-URL: Bug Tracker, https://github.com/Stranger6667/pyanyapi/issues
|
|
8
|
+
Project-URL: Funding, https://github.com/sponsors/Stranger6667
|
|
9
|
+
Project-URL: Source Code, https://github.com/Stranger6667/pyanyapi
|
|
10
|
+
Author-email: Dmitry Dygalo <dmitry@dygalo.dev>
|
|
11
|
+
Maintainer-email: Dmitry Dygalo <dmitry@dygalo.dev>
|
|
12
|
+
License-Expression: MIT
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Keywords: API,XPath,csv,interface,json,parsing,regexp,xml,yaml
|
|
15
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
16
|
+
Classifier: Environment :: Console
|
|
17
|
+
Classifier: Intended Audience :: Developers
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
25
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
26
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
27
|
+
Classifier: Topic :: Text Processing :: General
|
|
28
|
+
Classifier: Topic :: Utilities
|
|
29
|
+
Classifier: Typing :: Typed
|
|
30
|
+
Requires-Python: >=3.11
|
|
31
|
+
Requires-Dist: lxml>=4.9.1
|
|
32
|
+
Requires-Dist: pyyaml>=6.0
|
|
33
|
+
Provides-Extra: docs
|
|
34
|
+
Requires-Dist: pymdown-extensions; extra == 'docs'
|
|
35
|
+
Requires-Dist: zensical<0.2,>=0.1.1; extra == 'docs'
|
|
36
|
+
Provides-Extra: tests
|
|
37
|
+
Requires-Dist: coverage[toml]>=7.2; extra == 'tests'
|
|
38
|
+
Requires-Dist: pytest>=8; extra == 'tests'
|
|
39
|
+
Description-Content-Type: text/markdown
|
|
40
|
+
|
|
41
|
+
# PyAnyAPI
|
|
42
|
+
|
|
43
|
+
[](https://github.com/Stranger6667/pyanyapi/actions/workflows/build.yml)
|
|
44
|
+
[](https://codecov.io/github/Stranger6667/pyanyapi?branch=master)
|
|
45
|
+
[](https://pypi.org/project/pyanyapi/)
|
|
46
|
+
[](https://pypi.org/project/pyanyapi/)
|
|
47
|
+
[](https://pyanyapi.readthedocs.io/en/stable/)
|
|
48
|
+
[](https://opensource.org/licenses/MIT)
|
|
49
|
+
|
|
50
|
+
Tools for convenient interface creation over various types of data in a declarative way.
|
|
51
|
+
|
|
52
|
+
Declare the attributes you need as queries, and PyAnyAPI turns the content into an object with those attributes:
|
|
53
|
+
XPath for HTML and XML, `container > key` lookups for JSON and YAML, `row:column` for CSV and regular expressions for text.
|
|
54
|
+
|
|
55
|
+
## Installation
|
|
56
|
+
|
|
57
|
+
```
|
|
58
|
+
uv pip install pyanyapi
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Usage
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
from pyanyapi.parsers import HTMLParser
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class SimpleParser(HTMLParser):
|
|
68
|
+
settings = {"header": "string(.//h1/text())"}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
api = SimpleParser().parse("<html><body><h1>Value</h1></body></html>")
|
|
72
|
+
assert api.header == "Value"
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
JSON lookups and all results at once:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from pyanyapi.parsers import JSONParser
|
|
79
|
+
|
|
80
|
+
api = JSONParser({"id": "container > id", "second": "items > 1"}).parse(
|
|
81
|
+
'{"container": {"id": "123"}, "items": ["first", "second"]}'
|
|
82
|
+
)
|
|
83
|
+
assert api.parse_all() == {"id": "123", "second": "second"}
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Combine parsers when the content type is unknown upfront, e.g. JSON responses with HTML error pages:
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
from pyanyapi.parsers import HTMLParser, JSONParser
|
|
90
|
+
|
|
91
|
+
parser = JSONParser({"status": "status"}) & HTMLParser({"error": "string(//h1)"})
|
|
92
|
+
assert parser.parse('{"status": "ok"}').status == "ok"
|
|
93
|
+
assert parser.parse("<html><body><h1>Server error</h1></body></html>").error == "Server error"
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Features
|
|
97
|
+
|
|
98
|
+
- Parsers for HTML, XML, XML Objectify, JSON, YAML, CSV, AJAX (JSON with HTML values), regular expressions and substring checks.
|
|
99
|
+
- Settings inheritance, sub-parsers and "base-children" queries for result lists.
|
|
100
|
+
- Custom properties and methods via `@interface_property` and `@interface_method`.
|
|
101
|
+
- Optional stripping of whitespace in results.
|
|
102
|
+
- Type annotations, with a `py.typed` marker.
|
|
103
|
+
|
|
104
|
+
## Documentation
|
|
105
|
+
|
|
106
|
+
<https://pyanyapi.readthedocs.io/en/stable/>
|
|
107
|
+
|
|
108
|
+
## Python support
|
|
109
|
+
|
|
110
|
+
PyAnyAPI supports CPython 3.11 to 3.15 and PyPy 3.11.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
pyanyapi/__init__.py,sha256=0f0107OmHWiMBB2lMyXAwe33DJIDLvcJbNTwsGnDHr0,134
|
|
2
|
+
pyanyapi/decorators.py,sha256=CNv1FdVsUqqqt1j2J76fLJ0ay7pJx4AhRX2OSG4yIrw,582
|
|
3
|
+
pyanyapi/exceptions.py,sha256=ujZGlqV3Yg92aYkuNM3swwmrAg7Je3vL8DqZRnqeW1U,263
|
|
4
|
+
pyanyapi/helpers.py,sha256=HWpDOrFXENdboDNk95ykUvbEk5RfSQK2s2X2r1Ek8cw,869
|
|
5
|
+
pyanyapi/interfaces.py,sha256=Oi71-UZYu42YkT70vn7NokzBhPEPdK0EQDk7WOMMrLg,12787
|
|
6
|
+
pyanyapi/parsers.py,sha256=m5edRuREwUqJHJzZi8cRQRr91pFvh3KGrrXSwaz5zZM,6118
|
|
7
|
+
pyanyapi/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
pyanyapi-1.0.0.dist-info/METADATA,sha256=tL1SQnV-BbBhIcjRCk5fbj5GHP_wp69EHyoeWrEqUDU,4434
|
|
9
|
+
pyanyapi-1.0.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
10
|
+
pyanyapi-1.0.0.dist-info/licenses/LICENSE,sha256=cA6gAmjqsht2w7-IaDuCPWtprQcAxjEAtiG8rbG5yiQ,1080
|
|
11
|
+
pyanyapi-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2015 Dmitry Dygalo
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|