escrape-plan 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- escrape_plan/__init__.py +472 -0
- escrape_plan/extractors/__init__.py +51 -0
- escrape_plan/extractors/base.py +48 -0
- escrape_plan/extractors/read_html.py +107 -0
- escrape_plan/extractors/sqlite_table.py +205 -0
- escrape_plan/outputters/__init__.py +33 -0
- escrape_plan/outputters/base.py +29 -0
- escrape_plan/outputters/text_file.py +92 -0
- escrape_plan/outputters/text_to_sqlite.py +133 -0
- escrape_plan-0.0.1.dist-info/METADATA +49 -0
- escrape_plan-0.0.1.dist-info/RECORD +12 -0
- escrape_plan-0.0.1.dist-info/WHEEL +4 -0
escrape_plan/__init__.py
ADDED
|
@@ -0,0 +1,472 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Module for scraping data out of HTML documents.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections import OrderedDict
|
|
7
|
+
from functools import cached_property
|
|
8
|
+
from types import NoneType
|
|
9
|
+
|
|
10
|
+
import bs4
|
|
11
|
+
import yaml
|
|
12
|
+
from yaml.loader import SafeLoader
|
|
13
|
+
|
|
14
|
+
KNOWN_OUTPUTS = ['text_file', 'text_to_sqlite']
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def load_plan(file: str):
|
|
18
|
+
"""Loads a plan up from disk."""
|
|
19
|
+
|
|
20
|
+
with open(file, 'r') as fh:
|
|
21
|
+
return yaml.load(fh, Loader=SafeLoader)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class SchemaValidator:
|
|
25
|
+
"""A class that lets us define a config schema and then run validation checks against it."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, config: dict):
|
|
28
|
+
self.conforming_config = config
|
|
29
|
+
|
|
30
|
+
@cached_property
|
|
31
|
+
def config_schema(self) -> dict:
|
|
32
|
+
"""Must be implemented in subclasses. Must return a dict indicating the schema of the
|
|
33
|
+
configuration options for this Extractor.
|
|
34
|
+
|
|
35
|
+
:return: The Extractor config schema definition.
|
|
36
|
+
:rtype: dict
|
|
37
|
+
"""
|
|
38
|
+
raise NotImplementedError()
|
|
39
|
+
|
|
40
|
+
@cached_property
|
|
41
|
+
def config_schema_documentation(self) -> str:
|
|
42
|
+
docstr = (
|
|
43
|
+
f'The {self.__class__.__name__} can be configured using the following parameters:\n\n'
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
for field, properties in self.config_schema.items():
|
|
47
|
+
docstr += f' {field}:\n'
|
|
48
|
+
docstr += f' {properties["description"]}\n'
|
|
49
|
+
docstr += f' - Data Type: {properties["datatype"].__name__}\n'
|
|
50
|
+
docstr += f' - Required: {properties["required"]!s}\n'
|
|
51
|
+
|
|
52
|
+
if 'regex' in properties:
|
|
53
|
+
docstr += f' - Must match pattern: {properties["regex"]}\n'
|
|
54
|
+
|
|
55
|
+
docstr += '\n'
|
|
56
|
+
|
|
57
|
+
return docstr
|
|
58
|
+
|
|
59
|
+
def validate_config_schema(self):
|
|
60
|
+
"""Verifies that the config given for this extractor meets the requirements of its schema.
|
|
61
|
+
|
|
62
|
+
:raises ValueError: Some aspect of the config does not align with the schema.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
errors = {}
|
|
66
|
+
|
|
67
|
+
def add_error(field, err):
|
|
68
|
+
if field not in errors:
|
|
69
|
+
errors[field] = []
|
|
70
|
+
errors[field].append(err)
|
|
71
|
+
|
|
72
|
+
for field_name, field_value in self.conforming_config.items():
|
|
73
|
+
field_schema = self.config_schema.get(field_name)
|
|
74
|
+
|
|
75
|
+
# Determine if the field is defined at all
|
|
76
|
+
if field_schema is None:
|
|
77
|
+
add_error(field_name, f'Unknown field {field_name}')
|
|
78
|
+
else:
|
|
79
|
+
# Determine if the field's value is the right type
|
|
80
|
+
value_type = type(field_value)
|
|
81
|
+
expected_type = field_schema.get('datatype')
|
|
82
|
+
if not isinstance(field_value, expected_type) and value_type is not None:
|
|
83
|
+
add_error(
|
|
84
|
+
field_name,
|
|
85
|
+
f'Value of field {field_name} must be of type {expected_type}, but got type '
|
|
86
|
+
f' {value_type} instead',
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
# Determine if it matches the regex if one is given
|
|
90
|
+
regex_str = field_schema.get('regex')
|
|
91
|
+
if regex_str is not None:
|
|
92
|
+
if type(regex_str) is not str:
|
|
93
|
+
add_error(
|
|
94
|
+
field_name,
|
|
95
|
+
f'"regex" value must be a string, instead got {type(regex_str)}',
|
|
96
|
+
)
|
|
97
|
+
else:
|
|
98
|
+
regex = re.compile(regex_str)
|
|
99
|
+
if type(field_value) is not str:
|
|
100
|
+
add_error(field_name, f'Value {field_value} must be a string')
|
|
101
|
+
elif not regex.match(field_value):
|
|
102
|
+
add_error(
|
|
103
|
+
field_name,
|
|
104
|
+
f'Value for field {field_name} does not match pattern {regex_str}',
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
# Determine if all required fields are present
|
|
108
|
+
required_fields = [
|
|
109
|
+
name for name, value in self.config_schema.items() if value.get('required') is True
|
|
110
|
+
]
|
|
111
|
+
missing_fields = []
|
|
112
|
+
for field in required_fields:
|
|
113
|
+
if field not in self.conforming_config:
|
|
114
|
+
missing_fields.append(field)
|
|
115
|
+
if len(missing_fields) > 0:
|
|
116
|
+
add_error(
|
|
117
|
+
'missing_fields', f'Extractor config missing required fields: {missing_fields}'
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
if len(errors) > 0:
|
|
121
|
+
raise ValueError(f'Extractor configuration failed validation. Errors: {errors}')
|
|
122
|
+
|
|
123
|
+
@staticmethod
|
|
124
|
+
def validate_document_section_reference(section_ref: dict) -> bool:
|
|
125
|
+
"""Validates a reference to a configuration object that references a document section."""
|
|
126
|
+
|
|
127
|
+
# Fields which *must* be present in a section reference.
|
|
128
|
+
required_fields = ['type', 'section_id', 'data']
|
|
129
|
+
|
|
130
|
+
# Make sure we're not missing any required fields
|
|
131
|
+
missing_fields = [field for field in required_fields if field not in section_ref]
|
|
132
|
+
if len(missing_fields) > 0:
|
|
133
|
+
raise ValueError(
|
|
134
|
+
f'Content config is missing the following required fields: {missing_fields}'
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
# Make sure the type is valid
|
|
138
|
+
content_type = section_ref.get('type')
|
|
139
|
+
if content_type != 'section':
|
|
140
|
+
raise ValueError(f'Content type is "{content_type}"; must be "section".')
|
|
141
|
+
|
|
142
|
+
# Validate section configs
|
|
143
|
+
section_ref_required_fields = ['section_id', 'data']
|
|
144
|
+
missing_section_ref_fields = [
|
|
145
|
+
field for field in section_ref_required_fields if field not in section_ref
|
|
146
|
+
]
|
|
147
|
+
if len(missing_section_ref_fields) > 0:
|
|
148
|
+
raise ValueError(
|
|
149
|
+
'Section config is missing the following required fields: '
|
|
150
|
+
f'{missing_section_ref_fields}.'
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
@staticmethod
|
|
154
|
+
def validate_sqlite_table_schema(schema_ref: dict) -> bool:
|
|
155
|
+
"""Validates a configuration object that defines a SQLite table schema."""
|
|
156
|
+
|
|
157
|
+
# Info about what we support
|
|
158
|
+
required_fields = ['datatype', 'field']
|
|
159
|
+
optional_fields = ['unique', 'nullable', 'primary_key']
|
|
160
|
+
supported_fields = required_fields + optional_fields
|
|
161
|
+
supported_datatypes = ['TEXT', 'INTEGER', 'REAL']
|
|
162
|
+
|
|
163
|
+
# Make sure we're not missing any required fields
|
|
164
|
+
missing_fields = [field for field in required_fields if field not in schema_ref]
|
|
165
|
+
if len(missing_fields) > 0:
|
|
166
|
+
raise ValueError(
|
|
167
|
+
f'SQLite schema config is missing the following required fields: {missing_fields}'
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
# Detect unknown fields
|
|
171
|
+
unknown_fields = [field for field in schema_ref if field not in supported_fields]
|
|
172
|
+
if len(unknown_fields) > 0:
|
|
173
|
+
raise ValueError(f'SQLite schema config contains unknown fields: {unknown_fields}')
|
|
174
|
+
|
|
175
|
+
# Pull the fields and test their types
|
|
176
|
+
type_datatype = type(schema_ref.get('datatype'))
|
|
177
|
+
type_field = type(schema_ref.get('field'))
|
|
178
|
+
type_unique = type(schema_ref.get('unique'))
|
|
179
|
+
type_nullable = type(schema_ref.get('nullable'))
|
|
180
|
+
type_primary_key = type(schema_ref.get('primary_key'))
|
|
181
|
+
|
|
182
|
+
# datatype and field are required and must be strings
|
|
183
|
+
if type_datatype is not str:
|
|
184
|
+
raise ValueError(f'Field "datatype" must be a string, but is {type_datatype}')
|
|
185
|
+
if type_field is not str:
|
|
186
|
+
raise ValueError(f'Field "field" must be a string, but is {type_field}')
|
|
187
|
+
# These fields must be booleans, but they are not required, so we allow Nones
|
|
188
|
+
if type_unique is not bool and type_unique is not NoneType:
|
|
189
|
+
raise ValueError(f'Field "unique" must be a boolean, but is {type_unique}')
|
|
190
|
+
if type_nullable is not bool and type_nullable is not NoneType:
|
|
191
|
+
raise ValueError(f'Field "nullable" must be a boolean, but is {type_nullable}')
|
|
192
|
+
if type_primary_key is not bool and type_primary_key is not NoneType:
|
|
193
|
+
raise ValueError(f'Field "primary_key" must be a boolean, but is {type_primary_key}')
|
|
194
|
+
|
|
195
|
+
# Make sure the datatype field is valid
|
|
196
|
+
datatype = str(schema_ref.get('datatype')).upper()
|
|
197
|
+
if datatype not in supported_datatypes:
|
|
198
|
+
raise ValueError(
|
|
199
|
+
f'SQLite datatype {datatype} not supported, must be one of {supported_datatypes}'
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
return True
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
class Marker:
|
|
206
|
+
"""A Marker is a part of the document we expect to find. It identifies the beginning of a
|
|
207
|
+
section in the HTML document which we want to extract data from. Markers have at least one
|
|
208
|
+
extraction step that stores the data discovered after this marker and before the next one.
|
|
209
|
+
|
|
210
|
+
:param tag: The kind of HTML tag we're looking for, such as "p" or "h1".
|
|
211
|
+
:type tag: str
|
|
212
|
+
|
|
213
|
+
:param text: The inner text of the tag we're looking for.
|
|
214
|
+
:type text: str
|
|
215
|
+
|
|
216
|
+
:param id: The ID of the marker, which will become the ID of the Section it indicates.
|
|
217
|
+
:type id: str
|
|
218
|
+
|
|
219
|
+
:param extract: List of extraction configurations.
|
|
220
|
+
:type extract: list[dict]
|
|
221
|
+
"""
|
|
222
|
+
|
|
223
|
+
def __init__(self, tag: str, text: str, id: str, extract: list[dict]):
|
|
224
|
+
self.tag = tag
|
|
225
|
+
self.text = text
|
|
226
|
+
self.id = id
|
|
227
|
+
self.extractor_configs = extract
|
|
228
|
+
self.matched_element = None
|
|
229
|
+
|
|
230
|
+
def match(self, element: bs4.element.Tag) -> bool:
|
|
231
|
+
if element.name == self.tag and element.text == self.text:
|
|
232
|
+
self.matched_element = element
|
|
233
|
+
return True
|
|
234
|
+
return False
|
|
235
|
+
|
|
236
|
+
@staticmethod
|
|
237
|
+
def config_is_valid(marker_config: dict) -> tuple[bool, list[str]]:
|
|
238
|
+
"""Determines if the given Marker config dict is valid.
|
|
239
|
+
|
|
240
|
+
:param marker_config: A Marker config in dict form which might be parameterized into a
|
|
241
|
+
constructor call.
|
|
242
|
+
:type marker_config: dict
|
|
243
|
+
|
|
244
|
+
:return: A tuple of the form ``(is_valid, missing_fields)``. ``is_valid`` is a bool
|
|
245
|
+
indicating the validity of the given Marker config dict and its ability to be
|
|
246
|
+
parameterized into a constructor call. ``missing_fields`` is a list of required config
|
|
247
|
+
keys which must be present but are not. When ``is_valid`` is ``True``, this is an empty
|
|
248
|
+
list (``[]``).
|
|
249
|
+
:rtype: tuple[bool, list[str]]
|
|
250
|
+
"""
|
|
251
|
+
|
|
252
|
+
required_fields = ['tag', 'text', 'id', 'extract']
|
|
253
|
+
missing_fields = [field for field in required_fields if field not in marker_config]
|
|
254
|
+
is_valid = len(missing_fields) == 0
|
|
255
|
+
return (is_valid, missing_fields)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
class Section:
|
|
259
|
+
"""A Section is a piece of a document, some amount of content scraped from the HTML. It has an
|
|
260
|
+
ID, uniquely identifying this portion of content. It also has data, which is the output of the
|
|
261
|
+
extractors run on the content.
|
|
262
|
+
|
|
263
|
+
:param id: The unique ID of the Section.
|
|
264
|
+
:type id: str
|
|
265
|
+
|
|
266
|
+
:param data: The data extracted from this section of the document. Keys are the type of
|
|
267
|
+
extractor used to acquire the data. Values are whatever those extractors produce in whatever
|
|
268
|
+
format they produce it.
|
|
269
|
+
"""
|
|
270
|
+
|
|
271
|
+
def __init__(self, id: str, data: dict | None = None):
|
|
272
|
+
self.id = id
|
|
273
|
+
self.data = data or {}
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
class Document:
|
|
277
|
+
"""A Document is an internal comprehension of the HTML document being scraped based on the plan
|
|
278
|
+
designed for that document. The plan is a series of Markers which have data extraction rules.
|
|
279
|
+
|
|
280
|
+
:param src: The actual HTML of the document we're extracting from.
|
|
281
|
+
:type src: str
|
|
282
|
+
|
|
283
|
+
:param title: The title of the overall document.
|
|
284
|
+
:type title: str
|
|
285
|
+
|
|
286
|
+
:param plan: A series of Marker configurations -- things you expect to find in the document and
|
|
287
|
+
how you want to deal with them. Must have at least one marker defined.
|
|
288
|
+
:type plan: list[dict]
|
|
289
|
+
"""
|
|
290
|
+
|
|
291
|
+
def __init__(self, src: str, title: str, plan: list[dict], outputs: list[dict] = []):
|
|
292
|
+
self.src = src
|
|
293
|
+
self.soup = bs4.BeautifulSoup(self.src, 'lxml')
|
|
294
|
+
self.title = title
|
|
295
|
+
self.sections = None
|
|
296
|
+
self.output_configs = outputs
|
|
297
|
+
|
|
298
|
+
# Validate plan length
|
|
299
|
+
if len(plan) == 0:
|
|
300
|
+
raise ValueError('Plan must have at least one marker in it')
|
|
301
|
+
|
|
302
|
+
self.plan = plan
|
|
303
|
+
|
|
304
|
+
def scrape(self):
|
|
305
|
+
"""Step through the plan trying to extract data from its source accordingly. This function
|
|
306
|
+
populates the ``sections`` member."""
|
|
307
|
+
|
|
308
|
+
# --- Set up the scrape ---
|
|
309
|
+
|
|
310
|
+
# Start with an empty set of sections
|
|
311
|
+
self.sections = OrderedDict()
|
|
312
|
+
|
|
313
|
+
# The "current" marker is the one that heads up the section we're currently tracking
|
|
314
|
+
current_marker = None
|
|
315
|
+
|
|
316
|
+
# Common function to iterate on markers in the plan; returns None when iteration is done.
|
|
317
|
+
def __next_marker(idx: int, plan: list[dict]) -> tuple[int, Marker]:
|
|
318
|
+
"""Determines the next marker in the plan.
|
|
319
|
+
|
|
320
|
+
:param idx: The current plan marker's index.
|
|
321
|
+
:type idx: int
|
|
322
|
+
|
|
323
|
+
:param plan: This Document's plan, passed in so we have it in scope for reading in this
|
|
324
|
+
function.
|
|
325
|
+
:type plan: list[dict]
|
|
326
|
+
|
|
327
|
+
:return: Tuple of two values: the index of the next marker configuration in this
|
|
328
|
+
Document's plan and the Marker object created from that configuration.
|
|
329
|
+
:rtype: tuple[int, Marker]
|
|
330
|
+
"""
|
|
331
|
+
|
|
332
|
+
# Increment the marker index
|
|
333
|
+
idx += 1
|
|
334
|
+
|
|
335
|
+
# If the index is within bounds, create a Marker object from the config
|
|
336
|
+
if idx < len(plan):
|
|
337
|
+
marker_config_is_valid, missing_fields = Marker.config_is_valid(plan[idx])
|
|
338
|
+
if marker_config_is_valid:
|
|
339
|
+
marker = Marker(**plan[idx])
|
|
340
|
+
else:
|
|
341
|
+
raise ValueError(
|
|
342
|
+
f'Marker {plan[idx].get("id", "<no_id>")} is missing required configuration parameters:'
|
|
343
|
+
f' {", ".join(missing_fields)}'
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
# If it's out of bounds, the marker is None
|
|
347
|
+
else:
|
|
348
|
+
marker = None
|
|
349
|
+
|
|
350
|
+
# Return the incremented index and corresponding Marker
|
|
351
|
+
return idx, marker
|
|
352
|
+
|
|
353
|
+
# Common function to run extractors on a marker/section
|
|
354
|
+
def __extract(current_marker: Marker) -> Section:
|
|
355
|
+
"""Given the current Marker (the HTML tag we're currently looking for), creates all the
|
|
356
|
+
defined Extractors and tells them to extract data from the current set of elements we've
|
|
357
|
+
identified as belonging to the current section.
|
|
358
|
+
|
|
359
|
+
:param current_marker: The Marker we're currently parsing data for.
|
|
360
|
+
:type current_marker: Marker
|
|
361
|
+
|
|
362
|
+
:return: A proper Section object with the extracted data included.
|
|
363
|
+
:rtype: Section
|
|
364
|
+
"""
|
|
365
|
+
|
|
366
|
+
# Create a section, copying in the ID of the current marker
|
|
367
|
+
section = Section(id=current_marker.id)
|
|
368
|
+
|
|
369
|
+
# Run each defined extractor and store its data in the Section's data dictionary
|
|
370
|
+
from .extractors import construct_extractor
|
|
371
|
+
|
|
372
|
+
for extractor in [
|
|
373
|
+
construct_extractor(
|
|
374
|
+
config=extractor_config,
|
|
375
|
+
src=current_section_elements,
|
|
376
|
+
section_id=current_marker.id,
|
|
377
|
+
)
|
|
378
|
+
for extractor_config in current_marker.extractor_configs
|
|
379
|
+
]:
|
|
380
|
+
# Use the extractor type as the section's data key. While we should be able
|
|
381
|
+
# to use multiple extractors on the same set of elements, we should *not*
|
|
382
|
+
# have multiple extractors of the *same type*.
|
|
383
|
+
section.data[extractor.type] = extractor.extract(current_marker)
|
|
384
|
+
|
|
385
|
+
return section
|
|
386
|
+
|
|
387
|
+
# The "next" marker is the marker we expect to find next in the soup. To start, that's the
|
|
388
|
+
# first marker (zeroth index) in the list.
|
|
389
|
+
marker_idx = 0
|
|
390
|
+
next_marker = Marker(**self.plan[marker_idx])
|
|
391
|
+
|
|
392
|
+
# We begin with no section; when we find our first marker, we will start a new section
|
|
393
|
+
current_section_id = None
|
|
394
|
+
|
|
395
|
+
# Track the elements since the last time we found a marker
|
|
396
|
+
current_section_elements = []
|
|
397
|
+
|
|
398
|
+
# --- Document iteration ---
|
|
399
|
+
|
|
400
|
+
# Go through every element in the soup, checking each one against the current marker until
|
|
401
|
+
# we get a match. At the beginning, you will get things like an <html> tag and a <body> tag
|
|
402
|
+
# that contain the entire document, and which are pretty much useless. Because these
|
|
403
|
+
# elements won't match a sensible plan, these elements are naturally scrapped.
|
|
404
|
+
for element in self.soup.find_all():
|
|
405
|
+
# If the element is a match for the marker we expect next...
|
|
406
|
+
if next_marker:
|
|
407
|
+
if next_marker.match(element):
|
|
408
|
+
# If a section ID is set, that means we've already matched an element and started a
|
|
409
|
+
# section. We need to form a proper Section object and stash it in this Document.
|
|
410
|
+
if current_section_id:
|
|
411
|
+
# Run extractors on the current data set
|
|
412
|
+
self.sections[current_section_id] = __extract(current_marker=current_marker)
|
|
413
|
+
|
|
414
|
+
# Set the current marker to the thing we just found
|
|
415
|
+
current_marker = next_marker
|
|
416
|
+
|
|
417
|
+
# Set the current section ID to the now-current marker's ID
|
|
418
|
+
current_section_id = current_marker.id
|
|
419
|
+
|
|
420
|
+
# Iterate to the next marker
|
|
421
|
+
marker_idx, next_marker = __next_marker(marker_idx, self.plan)
|
|
422
|
+
# Clear out the elements we've been tracking
|
|
423
|
+
current_section_elements = []
|
|
424
|
+
|
|
425
|
+
# If a section ID is *not* set, that means we have yet to match our first
|
|
426
|
+
# marker. Therefore, *this* is our first marker and we should start tracking the
|
|
427
|
+
# contents of this section.
|
|
428
|
+
else:
|
|
429
|
+
# Set the current marker to the thing we just found
|
|
430
|
+
current_marker = next_marker
|
|
431
|
+
|
|
432
|
+
# Set the current section ID to the now-current marker's ID
|
|
433
|
+
current_section_id = current_marker.id
|
|
434
|
+
|
|
435
|
+
# Iterate to the next marker
|
|
436
|
+
marker_idx, next_marker = __next_marker(marker_idx, self.plan)
|
|
437
|
+
|
|
438
|
+
# Clear out the elements we've been tracking
|
|
439
|
+
current_section_elements = []
|
|
440
|
+
|
|
441
|
+
# If we are in a section but this is not a match for the next marker...
|
|
442
|
+
else:
|
|
443
|
+
# ...just tack it on to the current list of elements.
|
|
444
|
+
current_section_elements.append(element)
|
|
445
|
+
# If there is no next marker, then we are in our final section...
|
|
446
|
+
else:
|
|
447
|
+
# ...so just tack it on to the current list of elements.
|
|
448
|
+
current_section_elements.append(element)
|
|
449
|
+
|
|
450
|
+
# After iterating over everything, we need to do one final extraction, but only if we ever
|
|
451
|
+
# actually matched something.
|
|
452
|
+
if current_section_id:
|
|
453
|
+
self.sections[current_section_id] = __extract(current_marker=current_marker)
|
|
454
|
+
|
|
455
|
+
def output(self):
|
|
456
|
+
"""Runs through the document's output configs, attempting to create each output defined."""
|
|
457
|
+
|
|
458
|
+
for output_config in self.output_configs:
|
|
459
|
+
output_id = output_config.get('id')
|
|
460
|
+
if not output_id:
|
|
461
|
+
raise ValueError('Output does not have an `id` field.')
|
|
462
|
+
|
|
463
|
+
output_type = output_config.get('type')
|
|
464
|
+
if not output_type:
|
|
465
|
+
raise ValueError(f'Output "{output_id}" must specify a "type".')
|
|
466
|
+
if output_type not in KNOWN_OUTPUTS:
|
|
467
|
+
raise ValueError(f'Output "{output_id}" has an unknown output type: {output_type}.')
|
|
468
|
+
|
|
469
|
+
from .outputters import construct_outputter
|
|
470
|
+
|
|
471
|
+
outputter = construct_outputter(sections=self.sections, config=output_config)
|
|
472
|
+
outputter.output()
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from importlib import import_module
|
|
2
|
+
from typing import TYPE_CHECKING
|
|
3
|
+
|
|
4
|
+
if TYPE_CHECKING:
|
|
5
|
+
from bs4.element import Tag
|
|
6
|
+
|
|
7
|
+
from .base import Extractor
|
|
8
|
+
|
|
9
|
+
#: Mapping of strings (names of extractors) to the importable paths of the Extractor implementation
|
|
10
|
+
# classes they refer to. We use strings here and importlib later to avoid circular dependencies.
|
|
11
|
+
EXTRACTOR_TYPES = {
|
|
12
|
+
'read_html': {
|
|
13
|
+
'package': 'read_html',
|
|
14
|
+
'class': 'ReadHTMLExtractor',
|
|
15
|
+
},
|
|
16
|
+
'sqlite_table': {
|
|
17
|
+
'package': 'sqlite_table',
|
|
18
|
+
'class': 'SQLiteTableExtractor',
|
|
19
|
+
},
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def construct_extractor(config: dict, src: list[Tag], section_id: str) -> Extractor:
|
|
24
|
+
"""Given a configuration for an Extractor, constructs the correct extractor sub-type class.
|
|
25
|
+
|
|
26
|
+
:param config: Configuration for the extractor sub-type.
|
|
27
|
+
:type config: dict
|
|
28
|
+
|
|
29
|
+
:param src: The HTML tags that are a part of this Section of the Document.
|
|
30
|
+
:type src: bs4.element.Tag
|
|
31
|
+
|
|
32
|
+
:param section_id: The ID of this Section.
|
|
33
|
+
:type src: str
|
|
34
|
+
|
|
35
|
+
:raises KeyError: The config does not contain a "type" field.
|
|
36
|
+
|
|
37
|
+
:raises ValueError: The config contains an unknown "type" value.
|
|
38
|
+
|
|
39
|
+
:return: Fully constructed Extractor sub-type class.
|
|
40
|
+
:rtype: Extractor
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
extractor_type = config.get('type')
|
|
44
|
+
if not extractor_type:
|
|
45
|
+
raise KeyError('Field "type" is required in extractor configs')
|
|
46
|
+
if extractor_type not in EXTRACTOR_TYPES:
|
|
47
|
+
raise ValueError(f'Unknown extractor type: {extractor_type}')
|
|
48
|
+
__type = EXTRACTOR_TYPES[extractor_type]
|
|
49
|
+
__module = import_module(f'escrape_plan.extractors.{__type["package"]}')
|
|
50
|
+
__extractor = getattr(__module, __type['class'])
|
|
51
|
+
return __extractor(config=config, src=src, section_id=section_id)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import typing
|
|
2
|
+
from typing import TYPE_CHECKING
|
|
3
|
+
|
|
4
|
+
from bs4.element import Tag
|
|
5
|
+
|
|
6
|
+
from .. import SchemaValidator
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from . import Marker
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Extractor(SchemaValidator):
|
|
13
|
+
"""Base class for all extractors to use. Validates configurations against a defined schema.
|
|
14
|
+
Requires that subclasses implement the ``extract`` function and the ``config_schema`` property.
|
|
15
|
+
|
|
16
|
+
:param src: The HTML elements (already parsed by BeautifulSoup) we're extracting data from.
|
|
17
|
+
:type src: list[Tag]
|
|
18
|
+
|
|
19
|
+
:param config: The extraction configuration. This includes the ``type`` of config, which
|
|
20
|
+
determines how we validate the rest of the config.
|
|
21
|
+
:type config: dict
|
|
22
|
+
|
|
23
|
+
:param section_id: The ID of the section this extractor is parsing data from.
|
|
24
|
+
:type section_id: str
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(self, src: list[Tag], config: dict, section_id: str):
|
|
28
|
+
super().__init__(config=config)
|
|
29
|
+
self.src = src
|
|
30
|
+
self.config = config
|
|
31
|
+
self.section_id = section_id
|
|
32
|
+
self.type = None
|
|
33
|
+
self.validate_config_schema()
|
|
34
|
+
|
|
35
|
+
def extract(self, marker: Marker) -> typing.Any:
|
|
36
|
+
"""Must be implemented in subclasses. Extracts information from the ``src`` string however
|
|
37
|
+
it's necessary to. Returns any value, however the content should be referenced; this will
|
|
38
|
+
get stored in the document according to its plan.
|
|
39
|
+
|
|
40
|
+
:param marker: The BeautifulSoup Tag representing the marker that matched at the beginning
|
|
41
|
+
of this section.
|
|
42
|
+
:type marker: Marker
|
|
43
|
+
|
|
44
|
+
:return: The data extracted by this Extractor.
|
|
45
|
+
:rtype: typing.Any
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
raise NotImplementedError()
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
from functools import cached_property
|
|
2
|
+
|
|
3
|
+
from bs4.element import Tag
|
|
4
|
+
|
|
5
|
+
from escrape_plan import Marker
|
|
6
|
+
|
|
7
|
+
from .base import Extractor
|
|
8
|
+
|
|
9
|
+
KNOWN_TRANSFORMS = ['replace']
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ReadHTMLExtractor(Extractor):
|
|
13
|
+
"""The "read_html" extractor, which converts the HTML elements found in a Document into strings
|
|
14
|
+
of valid HTML snippets.
|
|
15
|
+
|
|
16
|
+
:param src: The HTML elements (already parsed by BeautifulSoup) we're extracting data from.
|
|
17
|
+
:type src: list[Tag]
|
|
18
|
+
|
|
19
|
+
:param config: The extraction configuration. Must comply with the :py:meth:`.config_schema`.
|
|
20
|
+
:type config: dict
|
|
21
|
+
|
|
22
|
+
:param section_id: The ID of this Section of the Document.
|
|
23
|
+
:type section_id: str
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def __init__(self, src: list[Tag], config: dict, section_id: str):
|
|
27
|
+
# Automatically handles validation against the config_schema
|
|
28
|
+
super().__init__(src, config, section_id)
|
|
29
|
+
self.type = 'read_html'
|
|
30
|
+
|
|
31
|
+
@cached_property
|
|
32
|
+
def config_schema(self) -> dict:
|
|
33
|
+
"""Schema definition for ReadHTMLExtractors.
|
|
34
|
+
|
|
35
|
+
:return: dict of field value constraints.
|
|
36
|
+
:rtype: dict
|
|
37
|
+
"""
|
|
38
|
+
return {
|
|
39
|
+
'type': {
|
|
40
|
+
'description': 'Kind of extractor (must be "read_html")',
|
|
41
|
+
'datatype': str,
|
|
42
|
+
'required': True,
|
|
43
|
+
'regex': '^read_html$',
|
|
44
|
+
},
|
|
45
|
+
'title': {
|
|
46
|
+
'description': 'Title of the section of the document this data is extracted to',
|
|
47
|
+
'datatype': str,
|
|
48
|
+
'required': True,
|
|
49
|
+
},
|
|
50
|
+
'keep_marker': {
|
|
51
|
+
'description': 'When True, keeps the matched Marker tag in the extracted data',
|
|
52
|
+
'datatype': bool,
|
|
53
|
+
'required': False,
|
|
54
|
+
},
|
|
55
|
+
'transforms': {
|
|
56
|
+
'description': 'List of transformations to apply to the content',
|
|
57
|
+
'datatype': list,
|
|
58
|
+
'required': False,
|
|
59
|
+
},
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
def extract(self, marker: Marker) -> str:
|
|
63
|
+
"""Returns the HTML that was given to the extractor in the constructor, but with any
|
|
64
|
+
defined transformations applied.
|
|
65
|
+
|
|
66
|
+
:param marker: The Marker representing the tag that matched at the beginning of this
|
|
67
|
+
section.
|
|
68
|
+
:type marker: Marker
|
|
69
|
+
|
|
70
|
+
:return: The HTML provided in the src.
|
|
71
|
+
:rtype: str
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
# Convert each tag to a string and concatenate them all
|
|
75
|
+
extracted_str = ''.join([str(tag) for tag in self.src])
|
|
76
|
+
|
|
77
|
+
keep_marker = self.config.get('keep_marker', True)
|
|
78
|
+
if keep_marker and marker.matched_element:
|
|
79
|
+
extracted_str = str(marker.matched_element) + extracted_str
|
|
80
|
+
|
|
81
|
+
# Apply transformations
|
|
82
|
+
transforms = self.config.get('transforms')
|
|
83
|
+
if transforms:
|
|
84
|
+
for transform in transforms:
|
|
85
|
+
# Determine and validate transform type
|
|
86
|
+
transform_type = transform.get('type')
|
|
87
|
+
if not transform_type:
|
|
88
|
+
raise ValueError('Transform must have a "type" field.')
|
|
89
|
+
|
|
90
|
+
if transform_type not in KNOWN_TRANSFORMS:
|
|
91
|
+
raise ValueError(
|
|
92
|
+
f'Unknown transform type: {transform_type}. Must be one of'
|
|
93
|
+
f' {", ".join(KNOWN_TRANSFORMS)}'
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
if transform_type == 'replace':
|
|
97
|
+
find = transform.get('find')
|
|
98
|
+
replacement = transform.get('replacement')
|
|
99
|
+
if find and replacement and len(find) > 0:
|
|
100
|
+
extracted_str = extracted_str.replace(find, replacement)
|
|
101
|
+
else:
|
|
102
|
+
raise ValueError(
|
|
103
|
+
'"find" and "replacement" are both required fields for transform type '
|
|
104
|
+
'replace.'
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
return extracted_str
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import sqlite3
|
|
2
|
+
from functools import cached_property
|
|
3
|
+
|
|
4
|
+
from bs4 import BeautifulSoup
|
|
5
|
+
from bs4.element import Tag
|
|
6
|
+
|
|
7
|
+
from escrape_plan import Marker
|
|
8
|
+
|
|
9
|
+
from .base import Extractor, SchemaValidator
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class SQLiteTableExtractor(Extractor):
|
|
13
|
+
"""
|
|
14
|
+
The "sqlite_table" extractor, which reads data from an HTML table and converts it into a schema
|
|
15
|
+
and a set of records in a SQLite file.
|
|
16
|
+
|
|
17
|
+
This extractor is inherently destructive. While you can specify a SQLite database file that has
|
|
18
|
+
other content in it, if this module finds a table in that database by the same name you've
|
|
19
|
+
specified in the config, **it will destroy every row in that table and replace it with data
|
|
20
|
+
found by this extractor.**
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(self, src: list[Tag], config: dict, section_id: str):
|
|
24
|
+
super().__init__(src, config, section_id)
|
|
25
|
+
self.type = 'sqlite_table'
|
|
26
|
+
|
|
27
|
+
# Localize some variables from the config
|
|
28
|
+
self.sqlite_filename = config.get('filename')
|
|
29
|
+
self.table_name = config.get('table')
|
|
30
|
+
self.schema = self.config.get('schema')
|
|
31
|
+
|
|
32
|
+
# Get all table tags in the src
|
|
33
|
+
all_table_tags = [tag for tag in src if tag.name == 'table']
|
|
34
|
+
|
|
35
|
+
# Check that there is actually a table in these tags
|
|
36
|
+
if len(all_table_tags) < 1:
|
|
37
|
+
raise ValueError(f'There are no tables in section {self.section_id}')
|
|
38
|
+
|
|
39
|
+
# Also check that no more than one table occur in these tags
|
|
40
|
+
if len(all_table_tags) > 1:
|
|
41
|
+
raise ValueError(f'More than one table in section {self.section_id}')
|
|
42
|
+
|
|
43
|
+
# Now we know there's one index in this list. This one <table> tag contains everything we
|
|
44
|
+
# need to parse through. Here we localize that one tag and re-soup it so we don't overparse.
|
|
45
|
+
self.table_soup = BeautifulSoup(str(all_table_tags[0]), 'lxml')
|
|
46
|
+
|
|
47
|
+
# Now check our schema config. We know it exists because it's a required field that's been
|
|
48
|
+
# validated in the superconstructor, but now we need to do validity checks on this data.
|
|
49
|
+
if len(self.schema) == 0:
|
|
50
|
+
raise ValueError('The configured schema is empty. Provide at least one field.')
|
|
51
|
+
|
|
52
|
+
# Run each schema entry through the special validator
|
|
53
|
+
for schema_ref in self.schema:
|
|
54
|
+
SchemaValidator.validate_sqlite_table_schema(schema_ref=schema_ref)
|
|
55
|
+
|
|
56
|
+
# Make sure we don't have more than one primary key
|
|
57
|
+
pk_fields = [schema_ref for schema_ref in self.schema if schema_ref.get('primary_key')]
|
|
58
|
+
if len(pk_fields) > 1:
|
|
59
|
+
raise ValueError('The schema cannot have more than one primary key.')
|
|
60
|
+
|
|
61
|
+
# The config all appears to be valid at this point, so we need to set up the database
|
|
62
|
+
# Here are some queries we'll use throughout this class.
|
|
63
|
+
|
|
64
|
+
# Construct the SQLite table schema from our config
|
|
65
|
+
table_schema_entries = []
|
|
66
|
+
for field in self.schema:
|
|
67
|
+
entry = f'"{field.get("field")}" {field.get("datatype")}'
|
|
68
|
+
if field.get('primary_key', False):
|
|
69
|
+
entry += ' PRIMARY KEY'
|
|
70
|
+
if field.get('unique', False):
|
|
71
|
+
entry += ' UNIQUE'
|
|
72
|
+
if not field.get('nullable', True):
|
|
73
|
+
entry += ' NOT NULL'
|
|
74
|
+
table_schema_entries.append(entry)
|
|
75
|
+
table_schema_str = ', '.join(table_schema_entries)
|
|
76
|
+
self.__query_create_table = f'CREATE TABLE {self.table_name} ({table_schema_str});'
|
|
77
|
+
self.__query_drop_table = f'DROP TABLE {self.table_name};'
|
|
78
|
+
self.__query_detect_table = (
|
|
79
|
+
f'SELECT COUNT(*) FROM sqlite_master WHERE type="table" and name="{self.table_name}";'
|
|
80
|
+
)
|
|
81
|
+
self.__querytpl_insert_row = (
|
|
82
|
+
f'INSERT INTO {self.table_name} '
|
|
83
|
+
f'({", ".join([field.get("field") for field in self.schema])}) '
|
|
84
|
+
f'VALUES ({("?, " * len(self.schema)).strip(", ")});'
|
|
85
|
+
)
|
|
86
|
+
self.__setup_database()
|
|
87
|
+
|
|
88
|
+
def __setup_database(self):
|
|
89
|
+
"""Sets up the SQLite table we are dumping this data into"""
|
|
90
|
+
|
|
91
|
+
# Do all of this in a connection context
|
|
92
|
+
with sqlite3.connect(self.sqlite_filename) as sqlite_conn:
|
|
93
|
+
cursor = sqlite_conn.cursor()
|
|
94
|
+
|
|
95
|
+
# Does the table already exist? Get a count of tables by this name.
|
|
96
|
+
cursor.execute(self.__query_detect_table)
|
|
97
|
+
table_ct = cursor.fetchone()[0]
|
|
98
|
+
|
|
99
|
+
# If the table already exists, drop it entirely
|
|
100
|
+
if table_ct > 0:
|
|
101
|
+
cursor.execute(self.__query_drop_table)
|
|
102
|
+
|
|
103
|
+
# Now create the table with the correct schema
|
|
104
|
+
cursor.execute(self.__query_create_table)
|
|
105
|
+
|
|
106
|
+
@cached_property
|
|
107
|
+
def config_schema(self) -> dict:
|
|
108
|
+
return {
|
|
109
|
+
'type': {
|
|
110
|
+
'description': 'Kind of extractor (must be "sqlite_table")',
|
|
111
|
+
'datatype': str,
|
|
112
|
+
'required': True,
|
|
113
|
+
'regex': '^sqlite_table$',
|
|
114
|
+
},
|
|
115
|
+
'filename': {
|
|
116
|
+
'description': 'SQLite database file location',
|
|
117
|
+
'datatype': str,
|
|
118
|
+
'required': True,
|
|
119
|
+
},
|
|
120
|
+
'table': {
|
|
121
|
+
'description': 'Name of the table in the SQLite database to store this data',
|
|
122
|
+
'datatype': str,
|
|
123
|
+
'required': True,
|
|
124
|
+
},
|
|
125
|
+
'schema': {
|
|
126
|
+
'description': 'Object indicating the schema of the table',
|
|
127
|
+
'datatype': list,
|
|
128
|
+
'required': True,
|
|
129
|
+
},
|
|
130
|
+
'ignore_first_row': {
|
|
131
|
+
'description': 'Whether the first row should be treated as headers and not data',
|
|
132
|
+
'datatype': bool,
|
|
133
|
+
'required': False,
|
|
134
|
+
},
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
def extract(self, marker: Marker) -> None:
|
|
138
|
+
"""Seeks out <table> data in the HTML source and attempts to convert it into a SQLite
|
|
139
|
+
database file.
|
|
140
|
+
"""
|
|
141
|
+
|
|
142
|
+
# The number of columns defined in the schema in our config
|
|
143
|
+
required_column_ct = len(self.schema)
|
|
144
|
+
|
|
145
|
+
# Do we skip the first row?
|
|
146
|
+
ignore_first_row = self.config.get('ignore_first_row', False)
|
|
147
|
+
|
|
148
|
+
# Perform all of the upcoming steps within a connection context
|
|
149
|
+
with sqlite3.connect(self.sqlite_filename) as sqlite_conn:
|
|
150
|
+
cursor = sqlite_conn.cursor()
|
|
151
|
+
|
|
152
|
+
# Grab all the <tr> elements in the table and go through them. Track whether we've
|
|
153
|
+
# parsed the first row.
|
|
154
|
+
first_row_parsed = False
|
|
155
|
+
for row in self.table_soup.find_all(name='tr'):
|
|
156
|
+
# Determine if we need to skip the first row
|
|
157
|
+
if not first_row_parsed:
|
|
158
|
+
first_row_parsed = True
|
|
159
|
+
if ignore_first_row:
|
|
160
|
+
continue
|
|
161
|
+
|
|
162
|
+
# We ignore <th> tags, only caring about <td> tags.
|
|
163
|
+
columns = row.find_all(name='td')
|
|
164
|
+
|
|
165
|
+
# There's a problem if we don't have exactly the same number of table columns as
|
|
166
|
+
# fields in our SQLite schema. This has to be checked for every row because <td>
|
|
167
|
+
# tags can have a `colspan` property that changes the number of columns in the row.
|
|
168
|
+
column_ct = len(columns)
|
|
169
|
+
if column_ct != required_column_ct:
|
|
170
|
+
raise ValueError(
|
|
171
|
+
f'Row has {column_ct} columns, but needs {required_column_ct}.'
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
# Go through the columns, forming a tuple of their type-verified values as we go, so
|
|
175
|
+
# we can later populate the insert query template with them. We have to match the
|
|
176
|
+
# table cells to our schema fields, so track the column index as we go.
|
|
177
|
+
field_idx = -1
|
|
178
|
+
query_values = []
|
|
179
|
+
for column in columns:
|
|
180
|
+
field_idx += 1
|
|
181
|
+
data = column.get_text().strip()
|
|
182
|
+
|
|
183
|
+
# Pull the SQL datatype for this column and try to conform the data to the
|
|
184
|
+
# corresponding Python type. If we can't, raise a ValueError with a better
|
|
185
|
+
# message.
|
|
186
|
+
col_schema = self.schema[field_idx]
|
|
187
|
+
target_sql_type = col_schema.get('datatype').upper()
|
|
188
|
+
target_python_type = {
|
|
189
|
+
'INTEGER': int,
|
|
190
|
+
'REAL': float,
|
|
191
|
+
'TEXT': str,
|
|
192
|
+
}.get(target_sql_type)
|
|
193
|
+
try:
|
|
194
|
+
data = target_python_type(data)
|
|
195
|
+
except ValueError:
|
|
196
|
+
raise ValueError(
|
|
197
|
+
f'Could not conform value {data} to datatype {target_sql_type}'
|
|
198
|
+
)
|
|
199
|
+
query_values.append(data)
|
|
200
|
+
|
|
201
|
+
# Execute the INSERT query
|
|
202
|
+
cursor.execute(self.__querytpl_insert_row, query_values)
|
|
203
|
+
|
|
204
|
+
# Commit the transaction once all the INSERTs have run
|
|
205
|
+
sqlite_conn.commit()
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from collections import OrderedDict
|
|
2
|
+
from importlib import import_module
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from .. import Section
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from .base import Outputter
|
|
9
|
+
|
|
10
|
+
#: Mapping of strings (names of outputters) to the importable paths of the Outputter implementation
|
|
11
|
+
# classes they refer to. We use strings here and ipmortlib later to avoid circular dependencies.
|
|
12
|
+
OUTPUTTER_TYPES = {
|
|
13
|
+
'text_file': {
|
|
14
|
+
'package': 'text_file',
|
|
15
|
+
'class': 'TextFileOutputter',
|
|
16
|
+
},
|
|
17
|
+
'text_to_sqlite': {
|
|
18
|
+
'package': 'text_to_sqlite',
|
|
19
|
+
'class': 'TextToSqliteOutputter',
|
|
20
|
+
},
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def construct_outputter(sections: OrderedDict[str, Section], config: dict) -> Outputter:
|
|
25
|
+
outputter_type = config.get('type')
|
|
26
|
+
if not outputter_type:
|
|
27
|
+
raise KeyError('Field "type" is required in outputter configs')
|
|
28
|
+
if outputter_type not in OUTPUTTER_TYPES:
|
|
29
|
+
raise ValueError(f'Unknown outputter type: {outputter_type}')
|
|
30
|
+
__type = OUTPUTTER_TYPES[outputter_type]
|
|
31
|
+
__module = import_module(f'escrape_plan.outputters.{__type["package"]}')
|
|
32
|
+
__outputter = getattr(__module, __type['class'])
|
|
33
|
+
return __outputter(sections=sections, config=config)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from .. import SchemaValidator
|
|
2
|
+
from . import Section
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class Outputter(SchemaValidator):
|
|
6
|
+
"""Base class for all outputters to use. Validates configurations against a defined schema.
|
|
7
|
+
Requires that subclasses implement the ``output`` function and the ``config_schema`` property.
|
|
8
|
+
|
|
9
|
+
:param sections: List of ``Section`` objects, the result of running a plan and extracting data.
|
|
10
|
+
:type sections: list[Section]
|
|
11
|
+
|
|
12
|
+
:param config: The outputter configuration. This includes an ``id`` for the outputter, the
|
|
13
|
+
``type`` of the outputter, and any other options required for that outputter.
|
|
14
|
+
:type config: dict
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def __init__(self, sections: list[Section], config: dict):
|
|
18
|
+
super().__init__(config=config)
|
|
19
|
+
self.config = config
|
|
20
|
+
self.sections = sections
|
|
21
|
+
self.type = None
|
|
22
|
+
self.validate_config_schema()
|
|
23
|
+
|
|
24
|
+
def output() -> None:
|
|
25
|
+
"""Must be ipmlemented in subclasses. Outputs data from the provided Sections according to
|
|
26
|
+
the given config.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
raise NotImplementedError()
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
from .base import Outputter, SchemaValidator, Section
|
|
2
|
+
from functools import cached_property
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class TextFileOutputter(Outputter):
|
|
6
|
+
"""The "text_file" outputter, which places string data extracted from a document into a file on
|
|
7
|
+
disk. Content can be restructured, omitted, etc. depending on the config.
|
|
8
|
+
|
|
9
|
+
:param sections: List of ``Section`` objects containing data to output.
|
|
10
|
+
:type sections: list[Section]
|
|
11
|
+
|
|
12
|
+
:param config: Configuration for outputting that data into text files.
|
|
13
|
+
:type config: dict
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
def __init__(self, sections: list[Section], config: dict):
|
|
17
|
+
super().__init__(sections=sections, config=config)
|
|
18
|
+
self.type = 'text_file'
|
|
19
|
+
|
|
20
|
+
@cached_property
|
|
21
|
+
def config_schema(self) -> dict:
|
|
22
|
+
return {
|
|
23
|
+
'id': {
|
|
24
|
+
'description': 'Unique ID for the outputter',
|
|
25
|
+
'datatype': str,
|
|
26
|
+
'required': True,
|
|
27
|
+
},
|
|
28
|
+
'type': {
|
|
29
|
+
'description': 'Kind of outputter (must be "text_file")',
|
|
30
|
+
'datatype': str,
|
|
31
|
+
'required': True,
|
|
32
|
+
'regex': '^text_file$',
|
|
33
|
+
},
|
|
34
|
+
'filename': {
|
|
35
|
+
'description': 'Place on disk to produce the output',
|
|
36
|
+
'datatype': str,
|
|
37
|
+
'required': True,
|
|
38
|
+
},
|
|
39
|
+
'content': {
|
|
40
|
+
'description': 'List of content to place into the output.',
|
|
41
|
+
'datatype': list,
|
|
42
|
+
'required': True,
|
|
43
|
+
},
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
def validate_config_schema(self):
|
|
47
|
+
"""Verifies that the config given for this outputter meets the requirements of its schema.
|
|
48
|
+
This function adds additional checks on top of the base validation provided by the
|
|
49
|
+
SchemaValidator super class.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
super().validate_config_schema()
|
|
53
|
+
|
|
54
|
+
content_configs = self.config.get('content', [])
|
|
55
|
+
for content_config in content_configs:
|
|
56
|
+
SchemaValidator.validate_document_section_reference(section_ref=content_config)
|
|
57
|
+
|
|
58
|
+
def output(self):
|
|
59
|
+
"""Produce the actual text files."""
|
|
60
|
+
|
|
61
|
+
# Localize the outputter settings for convenience. We know these are present because
|
|
62
|
+
# validate_config_schema raises an exception if they aren't.
|
|
63
|
+
output_id = self.config.get('id')
|
|
64
|
+
output_type = self.config.get('type')
|
|
65
|
+
filename = self.config.get('filename')
|
|
66
|
+
content_configs = self.config.get('content')
|
|
67
|
+
|
|
68
|
+
# Get a file handler
|
|
69
|
+
with open(filename, 'w') as output_file:
|
|
70
|
+
# Iterate over the content and output each entry
|
|
71
|
+
for content_config in content_configs:
|
|
72
|
+
# Determine what to do based on the type. We know that 'type' is present because we
|
|
73
|
+
# check for it in validate_config_schema.
|
|
74
|
+
content_type = content_config.get('type')
|
|
75
|
+
|
|
76
|
+
# Pull content from a document section.
|
|
77
|
+
if content_type == 'section':
|
|
78
|
+
# We know these fields exist because validate_config_schema makes sure of it.
|
|
79
|
+
section_id = content_config.get('section_id')
|
|
80
|
+
data_id = content_config.get('data')
|
|
81
|
+
|
|
82
|
+
# But we need to make sure that the section and data actually exist
|
|
83
|
+
if section_id not in self.sections:
|
|
84
|
+
raise ValueError(f'Section ID {section_id} does not exist in the document')
|
|
85
|
+
section = self.sections[section_id]
|
|
86
|
+
|
|
87
|
+
if data_id not in section.data:
|
|
88
|
+
raise ValueError(f'Data {data_id} does not exist in section {section_id}')
|
|
89
|
+
data = section.data[data_id]
|
|
90
|
+
|
|
91
|
+
# Make sure that everything we output is actually textual when we write it
|
|
92
|
+
output_file.write(str(data))
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import sqlite3
|
|
2
|
+
from functools import cached_property
|
|
3
|
+
|
|
4
|
+
from .base import Outputter, SchemaValidator, Section
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class TextToSqliteOutputter(Outputter):
|
|
8
|
+
"""Produces SQLite database files containing the specified contents. Each section gets inserted
|
|
9
|
+
into the specified table in the specified SQLite file following a very simple string storage
|
|
10
|
+
schema. **This outputter is destructive.** If you point it at a SQLite database that already
|
|
11
|
+
exists, it will overwrite any existing data it finds there. This is done on a per-section basis,
|
|
12
|
+
not a per-table or per-database basis. This can lead to confusing results. It is strongly
|
|
13
|
+
recommended that you use empty or non-existent database files when you run this outputter to
|
|
14
|
+
avoid mixed data, or else be very careful about how you configure your outputters.
|
|
15
|
+
|
|
16
|
+
:param sections: List of ``Section`` objects containing data to output.
|
|
17
|
+
:type sections: list[Section]
|
|
18
|
+
|
|
19
|
+
:param config: Configuration for outputting that data into SQLite files.
|
|
20
|
+
:type config: dict
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(self, sections: list[Section], config: dict):
|
|
24
|
+
super().__init__(sections=sections, config=config)
|
|
25
|
+
self.type = 'text_to_sqlite'
|
|
26
|
+
|
|
27
|
+
# Localize some config variables for convenience
|
|
28
|
+
self.table_name = self.config.get('table')
|
|
29
|
+
self.filename = self.config.get('filename')
|
|
30
|
+
|
|
31
|
+
# Define some SQLite queries we'll be using later
|
|
32
|
+
self.__query_create_table = (
|
|
33
|
+
f'CREATE TABLE IF NOT EXISTS {self.table_name} '
|
|
34
|
+
'(section_id TEXT PRIMARY_KEY, data TEXT);'
|
|
35
|
+
)
|
|
36
|
+
self.__querytpl_get_section = (
|
|
37
|
+
f'SELECT * FROM {self.table_name} WHERE section_id=:section_id;'
|
|
38
|
+
)
|
|
39
|
+
self.__querytpl_insert_section = (
|
|
40
|
+
f'INSERT INTO {self.table_name} (section_id, data) VALUES (:section_id, :data);'
|
|
41
|
+
)
|
|
42
|
+
self.__querytpl_update_section = (
|
|
43
|
+
f'UPDATE {self.table_name} SET data = :data WHERE section_id = :section_id;'
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
@cached_property
|
|
47
|
+
def config_schema(self) -> dict:
|
|
48
|
+
return {
|
|
49
|
+
'id': {
|
|
50
|
+
'description': 'Unique ID for the outputter',
|
|
51
|
+
'datatype': str,
|
|
52
|
+
'required': True,
|
|
53
|
+
},
|
|
54
|
+
'type': {
|
|
55
|
+
'description': 'Kind of outputter (must be "text_to_sqlite")',
|
|
56
|
+
'datatype': str,
|
|
57
|
+
'required': True,
|
|
58
|
+
'regex': '^text_to_sqlite$',
|
|
59
|
+
},
|
|
60
|
+
'filename': {
|
|
61
|
+
'description': 'SQLite file on disk to place the content',
|
|
62
|
+
'datatype': str,
|
|
63
|
+
'required': True,
|
|
64
|
+
},
|
|
65
|
+
'table': {
|
|
66
|
+
'description': 'Name of the SQLite table to create and place the content in',
|
|
67
|
+
'datatype': str,
|
|
68
|
+
'required': True,
|
|
69
|
+
},
|
|
70
|
+
'content': {
|
|
71
|
+
'description': 'List of content to place into the output.',
|
|
72
|
+
'datatype': list,
|
|
73
|
+
'required': True,
|
|
74
|
+
},
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
def validate_config_schema(self):
|
|
78
|
+
"""Verifies that the config given for this outputter meets the requirements of its schema.
|
|
79
|
+
This function adds additional checks on top of the base validation provided by the
|
|
80
|
+
SchemaValidator super class.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
super().validate_config_schema()
|
|
84
|
+
|
|
85
|
+
content_configs = self.config.get('content', [])
|
|
86
|
+
for content_config in content_configs:
|
|
87
|
+
SchemaValidator.validate_document_section_reference(section_ref=content_config)
|
|
88
|
+
|
|
89
|
+
def output(self):
|
|
90
|
+
"""Produce the SQLite files."""
|
|
91
|
+
|
|
92
|
+
# Pull the filename from the config. We know this exists because validate_config_schema
|
|
93
|
+
# ensures it
|
|
94
|
+
sqlite_filename = self.config.get('filename')
|
|
95
|
+
|
|
96
|
+
# Iterate over content definitions to create the data in the SQLite table.
|
|
97
|
+
content_configs = self.config.get('content', [])
|
|
98
|
+
|
|
99
|
+
with sqlite3.connect(sqlite_filename) as sqlite_conn:
|
|
100
|
+
cursor = sqlite_conn.cursor()
|
|
101
|
+
# Ensure the table exists
|
|
102
|
+
cursor.execute(self.__query_create_table)
|
|
103
|
+
|
|
104
|
+
for content_config in content_configs:
|
|
105
|
+
# Localize these config options for convenience, since we'll need them throughout this code
|
|
106
|
+
section_id = content_config.get('section_id')
|
|
107
|
+
section = self.sections.get(section_id)
|
|
108
|
+
if not section:
|
|
109
|
+
raise ValueError(f'No such section_id: {section_id}')
|
|
110
|
+
data_key = content_config.get('data')
|
|
111
|
+
data = str(section.data.get(data_key))
|
|
112
|
+
if not data:
|
|
113
|
+
raise ValueError(f'Section {section_id} does not have a {data_key} extractor')
|
|
114
|
+
|
|
115
|
+
# Determine if a record exists with this section ID
|
|
116
|
+
cursor.execute(
|
|
117
|
+
self.__querytpl_get_section,
|
|
118
|
+
{'section_id': section_id},
|
|
119
|
+
)
|
|
120
|
+
# If it doesn't exist, insert it
|
|
121
|
+
if len(cursor.fetchall()) == 0:
|
|
122
|
+
cursor.execute(
|
|
123
|
+
self.__querytpl_insert_section,
|
|
124
|
+
{'section_id': section_id, 'data': data},
|
|
125
|
+
)
|
|
126
|
+
else:
|
|
127
|
+
# If it does exist, update it
|
|
128
|
+
cursor.execute(
|
|
129
|
+
self.__querytpl_update_section,
|
|
130
|
+
{'section_id': section_id, 'data': data},
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
sqlite_conn.commit()
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: escrape-plan
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Tool for extracting data from HTML documents
|
|
5
|
+
Author: a.p.p
|
|
6
|
+
Author-email: a.p.p <a.phlaming.phoenix@gmail.com>
|
|
7
|
+
Requires-Dist: beautifulsoup4
|
|
8
|
+
Requires-Dist: lxml
|
|
9
|
+
Requires-Dist: pandoc
|
|
10
|
+
Requires-Dist: pyyaml
|
|
11
|
+
Requires-Dist: bpython>=0.26,<1 ; extra == 'dev'
|
|
12
|
+
Requires-Dist: ruff>=0.17.3,<1 ; extra == 'dev'
|
|
13
|
+
Requires-Dist: twine>=7.0.0,<8 ; extra == 'dev'
|
|
14
|
+
Requires-Python: >=3.14
|
|
15
|
+
Project-URL: homepage, https://gitlab.com/a.phlaming.phoenix/escrape-plan
|
|
16
|
+
Project-URL: repository, https://gitlab.com/a.phlaming.phoenix/escrape-plan
|
|
17
|
+
Provides-Extra: dev
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# eScrape Plan
|
|
21
|
+
|
|
22
|
+
A tool for extracting data from HTML pages in complex and extensible ways according to plans crafted to describe that data.
|
|
23
|
+
|
|
24
|
+
eScrape Plan starts at the top of an HTML document and compares every tag in a "flattened" sequence against a series of **markers**, which are HTML tags you expect to find. The content between those markers are stored internally as **sections** in a **document**. Each section can then be acted on by **extractors**, which pull data out of the HTML and store them. Further, after a document has been fully parsed and all of this data has been extracted, the data can be run through **outputters** to produce artifacts from the data.
|
|
25
|
+
|
|
26
|
+
Today, eScrape Plan can do things like:
|
|
27
|
+
|
|
28
|
+
- Parse a long document into smaller sections
|
|
29
|
+
- Categorize those sections internally by name/ID
|
|
30
|
+
- Run find/replace operations on that content (to do things like normalize newline characters in the source)
|
|
31
|
+
- Reorganize this content arbitrarily and produce new HTML files with any arrangement of that content
|
|
32
|
+
- Export textual/HTML content into SQLite databases for later querying
|
|
33
|
+
- Convert HTML `<table>` content into SQLite databases
|
|
34
|
+
|
|
35
|
+
> [!NOTE]
|
|
36
|
+
> **A Note from a.p.p on LLM Contributions**
|
|
37
|
+
>
|
|
38
|
+
> I would prefer not to use LLMs to generate code for this project for a variety of reasons that are not appropriate to list in detail here. That said, unfortunately, LLM usage is becoming unavoidable for me in my daily job developing software due to external pressures, and so it does behoove me to experiment with this technology.
|
|
39
|
+
>
|
|
40
|
+
> At this point, I remain unconvinced of the value of LLM generated code, but I have found some mixed value in using LLMs for code review. I will often have an LLM form merge requests and post code review comments for me. In these cases, there should always be an attribution to the bot. I only use open weight models running on my own hardware.
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
## Usage
|
|
44
|
+
|
|
45
|
+
I'll have [better documentation](https://gitlab.com/a.phlaming.phoenix/escrape-plan/-/work_items/11) on before too much longer. Until then...
|
|
46
|
+
|
|
47
|
+
```
|
|
48
|
+
pip install escrape-plan
|
|
49
|
+
```
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
escrape_plan/__init__.py,sha256=KN7abEZBncrysOlZl39ywFUdworAsd4goFvpqigeZZ4,20004
|
|
2
|
+
escrape_plan/extractors/__init__.py,sha256=u4EEPf6BG_BngyzgJixOrrK0OF1K1-i2BrBnpBL9FJ4,1746
|
|
3
|
+
escrape_plan/extractors/base.py,sha256=vCWofAcnVPhq5jI7nzYnz67e6okpjVZkPzEOTwZYE-0,1662
|
|
4
|
+
escrape_plan/extractors/read_html.py,sha256=YcKjO4WcRmZ1_GbNWzoDYxZPNhU2IYXFqpM0mF5lb84,3858
|
|
5
|
+
escrape_plan/extractors/sqlite_table.py,sha256=a9-Vx1bTzA1zH4Z4ovGtQCHuxqU5rhTnLJsp8rkREzo,8929
|
|
6
|
+
escrape_plan/outputters/__init__.py,sha256=xLX-eEuBvZg1fa2Z3aIF3tI6VIE2LxvUuRhla7TuM9E,1184
|
|
7
|
+
escrape_plan/outputters/base.py,sha256=Z7M2WXYrMeSq7_F1dDMzTrjoA6C_1-RhT7rQaOZFAA0,1062
|
|
8
|
+
escrape_plan/outputters/text_file.py,sha256=GHapPKjHAV7O2Fxjz08_qdyZ9LuhpqnccFbS2-b_VRQ,3825
|
|
9
|
+
escrape_plan/outputters/text_to_sqlite.py,sha256=92cI1cW7cNSO57BRIOsmxczu4lDyN5B9f-5669DD7Us,5537
|
|
10
|
+
escrape_plan-0.0.1.dist-info/WHEEL,sha256=MWx7psoF7KNST9G72pGtiBYuaXRXQXBk9pw5R6FzYko,80
|
|
11
|
+
escrape_plan-0.0.1.dist-info/METADATA,sha256=zOFfQTz8g71PZD0J_mcHRvpZzUU0Vbu1bTMvVY8MqD0,2725
|
|
12
|
+
escrape_plan-0.0.1.dist-info/RECORD,,
|