escrape-plan 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,472 @@
1
+ """
2
+ Module for scraping data out of HTML documents.
3
+ """
4
+
5
+ import re
6
+ from collections import OrderedDict
7
+ from functools import cached_property
8
+ from types import NoneType
9
+
10
+ import bs4
11
+ import yaml
12
+ from yaml.loader import SafeLoader
13
+
14
+ KNOWN_OUTPUTS = ['text_file', 'text_to_sqlite']
15
+
16
+
17
+ def load_plan(file: str):
18
+ """Loads a plan up from disk."""
19
+
20
+ with open(file, 'r') as fh:
21
+ return yaml.load(fh, Loader=SafeLoader)
22
+
23
+
24
+ class SchemaValidator:
25
+ """A class that lets us define a config schema and then run validation checks against it."""
26
+
27
+ def __init__(self, config: dict):
28
+ self.conforming_config = config
29
+
30
+ @cached_property
31
+ def config_schema(self) -> dict:
32
+ """Must be implemented in subclasses. Must return a dict indicating the schema of the
33
+ configuration options for this Extractor.
34
+
35
+ :return: The Extractor config schema definition.
36
+ :rtype: dict
37
+ """
38
+ raise NotImplementedError()
39
+
40
+ @cached_property
41
+ def config_schema_documentation(self) -> str:
42
+ docstr = (
43
+ f'The {self.__class__.__name__} can be configured using the following parameters:\n\n'
44
+ )
45
+
46
+ for field, properties in self.config_schema.items():
47
+ docstr += f' {field}:\n'
48
+ docstr += f' {properties["description"]}\n'
49
+ docstr += f' - Data Type: {properties["datatype"].__name__}\n'
50
+ docstr += f' - Required: {properties["required"]!s}\n'
51
+
52
+ if 'regex' in properties:
53
+ docstr += f' - Must match pattern: {properties["regex"]}\n'
54
+
55
+ docstr += '\n'
56
+
57
+ return docstr
58
+
59
+ def validate_config_schema(self):
60
+ """Verifies that the config given for this extractor meets the requirements of its schema.
61
+
62
+ :raises ValueError: Some aspect of the config does not align with the schema.
63
+ """
64
+
65
+ errors = {}
66
+
67
+ def add_error(field, err):
68
+ if field not in errors:
69
+ errors[field] = []
70
+ errors[field].append(err)
71
+
72
+ for field_name, field_value in self.conforming_config.items():
73
+ field_schema = self.config_schema.get(field_name)
74
+
75
+ # Determine if the field is defined at all
76
+ if field_schema is None:
77
+ add_error(field_name, f'Unknown field {field_name}')
78
+ else:
79
+ # Determine if the field's value is the right type
80
+ value_type = type(field_value)
81
+ expected_type = field_schema.get('datatype')
82
+ if not isinstance(field_value, expected_type) and value_type is not None:
83
+ add_error(
84
+ field_name,
85
+ f'Value of field {field_name} must be of type {expected_type}, but got type '
86
+ f' {value_type} instead',
87
+ )
88
+
89
+ # Determine if it matches the regex if one is given
90
+ regex_str = field_schema.get('regex')
91
+ if regex_str is not None:
92
+ if type(regex_str) is not str:
93
+ add_error(
94
+ field_name,
95
+ f'"regex" value must be a string, instead got {type(regex_str)}',
96
+ )
97
+ else:
98
+ regex = re.compile(regex_str)
99
+ if type(field_value) is not str:
100
+ add_error(field_name, f'Value {field_value} must be a string')
101
+ elif not regex.match(field_value):
102
+ add_error(
103
+ field_name,
104
+ f'Value for field {field_name} does not match pattern {regex_str}',
105
+ )
106
+
107
+ # Determine if all required fields are present
108
+ required_fields = [
109
+ name for name, value in self.config_schema.items() if value.get('required') is True
110
+ ]
111
+ missing_fields = []
112
+ for field in required_fields:
113
+ if field not in self.conforming_config:
114
+ missing_fields.append(field)
115
+ if len(missing_fields) > 0:
116
+ add_error(
117
+ 'missing_fields', f'Extractor config missing required fields: {missing_fields}'
118
+ )
119
+
120
+ if len(errors) > 0:
121
+ raise ValueError(f'Extractor configuration failed validation. Errors: {errors}')
122
+
123
+ @staticmethod
124
+ def validate_document_section_reference(section_ref: dict) -> bool:
125
+ """Validates a reference to a configuration object that references a document section."""
126
+
127
+ # Fields which *must* be present in a section reference.
128
+ required_fields = ['type', 'section_id', 'data']
129
+
130
+ # Make sure we're not missing any required fields
131
+ missing_fields = [field for field in required_fields if field not in section_ref]
132
+ if len(missing_fields) > 0:
133
+ raise ValueError(
134
+ f'Content config is missing the following required fields: {missing_fields}'
135
+ )
136
+
137
+ # Make sure the type is valid
138
+ content_type = section_ref.get('type')
139
+ if content_type != 'section':
140
+ raise ValueError(f'Content type is "{content_type}"; must be "section".')
141
+
142
+ # Validate section configs
143
+ section_ref_required_fields = ['section_id', 'data']
144
+ missing_section_ref_fields = [
145
+ field for field in section_ref_required_fields if field not in section_ref
146
+ ]
147
+ if len(missing_section_ref_fields) > 0:
148
+ raise ValueError(
149
+ 'Section config is missing the following required fields: '
150
+ f'{missing_section_ref_fields}.'
151
+ )
152
+
153
+ @staticmethod
154
+ def validate_sqlite_table_schema(schema_ref: dict) -> bool:
155
+ """Validates a configuration object that defines a SQLite table schema."""
156
+
157
+ # Info about what we support
158
+ required_fields = ['datatype', 'field']
159
+ optional_fields = ['unique', 'nullable', 'primary_key']
160
+ supported_fields = required_fields + optional_fields
161
+ supported_datatypes = ['TEXT', 'INTEGER', 'REAL']
162
+
163
+ # Make sure we're not missing any required fields
164
+ missing_fields = [field for field in required_fields if field not in schema_ref]
165
+ if len(missing_fields) > 0:
166
+ raise ValueError(
167
+ f'SQLite schema config is missing the following required fields: {missing_fields}'
168
+ )
169
+
170
+ # Detect unknown fields
171
+ unknown_fields = [field for field in schema_ref if field not in supported_fields]
172
+ if len(unknown_fields) > 0:
173
+ raise ValueError(f'SQLite schema config contains unknown fields: {unknown_fields}')
174
+
175
+ # Pull the fields and test their types
176
+ type_datatype = type(schema_ref.get('datatype'))
177
+ type_field = type(schema_ref.get('field'))
178
+ type_unique = type(schema_ref.get('unique'))
179
+ type_nullable = type(schema_ref.get('nullable'))
180
+ type_primary_key = type(schema_ref.get('primary_key'))
181
+
182
+ # datatype and field are required and must be strings
183
+ if type_datatype is not str:
184
+ raise ValueError(f'Field "datatype" must be a string, but is {type_datatype}')
185
+ if type_field is not str:
186
+ raise ValueError(f'Field "field" must be a string, but is {type_field}')
187
+ # These fields must be booleans, but they are not required, so we allow Nones
188
+ if type_unique is not bool and type_unique is not NoneType:
189
+ raise ValueError(f'Field "unique" must be a boolean, but is {type_unique}')
190
+ if type_nullable is not bool and type_nullable is not NoneType:
191
+ raise ValueError(f'Field "nullable" must be a boolean, but is {type_nullable}')
192
+ if type_primary_key is not bool and type_primary_key is not NoneType:
193
+ raise ValueError(f'Field "primary_key" must be a boolean, but is {type_primary_key}')
194
+
195
+ # Make sure the datatype field is valid
196
+ datatype = str(schema_ref.get('datatype')).upper()
197
+ if datatype not in supported_datatypes:
198
+ raise ValueError(
199
+ f'SQLite datatype {datatype} not supported, must be one of {supported_datatypes}'
200
+ )
201
+
202
+ return True
203
+
204
+
205
+ class Marker:
206
+ """A Marker is a part of the document we expect to find. It identifies the beginning of a
207
+ section in the HTML document which we want to extract data from. Markers have at least one
208
+ extraction step that stores the data discovered after this marker and before the next one.
209
+
210
+ :param tag: The kind of HTML tag we're looking for, such as "p" or "h1".
211
+ :type tag: str
212
+
213
+ :param text: The inner text of the tag we're looking for.
214
+ :type text: str
215
+
216
+ :param id: The ID of the marker, which will become the ID of the Section it indicates.
217
+ :type id: str
218
+
219
+ :param extract: List of extraction configurations.
220
+ :type extract: list[dict]
221
+ """
222
+
223
+ def __init__(self, tag: str, text: str, id: str, extract: list[dict]):
224
+ self.tag = tag
225
+ self.text = text
226
+ self.id = id
227
+ self.extractor_configs = extract
228
+ self.matched_element = None
229
+
230
+ def match(self, element: bs4.element.Tag) -> bool:
231
+ if element.name == self.tag and element.text == self.text:
232
+ self.matched_element = element
233
+ return True
234
+ return False
235
+
236
+ @staticmethod
237
+ def config_is_valid(marker_config: dict) -> tuple[bool, list[str]]:
238
+ """Determines if the given Marker config dict is valid.
239
+
240
+ :param marker_config: A Marker config in dict form which might be parameterized into a
241
+ constructor call.
242
+ :type marker_config: dict
243
+
244
+ :return: A tuple of the form ``(is_valid, missing_fields)``. ``is_valid`` is a bool
245
+ indicating the validity of the given Marker config dict and its ability to be
246
+ parameterized into a constructor call. ``missing_fields`` is a list of required config
247
+ keys which must be present but are not. When ``is_valid`` is ``True``, this is an empty
248
+ list (``[]``).
249
+ :rtype: tuple[bool, list[str]]
250
+ """
251
+
252
+ required_fields = ['tag', 'text', 'id', 'extract']
253
+ missing_fields = [field for field in required_fields if field not in marker_config]
254
+ is_valid = len(missing_fields) == 0
255
+ return (is_valid, missing_fields)
256
+
257
+
258
+ class Section:
259
+ """A Section is a piece of a document, some amount of content scraped from the HTML. It has an
260
+ ID, uniquely identifying this portion of content. It also has data, which is the output of the
261
+ extractors run on the content.
262
+
263
+ :param id: The unique ID of the Section.
264
+ :type id: str
265
+
266
+ :param data: The data extracted from this section of the document. Keys are the type of
267
+ extractor used to acquire the data. Values are whatever those extractors produce in whatever
268
+ format they produce it.
269
+ """
270
+
271
+ def __init__(self, id: str, data: dict | None = None):
272
+ self.id = id
273
+ self.data = data or {}
274
+
275
+
276
+ class Document:
277
+ """A Document is an internal comprehension of the HTML document being scraped based on the plan
278
+ designed for that document. The plan is a series of Markers which have data extraction rules.
279
+
280
+ :param src: The actual HTML of the document we're extracting from.
281
+ :type src: str
282
+
283
+ :param title: The title of the overall document.
284
+ :type title: str
285
+
286
+ :param plan: A series of Marker configurations -- things you expect to find in the document and
287
+ how you want to deal with them. Must have at least one marker defined.
288
+ :type plan: list[dict]
289
+ """
290
+
291
+ def __init__(self, src: str, title: str, plan: list[dict], outputs: list[dict] = []):
292
+ self.src = src
293
+ self.soup = bs4.BeautifulSoup(self.src, 'lxml')
294
+ self.title = title
295
+ self.sections = None
296
+ self.output_configs = outputs
297
+
298
+ # Validate plan length
299
+ if len(plan) == 0:
300
+ raise ValueError('Plan must have at least one marker in it')
301
+
302
+ self.plan = plan
303
+
304
+ def scrape(self):
305
+ """Step through the plan trying to extract data from its source accordingly. This function
306
+ populates the ``sections`` member."""
307
+
308
+ # --- Set up the scrape ---
309
+
310
+ # Start with an empty set of sections
311
+ self.sections = OrderedDict()
312
+
313
+ # The "current" marker is the one that heads up the section we're currently tracking
314
+ current_marker = None
315
+
316
+ # Common function to iterate on markers in the plan; returns None when iteration is done.
317
+ def __next_marker(idx: int, plan: list[dict]) -> tuple[int, Marker]:
318
+ """Determines the next marker in the plan.
319
+
320
+ :param idx: The current plan marker's index.
321
+ :type idx: int
322
+
323
+ :param plan: This Document's plan, passed in so we have it in scope for reading in this
324
+ function.
325
+ :type plan: list[dict]
326
+
327
+ :return: Tuple of two values: the index of the next marker configuration in this
328
+ Document's plan and the Marker object created from that configuration.
329
+ :rtype: tuple[int, Marker]
330
+ """
331
+
332
+ # Increment the marker index
333
+ idx += 1
334
+
335
+ # If the index is within bounds, create a Marker object from the config
336
+ if idx < len(plan):
337
+ marker_config_is_valid, missing_fields = Marker.config_is_valid(plan[idx])
338
+ if marker_config_is_valid:
339
+ marker = Marker(**plan[idx])
340
+ else:
341
+ raise ValueError(
342
+ f'Marker {plan[idx].get("id", "<no_id>")} is missing required configuration parameters:'
343
+ f' {", ".join(missing_fields)}'
344
+ )
345
+
346
+ # If it's out of bounds, the marker is None
347
+ else:
348
+ marker = None
349
+
350
+ # Return the incremented index and corresponding Marker
351
+ return idx, marker
352
+
353
+ # Common function to run extractors on a marker/section
354
+ def __extract(current_marker: Marker) -> Section:
355
+ """Given the current Marker (the HTML tag we're currently looking for), creates all the
356
+ defined Extractors and tells them to extract data from the current set of elements we've
357
+ identified as belonging to the current section.
358
+
359
+ :param current_marker: The Marker we're currently parsing data for.
360
+ :type current_marker: Marker
361
+
362
+ :return: A proper Section object with the extracted data included.
363
+ :rtype: Section
364
+ """
365
+
366
+ # Create a section, copying in the ID of the current marker
367
+ section = Section(id=current_marker.id)
368
+
369
+ # Run each defined extractor and store its data in the Section's data dictionary
370
+ from .extractors import construct_extractor
371
+
372
+ for extractor in [
373
+ construct_extractor(
374
+ config=extractor_config,
375
+ src=current_section_elements,
376
+ section_id=current_marker.id,
377
+ )
378
+ for extractor_config in current_marker.extractor_configs
379
+ ]:
380
+ # Use the extractor type as the section's data key. While we should be able
381
+ # to use multiple extractors on the same set of elements, we should *not*
382
+ # have multiple extractors of the *same type*.
383
+ section.data[extractor.type] = extractor.extract(current_marker)
384
+
385
+ return section
386
+
387
+ # The "next" marker is the marker we expect to find next in the soup. To start, that's the
388
+ # first marker (zeroth index) in the list.
389
+ marker_idx = 0
390
+ next_marker = Marker(**self.plan[marker_idx])
391
+
392
+ # We begin with no section; when we find our first marker, we will start a new section
393
+ current_section_id = None
394
+
395
+ # Track the elements since the last time we found a marker
396
+ current_section_elements = []
397
+
398
+ # --- Document iteration ---
399
+
400
+ # Go through every element in the soup, checking each one against the current marker until
401
+ # we get a match. At the beginning, you will get things like an <html> tag and a <body> tag
402
+ # that contain the entire document, and which are pretty much useless. Because these
403
+ # elements won't match a sensible plan, these elements are naturally scrapped.
404
+ for element in self.soup.find_all():
405
+ # If the element is a match for the marker we expect next...
406
+ if next_marker:
407
+ if next_marker.match(element):
408
+ # If a section ID is set, that means we've already matched an element and started a
409
+ # section. We need to form a proper Section object and stash it in this Document.
410
+ if current_section_id:
411
+ # Run extractors on the current data set
412
+ self.sections[current_section_id] = __extract(current_marker=current_marker)
413
+
414
+ # Set the current marker to the thing we just found
415
+ current_marker = next_marker
416
+
417
+ # Set the current section ID to the now-current marker's ID
418
+ current_section_id = current_marker.id
419
+
420
+ # Iterate to the next marker
421
+ marker_idx, next_marker = __next_marker(marker_idx, self.plan)
422
+ # Clear out the elements we've been tracking
423
+ current_section_elements = []
424
+
425
+ # If a section ID is *not* set, that means we have yet to match our first
426
+ # marker. Therefore, *this* is our first marker and we should start tracking the
427
+ # contents of this section.
428
+ else:
429
+ # Set the current marker to the thing we just found
430
+ current_marker = next_marker
431
+
432
+ # Set the current section ID to the now-current marker's ID
433
+ current_section_id = current_marker.id
434
+
435
+ # Iterate to the next marker
436
+ marker_idx, next_marker = __next_marker(marker_idx, self.plan)
437
+
438
+ # Clear out the elements we've been tracking
439
+ current_section_elements = []
440
+
441
+ # If we are in a section but this is not a match for the next marker...
442
+ else:
443
+ # ...just tack it on to the current list of elements.
444
+ current_section_elements.append(element)
445
+ # If there is no next marker, then we are in our final section...
446
+ else:
447
+ # ...so just tack it on to the current list of elements.
448
+ current_section_elements.append(element)
449
+
450
+ # After iterating over everything, we need to do one final extraction, but only if we ever
451
+ # actually matched something.
452
+ if current_section_id:
453
+ self.sections[current_section_id] = __extract(current_marker=current_marker)
454
+
455
+ def output(self):
456
+ """Runs through the document's output configs, attempting to create each output defined."""
457
+
458
+ for output_config in self.output_configs:
459
+ output_id = output_config.get('id')
460
+ if not output_id:
461
+ raise ValueError('Output does not have an `id` field.')
462
+
463
+ output_type = output_config.get('type')
464
+ if not output_type:
465
+ raise ValueError(f'Output "{output_id}" must specify a "type".')
466
+ if output_type not in KNOWN_OUTPUTS:
467
+ raise ValueError(f'Output "{output_id}" has an unknown output type: {output_type}.')
468
+
469
+ from .outputters import construct_outputter
470
+
471
+ outputter = construct_outputter(sections=self.sections, config=output_config)
472
+ outputter.output()
@@ -0,0 +1,51 @@
1
+ from importlib import import_module
2
+ from typing import TYPE_CHECKING
3
+
4
+ if TYPE_CHECKING:
5
+ from bs4.element import Tag
6
+
7
+ from .base import Extractor
8
+
9
+ #: Mapping of strings (names of extractors) to the importable paths of the Extractor implementation
10
+ # classes they refer to. We use strings here and importlib later to avoid circular dependencies.
11
+ EXTRACTOR_TYPES = {
12
+ 'read_html': {
13
+ 'package': 'read_html',
14
+ 'class': 'ReadHTMLExtractor',
15
+ },
16
+ 'sqlite_table': {
17
+ 'package': 'sqlite_table',
18
+ 'class': 'SQLiteTableExtractor',
19
+ },
20
+ }
21
+
22
+
23
+ def construct_extractor(config: dict, src: list[Tag], section_id: str) -> Extractor:
24
+ """Given a configuration for an Extractor, constructs the correct extractor sub-type class.
25
+
26
+ :param config: Configuration for the extractor sub-type.
27
+ :type config: dict
28
+
29
+ :param src: The HTML tags that are a part of this Section of the Document.
30
+ :type src: bs4.element.Tag
31
+
32
+ :param section_id: The ID of this Section.
33
+ :type src: str
34
+
35
+ :raises KeyError: The config does not contain a "type" field.
36
+
37
+ :raises ValueError: The config contains an unknown "type" value.
38
+
39
+ :return: Fully constructed Extractor sub-type class.
40
+ :rtype: Extractor
41
+ """
42
+
43
+ extractor_type = config.get('type')
44
+ if not extractor_type:
45
+ raise KeyError('Field "type" is required in extractor configs')
46
+ if extractor_type not in EXTRACTOR_TYPES:
47
+ raise ValueError(f'Unknown extractor type: {extractor_type}')
48
+ __type = EXTRACTOR_TYPES[extractor_type]
49
+ __module = import_module(f'escrape_plan.extractors.{__type["package"]}')
50
+ __extractor = getattr(__module, __type['class'])
51
+ return __extractor(config=config, src=src, section_id=section_id)
@@ -0,0 +1,48 @@
1
+ import typing
2
+ from typing import TYPE_CHECKING
3
+
4
+ from bs4.element import Tag
5
+
6
+ from .. import SchemaValidator
7
+
8
+ if TYPE_CHECKING:
9
+ from . import Marker
10
+
11
+
12
+ class Extractor(SchemaValidator):
13
+ """Base class for all extractors to use. Validates configurations against a defined schema.
14
+ Requires that subclasses implement the ``extract`` function and the ``config_schema`` property.
15
+
16
+ :param src: The HTML elements (already parsed by BeautifulSoup) we're extracting data from.
17
+ :type src: list[Tag]
18
+
19
+ :param config: The extraction configuration. This includes the ``type`` of config, which
20
+ determines how we validate the rest of the config.
21
+ :type config: dict
22
+
23
+ :param section_id: The ID of the section this extractor is parsing data from.
24
+ :type section_id: str
25
+ """
26
+
27
+ def __init__(self, src: list[Tag], config: dict, section_id: str):
28
+ super().__init__(config=config)
29
+ self.src = src
30
+ self.config = config
31
+ self.section_id = section_id
32
+ self.type = None
33
+ self.validate_config_schema()
34
+
35
+ def extract(self, marker: Marker) -> typing.Any:
36
+ """Must be implemented in subclasses. Extracts information from the ``src`` string however
37
+ it's necessary to. Returns any value, however the content should be referenced; this will
38
+ get stored in the document according to its plan.
39
+
40
+ :param marker: The BeautifulSoup Tag representing the marker that matched at the beginning
41
+ of this section.
42
+ :type marker: Marker
43
+
44
+ :return: The data extracted by this Extractor.
45
+ :rtype: typing.Any
46
+ """
47
+
48
+ raise NotImplementedError()
@@ -0,0 +1,107 @@
1
+ from functools import cached_property
2
+
3
+ from bs4.element import Tag
4
+
5
+ from escrape_plan import Marker
6
+
7
+ from .base import Extractor
8
+
9
+ KNOWN_TRANSFORMS = ['replace']
10
+
11
+
12
+ class ReadHTMLExtractor(Extractor):
13
+ """The "read_html" extractor, which converts the HTML elements found in a Document into strings
14
+ of valid HTML snippets.
15
+
16
+ :param src: The HTML elements (already parsed by BeautifulSoup) we're extracting data from.
17
+ :type src: list[Tag]
18
+
19
+ :param config: The extraction configuration. Must comply with the :py:meth:`.config_schema`.
20
+ :type config: dict
21
+
22
+ :param section_id: The ID of this Section of the Document.
23
+ :type section_id: str
24
+ """
25
+
26
+ def __init__(self, src: list[Tag], config: dict, section_id: str):
27
+ # Automatically handles validation against the config_schema
28
+ super().__init__(src, config, section_id)
29
+ self.type = 'read_html'
30
+
31
+ @cached_property
32
+ def config_schema(self) -> dict:
33
+ """Schema definition for ReadHTMLExtractors.
34
+
35
+ :return: dict of field value constraints.
36
+ :rtype: dict
37
+ """
38
+ return {
39
+ 'type': {
40
+ 'description': 'Kind of extractor (must be "read_html")',
41
+ 'datatype': str,
42
+ 'required': True,
43
+ 'regex': '^read_html$',
44
+ },
45
+ 'title': {
46
+ 'description': 'Title of the section of the document this data is extracted to',
47
+ 'datatype': str,
48
+ 'required': True,
49
+ },
50
+ 'keep_marker': {
51
+ 'description': 'When True, keeps the matched Marker tag in the extracted data',
52
+ 'datatype': bool,
53
+ 'required': False,
54
+ },
55
+ 'transforms': {
56
+ 'description': 'List of transformations to apply to the content',
57
+ 'datatype': list,
58
+ 'required': False,
59
+ },
60
+ }
61
+
62
+ def extract(self, marker: Marker) -> str:
63
+ """Returns the HTML that was given to the extractor in the constructor, but with any
64
+ defined transformations applied.
65
+
66
+ :param marker: The Marker representing the tag that matched at the beginning of this
67
+ section.
68
+ :type marker: Marker
69
+
70
+ :return: The HTML provided in the src.
71
+ :rtype: str
72
+ """
73
+
74
+ # Convert each tag to a string and concatenate them all
75
+ extracted_str = ''.join([str(tag) for tag in self.src])
76
+
77
+ keep_marker = self.config.get('keep_marker', True)
78
+ if keep_marker and marker.matched_element:
79
+ extracted_str = str(marker.matched_element) + extracted_str
80
+
81
+ # Apply transformations
82
+ transforms = self.config.get('transforms')
83
+ if transforms:
84
+ for transform in transforms:
85
+ # Determine and validate transform type
86
+ transform_type = transform.get('type')
87
+ if not transform_type:
88
+ raise ValueError('Transform must have a "type" field.')
89
+
90
+ if transform_type not in KNOWN_TRANSFORMS:
91
+ raise ValueError(
92
+ f'Unknown transform type: {transform_type}. Must be one of'
93
+ f' {", ".join(KNOWN_TRANSFORMS)}'
94
+ )
95
+
96
+ if transform_type == 'replace':
97
+ find = transform.get('find')
98
+ replacement = transform.get('replacement')
99
+ if find and replacement and len(find) > 0:
100
+ extracted_str = extracted_str.replace(find, replacement)
101
+ else:
102
+ raise ValueError(
103
+ '"find" and "replacement" are both required fields for transform type '
104
+ 'replace.'
105
+ )
106
+
107
+ return extracted_str
@@ -0,0 +1,205 @@
1
+ import sqlite3
2
+ from functools import cached_property
3
+
4
+ from bs4 import BeautifulSoup
5
+ from bs4.element import Tag
6
+
7
+ from escrape_plan import Marker
8
+
9
+ from .base import Extractor, SchemaValidator
10
+
11
+
12
+ class SQLiteTableExtractor(Extractor):
13
+ """
14
+ The "sqlite_table" extractor, which reads data from an HTML table and converts it into a schema
15
+ and a set of records in a SQLite file.
16
+
17
+ This extractor is inherently destructive. While you can specify a SQLite database file that has
18
+ other content in it, if this module finds a table in that database by the same name you've
19
+ specified in the config, **it will destroy every row in that table and replace it with data
20
+ found by this extractor.**
21
+ """
22
+
23
+ def __init__(self, src: list[Tag], config: dict, section_id: str):
24
+ super().__init__(src, config, section_id)
25
+ self.type = 'sqlite_table'
26
+
27
+ # Localize some variables from the config
28
+ self.sqlite_filename = config.get('filename')
29
+ self.table_name = config.get('table')
30
+ self.schema = self.config.get('schema')
31
+
32
+ # Get all table tags in the src
33
+ all_table_tags = [tag for tag in src if tag.name == 'table']
34
+
35
+ # Check that there is actually a table in these tags
36
+ if len(all_table_tags) < 1:
37
+ raise ValueError(f'There are no tables in section {self.section_id}')
38
+
39
+ # Also check that no more than one table occur in these tags
40
+ if len(all_table_tags) > 1:
41
+ raise ValueError(f'More than one table in section {self.section_id}')
42
+
43
+ # Now we know there's one index in this list. This one <table> tag contains everything we
44
+ # need to parse through. Here we localize that one tag and re-soup it so we don't overparse.
45
+ self.table_soup = BeautifulSoup(str(all_table_tags[0]), 'lxml')
46
+
47
+ # Now check our schema config. We know it exists because it's a required field that's been
48
+ # validated in the superconstructor, but now we need to do validity checks on this data.
49
+ if len(self.schema) == 0:
50
+ raise ValueError('The configured schema is empty. Provide at least one field.')
51
+
52
+ # Run each schema entry through the special validator
53
+ for schema_ref in self.schema:
54
+ SchemaValidator.validate_sqlite_table_schema(schema_ref=schema_ref)
55
+
56
+ # Make sure we don't have more than one primary key
57
+ pk_fields = [schema_ref for schema_ref in self.schema if schema_ref.get('primary_key')]
58
+ if len(pk_fields) > 1:
59
+ raise ValueError('The schema cannot have more than one primary key.')
60
+
61
+ # The config all appears to be valid at this point, so we need to set up the database
62
+ # Here are some queries we'll use throughout this class.
63
+
64
+ # Construct the SQLite table schema from our config
65
+ table_schema_entries = []
66
+ for field in self.schema:
67
+ entry = f'"{field.get("field")}" {field.get("datatype")}'
68
+ if field.get('primary_key', False):
69
+ entry += ' PRIMARY KEY'
70
+ if field.get('unique', False):
71
+ entry += ' UNIQUE'
72
+ if not field.get('nullable', True):
73
+ entry += ' NOT NULL'
74
+ table_schema_entries.append(entry)
75
+ table_schema_str = ', '.join(table_schema_entries)
76
+ self.__query_create_table = f'CREATE TABLE {self.table_name} ({table_schema_str});'
77
+ self.__query_drop_table = f'DROP TABLE {self.table_name};'
78
+ self.__query_detect_table = (
79
+ f'SELECT COUNT(*) FROM sqlite_master WHERE type="table" and name="{self.table_name}";'
80
+ )
81
+ self.__querytpl_insert_row = (
82
+ f'INSERT INTO {self.table_name} '
83
+ f'({", ".join([field.get("field") for field in self.schema])}) '
84
+ f'VALUES ({("?, " * len(self.schema)).strip(", ")});'
85
+ )
86
+ self.__setup_database()
87
+
88
+ def __setup_database(self):
89
+ """Sets up the SQLite table we are dumping this data into"""
90
+
91
+ # Do all of this in a connection context
92
+ with sqlite3.connect(self.sqlite_filename) as sqlite_conn:
93
+ cursor = sqlite_conn.cursor()
94
+
95
+ # Does the table already exist? Get a count of tables by this name.
96
+ cursor.execute(self.__query_detect_table)
97
+ table_ct = cursor.fetchone()[0]
98
+
99
+ # If the table already exists, drop it entirely
100
+ if table_ct > 0:
101
+ cursor.execute(self.__query_drop_table)
102
+
103
+ # Now create the table with the correct schema
104
+ cursor.execute(self.__query_create_table)
105
+
106
+ @cached_property
107
+ def config_schema(self) -> dict:
108
+ return {
109
+ 'type': {
110
+ 'description': 'Kind of extractor (must be "sqlite_table")',
111
+ 'datatype': str,
112
+ 'required': True,
113
+ 'regex': '^sqlite_table$',
114
+ },
115
+ 'filename': {
116
+ 'description': 'SQLite database file location',
117
+ 'datatype': str,
118
+ 'required': True,
119
+ },
120
+ 'table': {
121
+ 'description': 'Name of the table in the SQLite database to store this data',
122
+ 'datatype': str,
123
+ 'required': True,
124
+ },
125
+ 'schema': {
126
+ 'description': 'Object indicating the schema of the table',
127
+ 'datatype': list,
128
+ 'required': True,
129
+ },
130
+ 'ignore_first_row': {
131
+ 'description': 'Whether the first row should be treated as headers and not data',
132
+ 'datatype': bool,
133
+ 'required': False,
134
+ },
135
+ }
136
+
137
+ def extract(self, marker: Marker) -> None:
138
+ """Seeks out <table> data in the HTML source and attempts to convert it into a SQLite
139
+ database file.
140
+ """
141
+
142
+ # The number of columns defined in the schema in our config
143
+ required_column_ct = len(self.schema)
144
+
145
+ # Do we skip the first row?
146
+ ignore_first_row = self.config.get('ignore_first_row', False)
147
+
148
+ # Perform all of the upcoming steps within a connection context
149
+ with sqlite3.connect(self.sqlite_filename) as sqlite_conn:
150
+ cursor = sqlite_conn.cursor()
151
+
152
+ # Grab all the <tr> elements in the table and go through them. Track whether we've
153
+ # parsed the first row.
154
+ first_row_parsed = False
155
+ for row in self.table_soup.find_all(name='tr'):
156
+ # Determine if we need to skip the first row
157
+ if not first_row_parsed:
158
+ first_row_parsed = True
159
+ if ignore_first_row:
160
+ continue
161
+
162
+ # We ignore <th> tags, only caring about <td> tags.
163
+ columns = row.find_all(name='td')
164
+
165
+ # There's a problem if we don't have exactly the same number of table columns as
166
+ # fields in our SQLite schema. This has to be checked for every row because <td>
167
+ # tags can have a `colspan` property that changes the number of columns in the row.
168
+ column_ct = len(columns)
169
+ if column_ct != required_column_ct:
170
+ raise ValueError(
171
+ f'Row has {column_ct} columns, but needs {required_column_ct}.'
172
+ )
173
+
174
+ # Go through the columns, forming a tuple of their type-verified values as we go, so
175
+ # we can later populate the insert query template with them. We have to match the
176
+ # table cells to our schema fields, so track the column index as we go.
177
+ field_idx = -1
178
+ query_values = []
179
+ for column in columns:
180
+ field_idx += 1
181
+ data = column.get_text().strip()
182
+
183
+ # Pull the SQL datatype for this column and try to conform the data to the
184
+ # corresponding Python type. If we can't, raise a ValueError with a better
185
+ # message.
186
+ col_schema = self.schema[field_idx]
187
+ target_sql_type = col_schema.get('datatype').upper()
188
+ target_python_type = {
189
+ 'INTEGER': int,
190
+ 'REAL': float,
191
+ 'TEXT': str,
192
+ }.get(target_sql_type)
193
+ try:
194
+ data = target_python_type(data)
195
+ except ValueError:
196
+ raise ValueError(
197
+ f'Could not conform value {data} to datatype {target_sql_type}'
198
+ )
199
+ query_values.append(data)
200
+
201
+ # Execute the INSERT query
202
+ cursor.execute(self.__querytpl_insert_row, query_values)
203
+
204
+ # Commit the transaction once all the INSERTs have run
205
+ sqlite_conn.commit()
@@ -0,0 +1,33 @@
1
+ from collections import OrderedDict
2
+ from importlib import import_module
3
+ from typing import TYPE_CHECKING
4
+
5
+ from .. import Section
6
+
7
+ if TYPE_CHECKING:
8
+ from .base import Outputter
9
+
10
+ #: Mapping of strings (names of outputters) to the importable paths of the Outputter implementation
11
+ # classes they refer to. We use strings here and ipmortlib later to avoid circular dependencies.
12
+ OUTPUTTER_TYPES = {
13
+ 'text_file': {
14
+ 'package': 'text_file',
15
+ 'class': 'TextFileOutputter',
16
+ },
17
+ 'text_to_sqlite': {
18
+ 'package': 'text_to_sqlite',
19
+ 'class': 'TextToSqliteOutputter',
20
+ },
21
+ }
22
+
23
+
24
+ def construct_outputter(sections: OrderedDict[str, Section], config: dict) -> Outputter:
25
+ outputter_type = config.get('type')
26
+ if not outputter_type:
27
+ raise KeyError('Field "type" is required in outputter configs')
28
+ if outputter_type not in OUTPUTTER_TYPES:
29
+ raise ValueError(f'Unknown outputter type: {outputter_type}')
30
+ __type = OUTPUTTER_TYPES[outputter_type]
31
+ __module = import_module(f'escrape_plan.outputters.{__type["package"]}')
32
+ __outputter = getattr(__module, __type['class'])
33
+ return __outputter(sections=sections, config=config)
@@ -0,0 +1,29 @@
1
+ from .. import SchemaValidator
2
+ from . import Section
3
+
4
+
5
+ class Outputter(SchemaValidator):
6
+ """Base class for all outputters to use. Validates configurations against a defined schema.
7
+ Requires that subclasses implement the ``output`` function and the ``config_schema`` property.
8
+
9
+ :param sections: List of ``Section`` objects, the result of running a plan and extracting data.
10
+ :type sections: list[Section]
11
+
12
+ :param config: The outputter configuration. This includes an ``id`` for the outputter, the
13
+ ``type`` of the outputter, and any other options required for that outputter.
14
+ :type config: dict
15
+ """
16
+
17
+ def __init__(self, sections: list[Section], config: dict):
18
+ super().__init__(config=config)
19
+ self.config = config
20
+ self.sections = sections
21
+ self.type = None
22
+ self.validate_config_schema()
23
+
24
+ def output() -> None:
25
+ """Must be ipmlemented in subclasses. Outputs data from the provided Sections according to
26
+ the given config.
27
+ """
28
+
29
+ raise NotImplementedError()
@@ -0,0 +1,92 @@
1
+ from .base import Outputter, SchemaValidator, Section
2
+ from functools import cached_property
3
+
4
+
5
+ class TextFileOutputter(Outputter):
6
+ """The "text_file" outputter, which places string data extracted from a document into a file on
7
+ disk. Content can be restructured, omitted, etc. depending on the config.
8
+
9
+ :param sections: List of ``Section`` objects containing data to output.
10
+ :type sections: list[Section]
11
+
12
+ :param config: Configuration for outputting that data into text files.
13
+ :type config: dict
14
+ """
15
+
16
+ def __init__(self, sections: list[Section], config: dict):
17
+ super().__init__(sections=sections, config=config)
18
+ self.type = 'text_file'
19
+
20
+ @cached_property
21
+ def config_schema(self) -> dict:
22
+ return {
23
+ 'id': {
24
+ 'description': 'Unique ID for the outputter',
25
+ 'datatype': str,
26
+ 'required': True,
27
+ },
28
+ 'type': {
29
+ 'description': 'Kind of outputter (must be "text_file")',
30
+ 'datatype': str,
31
+ 'required': True,
32
+ 'regex': '^text_file$',
33
+ },
34
+ 'filename': {
35
+ 'description': 'Place on disk to produce the output',
36
+ 'datatype': str,
37
+ 'required': True,
38
+ },
39
+ 'content': {
40
+ 'description': 'List of content to place into the output.',
41
+ 'datatype': list,
42
+ 'required': True,
43
+ },
44
+ }
45
+
46
+ def validate_config_schema(self):
47
+ """Verifies that the config given for this outputter meets the requirements of its schema.
48
+ This function adds additional checks on top of the base validation provided by the
49
+ SchemaValidator super class.
50
+ """
51
+
52
+ super().validate_config_schema()
53
+
54
+ content_configs = self.config.get('content', [])
55
+ for content_config in content_configs:
56
+ SchemaValidator.validate_document_section_reference(section_ref=content_config)
57
+
58
+ def output(self):
59
+ """Produce the actual text files."""
60
+
61
+ # Localize the outputter settings for convenience. We know these are present because
62
+ # validate_config_schema raises an exception if they aren't.
63
+ output_id = self.config.get('id')
64
+ output_type = self.config.get('type')
65
+ filename = self.config.get('filename')
66
+ content_configs = self.config.get('content')
67
+
68
+ # Get a file handler
69
+ with open(filename, 'w') as output_file:
70
+ # Iterate over the content and output each entry
71
+ for content_config in content_configs:
72
+ # Determine what to do based on the type. We know that 'type' is present because we
73
+ # check for it in validate_config_schema.
74
+ content_type = content_config.get('type')
75
+
76
+ # Pull content from a document section.
77
+ if content_type == 'section':
78
+ # We know these fields exist because validate_config_schema makes sure of it.
79
+ section_id = content_config.get('section_id')
80
+ data_id = content_config.get('data')
81
+
82
+ # But we need to make sure that the section and data actually exist
83
+ if section_id not in self.sections:
84
+ raise ValueError(f'Section ID {section_id} does not exist in the document')
85
+ section = self.sections[section_id]
86
+
87
+ if data_id not in section.data:
88
+ raise ValueError(f'Data {data_id} does not exist in section {section_id}')
89
+ data = section.data[data_id]
90
+
91
+ # Make sure that everything we output is actually textual when we write it
92
+ output_file.write(str(data))
@@ -0,0 +1,133 @@
1
+ import sqlite3
2
+ from functools import cached_property
3
+
4
+ from .base import Outputter, SchemaValidator, Section
5
+
6
+
7
+ class TextToSqliteOutputter(Outputter):
8
+ """Produces SQLite database files containing the specified contents. Each section gets inserted
9
+ into the specified table in the specified SQLite file following a very simple string storage
10
+ schema. **This outputter is destructive.** If you point it at a SQLite database that already
11
+ exists, it will overwrite any existing data it finds there. This is done on a per-section basis,
12
+ not a per-table or per-database basis. This can lead to confusing results. It is strongly
13
+ recommended that you use empty or non-existent database files when you run this outputter to
14
+ avoid mixed data, or else be very careful about how you configure your outputters.
15
+
16
+ :param sections: List of ``Section`` objects containing data to output.
17
+ :type sections: list[Section]
18
+
19
+ :param config: Configuration for outputting that data into SQLite files.
20
+ :type config: dict
21
+ """
22
+
23
+ def __init__(self, sections: list[Section], config: dict):
24
+ super().__init__(sections=sections, config=config)
25
+ self.type = 'text_to_sqlite'
26
+
27
+ # Localize some config variables for convenience
28
+ self.table_name = self.config.get('table')
29
+ self.filename = self.config.get('filename')
30
+
31
+ # Define some SQLite queries we'll be using later
32
+ self.__query_create_table = (
33
+ f'CREATE TABLE IF NOT EXISTS {self.table_name} '
34
+ '(section_id TEXT PRIMARY_KEY, data TEXT);'
35
+ )
36
+ self.__querytpl_get_section = (
37
+ f'SELECT * FROM {self.table_name} WHERE section_id=:section_id;'
38
+ )
39
+ self.__querytpl_insert_section = (
40
+ f'INSERT INTO {self.table_name} (section_id, data) VALUES (:section_id, :data);'
41
+ )
42
+ self.__querytpl_update_section = (
43
+ f'UPDATE {self.table_name} SET data = :data WHERE section_id = :section_id;'
44
+ )
45
+
46
+ @cached_property
47
+ def config_schema(self) -> dict:
48
+ return {
49
+ 'id': {
50
+ 'description': 'Unique ID for the outputter',
51
+ 'datatype': str,
52
+ 'required': True,
53
+ },
54
+ 'type': {
55
+ 'description': 'Kind of outputter (must be "text_to_sqlite")',
56
+ 'datatype': str,
57
+ 'required': True,
58
+ 'regex': '^text_to_sqlite$',
59
+ },
60
+ 'filename': {
61
+ 'description': 'SQLite file on disk to place the content',
62
+ 'datatype': str,
63
+ 'required': True,
64
+ },
65
+ 'table': {
66
+ 'description': 'Name of the SQLite table to create and place the content in',
67
+ 'datatype': str,
68
+ 'required': True,
69
+ },
70
+ 'content': {
71
+ 'description': 'List of content to place into the output.',
72
+ 'datatype': list,
73
+ 'required': True,
74
+ },
75
+ }
76
+
77
+ def validate_config_schema(self):
78
+ """Verifies that the config given for this outputter meets the requirements of its schema.
79
+ This function adds additional checks on top of the base validation provided by the
80
+ SchemaValidator super class.
81
+ """
82
+
83
+ super().validate_config_schema()
84
+
85
+ content_configs = self.config.get('content', [])
86
+ for content_config in content_configs:
87
+ SchemaValidator.validate_document_section_reference(section_ref=content_config)
88
+
89
+ def output(self):
90
+ """Produce the SQLite files."""
91
+
92
+ # Pull the filename from the config. We know this exists because validate_config_schema
93
+ # ensures it
94
+ sqlite_filename = self.config.get('filename')
95
+
96
+ # Iterate over content definitions to create the data in the SQLite table.
97
+ content_configs = self.config.get('content', [])
98
+
99
+ with sqlite3.connect(sqlite_filename) as sqlite_conn:
100
+ cursor = sqlite_conn.cursor()
101
+ # Ensure the table exists
102
+ cursor.execute(self.__query_create_table)
103
+
104
+ for content_config in content_configs:
105
+ # Localize these config options for convenience, since we'll need them throughout this code
106
+ section_id = content_config.get('section_id')
107
+ section = self.sections.get(section_id)
108
+ if not section:
109
+ raise ValueError(f'No such section_id: {section_id}')
110
+ data_key = content_config.get('data')
111
+ data = str(section.data.get(data_key))
112
+ if not data:
113
+ raise ValueError(f'Section {section_id} does not have a {data_key} extractor')
114
+
115
+ # Determine if a record exists with this section ID
116
+ cursor.execute(
117
+ self.__querytpl_get_section,
118
+ {'section_id': section_id},
119
+ )
120
+ # If it doesn't exist, insert it
121
+ if len(cursor.fetchall()) == 0:
122
+ cursor.execute(
123
+ self.__querytpl_insert_section,
124
+ {'section_id': section_id, 'data': data},
125
+ )
126
+ else:
127
+ # If it does exist, update it
128
+ cursor.execute(
129
+ self.__querytpl_update_section,
130
+ {'section_id': section_id, 'data': data},
131
+ )
132
+
133
+ sqlite_conn.commit()
@@ -0,0 +1,49 @@
1
+ Metadata-Version: 2.3
2
+ Name: escrape-plan
3
+ Version: 0.0.1
4
+ Summary: Tool for extracting data from HTML documents
5
+ Author: a.p.p
6
+ Author-email: a.p.p <a.phlaming.phoenix@gmail.com>
7
+ Requires-Dist: beautifulsoup4
8
+ Requires-Dist: lxml
9
+ Requires-Dist: pandoc
10
+ Requires-Dist: pyyaml
11
+ Requires-Dist: bpython>=0.26,<1 ; extra == 'dev'
12
+ Requires-Dist: ruff>=0.17.3,<1 ; extra == 'dev'
13
+ Requires-Dist: twine>=7.0.0,<8 ; extra == 'dev'
14
+ Requires-Python: >=3.14
15
+ Project-URL: homepage, https://gitlab.com/a.phlaming.phoenix/escrape-plan
16
+ Project-URL: repository, https://gitlab.com/a.phlaming.phoenix/escrape-plan
17
+ Provides-Extra: dev
18
+ Description-Content-Type: text/markdown
19
+
20
+ # eScrape Plan
21
+
22
+ A tool for extracting data from HTML pages in complex and extensible ways according to plans crafted to describe that data.
23
+
24
+ eScrape Plan starts at the top of an HTML document and compares every tag in a "flattened" sequence against a series of **markers**, which are HTML tags you expect to find. The content between those markers are stored internally as **sections** in a **document**. Each section can then be acted on by **extractors**, which pull data out of the HTML and store them. Further, after a document has been fully parsed and all of this data has been extracted, the data can be run through **outputters** to produce artifacts from the data.
25
+
26
+ Today, eScrape Plan can do things like:
27
+
28
+ - Parse a long document into smaller sections
29
+ - Categorize those sections internally by name/ID
30
+ - Run find/replace operations on that content (to do things like normalize newline characters in the source)
31
+ - Reorganize this content arbitrarily and produce new HTML files with any arrangement of that content
32
+ - Export textual/HTML content into SQLite databases for later querying
33
+ - Convert HTML `<table>` content into SQLite databases
34
+
35
+ > [!NOTE]
36
+ > **A Note from a.p.p on LLM Contributions**
37
+ >
38
+ > I would prefer not to use LLMs to generate code for this project for a variety of reasons that are not appropriate to list in detail here. That said, unfortunately, LLM usage is becoming unavoidable for me in my daily job developing software due to external pressures, and so it does behoove me to experiment with this technology.
39
+ >
40
+ > At this point, I remain unconvinced of the value of LLM generated code, but I have found some mixed value in using LLMs for code review. I will often have an LLM form merge requests and post code review comments for me. In these cases, there should always be an attribution to the bot. I only use open weight models running on my own hardware.
41
+
42
+
43
+ ## Usage
44
+
45
+ I'll have [better documentation](https://gitlab.com/a.phlaming.phoenix/escrape-plan/-/work_items/11) on before too much longer. Until then...
46
+
47
+ ```
48
+ pip install escrape-plan
49
+ ```
@@ -0,0 +1,12 @@
1
+ escrape_plan/__init__.py,sha256=KN7abEZBncrysOlZl39ywFUdworAsd4goFvpqigeZZ4,20004
2
+ escrape_plan/extractors/__init__.py,sha256=u4EEPf6BG_BngyzgJixOrrK0OF1K1-i2BrBnpBL9FJ4,1746
3
+ escrape_plan/extractors/base.py,sha256=vCWofAcnVPhq5jI7nzYnz67e6okpjVZkPzEOTwZYE-0,1662
4
+ escrape_plan/extractors/read_html.py,sha256=YcKjO4WcRmZ1_GbNWzoDYxZPNhU2IYXFqpM0mF5lb84,3858
5
+ escrape_plan/extractors/sqlite_table.py,sha256=a9-Vx1bTzA1zH4Z4ovGtQCHuxqU5rhTnLJsp8rkREzo,8929
6
+ escrape_plan/outputters/__init__.py,sha256=xLX-eEuBvZg1fa2Z3aIF3tI6VIE2LxvUuRhla7TuM9E,1184
7
+ escrape_plan/outputters/base.py,sha256=Z7M2WXYrMeSq7_F1dDMzTrjoA6C_1-RhT7rQaOZFAA0,1062
8
+ escrape_plan/outputters/text_file.py,sha256=GHapPKjHAV7O2Fxjz08_qdyZ9LuhpqnccFbS2-b_VRQ,3825
9
+ escrape_plan/outputters/text_to_sqlite.py,sha256=92cI1cW7cNSO57BRIOsmxczu4lDyN5B9f-5669DD7Us,5537
10
+ escrape_plan-0.0.1.dist-info/WHEEL,sha256=MWx7psoF7KNST9G72pGtiBYuaXRXQXBk9pw5R6FzYko,80
11
+ escrape_plan-0.0.1.dist-info/METADATA,sha256=zOFfQTz8g71PZD0J_mcHRvpZzUU0Vbu1bTMvVY8MqD0,2725
12
+ escrape_plan-0.0.1.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: uv 0.13.7
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any