escrape-plan 0.0.2__tar.gz → 0.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {escrape_plan-0.0.2 → escrape_plan-0.0.4}/PKG-INFO +5 -5
- {escrape_plan-0.0.2 → escrape_plan-0.0.4}/pyproject.toml +5 -5
- {escrape_plan-0.0.2 → escrape_plan-0.0.4}/pyproject.toml.orig +7 -2
- {escrape_plan-0.0.2 → escrape_plan-0.0.4}/src/escrape_plan/__init__.py +32 -3
- {escrape_plan-0.0.2 → escrape_plan-0.0.4}/src/escrape_plan/extractors/__init__.py +7 -2
- {escrape_plan-0.0.2 → escrape_plan-0.0.4}/src/escrape_plan/extractors/base.py +2 -1
- {escrape_plan-0.0.2 → escrape_plan-0.0.4}/src/escrape_plan/extractors/read_html.py +6 -2
- {escrape_plan-0.0.2 → escrape_plan-0.0.4}/src/escrape_plan/extractors/sqlite_table.py +3 -1
- {escrape_plan-0.0.2 → escrape_plan-0.0.4}/src/escrape_plan/outputters/__init__.py +4 -2
- {escrape_plan-0.0.2 → escrape_plan-0.0.4}/src/escrape_plan/outputters/base.py +2 -1
- {escrape_plan-0.0.2 → escrape_plan-0.0.4}/src/escrape_plan/outputters/text_file.py +5 -2
- {escrape_plan-0.0.2 → escrape_plan-0.0.4}/src/escrape_plan/outputters/text_to_sqlite.py +7 -2
- {escrape_plan-0.0.2 → escrape_plan-0.0.4}/README.md +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: escrape-plan
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.4
|
|
4
4
|
Summary: Tool for extracting data from HTML documents
|
|
5
5
|
Author: a.p.p
|
|
6
6
|
Author-email: a.p.p <a.phlaming.phoenix@gmail.com>
|
|
7
|
-
Requires-Dist: beautifulsoup4
|
|
8
|
-
Requires-Dist: lxml
|
|
9
|
-
Requires-Dist: pandoc
|
|
10
|
-
Requires-Dist: pyyaml
|
|
7
|
+
Requires-Dist: beautifulsoup4>=4.15.0,<5
|
|
8
|
+
Requires-Dist: lxml>=6.1.3,<7
|
|
9
|
+
Requires-Dist: pandoc>=2.4,<3
|
|
10
|
+
Requires-Dist: pyyaml>=6.0.3,<7
|
|
11
11
|
Requires-Dist: bpython>=0.26,<1 ; extra == 'dev'
|
|
12
12
|
Requires-Dist: ruff>=0.17.3,<1 ; extra == 'dev'
|
|
13
13
|
Requires-Dist: twine>=7.0.0,<8 ; extra == 'dev'
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "escrape-plan"
|
|
3
|
-
version = "0.0.
|
|
3
|
+
version = "0.0.4"
|
|
4
4
|
description = "Tool for extracting data from HTML documents"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.14"
|
|
7
7
|
dependencies = [
|
|
8
|
-
"beautifulsoup4",
|
|
9
|
-
"lxml",
|
|
10
|
-
"pandoc",
|
|
11
|
-
"pyyaml",
|
|
8
|
+
"beautifulsoup4>=4.15.0,<5",
|
|
9
|
+
"lxml>=6.1.3,<7",
|
|
10
|
+
"pandoc>=2.4,<3",
|
|
11
|
+
"pyyaml>=6.0.3,<7",
|
|
12
12
|
]
|
|
13
13
|
|
|
14
14
|
[[project.authors]]
|
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "escrape-plan"
|
|
3
|
-
version = "0.0.
|
|
3
|
+
version = "0.0.4"
|
|
4
4
|
description = "Tool for extracting data from HTML documents"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.14"
|
|
7
7
|
authors = [{ name = "a.p.p", email = "a.phlaming.phoenix@gmail.com" }]
|
|
8
|
-
dependencies = [
|
|
8
|
+
dependencies = [
|
|
9
|
+
"beautifulsoup4>=4.15.0,<5",
|
|
10
|
+
"lxml>=6.1.3,<7",
|
|
11
|
+
"pandoc>=2.4,<3",
|
|
12
|
+
"pyyaml>=6.0.3,<7",
|
|
13
|
+
]
|
|
9
14
|
|
|
10
15
|
[project.optional-dependencies]
|
|
11
16
|
dev = ["bpython>=0.26,<1", "ruff>=0.17.3,<1", "twine>=7.0.0,<8"]
|
|
@@ -230,7 +230,7 @@ class Marker:
|
|
|
230
230
|
self.matched_element = None
|
|
231
231
|
|
|
232
232
|
def match(self, element: bs4.element.Tag) -> bool:
|
|
233
|
-
if element.name == self.tag and element.text.startswith(self.text):
|
|
233
|
+
if element.name == self.tag and element.text.strip().startswith(self.text):
|
|
234
234
|
self.matched_element = element
|
|
235
235
|
return True
|
|
236
236
|
return False
|
|
@@ -290,12 +290,20 @@ class Document:
|
|
|
290
290
|
:type plan: list[dict]
|
|
291
291
|
"""
|
|
292
292
|
|
|
293
|
-
def __init__(
|
|
293
|
+
def __init__(
|
|
294
|
+
self,
|
|
295
|
+
src: str,
|
|
296
|
+
title: str,
|
|
297
|
+
plan: list[dict],
|
|
298
|
+
outputs: list[dict] | None = None,
|
|
299
|
+
debug_logging: bool = False,
|
|
300
|
+
):
|
|
294
301
|
self.src = src
|
|
295
302
|
self.soup = bs4.BeautifulSoup(self.src, 'lxml')
|
|
296
303
|
self.title = title
|
|
297
304
|
self.sections = None
|
|
298
305
|
self.output_configs = outputs if outputs else []
|
|
306
|
+
self.debug_logging = debug_logging
|
|
299
307
|
|
|
300
308
|
# Validate plan length
|
|
301
309
|
if len(plan) == 0:
|
|
@@ -308,6 +316,8 @@ class Document:
|
|
|
308
316
|
populates the ``sections`` member."""
|
|
309
317
|
|
|
310
318
|
# --- Set up the scrape ---
|
|
319
|
+
if self.debug_logging:
|
|
320
|
+
print('Document scrape begins now')
|
|
311
321
|
|
|
312
322
|
# Start with an empty set of sections
|
|
313
323
|
self.sections = OrderedDict()
|
|
@@ -376,6 +386,7 @@ class Document:
|
|
|
376
386
|
config=extractor_config,
|
|
377
387
|
src=current_section_elements,
|
|
378
388
|
section_id=current_marker.id,
|
|
389
|
+
debug_logging=self.debug_logging,
|
|
379
390
|
)
|
|
380
391
|
for extractor_config in current_marker.extractor_configs
|
|
381
392
|
]:
|
|
@@ -404,12 +415,23 @@ class Document:
|
|
|
404
415
|
# that contain the entire document, and which are pretty much useless. Because these
|
|
405
416
|
# elements won't match a sensible plan, these elements are naturally scrapped.
|
|
406
417
|
for element in self.soup.find_all():
|
|
418
|
+
if self.debug_logging:
|
|
419
|
+
print(
|
|
420
|
+
f'- Evaluating <{element.name}> with text '
|
|
421
|
+
f'"{element.text.strip().replace("\n", " ")[:20]}..."'
|
|
422
|
+
)
|
|
423
|
+
|
|
407
424
|
# If the element is a match for the marker we expect next...
|
|
408
425
|
if next_marker:
|
|
409
426
|
if next_marker.match(element):
|
|
427
|
+
if self.debug_logging:
|
|
428
|
+
print(f'- Element matched the next marker: {next_marker.id}')
|
|
410
429
|
# If a section ID is set, that means we've already matched an element and started a
|
|
411
430
|
# section. We need to form a proper Section object and stash it in this Document.
|
|
412
431
|
if current_section_id:
|
|
432
|
+
if self.debug_logging:
|
|
433
|
+
print(f'- Extracting data from section {current_section_id}')
|
|
434
|
+
|
|
413
435
|
# Run extractors on the current data set
|
|
414
436
|
self.sections[current_section_id] = __extract(current_marker=current_marker)
|
|
415
437
|
|
|
@@ -428,6 +450,9 @@ class Document:
|
|
|
428
450
|
# marker. Therefore, *this* is our first marker and we should start tracking the
|
|
429
451
|
# contents of this section.
|
|
430
452
|
else:
|
|
453
|
+
if self.debug_logging:
|
|
454
|
+
print(f'- Found the first marker for section {next_marker.id}')
|
|
455
|
+
|
|
431
456
|
# Set the current marker to the thing we just found
|
|
432
457
|
current_marker = next_marker
|
|
433
458
|
|
|
@@ -452,6 +477,8 @@ class Document:
|
|
|
452
477
|
# After iterating over everything, we need to do one final extraction, but only if we ever
|
|
453
478
|
# actually matched something.
|
|
454
479
|
if current_section_id:
|
|
480
|
+
if self.debug_logging:
|
|
481
|
+
print(f'- Extracting data from the final section: {current_marker}')
|
|
455
482
|
self.sections[current_section_id] = __extract(current_marker=current_marker)
|
|
456
483
|
|
|
457
484
|
def output(self):
|
|
@@ -470,5 +497,7 @@ class Document:
|
|
|
470
497
|
|
|
471
498
|
from .outputters import construct_outputter
|
|
472
499
|
|
|
473
|
-
outputter = construct_outputter(
|
|
500
|
+
outputter = construct_outputter(
|
|
501
|
+
sections=self.sections, config=output_config, debug_logging=self.debug_logging
|
|
502
|
+
)
|
|
474
503
|
outputter.output()
|
|
@@ -20,7 +20,9 @@ EXTRACTOR_TYPES = {
|
|
|
20
20
|
}
|
|
21
21
|
|
|
22
22
|
|
|
23
|
-
def construct_extractor(
|
|
23
|
+
def construct_extractor(
|
|
24
|
+
config: dict, src: list[Tag], section_id: str, debug_logging: bool = False
|
|
25
|
+
) -> Extractor:
|
|
24
26
|
"""Given a configuration for an Extractor, constructs the correct extractor sub-type class.
|
|
25
27
|
|
|
26
28
|
:param config: Configuration for the extractor sub-type.
|
|
@@ -32,6 +34,9 @@ def construct_extractor(config: dict, src: list[Tag], section_id: str) -> Extrac
|
|
|
32
34
|
:param section_id: The ID of this Section.
|
|
33
35
|
:type src: str
|
|
34
36
|
|
|
37
|
+
:param debug_logging: Enables output to help debug a plan.
|
|
38
|
+
:type debug_logging: bool
|
|
39
|
+
|
|
35
40
|
:raises KeyError: The config does not contain a "type" field.
|
|
36
41
|
|
|
37
42
|
:raises ValueError: The config contains an unknown "type" value.
|
|
@@ -48,4 +53,4 @@ def construct_extractor(config: dict, src: list[Tag], section_id: str) -> Extrac
|
|
|
48
53
|
__type = EXTRACTOR_TYPES[extractor_type]
|
|
49
54
|
__module = import_module(f'escrape_plan.extractors.{__type["package"]}')
|
|
50
55
|
__extractor = getattr(__module, __type['class'])
|
|
51
|
-
return __extractor(config=config, src=src, section_id=section_id)
|
|
56
|
+
return __extractor(config=config, src=src, section_id=section_id, debug_logging=debug_logging)
|
|
@@ -24,11 +24,12 @@ class Extractor(SchemaValidator):
|
|
|
24
24
|
:type section_id: str
|
|
25
25
|
"""
|
|
26
26
|
|
|
27
|
-
def __init__(self, src: list[Tag], config: dict, section_id: str):
|
|
27
|
+
def __init__(self, src: list[Tag], config: dict, section_id: str, debug_logging: bool = False):
|
|
28
28
|
super().__init__(config=config)
|
|
29
29
|
self.src = src
|
|
30
30
|
self.config = config
|
|
31
31
|
self.section_id = section_id
|
|
32
|
+
self.debug_logging = debug_logging
|
|
32
33
|
self.type = None
|
|
33
34
|
self.validate_config_schema()
|
|
34
35
|
|
|
@@ -23,9 +23,9 @@ class ReadHTMLExtractor(Extractor):
|
|
|
23
23
|
:type section_id: str
|
|
24
24
|
"""
|
|
25
25
|
|
|
26
|
-
def __init__(self, src: list[Tag], config: dict, section_id: str):
|
|
26
|
+
def __init__(self, src: list[Tag], config: dict, section_id: str, debug_logging: bool = False):
|
|
27
27
|
# Automatically handles validation against the config_schema
|
|
28
|
-
super().__init__(src, config, section_id)
|
|
28
|
+
super().__init__(src, config, section_id, debug_logging)
|
|
29
29
|
self.type = 'read_html'
|
|
30
30
|
|
|
31
31
|
@cached_property
|
|
@@ -71,6 +71,8 @@ class ReadHTMLExtractor(Extractor):
|
|
|
71
71
|
:rtype: str
|
|
72
72
|
"""
|
|
73
73
|
|
|
74
|
+
if self.debug_logging:
|
|
75
|
+
print(f'- Extracting HTML text from section {marker.id}')
|
|
74
76
|
# Convert each tag to a string and concatenate them all
|
|
75
77
|
extracted_str = ''.join([str(tag) for tag in self.src])
|
|
76
78
|
|
|
@@ -93,6 +95,8 @@ class ReadHTMLExtractor(Extractor):
|
|
|
93
95
|
f' {", ".join(KNOWN_TRANSFORMS)}'
|
|
94
96
|
)
|
|
95
97
|
|
|
98
|
+
if self.debug_logging:
|
|
99
|
+
print(f'- Applying transformation {transform_type} to section {marker.id}')
|
|
96
100
|
if transform_type == 'replace':
|
|
97
101
|
find = transform.get('find')
|
|
98
102
|
replacement = transform.get('replacement')
|
|
@@ -20,7 +20,7 @@ class SQLiteTableExtractor(Extractor):
|
|
|
20
20
|
found by this extractor.**
|
|
21
21
|
"""
|
|
22
22
|
|
|
23
|
-
def __init__(self, src: list[Tag], config: dict, section_id: str):
|
|
23
|
+
def __init__(self, src: list[Tag], config: dict, section_id: str, debug_logging: bool = False):
|
|
24
24
|
super().__init__(src, config, section_id)
|
|
25
25
|
self.type = 'sqlite_table'
|
|
26
26
|
|
|
@@ -139,6 +139,8 @@ class SQLiteTableExtractor(Extractor):
|
|
|
139
139
|
database file.
|
|
140
140
|
"""
|
|
141
141
|
|
|
142
|
+
if self.debug_logging:
|
|
143
|
+
print(f'- Extracting HTML table to SQLite table in section {marker.id}')
|
|
142
144
|
# The number of columns defined in the schema in our config
|
|
143
145
|
required_column_ct = len(self.schema)
|
|
144
146
|
|
|
@@ -21,7 +21,9 @@ OUTPUTTER_TYPES = {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
|
|
24
|
-
def construct_outputter(
|
|
24
|
+
def construct_outputter(
|
|
25
|
+
sections: OrderedDict[str, Section], config: dict, debug_logging: bool = False
|
|
26
|
+
) -> Outputter:
|
|
25
27
|
outputter_type = config.get('type')
|
|
26
28
|
if not outputter_type:
|
|
27
29
|
raise KeyError('Field "type" is required in outputter configs')
|
|
@@ -30,4 +32,4 @@ def construct_outputter(sections: OrderedDict[str, Section], config: dict) -> Ou
|
|
|
30
32
|
__type = OUTPUTTER_TYPES[outputter_type]
|
|
31
33
|
__module = import_module(f'escrape_plan.outputters.{__type["package"]}')
|
|
32
34
|
__outputter = getattr(__module, __type['class'])
|
|
33
|
-
return __outputter(sections=sections, config=config)
|
|
35
|
+
return __outputter(sections=sections, config=config, debug_logging=debug_logging)
|
|
@@ -14,9 +14,10 @@ class Outputter(SchemaValidator):
|
|
|
14
14
|
:type config: dict
|
|
15
15
|
"""
|
|
16
16
|
|
|
17
|
-
def __init__(self, sections: list[Section], config: dict):
|
|
17
|
+
def __init__(self, sections: list[Section], config: dict, debug_logging: bool = False):
|
|
18
18
|
super().__init__(config=config)
|
|
19
19
|
self.config = config
|
|
20
|
+
self.debug_logging = debug_logging
|
|
20
21
|
self.sections = sections
|
|
21
22
|
self.type = None
|
|
22
23
|
self.validate_config_schema()
|
|
@@ -14,8 +14,8 @@ class TextFileOutputter(Outputter):
|
|
|
14
14
|
:type config: dict
|
|
15
15
|
"""
|
|
16
16
|
|
|
17
|
-
def __init__(self, sections: list[Section], config: dict):
|
|
18
|
-
super().__init__(sections=sections, config=config)
|
|
17
|
+
def __init__(self, sections: list[Section], config: dict, debug_logging: bool = False):
|
|
18
|
+
super().__init__(sections=sections, config=config, debug_logging=debug_logging)
|
|
19
19
|
self.type = 'text_file'
|
|
20
20
|
|
|
21
21
|
@cached_property
|
|
@@ -64,6 +64,9 @@ class TextFileOutputter(Outputter):
|
|
|
64
64
|
filename = self.config.get('filename')
|
|
65
65
|
content_configs = self.config.get('content')
|
|
66
66
|
|
|
67
|
+
if self.debug_logging:
|
|
68
|
+
print(f'Outputting text file: {filename}')
|
|
69
|
+
|
|
67
70
|
# Get a file handler
|
|
68
71
|
with open(filename, 'w') as output_file:
|
|
69
72
|
# Iterate over the content and output each entry
|
|
@@ -20,8 +20,8 @@ class TextToSqliteOutputter(Outputter):
|
|
|
20
20
|
:type config: dict
|
|
21
21
|
"""
|
|
22
22
|
|
|
23
|
-
def __init__(self, sections: list[Section], config: dict):
|
|
24
|
-
super().__init__(sections=sections, config=config)
|
|
23
|
+
def __init__(self, sections: list[Section], config: dict, debug_logging: bool = False):
|
|
24
|
+
super().__init__(sections=sections, config=config, debug_logging=debug_logging)
|
|
25
25
|
self.type = 'text_to_sqlite'
|
|
26
26
|
|
|
27
27
|
# Localize some config variables for convenience
|
|
@@ -96,6 +96,11 @@ class TextToSqliteOutputter(Outputter):
|
|
|
96
96
|
# Iterate over content definitions to create the data in the SQLite table.
|
|
97
97
|
content_configs = self.config.get('content', [])
|
|
98
98
|
|
|
99
|
+
if self.debug_logging:
|
|
100
|
+
print(
|
|
101
|
+
f'Outputting content to table {self.table_name} '
|
|
102
|
+
f'in SQLite database: {sqlite_filename}'
|
|
103
|
+
)
|
|
99
104
|
with sqlite3.connect(sqlite_filename) as sqlite_conn:
|
|
100
105
|
cursor = sqlite_conn.cursor()
|
|
101
106
|
# Ensure the table exists
|
|
File without changes
|