escrape-plan 0.0.2__tar.gz → 0.0.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,13 +1,13 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: escrape-plan
3
- Version: 0.0.2
3
+ Version: 0.0.4
4
4
  Summary: Tool for extracting data from HTML documents
5
5
  Author: a.p.p
6
6
  Author-email: a.p.p <a.phlaming.phoenix@gmail.com>
7
- Requires-Dist: beautifulsoup4
8
- Requires-Dist: lxml
9
- Requires-Dist: pandoc
10
- Requires-Dist: pyyaml
7
+ Requires-Dist: beautifulsoup4>=4.15.0,<5
8
+ Requires-Dist: lxml>=6.1.3,<7
9
+ Requires-Dist: pandoc>=2.4,<3
10
+ Requires-Dist: pyyaml>=6.0.3,<7
11
11
  Requires-Dist: bpython>=0.26,<1 ; extra == 'dev'
12
12
  Requires-Dist: ruff>=0.17.3,<1 ; extra == 'dev'
13
13
  Requires-Dist: twine>=7.0.0,<8 ; extra == 'dev'
@@ -1,14 +1,14 @@
1
1
  [project]
2
2
  name = "escrape-plan"
3
- version = "0.0.2"
3
+ version = "0.0.4"
4
4
  description = "Tool for extracting data from HTML documents"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.14"
7
7
  dependencies = [
8
- "beautifulsoup4",
9
- "lxml",
10
- "pandoc",
11
- "pyyaml",
8
+ "beautifulsoup4>=4.15.0,<5",
9
+ "lxml>=6.1.3,<7",
10
+ "pandoc>=2.4,<3",
11
+ "pyyaml>=6.0.3,<7",
12
12
  ]
13
13
 
14
14
  [[project.authors]]
@@ -1,11 +1,16 @@
1
1
  [project]
2
2
  name = "escrape-plan"
3
- version = "0.0.2"
3
+ version = "0.0.4"
4
4
  description = "Tool for extracting data from HTML documents"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.14"
7
7
  authors = [{ name = "a.p.p", email = "a.phlaming.phoenix@gmail.com" }]
8
- dependencies = ["beautifulsoup4", "lxml", "pandoc", "pyyaml"]
8
+ dependencies = [
9
+ "beautifulsoup4>=4.15.0,<5",
10
+ "lxml>=6.1.3,<7",
11
+ "pandoc>=2.4,<3",
12
+ "pyyaml>=6.0.3,<7",
13
+ ]
9
14
 
10
15
  [project.optional-dependencies]
11
16
  dev = ["bpython>=0.26,<1", "ruff>=0.17.3,<1", "twine>=7.0.0,<8"]
@@ -230,7 +230,7 @@ class Marker:
230
230
  self.matched_element = None
231
231
 
232
232
  def match(self, element: bs4.element.Tag) -> bool:
233
- if element.name == self.tag and element.text.startswith(self.text):
233
+ if element.name == self.tag and element.text.strip().startswith(self.text):
234
234
  self.matched_element = element
235
235
  return True
236
236
  return False
@@ -290,12 +290,20 @@ class Document:
290
290
  :type plan: list[dict]
291
291
  """
292
292
 
293
- def __init__(self, src: str, title: str, plan: list[dict], outputs: list[dict] | None = None):
293
+ def __init__(
294
+ self,
295
+ src: str,
296
+ title: str,
297
+ plan: list[dict],
298
+ outputs: list[dict] | None = None,
299
+ debug_logging: bool = False,
300
+ ):
294
301
  self.src = src
295
302
  self.soup = bs4.BeautifulSoup(self.src, 'lxml')
296
303
  self.title = title
297
304
  self.sections = None
298
305
  self.output_configs = outputs if outputs else []
306
+ self.debug_logging = debug_logging
299
307
 
300
308
  # Validate plan length
301
309
  if len(plan) == 0:
@@ -308,6 +316,8 @@ class Document:
308
316
  populates the ``sections`` member."""
309
317
 
310
318
  # --- Set up the scrape ---
319
+ if self.debug_logging:
320
+ print('Document scrape begins now')
311
321
 
312
322
  # Start with an empty set of sections
313
323
  self.sections = OrderedDict()
@@ -376,6 +386,7 @@ class Document:
376
386
  config=extractor_config,
377
387
  src=current_section_elements,
378
388
  section_id=current_marker.id,
389
+ debug_logging=self.debug_logging,
379
390
  )
380
391
  for extractor_config in current_marker.extractor_configs
381
392
  ]:
@@ -404,12 +415,23 @@ class Document:
404
415
  # that contain the entire document, and which are pretty much useless. Because these
405
416
  # elements won't match a sensible plan, these elements are naturally scrapped.
406
417
  for element in self.soup.find_all():
418
+ if self.debug_logging:
419
+ print(
420
+ f'- Evaluating <{element.name}> with text '
421
+ f'"{element.text.strip().replace("\n", " ")[:20]}..."'
422
+ )
423
+
407
424
  # If the element is a match for the marker we expect next...
408
425
  if next_marker:
409
426
  if next_marker.match(element):
427
+ if self.debug_logging:
428
+ print(f'- Element matched the next marker: {next_marker.id}')
410
429
  # If a section ID is set, that means we've already matched an element and started a
411
430
  # section. We need to form a proper Section object and stash it in this Document.
412
431
  if current_section_id:
432
+ if self.debug_logging:
433
+ print(f'- Extracting data from section {current_section_id}')
434
+
413
435
  # Run extractors on the current data set
414
436
  self.sections[current_section_id] = __extract(current_marker=current_marker)
415
437
 
@@ -428,6 +450,9 @@ class Document:
428
450
  # marker. Therefore, *this* is our first marker and we should start tracking the
429
451
  # contents of this section.
430
452
  else:
453
+ if self.debug_logging:
454
+ print(f'- Found the first marker for section {next_marker.id}')
455
+
431
456
  # Set the current marker to the thing we just found
432
457
  current_marker = next_marker
433
458
 
@@ -452,6 +477,8 @@ class Document:
452
477
  # After iterating over everything, we need to do one final extraction, but only if we ever
453
478
  # actually matched something.
454
479
  if current_section_id:
480
+ if self.debug_logging:
481
+ print(f'- Extracting data from the final section: {current_marker}')
455
482
  self.sections[current_section_id] = __extract(current_marker=current_marker)
456
483
 
457
484
  def output(self):
@@ -470,5 +497,7 @@ class Document:
470
497
 
471
498
  from .outputters import construct_outputter
472
499
 
473
- outputter = construct_outputter(sections=self.sections, config=output_config)
500
+ outputter = construct_outputter(
501
+ sections=self.sections, config=output_config, debug_logging=self.debug_logging
502
+ )
474
503
  outputter.output()
@@ -20,7 +20,9 @@ EXTRACTOR_TYPES = {
20
20
  }
21
21
 
22
22
 
23
- def construct_extractor(config: dict, src: list[Tag], section_id: str) -> Extractor:
23
+ def construct_extractor(
24
+ config: dict, src: list[Tag], section_id: str, debug_logging: bool = False
25
+ ) -> Extractor:
24
26
  """Given a configuration for an Extractor, constructs the correct extractor sub-type class.
25
27
 
26
28
  :param config: Configuration for the extractor sub-type.
@@ -32,6 +34,9 @@ def construct_extractor(config: dict, src: list[Tag], section_id: str) -> Extrac
32
34
  :param section_id: The ID of this Section.
33
35
  :type src: str
34
36
 
37
+ :param debug_logging: Enables output to help debug a plan.
38
+ :type debug_logging: bool
39
+
35
40
  :raises KeyError: The config does not contain a "type" field.
36
41
 
37
42
  :raises ValueError: The config contains an unknown "type" value.
@@ -48,4 +53,4 @@ def construct_extractor(config: dict, src: list[Tag], section_id: str) -> Extrac
48
53
  __type = EXTRACTOR_TYPES[extractor_type]
49
54
  __module = import_module(f'escrape_plan.extractors.{__type["package"]}')
50
55
  __extractor = getattr(__module, __type['class'])
51
- return __extractor(config=config, src=src, section_id=section_id)
56
+ return __extractor(config=config, src=src, section_id=section_id, debug_logging=debug_logging)
@@ -24,11 +24,12 @@ class Extractor(SchemaValidator):
24
24
  :type section_id: str
25
25
  """
26
26
 
27
- def __init__(self, src: list[Tag], config: dict, section_id: str):
27
+ def __init__(self, src: list[Tag], config: dict, section_id: str, debug_logging: bool = False):
28
28
  super().__init__(config=config)
29
29
  self.src = src
30
30
  self.config = config
31
31
  self.section_id = section_id
32
+ self.debug_logging = debug_logging
32
33
  self.type = None
33
34
  self.validate_config_schema()
34
35
 
@@ -23,9 +23,9 @@ class ReadHTMLExtractor(Extractor):
23
23
  :type section_id: str
24
24
  """
25
25
 
26
- def __init__(self, src: list[Tag], config: dict, section_id: str):
26
+ def __init__(self, src: list[Tag], config: dict, section_id: str, debug_logging: bool = False):
27
27
  # Automatically handles validation against the config_schema
28
- super().__init__(src, config, section_id)
28
+ super().__init__(src, config, section_id, debug_logging)
29
29
  self.type = 'read_html'
30
30
 
31
31
  @cached_property
@@ -71,6 +71,8 @@ class ReadHTMLExtractor(Extractor):
71
71
  :rtype: str
72
72
  """
73
73
 
74
+ if self.debug_logging:
75
+ print(f'- Extracting HTML text from section {marker.id}')
74
76
  # Convert each tag to a string and concatenate them all
75
77
  extracted_str = ''.join([str(tag) for tag in self.src])
76
78
 
@@ -93,6 +95,8 @@ class ReadHTMLExtractor(Extractor):
93
95
  f' {", ".join(KNOWN_TRANSFORMS)}'
94
96
  )
95
97
 
98
+ if self.debug_logging:
99
+ print(f'- Applying transformation {transform_type} to section {marker.id}')
96
100
  if transform_type == 'replace':
97
101
  find = transform.get('find')
98
102
  replacement = transform.get('replacement')
@@ -20,7 +20,7 @@ class SQLiteTableExtractor(Extractor):
20
20
  found by this extractor.**
21
21
  """
22
22
 
23
- def __init__(self, src: list[Tag], config: dict, section_id: str):
23
+ def __init__(self, src: list[Tag], config: dict, section_id: str, debug_logging: bool = False):
24
24
  super().__init__(src, config, section_id)
25
25
  self.type = 'sqlite_table'
26
26
 
@@ -139,6 +139,8 @@ class SQLiteTableExtractor(Extractor):
139
139
  database file.
140
140
  """
141
141
 
142
+ if self.debug_logging:
143
+ print(f'- Extracting HTML table to SQLite table in section {marker.id}')
142
144
  # The number of columns defined in the schema in our config
143
145
  required_column_ct = len(self.schema)
144
146
 
@@ -21,7 +21,9 @@ OUTPUTTER_TYPES = {
21
21
  }
22
22
 
23
23
 
24
- def construct_outputter(sections: OrderedDict[str, Section], config: dict) -> Outputter:
24
+ def construct_outputter(
25
+ sections: OrderedDict[str, Section], config: dict, debug_logging: bool = False
26
+ ) -> Outputter:
25
27
  outputter_type = config.get('type')
26
28
  if not outputter_type:
27
29
  raise KeyError('Field "type" is required in outputter configs')
@@ -30,4 +32,4 @@ def construct_outputter(sections: OrderedDict[str, Section], config: dict) -> Ou
30
32
  __type = OUTPUTTER_TYPES[outputter_type]
31
33
  __module = import_module(f'escrape_plan.outputters.{__type["package"]}')
32
34
  __outputter = getattr(__module, __type['class'])
33
- return __outputter(sections=sections, config=config)
35
+ return __outputter(sections=sections, config=config, debug_logging=debug_logging)
@@ -14,9 +14,10 @@ class Outputter(SchemaValidator):
14
14
  :type config: dict
15
15
  """
16
16
 
17
- def __init__(self, sections: list[Section], config: dict):
17
+ def __init__(self, sections: list[Section], config: dict, debug_logging: bool = False):
18
18
  super().__init__(config=config)
19
19
  self.config = config
20
+ self.debug_logging = debug_logging
20
21
  self.sections = sections
21
22
  self.type = None
22
23
  self.validate_config_schema()
@@ -14,8 +14,8 @@ class TextFileOutputter(Outputter):
14
14
  :type config: dict
15
15
  """
16
16
 
17
- def __init__(self, sections: list[Section], config: dict):
18
- super().__init__(sections=sections, config=config)
17
+ def __init__(self, sections: list[Section], config: dict, debug_logging: bool = False):
18
+ super().__init__(sections=sections, config=config, debug_logging=debug_logging)
19
19
  self.type = 'text_file'
20
20
 
21
21
  @cached_property
@@ -64,6 +64,9 @@ class TextFileOutputter(Outputter):
64
64
  filename = self.config.get('filename')
65
65
  content_configs = self.config.get('content')
66
66
 
67
+ if self.debug_logging:
68
+ print(f'Outputting text file: {filename}')
69
+
67
70
  # Get a file handler
68
71
  with open(filename, 'w') as output_file:
69
72
  # Iterate over the content and output each entry
@@ -20,8 +20,8 @@ class TextToSqliteOutputter(Outputter):
20
20
  :type config: dict
21
21
  """
22
22
 
23
- def __init__(self, sections: list[Section], config: dict):
24
- super().__init__(sections=sections, config=config)
23
+ def __init__(self, sections: list[Section], config: dict, debug_logging: bool = False):
24
+ super().__init__(sections=sections, config=config, debug_logging=debug_logging)
25
25
  self.type = 'text_to_sqlite'
26
26
 
27
27
  # Localize some config variables for convenience
@@ -96,6 +96,11 @@ class TextToSqliteOutputter(Outputter):
96
96
  # Iterate over content definitions to create the data in the SQLite table.
97
97
  content_configs = self.config.get('content', [])
98
98
 
99
+ if self.debug_logging:
100
+ print(
101
+ f'Outputting content to table {self.table_name} '
102
+ f'in SQLite database: {sqlite_filename}'
103
+ )
99
104
  with sqlite3.connect(sqlite_filename) as sqlite_conn:
100
105
  cursor = sqlite_conn.cursor()
101
106
  # Ensure the table exists
File without changes