markdownify 1.1.0__tar.gz → 1.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,7 +15,7 @@ jobs:
15
15
  runs-on: ubuntu-latest
16
16
 
17
17
  steps:
18
- - uses: actions/checkout@v2
18
+ - uses: actions/checkout@v4
19
19
  - name: Set up Python 3.8
20
20
  uses: actions/setup-python@v2
21
21
  with:
@@ -30,3 +30,22 @@ jobs:
30
30
  - name: Build
31
31
  run: |
32
32
  python -m build -nwsx .
33
+
34
+ types:
35
+
36
+ runs-on: ubuntu-latest
37
+
38
+ steps:
39
+ - uses: actions/checkout@v2
40
+ - name: Set up Python 3.8
41
+ uses: actions/setup-python@v2
42
+ with:
43
+ python-version: 3.8
44
+ - name: Install dependencies
45
+ run: |
46
+ python -m pip install --upgrade pip
47
+ pip install --upgrade setuptools setuptools_scm wheel build tox mypy types-beautifulsoup4
48
+ - name: Check types
49
+ run: |
50
+ mypy .
51
+ mypy --strict tests/types.py
@@ -13,7 +13,7 @@ jobs:
13
13
  runs-on: ubuntu-latest
14
14
 
15
15
  steps:
16
- - uses: actions/checkout@v2
16
+ - uses: actions/checkout@v4
17
17
  - name: Set up Python
18
18
  uses: actions/setup-python@v2
19
19
  with:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: markdownify
3
- Version: 1.1.0
3
+ Version: 1.2.0
4
4
  Summary: Convert HTML to markdown.
5
5
  Author-email: Matthew Tretter <m@tthewwithanm.com>
6
6
  Project-URL: Homepage, http://github.com/matthewwithanm/python-markdownify
@@ -134,7 +134,7 @@ code_language_callback
134
134
  When the HTML code contains ``pre`` tags that in some way provide the code
135
135
  language, for example as class, this callback can be used to extract the
136
136
  language from the tag and prefix it to the converted ``pre`` tag.
137
- The callback gets one single argument, an BeautifylSoup object, and returns
137
+ The callback gets one single argument, a BeautifulSoup object, and returns
138
138
  a string containing the code language, or ``None``.
139
139
  An example to use the class name as code language could be::
140
140
 
@@ -181,6 +181,23 @@ strip_document
181
181
  within the document are unaffected.
182
182
  Defaults to ``STRIP``.
183
183
 
184
+ strip_pre
185
+ Controls whether leading/trailing blank lines are removed from ``<pre>``
186
+ tags. Supported values are ``STRIP`` (all leading/trailing blank lines),
187
+ ``STRIP_ONE`` (one leading/trailing blank line), and ``None`` (neither).
188
+ Defaults to ``STRIP``.
189
+
190
+ bs4_options
191
+ Specify additional configuration options for the ``BeautifulSoup`` object
192
+ used to interpret the HTML markup. String and list values (such as ``lxml``
193
+ or ``html5lib``) are treated as ``features`` arguments to control parser
194
+ selection. Dictionary values (such as ``{"from_encoding": "iso-8859-8"}``)
195
+ are treated as full kwargs to be used for the BeautifulSoup constructor,
196
+ allowing specification of any parameter. For parameter details, see the
197
+ Beautiful Soup documentation at:
198
+
199
+ .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/bs4/doc/
200
+
184
201
  Options may be specified as kwargs to the ``markdownify`` function, or as a
185
202
  nested ``Options`` class in ``MarkdownConverter`` subclasses.
186
203
 
@@ -110,7 +110,7 @@ code_language_callback
110
110
  When the HTML code contains ``pre`` tags that in some way provide the code
111
111
  language, for example as class, this callback can be used to extract the
112
112
  language from the tag and prefix it to the converted ``pre`` tag.
113
- The callback gets one single argument, an BeautifylSoup object, and returns
113
+ The callback gets one single argument, a BeautifulSoup object, and returns
114
114
  a string containing the code language, or ``None``.
115
115
  An example to use the class name as code language could be::
116
116
 
@@ -157,6 +157,23 @@ strip_document
157
157
  within the document are unaffected.
158
158
  Defaults to ``STRIP``.
159
159
 
160
+ strip_pre
161
+ Controls whether leading/trailing blank lines are removed from ``<pre>``
162
+ tags. Supported values are ``STRIP`` (all leading/trailing blank lines),
163
+ ``STRIP_ONE`` (one leading/trailing blank line), and ``None`` (neither).
164
+ Defaults to ``STRIP``.
165
+
166
+ bs4_options
167
+ Specify additional configuration options for the ``BeautifulSoup`` object
168
+ used to interpret the HTML markup. String and list values (such as ``lxml``
169
+ or ``html5lib``) are treated as ``features`` arguments to control parser
170
+ selection. Dictionary values (such as ``{"from_encoding": "iso-8859-8"}``)
171
+ are treated as full kwargs to be used for the BeautifulSoup constructor,
172
+ allowing specification of any parameter. For parameter details, see the
173
+ Beautiful Soup documentation at:
174
+
175
+ .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/bs4/doc/
176
+
160
177
  Options may be specified as kwargs to the ``markdownify`` function, or as a
161
178
  nested ``Options`` class in ``MarkdownConverter`` subclasses.
162
179
 
@@ -11,6 +11,10 @@ re_whitespace = re.compile(r'[\t ]+')
11
11
  re_all_whitespace = re.compile(r'[\t \r\n]+')
12
12
  re_newline_whitespace = re.compile(r'[\t \r\n]*[\r\n][\t \r\n]*')
13
13
  re_html_heading = re.compile(r'h(\d+)')
14
+ re_pre_lstrip1 = re.compile(r'^ *\n')
15
+ re_pre_rstrip1 = re.compile(r'\n *$')
16
+ re_pre_lstrip = re.compile(r'^[ \n]*\n')
17
+ re_pre_rstrip = re.compile(r'[ \n]*$')
14
18
 
15
19
  # Pattern for creating convert_<tag> function names from tag names
16
20
  re_make_convert_fn_name = re.compile(r'[\[\]:-]')
@@ -37,6 +41,9 @@ re_escape_misc_hashes = re.compile(r'(\s|^)(#{1,6}(?:\s|$))')
37
41
  # confused with a list item
38
42
  re_escape_misc_list_items = re.compile(r'((?:\s|^)[0-9]{1,9})([.)](?:\s|$))')
39
43
 
44
+ # Find consecutive backtick sequences in a string
45
+ re_backtick_runs = re.compile(r'`+')
46
+
40
47
  # Heading styles
41
48
  ATX = 'atx'
42
49
  ATX_CLOSED = 'atx_closed'
@@ -51,10 +58,25 @@ BACKSLASH = 'backslash'
51
58
  ASTERISK = '*'
52
59
  UNDERSCORE = '_'
53
60
 
54
- # Document strip styles
61
+ # Document/pre strip styles
55
62
  LSTRIP = 'lstrip'
56
63
  RSTRIP = 'rstrip'
57
64
  STRIP = 'strip'
65
+ STRIP_ONE = 'strip_one'
66
+
67
+
68
+ def strip1_pre(text):
69
+ """Strip one leading and trailing newline from a <pre> string."""
70
+ text = re_pre_lstrip1.sub('', text)
71
+ text = re_pre_rstrip1.sub('', text)
72
+ return text
73
+
74
+
75
+ def strip_pre(text):
76
+ """Strip all leading and trailing newlines from a <pre> string."""
77
+ text = re_pre_lstrip.sub('', text)
78
+ text = re_pre_rstrip.sub('', text)
79
+ return text
58
80
 
59
81
 
60
82
  def chomp(text):
@@ -154,6 +176,7 @@ def _next_block_content_sibling(el):
154
176
  class MarkdownConverter(object):
155
177
  class DefaultOptions:
156
178
  autolinks = True
179
+ bs4_options = 'html.parser'
157
180
  bullets = '*+-' # An iterable of bullet types.
158
181
  code_language = ''
159
182
  code_language_callback = None
@@ -167,6 +190,7 @@ class MarkdownConverter(object):
167
190
  newline_style = SPACES
168
191
  strip = None
169
192
  strip_document = STRIP
193
+ strip_pre = STRIP
170
194
  strong_em_symbol = ASTERISK
171
195
  sub_symbol = ''
172
196
  sup_symbol = ''
@@ -187,11 +211,15 @@ class MarkdownConverter(object):
187
211
  raise ValueError('You may specify either tags to strip or tags to'
188
212
  ' convert, but not both.')
189
213
 
214
+ # If a string or list is passed to bs4_options, assume it is a 'features' specification
215
+ if not isinstance(self.options['bs4_options'], dict):
216
+ self.options['bs4_options'] = {'features': self.options['bs4_options']}
217
+
190
218
  # Initialize the conversion function cache
191
219
  self.convert_fn_cache = {}
192
220
 
193
221
  def convert(self, html):
194
- soup = BeautifulSoup(html, 'html.parser')
222
+ soup = BeautifulSoup(html, **self.options['bs4_options'])
195
223
  return self.convert_soup(soup)
196
224
 
197
225
  def convert_soup(self, soup):
@@ -362,16 +390,20 @@ class MarkdownConverter(object):
362
390
  if not self.should_convert_tag(tag_name):
363
391
  return None
364
392
 
365
- # Handle headings with _convert_hn() function
393
+ # Look for an explicitly defined conversion function by tag name first
394
+ convert_fn_name = "convert_%s" % re_make_convert_fn_name.sub("_", tag_name)
395
+ convert_fn = getattr(self, convert_fn_name, None)
396
+ if convert_fn:
397
+ return convert_fn
398
+
399
+ # If tag is any heading, handle with convert_hN() function
366
400
  match = re_html_heading.match(tag_name)
367
401
  if match:
368
- n = int(match.group(1))
369
- return lambda el, text, parent_tags: self._convert_hn(n, el, text, parent_tags)
402
+ n = int(match.group(1)) # get value of N from <hN>
403
+ return lambda el, text, parent_tags: self.convert_hN(n, el, text, parent_tags)
370
404
 
371
- # For other tags, look up their conversion function by tag name
372
- convert_fn_name = "convert_%s" % re_make_convert_fn_name.sub('_', tag_name)
373
- convert_fn = getattr(self, convert_fn_name, None)
374
- return convert_fn
405
+ # No conversion function was found
406
+ return None
375
407
 
376
408
  def should_convert_tag(self, tag):
377
409
  """Given a tag name, return whether to convert based on strip/convert options."""
@@ -451,10 +483,24 @@ class MarkdownConverter(object):
451
483
  return ' \n'
452
484
 
453
485
  def convert_code(self, el, text, parent_tags):
454
- if 'pre' in parent_tags:
486
+ if '_noformat' in parent_tags:
455
487
  return text
456
- converter = abstract_inline_conversion(lambda self: '`')
457
- return converter(self, el, text, parent_tags)
488
+
489
+ prefix, suffix, text = chomp(text)
490
+ if not text:
491
+ return ''
492
+
493
+ # Find the maximum number of consecutive backticks in the text, then
494
+ # delimit the code span with one more backtick than that
495
+ max_backticks = max((len(match) for match in re.findall(re_backtick_runs, text)), default=0)
496
+ markup_delimiter = '`' * (max_backticks + 1)
497
+
498
+ # If the maximum number of backticks is greater than zero, add a space
499
+ # to avoid interpretation of inside backticks as literals
500
+ if max_backticks > 0:
501
+ text = " " + text + " "
502
+
503
+ return '%s%s%s%s%s' % (prefix, markup_delimiter, text, markup_delimiter, suffix)
458
504
 
459
505
  convert_del = abstract_inline_conversion(lambda self: '~~')
460
506
 
@@ -509,12 +555,12 @@ class MarkdownConverter(object):
509
555
 
510
556
  return '\n\n%s\n' % text
511
557
 
512
- def _convert_hn(self, n, el, text, parent_tags):
513
- """ Method name prefixed with _ to prevent <hn> to call this """
558
+ def convert_hN(self, n, el, text, parent_tags):
559
+ # convert_hN() converts <hN> tags, where N is any integer
514
560
  if '_inline' in parent_tags:
515
561
  return text
516
562
 
517
- # prevent MemoryErrors in case of very large n
563
+ # Markdown does not support heading depths of n > 6
518
564
  n = max(1, min(6, n))
519
565
 
520
566
  style = self.options['heading_style'].lower()
@@ -647,8 +693,20 @@ class MarkdownConverter(object):
647
693
  if self.options['code_language_callback']:
648
694
  code_language = self.options['code_language_callback'](el) or code_language
649
695
 
696
+ if self.options['strip_pre'] == STRIP:
697
+ text = strip_pre(text) # remove all leading/trailing newlines
698
+ elif self.options['strip_pre'] == STRIP_ONE:
699
+ text = strip1_pre(text) # remove one leading/trailing newline
700
+ elif self.options['strip_pre'] is None:
701
+ pass # leave leading and trailing newlines as-is
702
+ else:
703
+ raise ValueError('Invalid value for strip_pre: %s' % self.options['strip_pre'])
704
+
650
705
  return '\n\n```%s\n%s\n```\n\n' % (code_language, text)
651
706
 
707
+ def convert_q(self, el, text, parent_tags):
708
+ return '"' + text + '"'
709
+
652
710
  def convert_script(self, el, text, parent_tags):
653
711
  return ''
654
712
 
@@ -677,13 +735,13 @@ class MarkdownConverter(object):
677
735
  def convert_td(self, el, text, parent_tags):
678
736
  colspan = 1
679
737
  if 'colspan' in el.attrs and el['colspan'].isdigit():
680
- colspan = int(el['colspan'])
738
+ colspan = max(1, min(1000, int(el['colspan'])))
681
739
  return ' ' + text.strip().replace("\n", " ") + ' |' * colspan
682
740
 
683
741
  def convert_th(self, el, text, parent_tags):
684
742
  colspan = 1
685
743
  if 'colspan' in el.attrs and el['colspan'].isdigit():
686
- colspan = int(el['colspan'])
744
+ colspan = max(1, min(1000, int(el['colspan'])))
687
745
  return ' ' + text.strip().replace("\n", " ") + ' |' * colspan
688
746
 
689
747
  def convert_tr(self, el, text, parent_tags):
@@ -704,7 +762,7 @@ class MarkdownConverter(object):
704
762
  full_colspan = 0
705
763
  for cell in cells:
706
764
  if 'colspan' in cell.attrs and cell['colspan'].isdigit():
707
- full_colspan += int(cell["colspan"])
765
+ full_colspan += max(1, min(1000, int(cell['colspan'])))
708
766
  else:
709
767
  full_colspan += 1
710
768
  if ((is_headrow
@@ -0,0 +1,77 @@
1
+ from _typeshed import Incomplete
2
+ from typing import Callable, Union
3
+
4
+ ATX: str
5
+ ATX_CLOSED: str
6
+ UNDERLINED: str
7
+ SETEXT = UNDERLINED
8
+ SPACES: str
9
+ BACKSLASH: str
10
+ ASTERISK: str
11
+ UNDERSCORE: str
12
+ LSTRIP: str
13
+ RSTRIP: str
14
+ STRIP: str
15
+ STRIP_ONE: str
16
+
17
+
18
+ def markdownify(
19
+ html: str,
20
+ autolinks: bool = ...,
21
+ bs4_options: str = ...,
22
+ bullets: str = ...,
23
+ code_language: str = ...,
24
+ code_language_callback: Union[Callable[[Incomplete], Union[str, None]], None] = ...,
25
+ convert: Union[list[str], None] = ...,
26
+ default_title: bool = ...,
27
+ escape_asterisks: bool = ...,
28
+ escape_underscores: bool = ...,
29
+ escape_misc: bool = ...,
30
+ heading_style: str = ...,
31
+ keep_inline_images_in: list[str] = ...,
32
+ newline_style: str = ...,
33
+ strip: Union[list[str], None] = ...,
34
+ strip_document: Union[str, None] = ...,
35
+ strip_pre: str = ...,
36
+ strong_em_symbol: str = ...,
37
+ sub_symbol: str = ...,
38
+ sup_symbol: str = ...,
39
+ table_infer_header: bool = ...,
40
+ wrap: bool = ...,
41
+ wrap_width: int = ...,
42
+ ) -> str: ...
43
+
44
+
45
+ class MarkdownConverter:
46
+ def __init__(
47
+ self,
48
+ autolinks: bool = ...,
49
+ bs4_options: str = ...,
50
+ bullets: str = ...,
51
+ code_language: str = ...,
52
+ code_language_callback: Union[Callable[[Incomplete], Union[str, None]], None] = ...,
53
+ convert: Union[list[str], None] = ...,
54
+ default_title: bool = ...,
55
+ escape_asterisks: bool = ...,
56
+ escape_underscores: bool = ...,
57
+ escape_misc: bool = ...,
58
+ heading_style: str = ...,
59
+ keep_inline_images_in: list[str] = ...,
60
+ newline_style: str = ...,
61
+ strip: Union[list[str], None] = ...,
62
+ strip_document: Union[str, None] = ...,
63
+ strip_pre: str = ...,
64
+ strong_em_symbol: str = ...,
65
+ sub_symbol: str = ...,
66
+ sup_symbol: str = ...,
67
+ table_infer_header: bool = ...,
68
+ wrap: bool = ...,
69
+ wrap_width: int = ...,
70
+ ) -> None:
71
+ ...
72
+
73
+ def convert(self, html: str) -> str:
74
+ ...
75
+
76
+ def convert_soup(self, soup: Incomplete) -> str:
77
+ ...
@@ -55,7 +55,9 @@ def main(argv=sys.argv[1:]):
55
55
  parser.add_argument('--no-escape-underscores', dest='escape_underscores',
56
56
  action='store_false',
57
57
  help="Do not escape '_' to '\\_' in text.")
58
- parser.add_argument('-i', '--keep-inline-images-in', nargs='*',
58
+ parser.add_argument('-i', '--keep-inline-images-in',
59
+ default=[],
60
+ nargs='*',
59
61
  help="Images are converted to their alt-text when the images are "
60
62
  "located inside headlines or table cells. If some inline images "
61
63
  "should be converted to markdown images instead, this option can "
@@ -68,6 +70,11 @@ def main(argv=sys.argv[1:]):
68
70
  parser.add_argument('-w', '--wrap', action='store_true',
69
71
  help="Wrap all text paragraphs at --wrap-width characters.")
70
72
  parser.add_argument('--wrap-width', type=int, default=80)
73
+ parser.add_argument('--bs4-options',
74
+ default='html.parser',
75
+ help="Specifies the parser that BeautifulSoup should use to parse "
76
+ "the HTML markup. Examples include 'html5.parser', 'lxml', and "
77
+ "'html5lib'.")
71
78
 
72
79
  args = parser.parse_args(argv)
73
80
  print(markdownify(**vars(args)))
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: markdownify
3
- Version: 1.1.0
3
+ Version: 1.2.0
4
4
  Summary: Convert HTML to markdown.
5
5
  Author-email: Matthew Tretter <m@tthewwithanm.com>
6
6
  Project-URL: Homepage, http://github.com/matthewwithanm/python-markdownify
@@ -134,7 +134,7 @@ code_language_callback
134
134
  When the HTML code contains ``pre`` tags that in some way provide the code
135
135
  language, for example as class, this callback can be used to extract the
136
136
  language from the tag and prefix it to the converted ``pre`` tag.
137
- The callback gets one single argument, an BeautifylSoup object, and returns
137
+ The callback gets one single argument, a BeautifulSoup object, and returns
138
138
  a string containing the code language, or ``None``.
139
139
  An example to use the class name as code language could be::
140
140
 
@@ -181,6 +181,23 @@ strip_document
181
181
  within the document are unaffected.
182
182
  Defaults to ``STRIP``.
183
183
 
184
+ strip_pre
185
+ Controls whether leading/trailing blank lines are removed from ``<pre>``
186
+ tags. Supported values are ``STRIP`` (all leading/trailing blank lines),
187
+ ``STRIP_ONE`` (one leading/trailing blank line), and ``None`` (neither).
188
+ Defaults to ``STRIP``.
189
+
190
+ bs4_options
191
+ Specify additional configuration options for the ``BeautifulSoup`` object
192
+ used to interpret the HTML markup. String and list values (such as ``lxml``
193
+ or ``html5lib``) are treated as ``features`` arguments to control parser
194
+ selection. Dictionary values (such as ``{"from_encoding": "iso-8859-8"}``)
195
+ are treated as full kwargs to be used for the BeautifulSoup constructor,
196
+ allowing specification of any parameter. For parameter details, see the
197
+ Beautiful Soup documentation at:
198
+
199
+ .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/bs4/doc/
200
+
184
201
  Options may be specified as kwargs to the ``markdownify`` function, or as a
185
202
  nested ``Options`` class in ``MarkdownConverter`` subclasses.
186
203
 
@@ -8,6 +8,7 @@ tox.ini
8
8
  .github/workflows/python-app.yml
9
9
  .github/workflows/python-publish.yml
10
10
  markdownify/__init__.py
11
+ markdownify/__init__.pyi
11
12
  markdownify/main.py
12
13
  markdownify.egg-info/PKG-INFO
13
14
  markdownify.egg-info/SOURCES.txt
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "markdownify"
7
- version = "1.1.0"
7
+ version = "1.2.0"
8
8
  authors = [{name = "Matthew Tretter", email = "m@tthewwithanm.com"}]
9
9
  description = "Convert HTML to markdown."
10
10
  readme = "README.rst"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes