pdftext 0.1.0__tar.gz

pdftext-0.1.0/LICENSE

@@ -0,0 +1,200 @@
+Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2024 Vikas Paruchuri
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pdftext-0.1.0/PKG-INFO

@@ -0,0 +1,155 @@
+Metadata-Version: 2.1
+Name: pdftext
+Version: 0.1.0
+Summary: Extract structured text from pdfs quickly
+Home-page: https://github.com/VikParuchuri/pdftext
+License: Apache-2.0
+Keywords: pdf,text,extraction
+Author: Vik Paruchuri
+Author-email: vik.paruchuri@gmail.com
+Requires-Python: >=3.9, !=2.7.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*, !=3.8.*
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: pydantic (>=2.7.1,<3.0.0)
+Requires-Dist: pydantic-settings (>=2.2.1,<3.0.0)
+Requires-Dist: pypdfium2 (>=4.29.0,<5.0.0)
+Requires-Dist: scikit-learn (>=1.4.2,<2.0.0)
+Project-URL: Repository, https://github.com/VikParuchuri/pdftext
+Description-Content-Type: text/markdown
+
+# PDFText
+
+Text extraction like [PyMuPDF]((https://github.com/pymupdf/PyMuPDF), but without the AGPL license.  PDFText extracts plain text or structured blocks and lines.  It's built on [pypdfium2](https://github.com/pypdfium2-team/pypdfium2), so it's [fast, accurate](#benchmarks), and Apache licensed.
+
+# Installation
+
+You'll need python 3.9+ first.  Then run `pip install pdftext`.
+
+# Usage
+
+- Inspect the settings in `pdftext/settings.py`.  You can override any settings with environment variables.
+
+## Plain text
+
+This command will write out a text file with the extracted plain text.
+
+```shell
+pdftext PDF_PATH --out_path output.txt
+```
+
+- `PDF_PATH` must be a single pdf file.
+- `--out_path` path to the output txt file.  If not specified, will write to stdout.
+- `--sort` will attempt to sort in reading order if specified.
+
+## JSON
+
+This command outputs structured blocks and lines with font and other information.
+
+```shell
+pdftext PDF_PATH --out_path output.txt --output_type json
+```
+
+- `PDF_PATH` must be a single pdf file.
+- `--out_path` path to the output txt file.  If not specified, will write to stdout.
+- `--output_type` specifies whether to write out plain text (default) or json
+- `--sort` will attempt to sort in reading order if specified.
+
+The output will be a json list, with each item in the list corresponding to a single page in the input pdf (in order).  Each page will include the following keys:
+
+- `bbox` - the page bbox, in `[x1, y1, x2, y2]` format
+- `rotation` - how much the page is rotated, in degrees (`0`, `90`, `180`, or `270`)
+- `page_idx` - the index of the page
+- `blocks` - the blocks that make up the text in the pdf.  Approximately equal to a paragraph.
+  - `bbox` - the block bbox, in `[x1, y1, x2, y2]` format
+  - `lines` - the lines inside the block
+    - `bbox` - the line bbox, in `[x1, y1, x2, y2]` format
+    - `chars` - the individual characters in the line
+      - `char` - the actual character, encoded in utf-8
+      - `rotation` - how much the character is rotated, in degrees
+      - `bbox` - the character bbox, in `[x1, y1, x2, y2]` format
+      - `char_idx` - the index of the character on the page (from `0` to number of characters, in original pdf order)
+      - `font` this is font info straight from the pdf, see [this pdfium code](https://pdfium.googlesource.com/pdfium/+/refs/heads/main/public/fpdf_text.h)
+        - `size` - the size of the font used for the character
+        - `weight` - font weight
+        - `name` - font name, may be None
+        - `flags` - font flags, in the format of the `PDF spec 1.7 Section 5.7.1 Font Descriptor Flags`
+
+# Programmatic usage
+
+Extract plain text:
+
+```python
+from pdftext.extraction import plain_text_output
+
+text = plain_text_output(PDF_PATH, sort=False)
+```
+
+Extract structured blocks and lines:
+
+```python
+from pdftext.extraction import dictionary_output
+
+text = dictionary_output(PDF_PATH)
+```
+
+If you want more customization, check out the `pdftext.extraction._get_pages` function for a starting point to dig deeper.  pdftext is a pretty thin wrapper around [pypdfium2](https://pypdfium2.readthedocs.io/en/stable/), so you might want to look at the documentation for that as well.
+
+# Benchmarks
+
+I benchmarked extraction speed and accuracy of [pymupdf](https://pymupdf.readthedocs.io/en/latest/), [pdfplumber](https://github.com/jsvine/pdfplumber), and pdftext.  I chose pymupdf because it extracts blocks and lines.  Pdfplumber extracts words and bboxes.  I did not benchmark pypdf, even though it is a great library, because it doesn't provide individual words/lines and bbox information.
+
+Here are the scores:
+
++------------+-------------------+-----------------------------------------+
+|  Library   | Time (s per page) | Alignment Score (% accuracy vs pymupdf) |
++------------+-------------------+-----------------------------------------+
+|  pymupdf   |       0.31        |                   --                    |
+|  pdftext   |       1.45        |                  95.64                  |
+| pdfplumber |       2.97        |                  89.88                  |
++------------+-------------------+-----------------------------------------+
+
+pdftext is approximately 2x slower than using pypdfium2 alone (if you were to extract all the same information).
+
+There are additional benchmarks for pypdfium2 and other tools [here](https://github.com/py-pdf/benchmarks).
+
+## Methodology
+
+I used a benchmark set of 200 pdfs extracted from [common crawl](https://huggingface.co/datasets/pixparse/pdfa-eng-wds), then processed by a team at HuggingFace.
+
+For each library, I used a detailed extraction method, to pull out font information, as well as just the words.  This ensured we were comparing similar performance numbers.
+
+For the alignment score, I extracted the text, then used the rapidfuzz library to find the alignment percentage.  I used the text extracted by pymupdf as the pseudo-ground truth.
+
+## Running benchmarks
+
+You can run the benchmarks yourself.  To do so, you have to first install pdftext manually.  The install assumes you have poetry and Python 3.9+ installed.
+
+```shell
+git clone https://github.com/VikParuchuri/pdftext.git
+cd pdftext
+poetry install
+python benchmark.py # Will download the benchmark pdfs automatically
+```
+
+The benchmark script has a few options:
+
+- `--max` this controls the maximum number of pdfs to benchmark
+- `--result_path` a folder to save the results.  A file called `results.json` will be created in the folder.
+
+# How it works
+
+PDFText is a very light wrapper around pypdfium2.  It first uses pypdfium2 to extract characters in order, along with font and other information.  Then it uses a simple decision tree algorithm to group characters into lines and blocks.  It then done some simple postprocessing to clean up the text.
+
+# Credits
+
+This is built on some amazing open source work, including:
+
+- [pypdfium2](https://github.com/pypdfium2-team/pypdfium2)
+- [scikit-learn](https://scikit-learn.org/stable/index.html)
+- [pypdf2](https://github.com/py-pdf/benchmarks) for very thorough and fair benchmarks
+
+Thank you to the [pymupdf](https://github.com/pymupdf/PyMuPDF) devs for creating such a great library - I just wish it had a simpler license!

pdftext-0.1.0/README.md

@@ -0,0 +1,132 @@
+# PDFText
+
+Text extraction like [PyMuPDF]((https://github.com/pymupdf/PyMuPDF), but without the AGPL license.  PDFText extracts plain text or structured blocks and lines.  It's built on [pypdfium2](https://github.com/pypdfium2-team/pypdfium2), so it's [fast, accurate](#benchmarks), and Apache licensed.
+
+# Installation
+
+You'll need python 3.9+ first.  Then run `pip install pdftext`.
+
+# Usage
+
+- Inspect the settings in `pdftext/settings.py`.  You can override any settings with environment variables.
+
+## Plain text
+
+This command will write out a text file with the extracted plain text.
+
+```shell
+pdftext PDF_PATH --out_path output.txt
+```
+
+- `PDF_PATH` must be a single pdf file.
+- `--out_path` path to the output txt file.  If not specified, will write to stdout.
+- `--sort` will attempt to sort in reading order if specified.
+
+## JSON
+
+This command outputs structured blocks and lines with font and other information.
+
+```shell
+pdftext PDF_PATH --out_path output.txt --output_type json
+```
+
+- `PDF_PATH` must be a single pdf file.
+- `--out_path` path to the output txt file.  If not specified, will write to stdout.
+- `--output_type` specifies whether to write out plain text (default) or json
+- `--sort` will attempt to sort in reading order if specified.
+
+The output will be a json list, with each item in the list corresponding to a single page in the input pdf (in order).  Each page will include the following keys:
+
+- `bbox` - the page bbox, in `[x1, y1, x2, y2]` format
+- `rotation` - how much the page is rotated, in degrees (`0`, `90`, `180`, or `270`)
+- `page_idx` - the index of the page
+- `blocks` - the blocks that make up the text in the pdf.  Approximately equal to a paragraph.
+  - `bbox` - the block bbox, in `[x1, y1, x2, y2]` format
+  - `lines` - the lines inside the block
+    - `bbox` - the line bbox, in `[x1, y1, x2, y2]` format
+    - `chars` - the individual characters in the line
+      - `char` - the actual character, encoded in utf-8
+      - `rotation` - how much the character is rotated, in degrees
+      - `bbox` - the character bbox, in `[x1, y1, x2, y2]` format
+      - `char_idx` - the index of the character on the page (from `0` to number of characters, in original pdf order)
+      - `font` this is font info straight from the pdf, see [this pdfium code](https://pdfium.googlesource.com/pdfium/+/refs/heads/main/public/fpdf_text.h)
+        - `size` - the size of the font used for the character
+        - `weight` - font weight
+        - `name` - font name, may be None
+        - `flags` - font flags, in the format of the `PDF spec 1.7 Section 5.7.1 Font Descriptor Flags`
+
+# Programmatic usage
+
+Extract plain text:
+
+```python
+from pdftext.extraction import plain_text_output
+
+text = plain_text_output(PDF_PATH, sort=False)
+```
+
+Extract structured blocks and lines:
+
+```python
+from pdftext.extraction import dictionary_output
+
+text = dictionary_output(PDF_PATH)
+```
+
+If you want more customization, check out the `pdftext.extraction._get_pages` function for a starting point to dig deeper.  pdftext is a pretty thin wrapper around [pypdfium2](https://pypdfium2.readthedocs.io/en/stable/), so you might want to look at the documentation for that as well.
+
+# Benchmarks
+
+I benchmarked extraction speed and accuracy of [pymupdf](https://pymupdf.readthedocs.io/en/latest/), [pdfplumber](https://github.com/jsvine/pdfplumber), and pdftext.  I chose pymupdf because it extracts blocks and lines.  Pdfplumber extracts words and bboxes.  I did not benchmark pypdf, even though it is a great library, because it doesn't provide individual words/lines and bbox information.
+
+Here are the scores:
+
++------------+-------------------+-----------------------------------------+
+|  Library   | Time (s per page) | Alignment Score (% accuracy vs pymupdf) |
++------------+-------------------+-----------------------------------------+
+|  pymupdf   |       0.31        |                   --                    |
+|  pdftext   |       1.45        |                  95.64                  |
+| pdfplumber |       2.97        |                  89.88                  |
++------------+-------------------+-----------------------------------------+
+
+pdftext is approximately 2x slower than using pypdfium2 alone (if you were to extract all the same information).
+
+There are additional benchmarks for pypdfium2 and other tools [here](https://github.com/py-pdf/benchmarks).
+
+## Methodology
+
+I used a benchmark set of 200 pdfs extracted from [common crawl](https://huggingface.co/datasets/pixparse/pdfa-eng-wds), then processed by a team at HuggingFace.
+
+For each library, I used a detailed extraction method, to pull out font information, as well as just the words.  This ensured we were comparing similar performance numbers.
+
+For the alignment score, I extracted the text, then used the rapidfuzz library to find the alignment percentage.  I used the text extracted by pymupdf as the pseudo-ground truth.
+
+## Running benchmarks
+
+You can run the benchmarks yourself.  To do so, you have to first install pdftext manually.  The install assumes you have poetry and Python 3.9+ installed.
+
+```shell
+git clone https://github.com/VikParuchuri/pdftext.git
+cd pdftext
+poetry install
+python benchmark.py # Will download the benchmark pdfs automatically
+```
+
+The benchmark script has a few options:
+
+- `--max` this controls the maximum number of pdfs to benchmark
+- `--result_path` a folder to save the results.  A file called `results.json` will be created in the folder.
+
+# How it works
+
+PDFText is a very light wrapper around pypdfium2.  It first uses pypdfium2 to extract characters in order, along with font and other information.  Then it uses a simple decision tree algorithm to group characters into lines and blocks.  It then done some simple postprocessing to clean up the text.
+
+# Credits
+
+This is built on some amazing open source work, including:
+
+- [pypdfium2](https://github.com/pypdfium2-team/pypdfium2)
+- [scikit-learn](https://scikit-learn.org/stable/index.html)
+- [pypdf2](https://github.com/py-pdf/benchmarks) for very thorough and fair benchmarks
+
+Thank you to the [pymupdf](https://github.com/pymupdf/PyMuPDF) devs for creating such a great library - I just wish it had a simpler license!

pdftext-0.1.0/extract_text.py

@@ -0,0 +1,31 @@
+import argparse
+import json
+
+from pdftext.extraction import plain_text_output, dictionary_output
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Extract plain text from PDF.  Not guaranteed to be in order.")
+    parser.add_argument("pdf_path", type=str, help="Path to the PDF file")
+    parser.add_argument("--out_path", type=str, help="Path to the output text file, defaults to stdout", default=None)
+    parser.add_argument("--output_type", type=str, help="Type of output to generate", default="plain_text")
+    parser.add_argument("--sort", action="store_true", help="Attempt to sort the text by reading order", default=False)
+    args = parser.parse_args()
+
+    assert args.output_type in ["plain_text", "json"], "Invalid output type, must be 'plain_text' or 'json'"
+
+    if args.output_type == "plain_text":
+        text = plain_text_output(args.pdf_path, sort=args.sort)
+    elif args.output_type == "json":
+        text = dictionary_output(args.pdf_path, sort=args.sort)
+        text = json.dumps(text)
+
+    if args.out_path is None:
+        print(text)
+    else:
+        with open(args.out_path, "w+") as f:
+            f.write(text)
+
+
+if __name__ == "__main__":
+    main()

pdftext-0.1.0/pdftext/extraction.py

@@ -0,0 +1,59 @@
+from typing import List
+
+from pdftext.inference import inference
+from pdftext.model import get_model
+from pdftext.pdf.chars import get_pdfium_chars
+from pdftext.pdf.utils import unnormalize_bbox
+from pdftext.postprocessing import merge_text, sort_blocks, postprocess_text
+
+
+def _get_pages(pdf_path, model=None):
+    if model is None:
+        model = get_model()
+    text_chars = get_pdfium_chars(pdf_path)
+    pages = inference(text_chars, model)
+    return pages
+
+
+def plain_text_output(pdf_path, sort=False, model=None) -> str:
+    text = paginated_plain_text_output(pdf_path, sort=sort, model=model)
+    return "\n".join(text)
+
+
+def paginated_plain_text_output(pdf_path, sort=False, model=None) -> List[str]:
+    pages = _get_pages(pdf_path, model)
+    text = []
+    for page in pages:
+        text.append(merge_text(page, sort=sort).strip())
+    return text
+
+
+def dictionary_output(pdf_path, sort=False, model=None):
+    pages = _get_pages(pdf_path, model)
+    for page in pages:
+        for block in page["blocks"]:
+            bad_keys = [key for key in block.keys() if key not in ["lines", "bbox"]]
+            for key in bad_keys:
+                del block[key]
+            for line in block["lines"]:
+                line_box = None
+                bad_keys = [key for key in line.keys() if key not in ["chars", "bbox"]]
+                for key in bad_keys:
+                    del line[key]
+                for char in line["chars"]:
+                    char["bbox"] = unnormalize_bbox(char["bbox"], page["bbox"])
+                    char["char"] = postprocess_text(char["char"])
+                    if line_box is None:
+                        line_box = char["bbox"]
+                    else:
+                        line_box = [
+                            min(line_box[0], char["bbox"][0]),
+                            min(line_box[1], char["bbox"][1]),
+                            max(line_box[2], char["bbox"][2]),
+                            max(line_box[3], char["bbox"][3]),
+                        ]
+                line["bbox"] = line_box
+            block["bbox"] = unnormalize_bbox(block["bbox"], page["bbox"])
+        if sort:
+            page["blocks"] = sort_blocks(page["blocks"])
+    return pages

pdftext-0.1.0/pdftext/inference.py

@@ -0,0 +1,152 @@
+from itertools import chain
+
+from pdftext.pdf.utils import LINE_BREAKS, TABS, SPACES
+
+
+def update_current(current, new_char):
+    bbox = new_char["bbox"]
+    if "bbox" not in current:
+        current_bbox = bbox
+        current["bbox"] = current_bbox
+    else:
+        current_bbox = current["bbox"]
+        current_bbox[0] = min(bbox[0], current_bbox[0])
+        current_bbox[1] = min(bbox[1], current_bbox[1])
+        current_bbox[2] = max(bbox[2], current_bbox[2])
+        current_bbox[3] = max(bbox[3], current_bbox[3])
+    current["center_x"] = (current_bbox[0] + current_bbox[2]) / 2
+    current["center_y"] = (current_bbox[1] + current_bbox[3]) / 2
+    return current
+
+
+def create_training_row(char_info, prev_char, currblock):
+    char = char_info["char"]
+    char_center_x = (char_info["bbox"][2] + char_info["bbox"][0]) / 2
+    char_center_y = (char_info["bbox"][3] + char_info["bbox"][1]) / 2
+    x_gap = char_info["bbox"][0] - prev_char["bbox"][2]
+    y_gap = char_info["bbox"][1] - prev_char["bbox"][3]
+    font_match = all(
+        [char_info["font"][key] == prev_char["font"][key] for key in ["name", "size", "weight", "flags"]] +
+        [char_info["rotation"] == prev_char["rotation"]]
+    )
+    is_space = any([
+        char in SPACES,
+        char in TABS,
+    ])
+
+    training_row = {
+        "is_newline": char in LINE_BREAKS,
+        "is_space": is_space,
+        "x_gap": x_gap,
+        "y_gap": y_gap,
+        "font_match": font_match,
+        "x_outer_gap": char_info["bbox"][2] - prev_char["bbox"][0],
+        "y_outer_gap": char_info["bbox"][3] - prev_char["bbox"][1],
+        "block_x_center_gap": char_center_x - currblock["center_x"],
+        "block_y_center_gap": char_center_y - currblock["center_y"],
+        "block_x_gap": char_info["bbox"][0] - currblock["bbox"][2],
+        "block_y_gap": char_info["bbox"][1] - currblock["bbox"][3]
+    }
+
+    return training_row
+
+
+def update_span(line, span):
+    line["spans"].append(span)
+    span = {"chars": []}
+    return span
+
+
+def update_line(block, line):
+    line["chars"] = list(chain.from_iterable(s["chars"] for s in line["spans"]))
+    del line["spans"]
+    block["lines"].append(line)
+    line = {"spans": []}
+    return line
+
+
+def update_block(blocks, block):
+    blocks["blocks"].append(block)
+    block = {"lines": []}
+    return block
+
+
+def infer_single_page(text_chars):
+    prev_char = None
+
+    blocks = {"blocks": []}
+    block = {"lines": []}
+    line = {"spans": []}
+    span = {"chars": []}
+    for i, char_info in enumerate(text_chars["chars"]):
+        if prev_char:
+            training_row = create_training_row(char_info, prev_char, block)
+            training_row = [v for _, v in sorted(training_row.items())]
+
+            prediction = yield training_row
+            if prediction == 0:
+                pass
+            elif prediction == 1:
+                span = update_span(line, span)
+            elif prediction == 2:
+                span = update_span(line, span)
+                line = update_line(block, line)
+            else:
+                span = update_span(line, span)
+                line = update_line(block, line)
+                block = update_block(blocks, block)
+
+        span["chars"].append(char_info)
+        block = update_current(block, char_info)
+
+        prev_char = char_info
+    if len(span["chars"]) > 0:
+        update_span(line, span)
+    if len(line["spans"]) > 0:
+        update_line(block, line)
+    if len(block["lines"]) > 0:
+        update_block(blocks, block)
+
+    blocks["page"] = text_chars["page"]
+    blocks["rotation"] = text_chars["rotation"]
+    blocks["bbox"] = text_chars["bbox"]
+    return blocks
+
+
+def inference(text_chars, model):
+    # Create generators and get first training row from each
+    generators = [infer_single_page(text_page) for text_page in text_chars]
+    next_prediction = {}
+
+    page_blocks = {}
+    while len(page_blocks) < len(generators):
+        training_data = {}
+        for page_idx, page_generator in enumerate(generators):
+            if page_idx in page_blocks:
+                continue
+
+            try:
+                if page_idx not in next_prediction:
+                    training_row = next(page_generator)
+                else:
+                    training_row = page_generator.send(next_prediction[page_idx])
+                    del next_prediction[page_idx]
+                training_data[page_idx] = training_row
+            except StopIteration as e:
+                blocks = e.value
+                page_blocks[page_idx] = blocks
+
+        if len(page_blocks) == len(generators):
+            break
+
+        training_list = sorted(training_data.items())
+        training_rows = [tl[1] for tl in training_list]
+        training_idxs = [tl[0] for tl in training_list]
+
+        predictions = model.predict(training_rows)
+        for pred, page_idx in zip(predictions, training_idxs):
+            next_prediction[page_idx] = pred
+    page_blocks = sorted(page_blocks.items())
+    page_blocks = [p[1] for p in page_blocks]
+    assert len(page_blocks) == len(text_chars)
+    return page_blocks

pdftext-0.1.0/pdftext/model.py

@@ -0,0 +1,7 @@
+import joblib
+from pdftext.settings import settings
+
+
+def get_model(model_path=settings.MODEL_PATH):
+    model = joblib.load(model_path)
+    return model

pdftext-0.1.0/pdftext/pdf/chars.py

@@ -0,0 +1,79 @@
+import decimal
+import math
+from typing import Dict
+
+import pypdfium2 as pdfium
+import pypdfium2.raw as pdfium_c
+
+from pdftext.pdf.utils import get_fontname, pdfium_page_bbox_to_device_bbox, page_bbox_to_device_bbox
+from pdftext.settings import settings
+
+
+def update_previous_fonts(text_chars: Dict, i: int, fontname: str, fontflags: int, prev_fontname: str, text_page, fontname_sample_freq: int):
+    min_update = max(0, i - fontname_sample_freq + 1) # Minimum index to update
+    regather_font_info = fontname != prev_fontname
+    for j in range(min_update, i): # Goes from min_update to i - 1
+        if regather_font_info:
+            fontname, fontflags = get_fontname(text_page, j)
+
+        # If we hit the region with the previous fontname, we can bail out
+        if fontname == prev_fontname:
+            break
+        text_chars["chars"][j]["font"]["name"] = fontname
+        text_chars["chars"][j]["font"]["flags"] = fontflags
+
+
+def get_pdfium_chars(pdf_path, fontname_sample_freq=settings.FONTNAME_SAMPLE_FREQ):
+    pdf = pdfium.PdfDocument(pdf_path)
+    blocks = []
+
+    for page_idx in range(len(pdf)):
+        page = pdf.get_page(page_idx)
+        text_page = page.get_textpage()
+
+        bbox = page.get_bbox()
+        page_width = math.ceil(bbox[2] - bbox[0])
+        page_height = math.ceil(abs(bbox[1] - bbox[3]))
+
+        text_chars = {
+            "chars": [],
+            "page": page_idx,
+            "rotation": page.get_rotation(),
+            "bbox": pdfium_page_bbox_to_device_bbox(page, bbox, page_width, page_height)
+        }
+
+        fontname = None
+        fontflags = None
+        total_chars = text_page.count_chars()
+        for i in range(total_chars):
+            char = pdfium_c.FPDFText_GetUnicode(text_page, i)
+            char = chr(char)
+            fontsize = round(pdfium_c.FPDFText_GetFontSize(text_page, i), 1)
+            fontweight = round(pdfium_c.FPDFText_GetFontWeight(text_page, i), 1)
+            if fontname is None or i % fontname_sample_freq == 0:
+                prev_fontname = fontname
+                fontname, fontflags = get_fontname(text_page, i)
+                update_previous_fonts(text_chars, i, fontname, fontflags, prev_fontname, text_page, fontname_sample_freq)
+
+            rotation = pdfium_c.FPDFText_GetCharAngle(text_page, i)
+            rotation = rotation * 180 / math.pi # convert from radians to degrees
+            coords = text_page.get_charbox(i, loose=True)
+            device_coords = page_bbox_to_device_bbox(page, coords, page_width, page_height, normalize=True)
+
+            char_info = {
+                "font": {
+                    "size": fontsize,
+                    "weight": fontweight,
+                    "name": fontname,
+                    "flags": fontflags
+                },
+                "rotation": rotation,
+                "char": char,
+                "bbox": device_coords,
+                "char_idx": i
+            }
+            text_chars["chars"].append(char_info)
+
+        text_chars["total_chars"] = total_chars
+        blocks.append(text_chars)
+    return blocks

pdftext-0.1.0/pdftext/pdf/utils.py

@@ -0,0 +1,97 @@
+import pypdfium2.raw as pdfium_c
+import ctypes
+import math
+
+from pdftext.settings import settings
+
+LINE_BREAKS = ["\n", "\u000D", "\u000A", "\u000C"]
+TABS = ["\t", "\u0009"]
+SPACES = [" ", "\ufffe", "\uFEFF", "\xa0"]
+HYPHEN = "-"
+WHITESPACE_CHARS = ["\n", "\r", "\f", "\t", " "]
+LIGATURES = {
+    "ff": "ff",
+    "ffi": "ffi",
+    "ffl": "ffl",
+    "fi": "fi",
+    "fl": "fl",
+    "st": "st",
+    "ſt": "st",
+}
+
+
+def char_count(textpage, *rect):
+    args = (textpage, *rect)
+    n_chars = pdfium_c.FPDFText_GetBoundedText(*args, None, 0)
+    if n_chars <= 0:
+        return 0
+    return n_chars
+
+
+def normalize_bbox(bbox, page_bound):
+    x1, y1, x2, y2 = bbox
+    x1 = x1 / page_bound[2]
+    y1 = y1 / page_bound[3]
+    x2 = x2 / page_bound[2]
+    y2 = y2 / page_bound[3]
+    return x1, y1, x2, y2
+
+
+def unnormalize_bbox(bbox, page_bound):
+    x1, y1, x2, y2 = bbox
+    x1 = round(x1 * page_bound[2], 1)
+    y1 = round(y1 * page_bound[3], 1)
+    x2 = round(x2 * page_bound[2], 1)
+    y2 = round(y2 * page_bound[3], 1)
+    return x1, y1, x2, y2
+
+
+def get_fontname(textpage, char_index):
+    n_bytes = settings.FONT_BUFFER_SIZE
+    buffer = ctypes.create_string_buffer(n_bytes)
+    # Re-interpret the type from char to unsigned short as required by the function
+    buffer_ptr = ctypes.cast(buffer, ctypes.POINTER(ctypes.c_ushort))
+    flag_buffer = ctypes.c_int()
+    flag_ptr = ctypes.pointer(flag_buffer)
+    font_info = pdfium_c.FPDFText_GetFontInfo(textpage, char_index, buffer_ptr, n_bytes, flag_ptr)
+    if font_info == 0:
+        return None, None
+    try:
+        decoded = buffer.value.decode("utf-8")
+    except Exception as e:
+        return None, None
+    return decoded, flag_buffer.value
+
+
+def page_to_device(page, x, y, page_width, page_height):
+    device_x = ctypes.c_int()
+    device_y = ctypes.c_int()
+    device_x_ptr = ctypes.pointer(device_x)
+    device_y_ptr = ctypes.pointer(device_y)
+    rotation = pdfium_c.FPDFPage_GetRotation(page)
+    width = math.ceil(page_width)
+    height = math.ceil(page_height)
+    pdfium_c.FPDF_PageToDevice(page, 0, 0, width, height, rotation, x, y, device_x_ptr, device_y_ptr)
+    x = device_x.value
+    y = device_y.value
+    return x, y
+
+
+def pdfium_page_bbox_to_device_bbox(page, bbox, page_width, page_height, normalize=False):
+    bbox_width = bbox[2] - bbox[0]
+    bbox_height = bbox[3] - bbox[1]
+    left_bottom = page_to_device(page, *bbox[:2], page_width, page_height)
+
+    dev_bbox = [left_bottom[0], left_bottom[1] - bbox_height, left_bottom[0] + bbox_width, left_bottom[1]]   # Convert to ltrb
+    if normalize:
+        dev_bbox = [dev_bbox[0] / page_width, dev_bbox[1] / page_height, dev_bbox[2] / page_width, dev_bbox[3] / page_height]
+    return dev_bbox
+
+
+def page_bbox_to_device_bbox(page, bbox, page_width, page_height, normalize=False):
+    left, bottom, right, top = bbox
+
+    dev_bbox = [left, page_height-top, right, page_height-bottom]
+    if normalize:
+        dev_bbox = [dev_bbox[0] / page_width, dev_bbox[1] / page_height, dev_bbox[2] / page_width, dev_bbox[3] / page_height]
+    return dev_bbox

pdftext-0.1.0/pdftext/postprocessing.py

@@ -0,0 +1,72 @@
+from typing import List, Dict
+import unicodedata
+
+from pdftext.pdf.utils import SPACES, LINE_BREAKS, TABS, WHITESPACE_CHARS, LIGATURES
+
+
+def postprocess_text(text: str) -> str:
+    text = replace_special_chars(text)
+    text = replace_control_chars(text)
+    text = replace_ligatures(text)
+    return text
+
+
+def replace_special_chars(text: str) -> str:
+    for item in SPACES:
+        text = text.replace(item, " ")
+    for item in LINE_BREAKS:
+        text = text.replace(item, "\n")
+    for item in TABS:
+        text = text.replace(item, "\t")
+    return text
+
+
+def replace_control_chars(text: str) -> str:
+    return "".join(char for char in text if unicodedata.category(char)[0] != "C" or char in WHITESPACE_CHARS)
+
+
+def replace_ligatures(text: str) -> str:
+    for ligature, replacement in LIGATURES.items():
+        text = text.replace(ligature, replacement)
+    return text
+
+
+def sort_blocks(blocks: List, tolerance=1.25) -> List:
+    # Sort blocks into best guess reading order
+    vertical_groups = {}
+    for block in blocks:
+        bbox = block["bbox"]
+        group_key = round(bbox[1] / tolerance) * tolerance
+        if group_key not in vertical_groups:
+            vertical_groups[group_key] = []
+        vertical_groups[group_key].append(block)
+
+    # Sort each group horizontally and flatten the groups into a single list
+    sorted_page_blocks = []
+    for _, group in sorted(vertical_groups.items()):
+        sorted_group = sorted(group, key=lambda x: x["bbox"][0])
+        sorted_page_blocks.extend(sorted_group)
+
+    return sorted_page_blocks
+
+
+def merge_text(page: Dict, sort=False) -> str:
+    text = ""
+    if sort:
+        page["blocks"] = sort_blocks(page["blocks"])
+
+    for block in page["blocks"]:
+        block_text = ""
+        for line in block["lines"]:
+            line_text = ""
+            for char in line["chars"]:
+                line_text += char["char"]
+            line_text = postprocess_text(line_text)
+            if line_text.endswith("\n"):
+                line_text = line_text[:-1].strip() + " "
+
+            block_text += line_text
+        if not block_text.endswith("\n"):
+            block_text += "\n\n"
+        text += block_text
+    return text

pdftext-0.1.0/pdftext/settings.py

@@ -0,0 +1,19 @@
+import os.path
+
+from pydantic_settings import BaseSettings
+
+
+class Settings(BaseSettings):
+    BASE_PATH: str = os.path.dirname(os.path.dirname(__file__))
+    MODEL_PATH: str = os.path.join(BASE_PATH, "models", "dt.joblib")
+
+    # How many characters to buffer when reading a font name
+    FONT_BUFFER_SIZE: int = 1024
+    FONTNAME_SAMPLE_FREQ: int = 10
+
+    # Benchmark
+    RESULTS_FOLDER: str = "results"
+    BENCH_DATASET_NAME: str = "vikp/pdf_bench"
+
+
+settings = Settings()

pdftext-0.1.0/pyproject.toml

@@ -0,0 +1,38 @@
+[tool.poetry]
+name = "pdftext"
+version = "0.1.0"
+description = "Extract structured text from pdfs quickly"
+authors = ["Vik Paruchuri <vik.paruchuri@gmail.com>"]
+license = "Apache-2.0"
+readme = "README.md"
+repository = "https://github.com/VikParuchuri/pdftext"
+keywords = ["pdf", "text", "extraction"]
+packages = [
+    {include = "pdftext"}
+]
+include = [
+    "extract_text.py",
+    "models/dt.joblib"
+]
+
+[tool.poetry.dependencies]
+python = ">=3.9,<3.13,!=3.9.7"
+pypdfium2 = "^4.29.0"
+scikit-learn = "^1.4.2"
+pydantic = "^2.7.1"
+pydantic-settings = "^2.2.1"
+
+[tool.poetry.group.dev.dependencies]
+pymupdf = "^1.24.2"
+datasets = "^2.19.0"
+pdfplumber = "^0.11.0"
+pillow = "^10.3.0"
+rapidfuzz = "^3.8.1"
+tabulate = "^0.9.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry.scripts]
+pdftext = "extract_text:main"