pdftext 0.2.0__tar.gz → 0.3.0__tar.gz

{pdftext-0.2.0 → pdftext-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pdftext
-Version: 0.2.0
+Version: 0.3.0
 Summary: Extract structured text from pdfs quickly
 Home-page: https://github.com/VikParuchuri/pdftext
 License: Apache-2.0
@@ -48,6 +48,7 @@ pdftext PDF_PATH --out_path output.txt
 - `PDF_PATH` must be a single pdf file.
 - `--out_path` path to the output txt file.  If not specified, will write to stdout.
 - `--sort` will attempt to sort in reading order if specified.
+- `--keep_hyphens` will keep hyphens in the output (they will be stripped and words joined otherwise)
 
 ## JSON
 
@@ -71,13 +72,14 @@ The output will be a json list, with each item in the list corresponding to a si
   - `bbox` - the block bbox, in `[x1, y1, x2, y2]` format
   - `lines` - the lines inside the block
     - `bbox` - the line bbox, in `[x1, y1, x2, y2]` format
-    - `chars` - the individual characters in the line
-      - `char` - the actual character, encoded in utf-8
-      - `rotation` - how much the character is rotated, in degrees
-      - `bbox` - the character bbox, in `[x1, y1, x2, y2]` format
-      - `char_idx` - the index of the character on the page (from `0` to number of characters, in original pdf order)
+    - `spans` - the individual text spans in the line (text spans have the same font/weight/etc)
+      - `text` - the text in the span, encoded in utf-8
+      - `rotation` - how much the span is rotated, in degrees
+      - `bbox` - the span bbox, in `[x1, y1, x2, y2]` format
+      - `char_start_idx` - the start index of the first span character in the pdf
+      - `char_end_idx` - the end index of the last span character in the pdf
       - `font` this is font info straight from the pdf, see [this pdfium code](https://pdfium.googlesource.com/pdfium/+/refs/heads/main/public/fpdf_text.h)
-        - `size` - the size of the font used for the character
+        - `size` - the size of the font used for the text
         - `weight` - font weight
         - `name` - font name, may be None
         - `flags` - font flags, in the format of the `PDF spec 1.7 Section 5.7.1 Font Descriptor Flags`

{pdftext-0.2.0 → pdftext-0.3.0}/README.md

@@ -25,6 +25,7 @@ pdftext PDF_PATH --out_path output.txt
 - `PDF_PATH` must be a single pdf file.
 - `--out_path` path to the output txt file.  If not specified, will write to stdout.
 - `--sort` will attempt to sort in reading order if specified.
+- `--keep_hyphens` will keep hyphens in the output (they will be stripped and words joined otherwise)
 
 ## JSON
 
@@ -48,13 +49,14 @@ The output will be a json list, with each item in the list corresponding to a si
   - `bbox` - the block bbox, in `[x1, y1, x2, y2]` format
   - `lines` - the lines inside the block
     - `bbox` - the line bbox, in `[x1, y1, x2, y2]` format
-    - `chars` - the individual characters in the line
-      - `char` - the actual character, encoded in utf-8
-      - `rotation` - how much the character is rotated, in degrees
-      - `bbox` - the character bbox, in `[x1, y1, x2, y2]` format
-      - `char_idx` - the index of the character on the page (from `0` to number of characters, in original pdf order)
+    - `spans` - the individual text spans in the line (text spans have the same font/weight/etc)
+      - `text` - the text in the span, encoded in utf-8
+      - `rotation` - how much the span is rotated, in degrees
+      - `bbox` - the span bbox, in `[x1, y1, x2, y2]` format
+      - `char_start_idx` - the start index of the first span character in the pdf
+      - `char_end_idx` - the end index of the last span character in the pdf
       - `font` this is font info straight from the pdf, see [this pdfium code](https://pdfium.googlesource.com/pdfium/+/refs/heads/main/public/fpdf_text.h)
-        - `size` - the size of the font used for the character
+        - `size` - the size of the font used for the text
         - `weight` - font weight
         - `name` - font name, may be None
         - `flags` - font flags, in the format of the `PDF spec 1.7 Section 5.7.1 Font Descriptor Flags`

{pdftext-0.2.0 → pdftext-0.3.0}/extract_text.py

@@ -10,12 +10,13 @@ def main():
     parser.add_argument("--out_path", type=str, help="Path to the output text file, defaults to stdout", default=None)
     parser.add_argument("--output_type", type=str, help="Type of output to generate", default="plain_text")
     parser.add_argument("--sort", action="store_true", help="Attempt to sort the text by reading order", default=False)
+    parser.add_argument("--keep_hyphens", action="store_true", help="Keep hyphens in words", default=False)
     args = parser.parse_args()
 
     assert args.output_type in ["plain_text", "json"], "Invalid output type, must be 'plain_text' or 'json'"
 
     if args.output_type == "plain_text":
-        text = plain_text_output(args.pdf_path, sort=args.sort)
+        text = plain_text_output(args.pdf_path, sort=args.sort, hyphens=args.keep_hyphens)
     elif args.output_type == "json":
         text = dictionary_output(args.pdf_path, sort=args.sort)
         text = json.dumps(text)

{pdftext-0.2.0 → pdftext-0.3.0}/pdftext/extraction.py

@@ -4,7 +4,7 @@ from pdftext.inference import inference
 from pdftext.model import get_model
 from pdftext.pdf.chars import get_pdfium_chars
 from pdftext.pdf.utils import unnormalize_bbox
-from pdftext.postprocessing import merge_text, sort_blocks, postprocess_text
+from pdftext.postprocessing import merge_text, sort_blocks, postprocess_text, handle_hyphens
 
 
 def _get_pages(pdf_path, model=None):
@@ -15,16 +15,16 @@ def _get_pages(pdf_path, model=None):
     return pages
 
 
-def plain_text_output(pdf_path, sort=False, model=None) -> str:
-    text = paginated_plain_text_output(pdf_path, sort=sort, model=model)
+def plain_text_output(pdf_path, sort=False, model=None, hyphens=False) -> str:
+    text = paginated_plain_text_output(pdf_path, sort=sort, model=model, hyphens=hyphens)
     return "\n".join(text)
 
 
-def paginated_plain_text_output(pdf_path, sort=False, model=None) -> List[str]:
+def paginated_plain_text_output(pdf_path, sort=False, model=None, hyphens=False) -> List[str]:
     pages = _get_pages(pdf_path, model)
     text = []
     for page in pages:
-        text.append(merge_text(page, sort=sort).strip())
+        text.append(merge_text(page, sort=sort, hyphens=hyphens).strip())
     return text
 
 
@@ -36,12 +36,14 @@ def dictionary_output(pdf_path, sort=False, model=None):
             for key in bad_keys:
                 del block[key]
             for line in block["lines"]:
-                bad_keys = [key for key in line.keys() if key not in ["chars", "bbox"]]
+                bad_keys = [key for key in line.keys() if key not in ["bbox", "spans"]]
                 for key in bad_keys:
                     del line[key]
-                for char in line["chars"]:
-                    char["bbox"] = unnormalize_bbox(char["bbox"], page["width"], page["height"])
-                    char["char"] = postprocess_text(char["char"])
+                for span in line["spans"]:
+                    span["bbox"] = unnormalize_bbox(span["bbox"], page["width"], page["height"])
+                    span["text"] = postprocess_text(span["text"])
+                    span["text"] = handle_hyphens(span["text"], keep_hyphens=True)
+
                 line["bbox"] = unnormalize_bbox(line["bbox"], page["width"], page["height"])
             block["bbox"] = unnormalize_bbox(block["bbox"], page["width"], page["height"])
         if sort:

{pdftext-0.2.0 → pdftext-0.3.0}/pdftext/inference.py

@@ -68,14 +68,25 @@ def create_training_row(char_info, prev_char, currblock, currline):
 
 
 def update_span(line, span):
+    if len(span["chars"]) > 0:
+        span["font"] = span["chars"][0]["font"]
+        span["rotation"] = span["chars"][0]["rotation"]
+        char_bboxes = [char["bbox"] for char in span["chars"]]
+        span["bbox"] = [min([bbox[0] for bbox in char_bboxes]),
+                        min([bbox[1] for bbox in char_bboxes]),
+                        max([bbox[2] for bbox in char_bboxes]),
+                        max([bbox[3] for bbox in char_bboxes])]
+        span["text"] = "".join([char["char"] for char in span["chars"]])
+        span["char_start_idx"] = span["chars"][0]["char_idx"]
+        span["char_end_idx"] = span["chars"][-1]["char_idx"]
+
+    del span["chars"]
     line["spans"].append(span)
     span = {"chars": []}
     return span
 
 
 def update_line(block, line):
-    line["chars"] = list(chain.from_iterable(s["chars"] for s in line["spans"]))
-    del line["spans"]
     block["lines"].append(line)
     line = {"spans": []}
     return line
@@ -89,6 +100,7 @@ def update_block(blocks, block):
 
 def infer_single_page(text_chars):
     prev_char = None
+    prev_font_info = None
 
     blocks = {
         "blocks": [],
@@ -102,6 +114,7 @@ def infer_single_page(text_chars):
     line = {"spans": []}
     span = {"chars": []}
     for i, char_info in enumerate(text_chars["chars"]):
+        font_info = f"{char_info['font']['name']}_{char_info['font']['size']}_{char_info['font']['weight']}_{char_info['font']['flags']}_{char_info['rotation']}"
         if prev_char:
             training_row = create_training_row(char_info, prev_char, block, line)
             sorted_keys = sorted(training_row.keys())
@@ -110,21 +123,22 @@ def infer_single_page(text_chars):
             prediction = yield training_row
             if prediction == 0:
                 pass
-            elif prediction == 1:
-                span = update_span(line, span)
-            elif prediction == 2:
+            elif prediction == 2 and prev_char["char"] in LINE_BREAKS:
                 span = update_span(line, span)
                 line = update_line(block, line)
-            else:
+                block = update_block(blocks, block)
+            elif prev_char["char"] in LINE_BREAKS and prediction == 1: # Look for newline character as a forcing signal for a new line
                 span = update_span(line, span)
                 line = update_line(block, line)
-                block = update_block(blocks, block)
+            elif prev_font_info != font_info:
+                span = update_span(line, span)
 
         span["chars"].append(char_info)
         update_current(line, char_info)
         update_current(block, char_info)
 
         prev_char = char_info
+        prev_font_info = font_info
     if len(span["chars"]) > 0:
         update_span(line, span)
     if len(line["spans"]) > 0:

{pdftext-0.2.0 → pdftext-0.3.0}/pdftext/pdf/chars.py

@@ -74,7 +74,7 @@ def get_pdfium_chars(pdf_path, fontname_sample_freq=settings.FONTNAME_SAMPLE_FRE
 
             rotation = pdfium_c.FPDFText_GetCharAngle(text_page, i)
             rotation = rotation * 180 / math.pi # convert from radians to degrees
-            coords = text_page.get_charbox(i, loose=False)
+            coords = text_page.get_charbox(i, loose=True)
             device_coords = page_bbox_to_device_bbox(page, coords, page_width, page_height, bl_origin, page_rotation, normalize=True)
 
             char_info = {

{pdftext-0.2.0 → pdftext-0.3.0}/pdftext/pdf/utils.py

@@ -4,20 +4,10 @@ import math
 
 from pdftext.settings import settings
 
-LINE_BREAKS = ["\n", "\u000D", "\u000A", "\u000C"]
-TABS = ["\t", "\u0009"]
+LINE_BREAKS = ["\n", "\u000D", "\u000A"]
+TABS = ["\t", "\u0009", "\x09"]
 SPACES = [" ", "\ufffe", "\uFEFF", "\xa0"]
-HYPHEN = "-"
 WHITESPACE_CHARS = ["\n", "\r", "\f", "\t", " "]
-LIGATURES = {
-    "ff": "ff",
-    "ffi": "ffi",
-    "ffl": "ffl",
-    "fi": "fi",
-    "fl": "fl",
-    "st": "st",
-    "ſt": "st",
-}
 
 
 def char_count(textpage, *rect):

{pdftext-0.2.0 → pdftext-0.3.0}/pdftext/postprocessing.py

@@ -1,16 +1,57 @@
 from typing import List, Dict
 import unicodedata
 
-from pdftext.pdf.utils import SPACES, LINE_BREAKS, TABS, WHITESPACE_CHARS, LIGATURES
+from pdftext.pdf.utils import SPACES, LINE_BREAKS, TABS, WHITESPACE_CHARS
+
+LIGATURES = {
+    "ff": "ff",
+    "ffi": "ffi",
+    "ffl": "ffl",
+    "fi": "fi",
+    "fl": "fl",
+    "st": "st",
+    "ſt": "st",
+}
+HYPHEN_CHAR = "\x02"
+REPLACEMENTS = {
+    "\r\n": "\n",
+}
 
 
 def postprocess_text(text: str) -> str:
+    for old, new in REPLACEMENTS.items():
+        text = text.replace(old, new)
     text = replace_special_chars(text)
     text = replace_control_chars(text)
     text = replace_ligatures(text)
     return text
 
 
+def handle_hyphens(text: str, keep_hyphens=False) -> str:
+    if keep_hyphens:
+        text = text.replace(HYPHEN_CHAR, "-")
+    elif len(text) == 0:
+        pass
+    else:
+        new_text = ""
+        found_hyphen = False
+        for i in range(len(text) - 1):
+            if text[i] == HYPHEN_CHAR and text[i+1] in LINE_BREAKS:
+                found_hyphen = True
+            elif found_hyphen:
+                if text[i] in LINE_BREAKS:
+                    pass
+                elif text[i] in SPACES:
+                    new_text = new_text.rstrip() + "\n"
+                    found_hyphen = False
+                else:
+                    new_text += text[i]
+            else:
+                new_text += text[i]
+        text = new_text
+    return text
+
+
 def replace_special_chars(text: str) -> str:
     for item in SPACES:
         text = text.replace(item, " ")
@@ -22,7 +63,7 @@ def replace_special_chars(text: str) -> str:
 
 
 def replace_control_chars(text: str) -> str:
-    return "".join(char for char in text if unicodedata.category(char)[0] != "C" or char in WHITESPACE_CHARS)
+    return "".join(char for char in text if (unicodedata.category(char)[0] != "C" or char == HYPHEN_CHAR or char in WHITESPACE_CHARS))
 
 
 def replace_ligatures(text: str) -> str:
@@ -50,7 +91,7 @@ def sort_blocks(blocks: List, tolerance=1.25) -> List:
     return sorted_page_blocks
 
 
-def merge_text(page: Dict, sort=False) -> str:
+def merge_text(page: Dict, sort=False, hyphens=False) -> str:
     text = ""
     if sort:
         page["blocks"] = sort_blocks(page["blocks"])
@@ -59,12 +100,13 @@ def merge_text(page: Dict, sort=False) -> str:
         block_text = ""
         for line in block["lines"]:
             line_text = ""
-            for char in line["chars"]:
-                line_text += char["char"]
+            for span in line["spans"]:
+                line_text += span["text"]
             line_text = postprocess_text(line_text)
             line_text = line_text.rstrip() + "\n"
 
             block_text += line_text
         block_text = block_text.rstrip() + "\n\n"
         text += block_text
+    text = handle_hyphens(text, keep_hyphens=hyphens)
     return text

{pdftext-0.2.0 → pdftext-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pdftext"
-version = "0.2.0"
+version = "0.3.0"
 description = "Extract structured text from pdfs quickly"
 authors = ["Vik Paruchuri <vik.paruchuri@gmail.com>"]
 license = "Apache-2.0"