pdftext 0.1.2__tar.gz → 0.2.0__tar.gz

{pdftext-0.1.2 → pdftext-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pdftext
-Version: 0.1.2
+Version: 0.2.0
 Summary: Extract structured text from pdfs quickly
 Home-page: https://github.com/VikParuchuri/pdftext
 License: Apache-2.0
@@ -82,6 +82,8 @@ The output will be a json list, with each item in the list corresponding to a si
         - `name` - font name, may be None
         - `flags` - font flags, in the format of the `PDF spec 1.7 Section 5.7.1 Font Descriptor Flags`
 
+If the pdf is rotated, the bboxes will be relative to the rotated page (they're rotated after being extracted).
+
 # Programmatic usage
 
 Extract plain text:

{pdftext-0.1.2 → pdftext-0.2.0}/README.md

@@ -59,6 +59,8 @@ The output will be a json list, with each item in the list corresponding to a si
         - `name` - font name, may be None
         - `flags` - font flags, in the format of the `PDF spec 1.7 Section 5.7.1 Font Descriptor Flags`
 
+If the pdf is rotated, the bboxes will be relative to the rotated page (they're rotated after being extracted).
+
 # Programmatic usage
 
 Extract plain text:

{pdftext-0.1.2 → pdftext-0.2.0}/pdftext/extraction.py

@@ -36,24 +36,14 @@ def dictionary_output(pdf_path, sort=False, model=None):
             for key in bad_keys:
                 del block[key]
             for line in block["lines"]:
-                line_box = None
                 bad_keys = [key for key in line.keys() if key not in ["chars", "bbox"]]
                 for key in bad_keys:
                     del line[key]
                 for char in line["chars"]:
-                    char["bbox"] = unnormalize_bbox(char["bbox"], page["bbox"])
+                    char["bbox"] = unnormalize_bbox(char["bbox"], page["width"], page["height"])
                     char["char"] = postprocess_text(char["char"])
-                    if line_box is None:
-                        line_box = char["bbox"]
-                    else:
-                        line_box = [
-                            min(line_box[0], char["bbox"][0]),
-                            min(line_box[1], char["bbox"][1]),
-                            max(line_box[2], char["bbox"][2]),
-                            max(line_box[3], char["bbox"][3]),
-                        ]
-                line["bbox"] = line_box
-            block["bbox"] = unnormalize_bbox(block["bbox"], page["bbox"])
+                line["bbox"] = unnormalize_bbox(line["bbox"], page["width"], page["height"])
+            block["bbox"] = unnormalize_bbox(block["bbox"], page["width"], page["height"])
         if sort:
             page["blocks"] = sort_blocks(page["blocks"])
     return pages

{pdftext-0.1.2 → pdftext-0.2.0}/pdftext/inference.py

@@ -90,7 +90,14 @@ def update_block(blocks, block):
 def infer_single_page(text_chars):
     prev_char = None
 
-    blocks = {"blocks": []}
+    blocks = {
+        "blocks": [],
+        "page": text_chars["page"],
+        "rotation": text_chars["rotation"],
+        "bbox": text_chars["bbox"],
+        "width": text_chars["width"],
+        "height": text_chars["height"],
+    }
     block = {"lines": []}
     line = {"spans": []}
     span = {"chars": []}
@@ -125,9 +132,6 @@ def infer_single_page(text_chars):
     if len(block["lines"]) > 0:
         update_block(blocks, block)
 
-    blocks["page"] = text_chars["page"]
-    blocks["rotation"] = text_chars["rotation"]
-    blocks["bbox"] = text_chars["bbox"]
     return blocks
 
 

{pdftext-0.1.2 → pdftext-0.2.0}/pdftext/pdf/chars.py

@@ -31,17 +31,32 @@ def get_pdfium_chars(pdf_path, fontname_sample_freq=settings.FONTNAME_SAMPLE_FRE
         page = pdf.get_page(page_idx)
         text_page = page.get_textpage()
         mediabox = page.get_mediabox()
-        bl_origin = mediabox[0] == 0 and mediabox[1] == 0
-
+        page_rotation = page.get_rotation()
         bbox = page.get_bbox()
-        page_width = math.ceil(bbox[2] - bbox[0])
+        page_width = math.ceil(abs(bbox[2] - bbox[0]))
+        page_height = math.ceil(abs(bbox[1] - bbox[3]))
+        bbox = pdfium_page_bbox_to_device_bbox(page, bbox, page_width, page_height, page_rotation)
+
+        # Recalculate page width and height with new bboxes
+        page_width = math.ceil(abs(bbox[2] - bbox[0]))
         page_height = math.ceil(abs(bbox[1] - bbox[3]))
 
+        # Flip width and height if rotated
+        if page_rotation == 90 or page_rotation == 270:
+            page_width, page_height = page_height, page_width
+
+        bl_origin = all([
+            mediabox[0] == 0,
+            mediabox[1] == 0
+        ])
+
         text_chars = {
             "chars": [],
             "page": page_idx,
-            "rotation": page.get_rotation(),
-            "bbox": pdfium_page_bbox_to_device_bbox(page, bbox, page_width, page_height)
+            "rotation": page_rotation,
+            "bbox": bbox,
+            "width": page_width,
+            "height": page_height,
         }
 
         fontname = None
@@ -59,8 +74,8 @@ def get_pdfium_chars(pdf_path, fontname_sample_freq=settings.FONTNAME_SAMPLE_FRE
 
             rotation = pdfium_c.FPDFText_GetCharAngle(text_page, i)
             rotation = rotation * 180 / math.pi # convert from radians to degrees
-            coords = text_page.get_charbox(i, loose=True)
-            device_coords = page_bbox_to_device_bbox(page, coords, page_width, page_height, bl_origin, normalize=True)
+            coords = text_page.get_charbox(i, loose=False)
+            device_coords = page_bbox_to_device_bbox(page, coords, page_width, page_height, bl_origin, page_rotation, normalize=True)
 
             char_info = {
                 "font": {

pdftext-0.2.0/pdftext/pdf/utils.py

@@ -0,0 +1,148 @@
+import pypdfium2.raw as pdfium_c
+import ctypes
+import math
+
+from pdftext.settings import settings
+
+LINE_BREAKS = ["\n", "\u000D", "\u000A", "\u000C"]
+TABS = ["\t", "\u0009"]
+SPACES = [" ", "\ufffe", "\uFEFF", "\xa0"]
+HYPHEN = "-"
+WHITESPACE_CHARS = ["\n", "\r", "\f", "\t", " "]
+LIGATURES = {
+    "ff": "ff",
+    "ffi": "ffi",
+    "ffl": "ffl",
+    "fi": "fi",
+    "fl": "fl",
+    "st": "st",
+    "ſt": "st",
+}
+
+
+def char_count(textpage, *rect):
+    args = (textpage, *rect)
+    n_chars = pdfium_c.FPDFText_GetBoundedText(*args, None, 0)
+    if n_chars <= 0:
+        return 0
+    return n_chars
+
+
+def normalize_bbox(bbox, page_bound):
+    x1, y1, x2, y2 = bbox
+    x1 = x1 / page_bound[2]
+    y1 = y1 / page_bound[3]
+    x2 = x2 / page_bound[2]
+    y2 = y2 / page_bound[3]
+    return x1, y1, x2, y2
+
+
+def unnormalize_bbox(bbox, page_width, page_height):
+    x1, y1, x2, y2 = bbox
+    x1 = round(x1 * page_width, 1)
+    y1 = round(y1 * page_height, 1)
+    x2 = round(x2 * page_width, 1)
+    y2 = round(y2 * page_height, 1)
+    return x1, y1, x2, y2
+
+
+def get_fontname(textpage, char_index):
+    n_bytes = settings.FONT_BUFFER_SIZE
+    buffer = ctypes.create_string_buffer(n_bytes)
+    # Re-interpret the type from char to unsigned short as required by the function
+    buffer_ptr = ctypes.cast(buffer, ctypes.POINTER(ctypes.c_ushort))
+    flag_buffer = ctypes.c_int()
+    flag_ptr = ctypes.pointer(flag_buffer)
+    font_info = pdfium_c.FPDFText_GetFontInfo(textpage, char_index, buffer_ptr, n_bytes, flag_ptr)
+    if font_info == 0:
+        return None, None
+    try:
+        decoded = buffer.value.decode("utf-8")
+    except Exception as e:
+        return None, None
+    return decoded, flag_buffer.value
+
+
+def page_to_device(page, x, y, page_width, page_height, page_rotation: int):
+    if page_rotation == 90:
+        page_rotation = 1
+    elif page_rotation == 180:
+        page_rotation = 2
+    elif page_rotation == 270:
+        page_rotation = 3
+    else:
+        page_rotation = 0
+    device_x = ctypes.c_int()
+    device_y = ctypes.c_int()
+    device_x_ptr = ctypes.pointer(device_x)
+    device_y_ptr = ctypes.pointer(device_y)
+    width = math.ceil(page_width)
+    height = math.ceil(page_height)
+    pdfium_c.FPDF_PageToDevice(page, 0, 0, width, height, page_rotation, x, y, device_x_ptr, device_y_ptr)
+    x = device_x.value
+    y = device_y.value
+    return x, y
+
+
+def pdfium_page_bbox_to_device_bbox(page, bbox, page_width, page_height, page_rotation):
+    left_bottom = page_to_device(page, *bbox[:2], page_width, page_height, page_rotation)
+    top_right = page_to_device(page, *bbox[2:], page_width, page_height, page_rotation)
+
+    dev_bbox = [left_bottom[0], top_right[1], top_right[0], left_bottom[1]]
+    return dev_bbox
+
+
+def fast_page_bbox_to_device_bbox(page, bbox, page_width, page_height):
+    left, bottom, right, top = bbox
+
+    dev_bbox = [left, page_height-top, right, page_height-bottom]
+    return dev_bbox
+
+
+def page_bbox_to_device_bbox(page, bbox, page_width: int, page_height: int, bl_origin: bool, page_rotation: int, normalize=False):
+    orig_page_height, orig_page_width = page_height, page_width
+    if page_rotation in [90, 270]:
+        orig_page_height, orig_page_width = page_width, page_height
+
+    if bl_origin:
+        bbox = fast_page_bbox_to_device_bbox(page, bbox, page_width, page_height)
+        if page_rotation > 0:
+            bbox = rotate_page_bbox(bbox, page_rotation, page_width, page_height)
+    else:
+        bbox = pdfium_page_bbox_to_device_bbox(page, bbox, orig_page_width, orig_page_height, page_rotation)
+        if page_rotation > 0:
+            bbox = rotate_pdfium_bbox(bbox, page_rotation, page_width, page_height)
+
+    if normalize:
+        bbox = [bbox[0] / page_width, bbox[1] / page_height, bbox[2] / page_width, bbox[3] / page_height]
+    return bbox
+
+
+def rotate_pdfium_bbox(bbox, angle_deg, width, height):
+    x1, y1, x2, y2 = bbox
+    if angle_deg == 90:
+        bbox = [y1, x1, y2, x2]
+        bbox = [bbox[2], height - bbox[1], bbox[0], height - bbox[3]]
+    elif angle_deg == 180:
+        bbox = [x2, y2, x1, y1]
+        bbox = [width - bbox[0], height - bbox[1], width - bbox[2], height - bbox[3]]
+    elif angle_deg == 270:
+        bbox = rotate_pdfium_bbox(bbox, 90, width, height)
+        bbox = rotate_pdfium_bbox(bbox, 180, width, height)
+
+    return bbox
+
+
+def rotate_page_bbox(bbox, angle_deg, width, height):
+    x1, y1, x2, y2 = bbox
+    if angle_deg == 90:
+        bbox = [y1, x1, y2, x2]
+        bbox = [height - bbox[2], bbox[1], height - bbox[0], bbox[3]]
+    elif angle_deg == 180:
+        bbox = [x2, y2, x1, y1]
+        bbox = [width - bbox[0], height - bbox[1], width - bbox[2], height - bbox[3]]
+    elif angle_deg == 270:
+        bbox = rotate_page_bbox(bbox, 90, width, height)
+        bbox = rotate_page_bbox(bbox, 180, width, height)
+
+    return bbox

{pdftext-0.1.2 → pdftext-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pdftext"
-version = "0.1.2"
+version = "0.2.0"
 description = "Extract structured text from pdfs quickly"
 authors = ["Vik Paruchuri <vik.paruchuri@gmail.com>"]
 license = "Apache-2.0"

pdftext-0.1.2/pdftext/pdf/utils.py

@@ -1,102 +0,0 @@
-import pypdfium2.raw as pdfium_c
-import ctypes
-import math
-
-from pdftext.settings import settings
-
-LINE_BREAKS = ["\n", "\u000D", "\u000A", "\u000C"]
-TABS = ["\t", "\u0009"]
-SPACES = [" ", "\ufffe", "\uFEFF", "\xa0"]
-HYPHEN = "-"
-WHITESPACE_CHARS = ["\n", "\r", "\f", "\t", " "]
-LIGATURES = {
-    "ff": "ff",
-    "ffi": "ffi",
-    "ffl": "ffl",
-    "fi": "fi",
-    "fl": "fl",
-    "st": "st",
-    "ſt": "st",
-}
-
-
-def char_count(textpage, *rect):
-    args = (textpage, *rect)
-    n_chars = pdfium_c.FPDFText_GetBoundedText(*args, None, 0)
-    if n_chars <= 0:
-        return 0
-    return n_chars
-
-
-def normalize_bbox(bbox, page_bound):
-    x1, y1, x2, y2 = bbox
-    x1 = x1 / page_bound[2]
-    y1 = y1 / page_bound[3]
-    x2 = x2 / page_bound[2]
-    y2 = y2 / page_bound[3]
-    return x1, y1, x2, y2
-
-
-def unnormalize_bbox(bbox, page_bound):
-    x1, y1, x2, y2 = bbox
-    x1 = round(x1 * page_bound[2], 1)
-    y1 = round(y1 * page_bound[3], 1)
-    x2 = round(x2 * page_bound[2], 1)
-    y2 = round(y2 * page_bound[3], 1)
-    return x1, y1, x2, y2
-
-
-def get_fontname(textpage, char_index):
-    n_bytes = settings.FONT_BUFFER_SIZE
-    buffer = ctypes.create_string_buffer(n_bytes)
-    # Re-interpret the type from char to unsigned short as required by the function
-    buffer_ptr = ctypes.cast(buffer, ctypes.POINTER(ctypes.c_ushort))
-    flag_buffer = ctypes.c_int()
-    flag_ptr = ctypes.pointer(flag_buffer)
-    font_info = pdfium_c.FPDFText_GetFontInfo(textpage, char_index, buffer_ptr, n_bytes, flag_ptr)
-    if font_info == 0:
-        return None, None
-    try:
-        decoded = buffer.value.decode("utf-8")
-    except Exception as e:
-        return None, None
-    return decoded, flag_buffer.value
-
-
-def page_to_device(page, x, y, page_width, page_height):
-    device_x = ctypes.c_int()
-    device_y = ctypes.c_int()
-    device_x_ptr = ctypes.pointer(device_x)
-    device_y_ptr = ctypes.pointer(device_y)
-    rotation = pdfium_c.FPDFPage_GetRotation(page)
-    width = math.ceil(page_width)
-    height = math.ceil(page_height)
-    pdfium_c.FPDF_PageToDevice(page, 0, 0, width, height, rotation, x, y, device_x_ptr, device_y_ptr)
-    x = device_x.value
-    y = device_y.value
-    return x, y
-
-
-def pdfium_page_bbox_to_device_bbox(page, bbox, page_width, page_height, normalize=False):
-    left_bottom = page_to_device(page, *bbox[:2], page_width, page_height)
-    top_right = page_to_device(page, *bbox[2:], page_width, page_height)
-
-    dev_bbox = [left_bottom[0], top_right[1], top_right[0], left_bottom[1]]
-    if normalize:
-        dev_bbox = [dev_bbox[0] / page_width, dev_bbox[1] / page_height, dev_bbox[2] / page_width, dev_bbox[3] / page_height]
-    return dev_bbox
-
-
-def fast_page_bbox_to_device_bbox(page, bbox, page_width, page_height, normalize=False):
-    left, bottom, right, top = bbox
-
-    dev_bbox = [left, page_height-top, right, page_height-bottom]
-    if normalize:
-        dev_bbox = [dev_bbox[0] / page_width, dev_bbox[1] / page_height, dev_bbox[2] / page_width, dev_bbox[3] / page_height]
-    return dev_bbox
-
-
-def page_bbox_to_device_bbox(page, bbox, page_width: int, page_height: int, bl_origin: bool, normalize=False):
-    if bl_origin:
-        return fast_page_bbox_to_device_bbox(page, bbox, page_width, page_height, normalize)
-    return pdfium_page_bbox_to_device_bbox(page, bbox, page_width, page_height, normalize)