extract-python 0.1.0__tar.gz → 0.3.0__tar.gz

extract_python-0.3.0/.dockerignore

@@ -0,0 +1,6 @@
+*
+!extract_python
+!uv.lock
+!pyproject.toml
+!README.md
+

extract_python-0.3.0/.github/workflows/publish.yml

@@ -0,0 +1,45 @@
+name: Publish extra-python
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  create-release:
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: 3.12
+      ASTRAL_VERSION: 0.11.6
+    steps:
+      - uses: actions/checkout@v6
+      - name: Create GH release
+        run: gh release create "$tag" --generate-notes
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ github.ref_name }}
+
+  publish-extra-python-to-pypi:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    environment:
+      name: pypi
+      url: https://pypi.org/p/extract-python
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: "0.10.8"
+      - name: Install Python 3.12
+        run: uv python install 3.12
+      - name: Build
+        run: uv build
+      - name: Publish
+        run: uv publish
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false

extract_python-0.3.0/.github/workflows/tests.yml

@@ -0,0 +1,79 @@
+name: Tests extract-python
+
+on:
+  push:
+    branches: [ 'main' ]
+  pull_request:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/ruff-action@v3
+        with:
+          args: "--version" # skips test by displaying the version
+      - name: Check formatting
+        run: ruff format --config qa/ruff.toml --check extract_python tests
+      - name: Lint test
+        run: ruff check --config qa/ruff.toml extract_python tests
+
+  tests-no-miner-u:
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: 3.12
+      ASTRAL_VERSION: 0.11.6
+    steps:
+      - uses: actions/checkout@v6
+      - name: Setup Python project
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: ${{ env.ASTRAL_VERSION }}
+          python-version: ${{ env.PYTHON_VERSION }}
+          enable-cache: true
+      - name: Install tesseract
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --fix-missing tesseract-ocr \
+              tesseract-ocr-eng \
+              tesseract-ocr-fra \
+              tesseract-ocr-deu \
+              tesseract-ocr-spa \
+              tesseract-ocr-lat \
+              tesseract-ocr-jpn \
+              libtesseract-dev \
+              libleptonica-dev \
+              pkg-config
+          echo "TESSDATA_PREFIX=$(sudo dpkg -L tesseract-ocr-eng | grep tessdata$)" >> $GITHUB_ENV
+      - name: Run tests
+        run: |
+          uv run --dev --extra docling --extra marker --frozen pytest -m "not miner_u" -vvv --cache-clear --show-capture=all -r A tests
+
+  tests-miner-u:
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: 3.12
+      ASTRAL_VERSION: 0.11.6
+    steps:
+      - uses: actions/checkout@v6
+      - name: Setup Python project
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: ${{ env.ASTRAL_VERSION }}
+          python-version: ${{ env.PYTHON_VERSION }}
+          enable-cache: true
+      - name: Run tests
+        run: |
+          uv run --dev --extra mineru --frozen pytest -m "miner_u" -vvv --cache-clear --show-capture=all -r A tests
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true

extract_python-0.3.0/.python-version

@@ -0,0 +1 @@
+3.12

extract_python-0.3.0/Dockerfile

@@ -0,0 +1,76 @@
+# syntax=docker/dockerfile:1.14.0
+FROM python:3.10-slim-bullseye AS python-base
+
+ENV HOME=/home/user
+WORKDIR $HOME
+RUN apt-get update && apt-get install -y curl
+
+RUN curl -LsSf https://astral.sh/uv/0.6.7/install.sh | sh
+ENV PATH="$HOME/.local/bin:$PATH"
+ENV UV_LINK_MODE=copy
+ENV UV_COMPILE_BYTECODE=1
+
+FROM python-base AS worker-base
+
+ARG dbmate_arch
+WORKDIR $HOME/src/app
+RUN curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v2.19.0/dbmate-linux-${dbmate_arch} \
+    && chmod +x /usr/local/bin/dbmate
+
+FROM worker-base AS worker-cpu
+# TODO: add more languages here
+RUN apt-get install -y tesseract-ocr \
+    tesseract-ocr-eng \
+    tesseract-ocr-fra \
+    tesseract-ocr-deu \
+    tesseract-ocr-spa \
+    tesseract-ocr-lat \
+    tesseract-ocr-jpn \
+    libtesseract-dev \
+    libleptonica-dev \
+    pkg-config
+ENV TESSDATA_PREFIX=/usr/share/tesseract-ocr/4.00/tessdata
+# We skip opencv since we already depend on opencv-python-headless which is the lib we need to use
+# Install deps first to optimize layer cache
+RUN --mount=type=cache,target=~/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync -v --frozen --no-editable --no-sources --no-install-project --no-install-package opencv-python --extra cpu
+RUN uv run --no-sync docling-tools models download -o ~/.cache/docling/models
+# Then copy code
+ADD uv.lock pyproject.toml README.md ./
+ADD extract_python ./extract_python/
+# Then install service
+RUN uv sync -v --frozen --no-editable --no-sources --no-install-package opencv-python --extra cpu
+
+RUN rm -rf ~/.cache/pip $(uv cache dir)
+
+ENTRYPOINT ["uv", "run", "--no-sync", "icij-worker", "workers", "start", "-g", "cpu", "extract_python.app:app"]
+
+
+FROM worker-base AS worker-miner-u
+RUN apt-get update && apt-get install -y wget
+# We skip opencv since we already depend on opencv-python-headless which is the lib we need to use
+# Install deps first to optimize layer cache
+RUN --mount=type=cache,target=~/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync -v --frozen --no-editable --no-sources --no-install-project --no-install-package opencv-python --extra miner-u
+# TODO: elegantly handle the version here...
+# Download models
+RUN wget "https://raw.githubusercontent.com/opendatalab/MinerU/refs/tags/magic_pdf-1.3.1-released/scripts/download_models_hf.py" -O download_models_hf.py \
+    && uv run python download_models_hf.py
+# Then copy code
+ADD uv.lock pyproject.toml README.md ./
+ADD extract_python ./extract_python/
+# Then install service
+RUN uv sync -v --frozen --no-editable --no-sources --no-install-package opencv-python --extra miner-u
+
+RUN rm -rf ~/.cache/pip $(uv cache dir)
+
+ENTRYPOINT ["uv", "run", "--no-sync", "icij-worker", "workers", "start", "-g", "miner-u", "extract_python.app:app"]
+
+FROM icij/task-service:icij-worker-0.17.21 AS http-service
+ADD uv.lock pyproject.toml README.md ./extract-python/
+ADD extract_python ./extract-python/extract_python/
+RUN uv pip install -e ./extract-python

{extract_python-0.1.0 → extract_python-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: extract-python
-Version: 0.1.0
+Version: 0.3.0
 Summary: Structured content extraction
 Project-URL: Homepage, https://github.com/ICIJ/extract-python
 Project-URL: Repository, https://github.com/ICIJ/extract-python

extract_python-0.3.0/benches/__init__.py

@@ -0,0 +1,5 @@
+from pathlib import Path
+
+ROOT_PATH = Path(__file__).parents[1]
+DATA_PATH = ROOT_PATH / "data"
+TEST_DATA_PATH = ROOT_PATH / "tests" / "data"

extract_python-0.3.0/benches/compare.ipynb

@@ -0,0 +1,117 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "312f3d3220d0b3a0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import shutil\n",
+    "\n",
+    "from extract_python.benches import DATA_PATH, TEST_DATA_PATH\n",
+    "from extract_python.benches.compare import (\n",
+    "    compare,\n",
+    ")\n",
+    "from extract_python.pipelines import DoclingPipeline, MarkerPipeline\n",
+    "from extract_python.objects import InputDoc, OutputFormat"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0a36e830-3fe8-4d65-9b46-f398300e22d9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pdfs = [TEST_DATA_PATH / \"computer_generated.pdf\", TEST_DATA_PATH / \"scanned.pdf\"]\n",
+    "work_dir = DATA_PATH / \"workdir\"\n",
+    "comparison_dir = work_dir / \"comparison\"\n",
+    "docs = [InputDoc.from_path(pdf_path) for pdf_path in pdfs]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6aba45d151eb7302",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from extract_python.pipelines import Pipeline\n",
+    "\n",
+    "pipelines: list[Pipeline] = [DoclingPipeline(), MarkerPipeline()]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d13e58e4c286134b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from extract_python.objects import Result\n",
+    "\n",
+    "if work_dir.exists():\n",
+    "    shutil.rmtree(work_dir)\n",
+    "\n",
+    "for p in pipelines:\n",
+    "    pipeline_dir = p.registered_name.lower().replace(\"pipeline\", \"\")\n",
+    "    output_path = work_dir / pipeline_dir\n",
+    "    output_path.mkdir(parents=True, exist_ok=True)\n",
+    "    mds: list[Result] = [\n",
+    "        r\n",
+    "        async for r in p.extract_content(  # noqa: PLE1142\n",
+    "            docs, output_format=OutputFormat.MARKDOWN, output_path=output_path\n",
+    "        )\n",
+    "    ]\n",
+    "    for md in mds:\n",
+    "        pages_path = output_path / md.output.path / \"artifacts\" / \"pages.json\"\n",
+    "        pages_path.write_text(md.output.pages.model_dump_json())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "75a4e7093ac41453",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from extract_python.benches.compare import discover_comparison\n",
+    "\n",
+    "if comparison_dir.exists():\n",
+    "    shutil.rmtree(comparison_dir)\n",
+    "\n",
+    "compare(discover_comparison(pdfs, work_dir), root=work_dir, output_path=comparison_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "814741bf4798974f",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

extract_python-0.3.0/benches/compare.py

@@ -0,0 +1,152 @@
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+import markdown2
+import pypdfium2
+from extract_python.pipelines.utils import chdir
+from html2image import Html2Image
+from PIL import Image, ImageDraw
+
+from extract_python.objects import BaseModel, OutputFormat, PageIndexes
+
+_WHITE_BACKGROUND_CSS = "body {background: white;}"
+
+
+class ComparisonItem(BaseModel):
+    ref: Path
+    compared: list[Path]
+
+
+def compare(comparisons: list[ComparisonItem], root: Path, output_path: Path) -> None:
+    output_path.mkdir(parents=True)
+    if not comparisons:
+        return
+    first_item = comparisons[0]
+    if not first_item.compared:
+        return
+    if (root / first_item.compared[0]).is_dir():
+        output_format = OutputFormat.MARKDOWN
+    else:
+        output_format = OutputFormat[first_item.compared[0].suffix]
+    match output_format:
+        case OutputFormat.MARKDOWN:
+            side_by_side_page_comp_fn = side_by_side_md_page_comp
+        case _:
+            raise ValueError(f"unsupported output format {output_format}")
+    for comparison in comparisons:
+        if not comparison.compared:
+            continue
+        # We flatten everything and will fail if 2 refs have the same file name, even
+        # if they have different paths. To be improved, potentially using nested
+        # structure or concatenating the path into a single dir name
+        comparison_dir = output_path / path_to_compared_name(comparison.ref)
+        comparison_dir.mkdir()
+        ref_path = root / comparison.ref
+        ref_pdf = pypdfium2.PdfDocument(ref_path)
+        # TODO: create TIFF or PDF
+        pages = _scan_pages(root, comparison)
+        for page_i, page_idxs in enumerate(pages):
+            pdf_page_im = ref_pdf.get_page(page_i).render().to_pil()
+            page_comparisons = []
+            for compared in comparison.compared:
+                compared_name = compared.parent.name
+                page_ix = page_idxs[compared_name]
+                page_comp_im = side_by_side_page_comp_fn(
+                    ref_im=pdf_page_im,
+                    compared_path=root / compared,
+                    page_ix=page_ix,
+                    compared_name=compared_name,
+                )
+                page_comparisons.append(page_comp_im)
+            page_comparison_path = comparison_dir / f"page_{page_i}.tiff"
+            page_comparisons[0].save(
+                page_comparison_path, save_all=True, append_images=page_comparisons[1:]
+            )
+            for p in page_comparisons:
+                p.close()
+
+
+def discover_comparison(refs: list[Path], root: Path) -> list[ComparisonItem]:
+    name_to_ref = {path_to_compared_name(r): r for r in refs}
+    comparisons = {r: [] for r in refs}
+    for d in root.iterdir():
+        if not d.is_dir():
+            continue
+        for parsing in d.iterdir():
+            ref = name_to_ref.get(parsing.name)
+            if ref is None:
+                continue
+            comparisons[ref].append(parsing.relative_to(root))
+    comparisons = [
+        ComparisonItem(ref=ref, compared=compared)
+        for ref, compared in comparisons.items()
+    ]
+    return comparisons
+
+
+def side_by_side_md_page_comp(
+    ref_im: Image,
+    compared_path: Path,
+    page_ix: tuple[int, int],
+    compared_name: str,
+) -> Image.Image:
+    md_files = list(compared_path.glob("*.md"))
+    if len(md_files) != 1:
+        msg = f"unexpected number of md files ({len(md_files)}) in {compared_path}"
+        raise ValueError(msg)
+    md_content = md_files[0].read_text()[page_ix[0] : page_ix[1]]
+    # change the current dir so that the browser renders images properly
+    with chdir(compared_path):
+        md_page_im = _render_md(md_content, compared_path, html_size=ref_im.size)
+    ref_im = _add_compared_name(ref_im, compared_name)
+    comparison_im = Image.new("RGB", (ref_im.width * 2, ref_im.height))
+    comparison_im.paste(ref_im, (0, 0))
+    comparison_im.paste(md_page_im, (ref_im.width, 0))
+    ref_im.close()
+    md_page_im.close()
+    return comparison_im
+
+
+def path_to_compared_name(path: Path) -> str:
+    return f"{path.stem}_{path.suffix.replace('.', '')}"
+
+
+def _add_compared_name(ref_im: Image, compared_name: str) -> Image:
+    with_name = ref_im.copy()
+    d = ImageDraw.Draw(with_name)
+    d.text((0, 0), compared_name, font_size=24, fill=(255, 0, 0))
+    return with_name
+
+
+def _render_md(md_content: str, md_path: Path, html_size: tuple[int, int]) -> Image:
+    with TemporaryDirectory() as tmpdir:
+        # TODO: check that we're handling images correctly,
+        #  maybe make md images absolute or something like this
+        hti = Html2Image(size=html_size, output_path=tmpdir)
+        html = markdown2.markdown(md_content)
+        html = html.replace('<img src="', f'<img src="file://{md_path.absolute()}/')
+        screen_files = hti.screenshot(html_str=html, css_str=_WHITE_BACKGROUND_CSS)
+        if len(screen_files) > 1:
+            msg = (
+                "unexpected state, found multiple screenshots, "
+                "either set a large html_size or find a way to combine them into a"
+                " single image"
+            )
+            raise RuntimeError(msg)
+        im_path = screen_files[0]
+        return Image.open(im_path)
+
+
+def _scan_pages(
+    root: Path, comparison: ComparisonItem
+) -> list[dict[str, tuple[int, int]]]:
+    all_pages = [
+        PageIndexes.model_validate_json(
+            (root / compared / "artifacts" / "pages.json").read_text()
+        ).root
+        for compared in comparison.compared
+    ]
+    all_pages = zip(*all_pages)
+    compared_names = (p.parent.name for p in comparison.compared)
+    pages = [dict(zip(compared_names, page_comp_ixs)) for page_comp_ixs in all_pages]
+    return pages

extract_python-0.3.0/benches/constants.py

@@ -0,0 +1,4 @@
+from pathlib import Path
+
+ROOT_PATH = Path(__file__).parents[1]
+DATA_PATH = ROOT_PATH / "data"

extract_python-0.3.0/data/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

extract_python-0.3.0/docker-compose.yml

@@ -0,0 +1,107 @@
+version: '3.7'
+
+x-tm-amqp-config-variables: &tm-amqp-config
+  TASK_MANAGER__RABBITMQ_HOST: rabbitmq
+
+x-postgres-storage-config: &tm-postgres-storage-config
+  TASK_MANAGER__BACKEND: amqp
+  # Change this to a FSKeyValueStorageConfig if you don't want to use postgres
+  TASK_MANAGER__STORAGE__HOST: postgres
+  TASK_MANAGER__STORAGE__PORT: 5432
+  TASK_MANAGER__STORAGE__PASSWORD: changeme
+
+x-worker-config-variables: &worker-config
+  ICIJ_WORKER_TYPE: amqp
+  ICIJ_WORKER_RABBITMQ_HOST: rabbitmq
+  ICIJ_WORKER_RABBITMQ_PORT: 5672
+
+x-async-app-variables: &async-app
+  EXTRACT_DATA_DIR: /usr/src/data
+  EXTRACT_WORK_DIR: /usr/src/data/workdir
+  EXTRACT_LOG_LEVEL: DEBUG
+
+
+services:
+  rabbitmq:
+    image: rabbitmq:3.12.0-management
+    container_name: extract-rabbitmq
+    healthcheck:
+      test: rabbitmq-diagnostics -q status
+      interval: 5s
+      timeout: 2s
+      retries: 10
+      start_period: 5s
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+
+  postgres:
+    image: postgres
+    container_name: extract-postgres
+    environment:
+      POSTGRES_PASSWORD: changeme
+    healthcheck:
+      test: pg_isready
+      interval: 2s
+      timeout: 2s
+      retries: 10
+      start_period: 5s
+    ports:
+      - "5435:5432"
+
+  http-service:
+    depends_on:
+      rabbitmq:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+    build:
+      context: .
+      target: http-service
+    container_name: extract-http-service
+    environment:
+      <<: [ *tm-amqp-config, *tm-postgres-storage-config ]
+      PORT: "8000"
+      HOST: "0.0.0.0"
+      LOG_LEVEL: DEBUG
+      TASK_MANAGER__APP_PATH: extract_app
+    healthcheck:
+      test: curl -f http://localhost:8000/health
+      interval: 5s
+      timeout: 2s
+      retries: 10
+      start_period: 5s
+    ports:
+      - "8000:8000"
+
+  extract-worker-cpu:
+    depends_on:
+      http-service:
+        condition: service_healthy
+    build:
+      context: .
+      args:
+        dbmate_arch: $DBMATE_ARCH
+      target: worker-cpu
+    environment:
+      <<: [ *worker-config, *async-app ]
+    volumes:
+      - type: bind
+        source: ./data
+        target: /usr/src/data
+
+  extract-worker-miner-u:
+    depends_on:
+      http-service:
+        condition: service_healthy
+    build:
+      context: .
+      args:
+        dbmate_arch: $DBMATE_ARCH
+      target: worker-miner-u
+    environment:
+      <<: [ *worker-config, *async-app ]
+    volumes:
+      - type: bind
+        source: ./data
+        target: /usr/src/data

extract_python-0.3.0/extract

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+function _export_globals() {
+    DBMATE_ARCH=$(dbmate_arch)
+    export DBMATE_ARCH
+}
+
+function _helpers() {
+    function dbmate_arch() {
+        local host_arch
+        if command -v arch >/dev/null 2>&1; then
+            host_arch=$(arch)
+        else
+            host_arch=$(uname -m)
+        fi
+        local dbmate_arch_
+        if [ "$host_arch" == "x86_64" ] ||[ "$host_arch" == "amd64" ]; then
+            dbmate_arch_="amd64"
+        elif [ "$host_arch" == "aarch64" ] || [ "$host_arch" == "arm64" ]; then
+            dbmate_arch_="arm64"
+        elif  [ "$host_arch" == "i386" ] ; then
+            dbmate_arch_="386"
+        else
+            _exit_with_message "Unsupported architecture $host_arch"
+        fi
+        echo "$dbmate_arch_"
+    }
+
+}
+
+function _main() {
+    set -e
+    function _exit_with_message() {
+        echo "$1"
+        exit "${2:-1}"
+    }
+    _helpers
+    _export_globals
+    docker compose "$@"
+}
+
+_main "$@"