raw2md 0.1.0__tar.gz

raw2md-0.1.0/.gitattributes

@@ -0,0 +1,34 @@
+# Normalize line endings: LF in the repository and in the working tree on every
+# platform. Gives deterministic, cross-platform checkouts (Windows + Linux) and
+# silences the "LF will be replaced by CRLF" warning.
+* text=auto eol=lf
+
+# Scripts that must keep CRLF to run correctly on Windows.
+*.bat text eol=crlf
+*.cmd text eol=crlf
+
+# Scripts that must stay LF to run on Linux.
+*.sh text eol=lf
+
+# The vendored KaTeX bundle is minified onto a single 270 KB line: a text diff
+# of it prints the whole file and says nothing.
+src/raw2md/katex.min.js -diff
+
+# Binary files: never normalize line endings, never diff as text.
+*.pdf  binary
+*.docx binary
+*.doc  binary
+*.djvu binary
+*.png  binary
+*.jpg  binary
+*.jpeg binary
+*.gif  binary
+*.bmp  binary
+*.tif  binary
+*.tiff binary
+*.ico  binary
+*.zip  binary
+*.gz   binary
+*.7z   binary
+*.ttf  binary
+*.otf  binary

raw2md-0.1.0/.github/scripts/check-smoke-result.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Check one smoke conversion result: the file exists, opens with a YAML header
+# that reports `ok`, and carries a body. The body is measured in non-blank
+# characters rather than lines, because a converter is free to join or split
+# lines; what separates a real conversion from an empty result that still
+# parses is how much text arrived.
+set -euo pipefail
+
+result="$1"
+minimum_body_chars="${2:-100}"
+
+test -f "$result" || {
+  echo "no result at $result" >&2
+  exit 1
+}
+
+head -n 1 "$result" | grep -qx -- '---' || {
+  echo "result does not open with a YAML header" >&2
+  exit 1
+}
+
+grep -qx 'status: ok' "$result" || {
+  echo "result is not status: ok" >&2
+  sed -n '1,20p' "$result" >&2
+  exit 1
+}
+
+body_chars=$(awk 'body { print } /^---$/ { if (++fence == 2) body = 1 }' "$result" |
+  tr -d '[:space:]' | wc -c)
+if [ "$body_chars" -lt "$minimum_body_chars" ]; then
+  echo "body holds $body_chars non-blank character(s), expected at least $minimum_body_chars" >&2
+  exit 1
+fi
+
+echo "$result: header ok, $body_chars non-blank body character(s)"

raw2md-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,123 @@
+name: CI
+
+on:
+  # No test reads these paths, and docs.yml lints the Markdown. A run holds
+  # four runners for minutes, and the next push queues behind it.
+  # tests/corpus/ Markdown is absent on purpose: the fast layer reads it. A
+  # new top-level folder joins the list only if no test reads it. The list is
+  # written twice: GitHub Actions does not resolve YAML anchors.
+  push:
+    branches: [main]
+    paths-ignore:
+      - 'docs/**'
+      - 'examples/**'
+      - 'README.md'
+      - 'AGENTS.md'
+      - 'LICENSE'
+      - 'THIRD_PARTY_NOTICES.md'
+      - '.markdownlint-cli2.jsonc'
+      - 'package.json'
+      - 'package-lock.json'
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - 'examples/**'
+      - 'README.md'
+      - 'AGENTS.md'
+      - 'LICENSE'
+      - 'THIRD_PARTY_NOTICES.md'
+      - '.markdownlint-cli2.jsonc'
+      - 'package.json'
+      - 'package-lock.json'
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  fast:
+    # Python versions fan out on fast Linux jobs, from the floor to the
+    # ceiling in pyproject.toml. One Windows job, the slowest of the set, runs
+    # the tests for the OS-specific behavior (the lock, the temp root, paths).
+    # The linters and the coverage report run once.
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-24.04
+            python: "3.11"
+          - os: ubuntu-24.04
+            python: "3.12"
+            lint: true
+          - os: ubuntu-24.04
+            python: "3.13"
+          - os: windows-2025
+            python: "3.12"
+    runs-on: ${{ matrix.os }}
+    # id-token lets the Codecov upload authenticate through OIDC instead of a
+    # repository secret. Permissions are per job, not per matrix entry, so the
+    # entries without coverage hold it too; none of them uses it.
+    permissions:
+      contents: read
+      id-token: write
+    env:
+      # The fast layer never imports torch, so the multi-GB wheel and its
+      # Linux-only CUDA and triton dependencies are skipped below; each is
+      # reachable in uv.lock only through torch. A new one after a torch
+      # upgrade only slows the job. UV_NO_SYNC stops `uv run` re-syncing them.
+      UV_NO_SYNC: "1"
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python }}
+
+      - run: >-
+          uv sync --locked
+          --no-install-package torch
+          --no-install-package cuda-bindings
+          --no-install-package cuda-pathfinder
+          --no-install-package cuda-toolkit
+          --no-install-package nvidia-cublas
+          --no-install-package nvidia-cuda-cupti
+          --no-install-package nvidia-cuda-nvrtc
+          --no-install-package nvidia-cuda-runtime
+          --no-install-package nvidia-cudnn-cu13
+          --no-install-package nvidia-cufft
+          --no-install-package nvidia-cufile
+          --no-install-package nvidia-curand
+          --no-install-package nvidia-cusolver
+          --no-install-package nvidia-cusparse
+          --no-install-package nvidia-cusparselt-cu13
+          --no-install-package nvidia-nccl-cu13
+          --no-install-package nvidia-nvjitlink
+          --no-install-package nvidia-nvshmem-cu13
+          --no-install-package nvidia-nvtx
+          --no-install-package triton
+
+      - if: matrix.lint
+        run: uv run ruff format --check .
+      - if: matrix.lint
+        run: uv run ruff check .
+      - if: matrix.lint
+        run: uv run mypy
+
+      - if: ${{ !matrix.lint }}
+        run: uv run pytest
+
+      - if: matrix.lint
+        run: uv run pytest --cov --cov-report=xml
+      - if: matrix.lint
+        run: uv run coverage report --format=markdown --skip-covered >> "$GITHUB_STEP_SUMMARY"
+      - if: matrix.lint
+        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
+        with:
+          use_oidc: true
+          files: coverage.xml
+          disable_search: true
+
+      # Opt-in locally, but seconds long, and the lock is OS-specific: every
+      # job runs it as a step of its own rather than a job billed separately.
+      - run: uv run pytest -m concurrency

raw2md-0.1.0/.github/workflows/docs.yml

@@ -0,0 +1,41 @@
+name: Docs
+
+# The Markdown lint alone: a prose-only commit needs nothing else, and one
+# short job holds a runner for a fraction of the time ci.yml would. The paths
+# are wider than those ci.yml ignores: Markdown lives under tests/ too, and a
+# lint config change re-lints everything. Written twice: Actions does not
+# resolve YAML anchors.
+on:
+  push:
+    branches: [main]
+    paths:
+      - '**.md'
+      - '.markdownlint-cli2.jsonc'
+      - 'package.json'
+      - 'package-lock.json'
+      - '.github/workflows/docs.yml'
+  pull_request:
+    paths:
+      - '**.md'
+      - '.markdownlint-cli2.jsonc'
+      - 'package.json'
+      - 'package-lock.json'
+      - '.github/workflows/docs.yml'
+
+concurrency:
+  group: docs-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  markdown:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: lts/*
+          cache: npm
+
+      - run: npm ci
+      - run: npm run lint:md

raw2md-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,123 @@
+name: Release
+
+# A pushed v* tag builds once, smoke-tests the built wheel, uploads to PyPI
+# and then makes the GitHub release, so a failed upload leaves no release. A
+# manual run is the dry run: build and smoke only. A manual run on a tag also
+# extracts the release notes, which shows that the annotation reaches the
+# runner.
+on:
+  push:
+    tags: ['v*']
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: release-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
+        with:
+          python-version: "3.12"
+
+      - name: Check the tag against the package version
+        if: github.event_name == 'push'
+        run: |
+          version=$(uv run --no-project python -c \
+            "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")
+          if [ "$GITHUB_REF_NAME" != "v$version" ]; then
+            echo "tag $GITHUB_REF_NAME does not match version $version" >&2
+            exit 1
+          fi
+
+      - name: Extract the release notes from the tag
+        # The checkout may store the tag without its annotation, so the tag is
+        # fetched again as it is on the remote.
+        if: github.ref_type == 'tag'
+        run: |
+          git fetch --force --depth=1 origin "+refs/tags/$GITHUB_REF_NAME:refs/tags/$GITHUB_REF_NAME"
+          if [ "$(git cat-file -t "refs/tags/$GITHUB_REF_NAME")" != "tag" ]; then
+            echo "tag $GITHUB_REF_NAME is not annotated; the release text comes from its message" >&2
+            exit 1
+          fi
+          mkdir notes
+          git for-each-ref --format='%(contents:subject)%0a%0a%(contents:body)' \
+            "refs/tags/$GITHUB_REF_NAME" > notes/notes.md
+          cat notes/notes.md >> "$GITHUB_STEP_SUMMARY"
+
+      # --no-sources builds what PyPI users get: without the CUDA torch pin.
+      - run: uv build --no-sources
+      - run: uvx twine check --strict dist/*
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error
+
+      - if: github.ref_type == 'tag'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: notes
+          path: notes/notes.md
+          if-no-files-found: error
+
+  smoke:
+    needs: build
+    uses: ./.github/workflows/smoke.yml
+    with:
+      wheel-artifact: dist
+
+  pypi:
+    needs: smoke
+    if: github.event_name == 'push'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    # The environment holds the manual approval and admits v* tags only.
+    environment:
+      name: pypi
+      url: https://pypi.org/project/raw2md/
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: dist
+          path: dist
+
+      - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
+
+  github-release:
+    needs: pypi
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: dist
+          path: dist
+
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: notes
+          path: notes
+
+      - name: Create the release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: >-
+          gh release create "$GITHUB_REF_NAME" dist/*
+          --repo "$GITHUB_REPOSITORY"
+          --title "$GITHUB_REF_NAME"
+          --notes-file notes/notes.md
+          --verify-tag

raw2md-0.1.0/.github/workflows/smoke.yml

@@ -0,0 +1,179 @@
+name: Smoke
+
+# It installs the tool as a user does, with CPU torch. A dependency release
+# breaks it more often than a commit does, hence the weekly run; a push runs it
+# only when the installed code or the smoke itself changes. The release calls
+# it on the wheel it built.
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'src/**'
+      - 'pyproject.toml'
+      - '.github/workflows/smoke.yml'
+      - '.github/scripts/**'
+      - 'tests/synthesis/gen_smoke.py'
+  schedule:
+    - cron: '17 5 * * 1'
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      wheel-artifact:
+        # Empty installs the checkout instead.
+        description: Name of the artifact that holds the wheel to install
+        type: string
+        default: ''
+
+concurrency:
+  group: smoke-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  install:
+    # `uv tool install` reads no lock file, so a dependency upgrade breaks a
+    # user here first. DOCX converts in a second without a GPU or models.
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-24.04, ubuntu-24.04-arm, windows-2025]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
+        with:
+          enable-cache: true
+          python-version: "3.12"
+
+      - name: Ensure pandoc
+        # The docx target needs pandoc 3.1.9+; the Ubuntu 24.04 package is
+        # 3.1.3, so a pinned release is installed instead.
+        env:
+          PANDOC_VERSION: 3.10.1
+        run: |
+          case "$RUNNER_OS" in
+            Linux)
+              case "$RUNNER_ARCH" in
+                X64) arch=amd64 ;;
+                ARM64) arch=arm64 ;;
+                *) echo "no pandoc release asset for $RUNNER_ARCH" >&2; exit 1 ;;
+              esac
+              curl -fsSL -o pandoc.deb \
+                "https://github.com/jgm/pandoc/releases/download/${PANDOC_VERSION}/pandoc-${PANDOC_VERSION}-1-${arch}.deb"
+              sudo dpkg -i pandoc.deb || (sudo apt-get update -qq && sudo apt-get install -f -y)
+              rm pandoc.deb
+              ;;
+            Windows)
+              choco install pandoc -y --no-progress --version="$PANDOC_VERSION"
+              echo "C:/Program Files/Pandoc" >> "$GITHUB_PATH"
+              ;;
+            *) echo "no pandoc recipe for $RUNNER_OS" >&2; exit 1 ;;
+          esac
+          pandoc --version | head -1
+
+      - if: inputs.wheel-artifact
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: ${{ inputs.wheel-artifact }}
+          path: dist
+
+      - name: Install raw2md the way a user does
+        # The project pins the CUDA build of torch on x64, and the pin wins
+        # over --torch-backend; --no-sources drops it, as a wheel install has
+        # no pin either.
+        env:
+          WHEEL_ARTIFACT: ${{ inputs.wheel-artifact }}
+        run: |
+          if [ -n "$WHEEL_ARTIFACT" ]; then
+            uv tool install dist/*.whl --torch-backend cpu
+          else
+            uv tool install . --no-sources --torch-backend cpu
+          fi
+          echo "$(uv tool dir --bin)" >> "$GITHUB_PATH"
+
+      - run: raw2md --version
+
+      - name: Report readiness
+        run: |
+          set +e
+          raw2md doctor
+          code=$?
+          set -e
+          # 3 is expected on a bare runner (no djvu chain, no API key); any
+          # other nonzero code is a real failure.
+          if [ "$code" != "0" ] && [ "$code" != "3" ]; then
+            exit "$code"
+          fi
+
+      - name: Convert a DOCX end to end
+        run: |
+          uv run --no-project --with pymupdf python tests/synthesis/gen_smoke.py docx smoke.docx
+          raw2md smoke.docx --output-dir out
+
+      - name: Check the result
+        run: bash .github/scripts/check-smoke-result.sh out/smoke.md
+
+  pdf:
+    # marker on a CPU runner, one platform only: gigabytes of models and
+    # minutes of inference. Proves the required engine runs without a GPU.
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
+        with:
+          enable-cache: true
+          python-version: "3.12"
+
+      # marker's model cache; bump the key suffix to refresh it.
+      - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/datalab
+          key: datalab-models-${{ runner.os }}-v1
+
+      - if: inputs.wheel-artifact
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: ${{ inputs.wheel-artifact }}
+          path: dist
+
+      - name: Install raw2md the way a user does
+        # The project pins the CUDA build of torch on x64, and the pin wins
+        # over --torch-backend; --no-sources drops it, as a wheel install has
+        # no pin either.
+        env:
+          WHEEL_ARTIFACT: ${{ inputs.wheel-artifact }}
+        run: |
+          if [ -n "$WHEEL_ARTIFACT" ]; then
+            uv tool install dist/*.whl --torch-backend cpu
+          else
+            uv tool install . --no-sources --torch-backend cpu
+          fi
+          echo "$(uv tool dir --bin)" >> "$GITHUB_PATH"
+
+      - name: Convert a one-page PDF without a GPU
+        run: |
+          uv run --no-project --with pymupdf python tests/synthesis/gen_smoke.py pdf smoke.pdf
+          raw2md smoke.pdf --cuda off --output-dir out
+
+      - name: Check the result
+        run: bash .github/scripts/check-smoke-result.sh out/smoke.md
+
+      # The stage counters live only in the run log.
+      - if: always()
+        run: cp ~/.raw2md/logs/raw2md.log ./pdf-smoke.log || true
+
+      - if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: pdf-smoke-log
+          path: pdf-smoke.log
+          if-no-files-found: warn

raw2md-0.1.0/.gitignore

@@ -0,0 +1,37 @@
+# OS
+.DS_Store
+Thumbs.db
+desktop.ini
+
+# Editors
+.vscode/
+.idea/
+*.swp
+*~
+
+# Temp
+*.tmp
+*.log
+
+# Node (markdownlint tooling)
+node_modules/
+
+# Python
+__pycache__/
+*.pyc
+.venv/
+*.egg-info/
+.coverage
+.coverage.*
+coverage.xml
+htmlcov/
+build/
+dist/
+
+# Round-trip generated formats (reproducible from fixtures + seed, never committed)
+tests/synthesis/generated/
+
+# Default download, run, and scorer folders of the benchmark runner
+mpdocbench-data/
+mpdocbench-run/
+mpdocbench-scorer/

raw2md-0.1.0/.markdownlint-cli2.jsonc

@@ -0,0 +1,31 @@
+{
+  // Markdownlint config for raw2md docs.
+  "config": {
+    "default": true,
+    // Line length is enforced by review, not the linter: a path, a command,
+    // and a table need a long line.
+    "MD013": false,
+    // Allow repeated headings as long as they are not siblings.
+    "MD024": { "siblings_only": true },
+    // Files may start with a comment or frontmatter rather than a heading.
+    "MD041": false,
+    // The README centers the logo and the badges, and Markdown cannot center:
+    // only these tags, with `align`, render on both GitHub and PyPI.
+    "MD033": { "allowed_elements": ["h1", "p", "img", "a"] },
+    // Table pipe spacing is a free formatting choice (the test corpus uses
+    // compact tables on purpose; table reformatting is not a defect).
+    "MD060": false
+  },
+  "globs": ["**/*.md"],
+  "ignores": [
+    "node_modules/**",
+    "**/.venv/**",
+    ".git/**",
+    ".pytest_cache/**",
+    "tests/synthesis/generated/**",
+    // Conversion results shown as they came out of raw2md, not authored text.
+    "examples/book.md",
+    "examples/thesis.md",
+    "examples/report.md"
+  ]
+}