convert-docs 0.1.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- convert_docs-0.3.0/PKG-INFO +92 -0
- convert_docs-0.3.0/README.md +84 -0
- {convert_docs-0.1.1 → convert_docs-0.3.0}/pyproject.toml +1 -1
- convert_docs-0.3.0/src/convert_docs/cli.py +288 -0
- convert_docs-0.1.1/PKG-INFO +0 -51
- convert_docs-0.1.1/README.md +0 -43
- convert_docs-0.1.1/src/convert_docs/cli.py +0 -174
- {convert_docs-0.1.1 → convert_docs-0.3.0}/.github/workflows/publish.yml +0 -0
- {convert_docs-0.1.1 → convert_docs-0.3.0}/.gitignore +0 -0
- {convert_docs-0.1.1 → convert_docs-0.3.0}/src/convert_docs/__init__.py +0 -0
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: convert-docs
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Recursively convert documents (pdf, docx, pptx, xlsx, ...) to Markdown via markitdown, mirroring the source folder structure into a destination folder.
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Requires-Dist: markitdown[docx,outlook,pdf,pptx,xls,xlsx]>=0.1.5
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
|
|
9
|
+
# convert-docs
|
|
10
|
+
|
|
11
|
+
Recursively convert documents (pdf, docx, pptx, xlsx, and more) to Markdown via
|
|
12
|
+
[markitdown](https://github.com/microsoft/markitdown), mirroring the source folder
|
|
13
|
+
structure into a destination folder.
|
|
14
|
+
|
|
15
|
+
## Install
|
|
16
|
+
|
|
17
|
+
Requires [uv](https://docs.astral.sh/uv/). If you don't have it yet:
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
# macOS / Linux
|
|
21
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
22
|
+
|
|
23
|
+
# Windows (PowerShell)
|
|
24
|
+
powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Then install the tool:
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
uv tool install convert-docs
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
No `uv`? Plain `pip` also works, as long as it's Python 3.10+: `pip install convert-docs`.
|
|
34
|
+
|
|
35
|
+
## Usage
|
|
36
|
+
|
|
37
|
+
Run with no flags for an interactive prompt (asks for source, then destination):
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
convert-docs
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Or pass flags to skip the prompts:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
convert-docs -s ~/Documents/Reports -o ~/Documents/Reports_MD
|
|
47
|
+
convert-docs -e pdf,docx # restrict which extensions get converted
|
|
48
|
+
convert-docs -f # force re-conversion, ignoring the up-to-date skip
|
|
49
|
+
convert-docs --last # re-run with the same source/destination/extensions as last time
|
|
50
|
+
convert-docs -j 8 # convert 8 files in parallel instead of the default 4
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
| Flag | Description |
|
|
54
|
+
|---|---|
|
|
55
|
+
| `-s, --source` | Source directory to scan (skips the interactive prompt) |
|
|
56
|
+
| `-o, --output` | Output directory, mirrors source structure (skips the interactive prompt) |
|
|
57
|
+
| `-e, --ext` | Comma-separated extensions to convert |
|
|
58
|
+
| `-f, --force` | Force re-conversion even if output `.md` already exists and is up to date |
|
|
59
|
+
| `-l, --last` | Reuse the source, destination, and extensions from the last run (skips prompts; cannot be combined with `-s`/`-o`) |
|
|
60
|
+
| `-j, --jobs` | Convert this many files in parallel using separate processes (default: `min(4, cpu_count)`; use `1` for sequential) |
|
|
61
|
+
|
|
62
|
+
Files with unsupported extensions are listed at the end instead of being silently
|
|
63
|
+
skipped, and any conversion failures are reported with the underlying error.
|
|
64
|
+
|
|
65
|
+
### Parallel conversion
|
|
66
|
+
|
|
67
|
+
Files convert using separate OS processes (real parallelism across CPU cores, not
|
|
68
|
+
`asyncio` — conversion is CPU-bound parsing work, not I/O waiting, so asyncio's
|
|
69
|
+
cooperative concurrency wouldn't actually help). Benchmarked on a 10-core Mac converting
|
|
70
|
+
12–30 real docx files: `-j 4` (the default) cut wall time by ~2.3x vs. sequential.
|
|
71
|
+
Pushing higher didn't help further — `-j 8` was consistently *slower* than `-j 4` in
|
|
72
|
+
testing, since each worker process pays a fixed startup cost importing `markitdown`'s
|
|
73
|
+
dependencies (onnxruntime, magika, pandas), which outweighs the extra parallelism for
|
|
74
|
+
typical batch sizes. If you're converting a very large batch (hundreds of files), it's
|
|
75
|
+
worth experimenting with higher `-j` values yourself; for typical folders the default
|
|
76
|
+
is a reasonable balance. Progress lines print in completion order, not scan order, since
|
|
77
|
+
files finish out of sequence when running in parallel.
|
|
78
|
+
|
|
79
|
+
### Re-running on new files
|
|
80
|
+
|
|
81
|
+
Every run saves its source/destination/extensions to `~/.config/convert-docs/last_run.json`.
|
|
82
|
+
Since conversion already skips files whose output is up to date, `convert-docs --last` is a
|
|
83
|
+
cheap way to pick up newly added files in a folder you've converted before — run it manually
|
|
84
|
+
whenever you want to sync, or wire it into a cron job / scheduled task on whatever interval
|
|
85
|
+
suits you (just point it at the tool's full path, e.g. `~/.local/bin/convert-docs --last`,
|
|
86
|
+
since scheduled jobs don't load your shell profile).
|
|
87
|
+
|
|
88
|
+
## Update
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
uv tool upgrade convert-docs
|
|
92
|
+
```
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# convert-docs
|
|
2
|
+
|
|
3
|
+
Recursively convert documents (pdf, docx, pptx, xlsx, and more) to Markdown via
|
|
4
|
+
[markitdown](https://github.com/microsoft/markitdown), mirroring the source folder
|
|
5
|
+
structure into a destination folder.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
Requires [uv](https://docs.astral.sh/uv/). If you don't have it yet:
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
# macOS / Linux
|
|
13
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
14
|
+
|
|
15
|
+
# Windows (PowerShell)
|
|
16
|
+
powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Then install the tool:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
uv tool install convert-docs
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
No `uv`? Plain `pip` also works, as long as it's Python 3.10+: `pip install convert-docs`.
|
|
26
|
+
|
|
27
|
+
## Usage
|
|
28
|
+
|
|
29
|
+
Run with no flags for an interactive prompt (asks for source, then destination):
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
convert-docs
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Or pass flags to skip the prompts:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
convert-docs -s ~/Documents/Reports -o ~/Documents/Reports_MD
|
|
39
|
+
convert-docs -e pdf,docx # restrict which extensions get converted
|
|
40
|
+
convert-docs -f # force re-conversion, ignoring the up-to-date skip
|
|
41
|
+
convert-docs --last # re-run with the same source/destination/extensions as last time
|
|
42
|
+
convert-docs -j 8 # convert 8 files in parallel instead of the default 4
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
| Flag | Description |
|
|
46
|
+
|---|---|
|
|
47
|
+
| `-s, --source` | Source directory to scan (skips the interactive prompt) |
|
|
48
|
+
| `-o, --output` | Output directory, mirrors source structure (skips the interactive prompt) |
|
|
49
|
+
| `-e, --ext` | Comma-separated extensions to convert |
|
|
50
|
+
| `-f, --force` | Force re-conversion even if output `.md` already exists and is up to date |
|
|
51
|
+
| `-l, --last` | Reuse the source, destination, and extensions from the last run (skips prompts; cannot be combined with `-s`/`-o`) |
|
|
52
|
+
| `-j, --jobs` | Convert this many files in parallel using separate processes (default: `min(4, cpu_count)`; use `1` for sequential) |
|
|
53
|
+
|
|
54
|
+
Files with unsupported extensions are listed at the end instead of being silently
|
|
55
|
+
skipped, and any conversion failures are reported with the underlying error.
|
|
56
|
+
|
|
57
|
+
### Parallel conversion
|
|
58
|
+
|
|
59
|
+
Files convert using separate OS processes (real parallelism across CPU cores, not
|
|
60
|
+
`asyncio` — conversion is CPU-bound parsing work, not I/O waiting, so asyncio's
|
|
61
|
+
cooperative concurrency wouldn't actually help). Benchmarked on a 10-core Mac converting
|
|
62
|
+
12–30 real docx files: `-j 4` (the default) cut wall time by ~2.3x vs. sequential.
|
|
63
|
+
Pushing higher didn't help further — `-j 8` was consistently *slower* than `-j 4` in
|
|
64
|
+
testing, since each worker process pays a fixed startup cost importing `markitdown`'s
|
|
65
|
+
dependencies (onnxruntime, magika, pandas), which outweighs the extra parallelism for
|
|
66
|
+
typical batch sizes. If you're converting a very large batch (hundreds of files), it's
|
|
67
|
+
worth experimenting with higher `-j` values yourself; for typical folders the default
|
|
68
|
+
is a reasonable balance. Progress lines print in completion order, not scan order, since
|
|
69
|
+
files finish out of sequence when running in parallel.
|
|
70
|
+
|
|
71
|
+
### Re-running on new files
|
|
72
|
+
|
|
73
|
+
Every run saves its source/destination/extensions to `~/.config/convert-docs/last_run.json`.
|
|
74
|
+
Since conversion already skips files whose output is up to date, `convert-docs --last` is a
|
|
75
|
+
cheap way to pick up newly added files in a folder you've converted before — run it manually
|
|
76
|
+
whenever you want to sync, or wire it into a cron job / scheduled task on whatever interval
|
|
77
|
+
suits you (just point it at the tool's full path, e.g. `~/.local/bin/convert-docs --last`,
|
|
78
|
+
since scheduled jobs don't load your shell profile).
|
|
79
|
+
|
|
80
|
+
## Update
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
uv tool upgrade convert-docs
|
|
84
|
+
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "convert-docs"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
description = "Recursively convert documents (pdf, docx, pptx, xlsx, ...) to Markdown via markitdown, mirroring the source folder structure into a destination folder."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.10"
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
"""Recursively convert documents to Markdown via markitdown, mirroring folder structure."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
DEFAULT_EXTENSIONS = [
|
|
11
|
+
"pdf", "docx", "pptx", "xlsx", "doc", "ppt", "xls", "csv",
|
|
12
|
+
"html", "htm", "txt", "epub", "json", "xml", "msg",
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
SEPARATOR = "-" * 40
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def config_dir() -> Path:
|
|
19
|
+
base = os.environ.get("XDG_CONFIG_HOME")
|
|
20
|
+
return (Path(base) if base else Path.home() / ".config") / "convert-docs"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def last_run_path() -> Path:
|
|
24
|
+
return config_dir() / "last_run.json"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def save_last_run(src_dir: Path, dest_dir: Path, extensions: list) -> None:
|
|
28
|
+
path = last_run_path()
|
|
29
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
30
|
+
path.write_text(json.dumps({
|
|
31
|
+
"source": str(src_dir),
|
|
32
|
+
"destination": str(dest_dir),
|
|
33
|
+
"extensions": extensions,
|
|
34
|
+
}, indent=2))
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def load_last_run() -> dict:
|
|
38
|
+
path = last_run_path()
|
|
39
|
+
if not path.exists():
|
|
40
|
+
print(
|
|
41
|
+
"Error: no previous run found. Run convert-docs normally first "
|
|
42
|
+
"(with -s/-o or interactively) before using --last.",
|
|
43
|
+
file=sys.stderr,
|
|
44
|
+
)
|
|
45
|
+
sys.exit(1)
|
|
46
|
+
try:
|
|
47
|
+
return json.loads(path.read_text())
|
|
48
|
+
except (json.JSONDecodeError, OSError) as exc:
|
|
49
|
+
print(f"Error: could not read saved config at {path}: {exc}", file=sys.stderr)
|
|
50
|
+
sys.exit(1)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def parse_extensions(raw: str) -> list:
|
|
54
|
+
return [e.strip().lower().lstrip(".") for e in raw.split(",") if e.strip()]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def strip_quotes(text: str) -> str:
|
|
58
|
+
if len(text) >= 2 and text[0] == text[-1] and text[0] in ("'", '"'):
|
|
59
|
+
return text[1:-1]
|
|
60
|
+
return text
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def prompt_source() -> Path:
|
|
64
|
+
while True:
|
|
65
|
+
raw = strip_quotes(input("Source folder path: ").strip())
|
|
66
|
+
candidate = Path(raw or ".").expanduser()
|
|
67
|
+
if candidate.is_dir():
|
|
68
|
+
return candidate.resolve()
|
|
69
|
+
print(f" '{raw}' is not a valid directory. Try again.", file=sys.stderr)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def prompt_destination(default: Path) -> Path:
|
|
73
|
+
print("Destination folder path (press Enter to use the default):")
|
|
74
|
+
print(f" default: {default}")
|
|
75
|
+
raw = strip_quotes(input("> ").strip())
|
|
76
|
+
if not raw:
|
|
77
|
+
return default
|
|
78
|
+
return Path(raw).expanduser().resolve()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def parse_args(argv=None) -> argparse.Namespace:
|
|
82
|
+
parser = argparse.ArgumentParser(
|
|
83
|
+
prog="convert-docs",
|
|
84
|
+
description=__doc__,
|
|
85
|
+
)
|
|
86
|
+
parser.add_argument(
|
|
87
|
+
"-s", "--source",
|
|
88
|
+
help="Source directory to scan (skips the interactive prompt)",
|
|
89
|
+
)
|
|
90
|
+
parser.add_argument(
|
|
91
|
+
"-o", "--output",
|
|
92
|
+
help="Output directory, mirrors source structure (skips the interactive prompt)",
|
|
93
|
+
)
|
|
94
|
+
parser.add_argument(
|
|
95
|
+
"-e", "--ext",
|
|
96
|
+
default=None,
|
|
97
|
+
help=f"Comma-separated extensions to convert (default: {','.join(DEFAULT_EXTENSIONS)})",
|
|
98
|
+
)
|
|
99
|
+
parser.add_argument(
|
|
100
|
+
"-f", "--force",
|
|
101
|
+
action="store_true",
|
|
102
|
+
help="Force re-conversion even if output .md already exists and is up to date",
|
|
103
|
+
)
|
|
104
|
+
parser.add_argument(
|
|
105
|
+
"-l", "--last",
|
|
106
|
+
action="store_true",
|
|
107
|
+
help="Reuse the source, destination, and extensions from the last run "
|
|
108
|
+
"(skips prompts; cannot be combined with -s/-o)",
|
|
109
|
+
)
|
|
110
|
+
parser.add_argument(
|
|
111
|
+
"-j", "--jobs",
|
|
112
|
+
type=int,
|
|
113
|
+
default=min(4, os.cpu_count() or 1),
|
|
114
|
+
help="Convert this many files in parallel using separate processes "
|
|
115
|
+
"(default: %(default)s; use 1 for sequential)",
|
|
116
|
+
)
|
|
117
|
+
return parser.parse_args(argv)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def resolve_source(args: argparse.Namespace) -> Path:
|
|
121
|
+
if not args.source:
|
|
122
|
+
return prompt_source()
|
|
123
|
+
src_dir = Path(args.source).expanduser()
|
|
124
|
+
if not src_dir.is_dir():
|
|
125
|
+
print(f"Error: source directory '{args.source}' does not exist.", file=sys.stderr)
|
|
126
|
+
sys.exit(1)
|
|
127
|
+
return src_dir.resolve()
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def resolve_destination(args: argparse.Namespace, src_dir: Path) -> Path:
|
|
131
|
+
default = src_dir / "Context"
|
|
132
|
+
if not args.output:
|
|
133
|
+
return prompt_destination(default)
|
|
134
|
+
return Path(args.output).expanduser().resolve()
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def collect_files(src_dir: Path, dest_dir: Path, ext_set: set) -> tuple:
|
|
138
|
+
target_files = []
|
|
139
|
+
other_files = []
|
|
140
|
+
for path in sorted(src_dir.rglob("*")):
|
|
141
|
+
if not path.is_file():
|
|
142
|
+
continue
|
|
143
|
+
if dest_dir == path or dest_dir in path.parents:
|
|
144
|
+
continue
|
|
145
|
+
suffix = path.suffix.lower().lstrip(".")
|
|
146
|
+
(target_files if suffix in ext_set else other_files).append(path)
|
|
147
|
+
return target_files, other_files
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
_worker_md = None
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _init_worker() -> None:
|
|
154
|
+
global _worker_md
|
|
155
|
+
from markitdown import MarkItDown
|
|
156
|
+
_worker_md = MarkItDown()
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _convert_one(path_str: str, out_path_str: str) -> tuple:
|
|
160
|
+
try:
|
|
161
|
+
result = _worker_md.convert(path_str)
|
|
162
|
+
Path(out_path_str).write_text(result.text_content, encoding="utf-8")
|
|
163
|
+
return (True, None)
|
|
164
|
+
except Exception as exc:
|
|
165
|
+
return (False, str(exc))
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def plan_conversions(target_files: list, src_dir: Path, dest_dir: Path, force: bool) -> tuple:
|
|
169
|
+
to_convert = []
|
|
170
|
+
skipped = []
|
|
171
|
+
for path in target_files:
|
|
172
|
+
rel_path = path.relative_to(src_dir)
|
|
173
|
+
out_path = (dest_dir / rel_path).with_suffix(".md")
|
|
174
|
+
if not force and out_path.exists() and out_path.stat().st_mtime >= path.stat().st_mtime:
|
|
175
|
+
skipped.append(str(rel_path))
|
|
176
|
+
continue
|
|
177
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
178
|
+
to_convert.append((path, rel_path, out_path))
|
|
179
|
+
return to_convert, skipped
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def convert_files(target_files: list, src_dir: Path, dest_dir: Path, force: bool, jobs: int = 1) -> tuple:
|
|
183
|
+
to_convert, skipped = plan_conversions(target_files, src_dir, dest_dir, force)
|
|
184
|
+
total = len(to_convert)
|
|
185
|
+
converted, failed = [], []
|
|
186
|
+
|
|
187
|
+
if total == 0:
|
|
188
|
+
return converted, failed, skipped
|
|
189
|
+
|
|
190
|
+
if jobs <= 1 or total == 1:
|
|
191
|
+
from markitdown import MarkItDown
|
|
192
|
+
md = MarkItDown()
|
|
193
|
+
for i, (path, rel_path, out_path) in enumerate(to_convert, start=1):
|
|
194
|
+
print(f"[{i}/{total}] {rel_path} ... ", end="", flush=True)
|
|
195
|
+
try:
|
|
196
|
+
result = md.convert(str(path))
|
|
197
|
+
out_path.write_text(result.text_content, encoding="utf-8")
|
|
198
|
+
print("OK")
|
|
199
|
+
converted.append(str(rel_path))
|
|
200
|
+
except Exception as exc:
|
|
201
|
+
print("FAILED")
|
|
202
|
+
failed.append(f"{rel_path} :: {exc}")
|
|
203
|
+
return converted, failed, skipped
|
|
204
|
+
|
|
205
|
+
with ProcessPoolExecutor(max_workers=jobs, initializer=_init_worker) as executor:
|
|
206
|
+
futures = {
|
|
207
|
+
executor.submit(_convert_one, str(path), str(out_path)): rel_path
|
|
208
|
+
for path, rel_path, out_path in to_convert
|
|
209
|
+
}
|
|
210
|
+
for i, future in enumerate(as_completed(futures), start=1):
|
|
211
|
+
rel_path = futures[future]
|
|
212
|
+
ok, err = future.result()
|
|
213
|
+
if ok:
|
|
214
|
+
print(f"[{i}/{total}] {rel_path} ... OK")
|
|
215
|
+
converted.append(str(rel_path))
|
|
216
|
+
else:
|
|
217
|
+
print(f"[{i}/{total}] {rel_path} ... FAILED")
|
|
218
|
+
failed.append(f"{rel_path} :: {err}")
|
|
219
|
+
|
|
220
|
+
return converted, failed, skipped
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def main(argv=None) -> int:
|
|
224
|
+
args = parse_args(argv)
|
|
225
|
+
|
|
226
|
+
if args.last and (args.source or args.output):
|
|
227
|
+
print("Error: --last cannot be combined with -s/--source or -o/--output.", file=sys.stderr)
|
|
228
|
+
return 1
|
|
229
|
+
|
|
230
|
+
if args.last:
|
|
231
|
+
last = load_last_run()
|
|
232
|
+
src_dir = Path(last["source"])
|
|
233
|
+
if not src_dir.is_dir():
|
|
234
|
+
print(f"Error: last-used source directory no longer exists: {src_dir}", file=sys.stderr)
|
|
235
|
+
return 1
|
|
236
|
+
src_dir = src_dir.resolve()
|
|
237
|
+
dest_dir = Path(last["destination"])
|
|
238
|
+
extensions = parse_extensions(args.ext) if args.ext else last.get("extensions", DEFAULT_EXTENSIONS)
|
|
239
|
+
else:
|
|
240
|
+
extensions = parse_extensions(args.ext) if args.ext else DEFAULT_EXTENSIONS
|
|
241
|
+
src_dir = resolve_source(args)
|
|
242
|
+
dest_dir = resolve_destination(args, src_dir)
|
|
243
|
+
|
|
244
|
+
if not dest_dir.exists():
|
|
245
|
+
print(f"Destination does not exist, creating: {dest_dir}")
|
|
246
|
+
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
247
|
+
dest_dir = dest_dir.resolve()
|
|
248
|
+
|
|
249
|
+
save_last_run(src_dir, dest_dir, extensions)
|
|
250
|
+
|
|
251
|
+
print(SEPARATOR)
|
|
252
|
+
print(f"Source folder: {src_dir}")
|
|
253
|
+
print(f"Destination folder: {dest_dir}")
|
|
254
|
+
print(f"Extensions: {', '.join(extensions)}")
|
|
255
|
+
print(SEPARATOR)
|
|
256
|
+
|
|
257
|
+
target_files, other_files = collect_files(src_dir, dest_dir, set(extensions))
|
|
258
|
+
print(f"Found {len(target_files)} convertible file(s), {len(other_files)} file(s) with unsupported extensions.")
|
|
259
|
+
print(SEPARATOR)
|
|
260
|
+
|
|
261
|
+
converted, failed, skipped = convert_files(target_files, src_dir, dest_dir, args.force, jobs=args.jobs)
|
|
262
|
+
|
|
263
|
+
print(SEPARATOR)
|
|
264
|
+
print("Summary:")
|
|
265
|
+
print(f" Converted: {len(converted)}")
|
|
266
|
+
print(f" Skipped (up to date): {len(skipped)}")
|
|
267
|
+
print(f" Failed: {len(failed)}")
|
|
268
|
+
print(f" Unsupported extension: {len(other_files)}")
|
|
269
|
+
|
|
270
|
+
if failed:
|
|
271
|
+
print()
|
|
272
|
+
print("Failed conversions:")
|
|
273
|
+
for item in failed:
|
|
274
|
+
print(f" - {item}")
|
|
275
|
+
|
|
276
|
+
if other_files:
|
|
277
|
+
print()
|
|
278
|
+
print("Unsupported / not attempted:")
|
|
279
|
+
for path in other_files:
|
|
280
|
+
print(f" - {path.relative_to(src_dir)}")
|
|
281
|
+
|
|
282
|
+
print(SEPARATOR)
|
|
283
|
+
print(f"Done. Output mirrored under: {dest_dir}")
|
|
284
|
+
return 1 if failed else 0
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
if __name__ == "__main__":
|
|
288
|
+
sys.exit(main())
|
convert_docs-0.1.1/PKG-INFO
DELETED
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: convert-docs
|
|
3
|
-
Version: 0.1.1
|
|
4
|
-
Summary: Recursively convert documents (pdf, docx, pptx, xlsx, ...) to Markdown via markitdown, mirroring the source folder structure into a destination folder.
|
|
5
|
-
Requires-Python: >=3.10
|
|
6
|
-
Requires-Dist: markitdown[docx,outlook,pdf,pptx,xls,xlsx]>=0.1.5
|
|
7
|
-
Description-Content-Type: text/markdown
|
|
8
|
-
|
|
9
|
-
# convert-docs
|
|
10
|
-
|
|
11
|
-
Recursively convert documents (pdf, docx, pptx, xlsx, and more) to Markdown via
|
|
12
|
-
[markitdown](https://github.com/microsoft/markitdown), mirroring the source folder
|
|
13
|
-
structure into a destination folder.
|
|
14
|
-
|
|
15
|
-
## Install
|
|
16
|
-
|
|
17
|
-
```bash
|
|
18
|
-
uv tool install convert-docs
|
|
19
|
-
```
|
|
20
|
-
|
|
21
|
-
## Usage
|
|
22
|
-
|
|
23
|
-
Run with no flags for an interactive prompt (asks for source, then destination):
|
|
24
|
-
|
|
25
|
-
```bash
|
|
26
|
-
convert-docs
|
|
27
|
-
```
|
|
28
|
-
|
|
29
|
-
Or pass flags to skip the prompts:
|
|
30
|
-
|
|
31
|
-
```bash
|
|
32
|
-
convert-docs -s ~/Documents/Reports -o ~/Documents/Reports_MD
|
|
33
|
-
convert-docs -e pdf,docx # restrict which extensions get converted
|
|
34
|
-
convert-docs -f # force re-conversion, ignoring the up-to-date skip
|
|
35
|
-
```
|
|
36
|
-
|
|
37
|
-
| Flag | Description |
|
|
38
|
-
|---|---|
|
|
39
|
-
| `-s, --source` | Source directory to scan (skips the interactive prompt) |
|
|
40
|
-
| `-o, --output` | Output directory, mirrors source structure (skips the interactive prompt) |
|
|
41
|
-
| `-e, --ext` | Comma-separated extensions to convert |
|
|
42
|
-
| `-f, --force` | Force re-conversion even if output `.md` already exists and is up to date |
|
|
43
|
-
|
|
44
|
-
Files with unsupported extensions are listed at the end instead of being silently
|
|
45
|
-
skipped, and any conversion failures are reported with the underlying error.
|
|
46
|
-
|
|
47
|
-
## Update
|
|
48
|
-
|
|
49
|
-
```bash
|
|
50
|
-
uv tool upgrade convert-docs
|
|
51
|
-
```
|
convert_docs-0.1.1/README.md
DELETED
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
# convert-docs
|
|
2
|
-
|
|
3
|
-
Recursively convert documents (pdf, docx, pptx, xlsx, and more) to Markdown via
|
|
4
|
-
[markitdown](https://github.com/microsoft/markitdown), mirroring the source folder
|
|
5
|
-
structure into a destination folder.
|
|
6
|
-
|
|
7
|
-
## Install
|
|
8
|
-
|
|
9
|
-
```bash
|
|
10
|
-
uv tool install convert-docs
|
|
11
|
-
```
|
|
12
|
-
|
|
13
|
-
## Usage
|
|
14
|
-
|
|
15
|
-
Run with no flags for an interactive prompt (asks for source, then destination):
|
|
16
|
-
|
|
17
|
-
```bash
|
|
18
|
-
convert-docs
|
|
19
|
-
```
|
|
20
|
-
|
|
21
|
-
Or pass flags to skip the prompts:
|
|
22
|
-
|
|
23
|
-
```bash
|
|
24
|
-
convert-docs -s ~/Documents/Reports -o ~/Documents/Reports_MD
|
|
25
|
-
convert-docs -e pdf,docx # restrict which extensions get converted
|
|
26
|
-
convert-docs -f # force re-conversion, ignoring the up-to-date skip
|
|
27
|
-
```
|
|
28
|
-
|
|
29
|
-
| Flag | Description |
|
|
30
|
-
|---|---|
|
|
31
|
-
| `-s, --source` | Source directory to scan (skips the interactive prompt) |
|
|
32
|
-
| `-o, --output` | Output directory, mirrors source structure (skips the interactive prompt) |
|
|
33
|
-
| `-e, --ext` | Comma-separated extensions to convert |
|
|
34
|
-
| `-f, --force` | Force re-conversion even if output `.md` already exists and is up to date |
|
|
35
|
-
|
|
36
|
-
Files with unsupported extensions are listed at the end instead of being silently
|
|
37
|
-
skipped, and any conversion failures are reported with the underlying error.
|
|
38
|
-
|
|
39
|
-
## Update
|
|
40
|
-
|
|
41
|
-
```bash
|
|
42
|
-
uv tool upgrade convert-docs
|
|
43
|
-
```
|
|
@@ -1,174 +0,0 @@
|
|
|
1
|
-
"""Recursively convert documents to Markdown via markitdown, mirroring folder structure."""
|
|
2
|
-
|
|
3
|
-
import argparse
|
|
4
|
-
import sys
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
|
|
7
|
-
DEFAULT_EXTENSIONS = [
|
|
8
|
-
"pdf", "docx", "pptx", "xlsx", "doc", "ppt", "xls", "csv",
|
|
9
|
-
"html", "htm", "txt", "epub", "json", "xml", "msg",
|
|
10
|
-
]
|
|
11
|
-
|
|
12
|
-
SEPARATOR = "-" * 40
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
def strip_quotes(text: str) -> str:
|
|
16
|
-
if len(text) >= 2 and text[0] == text[-1] and text[0] in ("'", '"'):
|
|
17
|
-
return text[1:-1]
|
|
18
|
-
return text
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def prompt_source() -> Path:
|
|
22
|
-
while True:
|
|
23
|
-
raw = strip_quotes(input("Source folder path: ").strip())
|
|
24
|
-
candidate = Path(raw or ".").expanduser()
|
|
25
|
-
if candidate.is_dir():
|
|
26
|
-
return candidate.resolve()
|
|
27
|
-
print(f" '{raw}' is not a valid directory. Try again.", file=sys.stderr)
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
def prompt_destination(default: Path) -> Path:
|
|
31
|
-
print("Destination folder path (press Enter to use the default):")
|
|
32
|
-
print(f" default: {default}")
|
|
33
|
-
raw = strip_quotes(input("> ").strip())
|
|
34
|
-
if not raw:
|
|
35
|
-
return default
|
|
36
|
-
return Path(raw).expanduser().resolve()
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def parse_args(argv=None) -> argparse.Namespace:
|
|
40
|
-
parser = argparse.ArgumentParser(
|
|
41
|
-
prog="convert-docs",
|
|
42
|
-
description=__doc__,
|
|
43
|
-
)
|
|
44
|
-
parser.add_argument(
|
|
45
|
-
"-s", "--source",
|
|
46
|
-
help="Source directory to scan (skips the interactive prompt)",
|
|
47
|
-
)
|
|
48
|
-
parser.add_argument(
|
|
49
|
-
"-o", "--output",
|
|
50
|
-
help="Output directory, mirrors source structure (skips the interactive prompt)",
|
|
51
|
-
)
|
|
52
|
-
parser.add_argument(
|
|
53
|
-
"-e", "--ext",
|
|
54
|
-
default=",".join(DEFAULT_EXTENSIONS),
|
|
55
|
-
help="Comma-separated extensions to convert (default: %(default)s)",
|
|
56
|
-
)
|
|
57
|
-
parser.add_argument(
|
|
58
|
-
"-f", "--force",
|
|
59
|
-
action="store_true",
|
|
60
|
-
help="Force re-conversion even if output .md already exists and is up to date",
|
|
61
|
-
)
|
|
62
|
-
return parser.parse_args(argv)
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def resolve_source(args: argparse.Namespace) -> Path:
|
|
66
|
-
if not args.source:
|
|
67
|
-
return prompt_source()
|
|
68
|
-
src_dir = Path(args.source).expanduser()
|
|
69
|
-
if not src_dir.is_dir():
|
|
70
|
-
print(f"Error: source directory '{args.source}' does not exist.", file=sys.stderr)
|
|
71
|
-
sys.exit(1)
|
|
72
|
-
return src_dir.resolve()
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
def resolve_destination(args: argparse.Namespace, src_dir: Path) -> Path:
|
|
76
|
-
default = src_dir / "Context"
|
|
77
|
-
if not args.output:
|
|
78
|
-
return prompt_destination(default)
|
|
79
|
-
return Path(args.output).expanduser().resolve()
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
def collect_files(src_dir: Path, dest_dir: Path, ext_set: set) -> tuple:
|
|
83
|
-
target_files = []
|
|
84
|
-
other_files = []
|
|
85
|
-
for path in sorted(src_dir.rglob("*")):
|
|
86
|
-
if not path.is_file():
|
|
87
|
-
continue
|
|
88
|
-
if dest_dir == path or dest_dir in path.parents:
|
|
89
|
-
continue
|
|
90
|
-
suffix = path.suffix.lower().lstrip(".")
|
|
91
|
-
(target_files if suffix in ext_set else other_files).append(path)
|
|
92
|
-
return target_files, other_files
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
def convert_files(target_files: list, src_dir: Path, dest_dir: Path, force: bool) -> tuple:
|
|
96
|
-
from markitdown import MarkItDown
|
|
97
|
-
|
|
98
|
-
md = MarkItDown()
|
|
99
|
-
total = len(target_files)
|
|
100
|
-
converted, failed, skipped = [], [], []
|
|
101
|
-
|
|
102
|
-
for i, path in enumerate(target_files, start=1):
|
|
103
|
-
rel_path = path.relative_to(src_dir)
|
|
104
|
-
out_path = (dest_dir / rel_path).with_suffix(".md")
|
|
105
|
-
print(f"[{i}/{total}] {rel_path} ... ", end="", flush=True)
|
|
106
|
-
|
|
107
|
-
if not force and out_path.exists() and out_path.stat().st_mtime >= path.stat().st_mtime:
|
|
108
|
-
print("skipped (up to date)")
|
|
109
|
-
skipped.append(str(rel_path))
|
|
110
|
-
continue
|
|
111
|
-
|
|
112
|
-
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
113
|
-
try:
|
|
114
|
-
result = md.convert(str(path))
|
|
115
|
-
out_path.write_text(result.text_content, encoding="utf-8")
|
|
116
|
-
print("OK")
|
|
117
|
-
converted.append(str(rel_path))
|
|
118
|
-
except Exception as exc:
|
|
119
|
-
print("FAILED")
|
|
120
|
-
failed.append(f"{rel_path} :: {exc}")
|
|
121
|
-
|
|
122
|
-
return converted, failed, skipped
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
def main(argv=None) -> int:
|
|
126
|
-
args = parse_args(argv)
|
|
127
|
-
extensions = [e.strip().lower().lstrip(".") for e in args.ext.split(",") if e.strip()]
|
|
128
|
-
|
|
129
|
-
src_dir = resolve_source(args)
|
|
130
|
-
dest_dir = resolve_destination(args, src_dir)
|
|
131
|
-
|
|
132
|
-
if not dest_dir.exists():
|
|
133
|
-
print(f"Destination does not exist, creating: {dest_dir}")
|
|
134
|
-
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
135
|
-
dest_dir = dest_dir.resolve()
|
|
136
|
-
|
|
137
|
-
print(SEPARATOR)
|
|
138
|
-
print(f"Source folder: {src_dir}")
|
|
139
|
-
print(f"Destination folder: {dest_dir}")
|
|
140
|
-
print(f"Extensions: {', '.join(extensions)}")
|
|
141
|
-
print(SEPARATOR)
|
|
142
|
-
|
|
143
|
-
target_files, other_files = collect_files(src_dir, dest_dir, set(extensions))
|
|
144
|
-
print(f"Found {len(target_files)} convertible file(s), {len(other_files)} file(s) with unsupported extensions.")
|
|
145
|
-
print(SEPARATOR)
|
|
146
|
-
|
|
147
|
-
converted, failed, skipped = convert_files(target_files, src_dir, dest_dir, args.force)
|
|
148
|
-
|
|
149
|
-
print(SEPARATOR)
|
|
150
|
-
print("Summary:")
|
|
151
|
-
print(f" Converted: {len(converted)}")
|
|
152
|
-
print(f" Skipped (up to date): {len(skipped)}")
|
|
153
|
-
print(f" Failed: {len(failed)}")
|
|
154
|
-
print(f" Unsupported extension: {len(other_files)}")
|
|
155
|
-
|
|
156
|
-
if failed:
|
|
157
|
-
print()
|
|
158
|
-
print("Failed conversions:")
|
|
159
|
-
for item in failed:
|
|
160
|
-
print(f" - {item}")
|
|
161
|
-
|
|
162
|
-
if other_files:
|
|
163
|
-
print()
|
|
164
|
-
print("Unsupported / not attempted:")
|
|
165
|
-
for path in other_files:
|
|
166
|
-
print(f" - {path.relative_to(src_dir)}")
|
|
167
|
-
|
|
168
|
-
print(SEPARATOR)
|
|
169
|
-
print(f"Done. Output mirrored under: {dest_dir}")
|
|
170
|
-
return 1 if failed else 0
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
if __name__ == "__main__":
|
|
174
|
-
sys.exit(main())
|
|
File without changes
|
|
File without changes
|
|
File without changes
|