any2md 0.4.0__tar.gz

any2md-0.4.0/.gitignore

@@ -0,0 +1,40 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+
+# Virtual environments
+venv/
+.venv/
+env/
+
+# Distribution / packaging
+dist/
+build/
+*.egg-info/
+*.egg
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Environment variables
+.env
+.env.local
+
+# Generated output
+output/
+Text/
+input/
+
+# ZERG / dev tooling
+.gsd/
+.zerg/
+.devcontainer/

any2md-0.4.0/CHANGELOG.md

@@ -0,0 +1,26 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
+
+## [Unreleased]
+
+### Added
+- HTML file conversion (.html, .htm) with BeautifulSoup pre-cleaning, trafilatura content extraction, and markdownify fallback
+- URL fetching — convert web pages directly by passing a URL as a positional argument
+- `--strip-links` flag to remove markdown hyperlinks from output, keeping only link text
+- .html/.htm support in `--input-dir` batch scanning
+- Package architecture (`any2md/` package with `converters` subpackage)
+- `python -m any2md` entry point
+- Shared utilities module (`any2md/utils.py`) with `strip_links()` and `url_to_filename()`
+- YAML frontmatter for HTML outputs includes `source_url` when converted from a URL
+
+### Changed
+- Refactored from single-file (`any2md.py`) to package architecture
+- `any2md.py` is now a thin wrapper for backward compatibility
+- Updated `SUPPORTED_EXTENSIONS` to include `.html` and `.htm`
+
+### Dependencies
+- Added `trafilatura` for HTML content extraction and URL fetching
+- Added `beautifulsoup4` for HTML pre-cleaning

any2md-0.4.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 rocklambros
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

any2md-0.4.0/PKG-INFO

@@ -0,0 +1,256 @@
+Metadata-Version: 2.4
+Name: any2md
+Version: 0.4.0
+Summary: Convert PDF, DOCX, and HTML files — or web pages by URL — to clean, LLM-optimized Markdown with YAML frontmatter.
+Project-URL: Homepage, https://github.com/rocklambros/any2md
+Project-URL: Issues, https://github.com/rocklambros/any2md/issues
+Author-email: rocklambros <rock@rockcyber.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Requires-Python: >=3.8
+Requires-Dist: beautifulsoup4
+Requires-Dist: mammoth
+Requires-Dist: markdownify
+Requires-Dist: pymupdf
+Requires-Dist: pymupdf4llm
+Requires-Dist: trafilatura
+Description-Content-Type: text/markdown
+
+# any2md
+
+Convert PDF, DOCX, and HTML files — or web pages by URL — to clean, LLM-optimized Markdown with YAML frontmatter.
+
+One command. Any format. Consistent, structured output ready for language models.
+
+## Quick Start
+
+```bash
+pip install any2md
+
+any2md report.pdf
+any2md https://example.com/article
+any2md --help
+```
+
+Output lands in `./Text/` by default:
+
+```markdown
+---
+title: "Quarterly Financial Report"
+source_file: "report.pdf"
+pages: 12
+type: pdf
+---
+
+# Quarterly Financial Report
+
+Document content here...
+```
+
+## Features
+
+| Feature | Description |
+|---------|-------------|
+| **Multi-format** | PDF, DOCX, HTML (.html, .htm) |
+| **URL fetching** | Pass any http/https URL as input |
+| **YAML frontmatter** | Title, source, page/word count, type |
+| **Batch processing** | Single file, directory scan, or mixed inputs |
+| **Auto-routing** | Dispatches to the correct converter by extension |
+| **Smart skip** | Won't overwrite existing files unless `--force` |
+| **Filename sanitization** | Spaces, special characters, unicode dashes handled |
+| **Title extraction** | Pulls the first H1–H3 heading automatically |
+| **Link stripping** | `--strip-links` removes hyperlinks, keeps text |
+
+## Installation
+
+Requires **Python 3.8+**.
+
+```bash
+pip install any2md
+```
+
+### From source
+
+```bash
+git clone https://github.com/rocklambros/any2md.git
+cd any2md
+pip install .
+```
+
+### Dependencies
+
+| Library | Purpose |
+|---------|---------|
+| [PyMuPDF](https://pymupdf.readthedocs.io/) + [pymupdf4llm](https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/) | PDF extraction |
+| [mammoth](https://github.com/mwilliamson/python-mammoth) + [markdownify](https://github.com/matthewwithanm/python-markdownify) | DOCX conversion |
+| [trafilatura](https://trafilatura.readthedocs.io/) + [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) | HTML/URL extraction |
+
+## Usage
+
+### Basic conversion
+
+```bash
+# Single file
+any2md report.pdf
+
+# Multiple files
+any2md report.pdf proposal.docx "meeting notes.pdf"
+
+# HTML file
+any2md page.html
+
+# Web page by URL
+any2md https://example.com/article
+
+# Mixed batch — PDFs, DOCX, HTML, and URLs together
+any2md doc.pdf page.html https://example.com
+```
+
+### Directory scanning
+
+```bash
+# Scan a specific directory
+any2md --input-dir ./documents
+
+# Convert everything in the current directory (default behavior)
+any2md
+```
+
+### Options
+
+```bash
+# Custom output directory
+any2md -o ./converted report.pdf
+
+# Overwrite existing files
+any2md --force
+
+# Strip hyperlinks from output
+any2md --strip-links doc.pdf
+
+# Combine options
+any2md -f -o ./out --strip-links docs/*.pdf docs/*.docx
+```
+
+### Alternative invocations
+
+```bash
+# Module mode (works without installing via pip)
+python -m any2md report.pdf
+
+# Legacy script (backward compatibility)
+python3 mdconv.py report.pdf
+```
+
+## Output Format
+
+Every converted file has YAML frontmatter followed by cleaned Markdown. The frontmatter fields vary by source format:
+
+**PDF** — includes page count:
+
+```markdown
+---
+title: "Quarterly Financial Report"
+source_file: "Q3 Report 2024.pdf"
+pages: 12
+type: pdf
+---
+```
+
+**DOCX** — includes word count:
+
+```markdown
+---
+title: "Project Proposal"
+source_file: "proposal.docx"
+word_count: 3847
+type: docx
+---
+```
+
+**HTML file** — includes word count:
+
+```markdown
+---
+title: "Page Title"
+source_file: "page.html"
+word_count: 1234
+type: html
+---
+```
+
+**URL** — records source URL instead of filename:
+
+```markdown
+---
+title: "Article Title"
+source_url: "https://example.com/article"
+word_count: 567
+type: html
+---
+```
+
+## CLI Reference
+
+```
+usage: any2md [-h] [--input-dir PATH] [--force] [--output-dir PATH] [--strip-links] [files ...]
+
+Convert PDF, DOCX, and HTML files to LLM-optimized Markdown.
+
+positional arguments:
+  files                 Files or URLs to convert. Supports PDF, DOCX, HTML
+                        files and http(s) URLs. If omitted, converts all
+                        supported files in the current directory.
+
+options:
+  -h, --help            show this help message and exit
+  --input-dir, -i PATH  Directory to scan for supported files (PDF, DOCX, HTML)
+  --force, -f           Overwrite existing .md files
+  --output-dir, -o PATH Output directory (default: ./Text)
+  --strip-links         Remove markdown links, keeping only the link text
+```
+
+## Architecture
+
+```
+User Input (files, URLs, flags)
+         │
+         ▼
+      cli.py ─── parse args, classify URLs vs file paths
+         │
+         ▼
+converters/__init__.py ─── dispatch by extension
+         │
+    ┌────┼────┐
+    ▼    ▼    ▼
+ pdf  docx  html ─── format-specific extraction
+    │    │    │
+    └────┼────┘
+         ▼
+      utils.py ─── clean, title-extract, sanitize, frontmatter
+         │
+         ▼
+      Output ─── YAML frontmatter + Markdown → output_dir/
+```
+
+### Extraction pipelines
+
+| Format | Pipeline |
+|--------|----------|
+| **PDF** | `pymupdf4llm.to_markdown()` → clean → frontmatter |
+| **DOCX** | `mammoth` (DOCX → HTML) → `markdownify` (HTML → Markdown) → clean → frontmatter |
+| **HTML/URL** | BS4 pre-clean → `trafilatura` extract (fallback: `markdownify`) → clean → frontmatter |
+
+### Adding a new format
+
+1. Create `any2md/converters/newformat.py` with a `convert_newformat(path, output_dir, force, strip_links_flag) → bool` function
+2. Add the extension and function to `CONVERTERS` in `any2md/converters/__init__.py`
+3. Add the extension to `SUPPORTED_EXTENSIONS`
+
+## License
+
+MIT

any2md-0.4.0/README.md

@@ -0,0 +1,234 @@
+# any2md
+
+Convert PDF, DOCX, and HTML files — or web pages by URL — to clean, LLM-optimized Markdown with YAML frontmatter.
+
+One command. Any format. Consistent, structured output ready for language models.
+
+## Quick Start
+
+```bash
+pip install any2md
+
+any2md report.pdf
+any2md https://example.com/article
+any2md --help
+```
+
+Output lands in `./Text/` by default:
+
+```markdown
+---
+title: "Quarterly Financial Report"
+source_file: "report.pdf"
+pages: 12
+type: pdf
+---
+
+# Quarterly Financial Report
+
+Document content here...
+```
+
+## Features
+
+| Feature | Description |
+|---------|-------------|
+| **Multi-format** | PDF, DOCX, HTML (.html, .htm) |
+| **URL fetching** | Pass any http/https URL as input |
+| **YAML frontmatter** | Title, source, page/word count, type |
+| **Batch processing** | Single file, directory scan, or mixed inputs |
+| **Auto-routing** | Dispatches to the correct converter by extension |
+| **Smart skip** | Won't overwrite existing files unless `--force` |
+| **Filename sanitization** | Spaces, special characters, unicode dashes handled |
+| **Title extraction** | Pulls the first H1–H3 heading automatically |
+| **Link stripping** | `--strip-links` removes hyperlinks, keeps text |
+
+## Installation
+
+Requires **Python 3.8+**.
+
+```bash
+pip install any2md
+```
+
+### From source
+
+```bash
+git clone https://github.com/rocklambros/any2md.git
+cd any2md
+pip install .
+```
+
+### Dependencies
+
+| Library | Purpose |
+|---------|---------|
+| [PyMuPDF](https://pymupdf.readthedocs.io/) + [pymupdf4llm](https://pymupdf.readthedocs.io/en/latest/pymupdf4llm/) | PDF extraction |
+| [mammoth](https://github.com/mwilliamson/python-mammoth) + [markdownify](https://github.com/matthewwithanm/python-markdownify) | DOCX conversion |
+| [trafilatura](https://trafilatura.readthedocs.io/) + [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) | HTML/URL extraction |
+
+## Usage
+
+### Basic conversion
+
+```bash
+# Single file
+any2md report.pdf
+
+# Multiple files
+any2md report.pdf proposal.docx "meeting notes.pdf"
+
+# HTML file
+any2md page.html
+
+# Web page by URL
+any2md https://example.com/article
+
+# Mixed batch — PDFs, DOCX, HTML, and URLs together
+any2md doc.pdf page.html https://example.com
+```
+
+### Directory scanning
+
+```bash
+# Scan a specific directory
+any2md --input-dir ./documents
+
+# Convert everything in the current directory (default behavior)
+any2md
+```
+
+### Options
+
+```bash
+# Custom output directory
+any2md -o ./converted report.pdf
+
+# Overwrite existing files
+any2md --force
+
+# Strip hyperlinks from output
+any2md --strip-links doc.pdf
+
+# Combine options
+any2md -f -o ./out --strip-links docs/*.pdf docs/*.docx
+```
+
+### Alternative invocations
+
+```bash
+# Module mode (works without installing via pip)
+python -m any2md report.pdf
+
+# Legacy script (backward compatibility)
+python3 mdconv.py report.pdf
+```
+
+## Output Format
+
+Every converted file has YAML frontmatter followed by cleaned Markdown. The frontmatter fields vary by source format:
+
+**PDF** — includes page count:
+
+```markdown
+---
+title: "Quarterly Financial Report"
+source_file: "Q3 Report 2024.pdf"
+pages: 12
+type: pdf
+---
+```
+
+**DOCX** — includes word count:
+
+```markdown
+---
+title: "Project Proposal"
+source_file: "proposal.docx"
+word_count: 3847
+type: docx
+---
+```
+
+**HTML file** — includes word count:
+
+```markdown
+---
+title: "Page Title"
+source_file: "page.html"
+word_count: 1234
+type: html
+---
+```
+
+**URL** — records source URL instead of filename:
+
+```markdown
+---
+title: "Article Title"
+source_url: "https://example.com/article"
+word_count: 567
+type: html
+---
+```
+
+## CLI Reference
+
+```
+usage: any2md [-h] [--input-dir PATH] [--force] [--output-dir PATH] [--strip-links] [files ...]
+
+Convert PDF, DOCX, and HTML files to LLM-optimized Markdown.
+
+positional arguments:
+  files                 Files or URLs to convert. Supports PDF, DOCX, HTML
+                        files and http(s) URLs. If omitted, converts all
+                        supported files in the current directory.
+
+options:
+  -h, --help            show this help message and exit
+  --input-dir, -i PATH  Directory to scan for supported files (PDF, DOCX, HTML)
+  --force, -f           Overwrite existing .md files
+  --output-dir, -o PATH Output directory (default: ./Text)
+  --strip-links         Remove markdown links, keeping only the link text
+```
+
+## Architecture
+
+```
+User Input (files, URLs, flags)
+         │
+         ▼
+      cli.py ─── parse args, classify URLs vs file paths
+         │
+         ▼
+converters/__init__.py ─── dispatch by extension
+         │
+    ┌────┼────┐
+    ▼    ▼    ▼
+ pdf  docx  html ─── format-specific extraction
+    │    │    │
+    └────┼────┘
+         ▼
+      utils.py ─── clean, title-extract, sanitize, frontmatter
+         │
+         ▼
+      Output ─── YAML frontmatter + Markdown → output_dir/
+```
+
+### Extraction pipelines
+
+| Format | Pipeline |
+|--------|----------|
+| **PDF** | `pymupdf4llm.to_markdown()` → clean → frontmatter |
+| **DOCX** | `mammoth` (DOCX → HTML) → `markdownify` (HTML → Markdown) → clean → frontmatter |
+| **HTML/URL** | BS4 pre-clean → `trafilatura` extract (fallback: `markdownify`) → clean → frontmatter |
+
+### Adding a new format
+
+1. Create `any2md/converters/newformat.py` with a `convert_newformat(path, output_dir, force, strip_links_flag) → bool` function
+2. Add the extension and function to `CONVERTERS` in `any2md/converters/__init__.py`
+3. Add the extension to `SUPPORTED_EXTENSIONS`
+
+## License
+
+MIT

any2md-0.4.0/any2md/__init__.py

@@ -0,0 +1,3 @@
+"""Convert PDF, DOCX, and HTML files to LLM-optimized Markdown."""
+
+__version__ = "0.4.0"

any2md-0.4.0/any2md/__main__.py

@@ -0,0 +1,4 @@
+"""Allow running any2md as a module: python -m any2md."""
+from any2md.cli import main
+
+main()

any2md-0.4.0/any2md/cli.py

@@ -0,0 +1,145 @@
+"""CLI entry point for any2md."""
+
+import argparse
+import sys
+import time
+from pathlib import Path
+
+from any2md.converters import convert_file, SUPPORTED_EXTENSIONS
+from any2md.converters.html import convert_html, fetch_url
+from any2md.utils import sanitize_filename, url_to_filename
+
+SCRIPT_DIR = Path.cwd()
+DEFAULT_OUTPUT_DIR = SCRIPT_DIR / "Text"
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Convert PDF, DOCX, and HTML files to LLM-optimized Markdown."
+    )
+    parser.add_argument(
+        "files",
+        nargs="*",
+        help="Files or URLs to convert. Supports PDF, DOCX, HTML files and http(s) URLs. "
+        "If omitted, converts all supported files in the current directory.",
+    )
+    parser.add_argument(
+        "--input-dir", "-i",
+        type=Path,
+        help="Directory to scan for supported files (PDF, DOCX, HTML).",
+    )
+    parser.add_argument(
+        "--force", "-f",
+        action="store_true",
+        help="Overwrite existing .md files.",
+    )
+    parser.add_argument(
+        "--output-dir", "-o",
+        type=Path,
+        default=DEFAULT_OUTPUT_DIR,
+        help=f"Output directory (default: {DEFAULT_OUTPUT_DIR}).",
+    )
+    parser.add_argument(
+        "--strip-links",
+        action="store_true",
+        help="Remove markdown links, keeping only the link text.",
+    )
+    args = parser.parse_args()
+
+    # Determine which files to process
+    if args.files and args.input_dir:
+        print("Error: cannot use both positional files and --input-dir.", file=sys.stderr)
+        sys.exit(1)
+
+    urls = []
+    file_paths = []
+
+    if args.files:
+        for f in args.files:
+            # URL detection
+            if f.startswith("http://") or f.startswith("https://"):
+                urls.append(f)
+                continue
+
+            p = Path(f)
+            if not p.is_absolute():
+                p = Path.cwd() / p
+            if not p.exists():
+                print(f"  NOT FOUND: {f}", file=sys.stderr)
+                continue
+            if p.suffix.lower() not in SUPPORTED_EXTENSIONS:
+                print(f"  UNSUPPORTED FORMAT: {f}", file=sys.stderr)
+                continue
+            file_paths.append(p)
+    elif args.input_dir:
+        if not args.input_dir.is_dir():
+            print(f"Error: not a directory: {args.input_dir}", file=sys.stderr)
+            sys.exit(1)
+        file_paths = sorted(
+            p for ext in SUPPORTED_EXTENSIONS
+            for p in args.input_dir.glob(f"*{ext}")
+        )
+    else:
+        file_paths = sorted(
+            p for ext in SUPPORTED_EXTENSIONS
+            for p in SCRIPT_DIR.glob(f"*{ext}")
+        )
+
+    if not file_paths and not urls:
+        print("No supported files to process.")
+        sys.exit(0)
+
+    total = len(file_paths) + len(urls)
+    print(f"Processing {total} file(s) → {args.output_dir}/\n")
+    start = time.time()
+    ok = 0
+    fail = 0
+    skip = 0
+
+    # Process URLs
+    for url in urls:
+        html_content, error = fetch_url(url)
+        if error:
+            print(f"  FAIL: {url} -- {error}", file=sys.stderr)
+            fail += 1
+            continue
+
+        out_name = url_to_filename(url)
+        out_exists = (args.output_dir / out_name).exists()
+        if out_exists and not args.force:
+            skip += 1
+
+        result = convert_html(
+            None,
+            args.output_dir,
+            force=args.force,
+            strip_links_flag=args.strip_links,
+            source_url=url,
+            html_content=html_content,
+        )
+        if result:
+            if not (out_exists and not args.force):
+                ok += 1
+        else:
+            fail += 1
+
+    # Process local files
+    for file_path in file_paths:
+        out_name = sanitize_filename(file_path.name)
+        out_exists = (args.output_dir / out_name).exists()
+        if out_exists and not args.force:
+            skip += 1
+        result = convert_file(
+            file_path,
+            args.output_dir,
+            force=args.force,
+            strip_links_flag=args.strip_links,
+        )
+        if result:
+            if not (out_exists and not args.force):
+                ok += 1
+        else:
+            fail += 1
+
+    elapsed = time.time() - start
+    print(f"\nDone in {elapsed:.1f}s: {ok} converted, {skip} skipped, {fail} failed.")

any2md-0.4.0/any2md/converters/__init__.py

@@ -0,0 +1,32 @@
+"""Converter dispatcher for any2md."""
+
+import sys
+from pathlib import Path
+
+from any2md.converters.pdf import convert_pdf
+from any2md.converters.docx import convert_docx
+from any2md.converters.html import convert_html
+
+SUPPORTED_EXTENSIONS = {".pdf", ".docx", ".html", ".htm"}
+
+CONVERTERS = {
+    ".pdf": convert_pdf,
+    ".docx": convert_docx,
+    ".html": convert_html,
+    ".htm": convert_html,
+}
+
+
+def convert_file(
+    file_path: Path,
+    output_dir: Path,
+    force: bool = False,
+    strip_links_flag: bool = False,
+) -> bool:
+    """Dispatch to the appropriate converter based on file extension."""
+    ext = file_path.suffix.lower()
+    converter = CONVERTERS.get(ext)
+    if converter is None:
+        print(f"  UNSUPPORTED: {file_path.name} (no converter for {ext})", file=sys.stderr)
+        return False
+    return converter(file_path, output_dir, force=force, strip_links_flag=strip_links_flag)

any2md-0.4.0/any2md/converters/docx.py

@@ -0,0 +1,72 @@
+"""DOCX to Markdown converter module."""
+
+import sys
+from pathlib import Path
+
+import mammoth
+import markdownify
+
+from any2md.utils import sanitize_filename, extract_title, clean_markdown, strip_links, escape_yaml_string
+
+
+def convert_docx(
+    docx_path: Path,
+    output_dir: Path,
+    force: bool = False,
+    strip_links_flag: bool = False,
+) -> bool:
+    """Convert a single DOCX to LLM-optimized Markdown.
+
+    Returns True on success, False on failure.
+    """
+    out_name = sanitize_filename(docx_path.name)
+    out_path = output_dir / out_name
+
+    if out_path.exists() and not force:
+        print(f"  SKIP (exists): {out_name}")
+        return True
+
+    try:
+        with open(docx_path, "rb") as f:
+            result = mammoth.convert_to_html(f)
+
+        md_text = markdownify.markdownify(
+            result.value,
+            heading_style="ATX",
+            strip=["img"],
+        )
+
+        # Clean markdown content
+        md_text = clean_markdown(md_text)
+
+        # Optionally strip links (before frontmatter)
+        if strip_links_flag:
+            md_text = strip_links(md_text)
+
+        # Extract title
+        title = extract_title(md_text, docx_path.stem)
+
+        # Word count (DOCX has no reliable page count)
+        word_count = len(md_text.split())
+
+        # Build frontmatter (escape values for valid YAML)
+        frontmatter = (
+            f'---\n'
+            f'title: "{escape_yaml_string(title)}"\n'
+            f'source_file: "{escape_yaml_string(docx_path.name)}"\n'
+            f'word_count: {word_count}\n'
+            f'type: docx\n'
+            f'---\n\n'
+        )
+
+        full_text = frontmatter + md_text
+
+        # Write output
+        output_dir.mkdir(parents=True, exist_ok=True)
+        out_path.write_text(full_text, encoding="utf-8")
+        print(f"  OK: {out_name} ({word_count} words)")
+        return True
+
+    except Exception as e:
+        print(f"  FAIL: {docx_path.name} -- {e}", file=sys.stderr)
+        return False

any2md-0.4.0/any2md/converters/html.py

@@ -0,0 +1,162 @@
+"""HTML to Markdown converter module."""
+
+from __future__ import annotations
+
+import sys
+import urllib.parse
+from pathlib import Path
+
+import trafilatura
+import markdownify
+from bs4 import BeautifulSoup
+
+from any2md.utils import (
+    sanitize_filename,
+    extract_title,
+    clean_markdown,
+    strip_links,
+    url_to_filename,
+    escape_yaml_string,
+)
+
+
+def fetch_url(url: str) -> tuple[str | None, str | None]:
+    """Fetch HTML content from a URL.
+
+    Only http and https schemes are accepted.
+
+    Returns (html_string, None) on success or (None, error_message) on failure.
+    """
+    parsed = urllib.parse.urlparse(url)
+    if parsed.scheme not in ("http", "https"):
+        return None, f"Unsupported URL scheme: {parsed.scheme!r} (only http/https allowed)"
+
+    try:
+        html = trafilatura.fetch_url(url)
+        if html is None:
+            return None, f"Failed to fetch URL: {url}"
+        return html, None
+    except Exception as e:
+        return None, f"Error fetching URL: {e}"
+
+
+def _bs4_preclean(html: str) -> str:
+    """Remove boilerplate HTML elements before conversion.
+
+    Strips script, style, nav, header, footer, aside, and iframe tags
+    along with their contents.
+    """
+    soup = BeautifulSoup(html, "html.parser")
+    for tag in soup.find_all(["script", "style", "nav", "header", "footer", "aside", "iframe"]):
+        tag.decompose()
+    return str(soup)
+
+
+def convert_html(
+    html_path: Path | None,
+    output_dir: Path,
+    force: bool = False,
+    strip_links_flag: bool = False,
+    source_url: str | None = None,
+    html_content: str | None = None,
+) -> bool:
+    """Convert HTML to LLM-optimized Markdown.
+
+    When *html_content* is provided it is used directly; otherwise the file
+    at *html_path* is read.  When *source_url* is set, frontmatter records
+    the URL instead of a local filename.
+
+    Returns True on success, False on failure.
+    """
+    # Determine output filename
+    if source_url:
+        out_name = url_to_filename(source_url)
+        name_for_error = source_url
+    elif html_path is not None:
+        out_name = sanitize_filename(html_path.name)
+        name_for_error = html_path.name
+    else:
+        raise ValueError("Either source_url or html_path must be provided")
+
+    out_path = output_dir / out_name
+
+    if out_path.exists() and not force:
+        print(f"  SKIP (exists): {out_name}")
+        return True
+
+    try:
+        # 1. Acquire HTML
+        if html_content is not None:
+            raw_html = html_content
+        elif html_path is not None:
+            try:
+                raw_html = html_path.read_text(encoding="utf-8")
+            except UnicodeDecodeError:
+                raw_html = html_path.read_text(encoding="latin-1")
+        else:
+            raise ValueError("Either html_content or html_path must be provided")
+
+        # 2. BS4 pre-clean
+        cleaned_html = _bs4_preclean(raw_html)
+
+        # 3. trafilatura extract
+        md_text = trafilatura.extract(
+            cleaned_html,
+            include_formatting=True,
+            include_links=True,
+        )
+
+        # 4. Fallback to markdownify if trafilatura returned nothing
+        if not md_text:
+            md_text = markdownify.markdownify(
+                cleaned_html,
+                heading_style="ATX",
+                strip=["img"],
+            )
+
+        # 5. Clean markdown
+        md_text = clean_markdown(md_text)
+
+        # 6. Optionally strip links
+        if strip_links_flag:
+            md_text = strip_links(md_text)
+
+        # 7. Extract title
+        if source_url:
+            fallback = urllib.parse.urlparse(source_url).netloc
+        elif html_path is not None:
+            fallback = html_path.stem
+        else:
+            fallback = "untitled"
+        title = extract_title(md_text, fallback)
+
+        # 8. Word count
+        word_count = len(md_text.split())
+
+        # 9. Build frontmatter (escape values for valid YAML)
+        if source_url:
+            source_field = f'source_url: "{escape_yaml_string(source_url)}"'
+        elif html_path is not None:
+            source_field = f'source_file: "{escape_yaml_string(html_path.name)}"'
+        else:
+            source_field = 'source_file: "unknown"'
+
+        frontmatter = (
+            f'---\n'
+            f'title: "{escape_yaml_string(title)}"\n'
+            f'{source_field}\n'
+            f'word_count: {word_count}\n'
+            f'type: html\n'
+            f'---\n\n'
+        )
+
+        # 10-11. Write output
+        full_text = frontmatter + md_text
+        output_dir.mkdir(parents=True, exist_ok=True)
+        out_path.write_text(full_text, encoding="utf-8")
+        print(f"  OK: {out_name} ({word_count} words)")
+        return True
+
+    except Exception as e:
+        print(f"  FAIL: {name_for_error} -- {e}", file=sys.stderr)
+        return False

any2md-0.4.0/any2md/converters/pdf.py

@@ -0,0 +1,73 @@
+"""PDF to Markdown converter."""
+
+import sys
+from pathlib import Path
+
+import pymupdf
+import pymupdf4llm
+
+from any2md.utils import sanitize_filename, extract_title, clean_markdown, strip_links, escape_yaml_string
+
+
+def convert_pdf(
+    pdf_path: Path,
+    output_dir: Path,
+    force: bool = False,
+    strip_links_flag: bool = False,
+) -> bool:
+    """Convert a single PDF to LLM-optimized Markdown.
+
+    Returns True on success, False on failure.
+    """
+    out_name = sanitize_filename(pdf_path.name)
+    out_path = output_dir / out_name
+
+    if out_path.exists() and not force:
+        print(f"  SKIP (exists): {out_name}")
+        return True
+
+    try:
+        # Get page count
+        doc = pymupdf.open(str(pdf_path))
+        page_count = len(doc)
+        doc.close()
+
+        # Convert to markdown
+        md_text = pymupdf4llm.to_markdown(
+            str(pdf_path),
+            write_images=False,
+            show_progress=False,
+            force_text=True,
+        )
+
+        # Clean markdown content
+        md_text = clean_markdown(md_text)
+
+        # Optionally strip links (before frontmatter)
+        if strip_links_flag:
+            md_text = strip_links(md_text)
+
+        # Extract title
+        title = extract_title(md_text, pdf_path.stem)
+
+        # Build frontmatter (escape values for valid YAML)
+        frontmatter = (
+            f'---\n'
+            f'title: "{escape_yaml_string(title)}"\n'
+            f'source_file: "{escape_yaml_string(pdf_path.name)}"\n'
+            f'pages: {page_count}\n'
+            f'type: pdf\n'
+            f'---\n\n'
+        )
+
+        full_text = frontmatter + md_text
+
+        # Write output
+        output_dir.mkdir(parents=True, exist_ok=True)
+        out_path.write_text(full_text, encoding="utf-8")
+        print(f"  OK: {out_name} ({page_count} pages)")
+        return True
+
+    except Exception as e:
+        print(f"  FAIL: {pdf_path.name} -- {e}", file=sys.stderr)
+        return False

any2md-0.4.0/any2md/utils.py

@@ -0,0 +1,86 @@
+"""Shared utility functions for any2md."""
+
+import re
+import urllib.parse
+from pathlib import Path
+
+
+def sanitize_filename(name: str) -> str:
+    """Convert a source filename to a sanitized .md filename.
+
+    Matches existing convention: spaces -> underscores, extension -> .md.
+    """
+    stem = Path(name).stem
+    # Replace spaces with underscores
+    stem = stem.replace(" ", "_")
+    # Replace characters problematic in filenames
+    stem = re.sub(r"[,;:'\"\u2014\u2013]", "", stem)
+    # Collapse multiple underscores
+    stem = re.sub(r"_+", "_", stem)
+    # Strip leading/trailing underscores
+    stem = stem.strip("_")
+    return stem + ".md"
+
+
+def extract_title(markdown_text: str, fallback: str) -> str:
+    """Extract the first markdown heading as the document title."""
+    match = re.search(r"^#{1,3}\s+(.+)", markdown_text, re.MULTILINE)
+    if match:
+        title = match.group(1).strip()
+        # Clean markdown formatting from title
+        title = re.sub(r"\*+", "", title)
+        title = re.sub(r"_+", " ", title)
+        title = title.strip()
+        if len(title) > 10:
+            return title
+    # Fallback: derive from filename
+    return fallback.replace("_", " ").strip()
+
+
+def clean_markdown(text: str) -> str:
+    """Clean up markdown for LLM consumption.
+
+    Reduces excessive whitespace while preserving structure.
+    """
+    # Collapse 3+ consecutive blank lines to 2
+    text = re.sub(r"\n{4,}", "\n\n\n", text)
+    # Remove trailing whitespace on each line
+    text = re.sub(r"[ \t]+$", "", text, flags=re.MULTILINE)
+    # Ensure file ends with single newline
+    text = text.rstrip() + "\n"
+    return text
+
+
+def escape_yaml_string(value: str) -> str:
+    """Escape a string for safe inclusion in double-quoted YAML values."""
+    return value.replace("\\", "\\\\").replace('"', '\\"')
+
+
+def strip_links(text: str) -> str:
+    """Replace markdown links with their display text.
+
+    Converts ``[text](url)`` to ``text``.
+    """
+    return re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text)
+
+
+def url_to_filename(url: str) -> str:
+    """Convert a URL to a sanitized .md filename.
+
+    Uses the netloc and path components, replacing dots and slashes
+    with underscores and collapsing duplicates.
+
+    Example::
+
+        >>> url_to_filename("https://example.com/blog/my-post")
+        'example_com_blog_my-post.md'
+    """
+    parsed = urllib.parse.urlparse(url)
+    raw = parsed.netloc + parsed.path
+    # Replace dots and slashes with underscores
+    raw = raw.replace(".", "_").replace("/", "_")
+    # Strip leading/trailing underscores
+    raw = raw.strip("_")
+    # Collapse multiple underscores
+    raw = re.sub(r"_+", "_", raw)
+    return raw + ".md"

any2md-0.4.0/pyproject.toml

@@ -0,0 +1,54 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "any2md"
+dynamic = ["version"]
+description = "Convert PDF, DOCX, and HTML files — or web pages by URL — to clean, LLM-optimized Markdown with YAML frontmatter."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.8"
+authors = [
+    { name = "rocklambros", email = "rock@rockcyber.com" },
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Topic :: Text Processing :: Markup :: Markdown",
+]
+dependencies = [
+    "pymupdf",
+    "pymupdf4llm",
+    "mammoth",
+    "markdownify",
+    "trafilatura",
+    "beautifulsoup4",
+]
+
+[project.scripts]
+any2md = "any2md.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/rocklambros/any2md"
+Issues = "https://github.com/rocklambros/any2md/issues"
+
+[tool.hatch.version]
+path = "any2md/__init__.py"
+
+[tool.hatch.build.targets.sdist]
+exclude = [
+    "/.github",
+    "/.claude",
+    "/.devcontainer",
+    "/.gsd",
+    "/.zerg",
+    "/input",
+    "/output",
+    "/Text",
+    "mdconv.py",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["any2md"]

any2md-0.4.0/requirements.txt

@@ -0,0 +1,6 @@
+pymupdf
+pymupdf4llm
+mammoth
+markdownify
+trafilatura
+beautifulsoup4