doculift-cli 0.1.0__py3-none-any.whl

doculift/__main__.py

@@ -0,0 +1,4 @@
+from .cli import cli
+
+if __name__ == "__main__":
+    cli()

doculift/app.py

@@ -0,0 +1,97 @@
+from flask import Flask, render_template, request, jsonify, send_from_directory
+import threading
+import uuid
+import os
+from .scraper import DocuLiftScraper
+
+app = Flask(__name__)
+
+# In-memory storage for jobs
+# Production should use Redis/Celery, but for this scale a global dict is fine.
+jobs = {}
+
+OUTPUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "outputs")
+if not os.path.exists(OUTPUT_DIR):
+    os.makedirs(OUTPUT_DIR)
+
+
+@app.route("/")
+def index():
+    return render_template("index.html")
+
+
+@app.route("/scrape", methods=["POST"])
+def start_scrape():
+    data = request.json
+    urls = data.get("urls", [])
+    output_format = data.get("format", "md")
+    max_pages = int(data.get("max_pages", 500))
+    scope_type = data.get("scope_type", "section")
+    extract_mode = data.get("extract_mode", "content")
+
+    if not urls:
+        return jsonify({"error": "No URLs provided"}), 400
+
+    job_id = str(uuid.uuid4())
+    scraper = DocuLiftScraper(
+        urls,
+        output_format=output_format,
+        max_pages=max_pages,
+        scope_type=scope_type,
+        extract_mode=extract_mode,
+    )
+
+    jobs[job_id] = {"scraper": scraper, "status": "pending", "progress": 0, "files": []}
+
+    # Start thread
+    thread = threading.Thread(target=run_scraper_task, args=(job_id, scraper))
+    thread.daemon = True
+    thread.start()
+
+    return jsonify({"job_id": job_id})
+
+
+def run_scraper_task(job_id, scraper):
+    try:
+        jobs[job_id]["status"] = "running"
+        job_dir = os.path.join(OUTPUT_DIR, job_id)
+        scraper.run(job_dir)
+
+        # Get list of files
+        files = os.listdir(job_dir)
+        jobs[job_id]["files"] = files
+        jobs[job_id]["status"] = "completed"
+        jobs[job_id]["progress"] = 100
+    except Exception as e:
+        jobs[job_id]["status"] = f"error: {str(e)}"
+
+
+@app.route("/status/<job_id>")
+def get_status(job_id):
+    job = jobs.get(job_id)
+    if not job:
+        return jsonify({"error": "Job not found"}), 404
+
+    return jsonify(
+        {
+            "status": job["scraper"].status,
+            "progress": job["scraper"].progress,
+            "is_finished": job["status"] in ["completed", "error"]
+            or "error" in job["status"],
+            "files": job["files"],
+            "job_id": job_id,
+            "urls_extracted": job["scraper"].urls_extracted,
+            "per_url_stats": job["scraper"].per_url_stats,
+        }
+    )
+
+
+@app.route("/download/<job_id>/<filename>")
+def download_file(job_id, filename):
+    return send_from_directory(os.path.join(OUTPUT_DIR, job_id), filename)
+
+
+if __name__ == "__main__":
+    # debug=True is disabled to prevent arbitrary code execution vulnerabilities
+    # in production-like environments, satisfying Bandit B201.
+    app.run(debug=False, port=5001)

doculift/cli.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+import click
+import os
+import sys
+
+# Ensure the scripts can import our local modules
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+# Rich configuration for beautiful terminal output
+from rich.console import Console  # noqa: E402
+from rich.progress import (  # noqa: E402
+    Progress,
+    SpinnerColumn,
+    BarColumn,
+    TimeElapsedColumn,
+)
+from rich.panel import Panel  # noqa: E402
+
+from .scraper import DocuLiftScraper  # noqa: E402
+from .app import app, OUTPUT_DIR  # noqa: E402
+
+console = Console()
+
+CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
+
+
+@click.group(context_settings=CONTEXT_SETTINGS)
+def cli():
+    """
+    \b
+    DocuLift - Effortlessly lift documentation for AI.
+
+    A hybrid tool to siphon documentation websites into clean Markdown files
+    or URL lists, optimized for Large Language Models.
+
+    \b
+    Examples:
+      # Start the local Web UI
+      doculift ui
+
+      # Extract full markdown content from a documentation section
+      doculift scrape https://docs.docker.com/reference/
+
+      # Just extract all the URLs from a section, up to 1000 pages
+      doculift scrape https://paketo.io/docs/ --mode urls --max-pages 1000
+    """
+    pass
+
+
+@cli.command()
+@click.option("--port", default=5001, help="Port to run the Flask web UI on.")
+def ui(port):
+    """Launch the DocuLift Web User Interface."""
+    console.print(
+        f"[bold green]🚀 Starting DocuLift Web UI "
+        f"on port {port}...[/bold green]"
+    )
+    app.run(debug=False, port=port)
+
+
+@cli.command()
+@click.argument("urls", nargs=-1, required=True)
+@click.option(
+    "-m",
+    "--mode",
+    type=click.Choice(["content", "urls"]),
+    default="content",
+    help='Extract full "content" (Markdown) or just "urls".',
+)
+@click.option(
+    "-s",
+    "--scope",
+    type=click.Choice(["section", "domain"]),
+    default="section",
+    help=(
+        'Restrict crawl to the starting "section" folder, '
+        'or allow the entire "domain".'
+    ),
+)
+@click.option(
+    "-f",
+    "--format",
+    "output_format",
+    default="md",
+    help="Output file format (default: md).",
+)
+@click.option(
+    "-p",
+    "--max-pages",
+    type=int,
+    default=500,
+    help="Maximum number of pages to scan PER starting URL.",
+)
+def scrape(urls, mode, scope, output_format, max_pages):
+    """
+    Scrape target URLs from the terminal.
+
+    URLS: One or more starting documentation URLs separated by spaces.
+    """
+    console.print(
+        Panel(
+            f"[bold blue]DocuLift CLI[/bold blue]\nTarget URLs: {len(urls)}"
+            f"\nMode: {mode.upper()}\nScope: {scope.capitalize()}",
+            expand=False,
+        )
+    )
+
+    # Initialize the scraper class directly
+    scraper = DocuLiftScraper(
+        start_urls=list(urls),
+        output_format=output_format,
+        max_pages=max_pages,
+        scope_type=scope,
+        extract_mode=mode,
+    )
+
+    # Use Rich to create a beautiful progress bar in the terminal
+    with Progress(
+        SpinnerColumn(),
+        "[progress.description]{task.description}",
+        BarColumn(),
+        "[progress.percentage]{task.percentage:>3.0f}%",
+        TimeElapsedColumn(),
+        console=console,
+    ) as progress:
+
+        # We need to run the scraper, but we also want progress updates.
+        # Since scraper.run() is synchronous and holds the thread, we will
+        # run it in a background thread and use the main thread to poll its
+        # status and update Rich.
+        import threading
+        import time
+
+        task = progress.add_task("[cyan]Initializing...", total=100)
+
+        def run_scraper():
+            try:
+                scraper.run(OUTPUT_DIR)
+            except Exception as e:
+                console.print(
+                    f"[bold red]Error during scraping: {e}[/bold red]"
+                )
+
+        thread = threading.Thread(target=run_scraper)
+        thread.daemon = True
+        thread.start()
+
+        # Poll and update until finished
+        while thread.is_alive():
+            progress.update(
+                task,
+                description=f"[cyan]{scraper.status}",
+                completed=scraper.progress,
+            )
+            time.sleep(0.5)
+
+        # Final update
+        progress.update(
+            task, description=f"[green]{scraper.status}", completed=100
+        )
+
+    # Print Summary
+    console.print("\n[bold green]✅ Mission Accomplished![/bold green]")
+    for url, count in scraper.per_url_stats.items():
+        label = "URLs extracted" if mode == "urls" else "Pages scraped"
+        console.print(
+            f"  [dim]•[/dim] {url} -> [bold cyan]{count}[/bold cyan] {label}"
+        )
+
+    console.print(
+        "\n[reset]Check the [bold yellow]outputs/[/bold yellow] directory "
+        "for your files.[/reset]\n"
+    )
+
+
+if __name__ == "__main__":
+    cli()