doculift-cli 0.1.0__py3-none-any.whl

doculift/scraper.py

@@ -0,0 +1,442 @@
+from bs4 import BeautifulSoup, NavigableString
+from urllib.parse import urljoin, urlparse
+from collections import deque
+import time
+import os
+import re
+
+from playwright.sync_api import sync_playwright
+
+
+class DocuLiftScraper:
+    def __init__(
+        self,
+        start_urls,
+        output_format="md",
+        max_pages=500,
+        max_chars_per_file=500000,
+        scope_type="section",
+        extract_mode="content",
+    ):
+        # Allow passing a single string or list
+        if isinstance(start_urls, str):
+            start_urls = [start_urls]
+        self.start_urls = [url.strip() for url in start_urls if url.strip()]
+        self.output_format = output_format.lower()
+        self.max_pages = max_pages
+        # 500KB approx char limit for NotebookLM
+        self.max_chars_per_file = max_chars_per_file
+        self.scope_type = scope_type
+        self.extract_mode = extract_mode
+        self.visited_urls = set()
+        self.pages_scraped = 0
+        self.urls_extracted = 0
+        self.per_url_stats = {}  # start_url -> count of pages/urls collected
+        self.status = "Initializing..."
+        self.progress = 0
+        self.max_chars_per_file = max_chars_per_file
+        self.visited_urls = set()
+        self.pages_scraped = 0
+        self.status = "Initializing..."
+        self.progress = 0
+
+    def clean_text(self, text):
+        if not text:
+            return ""
+        text = re.sub(r"[ \t]+", " ", text)
+        text = re.sub(r"\n\s*\n", "\n\n", text)
+        return text.strip()
+
+    def html_to_markdown(self, element, base_url=""):
+        if element is None:
+            return ""
+        if isinstance(element, NavigableString):
+            return self.clean_text(str(element))
+
+        tag_name = element.name
+        content = ""
+
+        if tag_name in ["ul", "ol"]:
+            list_md = "\n"
+            for i, child in enumerate(element.find_all("li", recursive=False)):
+                li_content = ""
+                for sub in child.children:
+                    li_content += self.html_to_markdown(sub, base_url) + " "
+                li_content = self.clean_text(li_content)
+                if tag_name == "ul":
+                    list_md += f"- {li_content}\n"
+                else:
+                    list_md += f"{i+1}. {li_content}\n"
+            return list_md + "\n"
+
+        for child in element.children:
+            child_text = self.html_to_markdown(child, base_url)
+            if child_text:
+                content += child_text + " "
+
+        content = self.clean_text(content)
+
+        if tag_name in ["h1", "h2", "h3", "h4", "h5", "h6"]:
+            level = (
+                int(tag_name[1]) if len(tag_name) > 1 and tag_name[1].isdigit() else 1
+            )
+            return f"\n\n{'#' * level} {content}\n\n"
+        elif tag_name == "p":
+            return f"\n\n{content}\n\n"
+        elif tag_name == "pre":
+            # Try to find code language class
+            lang = ""
+            code_elem = element.find("code")
+            if code_elem and code_elem.get("class"):
+                for cls in code_elem.get("class"):
+                    if cls.startswith("language-"):
+                        lang = cls.replace("language-", "")
+                        break
+            return f"\n\n```{lang}\n{element.get_text()}\n```\n\n"
+        elif tag_name == "code":
+            if element.parent.name == "pre":
+                return element.get_text()
+            return f"`{content}`"
+        elif tag_name == "a":
+            href = element.get("href", "")
+            # Resolve relative URLs if base_url is present (simplified)
+            if base_url and href and not href.startswith(("http", "mailto", "#")):
+                href = urljoin(base_url, href)
+            return f"[{content}]({href})"
+        elif tag_name == "img":
+            src = element.get("src", "")
+            if base_url and src and not src.startswith(("http", "data:")):
+                src = urljoin(base_url, src)
+            return f"![{element.get('alt', '')}]({src})"
+        elif tag_name == "blockquote":
+            return f"\n> {content}\n"
+        elif tag_name == "tr":
+            return content + "\n"
+        elif tag_name in ["td", "th"]:
+            return content + " | "
+
+        return content
+
+    def fetch_and_process(self, page, url, original_domain, scope_url):
+        try:
+            print(f"Navigating to {url}...")
+            # Set timeout to handle slow pages
+            try:
+                page.goto(url, timeout=45000)
+            except Exception as e:
+                print(f"Timeout on {url}, attempting to read anyway: {e}")
+
+            # Wait for meaningful content or network idle
+            try:
+                page.wait_for_load_state("networkidle", timeout=5000)
+            except Exception:  # nosec B110
+                pass
+
+            # Additional small wait for JS rendering
+            time.sleep(1)
+
+            html_content = page.content()
+            soup = BeautifulSoup(html_content, "html.parser")
+            extracted_links = set()
+
+            # Extract links *before* cleaning garbage to ensure navigation works
+            for link in soup.find_all("a", href=True):
+                href = link.get("href").strip()
+                # Skip anchor links on same page to avoid loops
+                if href.startswith("#"):
+                    continue
+
+                full_url = urljoin(url, href)
+                # Remove fragment for deduplication
+                full_url_no_frag = full_url.split("#")[0]
+
+                parsed = urlparse(full_url_no_frag)
+                clean_url = parsed.scheme + "://" + parsed.netloc + parsed.path
+
+                # Check Domain & Scope
+                if parsed.netloc == original_domain and clean_url.startswith(scope_url):
+                    extracted_links.add(clean_url)
+
+            # Define garbage to remove
+            for garbage in soup(
+                [
+                    "script",
+                    "style",
+                    "nav",
+                    "footer",
+                    "header",
+                    "aside",
+                    "form",
+                    "iframe",
+                    "noscript",
+                    "button",
+                ]
+            ):
+                garbage.decompose()
+
+            # Find main content area - heuristic
+            main_content = (
+                soup.find("main")
+                or soup.find("article")
+                or soup.find("div", class_="content")
+                or soup.find("div", id="content")
+                or soup.body
+                or soup
+            )
+
+            markdown_text = self.html_to_markdown(main_content, base_url=url)
+            page_title = soup.title.string.strip() if soup.title else "Untitled"
+
+            return markdown_text, extracted_links, page_title
+        except Exception as e:
+            print(f"Error processing {url}: {e}")
+            return None, [], ""
+
+    def generate_base_filename(self, url):
+        parsed = urlparse(url)
+        path = parsed.path.strip("/")
+        if not path:
+            filename = parsed.netloc.replace("www.", "")
+        else:
+            # Create readable filename from path
+            filename = re.sub(r"\W+", "_", path)
+
+        # Ensure it's not too long
+        if len(filename) > 50:
+            filename = filename[:50]
+
+        # Prepend part of domain if generic
+        if filename in ["docs", "api", "en", "v1"]:
+            domain_part = parsed.netloc.split(".")[0]
+            filename = f"{domain_part}_{filename}"
+
+        return filename
+
+    def determine_scope(self, start_url):
+        parsed = urlparse(start_url)
+
+        if self.scope_type == "domain":
+            # Broadest scope: Entire domain
+            return f"{parsed.scheme}://{parsed.netloc}/"
+
+        # "Smart Scoping" Logic (Section/Default)
+        path = parsed.path
+
+        # If url ends with overview, index, or is just a directory root
+        if (
+            path.endswith("/overview")
+            or path.endswith("/index")
+            or path.endswith("/index.html")
+        ):
+            # Go up one level
+            # e.g. /docs/en/overview -> /docs/en/
+            parent_path = path.rsplit("/", 1)[0]
+            scope = f"{parsed.scheme}://{parsed.netloc}{parent_path}/"
+            return scope
+        elif path.endswith("/"):
+            # Already a directory
+            return start_url
+        else:
+            # General page URL: trim last segment to get parent folder
+            # e.g. /docs/agents/tool-use -> /docs/agents/
+            parent_path = path.rsplit("/", 1)[0]
+            return f"{parsed.scheme}://{parsed.netloc}{parent_path}/"
+
+    def run_url_extraction(self, output_dir):
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        self.urls_extracted = 0
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True)
+            page = browser.new_page()
+
+            total_start_urls = len(self.start_urls)
+            for url_idx, start_url in enumerate(self.start_urls):
+                original_domain = urlparse(start_url).netloc
+                scope = self.determine_scope(start_url)
+                print(f"Collecting URLs from {start_url} with scope: {scope}")
+
+                # Per-URL visited set — isolates crawl per start URL
+                local_visited = {start_url}
+                queue = deque([start_url])
+                local_all_urls = [start_url]
+                url_count = 1
+                self.urls_extracted += 1
+                pages_scanned = 0
+
+                while queue and pages_scanned < self.max_pages:
+                    url = queue.popleft()
+                    self.status = f"Scanning: {url}"
+
+                    try:
+                        try:
+                            page.goto(url, timeout=45000)
+                        except Exception as e:
+                            print(f"Timeout on {url}: {e}")
+
+                        try:
+                            page.wait_for_load_state("networkidle", timeout=5000)
+                        except Exception:  # nosec B110
+                            pass
+
+                        time.sleep(1)
+
+                        html_content = page.content()
+                        soup = BeautifulSoup(html_content, "html.parser")
+
+                        for link in soup.find_all("a", href=True):
+                            href = link.get("href").strip()
+                            if href.startswith("#"):
+                                continue
+                            full_url = urljoin(url, href)
+                            full_url_no_frag = full_url.split("#")[0]
+                            parsed = urlparse(full_url_no_frag)
+                            clean_url = (
+                                parsed.scheme + "://" + parsed.netloc + parsed.path
+                            )
+                            if (
+                                parsed.netloc == original_domain
+                                and clean_url.startswith(scope)
+                            ):
+                                if (
+                                    clean_url not in local_visited
+                                    and clean_url not in self.visited_urls
+                                ):
+                                    local_visited.add(clean_url)
+                                    queue.append(clean_url)
+                                    local_all_urls.append(clean_url)
+                                    url_count += 1
+                                    self.urls_extracted += 1
+
+                        pages_scanned += 1
+                        self.pages_scraped += 1
+                        self.progress = int(
+                            (
+                                (url_idx + pages_scanned / self.max_pages)
+                                / total_start_urls
+                            )
+                            * 100
+                        )
+
+                    except Exception as e:
+                        print(f"Error scanning {url}: {e}")
+
+                # Merge this URL's visited pages into the global set
+                self.visited_urls.update(local_visited)
+                self.per_url_stats[start_url] = url_count
+
+                # Write collected URLs for THIS start_url to its own file
+                base_filename = self.generate_base_filename(start_url)
+                filename = f"{base_filename}_urls.txt"
+                filepath = os.path.join(output_dir, filename)
+                with open(filepath, "w", encoding="utf-8") as f:
+                    for u in local_all_urls:
+                        f.write(u + "\n")
+                print(f"Saved {filepath} ({url_count} URLs)")
+
+            browser.close()
+
+        self.status = "Completed"
+        self.progress = 100
+
+    def run(self, output_dir):
+        if self.extract_mode == "urls":
+            self.run_url_extraction(output_dir)
+            return
+
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        # Launch Playwright once
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True)
+            page = browser.new_page()
+
+            # Loop strictly through provided Start URLs
+            total_start_urls = len(self.start_urls)
+            for url_idx, start_url in enumerate(self.start_urls):
+                # Per-Start-URL isolation
+                original_domain = urlparse(start_url).netloc
+                scope = self.determine_scope(start_url)
+                print(f"Starting {start_url} with scope: {scope}")
+
+                # Per-URL visited set — isolates crawl per start URL
+                local_visited = {start_url}
+                queue = deque([start_url])
+
+                # Per-URL counters
+                pages_scraped_this_url = 0
+                current_file_idx = 1
+                current_buffer = ""
+                base_filename = self.generate_base_filename(start_url)
+
+                # BFS — max_pages applies per start URL
+                while queue and pages_scraped_this_url < self.max_pages:
+                    url = queue.popleft()
+                    self.status = f"Scraping: {url}"
+
+                    md_content, links, page_title = self.fetch_and_process(
+                        page, url, original_domain, scope
+                    )
+
+                    if md_content:
+                        header = f"\n\n---\n# Page: {page_title}\n# URL: {url}\n---\n\n"
+                        chunk = header + md_content
+
+                        if len(current_buffer) + len(chunk) > self.max_chars_per_file:
+                            self.save_buffer(
+                                output_dir,
+                                base_filename,
+                                current_file_idx,
+                                current_buffer,
+                            )
+                            current_file_idx += 1
+                            current_buffer = chunk
+                        else:
+                            current_buffer += chunk
+
+                        pages_scraped_this_url += 1
+                        self.pages_scraped += 1
+                        self.progress = int(
+                            (
+                                (url_idx + pages_scraped_this_url / self.max_pages)
+                                / total_start_urls
+                            )
+                            * 100
+                        )
+
+                        for link in links:
+                            if (
+                                link not in local_visited
+                                and link not in self.visited_urls
+                            ):
+                                local_visited.add(link)
+                                queue.append(link)
+
+                # Merge into global visited to prevent cross-URL duplication
+                self.visited_urls.update(local_visited)
+                self.per_url_stats[start_url] = pages_scraped_this_url
+
+                # Flush remaining buffer for this URL
+                if current_buffer:
+                    self.save_buffer(
+                        output_dir, base_filename, current_file_idx, current_buffer
+                    )
+
+            browser.close()
+
+        self.status = "Completed"
+        self.progress = 100
+
+    def save_buffer(self, output_dir, base_name, idx, content):
+        if not content.strip():
+            return
+
+        filename = f"{base_name}_part_{idx}.{self.output_format}"
+        filepath = os.path.join(output_dir, filename)
+
+        with open(filepath, "w", encoding="utf-8") as f:
+            f.write(content)
+        print(f"Saved {filepath} ({len(content)} chars)")