instapaper-scraper 1.0.0__py3-none-any.whl

instapaper_scraper/cli.py

@@ -0,0 +1,202 @@
+import sys
+import logging
+import argparse
+import requests
+from pathlib import Path
+from typing import Union
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
+from . import __version__
+from .auth import InstapaperAuthenticator
+from .api import InstapaperClient
+from .output import save_articles
+from .exceptions import ScraperStructureChanged
+
+
+def _resolve_path(
+    arg_path: str, working_dir_filename: str, user_dir_filename: Path
+) -> Path:
+    """Resolves a path based on CLI arg, working dir, and user config dir."""
+    if arg_path:
+        return Path(arg_path).expanduser()
+
+    working_dir_path = Path(working_dir_filename)
+    if working_dir_path.exists():
+        logging.info(f"Found {working_dir_filename} in working directory.")
+        return working_dir_path
+
+    return user_dir_filename
+
+
+def load_config(config_path_str: Union[str, None] = None) -> Union[dict, None]:
+    """
+    Loads configuration from a TOML file.
+    It checks the provided path, then config.toml in the project root,
+    and finally ~/.config/instapaper-scraper/config.toml.
+    """
+    app_name = "instapaper-scraper"
+    default_paths = [
+        Path("config.toml"),
+        Path.home() / ".config" / app_name / "config.toml",
+    ]
+
+    paths_to_check = []
+    if config_path_str:
+        paths_to_check.insert(0, Path(config_path_str).expanduser())
+    paths_to_check.extend(default_paths)
+
+    for path in paths_to_check:
+        if path.is_file():
+            try:
+                with open(path, "rb") as f:
+                    logging.info(f"Loading configuration from {path}")
+                    return tomllib.load(f)
+            except tomllib.TOMLDecodeError as e:
+                logging.error(f"Error decoding TOML file at {path}: {e}")
+                return None
+    logging.info("No configuration file found at any default location.")
+    return None
+
+
+def main():
+    """
+    Main entry point for the Instapaper scraper CLI.
+    """
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+
+    parser = argparse.ArgumentParser(description="Scrape Instapaper articles.")
+    parser.add_argument(
+        "-v",
+        "--version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+        help="Show program's version number and exit.",
+    )
+    parser.add_argument(
+        "--config-path",
+        help="Path to the configuration file.",
+    )
+    parser.add_argument(
+        "--format",
+        choices=["csv", "json", "sqlite"],
+        default="csv",
+        help="Output format (default: csv)",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        help="Output filename. If not provided, defaults to output/bookmarks.{format}",
+    )
+    parser.add_argument("--session-file", help="Path to the encrypted session file.")
+    parser.add_argument("--key-file", help="Path to the session key file.")
+    parser.add_argument("--username", help="Instapaper username.")
+    parser.add_argument("--password", help="Instapaper password.")
+    parser.add_argument(
+        "--limit",
+        type=int,
+        default=None,
+        help="Maximum number of pages to scrape (default: unlimited)",
+    )
+    parser.add_argument(
+        "--folder",
+        help="Folder key, ID, or slug to scrape. Use 'none' to disable folder mode.",
+    )
+    args = parser.parse_args()
+
+    config = load_config(args.config_path)
+    folders = config.get("folders", []) if config else []
+    selected_folder = None
+
+    if args.folder:
+        if args.folder.lower() == "none":
+            selected_folder = None
+        else:
+            if not config:
+                logging.error(
+                    "Configuration file not found or failed to load. The --folder option requires a configuration file."
+                )
+                sys.exit(1)
+            else:
+                for f in folders:
+                    if args.folder in (f.get("key"), str(f.get("id")), f.get("slug")):
+                        selected_folder = f
+                        break
+                if not selected_folder:
+                    # If folder is not in config, treat it as a folder ID
+                    selected_folder = {"id": args.folder}
+    elif folders:
+        print("Available folders:")
+        print("  0: none (non-folder mode)")
+        for i, folder in enumerate(folders):
+            display_name = folder.get("key") or folder.get("slug") or folder.get("id")
+            print(f"  {i+1}: {display_name}")
+
+        try:
+            choice = int(input("Select a folder (enter a number): "))
+            if 0 < choice <= len(folders):
+                selected_folder = folders[choice - 1]
+            elif choice != 0:
+                print("Invalid selection. Continuing in non-folder mode.")
+        except (ValueError, IndexError):
+            print("Invalid input. Continuing in non-folder mode.")
+
+    # Determine output filename
+    output_filename = args.output
+    if not output_filename:
+        if selected_folder and selected_folder.get("output_filename"):
+            output_filename = selected_folder["output_filename"]
+        elif not selected_folder and config and config.get("output_filename"):
+            output_filename = config["output_filename"]
+        else:
+            ext = "db" if args.format == "sqlite" else args.format
+            output_filename = f"output/bookmarks.{ext}"
+
+    session = requests.Session()
+
+    # Resolve session and key file paths
+    app_name = "instapaper-scraper"
+    user_config_dir = Path.home() / ".config" / app_name
+
+    session_file = _resolve_path(
+        args.session_file, ".instapaper_session", user_config_dir / ".instapaper_session"
+    )
+    key_file = _resolve_path(args.key_file, ".session_key", user_config_dir / ".session_key")
+
+    # 1. Authenticate
+    authenticator = InstapaperAuthenticator(
+        session,
+        session_file=session_file,
+        key_file=key_file,
+        username=args.username,
+        password=args.password,
+    )
+    if not authenticator.login():
+        sys.exit(1)  # Exit if login fails
+
+    # 2. Scrape Articles
+    client = InstapaperClient(session)
+    try:
+        folder_info = selected_folder if selected_folder else None
+        all_articles = client.get_all_articles(
+            limit=args.limit, folder_info=folder_info
+        )
+    except ScraperStructureChanged as e:
+        logging.error(f"Stopping scraper due to an unrecoverable error: {e}")
+        sys.exit(1)
+    except requests.exceptions.RequestException as e:
+        logging.error(f"An HTTP error occurred: {e}")
+        sys.exit(1)
+    except Exception as e:
+        logging.error(f"An unexpected error occurred during scraping: {e}")
+        sys.exit(1)
+
+    # 3. Save Articles
+    save_articles(all_articles, args.format, output_filename)
+
+
+if __name__ == "__main__":
+    main()

instapaper_scraper/exceptions.py

@@ -0,0 +1,4 @@
+class ScraperStructureChanged(Exception):
+    """Custom exception for when the scraper detects an HTML structure change."""
+
+    pass

instapaper_scraper/output.py

@@ -0,0 +1,88 @@
+import os
+import json
+import sqlite3
+import logging
+from typing import List, Dict, Any
+
+# Constants for file operations
+JSON_INDENT = 4
+
+# Constants for CSV output
+CSV_HEADER = "id,title,url\n"
+CSV_DELIMITER = ","
+CSV_ROW_FORMAT = "{id},{title},{url}\n"
+
+# Constants for SQLite output
+SQLITE_TABLE_NAME = "articles"
+SQLITE_ID_COL = "id"
+SQLITE_TITLE_COL = "title"
+SQLITE_URL_COL = "url"
+SQLITE_CREATE_TABLE_SQL = f"""
+        CREATE TABLE IF NOT EXISTS {SQLITE_TABLE_NAME} (
+            {SQLITE_ID_COL} TEXT PRIMARY KEY,
+            {SQLITE_TITLE_COL} TEXT NOT NULL,
+            {SQLITE_URL_COL} TEXT NOT NULL
+        )
+    """
+SQLITE_INSERT_SQL = f"INSERT OR REPLACE INTO {SQLITE_TABLE_NAME} ({SQLITE_ID_COL}, {SQLITE_TITLE_COL}, {SQLITE_URL_COL}) VALUES (:{SQLITE_ID_COL}, :{SQLITE_TITLE_COL}, :{SQLITE_URL_COL})"
+
+# Constants for logging messages
+LOG_NO_ARTICLES = "No articles found to save."
+LOG_SAVED_ARTICLES = "Saved {count} articles to {filename}"
+LOG_UNKNOWN_FORMAT = "Unknown output format: {format}"
+
+
+def save_to_csv(data: List[Dict[str, Any]], filename: str):
+    """Saves a list of articles to a CSV file."""
+    os.makedirs(os.path.dirname(filename), exist_ok=True)
+    with open(filename, "w", newline="", encoding="utf-8") as f:
+        f.write(CSV_HEADER)
+        for article in data:
+            # Basic CSV quoting for titles with commas
+            title = article[SQLITE_TITLE_COL]
+            if CSV_DELIMITER in title:
+                title = f'"{title}"'
+            f.write(
+                CSV_ROW_FORMAT.format(
+                    id=article[SQLITE_ID_COL], title=title, url=article[SQLITE_URL_COL]
+                )
+            )
+    logging.info(LOG_SAVED_ARTICLES.format(count=len(data), filename=filename))
+
+
+def save_to_json(data: List[Dict[str, Any]], filename: str):
+    """Saves a list of articles to a JSON file."""
+    os.makedirs(os.path.dirname(filename), exist_ok=True)
+    with open(filename, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=JSON_INDENT, ensure_ascii=False)
+    logging.info(LOG_SAVED_ARTICLES.format(count=len(data), filename=filename))
+
+
+def save_to_sqlite(data: List[Dict[str, Any]], db_name: str):
+    """Saves a list of articles to a SQLite database."""
+    os.makedirs(os.path.dirname(db_name), exist_ok=True)
+    conn = sqlite3.connect(db_name)
+    cursor = conn.cursor()
+    cursor.execute(SQLITE_CREATE_TABLE_SQL)
+    cursor.executemany(SQLITE_INSERT_SQL, data)
+    conn.commit()
+    conn.close()
+    logging.info(LOG_SAVED_ARTICLES.format(count=len(data), filename=db_name))
+
+
+def save_articles(data: List[Dict[str, Any]], format: str, filename: str):
+    """
+    Dispatches to the correct save function based on the format.
+    """
+    if not data:
+        logging.info(LOG_NO_ARTICLES)
+        return
+
+    if format == "csv":
+        save_to_csv(data, filename=filename)
+    elif format == "json":
+        save_to_json(data, filename=filename)
+    elif format == "sqlite":
+        save_to_sqlite(data, db_name=filename)
+    else:
+        logging.error(LOG_UNKNOWN_FORMAT.format(format=format))

instapaper_scraper-1.0.0.dist-info/METADATA

@@ -0,0 +1,280 @@
+Metadata-Version: 2.4
+Name: instapaper-scraper
+Version: 1.0.0
+Summary: A tool to scrape articles from Instapaper.
+Project-URL: Homepage, https://github.com/chriskyfung/InstapaperScraper
+Project-URL: Source, https://github.com/chriskyfung/InstapaperScraper
+Project-URL: Issues, https://github.com/chriskyfung/InstapaperScraper/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: End Users/Desktop
+Classifier: Topic :: Internet :: WWW/HTTP :: Browsers
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: beautifulsoup4~=4.14.2
+Requires-Dist: certifi~=2025.11.12
+Requires-Dist: charset-normalizer~=3.4.3
+Requires-Dist: cryptography~=46.0.3
+Requires-Dist: guara~=0.0.14
+Requires-Dist: idna~=3.11
+Requires-Dist: python-dotenv~=1.2.1
+Requires-Dist: requests~=2.32.5
+Requires-Dist: soupsieve~=2.8
+Requires-Dist: typing_extensions~=4.15.0
+Requires-Dist: urllib3~=2.5.0
+Requires-Dist: tomli~=2.0.1; python_version < "3.11"
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: types-requests; extra == "dev"
+Requires-Dist: types-beautifulsoup4; extra == "dev"
+Requires-Dist: requests-mock; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Dynamic: license-file
+
+# Instapaper Scraper
+
+![Python Version from PEP 621 TOML](https://img.shields.io/python/required-version-toml?tomlFilePath=https%3A%2F%2Fraw.githubusercontent.com%2Fchriskyfung%2FInstapaperScraper%2Frefs%2Fheads%2Fmaster%2Fpyproject.toml)
+[![CI](https://github.com/chriskyfung/InstapaperScraper/actions/workflows/ci.yml/badge.svg)](https://github.com/chriskyfung/InstapaperScraper/actions/workflows/ci.yml)
+[![PyPI version](https://img.shields.io/pypi/v/instapaper-scraper.svg)](https://pypi.org/project/instapaper-scraper/)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![GitHub License](https://img.shields.io/github/license/chriskyfung/InstapaperScraper)
+](https://www.gnu.org/licenses/gpl-3.0.en.html)
+[![codecov](https://codecov.io/gh/chriskyfung/InstapaperScraper/branch/main/graph/badge.svg?token=1OK83DVUA5)](https://codecov.io/gh/chriskyfung/InstapaperScraper)
+
+A Python tool to scrape all your saved Instapaper bookmarks and export them to various formats.
+
+## Features
+
+- Scrapes all bookmarks from your Instapaper account.
+- Supports scraping from specific folders.
+- Exports data to CSV, JSON, or a SQLite database.
+- Securely stores your session for future runs.
+- Modern, modular, and tested architecture.
+
+## Getting Started
+
+### 1. Requirements
+- Python 3.9+
+
+### 2. Installation
+
+This package is available on PyPI and can be installed with pip:
+
+```sh
+pip install instapaper-scraper
+```
+
+### 3. Usage
+
+Run the tool from the command line, specifying your desired output format:
+
+```sh
+# Scrape and export to the default CSV format
+instapaper-scraper
+
+# Scrape and export to JSON
+instapaper-scraper --format json
+
+# Scrape and export to a SQLite database with a custom name
+instapaper-scraper --format sqlite --output my_articles.db
+```
+
+## Configuration
+
+### Authentication
+
+The script authenticates using one of the following methods, in order of priority:
+
+1.  **Command-line Arguments**: Provide your username and password directly when running the script:
+
+    ```sh
+    instapaper-scraper --username your_username --password your_password
+    ```
+
+2.  **Session Files (`.session_key`, `.instapaper_session`)**: The script attempts to load these files in the following order:
+    a.  Path specified by `--session-file` or `--key-file` arguments.
+    b.  Files in the current working directory (e.g., `./.session_key`).
+    c.  Files in the user's configuration directory (`~/.config/instapaper-scraper/`).
+    After the first successful login, the script creates an encrypted `.instapaper_session` file and a `.session_key` file to reuse your session securely.
+
+3.  **Interactive Prompt**: If no other method is available, the script will prompt you for your username and password.
+
+> **Note on Security:** Your session file (`.instapaper_session`) and the encryption key (`.session_key`) are stored with secure permissions (read/write for the owner only) to protect your credentials.
+
+### Folder Configuration
+
+You can define and quickly access your Instapaper folders using a `config.toml` file. The scraper will look for this file in the following locations (in order of precedence):
+
+1.  The path specified by the `--config-path` argument.
+2.  `config.toml` in the current working directory.
+3.  `~/.config/instapaper-scraper/config.toml`
+
+Here is an example of `config.toml`:
+
+```toml
+# Default output filename for non-folder mode
+output_filename = "home-articles.csv"
+
+[[folders]]
+key = "ml"
+id = "1234567"
+slug = "machine-learning"
+output_filename = "ml-articles.json"
+
+[[folders]]
+key = "python"
+id = "7654321"
+slug = "python-programming"
+output_filename = "python-articles.db"
+```
+
+- **output_filename (top-level)**: The default output filename to use when not in folder mode.
+- **key**: A short alias for the folder.
+- **id**: The folder ID from the Instapaper URL.
+- **slug**: The human-readable part of the folder URL.
+- **output_filename (folder-specific)**: A preset output filename for scraped articles from this specific folder.
+
+When a `config.toml` file is present and no `--folder` argument is provided, the scraper will prompt you to select a folder. You can also specify a folder directly using the `--folder` argument with its key, ID, or slug. Use `--folder=none` to explicitly disable folder mode and scrape all articles.
+
+### Command-line Arguments
+
+| Argument              | Description                                                              |
+| --------------------- | ------------------------------------------------------------------------ |
+| `--config-path <path>`| Path to the configuration file. Searches `~/.config/instapaper-scraper/config.toml` and `config.toml` in the current directory by default. |
+| `--folder <value>`    | Specify a folder by key, ID, or slug from your `config.toml`. **Requires a configuration file to be loaded.** Use `none` to explicitly disable folder mode. If a configuration file is not found or fails to load, and this option is used (not set to `none`), the program will exit. |
+| `--format <format>`   | Output format (`csv`, `json`, `sqlite`). Default: `csv`.                 |
+| `--output <filename>` | Specify a custom output filename.                                        |
+| `--username <user>`   | Your Instapaper account username.                                        |
+| `--password <pass>`   | Your Instapaper account password.                                        |
+
+### Output Formats
+
+You can control the output format using the `--format` argument. The supported formats are:
+
+- `csv` (default): Exports data to `output/bookmarks.csv`.
+- `json`: Exports data to `output/bookmarks.json`.
+- `sqlite`: Exports data to an `articles` table in `output/bookmarks.db`.
+- `--output <filename>`: Specify a custom output filename.
+
+If the `--format` flag is omitted, the script will default to `csv`.
+
+#### Opening Articles in Instapaper
+
+The output data includes a unique `id` for each article. To open an article directly in Instapaper's reader view, append this ID to the base URL:
+`https://www.instapaper.com/read/<article_id>`
+
+## How It Works
+
+The tool is designed with a modular architecture for reliability and maintainability.
+
+1. **Authentication**: The `InstapaperAuthenticator` handles secure login and session management.
+2. **Scraping**: The `InstapaperClient` iterates through all pages of your bookmarks, fetching the metadata for each article with robust error handling and retries.
+3. **Data Collection**: All fetched articles are aggregated into a single list.
+4. **Export**: Finally, the collected data is written to a file in your chosen format (`.csv`, `.json`, or `.db`).
+
+## Example Output
+
+### CSV (`output/bookmarks.csv`)
+
+```csv
+id,title,url
+999901234,"Article 1",https://www.example.com/page-1/
+999002345,"Article 2",https://www.example.com/page-2/
+```
+
+### JSON (`output/bookmarks.json`)
+
+```json
+[
+    {
+        "id": "999901234",
+        "title": "Article 1",
+        "url": "https://www.example.com/page-1/"
+    },
+    {
+        "id": "999002345",
+        "title": "Article 2",
+        "url": "https://www.example.com/page-2/"
+    }
+]
+```
+
+### SQLite (`output/bookmarks.db`)
+
+A SQLite database file is created with an `articles` table containing `id`, `title`, and `url` columns.
+
+## Development & Testing
+
+This project uses `pytest` for testing, `black` for code formatting, and `ruff` for linting.
+
+### Setup
+
+To install the development dependencies:
+
+```sh
+pip install -e .[dev]
+```
+
+### Running the Scraper
+
+To run the scraper directly without installing the package:
+
+```sh
+python -m src.instapaper_scraper.cli
+```
+
+### Testing
+
+To run the tests, execute the following command from the project root:
+
+```sh
+pytest
+```
+
+To check test coverage:
+
+```sh
+pytest --cov=src/instapaper_scraper --cov-report=term-missing
+```
+
+### Code Quality
+
+To format the code with `black`:
+
+```sh
+black .
+```
+
+To check for linting errors with `ruff`:
+
+```sh
+ruff check .
+```
+
+To automatically fix linting errors:
+
+```sh
+ruff check . --fix
+```
+
+## Disclaimer
+
+This script requires valid Instapaper credentials. Use it responsibly and in accordance with Instapaper’s Terms of Service.
+
+## License
+
+This project is licensed under the terms of the GNU General Public License v3.0. See the [LICENSE](LICENSE) file for the full license text.

instapaper_scraper-1.0.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+instapaper_scraper/__init__.py,sha256=qdcT3tp4KLufWH1u6tOuPVUQaXwakQD0gdjkwY4ljfg,206
+instapaper_scraper/api.py,sha256=KvGxK2P35-3TsONPWcQTVBZT-q70p7hobeQ7E9PhXwA,11740
+instapaper_scraper/auth.py,sha256=DepQKDdVSm1dMFNIkpK_LIlaI0JllAYZb3_LJWhMe-g,7554
+instapaper_scraper/cli.py,sha256=Pxf1cAoLW9N-X1BP73HE0i2Qv7rPTaIyrPqG3cgdSTI,6860
+instapaper_scraper/exceptions.py,sha256=CptHoZe4NOhdjOoyXkZEMFgQC6oKtzjRljywwDEtsTg,134
+instapaper_scraper/output.py,sha256=0vQQ4AHZwFJg3O5O2zzvKUf0cOS1fTjXdivFqEHAun0,3081
+instapaper_scraper-1.0.0.dist-info/licenses/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
+instapaper_scraper-1.0.0.dist-info/METADATA,sha256=cI3CAWZOY-1R5h6NfNEJHUOlhJQ6D5vS1bQCQ_cu3OI,10376
+instapaper_scraper-1.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+instapaper_scraper-1.0.0.dist-info/entry_points.txt,sha256=7AvRgN5fvtas_Duxdz-JPbDN6A1Lq2GaTfTSv54afxA,67
+instapaper_scraper-1.0.0.dist-info/top_level.txt,sha256=kiU9nLkqPOVPLsP4QMHuBFjAmoIKfftYmGV05daLrcc,19
+instapaper_scraper-1.0.0.dist-info/RECORD,,

instapaper_scraper-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

instapaper_scraper-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+instapaper-scraper = instapaper_scraper.cli:main