crawlit 0.1.0__tar.gz

crawlit-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 The crawlit Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crawlit-0.1.0/MANIFEST.in

@@ -0,0 +1,24 @@
+include README.md
+include requirements.txt
+include pyproject.toml
+include LICENSE
+
+# Include documentation
+include *.md
+
+# Include examples
+recursive-include examples *
+
+# Include core modules
+recursive-include crawlit/crawler *
+recursive-include crawlit/output *
+
+# Include tests
+recursive-include tests *
+
+# Include any data files
+# If you have a data directory, uncomment this line
+# recursive-include crawlit/data *
+
+# Global excludes for common temporary/generated files
+global-exclude *.egg-info/*

crawlit-0.1.0/PKG-INFO

@@ -0,0 +1,205 @@
+Metadata-Version: 2.4
+Name: crawlit
+Version: 0.1.0
+Summary: Modular, Ethical Python Web Crawler
+Author-email: Swayam Dani <swayamdani@swayamdani.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/SwayamDani/crawlit
+Project-URL: Bug Tracker, https://github.com/SwayamDani/crawlit/issues
+Project-URL: Documentation, https://github.com/SwayamDani/crawlit/tree/main/docs
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.25.0
+Requires-Dist: beautifulsoup4>=4.9.0
+Requires-Dist: lxml>=4.6.0
+Provides-Extra: cli
+Dynamic: license-file
+Dynamic: requires-python
+
+# 🕷️ crawlit - Modular, Ethical Python Web Crawler
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/)
+
+A powerful, modular, and ethical web crawler built in Python. Designed for security testing, link extraction, and website structure mapping with a focus on clean architecture and extensibility.
+
+## 🚀 Features
+
+- **Modular Architecture**: Easily extend with custom modules and parsers
+- **Ethical Crawling**: Configurable robots.txt compliance and rate limiting
+- **Depth Control**: Set maximum crawl depth to prevent excessive resource usage
+- **Domain Filtering**: Restrict crawling to specific domains or subdomains
+- **Robust Error Handling**: Gracefully manage connection issues and malformed pages
+- **Multiple Output Formats**: Export results as JSON, CSV, or plain text
+- **Detailed Logging**: Comprehensive logging of all crawler activities
+- **Command Line Interface**: Simple, powerful CLI for easy usage
+- **Programmatic API**: Use as a library in your own Python code
+
+## 📋 Requirements
+
+- Python 3.8+
+- Dependencies (will be listed in requirements.txt)
+
+## 🛠️ Installation
+
+### From PyPI (recommended)
+
+```bash
+# Install the core library
+pip install crawlit
+
+# Install with CLI tool support
+pip install crawlit[cli]
+```
+
+### From Source
+
+```bash
+# Clone the repository
+git clone https://github.com/SwayamDani/crawlit.git
+cd crawlit
+
+# Create a virtual environment (recommended)
+python -m venv venv
+source venv/bin/activate  # On Windows: venv\Scripts\activate
+
+# Install dependencies
+pip install -r requirements.txt
+
+# Install in development mode
+pip install -e .
+```
+
+## 📘 Usage
+
+### API Documentation
+
+Full API documentation is available in the `docs` directory. To build and view the documentation:
+
+```bash
+# Install Sphinx and required packages
+pip install sphinx sphinx_rtd_theme sphinxcontrib-napoleon
+
+# Build the documentation
+cd docs
+make html  # On Windows: make.bat html
+
+# View the documentation
+# Open docs/_build/html/index.html in your browser
+```
+
+### As a Library in Your Python Code
+
+```python
+from crawlit import Crawler, save_results, generate_summary_report
+
+# Initialize the crawler with custom parameters
+crawler = Crawler(
+    start_url="https://example.com",
+    max_depth=3,
+    internal_only=True,
+    user_agent="MyCustomBot/1.0",
+    delay=0.5,
+    respect_robots=True
+)
+
+# Start crawling
+crawler.crawl()
+
+# Get and process results
+results = crawler.get_results()
+print(f"Crawled {len(results)} URLs")
+
+# Save results in different formats
+save_results(results, "json", "crawl_results.json", pretty=True)
+```
+
+See the `examples/programmatic_usage.py` file for a complete example.
+
+### Command Line Interface
+
+If you installed with `pip install crawlit[cli]`, you can use the command-line interface:
+
+```bash
+# Basic usage
+crawlit --url https://example.com
+
+# Advanced options
+crawlit --url https://example.com \
+        --depth 3 \
+        --output-format json \
+        --output results.json \
+        --delay 0.5 \
+        --user-agent "crawlit/1.0" \
+        --ignore-robots
+```
+
+### Command Line Arguments
+
+| Argument | Description | Default |
+|----------|-------------|---------|
+| `--url`, `-u` | Target website URL | Required |
+| `--depth`, `-d` | Maximum crawl depth | 3 |
+| `--output-format`, `-f` | Output format (json, csv, txt, html) | json |
+| `--output`, `-O` | File to save results | crawl_results.json |
+| `--pretty-json`, `-p` | Enable pretty-print JSON with indentation | False |
+| `--ignore-robots`, `-i` | Ignore robots.txt rules | False |
+| `--delay` | Delay between requests (seconds) | 0.1 |
+| `--user-agent`, `-a` | Custom User-Agent string | crawlit/1.0 |
+| `--allow-external`, `-e` | Allow crawling URLs outside initial domain | False |
+| `--summary`, `-s` | Show a summary of crawl results | False |
+| `--verbose`, `-v` | Verbose output | False |
+| `--help`, `-h` | Show help message | - |
+
+## 🏗️ Project Structure
+
+```
+crawlit/
+├── crawlit.py           # CLI entry point
+├── requirements.txt     # Project dependencies
+├── crawler/             # Core crawler modules
+│   ├── __init__.py
+│   ├── engine.py        # Core crawler logic
+│   ├── fetcher.py       # HTTP request handling
+│   ├── parser.py        # HTML parsing and link extraction
+│   └── robots.py        # Robots.txt parser
+├── output/              # Output formatters
+│   ├── __init__.py
+│   └── formatters.py    # Output formatting functions
+├── examples/            # Example usage
+│   └── programmatic_usage.py  # Example of using as a library
+└── tests/               # Unit and integration tests
+    └── __init__.py
+```
+
+## 📅 Project Timeline
+
+- **May 2025**: Initial structure and CLI setup
+- **June 2025**: Core functionality complete (HTTP handling, parsing, domain control)
+- **June 30, 2025**: Project completion target with all core features
+
+## 🤝 Contributing
+
+Contributions will be welcome after the core functionality is complete. Please check back after June 30, 2025, for contribution guidelines.
+
+## 📜 License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+## 👤 Author
+
+Built and maintained by Swayam Dani
+
+---
+
+**Note**: This project is under active development with completion targeted for June 30, 2025.

crawlit-0.1.0/README.md

@@ -0,0 +1,177 @@
+# 🕷️ crawlit - Modular, Ethical Python Web Crawler
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/)
+
+A powerful, modular, and ethical web crawler built in Python. Designed for security testing, link extraction, and website structure mapping with a focus on clean architecture and extensibility.
+
+## 🚀 Features
+
+- **Modular Architecture**: Easily extend with custom modules and parsers
+- **Ethical Crawling**: Configurable robots.txt compliance and rate limiting
+- **Depth Control**: Set maximum crawl depth to prevent excessive resource usage
+- **Domain Filtering**: Restrict crawling to specific domains or subdomains
+- **Robust Error Handling**: Gracefully manage connection issues and malformed pages
+- **Multiple Output Formats**: Export results as JSON, CSV, or plain text
+- **Detailed Logging**: Comprehensive logging of all crawler activities
+- **Command Line Interface**: Simple, powerful CLI for easy usage
+- **Programmatic API**: Use as a library in your own Python code
+
+## 📋 Requirements
+
+- Python 3.8+
+- Dependencies (will be listed in requirements.txt)
+
+## 🛠️ Installation
+
+### From PyPI (recommended)
+
+```bash
+# Install the core library
+pip install crawlit
+
+# Install with CLI tool support
+pip install crawlit[cli]
+```
+
+### From Source
+
+```bash
+# Clone the repository
+git clone https://github.com/SwayamDani/crawlit.git
+cd crawlit
+
+# Create a virtual environment (recommended)
+python -m venv venv
+source venv/bin/activate  # On Windows: venv\Scripts\activate
+
+# Install dependencies
+pip install -r requirements.txt
+
+# Install in development mode
+pip install -e .
+```
+
+## 📘 Usage
+
+### API Documentation
+
+Full API documentation is available in the `docs` directory. To build and view the documentation:
+
+```bash
+# Install Sphinx and required packages
+pip install sphinx sphinx_rtd_theme sphinxcontrib-napoleon
+
+# Build the documentation
+cd docs
+make html  # On Windows: make.bat html
+
+# View the documentation
+# Open docs/_build/html/index.html in your browser
+```
+
+### As a Library in Your Python Code
+
+```python
+from crawlit import Crawler, save_results, generate_summary_report
+
+# Initialize the crawler with custom parameters
+crawler = Crawler(
+    start_url="https://example.com",
+    max_depth=3,
+    internal_only=True,
+    user_agent="MyCustomBot/1.0",
+    delay=0.5,
+    respect_robots=True
+)
+
+# Start crawling
+crawler.crawl()
+
+# Get and process results
+results = crawler.get_results()
+print(f"Crawled {len(results)} URLs")
+
+# Save results in different formats
+save_results(results, "json", "crawl_results.json", pretty=True)
+```
+
+See the `examples/programmatic_usage.py` file for a complete example.
+
+### Command Line Interface
+
+If you installed with `pip install crawlit[cli]`, you can use the command-line interface:
+
+```bash
+# Basic usage
+crawlit --url https://example.com
+
+# Advanced options
+crawlit --url https://example.com \
+        --depth 3 \
+        --output-format json \
+        --output results.json \
+        --delay 0.5 \
+        --user-agent "crawlit/1.0" \
+        --ignore-robots
+```
+
+### Command Line Arguments
+
+| Argument | Description | Default |
+|----------|-------------|---------|
+| `--url`, `-u` | Target website URL | Required |
+| `--depth`, `-d` | Maximum crawl depth | 3 |
+| `--output-format`, `-f` | Output format (json, csv, txt, html) | json |
+| `--output`, `-O` | File to save results | crawl_results.json |
+| `--pretty-json`, `-p` | Enable pretty-print JSON with indentation | False |
+| `--ignore-robots`, `-i` | Ignore robots.txt rules | False |
+| `--delay` | Delay between requests (seconds) | 0.1 |
+| `--user-agent`, `-a` | Custom User-Agent string | crawlit/1.0 |
+| `--allow-external`, `-e` | Allow crawling URLs outside initial domain | False |
+| `--summary`, `-s` | Show a summary of crawl results | False |
+| `--verbose`, `-v` | Verbose output | False |
+| `--help`, `-h` | Show help message | - |
+
+## 🏗️ Project Structure
+
+```
+crawlit/
+├── crawlit.py           # CLI entry point
+├── requirements.txt     # Project dependencies
+├── crawler/             # Core crawler modules
+│   ├── __init__.py
+│   ├── engine.py        # Core crawler logic
+│   ├── fetcher.py       # HTTP request handling
+│   ├── parser.py        # HTML parsing and link extraction
+│   └── robots.py        # Robots.txt parser
+├── output/              # Output formatters
+│   ├── __init__.py
+│   └── formatters.py    # Output formatting functions
+├── examples/            # Example usage
+│   └── programmatic_usage.py  # Example of using as a library
+└── tests/               # Unit and integration tests
+    └── __init__.py
+```
+
+## 📅 Project Timeline
+
+- **May 2025**: Initial structure and CLI setup
+- **June 2025**: Core functionality complete (HTTP handling, parsing, domain control)
+- **June 30, 2025**: Project completion target with all core features
+
+## 🤝 Contributing
+
+Contributions will be welcome after the core functionality is complete. Please check back after June 30, 2025, for contribution guidelines.
+
+## 📜 License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+## 👤 Author
+
+Built and maintained by Swayam Dani
+
+---
+
+**Note**: This project is under active development with completion targeted for June 30, 2025.

crawlit-0.1.0/RELEASE_NOTES.md

@@ -0,0 +1,50 @@
+# Crawlit 0.1.0 - Release Notes
+
+We are pleased to announce the first public release of Crawlit - a modular, ethical Python web crawler.
+
+## Features
+
+- **Modular Architecture**: Easily extend with custom modules and parsers
+- **Ethical Crawling**: Configurable robots.txt compliance and rate limiting
+- **Depth Control**: Set maximum crawl depth to prevent excessive resource usage
+- **Domain Filtering**: Restrict crawling to specific domains or subdomains
+- **Robust Error Handling**: Gracefully manage connection issues and malformed pages
+- **Multiple Output Formats**: Export results as JSON, CSV, or plain text
+- **Detailed Logging**: Comprehensive logging of all crawler activities
+- **Command Line Interface**: Simple, powerful CLI for easy usage
+- **Programmatic API**: Use as a library in your own Python code
+
+## Installation
+
+```bash
+# Install the core library
+pip install crawlit
+
+# Install with CLI tool support
+pip install crawlit[cli]
+```
+
+## Documentation
+
+Comprehensive API documentation is now available in the `docs` directory. To build and view the documentation:
+
+```bash
+# Install Sphinx and required packages
+pip install sphinx sphinx_rtd_theme sphinxcontrib-napoleon
+
+# Build the documentation
+cd docs
+make html  # On Windows: make.bat html
+
+# View the documentation
+# Open docs/_build/html/index.html in your browser
+```
+
+## Known Issues
+
+- Limited support for JavaScript-rendered content
+- No advanced request throttling based on domain
+
+## Acknowledgments
+
+Thanks to all the early testers and contributors who helped make this release possible.

crawlit-0.1.0/crawlit/__init__.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+"""
+crawlit - Modular, Ethical Python Web Crawler
+
+A flexible web crawler library that can be used programmatically or via CLI.
+"""
+
+__version__ = '0.1.0'
+
+# Export core functionality
+from crawlit.crawler.engine import Crawler
+from crawlit.output.formatters import save_results, generate_summary_report
+
+# CLI functionality (but not executed on import)
+def cli_main():
+    """Entry point for the CLI interface when installed with [cli] option"""
+    from crawlit.crawlit import main
+    return main()
+
+__all__ = [
+    'Crawler',           # Main crawler engine
+    'save_results',      # Output formatters 
+    'generate_summary_report',
+    'cli_main'           # CLI entry point
+]

crawlit-0.1.0/crawlit/crawler/__init__.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+"""
+Crawler package containing the core modules for the crawlit web crawler
+"""
+
+from .engine import Crawler
+from .fetcher import fetch_page
+from .parser import extract_links, _process_url
+
+__all__ = [
+    'Crawler',
+    'fetch_page',
+    'extract_links',
+    '_process_url'
+]