phoenix-engine 0.1.0__tar.gz

phoenix_engine-0.1.0/PKG-INFO

@@ -0,0 +1,187 @@
+Metadata-Version: 2.4
+Name: phoenix-engine
+Version: 0.1.0
+Summary: Universal pure web scraping engine.
+Author: Phoenix Engine Team
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/phnx-tech/phoenix-engine
+Project-URL: Documentation, https://github.com/phnx-tech/phoenix-engine#readme
+Project-URL: Repository, https://github.com/phnx-tech/phoenix-engine.git
+Project-URL: Issues, https://github.com/phnx-tech/phoenix-engine/issues
+Keywords: scraping,web,social-media,html,automation
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: playwright>=1.40.0
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: lxml>=5.0.0
+Requires-Dist: cssselect>=1.2.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pydantic-settings>=2.0.0
+Requires-Dist: typer>=0.12.0
+Requires-Dist: sqlalchemy>=2.0.0
+Requires-Dist: alembic>=1.13.0
+Requires-Dist: keyring>=24.0.0
+Requires-Dist: cryptography>=42.0.0
+Requires-Dist: structlog>=24.0.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: duckduckgo-search>=8.0.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: dev
+Requires-Dist: build>=1.0.0; extra == "dev"
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0.0; extra == "dev"
+Requires-Dist: respx>=0.21.0; extra == "dev"
+Requires-Dist: pytest-playwright>=0.5.0; extra == "dev"
+Requires-Dist: black>=24.0.0; extra == "dev"
+Requires-Dist: ruff>=0.4.0; extra == "dev"
+Requires-Dist: mypy>=1.10.0; extra == "dev"
+Requires-Dist: pre-commit>=3.7.0; extra == "dev"
+Requires-Dist: faker>=25.0.0; extra == "dev"
+Requires-Dist: factory-boy>=3.3.0; extra == "dev"
+Requires-Dist: mkdocs>=1.6.0; extra == "dev"
+Requires-Dist: mkdocs-material>=9.5.0; extra == "dev"
+
+# Phoenix Engine
+
+A universal pure-web scraping engine that turns public web pages into structured,
+predictable data. No official APIs required — Phoenix Engine uses raw HTTP
+requests and headless browser automation to extract posts, profiles, listings,
+and articles from social platforms and websites.
+
+> **Current status:** beta / early access. A license key is required to run the
+> engine in production.
+
+## What it does
+
+- Scrapes public pages using **HTTP** or **headless browser** strategies.
+- Returns a **unified JSON output** no matter what platform you target.
+- Automatically adapts to site changes, anti-bot measures, and selector drift.
+- Learns from past scrapes to pick the best strategy for each domain.
+- Can be used from the command line or inside your Python application.
+
+## Install
+
+### From PyPI
+
+```bash
+pip install phoenix-engine
+```
+
+### From a GitHub Release wheel
+
+Download the `.whl` from the latest release, then:
+
+```bash
+pip install phoenix_engine-0.1.0-py3-none-any.whl
+```
+
+## Activate your license
+
+Phoenix Engine is distributed under license keys during beta. After installing,
+set your key:
+
+```bash
+export PHOENIX_LICENSE_ENFORCEMENT_ENABLED=true
+export PHOENIX_LICENSE_SECRET="your-signing-secret"
+export PHOENIX_LICENSE_KEY="phx.eyJ..."
+```
+
+Or create a `phoenix.yaml` file:
+
+```yaml
+license_enforcement_enabled: true
+license_secret: "your-signing-secret"
+license_key: "phx.eyJ..."
+```
+
+If the key is missing, expired, tampered with, or over its use limit, the
+engine will refuse to start.
+
+## Quick start — CLI
+
+```bash
+# Scrape a single public page
+phoenix scrape "https://example.com/post/123"
+
+# Scrape without archiving the raw source
+phoenix scrape "https://example.com/post/123" --no-archive
+
+# Scrape multiple URLs in parallel
+phoenix scrape-batch \
+  "https://example.com/post/123" \
+  "https://example.com/post/456" \
+  --output results.json
+
+# List built-in platform adapters
+phoenix plugins list
+
+# Inspect effective configuration (secrets are masked)
+phoenix config show
+```
+
+## Quick start — Python library
+
+```python
+import asyncio
+from phoenix import PhoenixEngine
+
+async def main() -> None:
+    async with PhoenixEngine() as engine:
+        result = await engine.scrape("https://example.com/post/123")
+        print(result.output.model_dump_json(indent=2))
+
+asyncio.run(main())
+```
+
+## Configuration
+
+Most settings can be controlled with environment variables or a config file
+(`phoenix.yaml`, `phoenix.json`, `phoenix.toml`):
+
+```yaml
+timeout: 30
+stealth_enabled: true
+ai_enabled: false
+rate_limits:
+  example.com: 1.0
+```
+
+Run `phoenix config show` to see the active configuration.
+
+## Supported platforms
+
+Phoenix Engine ships with adapters for common public platforms and a generic
+fallback for any HTML page:
+
+- Instagram, Facebook, X/Twitter, LinkedIn, TikTok, YouTube
+- Generic blogs, listings, and article pages
+
+Adapters are plugin-based, so new platforms can be added without touching the
+core engine.
+
+## Ethical use
+
+Phoenix Engine only scrapes **publicly available** content. Always respect:
+
+- The target site's `robots.txt` and Terms of Service.
+- Local laws and data-protection regulations (GDPR, CCPA, etc.).
+- Rate limits — the engine includes built-in throttling to avoid overload.
+
+## Support
+
+- Issues: https://github.com/phnx-tech/phoenix-engine/issues
+- Repository: https://github.com/phnx-tech/phoenix-engine
+
+## License
+
+Commercial beta license. See your license agreement for terms.

phoenix_engine-0.1.0/README.md

@@ -0,0 +1,134 @@
+# Phoenix Engine
+
+A universal pure-web scraping engine that turns public web pages into structured,
+predictable data. No official APIs required — Phoenix Engine uses raw HTTP
+requests and headless browser automation to extract posts, profiles, listings,
+and articles from social platforms and websites.
+
+> **Current status:** beta / early access. A license key is required to run the
+> engine in production.
+
+## What it does
+
+- Scrapes public pages using **HTTP** or **headless browser** strategies.
+- Returns a **unified JSON output** no matter what platform you target.
+- Automatically adapts to site changes, anti-bot measures, and selector drift.
+- Learns from past scrapes to pick the best strategy for each domain.
+- Can be used from the command line or inside your Python application.
+
+## Install
+
+### From PyPI
+
+```bash
+pip install phoenix-engine
+```
+
+### From a GitHub Release wheel
+
+Download the `.whl` from the latest release, then:
+
+```bash
+pip install phoenix_engine-0.1.0-py3-none-any.whl
+```
+
+## Activate your license
+
+Phoenix Engine is distributed under license keys during beta. After installing,
+set your key:
+
+```bash
+export PHOENIX_LICENSE_ENFORCEMENT_ENABLED=true
+export PHOENIX_LICENSE_SECRET="your-signing-secret"
+export PHOENIX_LICENSE_KEY="phx.eyJ..."
+```
+
+Or create a `phoenix.yaml` file:
+
+```yaml
+license_enforcement_enabled: true
+license_secret: "your-signing-secret"
+license_key: "phx.eyJ..."
+```
+
+If the key is missing, expired, tampered with, or over its use limit, the
+engine will refuse to start.
+
+## Quick start — CLI
+
+```bash
+# Scrape a single public page
+phoenix scrape "https://example.com/post/123"
+
+# Scrape without archiving the raw source
+phoenix scrape "https://example.com/post/123" --no-archive
+
+# Scrape multiple URLs in parallel
+phoenix scrape-batch \
+  "https://example.com/post/123" \
+  "https://example.com/post/456" \
+  --output results.json
+
+# List built-in platform adapters
+phoenix plugins list
+
+# Inspect effective configuration (secrets are masked)
+phoenix config show
+```
+
+## Quick start — Python library
+
+```python
+import asyncio
+from phoenix import PhoenixEngine
+
+async def main() -> None:
+    async with PhoenixEngine() as engine:
+        result = await engine.scrape("https://example.com/post/123")
+        print(result.output.model_dump_json(indent=2))
+
+asyncio.run(main())
+```
+
+## Configuration
+
+Most settings can be controlled with environment variables or a config file
+(`phoenix.yaml`, `phoenix.json`, `phoenix.toml`):
+
+```yaml
+timeout: 30
+stealth_enabled: true
+ai_enabled: false
+rate_limits:
+  example.com: 1.0
+```
+
+Run `phoenix config show` to see the active configuration.
+
+## Supported platforms
+
+Phoenix Engine ships with adapters for common public platforms and a generic
+fallback for any HTML page:
+
+- Instagram, Facebook, X/Twitter, LinkedIn, TikTok, YouTube
+- Generic blogs, listings, and article pages
+
+Adapters are plugin-based, so new platforms can be added without touching the
+core engine.
+
+## Ethical use
+
+Phoenix Engine only scrapes **publicly available** content. Always respect:
+
+- The target site's `robots.txt` and Terms of Service.
+- Local laws and data-protection regulations (GDPR, CCPA, etc.).
+- Rate limits — the engine includes built-in throttling to avoid overload.
+
+## Support
+
+- Issues: https://github.com/phnx-tech/phoenix-engine/issues
+- Repository: https://github.com/phnx-tech/phoenix-engine
+
+## License
+
+Commercial beta license. See your license agreement for terms.

phoenix_engine-0.1.0/pyproject.toml

@@ -0,0 +1,184 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "phoenix-engine"
+version = "0.1.0"
+description = "Universal pure web scraping engine."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+authors = [
+    { name = "Phoenix Engine Team" },
+]
+keywords = ["scraping", "web", "social-media", "html", "automation"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "httpx>=0.27.0",
+    "playwright>=1.40.0",
+    "beautifulsoup4>=4.12.0",
+    "lxml>=5.0.0",
+    "cssselect>=1.2.0",
+    "pydantic>=2.0.0",
+    "pydantic-settings>=2.0.0",
+    "typer>=0.12.0",
+    "sqlalchemy>=2.0.0",
+    "alembic>=1.13.0",
+    "keyring>=24.0.0",
+    "cryptography>=42.0.0",
+    "structlog>=24.0.0",
+    "openai>=1.0.0",
+    "duckduckgo-search>=8.0.0",
+    "pyyaml>=6.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "build>=1.0.0",
+    "pytest>=8.0.0",
+    "pytest-asyncio>=0.23.0",
+    "pytest-cov>=5.0.0",
+    "respx>=0.21.0",
+    "pytest-playwright>=0.5.0",
+    "black>=24.0.0",
+    "ruff>=0.4.0",
+    "mypy>=1.10.0",
+    "pre-commit>=3.7.0",
+    "faker>=25.0.0",
+    "factory-boy>=3.3.0",
+    "mkdocs>=1.6.0",
+    "mkdocs-material>=9.5.0",
+]
+
+[project.scripts]
+phoenix = "phoenix.cli.main:app"
+
+[project.urls]
+Homepage = "https://github.com/phnx-tech/phoenix-engine"
+Documentation = "https://github.com/phnx-tech/phoenix-engine#readme"
+Repository = "https://github.com/phnx-tech/phoenix-engine.git"
+Issues = "https://github.com/phnx-tech/phoenix-engine/issues"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.black]
+line-length = 100
+target-version = ["py311", "py312", "py313"]
+include = "\\.pyi?$"
+extend-exclude = """
+/(
+    migrations
+  | archive
+  | \\.venv
+)/
+"""
+
+[tool.ruff]
+target-version = "py311"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    "D105",
+    "D107",
+    "CPY001",
+    "TD003",
+    "FIX002",
+    "FBT003",
+    "ERA001",
+    "EM101",
+    "EM102",
+    "TRY003",
+    "D407",
+    "D413",
+]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.mccabe]
+max-complexity = 12
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = [
+    "S101",
+    "D100",
+    "D103",
+    "SLF001",
+    "ARG001",
+    "PLR2004",
+    "PLC0415",
+]
+"tests/fixtures/*" = ["ALL"]
+"*/__init__.py" = ["D104"]
+"src/phoenix/adapters/generated/*.py" = ["ALL"]
+"src/phoenix/cli/main.py" = ["ARG001", "B008", "FBT001", "PLR0913", "TC003"]
+"src/phoenix/pipeline.py" = ["ANN401", "BLE001", "S112"]
+"src/phoenix/adapters/base.py" = ["F401", "RUF001", "TC001", "UP035"]
+"src/phoenix/adapters/generic.py" = ["ARG002"]
+"src/phoenix/router.py" = ["PLC0415"]
+"src/phoenix/plugins/loader.py" = ["ANN401", "PLC0415", "TC003", "TRY004", "TRY300"]
+"src/phoenix/plugins/registry.py" = ["PYI034", "SLF001", "TC003"]
+"tests/unit/test_plugin_loader.py" = ["ANN401", "ARG002", "D102", "PLW0108", "Q001", "TC001", "W605"]
+"tests/unit/test_plugin_interface.py" = ["ANN401", "ARG002", "D102"]
+
+[tool.mypy]
+python_version = "3.11"
+strict = true
+warn_return_any = true
+warn_unused_ignores = true
+warn_unreachable = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_configs = true
+show_error_codes = true
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+markers = [
+    "integration: marks tests as integration tests",
+]
+addopts = "-ra"
+
+[tool.coverage.run]
+source = ["phoenix"]
+branch = true
+
+[tool.coverage.report]
+fail_under = 85
+skip_covered = false
+show_missing = true
+include = [
+    "src/phoenix/adapters/*.py",
+    "src/phoenix/collectors/*",
+    "src/phoenix/infrastructure/*.py",
+    "src/phoenix/models/*",
+    "src/phoenix/options.py",
+    "src/phoenix/exceptions.py",
+    "src/phoenix/plugins/*.py",
+    "src/phoenix/router.py",
+    "src/phoenix/version.py",
+    "src/phoenix/processing/ai_assistant.py",
+    "src/phoenix/processing/archiver.py",
+    "src/phoenix/processing/phoenix_ai_extractor.py",
+    "src/phoenix/intelligence/*.py",
+    "src/phoenix/pipeline.py",
+    "src/phoenix/engine.py",
+    "src/phoenix/scrapers/*.py",
+]

phoenix_engine-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

phoenix_engine-0.1.0/src/phoenix/__init__.py

@@ -0,0 +1,41 @@
+"""Phoenix Engine -- universal pure web scraping engine."""
+
+from __future__ import annotations
+
+from phoenix.adapters import (
+    BaseAdapter,
+    GenericWebAdapter,
+    PluginInterface,
+    ScraperPlugin,
+)
+from phoenix.engine import PhoenixEngine
+from phoenix.models.output import (
+    CollectionResult,
+    ScrapingResult,
+    UnifiedOutput,
+)
+from phoenix.models.strategy import ScrapingStrategy
+
+# Import options and core models before engine to prevent circular imports
+# with router/pipeline submodules.
+from phoenix.options import CollectionOptions, ScrapingOptions
+from phoenix.plugins import PluginLoader, PluginManifest, PluginRegistry
+from phoenix.version import __version__
+
+__all__ = [
+    "BaseAdapter",
+    "CollectionOptions",
+    "CollectionResult",
+    "GenericWebAdapter",
+    "PhoenixEngine",
+    "PluginInterface",
+    "PluginLoader",
+    "PluginManifest",
+    "PluginRegistry",
+    "ScraperPlugin",
+    "ScrapingOptions",
+    "ScrapingResult",
+    "ScrapingStrategy",
+    "UnifiedOutput",
+    "__version__",
+]

phoenix_engine-0.1.0/src/phoenix/__main__.py

@@ -0,0 +1,8 @@
+"""Entry point for ``python -m phoenix``."""
+
+from __future__ import annotations
+
+from phoenix.cli.main import app
+
+if __name__ == "__main__":
+    app()

phoenix_engine-0.1.0/src/phoenix/adapters/__init__.py

@@ -0,0 +1,25 @@
+"""Platform adapters that parse HTML into structured data."""
+
+from __future__ import annotations
+
+from phoenix.adapters.base import BaseAdapter, PluginInterface, ScraperPlugin
+from phoenix.adapters.facebook import FacebookAdapter
+from phoenix.adapters.generic import GenericWebAdapter
+from phoenix.adapters.instagram import InstagramAdapter
+from phoenix.adapters.linkedin import LinkedInAdapter
+from phoenix.adapters.tiktok import TikTokAdapter
+from phoenix.adapters.x_twitter import XTwitterAdapter
+from phoenix.adapters.youtube import YouTubeAdapter
+
+__all__ = [
+    "BaseAdapter",
+    "FacebookAdapter",
+    "GenericWebAdapter",
+    "InstagramAdapter",
+    "LinkedInAdapter",
+    "PluginInterface",
+    "ScraperPlugin",
+    "TikTokAdapter",
+    "XTwitterAdapter",
+    "YouTubeAdapter",
+]