scrapely-python 0.1.0__tar.gz

scrapely_python-0.1.0/PKG-INFO

@@ -0,0 +1,83 @@
+Metadata-Version: 2.4
+Name: scrapely-python
+Version: 0.1.0
+Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
+Author-email: Scrapely <support@scrape.ly>
+License: MIT
+Project-URL: Homepage, https://scrape.ly
+Project-URL: Documentation, https://docs.scrape.ly
+Project-URL: Repository, https://github.com/SandeepDev1/scrapely
+Keywords: scraping,actor,serverless,apify,scrapely
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: httpx>=0.25.0
+Requires-Dist: crawlee<2.0.0,>=1.8.0
+Requires-Dist: websockets>=12.0
+Requires-Dist: scrapely-client-python<2,>=1.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: cryptography>=42.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=1.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: isort>=5.12.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: playwright>=1.40.0; extra == "dev"
+Requires-Dist: browserforge>=1.2.0; extra == "dev"
+
+# Scrapely Python SDK
+
+Build Python Actors for the [Scrapely](https://scrape.ly) platform with lifecycle helpers, Crawlee-compatible storage, platform events, remote browser CDP, run control, webhooks, and pay-per-event accounting.
+
+## Install
+
+Scrapely requires Python 3.9 or newer.
+
+```bash
+python -m pip install scrapely
+```
+
+Install the appropriate Crawlee/Playwright browser dependencies separately when your Actor uses a browser.
+
+## Quick start
+
+```python
+import asyncio
+
+from scrapely import Actor
+
+
+async def main() -> None:
+    actor_input = await Actor.get_input_or_throw()
+    await Actor.push_data({
+        "url": actor_input["url"],
+        "status": "processed",
+    })
+
+
+if __name__ == "__main__":
+    asyncio.run(Actor.main(main))
+```
+
+`Actor.main()` initializes the Actor, awaits your callback, closes events and storage, and exits the process. For embedded applications and tests, use `async with Actor:` instead.
+
+## Highlights
+
+- Local file-system storage and Scrapely API-backed datasets, key-value stores, and request queues
+- Crawlee request loaders and storage objects
+- Platform event WebSocket integration and state persistence
+- Automatic remote Chromium CDP wiring through `CDP_URL`
+- Actor start, call, abort, reboot, and metamorph operations
+- Ad-hoc webhooks, proxy configuration, and charging/pricing types
+- Apify-compatible environment aliases and asynchronous client access
+
+See the [complete Python SDK guide](../docs/sdk-python.md) for lifecycle details, option objects, storage selection, events, CDP behavior, run control, environment variables, errors, and the full public export list.

scrapely_python-0.1.0/README.md

@@ -0,0 +1,47 @@
+# Scrapely Python SDK
+
+Build Python Actors for the [Scrapely](https://scrape.ly) platform with lifecycle helpers, Crawlee-compatible storage, platform events, remote browser CDP, run control, webhooks, and pay-per-event accounting.
+
+## Install
+
+Scrapely requires Python 3.9 or newer.
+
+```bash
+python -m pip install scrapely
+```
+
+Install the appropriate Crawlee/Playwright browser dependencies separately when your Actor uses a browser.
+
+## Quick start
+
+```python
+import asyncio
+
+from scrapely import Actor
+
+
+async def main() -> None:
+    actor_input = await Actor.get_input_or_throw()
+    await Actor.push_data({
+        "url": actor_input["url"],
+        "status": "processed",
+    })
+
+
+if __name__ == "__main__":
+    asyncio.run(Actor.main(main))
+```
+
+`Actor.main()` initializes the Actor, awaits your callback, closes events and storage, and exits the process. For embedded applications and tests, use `async with Actor:` instead.
+
+## Highlights
+
+- Local file-system storage and Scrapely API-backed datasets, key-value stores, and request queues
+- Crawlee request loaders and storage objects
+- Platform event WebSocket integration and state persistence
+- Automatic remote Chromium CDP wiring through `CDP_URL`
+- Actor start, call, abort, reboot, and metamorph operations
+- Ad-hoc webhooks, proxy configuration, and charging/pricing types
+- Apify-compatible environment aliases and asynchronous client access
+
+See the [complete Python SDK guide](../docs/sdk-python.md) for lifecycle details, option objects, storage selection, events, CDP behavior, run control, environment variables, errors, and the full public export list.

scrapely_python-0.1.0/pyproject.toml

@@ -0,0 +1,85 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "scrapely-python"
+version = "0.1.0"
+description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
+readme = "README.md"
+license = {text = "MIT"}
+authors = [
+    {name = "Scrapely", email = "support@scrape.ly"}
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Internet :: WWW/HTTP :: Dynamic Content",
+]
+keywords = ["scraping", "actor", "serverless", "apify", "scrapely"]
+requires-python = ">=3.11"
+dependencies = [
+    "httpx>=0.25.0",
+    "crawlee>=1.8.0,<2.0.0",
+    "websockets>=12.0",
+    "scrapely-client-python>=1.0.0,<2",
+    "pydantic>=2.0.0",
+    "cryptography>=42.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-asyncio>=1.0.0",
+    "black>=23.0.0",
+    "isort>=5.12.0",
+    "mypy>=1.0.0",
+    "playwright>=1.40.0",
+    "browserforge>=1.2.0",
+]
+
+
+[project.urls]
+Homepage = "https://scrape.ly"
+Documentation = "https://docs.scrape.ly"
+Repository = "https://github.com/SandeepDev1/scrapely"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+scrapely = ["py.typed"]
+
+[tool.black]
+line-length = 100
+target-version = ["py311", "py312"]
+
+[tool.isort]
+profile = "black"
+line_length = 100
+
+[tool.mypy]
+python_version = "3.11"
+plugins = ["pydantic.mypy"]
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+disallow_untyped_decorators = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_no_return = true
+warn_unreachable = true
+strict_equality = true
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]

scrapely_python-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

scrapely_python-0.1.0/src/scrapely/__init__.py

@@ -0,0 +1,246 @@
+"""
+Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform.
+
+Usage:
+    from scrapely import Actor, Dataset
+
+    async def main():
+        async with Actor:
+            actor_input = await Actor.get_input()
+            await Actor.push_data({'url': actor_input['url']})
+"""
+
+# Side-effect: patch Crawlee Playwright browser pool to use remote CDP (CDP_URL).
+# Must run on package import so `from scrapely import Actor` enables the patch
+# before PlaywrightCrawler launches a browser (Node SDK parity).
+from .patch_browser_pool import set_cdp_url  # noqa: F401
+
+from crawlee import Request
+
+from crawlee.request_loaders import (
+    RequestLoader,
+    RequestManager,
+    RequestManagerTandem,
+    SitemapRequestLoader,
+)
+from crawlee.storages import Dataset, KeyValueStore, RequestQueue
+from crawlee.storage_clients.models import DatasetItemsListPage
+
+from .actor import Actor
+from .configuration import Configuration
+from .charging_client import (
+    USES_PUSH_DATA_INTERCEPTION,
+    create_charging_client,
+    push_data_charging_context,
+)
+from .storage import (
+    ApiStorageClient,
+    FileSystemStorageClient,
+    MemoryStorageClient,
+    ScrapelyStorageClient,
+    SmartScrapelyStorageClient,
+    StorageClient,
+    set_api_storage,
+)
+from .request_loaders import ScrapelyRequestList
+from crawlee.events import EventListener
+from .events import (
+    AbortingEvent,
+    ActorEventTypes,
+    CrawleeEventManager,
+    DeprecatedEvent,
+    Event,
+    EventAbortingData,
+    EventExitData,
+    EventManager,
+    EventMessage,
+    EventMigratingData,
+    EventPersistStateData,
+    EventSystemInfoData,
+    EventWithoutData,
+    ExitEvent,
+    LocalEventManager,
+    MigratingEvent,
+    PersistStateEvent,
+    ScrapelyEventManager,
+    SystemInfoEvent,
+    SystemInfoEventData,
+    UnknownEvent,
+)
+from .proxy_configuration import ProxyConfiguration
+from .browser_configuration import BrowserConfiguration
+from .charging import (
+    ChargingManager,
+    ChargeUncertainError,
+    ChargeRejectedError,
+    has_default_dataset_item_event_in_pricing,
+    is_default_dataset_item_event,
+    merge_charge_results,
+    push_data_and_charge,
+    resolve_default_dataset_item_event_name,
+    APIFY_DEFAULT_DATASET_ITEM_EVENT,
+    SCRAPELY_DEFAULT_DATASET_ITEM_EVENT,
+    DEFAULT_DATASET_ITEM_EVENT,
+)
+
+from .types import (
+    ActorInput,
+    ActorPermissionLevel,
+    ActorRun,
+    ApifyEnv,
+    AbortOptions,
+    CallOptions,
+    ExitOptions,
+    InitOptions,
+    MetamorphOptions,
+    OpenStorageOptions,
+    ProxyInfo,
+    RebootOptions,
+    RecordOptions,
+    SetStatusMessageOptions,
+    StartOptions,
+    WebhookOptions,
+)
+
+from .webhook import (
+    Webhook,
+    WebhookEventType,
+)
+from .pricing_types import (
+    PricingModel,
+    FreeActorPricingInfo,
+    FlatPricePerMonthActorPricingInfo,
+    PricePerDatasetItemActorPricingInfo,
+    PayPerEventActorPricingInfo,
+    PricingPerEvent,
+    ActorPricingInfo,
+    ActorChargeEvent,
+    ChargeResult,
+    ActorPricingInfoResult,
+)
+from .errors import (
+    ScrapelyClientError,
+    ScrapelyApiError,
+    InvalidResponseBodyError,
+    InvalidRequestError,
+    UnauthorizedError,
+    ForbiddenError,
+    NotFoundError,
+    ConflictError,
+    RateLimitError,
+    ServerError,
+    raise_for_status,
+)
+
+__version__ = '0.1.0'
+__all__ = [
+    # Core
+    'Actor',
+    'Configuration',
+    'set_cdp_url',
+    # ApifyClient
+    'create_charging_client',
+    'USES_PUSH_DATA_INTERCEPTION',
+    'push_data_charging_context',
+
+    # Storage
+    'Dataset',
+    'KeyValueStore',
+    'RequestQueue',
+    'Request',
+    'DatasetItemsListPage',
+    # Storage clients
+    'ApiStorageClient',
+    'ScrapelyStorageClient',
+    'SmartScrapelyStorageClient',
+    'FileSystemStorageClient',
+    'MemoryStorageClient',
+    'StorageClient',
+    'set_api_storage',
+    # Request loaders
+    'ScrapelyRequestList',
+    'RequestLoader',
+    'RequestManager',
+    'RequestManagerTandem',
+    'SitemapRequestLoader',
+    # Events
+    'Event',
+    'EventManager',
+    'ScrapelyEventManager',
+    'CrawleeEventManager',
+    'LocalEventManager',
+    # Event data types
+    'SystemInfoEventData',
+    'EventAbortingData',
+    'EventExitData',
+    'EventListener',
+    'EventMigratingData',
+    'EventPersistStateData',
+    'EventSystemInfoData',
+    # Event types
+    'PersistStateEvent',
+    'SystemInfoEvent',
+    'MigratingEvent',
+    'AbortingEvent',
+    'ExitEvent',
+    'EventWithoutData',
+    'DeprecatedEvent',
+    'UnknownEvent',
+    'EventMessage',
+    'ActorEventTypes',
+    # Proxy
+    'ProxyConfiguration',
+    'ProxyInfo',
+    # Remote browser
+    'BrowserConfiguration',
+    # Charging
+    'ChargingManager',
+    'ChargeResult',
+    'ChargeUncertainError',
+    'ChargeRejectedError',
+    'merge_charge_results',
+    'push_data_and_charge',
+
+    # Types
+    'ActorInput',
+    'ActorPermissionLevel',
+    'ActorRun',
+    'ApifyEnv',
+    'AbortOptions',
+    'CallOptions',
+    'ExitOptions',
+    'InitOptions',
+    'MetamorphOptions',
+    'OpenStorageOptions',
+    'RebootOptions',
+    'RecordOptions',
+    'SetStatusMessageOptions',
+    'StartOptions',
+    'WebhookOptions',
+
+    # Webhook
+    'Webhook',
+    'WebhookEventType',
+    # Pricing
+    'PricingModel',
+    'FreeActorPricingInfo',
+    'FlatPricePerMonthActorPricingInfo',
+    'PricePerDatasetItemActorPricingInfo',
+    'PayPerEventActorPricingInfo',
+    'PricingPerEvent',
+    'ActorPricingInfo',
+    'ActorChargeEvent',
+    'ActorPricingInfoResult',
+    # Errors
+    'ScrapelyClientError',
+    'ScrapelyApiError',
+    'InvalidResponseBodyError',
+    'InvalidRequestError',
+    'UnauthorizedError',
+    'ForbiddenError',
+    'NotFoundError',
+    'ConflictError',
+    'RateLimitError',
+    'ServerError',
+    'raise_for_status',
+]