scrapely-python 0.1.0__py3-none-any.whl

scrapely/types.py

@@ -0,0 +1,269 @@
+"""
+Type definitions for the Scrapely Python SDK.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Dict, List, Optional, Union
+
+# Single source of truth for ChargeResult lives in pricing_types (pydantic model
+# used by ChargingManager); re-exported here for backwards compatibility.
+from .pricing_types import ChargeResult  # noqa: F401  (re-export)
+
+
+class ActorPermissionLevel(str, Enum):
+    """Actor permission level - matches Apify's ACTOR_PERMISSION_LEVEL enum."""
+    NONE = 'NONE'
+    RUN = 'RUN'
+    WRITE = 'WRITE'
+    ADMIN = 'ADMIN'
+
+
+@dataclass
+class ActorInput:
+    """Value types returned by Actor.get_input()."""
+    data: Dict[str, Any] = field(default_factory=dict)
+    
+    def get(self, key: str, default: Any = None) -> Any:
+        return self.data.get(key, default)
+    
+    def __getitem__(self, key: str) -> Any:
+        return self.data[key]
+    
+    def __contains__(self, key: str) -> bool:
+        return key in self.data
+
+
+@dataclass
+class ApifyEnv:
+    """Parsed representation of the environment variables.
+    
+    This object is returned by the Actor.get_env() function.
+    Fully compatible with Apify's ApifyEnv interface.
+    
+    Supports both attribute access (env.memory_mbytes) and
+    dictionary-style subscript access (env['memory_mbytes']).
+    """
+    actor_id: Optional[str] = None
+    actor_build_id: Optional[str] = None
+    actor_build_number: Optional[str] = None
+    actor_build_tags: Optional[str] = None
+    actor_events_ws_url: Optional[str] = None
+    actor_full_name: Optional[str] = None
+    actor_max_paid_dataset_items: Optional[int] = None
+    actor_permission_level: Optional[ActorPermissionLevel] = None
+    actor_pricing_info: Optional[str] = None
+    actor_run_id: Optional[str] = None
+    actor_storages_json: Optional[str] = None
+    actor_task_id: Optional[str] = None
+    api_base_url: Optional[str] = None
+    api_public_base_url: Optional[str] = None
+    charged_event_counts: Optional[str] = None
+    chrome_executable_path: Optional[str] = None
+    container_port: Optional[int] = None
+    container_url: Optional[str] = None
+    dedicated_cpus: Optional[str] = None
+    default_dataset_id: Optional[str] = None
+    default_key_value_store_id: Optional[str] = None
+    default_request_queue_id: Optional[str] = None
+    disable_outdated_warning: Optional[int] = None
+    fact: Optional[str] = None
+    headless: Optional[str] = None
+    input_key: Optional[str] = None
+    input_secrets_private_key_file: Optional[str] = None
+    input_secrets_private_key_passphrase: Optional[str] = None
+    is_at_home: Optional[str] = None
+    local_storage_dir: Optional[str] = None
+    log_format: Optional[str] = None
+    log_level: Optional[str] = None
+    memory_mbytes: Optional[int] = None
+    meta_origin: Optional[str] = None
+    metamorph_after_sleep_millis: Optional[int] = None
+    persist_state_interval_millis: Optional[int] = None
+    proxy_hostname: Optional[str] = None
+    proxy_password: Optional[str] = None
+    proxy_port: Optional[str] = None
+    proxy_status_url: Optional[str] = None
+    purge_on_start: Optional[str] = None
+    sdk_latest_version: Optional[str] = None
+    started_at: Optional[datetime] = None
+    standby_port: Optional[int] = None
+    standby_url: Optional[str] = None
+    system_info_interval_millis: Optional[str] = None
+    timeout_at: Optional[datetime] = None
+    token: Optional[str] = None
+    user_id: Optional[str] = None
+    user_is_paying: Optional[str] = None
+    user_pricing_tier: Optional[str] = None
+    workflow_key: Optional[str] = None
+    xvfb: Optional[str] = None
+    # Scrapely extensions / ACTOR_* extras
+    max_total_charge_usd: Optional[float] = None
+    user_plan: Optional[str] = None
+    restart_on_error: Optional[str] = None
+
+
+    def __getitem__(self, key: str) -> Any:
+        """Enable dictionary-style access: env['memory_mbytes']."""
+        return getattr(self, key)
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        """Enable dictionary-style assignment: env['memory_mbytes'] = 1024."""
+        setattr(self, key, value)
+
+    def keys(self) -> List[str]:
+        """Return all field names that have values set (non-None)."""
+        return [f.name for f in self.__dataclass_fields__.values() 
+                if getattr(self, f.name) is not None]
+
+
+@dataclass
+class WebhookOptions:
+    """Options for Actor.add_webhook()."""
+    event_types: List[str] = field(default_factory=list)
+    request_url: str = ''
+    payload_template: Optional[str] = None
+    idempotency_key: Optional[str] = None
+    headers_template: Optional[str] = None
+    description: Optional[str] = None
+    ignore_ssl_errors: bool = False
+    do_not_retry: bool = False
+    should_interpolate_strings: bool = False
+    is_apify_integration: bool = False
+
+
+@dataclass
+class InitOptions:
+    """Options for Actor.init()."""
+    configuration: Optional[Dict[str, Any]] = None
+    # ApiStorageConfig dict or a Crawlee StorageClient (tests / custom)
+    storage: Optional[Any] = None
+    graceful_shutdown: bool = True
+    graceful_shutdown_delay_millis: int = 0
+    # Set up the default log output (Apify Actor(configure_logging=True) parity)
+    configure_logging: bool = True
+
+
+@dataclass
+class ExitOptions:
+    """Options for Actor.exit()."""
+    status_message: Optional[str] = None
+    timeout_secs: int = 30
+    exit_code: int = 0
+    exit: bool = True
+
+
+@dataclass
+class RecordOptions:
+    """Options for Actor.set_value() / KeyValueStore.set_value()."""
+    content_type: Optional[str] = None
+    do_not_retry_timeouts: bool = False
+    timeout_secs: Optional[int] = None
+
+
+@dataclass
+class SetStatusMessageOptions:
+    """Options for Actor.set_status_message()."""
+    is_status_message_terminal: bool = False
+    level: str = 'INFO'  # 'DEBUG', 'INFO', 'WARNING', 'ERROR'
+
+
+@dataclass
+class MetamorphOptions:
+    """Options for Actor.metamorph()."""
+    build: Optional[str] = None
+
+
+@dataclass
+class RebootOptions:
+    """Options for Actor.reboot()."""
+    custom_after_sleep_millis: Optional[int] = None
+
+
+@dataclass
+class OpenStorageOptions:
+    """Options for Actor.open_dataset(), open_key_value_store(), and open_request_queue()."""
+    force_cloud: bool = False
+    config: Optional[Any] = None
+
+
+@dataclass
+class CallOptions:
+    """Options for Actor.call() / Actor.call_task()."""
+    token: Optional[str] = None
+    build: Optional[str] = None
+    max_total_charge_usd: Optional[float] = None
+    restart_on_error: Optional[bool] = None
+    memory_mbytes: Optional[int] = None
+    timeout: Optional[Union[int, str]] = None  # number or 'inherit'
+    force_permission_level: Optional[ActorPermissionLevel] = None
+    webhooks: Optional[List[WebhookOptions]] = None
+
+
+@dataclass
+class StartOptions:
+    """Options for Actor.start()."""
+    token: Optional[str] = None
+    build: Optional[str] = None
+    max_total_charge_usd: Optional[float] = None
+    restart_on_error: Optional[bool] = None
+    memory_mbytes: Optional[int] = None
+    timeout: Optional[Union[int, str]] = None  # number or 'inherit'
+    force_permission_level: Optional[ActorPermissionLevel] = None
+    webhooks: Optional[List[WebhookOptions]] = None
+
+
+@dataclass
+class AbortOptions:
+    """Options for Actor.abort()."""
+    token: Optional[str] = None
+    status_message: Optional[str] = None
+    gracefully: Optional[bool] = None
+
+
+@dataclass
+class ActorRun:
+    """Information about an Actor run."""
+    id: str
+    actor_id: str
+    status: str
+    started_at: Optional[datetime] = None
+    finished_at: Optional[datetime] = None
+    duration_millis: Optional[int] = None
+    build_id: Optional[str] = None
+    build_number: Optional[str] = None
+    default_dataset_id: Optional[str] = None
+    default_key_value_store_id: Optional[str] = None
+    default_request_queue_id: Optional[str] = None
+    exit_code: Optional[int] = None
+    status_message: Optional[str] = None
+    input: Optional[Dict[str, Any]] = None
+    output: Optional[Dict[str, Any]] = None
+    usage_total_usd: Optional[float] = None
+    usage: Optional[Dict[str, Any]] = None
+
+
+@dataclass
+class ProxyInfo:
+    """Information about a proxy configuration.
+    
+    Compatible with crawlee's ProxyInfo to support crawlee's _basic_crawler.py
+    which calls proxy_configuration.new_proxy_info(request=..., session=...).
+    """
+    url: str
+    scheme: str  # Required by crawlee
+    hostname: str
+    port: int
+    username: str = ''
+    password: str = ''
+    session_id: Optional[str] = None  # Required by crawlee
+    proxy_tier: Optional[int] = None  # Required by crawlee
+    groups: List[str] = field(default_factory=list)
+    country_code: Optional[str] = None
+    subdivision_code: Optional[str] = None
+
+
+# Type aliases
+Dictionary = Dict[str, Any]
+JsonValue = Union[None, bool, int, float, str, List['JsonValue'], Dict[str, 'JsonValue']]

scrapely/webhook.py

@@ -0,0 +1,82 @@
+"""
+Webhook types for the Scrapely Python SDK.
+
+This module defines the Webhook class and related types for creating
+ad-hoc webhooks on the Scrapely platform.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+
+class WebhookEventType(str, Enum):
+    """Event types that can trigger a webhook."""
+    ACTOR_RUN_CREATED = 'ACTOR.RUN.CREATED'
+    ACTOR_RUN_SUCCEEDED = 'ACTOR.RUN.SUCCEEDED'
+    ACTOR_RUN_FAILED = 'ACTOR.RUN.FAILED'
+    ACTOR_RUN_ABORTED = 'ACTOR.RUN.ABORTED'
+    ACTOR_RUN_TIMED_OUT = 'ACTOR.RUN.TIMED_OUT'
+
+
+@dataclass
+class Webhook:
+    """An Apify webhook definition used by the Actor SDK.
+    
+    The same instance can be passed as an ad-hoc webhook to Actor.start() / Actor.call(),
+    or to Actor.add_webhook(), which registers it as an ad-hoc webhook for the current run
+    (the condition.actor_run_id is set automatically).
+    """
+    
+    event_types: List[WebhookEventType]
+    request_url: str
+    payload_template: Optional[str] = None
+    headers_template: Optional[str] = None
+    idempotency_key: Optional[str] = None
+    do_not_retry: Optional[bool] = None
+    ignore_ssl_errors: Optional[bool] = None
+    # Server-assigned identifier, populated by Actor.add_webhook() / the API.
+    id: Optional[str] = None
+    
+    def __post_init__(self) -> None:
+        """Validate the webhook after initialization."""
+        if not self.event_types:
+            raise ValueError('At least one event type must be specified')
+        if not self.request_url:
+            raise ValueError('Request URL must be specified')
+    
+    def to_dict(self, actor_run_id: Optional[str] = None) -> Dict[str, Any]:
+        """Convert webhook to dictionary for API calls.
+        
+        Args:
+            actor_run_id: Optional run ID to bind the webhook to.
+            
+        Returns:
+            Dictionary representation for the API.
+        """
+        result: Dict[str, Any] = {
+            'eventTypes': [e.value for e in self.event_types],
+            'requestUrl': self.request_url,
+        }
+        
+        if self.payload_template:
+            result['payloadTemplate'] = self.payload_template
+        if self.headers_template:
+            result['headersTemplate'] = self.headers_template
+        if self.idempotency_key:
+            result['idempotencyKey'] = self.idempotency_key
+        if self.do_not_retry is not None:
+            result['doNotRetry'] = self.do_not_retry
+        if self.ignore_ssl_errors is not None:
+            result['ignoreSslErrors'] = self.ignore_ssl_errors
+        
+        # Add condition for the current run
+        if actor_run_id:
+            result['condition'] = {'actorRunId': actor_run_id}
+        
+        return result
+
+
+__all__ = ['Webhook', 'WebhookEventType']

scrapely_python-0.1.0.dist-info/METADATA

@@ -0,0 +1,83 @@
+Metadata-Version: 2.4
+Name: scrapely-python
+Version: 0.1.0
+Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
+Author-email: Scrapely <support@scrape.ly>
+License: MIT
+Project-URL: Homepage, https://scrape.ly
+Project-URL: Documentation, https://docs.scrape.ly
+Project-URL: Repository, https://github.com/SandeepDev1/scrapely
+Keywords: scraping,actor,serverless,apify,scrapely
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: httpx>=0.25.0
+Requires-Dist: crawlee<2.0.0,>=1.8.0
+Requires-Dist: websockets>=12.0
+Requires-Dist: scrapely-client-python<2,>=1.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: cryptography>=42.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=1.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: isort>=5.12.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: playwright>=1.40.0; extra == "dev"
+Requires-Dist: browserforge>=1.2.0; extra == "dev"
+
+# Scrapely Python SDK
+
+Build Python Actors for the [Scrapely](https://scrape.ly) platform with lifecycle helpers, Crawlee-compatible storage, platform events, remote browser CDP, run control, webhooks, and pay-per-event accounting.
+
+## Install
+
+Scrapely requires Python 3.9 or newer.
+
+```bash
+python -m pip install scrapely
+```
+
+Install the appropriate Crawlee/Playwright browser dependencies separately when your Actor uses a browser.
+
+## Quick start
+
+```python
+import asyncio
+
+from scrapely import Actor
+
+
+async def main() -> None:
+    actor_input = await Actor.get_input_or_throw()
+    await Actor.push_data({
+        "url": actor_input["url"],
+        "status": "processed",
+    })
+
+
+if __name__ == "__main__":
+    asyncio.run(Actor.main(main))
+```
+
+`Actor.main()` initializes the Actor, awaits your callback, closes events and storage, and exits the process. For embedded applications and tests, use `async with Actor:` instead.
+
+## Highlights
+
+- Local file-system storage and Scrapely API-backed datasets, key-value stores, and request queues
+- Crawlee request loaders and storage objects
+- Platform event WebSocket integration and state persistence
+- Automatic remote Chromium CDP wiring through `CDP_URL`
+- Actor start, call, abort, reboot, and metamorph operations
+- Ad-hoc webhooks, proxy configuration, and charging/pricing types
+- Apify-compatible environment aliases and asynchronous client access
+
+See the [complete Python SDK guide](../docs/sdk-python.md) for lifecycle details, option objects, storage selection, events, CDP behavior, run control, environment variables, errors, and the full public export list.

scrapely_python-0.1.0.dist-info/RECORD

@@ -0,0 +1,43 @@
+scrapely/__init__.py,sha256=EqZTSyL6fkCFXkbp2iej7Umi6y_F19aC7MqxOaUbB68,5745
+scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
+scrapely/actor.py,sha256=WzvD7gvCKRMyIN_Ei_I-MfIBL7Darn_B40GQJIgAn04,74960
+scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
+scrapely/charging.py,sha256=ckqsGbGuzpdw5PgEt6Apx3cwR46sA8V2WwUDSqSSfz8,46887
+scrapely/charging_client.py,sha256=tDtlPwLSVFvoRa2NBgNjyfHkAUM77SSIyYT2ACgpDm8,3710
+scrapely/configuration.py,sha256=YUnbft9JeWlVTgWalr-CgJCiyJ_e_rgoQItVPvjeWRs,15406
+scrapely/errors.py,sha256=DCAlVvE3jToZuPEG2HVDL7i8D8hLboomQ5JpSOcsXoQ,4107
+scrapely/log.py,sha256=iVKoQdMLQXEWz1STzbSXrNUxsS_FNwiIdqoEg7CzeqU,1440
+scrapely/patch_browser_pool.py,sha256=sRAjuzKoOEgn0X8695c_9qioRALzkTWXoDlorVnK9jg,37426
+scrapely/pricing_types.py,sha256=J6wCddZd-U1MYeOtaaEdeHSsp1HBgiQLfvSInBOvhIo,5305
+scrapely/proxy_configuration.py,sha256=lvfOvuR8tOE9uiMOS_nw6TEj11xy7GYGa56iYvPwtmA,34654
+scrapely/py.typed,sha256=XtWlywqUjc6x_M2gAXlDtYXbAsYkCjKZEjFyJvaHvm4,105
+scrapely/types.py,sha256=ceVuHvhYbPai2CL3vFmXLTMoGaMR27IefJFgWhzd7ws,8900
+scrapely/webhook.py,sha256=4WOM42ypOSOs4lKtP5ofxAh6lX-7c8p5U9lOfuFzx-g,2830
+scrapely/crypto/__init__.py,sha256=MbMPs7Eq_fGSVjOZxNts_JckDCUtxCVqdkHU0VCfN4M,481
+scrapely/crypto/_secrets.py,sha256=-ETpq1yCMaW-mheTGedFaQGGhu_pGT9_VMdJ3--V1rk,7614
+scrapely/events/__init__.py,sha256=8vbbxxqkV0DrQicvFkwFwacUzi96XeW-b6KQzpUBuSM,1239
+scrapely/events/_types.py,sha256=UHNP1dVVG0geJqhlhivQHK00nErYJ0gaQ-PxuGBujXo,5754
+scrapely/events/event_manager.py,sha256=CiI-Fiw0vaKx1iGEiXRG9Df08WDgwFDOT-DX3IIHZJg,12521
+scrapely/request_loaders/__init__.py,sha256=aNgr4pRt3MziUbU6UQ_bjTT3moMNvvyUsPeCJyMQcwU,196
+scrapely/request_loaders/_scrapely_request_list.py,sha256=cq7xx7Ai4qOhcqGV2TLqPR3o53FPCPKpVxWEMw5qKaM,7367
+scrapely/storage/__init__.py,sha256=diAhxEvOXjc3YVhZvzkO3MttJUXQWJ5ogxstiApsh7U,1405
+scrapely/storage/_metadata.py,sha256=WWawBnVN0b1iK2K-SlxZXUc6czb5wXWf8iC_0tzkgAQ,4764
+scrapely/storage/api_client.py,sha256=wrRxQ8h4jKdWY50HI0gXS-XHaFpzWPbVyLQ1nOYnsnA,11305
+scrapely/storage/client.py,sha256=M3o_a_9azHNn6eL_UyEeVz-VQJvnT_th1HJPuCA4PkE,21746
+scrapely/storage/patch_storage.py,sha256=o3Fv0BByeZ1T9HhkzhY6bjolOFsIQcnqBoHq1EdG8Fw,8758
+scrapely/storage/scrapely_dataset_client.py,sha256=gRvlCDw9c0yO5GE5LQOa1xAyXc9pnb-E37Eq7fHqHCM,10338
+scrapely/storage/scrapely_kvs_client.py,sha256=ItQhe2rcSl20NkSmegfqHiRyKncH4Qg9BjQVpF6jHfk,7403
+scrapely/storage/scrapely_rq_client.py,sha256=0MoLZSIlzdqaQW_wypraVmHIsQimf_fV4-LCA_lLKJc,18601
+scrapely/storage/scrapely_storage_client.py,sha256=N-p2ISICuKQ8RNzmN-JiXcj_lTttULLOGl9lpLl62qI,6567
+scrapely/storage/smart_scrapely_storage_client.py,sha256=OipItBvlDz83_5ctQirdC8yRSv321d_nrZTqReTwmOU,4245
+scrapely/storage/types.py,sha256=7eC9tL_eiEL2neaK_Ri1VZ0aolrvHlkhwgW9HZ94z64,20844
+scrapely/storage/utils.py,sha256=ngFWCNM48Kx6tdsvR05Pl3iWgiNPwTL3YlEpeRBm57w,1425
+scrapely/storage/_file_system/__init__.py,sha256=dSJ28uDrA--dz6e-vqk7MpE2nYOsXVjkzIlmGvoHTZ8,318
+scrapely/storage/_file_system/_key_value_store_client.py,sha256=4Ipgbh4qEqAkwQJ9fXTujFR0Uu4nsw_7jvX6UkIk-Is,6094
+scrapely/storage/_file_system/_storage_client.py,sha256=aaGJ4P0QAtIgEMl6yHpuobeWFD5M4rMC5HwhOOvuGuo,1794
+scrapely/storage_clients/__init__.py,sha256=cs1YX7JV2_GKBXeiiJgkQFali08b5_y_jH6Yw7iROAA,656
+scrapely/storages/__init__.py,sha256=4PAELhJNZ3exJ7Uouicn-bCQ58NafKjhLBixQ8ATQc4,173
+scrapely_python-0.1.0.dist-info/METADATA,sha256=AbI8nacnK9KwHjwfREv7B3uBfSMnFQuFpyzwaU-33Ag,3137
+scrapely_python-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+scrapely_python-0.1.0.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
+scrapely_python-0.1.0.dist-info/RECORD,,

scrapely_python-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

scrapely_python-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+scrapely