scrapely-python 0.1.0__py3-none-any.whl

scrapely/crypto/_secrets.py

@@ -0,0 +1,210 @@
+"""Input secret decryption for the Scrapely Python SDK.
+
+Port of the Node SDK's ``src/crypto/secrets.ts``. The platform encrypts secret
+Actor input values with RSA-OAEP (SHA-256) and delivers the RSA private key to
+the container as a passphrase-encrypted file (AES-256-GCM with a PBKDF2-derived
+key). This module decrypts the private key, decrypts each secret field, and
+merges the plaintext values back into the Actor input.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict
+
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import padding, rsa
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+logger = logging.getLogger(__name__)
+
+# ─── Constants (must match the Node SDK's crypto/secrets.ts) ──────────────────
+
+_IV_LENGTH = 12
+_AUTH_TAG_LENGTH = 16
+_PBKDF2_ITERATIONS = 100_000
+_PBKDF2_HASH = 'sha256'
+_SALT_LENGTH = 32
+_KEY_LENGTH = 32
+
+_OAEP_PADDING = padding.OAEP(
+    mgf=padding.MGF1(algorithm=hashes.SHA256()),
+    algorithm=hashes.SHA256(),
+    label=None,
+)
+
+
+def decrypt_private_key_file(private_key_file_path: str, passphrase: str) -> str:
+    """Read the private key file from disk and decrypt it with the run passphrase.
+
+    The file layout is ``salt(32) | iv(12) | authTag(16) | ciphertext``, where the
+    AES-256-GCM key is derived from the passphrase with PBKDF2-HMAC-SHA256.
+
+    Args:
+        private_key_file_path: Path to the encrypted private key file
+            (e.g. ``/run/secrets/private_key.pem``).
+        passphrase: The run passphrase from ``APIFY_INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE``.
+
+    Returns:
+        The decrypted private key as a PEM string.
+    """
+    encrypted_data = Path(private_key_file_path).read_bytes()
+
+    salt = encrypted_data[:_SALT_LENGTH]
+    iv = encrypted_data[_SALT_LENGTH:_SALT_LENGTH + _IV_LENGTH]
+    auth_tag = encrypted_data[_SALT_LENGTH + _IV_LENGTH:_SALT_LENGTH + _IV_LENGTH + _AUTH_TAG_LENGTH]
+    encrypted = encrypted_data[_SALT_LENGTH + _IV_LENGTH + _AUTH_TAG_LENGTH:]
+
+    key = hashlib.pbkdf2_hmac(
+        _PBKDF2_HASH,
+        passphrase.encode('utf-8'),
+        salt,
+        _PBKDF2_ITERATIONS,
+        dklen=_KEY_LENGTH,
+    )
+
+    # The ``cryptography`` AESGCM helper expects ciphertext || tag (the file
+    # stores tag before the ciphertext, like Node's setAuthTag() flow).
+    decrypted = AESGCM(key).decrypt(iv, encrypted + auth_tag, None)
+
+    return decrypted.decode('utf-8')
+
+
+def decrypt_secret_value(encrypted_base64: str, private_key_pem: str) -> Any:
+    """Decrypt a single secret value encrypted with RSA-OAEP (SHA-256).
+
+    Args:
+        encrypted_base64: Base64-encoded encrypted data.
+        private_key_pem: The decrypted private key PEM.
+
+    Returns:
+        The decrypted value; JSON-shaped plaintext is parsed into Python objects,
+        anything else is returned as a plain string.
+    """
+    encrypted = base64.b64decode(encrypted_base64)
+
+    private_key = serialization.load_pem_private_key(private_key_pem.encode('utf-8'), password=None)
+    if not isinstance(private_key, rsa.RSAPrivateKey):
+        raise ValueError('The decrypted private key is not an RSA key')
+    decrypted_data = private_key.decrypt(encrypted, _OAEP_PADDING)
+
+    plaintext = decrypted_data.decode('utf-8')
+
+    # Try to parse as JSON (for objects, arrays, numbers, booleans)
+    try:
+        return json.loads(plaintext)
+    except json.JSONDecodeError:
+        return plaintext
+
+
+def decrypt_all_secrets(
+    encrypted_secrets: Dict[str, str],
+    private_key_pem: str,
+) -> Dict[str, Any]:
+    """Decrypt all secret values from the encrypted input payload.
+
+    Args:
+        encrypted_secrets: Map of field name to base64 encrypted value.
+        private_key_pem: The decrypted private key PEM.
+
+    Returns:
+        Map of field name to decrypted value; fields that fail to decrypt are
+        skipped with a warning (matching the Node SDK behavior).
+    """
+    decrypted: Dict[str, Any] = {}
+
+    for field_name, encrypted_base64 in encrypted_secrets.items():
+        try:
+            decrypted[field_name] = decrypt_secret_value(encrypted_base64, private_key_pem)
+        except Exception as err:  # noqa: BLE001 - a bad field must not kill the run
+            logger.warning('Failed to decrypt secret field "%s": %s', field_name, err)
+
+    return decrypted
+
+
+def merge_decrypted_input(
+    input_data: Dict[str, Any],
+    decrypted_secrets: Dict[str, Any],
+) -> Dict[str, Any]:
+    """Merge decrypted secret values into the original input object.
+
+    Supports nested paths like ``credentials.apiKey`` or ``database.password``.
+
+    Args:
+        input_data: The original input object (with secrets stripped/masked).
+        decrypted_secrets: Map of field name to decrypted value.
+
+    Returns:
+        A new input object with secrets populated.
+    """
+    merged = dict(input_data)
+
+    for field_name, value in decrypted_secrets.items():
+        if '.' in field_name:
+            # Handle nested paths
+            parts = field_name.split('.')
+            current = merged
+            for part in parts[:-1]:
+                if not isinstance(current.get(part), dict):
+                    current[part] = {}
+                current = current[part]
+            current[parts[-1]] = value
+        else:
+            merged[field_name] = value
+
+    return merged
+
+
+def decrypt_secret_inputs(input_data: Dict[str, Any]) -> Dict[str, Any]:
+    """Full decryption flow: read the private key file, decrypt it with the passphrase,
+    then decrypt all secrets and merge them into the input.
+
+    This is called from ``Actor.get_input()`` when
+    ``APIFY_INPUT_SECRETS_PRIVATE_KEY_FILE`` is set.
+
+    Args:
+        input_data: Current Actor input object.
+
+    Returns:
+        The input object with decrypted secret values merged in; on failure the
+        input is returned as-is so the Actor can still run.
+    """
+    private_key_file = os.environ.get('APIFY_INPUT_SECRETS_PRIVATE_KEY_FILE')
+    passphrase = os.environ.get('APIFY_INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE')
+
+    if not private_key_file or not passphrase:
+        return input_data  # No secrets to decrypt
+
+    logger.info('Decrypting secret inputs...')
+
+    try:
+        # 1. Read and decrypt the private key file
+        private_key_pem = decrypt_private_key_file(private_key_file, passphrase)
+        logger.info('Private key decrypted successfully')
+
+        # 2. Get the encrypted secrets from the input
+        # The encrypted secrets are stored in a special __encrypted_secrets property
+        encrypted_secrets = input_data.get('__encrypted_secrets')
+
+        if not encrypted_secrets or not isinstance(encrypted_secrets, dict) or not encrypted_secrets:
+            logger.info('No encrypted secrets found in input')
+            return input_data
+
+        # 3. Decrypt all secret values
+        decrypted_secrets = decrypt_all_secrets(encrypted_secrets, private_key_pem)
+        logger.info('Decrypted %d secret field(s)', len(decrypted_secrets))
+
+        # 4. Merge decrypted values into input and remove the __encrypted_secrets key
+        merged = merge_decrypted_input(input_data, decrypted_secrets)
+        merged.pop('__encrypted_secrets', None)
+        merged.pop('__encrypted_secret_fields', None)
+
+        return merged
+    except Exception as err:  # noqa: BLE001 - secrets won't be available but actor can still run
+        logger.error('Failed to decrypt secret inputs: %s', err)
+        return input_data

scrapely/errors.py

@@ -0,0 +1,153 @@
+"""
+Error classes for the Scrapely Python SDK.
+
+This module provides a hierarchy of error classes for handling API errors,
+following the same pattern as the Apify SDK.
+
+Hierarchy:
+    ScrapelyClientError
+    ├── ScrapelyApiError
+    │   ├── InvalidRequestError (400)
+    │   ├── UnauthorizedError (401)
+    │   ├── ForbiddenError (403)
+    │   ├── NotFoundError (404)
+    │   ├── ConflictError (409)
+    │   ├── RateLimitError (429)
+    │   └── ServerError (5xx)
+    └── InvalidResponseBodyError
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Dict, Optional, Type
+
+if TYPE_CHECKING:
+    import httpx
+
+
+class ScrapelyClientError(Exception):
+    """Base class for all Scrapely API client errors."""
+
+    pass
+
+
+class InvalidResponseBodyError(ScrapelyClientError):
+    """Error raised when a response body cannot be parsed."""
+
+    def __init__(self, response: 'httpx.Response') -> None:
+        self.response = response
+        super().__init__(f'Invalid response body: {response.text[:200]}')
+
+
+class ScrapelyApiError(ScrapelyClientError):
+    """Error raised when the Scrapely API returns an error response."""
+
+    def __init__(
+        self,
+        response: 'httpx.Response',
+        attempt: int,
+        *,
+        method: str = 'GET',
+    ) -> None:
+        self.response = response
+        self.attempt = attempt
+        self.method = method
+        self.status_code = response.status_code
+
+        self.type: Optional[str] = None
+        self.message: Optional[str] = None
+        self.data: Optional[Dict[str, Any]] = None
+
+        try:
+            body = response.json()
+            self.type = body.get('type')
+            self.message = body.get('message')
+            self.data = body.get('data')
+        except Exception:
+            pass
+
+        error_parts = [f'HTTP {response.status_code}']
+        if self.type:
+            error_parts.append(f'({self.type})')
+        if self.message:
+            error_parts.append(f'- {self.message}')
+        error_parts.append(f'[attempt {attempt}]')
+
+        super().__init__(' '.join(error_parts))
+
+
+class InvalidRequestError(ScrapelyApiError):
+    """Raised when the Scrapely API returns an HTTP 400 Bad Request response."""
+    pass
+
+
+class UnauthorizedError(ScrapelyApiError):
+    """Raised when the Scrapely API returns an HTTP 401 Unauthorized response."""
+    pass
+
+
+class ForbiddenError(ScrapelyApiError):
+    """Raised when the Scrapely API returns an HTTP 403 Forbidden response."""
+    pass
+
+
+class NotFoundError(ScrapelyApiError):
+    """Raised when the Scrapely API returns an HTTP 404 Not Found response."""
+    pass
+
+
+class ConflictError(ScrapelyApiError):
+    """Raised when the Scrapely API returns an HTTP 409 Conflict response."""
+    pass
+
+
+class RateLimitError(ScrapelyApiError):
+    """Raised when the Scrapely API returns an HTTP 429 Too Many Requests response."""
+    pass
+
+
+class ServerError(ScrapelyApiError):
+    """Raised when the Scrapely API returns an HTTP 5xx response."""
+    pass
+
+
+_STATUS_CODE_TO_ERROR: Dict[int, Type[ScrapelyApiError]] = {
+    400: InvalidRequestError,
+    401: UnauthorizedError,
+    403: ForbiddenError,
+    404: NotFoundError,
+    409: ConflictError,
+    429: RateLimitError,
+}
+
+
+def raise_for_status(response: 'httpx.Response', attempt: int = 1, *, method: str = 'GET') -> None:
+    """Raise an appropriate error for the given HTTP response."""
+    if response.status_code < 400:
+        return
+
+    status_code = response.status_code
+    error_class = _STATUS_CODE_TO_ERROR.get(status_code)
+
+    if error_class is None and 500 <= status_code < 600:
+        error_class = ServerError
+
+    if error_class is None:
+        error_class = ScrapelyApiError
+
+    raise error_class(response, attempt, method=method)
+
+
+__all__ = [
+    'ScrapelyClientError',
+    'ScrapelyApiError',
+    'InvalidResponseBodyError',
+    'InvalidRequestError',
+    'UnauthorizedError',
+    'ForbiddenError',
+    'NotFoundError',
+    'ConflictError',
+    'RateLimitError',
+    'ServerError',
+    'raise_for_status',
+]

scrapely/events/__init__.py

@@ -0,0 +1,57 @@
+"""
+Event management for Scrapely Actors.
+
+Exports all event types and data models matching the Apify SDK.
+"""
+
+from crawlee.events import EventManager as CrawleeEventManager
+from crawlee.events import LocalEventManager
+from crawlee.events._types import (
+    EventAbortingData,
+    EventExitData,
+    EventMigratingData,
+    EventPersistStateData,
+    EventSystemInfoData,
+)
+
+from ._types import (
+    AbortingEvent,
+    ActorEventTypes,
+    DeprecatedEvent,
+    Event,
+    EventMessage,
+    EventWithoutData,
+    ExitEvent,
+    MigratingEvent,
+    PersistStateEvent,
+    SystemInfoEvent,
+    SystemInfoEventData,
+    UnknownEvent,
+)
+from .event_manager import EventManager, ScrapelyEventManager
+
+__all__ = [
+    'Event',
+    'EventManager',
+    'ScrapelyEventManager',
+    'CrawleeEventManager',
+    'LocalEventManager',
+    # Event data types
+    'SystemInfoEventData',
+    'EventAbortingData',
+    'EventExitData',
+    'EventMigratingData',
+    'EventPersistStateData',
+    'EventSystemInfoData',
+    # Event types
+    'PersistStateEvent',
+    'SystemInfoEvent',
+    'MigratingEvent',
+    'AbortingEvent',
+    'ExitEvent',
+    'EventWithoutData',
+    'DeprecatedEvent',
+    'UnknownEvent',
+    'EventMessage',
+    'ActorEventTypes',
+]

scrapely/events/_types.py

@@ -0,0 +1,194 @@
+"""Scrapely event types - extends Crawlee's event types with Scrapely-specific data models."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Annotated, Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic.alias_generators import to_camel
+
+from crawlee.events._types import (
+    Event,
+    EventAbortingData,
+    EventExitData,
+    EventMigratingData,
+    EventPersistStateData,
+    EventSystemInfoData,
+)
+
+# Re-export Event for convenience
+__all__ = [
+    'Event',
+    'SystemInfoEventData',
+    'PersistStateEvent',
+    'SystemInfoEvent',
+    'MigratingEvent',
+    'AbortingEvent',
+    'ExitEvent',
+    'EventWithoutData',
+    'DeprecatedEvent',
+    'UnknownEvent',
+    'EventMessage',
+    'ActorEventTypes',
+]
+
+ActorEventTypes = Literal['systemInfo', 'persistState', 'migrating', 'aborting']
+"""Event types emitted by the Scrapely platform during an Actor run.
+
+This is the Scrapely-specific subset of `Event`. For the full set (including framework-level events
+like `SESSION_RETIRED` or `BROWSER_LAUNCHED`), use `Event` from `scrapely`.
+"""
+
+
+class SystemInfoEventData(BaseModel):
+    """Resource usage metrics carried by a `systemInfo` event."""
+
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
+    mem_avg_bytes: float
+    """Average memory usage over the measured interval, in bytes."""
+
+    mem_current_bytes: float
+    """Current memory usage, in bytes."""
+
+    mem_max_bytes: float
+    """Peak memory usage observed so far, in bytes."""
+
+    cpu_avg_usage: float
+    """Average CPU usage over the measured interval, in percent."""
+
+    cpu_max_usage: float
+    """Peak CPU usage observed so far, in percent."""
+
+    cpu_current_usage: float
+    """Current CPU usage, in percent."""
+
+    is_cpu_overloaded: bool
+    """Whether the CPU is currently overloaded."""
+
+    created_at: datetime
+    """Timestamp when the metrics were collected."""
+
+    def to_crawlee_format(self, dedicated_cpus: float) -> EventSystemInfoData:
+        """Convert to Crawlee's EventSystemInfoData format."""
+        return EventSystemInfoData.model_validate(
+            {
+                'cpu_info': {
+                    'used_ratio': (self.cpu_current_usage / 100) / dedicated_cpus,
+                    'created_at': self.created_at,
+                },
+                'memory_info': {
+                    'total_size': self.mem_max_bytes,
+                    'current_size': self.mem_current_bytes,
+                    'created_at': self.created_at,
+                },
+            }
+        )
+
+
+class PersistStateEvent(BaseModel):
+    """A `persistState` event instructing the Actor to persist its state."""
+
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
+    name: Literal[Event.PERSIST_STATE]
+    """The event name."""
+
+    data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))]
+    """The event payload."""
+
+
+class SystemInfoEvent(BaseModel):
+    """A `systemInfo` event carrying the Actor's resource usage metrics."""
+
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
+    name: Literal[Event.SYSTEM_INFO]
+    """The event name."""
+
+    data: SystemInfoEventData
+    """The event payload."""
+
+
+class MigratingEvent(BaseModel):
+    """A `migrating` event signalling the Actor is about to be migrated to another host."""
+
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
+    name: Literal[Event.MIGRATING]
+    """The event name."""
+
+    data: Annotated[EventMigratingData, Field(default_factory=EventMigratingData)]
+    """The event payload."""
+
+
+class AbortingEvent(BaseModel):
+    """An `aborting` event signalling the Actor run is being aborted."""
+
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
+    name: Literal[Event.ABORTING]
+    """The event name."""
+
+    data: Annotated[EventAbortingData, Field(default_factory=EventAbortingData)]
+    """The event payload."""
+
+
+class ExitEvent(BaseModel):
+    """An `exit` event signalling the Actor process is about to exit."""
+
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
+    name: Literal[Event.EXIT]
+    """The event name."""
+
+    data: Annotated[EventExitData, Field(default_factory=EventExitData)]
+    """The event payload."""
+
+
+class EventWithoutData(BaseModel):
+    """A framework-level event that carries no payload (e.g. browser and page lifecycle events)."""
+
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
+    name: Literal[
+        Event.SESSION_RETIRED,
+        Event.BROWSER_LAUNCHED,
+        Event.BROWSER_RETIRED,
+        Event.BROWSER_CLOSED,
+        Event.PAGE_CREATED,
+        Event.PAGE_CLOSED,
+    ]
+    """The event name."""
+
+    data: Any = None
+    """The event payload, always empty for this event."""
+
+
+class DeprecatedEvent(BaseModel):
+    """A deprecated event kept for backward compatibility (e.g. `cpuInfo`)."""
+
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
+    name: Literal['cpuInfo']
+    """The event name."""
+
+    data: Annotated[dict[str, Any], Field(default_factory=dict)]
+    """The event payload."""
+
+
+class UnknownEvent(BaseModel):
+    """A fallback for any event whose name is not recognized by the SDK."""
+
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
+    name: str
+    """The event name."""
+
+    data: Annotated[dict[str, Any], Field(default_factory=dict)]
+    """The event payload."""
+
+
+EventMessage = PersistStateEvent | SystemInfoEvent | MigratingEvent | AbortingEvent | ExitEvent | EventWithoutData
+"""Union of all recognized event message types."""