scrapely-python 0.1.31__tar.gz → 0.1.33__tar.gz

scrapely_python-0.1.33/CHANGELOG.md

@@ -0,0 +1,24 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## [0.1.33] - 2026-10-22
+
+- Debug captures (`Actor.debug_capture`, `Actor.debug_capture_sync`) keep payloads over 256 KiB whole. They are stored gzip-compressed and base64-encoded as `{key, payload, encoding: 'gzip-base64', length, truncated}` instead of being cut at 262,144 characters; up to 8 Mi characters are kept. A payload that does not compress enough keeps its first 256 KiB, now marked `truncated: True` with its full `length`. Dict and list payloads are dumped as compact JSON, and `bytes` payloads are stored as UTF-8 text (they used to be stored as their `repr`). Captures are only active on the platform's own diagnostic runs (`SCRAPELY_DEBUG_CAPTURE=1`).
+
+## [0.1.32] - 2026-10-20
+
+- Added `scrapely.scrapy`, the Scrapy integration: `run_scrapy_actor`, `initialize_logging`, `apply_apify_settings`, a scheduler backed by the run's request queue, a pipeline that pushes items to the default dataset, a proxy middleware, an HTTP cache storage and an extension that stops the crawl gracefully when the run is aborted. Queued Scrapy requests and cached responses are stored as JSON, never pickled, and a stored request only rebuilds as a `scrapy.Request` subclass. A request leaves the queue as handled only once Scrapy has finished it, so a migrated or aborted run does not lose or repeat it. Scrapy is imported only when these are used.
+
+## [0.1.31] - 2026-10-20
+
+- The storage client percent-encodes request ids and key-value store record keys as one URL path segment and refuses `''`, `.` and `..`. A request id such as `..` read from a queue used to send the call to another resource.
+
+## [0.1.30] - 2026-10-20
+
+- Removed `apify_margin_percentage` from the pricing info models; the API no longer returns it.
+- Added Apache-2.0 change notices to the storage clients, storage utils and log module derived from apify-sdk-python, and listed them in NOTICE.
+
+## [0.1.29] - 2026-10-19
+
+- Initial public release.

{scrapely_python-0.1.31 → scrapely_python-0.1.33}/NOTICE

@@ -10,6 +10,7 @@ Version 2.0 (see LICENSE-APACHE). They have been modified by TeraShift GmbH:
   - src/scrapely/events/event_manager.py
   - src/scrapely/log.py
   - src/scrapely/request_loaders/_scrapely_request_list.py
+  - src/scrapely/scrapy/ (all files)
   - src/scrapely/storage/_file_system/_key_value_store_client.py
   - src/scrapely/storage/_file_system/_storage_client.py
   - src/scrapely/storage/smart_scrapely_storage_client.py

{scrapely_python-0.1.31 → scrapely_python-0.1.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.31
+Version: 0.1.33
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.31 → scrapely_python-0.1.33}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.31"
+version = "0.1.33"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.31 → scrapely_python-0.1.33}/src/scrapely/__init__.py

@@ -227,7 +227,7 @@ if TYPE_CHECKING:  # Static analysers and IDEs see the lazy names as plain impor
         set_api_storage,
     )
 
-__version__ = '0.1.31'
+__version__ = '0.1.33'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.31 → scrapely_python-0.1.33}/src/scrapely/actor.py

@@ -925,7 +925,14 @@ class Actor(metaclass=_ActorMeta):
 
     # Debug capture: raw source payloads for platform diagnostics. Only active when the platform
     # sets SCRAPELY_DEBUG_CAPTURE=1 on its own diagnostic runs; a no-op in every other run.
+    # A payload up to _DEBUG_CAPTURE_MAX_BYTES characters is stored as it is; a larger one gzip-compressed
+    # (base64, at most _DEBUG_CAPTURE_MAX_BYTES characters again), keeping up to _DEBUG_CAPTURE_MAX_CHARS
+    # characters / _DEBUG_CAPTURE_MAX_ENCODED bytes of it. A plain cut at 256 KiB lost the part a check
+    # needed when a large API document starts with site navigation (the seller section of a 581 KB
+    # listing JSON sat at ~333 KB).
     _DEBUG_CAPTURE_MAX_BYTES = 256 * 1024
+    _DEBUG_CAPTURE_MAX_CHARS = 8 * 1024 * 1024
+    _DEBUG_CAPTURE_MAX_ENCODED = 8 * 1024 * 1024
     _DEBUG_CAPTURE_MAX_RECORDS = 500
     _debug_capture_count = 0
 
@@ -967,15 +974,33 @@ class Actor(metaclass=_ActorMeta):
         if cls._debug_capture_count >= cls._DEBUG_CAPTURE_MAX_RECORDS:
             return
         try:
-            text = payload if isinstance(payload, str) else json.dumps(payload, default=str, ensure_ascii=False)
-            if isinstance(text, bytes):
-                text = text.decode('utf-8', 'replace')
-            text = text[: cls._DEBUG_CAPTURE_MAX_BYTES]
+            if isinstance(payload, (bytes, bytearray)):
+                text = bytes(payload).decode('utf-8', 'replace')
+            elif isinstance(payload, str):
+                text = payload
+            else:
+                text = json.dumps(payload, default=str, ensure_ascii=False, separators=(',', ':'))
+            record = cls._debug_capture_record(str(key), text)
             cls._debug_capture_count += 1
-            await cls.set_value(cls.debug_capture_record_key(key), {'key': str(key), 'payload': text})
+            await cls.set_value(cls.debug_capture_record_key(key), record)
         except Exception as err:  # never break the run
             logging.getLogger('scrapely').debug('debug_capture failed: %s', err)
 
+    @classmethod
+    def _debug_capture_record(cls, key: str, text: str) -> Dict[str, Any]:
+        """The record for one capture: plain when small, else gzip-base64 (plain head as the last resort)."""
+        limit = cls._DEBUG_CAPTURE_MAX_BYTES
+        if len(text) <= limit:
+            return {'key': key, 'payload': text}
+        import gzip
+        raw = text[: cls._DEBUG_CAPTURE_MAX_CHARS].encode('utf-8')
+        kept = raw[: cls._DEBUG_CAPTURE_MAX_ENCODED]
+        packed = base64.b64encode(gzip.compress(kept, 6)).decode('ascii')
+        if len(packed) <= limit:
+            truncated = len(text) > cls._DEBUG_CAPTURE_MAX_CHARS or len(kept) < len(raw)
+            return {'key': key, 'payload': packed, 'encoding': 'gzip-base64', 'length': len(text), 'truncated': truncated}
+        return {'key': key, 'payload': text[:limit], 'length': len(text), 'truncated': True}
+
     @classmethod
     def _resolve_open_args(
         cls,

scrapely_python-0.1.33/src/scrapely/scrapy/__init__.py

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: Apache-2.0
+# Derived from apify-sdk-python (src/apify/scrapy/__init__.py),
+# Copyright Apify Technologies s.r.o., licensed under the Apache License 2.0.
+# Modified by TeraShift GmbH. See LICENSE-APACHE and NOTICE.
+from crawlee._utils.try_import import install_import_hook as _install_import_hook
+from crawlee._utils.try_import import try_import as _try_import
+
+_install_import_hook(__name__)
+
+# The following imports use try_import to handle optional dependencies, as they may not always be available.
+
+with _try_import(__name__, 'run_scrapy_actor'):
+    from ._actor_runner import run_scrapy_actor
+
+with _try_import(__name__, 'initialize_logging'):
+    from ._logging_config import initialize_logging
+
+with _try_import(__name__, 'to_apify_request', 'to_scrapy_request'):
+    from .requests import to_apify_request, to_scrapy_request
+
+with _try_import(__name__, 'ApifyScheduler'):
+    from .scheduler import ApifyScheduler
+
+with _try_import(__name__, 'apply_apify_settings', 'get_basic_auth_header'):
+    from .utils import apply_apify_settings, get_basic_auth_header
+
+
+__all__ = [
+    'ApifyScheduler',
+    'apply_apify_settings',
+    'get_basic_auth_header',
+    'initialize_logging',
+    'run_scrapy_actor',
+    'to_apify_request',
+    'to_scrapy_request',
+]

scrapely_python-0.1.33/src/scrapely/scrapy/_actor_runner.py

@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: Apache-2.0
+# Derived from apify-sdk-python (src/apify/scrapy/_actor_runner.py),
+# Copyright Apify Technologies s.r.o., licensed under the Apache License 2.0.
+# Modified by TeraShift GmbH. See LICENSE-APACHE and NOTICE.
+from __future__ import annotations
+
+import asyncio
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Coroutine
+
+
+def run_scrapy_actor(coro: Coroutine) -> None:
+    """Start Twisted's reactor and execute the provided Actor coroutine.
+
+    This function installs Twisted's asyncio-compatible reactor, then initiates it and runs the given asyncio
+    coroutine (typically the Actor's main) by converting it to a Deferred. This bridges the asyncio and Twisted
+    event loops, enabling the Scrapely and Scrapy integration to work together.
+    """
+    from scrapy.utils.reactor import install_reactor  # noqa: PLC0415
+    from twisted.internet.error import ReactorAlreadyInstalledError  # noqa: PLC0415
+
+    try:
+        install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
+    except ReactorAlreadyInstalledError:
+        from twisted.internet import reactor as installed_reactor  # noqa: PLC0415
+
+        reactor_cls = type(installed_reactor)
+        if (
+            reactor_cls.__module__ != 'twisted.internet.asyncioreactor'
+            or reactor_cls.__name__ != 'AsyncioSelectorReactor'
+        ):
+            raise RuntimeError(
+                'A Twisted reactor is already installed and it is not AsyncioSelectorReactor. '
+                'Make sure that run_scrapy_actor() is called before importing any Scrapy or Twisted '
+                'modules that install a reactor (e.g. scrapy.crawler).'
+            ) from None
+
+    from twisted.internet.defer import Deferred  # noqa: PLC0415
+    from twisted.internet.task import react  # noqa: PLC0415
+
+    react(lambda _reactor: Deferred.fromFuture(asyncio.ensure_future(coro)))

scrapely_python-0.1.33/src/scrapely/scrapy/_async_thread.py

@@ -0,0 +1,209 @@
+# SPDX-License-Identifier: Apache-2.0
+# Derived from apify-sdk-python (src/apify/scrapy/_async_thread.py),
+# Copyright Apify Technologies s.r.o., licensed under the Apache License 2.0.
+# Modified by TeraShift GmbH. See LICENSE-APACHE and NOTICE.
+from __future__ import annotations
+
+import asyncio
+import threading
+from concurrent import futures
+from datetime import timedelta
+from logging import getLogger
+from typing import TYPE_CHECKING, Any, Literal
+
+if TYPE_CHECKING:
+    from collections.abc import Coroutine
+
+logger = getLogger(__name__)
+
+SUBMITTED_PRUNE_THRESHOLD = 128
+"""How many `submit_coro` futures may pile up before the finished ones are dropped from the tracking list."""
+
+
+class AsyncThread:
+    """Run an asyncio event loop in a dedicated background thread.
+
+    This lets synchronous Scrapy callbacks drive asynchronous Scrapely and Crawlee coroutines. The
+    scheduler and the HTTP cache storage each own their own `AsyncThread`, so the request queue and
+    the key-value store never share an event loop; they only share the read-only global
+    `Configuration`. A single shared loop would also work but would couple their lifecycles.
+    """
+
+    def __init__(self, default_timeout: timedelta = timedelta(seconds=60)) -> None:
+        self._default_timeout = default_timeout
+        self._eventloop = asyncio.new_event_loop()
+
+        self._submitted: list[futures.Future] = []
+        """Futures of the coroutines submitted via `submit_coro` that may still be running."""
+
+        # Start the event loop in a dedicated daemon thread.
+        self._thread = threading.Thread(
+            target=self._start_event_loop,
+            daemon=True,
+        )
+        self._thread.start()
+
+    def run_coro(
+        self,
+        coro: Coroutine,
+        timeout: timedelta | Literal['default'] = 'default',
+    ) -> Any:
+        """Run a coroutine on an event loop running in a separate thread.
+
+        This method schedules the coroutine to run on the event loop and blocks until the coroutine completes
+        or the specified timeout is reached.
+
+        Args:
+            coro: The coroutine to run.
+            timeout: The maximum time to wait for the coroutine to finish. Pass `'default'` to use the
+                `default_timeout` passed to the constructor.
+
+        Returns:
+            The result returned by the coroutine.
+
+        Raises:
+            RuntimeError: If the event loop has been closed.
+            TimeoutError: If the coroutine does not complete within the timeout.
+            Exception: Any exception raised during coroutine execution.
+        """
+        if timeout == 'default':
+            timeout = self._default_timeout
+
+        if self._eventloop.is_closed():
+            raise RuntimeError(f'The coroutine {coro} cannot be executed because the event loop is closed.')
+
+        # Submit the coroutine to the event loop running in the other thread.
+        future = asyncio.run_coroutine_threadsafe(coro, self._eventloop)
+        try:
+            # Wait for the coroutine's result until the specified timeout.
+            return future.result(timeout=timeout.total_seconds())
+        except futures.TimeoutError:
+            # `future.result` gave up, but the coroutine keeps running on the loop; cancel it so it does
+            # not outlive the timeout. The propagated error is logged once by the caller (or Scrapy), so
+            # this method does not log it itself.
+            future.cancel()
+            raise
+
+    def submit_coro(self, coro: Coroutine) -> futures.Future:
+        """Schedule a coroutine on the event loop without waiting for its result.
+
+        Use this for work nothing depends on, so the calling thread is not blocked by the round trip. Failures
+        are logged, as there is no caller to propagate them to, and `close` cancels whatever is still pending -
+        call `wait_for_submitted` first if that matters.
+
+        Args:
+            coro: The coroutine to run.
+
+        Returns:
+            The future of the scheduled coroutine, for callers that want to inspect its outcome later.
+
+        Raises:
+            RuntimeError: If the event loop has been closed.
+        """
+        if self._eventloop.is_closed():
+            raise RuntimeError(f'The coroutine {coro} cannot be executed because the event loop is closed.')
+
+        # Callers may go a long time between `wait_for_submitted` calls, so without pruning the list would hold
+        # every coroutine ever submitted, with its result.
+        if len(self._submitted) >= SUBMITTED_PRUNE_THRESHOLD:
+            self._submitted = [submitted for submitted in self._submitted if not submitted.done()]
+
+        future = asyncio.run_coroutine_threadsafe(coro, self._eventloop)
+        future.add_done_callback(self._log_failure)
+        self._submitted.append(future)
+
+        return future
+
+    def wait_for_submitted(self, timeout: timedelta | None = None) -> None:
+        """Block until the coroutines submitted via `submit_coro` have finished.
+
+        Use this before anything that would observe their effects, or before `close`, which cancels whatever is
+        still running. Coroutines that do not finish within the timeout stay tracked for the next call.
+
+        Args:
+            timeout: The maximum time to wait for the submitted coroutines. Pass `None` to use the
+                `default_timeout` passed to the constructor.
+        """
+        if timeout is None:
+            timeout = self._default_timeout
+
+        self._submitted = list(futures.wait(self._submitted, timeout=timeout.total_seconds()).not_done)
+
+        # Callers rely on the effects having landed, so a timeout has to be visible.
+        if self._submitted:
+            logger.warning(f'{len(self._submitted)} submitted coroutines did not finish within the timeout.')
+
+    def close(self, timeout: timedelta | None = None) -> None:
+        """Close the event loop and its thread gracefully.
+
+        This method cancels all pending tasks, stops the event loop, and waits for the thread to exit.
+        If the thread does not exit within the given timeout, a forced shutdown is attempted.
+
+        Args:
+            timeout: The maximum time to wait for the event loop thread to exit. Pass `None` to use the
+                `default_timeout` passed to the constructor.
+        """
+        if timeout is None:
+            timeout = self._default_timeout
+
+        # A repeated close (e.g. a retried shutdown) would call into the already-closed loop and raise
+        # `RuntimeError: Event loop is closed`. The loop closes itself once it stops, so a second close
+        # is a no-op.
+        if self._eventloop.is_closed():
+            return
+
+        try:
+            if self._eventloop.is_running():
+                # Cancel all pending tasks in the event loop, honouring the caller's timeout.
+                self.run_coro(self._shutdown_tasks(), timeout=timeout)
+        finally:
+            # Stop the loop and join its thread even if cancelling the pending tasks above raised or timed
+            # out. Skipping this would leave the loop running and leak its thread.
+            self._eventloop.call_soon_threadsafe(self._eventloop.stop)
+
+            # Wait for the event loop thread to finish execution.
+            self._thread.join(timeout=timeout.total_seconds())
+
+            # If the thread is still running after the timeout, force a shutdown.
+            if self._thread.is_alive():
+                logger.warning('Event loop thread did not exit cleanly! Forcing shutdown...')
+                self._force_exit_event_loop()
+
+    @staticmethod
+    def _log_failure(future: futures.Future) -> None:
+        """Log the failure of a coroutine submitted via `submit_coro`."""
+        if future.cancelled():
+            return
+
+        if (exc := future.exception()) is not None:
+            logger.error('A coroutine submitted to the event loop failed.', exc_info=exc)
+
+    def _start_event_loop(self) -> None:
+        """Set up and run the asyncio event loop in the dedicated thread."""
+        asyncio.set_event_loop(self._eventloop)
+        try:
+            self._eventloop.run_forever()
+        finally:
+            self._eventloop.close()
+            logger.debug('Asyncio event loop has been closed.')
+
+    async def _shutdown_tasks(self) -> None:
+        """Cancel all pending tasks in the event loop."""
+        # Retrieve all tasks for the event loop, excluding the current task.
+        tasks = [task for task in asyncio.all_tasks(self._eventloop) if task is not asyncio.current_task()]
+
+        # Cancel each pending task.
+        for task in tasks:
+            task.cancel()
+
+        # Wait until all tasks have been cancelled or finished.
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+    def _force_exit_event_loop(self) -> None:
+        """Shut down the event loop and its thread forcefully."""
+        try:
+            logger.info('Forced shutdown of the event loop and its thread...')
+            self._eventloop.call_soon_threadsafe(self._eventloop.stop)
+            self._thread.join(timeout=5)
+        except Exception:
+            logger.exception('Exception occurred during forced event loop shutdown.')

scrapely_python-0.1.33/src/scrapely/scrapy/_logging_config.py

@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: Apache-2.0
+# Derived from apify-sdk-python (src/apify/scrapy/_logging_config.py),
+# Copyright Apify Technologies s.r.o., licensed under the Apache License 2.0.
+# Modified by TeraShift GmbH. See LICENSE-APACHE and NOTICE.
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from scrapy.utils import log as scrapy_logging
+from scrapy.utils.project import get_project_settings
+
+from scrapely.log import ActorLogFormatter
+
+# Define logger names.
+_PRIMARY_LOGGERS = ['scrapely', 'scrapely_client', 'scrapy']
+_SUPPLEMENTAL_LOGGERS = ['filelock', 'hpack', 'httpcore', 'protego', 'twisted']
+_ALL_LOGGERS = _PRIMARY_LOGGERS + _SUPPLEMENTAL_LOGGERS
+
+# Mutable state shared with the Scrapy monkey-patch below. `initialize_logging` refreshes
+# `level`/`handler` on each call; the patch (installed once) reads them so it always applies the
+# latest configuration rather than values captured the first time it ran.
+_state: dict[str, Any] = {'level': 'INFO', 'handler': None, 'patched': False}
+
+
+def _configure_logger(name: str | None, logging_level: str, handler: logging.Handler) -> None:
+    """Clear and reconfigure the logger."""
+    logger = logging.getLogger(name)
+    logger.handlers.clear()
+    logger.setLevel(logging_level)
+    logger.addHandler(handler)
+    logger.propagate = False
+
+
+def _configure_all_loggers() -> None:
+    """Apply the Scrapely handler and level to the root logger and all defined loggers."""
+    handler = _state['handler']
+    if handler is None:
+        return
+    for logger_name in [None, *_ALL_LOGGERS]:
+        _configure_logger(logger_name, _state['level'], handler)
+
+
+def initialize_logging() -> None:
+    """Configure logging for Scrapely Actors and adjust Scrapy's logging settings."""
+    # Retrieve Scrapy project settings and determine the logging level.
+    settings = get_project_settings()
+    _state['level'] = settings.get('LOG_LEVEL', 'INFO')  # Default to INFO.
+
+    # Create a custom handler with the Scrapely log formatter.
+    handler = logging.StreamHandler()
+    handler.setFormatter(ActorLogFormatter(include_logger_name=True))
+    _state['handler'] = handler
+
+    # Configure the root logger and all other defined loggers.
+    _configure_all_loggers()
+
+    # Monkey-patch Scrapy's logging to re-apply our settings whenever it reconfigures logging.
+    # Install the wrapper at most once, otherwise repeated calls would nest wrappers.
+    if _state['patched']:
+        return
+
+    original_configure_logging = scrapy_logging.configure_logging
+
+    def new_configure_logging(*args: Any, **kwargs: Any) -> None:
+        original_configure_logging(*args, **kwargs)
+        _configure_all_loggers()
+
+    scrapy_logging.configure_logging = new_configure_logging  # ty: ignore[invalid-assignment]
+    _state['patched'] = True

scrapely_python-0.1.33/src/scrapely/scrapy/_serialization.py

@@ -0,0 +1,147 @@
+# SPDX-License-Identifier: Apache-2.0
+# Derived from apify-sdk-python (src/apify/scrapy/_serialization.py),
+# Copyright Apify Technologies s.r.o., licensed under the Apache License 2.0.
+# Modified by TeraShift GmbH. See LICENSE-APACHE and NOTICE.
+"""JSON serialization of Scrapy requests and cached responses for storage on the Scrapely platform.
+
+Scrapy requests and cached responses are stored in the request queue and key-value store, which hold JSON.
+They are therefore serialized as JSON here (never pickled: unpickling stored data could run arbitrary code).
+
+Only `body` (`bytes`) and `headers` (`{bytes: [bytes]}`) are not natively JSON-serializable. Both live at fixed keys
+and are base64-encoded in place. A `str` `body` is encoded as UTF-8 bytes and deserialized as `bytes`, matching
+Scrapy, which always stores `body` as `bytes`.
+
+Pydantic models, such as Crawlee's `UserData`, are dumped with `model_dump(mode='json')`, which converts
+non-JSON-native fields into JSON-compatible values. For example, `datetime` fields are stored as ISO-8601 strings.
+
+All other values, notably `meta` and `cb_kwargs`, must already be JSON-serializable. Non-JSON-serializable values,
+such as `datetime`, `set`, `Decimal`, or arbitrary objects, fail serialization with a clear error naming the
+offending value. The request is skipped rather than stored in a corrupted form.
+
+No in-band sentinel is used, so user values cannot collide with the encoding.
+
+Known JSON limitations: values in `meta` and `cb_kwargs` are subject to JSON coercions. A `tuple` round-trips as
+a `list`, and non-string `dict` keys round-trip as strings; for example, `{1: 'a'}` becomes `{'1': 'a'}`.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+from typing import Any
+
+from pydantic import BaseModel
+
+# Cap the offending value's repr in a serialization error message so a huge value cannot bloat the log.
+_MAX_ERROR_VALUE_REPR_LEN = 200
+
+
+def encode_to_json(data: dict[str, Any]) -> str:
+    """Serialize a Scrapy request/response dict to a JSON string.
+
+    The `body` and `headers` fields are base64-encoded in place (a `str` `body` via its UTF-8 bytes); pydantic
+    models are dumped to plain dicts. A `TypeError` is raised if any other value cannot be JSON-encoded.
+
+    Args:
+        data: The dict to serialize, e.g. the output of `scrapy.Request.to_dict()`.
+
+    Returns:
+        The JSON-encoded string.
+    """
+    if not isinstance(data, dict):
+        raise TypeError(f'Expected a dict to serialize, got {type(data)}')
+
+    safe = dict(data)
+
+    # `body` is base64-encoded so binary payloads survive; a `str` body is taken as its UTF-8 bytes, which keeps
+    # encode/decode symmetric (decode always base64-decodes `body` back to `bytes`).
+    body = safe.get('body')
+    if isinstance(body, (bytes, str)):
+        raw_body = body.encode('utf-8') if isinstance(body, str) else body
+        safe['body'] = base64.b64encode(raw_body).decode('ascii')
+
+    if isinstance(safe.get('headers'), dict):
+        safe['headers'] = _encode_headers(safe['headers'])
+
+    try:
+        # `ensure_ascii=False` keeps non-ASCII URLs/meta as their UTF-8 form instead of `\uXXXX` escapes, which
+        # would otherwise roughly double the size of non-Latin text in storage.
+        return json.dumps(safe, default=_json_default, ensure_ascii=False)
+    # `ValueError` covers pydantic's `PydanticSerializationError`, raised when a model field cannot be dumped
+    # to JSON even in JSON mode.
+    except (TypeError, ValueError) as exc:
+        raise TypeError(
+            'Failed to JSON-serialize a Scrapy request/response for storage on the Scrapely platform. '
+            'All values in `meta` and `cb_kwargs` must be JSON-serializable (str, int, float, bool, None, '
+            'list, dict, or a pydantic model).'
+        ) from exc
+
+
+def decode_from_json(text: str) -> Any:
+    """Reconstruct a Scrapy request/response dict from a string produced by `encode_to_json`.
+
+    The base64-encoded `body` and `headers` fields are decoded back to their `bytes` representation.
+
+    Args:
+        text: The JSON-encoded string.
+
+    Returns:
+        The decoded object (a dict for valid request/response payloads).
+    """
+    data = json.loads(text)
+    if not isinstance(data, dict):
+        return data
+
+    # `validate=True` makes a non-base64 body raise loudly instead of silently decoding to garbage.
+    if isinstance(data.get('body'), str):
+        data['body'] = base64.b64decode(data['body'], validate=True)
+
+    if isinstance(data.get('headers'), dict):
+        data['headers'] = _decode_headers(data['headers'])
+
+    return data
+
+
+def _json_default(obj: Any) -> Any:
+    """Fallback for values `json.dumps` cannot serialize: pydantic models are dumped, anything else raises.
+
+    The error names the offending value (type and a truncated repr) so a failed serialization points straight
+    at the bad `meta`/`cb_kwargs` entry instead of just reporting that something failed.
+    """
+    if isinstance(obj, BaseModel):
+        return obj.model_dump(mode='json', by_alias=True)
+    value_repr = repr(obj)
+    if len(value_repr) > _MAX_ERROR_VALUE_REPR_LEN:
+        value_repr = value_repr[:_MAX_ERROR_VALUE_REPR_LEN] + '...'
+    raise TypeError(f'Object of type {type(obj).__name__} is not JSON-serializable: {value_repr}')
+
+
+def _encode_headers(headers: dict[Any, Any]) -> dict[str, list[str]]:
+    """Encode a Scrapy `{bytes: [bytes]}` headers mapping to a JSON-safe `{str: [base64-str]}`."""
+    encoded: dict[str, list[str]] = {}
+    for key, value in headers.items():
+        str_key = key.decode('latin-1') if isinstance(key, bytes) else key
+        values = value if isinstance(value, (list, tuple)) else [value]
+        encoded[str_key] = [_b64encode_value(item) for item in values]
+    return encoded
+
+
+def _decode_headers(headers: dict[str, Any]) -> dict[bytes, list[bytes]]:
+    """Reverse `_encode_headers`, restoring the `{bytes: [bytes]}` mapping Scrapy expects."""
+    decoded: dict[bytes, list[bytes]] = {}
+    for key, value in headers.items():
+        bytes_key = key.encode('latin-1') if isinstance(key, str) else key
+        values = value if isinstance(value, list) else [value]
+        decoded[bytes_key] = [base64.b64decode(item, validate=True) for item in values]
+    return decoded
+
+
+def _b64encode_value(value: Any) -> str:
+    """Base64-encode a single header value.
+
+    Scrapy stores header values as `bytes`; a `str` is encoded as its UTF-8 bytes. Any other type is coerced with
+    `str()` as a lenient last resort. That coercion is lossy (e.g. `5` becomes `b'5'`), but Scrapy does not produce
+    non-`bytes`/`str` header values, so it is not hit on the real path.
+    """
+    raw = value if isinstance(value, bytes) else str(value).encode('utf-8')
+    return base64.b64encode(raw).decode('ascii')

scrapely_python-0.1.33/src/scrapely/scrapy/_warnings.py

@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: Apache-2.0
+# Derived from apify-sdk-python (src/apify/scrapy/_warnings.py),
+# Copyright Apify Technologies s.r.o., licensed under the Apache License 2.0.
+# Modified by TeraShift GmbH. See LICENSE-APACHE and NOTICE.
+from __future__ import annotations
+
+import logging
+
+from crawlee._utils.log import LoggerOnce
+
+logger_once = LoggerOnce(logging.getLogger(__name__))
+"""Process-wide deduplication: the scheduler and the graceful-stop extension share the one message below."""
+
+
+def warn_about_uninitialized_actor() -> None:
+    """Warn, once per process, that the integration cannot react to the Actor run being migrated or aborted.
+
+    The scheduler and the graceful-stop extension both need an initialized Actor to listen for its events, and both
+    run without one in a plain `scrapy crawl`. One message covers them both.
+    """
+    logger_once.log(
+        'The Actor is not initialized, so the Scrapy integration cannot react to a migration or an abort of the '
+        'Actor run; the requests Scrapy is working on when the run is interrupted stay pending in the request queue.',
+        key='uninitialized-actor',
+        level=logging.WARNING,
+    )

scrapely_python-0.1.33/src/scrapely/scrapy/extensions/__init__.py

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: Apache-2.0
+# Derived from apify-sdk-python (src/apify/scrapy/extensions/__init__.py),
+# Copyright Apify Technologies s.r.o., licensed under the Apache License 2.0.
+# Modified by TeraShift GmbH. See LICENSE-APACHE and NOTICE.
+from scrapely.scrapy.extensions._graceful_stop import ApifyGracefulStopExtension
+from scrapely.scrapy.extensions._httpcache import ApifyCacheStorage
+
+__all__ = ['ApifyCacheStorage', 'ApifyGracefulStopExtension']