scrapely-client-python 3.1.3__py3-none-any.whl

scrapely_client/_statistics.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass, field
+
+
+@dataclass
+class ClientStatistics:
+    """Statistics about API client usage and rate limit errors."""
+
+    calls: int = 0
+    """Total number of API method calls made by the client."""
+
+    requests: int = 0
+    """Total number of HTTP requests sent, including retries."""
+
+    rate_limit_errors: defaultdict[int, int] = field(default_factory=lambda: defaultdict(int))
+    """List tracking which retry attempts encountered rate limit (429) errors."""
+
+    def add_rate_limit_error(self, attempt: int) -> None:
+        """Add rate limit error for specific attempt.
+
+        Args:
+            attempt: The attempt number (1-based indexing).
+        """
+        if attempt < 1:
+            raise ValueError('Attempt must be greater than 0')
+
+        self.rate_limit_errors[attempt - 1] += 1

scrapely_client/_status_message_watcher.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+import asyncio
+import threading
+import time
+from asyncio import Task
+from datetime import timedelta
+from threading import Thread
+from typing import TYPE_CHECKING, Self
+
+from scrapely_client._docs import docs_group
+from scrapely_client._utils.time import to_seconds
+
+if TYPE_CHECKING:
+    import logging
+    from types import TracebackType
+
+    from scrapely_client._models import Run
+    from scrapely_client._resource_clients import RunClient, RunClientAsync
+
+
+class StatusMessageWatcherBase:
+    """Base class for polling and logging Actor run status messages."""
+
+    _force_propagate = False
+    # This is final sleep time to try to get the last status and status message of finished Actor run.
+    # The status and status message can get set on the Actor run with a delay. Sleep time does not guarantee that the
+    # final message will be captured, but increases the chances of that.
+    _final_sleep_time_s = 6
+
+    def __init__(self, *, to_logger: logging.Logger, check_period: timedelta = timedelta(seconds=5)) -> None:
+        if self._force_propagate:
+            to_logger.propagate = True
+        self._to_logger = to_logger
+        self._check_period = to_seconds(check_period)
+        self._last_status_message = ''
+
+    def _log_run_data(self, run_data: Run | None) -> bool:
+        """Get relevant run data, log them if changed and return `True` if more data is expected.
+
+        Args:
+            run_data: The Run model that contains the run data.
+
+        Returns:
+              `True` if more data is expected, `False` otherwise.
+        """
+        if run_data is not None:
+            status = run_data.status or 'Unknown status'
+            status_message = run_data.status_message or ''
+            new_status_message = f'Status: {status}, Message: {status_message}'
+
+            if new_status_message != self._last_status_message:
+                self._last_status_message = new_status_message
+                self._to_logger.info(new_status_message)
+
+            return not (run_data.is_status_message_terminal or False)
+        return True
+
+
+@docs_group('Other')
+class StatusMessageWatcherAsync(StatusMessageWatcherBase):
+    """Polls and logs Actor run status messages in an asyncio task.
+
+    The status message and status of the Actor run are polled at a fixed interval and forwarded to the provided logger
+    whenever they change. There is no guarantee that every intermediate status message will be captured, especially
+    when messages change rapidly.
+
+    Can be used as an async context manager, which automatically starts and cancels the polling task. Alternatively,
+    call `start` and `stop` manually. Obtain an instance via `RunClientAsync.get_status_message_watcher`.
+    """
+
+    def __init__(
+        self, *, run_client: RunClientAsync, to_logger: logging.Logger, check_period: timedelta = timedelta(seconds=1)
+    ) -> None:
+        """Initialize `StatusMessageWatcherAsync`.
+
+        Args:
+            run_client: The run client used to poll the Actor run status and status message.
+            to_logger: The logger to which the status messages will be forwarded.
+            check_period: How often to poll the status message.
+        """
+        super().__init__(to_logger=to_logger, check_period=check_period)
+        self._run_client = run_client
+        self._logging_task: Task | None = None
+
+    def start(self) -> Task:
+        """Start the polling task.
+
+        The caller is responsible for cleanup by calling the `stop` method when done.
+        """
+        if self._logging_task and not self._logging_task.done():
+            raise RuntimeError('Logging task already active')
+        self._logging_task = asyncio.create_task(self._log_changed_status_message())
+        return self._logging_task
+
+    async def stop(self) -> None:
+        """Stop the logging task."""
+        if not self._logging_task:
+            raise RuntimeError('Logging task is not active')
+
+        self._logging_task.cancel()
+        try:
+            await self._logging_task
+        except asyncio.CancelledError:
+            pass
+        finally:
+            self._logging_task = None
+
+    async def __aenter__(self) -> Self:
+        """Start the logging task within the context. Exiting the context will cancel the logging task."""
+        self.start()
+        return self
+
+    async def __aexit__(
+        self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
+    ) -> None:
+        """Cancel the logging task."""
+        if exc_type is None:
+            await asyncio.sleep(self._final_sleep_time_s)
+        await self.stop()
+
+    async def _log_changed_status_message(self) -> None:
+        while True:
+            run_data = await self._run_client.get()
+            if not self._log_run_data(run_data):
+                break
+            await asyncio.sleep(self._check_period)
+
+
+@docs_group('Other')
+class StatusMessageWatcher(StatusMessageWatcherBase):
+    """Polls and logs Actor run status messages in a background thread.
+
+    The status message and status of the Actor run are polled at a fixed interval and forwarded to the provided logger
+    whenever they change. There is no guarantee that every intermediate status message will be captured, especially
+    when messages change rapidly.
+
+    Can be used as a context manager, which automatically starts and stops the polling thread. Alternatively,
+    call `start` and `stop` manually. Obtain an instance via `RunClient.get_status_message_watcher`.
+    """
+
+    def __init__(
+        self, *, run_client: RunClient, to_logger: logging.Logger, check_period: timedelta = timedelta(seconds=1)
+    ) -> None:
+        """Initialize `StatusMessageWatcher`.
+
+        Args:
+            run_client: The run client used to poll the Actor run status and status message.
+            to_logger: The logger to which the status messages will be forwarded.
+            check_period: How often to poll the status message.
+        """
+        super().__init__(to_logger=to_logger, check_period=check_period)
+        self._run_client = run_client
+        self._logging_thread: Thread | None = None
+        self._stop_logging = False
+
+    def start(self) -> Thread:
+        """Start the polling thread.
+
+        The caller is responsible for cleanup by calling the `stop` method when done.
+        """
+        if self._logging_thread:
+            raise RuntimeError('Logging thread already active')
+        self._stop_logging = False
+        self._logging_thread = threading.Thread(target=self._log_changed_status_message)
+        self._logging_thread.start()
+        return self._logging_thread
+
+    def stop(self) -> None:
+        """Signal the logging thread to stop logging and wait for it to finish."""
+        if not self._logging_thread:
+            raise RuntimeError('Logging thread is not active')
+        self._stop_logging = True
+        self._logging_thread.join()
+        self._logging_thread = None
+        self._stop_logging = False
+
+    def __enter__(self) -> Self:
+        """Start the logging thread within the context. Exiting the context will stop the logging thread."""
+        self.start()
+        return self
+
+    def __exit__(
+        self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
+    ) -> None:
+        """Stop the logging thread."""
+        if exc_type is None:
+            time.sleep(self._final_sleep_time_s)
+        self.stop()
+
+    def _log_changed_status_message(self) -> None:
+        while True:
+            if not self._log_run_data(self._run_client.get()):
+                break
+            if self._stop_logging:
+                break
+            time.sleep(self._check_period)

scrapely_client/_streamed_log.py

@@ -0,0 +1,250 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+import threading
+from asyncio import Task
+from datetime import UTC, datetime
+from threading import Thread
+from typing import TYPE_CHECKING, ClassVar, Self, cast
+
+import impit
+
+from scrapely_client._docs import docs_group
+
+if TYPE_CHECKING:
+    from types import TracebackType
+
+    from scrapely_client._resource_clients import LogClient, LogClientAsync
+    from scrapely_client.types import Timeout
+
+
+class StreamedLogBase:
+    """Base class for streaming and buffering chunked Actor run logs."""
+
+    _force_propagate = False
+    """Test related flag to enable propagation of logs to the `caplog` fixture during tests."""
+
+    _stream_timeout: ClassVar[Timeout] = 'no_timeout'
+    """Timeout for the log-stream long-poll request, which stays open for the whole Actor run.
+
+    impit applies its `timeout` to the whole request including the streamed body, so any bounded value truncates a
+    longer run mid-stream and raises `impit.TimeoutException` (#1040). `no_timeout` maps to impit's ~24h cap, which
+    is effectively unbounded for real runs and mirrors the JS client.
+    """
+
+    def __init__(self, to_logger: logging.Logger, *, from_start: bool = True) -> None:
+        if self._force_propagate:
+            to_logger.propagate = True
+        self._to_logger = to_logger
+        self._stream_buffer = list[bytes]()
+        self._split_marker = re.compile(rb'(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)')
+        self._relevancy_time_limit: datetime | None = None if from_start else datetime.now(tz=UTC)
+
+    def _process_new_data(self, data: bytes) -> None:
+        new_chunk = data
+        self._stream_buffer.append(new_chunk)
+        if re.findall(self._split_marker, new_chunk):
+            # If complete split marker was found in new chunk, then log the buffer.
+            self._log_buffer_content(include_last_part=False)
+
+    def _log_buffer_content(self, *, include_last_part: bool = False) -> None:
+        """Merge the whole buffer and split it into parts based on the marker.
+
+        Log the messages created from the split parts and remove them from buffer.
+        The last part could be incomplete, and so it can be left unprocessed in the buffer until later.
+        """
+        all_parts = re.split(self._split_marker, b''.join(self._stream_buffer))[1:]  # The First split is empty
+        if include_last_part:
+            message_markers = all_parts[0::2]
+            message_contents = all_parts[1::2]
+            self._stream_buffer = []
+        else:
+            message_markers = all_parts[0:-2:2]
+            message_contents = all_parts[1:-2:2]
+            # The last two parts (marker and message) are possibly not complete and will be left in the buffer
+            self._stream_buffer = all_parts[-2:]
+
+        for marker, content in zip(message_markers, message_contents, strict=False):
+            decoded_marker = marker.decode('utf-8')
+            decoded_content = content.decode('utf-8')
+            if self._relevancy_time_limit:
+                log_time = datetime.fromisoformat(decoded_marker)
+                if log_time < self._relevancy_time_limit:
+                    # Skip irrelevant logs
+                    continue
+            message = decoded_marker + decoded_content
+            self._to_logger.log(level=self._guess_log_level_from_message(message), msg=message.strip())
+
+    @staticmethod
+    def _guess_log_level_from_message(message: str) -> int:
+        """Guess the log level from the message."""
+        # Using only levels explicitly mentioned in the logging module
+        known_levels = ('CRITICAL', 'FATAL', 'ERROR', 'WARN', 'WARNING', 'INFO', 'DEBUG', 'NOTSET')
+        for level in known_levels:
+            if level in message:
+                # `getLevelName` returns an `int` when string is passed as input.
+                return cast('int', logging.getLevelName(level))
+        # Unknown log level. Fall back to the default.
+        return logging.INFO
+
+
+@docs_group('Other')
+class StreamedLog(StreamedLogBase):
+    """Streams Actor run log output to a Python logger in a background thread.
+
+    The log stream is consumed in a background thread and each log message is forwarded to the provided logger with
+    an appropriate log level inferred from the message content.
+
+    Can be used as a context manager, which automatically starts and stops the streaming thread. Alternatively,
+    call `start` and `stop` manually. Obtain an instance via `RunClient.get_streamed_log`.
+    """
+
+    def __init__(self, log_client: LogClient, *, to_logger: logging.Logger, from_start: bool = True) -> None:
+        """Initialize `StreamedLog`.
+
+        Args:
+            log_client: The log client used to stream raw log data from the Actor run.
+            to_logger: The logger to which the log messages will be forwarded.
+            from_start: If `True`, all logs from the start of the Actor run will be streamed. If `False`, only newly
+                arrived logs will be streamed. This can be useful for long-running Actors in stand-by mode where only
+                recent logs are relevant.
+        """
+        super().__init__(to_logger=to_logger, from_start=from_start)
+        self._log_client = log_client
+        self._streaming_thread: Thread | None = None
+        self._stop_logging = False
+
+    def start(self) -> Thread:
+        """Start the streaming thread.
+
+        The caller is responsible for cleanup by calling the `stop` method when done.
+        """
+        if self._streaming_thread:
+            raise RuntimeError('Streaming thread already active')
+        self._stop_logging = False
+        # A daemon thread so a stream still blocked on a read can never hold up interpreter shutdown.
+        self._streaming_thread = threading.Thread(target=self._stream_log, daemon=True)
+        self._streaming_thread.start()
+        return self._streaming_thread
+
+    def stop(self) -> None:
+        """Signal the streaming thread to stop logging and wait for it to finish."""
+        if not self._streaming_thread:
+            raise RuntimeError('Streaming thread is not active')
+        self._stop_logging = True
+        self._streaming_thread.join()
+        self._streaming_thread = None
+        self._stop_logging = False
+
+    def __enter__(self) -> Self:
+        """Start the streaming thread within the context. Exiting the context will finish the streaming thread."""
+        self.start()
+        return self
+
+    def __exit__(
+        self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
+    ) -> None:
+        """Stop the streaming thread."""
+        self.stop()
+
+    def _stream_log(self) -> None:
+        try:
+            with self._log_client.stream(raw=True, timeout=self._stream_timeout) as log_stream:
+                if not log_stream:
+                    return
+                try:
+                    for data in log_stream.iter_bytes():
+                        self._process_new_data(data)
+                        if self._stop_logging:
+                            break
+                finally:
+                    # Flush the last buffered part even if the read timed out or was stopped.
+                    self._log_buffer_content(include_last_part=True)
+        except impit.TimeoutException:
+            # With `no_timeout` this fires only if the run outlives impit's ~24h cap or the connection stalls.
+            # The stream cannot continue, so warn and let the thread end instead of leaking a traceback (#1040).
+            self._to_logger.warning('Log streaming stopped: the log stream request timed out.')
+        except Exception:
+            # Any other failure in log redirection must not escape the background thread; log it instead.
+            self._to_logger.exception('Log redirection stopped due to unexpected error:')
+
+
+@docs_group('Other')
+class StreamedLogAsync(StreamedLogBase):
+    """Streams Actor run log output to a Python logger in an asyncio task.
+
+    The log stream is consumed in a background asyncio task and each log message is forwarded to the provided logger
+    with an appropriate log level inferred from the message content.
+
+    Can be used as an async context manager, which automatically starts and cancels the streaming task. Alternatively,
+    call `start` and `stop` manually. Obtain an instance via `RunClientAsync.get_streamed_log`.
+    """
+
+    def __init__(self, log_client: LogClientAsync, *, to_logger: logging.Logger, from_start: bool = True) -> None:
+        """Initialize `StreamedLogAsync`.
+
+        Args:
+            log_client: The async log client used to stream raw log data from the Actor run.
+            to_logger: The logger to which the log messages will be forwarded.
+            from_start: If `True`, all logs from the start of the Actor run will be streamed. If `False`, only newly
+                arrived logs will be streamed. This can be useful for long-running Actors in stand-by mode where only
+                recent logs are relevant.
+        """
+        super().__init__(to_logger=to_logger, from_start=from_start)
+        self._log_client = log_client
+        self._streaming_task: Task | None = None
+
+    def start(self) -> Task:
+        """Start the streaming task.
+
+        The caller is responsible for cleanup by calling the `stop` method when done.
+        """
+        if self._streaming_task and not self._streaming_task.done():
+            raise RuntimeError('Streaming task already active')
+        self._streaming_task = asyncio.create_task(self._stream_log())
+        return self._streaming_task
+
+    async def stop(self) -> None:
+        """Stop the streaming task."""
+        if not self._streaming_task:
+            raise RuntimeError('Streaming task is not active')
+
+        self._streaming_task.cancel()
+        try:
+            await self._streaming_task
+        except asyncio.CancelledError:
+            pass
+        finally:
+            self._streaming_task = None
+
+    async def __aenter__(self) -> Self:
+        """Start the streaming task within the context. Exiting the context will cancel the streaming task."""
+        self.start()
+        return self
+
+    async def __aexit__(
+        self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
+    ) -> None:
+        """Cancel the streaming task."""
+        await self.stop()
+
+    async def _stream_log(self) -> None:
+        try:
+            async with self._log_client.stream(raw=True, timeout=self._stream_timeout) as log_stream:
+                if not log_stream:
+                    return
+                try:
+                    async for data in log_stream.aiter_bytes():
+                        self._process_new_data(data)
+                finally:
+                    # Flush the last buffered part even if the task is cancelled by `stop()`.
+                    self._log_buffer_content(include_last_part=True)
+        except impit.TimeoutException:
+            # As in `StreamedLog._stream_log`, impit's whole-request timeout on the long-lived stream is an
+            # expected terminal condition, not an error, so log a warning and end the task instead of a traceback.
+            self._to_logger.warning('Log streaming stopped: the log stream request timed out.')
+        except Exception:
+            # Exception in log redirection should not propagate further.
+            self._to_logger.exception('Log redirection stopped due to unexpected error:')