crawlee 0.0.1__tar.gz

crawlee-0.0.1/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023 Apify Technologies s.r.o.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

crawlee-0.0.1/PKG-INFO

@@ -0,0 +1,34 @@
+Metadata-Version: 2.1
+Name: crawlee
+Version: 0.0.1
+Summary: Crawlee for Python
+Home-page: https://todo.com/
+License: Apache-2.0
+Keywords: apify,automation,chrome,crawlee,crawler,headless,scraper,scraping
+Author: Apify Technologies s.r.o.
+Author-email: support@apify.com
+Requires-Python: >=3.9,<4.0
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries
+Requires-Dist: colorama (>=0.4.6,<0.5.0)
+Requires-Dist: more_itertools (>=10.2.0,<11.0.0)
+Requires-Dist: pyee (>=11.1.0,<12.0.0)
+Requires-Dist: typing-extensions (>=4.1.0,<5.0.0)
+Project-URL: Apify Homepage, https://apify.com
+Project-URL: Changelog, https://github.com/apify/crawlee-py/blob/master/CHANGELOG.md
+Project-URL: Documentation, https://todo.com/
+Project-URL: Issue tracker, https://github.com/apify/crawlee-py/issues
+Project-URL: Repository, https://github.com/apify/crawlee-py
+Project-URL: Source, https://github.com/apify/crawlee-py
+Description-Content-Type: text/markdown
+
+# Crawlee Python
+

crawlee-0.0.1/README.md

@@ -0,0 +1 @@
+# Crawlee Python

crawlee-0.0.1/pyproject.toml

@@ -0,0 +1,156 @@
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+name = "crawlee"
+version = "0.0.1"
+description = "Crawlee for Python"
+authors = ["Apify Technologies s.r.o. <support@apify.com>"]
+license = "Apache-2.0"
+readme = "README.md"
+homepage = "https://todo.com/"
+repository = "https://github.com/apify/crawlee-py"
+packages = [{ include = "crawlee", from = "src" }]
+classifiers = [
+    "Development Status :: 2 - Pre-Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Libraries",
+]
+keywords = [
+    "apify",
+    "automation",
+    "chrome",
+    "crawlee",
+    "crawler",
+    "headless",
+    "scraper",
+    "scraping",
+]
+
+[tool.poetry.urls]
+"Homepage" = "https://todo.com/"
+"Documentation" = "https://todo.com/"
+"Source" = "https://github.com/apify/crawlee-py"
+"Issue tracker" = "https://github.com/apify/crawlee-py/issues"
+"Changelog" = "https://github.com/apify/crawlee-py/blob/master/CHANGELOG.md"
+"Apify Homepage" = "https://apify.com"
+
+# We use inclusive ordered comparison clause for external packages intentionally in order to enhance the Crawlee's
+# compatibility with external packages. This decision was discussed in detail in the following PR:
+# https://github.com/apify/apify-sdk-python/pull/154.
+[tool.poetry.dependencies]
+python = "^3.9"
+colorama = "^0.4.6"
+more_itertools = "^10.2.0"
+pyee = "^11.1.0"
+typing-extensions = "^4.1.0"
+
+[tool.poetry.group.dev.dependencies]
+build = "~1.0.3"
+filelock = "~3.13.1"
+mypy = "~1.8.0"
+pre-commit = "~3.4.0"
+pydoc-markdown = "~4.8.2"
+pytest = "~7.4.4"
+pytest-asyncio = "~0.23.4"
+pytest-cov = "~4.1.0"
+pytest-only = "~2.0.0"
+pytest-timeout = "~2.2.0"
+pytest-xdist = "~3.5.0"
+respx = "~0.20.1"
+ruff = "~0.1.13"
+twine = "~4.0.2"
+types-colorama = "~0.4.15.20240106"
+
+[tool.ruff]
+line-length = 120
+select = ["ALL"]
+ignore = [
+    "ANN401",  # Dynamically typed expressions (typing.Any) are disallowed in {filename}
+    "BLE001",  # Do not catch blind exception
+    "C901",    # `{name}` is too complex
+    "COM812",  # This rule may cause conflicts when used with the formatter
+    "D100",    # Missing docstring in public module
+    "D104",    # Missing docstring in public package
+    "D107",    # Missing docstring in `__init__`
+    "EM",      # flake8-errmsg
+    "G004",    # Logging statement uses f-string
+    "ISC001",  # This rule may cause conflicts when used with the formatter
+    "FIX",     # flake8-fixme
+    "PGH003",  # Use specific rule codes when ignoring type issues
+    "PLR0911", # Too many return statements
+    "PLR0913", # Too many arguments in function definition
+    "PLR0915", # Too many statements
+    "PTH",     # flake8-use-pathlib
+    "PYI034",  # `__aenter__` methods in classes like `{name}` usually return `self` at runtime
+    "PYI036",  # The second argument in `__aexit__` should be annotated with `object` or `BaseException | None`
+    "S102",    # Use of `exec` detected
+    "S105",    # Possible hardcoded password assigned to
+    "S106",    # Possible hardcoded password assigned to argument: "{name}"
+    "S301",    # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue
+    "S303",    # Use of insecure MD2, MD4, MD5, or SHA1 hash function
+    "S311",    # Standard pseudo-random generators are not suitable for cryptographic purposes
+    "TD002",   # Missing author in TODO; try: `# TODO(<author_name>): ...` or `# TODO @<author_name>: ...
+    "TRY003",  # Avoid specifying long messages outside the exception class
+]
+
+[tool.ruff.format]
+quote-style = "single"
+indent-style = "space"
+
+[tool.ruff.lint.per-file-ignores]
+"**/__init__.py" = [
+    "F401", # Unused imports
+]
+"**/{scripts}/*" = [
+    "D",       # Everything from the pydocstyle
+    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
+    "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable
+    "T20",     # flake8-print
+]
+"**/{tests}/*" = [
+    "D",       # Everything from the pydocstyle
+    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
+    "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable
+    "S101",    # Use of assert detected
+    "T20",     # flake8-print
+    "TRY301",  # Abstract `raise` to an inner function
+]
+
+[tool.ruff.lint.flake8-quotes]
+docstring-quotes = "double"
+inline-quotes = "single"
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.pytest]
+asyncio_mode = "auto"
+timeout = 1200
+
+[tool.mypy]
+python_version = "3.9"
+files = ["scripts", "src", "tests"]
+check_untyped_defs = true
+disallow_incomplete_defs = true
+disallow_untyped_calls = true
+disallow_untyped_decorators = true
+disallow_untyped_defs = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_return_any = true
+warn_unreachable = true
+warn_unused_ignores = true
+
+[tool.mypy-scrapy]
+ignore_missing_imports = true
+
+[tool.mypy-sortedcollections]
+ignore_missing_imports = true

crawlee-0.0.1/src/crawlee/autoscaling/__init__.py

@@ -0,0 +1,2 @@
+from .snapshotter import Snapshotter
+from .system_status import SystemStatus

crawlee-0.0.1/src/crawlee/autoscaling/snapshotter.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from logging import getLogger
+from typing import TYPE_CHECKING, Sequence, Union
+
+if TYPE_CHECKING:
+    from datetime import datetime
+
+logger = getLogger(__name__)
+
+
+@dataclass
+class MemorySnapshot:
+    """A snapshot of memory usage."""
+
+    created_at: datetime
+    is_overloaded: bool
+    used_bytes: int | None
+
+
+@dataclass
+class CpuSnapshot:
+    """A snapshot of CPU usage."""
+
+    created_at: datetime
+    is_overloaded: bool
+    used_ratio: float
+    ticks: dict | None = None
+
+
+@dataclass
+class EventLoopSnapshot:
+    """A snapshot of event loop usage."""
+
+    created_at: datetime
+    is_overloaded: bool
+    exceeded_millis: float
+
+
+@dataclass
+class ClientSnapshot:
+    """A snapshot of client usage."""
+
+    created_at: datetime
+    is_overloaded: bool
+    rate_limit_error_count: int
+
+
+Snapshot = Union[MemorySnapshot, CpuSnapshot, EventLoopSnapshot, ClientSnapshot]
+
+
+class Snapshotter:
+    """Creates snapshots of system resources at given intervals.
+
+    Creates snapshots of system resources at given intervals and marks the resource as either overloaded or not during
+    the last interval. Keeps a history of the snapshots. It tracks the following resources: Memory, EventLoop, API
+    and CPU. The class is used by the `AutoscaledPool` class.
+
+    When running on the Apify platform, the CPU and memory statistics are provided by the platform, as collected from
+    the running Docker container. When running locally, `Snapshotter` makes its own statistics by querying the OS.
+
+    CPU becomes overloaded locally when its current use exceeds the `maxUsedCpuRatio` option or when Apify platform
+    marks it as overloaded.
+
+    Memory becomes overloaded if its current use exceeds the `max_used_memory_ratio` option. It's computed using
+    the total memory available to the container when running on the Apify platform and a quarter of total system
+    memory when running locally. Max total memory when running locally may be overridden by using
+    the `CRAWLEE_MEMORY_MBYTES` environment variable.
+
+    Event loop becomes overloaded if it slows down by more than the `max_blocked_millis` option.
+
+    Client becomes overloaded when rate limit errors (429 - Too Many Requests) exceeds the `max_client_errors` option,
+    typically received from the request queue, exceed the set limit within the set interval.
+    """
+
+    def get_memory_sample(
+        self: Snapshotter,
+        sample_duration_millis: float | None = None,
+    ) -> Sequence[MemorySnapshot]:
+        """Returns a sample of the latest memory snapshots.
+
+        Args:
+            sample_duration_millis: The size of the sample in milliseconds. Defaults to None.
+            If omitted, it returns a full snapshot history.
+
+        Returns:
+            A sample of memory snapshots.
+        """
+
+    def get_event_loop_sample(
+        self: Snapshotter,
+        sample_duration_millis: float | None = None,
+    ) -> Sequence[EventLoopSnapshot]:
+        """Returns a sample of the latest event loop snapshots.
+
+        Args:
+            sample_duration_millis: The size of the sample in milliseconds. Defaults to None.
+                If omitted, it returns a full snapshot history.
+
+        Returns:
+            A sample of event loop snapshots.
+        """
+
+    def get_cpu_sample(
+        self: Snapshotter,
+        sample_duration_millis: float | None = None,
+    ) -> Sequence[CpuSnapshot]:
+        """Returns a sample of the latest CPU snapshots.
+
+        Args:
+            sample_duration_millis: The size of the sample in milliseconds. Defaults to None.
+                If omitted, it returns a full snapshot history.
+
+        Returns:
+            A sample of CPU snapshots.
+        """
+
+    def get_client_sample(
+        self: Snapshotter,
+        sample_duration_millis: float | None = None,
+    ) -> Sequence[ClientSnapshot]:
+        """Returns a sample of the latest client snapshots.
+
+        Args:
+            sample_duration_millis: The size of the sample in milliseconds. Defaults to None.
+                If omitted, it returns a full snapshot history.
+
+        Returns:
+            A sample of client snapshots.
+        """

crawlee-0.0.1/src/crawlee/autoscaling/system_status.py

@@ -0,0 +1,237 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from logging import getLogger
+from typing import TYPE_CHECKING, Sequence
+
+from more_itertools import pairwise
+
+from crawlee.utils import weighted_avg
+
+if TYPE_CHECKING:
+    from datetime import datetime
+
+    from crawlee.autoscaling.snapshotter import Snapshot, Snapshotter
+
+logger = getLogger(__name__)
+
+
+@dataclass
+class LoadRatioInfo:
+    """Represents the load ratio of a resource."""
+
+    is_overloaded: bool
+    limit_ratio: float
+    actual_ratio: float
+
+
+@dataclass
+class SystemInfo:
+    """Represents the current status of the system."""
+
+    is_system_idle: bool  # Indicates whether the system is currently idle or overloaded
+    mem_info: LoadRatioInfo
+    event_loop_info: LoadRatioInfo
+    cpu_info: LoadRatioInfo
+    client_info: LoadRatioInfo
+    mem_current_bytes: int | None = None  # Platform only property
+    cpu_current_usage: int | None = None  # Platform only property
+    is_cpu_overloaded: bool | None = None  # Platform only property
+    created_at: datetime | None = None
+
+
+@dataclass
+class FinalStatistics:
+    """Represents final statistics."""
+
+    requests_finished: int
+    requests_failed: int
+    retry_histogram: list[int]
+    request_avg_failed_duration_millis: float
+    request_avg_finished_duration_millis: float
+    requests_finished_per_minute: float
+    requests_failed_per_minute: float
+    request_total_duration_millis: float
+    requests_total: int
+    crawler_runtime_millis: float
+
+
+class SystemStatus:
+    """Provides a simple interface to reading system status from a `Snapshotter` instance.
+
+    It only exposes two functions `SystemStatus.get_current_status` and `SystemStatus.get_historical_status`.
+    The system status is calculated using a weighted average of overloaded messages in the snapshots, with the weights
+    being the time intervals between the snapshots. Each resource is calculated separately, and the system
+    is overloaded whenever at least one resource is overloaded. The class is used by the `AutoscaledPool` class.
+
+    `SystemStatus.get_current_status` returns a boolean that represents the current status of the system. The length
+    of the current timeframe in seconds is configurable by the `currentHistorySecs` option and represents the max age
+    of snapshots to be considered for the calculation.
+
+    `SystemStatus.get_historical_status` returns a boolean that represents the long-term status of the system. It
+    considers the full snapshot history available in the `Snapshotter` instance.
+
+    Attributes:
+        snapshotter: The `Snapshotter` instance to be queried for `SystemStatus`.
+
+        current_history_secs: Defines max age of snapshots used in the `SystemStatus.get_current_status` measurement.
+
+        max_memory_overloaded_ratio: Sets the maximum ratio of overloaded snapshots in a memory sample.
+            If the sample exceeds this ratio, the system will be overloaded.
+
+        max_event_loop_overloaded_ratio: Sets the maximum ratio of overloaded snapshots in an event loop sample.
+            If the sample exceeds this ratio, the system will be overloaded.
+
+        max_cpu_overloaded_ratio: Sets the maximum ratio of overloaded snapshots in a CPU sample. If the sample
+            exceeds this ratio, the system will be overloaded.
+
+        max_client_overloaded_ratio: Sets the maximum ratio of overloaded snapshots in a Client sample.
+            If the sample exceeds this ratio, the system will be overloaded.
+    """
+
+    def __init__(
+        self: SystemStatus,
+        snapshotter: Snapshotter,
+        current_history_secs: int = 5,
+        max_memory_overloaded_ratio: float = 0.2,
+        max_event_loop_overloaded_ratio: float = 0.6,
+        max_cpu_overloaded_ratio: float = 0.4,
+        max_client_overloaded_ratio: float = 0.3,
+    ) -> None:
+        self.snapshotter = snapshotter
+        self.current_history_secs = current_history_secs
+        self.max_memory_overloaded_ratio = max_memory_overloaded_ratio
+        self.max_event_loop_overloaded_ratio = max_event_loop_overloaded_ratio
+        self.max_cpu_overloaded_ratio = max_cpu_overloaded_ratio
+        self.max_client_overloaded_ratio = max_client_overloaded_ratio
+
+    def get_current_status(self: SystemStatus) -> SystemInfo:
+        """Get the current system status.
+
+        Returns a `SystemInfo` object where the `is_system_idle` property is `False` if the system has been overloaded
+        in the last `current_history_secs` seconds, and `True` otherwise.
+
+        Returns:
+            An object representing the current system status.
+        """
+        return self._is_system_idle(self.current_history_secs)
+
+    def get_historical_status(self: SystemStatus) -> SystemInfo:
+        """Get the historical system status.
+
+        Returns a `SystemInfo` where the `is_system_idle` property is set to `False` if the system has been overloaded
+        in the full history of the `Snapshotter` (which is configurable in the `Snapshotter`, and `True` otherwise.
+
+        Returns:
+            An object representing the historical system status.
+        """
+        return self._is_system_idle()
+
+    def _is_system_idle(self: SystemStatus, sample_duration_millis: int | None = None) -> SystemInfo:
+        """Determine if the system is currently idle or overloaded.
+
+        Args:
+            sample_duration_millis: The duration (in milliseconds) within which to analyze system status.
+
+        Returns:
+            An object representing the system status with an `is_system_idle` property set to `True` if the system
+            has not been overloaded within the specified time duration, and `False` otherwise.
+        """
+        mem_info = self._is_memory_overloaded(sample_duration_millis)
+        event_loop_info = self._is_event_loop_overloaded(sample_duration_millis)
+        cpu_info = self._is_cpu_overloaded(sample_duration_millis)
+        client_info = self._is_client_overloaded(sample_duration_millis)
+
+        return SystemInfo(
+            is_system_idle=(
+                not mem_info.is_overloaded
+                and not event_loop_info.is_overloaded
+                and not cpu_info.is_overloaded
+                and not client_info.is_overloaded
+            ),
+            mem_info=mem_info,
+            event_loop_info=event_loop_info,
+            cpu_info=cpu_info,
+            client_info=client_info,
+        )
+
+    def _is_memory_overloaded(self: SystemStatus, sample_duration_millis: int | None = None) -> LoadRatioInfo:
+        """Determine if memory has been overloaded within a specified time duration.
+
+        Args:
+            sample_duration_millis: The duration (in milliseconds) within which to analyze memory snapshots.
+
+        Returns:
+            An object with an `is_overloaded` property set to `True` if memory has been overloaded within the specified
+            time duration. Otherwise, `is_overloaded` is set to `False`.
+        """
+        sample = self.snapshotter.get_memory_sample(sample_duration_millis)
+        return self._is_sample_overloaded(sample, self.max_memory_overloaded_ratio)
+
+    def _is_event_loop_overloaded(self: SystemStatus, sample_duration_millis: int | None = None) -> LoadRatioInfo:
+        """Determine if the event loop has been overloaded within a specified time duration.
+
+        Args:
+            sample_duration_millis: The duration (in milliseconds) within which to analyze event loop snapshots.
+
+        Returns:
+            An object with an `is_overloaded` property set to `True` if the event loop has been overloaded within
+            the specified time duration. Otherwise, `is_overloaded` is set to `False`.
+        """
+        sample = self.snapshotter.get_event_loop_sample(sample_duration_millis)
+        return self._is_sample_overloaded(sample, self.max_event_loop_overloaded_ratio)
+
+    def _is_cpu_overloaded(self: SystemStatus, sample_duration_millis: int | None = None) -> LoadRatioInfo:
+        """Determine if the CPU has been overloaded within a specified time duration.
+
+        Args:
+            sample_duration_millis: The duration (in milliseconds) within which to analyze CPU snapshots.
+
+        Returns:
+            An object with an `is_overloaded` property set to `True` if the CPU has been overloaded within
+            the specified time duration. Otherwise, `is_overloaded` is set to `False`.
+        """
+        sample = self.snapshotter.get_cpu_sample(sample_duration_millis)
+        return self._is_sample_overloaded(sample, self.max_cpu_overloaded_ratio)
+
+    def _is_client_overloaded(self: SystemStatus, sample_duration_millis: int | None = None) -> LoadRatioInfo:
+        """Determine if the client has been overloaded within a specified time duration.
+
+        Args:
+            sample_duration_millis: The duration (in milliseconds) within which to analyze client snapshots.
+
+        Returns:
+            An object with an `is_overloaded` property set to `True` if the client has been overloaded within
+            the specified time duration. Otherwise, `is_overloaded` is set to `False`.
+        """
+        sample = self.snapshotter.get_client_sample(sample_duration_millis)
+        return self._is_sample_overloaded(sample, self.max_client_overloaded_ratio)
+
+    def _is_sample_overloaded(self: SystemStatus, sample: Sequence[Snapshot], ratio: float) -> LoadRatioInfo:
+        """Determine if a sample of snapshot data is overloaded based on a specified ratio.
+
+        Args:
+            sample: A sequence of snapshot data to analyze.
+            ratio: The ratio threshold to consider the sample as overloaded.
+
+        Returns:
+            An object with an `is_overloaded` property set to `True` if the sample is considered overloaded based
+            on the specified ratio. Otherwise, `is_overloaded` is set to `False`.
+        """
+        if not sample:
+            return LoadRatioInfo(is_overloaded=False, limit_ratio=ratio, actual_ratio=0)
+
+        weights, values = [], []
+
+        for previous, current in pairwise(sample):
+            weight = (current.created_at - previous.created_at).total_seconds() or 0.001  # Avoid zero
+            weights.append(weight)
+            values.append(float(current.is_overloaded))
+
+        w_avg = values[0] if len(sample) == 1 else weighted_avg(values, weights)
+
+        return LoadRatioInfo(
+            is_overloaded=w_avg > ratio,
+            limit_ratio=ratio,
+            actual_ratio=round(w_avg, 3),
+        )

crawlee-0.0.1/src/crawlee/events/__init__.py

@@ -0,0 +1 @@
+from .event_manager import EventManager

crawlee-0.0.1/src/crawlee/events/event_manager.py

@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+import asyncio
+from collections import defaultdict
+from contextlib import suppress
+from functools import wraps
+from inspect import iscoroutinefunction
+from logging import getLogger
+from typing import TYPE_CHECKING
+
+from pyee.asyncio import AsyncIOEventEmitter
+
+if TYPE_CHECKING:
+    from datetime import timedelta
+
+    from crawlee.events.types import Event, EventData, Listener, WrappedListener
+
+logger = getLogger(__name__)
+
+
+class EventManager:
+    """Event manager for registering, emitting, and managing event listeners.
+
+    Event manager allows you to register event listeners, emit events, and wait for event listeners to complete
+    their execution. It is built on top of the `pyee.asyncio.AsyncIOEventEmitter` class.
+
+    Attributes:
+        _event_emitter: The event emitter which is used to emit events and call the event listeners.
+
+        _listener_tasks: The set of asyncio tasks which are currently executing the event listeners.
+
+        _listeners_to_wrappers: The mapping between events and listeners, and the wrapped listeners which are
+            registered with the event emitter.
+    """
+
+    def __init__(self: EventManager) -> None:
+        logger.debug('Calling LocalEventManager.__init__()...')
+        self._event_emitter = AsyncIOEventEmitter()
+
+        # Listeners are wrapped in a asyncio.Task, store their references here so that we can wait for them to finish
+        self._listener_tasks: set[asyncio.Task] = set()
+
+        # Store the mapping between events and listeners like this:
+        #   event -> listener -> [wrapped_listener_1, wrapped_listener_2, ...]
+        self._listeners_to_wrappers: dict[Event, dict[Listener, list[WrappedListener]]] = defaultdict(
+            lambda: defaultdict(list),
+        )
+
+    async def close(self: EventManager, *, timeout: timedelta | None = None) -> None:
+        """Close the event manager.
+
+        This will stop listening for the events, and it will wait for all the event listeners to finish.
+
+        Args:
+            timeout: Optional timeout after which the pending event listeners are canceled.
+        """
+        logger.debug('Calling LocalEventManager.close()...')
+        await self.wait_for_all_listeners_to_complete(timeout=timeout)
+        self._event_emitter.remove_all_listeners()
+
+    def on(self: EventManager, *, event: Event, listener: Listener) -> None:
+        """Add an event listener to the event manager.
+
+        Args:
+            event: The Actor event for which to listen to.
+            listener: The function (sync or async) which is to be called when the event is emitted.
+        """
+        logger.debug('Calling LocalEventManager.on()...')
+
+        @wraps(listener)
+        async def listener_wrapper(event_data: EventData) -> None:
+            logger.debug('Calling LocalEventManager.on.listener_wrapper()...')
+
+            # If the listener is a coroutine function, just call it, otherwise, run it in a separate thread
+            # to avoid blocking the event loop
+            coroutine = (
+                listener(event_data) if iscoroutinefunction(listener) else asyncio.to_thread(listener, event_data)
+            )
+
+            listener_task = asyncio.create_task(coroutine, name=f'Task-{event.value}-{listener.__name__}')
+            self._listener_tasks.add(listener_task)
+
+            try:
+                await listener_task
+            except Exception:
+                # We need to swallow the exception and just log it here, otherwise it could break the event emitter
+                logger.exception(
+                    'Exception in the event listener',
+                    extra={'event_name': event.value, 'listener_name': listener.__name__},
+                )
+            finally:
+                logger.debug('LocalEventManager.on.listener_wrapper(): Removing listener task from the set...')
+                self._listener_tasks.remove(listener_task)
+
+        self._listeners_to_wrappers[event][listener].append(listener_wrapper)
+        self._event_emitter.add_listener(event.value, listener_wrapper)
+
+    def off(self: EventManager, *, event: Event, listener: Listener | None = None) -> None:
+        """Remove a listener, or all listeners, from an Actor event.
+
+        Args:
+            event: The Actor event for which to remove listeners.
+            listener: The listener which is supposed to be removed. If not passed, all listeners of this event
+                are removed.
+        """
+        logger.debug('Calling LocalEventManager.off()...')
+
+        if listener:
+            for listener_wrapper in self._listeners_to_wrappers[event][listener]:
+                self._event_emitter.remove_listener(event.value, listener_wrapper)
+            self._listeners_to_wrappers[event][listener] = []
+        else:
+            self._listeners_to_wrappers[event] = defaultdict(list)
+            self._event_emitter.remove_all_listeners(event.value)
+
+    def emit(self: EventManager, *, event: Event, event_data: EventData) -> None:
+        """Emit an event.
+
+        Args:
+            event: The event which will be emitted.
+            event_data: The data which will be passed to the event listeners.
+        """
+        logger.debug('Calling LocalEventManager.emit()...')
+        self._event_emitter.emit(event.value, event_data)
+
+    async def wait_for_all_listeners_to_complete(self: EventManager, *, timeout: timedelta | None = None) -> None:
+        """Wait for all currently executing event listeners to complete.
+
+        Args:
+            timeout: The maximum time to wait for the event listeners to finish. If they do not complete within
+                the specified timeout, they will be canceled.
+        """
+        logger.debug('Calling LocalEventManager.wait_for_all_listeners_to_complete()...')
+
+        async def wait_for_listeners() -> None:
+            """Gathers all listener tasks and awaits their completion, logging any exceptions encountered."""
+            results = await asyncio.gather(*self._listener_tasks, return_exceptions=True)
+            for result in results:
+                if isinstance(result, Exception):
+                    logger.exception('Event listener raised an exception.', exc_info=result)
+
+        tasks = [asyncio.create_task(wait_for_listeners(), name=f'Task-{wait_for_listeners.__name__}')]
+        timeout_secs = timeout.total_seconds() if timeout else None
+
+        try:
+            _, pending = await asyncio.wait(tasks, timeout=timeout_secs)
+            if pending:
+                logger.warning('Waiting timeout reached; canceling unfinished event listeners.')
+        except asyncio.CancelledError:
+            logger.warning('Asyncio wait was cancelled; canceling unfinished event listeners.')
+            raise
+        finally:
+            for task in tasks:
+                if not task.done():
+                    task.cancel()
+                    with suppress(asyncio.CancelledError):
+                        await task

crawlee-0.0.1/src/crawlee/events/types.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Coroutine
+from dataclasses import dataclass
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Union
+
+if TYPE_CHECKING:
+    from crawlee.autoscaling.system_status import SystemInfo
+
+
+class Event(Enum):
+    """Enum of all possible events that can be emitted."""
+
+    PERSIST_STATE = 'persistState'
+    SYSTEM_INFO = 'systemInfo'
+    MIGRATING = 'migrating'
+    ABORTING = 'aborting'
+    EXIT = 'exit'
+
+
+@dataclass
+class EventPersistStateData:
+    """Data for the persist state event."""
+
+    is_migrating: bool
+
+
+@dataclass
+class EventSystemInfoData:
+    """Data for the system info event."""
+
+    system_info: SystemInfo
+
+
+@dataclass
+class EventMigratingData:
+    """Data for the migrating event."""
+
+
+@dataclass
+class EventAbortingData:
+    """Data for the aborting event."""
+
+
+@dataclass
+class EventExitData:
+    """Data for the exit event."""
+
+
+EventData = Union[EventPersistStateData, EventSystemInfoData, EventMigratingData, EventAbortingData, EventExitData]
+SyncListener = Callable[..., None]
+AsyncListener = Callable[..., Coroutine[Any, Any, None]]
+Listener = Union[SyncListener, AsyncListener]
+WrappedListener = Callable[..., Coroutine[Any, Any, None]]

crawlee-0.0.1/src/crawlee/log_config.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import json
+import logging
+import textwrap
+import traceback
+from typing import Any
+
+from colorama import Fore, Style, just_fix_windows_console
+
+just_fix_windows_console()
+
+_LOG_NAME_COLOR = Fore.LIGHTBLACK_EX
+
+_LOG_LEVEL_COLOR = {
+    logging.DEBUG: Fore.BLUE,
+    logging.INFO: Fore.GREEN,
+    logging.WARNING: Fore.YELLOW,
+    logging.ERROR: Fore.RED,
+    logging.CRITICAL: Fore.RED,
+}
+
+_LOG_LEVEL_SHORT_ALIAS = {
+    logging.DEBUG: 'DEBUG',
+    logging.INFO: 'INFO ',
+    logging.WARNING: 'WARN ',
+    logging.ERROR: 'ERROR',
+}
+
+# So that all the log messages have the same alignment
+_LOG_MESSAGE_INDENT = ' ' * 6
+
+
+class CrawleeLogFormatter(logging.Formatter):
+    """Log formatter that prints out the log message nicely formatted, with colored level and stringified extra fields.
+
+    It formats the log records so that they:
+        - start with the level (colorized, and padded to 5 chars so that it is nicely aligned)
+        - then have the actual log message, if it's multiline then it's nicely indented
+        - then have the stringified extra log fields
+        - then, if an exception is a part of the log record, prints the formatted exception.
+    """
+
+    # The fields that are added to the log record with `logger.log(..., extra={...})` are just merged in the log record
+    # with the other log record properties, and you can't get them in some nice, isolated way. So, to get the extra
+    # fields, we just compare all the properties present in the log record with properties present in an empty log
+    # record, and extract all the extra ones not present in the empty log record.
+    empty_record = logging.LogRecord('dummy', 0, 'dummy', 0, 'dummy', None, None)
+
+    def __init__(
+        self: CrawleeLogFormatter,
+        include_logger_name: bool = False,  # noqa: FBT001, FBT002
+        *args: Any,
+        **kwargs: Any,
+    ) -> None:
+        """Create a new instance.
+
+        Args:
+            include_logger_name: Include logger name at the beginning of the log line. Defaults to False.
+            args: Arguments passed to the parent class.
+            kwargs: Keyword arguments passed to the parent class.
+        """
+        super().__init__(*args, **kwargs)
+        self.include_logger_name = include_logger_name
+
+    def _get_extra_fields(self: CrawleeLogFormatter, record: logging.LogRecord) -> dict[str, Any]:
+        extra_fields: dict[str, Any] = {}
+        for key, value in record.__dict__.items():
+            if key not in self.empty_record.__dict__:
+                extra_fields[key] = value
+
+        return extra_fields
+
+    def format(self: CrawleeLogFormatter, record: logging.LogRecord) -> str:
+        """Format the log record nicely.
+
+        This formats the log record so that it:
+            - starts with the level (colorized, and padded to 5 chars so that it is nicely aligned)
+            - then has the actual log message, if it's multiline then it's nicely indented
+            - then has the stringified extra log fields
+            - then, if an exception is a part of the log record, prints the formatted exception.
+        """
+        logger_name_string = f'{_LOG_NAME_COLOR}[{record.name}]{Style.RESET_ALL} '
+
+        # Colorize the log level, and shorten it to 6 chars tops
+        level_color_code = _LOG_LEVEL_COLOR.get(record.levelno, '')
+        level_short_alias = _LOG_LEVEL_SHORT_ALIAS.get(record.levelno, record.levelname)
+        level_string = f'{level_color_code}{level_short_alias}{Style.RESET_ALL} '
+
+        # Format the exception, if there is some
+        # Basically just print the traceback and indent it a bit
+        exception_string = ''
+        if record.exc_info:
+            exc_info = record.exc_info
+            record.exc_info = None
+            exception_string = ''.join(traceback.format_exception(*exc_info)).rstrip()
+            exception_string = '\n' + textwrap.indent(exception_string, _LOG_MESSAGE_INDENT)
+
+        # Format the extra log record fields, if there were some
+        # Just stringify them to JSON and color them gray
+        extra_string = ''
+        extra = self._get_extra_fields(record)
+        if extra:
+            extra_string = (
+                f' {Fore.LIGHTBLACK_EX}({json.dumps(extra, ensure_ascii=False, default=str)}){Style.RESET_ALL}'
+            )
+
+        # Format the actual log message, and indent everything but the first line
+        log_string = super().format(record)
+        log_string = textwrap.indent(log_string, _LOG_MESSAGE_INDENT).lstrip()
+
+        if self.include_logger_name:
+            # Include logger name at the beginning of the log line
+            return f'{logger_name_string}{level_string}{log_string}{extra_string}{exception_string}'
+
+        return f'{level_string}{log_string}{extra_string}{exception_string}'

crawlee-0.0.1/src/crawlee/utils.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+
+def weighted_avg(values: list[float], weights: list[float]) -> float:
+    """Computes a weighted average of an array of numbers, complemented by an array of weights.
+
+    Args:
+        values: List of values.
+        weights: List of weights.
+
+    Raises:
+        ValueError: If total weight is zero.
+
+    Returns:
+        float: Weighted average.
+    """
+    result = sum(value * weight for value, weight in zip(values, weights, strict=True))
+    total_weight = sum(weights)
+
+    if total_weight == 0:
+        raise ValueError('Total weight cannot be zero')
+
+    return result / total_weight