apify 2.0.0a1__tar.gz → 2.0.0b2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of apify might be problematic. Click here for more details.

Files changed (37) hide show
  1. {apify-2.0.0a1 → apify-2.0.0b2}/PKG-INFO +16 -10
  2. {apify-2.0.0a1 → apify-2.0.0b2}/pyproject.toml +35 -26
  3. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/__init__.py +1 -12
  4. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/_actor.py +34 -60
  5. apify-2.0.0a1/src/apify/log.py → apify-2.0.0b2/src/apify/_log.py +1 -1
  6. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/_platform_event_manager.py +2 -11
  7. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/_proxy_configuration.py +1 -1
  8. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/apify_storage_client/_dataset_client.py +12 -14
  9. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/scrapy/middlewares/apify_proxy.py +6 -4
  10. apify-2.0.0a1/src/apify/_models.py +0 -110
  11. apify-2.0.0a1/src/apify/py.typed +0 -0
  12. apify-2.0.0a1/src/apify/scrapy/middlewares/py.typed +0 -0
  13. apify-2.0.0a1/src/apify/scrapy/pipelines/py.typed +0 -0
  14. apify-2.0.0a1/src/apify/scrapy/py.typed +0 -0
  15. apify-2.0.0a1/src/apify/storages/py.typed +0 -0
  16. {apify-2.0.0a1 → apify-2.0.0b2}/LICENSE +0 -0
  17. {apify-2.0.0a1 → apify-2.0.0b2}/README.md +0 -0
  18. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/_configuration.py +0 -0
  19. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/_consts.py +0 -0
  20. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/_crypto.py +0 -0
  21. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/_utils.py +0 -0
  22. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/apify_storage_client/__init__.py +0 -0
  23. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/apify_storage_client/_apify_storage_client.py +0 -0
  24. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/apify_storage_client/_dataset_collection_client.py +0 -0
  25. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/apify_storage_client/_key_value_store_client.py +0 -0
  26. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/apify_storage_client/_key_value_store_collection_client.py +0 -0
  27. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/apify_storage_client/_request_queue_client.py +0 -0
  28. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/apify_storage_client/_request_queue_collection_client.py +0 -0
  29. {apify-2.0.0a1/src/apify/apify_storage_client → apify-2.0.0b2/src/apify}/py.typed +0 -0
  30. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/scrapy/__init__.py +0 -0
  31. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/scrapy/middlewares/__init__.py +0 -0
  32. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/scrapy/pipelines/__init__.py +0 -0
  33. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/scrapy/pipelines/actor_dataset_push.py +0 -0
  34. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/scrapy/requests.py +0 -0
  35. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/scrapy/scheduler.py +0 -0
  36. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/scrapy/utils.py +0 -0
  37. {apify-2.0.0a1 → apify-2.0.0b2}/src/apify/storages/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: apify
3
- Version: 2.0.0a1
3
+ Version: 2.0.0b2
4
4
  Summary: Apify SDK for Python
5
5
  License: Apache-2.0
6
6
  Keywords: apify,sdk,automation,chrome,crawlee,crawler,headless,scraper,scraping
@@ -18,15 +18,21 @@ Classifier: Programming Language :: Python :: 3.11
18
18
  Classifier: Programming Language :: Python :: 3.12
19
19
  Classifier: Topic :: Software Development :: Libraries
20
20
  Provides-Extra: scrapy
21
- Requires-Dist: apify-client (>=1.7.1)
22
- Requires-Dist: apify-shared (>=1.1.2)
23
- Requires-Dist: crawlee (>=0.3.0)
24
- Requires-Dist: cryptography (>=42.0.0)
25
- Requires-Dist: httpx (>=0.27.0)
26
- Requires-Dist: lazy-object-proxy (>=1.10.0)
27
- Requires-Dist: scrapy (>=2.11.0) ; extra == "scrapy"
28
- Requires-Dist: typing-extensions (>=4.1.0)
29
- Requires-Dist: websockets (>=10.0)
21
+ Requires-Dist: aiofiles (>=22.1.0,<23.0.0)
22
+ Requires-Dist: aioshutil (>=1.0,<2.0)
23
+ Requires-Dist: apify-client (>=1.7.1,<2.0.0)
24
+ Requires-Dist: apify-shared (>=1.1.2,<2.0.0)
25
+ Requires-Dist: colorama (>=0.4.6,<0.5.0)
26
+ Requires-Dist: crawlee (>=0.3.0,<0.4.0)
27
+ Requires-Dist: cryptography (>=42.0.4,<43.0.0)
28
+ Requires-Dist: httpx (>=0.27.0,<0.28.0)
29
+ Requires-Dist: lazy-object-proxy (>=1.10.0,<2.0.0)
30
+ Requires-Dist: psutil (>=6.0.0,<7.0.0)
31
+ Requires-Dist: pyee (>=11.0.0,<12.0.0)
32
+ Requires-Dist: scrapy (>=2.11.0,<3.0.0) ; extra == "scrapy"
33
+ Requires-Dist: sortedcollections (>=2.0.0,<3.0.0)
34
+ Requires-Dist: typing-extensions (>=4.1.0,<5.0.0)
35
+ Requires-Dist: websockets (>=10.1,<11.0)
30
36
  Project-URL: Apify Homepage, https://apify.com
31
37
  Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog
32
38
  Project-URL: Documentation, https://docs.apify.com/sdk/python/
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
4
4
 
5
5
  [tool.poetry]
6
6
  name = "apify"
7
- version = "2.0.0a1"
7
+ version = "2.0.0b2"
8
8
  description = "Apify SDK for Python"
9
9
  authors = ["Apify Technologies s.r.o. <support@apify.com>"]
10
10
  license = "Apache-2.0"
@@ -46,32 +46,41 @@ keywords = [
46
46
  # https://github.com/apify/apify-sdk-python/pull/154.
47
47
  [tool.poetry.dependencies]
48
48
  python = "^3.9"
49
- apify-client = ">=1.7.1"
50
- apify-shared = ">=1.1.2"
51
- crawlee = ">=0.3.0"
52
- cryptography = ">=42.0.0"
53
- httpx = ">=0.27.0"
54
- lazy-object-proxy = ">=1.10.0"
55
- scrapy = { version = ">=2.11.0", optional = true }
56
- typing-extensions = ">=4.1.0"
57
- websockets = ">=10.0"
49
+ aiofiles = "^22.1.0"
50
+ aioshutil = "^1.0"
51
+ apify-client = "^1.7.1"
52
+ apify-shared = "^1.1.2"
53
+ colorama = "^0.4.6"
54
+ crawlee = "^0.3.0"
55
+ cryptography = "^42.0.4"
56
+ httpx = "^0.27.0"
57
+ lazy-object-proxy = "^1.10.0"
58
+ psutil = "^6.0.0"
59
+ pyee = "^11.0.0"
60
+ scrapy = { version = "^2.11.0", optional = true }
61
+ sortedcollections = "^2.0.0"
62
+ typing-extensions = "^4.1.0"
63
+ websockets = "^10.1"
58
64
 
59
65
  [tool.poetry.group.dev.dependencies]
60
- build = "~1.2.0"
61
- filelock = "~3.15.0"
62
- griffe = "~1.2.0"
63
- mypy = "~1.11.0"
64
- pre-commit = "~3.8.0"
65
- pydoc-markdown = "~4.8.0"
66
- pytest = "~8.3.0"
67
- pytest-asyncio = "~0.24.0"
68
- pytest-cov = "~5.0.0"
69
- pytest-only = "~2.1.0"
70
- pytest-timeout = "~2.3.0"
71
- pytest-xdist = "~3.6.0"
72
- respx = "~0.21.0"
73
- ruff = "~0.6.0"
74
- setuptools = "~74.0.0" # setuptools are used by pytest but not explicitly required
66
+ build = "^1.2.0"
67
+ filelock = "^3.15.0"
68
+ mypy = "^1.11.0"
69
+ pre-commit = "^3.8.0"
70
+ pydoc-markdown = "^4.8.0"
71
+ pytest = "^8.3.0"
72
+ pytest-asyncio = "^0.24.0"
73
+ pytest-cov = "^5.0.0"
74
+ pytest-only = "^2.1.0"
75
+ pytest-timeout = "^2.3.0"
76
+ pytest-xdist = "^3.6.0"
77
+ respx = "^0.21.0"
78
+ ruff = "^0.6.0"
79
+ setuptools = "^74.0.0" # setuptools are used by pytest but not explicitly required
80
+ twine = "^5.1.0"
81
+ types-aiofiles = "^24.1.0.20240626"
82
+ types-colorama = "^0.4.15.20240311"
83
+ types-psutil = "^6.0.0.20240621"
75
84
 
76
85
  [tool.poetry.extras]
77
86
  scrapy = ["scrapy"]
@@ -182,7 +191,7 @@ warn_unused_ignores = true
182
191
  exclude = []
183
192
 
184
193
  [[tool.mypy.overrides]]
185
- module = ['scrapy', 'scrapy.*', 'lazy_object_proxy']
194
+ module = ['scrapy', 'scrapy.*', 'sortedcollections', 'lazy_object_proxy']
186
195
  ignore_missing_imports = true
187
196
 
188
197
  [tool.coverage.report]
@@ -1,22 +1,11 @@
1
1
  from importlib import metadata
2
2
 
3
- from apify_shared.consts import WebhookEventType
4
3
  from crawlee.events._types import Event
5
4
 
6
5
  from apify._actor import Actor
7
6
  from apify._configuration import Configuration
8
- from apify._models import Webhook
9
7
  from apify._proxy_configuration import ProxyConfiguration, ProxyInfo
10
8
 
11
9
  __version__ = metadata.version('apify')
12
10
 
13
- __all__ = [
14
- 'Actor',
15
- 'Event',
16
- 'Configuration',
17
- 'ProxyConfiguration',
18
- 'ProxyInfo',
19
- 'Webhook',
20
- 'WebhookEventType',
21
- '__version__',
22
- ]
11
+ __all__ = ['Actor', 'Event', 'Configuration', 'ProxyConfiguration', 'ProxyInfo', '__version__']
@@ -11,7 +11,7 @@ from pydantic import AliasChoices
11
11
  from typing_extensions import Self
12
12
 
13
13
  from apify_client import ApifyClientAsync
14
- from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars
14
+ from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars, WebhookEventType
15
15
  from apify_shared.utils import ignore_docs, maybe_extract_enum_member_value
16
16
  from crawlee import service_container
17
17
  from crawlee.events._types import Event, EventPersistStateData
@@ -19,12 +19,11 @@ from crawlee.events._types import Event, EventPersistStateData
19
19
  from apify._configuration import Configuration
20
20
  from apify._consts import EVENT_LISTENERS_TIMEOUT
21
21
  from apify._crypto import decrypt_input_secrets, load_private_key
22
- from apify._models import ActorRun
22
+ from apify._log import logger
23
23
  from apify._platform_event_manager import EventManager, LocalEventManager, PlatformEventManager
24
24
  from apify._proxy_configuration import ProxyConfiguration
25
25
  from apify._utils import get_system_info, is_running_in_ipython
26
26
  from apify.apify_storage_client import ApifyStorageClient
27
- from apify.log import logger
28
27
  from apify.storages import Dataset, KeyValueStore, RequestQueue
29
28
 
30
29
  if TYPE_CHECKING:
@@ -33,8 +32,6 @@ if TYPE_CHECKING:
33
32
 
34
33
  from crawlee.proxy_configuration import _NewUrlFunction
35
34
 
36
- from apify._models import Webhook
37
-
38
35
 
39
36
  MainReturnType = TypeVar('MainReturnType')
40
37
 
@@ -536,8 +533,8 @@ class _ActorType:
536
533
  memory_mbytes: int | None = None,
537
534
  timeout: timedelta | None = None,
538
535
  wait_for_finish: int | None = None,
539
- webhooks: list[Webhook] | None = None,
540
- ) -> ActorRun:
536
+ webhooks: list[dict] | None = None,
537
+ ) -> dict:
541
538
  """Run an Actor on the Apify platform.
542
539
 
543
540
  Unlike `Actor.call`, this method just starts the run without waiting for finish.
@@ -558,6 +555,10 @@ class _ActorType:
558
555
  webhooks: Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks) associated with
559
556
  the Actor run which can be used to receive a notification, e.g. when the Actor finished or failed.
560
557
  If you already have a webhook set up for the Actor or task, you do not have to add it again here.
558
+ Each webhook is represented by a dictionary containing these items:
559
+ * `event_types`: list of `WebhookEventType` values which trigger the webhook
560
+ * `request_url`: URL to which to send the webhook HTTP request
561
+ * `payload_template` (optional): Optional template for the request payload
561
562
 
562
563
  Returns:
563
564
  Info about the started Actor run
@@ -566,25 +567,16 @@ class _ActorType:
566
567
 
567
568
  client = self.new_client(token=token) if token else self._apify_client
568
569
 
569
- if webhooks:
570
- serialized_webhooks = [
571
- hook.model_dump(by_alias=True, exclude_unset=True, exclude_defaults=True) for hook in webhooks
572
- ]
573
- else:
574
- serialized_webhooks = None
575
-
576
- api_result = await client.actor(actor_id).start(
570
+ return await client.actor(actor_id).start(
577
571
  run_input=run_input,
578
572
  content_type=content_type,
579
573
  build=build,
580
574
  memory_mbytes=memory_mbytes,
581
575
  timeout_secs=int(timeout.total_seconds()) if timeout is not None else None,
582
576
  wait_for_finish=wait_for_finish,
583
- webhooks=serialized_webhooks,
577
+ webhooks=webhooks,
584
578
  )
585
579
 
586
- return ActorRun.model_validate(api_result)
587
-
588
580
  async def abort(
589
581
  self,
590
582
  run_id: str,
@@ -592,7 +584,7 @@ class _ActorType:
592
584
  token: str | None = None,
593
585
  status_message: str | None = None,
594
586
  gracefully: bool | None = None,
595
- ) -> ActorRun:
587
+ ) -> dict:
596
588
  """Abort given Actor run on the Apify platform using the current user account.
597
589
 
598
590
  The user account is determined by the `APIFY_TOKEN` environment variable.
@@ -615,9 +607,7 @@ class _ActorType:
615
607
  if status_message:
616
608
  await client.run(run_id).update(status_message=status_message)
617
609
 
618
- api_result = await client.run(run_id).abort(gracefully=gracefully)
619
-
620
- return ActorRun.model_validate(api_result)
610
+ return await client.run(run_id).abort(gracefully=gracefully)
621
611
 
622
612
  async def call(
623
613
  self,
@@ -629,9 +619,9 @@ class _ActorType:
629
619
  build: str | None = None,
630
620
  memory_mbytes: int | None = None,
631
621
  timeout: timedelta | None = None,
632
- webhooks: list[Webhook] | None = None,
622
+ webhooks: list[dict] | None = None,
633
623
  wait: timedelta | None = None,
634
- ) -> ActorRun | None:
624
+ ) -> dict | None:
635
625
  """Start an Actor on the Apify Platform and wait for it to finish before returning.
636
626
 
637
627
  It waits indefinitely, unless the wait argument is provided.
@@ -660,25 +650,16 @@ class _ActorType:
660
650
 
661
651
  client = self.new_client(token=token) if token else self._apify_client
662
652
 
663
- if webhooks:
664
- serialized_webhooks = [
665
- hook.model_dump(by_alias=True, exclude_unset=True, exclude_defaults=True) for hook in webhooks
666
- ]
667
- else:
668
- serialized_webhooks = None
669
-
670
- api_result = await client.actor(actor_id).call(
653
+ return await client.actor(actor_id).call(
671
654
  run_input=run_input,
672
655
  content_type=content_type,
673
656
  build=build,
674
657
  memory_mbytes=memory_mbytes,
675
658
  timeout_secs=int(timeout.total_seconds()) if timeout is not None else None,
676
- webhooks=serialized_webhooks,
659
+ webhooks=webhooks,
677
660
  wait_secs=int(wait.total_seconds()) if wait is not None else None,
678
661
  )
679
662
 
680
- return ActorRun.model_validate(api_result)
681
-
682
663
  async def call_task(
683
664
  self,
684
665
  task_id: str,
@@ -687,10 +668,10 @@ class _ActorType:
687
668
  build: str | None = None,
688
669
  memory_mbytes: int | None = None,
689
670
  timeout: timedelta | None = None,
690
- webhooks: list[Webhook] | None = None,
671
+ webhooks: list[dict] | None = None,
691
672
  wait: timedelta | None = None,
692
673
  token: str | None = None,
693
- ) -> ActorRun | None:
674
+ ) -> dict | None:
694
675
  """Start an Actor task on the Apify Platform and wait for it to finish before returning.
695
676
 
696
677
  It waits indefinitely, unless the wait argument is provided.
@@ -722,24 +703,15 @@ class _ActorType:
722
703
 
723
704
  client = self.new_client(token=token) if token else self._apify_client
724
705
 
725
- if webhooks:
726
- serialized_webhooks = [
727
- hook.model_dump(by_alias=True, exclude_unset=True, exclude_defaults=True) for hook in webhooks
728
- ]
729
- else:
730
- serialized_webhooks = None
731
-
732
- api_result = await client.task(task_id).call(
706
+ return await client.task(task_id).call(
733
707
  task_input=task_input,
734
708
  build=build,
735
709
  memory_mbytes=memory_mbytes,
736
710
  timeout_secs=int(timeout.total_seconds()) if timeout is not None else None,
737
- webhooks=serialized_webhooks,
711
+ webhooks=webhooks,
738
712
  wait_secs=int(wait.total_seconds()) if wait is not None else None,
739
713
  )
740
714
 
741
- return ActorRun.model_validate(api_result)
742
-
743
715
  async def metamorph(
744
716
  self,
745
717
  target_actor_id: str,
@@ -824,12 +796,14 @@ class _ActorType:
824
796
 
825
797
  async def add_webhook(
826
798
  self,
827
- webhook: Webhook,
828
799
  *,
800
+ event_types: list[WebhookEventType],
801
+ request_url: str,
802
+ payload_template: str | None = None,
829
803
  ignore_ssl_errors: bool | None = None,
830
804
  do_not_retry: bool | None = None,
831
805
  idempotency_key: str | None = None,
832
- ) -> None:
806
+ ) -> dict | None:
833
807
  """Create an ad-hoc webhook for the current Actor run.
834
808
 
835
809
  This webhook lets you receive a notification when the Actor run finished or failed.
@@ -840,7 +814,9 @@ class _ActorType:
840
814
  For more information about Apify Actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).
841
815
 
842
816
  Args:
843
- webhook: The webhook to be added
817
+ event_types: List of event types that should trigger the webhook. At least one is required.
818
+ request_url: URL that will be invoked once the webhook is triggered.
819
+ payload_template: Specification of the payload that will be sent to request_url
844
820
  ignore_ssl_errors: Whether the webhook should ignore SSL errors returned by request_url
845
821
  do_not_retry: Whether the webhook should retry sending the payload to request_url upon failure.
846
822
  idempotency_key: A unique identifier of a webhook. You can use it to ensure that you won't create
@@ -853,17 +829,17 @@ class _ActorType:
853
829
 
854
830
  if not self.is_at_home():
855
831
  self.log.error('Actor.add_webhook() is only supported when running on the Apify platform.')
856
- return
832
+ return None
857
833
 
858
834
  # If is_at_home() is True, config.actor_run_id is always set
859
835
  if not self._configuration.actor_run_id:
860
836
  raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.')
861
837
 
862
- await self._apify_client.webhooks().create(
838
+ return await self._apify_client.webhooks().create(
863
839
  actor_run_id=self._configuration.actor_run_id,
864
- event_types=webhook.event_types,
865
- request_url=webhook.request_url,
866
- payload_template=webhook.payload_template,
840
+ event_types=event_types,
841
+ request_url=request_url,
842
+ payload_template=payload_template,
867
843
  ignore_ssl_errors=ignore_ssl_errors,
868
844
  do_not_retry=do_not_retry,
869
845
  idempotency_key=idempotency_key,
@@ -874,7 +850,7 @@ class _ActorType:
874
850
  status_message: str,
875
851
  *,
876
852
  is_terminal: bool | None = None,
877
- ) -> ActorRun | None:
853
+ ) -> dict | None:
878
854
  """Set the status message for the current Actor run.
879
855
 
880
856
  Args:
@@ -895,12 +871,10 @@ class _ActorType:
895
871
  if not self._configuration.actor_run_id:
896
872
  raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.')
897
873
 
898
- api_result = await self._apify_client.run(self._configuration.actor_run_id).update(
874
+ return await self._apify_client.run(self._configuration.actor_run_id).update(
899
875
  status_message=status_message, is_status_message_terminal=is_terminal
900
876
  )
901
877
 
902
- return ActorRun.model_validate(api_result)
903
-
904
878
  async def create_proxy_configuration(
905
879
  self,
906
880
  *,
@@ -11,5 +11,5 @@ logger_name = __name__.split('.')[0]
11
11
  logger = logging.getLogger(logger_name)
12
12
 
13
13
 
14
- class ActorLogFormatter(CrawleeLogFormatter): # noqa: D101 Inherited from parent class
14
+ class ActorLogFormatter(CrawleeLogFormatter): # Inherited from parent class
15
15
  pass
@@ -20,7 +20,7 @@ from crawlee.events._types import (
20
20
  EventSystemInfoData,
21
21
  )
22
22
 
23
- from apify.log import logger
23
+ from apify._log import logger
24
24
 
25
25
  if TYPE_CHECKING:
26
26
  from types import TracebackType
@@ -94,11 +94,6 @@ class EventWithoutData(BaseModel):
94
94
  data: Any = None
95
95
 
96
96
 
97
- class DeprecatedEvent(BaseModel):
98
- name: Literal['cpuInfo']
99
- data: Annotated[dict[str, Any], Field(default_factory=dict)]
100
-
101
-
102
97
  class UnknownEvent(BaseModel):
103
98
  name: str
104
99
  data: Annotated[dict[str, Any], Field(default_factory=dict)]
@@ -114,13 +109,12 @@ EventMessage = Union[
114
109
  ]
115
110
 
116
111
 
117
- event_data_adapter: TypeAdapter[EventMessage | DeprecatedEvent | UnknownEvent] = TypeAdapter(
112
+ event_data_adapter: TypeAdapter[EventMessage | UnknownEvent] = TypeAdapter(
118
113
  Union[
119
114
  Annotated[
120
115
  EventMessage,
121
116
  Discriminator('name'),
122
117
  ],
123
- DeprecatedEvent,
124
118
  UnknownEvent,
125
119
  ]
126
120
  )
@@ -195,9 +189,6 @@ class PlatformEventManager(EventManager):
195
189
  try:
196
190
  parsed_message = event_data_adapter.validate_json(message)
197
191
 
198
- if isinstance(parsed_message, DeprecatedEvent):
199
- continue
200
-
201
192
  if isinstance(parsed_message, UnknownEvent):
202
193
  logger.info(
203
194
  f'Unknown message received: event_name={parsed_message.name}, '
@@ -16,7 +16,7 @@ from crawlee.proxy_configuration import ProxyInfo as CrawleeProxyInfo
16
16
  from crawlee.proxy_configuration import _NewUrlFunction
17
17
 
18
18
  from apify._configuration import Configuration
19
- from apify.log import logger
19
+ from apify._log import logger
20
20
 
21
21
  if TYPE_CHECKING:
22
22
  from apify_client import ApifyClientAsync
@@ -60,20 +60,18 @@ class DatasetClient(BaseDatasetClient):
60
60
  view: str | None = None,
61
61
  ) -> DatasetItemsListPage:
62
62
  return DatasetItemsListPage.model_validate(
63
- vars(
64
- await self._client.list_items(
65
- offset=offset,
66
- limit=limit,
67
- clean=clean,
68
- desc=desc,
69
- fields=fields,
70
- omit=omit,
71
- unwind=unwind,
72
- skip_empty=skip_empty,
73
- skip_hidden=skip_hidden,
74
- flatten=flatten,
75
- view=view,
76
- )
63
+ await self._client.list_items(
64
+ offset=offset,
65
+ limit=limit,
66
+ clean=clean,
67
+ desc=desc,
68
+ fields=fields,
69
+ omit=omit,
70
+ unwind=unwind,
71
+ skip_empty=skip_empty,
72
+ skip_hidden=skip_hidden,
73
+ flatten=flatten,
74
+ view=view,
77
75
  )
78
76
  )
79
77
 
@@ -93,7 +93,7 @@ class ApifyHttpProxyMiddleware:
93
93
  request: Request,
94
94
  exception: Exception,
95
95
  spider: Spider,
96
- ) -> None:
96
+ ) -> None | Request:
97
97
  """Process an exception that occurs during request processing.
98
98
 
99
99
  Args:
@@ -102,9 +102,8 @@ class ApifyHttpProxyMiddleware:
102
102
  spider: Scrapy Spider object.
103
103
 
104
104
  Returns:
105
- Returning None, meaning Scrapy will continue processing this exception, executing any other
106
- process_exception() methods of installed middleware, until no middleware is left and the default
107
- exception handling kicks in.
105
+ If a TunnelError occurs, return the request object to halt its processing in the middleware pipeline.
106
+ Return None otherwise to allow the continuation of request processing.
108
107
  """
109
108
  Actor.log.debug(
110
109
  f'ApifyHttpProxyMiddleware.process_exception: request={request}, exception={exception}, spider={spider}',
@@ -115,6 +114,9 @@ class ApifyHttpProxyMiddleware:
115
114
  f'ApifyHttpProxyMiddleware: TunnelError occurred for request="{request}", '
116
115
  'reason="{exception}", skipping...'
117
116
  )
117
+ return request
118
+
119
+ return None
118
120
 
119
121
  async def _get_new_proxy_url(self: ApifyHttpProxyMiddleware) -> ParseResult:
120
122
  """Get a new proxy URL.
@@ -1,110 +0,0 @@
1
- # ruff: noqa: TCH001 TCH002 TCH003 (Pydantic)
2
- from __future__ import annotations
3
-
4
- from datetime import datetime, timedelta
5
- from typing import Annotated
6
-
7
- from pydantic import BaseModel, BeforeValidator, ConfigDict, Field
8
-
9
- from apify_shared.consts import ActorJobStatus, MetaOrigin, WebhookEventType
10
- from crawlee._utils.models import timedelta_ms
11
- from crawlee._utils.urls import validate_http_url
12
-
13
-
14
- class Webhook(BaseModel):
15
- __model_config__ = ConfigDict(populate_by_name=True)
16
-
17
- event_types: Annotated[
18
- list[WebhookEventType],
19
- Field(description='Event types that should trigger the webhook'),
20
- ]
21
- request_url: Annotated[
22
- str,
23
- Field(description='URL that the webhook should call'),
24
- BeforeValidator(validate_http_url),
25
- ]
26
- payload_template: Annotated[
27
- str | None,
28
- Field(description='Template for the payload sent by the webook'),
29
- ] = None
30
-
31
-
32
- class ActorRunMeta(BaseModel):
33
- __model_config__ = ConfigDict(populate_by_name=True)
34
-
35
- origin: Annotated[MetaOrigin, Field()]
36
-
37
-
38
- class ActorRunStats(BaseModel):
39
- __model_config__ = ConfigDict(populate_by_name=True)
40
-
41
- input_body_len: Annotated[int, Field(alias='inputBodyLen')]
42
- restart_count: Annotated[int, Field(alias='restartCount')]
43
- resurrect_count: Annotated[int, Field(alias='resurrectCount')]
44
- mem_avg_bytes: Annotated[float | None, Field(alias='memAvgBytes')] = None
45
- mem_max_bytes: Annotated[int | None, Field(alias='memMaxBytes')] = None
46
- mem_current_bytes: Annotated[int | None, Field(alias='memCurrentBytes')] = None
47
- cpu_avg_usage: Annotated[float | None, Field(alias='cpuAvgUsage')] = None
48
- cpu_max_usage: Annotated[float | None, Field(alias='cpuMaxUsage')] = None
49
- cpu_current_usage: Annotated[float | None, Field(alias='cpuCurrentUsage')] = None
50
- net_rx_bytes: Annotated[int | None, Field(alias='netRxBytes')] = None
51
- net_tx_bytes: Annotated[int | None, Field(alias='netTxBytes')] = None
52
- duration: Annotated[timedelta_ms | None, Field(alias='durationMillis')] = None
53
- run_time: Annotated[timedelta | None, Field(alias='runTimeSecs')] = None
54
- metamorph: Annotated[int | None, Field(alias='metamorph')] = None
55
- compute_units: Annotated[float, Field(alias='computeUnits')]
56
-
57
-
58
- class ActorRunOptions(BaseModel):
59
- __model_config__ = ConfigDict(populate_by_name=True)
60
-
61
- build: str
62
- timeout: Annotated[timedelta, Field(alias='timeoutSecs')]
63
- memory_mbytes: Annotated[int, Field(alias='memoryMbytes')]
64
- disk_mbytes: Annotated[int, Field(alias='diskMbytes')]
65
-
66
-
67
- class ActorRunUsage(BaseModel):
68
- __model_config__ = ConfigDict(populate_by_name=True)
69
-
70
- actor_compute_units: Annotated[float | None, Field(alias='ACTOR_COMPUTE_UNITS')] = None
71
- dataset_reads: Annotated[float | None, Field(alias='DATASET_READS')] = None
72
- dataset_writes: Annotated[float | None, Field(alias='DATASET_WRITES')] = None
73
- key_value_store_reads: Annotated[float | None, Field(alias='KEY_VALUE_STORE_READS')] = None
74
- key_value_store_writes: Annotated[float | None, Field(alias='KEY_VALUE_STORE_WRITES')] = None
75
- key_value_store_lists: Annotated[float | None, Field(alias='KEY_VALUE_STORE_LISTS')] = None
76
- request_queue_reads: Annotated[float | None, Field(alias='REQUEST_QUEUE_READS')] = None
77
- request_queue_writes: Annotated[float | None, Field(alias='REQUEST_QUEUE_WRITES')] = None
78
- data_transfer_internal_gbytes: Annotated[float | None, Field(alias='DATA_TRANSFER_INTERNAL_GBYTES')] = None
79
- data_transfer_external_gbytes: Annotated[float | None, Field(alias='DATA_TRANSFER_EXTERNAL_GBYTES')] = None
80
- proxy_residential_transfer_gbytes: Annotated[float | None, Field(alias='PROXY_RESIDENTIAL_TRANSFER_GBYTES')] = None
81
- proxy_serps: Annotated[float | None, Field(alias='PROXY_SERPS')] = None
82
-
83
-
84
- class ActorRun(BaseModel):
85
- __model_config__ = ConfigDict(populate_by_name=True)
86
-
87
- id: Annotated[str, Field(alias='id')]
88
- act_id: Annotated[str, Field(alias='actId')]
89
- user_id: Annotated[str, Field(alias='userId')]
90
- actor_task_id: Annotated[str | None, Field(alias='actorTaskId')] = None
91
- started_at: Annotated[datetime, Field(alias='startedAt')]
92
- finished_at: Annotated[datetime | None, Field(alias='finishedAt')] = None
93
- status: Annotated[ActorJobStatus, Field(alias='status')]
94
- status_message: Annotated[str | None, Field(alias='statusMessage')] = None
95
- is_status_message_terminal: Annotated[bool | None, Field(alias='isStatusMessageTerminal')] = None
96
- meta: Annotated[ActorRunMeta, Field(alias='meta')]
97
- stats: Annotated[ActorRunStats, Field(alias='stats')]
98
- options: Annotated[ActorRunOptions, Field(alias='options')]
99
- build_id: Annotated[str, Field(alias='buildId')]
100
- exit_code: Annotated[int | None, Field(alias='exitCode')] = None
101
- default_key_value_store_id: Annotated[str, Field(alias='defaultKeyValueStoreId')]
102
- default_dataset_id: Annotated[str, Field(alias='defaultDatasetId')]
103
- default_request_queue_id: Annotated[str, Field(alias='defaultRequestQueueId')]
104
- build_number: Annotated[str | None, Field(alias='buildNumber')] = None
105
- container_url: Annotated[str, Field(alias='containerUrl')]
106
- is_container_server_ready: Annotated[bool | None, Field(alias='isContainerServerReady')] = None
107
- git_branch_name: Annotated[str | None, Field(alias='gitBranchName')] = None
108
- usage: Annotated[ActorRunUsage | None, Field(alias='usage')] = None
109
- usage_total_usd: Annotated[float | None, Field(alias='usageTotalUsd')] = None
110
- usage_usd: Annotated[ActorRunUsage | None, Field(alias='usageUsd')] = None
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes