scrapely-python 0.1.0__py3-none-any.whl

scrapely/py.typed

@@ -0,0 +1,2 @@
+# Marker file for PEP 561.
+# The presence of this file indicates that the package supports type checking.

scrapely/request_loaders/__init__.py

@@ -0,0 +1,5 @@
+"""Scrapely request loaders - extends Crawlee's request loaders with Scrapely-specific functionality."""
+
+from ._scrapely_request_list import ScrapelyRequestList
+
+__all__ = ['ScrapelyRequestList']

scrapely/request_loaders/_scrapely_request_list.py

@@ -0,0 +1,169 @@
+"""ScrapelyRequestList - a request list that parses the standard Scrapely/Apify requestListSources input format."""
+
+from __future__ import annotations
+
+import asyncio
+import re
+from copy import deepcopy
+from itertools import chain
+from typing import Annotated, Any
+
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
+from pydantic.alias_generators import to_camel
+
+from crawlee._types import HttpMethod, JsonSerializable
+from crawlee.http_clients import HttpClient, ImpitHttpClient
+from crawlee import Request
+from crawlee.request_loaders import RequestList
+
+# Regex for extracting URLs from remote text files (matches URLs without commas)
+URL_NO_COMMAS_REGEX = re.compile(
+    r'https?:\/\/(www\.)?([^\W_]|[^\W_][-\w0-9@:%._+~#=]{0,254}[^\W_])\.[a-z]{2,63}(:\d{1,5})?(\/[-\w@:%+.~#?&/=()]*)?'
+)
+
+
+class _RequestDetails(BaseModel):
+    """Common request details that can be applied to all requests from a source entry."""
+
+    model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
+
+    method: HttpMethod = 'GET'
+    payload: str = ''
+    headers: Annotated[dict[str, str], Field(default_factory=dict)]
+    user_data: Annotated[dict[str, JsonSerializable], Field(default_factory=dict)]
+
+
+class _RequestsFromUrlInput(_RequestDetails):
+    """Input entry for a remote URL list (requestsFromUrl)."""
+
+    requests_from_url: str
+
+
+class _SimpleUrlInput(_RequestDetails):
+    """Input entry for a direct URL."""
+
+    url: str
+
+
+# Pydantic type adapter for parsing the requestListSources input format
+url_input_adapter = TypeAdapter(list[_RequestsFromUrlInput | _SimpleUrlInput])
+
+
+class ScrapelyRequestList(RequestList):
+    """A request list that can be constructed from the standard Scrapely/Apify ``requestListSources`` input format.
+
+    This extends the Crawlee ``RequestList`` with the ability to parse the request list sources input commonly
+    used in Scrapely Actors. It supports two kinds of entries:
+
+    - **Direct URLs** - entries with a ``url`` key are converted to requests directly.
+    - **Remote URL lists** - entries with a ``requestsFromUrl`` key point to a remote resource (e.g. a plain-text
+      file). The resource is fetched and all URLs found in the response body are extracted and converted to requests.
+
+    Both kinds of entries can optionally specify ``method``, ``payload``, ``headers``, and ``userData`` fields that
+    will be applied to every request created from that entry.
+
+    Usage:
+        ```python
+        from scrapely import Actor
+        from scrapely.request_loaders import ScrapelyRequestList
+
+        async with Actor:
+            actor_input = await Actor.get_input() or {}
+            request_list = await ScrapelyRequestList.open(
+                request_list_sources_input=actor_input.get('requestListSources', []),
+            )
+        ```
+    """
+
+    @classmethod
+    async def open(
+        cls,
+        *,
+        name: str | None = None,
+        request_list_sources_input: list[dict[str, Any]] | None = None,
+        http_client: HttpClient | None = None,
+    ) -> ScrapelyRequestList:
+        """Create a new ``ScrapelyRequestList`` from the standard Scrapely request list sources input.
+
+        Each entry in ``request_list_sources_input`` is a dict with either a ``url`` key (for a direct URL) or
+        a ``requestsFromUrl`` key (for a remote resource whose response body is scanned for URLs). Optional keys
+        ``method``, ``payload``, ``headers``, and ``userData`` are applied to every request produced from that entry.
+
+        Args:
+            name: An optional name for the request list, used for state persistence.
+            request_list_sources_input: A list of request source dicts in the standard Scrapely format. Each dict
+                must contain either a ``url`` key or a ``requestsFromUrl`` key. If ``None`` or empty, an empty
+                request list is returned.
+            http_client: HTTP client used to fetch remote URL lists (entries with ``requestsFromUrl``). Defaults to
+                ``ImpitHttpClient`` if not provided.
+
+        Returns:
+            A new ``ScrapelyRequestList`` populated with the resolved requests.
+        """
+        request_list_sources_input = request_list_sources_input or []
+
+        if not http_client:
+            http_client = ImpitHttpClient()
+
+        url_inputs = url_input_adapter.validate_python(request_list_sources_input)
+
+        simple_url_inputs = [url_input for url_input in url_inputs if isinstance(url_input, _SimpleUrlInput)]
+        remote_url_inputs = [url_input for url_input in url_inputs if isinstance(url_input, _RequestsFromUrlInput)]
+
+        simple_url_requests = cls._create_requests_from_input(simple_url_inputs)
+        remote_url_requests = await cls._fetch_requests_from_url(remote_url_inputs, http_client)
+
+        return ScrapelyRequestList(name=name, requests=simple_url_requests + remote_url_requests)
+
+    @classmethod
+    async def _fetch_requests_from_url(
+        cls,
+        remote_url_requests_inputs: list[_RequestsFromUrlInput],
+        http_client: HttpClient,
+    ) -> list[Request]:
+        """Fetch all remote URL sources concurrently and return the extracted requests.
+
+        For each entry, a GET request is sent to the ``requests_from_url`` URL. All URLs matching
+        ``URL_NO_COMMAS_REGEX`` are extracted from the response body and turned into ``Request`` objects,
+        inheriting ``method``, ``payload``, ``headers``, and ``user_data`` from the source entry.
+        """
+        async with asyncio.TaskGroup() as tg:
+            tasks = [
+                tg.create_task(cls._process_remote_url(request_input, http_client))
+                for request_input in remote_url_requests_inputs
+            ]
+        return list(chain.from_iterable(task.result() for task in tasks))
+
+    @staticmethod
+    def _create_requests_from_input(simple_url_inputs: list[_SimpleUrlInput]) -> list[Request]:
+        """Create ``Request`` objects from simple URL inputs."""
+        return [
+            Request.from_url(
+                method=request_input.method,
+                url=request_input.url,
+                payload=request_input.payload.encode('utf-8'),
+                headers=request_input.headers,
+                user_data=request_input.user_data,
+            )
+            for request_input in simple_url_inputs
+        ]
+
+    @staticmethod
+    async def _process_remote_url(request_input: _RequestsFromUrlInput, http_client: HttpClient) -> list[Request]:
+        """Fetch a remote URL and extract links from the response body."""
+        http_response = await http_client.send_request(method='GET', url=request_input.requests_from_url)
+        response_body = await http_response.read()
+        matches = re.finditer(URL_NO_COMMAS_REGEX, response_body.decode('utf-8', errors='replace'))
+
+        return [
+            Request.from_url(
+                url=match.group(0),
+                method=request_input.method,
+                payload=request_input.payload.encode('utf-8'),
+                headers=request_input.headers,
+                # Deep-copy so Request.from_url (which writes __crawlee into the dict) cannot corrupt
+                # the shared input, and nested JSON values are not aliased across the requests.
+                user_data=deepcopy(request_input.user_data),
+            )
+            for match in matches
+        ]

scrapely/storage/__init__.py

@@ -0,0 +1,48 @@
+"""Scrapely storage: Crawlee StorageClient backends + HTTP helpers."""
+
+from crawlee import Request
+from crawlee.storage_clients import MemoryStorageClient, StorageClient
+from crawlee.storages import Dataset, KeyValueStore, RequestQueue
+
+from .api_client import ApiClient, ApiClientError
+from .client import ApiStorageClient
+from ._file_system import ScrapelyFileSystemStorageClient
+from .patch_storage import (
+    disconnect_storage,
+    get_api_storage_factory,
+    get_auto_saved_value,
+    get_smart_storage_client,
+    is_api_storage_configured,
+    persist_autosaved_values,
+    set_api_storage,
+    set_storage_client,
+)
+from .scrapely_storage_client import ScrapelyStorageClient
+from .smart_scrapely_storage_client import SmartScrapelyStorageClient
+
+# Apify-style public alias
+FileSystemStorageClient = ScrapelyFileSystemStorageClient
+
+__all__ = [
+    'ApiClient',
+    'ApiClientError',
+    'ApiStorageClient',
+    'ScrapelyStorageClient',
+    'SmartScrapelyStorageClient',
+    'FileSystemStorageClient',
+    'ScrapelyFileSystemStorageClient',
+    'MemoryStorageClient',
+    'StorageClient',
+    'set_api_storage',
+    'set_storage_client',
+    'get_api_storage_factory',
+    'get_smart_storage_client',
+    'is_api_storage_configured',
+    'disconnect_storage',
+    'get_auto_saved_value',
+    'persist_autosaved_values',
+    'Dataset',
+    'KeyValueStore',
+    'RequestQueue',
+    'Request',
+]

scrapely/storage/_file_system/__init__.py

@@ -0,0 +1,9 @@
+"""Scrapely file-system storage clients (Apify FileSystemStorageClient shape)."""
+
+from ._key_value_store_client import ScrapelyFileSystemKeyValueStoreClient
+from ._storage_client import ScrapelyFileSystemStorageClient
+
+__all__ = [
+    'ScrapelyFileSystemKeyValueStoreClient',
+    'ScrapelyFileSystemStorageClient',
+]

scrapely/storage/_file_system/_key_value_store_client.py

@@ -0,0 +1,155 @@
+"""File-system KVS client that preserves INPUT on purge (Apify parity)."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from itertools import chain
+from pathlib import Path
+from typing import Any, Optional
+
+from typing_extensions import Self, override
+
+from crawlee._consts import METADATA_FILENAME
+from crawlee._utils.file import atomic_write, infer_mime_type, json_dumps
+from crawlee.configuration import Configuration as CrawleeConfiguration
+from crawlee.storage_clients._file_system import FileSystemKeyValueStoreClient
+from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata
+
+from scrapely.configuration import Configuration as ScrapelyConfiguration
+
+logger = logging.getLogger(__name__)
+
+
+class ScrapelyFileSystemKeyValueStoreClient(FileSystemKeyValueStoreClient):
+    """File-system KVS that keeps metadata and INPUT when purged."""
+
+    def __init__(
+        self,
+        *,
+        metadata: KeyValueStoreMetadata,
+        path_to_kvs: Path,
+        lock: asyncio.Lock,
+    ) -> None:
+        super().__init__(metadata=metadata, path_to_kvs=path_to_kvs, lock=lock)
+        global_configuration = ScrapelyConfiguration.get_global_config()
+        self._input_key = global_configuration.input_key
+        self._input_key_filename = global_configuration.input_key
+
+    @override
+    @classmethod
+    async def open(
+        cls,
+        *,
+        id: Optional[str],
+        name: Optional[str],
+        alias: Optional[str],
+        configuration: CrawleeConfiguration,
+    ) -> Self:
+        client = await super().open(id=id, name=name, alias=alias, configuration=configuration)
+
+        scrapely_config = ScrapelyConfiguration.get_global_config()
+        client._input_key = scrapely_config.input_key  # noqa: SLF001
+        input_key_filename = cls._get_input_key_file_name(
+            path_to_kvs=client.path_to_kvs,
+            input_key=scrapely_config.input_key,
+        )
+        client._input_key_filename = input_key_filename  # noqa: SLF001
+        input_file_path = client.path_to_kvs / input_key_filename
+        input_file_metadata_path = client.path_to_kvs / f'{input_file_path.name}.{METADATA_FILENAME}'
+        if input_file_path.exists() and not input_file_metadata_path.exists():
+            await cls._create_missing_metadata_for_input_file(
+                key=scrapely_config.input_key,
+                record_path=input_file_path,
+            )
+
+        return client
+
+    @override
+    async def purge(self) -> None:
+        """Delete all files except store metadata and INPUT (+ its metadata sidecar)."""
+        async with self._lock:
+            files_to_keep = {
+                self._input_key_filename,
+                f'{self._input_key_filename}.{METADATA_FILENAME}',
+                METADATA_FILENAME,
+            }
+
+            async with asyncio.TaskGroup() as tg:
+                for file_path in self.path_to_kvs.glob('*'):
+                    if file_path.name in files_to_keep:
+                        continue
+                    if file_path.is_file():
+                        tg.create_task(asyncio.to_thread(file_path.unlink, missing_ok=True))
+
+            await self._update_metadata(
+                update_accessed_at=True,
+                update_modified_at=True,
+            )
+
+    @override
+    async def get_value(self, *, key: str) -> Optional[KeyValueStoreRecord]:
+        return await super().get_value(key=self._resolve_input_key(key))
+
+    @override
+    async def set_value(self, *, key: str, value: Any, content_type: Optional[str] = None) -> None:
+        await super().set_value(key=self._resolve_input_key(key), value=value, content_type=content_type)
+
+    @override
+    async def record_exists(self, *, key: str) -> bool:
+        return await super().record_exists(key=self._resolve_input_key(key))
+
+    @override
+    async def delete_value(self, *, key: str) -> None:
+        await super().delete_value(key=self._resolve_input_key(key))
+
+    @override
+    async def get_public_url(self, *, key: str) -> str:
+        return await super().get_public_url(key=self._resolve_input_key(key))
+
+    def _resolve_input_key(self, key: str) -> str:
+        return self._input_key_filename if key == self._input_key else key
+
+    @staticmethod
+    async def _create_missing_metadata_for_input_file(key: str, record_path: Path) -> None:
+        try:
+            content = await asyncio.to_thread(record_path.read_bytes)
+        except FileNotFoundError:
+            logger.warning('Input file disappeared from path: "%s"', record_path)
+            return
+
+        size = len(content)
+        if record_path.suffix == '.json':
+            value: Any = json.loads(content.decode('utf-8'))
+        elif record_path.suffix == '.txt':
+            value = content.decode('utf-8')
+        elif record_path.suffix == '':
+            try:
+                value = json.loads(content.decode('utf-8'))
+            except json.JSONDecodeError:
+                value = content
+        else:
+            value = content
+
+        content_type = infer_mime_type(value)
+        record_metadata = KeyValueStoreRecordMetadata(key=key, content_type=content_type, size=size)
+        record_metadata_filepath = record_path.with_name(f'{record_path.name}.{METADATA_FILENAME}')
+        record_metadata_content = await json_dumps(record_metadata.model_dump())
+        await atomic_write(record_metadata_filepath, record_metadata_content)
+
+    @staticmethod
+    def _get_input_key_file_name(path_to_kvs: Path, input_key: str) -> str:
+        found_input_files = set()
+        for file_path in chain(path_to_kvs.glob(f'{input_key}.*'), path_to_kvs.glob(f'{input_key}')):
+            if str(file_path).endswith(METADATA_FILENAME):
+                continue
+            found_input_files.add(file_path.name)
+
+        if len(found_input_files) > 1:
+            raise RuntimeError(f'Only one input file is allowed. Following input files found: {found_input_files}')
+
+        if len(found_input_files) == 1:
+            return found_input_files.pop()
+
+        return input_key

scrapely/storage/_file_system/_storage_client.py

@@ -0,0 +1,50 @@
+"""Scrapely file-system StorageClient (ApifyFileSystemStorageClient shape)."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Optional
+
+from typing_extensions import override
+
+from crawlee.configuration import Configuration
+from crawlee.storage_clients import FileSystemStorageClient
+
+from ._key_value_store_client import ScrapelyFileSystemKeyValueStoreClient
+
+if TYPE_CHECKING:
+    from collections.abc import Hashable
+
+    from crawlee.storage_clients._file_system import FileSystemKeyValueStoreClient
+
+
+class ScrapelyFileSystemStorageClient(FileSystemStorageClient):
+    """File-system storage client with Scrapely KVS purge that preserves INPUT.
+
+    The only difference from Crawlee's ``FileSystemStorageClient`` is that it uses
+    ``ScrapelyFileSystemKeyValueStoreClient`` for key-value stores, which overrides
+    ``purge`` to keep the metadata file and the INPUT file.
+    """
+
+    @override
+    def get_storage_client_cache_key(self, configuration: Configuration) -> Hashable:
+        # Same cache key as FileSystemStorageClient to avoid double-purge.
+        return FileSystemStorageClient().get_storage_client_cache_key(configuration)
+
+    @override
+    async def create_kvs_client(
+        self,
+        *,
+        id: Optional[str] = None,
+        name: Optional[str] = None,
+        alias: Optional[str] = None,
+        configuration: Optional[Configuration] = None,
+    ) -> FileSystemKeyValueStoreClient:
+        configuration = configuration or Configuration.get_global_configuration()
+        client = await ScrapelyFileSystemKeyValueStoreClient.open(
+            id=id,
+            name=name,
+            alias=alias,
+            configuration=configuration,
+        )
+        await self._purge_if_needed(client, configuration)
+        return client

scrapely/storage/_metadata.py

@@ -0,0 +1,138 @@
+"""HTTP helpers shared by Scrapely Crawlee storage clients."""
+
+from __future__ import annotations
+
+import base64
+import json
+import logging
+from datetime import datetime, timezone
+from typing import Any, Literal, Mapping, Optional, cast
+
+from crawlee.storage_clients.models import (
+    DatasetMetadata,
+    KeyValueStoreMetadata,
+    RequestQueueMetadata,
+)
+
+logger = logging.getLogger(__name__)
+
+StorageType = Literal['datasets', 'keyValueStores', 'requestQueues']
+
+
+def parse_api_datetime(value: Any) -> datetime:
+    if isinstance(value, datetime):
+        if value.tzinfo is None:
+            return value.replace(tzinfo=timezone.utc)
+        return value
+    if isinstance(value, str) and value:
+        try:
+            parsed = datetime.fromisoformat(value.replace('Z', '+00:00'))
+            if parsed.tzinfo is None:
+                return parsed.replace(tzinfo=timezone.utc)
+            return parsed
+        except ValueError:
+            pass
+    return datetime.now(timezone.utc)
+
+
+def to_dataset_metadata(data: Mapping[str, Any], fallback_id: str = '') -> DatasetMetadata:
+    return DatasetMetadata(
+        id=str(data.get('id') or fallback_id),
+        name=data.get('name'),
+        created_at=parse_api_datetime(data.get('createdAt')),
+        accessed_at=parse_api_datetime(data.get('accessedAt')),
+        modified_at=parse_api_datetime(data.get('modifiedAt')),
+        item_count=int(data.get('itemCount') or 0),
+    )
+
+
+def to_kvs_metadata(data: Mapping[str, Any], fallback_id: str = '') -> KeyValueStoreMetadata:
+    return KeyValueStoreMetadata(
+        id=str(data.get('id') or fallback_id),
+        name=data.get('name'),
+        created_at=parse_api_datetime(data.get('createdAt')),
+        accessed_at=parse_api_datetime(data.get('accessedAt')),
+        modified_at=parse_api_datetime(data.get('modifiedAt')),
+    )
+
+
+def to_rq_metadata(data: Mapping[str, Any], fallback_id: str = '') -> RequestQueueMetadata:
+    return RequestQueueMetadata(
+        id=str(data.get('id') or fallback_id),
+        name=data.get('name'),
+        created_at=parse_api_datetime(data.get('createdAt')),
+        accessed_at=parse_api_datetime(data.get('accessedAt')),
+        modified_at=parse_api_datetime(data.get('modifiedAt')),
+        had_multiple_clients=bool(data.get('hadMultipleClients') or False),
+        handled_request_count=int(data.get('handledRequestCount') or 0),
+        pending_request_count=int(data.get('pendingRequestCount') or 0),
+        total_request_count=int(data.get('totalRequestCount') or 0),
+    )
+
+
+def parse_actor_storages_json(raw: Optional[str]) -> Optional[dict[str, Any]]:
+    """Parse ``ACTOR_STORAGES_JSON`` (plain JSON or base64-encoded JSON)."""
+    if not raw:
+        return None
+    try:
+        return cast(dict[str, Any], json.loads(raw))
+    except json.JSONDecodeError:
+        pass
+    try:
+        decoded = base64.b64decode(raw).decode('utf-8')
+        return cast(dict[str, Any], json.loads(decoded))
+    except Exception:
+        logger.debug('Failed to parse ACTOR_STORAGES_JSON', exc_info=True)
+        return None
+
+
+def resolve_alias_from_storages_json(
+    alias: str,
+    storage_type: StorageType,
+    actor_storages_json: Optional[str] = None,
+) -> Optional[str]:
+    """Resolve an alias to a storage id via ``ACTOR_STORAGES_JSON`` (TS parity).
+
+    Returns the resolved id, or ``None`` if not found (caller should treat alias as name).
+    """
+    parsed = parse_actor_storages_json(actor_storages_json)
+    if not parsed:
+        return None
+    bucket = parsed.get(storage_type)
+    if isinstance(bucket, dict):
+        value = bucket.get(alias)
+        if isinstance(value, str) and value:
+            return value
+    return None
+
+
+def resolve_storage_id(
+    *,
+    id: Optional[str],
+    name: Optional[str],
+    alias: Optional[str],
+    default_id: Optional[str],
+    storage_type: Optional[StorageType] = None,
+    actor_storages_json: Optional[str] = None,
+) -> tuple[Optional[str], Optional[str]]:
+    """Return (id, name) for get-or-create.
+
+    Alias is resolved via ``ACTOR_STORAGES_JSON`` when possible; otherwise treated as a name
+    (same as TypeScript Actor outside the platform).
+    """
+    provided = [v for v in (id, name, alias) if v is not None]
+    if len(provided) > 1:
+        raise ValueError('Only one of id, name, or alias may be provided')
+    if alias is not None:
+        if storage_type is not None:
+            resolved = resolve_alias_from_storages_json(
+                alias,
+                storage_type,
+                actor_storages_json,
+            )
+            if resolved is not None:
+                return resolved, None
+        return None, alias
+    if id is not None or name is not None:
+        return id, name
+    return default_id, None