apify 1.7.0b1__py3-none-any.whl → 2.2.0b14__py3-none-any.whl

apify/_memory_storage/file_storage_utils.py

@@ -1,71 +0,0 @@
-from __future__ import annotations
-
-import os
-
-import aiofiles
-from aiofiles.os import makedirs
-from apify_shared.utils import json_dumps
-
-from apify._utils import force_remove
-
-
-async def update_metadata(*, data: dict, entity_directory: str, write_metadata: bool) -> None:
-    # Skip writing the actual metadata file. This is done after ensuring the directory exists so we have the directory present
-    if not write_metadata:
-        return
-
-    # Ensure the directory for the entity exists
-    await makedirs(entity_directory, exist_ok=True)
-
-    # Write the metadata to the file
-    file_path = os.path.join(entity_directory, '__metadata__.json')
-    async with aiofiles.open(file_path, mode='wb') as f:
-        await f.write(json_dumps(data).encode('utf-8'))
-
-
-async def _update_dataset_items(
-    *,
-    data: list[tuple[str, dict]],
-    entity_directory: str,
-    persist_storage: bool,
-) -> None:
-    # Skip writing files to the disk if the client has the option set to false
-    if not persist_storage:
-        return
-
-    # Ensure the directory for the entity exists
-    await makedirs(entity_directory, exist_ok=True)
-
-    # Save all the new items to the disk
-    for idx, item in data:
-        file_path = os.path.join(entity_directory, f'{idx}.json')
-        async with aiofiles.open(file_path, mode='wb') as f:
-            await f.write(json_dumps(item).encode('utf-8'))
-
-
-async def update_request_queue_item(
-    *,
-    request_id: str,
-    request: dict,
-    entity_directory: str,
-    persist_storage: bool,
-) -> None:
-    # Skip writing files to the disk if the client has the option set to false
-    if not persist_storage:
-        return
-
-    # Ensure the directory for the entity exists
-    await makedirs(entity_directory, exist_ok=True)
-
-    # Write the request to the file
-    file_path = os.path.join(entity_directory, f'{request_id}.json')
-    async with aiofiles.open(file_path, mode='wb') as f:
-        await f.write(json_dumps(request).encode('utf-8'))
-
-
-async def delete_request(*, request_id: str, entity_directory: str) -> None:
-    # Ensure the directory for the entity exists
-    await makedirs(entity_directory, exist_ok=True)
-
-    file_path = os.path.join(entity_directory, f'{request_id}.json')
-    await force_remove(file_path)

apify/_memory_storage/memory_storage_client.py

@@ -1,219 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import contextlib
-import os
-from pathlib import Path
-
-import aioshutil
-from aiofiles import ospath
-from aiofiles.os import rename, scandir
-from apify_shared.consts import ApifyEnvVars
-from apify_shared.utils import ignore_docs
-
-from apify._memory_storage.resource_clients.dataset import DatasetClient
-from apify._memory_storage.resource_clients.dataset_collection import DatasetCollectionClient
-from apify._memory_storage.resource_clients.key_value_store import KeyValueStoreClient
-from apify._memory_storage.resource_clients.key_value_store_collection import KeyValueStoreCollectionClient
-from apify._memory_storage.resource_clients.request_queue import RequestQueueClient
-from apify._memory_storage.resource_clients.request_queue_collection import RequestQueueCollectionClient
-from apify._utils import maybe_parse_bool
-
-"""
-Memory storage emulates data storages that are available on the Apify platform.
-Specifically, it emulates clients for datasets, key-value stores and request queues.
-The data are held in-memory and persisted locally if `persist_storage` is True.
-The metadata of the storages is also persisted if `write_metadata` is True.
-"""
-
-
-@ignore_docs
-class MemoryStorageClient:
-    """Class representing an in-memory storage."""
-
-    _local_data_directory: str
-    _datasets_directory: str
-    _key_value_stores_directory: str
-    _request_queues_directory: str
-    _write_metadata: bool
-    _persist_storage: bool
-    _datasets_handled: list[DatasetClient]
-    _key_value_stores_handled: list[KeyValueStoreClient]
-    _request_queues_handled: list[RequestQueueClient]
-
-    _purged_on_start: bool = False
-    _purge_lock: asyncio.Lock
-
-    """Indicates whether a purge was already performed on this instance"""
-
-    def __init__(
-        self: MemoryStorageClient,
-        *,
-        local_data_directory: str | None = None,
-        write_metadata: bool | None = None,
-        persist_storage: bool | None = None,
-    ) -> None:
-        """Initialize the MemoryStorageClient.
-
-        Args:
-            local_data_directory (str, optional): A local directory where all data will be persisted
-            persist_storage (bool, optional): Whether to persist the data to the `local_data_directory` or just keep them in memory
-            write_metadata (bool, optional): Whether to persist metadata of the storages as well
-        """
-        self._local_data_directory = local_data_directory or os.getenv(ApifyEnvVars.LOCAL_STORAGE_DIR) or './storage'
-        self._datasets_directory = os.path.join(self._local_data_directory, 'datasets')
-        self._key_value_stores_directory = os.path.join(self._local_data_directory, 'key_value_stores')
-        self._request_queues_directory = os.path.join(self._local_data_directory, 'request_queues')
-        self._write_metadata = write_metadata if write_metadata is not None else '*' in os.getenv('DEBUG', '')
-        self._persist_storage = persist_storage if persist_storage is not None else maybe_parse_bool(os.getenv(ApifyEnvVars.PERSIST_STORAGE, 'true'))
-        self._datasets_handled = []
-        self._key_value_stores_handled = []
-        self._request_queues_handled = []
-        self._purge_lock = asyncio.Lock()
-
-    def datasets(self: MemoryStorageClient) -> DatasetCollectionClient:
-        """Retrieve the sub-client for manipulating datasets."""
-        return DatasetCollectionClient(base_storage_directory=self._datasets_directory, memory_storage_client=self)
-
-    def dataset(self: MemoryStorageClient, dataset_id: str) -> DatasetClient:
-        """Retrieve the sub-client for manipulating a single dataset.
-
-        Args:
-            dataset_id (str): ID of the dataset to be manipulated
-        """
-        return DatasetClient(base_storage_directory=self._datasets_directory, memory_storage_client=self, id=dataset_id)
-
-    def key_value_stores(self: MemoryStorageClient) -> KeyValueStoreCollectionClient:
-        """Retrieve the sub-client for manipulating key-value stores."""
-        return KeyValueStoreCollectionClient(base_storage_directory=self._key_value_stores_directory, memory_storage_client=self)
-
-    def key_value_store(self: MemoryStorageClient, key_value_store_id: str) -> KeyValueStoreClient:
-        """Retrieve the sub-client for manipulating a single key-value store.
-
-        Args:
-            key_value_store_id (str): ID of the key-value store to be manipulated
-        """
-        return KeyValueStoreClient(base_storage_directory=self._key_value_stores_directory, memory_storage_client=self, id=key_value_store_id)
-
-    def request_queues(self: MemoryStorageClient) -> RequestQueueCollectionClient:
-        """Retrieve the sub-client for manipulating request queues."""
-        return RequestQueueCollectionClient(base_storage_directory=self._request_queues_directory, memory_storage_client=self)
-
-    def request_queue(
-        self: MemoryStorageClient,
-        request_queue_id: str,
-        *,
-        client_key: str | None = None,  # noqa: ARG002
-    ) -> RequestQueueClient:
-        """Retrieve the sub-client for manipulating a single request queue.
-
-        Args:
-            request_queue_id (str): ID of the request queue to be manipulated
-            client_key (str): A unique identifier of the client accessing the request queue
-        """
-        return RequestQueueClient(base_storage_directory=self._request_queues_directory, memory_storage_client=self, id=request_queue_id)
-
-    async def _purge_on_start(self: MemoryStorageClient) -> None:
-        # Optimistic, non-blocking check
-        if self._purged_on_start is True:
-            return
-
-        async with self._purge_lock:
-            # Another check under the lock just to be sure
-            if self._purged_on_start is True:
-                return  # type: ignore[unreachable] # Mypy doesn't understand that the _purged_on_start can change while we're getting the async lock
-
-            await self._purge()
-            self._purged_on_start = True
-
-    async def _purge(self: MemoryStorageClient) -> None:
-        """Clean up the default storage directories before the run starts.
-
-        Specifically, `purge` cleans up:
-         - local directory containing the default dataset;
-         - all records from the default key-value store in the local directory, except for the "INPUT" key;
-         - local directory containing the default request queue.
-        """
-        # Key-value stores
-        if await ospath.exists(self._key_value_stores_directory):
-            key_value_store_folders = await scandir(self._key_value_stores_directory)
-            for key_value_store_folder in key_value_store_folders:
-                if key_value_store_folder.name.startswith('__APIFY_TEMPORARY') or key_value_store_folder.name.startswith('__OLD'):
-                    await self._batch_remove_files(key_value_store_folder.path)
-                elif key_value_store_folder.name == 'default':
-                    await self._handle_default_key_value_store(key_value_store_folder.path)
-
-        # Datasets
-        if await ospath.exists(self._datasets_directory):
-            dataset_folders = await scandir(self._datasets_directory)
-            for dataset_folder in dataset_folders:
-                if dataset_folder.name == 'default' or dataset_folder.name.startswith('__APIFY_TEMPORARY'):
-                    await self._batch_remove_files(dataset_folder.path)
-        # Request queues
-        if await ospath.exists(self._request_queues_directory):
-            request_queue_folders = await scandir(self._request_queues_directory)
-            for request_queue_folder in request_queue_folders:
-                if request_queue_folder.name == 'default' or request_queue_folder.name.startswith('__APIFY_TEMPORARY'):
-                    await self._batch_remove_files(request_queue_folder.path)
-
-    async def _handle_default_key_value_store(self: MemoryStorageClient, folder: str) -> None:
-        """Remove everything from the default key-value store folder except `possible_input_keys`."""
-        folder_exists = await ospath.exists(folder)
-        temporary_path = os.path.normpath(os.path.join(folder, '../__APIFY_MIGRATING_KEY_VALUE_STORE__'))
-
-        # For optimization, we want to only attempt to copy a few files from the default key-value store
-        possible_input_keys = [
-            'INPUT',
-            'INPUT.json',
-            'INPUT.bin',
-            'INPUT.txt',
-        ]
-
-        if folder_exists:
-            # Create a temporary folder to save important files in
-            Path(temporary_path).mkdir(parents=True, exist_ok=True)
-
-            # Go through each file and save the ones that are important
-            for entity in possible_input_keys:
-                original_file_path = os.path.join(folder, entity)
-                temp_file_path = os.path.join(temporary_path, entity)
-                with contextlib.suppress(Exception):
-                    await rename(original_file_path, temp_file_path)
-
-            # Remove the original folder and all its content
-            counter = 0
-            temp_path_for_old_folder = os.path.normpath(os.path.join(folder, f'../__OLD_DEFAULT_{counter}__'))
-            done = False
-            try:
-                while not done:
-                    await rename(folder, temp_path_for_old_folder)
-                    done = True
-            except Exception:
-                counter += 1
-                temp_path_for_old_folder = os.path.normpath(os.path.join(folder, f'../__OLD_DEFAULT_{counter}__'))
-
-            # Replace the temporary folder with the original folder
-            await rename(temporary_path, folder)
-
-            # Remove the old folder
-            await self._batch_remove_files(temp_path_for_old_folder)
-
-    async def _batch_remove_files(self: MemoryStorageClient, folder: str, counter: int = 0) -> None:
-        folder_exists = await ospath.exists(folder)
-
-        if folder_exists:
-            temporary_folder = (
-                folder
-                if os.path.basename(folder).startswith('__APIFY_TEMPORARY_')
-                else os.path.normpath(os.path.join(folder, f'../__APIFY_TEMPORARY_{counter}__'))
-            )
-
-            try:
-                # Rename the old folder to the new one to allow background deletions
-                await rename(folder, temporary_folder)
-            except Exception:
-                # Folder exists already, try again with an incremented counter
-                return await self._batch_remove_files(folder, counter + 1)
-
-            await aioshutil.rmtree(temporary_folder, ignore_errors=True)
-        return None

apify/_memory_storage/resource_clients/__init__.py

@@ -1,19 +0,0 @@
-from .base_resource_client import BaseResourceClient
-from .base_resource_collection_client import BaseResourceCollectionClient
-from .dataset import DatasetClient
-from .dataset_collection import DatasetCollectionClient
-from .key_value_store import KeyValueStoreClient
-from .key_value_store_collection import KeyValueStoreCollectionClient
-from .request_queue import RequestQueueClient
-from .request_queue_collection import RequestQueueCollectionClient
-
-__all__ = [
-    'BaseResourceClient',
-    'BaseResourceCollectionClient',
-    'DatasetClient',
-    'DatasetCollectionClient',
-    'KeyValueStoreClient',
-    'KeyValueStoreCollectionClient',
-    'RequestQueueClient',
-    'RequestQueueCollectionClient',
-]

apify/_memory_storage/resource_clients/base_resource_client.py

@@ -1,141 +0,0 @@
-from __future__ import annotations
-
-import json
-import os
-from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING
-
-from apify_shared.utils import ignore_docs
-
-if TYPE_CHECKING:
-    from typing_extensions import Self
-
-    from apify._memory_storage.memory_storage_client import MemoryStorageClient
-
-
-@ignore_docs
-class BaseResourceClient(ABC):
-    """Base class for resource clients."""
-
-    _id: str
-    _name: str | None
-    _resource_directory: str
-
-    @abstractmethod
-    def __init__(
-        self: BaseResourceClient,
-        *,
-        base_storage_directory: str,
-        memory_storage_client: MemoryStorageClient,
-        id: str | None = None,  # noqa: A002
-        name: str | None = None,
-    ) -> None:
-        """Initialize the BaseResourceClient."""
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @abstractmethod
-    async def get(self: BaseResourceClient) -> dict | None:
-        """Retrieve the storage.
-
-        Returns:
-            dict, optional: The retrieved storage, or None, if it does not exist
-        """
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @classmethod
-    @abstractmethod
-    def _get_storages_dir(cls: type[BaseResourceClient], memory_storage_client: MemoryStorageClient) -> str:
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @classmethod
-    @abstractmethod
-    def _get_storage_client_cache(
-        cls,  # noqa: ANN102 # type annotated cls does not work with Self as a return type
-        memory_storage_client: MemoryStorageClient,
-    ) -> list[Self]:
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @abstractmethod
-    def _to_resource_info(self: BaseResourceClient) -> dict:
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @classmethod
-    @abstractmethod
-    def _create_from_directory(
-        cls,  # noqa: ANN102 # type annotated cls does not work with Self as a return type
-        storage_directory: str,
-        memory_storage_client: MemoryStorageClient,
-        id: str | None = None,  # noqa: A002
-        name: str | None = None,
-    ) -> Self:
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @classmethod
-    def _find_or_create_client_by_id_or_name(
-        cls,  # noqa: ANN102 # type annotated cls does not work with Self as a return type
-        memory_storage_client: MemoryStorageClient,
-        id: str | None = None,  # noqa: A002
-        name: str | None = None,
-    ) -> Self | None:
-        assert id is not None or name is not None  # noqa: S101
-
-        storage_client_cache = cls._get_storage_client_cache(memory_storage_client)
-        storages_dir = cls._get_storages_dir(memory_storage_client)
-
-        # First check memory cache
-        found = next(
-            (
-                storage_client
-                for storage_client in storage_client_cache
-                if storage_client._id == id or (storage_client._name and name and storage_client._name.lower() == name.lower())
-            ),
-            None,
-        )
-
-        if found is not None:
-            return found
-
-        storage_path = None
-
-        # First try to find the storage by looking up the directory by name
-        if name:
-            possible_storage_path = os.path.join(storages_dir, name)
-            if os.access(possible_storage_path, os.F_OK):
-                storage_path = possible_storage_path
-
-        # If it's not found, try going through the storages dir and finding it by metadata
-        if not storage_path and os.access(storages_dir, os.F_OK):
-            for entry in os.scandir(storages_dir):
-                if not entry.is_dir():
-                    continue
-                metadata_path = os.path.join(entry.path, '__metadata__.json')
-                if not os.access(metadata_path, os.F_OK):
-                    continue
-                with open(metadata_path, encoding='utf-8') as metadata_file:
-                    metadata = json.load(metadata_file)
-                if id and id == metadata.get('id'):
-                    storage_path = entry.path
-                    name = metadata.get(name)
-                    break
-                if name and name == metadata.get('name'):
-                    storage_path = entry.path
-                    id = metadata.get(id)  # noqa: A001
-                    break
-
-        # As a last resort, try to check if the accessed storage is the default one,
-        # and the folder has no metadata
-        # TODO: make this respect the APIFY_DEFAULT_XXX_ID env var
-        # https://github.com/apify/apify-sdk-python/issues/149
-        if id == 'default':
-            possible_storage_path = os.path.join(storages_dir, id)
-            if os.access(possible_storage_path, os.F_OK):
-                storage_path = possible_storage_path
-
-        if not storage_path:
-            return None
-
-        resource_client = cls._create_from_directory(storage_path, memory_storage_client, id, name)
-
-        storage_client_cache.append(resource_client)
-
-        return resource_client

apify/_memory_storage/resource_clients/base_resource_collection_client.py

@@ -1,114 +0,0 @@
-from __future__ import annotations
-
-from abc import ABC, abstractmethod
-from operator import itemgetter
-from typing import TYPE_CHECKING, Generic, TypeVar, cast
-
-from apify_shared.models import ListPage
-from apify_shared.utils import ignore_docs
-
-from apify._memory_storage.file_storage_utils import update_metadata
-from apify._memory_storage.resource_clients.base_resource_client import BaseResourceClient
-
-if TYPE_CHECKING:
-    from apify._memory_storage.memory_storage_client import MemoryStorageClient
-
-
-ResourceClientType = TypeVar('ResourceClientType', bound=BaseResourceClient, contravariant=True)  # noqa: PLC0105
-
-
-@ignore_docs
-class BaseResourceCollectionClient(ABC, Generic[ResourceClientType]):
-    """Base class for resource collection clients."""
-
-    _base_storage_directory: str
-    _memory_storage_client: MemoryStorageClient
-
-    def __init__(
-        self: BaseResourceCollectionClient,
-        *,
-        base_storage_directory: str,
-        memory_storage_client: MemoryStorageClient,
-    ) -> None:
-        """Initialize the DatasetCollectionClient with the passed arguments."""
-        self._base_storage_directory = base_storage_directory
-        self._memory_storage_client = memory_storage_client
-
-    @abstractmethod
-    def _get_storage_client_cache(self: BaseResourceCollectionClient) -> list[ResourceClientType]:
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @abstractmethod
-    def _get_resource_client_class(self: BaseResourceCollectionClient) -> type[ResourceClientType]:
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @abstractmethod
-    async def list(self: BaseResourceCollectionClient) -> ListPage:
-        """List the available storages.
-
-        Returns:
-            ListPage: The list of available storages matching the specified filters.
-        """
-        storage_client_cache = self._get_storage_client_cache()
-
-        items = [storage._to_resource_info() for storage in storage_client_cache]
-
-        return ListPage(
-            {
-                'total': len(items),
-                'count': len(items),
-                'offset': 0,
-                'limit': len(items),
-                'desc': False,
-                'items': sorted(items, key=itemgetter('createdAt')),
-            }
-        )
-
-    @abstractmethod
-    async def get_or_create(
-        self: BaseResourceCollectionClient,
-        *,
-        name: str | None = None,
-        schema: dict | None = None,
-        _id: str | None = None,
-    ) -> dict:
-        """Retrieve a named storage, or create a new one when it doesn't exist.
-
-        Args:
-            name (str, optional): The name of the storage to retrieve or create.
-            schema (Dict, optional): The schema of the storage
-
-        Returns:
-            dict: The retrieved or newly-created storage.
-        """
-        resource_client_class = self._get_resource_client_class()
-        storage_client_cache = self._get_storage_client_cache()
-
-        if name or _id:
-            found = resource_client_class._find_or_create_client_by_id_or_name(
-                memory_storage_client=self._memory_storage_client,
-                name=name,
-                id=_id,
-            )
-            if found:
-                resource_info = found._to_resource_info()
-                return cast(dict, resource_info)
-
-        new_resource = resource_client_class(
-            id=_id,
-            name=name,
-            base_storage_directory=self._base_storage_directory,
-            memory_storage_client=self._memory_storage_client,
-        )
-        storage_client_cache.append(new_resource)
-
-        resource_info = new_resource._to_resource_info()
-
-        # Write to the disk
-        await update_metadata(
-            data=resource_info,
-            entity_directory=new_resource._resource_directory,
-            write_metadata=self._memory_storage_client._write_metadata,
-        )
-
-        return cast(dict, resource_info)