apify 1.7.3b4__py3-none-any.whl → 2.0.0__py3-none-any.whl

apify/_memory_storage/resource_clients/base_resource_client.py

@@ -1,141 +0,0 @@
-from __future__ import annotations
-
-import json
-import os
-from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING
-
-from apify_shared.utils import ignore_docs
-
-if TYPE_CHECKING:
-    from typing_extensions import Self
-
-    from apify._memory_storage.memory_storage_client import MemoryStorageClient
-
-
-@ignore_docs
-class BaseResourceClient(ABC):
-    """Base class for resource clients."""
-
-    _id: str
-    _name: str | None
-    _resource_directory: str
-
-    @abstractmethod
-    def __init__(
-        self: BaseResourceClient,
-        *,
-        base_storage_directory: str,
-        memory_storage_client: MemoryStorageClient,
-        id: str | None = None,  # noqa: A002
-        name: str | None = None,
-    ) -> None:
-        """Initialize the BaseResourceClient."""
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @abstractmethod
-    async def get(self: BaseResourceClient) -> dict | None:
-        """Retrieve the storage.
-
-        Returns:
-            dict, optional: The retrieved storage, or None, if it does not exist
-        """
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @classmethod
-    @abstractmethod
-    def _get_storages_dir(cls: type[BaseResourceClient], memory_storage_client: MemoryStorageClient) -> str:
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @classmethod
-    @abstractmethod
-    def _get_storage_client_cache(
-        cls,
-        memory_storage_client: MemoryStorageClient,
-    ) -> list[Self]:
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @abstractmethod
-    def _to_resource_info(self: BaseResourceClient) -> dict:
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @classmethod
-    @abstractmethod
-    def _create_from_directory(
-        cls,
-        storage_directory: str,
-        memory_storage_client: MemoryStorageClient,
-        id: str | None = None,  # noqa: A002
-        name: str | None = None,
-    ) -> Self:
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @classmethod
-    def _find_or_create_client_by_id_or_name(
-        cls,
-        memory_storage_client: MemoryStorageClient,
-        id: str | None = None,  # noqa: A002
-        name: str | None = None,
-    ) -> Self | None:
-        assert id is not None or name is not None  # noqa: S101
-
-        storage_client_cache = cls._get_storage_client_cache(memory_storage_client)
-        storages_dir = cls._get_storages_dir(memory_storage_client)
-
-        # First check memory cache
-        found = next(
-            (
-                storage_client
-                for storage_client in storage_client_cache
-                if storage_client._id == id or (storage_client._name and name and storage_client._name.lower() == name.lower())
-            ),
-            None,
-        )
-
-        if found is not None:
-            return found
-
-        storage_path = None
-
-        # First try to find the storage by looking up the directory by name
-        if name:
-            possible_storage_path = os.path.join(storages_dir, name)
-            if os.access(possible_storage_path, os.F_OK):
-                storage_path = possible_storage_path
-
-        # If it's not found, try going through the storages dir and finding it by metadata
-        if not storage_path and os.access(storages_dir, os.F_OK):
-            for entry in os.scandir(storages_dir):
-                if not entry.is_dir():
-                    continue
-                metadata_path = os.path.join(entry.path, '__metadata__.json')
-                if not os.access(metadata_path, os.F_OK):
-                    continue
-                with open(metadata_path, encoding='utf-8') as metadata_file:
-                    metadata = json.load(metadata_file)
-                if id and id == metadata.get('id'):
-                    storage_path = entry.path
-                    name = metadata.get(name)
-                    break
-                if name and name == metadata.get('name'):
-                    storage_path = entry.path
-                    id = metadata.get(id)  # noqa: A001
-                    break
-
-        # As a last resort, try to check if the accessed storage is the default one,
-        # and the folder has no metadata
-        # TODO: make this respect the APIFY_DEFAULT_XXX_ID env var
-        # https://github.com/apify/apify-sdk-python/issues/149
-        if id == 'default':
-            possible_storage_path = os.path.join(storages_dir, id)
-            if os.access(possible_storage_path, os.F_OK):
-                storage_path = possible_storage_path
-
-        if not storage_path:
-            return None
-
-        resource_client = cls._create_from_directory(storage_path, memory_storage_client, id, name)
-
-        storage_client_cache.append(resource_client)
-
-        return resource_client

apify/_memory_storage/resource_clients/base_resource_collection_client.py

@@ -1,114 +0,0 @@
-from __future__ import annotations
-
-from abc import ABC, abstractmethod
-from operator import itemgetter
-from typing import TYPE_CHECKING, Generic, TypeVar, cast
-
-from apify_shared.models import ListPage
-from apify_shared.utils import ignore_docs
-
-from apify._memory_storage.file_storage_utils import update_metadata
-from apify._memory_storage.resource_clients.base_resource_client import BaseResourceClient
-
-if TYPE_CHECKING:
-    from apify._memory_storage.memory_storage_client import MemoryStorageClient
-
-
-ResourceClientType = TypeVar('ResourceClientType', bound=BaseResourceClient, contravariant=True)  # noqa: PLC0105
-
-
-@ignore_docs
-class BaseResourceCollectionClient(ABC, Generic[ResourceClientType]):
-    """Base class for resource collection clients."""
-
-    _base_storage_directory: str
-    _memory_storage_client: MemoryStorageClient
-
-    def __init__(
-        self: BaseResourceCollectionClient,
-        *,
-        base_storage_directory: str,
-        memory_storage_client: MemoryStorageClient,
-    ) -> None:
-        """Initialize the DatasetCollectionClient with the passed arguments."""
-        self._base_storage_directory = base_storage_directory
-        self._memory_storage_client = memory_storage_client
-
-    @abstractmethod
-    def _get_storage_client_cache(self: BaseResourceCollectionClient) -> list[ResourceClientType]:
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @abstractmethod
-    def _get_resource_client_class(self: BaseResourceCollectionClient) -> type[ResourceClientType]:
-        raise NotImplementedError('You must override this method in the subclass!')
-
-    @abstractmethod
-    async def list(self: BaseResourceCollectionClient) -> ListPage:
-        """List the available storages.
-
-        Returns:
-            ListPage: The list of available storages matching the specified filters.
-        """
-        storage_client_cache = self._get_storage_client_cache()
-
-        items = [storage._to_resource_info() for storage in storage_client_cache]
-
-        return ListPage(
-            {
-                'total': len(items),
-                'count': len(items),
-                'offset': 0,
-                'limit': len(items),
-                'desc': False,
-                'items': sorted(items, key=itemgetter('createdAt')),
-            }
-        )
-
-    @abstractmethod
-    async def get_or_create(
-        self: BaseResourceCollectionClient,
-        *,
-        name: str | None = None,
-        schema: dict | None = None,
-        _id: str | None = None,
-    ) -> dict:
-        """Retrieve a named storage, or create a new one when it doesn't exist.
-
-        Args:
-            name (str, optional): The name of the storage to retrieve or create.
-            schema (Dict, optional): The schema of the storage
-
-        Returns:
-            dict: The retrieved or newly-created storage.
-        """
-        resource_client_class = self._get_resource_client_class()
-        storage_client_cache = self._get_storage_client_cache()
-
-        if name or _id:
-            found = resource_client_class._find_or_create_client_by_id_or_name(
-                memory_storage_client=self._memory_storage_client,
-                name=name,
-                id=_id,
-            )
-            if found:
-                resource_info = found._to_resource_info()
-                return cast(dict, resource_info)
-
-        new_resource = resource_client_class(
-            id=_id,
-            name=name,
-            base_storage_directory=self._base_storage_directory,
-            memory_storage_client=self._memory_storage_client,
-        )
-        storage_client_cache.append(new_resource)
-
-        resource_info = new_resource._to_resource_info()
-
-        # Write to the disk
-        await update_metadata(
-            data=resource_info,
-            entity_directory=new_resource._resource_directory,
-            write_metadata=self._memory_storage_client._write_metadata,
-        )
-
-        return cast(dict, resource_info)