apify 3.0.1b1__py3-none-any.whl → 3.0.1b2__py3-none-any.whl

apify/_configuration.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from datetime import datetime, timedelta
 from decimal import Decimal
 from logging import getLogger
+from pathlib import Path
 from typing import Annotated, Any
 
 from pydantic import AliasChoices, BeforeValidator, Field, model_validator
@@ -421,6 +422,14 @@ class Configuration(CrawleeConfiguration):
             logger.warning('Actor is running on the Apify platform, `disable_browser_sandbox` was changed to True.')
         return self
 
+    @property
+    def canonical_input_key(self) -> str:
+        return str(Path(self.input_key).with_suffix('.json'))
+
+    @property
+    def input_key_candidates(self) -> set[str]:
+        return {self.input_key, self.canonical_input_key, Path(self.canonical_input_key).stem}
+
     @classmethod
     def get_global_configuration(cls) -> Configuration:
         """Retrieve the global instance of the configuration.

apify/storage_clients/_file_system/_key_value_store_client.py

@@ -1,14 +1,19 @@
 import asyncio
 import json
-from pathlib import Path
+import logging
 
-from typing_extensions import override
+from more_itertools import flatten
+from typing_extensions import Self, override
 
 from crawlee._consts import METADATA_FILENAME
+from crawlee.configuration import Configuration as CrawleeConfiguration
 from crawlee.storage_clients._file_system import FileSystemKeyValueStoreClient
+from crawlee.storage_clients.models import KeyValueStoreRecord
 
 from apify._configuration import Configuration
 
+logger = logging.getLogger(__name__)
+
 
 class ApifyFileSystemKeyValueStoreClient(FileSystemKeyValueStoreClient):
     """Apify-specific implementation of the `FileSystemKeyValueStoreClient`.
@@ -17,6 +22,22 @@ class ApifyFileSystemKeyValueStoreClient(FileSystemKeyValueStoreClient):
     directory, except for the metadata file and the `INPUT.json` file.
     """
 
+    @override
+    @classmethod
+    async def open(
+        cls,
+        *,
+        id: str | None,
+        name: str | None,
+        alias: str | None,
+        configuration: CrawleeConfiguration,
+    ) -> Self:
+        client = await super().open(id=id, name=name, alias=alias, configuration=configuration)
+
+        await client._sanitize_input_json_files()  # noqa: SLF001 - it's okay, this is a factory method
+
+        return client
+
     @override
     async def purge(self) -> None:
         """Purges the key-value store by deleting all its contents.
@@ -24,16 +45,16 @@ class ApifyFileSystemKeyValueStoreClient(FileSystemKeyValueStoreClient):
         It deletes all files in the key-value store directory, except for the metadata file and
         the `INPUT.json` file. It also updates the metadata to reflect that the store has been purged.
         """
-        kvs_input_key = Configuration.get_global_configuration().input_key
-
-        # First try to find the alternative format of the input file and process it if it exists.
-        for file_path in self.path_to_kvs.glob('*'):
-            if file_path.name == f'{kvs_input_key}.json':
-                await self._process_input_json(file_path)
+        configuration = Configuration.get_global_configuration()
 
         async with self._lock:
+            files_to_keep = set(
+                flatten([key, f'{key}.{METADATA_FILENAME}'] for key in configuration.input_key_candidates)
+            )
+            files_to_keep.add(METADATA_FILENAME)
+
             for file_path in self.path_to_kvs.glob('*'):
-                if file_path.name in {METADATA_FILENAME, kvs_input_key, f'{kvs_input_key}.{METADATA_FILENAME}'}:
+                if file_path.name in files_to_keep:
                     continue
                 if file_path.is_file():
                     await asyncio.to_thread(file_path.unlink, missing_ok=True)
@@ -43,15 +64,40 @@ class ApifyFileSystemKeyValueStoreClient(FileSystemKeyValueStoreClient):
                 update_modified_at=True,
             )
 
-    async def _process_input_json(self, path: Path) -> None:
-        """Process simple input json file to format expected by the FileSystemKeyValueStoreClient.
+    async def _sanitize_input_json_files(self) -> None:
+        """Handle missing metadata for input files."""
+        configuration = Configuration.get_global_configuration()
+        alternative_keys = configuration.input_key_candidates - {configuration.canonical_input_key}
 
-        For example: INPUT.json -> INPUT, INPUT.json.metadata
-        """
-        try:
-            f = await asyncio.to_thread(path.open)
-            input_data = json.load(f)
-        finally:
-            f.close()
-        await asyncio.to_thread(path.unlink, missing_ok=True)
-        await self.set_value(key=path.stem, value=input_data)
+        if (self.path_to_kvs / configuration.canonical_input_key).exists():
+            # Refresh metadata to prevent inconsistencies
+            input_data = await asyncio.to_thread(
+                lambda: json.loads((self.path_to_kvs / configuration.canonical_input_key).read_text())
+            )
+            await self.set_value(key=configuration.canonical_input_key, value=input_data)
+
+            for alternative_key in alternative_keys:
+                if (alternative_input_file := self.path_to_kvs / alternative_key).exists():
+                    logger.warning(f'Redundant input file found: {alternative_input_file}')
+        else:
+            for alternative_key in alternative_keys:
+                alternative_input_file = self.path_to_kvs / alternative_key
+
+                # Only process files that actually exist
+                if alternative_input_file.exists():
+                    # Refresh metadata to prevent inconsistencies
+                    with alternative_input_file.open() as f:
+                        input_data = await asyncio.to_thread(lambda: json.load(f))
+                    await self.set_value(key=alternative_key, value=input_data)
+
+    @override
+    async def get_value(self, *, key: str) -> KeyValueStoreRecord | None:
+        configuration = Configuration.get_global_configuration()
+
+        if key in configuration.input_key_candidates:
+            for candidate in configuration.input_key_candidates:
+                value = await super().get_value(key=candidate)
+                if value is not None:
+                    return value
+
+        return await super().get_value(key=key)

{apify-3.0.1b1.dist-info → apify-3.0.1b2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 3.0.1b1
+Version: 3.0.1b2
 Summary: Apify SDK for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog
@@ -228,7 +228,7 @@ Requires-Python: >=3.10
 Requires-Dist: apify-client<3.0.0,>=2.0.0
 Requires-Dist: apify-shared<3.0.0,>=2.0.0
 Requires-Dist: cachetools>=5.5.0
-Requires-Dist: crawlee<2.0.0,>=1.0.0
+Requires-Dist: crawlee<2.0.0,>=1.0.2
 Requires-Dist: cryptography>=42.0.0
 Requires-Dist: impit>=0.6.1
 Requires-Dist: lazy-object-proxy>=1.11.0

{apify-3.0.1b1.dist-info → apify-3.0.1b2.dist-info}/RECORD

@@ -1,7 +1,7 @@
 apify/__init__.py,sha256=HpgKg2FZWJuSPfDygzJ62psylhw4NN4tKFnoYUIhcd4,838
 apify/_actor.py,sha256=DYHoyBAu6hDLs0BcTZL-IQveLK8gPTWvb6AgDnJc3EA,54755
 apify/_charging.py,sha256=KjZ2DnEMS0Tt8ibizmmt0RwBq8FOAsD1z-hKFgdazcY,13143
-apify/_configuration.py,sha256=gq_UfWTgcP1_0kEMLhXVg33SgSxXjShbuzoXyCFfK0w,14682
+apify/_configuration.py,sha256=7ZHhgRp98kr35zx4k4EB2aImq7Dq1FJjPg7r5bucv_M,14984
 apify/_consts.py,sha256=CjhyEJ4Mi0lcIrzfqz8dN7nPJWGjCeBrrXQy1PZ6zRI,440
 apify/_crypto.py,sha256=tqUs13QkemDtGzvU41pIA2HUEawpDlgzqbwKjm4I8kM,6852
 apify/_models.py,sha256=EzU-inWeJ7T5HNVYEwnYb79W-q4OAPhtrYctfRYzpTE,7848
@@ -45,13 +45,13 @@ apify/storage_clients/_apify/_storage_client.py,sha256=hFl_PuX1UgOydBD6pieZ0u2NW
 apify/storage_clients/_apify/_utils.py,sha256=ywXoSM69amRokUZcshbAvQLIcSZq4L-bpYIGyeFxCGQ,7696
 apify/storage_clients/_apify/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 apify/storage_clients/_file_system/__init__.py,sha256=rDbXatXV9wHKPhKTrXDzWnexhTm7sIJQWucMi-P-SD4,130
-apify/storage_clients/_file_system/_key_value_store_client.py,sha256=fnSJ1EIOPCGfcE6e5S3Tux9VbnMVLCJjugkaQoH_9yo,2267
+apify/storage_clients/_file_system/_key_value_store_client.py,sha256=gxM3ap67PnY80Rd7P3onPAf2pksYpU0LoAlJdayEMdU,4179
 apify/storage_clients/_file_system/_storage_client.py,sha256=rcwpKYlrWzvlSA2xoxftg-EZAi_iGZ3vOCbu0C5lKDE,1396
 apify/storage_clients/_smart_apify/__init__.py,sha256=614B2AaWY-dx6RQ6mod7VVR8gFh75-_jnq5BeDD7hSc,53
 apify/storage_clients/_smart_apify/_storage_client.py,sha256=GCPmVe_xWAFcO2Cuej4su4i97_d33Q9Ih_Sc5xW2Wa4,4674
 apify/storages/__init__.py,sha256=-9tEYJVabVs_eRVhUehxN58GH0UG8OfuGjGwuDieP2M,122
 apify/storages/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify-3.0.1b1.dist-info/METADATA,sha256=Qy-fnT_4BnuEpoIhk_Aa0vIl6GVQtkqkk8diacKkzA0,22582
-apify-3.0.1b1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-apify-3.0.1b1.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
-apify-3.0.1b1.dist-info/RECORD,,
+apify-3.0.1b2.dist-info/METADATA,sha256=2NLa54gUwW-FF9Rf6Qc4XQo2JsAVOvPaThJljzT4OZo,22582
+apify-3.0.1b2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+apify-3.0.1b2.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
+apify-3.0.1b2.dist-info/RECORD,,