nucliadb-utils 6.9.5.post5447__py3-none-any.whl → 6.10.0.post5617__py3-none-any.whl

nucliadb_utils/audit/audit.py

@@ -97,6 +97,17 @@ class AuditStorage:
     ):
         raise NotImplementedError
 
+    def retrieve(
+        self,
+        kbid: str,
+        user: str,
+        client: int,
+        origin: str,
+        retrieval_time: float,
+    ):
+        # TODO(decoupled-ask): implement audit for /retrieve
+        ...
+
     def report_storage(self, kbid: str, paragraphs: int, fields: int, bytes: int):
         raise NotImplementedError
 

nucliadb_utils/authentication.py

@@ -17,7 +17,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-import asyncio
 import functools
 import inspect
 import typing
@@ -137,7 +136,7 @@ def requires(
 
             return websocket_wrapper
 
-        elif asyncio.iscoroutinefunction(func):
+        elif inspect.iscoroutinefunction(func):
             # Handle async request/response functions.
             @functools.wraps(func)
             async def async_wrapper(*args: typing.Any, **kwargs: typing.Any) -> Response:

nucliadb_utils/const.py

@@ -41,4 +41,4 @@ class Features:
     SKIP_EXTERNAL_INDEX = "nucliadb_skip_external_index"
     LOG_REQUEST_PAYLOADS = "nucliadb_log_request_payloads"
     IGNORE_EXTRACTED_IN_SEARCH = "nucliadb_ignore_extracted_in_search"
-    REBALANCE_ENABLED = "nucliadb_rebalance"
+    ASK_DECOUPLED = "nucliadb_ask_decoupled"

nucliadb_utils/featureflagging.py

@@ -30,6 +30,9 @@ from nucliadb_utils.settings import nuclia_settings, running_settings
 class Settings(pydantic_settings.BaseSettings):
     flag_settings_url: Optional[str] = None
 
+    # temporary flag to test this FF enabled/disabled easily
+    disable_ask_decoupled_ff: bool = False
+
 
 DEFAULT_FLAG_DATA: dict[str, Any] = {
     # These are just defaults to use for local dev and tests
@@ -45,7 +48,10 @@ DEFAULT_FLAG_DATA: dict[str, Any] = {
         "rollout": 0,
         "variants": {"environment": ["local"]},
     },
-    const.Features.REBALANCE_ENABLED: {"rollout": 0, "variants": {"environment": ["local"]}},
+    const.Features.ASK_DECOUPLED: {
+        "rollout": 0,
+        "variants": {"environment": [] if Settings().disable_ask_decoupled_ff else ["local"]},
+    },
 }
 
 

nucliadb_utils/storages/s3.py

@@ -55,12 +55,12 @@ RETRIABLE_EXCEPTIONS = (
 POLICY_DELETE = {
     "Rules": [
         {
-            "Expiration": {"Days": 1},
+            "Expiration": {"Days": 7},
             "ID": "FullDelete",
             "Filter": {"Prefix": ""},
             "Status": "Enabled",
-            "NoncurrentVersionExpiration": {"NoncurrentDays": 1},
-            "AbortIncompleteMultipartUpload": {"DaysAfterInitiation": 1},
+            "NoncurrentVersionExpiration": {"NoncurrentDays": 7},
+            "AbortIncompleteMultipartUpload": {"DaysAfterInitiation": 7},
         },
         {
             "Expiration": {"ExpiredObjectDeleteMarker": True},

nucliadb_utils/storages/storage.py

@@ -370,7 +370,7 @@ class Storage(abc.ABC, metaclass=abc.ABCMeta):
         cf.source = self.source  # type: ignore
 
         if md5 is None:
-            md5hash = hashlib.md5(decoded_payload).digest()
+            md5hash = hashlib.md5(decoded_payload, usedforsecurity=False).digest()
             cf.md5 = md5hash.decode()
         else:
             cf.md5 = md5
@@ -437,19 +437,18 @@ class Storage(abc.ABC, metaclass=abc.ABCMeta):
         bucket: str,
         key: str,
         range: Optional[Range] = None,
-    ):
+    ) -> AsyncGenerator[bytes, None]:
         destination: StorageField = self.field_klass(storage=self, bucket=bucket, fullkey=key)
         try:
             async for data in destination.iter_data(range=range):
                 yield data
         except KeyError:
-            yield None
+            pass
 
     async def downloadbytes(self, bucket: str, key: str) -> BytesIO:
         result = BytesIO()
         async for data in self.download(bucket, key):
-            if data is not None:
-                result.write(data)
+            result.write(data)
 
         result.seek(0)
         return result
@@ -467,18 +466,15 @@ class Storage(abc.ABC, metaclass=abc.ABCMeta):
         # this is covered by other tests
         if cf.source == self.source:
             async for data in self.download(cf.bucket_name, cf.uri):
-                if data is not None:
-                    yield data
+                yield data
         elif cf.source == CloudFile.FLAPS:
             flaps_storage = await get_nuclia_storage()
             async for data in flaps_storage.download(cf):
-                if data is not None:
-                    yield data
+                yield data
         elif cf.source == CloudFile.LOCAL:
             local_storage = get_local_storage()
             async for data in local_storage.download(cf.bucket_name, cf.uri):
-                if data is not None:
-                    yield data
+                yield data
 
     async def upload_pb(self, sf: StorageField, payload: Any):
         await self.upload_object(sf.bucket, sf.key, payload.SerializeToString())

nucliadb_utils/utilities.py

@@ -19,8 +19,8 @@
 
 from __future__ import annotations
 
-import asyncio
 import hashlib
+import inspect
 import logging
 from concurrent.futures.thread import ThreadPoolExecutor
 from enum import Enum
@@ -273,7 +273,7 @@ def clear_global_cache():
 async def finalize_utilities():
     to_delete = []
     for key, util in MAIN.items():
-        if hasattr(util, "finalize") and asyncio.iscoroutinefunction(util.finalize):
+        if hasattr(util, "finalize") and inspect.iscoroutinefunction(util.finalize):
             await util.finalize()
         elif hasattr(util, "finalize"):
             util.finalize()
@@ -420,9 +420,11 @@ def has_feature(
         context = {}
     if headers is not None:
         if X_USER_HEADER in headers:
-            context["user_id_md5"] = hashlib.md5(headers[X_USER_HEADER].encode("utf-8")).hexdigest()
+            context["user_id_sha256"] = hashlib.sha256(
+                headers[X_USER_HEADER].encode("utf-8")
+            ).hexdigest()
         if X_ACCOUNT_HEADER in headers:
-            context["account_id_md5"] = hashlib.md5(headers[X_ACCOUNT_HEADER].encode()).hexdigest()
+            context["account_id_sha256"] = hashlib.sha256(headers[X_ACCOUNT_HEADER].encode()).hexdigest()
         if X_ACCOUNT_TYPE_HEADER in headers:
             context["account_type"] = headers[X_ACCOUNT_TYPE_HEADER]
     return get_feature_flags().enabled(name, default=default, context=context)

{nucliadb_utils-6.9.5.post5447.dist-info → nucliadb_utils-6.10.0.post5617.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb_utils
-Version: 6.9.5.post5447
+Version: 6.10.0.post5617
 Summary: NucliaDB util library
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -25,10 +25,10 @@ Requires-Dist: mmh3>=3.0.0
 Requires-Dist: nats-py[nkeys]>=2.6.0
 Requires-Dist: PyNaCl
 Requires-Dist: pyjwt>=2.4.0
-Requires-Dist: mrflagly>=0.2.9
-Requires-Dist: nidx-protos>=6.9.5.post5447
-Requires-Dist: nucliadb-protos>=6.9.5.post5447
-Requires-Dist: nucliadb-telemetry>=6.9.5.post5447
+Requires-Dist: mrflagly>=0.2.14
+Requires-Dist: nidx-protos>=6.10.0.post5617
+Requires-Dist: nucliadb-protos>=6.10.0.post5617
+Requires-Dist: nucliadb-telemetry>=6.10.0.post5617
 Provides-Extra: cache
 Requires-Dist: redis>=4.3.4; extra == "cache"
 Requires-Dist: orjson>=3.6.7; extra == "cache"

{nucliadb_utils-6.9.5.post5447.dist-info → nucliadb_utils-6.10.0.post5617.dist-info}/RECORD

@@ -1,10 +1,10 @@
 nucliadb_utils/__init__.py,sha256=EvBCH1iTODe-AgXm48aj4kVUt_Std3PeL8QnwimR5wI,895
 nucliadb_utils/asyncio_utils.py,sha256=h8Y-xpcFFRgNzaiIW0eidz7griAQa7ggbNk34-tAt2c,2888
-nucliadb_utils/authentication.py,sha256=5_b323v2ylJaJvM_0coeSQEtnD-p9IGD-6CPA6IXhik,6471
-nucliadb_utils/const.py,sha256=YtWadXGm044MbwLcfEImNC6skh3e4LKZDu6hjlO0qMU,1521
+nucliadb_utils/authentication.py,sha256=7fkEvJCvOg3SLiPx3FdKIGSt8ca7ilpLCN1QInjgOHo,6456
+nucliadb_utils/const.py,sha256=F1kZzTSbJvg9YYGhSi-H6KJsfTkTiX8PER4RBAOkq40,1521
 nucliadb_utils/debug.py,sha256=Q56Nx9Dp7V2ae3CU2H0ztaZcHTJXdlflPLKLeOPZ170,2436
 nucliadb_utils/exceptions.py,sha256=y_3wk77WLVUtdo-5FtbBsdSkCtK_DsJkdWb5BoPn3qo,1094
-nucliadb_utils/featureflagging.py,sha256=ctd9Nqm_nhoedMIV2GC819-cSP5GlkLYXCRE0DbwxYU,2353
+nucliadb_utils/featureflagging.py,sha256=3VyZT8byTkMe7tFWqlxhLdeuIX9713Xy9KDL1A1sSVE,2524
 nucliadb_utils/grpc.py,sha256=apu0uePnkGHCAT7GRQ9YZfRYyFj26kJ440i8jitbM3U,3314
 nucliadb_utils/helpers.py,sha256=eed7_E1MKh9eW3CpqOXka3OvLw5C9eJGC_R-1MPYdfY,3336
 nucliadb_utils/nats.py,sha256=U21Cfg36_IHd3ZLXEC4eZ7nZ1Soh_ZNFFwjryNyd2-8,15248
@@ -15,9 +15,9 @@ nucliadb_utils/settings.py,sha256=H9yKrHPR5emTxai-D4owg4CjE4_-E0qR0HyuHERQNH4,84
 nucliadb_utils/signals.py,sha256=lo_Mk12NIX5Au--3H3WObvDOXq_OMurql2qiC2TnAao,2676
 nucliadb_utils/store.py,sha256=kQ35HemE0v4_Qg6xVqNIJi8vSFAYQtwI3rDtMsNy62Y,890
 nucliadb_utils/transaction.py,sha256=l3ZvrITYMnAs_fv1OOC-1nDZxWPG5qmbBhzvuC3DUzQ,8039
-nucliadb_utils/utilities.py,sha256=SjPnCwCUH_lWUKSOZQp9vIcTYmLP0yL_UC8nPwlnds4,15817
+nucliadb_utils/utilities.py,sha256=yPCZFucWIvtvEaGJLfmLIjWY3Y1qhoTLH16xlaqZbyU,15859
 nucliadb_utils/audit/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb_utils/audit/audit.py,sha256=xmJJiAGG8rPGADwD9gXN9-QJ80GeGvqmY-kCwEf6PiQ,3598
+nucliadb_utils/audit/audit.py,sha256=rDMVIV-W5-TayFrqinMMunrm_nRH9yHUICHiB9btJRo,3822
 nucliadb_utils/audit/basic.py,sha256=fcCYvoFSGVbbB8cSCnm95bN2rf1AAeuWhGfh5no0S-Y,4246
 nucliadb_utils/audit/stream.py,sha256=A0PK-uHW-gXxrDsggbWwbcf2x1CWfP2rYf60wSixg0k,17670
 nucliadb_utils/cache/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu4UBh8,833
@@ -41,9 +41,9 @@ nucliadb_utils/storages/gcs.py,sha256=VyT72My34N4pEMmrQc5wdAMNLiuqpYl8OW3d50cJfS
 nucliadb_utils/storages/local.py,sha256=2aCHpZymORG_dUc1FDq0VFcgQulu0w2pZiUaj9dphFs,11686
 nucliadb_utils/storages/nuclia.py,sha256=vEv94xAT7QM2g80S25QyrOw2pzvP2BAX-ADgZLtuCVc,2097
 nucliadb_utils/storages/object_store.py,sha256=2PueRP5Q3XOuWgKhj6B9Kp2fyBql5np0T400YRUbqn4,4535
-nucliadb_utils/storages/s3.py,sha256=EUqlNoJW32AI6jpETbDla3teYbxlz8RFTfxSdHgWZdo,21878
+nucliadb_utils/storages/s3.py,sha256=R-ZksX6e0uvp3AxHdG0rGNgvZ5XMnOR_OwGRc_F3o3M,21878
 nucliadb_utils/storages/settings.py,sha256=mepN3wbLGL0Pv5yI6D-sNjSAFinEWT7aRi6N3eClNDg,1384
-nucliadb_utils/storages/storage.py,sha256=aOJnx6-WX8U3AAqPL_sWPCghIzlr8e3GKGi8z3-mtqw,22024
+nucliadb_utils/storages/storage.py,sha256=HKY-2gTMC-KCsJn9jW0OO2yvwkwl5IusLPahDHsIgno,21912
 nucliadb_utils/storages/utils.py,sha256=F4Iboa_0_bhDQr-JOKD9sGPld_-hKwJW5ptyZdn9Oag,1505
 nucliadb_utils/tests/__init__.py,sha256=Oo9CAE7B0eW5VHn8sHd6o30SQzOWUhktLPRXdlDOleA,1456
 nucliadb_utils/tests/asyncbenchmark.py,sha256=vrX_x9ifCXi18PfNShc23w9x_VUiB_Ph-2nuolh9z3Q,10707
@@ -53,7 +53,7 @@ nucliadb_utils/tests/gcs.py,sha256=JNqp5ymeNNU9Ci8rNYTh7-VqP4fjybElhyB3ap7EV1c,4
 nucliadb_utils/tests/local.py,sha256=z9E11_ol1mu7N8Y6PkjKl-WMPPMl7JqQbDj3uhVa1A0,1933
 nucliadb_utils/tests/nats.py,sha256=rbTaC6kv-u6SdZ7N-XBEGS40XCRiUmFUsKHIYWJfxTs,2325
 nucliadb_utils/tests/s3.py,sha256=kz9ULxrAYLVslZ59I8dtweZ9DJz5R8Ioy2XYrveZzHw,3829
-nucliadb_utils-6.9.5.post5447.dist-info/METADATA,sha256=6Xl5Km6XkK8mFKdFWW6ebDu_r22nBt-W6YPUGq7WoyI,2174
-nucliadb_utils-6.9.5.post5447.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb_utils-6.9.5.post5447.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
-nucliadb_utils-6.9.5.post5447.dist-info/RECORD,,
+nucliadb_utils-6.10.0.post5617.dist-info/METADATA,sha256=jQ6YCYc6UPaNF3oaztE-iyHgdwujPX_Os8_6AdfIzB0,2179
+nucliadb_utils-6.10.0.post5617.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb_utils-6.10.0.post5617.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
+nucliadb_utils-6.10.0.post5617.dist-info/RECORD,,