redis-allocator 0.4.3__py3-none-any.whl → 0.4.5__py3-none-any.whl

redis_allocator/_version.py

@@ -1 +1 @@
-__version__ = '0.4.3'
+__version__ = '0.4.5'

redis_allocator/allocator.py

@@ -632,23 +632,51 @@ class RedisAllocator(RedisLockPool, Generic[U]):
         local poolItemsKey = pool_str()
         local headKey      = pool_pointer_str(true)
         local tailKey      = pool_pointer_str(false)
-        local function push_to_tail(itemName, expiry)  -- push the item to the free list
-            local tail = redis.call("GET", tailKey)
-            local head = redis.call("GET", headKey)
-            if not tail then
-                tail = ""
+        local function current_pointer(pointerKey)
+            local pointer = redis.call("GET", pointerKey)
+            if not pointer then
+                return ""
+            end
+            return pointer
+        end
+        local function current_head()
+            return current_pointer(headKey)
+        end
+        local function current_tail()
+            return current_pointer(tailKey)
+        end
+        local function set_current_head(pointer)
+            redis.call("SET", headKey, pointer)
+        end
+        local function set_current_tail(pointer)
+            redis.call("SET", tailKey, pointer)
+        end
+
+        local function push_to_tail(itemName, expiry)  -- push the item to the free list, item should not be in the free list
+            local head = current_head()
+            local tail = current_tail()
+            ---- this is the debug code
+            local itemVal = redis.call("HGET", poolItemsKey, itemName)
+            if itemVal then
+                local prev, next, expiry = split_pool_value(itemVal)
+                assert(prev == "#ALLOCATED" or prev == "", "item should be allocated or free")
+                assert(next == "" or next == "#ALLOCATED", "item should be the last item in the free list")
             end
+            -- set the item points to the tail
             redis.call("HSET", poolItemsKey, itemName, join_pool_value(tail, "", expiry))
-            if tail == "" then  -- the free list is empty chain
-                redis.call("SET", headKey, itemName)
+            if tail == "" or head == "" then  -- the free list is empty chain
+                -- assert(tail == "" and head == "", "head or tail should not be empty")
+                set_current_head(itemName)
             else
                 local tailVal = redis.call("HGET", poolItemsKey, tail)
-                local prev, next, expiry = split_pool_value(tailVal)
-                assert(next == "" or next == '#ALLOCATED', "tail is not the last item in the free list")
-                redis.call("HSET", poolItemsKey, tail, join_pool_value(prev, itemName, expiry))
+                assert(tailVal, "tail value should not be nil")
+                local tailPrev, tailNext, tailExpiry = split_pool_value(tailVal)
+                assert(tailNext == "", "tail should be the last item in the free list")
+                redis.call("HSET", poolItemsKey, tail, join_pool_value(tailPrev, itemName, tailExpiry))
             end
-            redis.call("SET", tailKey, itemName)  -- set the tail point to the new item
+            set_current_tail(itemName)
         end
+
         -- Ensure the new head node is well-formed (prev="") and update tail/head
         local function set_item_head_nil(nextItemName)
             if nextItemName == "" then
@@ -671,94 +699,80 @@ class RedisAllocator(RedisLockPool, Generic[U]):
                 redis.call("HSET", poolItemsKey, nextItemName, join_pool_value("", nextNext, nextExpiry))
             end
         end
+
         local function pop_from_head()  -- pop the item from the free list
-            local head = redis.call("GET", headKey)
+            local head = current_head()
             if not head or head == "" then  -- the free list is empty
                 return nil, -1
             end
             local headVal = redis.call("HGET", poolItemsKey, head)
-            assert(headVal ~= nil, "head should not nil")
+            assert(headVal, "head value should not be nil")
             local headPrev, headNext, headExpiry = split_pool_value(headVal)
             -- Check if the head item has expired or is locked
             if is_expiry_invalid(headExpiry) then  -- the item has expired
                 redis.call("HDEL", poolItemsKey, head)
-                redis.call("SET", headKey, headNext)
-                set_item_head_nil(headNext)
+                set_current_head(headNext)
+                -- set_item_head_nil(headNext)
                 return pop_from_head()
             elseif redis.call("EXISTS", key_str(head)) > 0 then  -- the item is locked
                 redis.call("HSET", poolItemsKey, head, join_pool_value("#ALLOCATED", "#ALLOCATED", headExpiry))
-                redis.call("SET", headKey, headNext)
-                set_item_head_nil(headNext)
+                set_current_head(headNext)
+                -- set_item_head_nil(headNext)
                 return pop_from_head()
             elseif headNext == "" then  -- the item is the last in the free list
                 redis.call("SET", headKey, "")
                 redis.call("SET", tailKey, "")
             else
                 local nextVal = redis.call("HGET", poolItemsKey, headNext)
+                assert(nextVal, "next value should not be nil")
                 local nextPrev, nextNext, nextExpiry = split_pool_value(nextVal)
                 redis.call("HSET", poolItemsKey, headNext, join_pool_value("", nextNext, nextExpiry))
                 redis.call("SET", headKey, headNext)
             end
             redis.call("HSET", poolItemsKey, head, join_pool_value("#ALLOCATED", "#ALLOCATED", headExpiry))
-            -- If we removed the current head, update head pointer
-            local savedHead = redis.call("GET", headKey)
-            if savedHead == itemName then
-                redis.call("SET", headKey, next or "")
-            end
             return head, headExpiry
         end
+
         local function set_item_allocated(itemName, val)
             if not val then
                 val = redis.call("HGET", pool_str(), itemName)
             end
-            if val then
-                local prev, next, expiry = split_pool_value(val)
-                if prev ~= "#ALLOCATED" then
-                    if is_expiry_invalid(expiry) then
-                        redis.call("HDEL", poolItemsKey, itemName)
-                    end
-                    if prev ~= "" then
-                        local prevVal = redis.call("HGET", poolItemsKey, prev)
-                        if prevVal then
-                            local prevPrev, prevNext, prevExpiry = split_pool_value(prevVal)
-                            redis.call("HSET", poolItemsKey, prev, join_pool_value(prevPrev, next, prevExpiry))
-                        end
-                    else
-                        redis.call("SET", headKey, next or "")
-                    end
-                    if next ~= "" then
-                        local nextVal = redis.call("HGET", poolItemsKey, next)
-                        if nextVal then
-                            local nextPrev, nextNext, nextExpiry = split_pool_value(nextVal)
-                            redis.call("HSET", poolItemsKey, next, join_pool_value(prev, nextNext, nextExpiry))
-                        end
-                    else
-                        redis.call("SET", tailKey, prev or "")
-                    end
-                    redis.call("HSET", poolItemsKey, itemName, join_pool_value("#ALLOCATED", "#ALLOCATED", expiry))
-                    -- If we removed the current head, update head pointer
-                    local savedHead = redis.call("GET", headKey)
-                    if savedHead == itemName then
-                        redis.call("SET", headKey, next or "")
-                    end
+            assert(val, "val should not be nil")
+            local prev, next, expiry = split_pool_value(val)
+            if prev ~= "#ALLOCATED" then
+                assert(next ~= "#ALLOCATED", "next item should not be allocated")
+                if is_expiry_invalid(expiry) then
+                    redis.call("HDEL", poolItemsKey, itemName)
                 end
-            end
-            -- Invariant fix: ensure current head's prev pointer is always "".
-            local currentHead = redis.call("GET", headKey)
-            if currentHead and currentHead ~= "" then
-                local currentHeadVal = redis.call("HGET", poolItemsKey, currentHead)
-                if currentHeadVal then
-                    local curPrev, curNext, curExpiry = split_pool_value(currentHeadVal)
-                    if curPrev ~= "" then
-                        redis.call("HSET", poolItemsKey, currentHead, join_pool_value("", curNext, curExpiry))
-                    end
+                if prev ~= "" then
+                    local prevVal = redis.call("HGET", poolItemsKey, prev)
+                    assert(prevVal, "prev value should not be nil", prev, prevVal)
+                    local prevPrev, prevNext, prevExpiry = split_pool_value(prevVal)
+                    redis.call("HSET", poolItemsKey, prev, join_pool_value(prevPrev, next, prevExpiry))
+                else
+                    redis.call("SET", headKey, next)
+                end
+                if next ~= "" then
+                    local nextVal = redis.call("HGET", poolItemsKey, next)
+                    assert(nextVal, "next value should not be nil")
+                    local nextPrev, nextNext, nextExpiry = split_pool_value(nextVal)
+                    redis.call("HSET", poolItemsKey, next, join_pool_value(prev, nextNext, nextExpiry))
+                else
+                    redis.call("SET", tailKey, prev)
                 end
+                redis.call("HSET", poolItemsKey, itemName, join_pool_value("#ALLOCATED", "#ALLOCATED", expiry))
+                -- If we removed the current head, update head pointer
+                -- local savedHead = redis.call("GET", headKey)
+                -- if savedHead == itemName then
+                --     redis.call("SET", headKey, next or "")
+                -- end
+            else
+                assert(next == "#ALLOCATED", "next item should also be allocated")
             end
         end
-        local function check_item_health(itemName, value)
-            if not value then
-                value = redis.call("HGET", pool_str(), itemName)
-            end
+
+        local function check_item_health(itemName)
+            local value = redis.call("HGET", pool_str(), itemName)
             assert(value, "value should not be nil")
             local prev, next, expiry = split_pool_value(value)
             if is_expiry_invalid(expiry) then  -- Check if the item has expired
@@ -777,6 +791,37 @@ class RedisAllocator(RedisLockPool, Generic[U]):
                 end
             end
         end
+
+        -- Return an array of item names in the free list order (head -> tail)
+        -- Does NOT modify the list. Uses a bounded loop (at most pool size)
+        -- to avoid infinite traversal when the list structure is corrupted.
+        local function get_free_list()
+            local items = {{}}
+            local current = redis.call("GET", headKey)
+            if not current or current == "" then
+                return items  -- empty list
+            end
+            local max_iters = tonumber(redis.call("HLEN", poolItemsKey))
+            if not max_iters or max_iters <= 0 then
+                return items
+            end
+            for i = 1, max_iters do
+                if not current or current == "" then
+                    break
+                end
+                table.insert(items, current)
+                local val = redis.call("HGET", poolItemsKey, current)
+                if not val then
+                    break  -- corrupted pointer
+                end
+                local _prev, nxt, _expiry = split_pool_value(val)
+                if nxt == "" or nxt == "#ALLOCATED" then
+                    break -- reached tail or allocated marker
+                end
+                current = nxt
+            end
+            return items
+        end
         '''
 
     @cached_property
@@ -1154,6 +1199,16 @@ class RedisAllocator(RedisLockPool, Generic[U]):
         """
         self.free_keys(obj.key, timeout=timeout)
 
+    @cached_property
+    def _get_free_list_script(self):
+        return self.redis.register_script(f'''
+        {self._lua_required_string}
+        return get_free_list()
+        ''')
+
+    def get_free_list(self):
+        return self._get_free_list_script()
+
     def _gc_cursor_str(self):
         """Get the Redis key for the garbage collection cursor.
 
@@ -1195,10 +1250,11 @@ class RedisAllocator(RedisLockPool, Generic[U]):
         local scanResult = redis.call("HSCAN", pool_str(), get_cursor(), "COUNT", n)
         local newCursor  = scanResult[1]
         local kvList     = scanResult[2]
+        local t = ""
         for i = 1, #kvList, 2 do
             local itemName = kvList[i]
-            local val      = kvList[i + 1]
-            check_item_health(itemName, val)
+            -- local val      = kvList[i + 1]
+            check_item_health(itemName)
         end
         set_cursor(newCursor)
         ''')

{redis_allocator-0.4.3.dist-info → redis_allocator-0.4.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redis-allocator
-Version: 0.4.3
+Version: 0.4.5
 Summary: Redis-based resource allocation system.
 Home-page: https://github.com/invoker-bot/RedisAllocator-python
 Author: Invoker Bot

{redis_allocator-0.4.3.dist-info → redis_allocator-0.4.5.dist-info}/RECORD

@@ -1,15 +1,15 @@
 redis_allocator/__init__.py,sha256=TVjUm-8YEu_MQD_PkfeIKiVknpCJBrUY9cWN1LlaZcU,1016
-redis_allocator/_version.py,sha256=GI8jYEksCyU_GLvu0eW6pPFjDcmTuWNQVavfvCX2KuQ,22
-redis_allocator/allocator.py,sha256=WZ3UkdAIq8x-hywA_8jyQcPLRu17G-QgqRCdFjfo6tg,51029
+redis_allocator/_version.py,sha256=jMc7UM2pASY9F8z0XMALQbSXxGCAxWPyeXQlarwD0VI,22
+redis_allocator/allocator.py,sha256=VkJk6ioN6hwe227lMKjNdyeO0QkJ8sajTDPkEQgDcWg,52964
 redis_allocator/lock.py,sha256=fqf6WUWHKYenEArWopMIF6kWEnDfADC-bZvnQImsQVo,27400
 redis_allocator/task_queue.py,sha256=8DjNr2uxhzCsHatV_CHOeGh7_K9pqQZFApSbe2blRO0,14989
-redis_allocator-0.4.3.dist-info/licenses/LICENSE,sha256=Wt4X1rHpffQfEiyWcDUx8BMLjXxfPqaiYZ7Lgsj7L4c,1068
+redis_allocator-0.4.5.dist-info/licenses/LICENSE,sha256=Wt4X1rHpffQfEiyWcDUx8BMLjXxfPqaiYZ7Lgsj7L4c,1068
 tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/conftest.py,sha256=Ts82uylQSzP_GcaN0E02o3xcFdjw20cXNzh3RAdYKW4,3967
-tests/test_allocator.py,sha256=_aoSDkzyjiUyzw6WoO2bMV1Ow9ymLOOMSJrp7TENU70,22139
+tests/test_allocator.py,sha256=zOBdaOhlpLsv63J4QZ9vOhL1k1gu_vO68x4CpPDa9_4,23350
 tests/test_lock.py,sha256=MDMRNN46VhWqkHUIhYOMEDgZkFFCW_WjwRLTOjkFF-Q,46952
 tests/test_task_queue.py,sha256=Fh5naikFajfOvL6GngEy_TPfOYCYZolZfVwtR6T4dTY,31710
-redis_allocator-0.4.3.dist-info/METADATA,sha256=yHNCfaUB9TQQSYMkWWdnhmuI72tT5ESvod2SyCyuFNU,21727
-redis_allocator-0.4.3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-redis_allocator-0.4.3.dist-info/top_level.txt,sha256=0hXzU7sK5FCeSolTEYxThOt3HOybnwaXv1FLRJvHVgI,22
-redis_allocator-0.4.3.dist-info/RECORD,,
+redis_allocator-0.4.5.dist-info/METADATA,sha256=qNua-7xwZmj7Vyexpugt2h65v1c8Tn7SCKfRJc6CLZI,21727
+redis_allocator-0.4.5.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+redis_allocator-0.4.5.dist-info/top_level.txt,sha256=0hXzU7sK5FCeSolTEYxThOt3HOybnwaXv1FLRJvHVgI,22
+redis_allocator-0.4.5.dist-info/RECORD,,

{redis_allocator-0.4.3.dist-info → redis_allocator-0.4.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

tests/test_allocator.py

@@ -9,6 +9,7 @@ This module tests the functionality of:
 import time
 import datetime
 import threading
+# from itertools import cycle
 from operator import xor
 from pytest_mock import MockFixture
 from redis import Redis
@@ -277,6 +278,7 @@ class TestRedisAllocator:
             assert not redis_allocator.is_locked(key)
             assert not redis_client.exists(redis_allocator._key_str(key))
 
+        # print(self.get_redis_pool_state(redis_allocator, redis_client))
         allocated_key = redis_allocator.malloc_key(timeout=30)
         assert allocated_key == "key1"
         assert xor(shared, redis_allocator.is_locked(allocated_key))
@@ -427,7 +429,11 @@ class TestRedisAllocator:
             )
             time.tick(600)
             allocator_with_policy.gc()
+            print(self.get_redis_pool_state(allocator_with_policy, redis_client))
+            redis_client.register_script("print('-------------------------------')")()
             allocator_with_policy.gc()  # some times should be called twice to remove the expired items
+            redis_client.register_script("print('-------------------------------')")()
+            print(self.get_redis_pool_state(allocator_with_policy, redis_client))
             allocator_with_policy.policy.refresh_pool(allocator_with_policy)
             assert len(allocator_with_policy) == 4
             allocator_with_policy.gc()
@@ -491,3 +497,20 @@ class TestRedisAllocator:
                 else:
                     state = ['key1', 'key2'], ['key3']
             assert self.get_redis_pool_state(redis_allocator, redis_client) == self.generate_pool_state(redis_client, *state)
+
+
+    # def test_allocator_health_check(self, redis_allocator: RedisAllocator, mocker: MockFixture):
+    #     ''''''
+    #     with freeze_time("2024-01-01") as time:
+    #         for i in range(100):
+    #             obj = _TestObject(name = f"test_object_{i}")
+    #             mocker.patch.object(obj, 'is_healthy', side_effect = cycle([True, False]))
+    #             
+    #             redis_allocator.malloc(obj = obj, timeout = 30)
+    #             time.tick(30)
+    #             redis_allocator.gc()
+    #             unh, h = redis_allocator.health_check()
+    #             assert len(redis_allocator.get_free_list()) == h
+    #         # time.tick(100)
+    #         # redis_allocator.gc()
+    #         # assert not redis_allocator.is_locked("key1")