redis-allocator 0.4.2__tar.gz → 0.4.4__tar.gz

{redis_allocator-0.4.2/redis_allocator.egg-info → redis_allocator-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redis-allocator
-Version: 0.4.2
+Version: 0.4.4
 Summary: Redis-based resource allocation system.
 Home-page: https://github.com/invoker-bot/RedisAllocator-python
 Author: Invoker Bot

{redis_allocator-0.4.2 → redis_allocator-0.4.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.4.2"
+version = "0.4.4"
 tag_format = "v$version"
 
 [tool.commitizen.version_files]

redis_allocator-0.4.4/redis_allocator/_version.py

@@ -0,0 +1 @@
+__version__ = '0.4.4'

{redis_allocator-0.4.2 → redis_allocator-0.4.4}/redis_allocator/allocator.py

@@ -632,95 +632,153 @@ class RedisAllocator(RedisLockPool, Generic[U]):
         local poolItemsKey = pool_str()
         local headKey      = pool_pointer_str(true)
         local tailKey      = pool_pointer_str(false)
-        local function push_to_tail(itemName, expiry)  -- push the item to the free list
-            local tail = redis.call("GET", tailKey)
-            if not tail then
-                tail = ""
+        local function current_pointer(pointerKey)
+            local pointer = redis.call("GET", pointerKey)
+            if not pointer then
+                return ""
             end
+            return pointer
+        end
+        local function current_head()
+            return current_pointer(headKey)
+        end
+        local function current_tail()
+            return current_pointer(tailKey)
+        end
+        local function set_current_head(pointer)
+            redis.call("SET", headKey, pointer)
+        end
+        local function set_current_tail(pointer)
+            redis.call("SET", tailKey, pointer)
+        end
+
+        local function push_to_tail(itemName, expiry)  -- push the item to the free list, item should not be in the free list
+            local head = current_head()
+            local tail = current_tail()
+            ---- this is the debug code
+            local itemVal = redis.call("HGET", poolItemsKey, itemName)
+            if itemVal then
+                local prev, next, expiry = split_pool_value(itemVal)
+                assert(prev == "#ALLOCATED" or prev == "", "item should be allocated or free")
+                assert(next == "" or next == "#ALLOCATED", "item should be the last item in the free list")
+            end
+            -- set the item points to the tail
             redis.call("HSET", poolItemsKey, itemName, join_pool_value(tail, "", expiry))
-            if tail == "" then  -- the free list is empty chain
-                redis.call("SET", headKey, itemName)
+            if tail == "" or head == "" then  -- the free list is empty chain
+                assert(tail == "" and head == "", "head or tail should not be empty")
+                set_current_head(itemName)
             else
                 local tailVal = redis.call("HGET", poolItemsKey, tail)
-                local prev, next, expiry = split_pool_value(tailVal)
-                assert(next == "", "tail is not the last item in the free list")
-                redis.call("HSET", poolItemsKey, tail, join_pool_value(prev, itemName, expiry))
+                assert(tailVal, "tail value should not be nil")
+                local tailPrev, tailNext, tailExpiry = split_pool_value(tailVal)
+                assert(tailNext == "", "tail should be the last item in the free list")
+                redis.call("HSET", poolItemsKey, tail, join_pool_value(tailPrev, itemName, tailExpiry))
+            end
+            set_current_tail(itemName)
+        end
+
+        -- Ensure the new head node is well-formed (prev="") and update tail/head
+        local function set_item_head_nil(nextItemName)
+            if nextItemName == "" then
+                -- list becomes empty
+                redis.call("SET", headKey, "")
+                redis.call("SET", tailKey, "")
+                return
+            end
+
+            local nextVal = redis.call("HGET", poolItemsKey, nextItemName)
+            if not nextVal then
+                -- corrupted pointer, clear list
+                redis.call("SET", headKey, "")
+                redis.call("SET", tailKey, "")
+                return
+            end
+
+            local _prev, nextNext, nextExpiry = split_pool_value(nextVal)
+            if _prev ~= "" then
+                redis.call("HSET", poolItemsKey, nextItemName, join_pool_value("", nextNext, nextExpiry))
             end
-            redis.call("SET", tailKey, itemName)  -- set the tail point to the new item
         end
+
         local function pop_from_head()  -- pop the item from the free list
-            local head = redis.call("GET", headKey)
+            local head = current_head()
             if not head or head == "" then  -- the free list is empty
                 return nil, -1
             end
             local headVal = redis.call("HGET", poolItemsKey, head)
-            assert(headVal ~= nil, "head should not nil")
+            assert(headVal, "head value should not be nil")
             local headPrev, headNext, headExpiry = split_pool_value(headVal)
             -- Check if the head item has expired or is locked
             if is_expiry_invalid(headExpiry) then  -- the item has expired
                 redis.call("HDEL", poolItemsKey, head)
-                redis.call("SET", headKey, headNext)
+                set_current_head(headNext)
+                -- set_item_head_nil(headNext)
                 return pop_from_head()
-            end
-            if redis.call("EXISTS", key_str(head)) > 0 then  -- the item is locked
+            elseif redis.call("EXISTS", key_str(head)) > 0 then  -- the item is locked
                 redis.call("HSET", poolItemsKey, head, join_pool_value("#ALLOCATED", "#ALLOCATED", headExpiry))
-                redis.call("SET", headKey, headNext)
+                set_current_head(headNext)
+                -- set_item_head_nil(headNext)
                 return pop_from_head()
-            end
-            local prev, next, expiry = split_pool_value(headVal)
-            if next == "" then  -- the item is the last in the free list
+            elseif headNext == "" then  -- the item is the last in the free list
                 redis.call("SET", headKey, "")
                 redis.call("SET", tailKey, "")
             else
-                local nextVal = redis.call("HGET", poolItemsKey, next)
+                local nextVal = redis.call("HGET", poolItemsKey, headNext)
+                assert(nextVal, "next value should not be nil")
                 local nextPrev, nextNext, nextExpiry = split_pool_value(nextVal)
-                redis.call("HSET", poolItemsKey, next, join_pool_value("", nextNext, nextExpiry))
-                redis.call("SET", headKey, next)
+                redis.call("HSET", poolItemsKey, headNext, join_pool_value("", nextNext, nextExpiry))
+                redis.call("SET", headKey, headNext)
             end
-            redis.call("HSET", poolItemsKey, head, join_pool_value("#ALLOCATED", "#ALLOCATED", expiry))
+            redis.call("HSET", poolItemsKey, head, join_pool_value("#ALLOCATED", "#ALLOCATED", headExpiry))
             return head, headExpiry
         end
+
         local function set_item_allocated(itemName, val)
             if not val then
-                val = redis.call("HGET", poolItemsKey, itemName)
+                val = redis.call("HGET", pool_str(), itemName)
             end
-            if val then
-                local prev, next, expiry = split_pool_value(val)
-                if prev ~= "#ALLOCATED" then
-                    if is_expiry_invalid(expiry) then
-                        redis.call("HDEL", poolItemsKey, itemName)
-                    end
-                    if prev ~= "" then
-                        local prevVal = redis.call("HGET", poolItemsKey, prev)
-                        if prevVal then
-                            local prevPrev, prevNext, prevExpiry = split_pool_value(prevVal)
-                            redis.call("HSET", poolItemsKey, prev, join_pool_value(prevPrev, next, prevExpiry))
-                        end
-                    else
-                        redis.call("SET", headKey, next or "")
-                    end
-                    if next ~= "" then
-                        local nextVal = redis.call("HGET", poolItemsKey, next)
-                        if nextVal then
-                            local nextPrev, nextNext, nextExpiry = split_pool_value(nextVal)
-                            redis.call("HSET", poolItemsKey, next, join_pool_value(prev, nextNext, nextExpiry))
-                        end
-                    else
-                        redis.call("SET", tailKey, prev or "")
-                    end
-                    redis.call("HSET", poolItemsKey, itemName, join_pool_value("#ALLOCATED", "#ALLOCATED", expiry))
+            assert(val, "val should not be nil")
+            local prev, next, expiry = split_pool_value(val)
+            if prev ~= "#ALLOCATED" then
+                assert(next ~= "#ALLOCATED", "next item should not be allocated")
+                if is_expiry_invalid(expiry) then
+                    redis.call("HDEL", poolItemsKey, itemName)
+                end
+                if prev ~= "" then
+                    local prevVal = redis.call("HGET", poolItemsKey, prev)
+                    assert(prevVal, "prev value should not be nil", prev, prevVal)
+                    local prevPrev, prevNext, prevExpiry = split_pool_value(prevVal)
+                    redis.call("HSET", poolItemsKey, prev, join_pool_value(prevPrev, next, prevExpiry))
+                else
+                    redis.call("SET", headKey, next)
+                end
+                if next ~= "" then
+                    local nextVal = redis.call("HGET", poolItemsKey, next)
+                    assert(nextVal, "next value should not be nil")
+                    local nextPrev, nextNext, nextExpiry = split_pool_value(nextVal)
+                    redis.call("HSET", poolItemsKey, next, join_pool_value(prev, nextNext, nextExpiry))
+                else
+                    redis.call("SET", tailKey, prev)
                 end
+                redis.call("HSET", poolItemsKey, itemName, join_pool_value("#ALLOCATED", "#ALLOCATED", expiry))
+                -- If we removed the current head, update head pointer
+                -- local savedHead = redis.call("GET", headKey)
+                -- if savedHead == itemName then
+                --     redis.call("SET", headKey, next or "")
+                -- end
+            else
+                assert(next == "#ALLOCATED", "next item should also be allocated")
             end
         end
-        local function check_item_health(itemName, value)
-            if not value then
-                value = redis.call("HGET", pool_str(), itemName)
-            end
+
+        local function check_item_health(itemName)
+            local value = redis.call("HGET", pool_str(), itemName)
             assert(value, "value should not be nil")
             local prev, next, expiry = split_pool_value(value)
             if is_expiry_invalid(expiry) then  -- Check if the item has expired
-                set_item_allocated(itemName)
+                set_item_allocated(itemName, value)
                 redis.call("HDEL", poolItemsKey, itemName)
+                return
             end
             local locked = redis.call("EXISTS", key_str(itemName)) > 0
             if prev == "#ALLOCATED" then
@@ -729,10 +787,41 @@ class RedisAllocator(RedisLockPool, Generic[U]):
                 end
             else
                 if locked then
-                    set_item_allocated(itemName)
+                    set_item_allocated(itemName, value)
                 end
             end
         end
+
+        -- Return an array of item names in the free list order (head -> tail)
+        -- Does NOT modify the list. Uses a bounded loop (at most pool size)
+        -- to avoid infinite traversal when the list structure is corrupted.
+        local function get_free_list()
+            local items = {{}}
+            local current = redis.call("GET", headKey)
+            if not current or current == "" then
+                return items  -- empty list
+            end
+            local max_iters = tonumber(redis.call("HLEN", poolItemsKey))
+            if not max_iters or max_iters <= 0 then
+                return items
+            end
+            for i = 1, max_iters do
+                if not current or current == "" then
+                    break
+                end
+                table.insert(items, current)
+                local val = redis.call("HGET", poolItemsKey, current)
+                if not val then
+                    break  -- corrupted pointer
+                end
+                local _prev, nxt, _expiry = split_pool_value(val)
+                if nxt == "" or nxt == "#ALLOCATED" then
+                    break -- reached tail or allocated marker
+                end
+                current = nxt
+            end
+            return items
+        end
         '''
 
     @cached_property
@@ -1110,6 +1199,16 @@ class RedisAllocator(RedisLockPool, Generic[U]):
         """
         self.free_keys(obj.key, timeout=timeout)
 
+    @cached_property
+    def _get_free_list_script(self):
+        return self.redis.register_script(f'''
+        {self._lua_required_string}
+        return get_free_list()
+        ''')
+
+    def get_free_list(self):
+        return self._get_free_list_script()
+
     def _gc_cursor_str(self):
         """Get the Redis key for the garbage collection cursor.
 
@@ -1151,10 +1250,11 @@ class RedisAllocator(RedisLockPool, Generic[U]):
         local scanResult = redis.call("HSCAN", pool_str(), get_cursor(), "COUNT", n)
         local newCursor  = scanResult[1]
         local kvList     = scanResult[2]
+        local t = ""
         for i = 1, #kvList, 2 do
             local itemName = kvList[i]
-            local val      = kvList[i + 1]
-            check_item_health(itemName, val)
+            -- local val      = kvList[i + 1]
+            check_item_health(itemName)
         end
         set_cursor(newCursor)
         ''')

{redis_allocator-0.4.2 → redis_allocator-0.4.4/redis_allocator.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redis-allocator
-Version: 0.4.2
+Version: 0.4.4
 Summary: Redis-based resource allocation system.
 Home-page: https://github.com/invoker-bot/RedisAllocator-python
 Author: Invoker Bot

{redis_allocator-0.4.2 → redis_allocator-0.4.4}/tests/test_allocator.py

@@ -9,6 +9,7 @@ This module tests the functionality of:
 import time
 import datetime
 import threading
+# from itertools import cycle
 from operator import xor
 from pytest_mock import MockFixture
 from redis import Redis
@@ -277,6 +278,7 @@ class TestRedisAllocator:
             assert not redis_allocator.is_locked(key)
             assert not redis_client.exists(redis_allocator._key_str(key))
 
+        # print(self.get_redis_pool_state(redis_allocator, redis_client))
         allocated_key = redis_allocator.malloc_key(timeout=30)
         assert allocated_key == "key1"
         assert xor(shared, redis_allocator.is_locked(allocated_key))
@@ -427,7 +429,11 @@ class TestRedisAllocator:
             )
             time.tick(600)
             allocator_with_policy.gc()
+            print(self.get_redis_pool_state(allocator_with_policy, redis_client))
+            redis_client.register_script("print('-------------------------------')")()
             allocator_with_policy.gc()  # some times should be called twice to remove the expired items
+            redis_client.register_script("print('-------------------------------')")()
+            print(self.get_redis_pool_state(allocator_with_policy, redis_client))
             allocator_with_policy.policy.refresh_pool(allocator_with_policy)
             assert len(allocator_with_policy) == 4
             allocator_with_policy.gc()
@@ -491,3 +497,20 @@ class TestRedisAllocator:
                 else:
                     state = ['key1', 'key2'], ['key3']
             assert self.get_redis_pool_state(redis_allocator, redis_client) == self.generate_pool_state(redis_client, *state)
+
+
+    # def test_allocator_health_check(self, redis_allocator: RedisAllocator, mocker: MockFixture):
+    #     ''''''
+    #     with freeze_time("2024-01-01") as time:
+    #         for i in range(100):
+    #             obj = _TestObject(name = f"test_object_{i}")
+    #             mocker.patch.object(obj, 'is_healthy', side_effect = cycle([True, False]))
+    #             
+    #             redis_allocator.malloc(obj = obj, timeout = 30)
+    #             time.tick(30)
+    #             redis_allocator.gc()
+    #             unh, h = redis_allocator.health_check()
+    #             assert len(redis_allocator.get_free_list()) == h
+    #         # time.tick(100)
+    #         # redis_allocator.gc()
+    #         # assert not redis_allocator.is_locked("key1")

redis_allocator-0.4.2/redis_allocator/_version.py

@@ -1 +0,0 @@
-__version__ = '0.4.2'