swapcollection 1.0.0__py3-none-any.whl

swapcollection/__init__.py

@@ -0,0 +1,3 @@
+from .swapcollection import SwapDict, SwapList
+
+__all__ = ["SwapDict", "SwapList"]

swapcollection/swapcollection.py

@@ -0,0 +1,234 @@
+from sqlite_utils import Database
+from sqlite_utils.db import NotFoundError
+import uuid
+from collections import UserDict, UserList
+import pickle
+
+class SwapDict(UserDict):
+    PREFIX = "swapdict_"
+    PATH = "swapcollections_cache.db"
+        
+    def __init__(self, data={}, size_threshold: int = 1048576):
+        self.db = Database(self.PATH)
+        self.table = self.db["DICT"]
+        self.size_threshold = size_threshold
+
+        if not self.table.exists():
+            self.table.create(
+                {
+                    "id": str,
+                    "value": bytes,
+                },
+                pk="id",
+            )
+
+        super().__init__(data) # 内部で__setitem__を呼んで処理するので、ここでは何もしない
+
+    def _setprocess(self, value):
+        id = self.PREFIX + str(uuid.uuid4())
+        self.table.insert({"id": id, "value": pickle.dumps(value)})
+        return str(id)
+
+    def _getprocess(self, id):
+        result = self.table.get(id)
+        if result:
+            return pickle.loads(result["value"])
+        raise NotFoundError(f"id {id} not found")
+
+    def __setitem__(self, key, value):
+        if len(pickle.dumps(value)) >= self.size_threshold:
+            value = self._setprocess(value)
+        return super().__setitem__(key, value)
+
+    def __getitem__(self, key):
+        value = super().__getitem__(key)
+        if type(value) == str:
+            if value.startswith(self.PREFIX):
+                return self._getprocess(value)
+        return value
+
+    def __repr__(self):
+        self.repr_data = UserDict()
+        for key, value in self.items():
+            if type(value) == str:
+                if value.startswith(self.PREFIX):
+                    value = self._getprocess(value)
+            self.repr_data[key] = value
+        return repr(self.repr_data)
+
+class SwapList(UserList):
+    PREFIX = "swaplist_"
+    PATH = "swapcollections_cache.db"
+    
+    def __init__(self, data=None, size_threshold: int = 1048576):
+        self.db = Database(self.PATH)
+        self.table = self.db["LIST"]
+        self.size_threshold = size_threshold
+
+        if not self.table.exists():
+            self.table.create(
+                {
+                    "id": str,
+                    "value": bytes,
+                },
+                pk="id",
+            )
+
+        tmp_data = []
+        for i in (data or []):
+            if len(pickle.dumps(i)) >= self.size_threshold:
+                i = self._setprocess(i)
+            tmp_data.append(i)
+
+        super().__init__(tmp_data)
+
+    def _setprocess(self, value):
+        id = self.PREFIX + str(uuid.uuid4())
+        self.table.insert({"id": id, "value": pickle.dumps(value)})
+        return id
+
+    def _getprocess(self, id):
+        result = self.table.get(id)
+        if result:
+            return pickle.loads(result["value"])
+        raise NotFoundError(f"id {id} not found")
+
+    def __setitem__(self, i, value):
+        if isinstance(i, slice):
+            processed = []
+            for v in value:
+                if len(pickle.dumps(v)) >= self.size_threshold:
+                    v = self._setprocess(v)
+                processed.append(v)
+            return super().__setitem__(i, processed)
+        if len(pickle.dumps(value)) >= self.size_threshold:
+            value = self._setprocess(value)
+        return super().__setitem__(i, value)
+
+    def __eq__(self, other):
+        try:
+            return list(self) == list(other)
+        except TypeError:
+            return NotImplemented
+    
+    def __getitem__(self, i):
+        value = super().__getitem__(i)
+        if type(value) == str:            
+            if value.startswith(self.PREFIX):                
+                return self._getprocess(value)
+        return value
+    
+    def _resolve(self, value):
+        """Return the original value if it's a spill ID, otherwise return as-is."""
+        if isinstance(value, str) and value.startswith(self.PREFIX):
+            return self._getprocess(value)
+        return value
+
+    def __contains__(self, item):
+        for i in range(len(self)):
+            if self[i] == item:
+                return True
+        return False
+
+    def pop(self, i=-1):
+        resolved = self[i]
+        self.data.pop(i)
+        return resolved
+
+    def index(self, item, *args):
+        start = args[0] if len(args) > 0 else 0
+        stop = args[1] if len(args) > 1 else len(self)
+        for i in range(start, stop):
+            if self[i] == item:
+                return i
+        raise ValueError(f"{item!r} is not in list")
+
+    def count(self, item):
+        cnt = 0
+        for i in range(len(self)):
+            if self[i] == item:
+                cnt += 1
+        return cnt
+
+    def remove(self, item):
+        idx = self.index(item)
+        del self[idx]
+
+    def sort(self, *args, **kwds):
+        resolved = [self._resolve(v) for v in self.data]
+        resolved.sort(*args, **kwds)
+        self.data.clear()
+        for value in resolved:
+            if len(pickle.dumps(value)) >= self.size_threshold:
+                value = self._setprocess(value)
+            self.data.append(value)
+
+    def append(self, value):
+        if len(pickle.dumps(value)) >= self.size_threshold:
+            value = self._setprocess(value)
+        return super().append(value)
+
+    def extend(self, value):
+        tmp_value = []
+        for i in value:
+            if len(pickle.dumps(i)) >= self.size_threshold:
+                i = self._setprocess(i)
+            tmp_value.append(i)
+        return super().extend(tmp_value)
+
+    def __repr__(self):
+        self.repr_data = UserList()
+        for i in self:
+            if type(i) == str:
+                if i.startswith(self.PREFIX):
+                    i = self._getprocess(i)
+            self.repr_data.append(i)
+        return repr(self.repr_data)
+
+if __name__ == "__main__":
+
+    class TestObj:
+        def __init__(self, a, b):
+            self.a = a
+            self.b = b
+            self.c = a + b
+        
+        def test(self):
+            return self.c
+
+    testobj = TestObj(1, 2)
+
+    tmp = SwapList(
+        [
+            "hello", 
+            {"hello": "world"},
+            testobj,
+            1977,
+        ],
+        size_threshold=1,
+    )
+
+    tmp.append("hello")
+    tmp.extend([{"hello": "world"}, testobj, 1977])
+    tmp[2:4] = ["hello", {"hello": "world"}, testobj, 1977]
+    print(tmp)
+
+
+    for i in tmp:
+        print(i)
+
+    # tmp2 = SwapList(
+    #     [
+    #         "small",
+    #         {"hello": "world"},
+    #         "large",
+    #         {"hello": "world", "hello": "world", "hello": "world"},
+    #     ],
+    #     size_threshold=1,
+    # )
+
+    # print(tmp2)
+    # print(tmp2[0])
+    # print(tmp2[1])
+    # print(tmp2[2])    
+    # print(tmp2[3])

swapcollection-1.0.0.dist-info/METADATA

@@ -0,0 +1,68 @@
+Metadata-Version: 2.4
+Name: swapcollection
+Version: 1.0.0
+Summary: A library that automatically offloads large pickle-able objects stored in a dict/list to SQLite, reducing memory usage. It can be used with the same syntax as a regular dict/list.
+Author: llinghyami
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/llinghyami/swapcollection
+Project-URL: Repository, https://github.com/llinghyami/swapcollection.git
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: sqlite-utils
+Dynamic: license-file
+
+> **English** · [日本語 (Japanese)](README.md)
+
+# swapcollection
+
+A library that automatically offloads large objects stored in a
+`dict`/`list` to SQLite, reducing memory usage. It can be used with
+the same syntax as a regular `dict`/`list`.
+
+## Quick Start
+
+```bash
+pip install swapcollection
+```
+
+```python
+from swapcollection import SwapDict, SwapList
+
+# Spill values >= 1 MB (pickle size) to SQLite
+data = SwapDict(size_threshold=1024)
+
+data["small"] = b"hello"
+data["large"] = b"x" * 100_000
+
+# Access transparently — same as a normal dict
+print(data["small"])
+print(len(data["large"]))
+```
+
+## How It Works
+
+`SwapDict` / `SwapList` automatically offloads values whose pickled size
+meets or exceeds `size_threshold` to a SQLite database. Any pickle-able
+object (not just `bytes`) is eligible.
+
+| Value | Storage |
+|---|---|
+| Pickle size < `size_threshold` | In-memory |
+| Pickle size >= `size_threshold` | SQLite (transparent) |
+
+Spilled values are retrieved, updated, and deleted using the same `dict`/`list`
+interface — no special API calls needed.
+
+## Caveats
+
+- `size_threshold` is the pickle-size threshold in bytes; values >= this size
+  are spilled.
+- `size_threshold` cannot be changed after instantiation.
+- Any pickle-able object type (not only `bytes`) can be spilled.
+- Deleting the SQLite database will cause errors on previously spilled items.
+
+## License
+
+MIT

swapcollection-1.0.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+swapcollection/__init__.py,sha256=OXj6fYQu7-Bj-Msmx3f1O7K6VubrOTWtEE4isSshR-s,83
+swapcollection/swapcollection.py,sha256=-Dp_SD1uqqUdEpbmWPjU2nsTEVYo7acTgx_NGoe_tR8,6769
+swapcollection-1.0.0.dist-info/licenses/LICENSE,sha256=BM5MwGCc7-gpyfEiiWcqbmDcpyTVco9NLSM6uY-adu4,1067
+swapcollection-1.0.0.dist-info/METADATA,sha256=n8fyltJ3mQ0_Hr0FDZ0KKC4pZlAeCQ2WGXKWaIZ2kKk,2006
+swapcollection-1.0.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+swapcollection-1.0.0.dist-info/top_level.txt,sha256=XB7bFNMYsk7b4MZP9yexv2oEMPofADUvpZ_qUCnscvw,15
+swapcollection-1.0.0.dist-info/RECORD,,

swapcollection-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

swapcollection-1.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 renren5977
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

swapcollection-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+swapcollection