collections-cache 0.2.2__tar.gz → 2025.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {collections_cache-0.2.2 → collections_cache-2025.3.3}/PKG-INFO +1 -1
- {collections_cache-0.2.2 → collections_cache-2025.3.3}/collections_cache/collections_cache.py +24 -11
- {collections_cache-0.2.2 → collections_cache-2025.3.3}/pyproject.toml +1 -1
- {collections_cache-0.2.2 → collections_cache-2025.3.3}/LICENSE +0 -0
- {collections_cache-0.2.2 → collections_cache-2025.3.3}/README.md +0 -0
- {collections_cache-0.2.2 → collections_cache-2025.3.3}/collections_cache/__init__.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: collections-cache
|
3
|
-
Version:
|
3
|
+
Version: 2025.3.3
|
4
4
|
Summary: collections-cache is a Python package for managing data collections across multiple SQLite databases. It allows efficient storage, retrieval, and updating of key-value pairs, supporting various data types serialized with pickle. The package uses parallel processing for fast access and manipulation of large collections.
|
5
5
|
License: MIT
|
6
6
|
Author: Luiz-Trindade
|
{collections_cache-0.2.2 → collections_cache-2025.3.3}/collections_cache/collections_cache.py
RENAMED
@@ -3,8 +3,8 @@ import sqlite3
|
|
3
3
|
from random import choice
|
4
4
|
from itertools import chain
|
5
5
|
from os import cpu_count, path, makedirs, scandir
|
6
|
-
|
7
|
-
from concurrent.futures import ThreadPoolExecutor as
|
6
|
+
from concurrent.futures import ProcessPoolExecutor as Pool
|
7
|
+
from concurrent.futures import ThreadPoolExecutor as Thread
|
8
8
|
|
9
9
|
class Collection_Cache:
|
10
10
|
def __init__(self, collection_name):
|
@@ -14,12 +14,14 @@ class Collection_Cache:
|
|
14
14
|
self.collection_dir = path.join("./Collections", self.collection_name)
|
15
15
|
self.databases_list = []
|
16
16
|
self.keys_databases = {}
|
17
|
+
|
17
18
|
# Init methods
|
18
19
|
self.create_collection()
|
19
20
|
self.get_all_databases()
|
20
21
|
|
21
22
|
def create_collection(self):
|
22
23
|
makedirs(self.collection_dir, exist_ok=True)
|
24
|
+
|
23
25
|
for core in range(self.cpu_cores):
|
24
26
|
db_path = path.join(self.collection_dir, f"database_{core}.db")
|
25
27
|
self.initialize_databases(db_path)
|
@@ -38,6 +40,7 @@ class Collection_Cache:
|
|
38
40
|
def get_all_databases(self):
|
39
41
|
with scandir(self.collection_dir) as contents:
|
40
42
|
self.databases_list = [path.join(self.collection_dir, content.name) for content in contents]
|
43
|
+
|
41
44
|
with Pool(self.cpu_cores) as pool:
|
42
45
|
self.keys_databases = dict(chain.from_iterable(pool.map(self.get_all_keys, self.databases_list)))
|
43
46
|
|
@@ -62,6 +65,7 @@ class Collection_Cache:
|
|
62
65
|
conn.commit()
|
63
66
|
conn.close()
|
64
67
|
self.add_to_keys_database(key, database_to_insert)
|
68
|
+
|
65
69
|
else:
|
66
70
|
database_to_update = self.keys_databases[key]
|
67
71
|
conn = sqlite3.connect(database_to_update)
|
@@ -73,19 +77,22 @@ class Collection_Cache:
|
|
73
77
|
|
74
78
|
def set_multi_keys(self, keys_and_values: dict[str, any]):
|
75
79
|
"""Experimental. Set multiple keys and values at the same time."""
|
76
|
-
|
77
|
-
|
80
|
+
|
81
|
+
with Thread(self.cpu_cores) as thread:
|
82
|
+
thread.map(lambda kv: self.set_key(kv[0], kv[1]), keys_and_values.items())
|
78
83
|
|
79
84
|
def add_to_keys_database(self, key, database):
|
80
85
|
self.keys_databases[key] = database
|
81
86
|
|
82
87
|
def delete_to_keys_database(self, key):
|
83
88
|
"""Removes the key from the dictionary of stored keys"""
|
89
|
+
|
84
90
|
if key in self.keys_databases:
|
85
91
|
del self.keys_databases[key]
|
86
92
|
|
87
93
|
def get_key(self, key: str):
|
88
94
|
"""Used to obtain the value stored by the key"""
|
95
|
+
|
89
96
|
try:
|
90
97
|
database_to_search = self.keys_databases[key]
|
91
98
|
conn = sqlite3.connect(database_to_search)
|
@@ -95,11 +102,13 @@ class Collection_Cache:
|
|
95
102
|
result = cursor.fetchall()
|
96
103
|
conn.close()
|
97
104
|
return pickle.loads(result[0][0])
|
105
|
+
|
98
106
|
except Exception as error:
|
99
107
|
return error
|
100
108
|
|
101
109
|
def delete_key(self, key: str):
|
102
110
|
"""Used to delete the value stored by the key"""
|
111
|
+
|
103
112
|
try:
|
104
113
|
database_to_delete = self.keys_databases[key]
|
105
114
|
conn = sqlite3.connect(database_to_delete)
|
@@ -109,19 +118,23 @@ class Collection_Cache:
|
|
109
118
|
conn.commit()
|
110
119
|
conn.close()
|
111
120
|
self.delete_to_keys_database(key)
|
121
|
+
|
112
122
|
except KeyError:
|
113
123
|
return f"Key '{key}' not found."
|
124
|
+
|
114
125
|
except Exception as error:
|
115
126
|
return error
|
116
127
|
|
117
128
|
def configure_connection(self, conn):
|
118
|
-
conn.
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
129
|
+
conn.executescript("""
|
130
|
+
PRAGMA auto_vacuum = FULL;
|
131
|
+
PRAGMA journal_mode = WAL;
|
132
|
+
PRAGMA synchronous = NORMAL;
|
133
|
+
PRAGMA wal_autocheckpoint = 1000;
|
134
|
+
PRAGMA cache_size = -2000;
|
135
|
+
PRAGMA temp_store = MEMORY;
|
136
|
+
PRAGMA optimize;
|
137
|
+
""")
|
125
138
|
|
126
139
|
def keys(self):
|
127
140
|
"""Returns all stored keys"""
|
@@ -1,6 +1,6 @@
|
|
1
1
|
[tool.poetry]
|
2
2
|
name = "collections-cache"
|
3
|
-
version = "
|
3
|
+
version = "2025.3.3"
|
4
4
|
description = "collections-cache is a Python package for managing data collections across multiple SQLite databases. It allows efficient storage, retrieval, and updating of key-value pairs, supporting various data types serialized with pickle. The package uses parallel processing for fast access and manipulation of large collections."
|
5
5
|
authors = ["Luiz-Trindade <luiz.gabriel.m.trindade@gmail.com>"]
|
6
6
|
license = "MIT"
|
File without changes
|
File without changes
|
File without changes
|