filetracker 2.2.0__tar.gz → 2.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- filetracker-2.3.3/.github/CODEOWNERS +1 -0
- filetracker-2.3.3/.github/workflows/pytest.yml +39 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/.github/workflows/release.yml +10 -1
- {filetracker-2.2.0 → filetracker-2.3.3}/PKG-INFO +7 -7
- {filetracker-2.2.0 → filetracker-2.3.3}/PROTOCOL.md +1 -1
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/client.py +10 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/data_store.py +2 -1
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/remote_data_store.py +20 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/servers/base.py +0 -42
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/servers/files.py +7 -3
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/servers/run.py +4 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/servers/storage.py +40 -11
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/tests/interaction_test.py +29 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/tests/parallel_test.py +59 -7
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/tests/protocol_test.py +26 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker.egg-info/PKG-INFO +7 -7
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker.egg-info/requires.txt +3 -4
- {filetracker-2.2.0 → filetracker-2.3.3}/pyproject.toml +6 -7
- {filetracker-2.2.0 → filetracker-2.3.3}/tox.ini +2 -1
- filetracker-2.2.0/.github/CODEOWNERS +0 -2
- filetracker-2.2.0/.github/workflows/pytest.yml +0 -40
- {filetracker-2.2.0 → filetracker-2.3.3}/.gitignore +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/.travis.yml +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/LICENSE +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/MIGRATING.md +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/README.md +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/__init__.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/__init__.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/dummy.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/local_data_store.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/local_data_store_test.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/lock_manager.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/shell.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/scripts/__init__.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/scripts/cachecleaner.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/scripts/migrate.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/scripts/migrate_test.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/scripts/progress_bar.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/scripts/recover.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/scripts/recover_test.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/servers/__init__.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/servers/migration.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/servers/storage_test.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/tests/__init__.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/tests/migration_test.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/utils.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker.egg-info/SOURCES.txt +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker.egg-info/dependency_links.txt +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker.egg-info/entry_points.txt +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/filetracker.egg-info/top_level.txt +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/rst/.gitignore +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/rst/Makefile +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/rst/README +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/rst/source/_static/README +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/rst/source/conf.py +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/rst/source/index.rst +0 -0
- {filetracker-2.2.0 → filetracker-2.3.3}/setup.cfg +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
* @twalen
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
name: Unit tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- master
|
|
7
|
+
pull_request:
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
build:
|
|
12
|
+
strategy:
|
|
13
|
+
fail-fast: false
|
|
14
|
+
matrix:
|
|
15
|
+
include:
|
|
16
|
+
- python-version: "3.13"
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v3
|
|
21
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
22
|
+
uses: actions/setup-python@v4
|
|
23
|
+
with:
|
|
24
|
+
python-version: ${{ matrix.python-version }}
|
|
25
|
+
|
|
26
|
+
- name: Install apt dependencies
|
|
27
|
+
run: |
|
|
28
|
+
sudo apt-get update
|
|
29
|
+
sudo apt-get install -y libdb-dev
|
|
30
|
+
|
|
31
|
+
- name: Install pip dependencies
|
|
32
|
+
run: |
|
|
33
|
+
python -m pip install --upgrade pip wheel setuptools
|
|
34
|
+
pip install -e .[server,tests]
|
|
35
|
+
pip install pytest
|
|
36
|
+
|
|
37
|
+
- name: Run tests
|
|
38
|
+
run: |
|
|
39
|
+
pytest -vv filetracker/tests
|
|
@@ -15,7 +15,16 @@ jobs:
|
|
|
15
15
|
- name: Set up Python
|
|
16
16
|
uses: actions/setup-python@v4
|
|
17
17
|
with:
|
|
18
|
-
python-version: 3.
|
|
18
|
+
python-version: 3.13
|
|
19
|
+
- name: Check version matches tag
|
|
20
|
+
if: startsWith(github.ref, 'refs/tags')
|
|
21
|
+
run: |
|
|
22
|
+
tag_version="${GITHUB_REF_NAME#v}"
|
|
23
|
+
project_version=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")
|
|
24
|
+
if [ "$tag_version" != "$project_version" ]; then
|
|
25
|
+
echo "::error::Tag $GITHUB_REF_NAME does not match version $project_version in pyproject.toml"
|
|
26
|
+
exit 1
|
|
27
|
+
fi
|
|
19
28
|
- name: Install build
|
|
20
29
|
run: |
|
|
21
30
|
python3 -m pip install build
|
|
@@ -1,26 +1,26 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: filetracker
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.3.3
|
|
4
4
|
Summary: Filetracker caching file storage
|
|
5
5
|
Author-email: SIO2 Project Team <sio2@sio2project.mimuw.edu.pl>
|
|
6
6
|
License: GPL
|
|
7
7
|
Project-URL: homepage, https://github.com/sio2project/filetracker
|
|
8
8
|
Classifier: Framework :: Django
|
|
9
9
|
Classifier: Programming Language :: Python :: 3
|
|
10
|
-
Requires-Python: >=3.
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
11
|
Description-Content-Type: text/markdown
|
|
12
12
|
License-File: LICENSE
|
|
13
13
|
Requires-Dist: bsddb3==6.2.7
|
|
14
|
-
Requires-Dist:
|
|
15
|
-
Requires-Dist: gunicorn==19.9.0
|
|
14
|
+
Requires-Dist: gunicorn==23.0.0
|
|
16
15
|
Requires-Dist: progressbar2
|
|
17
16
|
Requires-Dist: requests
|
|
18
17
|
Requires-Dist: six
|
|
19
18
|
Provides-Extra: server
|
|
20
|
-
Requires-Dist: gevent==
|
|
21
|
-
Requires-Dist: greenlet==2.0.2; extra == "server"
|
|
19
|
+
Requires-Dist: gevent==25.9.1; extra == "server"
|
|
22
20
|
Provides-Extra: tests
|
|
23
21
|
Requires-Dist: pytest; extra == "tests"
|
|
22
|
+
Requires-Dist: psutil; extra == "tests"
|
|
23
|
+
Dynamic: license-file
|
|
24
24
|
|
|
25
25
|
# filetracker
|
|
26
26
|
|
|
@@ -97,7 +97,7 @@ slashes.
|
|
|
97
97
|
|
|
98
98
|
The response is plain-text, with one line for every file path.
|
|
99
99
|
|
|
100
|
-
Version cutoff
|
|
100
|
+
Version cutoff may be specified in RFC 2822 format through
|
|
101
101
|
`?last_modified=` query parameter. Only files with modification time
|
|
102
102
|
older than this version will be listed.
|
|
103
103
|
|
|
@@ -338,3 +338,13 @@ class Client(object):
|
|
|
338
338
|
if self.local_store:
|
|
339
339
|
result.extend(self.local_store.list_files())
|
|
340
340
|
return result
|
|
341
|
+
|
|
342
|
+
def list_remote_files(self, version_cutoff_timestamp=None, subpath="", absolute_paths=False):
|
|
343
|
+
"""Returns list of all stored remote files under `subpath`
|
|
344
|
+
not newer than `version_cutoff_timestamp`.
|
|
345
|
+
|
|
346
|
+
Each element of this list is just a string with the full path.
|
|
347
|
+
"""
|
|
348
|
+
if not self.remote_store:
|
|
349
|
+
return []
|
|
350
|
+
return self.remote_store.list_files(version_cutoff_timestamp, subpath, absolute_paths)
|
|
@@ -43,7 +43,8 @@ class DataStore(object):
|
|
|
43
43
|
Works like :meth:`add_stream`, but ``filename`` is the name of
|
|
44
44
|
an existing file in the filesystem.
|
|
45
45
|
"""
|
|
46
|
-
|
|
46
|
+
with open(filename, 'rb') as f:
|
|
47
|
+
return self.add_stream(name, f)
|
|
47
48
|
|
|
48
49
|
def exists(self, name):
|
|
49
50
|
"""Returns ``True`` if the file exists, ``False`` otherwise.
|
|
@@ -40,6 +40,9 @@ SERVER_ACCEPTS_SHA256_DIGEST = 3
|
|
|
40
40
|
# The server supports deleting files
|
|
41
41
|
SERVER_ACCEPTS_DELETE = 4
|
|
42
42
|
|
|
43
|
+
# The server supports listing files
|
|
44
|
+
SERVER_ACCEPTS_LIST = 5
|
|
45
|
+
|
|
43
46
|
_PROTOCOL_CAPABILITIES = {
|
|
44
47
|
1: [
|
|
45
48
|
SERVER_REQUIRES_VERSION_HEADER,
|
|
@@ -48,6 +51,7 @@ _PROTOCOL_CAPABILITIES = {
|
|
|
48
51
|
SERVER_ACCEPTS_GZIP,
|
|
49
52
|
SERVER_ACCEPTS_SHA256_DIGEST,
|
|
50
53
|
SERVER_ACCEPTS_DELETE,
|
|
54
|
+
SERVER_ACCEPTS_LIST,
|
|
51
55
|
],
|
|
52
56
|
}
|
|
53
57
|
|
|
@@ -217,6 +221,22 @@ class RemoteDataStore(DataStore):
|
|
|
217
221
|
response = requests.delete(url, headers=headers)
|
|
218
222
|
response.raise_for_status()
|
|
219
223
|
|
|
224
|
+
@_verbose_http_errors
|
|
225
|
+
def list_files(self, version, subpath, absolute_paths):
|
|
226
|
+
if not self._has_capability(SERVER_ACCEPTS_LIST):
|
|
227
|
+
return
|
|
228
|
+
url = self.base_url + '/list' + pathname2url(subpath)
|
|
229
|
+
url, headers = self._add_version_to_request(url, {}, version)
|
|
230
|
+
response = requests.get(url, headers=headers)
|
|
231
|
+
response.raise_for_status()
|
|
232
|
+
result = response.content.decode('utf-8').split('\n')
|
|
233
|
+
assert len(result.pop()) == 0
|
|
234
|
+
if absolute_paths and subpath and subpath != "/":
|
|
235
|
+
prefix = subpath.rstrip("/").lstrip("/") + "/"
|
|
236
|
+
return [prefix + path for path in result]
|
|
237
|
+
else:
|
|
238
|
+
return result
|
|
239
|
+
|
|
220
240
|
def _add_version_to_request(self, url, headers, version):
|
|
221
241
|
"""Adds version to either url or headers, depending on protocol."""
|
|
222
242
|
if self._has_capability(SERVER_REQUIRES_VERSION_HEADER):
|
|
@@ -86,48 +86,6 @@ def get_endpoint_and_path(environ):
|
|
|
86
86
|
return components[0], '/'.join(components[1:])
|
|
87
87
|
|
|
88
88
|
|
|
89
|
-
# Deprecated, run.py should be used in all cases.
|
|
90
|
-
def start_cgi(server):
|
|
91
|
-
from flup.server.cgi import WSGIServer
|
|
92
|
-
|
|
93
|
-
WSGIServer(server).run()
|
|
94
|
-
sys.exit(0)
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
# Deprecated, run.py should be used in all cases.
|
|
98
|
-
def start_fcgi(server):
|
|
99
|
-
from flup.server.fcgi import WSGIServer
|
|
100
|
-
|
|
101
|
-
WSGIServer(server).run()
|
|
102
|
-
sys.exit(0)
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
# Deprecated, run.py should be used in all cases.
|
|
106
|
-
def start_standalone(server, port=8000):
|
|
107
|
-
from wsgiref.simple_server import make_server
|
|
108
|
-
|
|
109
|
-
httpd = make_server('', port, server)
|
|
110
|
-
print("Serving on port %d..." % port)
|
|
111
|
-
httpd.serve_forever()
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
def main(server):
|
|
115
|
-
"""A convenience ``main`` method for running WSGI-compatible HTTP
|
|
116
|
-
application as CGI, FCGI or standalone (with auto-detection)."""
|
|
117
|
-
|
|
118
|
-
if 'REQUEST_METHOD' in os.environ:
|
|
119
|
-
start_cgi(server)
|
|
120
|
-
|
|
121
|
-
stdin_sock = socket.fromfd(0, socket.AF_UNIX, socket.SOCK_STREAM)
|
|
122
|
-
try:
|
|
123
|
-
stdin_sock.getpeername()
|
|
124
|
-
except socket.error as e:
|
|
125
|
-
if e.errno == errno.ENOTCONN:
|
|
126
|
-
start_fcgi(server)
|
|
127
|
-
|
|
128
|
-
start_standalone(server)
|
|
129
|
-
|
|
130
|
-
|
|
131
89
|
def _rindex(l, value):
|
|
132
90
|
"""Same as str.rindex, but for lists."""
|
|
133
91
|
return len(l) - l[::-1].index(value) - 1
|
|
@@ -156,9 +156,13 @@ class FiletrackerServer(base.Server):
|
|
|
156
156
|
query_params = self.parse_query_params(environ)
|
|
157
157
|
|
|
158
158
|
last_modified = query_params.get('last_modified', (None,))[0]
|
|
159
|
-
if
|
|
160
|
-
last_modified =
|
|
161
|
-
|
|
159
|
+
if last_modified:
|
|
160
|
+
last_modified = email.utils.parsedate_tz(last_modified)
|
|
161
|
+
last_modified = email.utils.mktime_tz(last_modified)
|
|
162
|
+
else:
|
|
163
|
+
last_modified = time.time()
|
|
164
|
+
last_modified = int(last_modified)
|
|
165
|
+
|
|
162
166
|
logger.debug('Handling GET /list/%s (@%d)', path, last_modified)
|
|
163
167
|
|
|
164
168
|
root_dir = os.path.join(self.dir, path)
|
|
@@ -174,6 +174,10 @@ def main(args=None):
|
|
|
174
174
|
log_config['loggers']['gunicorn.error']['level'] = options.log_level
|
|
175
175
|
log_config['loggers']['gunicorn.access']['level'] = options.log_level
|
|
176
176
|
log_config['loggers']['']['level'] = options.log_level
|
|
177
|
+
log_config['root'] = {
|
|
178
|
+
'level': "WARNING",
|
|
179
|
+
'handlers': [],
|
|
180
|
+
}
|
|
177
181
|
|
|
178
182
|
logging.config.dictConfig(log_config)
|
|
179
183
|
|
|
@@ -47,6 +47,7 @@ from filetracker.utils import file_digest
|
|
|
47
47
|
|
|
48
48
|
_LOCK_RETRIES = 20
|
|
49
49
|
_LOCK_SLEEP_TIME_S = 1
|
|
50
|
+
_DB_DEADLOCK_RETRIES = 50
|
|
50
51
|
|
|
51
52
|
|
|
52
53
|
logger = logging.getLogger(__name__)
|
|
@@ -81,6 +82,8 @@ class FileStorage(object):
|
|
|
81
82
|
|
|
82
83
|
# https://docs.oracle.com/cd/E17076_05/html/programmer_reference/transapp_env_open.html
|
|
83
84
|
self.db_env = bsddb3.db.DBEnv()
|
|
85
|
+
# Enable running the deadlock detector when lock conflicts are automatically detected.
|
|
86
|
+
self.db_env.set_lk_detect(bsddb3.db.DB_LOCK_DEFAULT)
|
|
84
87
|
try:
|
|
85
88
|
self.db_env.open(
|
|
86
89
|
self.db_dir,
|
|
@@ -182,9 +185,8 @@ class FileStorage(object):
|
|
|
182
185
|
logger.debug('Acquired lock for blob %s.', digest)
|
|
183
186
|
digest_bytes = digest.encode()
|
|
184
187
|
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
link_count = int(self.db.get(digest_bytes, 0, txn=txn))
|
|
188
|
+
def transaction_contents(txn):
|
|
189
|
+
link_count = int(self.db.get(digest_bytes, 0, txn=txn, flags=bsddb3.db.DB_RMW))
|
|
188
190
|
new_count = str(link_count + 1).encode()
|
|
189
191
|
self.db.put(digest_bytes, new_count, txn=txn)
|
|
190
192
|
|
|
@@ -194,9 +196,11 @@ class FileStorage(object):
|
|
|
194
196
|
str(logical_size).encode(),
|
|
195
197
|
txn=txn,
|
|
196
198
|
)
|
|
197
|
-
|
|
199
|
+
return link_count
|
|
198
200
|
|
|
199
|
-
|
|
201
|
+
link_count = self._call_in_transaction_with_retries(
|
|
202
|
+
transaction_contents, "adding link"
|
|
203
|
+
)
|
|
200
204
|
|
|
201
205
|
# Create a new blob if this isn't a duplicate.
|
|
202
206
|
if link_count == 0:
|
|
@@ -268,12 +272,11 @@ class FileStorage(object):
|
|
|
268
272
|
|
|
269
273
|
with _exclusive_lock(self._lock_path('blobs', digest)):
|
|
270
274
|
logger.debug('Acquired lock for blob %s.', digest)
|
|
271
|
-
should_delete_blob = False
|
|
272
275
|
|
|
273
|
-
|
|
274
|
-
|
|
276
|
+
def transaction_contents(txn):
|
|
277
|
+
should_delete_blob = False
|
|
275
278
|
digest_bytes = digest.encode()
|
|
276
|
-
link_count = self.db.get(digest_bytes, txn=txn)
|
|
279
|
+
link_count = self.db.get(digest_bytes, txn=txn, flags=bsddb3.db.DB_RMW)
|
|
277
280
|
if link_count is None:
|
|
278
281
|
raise RuntimeError("File exists but has no key in db")
|
|
279
282
|
|
|
@@ -288,9 +291,11 @@ class FileStorage(object):
|
|
|
288
291
|
else:
|
|
289
292
|
new_count = str(link_count - 1).encode()
|
|
290
293
|
self.db.put(digest_bytes, new_count, txn=txn)
|
|
291
|
-
|
|
294
|
+
return should_delete_blob
|
|
292
295
|
|
|
293
|
-
|
|
296
|
+
should_delete_blob = self._call_in_transaction_with_retries(
|
|
297
|
+
transaction_contents, "deleting link"
|
|
298
|
+
)
|
|
294
299
|
|
|
295
300
|
os.unlink(link_path)
|
|
296
301
|
logger.debug('Deleted link %s.', name)
|
|
@@ -339,6 +344,30 @@ class FileStorage(object):
|
|
|
339
344
|
else:
|
|
340
345
|
txn.commit()
|
|
341
346
|
|
|
347
|
+
def _call_in_transaction_with_retries(self, func, description):
|
|
348
|
+
retries = 0
|
|
349
|
+
result = None
|
|
350
|
+
while True:
|
|
351
|
+
try:
|
|
352
|
+
with self._db_transaction() as txn:
|
|
353
|
+
logger.debug('Started DB transaction ({}).'.format(description))
|
|
354
|
+
result = func(txn)
|
|
355
|
+
logger.debug('Commiting DB transaction ({}).'.format(description))
|
|
356
|
+
except bsddb3.db.DBLockDeadlockError:
|
|
357
|
+
retries += 1
|
|
358
|
+
if retries > _DB_DEADLOCK_RETRIES:
|
|
359
|
+
logger.error('BSDDB deadlock detected, retry limit exceeded ({}).'.format(description))
|
|
360
|
+
raise
|
|
361
|
+
logger.warning(
|
|
362
|
+
'BSDDB deadlock detected in transaction ({}), retry no {}.'.format(
|
|
363
|
+
description, retries,
|
|
364
|
+
)
|
|
365
|
+
)
|
|
366
|
+
continue
|
|
367
|
+
break
|
|
368
|
+
logger.debug('Committed DB transaction ({}).'.format(description))
|
|
369
|
+
return result
|
|
370
|
+
|
|
342
371
|
def _digest_for_link(self, name):
|
|
343
372
|
link = self._link_path(name)
|
|
344
373
|
blob_path = os.readlink(link)
|
|
@@ -148,6 +148,35 @@ class InteractionTest(unittest.TestCase):
|
|
|
148
148
|
|
|
149
149
|
self.assertNotEqual(version_a, version_b)
|
|
150
150
|
|
|
151
|
+
def test_list_remote_files(self):
|
|
152
|
+
src_file = os.path.join(self.temp_dir, 'older.txt')
|
|
153
|
+
with open(src_file, 'wb') as sf:
|
|
154
|
+
sf.write(b'these tests are so bad')
|
|
155
|
+
|
|
156
|
+
self.client.put_file('/A@1', src_file)
|
|
157
|
+
self.client.put_file('/B@5', src_file)
|
|
158
|
+
self.client.put_file('/C/D@10', src_file)
|
|
159
|
+
|
|
160
|
+
def check(expected, *args):
|
|
161
|
+
result_raw = self.client.list_remote_files(*args)
|
|
162
|
+
# Filter out files from other tests.
|
|
163
|
+
result = sorted(filter(lambda x: not x.endswith(".txt"), result_raw))
|
|
164
|
+
self.assertEqual(result, sorted(expected))
|
|
165
|
+
|
|
166
|
+
check(["A", "B", "C/D"])
|
|
167
|
+
check(["A"], 4)
|
|
168
|
+
check(["A", "B"], 5)
|
|
169
|
+
check(["A", "B"], 9)
|
|
170
|
+
check(["A", "B", "C/D"], 10)
|
|
171
|
+
with self.assertRaises(FiletrackerError):
|
|
172
|
+
check(["D"], 10, "C")
|
|
173
|
+
check(["D"], 10, "/C")
|
|
174
|
+
check(["D"], 10, "/C/")
|
|
175
|
+
# absolute_paths=True
|
|
176
|
+
check(["C/D"], 10, "/C", True)
|
|
177
|
+
check(["C/D"], 10, "/C/", True)
|
|
178
|
+
check([], 9, "/C")
|
|
179
|
+
|
|
151
180
|
def test_put_older_should_fail(self):
|
|
152
181
|
"""This test assumes file version is stored in mtime."""
|
|
153
182
|
src_file = os.path.join(self.temp_dir, 'older.txt')
|
|
@@ -4,8 +4,9 @@ from __future__ import absolute_import
|
|
|
4
4
|
from __future__ import division
|
|
5
5
|
from __future__ import print_function
|
|
6
6
|
|
|
7
|
-
from multiprocessing import Process
|
|
7
|
+
from multiprocessing import Barrier, Process
|
|
8
8
|
import os
|
|
9
|
+
import psutil
|
|
9
10
|
import shutil
|
|
10
11
|
import tempfile
|
|
11
12
|
import time
|
|
@@ -25,6 +26,17 @@ _CLIENT_WAIT_S = 0.1
|
|
|
25
26
|
_FILE_SIZE = 6 * 1024 * 1024
|
|
26
27
|
_PARALLEL_CLIENTS = 5
|
|
27
28
|
_TEST_PORT_NUMBER = 45745
|
|
29
|
+
_COPIES_TO_UPLOAD = 500
|
|
30
|
+
_SUBPROCESS_TIMEOUT_S = 20
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def kill_process_tree(pid):
|
|
34
|
+
parent = psutil.Process(pid)
|
|
35
|
+
for p in parent.children(recursive=True) + [parent]:
|
|
36
|
+
try:
|
|
37
|
+
p.kill()
|
|
38
|
+
except psutil.NoSuchProcess:
|
|
39
|
+
pass
|
|
28
40
|
|
|
29
41
|
|
|
30
42
|
class ParallelTest(unittest.TestCase):
|
|
@@ -52,19 +64,23 @@ class ParallelTest(unittest.TestCase):
|
|
|
52
64
|
|
|
53
65
|
@classmethod
|
|
54
66
|
def tearDownClass(cls):
|
|
55
|
-
cls.server_process.
|
|
67
|
+
kill_process_tree(cls.server_process.pid)
|
|
68
|
+
cls.server_process.join()
|
|
56
69
|
shutil.rmtree(cls.server_dir)
|
|
57
70
|
shutil.rmtree(cls.temp_dir)
|
|
58
71
|
|
|
59
72
|
def setUp(self):
|
|
60
73
|
# Shortcuts for convenience
|
|
61
|
-
|
|
62
|
-
self.
|
|
63
|
-
self.
|
|
74
|
+
cls = self.__class__
|
|
75
|
+
self.server_dir = cls.server_dir
|
|
76
|
+
self.temp_dir = cls.temp_dir
|
|
77
|
+
self.clients = cls.clients
|
|
64
78
|
|
|
65
79
|
# For non-parallel requests
|
|
66
|
-
self.client =
|
|
80
|
+
self.client = cls.clients[0]
|
|
81
|
+
|
|
67
82
|
|
|
83
|
+
class ParallelTestSameFile(ParallelTest):
|
|
68
84
|
def test_only_last_parallel_upload_of_same_file_should_succeed(self):
|
|
69
85
|
processes = []
|
|
70
86
|
|
|
@@ -98,7 +114,43 @@ class ParallelTest(unittest.TestCase):
|
|
|
98
114
|
self.assertEqual(f.read(), lf.read())
|
|
99
115
|
|
|
100
116
|
|
|
117
|
+
class ParallelTestDeadlocks(ParallelTest):
|
|
118
|
+
def test_bsddb_deadlocks(self):
|
|
119
|
+
processes = []
|
|
120
|
+
|
|
121
|
+
# Initialize different files for every client.
|
|
122
|
+
for i in range(len(self.clients)):
|
|
123
|
+
temp_file = os.path.join(self.temp_dir, 'foo{}.txt'.format(i))
|
|
124
|
+
text = str(i).encode()
|
|
125
|
+
with open(temp_file, 'wb') as tf:
|
|
126
|
+
tf.write(text)
|
|
127
|
+
|
|
128
|
+
# The deadlocks are visible even without this barrier.
|
|
129
|
+
barrier = Barrier(len(self.clients))
|
|
130
|
+
|
|
131
|
+
def job(id, barrier):
|
|
132
|
+
temp_file = os.path.join(self.temp_dir, 'foo{}.txt'.format(id))
|
|
133
|
+
for i in range(0, _COPIES_TO_UPLOAD):
|
|
134
|
+
ft_name = '/foo{}.{}.txt'.format(id, i)
|
|
135
|
+
barrier.wait()
|
|
136
|
+
client.put_file(ft_name, temp_file, compress_hint=False)
|
|
137
|
+
|
|
138
|
+
for i, client in enumerate(self.clients):
|
|
139
|
+
process = Process(target=lambda: job(i, barrier))
|
|
140
|
+
process.start()
|
|
141
|
+
processes.append(process)
|
|
142
|
+
|
|
143
|
+
for process in processes:
|
|
144
|
+
process.join(timeout=_SUBPROCESS_TIMEOUT_S)
|
|
145
|
+
self.assertFalse(process.is_alive())
|
|
146
|
+
self.assertEqual(process.exitcode, 0)
|
|
147
|
+
process.join()
|
|
148
|
+
|
|
149
|
+
# Put one final file to check for e.g. corruption.
|
|
150
|
+
client.put_file('/foo_last', temp_file, compress_hint=False)
|
|
151
|
+
|
|
152
|
+
|
|
101
153
|
def _start_server(server_dir):
|
|
102
154
|
server_main(
|
|
103
|
-
['-p', str(_TEST_PORT_NUMBER), '-d', server_dir, '-D', '--workers', '
|
|
155
|
+
['-p', str(_TEST_PORT_NUMBER), '-d', server_dir, '-D', '--workers', '3']
|
|
104
156
|
)
|
|
@@ -5,6 +5,7 @@ from __future__ import division
|
|
|
5
5
|
from __future__ import print_function
|
|
6
6
|
|
|
7
7
|
from multiprocessing import Process
|
|
8
|
+
import email.utils
|
|
8
9
|
import os
|
|
9
10
|
import shutil
|
|
10
11
|
import tempfile
|
|
@@ -67,6 +68,31 @@ class ProtocolTest(unittest.TestCase):
|
|
|
67
68
|
self.assertEqual(lines.count('list_a.txt'), 1)
|
|
68
69
|
self.assertEqual(lines.count('list_b.txt'), 1)
|
|
69
70
|
|
|
71
|
+
def test_list_files_version_cutoff_should_work(self):
|
|
72
|
+
src_file = os.path.join(self.temp_dir, 'list.txt')
|
|
73
|
+
with open(src_file, 'wb') as sf:
|
|
74
|
+
sf.write(b'hello list')
|
|
75
|
+
|
|
76
|
+
self.client.put_file('/A@1', src_file)
|
|
77
|
+
self.client.put_file('/B@100', src_file)
|
|
78
|
+
self.client.put_file('/C@200', src_file)
|
|
79
|
+
|
|
80
|
+
def check(timestamp, expected):
|
|
81
|
+
date = email.utils.formatdate(timestamp)
|
|
82
|
+
params = {'last_modified': date}
|
|
83
|
+
res = requests.get('http://127.0.0.1:{}/list/'.format(_TEST_PORT_NUMBER), params=params)
|
|
84
|
+
self.assertEqual(res.status_code, 200)
|
|
85
|
+
# Filter out files from other tests.
|
|
86
|
+
lines = [l for l in res.text.split('\n') if l]
|
|
87
|
+
self.assertCountEqual(lines, expected)
|
|
88
|
+
|
|
89
|
+
check(0, [])
|
|
90
|
+
check(1, ['A'])
|
|
91
|
+
check(99, ['A'])
|
|
92
|
+
check(100, ['A', 'B'])
|
|
93
|
+
check(199, ['A', 'B'])
|
|
94
|
+
check(200, ['A', 'B', 'C'])
|
|
95
|
+
|
|
70
96
|
def test_list_files_in_subdirectory_should_work(self):
|
|
71
97
|
src_file = os.path.join(self.temp_dir, 'list_sub.txt')
|
|
72
98
|
with open(src_file, 'wb') as sf:
|
|
@@ -1,26 +1,26 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: filetracker
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.3.3
|
|
4
4
|
Summary: Filetracker caching file storage
|
|
5
5
|
Author-email: SIO2 Project Team <sio2@sio2project.mimuw.edu.pl>
|
|
6
6
|
License: GPL
|
|
7
7
|
Project-URL: homepage, https://github.com/sio2project/filetracker
|
|
8
8
|
Classifier: Framework :: Django
|
|
9
9
|
Classifier: Programming Language :: Python :: 3
|
|
10
|
-
Requires-Python: >=3.
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
11
|
Description-Content-Type: text/markdown
|
|
12
12
|
License-File: LICENSE
|
|
13
13
|
Requires-Dist: bsddb3==6.2.7
|
|
14
|
-
Requires-Dist:
|
|
15
|
-
Requires-Dist: gunicorn==19.9.0
|
|
14
|
+
Requires-Dist: gunicorn==23.0.0
|
|
16
15
|
Requires-Dist: progressbar2
|
|
17
16
|
Requires-Dist: requests
|
|
18
17
|
Requires-Dist: six
|
|
19
18
|
Provides-Extra: server
|
|
20
|
-
Requires-Dist: gevent==
|
|
21
|
-
Requires-Dist: greenlet==2.0.2; extra == "server"
|
|
19
|
+
Requires-Dist: gevent==25.9.1; extra == "server"
|
|
22
20
|
Provides-Extra: tests
|
|
23
21
|
Requires-Dist: pytest; extra == "tests"
|
|
22
|
+
Requires-Dist: psutil; extra == "tests"
|
|
23
|
+
Dynamic: license-file
|
|
24
24
|
|
|
25
25
|
# filetracker
|
|
26
26
|
|
|
@@ -15,7 +15,7 @@ authors = [
|
|
|
15
15
|
]
|
|
16
16
|
description = "Filetracker caching file storage"
|
|
17
17
|
readme = "README.md"
|
|
18
|
-
requires-python = ">=3.
|
|
18
|
+
requires-python = ">=3.9"
|
|
19
19
|
license = {text = "GPL"}
|
|
20
20
|
classifiers = [
|
|
21
21
|
"Framework :: Django",
|
|
@@ -23,24 +23,23 @@ classifiers = [
|
|
|
23
23
|
]
|
|
24
24
|
dependencies = [
|
|
25
25
|
"bsddb3==6.2.7",
|
|
26
|
-
"
|
|
27
|
-
"gunicorn==19.9.0",
|
|
26
|
+
"gunicorn==23.0.0",
|
|
28
27
|
"progressbar2",
|
|
29
28
|
"requests",
|
|
30
29
|
"six",
|
|
31
30
|
]
|
|
32
|
-
version = "2.
|
|
31
|
+
version = "2.3.3"
|
|
33
32
|
|
|
34
33
|
[project.urls]
|
|
35
34
|
"homepage" = "https://github.com/sio2project/filetracker"
|
|
36
35
|
|
|
37
36
|
[project.optional-dependencies]
|
|
38
37
|
server = [
|
|
39
|
-
"gevent==
|
|
40
|
-
"greenlet==2.0.2",
|
|
38
|
+
"gevent==25.9.1"
|
|
41
39
|
]
|
|
42
40
|
tests = [
|
|
43
41
|
"pytest",
|
|
42
|
+
"psutil",
|
|
44
43
|
]
|
|
45
44
|
|
|
46
45
|
[project.scripts]
|
|
@@ -48,4 +47,4 @@ filetracker = "filetracker.client.shell:main"
|
|
|
48
47
|
filetracker-server = "filetracker.servers.run:main"
|
|
49
48
|
filetracker-cache-cleaner = "filetracker.scripts.cachecleaner:main"
|
|
50
49
|
filetracker-migrate = "filetracker.scripts.migrate:main"
|
|
51
|
-
filetracker-recover = "filetracker.scripts.recover:main"
|
|
50
|
+
filetracker-recover = "filetracker.scripts.recover:main"
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
name: Unit tests
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
branches:
|
|
6
|
-
- master
|
|
7
|
-
pull_request:
|
|
8
|
-
workflow_dispatch:
|
|
9
|
-
|
|
10
|
-
jobs:
|
|
11
|
-
build:
|
|
12
|
-
strategy:
|
|
13
|
-
fail-fast: false
|
|
14
|
-
matrix:
|
|
15
|
-
include:
|
|
16
|
-
- python-version: "3.8"
|
|
17
|
-
- python-version: "3.9"
|
|
18
|
-
runs-on: ubuntu-latest
|
|
19
|
-
|
|
20
|
-
steps:
|
|
21
|
-
- uses: actions/checkout@v3
|
|
22
|
-
- name: Set up Python ${{ matrix.python-version }}
|
|
23
|
-
uses: actions/setup-python@v4
|
|
24
|
-
with:
|
|
25
|
-
python-version: ${{ matrix.python-version }}
|
|
26
|
-
|
|
27
|
-
- name: Install apt dependencies
|
|
28
|
-
run: |
|
|
29
|
-
sudo apt-get update
|
|
30
|
-
sudo apt-get install -y libdb-dev
|
|
31
|
-
|
|
32
|
-
- name: Install pip dependencies
|
|
33
|
-
run: |
|
|
34
|
-
python -m pip install --upgrade pip wheel setuptools
|
|
35
|
-
pip install -e .[server]
|
|
36
|
-
pip install pytest
|
|
37
|
-
|
|
38
|
-
- name: Run tests
|
|
39
|
-
run: |
|
|
40
|
-
pytest -vv filetracker/tests
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|