filetracker 2.2.0__tar.gz → 2.3.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. filetracker-2.3.3/.github/CODEOWNERS +1 -0
  2. filetracker-2.3.3/.github/workflows/pytest.yml +39 -0
  3. {filetracker-2.2.0 → filetracker-2.3.3}/.github/workflows/release.yml +10 -1
  4. {filetracker-2.2.0 → filetracker-2.3.3}/PKG-INFO +7 -7
  5. {filetracker-2.2.0 → filetracker-2.3.3}/PROTOCOL.md +1 -1
  6. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/client.py +10 -0
  7. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/data_store.py +2 -1
  8. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/remote_data_store.py +20 -0
  9. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/servers/base.py +0 -42
  10. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/servers/files.py +7 -3
  11. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/servers/run.py +4 -0
  12. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/servers/storage.py +40 -11
  13. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/tests/interaction_test.py +29 -0
  14. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/tests/parallel_test.py +59 -7
  15. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/tests/protocol_test.py +26 -0
  16. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker.egg-info/PKG-INFO +7 -7
  17. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker.egg-info/requires.txt +3 -4
  18. {filetracker-2.2.0 → filetracker-2.3.3}/pyproject.toml +6 -7
  19. {filetracker-2.2.0 → filetracker-2.3.3}/tox.ini +2 -1
  20. filetracker-2.2.0/.github/CODEOWNERS +0 -2
  21. filetracker-2.2.0/.github/workflows/pytest.yml +0 -40
  22. {filetracker-2.2.0 → filetracker-2.3.3}/.gitignore +0 -0
  23. {filetracker-2.2.0 → filetracker-2.3.3}/.travis.yml +0 -0
  24. {filetracker-2.2.0 → filetracker-2.3.3}/LICENSE +0 -0
  25. {filetracker-2.2.0 → filetracker-2.3.3}/MIGRATING.md +0 -0
  26. {filetracker-2.2.0 → filetracker-2.3.3}/README.md +0 -0
  27. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/__init__.py +0 -0
  28. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/__init__.py +0 -0
  29. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/dummy.py +0 -0
  30. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/local_data_store.py +0 -0
  31. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/local_data_store_test.py +0 -0
  32. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/lock_manager.py +0 -0
  33. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/client/shell.py +0 -0
  34. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/scripts/__init__.py +0 -0
  35. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/scripts/cachecleaner.py +0 -0
  36. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/scripts/migrate.py +0 -0
  37. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/scripts/migrate_test.py +0 -0
  38. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/scripts/progress_bar.py +0 -0
  39. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/scripts/recover.py +0 -0
  40. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/scripts/recover_test.py +0 -0
  41. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/servers/__init__.py +0 -0
  42. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/servers/migration.py +0 -0
  43. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/servers/storage_test.py +0 -0
  44. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/tests/__init__.py +0 -0
  45. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/tests/migration_test.py +0 -0
  46. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker/utils.py +0 -0
  47. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker.egg-info/SOURCES.txt +0 -0
  48. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker.egg-info/dependency_links.txt +0 -0
  49. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker.egg-info/entry_points.txt +0 -0
  50. {filetracker-2.2.0 → filetracker-2.3.3}/filetracker.egg-info/top_level.txt +0 -0
  51. {filetracker-2.2.0 → filetracker-2.3.3}/rst/.gitignore +0 -0
  52. {filetracker-2.2.0 → filetracker-2.3.3}/rst/Makefile +0 -0
  53. {filetracker-2.2.0 → filetracker-2.3.3}/rst/README +0 -0
  54. {filetracker-2.2.0 → filetracker-2.3.3}/rst/source/_static/README +0 -0
  55. {filetracker-2.2.0 → filetracker-2.3.3}/rst/source/conf.py +0 -0
  56. {filetracker-2.2.0 → filetracker-2.3.3}/rst/source/index.rst +0 -0
  57. {filetracker-2.2.0 → filetracker-2.3.3}/setup.cfg +0 -0
@@ -0,0 +1 @@
1
+ * @twalen
@@ -0,0 +1,39 @@
1
+ name: Unit tests
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - master
7
+ pull_request:
8
+ workflow_dispatch:
9
+
10
+ jobs:
11
+ build:
12
+ strategy:
13
+ fail-fast: false
14
+ matrix:
15
+ include:
16
+ - python-version: "3.13"
17
+ runs-on: ubuntu-latest
18
+
19
+ steps:
20
+ - uses: actions/checkout@v3
21
+ - name: Set up Python ${{ matrix.python-version }}
22
+ uses: actions/setup-python@v4
23
+ with:
24
+ python-version: ${{ matrix.python-version }}
25
+
26
+ - name: Install apt dependencies
27
+ run: |
28
+ sudo apt-get update
29
+ sudo apt-get install -y libdb-dev
30
+
31
+ - name: Install pip dependencies
32
+ run: |
33
+ python -m pip install --upgrade pip wheel setuptools
34
+ pip install -e .[server,tests]
35
+ pip install pytest
36
+
37
+ - name: Run tests
38
+ run: |
39
+ pytest -vv filetracker/tests
@@ -15,7 +15,16 @@ jobs:
15
15
  - name: Set up Python
16
16
  uses: actions/setup-python@v4
17
17
  with:
18
- python-version: 3.8
18
+ python-version: 3.13
19
+ - name: Check version matches tag
20
+ if: startsWith(github.ref, 'refs/tags')
21
+ run: |
22
+ tag_version="${GITHUB_REF_NAME#v}"
23
+ project_version=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")
24
+ if [ "$tag_version" != "$project_version" ]; then
25
+ echo "::error::Tag $GITHUB_REF_NAME does not match version $project_version in pyproject.toml"
26
+ exit 1
27
+ fi
19
28
  - name: Install build
20
29
  run: |
21
30
  python3 -m pip install build
@@ -1,26 +1,26 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: filetracker
3
- Version: 2.2.0
3
+ Version: 2.3.3
4
4
  Summary: Filetracker caching file storage
5
5
  Author-email: SIO2 Project Team <sio2@sio2project.mimuw.edu.pl>
6
6
  License: GPL
7
7
  Project-URL: homepage, https://github.com/sio2project/filetracker
8
8
  Classifier: Framework :: Django
9
9
  Classifier: Programming Language :: Python :: 3
10
- Requires-Python: >=3.7
10
+ Requires-Python: >=3.9
11
11
  Description-Content-Type: text/markdown
12
12
  License-File: LICENSE
13
13
  Requires-Dist: bsddb3==6.2.7
14
- Requires-Dist: flup6
15
- Requires-Dist: gunicorn==19.9.0
14
+ Requires-Dist: gunicorn==23.0.0
16
15
  Requires-Dist: progressbar2
17
16
  Requires-Dist: requests
18
17
  Requires-Dist: six
19
18
  Provides-Extra: server
20
- Requires-Dist: gevent==22.10.2; extra == "server"
21
- Requires-Dist: greenlet==2.0.2; extra == "server"
19
+ Requires-Dist: gevent==25.9.1; extra == "server"
22
20
  Provides-Extra: tests
23
21
  Requires-Dist: pytest; extra == "tests"
22
+ Requires-Dist: psutil; extra == "tests"
23
+ Dynamic: license-file
24
24
 
25
25
  # filetracker
26
26
 
@@ -97,7 +97,7 @@ slashes.
97
97
 
98
98
  The response is plain-text, with one line for every file path.
99
99
 
100
- Version cutoff must be specified in RFC 2822 format through
100
+ Version cutoff may be specified in RFC 2822 format through
101
101
  `?last_modified=` query parameter. Only files with modification time
102
102
  older than this version will be listed.
103
103
 
@@ -338,3 +338,13 @@ class Client(object):
338
338
  if self.local_store:
339
339
  result.extend(self.local_store.list_files())
340
340
  return result
341
+
342
+ def list_remote_files(self, version_cutoff_timestamp=None, subpath="", absolute_paths=False):
343
+ """Returns list of all stored remote files under `subpath`
344
+ not newer than `version_cutoff_timestamp`.
345
+
346
+ Each element of this list is just a string with the full path.
347
+ """
348
+ if not self.remote_store:
349
+ return []
350
+ return self.remote_store.list_files(version_cutoff_timestamp, subpath, absolute_paths)
@@ -43,7 +43,8 @@ class DataStore(object):
43
43
  Works like :meth:`add_stream`, but ``filename`` is the name of
44
44
  an existing file in the filesystem.
45
45
  """
46
- return self.add_stream(name, open(filename, 'rb'))
46
+ with open(filename, 'rb') as f:
47
+ return self.add_stream(name, f)
47
48
 
48
49
  def exists(self, name):
49
50
  """Returns ``True`` if the file exists, ``False`` otherwise.
@@ -40,6 +40,9 @@ SERVER_ACCEPTS_SHA256_DIGEST = 3
40
40
  # The server supports deleting files
41
41
  SERVER_ACCEPTS_DELETE = 4
42
42
 
43
+ # The server supports listing files
44
+ SERVER_ACCEPTS_LIST = 5
45
+
43
46
  _PROTOCOL_CAPABILITIES = {
44
47
  1: [
45
48
  SERVER_REQUIRES_VERSION_HEADER,
@@ -48,6 +51,7 @@ _PROTOCOL_CAPABILITIES = {
48
51
  SERVER_ACCEPTS_GZIP,
49
52
  SERVER_ACCEPTS_SHA256_DIGEST,
50
53
  SERVER_ACCEPTS_DELETE,
54
+ SERVER_ACCEPTS_LIST,
51
55
  ],
52
56
  }
53
57
 
@@ -217,6 +221,22 @@ class RemoteDataStore(DataStore):
217
221
  response = requests.delete(url, headers=headers)
218
222
  response.raise_for_status()
219
223
 
224
+ @_verbose_http_errors
225
+ def list_files(self, version, subpath, absolute_paths):
226
+ if not self._has_capability(SERVER_ACCEPTS_LIST):
227
+ return
228
+ url = self.base_url + '/list' + pathname2url(subpath)
229
+ url, headers = self._add_version_to_request(url, {}, version)
230
+ response = requests.get(url, headers=headers)
231
+ response.raise_for_status()
232
+ result = response.content.decode('utf-8').split('\n')
233
+ assert len(result.pop()) == 0
234
+ if absolute_paths and subpath and subpath != "/":
235
+ prefix = subpath.rstrip("/").lstrip("/") + "/"
236
+ return [prefix + path for path in result]
237
+ else:
238
+ return result
239
+
220
240
  def _add_version_to_request(self, url, headers, version):
221
241
  """Adds version to either url or headers, depending on protocol."""
222
242
  if self._has_capability(SERVER_REQUIRES_VERSION_HEADER):
@@ -86,48 +86,6 @@ def get_endpoint_and_path(environ):
86
86
  return components[0], '/'.join(components[1:])
87
87
 
88
88
 
89
- # Deprecated, run.py should be used in all cases.
90
- def start_cgi(server):
91
- from flup.server.cgi import WSGIServer
92
-
93
- WSGIServer(server).run()
94
- sys.exit(0)
95
-
96
-
97
- # Deprecated, run.py should be used in all cases.
98
- def start_fcgi(server):
99
- from flup.server.fcgi import WSGIServer
100
-
101
- WSGIServer(server).run()
102
- sys.exit(0)
103
-
104
-
105
- # Deprecated, run.py should be used in all cases.
106
- def start_standalone(server, port=8000):
107
- from wsgiref.simple_server import make_server
108
-
109
- httpd = make_server('', port, server)
110
- print("Serving on port %d..." % port)
111
- httpd.serve_forever()
112
-
113
-
114
- def main(server):
115
- """A convenience ``main`` method for running WSGI-compatible HTTP
116
- application as CGI, FCGI or standalone (with auto-detection)."""
117
-
118
- if 'REQUEST_METHOD' in os.environ:
119
- start_cgi(server)
120
-
121
- stdin_sock = socket.fromfd(0, socket.AF_UNIX, socket.SOCK_STREAM)
122
- try:
123
- stdin_sock.getpeername()
124
- except socket.error as e:
125
- if e.errno == errno.ENOTCONN:
126
- start_fcgi(server)
127
-
128
- start_standalone(server)
129
-
130
-
131
89
  def _rindex(l, value):
132
90
  """Same as str.rindex, but for lists."""
133
91
  return len(l) - l[::-1].index(value) - 1
@@ -156,9 +156,13 @@ class FiletrackerServer(base.Server):
156
156
  query_params = self.parse_query_params(environ)
157
157
 
158
158
  last_modified = query_params.get('last_modified', (None,))[0]
159
- if not last_modified:
160
- last_modified = int(time.time())
161
-
159
+ if last_modified:
160
+ last_modified = email.utils.parsedate_tz(last_modified)
161
+ last_modified = email.utils.mktime_tz(last_modified)
162
+ else:
163
+ last_modified = time.time()
164
+ last_modified = int(last_modified)
165
+
162
166
  logger.debug('Handling GET /list/%s (@%d)', path, last_modified)
163
167
 
164
168
  root_dir = os.path.join(self.dir, path)
@@ -174,6 +174,10 @@ def main(args=None):
174
174
  log_config['loggers']['gunicorn.error']['level'] = options.log_level
175
175
  log_config['loggers']['gunicorn.access']['level'] = options.log_level
176
176
  log_config['loggers']['']['level'] = options.log_level
177
+ log_config['root'] = {
178
+ 'level': "WARNING",
179
+ 'handlers': [],
180
+ }
177
181
 
178
182
  logging.config.dictConfig(log_config)
179
183
 
@@ -47,6 +47,7 @@ from filetracker.utils import file_digest
47
47
 
48
48
  _LOCK_RETRIES = 20
49
49
  _LOCK_SLEEP_TIME_S = 1
50
+ _DB_DEADLOCK_RETRIES = 50
50
51
 
51
52
 
52
53
  logger = logging.getLogger(__name__)
@@ -81,6 +82,8 @@ class FileStorage(object):
81
82
 
82
83
  # https://docs.oracle.com/cd/E17076_05/html/programmer_reference/transapp_env_open.html
83
84
  self.db_env = bsddb3.db.DBEnv()
85
+ # Enable running the deadlock detector when lock conflicts are automatically detected.
86
+ self.db_env.set_lk_detect(bsddb3.db.DB_LOCK_DEFAULT)
84
87
  try:
85
88
  self.db_env.open(
86
89
  self.db_dir,
@@ -182,9 +185,8 @@ class FileStorage(object):
182
185
  logger.debug('Acquired lock for blob %s.', digest)
183
186
  digest_bytes = digest.encode()
184
187
 
185
- with self._db_transaction() as txn:
186
- logger.debug('Started DB transaction (adding link).')
187
- link_count = int(self.db.get(digest_bytes, 0, txn=txn))
188
+ def transaction_contents(txn):
189
+ link_count = int(self.db.get(digest_bytes, 0, txn=txn, flags=bsddb3.db.DB_RMW))
188
190
  new_count = str(link_count + 1).encode()
189
191
  self.db.put(digest_bytes, new_count, txn=txn)
190
192
 
@@ -194,9 +196,11 @@ class FileStorage(object):
194
196
  str(logical_size).encode(),
195
197
  txn=txn,
196
198
  )
197
- logger.debug('Commiting DB transaction (adding link).')
199
+ return link_count
198
200
 
199
- logger.debug('Committed DB transaction (adding link).')
201
+ link_count = self._call_in_transaction_with_retries(
202
+ transaction_contents, "adding link"
203
+ )
200
204
 
201
205
  # Create a new blob if this isn't a duplicate.
202
206
  if link_count == 0:
@@ -268,12 +272,11 @@ class FileStorage(object):
268
272
 
269
273
  with _exclusive_lock(self._lock_path('blobs', digest)):
270
274
  logger.debug('Acquired lock for blob %s.', digest)
271
- should_delete_blob = False
272
275
 
273
- with self._db_transaction() as txn:
274
- logger.debug('Started DB transaction (deleting link).')
276
+ def transaction_contents(txn):
277
+ should_delete_blob = False
275
278
  digest_bytes = digest.encode()
276
- link_count = self.db.get(digest_bytes, txn=txn)
279
+ link_count = self.db.get(digest_bytes, txn=txn, flags=bsddb3.db.DB_RMW)
277
280
  if link_count is None:
278
281
  raise RuntimeError("File exists but has no key in db")
279
282
 
@@ -288,9 +291,11 @@ class FileStorage(object):
288
291
  else:
289
292
  new_count = str(link_count - 1).encode()
290
293
  self.db.put(digest_bytes, new_count, txn=txn)
291
- logger.debug('Committing DB transaction (deleting link).')
294
+ return should_delete_blob
292
295
 
293
- logger.debug('Committed DB transaction (deleting link).')
296
+ should_delete_blob = self._call_in_transaction_with_retries(
297
+ transaction_contents, "deleting link"
298
+ )
294
299
 
295
300
  os.unlink(link_path)
296
301
  logger.debug('Deleted link %s.', name)
@@ -339,6 +344,30 @@ class FileStorage(object):
339
344
  else:
340
345
  txn.commit()
341
346
 
347
+ def _call_in_transaction_with_retries(self, func, description):
348
+ retries = 0
349
+ result = None
350
+ while True:
351
+ try:
352
+ with self._db_transaction() as txn:
353
+ logger.debug('Started DB transaction ({}).'.format(description))
354
+ result = func(txn)
355
+ logger.debug('Commiting DB transaction ({}).'.format(description))
356
+ except bsddb3.db.DBLockDeadlockError:
357
+ retries += 1
358
+ if retries > _DB_DEADLOCK_RETRIES:
359
+ logger.error('BSDDB deadlock detected, retry limit exceeded ({}).'.format(description))
360
+ raise
361
+ logger.warning(
362
+ 'BSDDB deadlock detected in transaction ({}), retry no {}.'.format(
363
+ description, retries,
364
+ )
365
+ )
366
+ continue
367
+ break
368
+ logger.debug('Committed DB transaction ({}).'.format(description))
369
+ return result
370
+
342
371
  def _digest_for_link(self, name):
343
372
  link = self._link_path(name)
344
373
  blob_path = os.readlink(link)
@@ -148,6 +148,35 @@ class InteractionTest(unittest.TestCase):
148
148
 
149
149
  self.assertNotEqual(version_a, version_b)
150
150
 
151
+ def test_list_remote_files(self):
152
+ src_file = os.path.join(self.temp_dir, 'older.txt')
153
+ with open(src_file, 'wb') as sf:
154
+ sf.write(b'these tests are so bad')
155
+
156
+ self.client.put_file('/A@1', src_file)
157
+ self.client.put_file('/B@5', src_file)
158
+ self.client.put_file('/C/D@10', src_file)
159
+
160
+ def check(expected, *args):
161
+ result_raw = self.client.list_remote_files(*args)
162
+ # Filter out files from other tests.
163
+ result = sorted(filter(lambda x: not x.endswith(".txt"), result_raw))
164
+ self.assertEqual(result, sorted(expected))
165
+
166
+ check(["A", "B", "C/D"])
167
+ check(["A"], 4)
168
+ check(["A", "B"], 5)
169
+ check(["A", "B"], 9)
170
+ check(["A", "B", "C/D"], 10)
171
+ with self.assertRaises(FiletrackerError):
172
+ check(["D"], 10, "C")
173
+ check(["D"], 10, "/C")
174
+ check(["D"], 10, "/C/")
175
+ # absolute_paths=True
176
+ check(["C/D"], 10, "/C", True)
177
+ check(["C/D"], 10, "/C/", True)
178
+ check([], 9, "/C")
179
+
151
180
  def test_put_older_should_fail(self):
152
181
  """This test assumes file version is stored in mtime."""
153
182
  src_file = os.path.join(self.temp_dir, 'older.txt')
@@ -4,8 +4,9 @@ from __future__ import absolute_import
4
4
  from __future__ import division
5
5
  from __future__ import print_function
6
6
 
7
- from multiprocessing import Process
7
+ from multiprocessing import Barrier, Process
8
8
  import os
9
+ import psutil
9
10
  import shutil
10
11
  import tempfile
11
12
  import time
@@ -25,6 +26,17 @@ _CLIENT_WAIT_S = 0.1
25
26
  _FILE_SIZE = 6 * 1024 * 1024
26
27
  _PARALLEL_CLIENTS = 5
27
28
  _TEST_PORT_NUMBER = 45745
29
+ _COPIES_TO_UPLOAD = 500
30
+ _SUBPROCESS_TIMEOUT_S = 20
31
+
32
+
33
+ def kill_process_tree(pid):
34
+ parent = psutil.Process(pid)
35
+ for p in parent.children(recursive=True) + [parent]:
36
+ try:
37
+ p.kill()
38
+ except psutil.NoSuchProcess:
39
+ pass
28
40
 
29
41
 
30
42
  class ParallelTest(unittest.TestCase):
@@ -52,19 +64,23 @@ class ParallelTest(unittest.TestCase):
52
64
 
53
65
  @classmethod
54
66
  def tearDownClass(cls):
55
- cls.server_process.terminate()
67
+ kill_process_tree(cls.server_process.pid)
68
+ cls.server_process.join()
56
69
  shutil.rmtree(cls.server_dir)
57
70
  shutil.rmtree(cls.temp_dir)
58
71
 
59
72
  def setUp(self):
60
73
  # Shortcuts for convenience
61
- self.server_dir = ParallelTest.server_dir
62
- self.temp_dir = ParallelTest.temp_dir
63
- self.clients = ParallelTest.clients
74
+ cls = self.__class__
75
+ self.server_dir = cls.server_dir
76
+ self.temp_dir = cls.temp_dir
77
+ self.clients = cls.clients
64
78
 
65
79
  # For non-parallel requests
66
- self.client = ParallelTest.clients[0]
80
+ self.client = cls.clients[0]
81
+
67
82
 
83
+ class ParallelTestSameFile(ParallelTest):
68
84
  def test_only_last_parallel_upload_of_same_file_should_succeed(self):
69
85
  processes = []
70
86
 
@@ -98,7 +114,43 @@ class ParallelTest(unittest.TestCase):
98
114
  self.assertEqual(f.read(), lf.read())
99
115
 
100
116
 
117
+ class ParallelTestDeadlocks(ParallelTest):
118
+ def test_bsddb_deadlocks(self):
119
+ processes = []
120
+
121
+ # Initialize different files for every client.
122
+ for i in range(len(self.clients)):
123
+ temp_file = os.path.join(self.temp_dir, 'foo{}.txt'.format(i))
124
+ text = str(i).encode()
125
+ with open(temp_file, 'wb') as tf:
126
+ tf.write(text)
127
+
128
+ # The deadlocks are visible even without this barrier.
129
+ barrier = Barrier(len(self.clients))
130
+
131
+ def job(id, barrier):
132
+ temp_file = os.path.join(self.temp_dir, 'foo{}.txt'.format(id))
133
+ for i in range(0, _COPIES_TO_UPLOAD):
134
+ ft_name = '/foo{}.{}.txt'.format(id, i)
135
+ barrier.wait()
136
+ client.put_file(ft_name, temp_file, compress_hint=False)
137
+
138
+ for i, client in enumerate(self.clients):
139
+ process = Process(target=lambda: job(i, barrier))
140
+ process.start()
141
+ processes.append(process)
142
+
143
+ for process in processes:
144
+ process.join(timeout=_SUBPROCESS_TIMEOUT_S)
145
+ self.assertFalse(process.is_alive())
146
+ self.assertEqual(process.exitcode, 0)
147
+ process.join()
148
+
149
+ # Put one final file to check for e.g. corruption.
150
+ client.put_file('/foo_last', temp_file, compress_hint=False)
151
+
152
+
101
153
  def _start_server(server_dir):
102
154
  server_main(
103
- ['-p', str(_TEST_PORT_NUMBER), '-d', server_dir, '-D', '--workers', '6']
155
+ ['-p', str(_TEST_PORT_NUMBER), '-d', server_dir, '-D', '--workers', '3']
104
156
  )
@@ -5,6 +5,7 @@ from __future__ import division
5
5
  from __future__ import print_function
6
6
 
7
7
  from multiprocessing import Process
8
+ import email.utils
8
9
  import os
9
10
  import shutil
10
11
  import tempfile
@@ -67,6 +68,31 @@ class ProtocolTest(unittest.TestCase):
67
68
  self.assertEqual(lines.count('list_a.txt'), 1)
68
69
  self.assertEqual(lines.count('list_b.txt'), 1)
69
70
 
71
+ def test_list_files_version_cutoff_should_work(self):
72
+ src_file = os.path.join(self.temp_dir, 'list.txt')
73
+ with open(src_file, 'wb') as sf:
74
+ sf.write(b'hello list')
75
+
76
+ self.client.put_file('/A@1', src_file)
77
+ self.client.put_file('/B@100', src_file)
78
+ self.client.put_file('/C@200', src_file)
79
+
80
+ def check(timestamp, expected):
81
+ date = email.utils.formatdate(timestamp)
82
+ params = {'last_modified': date}
83
+ res = requests.get('http://127.0.0.1:{}/list/'.format(_TEST_PORT_NUMBER), params=params)
84
+ self.assertEqual(res.status_code, 200)
85
+ # Filter out files from other tests.
86
+ lines = [l for l in res.text.split('\n') if l]
87
+ self.assertCountEqual(lines, expected)
88
+
89
+ check(0, [])
90
+ check(1, ['A'])
91
+ check(99, ['A'])
92
+ check(100, ['A', 'B'])
93
+ check(199, ['A', 'B'])
94
+ check(200, ['A', 'B', 'C'])
95
+
70
96
  def test_list_files_in_subdirectory_should_work(self):
71
97
  src_file = os.path.join(self.temp_dir, 'list_sub.txt')
72
98
  with open(src_file, 'wb') as sf:
@@ -1,26 +1,26 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: filetracker
3
- Version: 2.2.0
3
+ Version: 2.3.3
4
4
  Summary: Filetracker caching file storage
5
5
  Author-email: SIO2 Project Team <sio2@sio2project.mimuw.edu.pl>
6
6
  License: GPL
7
7
  Project-URL: homepage, https://github.com/sio2project/filetracker
8
8
  Classifier: Framework :: Django
9
9
  Classifier: Programming Language :: Python :: 3
10
- Requires-Python: >=3.7
10
+ Requires-Python: >=3.9
11
11
  Description-Content-Type: text/markdown
12
12
  License-File: LICENSE
13
13
  Requires-Dist: bsddb3==6.2.7
14
- Requires-Dist: flup6
15
- Requires-Dist: gunicorn==19.9.0
14
+ Requires-Dist: gunicorn==23.0.0
16
15
  Requires-Dist: progressbar2
17
16
  Requires-Dist: requests
18
17
  Requires-Dist: six
19
18
  Provides-Extra: server
20
- Requires-Dist: gevent==22.10.2; extra == "server"
21
- Requires-Dist: greenlet==2.0.2; extra == "server"
19
+ Requires-Dist: gevent==25.9.1; extra == "server"
22
20
  Provides-Extra: tests
23
21
  Requires-Dist: pytest; extra == "tests"
22
+ Requires-Dist: psutil; extra == "tests"
23
+ Dynamic: license-file
24
24
 
25
25
  # filetracker
26
26
 
@@ -1,13 +1,12 @@
1
1
  bsddb3==6.2.7
2
- flup6
3
- gunicorn==19.9.0
2
+ gunicorn==23.0.0
4
3
  progressbar2
5
4
  requests
6
5
  six
7
6
 
8
7
  [server]
9
- gevent==22.10.2
10
- greenlet==2.0.2
8
+ gevent==25.9.1
11
9
 
12
10
  [tests]
13
11
  pytest
12
+ psutil
@@ -15,7 +15,7 @@ authors = [
15
15
  ]
16
16
  description = "Filetracker caching file storage"
17
17
  readme = "README.md"
18
- requires-python = ">=3.7"
18
+ requires-python = ">=3.9"
19
19
  license = {text = "GPL"}
20
20
  classifiers = [
21
21
  "Framework :: Django",
@@ -23,24 +23,23 @@ classifiers = [
23
23
  ]
24
24
  dependencies = [
25
25
  "bsddb3==6.2.7",
26
- "flup6",
27
- "gunicorn==19.9.0",
26
+ "gunicorn==23.0.0",
28
27
  "progressbar2",
29
28
  "requests",
30
29
  "six",
31
30
  ]
32
- version = "2.2.0"
31
+ version = "2.3.3"
33
32
 
34
33
  [project.urls]
35
34
  "homepage" = "https://github.com/sio2project/filetracker"
36
35
 
37
36
  [project.optional-dependencies]
38
37
  server = [
39
- "gevent==22.10.2",
40
- "greenlet==2.0.2",
38
+ "gevent==25.9.1"
41
39
  ]
42
40
  tests = [
43
41
  "pytest",
42
+ "psutil",
44
43
  ]
45
44
 
46
45
  [project.scripts]
@@ -48,4 +47,4 @@ filetracker = "filetracker.client.shell:main"
48
47
  filetracker-server = "filetracker.servers.run:main"
49
48
  filetracker-cache-cleaner = "filetracker.scripts.cachecleaner:main"
50
49
  filetracker-migrate = "filetracker.scripts.migrate:main"
51
- filetracker-recover = "filetracker.scripts.recover:main"
50
+ filetracker-recover = "filetracker.scripts.recover:main"
@@ -1,8 +1,9 @@
1
1
  [tox]
2
- envlist = py38,py39
2
+ envlist = py313
3
3
 
4
4
  [testenv]
5
5
  extras = server
6
+ tests
6
7
  deps = pytest
7
8
  pytest-cov
8
9
  setenv =
@@ -1,2 +0,0 @@
1
- * @twalen
2
- * @wolf480pl
@@ -1,40 +0,0 @@
1
- name: Unit tests
2
-
3
- on:
4
- push:
5
- branches:
6
- - master
7
- pull_request:
8
- workflow_dispatch:
9
-
10
- jobs:
11
- build:
12
- strategy:
13
- fail-fast: false
14
- matrix:
15
- include:
16
- - python-version: "3.8"
17
- - python-version: "3.9"
18
- runs-on: ubuntu-latest
19
-
20
- steps:
21
- - uses: actions/checkout@v3
22
- - name: Set up Python ${{ matrix.python-version }}
23
- uses: actions/setup-python@v4
24
- with:
25
- python-version: ${{ matrix.python-version }}
26
-
27
- - name: Install apt dependencies
28
- run: |
29
- sudo apt-get update
30
- sudo apt-get install -y libdb-dev
31
-
32
- - name: Install pip dependencies
33
- run: |
34
- python -m pip install --upgrade pip wheel setuptools
35
- pip install -e .[server]
36
- pip install pytest
37
-
38
- - name: Run tests
39
- run: |
40
- pytest -vv filetracker/tests
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes