donald 0.0.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- Donald-0.0.0.dist-info/DESCRIPTION.rst +169 -0
- Donald-0.0.0.dist-info/METADATA +192 -0
- Donald-0.0.0.dist-info/RECORD +11 -0
- Donald-0.0.0.dist-info/WHEEL +6 -0
- Donald-0.0.0.dist-info/metadata.json +1 -0
- Donald-0.0.0.dist-info/top_level.txt +1 -0
- donald/__init__.py +14 -0
- donald/core.py +122 -0
- donald/queue.py +78 -0
- donald/utils.py +37 -0
- donald/worker.py +61 -0
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
Peewee Migrate
|
|
2
|
+
##############
|
|
3
|
+
|
|
4
|
+
.. _description:
|
|
5
|
+
|
|
6
|
+
Peewee Migrate -- A simple migration engine for Peewee
|
|
7
|
+
|
|
8
|
+
.. _badges:
|
|
9
|
+
|
|
10
|
+
.. image:: http://img.shields.io/travis/klen/peewee_migrate.svg?style=flat-square
|
|
11
|
+
:target: http://travis-ci.org/klen/peewee_migrate
|
|
12
|
+
:alt: Build Status
|
|
13
|
+
|
|
14
|
+
.. image:: http://img.shields.io/coveralls/klen/peewee_migrate.svg?style=flat-square
|
|
15
|
+
:target: https://coveralls.io/r/klen/pewee_migrate
|
|
16
|
+
:alt: Coverals
|
|
17
|
+
|
|
18
|
+
.. image:: http://img.shields.io/pypi/v/peewee_migrate.svg?style=flat-square
|
|
19
|
+
:target: https://pypi.python.org/pypi/peewee_migrate
|
|
20
|
+
:alt: Version
|
|
21
|
+
|
|
22
|
+
.. image:: http://img.shields.io/pypi/dm/peewee_migrate.svg?style=flat-square
|
|
23
|
+
:target: https://pypi.python.org/pypi/peewee_migrate
|
|
24
|
+
:alt: Downloads
|
|
25
|
+
|
|
26
|
+
.. _contents:
|
|
27
|
+
|
|
28
|
+
.. contents::
|
|
29
|
+
|
|
30
|
+
.. _requirements:
|
|
31
|
+
|
|
32
|
+
Requirements
|
|
33
|
+
=============
|
|
34
|
+
|
|
35
|
+
- python 2.7,3.3,3.4
|
|
36
|
+
|
|
37
|
+
.. _installation:
|
|
38
|
+
|
|
39
|
+
Installation
|
|
40
|
+
=============
|
|
41
|
+
|
|
42
|
+
**Peewee Migrate** should be installed using pip: ::
|
|
43
|
+
|
|
44
|
+
pip install peewee_migrate
|
|
45
|
+
|
|
46
|
+
.. _usage:
|
|
47
|
+
|
|
48
|
+
Usage
|
|
49
|
+
=====
|
|
50
|
+
|
|
51
|
+
Do you want Flask_ integration? Look at Flask-PW_.
|
|
52
|
+
|
|
53
|
+
>From shell
|
|
54
|
+
----------
|
|
55
|
+
|
|
56
|
+
Getting help: ::
|
|
57
|
+
|
|
58
|
+
$ pw_migrate --help
|
|
59
|
+
|
|
60
|
+
Usage: pw_migrate [OPTIONS] COMMAND [ARGS]...
|
|
61
|
+
|
|
62
|
+
Options:
|
|
63
|
+
--help Show this message and exit.
|
|
64
|
+
|
|
65
|
+
Commands:
|
|
66
|
+
create Create migration.
|
|
67
|
+
migrate Run migrations.
|
|
68
|
+
rollback Rollback migration.
|
|
69
|
+
|
|
70
|
+
Create migration: ::
|
|
71
|
+
|
|
72
|
+
$ pw_migrate create --help
|
|
73
|
+
|
|
74
|
+
Usage: pw_migrate create [OPTIONS] NAME
|
|
75
|
+
|
|
76
|
+
Create migration.
|
|
77
|
+
|
|
78
|
+
Options:
|
|
79
|
+
--auto TEXT Create migrations automatically. Set path to your models module.
|
|
80
|
+
--database TEXT Database connection
|
|
81
|
+
--directory TEXT Directory where migrations are stored
|
|
82
|
+
-v, --verbose
|
|
83
|
+
--help Show this message and exit.
|
|
84
|
+
|
|
85
|
+
Run migrations: ::
|
|
86
|
+
|
|
87
|
+
$ pw_migrate migrate --help
|
|
88
|
+
|
|
89
|
+
Usage: pw_migrate migrate [OPTIONS]
|
|
90
|
+
|
|
91
|
+
Run migrations.
|
|
92
|
+
|
|
93
|
+
Options:
|
|
94
|
+
--name TEXT Select migration
|
|
95
|
+
--database TEXT Database connection
|
|
96
|
+
--directory TEXT Directory where migrations are stored
|
|
97
|
+
-v, --verbose
|
|
98
|
+
--help Show this message and exit.
|
|
99
|
+
|
|
100
|
+
>From python
|
|
101
|
+
-----------
|
|
102
|
+
::
|
|
103
|
+
|
|
104
|
+
from peewee_migrate import Router
|
|
105
|
+
from peewee import SqliteDatabase
|
|
106
|
+
|
|
107
|
+
router = Router(SqliteDatabase('test.db'))
|
|
108
|
+
|
|
109
|
+
# Create migration
|
|
110
|
+
router.create('migration_name')
|
|
111
|
+
|
|
112
|
+
# Run migration/migrations
|
|
113
|
+
router.run('migration_name')
|
|
114
|
+
|
|
115
|
+
# Run all unapplied migrations
|
|
116
|
+
router.run()
|
|
117
|
+
|
|
118
|
+
Migration files
|
|
119
|
+
---------------
|
|
120
|
+
|
|
121
|
+
By default, migration files are looked up in ``os.getcwd()/migrations`` directory, but custom directory can be given.
|
|
122
|
+
|
|
123
|
+
Migration files are sorted and applied in ascending order per their filename.
|
|
124
|
+
|
|
125
|
+
Each migration file must specify ``migrate()`` function and may specify ``rollback()`` function::
|
|
126
|
+
|
|
127
|
+
def migrate(migrator, database, fake=False, **kwargs):
|
|
128
|
+
pass
|
|
129
|
+
|
|
130
|
+
def rollback(migrator, database, fake=False, **kwargs):
|
|
131
|
+
pass
|
|
132
|
+
|
|
133
|
+
.. _bugtracker:
|
|
134
|
+
|
|
135
|
+
Bug tracker
|
|
136
|
+
===========
|
|
137
|
+
|
|
138
|
+
If you have any suggestions, bug reports or
|
|
139
|
+
annoyances please report them to the issue tracker
|
|
140
|
+
at https://github.com/klen/peewee_migrate/issues
|
|
141
|
+
|
|
142
|
+
.. _contributing:
|
|
143
|
+
|
|
144
|
+
Contributing
|
|
145
|
+
============
|
|
146
|
+
|
|
147
|
+
Development of starter happens at github: https://github.com/klen/peewee_migrate
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
Contributors
|
|
151
|
+
=============
|
|
152
|
+
|
|
153
|
+
* klen_ (Kirill Klenov)
|
|
154
|
+
|
|
155
|
+
.. _license:
|
|
156
|
+
|
|
157
|
+
License
|
|
158
|
+
=======
|
|
159
|
+
|
|
160
|
+
Licensed under a `BSD license`_.
|
|
161
|
+
|
|
162
|
+
.. _links:
|
|
163
|
+
|
|
164
|
+
.. _BSD license: http://www.linfo.org/bsdlicense.html
|
|
165
|
+
.. _klen: https://klen.github.io/
|
|
166
|
+
.. _Flask: http://flask.pocoo.org/
|
|
167
|
+
.. _Flask-PW: https://github.com/klen/flask-pw
|
|
168
|
+
|
|
169
|
+
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
Metadata-Version: 2.0
|
|
2
|
+
Name: Donald
|
|
3
|
+
Version: 0.0.0
|
|
4
|
+
Summary: Donald is here
|
|
5
|
+
Home-page: https://github.com/klen/donald
|
|
6
|
+
Author: Kirill Klenov
|
|
7
|
+
Author-email: horneds@gmail.com
|
|
8
|
+
License: BSD
|
|
9
|
+
Keywords: asyncio,tasks,queue
|
|
10
|
+
Platform: Any
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
14
|
+
Classifier: Natural Language :: English
|
|
15
|
+
Classifier: Natural Language :: Russian
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
|
+
Classifier: Topic :: Software Development :: Testing
|
|
21
|
+
Classifier: Topic :: Utilities
|
|
22
|
+
Requires-Dist: aioamqp
|
|
23
|
+
|
|
24
|
+
Peewee Migrate
|
|
25
|
+
##############
|
|
26
|
+
|
|
27
|
+
.. _description:
|
|
28
|
+
|
|
29
|
+
Peewee Migrate -- A simple migration engine for Peewee
|
|
30
|
+
|
|
31
|
+
.. _badges:
|
|
32
|
+
|
|
33
|
+
.. image:: http://img.shields.io/travis/klen/peewee_migrate.svg?style=flat-square
|
|
34
|
+
:target: http://travis-ci.org/klen/peewee_migrate
|
|
35
|
+
:alt: Build Status
|
|
36
|
+
|
|
37
|
+
.. image:: http://img.shields.io/coveralls/klen/peewee_migrate.svg?style=flat-square
|
|
38
|
+
:target: https://coveralls.io/r/klen/pewee_migrate
|
|
39
|
+
:alt: Coverals
|
|
40
|
+
|
|
41
|
+
.. image:: http://img.shields.io/pypi/v/peewee_migrate.svg?style=flat-square
|
|
42
|
+
:target: https://pypi.python.org/pypi/peewee_migrate
|
|
43
|
+
:alt: Version
|
|
44
|
+
|
|
45
|
+
.. image:: http://img.shields.io/pypi/dm/peewee_migrate.svg?style=flat-square
|
|
46
|
+
:target: https://pypi.python.org/pypi/peewee_migrate
|
|
47
|
+
:alt: Downloads
|
|
48
|
+
|
|
49
|
+
.. _contents:
|
|
50
|
+
|
|
51
|
+
.. contents::
|
|
52
|
+
|
|
53
|
+
.. _requirements:
|
|
54
|
+
|
|
55
|
+
Requirements
|
|
56
|
+
=============
|
|
57
|
+
|
|
58
|
+
- python 2.7,3.3,3.4
|
|
59
|
+
|
|
60
|
+
.. _installation:
|
|
61
|
+
|
|
62
|
+
Installation
|
|
63
|
+
=============
|
|
64
|
+
|
|
65
|
+
**Peewee Migrate** should be installed using pip: ::
|
|
66
|
+
|
|
67
|
+
pip install peewee_migrate
|
|
68
|
+
|
|
69
|
+
.. _usage:
|
|
70
|
+
|
|
71
|
+
Usage
|
|
72
|
+
=====
|
|
73
|
+
|
|
74
|
+
Do you want Flask_ integration? Look at Flask-PW_.
|
|
75
|
+
|
|
76
|
+
>From shell
|
|
77
|
+
----------
|
|
78
|
+
|
|
79
|
+
Getting help: ::
|
|
80
|
+
|
|
81
|
+
$ pw_migrate --help
|
|
82
|
+
|
|
83
|
+
Usage: pw_migrate [OPTIONS] COMMAND [ARGS]...
|
|
84
|
+
|
|
85
|
+
Options:
|
|
86
|
+
--help Show this message and exit.
|
|
87
|
+
|
|
88
|
+
Commands:
|
|
89
|
+
create Create migration.
|
|
90
|
+
migrate Run migrations.
|
|
91
|
+
rollback Rollback migration.
|
|
92
|
+
|
|
93
|
+
Create migration: ::
|
|
94
|
+
|
|
95
|
+
$ pw_migrate create --help
|
|
96
|
+
|
|
97
|
+
Usage: pw_migrate create [OPTIONS] NAME
|
|
98
|
+
|
|
99
|
+
Create migration.
|
|
100
|
+
|
|
101
|
+
Options:
|
|
102
|
+
--auto TEXT Create migrations automatically. Set path to your models module.
|
|
103
|
+
--database TEXT Database connection
|
|
104
|
+
--directory TEXT Directory where migrations are stored
|
|
105
|
+
-v, --verbose
|
|
106
|
+
--help Show this message and exit.
|
|
107
|
+
|
|
108
|
+
Run migrations: ::
|
|
109
|
+
|
|
110
|
+
$ pw_migrate migrate --help
|
|
111
|
+
|
|
112
|
+
Usage: pw_migrate migrate [OPTIONS]
|
|
113
|
+
|
|
114
|
+
Run migrations.
|
|
115
|
+
|
|
116
|
+
Options:
|
|
117
|
+
--name TEXT Select migration
|
|
118
|
+
--database TEXT Database connection
|
|
119
|
+
--directory TEXT Directory where migrations are stored
|
|
120
|
+
-v, --verbose
|
|
121
|
+
--help Show this message and exit.
|
|
122
|
+
|
|
123
|
+
>From python
|
|
124
|
+
-----------
|
|
125
|
+
::
|
|
126
|
+
|
|
127
|
+
from peewee_migrate import Router
|
|
128
|
+
from peewee import SqliteDatabase
|
|
129
|
+
|
|
130
|
+
router = Router(SqliteDatabase('test.db'))
|
|
131
|
+
|
|
132
|
+
# Create migration
|
|
133
|
+
router.create('migration_name')
|
|
134
|
+
|
|
135
|
+
# Run migration/migrations
|
|
136
|
+
router.run('migration_name')
|
|
137
|
+
|
|
138
|
+
# Run all unapplied migrations
|
|
139
|
+
router.run()
|
|
140
|
+
|
|
141
|
+
Migration files
|
|
142
|
+
---------------
|
|
143
|
+
|
|
144
|
+
By default, migration files are looked up in ``os.getcwd()/migrations`` directory, but custom directory can be given.
|
|
145
|
+
|
|
146
|
+
Migration files are sorted and applied in ascending order per their filename.
|
|
147
|
+
|
|
148
|
+
Each migration file must specify ``migrate()`` function and may specify ``rollback()`` function::
|
|
149
|
+
|
|
150
|
+
def migrate(migrator, database, fake=False, **kwargs):
|
|
151
|
+
pass
|
|
152
|
+
|
|
153
|
+
def rollback(migrator, database, fake=False, **kwargs):
|
|
154
|
+
pass
|
|
155
|
+
|
|
156
|
+
.. _bugtracker:
|
|
157
|
+
|
|
158
|
+
Bug tracker
|
|
159
|
+
===========
|
|
160
|
+
|
|
161
|
+
If you have any suggestions, bug reports or
|
|
162
|
+
annoyances please report them to the issue tracker
|
|
163
|
+
at https://github.com/klen/peewee_migrate/issues
|
|
164
|
+
|
|
165
|
+
.. _contributing:
|
|
166
|
+
|
|
167
|
+
Contributing
|
|
168
|
+
============
|
|
169
|
+
|
|
170
|
+
Development of starter happens at github: https://github.com/klen/peewee_migrate
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
Contributors
|
|
174
|
+
=============
|
|
175
|
+
|
|
176
|
+
* klen_ (Kirill Klenov)
|
|
177
|
+
|
|
178
|
+
.. _license:
|
|
179
|
+
|
|
180
|
+
License
|
|
181
|
+
=======
|
|
182
|
+
|
|
183
|
+
Licensed under a `BSD license`_.
|
|
184
|
+
|
|
185
|
+
.. _links:
|
|
186
|
+
|
|
187
|
+
.. _BSD license: http://www.linfo.org/bsdlicense.html
|
|
188
|
+
.. _klen: https://klen.github.io/
|
|
189
|
+
.. _Flask: http://flask.pocoo.org/
|
|
190
|
+
.. _Flask-PW: https://github.com/klen/flask-pw
|
|
191
|
+
|
|
192
|
+
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Donald-0.0.0.dist-info/DESCRIPTION.rst,sha256=2uQFIcvd6TuVJBSFezuIrWFjXPY_Vt8yklXscwgnzM0,3558
|
|
2
|
+
Donald-0.0.0.dist-info/METADATA,sha256=mmtZuUMm6fOQmU02o_rE7RLTTSIShwuYnpj8QvwnC6Y,4322
|
|
3
|
+
Donald-0.0.0.dist-info/RECORD,,
|
|
4
|
+
Donald-0.0.0.dist-info/WHEEL,sha256=5wvfB7GvgZAbKBSE9uX9Zbi6LCL-_KgezgHblXhCRnM,113
|
|
5
|
+
Donald-0.0.0.dist-info/metadata.json,sha256=bwMZEU_693JI937JGWfb3Qpv-OF4hkXKJkafBUg6qMg,943
|
|
6
|
+
Donald-0.0.0.dist-info/top_level.txt,sha256=IML3k_c4jhaPTa4bnewn5WmdlRGXCabkwLQUIFQ0QRM,7
|
|
7
|
+
donald/__init__.py,sha256=_ozYEU5AkMibcdONNmYcPq1rpBTQ590BD2SWKpNlhpc,247
|
|
8
|
+
donald/core.py,sha256=sCDtwvfADUJsAM4HGetWVAlmFkDth1HR5RjpXBLpmLQ,3540
|
|
9
|
+
donald/queue.py,sha256=WWSehfUMu8gQ3dO9MSJy5K5lmVjlANUYF6KSa5sIYKc,2470
|
|
10
|
+
donald/utils.py,sha256=zmDSEphDWbgowaXs-Or9a52fBBrAFfAQfFd9AgKG-MI,864
|
|
11
|
+
donald/worker.py,sha256=hNe4imtJslnlVO1cMqlZCUzJFenkA6p-qFSp9N6MEAI,1991
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"classifiers": ["Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Natural Language :: English", "Natural Language :: Russian", "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Testing", "Topic :: Utilities"], "extensions": {"python.details": {"contacts": [{"email": "horneds@gmail.com", "name": "Kirill Klenov", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}, "project_urls": {"Home": "https://github.com/klen/donald"}}}, "extras": [], "generator": "bdist_wheel (0.30.0.a0)", "keywords": ["asyncio", "tasks", "queue"], "license": "BSD", "metadata_version": "2.0", "name": "Donald", "platform": "Any", "run_requires": [{"requires": ["aioamqp"]}], "summary": "Donald is here", "version": "0.0.0"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
donald
|
donald/__init__.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
logger = logging.getLogger(__name__)
|
|
5
|
+
logger.addHandler(logging.StreamHandler())
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
from .core import Donald # noqa
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
__version__ = "0.0.0"
|
|
12
|
+
__project__ = "Donald"
|
|
13
|
+
__author__ = "Kirill Klenov <horneds@gmail.com>"
|
|
14
|
+
__license__ = "BSD"
|
donald/core.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Main Donald class."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import atexit
|
|
5
|
+
import datetime
|
|
6
|
+
|
|
7
|
+
from . import logger
|
|
8
|
+
from .queue import Queue
|
|
9
|
+
from .utils import AsyncMixin, AttrDict
|
|
10
|
+
from .worker import AsyncThreadWorker, call_with_loop
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Donald(AsyncMixin):
|
|
14
|
+
|
|
15
|
+
"""I'am on Donald."""
|
|
16
|
+
|
|
17
|
+
defaults = dict(
|
|
18
|
+
|
|
19
|
+
# Run tasks imediatelly
|
|
20
|
+
always_eager=False,
|
|
21
|
+
|
|
22
|
+
# Number of workers
|
|
23
|
+
num_threads=4,
|
|
24
|
+
|
|
25
|
+
# logging level
|
|
26
|
+
loglevel='INFO',
|
|
27
|
+
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
def __init__(self, loop=None, **params):
|
|
31
|
+
"""Initialize donald parameters."""
|
|
32
|
+
self.params = AttrDict(self.defaults)
|
|
33
|
+
self.params.update(params)
|
|
34
|
+
|
|
35
|
+
logger.setLevel(self.params['loglevel'].upper())
|
|
36
|
+
|
|
37
|
+
# Base loop and queue
|
|
38
|
+
self._loop = loop or asyncio.get_event_loop()
|
|
39
|
+
num_threads = 1 if self.params.num_threads < 1 else self.params.num_threads
|
|
40
|
+
self._threads = tuple(AsyncThreadWorker() for _ in range(num_threads))
|
|
41
|
+
self._tqueue = asyncio.Queue(maxsize=len(self._threads), loop=self._loop)
|
|
42
|
+
self._schedules = []
|
|
43
|
+
self._closing = False
|
|
44
|
+
|
|
45
|
+
self.queue = Queue(self, loop=self._loop)
|
|
46
|
+
self.params.always_eager = self.params.always_eager or not len(self._threads)
|
|
47
|
+
|
|
48
|
+
def start(self):
|
|
49
|
+
"""Start workers.
|
|
50
|
+
|
|
51
|
+
:returns: A coroutine
|
|
52
|
+
"""
|
|
53
|
+
logger.warn('Start Donald')
|
|
54
|
+
|
|
55
|
+
atexit.register(self.stop)
|
|
56
|
+
|
|
57
|
+
futures = []
|
|
58
|
+
for t in self._threads:
|
|
59
|
+
f = asyncio.wrap_future(t.start(), loop=self._loop)
|
|
60
|
+
f.add_done_callback(lambda f: self._tqueue.put_nowait(t))
|
|
61
|
+
futures.append(f)
|
|
62
|
+
|
|
63
|
+
self.closing = False
|
|
64
|
+
return asyncio.wait(futures)
|
|
65
|
+
|
|
66
|
+
def stop(self):
|
|
67
|
+
"""Stop workers. Disconnect from queue. Cancel schedules.
|
|
68
|
+
|
|
69
|
+
The method could be called syncroniously too.
|
|
70
|
+
|
|
71
|
+
:returns: A future
|
|
72
|
+
"""
|
|
73
|
+
if self.is_closed() or self.closing:
|
|
74
|
+
return False
|
|
75
|
+
|
|
76
|
+
logger.warn('Stop Donald.')
|
|
77
|
+
|
|
78
|
+
for task in self._schedules:
|
|
79
|
+
task.cancel()
|
|
80
|
+
|
|
81
|
+
# Stop queue
|
|
82
|
+
tasks = [asyncio.ensure_future(self.queue.stop())]
|
|
83
|
+
|
|
84
|
+
# Stop workers
|
|
85
|
+
tasks += [
|
|
86
|
+
asyncio.ensure_future(asyncio.wrap_future(t.stop(), loop=self.loop))
|
|
87
|
+
for t in self._threads]
|
|
88
|
+
|
|
89
|
+
self.closing = True
|
|
90
|
+
return asyncio.wait([self.queue.stop()])
|
|
91
|
+
|
|
92
|
+
@asyncio.coroutine
|
|
93
|
+
def submit(self, func, *args, **kwargs):
|
|
94
|
+
"""Submit given function/coroutine."""
|
|
95
|
+
if self.params.always_eager:
|
|
96
|
+
future = call_with_loop(self._loop, func, *args, **kwargs)
|
|
97
|
+
|
|
98
|
+
else:
|
|
99
|
+
worker = yield from self._tqueue.get()
|
|
100
|
+
future, waiter = worker.submit(func, *args, **kwargs)
|
|
101
|
+
waiter = asyncio.wrap_future(waiter, loop=self._loop)
|
|
102
|
+
waiter.add_done_callback(lambda f: self._tqueue.put_nowait(worker))
|
|
103
|
+
|
|
104
|
+
return (yield from asyncio.wrap_future(future, loop=self._loop))
|
|
105
|
+
|
|
106
|
+
def schedule(self, interval, func, *args, **kwargs):
|
|
107
|
+
"""Run given func/coro periodically."""
|
|
108
|
+
if isinstance(interval, datetime.timedelta):
|
|
109
|
+
interval = interval.total_seconds()
|
|
110
|
+
|
|
111
|
+
if not isinstance(interval, float):
|
|
112
|
+
interval = float(interval)
|
|
113
|
+
|
|
114
|
+
async def scheduler():
|
|
115
|
+
while self.is_running():
|
|
116
|
+
await self.submit(func, *args, **kwargs)
|
|
117
|
+
await asyncio.sleep(interval)
|
|
118
|
+
|
|
119
|
+
self._schedules.append(asyncio.ensure_future(scheduler(), loop=self._loop))
|
|
120
|
+
|
|
121
|
+
def __str__(self):
|
|
122
|
+
return 'Donald [%s]' % self.params['num_threads']
|
donald/queue.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import aioamqp
|
|
2
|
+
import asyncio
|
|
3
|
+
import pickle
|
|
4
|
+
import uuid
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
from . import logger
|
|
8
|
+
from .utils import AsyncMixin
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Queue(AsyncMixin):
|
|
12
|
+
|
|
13
|
+
def __init__(self, coro, loop=None, exchange='donald', queue='donald', **params):
|
|
14
|
+
self.params = params
|
|
15
|
+
|
|
16
|
+
self._coro = coro
|
|
17
|
+
self._loop = loop or asyncio.get_event_loop()
|
|
18
|
+
self._queue = queue
|
|
19
|
+
self._transport = None
|
|
20
|
+
self._protocol = None
|
|
21
|
+
self._channel = None
|
|
22
|
+
self._connected = False
|
|
23
|
+
|
|
24
|
+
def is_connected(self):
|
|
25
|
+
return self._connected
|
|
26
|
+
|
|
27
|
+
@asyncio.coroutine
|
|
28
|
+
def start(self, listen=True):
|
|
29
|
+
"""Connect to message queue."""
|
|
30
|
+
logger.warn('Connect to queue.')
|
|
31
|
+
try:
|
|
32
|
+
self._transport, self._protocol = yield from aioamqp.connect(**self.params)
|
|
33
|
+
self._channel = yield from self._protocol.channel()
|
|
34
|
+
|
|
35
|
+
yield from self._channel.queue_declare(queue_name=self._queue, durable=True)
|
|
36
|
+
yield from self._channel.basic_qos(
|
|
37
|
+
prefetch_count=1, prefetch_size=0, connection_global=False)
|
|
38
|
+
self._connected = True
|
|
39
|
+
if listen:
|
|
40
|
+
yield from self.listen()
|
|
41
|
+
|
|
42
|
+
except aioamqp.AmqpClosedConnection as exc:
|
|
43
|
+
logger.exception(exc)
|
|
44
|
+
logger.error('Connection is closed.')
|
|
45
|
+
|
|
46
|
+
def listen(self):
|
|
47
|
+
"""Run tasks from self.queue.
|
|
48
|
+
|
|
49
|
+
:returns: A coroutine
|
|
50
|
+
"""
|
|
51
|
+
return self._channel.basic_consume(self.callback, queue_name=self._queue)
|
|
52
|
+
|
|
53
|
+
@asyncio.coroutine
|
|
54
|
+
def stop(self):
|
|
55
|
+
if not self.is_connected() or self.is_closed():
|
|
56
|
+
return False
|
|
57
|
+
|
|
58
|
+
logger.warn('Disconnect from queue.')
|
|
59
|
+
|
|
60
|
+
yield from self._protocol.close()
|
|
61
|
+
self._transport.close()
|
|
62
|
+
self._connected = False
|
|
63
|
+
|
|
64
|
+
@asyncio.coroutine
|
|
65
|
+
def submit(self, func, *args, **kwargs):
|
|
66
|
+
logger.info('Submit task to queue.')
|
|
67
|
+
yield from self._channel.basic_publish(
|
|
68
|
+
payload=pickle.dumps((func, args, kwargs)),
|
|
69
|
+
exchange_name='', routing_key=self._queue,
|
|
70
|
+
properties=dict(delivery_mode=2, message_id=str(uuid.uuid4()))
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
@asyncio.coroutine
|
|
74
|
+
def callback(self, channel, body, envelope, properties):
|
|
75
|
+
func, args, kwargs = pickle.loads(body)
|
|
76
|
+
result = yield from self._coro.submit(func, *args, **kwargs)
|
|
77
|
+
print('Get result %r %r' % (result, properties.message_id))
|
|
78
|
+
yield from channel.basic_client_ack(delivery_tag=envelope.delivery_tag)
|
donald/utils.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from concurrent.futures import Future
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class AsyncMixin:
|
|
5
|
+
|
|
6
|
+
def is_running(self):
|
|
7
|
+
return self._loop and self._loop.is_running()
|
|
8
|
+
|
|
9
|
+
def is_closed(self):
|
|
10
|
+
return not self._loop or self._loop.is_closed()
|
|
11
|
+
|
|
12
|
+
@property
|
|
13
|
+
def loop(self):
|
|
14
|
+
return self._loop
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class AttrDict(dict):
|
|
18
|
+
|
|
19
|
+
def __init__(self, *args, **kwargs):
|
|
20
|
+
self.__dict__ = self
|
|
21
|
+
super(AttrDict, self).__init__(*args, **kwargs)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class CallableFuture(Future):
|
|
25
|
+
|
|
26
|
+
def __init__(self, func, *args, **kwargs):
|
|
27
|
+
super(CallableFuture, self).__init__()
|
|
28
|
+
self._func = func
|
|
29
|
+
self._args = args
|
|
30
|
+
self._kwargs = kwargs
|
|
31
|
+
|
|
32
|
+
def __call__(self):
|
|
33
|
+
with self._condition:
|
|
34
|
+
try:
|
|
35
|
+
self.set_result(self._func(*self._args, **self._kwargs))
|
|
36
|
+
except Exception as exc:
|
|
37
|
+
self.set_exception(exc)
|
donald/worker.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import threading
|
|
3
|
+
|
|
4
|
+
from . import logger
|
|
5
|
+
from .utils import AsyncMixin, CallableFuture, Future
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class AsyncThreadWorker(AsyncMixin, threading.Thread):
|
|
9
|
+
|
|
10
|
+
def __init__(self, *args, **kwargs):
|
|
11
|
+
super(AsyncThreadWorker, self).__init__(*args, **kwargs)
|
|
12
|
+
self._loop = None
|
|
13
|
+
|
|
14
|
+
def start(self):
|
|
15
|
+
self._future = Future()
|
|
16
|
+
super(AsyncThreadWorker, self).start()
|
|
17
|
+
return self._future
|
|
18
|
+
|
|
19
|
+
def run(self):
|
|
20
|
+
logger.info('Start loop for worker: %d' % id(self))
|
|
21
|
+
self._loop = asyncio.new_event_loop()
|
|
22
|
+
asyncio.set_event_loop(self._loop)
|
|
23
|
+
self._loop.call_soon_threadsafe(
|
|
24
|
+
asyncio.futures._set_result_unless_cancelled, self._future, True)
|
|
25
|
+
self._loop.run_forever()
|
|
26
|
+
|
|
27
|
+
def stop(self):
|
|
28
|
+
if self.is_closed():
|
|
29
|
+
return False
|
|
30
|
+
logger.info('Stop loop for worker: %d' % id(self))
|
|
31
|
+
for task in asyncio.Task.all_tasks(loop=self._loop):
|
|
32
|
+
task.cancel()
|
|
33
|
+
|
|
34
|
+
fut = CallableFuture(self._loop.stop)
|
|
35
|
+
self._loop.call_soon_threadsafe(fut)
|
|
36
|
+
return fut
|
|
37
|
+
|
|
38
|
+
def submit(self, func, *args, **kwargs):
|
|
39
|
+
"""Submit function/coroutine to current loop."""
|
|
40
|
+
if not self.is_running():
|
|
41
|
+
raise RuntimeError('Worker loop is stopped.')
|
|
42
|
+
|
|
43
|
+
logger.info('Submit task with worker: %d' % id(self))
|
|
44
|
+
job = call_with_loop(self._loop, func, *args, **kwargs)
|
|
45
|
+
waiter = self._loop.create_future()
|
|
46
|
+
self._loop.call_soon(
|
|
47
|
+
asyncio.futures._set_result_unless_cancelled, waiter, True)
|
|
48
|
+
return job, waiter
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def call_with_loop(loop, func, *args, **kwargs):
|
|
52
|
+
"""Call given coro/function with given loop."""
|
|
53
|
+
if asyncio.iscoroutine(func):
|
|
54
|
+
return asyncio.run_coroutine_threadsafe(func, loop)
|
|
55
|
+
|
|
56
|
+
if asyncio.iscoroutinefunction(func):
|
|
57
|
+
return asyncio.run_coroutine_threadsafe(func(*args, **kwargs), loop)
|
|
58
|
+
|
|
59
|
+
fut = CallableFuture(func, *args, **kwargs)
|
|
60
|
+
loop.call_soon_threadsafe(fut)
|
|
61
|
+
return fut
|