deva 1.2.1__tar.gz → 1.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deva-1.2.1 → deva-1.2.3}/PKG-INFO +1 -1
- {deva-1.2.1 → deva-1.2.3}/deva/__init__.py +1 -1
- {deva-1.2.1 → deva-1.2.3}/deva/core.py +36 -33
- {deva-1.2.1 → deva-1.2.3}/deva/sample/crawler/asynchttp.py +2 -2
- {deva-1.2.1 → deva-1.2.3}/deva/sample/my_future.py +1 -1
- deva-1.2.3/deva/sample/search/cpiplog.db +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/sample/search/search_sql.py +2 -2
- {deva-1.2.1 → deva-1.2.3}/deva/sample/webview/stream_page.py +1 -1
- {deva-1.2.1 → deva-1.2.3}/deva/when.py +2 -2
- {deva-1.2.1 → deva-1.2.3}/deva.egg-info/PKG-INFO +1 -1
- {deva-1.2.1 → deva-1.2.3}/deva.egg-info/SOURCES.txt +1 -0
- {deva-1.2.1 → deva-1.2.3}/setup.py +1 -1
- {deva-1.2.1 → deva-1.2.3}/MANIFEST.in +0 -0
- {deva-1.2.1 → deva-1.2.3}/README.rst +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/bus.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/compute.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/dask.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/endpoints.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/future.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/graph.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/lambdas.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/monitor.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/namespace.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/page.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/pipe.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/river.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/sample/.DS_Store +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/sample/bus/bus_in.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/sample/bus/bus_out.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/sample/crawler/.DS_Store +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/sample/matmul.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/sample/search/.DS_Store +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/sample/search/search_stream.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/sample/when/scheduler.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/sample/when/timer.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/sample/when/worker.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/search.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/sources.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/state.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/store.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/templates/monitor.html +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/templates/stream.html +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/templates/streams.html +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/templates/xterm.css +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/templates/xterm.js +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/topic.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/utils/__init__.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/utils/simhash.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/utils/sqlitedict.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva/utils/whooshalchemy.py +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva.egg-info/dependency_links.txt +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva.egg-info/not-zip-safe +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva.egg-info/requires.txt +0 -0
- {deva-1.2.1 → deva-1.2.3}/deva.egg-info/top_level.txt +0 -0
- {deva-1.2.1 → deva-1.2.3}/setup.cfg +0 -0
|
@@ -399,27 +399,28 @@ class Stream(object):
|
|
|
399
399
|
This is typically done only at source Streams but can theortically be
|
|
400
400
|
done at any point
|
|
401
401
|
"""
|
|
402
|
-
ts_async = getattr(thread_state, 'asynchronous', False)
|
|
403
|
-
if self.loop is None or asynchronous or self.asynchronous or ts_async:
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
else:
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
402
|
+
# ts_async = getattr(thread_state, 'asynchronous', False)
|
|
403
|
+
# if self.loop is None or asynchronous or self.asynchronous or ts_async:
|
|
404
|
+
# if not ts_async:
|
|
405
|
+
# thread_state.asynchronous = True
|
|
406
|
+
# try:
|
|
407
|
+
# result = self.update(x)
|
|
408
|
+
# if self.loop:
|
|
409
|
+
# return gen.convert_yielded(result)
|
|
410
|
+
# finally:
|
|
411
|
+
# thread_state.asynchronous = ts_async
|
|
412
|
+
# else:
|
|
413
|
+
# @gen.coroutine
|
|
414
|
+
# def _():
|
|
415
|
+
# thread_state.asynchronous = True
|
|
416
|
+
# try:
|
|
417
|
+
# result = yield self.update(x)
|
|
418
|
+
# finally:
|
|
419
|
+
# del thread_state.asynchronous
|
|
420
|
+
|
|
421
|
+
# raise gen.Return(result)
|
|
422
|
+
# sync(self.loop, _)
|
|
423
|
+
self.update(x)
|
|
423
424
|
|
|
424
425
|
def update(self, x, who=None):
|
|
425
426
|
try:
|
|
@@ -544,19 +545,16 @@ class Stream(object):
|
|
|
544
545
|
from .graph import visualize
|
|
545
546
|
return visualize(self, filename, source_node=source_node, **kwargs)
|
|
546
547
|
|
|
547
|
-
def attend(self, x):
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
548
|
+
# def attend(self, x):
|
|
549
|
+
# """#async等将执行结果入流"""
|
|
550
|
+
# assert isinstance(x, gen.Awaitable)
|
|
551
|
+
# futs = gen.convert_yielded(x)
|
|
552
|
+
# self.loop = self.loop or get_io_loop()
|
|
553
|
+
# self.loop.add_future(futs, lambda x: self._emit(x.result()))
|
|
553
554
|
|
|
554
555
|
def __ror__(self, x): # |
|
|
555
556
|
"""emit value to stream ,end,return emit result"""
|
|
556
|
-
|
|
557
|
-
self.attend(x)
|
|
558
|
-
else:
|
|
559
|
-
self.emit(x, asynchronous=False)
|
|
557
|
+
self.emit(x, asynchronous=False)
|
|
560
558
|
return x
|
|
561
559
|
|
|
562
560
|
def __rrshift__(self, x): # stream左边的>>
|
|
@@ -1205,12 +1203,17 @@ class Deva():
|
|
|
1205
1203
|
# loop.make_current()
|
|
1206
1204
|
# loop.start()
|
|
1207
1205
|
|
|
1208
|
-
loop = get_io_loop(asynchronous=False)
|
|
1209
|
-
loop.instance().start()
|
|
1206
|
+
# loop = get_io_loop(asynchronous=False)
|
|
1207
|
+
# loop.instance().start()
|
|
1208
|
+
|
|
1210
1209
|
# import asyncio
|
|
1211
1210
|
# l = asyncio.new_event_loop()
|
|
1212
1211
|
|
|
1213
1212
|
# l.run_forever()
|
|
1213
|
+
try:
|
|
1214
|
+
IOLoop().start()
|
|
1215
|
+
except KeyboardInterrupt:
|
|
1216
|
+
exit()
|
|
1214
1217
|
|
|
1215
1218
|
|
|
1216
1219
|
print(os.getpid())
|
|
@@ -2,12 +2,12 @@ from deva import *
|
|
|
2
2
|
|
|
3
3
|
h = http()
|
|
4
4
|
h.map(lambda r: (r.url, r.html.search('<title>{}</title>')[0])) >> log
|
|
5
|
-
'http://
|
|
5
|
+
'http://secsay.com' >> h
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
s = Stream()
|
|
9
9
|
s.rate_limit(1).http(workers=20).map(lambda r: (
|
|
10
10
|
r.url, r.html.search('<title>{}</title>')[0])) >> warn
|
|
11
|
-
'http://
|
|
11
|
+
'http://secsay.com' >> s
|
|
12
12
|
|
|
13
13
|
Deva.run()
|
|
Binary file
|
|
@@ -150,9 +150,9 @@ class timer(Stream):
|
|
|
150
150
|
self.stop()
|
|
151
151
|
|
|
152
152
|
if self.thread:
|
|
153
|
-
self.thread_pool.submit(lambda: self.
|
|
153
|
+
self.thread_pool.submit(lambda: self.emit(self.func()))
|
|
154
154
|
else:
|
|
155
|
-
self.
|
|
155
|
+
self.emit(self.func())
|
|
156
156
|
yield gen.sleep(self.interval)
|
|
157
157
|
if self.stopped:
|
|
158
158
|
break
|
|
@@ -35,6 +35,7 @@ deva/sample/bus/bus_out.py
|
|
|
35
35
|
deva/sample/crawler/.DS_Store
|
|
36
36
|
deva/sample/crawler/asynchttp.py
|
|
37
37
|
deva/sample/search/.DS_Store
|
|
38
|
+
deva/sample/search/cpiplog.db
|
|
38
39
|
deva/sample/search/search_sql.py
|
|
39
40
|
deva/sample/search/search_stream.py
|
|
40
41
|
deva/sample/webview/stream_page.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|