stream2py 1.0.44__tar.gz → 1.0.45__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {stream2py-1.0.44 → stream2py-1.0.45}/PKG-INFO +1 -1
  2. {stream2py-1.0.44 → stream2py-1.0.45}/pyproject.toml +1 -1
  3. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/buffer_reader.py +37 -1
  4. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/source_reader.py +26 -0
  5. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/stream_buffer.py +28 -1
  6. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/tests/test_buffer_reader_stopiteration.py +3 -1
  7. stream2py-1.0.45/stream2py/tests/test_zz_range_join.py +78 -0
  8. {stream2py-1.0.44 → stream2py-1.0.45}/.gitattributes +0 -0
  9. {stream2py-1.0.44 → stream2py-1.0.45}/.github/workflows/ci.yml +0 -0
  10. {stream2py-1.0.44 → stream2py-1.0.45}/.gitignore +0 -0
  11. {stream2py-1.0.44 → stream2py-1.0.45}/ISSUE_ANALYSIS.md +0 -0
  12. {stream2py-1.0.44 → stream2py-1.0.45}/ISSUE_COMMENTS.md +0 -0
  13. {stream2py-1.0.44 → stream2py-1.0.45}/LICENSE +0 -0
  14. {stream2py-1.0.44 → stream2py-1.0.45}/PR_DESCRIPTION.md +0 -0
  15. {stream2py-1.0.44 → stream2py-1.0.45}/README.md +0 -0
  16. {stream2py-1.0.44 → stream2py-1.0.45}/README.rst +0 -0
  17. {stream2py-1.0.44 → stream2py-1.0.45}/RESOLVED_ISSUES.md +0 -0
  18. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/.gitignore +0 -0
  19. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/Makefile +0 -0
  20. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/conf.py +0 -0
  21. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/index.rst +0 -0
  22. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/make.bat +0 -0
  23. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/buffer_reader.rst +0 -0
  24. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/protocols.rst +0 -0
  25. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/simply.rst +0 -0
  26. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/source_reader.rst +0 -0
  27. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/sources/http.rst +0 -0
  28. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/sources.rst +0 -0
  29. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/stream_buffer.rst +0 -0
  30. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/stream_source.rst +0 -0
  31. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/tests/test_buffer_reader.rst +0 -0
  32. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/tests/test_lock_sorted_deque.rst +0 -0
  33. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/tests/test_source_reader.rst +0 -0
  34. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/tests/test_stream_buffer.rst +0 -0
  35. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/tests/test_stream_source.rst +0 -0
  36. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/tests/test_util.rst +0 -0
  37. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/tests/utils_for_testing.rst +0 -0
  38. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/util.rst +0 -0
  39. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/utility/locked_sorted_deque.rst +0 -0
  40. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/utility/reader_writer_lock.rst +0 -0
  41. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/utility/sorted_collection.rst +0 -0
  42. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/utility/sorted_deque.rst +0 -0
  43. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py/utility/typing_hints.rst +0 -0
  44. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/module_docs/stream2py.rst +0 -0
  45. {stream2py-1.0.44 → stream2py-1.0.45}/docsrc/table_of_contents.rst +0 -0
  46. {stream2py-1.0.44 → stream2py-1.0.45}/pack.py +0 -0
  47. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/__init__.py +0 -0
  48. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/examples/source_reader.py +0 -0
  49. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/examples/stream_source.py +0 -0
  50. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/exceptions.py +0 -0
  51. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/protocols.py +0 -0
  52. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/simply.py +0 -0
  53. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/sources/__init__.py +0 -0
  54. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/sources/http.py +0 -0
  55. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/stream_source.py +0 -0
  56. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/tests/__init__.py +0 -0
  57. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/tests/test_buffer_reader.py +0 -0
  58. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/tests/test_buffer_reader_blocking.py +0 -0
  59. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/tests/test_lock_sorted_deque.py +0 -0
  60. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/tests/test_quick_source_reader.py +0 -0
  61. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/tests/test_reader_without_context.py +0 -0
  62. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/tests/test_source_reader.py +0 -0
  63. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/tests/test_stream_buffer.py +0 -0
  64. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/tests/test_stream_source.py +0 -0
  65. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/tests/test_util.py +0 -0
  66. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/tests/utils_for_testing.py +0 -0
  67. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/util.py +0 -0
  68. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/utility/__init__.py +0 -0
  69. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/utility/locked_sorted_deque.py +0 -0
  70. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/utility/reader_writer_lock.py +0 -0
  71. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/utility/sorted_collection.py +0 -0
  72. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/utility/sorted_deque.py +0 -0
  73. {stream2py-1.0.44 → stream2py-1.0.45}/stream2py/utility/typing_hints.py +0 -0
  74. {stream2py-1.0.44 → stream2py-1.0.45}/things_you_can_do.md +0 -0
  75. {stream2py-1.0.44 → stream2py-1.0.45}/use_cases.md +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: stream2py
3
- Version: 1.0.44
3
+ Version: 1.0.45
4
4
  Summary: Bring data streams to python with ease
5
5
  Project-URL: Homepage, https://github.com/i2mint/stream2py
6
6
  Project-URL: Documentation, https://i2mint.github.io/stream2py/
@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
6
6
 
7
7
  [project]
8
8
  name = "stream2py"
9
- version = "1.0.44"
9
+ version = "1.0.45"
10
10
  description = "Bring data streams to python with ease"
11
11
  readme = "README.md"
12
12
  requires-python = ">=3.10"
@@ -122,6 +122,8 @@ class BufferReader:
122
122
  self._buffer = buffer
123
123
  self._last_item = None
124
124
  self._last_key = None
125
+ self._join_items = None # set by the buffer from the source reader, if offered
126
+ self._slice_item = None
125
127
  self._stop_event = stop_event
126
128
  self._sleep_time_on_iter_none_s = 0.1
127
129
  self.read_size = read_size # read_size used by __next__
@@ -237,6 +239,7 @@ class BufferReader:
237
239
  only_new_items=False,
238
240
  start_le=False,
239
241
  stop_ge=False,
242
+ join=False,
240
243
  ):
241
244
  """Enables:
242
245
  1. Get last n minutes
@@ -255,8 +258,26 @@ class BufferReader:
255
258
  i.e. keys=[0, 10, 20], start=9 will include key=0
256
259
  :param stop_ge: if True, raise ValueError when there is no key greater than or equal to
257
260
  stop in buffer, if ignore_no_item_found is also True, return None instead of ValueError
258
- :return: list of items in range
261
+ :param join: if True, return ONE item instead of a list: the items in range
262
+ trimmed to exactly ``[start, stop]`` and joined, using the source reader's
263
+ ``slice_item`` and ``join_items`` (both must be implemented, otherwise
264
+ NotImplementedError is raised before anything is read). Implies
265
+ ``start_le=True`` so the chunk containing ``start`` is included, and is
266
+ incompatible with ``step``. The ``last_item`` cursor still moves to the
267
+ last *buffer* item, not the trimmed one. With ``only_new_items`` the
268
+ joined item starts after the cursor. When nothing is found the result is
269
+ the same as without ``join`` (``[]``, ``None`` or an exception).
270
+ :return: list of items in range (a single joined item if ``join``)
259
271
  """
272
+ if join:
273
+ if step is not None:
274
+ raise ValueError("range(join=True) doesn't support step")
275
+ if self._join_items is None or self._slice_item is None:
276
+ raise NotImplementedError(
277
+ "range(join=True) needs the source reader to implement both "
278
+ "join_items(items) and slice_item(item, start, stop)"
279
+ )
280
+ start_le = True
260
281
  if ignore_no_item_found is None:
261
282
  ignore_no_item_found = self._read_kwargs.get("ignore_no_item_found", False)
262
283
  with self._buffer.reader_lock() as reader:
@@ -275,6 +296,9 @@ class BufferReader:
275
296
  _start = start if start > _next_key else _next_key
276
297
  else:
277
298
  _start = start
299
+ trim_start = (
300
+ _start # where the joined item must begin, before start_le widens
301
+ )
278
302
  if start_le is True:
279
303
  with suppress(
280
304
  ValueError
@@ -299,8 +323,20 @@ class BufferReader:
299
323
  if ignore_no_item_found:
300
324
  return None
301
325
  raise e
326
+ if join and items:
327
+ return self._trim_and_join(items, trim_start, stop)
302
328
  return items
303
329
 
330
+ def _trim_and_join(self, items, start, stop):
331
+ """Trim the first and last of ``items`` to ``[start, stop]`` and join them.
332
+
333
+ Only the ends can overhang the query (the keys are ordered), and a single
334
+ item is both ends at once."""
335
+ items = list(items)
336
+ items[0] = self._slice_item(items[0], start, None)
337
+ items[-1] = self._slice_item(items[-1], None, stop)
338
+ return self._join_items(items)
339
+
304
340
  def tail(self, *, peek=False, ignore_no_item_found=False, only_new_items=False):
305
341
  """Finds the last item in buffer. Raise ValueError if no item found.
306
342
 
@@ -129,6 +129,32 @@ class SourceReader(Source, metaclass=ABCMeta):
129
129
  "Implement the 'key' method to convert data into a comparable value to sort by"
130
130
  )
131
131
 
132
+ def join_items(self, items: list) -> Any:
133
+ """(Optional) Join consecutive data items into one data item.
134
+
135
+ Implement together with ``slice_item`` to let ``BufferReader.range(...,
136
+ join=True)`` return the exact span asked for, as a single item (e.g. several
137
+ waveform chunks concatenated). Not abstract: sources whose items can't be
138
+ joined just don't override it.
139
+
140
+ :param items: non-empty list of data items, in key order
141
+ :return: one data item
142
+ """
143
+ raise NotImplementedError("Implement 'join_items' to support range(join=True)")
144
+
145
+ def slice_item(self, item: Any, start=None, stop=None) -> Any:
146
+ """(Optional) The part of ``item`` whose keys fall within ``[start, stop]``.
147
+
148
+ ``None`` for a bound means unbounded on that side; an item that needs no
149
+ trimming can be returned as is. See ``join_items``. Must be pure (items in the
150
+ buffer are shared, never mutate them) and keyed on absolute keys: it is applied
151
+ once per bound, so a lone item is sliced twice (``start`` then ``stop``).
152
+
153
+ :param item: a data item as returned by ``read``
154
+ :return: a data item of the same kind, possibly shorter
155
+ """
156
+ raise NotImplementedError("Implement 'slice_item' to support range(join=True)")
157
+
132
158
  @property
133
159
  def closed(self) -> bool:
134
160
  """True if the source reader is closed, False if it's open.
@@ -24,6 +24,21 @@ DFLT_SLEEP_TIME_ON_READ_NONE_S = 0.3
24
24
  DFLT_MAX_LEN = 10000
25
25
 
26
26
 
27
+ def _overridden(source_reader, name):
28
+ """The bound method ``name`` if the source reader really implements it, else None.
29
+
30
+ ``SourceReader`` defines ``join_items``/``slice_item`` as raising stubs, so
31
+ presence on the object doesn't mean support."""
32
+ method = getattr(source_reader, name, None)
33
+ if method is None:
34
+ return None
35
+ from stream2py.source_reader import SourceReader
36
+
37
+ if getattr(type(source_reader), name, None) is getattr(SourceReader, name, None):
38
+ return None
39
+ return method
40
+
41
+
27
42
  class _SourceBuffer:
28
43
  def __init__(
29
44
  self,
@@ -33,6 +48,8 @@ class _SourceBuffer:
33
48
  key=None,
34
49
  maxlen: int = 10000,
35
50
  buffer_reader_class: Type[BufferReader] = BufferReader,
51
+ join_items=None,
52
+ slice_item=None,
36
53
  ):
37
54
  """StreamBuffer helper class
38
55
  Thread safe buffer for reading and writing data items
@@ -46,6 +63,8 @@ class _SourceBuffer:
46
63
  self._buffer = RWLockSortedDeque([], key=key, maxlen=maxlen)
47
64
  self._source_reader_info = source_reader_info
48
65
  self.buffer_reader_class = buffer_reader_class
66
+ self._join_items = join_items
67
+ self._slice_item = slice_item
49
68
 
50
69
  def __len__(self):
51
70
  return len(self._buffer)
@@ -67,16 +86,22 @@ class _SourceBuffer:
67
86
  writer.drop(n)
68
87
 
69
88
  def mk_reader(self, **read_kwargs):
70
- return self.buffer_reader_class(
89
+ reader = self.buffer_reader_class(
71
90
  buffer=self._buffer,
72
91
  source_reader_info=self._source_reader_info,
73
92
  stop_event=self._stop_event,
74
93
  **read_kwargs,
75
94
  )
95
+ # set after construction so custom buffer_reader_classes need not know about them
96
+ reader._join_items = self._join_items
97
+ reader._slice_item = self._slice_item
98
+ return reader
76
99
 
77
100
  def attach_reader(self, reader):
78
101
  reader._buffer = self._buffer
79
102
  reader._stop_event = self._stop_event
103
+ reader._join_items = self._join_items
104
+ reader._slice_item = self._slice_item
80
105
 
81
106
 
82
107
  class StreamBuffer:
@@ -293,6 +318,8 @@ class StreamBuffer:
293
318
  buffer_reader_class=getattr(
294
319
  self.source_reader, "buffer_reader_class", BufferReader
295
320
  ),
321
+ join_items=_overridden(self.source_reader, "join_items"),
322
+ slice_item=_overridden(self.source_reader, "slice_item"),
296
323
  )
297
324
 
298
325
  # def _mk_contextualized_iterator(self):
@@ -39,7 +39,9 @@ def test_next_returns_none_when_no_data_but_running():
39
39
 
40
40
  # Read all available data quickly
41
41
  import time
42
- time.sleep(0.1) # Let some data accumulate
42
+ deadline = time.time() + 5 # wait for data rather than hope 0.1s is enough on a loaded runner
43
+ while not len(buffer.source_buffer) and time.time() < deadline:
44
+ time.sleep(0.01)
43
45
 
44
46
  # Read until we get None (no more data available yet)
45
47
  items = []
@@ -0,0 +1,78 @@
1
+ """range(join=True) (i2mint/stream2py#14).
2
+
3
+ Deliberately thread-free: the buffer and reader are driven directly, so these tests
4
+ can't compete for CPU with the timing-sensitive threaded tests around them."""
5
+ import threading
6
+
7
+ import pytest
8
+
9
+ from stream2py import SourceReader
10
+ from stream2py.stream_buffer import _SourceBuffer, _overridden
11
+
12
+
13
+ class Chunks(SourceReader):
14
+ """Items are (first_key, [values...]); value i sits at key first_key + i."""
15
+
16
+ def open(self):
17
+ pass
18
+
19
+ def read(self):
20
+ return None
21
+
22
+ def close(self):
23
+ pass
24
+
25
+ def key(self, data):
26
+ return data[0]
27
+
28
+ def slice_item(self, item, start=None, stop=None):
29
+ first, values = item
30
+ lo = 0 if start is None else max(start - first, 0)
31
+ hi = len(values) if stop is None else stop - first + 1
32
+ return (first + lo, values[lo:hi])
33
+
34
+ def join_items(self, items):
35
+ return (items[0][0], [v for _, vs in items for v in vs])
36
+
37
+
38
+ class NoJoin(Chunks):
39
+ slice_item = SourceReader.slice_item
40
+ join_items = SourceReader.join_items
41
+
42
+
43
+ def _reader(source, n=5, size=4):
44
+ sb = _SourceBuffer(
45
+ {},
46
+ threading.Event(),
47
+ key=source.key,
48
+ join_items=_overridden(source, "join_items"),
49
+ slice_item=_overridden(source, "slice_item"),
50
+ )
51
+ for i in range(n):
52
+ sb.append((i * size, list(range(i * size, (i + 1) * size))))
53
+ return sb.mk_reader()
54
+
55
+
56
+ def test_range_join_trims_and_joins_chunks():
57
+ reader = _reader(Chunks())
58
+ assert reader.range(6, 13, join=True) == (6, list(range(6, 14)))
59
+ assert reader.last_key == 12 # cursor stays on a real buffer item
60
+ assert reader.range(5, 5, join=True, peek=True) == (5, [5])
61
+ assert reader.range(0, 19, join=True) == (0, list(range(20)))
62
+ # default behaviour is unchanged: whole chunks, from the first key >= start
63
+ assert [k for k, _ in reader.range(6, 13)] == [8, 12]
64
+ with pytest.raises(ValueError):
65
+ reader.range(0, 5, step=2, join=True)
66
+
67
+
68
+ def test_range_join_only_new_items_starts_after_cursor():
69
+ reader = _reader(Chunks())
70
+ reader.range(0, 5) # cursor now on the chunk with key 4
71
+ assert reader.last_key == 4
72
+ assert reader.range(0, 13, join=True, only_new_items=True)[0] == 8
73
+
74
+
75
+ def test_range_join_needs_the_source_to_implement_it():
76
+ assert _overridden(NoJoin(), "join_items") is None
77
+ with pytest.raises(NotImplementedError):
78
+ _reader(NoJoin()).range(0, 5, join=True)
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes