quasardb 3.14.2.dev3__cp311-cp311-macosx_11_0_arm64.whl → 3.14.2.dev5__cp311-cp311-macosx_11_0_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of quasardb might be problematic. Click here for more details.
- quasardb/CMakeFiles/CMakeDirectoryInformation.cmake +2 -2
- quasardb/Makefile +20 -20
- quasardb/__init__.py +21 -7
- quasardb/cmake_install.cmake +10 -4
- quasardb/date/CMakeFiles/CMakeDirectoryInformation.cmake +2 -2
- quasardb/date/CMakeFiles/Export/a52b05f964b070ee926bcad51d3288af/dateTargets.cmake +2 -2
- quasardb/date/Makefile +20 -20
- quasardb/date/cmake_install.cmake +10 -4
- quasardb/date/dateConfigVersion.cmake +0 -0
- quasardb/date/dateTargets.cmake +2 -2
- quasardb/extensions/writer.py +59 -61
- quasardb/firehose.py +24 -22
- quasardb/libqdb_api.dylib +0 -0
- quasardb/numpy/__init__.py +181 -120
- quasardb/pandas/__init__.py +145 -95
- quasardb/pool.py +13 -2
- quasardb/pybind11/CMakeFiles/CMakeDirectoryInformation.cmake +2 -2
- quasardb/pybind11/Makefile +20 -20
- quasardb/pybind11/cmake_install.cmake +7 -1
- quasardb/quasardb.cpython-311-darwin.so +0 -0
- quasardb/range-v3/CMakeFiles/CMakeDirectoryInformation.cmake +2 -2
- quasardb/range-v3/CMakeFiles/Export/d94ef200eca10a819b5858b33e808f5b/range-v3-targets.cmake +2 -2
- quasardb/range-v3/CMakeFiles/range.v3.headers.dir/build.make +18 -15
- quasardb/range-v3/Makefile +25 -25
- quasardb/range-v3/cmake_install.cmake +13 -7
- quasardb/range-v3/range-v3-config-version.cmake +0 -0
- quasardb/range-v3/range-v3-config.cmake +2 -2
- quasardb/stats.py +92 -80
- quasardb/table_cache.py +5 -1
- {quasardb-3.14.2.dev3.dist-info → quasardb-3.14.2.dev5.dist-info}/METADATA +13 -8
- quasardb-3.14.2.dev5.dist-info/RECORD +45 -0
- {quasardb-3.14.2.dev3.dist-info → quasardb-3.14.2.dev5.dist-info}/WHEEL +1 -1
- quasardb/CMakeLists.txt +0 -517
- quasardb/batch_column.hpp +0 -80
- quasardb/batch_inserter.hpp +0 -248
- quasardb/blob.hpp +0 -150
- quasardb/cluster.cpp +0 -102
- quasardb/cluster.hpp +0 -593
- quasardb/concepts.hpp +0 -322
- quasardb/continuous.cpp +0 -199
- quasardb/continuous.hpp +0 -109
- quasardb/convert/array.hpp +0 -299
- quasardb/convert/point.hpp +0 -330
- quasardb/convert/range.hpp +0 -282
- quasardb/convert/unicode.hpp +0 -598
- quasardb/convert/util.hpp +0 -22
- quasardb/convert/value.hpp +0 -782
- quasardb/convert.hpp +0 -38
- quasardb/detail/invoke.hpp +0 -0
- quasardb/detail/qdb_resource.hpp +0 -129
- quasardb/detail/retry.cpp +0 -30
- quasardb/detail/retry.hpp +0 -147
- quasardb/detail/sleep.hpp +0 -53
- quasardb/detail/ts_column.hpp +0 -224
- quasardb/detail/writer.cpp +0 -440
- quasardb/detail/writer.hpp +0 -550
- quasardb/direct_blob.hpp +0 -108
- quasardb/direct_handle.hpp +0 -83
- quasardb/direct_integer.hpp +0 -94
- quasardb/dispatch.hpp +0 -157
- quasardb/double.hpp +0 -87
- quasardb/entry.hpp +0 -273
- quasardb/error.hpp +0 -393
- quasardb/handle.cpp +0 -29
- quasardb/handle.hpp +0 -98
- quasardb/integer.hpp +0 -88
- quasardb/logger.cpp +0 -106
- quasardb/logger.hpp +0 -228
- quasardb/masked_array.hpp +0 -658
- quasardb/metrics.cpp +0 -103
- quasardb/metrics.hpp +0 -112
- quasardb/module.cpp +0 -92
- quasardb/module.hpp +0 -24
- quasardb/node.hpp +0 -132
- quasardb/numpy.cpp +0 -6
- quasardb/numpy.hpp +0 -489
- quasardb/object_tracker.hpp +0 -282
- quasardb/options.hpp +0 -273
- quasardb/perf.hpp +0 -336
- quasardb/properties.cpp +0 -41
- quasardb/properties.hpp +0 -85
- quasardb/pytypes.hpp +0 -221
- quasardb/query.cpp +0 -420
- quasardb/query.hpp +0 -92
- quasardb/reader.cpp +0 -282
- quasardb/reader.hpp +0 -256
- quasardb/remove_cvref.hpp +0 -31
- quasardb/string.hpp +0 -160
- quasardb/table.cpp +0 -257
- quasardb/table.hpp +0 -366
- quasardb/tag.hpp +0 -77
- quasardb/timestamp.hpp +0 -97
- quasardb/traits.hpp +0 -642
- quasardb/ts_iterator.hpp +0 -193
- quasardb/utils/blob_deque.hpp +0 -96
- quasardb/utils/ostream.hpp +0 -17
- quasardb/utils/permutation.hpp +0 -50
- quasardb/utils/stable_sort.hpp +0 -25
- quasardb/utils/unzip_view.hpp +0 -89
- quasardb/utils.cpp +0 -28
- quasardb/utils.hpp +0 -174
- quasardb/writer.hpp +0 -354
- quasardb-3.14.2.dev3.dist-info/RECORD +0 -115
- {quasardb-3.14.2.dev3.dist-info → quasardb-3.14.2.dev5.dist-info}/LICENSE.md +0 -0
- {quasardb-3.14.2.dev3.dist-info → quasardb-3.14.2.dev5.dist-info}/top_level.txt +0 -0
quasardb/pandas/__init__.py
CHANGED
|
@@ -36,15 +36,18 @@ import quasardb.table_cache as table_cache
|
|
|
36
36
|
import quasardb.numpy as qdbnp
|
|
37
37
|
|
|
38
38
|
|
|
39
|
-
logger = logging.getLogger(
|
|
39
|
+
logger = logging.getLogger("quasardb.pandas")
|
|
40
|
+
|
|
40
41
|
|
|
41
42
|
class PandasRequired(ImportError):
|
|
42
43
|
"""
|
|
43
44
|
Exception raised when trying to use QuasarDB pandas integration, but
|
|
44
45
|
pandas has not been installed.
|
|
45
46
|
"""
|
|
47
|
+
|
|
46
48
|
pass
|
|
47
49
|
|
|
50
|
+
|
|
48
51
|
try:
|
|
49
52
|
import numpy as np
|
|
50
53
|
import numpy.ma as ma
|
|
@@ -53,36 +56,34 @@ try:
|
|
|
53
56
|
from pandas.core.base import PandasObject
|
|
54
57
|
|
|
55
58
|
except ImportError:
|
|
56
|
-
raise PandasRequired(
|
|
57
|
-
"The pandas library is required to handle pandas data formats")
|
|
59
|
+
raise PandasRequired("The pandas library is required to handle pandas data formats")
|
|
58
60
|
|
|
59
61
|
|
|
60
62
|
# Constant mapping of numpy dtype to QuasarDB column type
|
|
61
63
|
# TODO(leon): support this natively in qdb C api ? we have everything we need
|
|
62
64
|
# to understand dtypes.
|
|
63
65
|
_dtype_map = {
|
|
64
|
-
np.dtype(
|
|
65
|
-
np.dtype(
|
|
66
|
-
np.dtype(
|
|
67
|
-
np.dtype(
|
|
68
|
-
np.dtype(
|
|
69
|
-
np.dtype(
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
'string': quasardb.ColumnType.String,
|
|
83
|
-
'datetime64': quasardb.ColumnType.Timestamp
|
|
66
|
+
np.dtype("int64"): quasardb.ColumnType.Int64,
|
|
67
|
+
np.dtype("int32"): quasardb.ColumnType.Int64,
|
|
68
|
+
np.dtype("float64"): quasardb.ColumnType.Double,
|
|
69
|
+
np.dtype("object"): quasardb.ColumnType.String,
|
|
70
|
+
np.dtype("M8[ns]"): quasardb.ColumnType.Timestamp,
|
|
71
|
+
np.dtype("datetime64[ns]"): quasardb.ColumnType.Timestamp,
|
|
72
|
+
"int64": quasardb.ColumnType.Int64,
|
|
73
|
+
"int32": quasardb.ColumnType.Int64,
|
|
74
|
+
"float32": quasardb.ColumnType.Double,
|
|
75
|
+
"float64": quasardb.ColumnType.Double,
|
|
76
|
+
"timestamp": quasardb.ColumnType.Timestamp,
|
|
77
|
+
"string": quasardb.ColumnType.String,
|
|
78
|
+
"bytes": quasardb.ColumnType.Blob,
|
|
79
|
+
"floating": quasardb.ColumnType.Double,
|
|
80
|
+
"integer": quasardb.ColumnType.Int64,
|
|
81
|
+
"bytes": quasardb.ColumnType.Blob,
|
|
82
|
+
"string": quasardb.ColumnType.String,
|
|
83
|
+
"datetime64": quasardb.ColumnType.Timestamp,
|
|
84
84
|
}
|
|
85
85
|
|
|
86
|
+
|
|
86
87
|
def read_series(table, col_name, ranges=None):
|
|
87
88
|
"""
|
|
88
89
|
Read a Pandas Timeseries from a single column.
|
|
@@ -108,29 +109,24 @@ def read_series(table, col_name, ranges=None):
|
|
|
108
109
|
quasardb.ColumnType.Symbol: table.string_get_ranges,
|
|
109
110
|
}
|
|
110
111
|
|
|
111
|
-
kwargs = {
|
|
112
|
-
'column': col_name
|
|
113
|
-
}
|
|
112
|
+
kwargs = {"column": col_name}
|
|
114
113
|
|
|
115
114
|
if ranges is not None:
|
|
116
|
-
kwargs[
|
|
115
|
+
kwargs["ranges"] = ranges
|
|
117
116
|
|
|
118
117
|
# Dispatch based on column type
|
|
119
118
|
t = table.column_type_by_id(col_name)
|
|
120
119
|
|
|
121
|
-
logger.info(
|
|
122
|
-
|
|
120
|
+
logger.info(
|
|
121
|
+
"reading Series from column %s.%s with type %s", table.get_name(), col_name, t
|
|
122
|
+
)
|
|
123
123
|
|
|
124
124
|
res = (read_with[t])(**kwargs)
|
|
125
125
|
|
|
126
126
|
return Series(res[1], index=res[0])
|
|
127
127
|
|
|
128
128
|
|
|
129
|
-
def write_series(series,
|
|
130
|
-
table,
|
|
131
|
-
col_name,
|
|
132
|
-
infer_types=True,
|
|
133
|
-
dtype=None):
|
|
129
|
+
def write_series(series, table, col_name, infer_types=True, dtype=None):
|
|
134
130
|
"""
|
|
135
131
|
Writes a Pandas Timeseries to a single column.
|
|
136
132
|
|
|
@@ -148,34 +144,38 @@ def write_series(series,
|
|
|
148
144
|
Column name to store data in.
|
|
149
145
|
"""
|
|
150
146
|
|
|
151
|
-
logger.debug(
|
|
147
|
+
logger.debug(
|
|
148
|
+
"write_series, table=%s, col_name=%s, infer_types=%s, dtype=%s",
|
|
149
|
+
table.get_name(),
|
|
150
|
+
col_name,
|
|
151
|
+
infer_types,
|
|
152
|
+
dtype,
|
|
153
|
+
)
|
|
152
154
|
|
|
153
155
|
data = None
|
|
154
156
|
index = None
|
|
155
157
|
|
|
156
|
-
data = ma.masked_array(series.to_numpy(copy=False),
|
|
157
|
-
mask=series.isna())
|
|
158
|
+
data = ma.masked_array(series.to_numpy(copy=False), mask=series.isna())
|
|
158
159
|
|
|
159
160
|
if infer_types is True:
|
|
160
|
-
index = series.index.to_numpy(
|
|
161
|
+
index = series.index.to_numpy("datetime64[ns]", copy=False)
|
|
161
162
|
else:
|
|
162
163
|
index = series.index.to_numpy(copy=False)
|
|
163
164
|
|
|
164
165
|
assert data is not None
|
|
165
166
|
assert index is not None
|
|
166
167
|
|
|
167
|
-
return qdbnp.write_array(
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
numpy=True):
|
|
168
|
+
return qdbnp.write_array(
|
|
169
|
+
data=data,
|
|
170
|
+
index=index,
|
|
171
|
+
table=table,
|
|
172
|
+
column=col_name,
|
|
173
|
+
dtype=dtype,
|
|
174
|
+
infer_types=infer_types,
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def query(cluster: quasardb.Cluster, query, index=None, blobs=False, numpy=True):
|
|
179
179
|
"""
|
|
180
180
|
Execute a query and return the results as DataFrames. Returns a dict of
|
|
181
181
|
tablename / DataFrame pairs.
|
|
@@ -205,21 +205,33 @@ def query(cluster: quasardb.Cluster,
|
|
|
205
205
|
df.set_index(index, inplace=True)
|
|
206
206
|
return df
|
|
207
207
|
|
|
208
|
-
|
|
208
|
+
|
|
209
|
+
def stream_dataframes(
|
|
210
|
+
conn: quasardb.Cluster,
|
|
211
|
+
tables: list,
|
|
212
|
+
*,
|
|
213
|
+
batch_size: int = 2**16,
|
|
214
|
+
column_names: list = None,
|
|
215
|
+
ranges: list = None,
|
|
216
|
+
):
|
|
209
217
|
"""
|
|
210
218
|
Read a Pandas Dataframe from a QuasarDB Timeseries table. Returns a generator with dataframes of size `batch_size`, which is useful
|
|
211
219
|
when traversing a large dataset which does not fit into memory.
|
|
212
220
|
|
|
221
|
+
Accepts the same parameters as `stream_dataframes`.
|
|
222
|
+
|
|
213
223
|
Parameters:
|
|
214
224
|
-----------
|
|
215
225
|
|
|
216
|
-
|
|
217
|
-
|
|
226
|
+
conn : quasardb.Cluster
|
|
227
|
+
Connection to the QuasarDB database.
|
|
228
|
+
|
|
229
|
+
tables : list[str | quasardb.Table]
|
|
230
|
+
QuasarDB tables to stream, as a list of strings or quasardb table objects.
|
|
218
231
|
|
|
219
232
|
batch_size : int
|
|
220
|
-
The amount of rows to fetch in a single read operation. If unset,
|
|
221
|
-
|
|
222
|
-
yields on dataframe at a time.
|
|
233
|
+
The amount of rows to fetch in a single read operation. If unset, uses 2^16 (65536) rows
|
|
234
|
+
as batch size by default.
|
|
223
235
|
|
|
224
236
|
column_names : optional list
|
|
225
237
|
List of columns to read in dataframe. The timestamp column '$timestamp' is
|
|
@@ -234,44 +246,86 @@ def stream_dataframe(table : quasardb.Table, *, batch_size : int = 0, column_nam
|
|
|
234
246
|
"""
|
|
235
247
|
# Sanitize batch_size
|
|
236
248
|
if batch_size == None:
|
|
237
|
-
batch_size =
|
|
249
|
+
batch_size = 2**16
|
|
238
250
|
elif not isinstance(batch_size, int):
|
|
239
|
-
raise TypeError(
|
|
251
|
+
raise TypeError(
|
|
252
|
+
"batch_size should be an integer, but got: {} with value {}".format(
|
|
253
|
+
type(batch_size), str(batch_size)
|
|
254
|
+
)
|
|
255
|
+
)
|
|
240
256
|
|
|
241
|
-
kwargs = {}
|
|
257
|
+
kwargs = {"batch_size": batch_size}
|
|
242
258
|
|
|
243
259
|
if column_names:
|
|
244
|
-
kwargs[
|
|
260
|
+
kwargs["column_names"] = column_names
|
|
245
261
|
|
|
246
262
|
if ranges:
|
|
247
|
-
kwargs[
|
|
263
|
+
kwargs["ranges"] = ranges
|
|
264
|
+
|
|
265
|
+
coerce_table_name_fn = lambda x: x if isinstance(x, str) else x.get_name()
|
|
266
|
+
kwargs["table_names"] = [coerce_table_name_fn(x) for x in tables]
|
|
248
267
|
|
|
249
|
-
with
|
|
268
|
+
with conn.reader(**kwargs) as reader:
|
|
250
269
|
for batch in reader:
|
|
251
270
|
# We always expect the timestamp column, and set this as the index
|
|
252
|
-
assert
|
|
271
|
+
assert "$timestamp" in batch
|
|
253
272
|
|
|
254
|
-
idx = pd.Index(batch.pop(
|
|
273
|
+
idx = pd.Index(batch.pop("$timestamp"), copy=False, name="$timestamp")
|
|
255
274
|
df = pd.DataFrame(batch, index=idx)
|
|
256
275
|
|
|
257
276
|
yield df
|
|
258
277
|
|
|
259
278
|
|
|
260
|
-
def
|
|
279
|
+
def stream_dataframe(conn: quasardb.Cluster, table, **kwargs):
|
|
280
|
+
"""
|
|
281
|
+
Read a single table and return a stream of dataframes. This is a convenience function that wraps around
|
|
282
|
+
`stream_dataframes`.
|
|
283
|
+
"""
|
|
284
|
+
kwargs["tables"] = [table]
|
|
285
|
+
|
|
286
|
+
# For backwards compatibility, we drop the `$table` column returned: this is not strictly
|
|
287
|
+
# necessary, but it also is somewhat reasonable to drop it when we're reading from a single
|
|
288
|
+
# table, which is the case here.
|
|
289
|
+
clean_df_fn = lambda df: df.drop(columns=["$table"])
|
|
290
|
+
|
|
291
|
+
return (clean_df_fn(df) for df in stream_dataframes(conn, **kwargs))
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def read_dataframe(conn: quasardb.Cluster, table, **kwargs):
|
|
261
295
|
"""
|
|
262
|
-
Read a Pandas Dataframe from a QuasarDB Timeseries table. Wraps around
|
|
296
|
+
Read a Pandas Dataframe from a QuasarDB Timeseries table. Wraps around stream_dataframes(), and
|
|
263
297
|
returns everything as a single dataframe. batch_size is always explicitly set to 0.
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
Parameters:
|
|
301
|
+
-----------
|
|
302
|
+
|
|
303
|
+
conn : quasardb.Cluster
|
|
304
|
+
Connection to the QuasarDB database.
|
|
305
|
+
|
|
306
|
+
table : str | quasardb.Table
|
|
307
|
+
QuasarDB table to stream, either as a string or a table object. When re-executing the same function
|
|
308
|
+
multiple times on the same tables, providing the table as an object has a performance benefit.
|
|
309
|
+
|
|
264
310
|
"""
|
|
265
311
|
|
|
266
|
-
if
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
kwargs[
|
|
312
|
+
if (
|
|
313
|
+
"batch_size" in kwargs
|
|
314
|
+
and kwargs["batch_size"] != 0
|
|
315
|
+
and kwargs["batch_size"] != None
|
|
316
|
+
):
|
|
317
|
+
logger.warn(
|
|
318
|
+
"Providing a batch size with read_dataframe is unsupported, overriding batch_size to 65536."
|
|
319
|
+
)
|
|
320
|
+
logger.warn(
|
|
321
|
+
"If you wish to traverse the data in smaller batches, please use: stream_dataframe()."
|
|
322
|
+
)
|
|
323
|
+
kwargs["batch_size"] = 2**16
|
|
270
324
|
|
|
271
325
|
# Note that this is *lazy*, dfs is a generator, not a list -- as such, dataframes will be
|
|
272
326
|
# fetched on-demand, which means that an error could occur in the middle of processing
|
|
273
327
|
# dataframes.
|
|
274
|
-
dfs = stream_dataframe(table, **kwargs)
|
|
328
|
+
dfs = stream_dataframe(conn, table, **kwargs)
|
|
275
329
|
|
|
276
330
|
return pd.concat(dfs)
|
|
277
331
|
|
|
@@ -294,18 +348,12 @@ def _extract_columns(df, cinfos):
|
|
|
294
348
|
|
|
295
349
|
if cname in df.columns:
|
|
296
350
|
arr = df[cname].array
|
|
297
|
-
ret[cname] = ma.masked_array(arr.to_numpy(copy=False),
|
|
298
|
-
mask=arr.isna())
|
|
351
|
+
ret[cname] = ma.masked_array(arr.to_numpy(copy=False), mask=arr.isna())
|
|
299
352
|
|
|
300
353
|
return ret
|
|
301
354
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
cluster,
|
|
305
|
-
*,
|
|
306
|
-
create = False,
|
|
307
|
-
shard_size = None,
|
|
308
|
-
**kwargs):
|
|
355
|
+
|
|
356
|
+
def write_dataframes(dfs, cluster, *, create=False, shard_size=None, **kwargs):
|
|
309
357
|
"""
|
|
310
358
|
Store dataframes into a table. Any additional parameters not documented here
|
|
311
359
|
are passed to numpy.write_arrays(). Please consult the pydoc of that function
|
|
@@ -337,7 +385,7 @@ def write_dataframes(
|
|
|
337
385
|
|
|
338
386
|
# If the tables are provided as strings, we look them up.
|
|
339
387
|
dfs_ = []
|
|
340
|
-
for
|
|
388
|
+
for table, df in dfs:
|
|
341
389
|
if isinstance(table, str):
|
|
342
390
|
table = table_cache.lookup(table, cluster)
|
|
343
391
|
|
|
@@ -356,7 +404,9 @@ def write_dataframes(
|
|
|
356
404
|
cinfos = [(x.name, x.type) for x in table.list_columns()]
|
|
357
405
|
|
|
358
406
|
if not df.index.is_monotonic_increasing:
|
|
359
|
-
logger.warn(
|
|
407
|
+
logger.warn(
|
|
408
|
+
"dataframe index is unsorted, resorting dataframe based on index"
|
|
409
|
+
)
|
|
360
410
|
df = df.sort_index().reindex()
|
|
361
411
|
|
|
362
412
|
# We pass everything else to our qdbnp.write_arrays function, as generally speaking
|
|
@@ -365,21 +415,14 @@ def write_dataframes(
|
|
|
365
415
|
# is sparse, most notably forcing sparse integer arrays to floating points.
|
|
366
416
|
|
|
367
417
|
data = _extract_columns(df, cinfos)
|
|
368
|
-
data[
|
|
369
|
-
dtype='datetime64[ns]')
|
|
418
|
+
data["$timestamp"] = df.index.to_numpy(copy=False, dtype="datetime64[ns]")
|
|
370
419
|
|
|
371
420
|
data_by_table.append((table, data))
|
|
372
421
|
|
|
373
|
-
return qdbnp.write_arrays(data_by_table, cluster,
|
|
374
|
-
|
|
375
|
-
index=None,
|
|
376
|
-
**kwargs)
|
|
422
|
+
return qdbnp.write_arrays(data_by_table, cluster, table=None, index=None, **kwargs)
|
|
423
|
+
|
|
377
424
|
|
|
378
|
-
def write_dataframe(
|
|
379
|
-
df,
|
|
380
|
-
cluster,
|
|
381
|
-
table,
|
|
382
|
-
**kwargs):
|
|
425
|
+
def write_dataframe(df, cluster, table, **kwargs):
|
|
383
426
|
"""
|
|
384
427
|
Store a single dataframe into a table. Takes the same arguments as `write_dataframes`, except only
|
|
385
428
|
a single df/table combination.
|
|
@@ -391,7 +434,9 @@ def write_pinned_dataframe(*args, **kwargs):
|
|
|
391
434
|
"""
|
|
392
435
|
Legacy wrapper around write_dataframe()
|
|
393
436
|
"""
|
|
394
|
-
logger.warn(
|
|
437
|
+
logger.warn(
|
|
438
|
+
"write_pinned_dataframe is deprecated and will be removed in a future release."
|
|
439
|
+
)
|
|
395
440
|
logger.warn("Please use write_dataframe directly instead")
|
|
396
441
|
return write_dataframe(*args, **kwargs)
|
|
397
442
|
|
|
@@ -405,7 +450,12 @@ def _create_table_from_df(df, table, shard_size=None):
|
|
|
405
450
|
for c in df.columns:
|
|
406
451
|
dt = dtypes[c]
|
|
407
452
|
ct = _dtype_to_column_type(df[c].dtype, dt)
|
|
408
|
-
logger.debug(
|
|
453
|
+
logger.debug(
|
|
454
|
+
"probed pandas dtype %s to inferred dtype %s and map to quasardb column type %s",
|
|
455
|
+
df[c].dtype,
|
|
456
|
+
dt,
|
|
457
|
+
ct,
|
|
458
|
+
)
|
|
409
459
|
cols.append(quasardb.ColumnInfo(ct, c))
|
|
410
460
|
|
|
411
461
|
try:
|
quasardb/pool.py
CHANGED
|
@@ -4,11 +4,13 @@ import threading
|
|
|
4
4
|
import functools
|
|
5
5
|
import weakref
|
|
6
6
|
|
|
7
|
-
logger = logging.getLogger(
|
|
7
|
+
logger = logging.getLogger("quasardb.pool")
|
|
8
|
+
|
|
8
9
|
|
|
9
10
|
def _create_conn(**kwargs):
|
|
10
11
|
return quasardb.Cluster(**kwargs)
|
|
11
12
|
|
|
13
|
+
|
|
12
14
|
class SessionWrapper(object):
|
|
13
15
|
def __init__(self, pool, conn):
|
|
14
16
|
self._conn = conn
|
|
@@ -45,6 +47,7 @@ class SessionWrapper(object):
|
|
|
45
47
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
46
48
|
self._pool.release(self._conn)
|
|
47
49
|
|
|
50
|
+
|
|
48
51
|
class Pool(object):
|
|
49
52
|
"""
|
|
50
53
|
A connection pool. This class should not be initialized directly, but
|
|
@@ -135,6 +138,7 @@ class Pool(object):
|
|
|
135
138
|
logger.info("Putting connection back onto pool")
|
|
136
139
|
return self._do_release(conn)
|
|
137
140
|
|
|
141
|
+
|
|
138
142
|
class SingletonPool(Pool):
|
|
139
143
|
"""
|
|
140
144
|
Implementation of our connection pool that maintains just a single connection
|
|
@@ -190,6 +194,7 @@ class SingletonPool(Pool):
|
|
|
190
194
|
|
|
191
195
|
__instance = None
|
|
192
196
|
|
|
197
|
+
|
|
193
198
|
def initialize(*args, **kwargs):
|
|
194
199
|
"""
|
|
195
200
|
Initialize a new global SingletonPool. Forwards all arguments to the constructor of
|
|
@@ -215,6 +220,7 @@ def initialize(*args, **kwargs):
|
|
|
215
220
|
global __instance
|
|
216
221
|
__instance = SingletonPool(*args, **kwargs)
|
|
217
222
|
|
|
223
|
+
|
|
218
224
|
def instance() -> SingletonPool:
|
|
219
225
|
"""
|
|
220
226
|
Singleton accessor. Instance must have been initialized using initialize()
|
|
@@ -226,9 +232,12 @@ def instance() -> SingletonPool:
|
|
|
226
232
|
|
|
227
233
|
"""
|
|
228
234
|
global __instance
|
|
229
|
-
assert
|
|
235
|
+
assert (
|
|
236
|
+
__instance is not None
|
|
237
|
+
), "Global connection pool uninitialized: please initialize by calling the initialize() function."
|
|
230
238
|
return __instance
|
|
231
239
|
|
|
240
|
+
|
|
232
241
|
def _inject_conn_arg(conn, arg, args, kwargs):
|
|
233
242
|
"""
|
|
234
243
|
Decorator utility function. Takes the argument provided to the decorator
|
|
@@ -251,6 +260,7 @@ def _inject_conn_arg(conn, arg, args, kwargs):
|
|
|
251
260
|
|
|
252
261
|
return (args, kwargs)
|
|
253
262
|
|
|
263
|
+
|
|
254
264
|
def with_conn(_fn=None, *, arg=0):
|
|
255
265
|
"""
|
|
256
266
|
Decorator function that handles connection assignment, release and invocation for you.
|
|
@@ -278,6 +288,7 @@ def with_conn(_fn=None, *, arg=0):
|
|
|
278
288
|
conn.query(...)
|
|
279
289
|
```
|
|
280
290
|
"""
|
|
291
|
+
|
|
281
292
|
def inner(fn):
|
|
282
293
|
def wrapper(*args, **kwargs):
|
|
283
294
|
pool = instance()
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# CMAKE generated file: DO NOT EDIT!
|
|
2
|
-
# Generated by "Unix Makefiles" Generator, CMake Version 3.
|
|
2
|
+
# Generated by "Unix Makefiles" Generator, CMake Version 3.31
|
|
3
3
|
|
|
4
4
|
# Relative path conversion top directories.
|
|
5
5
|
set(CMAKE_RELATIVE_PATH_TOP_SOURCE "/Users/teamcity/buildAgent/work/938b0bdf6727d1ad/thirdparty")
|
|
6
|
-
set(CMAKE_RELATIVE_PATH_TOP_BINARY "/Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/lib.macosx-
|
|
6
|
+
set(CMAKE_RELATIVE_PATH_TOP_BINARY "/Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/lib.macosx-15.0-arm64-cpython-311/quasardb")
|
|
7
7
|
|
|
8
8
|
# Force unix paths in dependencies.
|
|
9
9
|
set(CMAKE_FORCE_UNIX_PATHS 1)
|
quasardb/pybind11/Makefile
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# CMAKE generated file: DO NOT EDIT!
|
|
2
|
-
# Generated by "Unix Makefiles" Generator, CMake Version 3.
|
|
2
|
+
# Generated by "Unix Makefiles" Generator, CMake Version 3.31
|
|
3
3
|
|
|
4
4
|
# Default target executed when no arguments are given to make.
|
|
5
5
|
default_target: all
|
|
@@ -48,10 +48,10 @@ cmake_force:
|
|
|
48
48
|
SHELL = /bin/sh
|
|
49
49
|
|
|
50
50
|
# The CMake executable.
|
|
51
|
-
CMAKE_COMMAND = /
|
|
51
|
+
CMAKE_COMMAND = /Users/teamcity/buildAgent/temp/buildTmp/build-env-k6rf7kt7/lib/python3.11/site-packages/cmake/data/bin/cmake
|
|
52
52
|
|
|
53
53
|
# The command to remove a file.
|
|
54
|
-
RM = /
|
|
54
|
+
RM = /Users/teamcity/buildAgent/temp/buildTmp/build-env-k6rf7kt7/lib/python3.11/site-packages/cmake/data/bin/cmake -E rm -f
|
|
55
55
|
|
|
56
56
|
# Escaping for special characters.
|
|
57
57
|
EQUALS = =
|
|
@@ -60,7 +60,7 @@ EQUALS = =
|
|
|
60
60
|
CMAKE_SOURCE_DIR = /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/quasardb
|
|
61
61
|
|
|
62
62
|
# The top-level build directory on which CMake was run.
|
|
63
|
-
CMAKE_BINARY_DIR = /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-
|
|
63
|
+
CMAKE_BINARY_DIR = /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-15.0-arm64-cpython-311
|
|
64
64
|
|
|
65
65
|
#=============================================================================
|
|
66
66
|
# Targets provided globally by CMake.
|
|
@@ -68,7 +68,7 @@ CMAKE_BINARY_DIR = /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.m
|
|
|
68
68
|
# Special rule for the target edit_cache
|
|
69
69
|
edit_cache:
|
|
70
70
|
@$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Running CMake cache editor..."
|
|
71
|
-
/
|
|
71
|
+
/Users/teamcity/buildAgent/temp/buildTmp/build-env-k6rf7kt7/lib/python3.11/site-packages/cmake/data/bin/ccmake -S$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR)
|
|
72
72
|
.PHONY : edit_cache
|
|
73
73
|
|
|
74
74
|
# Special rule for the target edit_cache
|
|
@@ -78,7 +78,7 @@ edit_cache/fast: edit_cache
|
|
|
78
78
|
# Special rule for the target rebuild_cache
|
|
79
79
|
rebuild_cache:
|
|
80
80
|
@$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Running CMake to regenerate build system..."
|
|
81
|
-
/
|
|
81
|
+
/Users/teamcity/buildAgent/temp/buildTmp/build-env-k6rf7kt7/lib/python3.11/site-packages/cmake/data/bin/cmake --regenerate-during-build -S$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR)
|
|
82
82
|
.PHONY : rebuild_cache
|
|
83
83
|
|
|
84
84
|
# Special rule for the target rebuild_cache
|
|
@@ -97,49 +97,49 @@ list_install_components/fast: list_install_components
|
|
|
97
97
|
# Special rule for the target install
|
|
98
98
|
install: preinstall
|
|
99
99
|
@$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Install the project..."
|
|
100
|
-
/
|
|
100
|
+
/Users/teamcity/buildAgent/temp/buildTmp/build-env-k6rf7kt7/lib/python3.11/site-packages/cmake/data/bin/cmake -P cmake_install.cmake
|
|
101
101
|
.PHONY : install
|
|
102
102
|
|
|
103
103
|
# Special rule for the target install
|
|
104
104
|
install/fast: preinstall/fast
|
|
105
105
|
@$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Install the project..."
|
|
106
|
-
/
|
|
106
|
+
/Users/teamcity/buildAgent/temp/buildTmp/build-env-k6rf7kt7/lib/python3.11/site-packages/cmake/data/bin/cmake -P cmake_install.cmake
|
|
107
107
|
.PHONY : install/fast
|
|
108
108
|
|
|
109
109
|
# Special rule for the target install/local
|
|
110
110
|
install/local: preinstall
|
|
111
111
|
@$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Installing only the local directory..."
|
|
112
|
-
/
|
|
112
|
+
/Users/teamcity/buildAgent/temp/buildTmp/build-env-k6rf7kt7/lib/python3.11/site-packages/cmake/data/bin/cmake -DCMAKE_INSTALL_LOCAL_ONLY=1 -P cmake_install.cmake
|
|
113
113
|
.PHONY : install/local
|
|
114
114
|
|
|
115
115
|
# Special rule for the target install/local
|
|
116
116
|
install/local/fast: preinstall/fast
|
|
117
117
|
@$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Installing only the local directory..."
|
|
118
|
-
/
|
|
118
|
+
/Users/teamcity/buildAgent/temp/buildTmp/build-env-k6rf7kt7/lib/python3.11/site-packages/cmake/data/bin/cmake -DCMAKE_INSTALL_LOCAL_ONLY=1 -P cmake_install.cmake
|
|
119
119
|
.PHONY : install/local/fast
|
|
120
120
|
|
|
121
121
|
# Special rule for the target install/strip
|
|
122
122
|
install/strip: preinstall
|
|
123
123
|
@$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Installing the project stripped..."
|
|
124
|
-
/
|
|
124
|
+
/Users/teamcity/buildAgent/temp/buildTmp/build-env-k6rf7kt7/lib/python3.11/site-packages/cmake/data/bin/cmake -DCMAKE_INSTALL_DO_STRIP=1 -P cmake_install.cmake
|
|
125
125
|
.PHONY : install/strip
|
|
126
126
|
|
|
127
127
|
# Special rule for the target install/strip
|
|
128
128
|
install/strip/fast: preinstall/fast
|
|
129
129
|
@$(CMAKE_COMMAND) -E cmake_echo_color "--switch=$(COLOR)" --cyan "Installing the project stripped..."
|
|
130
|
-
/
|
|
130
|
+
/Users/teamcity/buildAgent/temp/buildTmp/build-env-k6rf7kt7/lib/python3.11/site-packages/cmake/data/bin/cmake -DCMAKE_INSTALL_DO_STRIP=1 -P cmake_install.cmake
|
|
131
131
|
.PHONY : install/strip/fast
|
|
132
132
|
|
|
133
133
|
# The main all target
|
|
134
134
|
all: cmake_check_build_system
|
|
135
|
-
cd /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-
|
|
136
|
-
cd /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-
|
|
137
|
-
$(CMAKE_COMMAND) -E cmake_progress_start /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-
|
|
135
|
+
cd /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-15.0-arm64-cpython-311 && $(CMAKE_COMMAND) -E cmake_progress_start /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-15.0-arm64-cpython-311/CMakeFiles /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/lib.macosx-15.0-arm64-cpython-311/quasardb/pybind11//CMakeFiles/progress.marks
|
|
136
|
+
cd /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-15.0-arm64-cpython-311 && $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/lib.macosx-15.0-arm64-cpython-311/quasardb/pybind11/all
|
|
137
|
+
$(CMAKE_COMMAND) -E cmake_progress_start /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-15.0-arm64-cpython-311/CMakeFiles 0
|
|
138
138
|
.PHONY : all
|
|
139
139
|
|
|
140
140
|
# The main clean target
|
|
141
141
|
clean:
|
|
142
|
-
cd /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-
|
|
142
|
+
cd /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-15.0-arm64-cpython-311 && $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/lib.macosx-15.0-arm64-cpython-311/quasardb/pybind11/clean
|
|
143
143
|
.PHONY : clean
|
|
144
144
|
|
|
145
145
|
# The main clean target
|
|
@@ -148,17 +148,17 @@ clean/fast: clean
|
|
|
148
148
|
|
|
149
149
|
# Prepare targets for installation.
|
|
150
150
|
preinstall: all
|
|
151
|
-
cd /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-
|
|
151
|
+
cd /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-15.0-arm64-cpython-311 && $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/lib.macosx-15.0-arm64-cpython-311/quasardb/pybind11/preinstall
|
|
152
152
|
.PHONY : preinstall
|
|
153
153
|
|
|
154
154
|
# Prepare targets for installation.
|
|
155
155
|
preinstall/fast:
|
|
156
|
-
cd /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-
|
|
156
|
+
cd /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-15.0-arm64-cpython-311 && $(MAKE) $(MAKESILENT) -f CMakeFiles/Makefile2 /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/lib.macosx-15.0-arm64-cpython-311/quasardb/pybind11/preinstall
|
|
157
157
|
.PHONY : preinstall/fast
|
|
158
158
|
|
|
159
159
|
# clear depends
|
|
160
160
|
depend:
|
|
161
|
-
cd /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-
|
|
161
|
+
cd /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-15.0-arm64-cpython-311 && $(CMAKE_COMMAND) -S$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 1
|
|
162
162
|
.PHONY : depend
|
|
163
163
|
|
|
164
164
|
# Help Target
|
|
@@ -184,6 +184,6 @@ help:
|
|
|
184
184
|
# No rule that depends on this can have commands that come from listfiles
|
|
185
185
|
# because they might be regenerated.
|
|
186
186
|
cmake_check_build_system:
|
|
187
|
-
cd /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-
|
|
187
|
+
cd /Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/temp.macosx-15.0-arm64-cpython-311 && $(CMAKE_COMMAND) -S$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 0
|
|
188
188
|
.PHONY : cmake_check_build_system
|
|
189
189
|
|
|
@@ -32,8 +32,14 @@ if(NOT DEFINED CMAKE_CROSSCOMPILING)
|
|
|
32
32
|
set(CMAKE_CROSSCOMPILING "FALSE")
|
|
33
33
|
endif()
|
|
34
34
|
|
|
35
|
-
# Set
|
|
35
|
+
# Set path to fallback-tool for dependency-resolution.
|
|
36
36
|
if(NOT DEFINED CMAKE_OBJDUMP)
|
|
37
37
|
set(CMAKE_OBJDUMP "/usr/local/clang16/bin/llvm-objdump")
|
|
38
38
|
endif()
|
|
39
39
|
|
|
40
|
+
string(REPLACE ";" "\n" CMAKE_INSTALL_MANIFEST_CONTENT
|
|
41
|
+
"${CMAKE_INSTALL_MANIFEST_FILES}")
|
|
42
|
+
if(CMAKE_INSTALL_LOCAL_ONLY)
|
|
43
|
+
file(WRITE "/Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/lib.macosx-15.0-arm64-cpython-311/quasardb/pybind11/install_local_manifest.txt"
|
|
44
|
+
"${CMAKE_INSTALL_MANIFEST_CONTENT}")
|
|
45
|
+
endif()
|
|
Binary file
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# CMAKE generated file: DO NOT EDIT!
|
|
2
|
-
# Generated by "Unix Makefiles" Generator, CMake Version 3.
|
|
2
|
+
# Generated by "Unix Makefiles" Generator, CMake Version 3.31
|
|
3
3
|
|
|
4
4
|
# Relative path conversion top directories.
|
|
5
5
|
set(CMAKE_RELATIVE_PATH_TOP_SOURCE "/Users/teamcity/buildAgent/work/938b0bdf6727d1ad/thirdparty")
|
|
6
|
-
set(CMAKE_RELATIVE_PATH_TOP_BINARY "/Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/lib.macosx-
|
|
6
|
+
set(CMAKE_RELATIVE_PATH_TOP_BINARY "/Users/teamcity/buildAgent/work/938b0bdf6727d1ad/build/lib.macosx-15.0-arm64-cpython-311/quasardb")
|
|
7
7
|
|
|
8
8
|
# Force unix paths in dependencies.
|
|
9
9
|
set(CMAKE_FORCE_UNIX_PATHS 1)
|