datachain 0.16.1__py3-none-any.whl → 0.16.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of datachain might be problematic. Click here for more details.

Files changed (7) hide show
  1. datachain/lib/dc/records.py +4 -2
  2. {datachain-0.16.1.dist-info → datachain-0.16.2.dist-info}/METADATA +1 -1
  3. {datachain-0.16.1.dist-info → datachain-0.16.2.dist-info}/RECORD +7 -7
  4. {datachain-0.16.1.dist-info → datachain-0.16.2.dist-info}/WHEEL +0 -0
  5. {datachain-0.16.1.dist-info → datachain-0.16.2.dist-info}/entry_points.txt +0 -0
  6. {datachain-0.16.1.dist-info → datachain-0.16.2.dist-info}/licenses/LICENSE +0 -0
  7. {datachain-0.16.1.dist-info → datachain-0.16.2.dist-info}/top_level.txt +0 -0
datachain/lib/dc/records.py CHANGED
@@ -38,8 +38,9 @@ def read_records(
38
38
  single_record = dc.read_records(dc.DEFAULT_FILE_RECORD)
39
39
  ```
40
40
  """
41
- from datachain.query.dataset import adjust_outputs, get_col_types
41
+ from datachain.query.dataset import INSERT_BATCH_SIZE, adjust_outputs, get_col_types
42
42
  from datachain.sql.types import SQLType
43
+ from datachain.utils import batched
43
44
 
44
45
  from .datasets import read_dataset
45
46
 
@@ -89,6 +90,7 @@ def read_records(
89
90
  {c.name: c.type for c in columns if isinstance(c.type, SQLType)},
90
91
  )
91
92
  records = (adjust_outputs(warehouse, record, col_types) for record in to_insert)
92
- warehouse.insert_rows(table, records)
93
+ for chunk in batched(records, INSERT_BATCH_SIZE):
94
+ warehouse.insert_rows(table, chunk)
93
95
  warehouse.insert_rows_done(table)
94
96
  return read_dataset(name=dsr.name, session=session, settings=settings)
{datachain-0.16.1.dist-info → datachain-0.16.2.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datachain
3
- Version: 0.16.1
3
+ Version: 0.16.2
4
4
  Summary: Wrangle unstructured AI data at scale
5
5
  Author-email: Dmitry Petrov <support@dvc.org>
6
6
  License-Expression: Apache-2.0
{datachain-0.16.1.dist-info → datachain-0.16.2.dist-info}/RECORD RENAMED
@@ -104,7 +104,7 @@ datachain/lib/dc/json.py,sha256=ZUThPDAaP2gBFIL5vsQTwKBcuN_dhvC_O44wdDv0jEc,2683
104
104
  datachain/lib/dc/listings.py,sha256=2na9v63xO1vPUNaoBSzA-TSN49V7zQAb-4iS1wOPLFE,1029
105
105
  datachain/lib/dc/pandas.py,sha256=ObueUXDUFKJGu380GmazdG02ARpKAHPhSaymfmOH13E,1489
106
106
  datachain/lib/dc/parquet.py,sha256=zYcSgrWwyEDW9UxGUSVdIVsCu15IGEf0xL8KfWQqK94,1782
107
- datachain/lib/dc/records.py,sha256=Z6EWy6c6hf87cWiDlQduvrDgOHMLwqF22g-XksOnXsU,2884
107
+ datachain/lib/dc/records.py,sha256=NKlKidKVwtUYa_a8oIoWjZrcUfPDvsWc-eDw5chkW8c,2999
108
108
  datachain/lib/dc/storage.py,sha256=QLf3-xMV2Gmy3AA8qF9WqAsb7R8Rk87l4s5hBoiCH98,5285
109
109
  datachain/lib/dc/utils.py,sha256=VawOAlJSvAtZbsMg33s5tJe21TRx1Km3QggI1nN6tnw,3984
110
110
  datachain/lib/dc/values.py,sha256=cBQubhmPNEDMJldUXzGh-UKbdim4P6O2B91Gp39roKw,1389
@@ -151,9 +151,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
151
151
  datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
152
152
  datachain/toolkit/split.py,sha256=ktGWzY4kyzjWyR86dhvzw-Zhl0lVk_LOX3NciTac6qo,2914
153
153
  datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
154
- datachain-0.16.1.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
155
- datachain-0.16.1.dist-info/METADATA,sha256=9YPqP6Sthuf_fuxFX3miQyp9MEjRq8j2DqubLXvZg0k,11328
156
- datachain-0.16.1.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
157
- datachain-0.16.1.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
158
- datachain-0.16.1.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
159
- datachain-0.16.1.dist-info/RECORD,,
154
+ datachain-0.16.2.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
155
+ datachain-0.16.2.dist-info/METADATA,sha256=K77kCEjBd5SdIiWy_jKbMHCgMuZZlSRjFj8Ba1zUsCw,11328
156
+ datachain-0.16.2.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
157
+ datachain-0.16.2.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
158
+ datachain-0.16.2.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
159
+ datachain-0.16.2.dist-info/RECORD,,