datachain 0.18.10__py3-none-any.whl → 0.18.11__py3-none-any.whl

datachain/data_storage/sqlite.py

@@ -109,12 +109,14 @@ class SQLiteDatabaseEngine(DatabaseEngine):
         metadata: "MetaData",
         db: sqlite3.Connection,
         db_file: Optional[str] = None,
+        max_variable_number: Optional[int] = 999,
     ):
         self.engine = engine
         self.metadata = metadata
         self.db = db
         self.db_file = db_file
         self.is_closed = False
+        self.max_variable_number = max_variable_number
 
     @classmethod
     def from_db_file(cls, db_file: Optional[str] = None) -> "SQLiteDatabaseEngine":
@@ -123,7 +125,7 @@ class SQLiteDatabaseEngine(DatabaseEngine):
     @staticmethod
     def _connect(
         db_file: Optional[str] = None,
-    ) -> tuple["Engine", "MetaData", sqlite3.Connection, str]:
+    ) -> tuple["Engine", "MetaData", sqlite3.Connection, str, int]:
         try:
             if db_file == ":memory:":
                 # Enable multithreaded usage of the same in-memory db
@@ -150,6 +152,13 @@ class SQLiteDatabaseEngine(DatabaseEngine):
             db.execute("PRAGMA journal_mode = WAL")
             db.execute("PRAGMA synchronous = NORMAL")
             db.execute("PRAGMA case_sensitive_like = ON")
+
+            max_variable_number = 999  # minimum in old SQLite versions
+            for row in db.execute("PRAGMA compile_options;").fetchall():
+                option = row[0]
+                if option.startswith("MAX_VARIABLE_NUMBER="):
+                    max_variable_number = int(option.split("=")[1])
+
             if os.environ.get("DEBUG_SHOW_SQL_QUERIES"):
                 import sys
 
@@ -157,7 +166,7 @@ class SQLiteDatabaseEngine(DatabaseEngine):
 
             load_usearch_extension(db)
 
-            return engine, MetaData(), db, db_file
+            return engine, MetaData(), db, db_file, max_variable_number
         except RuntimeError:
             raise DataChainError("Can't connect to SQLite DB") from None
 
@@ -180,11 +189,14 @@ class SQLiteDatabaseEngine(DatabaseEngine):
     def _reconnect(self) -> None:
         if not self.is_closed:
             raise RuntimeError("Cannot reconnect on still-open DB!")
-        engine, metadata, db, db_file = self._connect(db_file=self.db_file)
+        engine, metadata, db, db_file, max_variable_number = self._connect(
+            db_file=self.db_file
+        )
         self.engine = engine
         self.metadata = metadata
         self.db = db
         self.db_file = db_file
+        self.max_variable_number = max_variable_number
         self.is_closed = False
 
     def get_table(self, name: str) -> Table:
@@ -231,13 +243,27 @@ class SQLiteDatabaseEngine(DatabaseEngine):
         return self.db.execute(sql, parameters)
 
     def insert_dataframe(self, table_name: str, df) -> int:
+        # Dynamically calculates chunksize by dividing max variable limit in a
+        # single SQL insert with number of columns in dataframe.
+        # This way we avoid error: sqlite3.OperationalError: too many SQL variables,
+        num_columns = df.shape[1]
+        if num_columns == 0:
+            num_columns = 1
+
+        if self.max_variable_number < num_columns:
+            raise RuntimeError(
+                "Number of columns exceeds DB maximum variables when inserting data"
+            )
+
+        chunksize = self.max_variable_number // num_columns
+
         return df.to_sql(
             table_name,
             self.db,
             if_exists="append",
             index=False,
             method="multi",
-            chunksize=1000,
+            chunksize=chunksize,
         )
 
     def cursor(self, factory=None):

{datachain-0.18.10.dist-info → datachain-0.18.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.18.10
+Version: 0.18.11
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.18.10.dist-info → datachain-0.18.11.dist-info}/RECORD

@@ -50,7 +50,7 @@ datachain/data_storage/job.py,sha256=9r0OGwh22bHNIvLHqg8_-eJSP1YYB-BN5HOla5TdCxw
 datachain/data_storage/metastore.py,sha256=1PaRTQbL7kjcU1BVjiLjXJLrrLzQtUvpqLmm0pwc1rU,39882
 datachain/data_storage/schema.py,sha256=asZYz1cg_WKfe2Q-k5W51E2z2CzHU5B4QEDZDMFr8yo,9346
 datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2krunWUum5o,927
-datachain/data_storage/sqlite.py,sha256=bwZAB_NUMT2WMv5tPQnnLFA0P-PiQtxzSaQ1q6xDxOU,24590
+datachain/data_storage/sqlite.py,sha256=BB8x7jtBmHK9lwn2zTo4HgfTKWGF43JxOsGr38J8YV8,25698
 datachain/data_storage/warehouse.py,sha256=imPm4R2V7TkqgGNSO2FGnKu03axU9UVLMfdUPfpwgHE,31747
 datachain/diff/__init__.py,sha256=-OFZzgOplqO84iWgGY7kfe60NXaWR9JRIh9T-uJboAM,9668
 datachain/fs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -153,9 +153,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ktGWzY4kyzjWyR86dhvzw-Zhl0lVk_LOX3NciTac6qo,2914
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.18.10.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.18.10.dist-info/METADATA,sha256=Vjkb16V4J8lNJphVuqD2DZ_V_7BLIf8YPRlvJNtsLaM,11320
-datachain-0.18.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datachain-0.18.10.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.18.10.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.18.10.dist-info/RECORD,,
+datachain-0.18.11.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.18.11.dist-info/METADATA,sha256=TgOokr9DxfY4A1mq7-5APy8DTHUqFEf2FslYxASH1IA,11320
+datachain-0.18.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datachain-0.18.11.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.18.11.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.18.11.dist-info/RECORD,,