unstructured-ingest 0.4.0__py3-none-any.whl → 0.4.2__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/databricks/volumes_table.py

@@ -3,10 +3,11 @@ import os
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Generator, Optional
+from typing import Any, Generator
 
 from pydantic import Field
 
+from unstructured_ingest.utils.data_prep import write_data
 from unstructured_ingest.v2.interfaces import FileData, Uploader, UploaderConfig
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
@@ -14,9 +15,9 @@ from unstructured_ingest.v2.processes.connector_registry import (
 )
 from unstructured_ingest.v2.processes.connectors.databricks.volumes import DatabricksPathMixin
 from unstructured_ingest.v2.processes.connectors.sql.databricks_delta_tables import (
-    DatabrickDeltaTablesConnectionConfig,
-    DatabrickDeltaTablesUploadStager,
-    DatabrickDeltaTablesUploadStagerConfig,
+    DatabricksDeltaTablesConnectionConfig,
+    DatabricksDeltaTablesUploadStager,
+    DatabricksDeltaTablesUploadStagerConfig,
 )
 
 CONNECTOR_TYPE = "databricks_volume_delta_tables"
@@ -28,17 +29,16 @@ class DatabricksVolumeDeltaTableUploaderConfig(UploaderConfig, DatabricksPathMix
 
 
 @dataclass
-class DatabricksVolumeDeltaTableStager(DatabrickDeltaTablesUploadStager):
-    def write_output(self, output_path: Path, data: list[dict], indent: Optional[int] = 2) -> None:
+class DatabricksVolumeDeltaTableStager(DatabricksDeltaTablesUploadStager):
+    def write_output(self, output_path: Path, data: list[dict]) -> None:
         # To avoid new line issues when migrating from volumes into delta tables, omit indenting
         # and always write it as a json file
-        with output_path.with_suffix(".json").open("w") as f:
-            json.dump(data, f)
+        write_data(path=output_path.with_suffix(".json"), data=data, indent=None)
 
 
 @dataclass
 class DatabricksVolumeDeltaTableUploader(Uploader):
-    connection_config: DatabrickDeltaTablesConnectionConfig
+    connection_config: DatabricksDeltaTablesConnectionConfig
     upload_config: DatabricksVolumeDeltaTableUploaderConfig
     connector_type: str = CONNECTOR_TYPE
 
@@ -78,7 +78,10 @@ class DatabricksVolumeDeltaTableUploader(Uploader):
     @contextmanager
     def get_cursor(self, **connect_kwargs) -> Generator[Any, None, None]:
         with self.connection_config.get_cursor(**connect_kwargs) as cursor:
+            logger.debug(f"executing: USE CATALOG: '{self.upload_config.catalog}'")
             cursor.execute(f"USE CATALOG '{self.upload_config.catalog}'")
+            logger.debug(f"executing: USE DATABASE: {self.upload_config.database}")
+            cursor.execute(f"USE DATABASE {self.upload_config.database}")
             yield cursor
 
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
@@ -98,9 +101,9 @@ class DatabricksVolumeDeltaTableUploader(Uploader):
 
 
 databricks_volumes_delta_tables_destination_entry = DestinationRegistryEntry(
-    connection_config=DatabrickDeltaTablesConnectionConfig,
+    connection_config=DatabricksDeltaTablesConnectionConfig,
     uploader=DatabricksVolumeDeltaTableUploader,
     uploader_config=DatabricksVolumeDeltaTableUploaderConfig,
     upload_stager=DatabricksVolumeDeltaTableStager,
-    upload_stager_config=DatabrickDeltaTablesUploadStagerConfig,
+    upload_stager_config=DatabricksDeltaTablesUploadStagerConfig,
 )

unstructured_ingest/v2/processes/connectors/duckdb/base.py

@@ -4,7 +4,7 @@ from typing import Any
 
 import pandas as pd
 
-from unstructured_ingest.utils.data_prep import get_data
+from unstructured_ingest.utils.data_prep import get_data, write_data
 from unstructured_ingest.v2.interfaces import FileData, UploadStager
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
@@ -96,5 +96,5 @@ class BaseDuckDBUploadStager(UploadStager):
             df[column] = df[column].apply(str)
 
         data = df.to_dict(orient="records")
-        self.write_output(output_path=output_path, data=data)
+        write_data(path=output_path, data=data)
         return output_path

unstructured_ingest/v2/processes/connectors/fsspec/azure.py

@@ -128,22 +128,22 @@ class AzureIndexer(FsspecIndexer):
     def sterilize_info(self, file_data: dict) -> dict:
         return sterilize_dict(data=file_data, default=azure_json_serial)
 
-    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
-        path = file_data["name"]
+    def get_metadata(self, file_info: dict) -> FileDataSourceMetadata:
+        path = file_info["name"]
         date_created = (
-            str(file_data.get("creation_time").timestamp())
-            if "creation_time" in file_data
+            str(file_info.get("creation_time").timestamp())
+            if "creation_time" in file_info
             else None
         )
         date_modified = (
-            str(file_data.get("last_modified").timestamp())
-            if "last_modified" in file_data
+            str(file_info.get("last_modified").timestamp())
+            if "last_modified" in file_info
             else None
         )
 
-        file_size = file_data.get("size") if "size" in file_data else None
+        file_size = file_info.get("size") if "size" in file_info else None
 
-        version = file_data.get("etag")
+        version = file_info.get("etag")
         record_locator = {
             "protocol": self.index_config.protocol,
             "remote_file_path": self.index_config.remote_url,

unstructured_ingest/v2/processes/connectors/fsspec/box.py

@@ -104,22 +104,22 @@ class BoxIndexer(FsspecIndexer):
     index_config: BoxIndexerConfig
     connector_type: str = CONNECTOR_TYPE
 
-    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
-        path = file_data["name"]
+    def get_metadata(self, file_info: dict) -> FileDataSourceMetadata:
+        path = file_info["name"]
         date_created = None
         date_modified = None
-        if modified_at_str := file_data.get("modified_at"):
+        if modified_at_str := file_info.get("modified_at"):
             date_modified = str(parser.parse(modified_at_str).timestamp())
-        if created_at_str := file_data.get("created_at"):
+        if created_at_str := file_info.get("created_at"):
             date_created = str(parser.parse(created_at_str).timestamp())
 
-        file_size = file_data.get("size") if "size" in file_data else None
+        file_size = file_info.get("size") if "size" in file_info else None
 
-        version = file_data.get("id")
+        version = file_info.get("id")
         record_locator = {
             "protocol": self.index_config.protocol,
             "remote_file_path": self.index_config.remote_url,
-            "file_id": file_data.get("id"),
+            "file_id": file_info.get("id"),
         }
         return FileDataSourceMetadata(
             date_created=date_created,

unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py

@@ -93,15 +93,15 @@ class DropboxIndexer(FsspecIndexer):
     index_config: DropboxIndexerConfig
     connector_type: str = CONNECTOR_TYPE
 
-    def get_path(self, file_data: dict) -> str:
-        return file_data["name"]
+    def get_path(self, file_info: dict) -> str:
+        return file_info["name"]
 
-    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
-        path = file_data["name"].lstrip("/")
+    def get_metadata(self, file_info: dict) -> FileDataSourceMetadata:
+        path = file_info["name"].lstrip("/")
         date_created = None
         date_modified = None
-        server_modified = file_data.get("server_modified")
-        client_modified = file_data.get("client_modified")
+        server_modified = file_info.get("server_modified")
+        client_modified = file_info.get("client_modified")
         if server_modified and client_modified and server_modified > client_modified:
             date_created = str(client_modified.timestamp())
             date_modified = str(server_modified.timestamp())
@@ -109,13 +109,13 @@ class DropboxIndexer(FsspecIndexer):
             date_created = str(server_modified.timestamp())
             date_modified = str(client_modified.timestamp())
 
-        file_size = file_data.get("size") if "size" in file_data else None
+        file_size = file_info.get("size") if "size" in file_info else None
 
-        version = file_data.get("content_hash")
+        version = file_info.get("content_hash")
         record_locator = {
             "protocol": self.index_config.protocol,
             "remote_file_path": self.index_config.remote_url,
-            "file_id": file_data.get("id"),
+            "file_id": file_info.get("id"),
         }
         return FileDataSourceMetadata(
             date_created=date_created,

unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -119,7 +119,7 @@ class FsspecIndexer(Indexer):
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise self.wrap_error(e=e)
 
-    def get_file_data(self) -> list[dict[str, Any]]:
+    def get_file_info(self) -> list[dict[str, Any]]:
         if not self.index_config.recursive:
             # fs.ls does not walk directories
             # directories that are listed in cloud storage can cause problems
@@ -156,24 +156,56 @@ class FsspecIndexer(Indexer):
 
         return random.sample(files, n)
 
-    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+    def get_metadata(self, file_info: dict) -> FileDataSourceMetadata:
         raise NotImplementedError()
 
-    def get_path(self, file_data: dict) -> str:
-        return file_data["name"]
+    def get_path(self, file_info: dict) -> str:
+        return file_info["name"]
 
     def sterilize_info(self, file_data: dict) -> dict:
         return sterilize_dict(data=file_data)
 
+    def create_init_file_data(self, remote_filepath: Optional[str] = None) -> FileData:
+        # Create initial file data that requires no network calls and is constructed purely
+        # with information that exists in the config
+        remote_filepath = remote_filepath or self.index_config.remote_url
+        path_without_protocol = remote_filepath.split("://")[1]
+        rel_path = remote_filepath.replace(path_without_protocol, "").lstrip("/")
+        return FileData(
+            identifier=str(uuid5(NAMESPACE_DNS, remote_filepath)),
+            connector_type=self.connector_type,
+            display_name=remote_filepath,
+            source_identifiers=SourceIdentifiers(
+                filename=Path(remote_filepath).name,
+                rel_path=rel_path or None,
+                fullpath=remote_filepath,
+            ),
+            metadata=FileDataSourceMetadata(url=remote_filepath),
+        )
+
+    def hydrate_file_data(self, init_file_data: FileData):
+        # Get file info
+        with self.connection_config.get_client(protocol=self.index_config.protocol) as client:
+            files = client.ls(self.index_config.path_without_protocol, detail=True)
+        filtered_files = [
+            file for file in files if file.get("size") > 0 and file.get("type") == "file"
+        ]
+        if not filtered_files:
+            raise ValueError(f"{init_file_data} did not reference any valid file")
+        if len(filtered_files) > 1:
+            raise ValueError(f"{init_file_data} referenced more than one file")
+        file_info = filtered_files[0]
+        init_file_data.additional_metadata = self.get_metadata(file_info=file_info)
+
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
-        files = self.get_file_data()
-        for file_data in files:
-            file_path = self.get_path(file_data=file_data)
+        files = self.get_file_info()
+        for file_info in files:
+            file_path = self.get_path(file_info=file_info)
             # Note: we remove any remaining leading slashes (Box introduces these)
             # to get a valid relative path
             rel_path = file_path.replace(self.index_config.path_without_protocol, "").lstrip("/")
 
-            additional_metadata = self.sterilize_info(file_data=file_data)
+            additional_metadata = self.sterilize_info(file_data=file_info)
             additional_metadata["original_file_path"] = file_path
             yield FileData(
                 identifier=str(uuid5(NAMESPACE_DNS, file_path)),
@@ -183,7 +215,7 @@ class FsspecIndexer(Indexer):
                     rel_path=rel_path or None,
                     fullpath=file_path,
                 ),
-                metadata=self.get_metadata(file_data=file_data),
+                metadata=self.get_metadata(file_info=file_info),
                 additional_metadata=additional_metadata,
                 display_name=file_path,
             )

unstructured_ingest/v2/processes/connectors/fsspec/gcs.py

@@ -131,22 +131,22 @@ class GcsIndexer(FsspecIndexer):
     index_config: GcsIndexerConfig
     connector_type: str = CONNECTOR_TYPE
 
-    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
-        path = file_data["name"]
+    def get_metadata(self, file_info: dict) -> FileDataSourceMetadata:
+        path = file_info["name"]
         date_created = None
         date_modified = None
-        if modified_at_str := file_data.get("updated"):
+        if modified_at_str := file_info.get("updated"):
             date_modified = str(parser.parse(modified_at_str).timestamp())
-        if created_at_str := file_data.get("timeCreated"):
+        if created_at_str := file_info.get("timeCreated"):
             date_created = str(parser.parse(created_at_str).timestamp())
 
-        file_size = file_data.get("size") if "size" in file_data else None
+        file_size = file_info.get("size") if "size" in file_info else None
 
-        version = file_data.get("etag")
+        version = file_info.get("etag")
         record_locator = {
             "protocol": self.index_config.protocol,
             "remote_file_path": self.index_config.remote_url,
-            "file_id": file_data.get("id"),
+            "file_id": file_info.get("id"),
         }
         return FileDataSourceMetadata(
             date_created=date_created,

unstructured_ingest/v2/processes/connectors/fsspec/s3.py

@@ -110,22 +110,22 @@ class S3Indexer(FsspecIndexer):
     def wrap_error(self, e: Exception) -> Exception:
         return self.connection_config.wrap_error(e=e)
 
-    def get_path(self, file_data: dict) -> str:
-        return file_data["Key"]
+    def get_path(self, file_info: dict) -> str:
+        return file_info["Key"]
 
-    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
-        path = file_data["Key"]
+    def get_metadata(self, file_info: dict) -> FileDataSourceMetadata:
+        path = file_info["Key"]
         date_created = None
         date_modified = None
-        modified = file_data.get("LastModified")
+        modified = file_info.get("LastModified")
         if modified:
             date_created = str(modified.timestamp())
             date_modified = str(modified.timestamp())
 
-        file_size = file_data.get("size") if "size" in file_data else None
-        file_size = file_size or file_data.get("Size")
+        file_size = file_info.get("size") if "size" in file_info else None
+        file_size = file_size or file_info.get("Size")
 
-        version = file_data.get("ETag").rstrip('"').lstrip('"') if "ETag" in file_data else None
+        version = file_info.get("ETag").rstrip('"').lstrip('"') if "ETag" in file_info else None
         metadata: dict[str, str] = {}
         with contextlib.suppress(AttributeError):
             with self.connection_config.get_client(protocol=self.index_config.protocol) as client:

unstructured_ingest/v2/processes/connectors/fsspec/sftp.py

@@ -107,12 +107,12 @@ class SftpIndexer(FsspecIndexer):
             file.identifier = new_identifier
             yield file
 
-    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
-        path = file_data["name"]
-        date_created = str(file_data.get("time").timestamp()) if "time" in file_data else None
-        date_modified = str(file_data.get("mtime").timestamp()) if "mtime" in file_data else None
+    def get_metadata(self, file_info: dict) -> FileDataSourceMetadata:
+        path = file_info["name"]
+        date_created = str(file_info.get("time").timestamp()) if "time" in file_info else None
+        date_modified = str(file_info.get("mtime").timestamp()) if "mtime" in file_info else None
 
-        file_size = file_data.get("size") if "size" in file_data else None
+        file_size = file_info.get("size") if "size" in file_info else None
 
         record_locator = {
             "protocol": self.index_config.protocol,

unstructured_ingest/v2/processes/connectors/sql/__init__.py

@@ -15,11 +15,14 @@ from .snowflake import CONNECTOR_TYPE as SNOWFLAKE_CONNECTOR_TYPE
 from .snowflake import snowflake_destination_entry, snowflake_source_entry
 from .sqlite import CONNECTOR_TYPE as SQLITE_CONNECTOR_TYPE
 from .sqlite import sqlite_destination_entry, sqlite_source_entry
+from .vastdb import CONNECTOR_TYPE as VASTDB_CONNECTOR_TYPE
+from .vastdb import vastdb_destination_entry, vastdb_source_entry
 
 add_source_entry(source_type=SQLITE_CONNECTOR_TYPE, entry=sqlite_source_entry)
 add_source_entry(source_type=POSTGRES_CONNECTOR_TYPE, entry=postgres_source_entry)
 add_source_entry(source_type=SNOWFLAKE_CONNECTOR_TYPE, entry=snowflake_source_entry)
 add_source_entry(source_type=SINGLESTORE_CONNECTOR_TYPE, entry=singlestore_source_entry)
+add_source_entry(source_type=VASTDB_CONNECTOR_TYPE, entry=vastdb_source_entry)
 
 add_destination_entry(destination_type=SQLITE_CONNECTOR_TYPE, entry=sqlite_destination_entry)
 add_destination_entry(destination_type=POSTGRES_CONNECTOR_TYPE, entry=postgres_destination_entry)
@@ -31,3 +34,4 @@ add_destination_entry(
     destination_type=DATABRICKS_DELTA_TABLES_CONNECTOR_TYPE,
     entry=databricks_delta_tables_destination_entry,
 )
+add_destination_entry(destination_type=VASTDB_CONNECTOR_TYPE, entry=vastdb_destination_entry)

unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py

@@ -31,7 +31,7 @@ if TYPE_CHECKING:
 CONNECTOR_TYPE = "databricks_delta_tables"
 
 
-class DatabrickDeltaTablesAccessConfig(SQLAccessConfig):
+class DatabricksDeltaTablesAccessConfig(SQLAccessConfig):
     token: Optional[str] = Field(default=None, description="Databricks Personal Access Token")
     client_id: Optional[str] = Field(default=None, description="Client ID of the OAuth app.")
     client_secret: Optional[str] = Field(
@@ -39,8 +39,8 @@ class DatabrickDeltaTablesAccessConfig(SQLAccessConfig):
     )
 
 
-class DatabrickDeltaTablesConnectionConfig(SQLConnectionConfig):
-    access_config: Secret[DatabrickDeltaTablesAccessConfig]
+class DatabricksDeltaTablesConnectionConfig(SQLConnectionConfig):
+    access_config: Secret[DatabricksDeltaTablesAccessConfig]
     server_hostname: str = Field(description="server hostname connection config value")
     http_path: str = Field(description="http path connection config value")
     user_agent: str = "unstructuredio_oss"
@@ -102,24 +102,24 @@ class DatabrickDeltaTablesConnectionConfig(SQLConnectionConfig):
             yield cursor
 
 
-class DatabrickDeltaTablesUploadStagerConfig(SQLUploadStagerConfig):
+class DatabricksDeltaTablesUploadStagerConfig(SQLUploadStagerConfig):
     pass
 
 
-class DatabrickDeltaTablesUploadStager(SQLUploadStager):
-    upload_stager_config: DatabrickDeltaTablesUploadStagerConfig
+class DatabricksDeltaTablesUploadStager(SQLUploadStager):
+    upload_stager_config: DatabricksDeltaTablesUploadStagerConfig
 
 
-class DatabrickDeltaTablesUploaderConfig(SQLUploaderConfig):
+class DatabricksDeltaTablesUploaderConfig(SQLUploaderConfig):
     catalog: str = Field(description="Name of the catalog in the Databricks Unity Catalog service")
     database: str = Field(description="Database name", default="default")
     table_name: str = Field(description="Table name")
 
 
 @dataclass
-class DatabrickDeltaTablesUploader(SQLUploader):
-    upload_config: DatabrickDeltaTablesUploaderConfig
-    connection_config: DatabrickDeltaTablesConnectionConfig
+class DatabricksDeltaTablesUploader(SQLUploader):
+    upload_config: DatabricksDeltaTablesUploaderConfig
+    connection_config: DatabricksDeltaTablesConnectionConfig
     connector_type: str = CONNECTOR_TYPE
 
     @contextmanager
@@ -205,9 +205,9 @@ class DatabrickDeltaTablesUploader(SQLUploader):
 
 
 databricks_delta_tables_destination_entry = DestinationRegistryEntry(
-    connection_config=DatabrickDeltaTablesConnectionConfig,
-    uploader=DatabrickDeltaTablesUploader,
-    uploader_config=DatabrickDeltaTablesUploaderConfig,
-    upload_stager=DatabrickDeltaTablesUploadStager,
-    upload_stager_config=DatabrickDeltaTablesUploadStagerConfig,
+    connection_config=DatabricksDeltaTablesConnectionConfig,
+    uploader=DatabricksDeltaTablesUploader,
+    uploader_config=DatabricksDeltaTablesUploaderConfig,
+    upload_stager=DatabricksDeltaTablesUploadStager,
+    upload_stager_config=DatabricksDeltaTablesUploadStagerConfig,
 )

unstructured_ingest/v2/processes/connectors/sql/singlestore.py

@@ -3,6 +3,7 @@ from contextlib import contextmanager
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
+import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.v2.logger import logger
@@ -139,7 +140,7 @@ class SingleStoreUploader(SQLUploader):
                 if isinstance(value, (list, dict)):
                     value = json.dumps(value)
                 if column_name in _DATE_COLUMNS:
-                    if value is None:
+                    if value is None or pd.isna(value):
                         parsed.append(None)
                     else:
                         parsed.append(parse_date_string(value))

unstructured_ingest/v2/processes/connectors/sql/sql.py

@@ -14,7 +14,7 @@ from dateutil import parser
 from pydantic import BaseModel, Field, Secret
 
 from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
-from unstructured_ingest.utils.data_prep import get_data, get_data_df, split_dataframe
+from unstructured_ingest.utils.data_prep import get_data, get_data_df, split_dataframe, write_data
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
@@ -292,6 +292,9 @@ class SQLUploadStager(UploadStager):
             df[column] = df[column].apply(str)
         return df
 
+    def write_output(self, output_path: Path, data: list[dict]) -> None:
+        write_data(path=output_path, data=data)
+
     def run(
         self,
         elements_filepath: Path,
@@ -332,6 +335,7 @@ class SQLUploader(Uploader):
     upload_config: SQLUploaderConfig
     connection_config: SQLConnectionConfig
     values_delimiter: str = "?"
+    _columns: list[str] = field(init=False, default=None)
 
     def precheck(self) -> None:
         try:
@@ -354,7 +358,7 @@ class SQLUploader(Uploader):
             parsed = []
             for column_name, value in zip(columns, row):
                 if column_name in _DATE_COLUMNS:
-                    if value is None:
+                    if value is None or pd.isna(value):  # pandas is nan
                         parsed.append(None)
                     else:
                         parsed.append(parse_date_string(value))
@@ -364,8 +368,9 @@ class SQLUploader(Uploader):
         return output
 
     def _fit_to_schema(self, df: pd.DataFrame) -> pd.DataFrame:
+        table_columns = self.get_table_columns()
         columns = set(df.columns)
-        schema_fields = set(columns)
+        schema_fields = set(table_columns)
         columns_to_drop = columns - schema_fields
         missing_columns = schema_fields - columns
 
@@ -395,8 +400,8 @@ class SQLUploader(Uploader):
                 f"record id column "
                 f"{self.upload_config.record_id_key}, skipping delete"
             )
+        df = self._fit_to_schema(df=df)
         df.replace({np.nan: None}, inplace=True)
-        self._fit_to_schema(df=df)
 
         columns = list(df.columns)
         stmt = "INSERT INTO {table_name} ({columns}) VALUES({values})".format(
@@ -424,9 +429,11 @@ class SQLUploader(Uploader):
                 cursor.executemany(stmt, values)
 
     def get_table_columns(self) -> list[str]:
-        with self.get_cursor() as cursor:
-            cursor.execute(f"SELECT * from {self.upload_config.table_name}")
-            return [desc[0] for desc in cursor.description]
+        if self._columns is None:
+            with self.get_cursor() as cursor:
+                cursor.execute(f"SELECT * from {self.upload_config.table_name} LIMIT 1")
+                self._columns = [desc[0] for desc in cursor.description]
+        return self._columns
 
     def can_delete(self) -> bool:
         return self.upload_config.record_id_key in self.get_table_columns()

unstructured_ingest/v2/processes/connectors/sql/sqlite.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator
 
+import pandas as pd
 from pydantic import Field, Secret, model_validator
 
 from unstructured_ingest.v2.logger import logger
@@ -141,7 +142,7 @@ class SQLiteUploader(SQLUploader):
                 if isinstance(value, (list, dict)):
                     value = json.dumps(value)
                 if column_name in _DATE_COLUMNS:
-                    if value is None:
+                    if value is None or pd.isna(value):
                         parsed.append(None)
                     else:
                         parsed.append(parse_date_string(value))