unstructured-ingest 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/slack.py

@@ -0,0 +1,248 @@
+import hashlib
+import time
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Generator, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.error import SourceConnectionError
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    Indexer,
+    IndexerConfig,
+    download_responses,
+)
+from unstructured_ingest.v2.interfaces.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
+from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
+
+if TYPE_CHECKING:
+    from slack_sdk import WebClient
+    from slack_sdk.web.async_client import AsyncWebClient
+
+# NOTE: Pagination limit set to the upper end of the recommended range
+# https://api.slack.com/apis/pagination#facts
+PAGINATION_LIMIT = 200
+
+CONNECTOR_TYPE = "slack"
+
+
+class SlackAccessConfig(AccessConfig):
+    token: str = Field(
+        description="Bot token used to access Slack API, must have channels:history scope for the"
+        " bot user."
+    )
+
+
+class SlackConnectionConfig(ConnectionConfig):
+    access_config: Secret[SlackAccessConfig]
+
+    @requires_dependencies(["slack_sdk"], extras="slack")
+    @SourceConnectionError.wrap
+    def get_client(self) -> "WebClient":
+        from slack_sdk import WebClient
+
+        return WebClient(token=self.access_config.get_secret_value().token)
+
+    @requires_dependencies(["slack_sdk"], extras="slack")
+    @SourceConnectionError.wrap
+    def get_async_client(self) -> "AsyncWebClient":
+        from slack_sdk.web.async_client import AsyncWebClient
+
+        return AsyncWebClient(token=self.access_config.get_secret_value().token)
+
+
+class SlackIndexerConfig(IndexerConfig):
+    channels: list[str] = Field(
+        description="Comma-delimited list of Slack channel IDs to pull messages from, can be"
+        " both public or private channels."
+    )
+    start_date: Optional[datetime] = Field(
+        default=None,
+        description="Start date/time in formats YYYY-MM-DD[T]HH:MM[:SS[.ffffff]][Z or [±]HH[:]MM]"
+        " or YYYY-MM-DD",
+    )
+    end_date: Optional[datetime] = Field(
+        default=None,
+        description="End date/time in formats YYYY-MM-DD[T]HH:MM[:SS[.ffffff]][Z or [±]HH[:]MM]"
+        " or YYYY-MM-DD",
+    )
+
+
+@dataclass
+class SlackIndexer(Indexer):
+    index_config: SlackIndexerConfig
+    connection_config: SlackConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        client = self.connection_config.get_client()
+        for channel in self.index_config.channels:
+            messages = []
+            oldest = (
+                str(self.index_config.start_date.timestamp())
+                if self.index_config.start_date is not None
+                else None
+            )
+            latest = (
+                str(self.index_config.end_date.timestamp())
+                if self.index_config.end_date is not None
+                else None
+            )
+            for conversation_history in client.conversations_history(
+                channel=channel,
+                oldest=oldest,
+                latest=latest,
+                limit=PAGINATION_LIMIT,
+            ):
+                messages = conversation_history.get("messages", [])
+                if messages:
+                    yield self._messages_to_file_data(messages, channel)
+
+    def _messages_to_file_data(
+        self,
+        messages: list[dict],
+        channel: str,
+    ) -> FileData:
+        ts_oldest = min((message["ts"] for message in messages), key=lambda m: float(m))
+        ts_newest = max((message["ts"] for message in messages), key=lambda m: float(m))
+
+        identifier_base = f"{channel}-{ts_oldest}-{ts_newest}"
+        identifier = hashlib.sha256(identifier_base.encode("utf-8")).hexdigest()
+        filename = identifier[:16]
+
+        return FileData(
+            identifier=identifier,
+            connector_type=CONNECTOR_TYPE,
+            source_identifiers=SourceIdentifiers(
+                filename=f"{filename}.xml", fullpath=f"{filename}.xml"
+            ),
+            metadata=FileDataSourceMetadata(
+                date_created=ts_oldest,
+                date_modified=ts_newest,
+                date_processed=str(time.time()),
+                record_locator={
+                    "channel": channel,
+                    "oldest": ts_oldest,
+                    "latest": ts_newest,
+                },
+            ),
+        )
+
+    @SourceConnectionError.wrap
+    def precheck(self) -> None:
+        client = self.connection_config.get_client()
+        for channel in self.index_config.channels:
+            # NOTE: Querying conversations history guarantees that the bot is in the channel
+            client.conversations_history(channel=channel, limit=1)
+
+
+class SlackDownloaderConfig(DownloaderConfig):
+    pass
+
+
+@dataclass
+class SlackDownloader(Downloader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: SlackConnectionConfig
+    download_config: SlackDownloaderConfig = field(default_factory=SlackDownloaderConfig)
+
+    def run(self, file_data, **kwargs):
+        raise NotImplementedError
+
+    async def run_async(self, file_data: FileData, **kwargs) -> download_responses:
+        # NOTE: Indexer should provide source identifiers required to generate the download path
+        download_path = self.get_download_path(file_data)
+        if download_path is None:
+            logger.error(
+                "Generated download path is None, source_identifiers might be missing"
+                "from FileData."
+            )
+            raise ValueError("Generated invalid download path.")
+
+        await self._download_conversation(file_data, download_path)
+        return self.generate_download_response(file_data, download_path)
+
+    def is_async(self):
+        return True
+
+    async def _download_conversation(self, file_data: FileData, download_path: Path) -> None:
+        # NOTE: Indexer should supply the record locator in metadata
+        if (
+            file_data.metadata.record_locator is None
+            or "channel" not in file_data.metadata.record_locator
+            or "oldest" not in file_data.metadata.record_locator
+            or "latest" not in file_data.metadata.record_locator
+        ):
+            logger.error(
+                f"Invalid record locator in metadata: {file_data.metadata.record_locator}."
+                "Keys 'channel', 'oldest' and 'latest' must be present."
+            )
+            raise ValueError("Invalid record locator.")
+
+        client = self.connection_config.get_async_client()
+        messages = []
+        async for conversation_history in await client.conversations_history(
+            channel=file_data.metadata.record_locator["channel"],
+            oldest=file_data.metadata.record_locator["oldest"],
+            latest=file_data.metadata.record_locator["latest"],
+            limit=PAGINATION_LIMIT,
+            # NOTE: In order to get the exact same range of messages as indexer, it provides
+            # timestamps of oldest and newest messages, inclusive=True is necessary to include them
+            inclusive=True,
+        ):
+            messages += conversation_history.get("messages", [])
+
+        conversation = []
+        for message in messages:
+            thread_messages = []
+            async for conversations_replies in await client.conversations_replies(
+                channel=file_data.metadata.record_locator["channel"],
+                ts=message["ts"],
+                limit=PAGINATION_LIMIT,
+            ):
+                thread_messages += conversations_replies.get("messages", [])
+
+            # NOTE: Replies contains the whole thread, including the message references by the `ts`
+            # parameter even if it's the only message (there were no replies).
+            # Reference: https://api.slack.com/methods/conversations.replies#markdown
+            conversation.append(thread_messages)
+
+        conversation_xml = self._conversation_to_xml(conversation)
+        download_path.parent.mkdir(exist_ok=True, parents=True)
+        conversation_xml.write(download_path, encoding="utf-8", xml_declaration=True)
+
+    def _conversation_to_xml(self, conversation: list[list[dict]]) -> ET.ElementTree:
+        root = ET.Element("messages")
+
+        for thread in conversation:
+            message, *replies = thread
+            message_elem = ET.SubElement(root, "message")
+            text_elem = ET.SubElement(message_elem, "text")
+            text_elem.text = message.get("text")
+
+            for reply in replies:
+                reply_msg = reply.get("text", "")
+                text_elem.text = "".join([str(text_elem.text), " <reply> ", reply_msg])
+
+        return ET.ElementTree(root)
+
+
+slack_source_entry = SourceRegistryEntry(
+    indexer=SlackIndexer,
+    indexer_config=SlackIndexerConfig,
+    downloader=SlackDownloader,
+    downloader_config=DownloaderConfig,
+    connection_config=SlackConnectionConfig,
+)

unstructured_ingest/v2/processes/connectors/sql/__init__.py

@@ -2,12 +2,20 @@ from __future__ import annotations
 
 from unstructured_ingest.v2.processes.connector_registry import (
     add_destination_entry,
+    add_source_entry,
 )
 
 from .postgres import CONNECTOR_TYPE as POSTGRES_CONNECTOR_TYPE
-from .postgres import postgres_destination_entry
+from .postgres import postgres_destination_entry, postgres_source_entry
+from .snowflake import CONNECTOR_TYPE as SNOWFLAKE_CONNECTOR_TYPE
+from .snowflake import snowflake_destination_entry, snowflake_source_entry
 from .sqlite import CONNECTOR_TYPE as SQLITE_CONNECTOR_TYPE
-from .sqlite import sqlite_destination_entry
+from .sqlite import sqlite_destination_entry, sqlite_source_entry
+
+add_source_entry(source_type=SQLITE_CONNECTOR_TYPE, entry=sqlite_source_entry)
+add_source_entry(source_type=POSTGRES_CONNECTOR_TYPE, entry=postgres_source_entry)
+add_source_entry(source_type=SNOWFLAKE_CONNECTOR_TYPE, entry=snowflake_source_entry)
 
 add_destination_entry(destination_type=SQLITE_CONNECTOR_TYPE, entry=sqlite_destination_entry)
 add_destination_entry(destination_type=POSTGRES_CONNECTOR_TYPE, entry=postgres_destination_entry)
+add_destination_entry(destination_type=SNOWFLAKE_CONNECTOR_TYPE, entry=snowflake_destination_entry)

unstructured_ingest/v2/processes/connectors/sql/postgres.py

@@ -1,15 +1,16 @@
+from contextlib import contextmanager
 from dataclasses import dataclass, field
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any, Generator, Optional
 
-import numpy as np
-import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import FileData
 from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
 from unstructured_ingest.v2.processes.connectors.sql.sql import (
     _DATE_COLUMNS,
     SQLAccessConfig,
@@ -27,6 +28,7 @@ from unstructured_ingest.v2.processes.connectors.sql.sql import (
 
 if TYPE_CHECKING:
     from psycopg2.extensions import connection as PostgresConnection
+    from psycopg2.extensions import cursor as PostgresCursor
 
 CONNECTOR_TYPE = "postgres"
 
@@ -48,18 +50,33 @@ class PostgresConnectionConfig(SQLConnectionConfig):
     port: Optional[int] = Field(default=5432, description="DB host connection port")
     connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
+    @contextmanager
     @requires_dependencies(["psycopg2"], extras="postgres")
-    def get_connection(self) -> "PostgresConnection":
+    def get_connection(self) -> Generator["PostgresConnection", None, None]:
         from psycopg2 import connect
 
         access_config = self.access_config.get_secret_value()
-        return connect(
+        connection = connect(
             user=self.username,
             password=access_config.password,
             dbname=self.database,
             host=self.host,
             port=self.port,
         )
+        try:
+            yield connection
+        finally:
+            connection.commit()
+            connection.close()
+
+    @contextmanager
+    def get_cursor(self) -> Generator["PostgresCursor", None, None]:
+        with self.get_connection() as connection:
+            cursor = connection.cursor()
+            try:
+                yield cursor
+            finally:
+                cursor.close()
 
 
 class PostgresIndexerConfig(SQLIndexerConfig):
@@ -72,16 +89,6 @@ class PostgresIndexer(SQLIndexer):
     index_config: PostgresIndexerConfig
     connector_type: str = CONNECTOR_TYPE
 
-    def _get_doc_ids(self) -> list[str]:
-        connection = self.connection_config.get_connection()
-        with connection.cursor() as cursor:
-            cursor.execute(
-                f"SELECT {self.index_config.id_column} FROM {self.index_config.table_name}"
-            )
-            results = cursor.fetchall()
-            ids = [result[0] for result in results]
-            return ids
-
 
 class PostgresDownloaderConfig(SQLDownloaderConfig):
     pass
@@ -97,8 +104,7 @@ class PostgresDownloader(SQLDownloader):
         table_name = file_data.additional_metadata["table_name"]
         id_column = file_data.additional_metadata["id_column"]
         ids = file_data.additional_metadata["ids"]
-        connection = self.connection_config.get_connection()
-        with connection.cursor() as cursor:
+        with self.connection_config.get_cursor() as cursor:
             fields = ",".join(self.download_config.fields) if self.download_config.fields else "*"
             query = "SELECT {fields} FROM {table_name} WHERE {id_column} in ({ids})".format(
                 fields=fields,
@@ -130,6 +136,7 @@ class PostgresUploader(SQLUploader):
     upload_config: PostgresUploaderConfig = field(default_factory=PostgresUploaderConfig)
     connection_config: PostgresConnectionConfig
     connector_type: str = CONNECTOR_TYPE
+    values_delimiter: str = "%s"
 
     def prepare_data(
         self, columns: list[str], data: tuple[tuple[Any, ...], ...]
@@ -148,25 +155,14 @@ class PostgresUploader(SQLUploader):
             output.append(tuple(parsed))
         return output
 
-    def upload_contents(self, path: Path) -> None:
-        df = pd.read_json(path, orient="records", lines=True)
-        logger.debug(f"uploading {len(df)} entries to {self.connection_config.database} ")
-        df.replace({np.nan: None}, inplace=True)
-
-        columns = tuple(df.columns)
-        stmt = f"INSERT INTO {self.upload_config.table_name} ({','.join(columns)}) \
-                VALUES({','.join(['%s' for x in columns])})"  # noqa E501
-
-        for rows in pd.read_json(
-            path, orient="records", lines=True, chunksize=self.upload_config.batch_size
-        ):
-            with self.connection_config.get_connection() as conn:
-                values = self.prepare_data(columns, tuple(rows.itertuples(index=False, name=None)))
-                with conn.cursor() as cur:
-                    cur.executemany(stmt, values)
-
-                conn.commit()
 
+postgres_source_entry = SourceRegistryEntry(
+    connection_config=PostgresConnectionConfig,
+    indexer_config=PostgresIndexerConfig,
+    indexer=PostgresIndexer,
+    downloader_config=PostgresDownloaderConfig,
+    downloader=PostgresDownloader,
+)
 
 postgres_destination_entry = DestinationRegistryEntry(
     connection_config=PostgresConnectionConfig,

unstructured_ingest/v2/processes/connectors/sql/snowflake.py

@@ -0,0 +1,164 @@
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Generator, Optional
+
+import numpy as np
+import pandas as pd
+from pydantic import Field, Secret
+
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.sql.postgres import (
+    PostgresDownloader,
+    PostgresDownloaderConfig,
+    PostgresIndexer,
+    PostgresIndexerConfig,
+    PostgresUploader,
+    PostgresUploaderConfig,
+    PostgresUploadStager,
+    PostgresUploadStagerConfig,
+)
+from unstructured_ingest.v2.processes.connectors.sql.sql import SQLAccessConfig, SQLConnectionConfig
+
+if TYPE_CHECKING:
+    from snowflake.connector import SnowflakeConnection
+    from snowflake.connector.cursor import SnowflakeCursor
+
+CONNECTOR_TYPE = "snowflake"
+
+
+class SnowflakeAccessConfig(SQLAccessConfig):
+    password: Optional[str] = Field(default=None, description="DB password")
+
+
+class SnowflakeConnectionConfig(SQLConnectionConfig):
+    access_config: Secret[SnowflakeAccessConfig] = Field(
+        default=SnowflakeAccessConfig(), validate_default=True
+    )
+    account: str = Field(
+        default=None,
+        description="Your account identifier. The account identifier "
+        "does not include the snowflakecomputing.com suffix.",
+    )
+    user: Optional[str] = Field(default=None, description="DB username")
+    host: Optional[str] = Field(default=None, description="DB host")
+    port: Optional[int] = Field(default=443, description="DB host connection port")
+    database: str = Field(
+        default=None,
+        description="Database name.",
+    )
+    schema: str = Field(
+        default=None,
+        description="Database schema.",
+    )
+    role: str = Field(
+        default=None,
+        description="Database role.",
+    )
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+
+    @contextmanager
+    @requires_dependencies(["snowflake"], extras="snowflake")
+    def get_connection(self) -> Generator["SnowflakeConnection", None, None]:
+        # https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-api#label-snowflake-connector-methods-connect
+        from snowflake.connector import connect
+
+        connect_kwargs = self.model_dump()
+        connect_kwargs.pop("access_configs", None)
+        connect_kwargs["password"] = self.access_config.get_secret_value().password
+        # https://peps.python.org/pep-0249/#paramstyle
+        connect_kwargs["paramstyle"] = "qmark"
+        connection = connect(**connect_kwargs)
+        try:
+            yield connection
+        finally:
+            connection.commit()
+            connection.close()
+
+    @contextmanager
+    def get_cursor(self) -> Generator["SnowflakeCursor", None, None]:
+        with self.get_connection() as connection:
+            cursor = connection.cursor()
+            try:
+                yield cursor
+            finally:
+                cursor.close()
+
+
+class SnowflakeIndexerConfig(PostgresIndexerConfig):
+    pass
+
+
+@dataclass
+class SnowflakeIndexer(PostgresIndexer):
+    connection_config: SnowflakeConnectionConfig
+    index_config: SnowflakeIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+class SnowflakeDownloaderConfig(PostgresDownloaderConfig):
+    pass
+
+
+@dataclass
+class SnowflakeDownloader(PostgresDownloader):
+    connection_config: SnowflakeConnectionConfig
+    download_config: SnowflakeDownloaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+class SnowflakeUploadStagerConfig(PostgresUploadStagerConfig):
+    pass
+
+
+class SnowflakeUploadStager(PostgresUploadStager):
+    upload_stager_config: SnowflakeUploadStagerConfig
+
+
+class SnowflakeUploaderConfig(PostgresUploaderConfig):
+    pass
+
+
+@dataclass
+class SnowflakeUploader(PostgresUploader):
+    upload_config: SnowflakeUploaderConfig = field(default_factory=SnowflakeUploaderConfig)
+    connection_config: SnowflakeConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+    values_delimiter: str = "?"
+
+    def upload_contents(self, path: Path) -> None:
+        df = pd.read_json(path, orient="records", lines=True)
+        df.replace({np.nan: None}, inplace=True)
+
+        columns = list(df.columns)
+        stmt = f"INSERT INTO {self.upload_config.table_name} ({','.join(columns)}) VALUES({','.join([self.values_delimiter for x in columns])})"  # noqa E501
+
+        for rows in pd.read_json(
+            path, orient="records", lines=True, chunksize=self.upload_config.batch_size
+        ):
+            with self.connection_config.get_cursor() as cursor:
+                values = self.prepare_data(columns, tuple(rows.itertuples(index=False, name=None)))
+                # TODO: executemany break on 'Binding data in type (list) is not supported'
+                for val in values:
+                    cursor.execute(stmt, val)
+
+
+snowflake_source_entry = SourceRegistryEntry(
+    connection_config=SnowflakeConnectionConfig,
+    indexer_config=SnowflakeIndexerConfig,
+    indexer=SnowflakeIndexer,
+    downloader_config=SnowflakeDownloaderConfig,
+    downloader=SnowflakeDownloader,
+)
+
+snowflake_destination_entry = DestinationRegistryEntry(
+    connection_config=SnowflakeConnectionConfig,
+    uploader=SnowflakeUploader,
+    uploader_config=SnowflakeUploaderConfig,
+    upload_stager=SnowflakeUploadStager,
+    upload_stager_config=SnowflakeUploadStagerConfig,
+)

unstructured_ingest/v2/processes/connectors/sql/sql.py

@@ -3,12 +3,14 @@ import json
 import sys
 import uuid
 from abc import ABC, abstractmethod
+from contextlib import contextmanager
 from dataclasses import dataclass, field, replace
 from datetime import date, datetime
 from pathlib import Path
 from time import time
 from typing import Any, Generator, Union
 
+import numpy as np
 import pandas as pd
 from dateutil import parser
 from pydantic import Field, Secret
@@ -94,7 +96,13 @@ class SQLConnectionConfig(ConnectionConfig, ABC):
     access_config: Secret[SQLAccessConfig] = Field(default=SQLAccessConfig(), validate_default=True)
 
     @abstractmethod
-    def get_connection(self) -> Any:
+    @contextmanager
+    def get_connection(self) -> Generator[Any, None, None]:
+        pass
+
+    @abstractmethod
+    @contextmanager
+    def get_cursor(self) -> Generator[Any, None, None]:
         pass
 
 
@@ -108,16 +116,19 @@ class SQLIndexer(Indexer, ABC):
     connection_config: SQLConnectionConfig
     index_config: SQLIndexerConfig
 
-    @abstractmethod
     def _get_doc_ids(self) -> list[str]:
-        pass
+        with self.connection_config.get_cursor() as cursor:
+            cursor.execute(
+                f"SELECT {self.index_config.id_column} FROM {self.index_config.table_name}"
+            )
+            results = cursor.fetchall()
+            ids = [result[0] for result in results]
+            return ids
 
     def precheck(self) -> None:
         try:
-            connection = self.connection_config.get_connection()
-            cursor = connection.cursor()
-            cursor.execute("SELECT 1;")
-            cursor.close()
+            with self.connection_config.get_cursor() as cursor:
+                cursor.execute("SELECT 1;")
         except Exception as e:
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise SourceConnectionError(f"failed to validate connection: {e}")
@@ -198,7 +209,7 @@ class SQLDownloader(Downloader, ABC):
             f"Downloading results from table {table_name} and id {record_id} to {download_path}"
         )
         download_path.parent.mkdir(parents=True, exist_ok=True)
-        result.to_csv(download_path)
+        result.to_csv(download_path, index=False)
         copied_file_data = replace(file_data)
         copied_file_data.identifier = filename_id
         copied_file_data.doc_type = "file"
@@ -285,6 +296,7 @@ class SQLUploaderConfig(UploaderConfig):
 class SQLUploader(Uploader):
     upload_config: SQLUploaderConfig
     connection_config: SQLConnectionConfig
+    values_delimiter: str = "?"
 
     def precheck(self) -> None:
         try:
@@ -302,9 +314,25 @@ class SQLUploader(Uploader):
     ) -> list[tuple[Any, ...]]:
         pass
 
-    @abstractmethod
     def upload_contents(self, path: Path) -> None:
-        pass
+        df = pd.read_json(path, orient="records", lines=True)
+        df.replace({np.nan: None}, inplace=True)
+
+        columns = list(df.columns)
+        stmt = f"INSERT INTO {self.upload_config.table_name} ({','.join(columns)}) VALUES({','.join([self.values_delimiter for x in columns])})"  # noqa E501
+
+        for rows in pd.read_json(
+            path, orient="records", lines=True, chunksize=self.upload_config.batch_size
+        ):
+            with self.connection_config.get_cursor() as cursor:
+                values = self.prepare_data(columns, tuple(rows.itertuples(index=False, name=None)))
+                # for val in values:
+                #     try:
+                #         cursor.execute(stmt, val)
+                #     except Exception as e:
+                #         print(f"Error: {e}")
+                #         print(f"failed to write {len(columns)}, {len(val)}: {stmt} -> {val}")
+                cursor.executemany(stmt, values)
 
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
         self.upload_contents(path=path)