unstructured-ingest 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import random
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Optional, TypeVar
@@ -63,6 +64,7 @@ class FileConfig(BaseModel):
 
 class FsspecIndexerConfig(FileConfig, IndexerConfig):
     recursive: bool = False
+    sample_n_files: Optional[int] = None
 
 
 class FsspecAccessConfig(AccessConfig):
@@ -128,8 +130,23 @@ class FsspecIndexer(Indexer):
         filtered_files = [
             file for file in files if file.get("size") > 0 and file.get("type") == "file"
         ]
+
+        if self.index_config.sample_n_files:
+            filtered_files = self.sample_n_files(filtered_files, self.index_config.sample_n_files)
+
         return filtered_files
 
+    def sample_n_files(self, files: list[dict[str, Any]], n) -> list[dict[str, Any]]:
+        if len(files) <= n:
+            logger.warning(
+                f"number of files to be sampled={n} is not smaller than the number"
+                f" of files found ({len(files)}). Returning all of the files as the"
+                " sample."
+            )
+            return files
+
+        return random.sample(files, n)
+
     def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
         raise NotImplementedError()
 

unstructured_ingest/v2/processes/connectors/kdbai.py

@@ -26,7 +26,7 @@ from unstructured_ingest.v2.processes.connector_registry import (
 )
 
 if TYPE_CHECKING:
-    from kdbai_client import Session, Table
+    from kdbai_client import Database, Session, Table
 
 CONNECTOR_TYPE = "kdbai"
 
@@ -99,6 +99,9 @@ class KdbaiUploadStager(UploadStager):
 
 
 class KdbaiUploaderConfig(UploaderConfig):
+    database_name: str = Field(
+        default="default", description="The name of the KDBAI database to write into."
+    )
     table_name: str = Field(description="The name of the KDBAI table to write into.")
     batch_size: int = Field(default=100, description="Number of records per batch")
 
@@ -111,24 +114,29 @@ class KdbaiUploader(Uploader):
 
     def precheck(self) -> None:
         try:
-            self.get_table()
+            self.get_database()
         except Exception as e:
             logger.error(f"Failed to validate connection {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
-    def get_table(self) -> "Table":
+    def get_database(self) -> "Database":
         session: Session = self.connection_config.get_session()
-        table = session.table(self.upload_config.table_name)
+        db = session.database(self.upload_config.database_name)
+        return db
+
+    def get_table(self) -> "Table":
+        db = self.get_database()
+        table = db.table(self.upload_config.table_name)
         return table
 
     def upsert_batch(self, batch: pd.DataFrame):
         table = self.get_table()
-        table.insert(data=batch)
+        table.insert(batch)
 
     def process_dataframe(self, df: pd.DataFrame):
         logger.debug(
             f"uploading {len(df)} entries to {self.connection_config.endpoint} "
-            f"db in table {self.upload_config.table_name}"
+            f"db {self.upload_config.database_name} in table {self.upload_config.table_name}"
         )
         for _, batch_df in df.groupby(np.arange(len(df)) // self.upload_config.batch_size):
             self.upsert_batch(batch=batch_df)

unstructured_ingest/v2/processes/connectors/slack.py

@@ -0,0 +1,248 @@
+import hashlib
+import time
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Generator, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.error import SourceConnectionError
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    Indexer,
+    IndexerConfig,
+    download_responses,
+)
+from unstructured_ingest.v2.interfaces.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
+from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
+
+if TYPE_CHECKING:
+    from slack_sdk import WebClient
+    from slack_sdk.web.async_client import AsyncWebClient
+
+# NOTE: Pagination limit set to the upper end of the recommended range
+# https://api.slack.com/apis/pagination#facts
+PAGINATION_LIMIT = 200
+
+CONNECTOR_TYPE = "slack"
+
+
+class SlackAccessConfig(AccessConfig):
+    token: str = Field(
+        description="Bot token used to access Slack API, must have channels:history scope for the"
+        " bot user."
+    )
+
+
+class SlackConnectionConfig(ConnectionConfig):
+    access_config: Secret[SlackAccessConfig]
+
+    @requires_dependencies(["slack_sdk"], extras="slack")
+    @SourceConnectionError.wrap
+    def get_client(self) -> "WebClient":
+        from slack_sdk import WebClient
+
+        return WebClient(token=self.access_config.get_secret_value().token)
+
+    @requires_dependencies(["slack_sdk"], extras="slack")
+    @SourceConnectionError.wrap
+    def get_async_client(self) -> "AsyncWebClient":
+        from slack_sdk.web.async_client import AsyncWebClient
+
+        return AsyncWebClient(token=self.access_config.get_secret_value().token)
+
+
+class SlackIndexerConfig(IndexerConfig):
+    channels: list[str] = Field(
+        description="Comma-delimited list of Slack channel IDs to pull messages from, can be"
+        " both public or private channels."
+    )
+    start_date: Optional[datetime] = Field(
+        default=None,
+        description="Start date/time in formats YYYY-MM-DD[T]HH:MM[:SS[.ffffff]][Z or [±]HH[:]MM]"
+        " or YYYY-MM-DD",
+    )
+    end_date: Optional[datetime] = Field(
+        default=None,
+        description="End date/time in formats YYYY-MM-DD[T]HH:MM[:SS[.ffffff]][Z or [±]HH[:]MM]"
+        " or YYYY-MM-DD",
+    )
+
+
+@dataclass
+class SlackIndexer(Indexer):
+    index_config: SlackIndexerConfig
+    connection_config: SlackConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        client = self.connection_config.get_client()
+        for channel in self.index_config.channels:
+            messages = []
+            oldest = (
+                str(self.index_config.start_date.timestamp())
+                if self.index_config.start_date is not None
+                else None
+            )
+            latest = (
+                str(self.index_config.end_date.timestamp())
+                if self.index_config.end_date is not None
+                else None
+            )
+            for conversation_history in client.conversations_history(
+                channel=channel,
+                oldest=oldest,
+                latest=latest,
+                limit=PAGINATION_LIMIT,
+            ):
+                messages = conversation_history.get("messages", [])
+                if messages:
+                    yield self._messages_to_file_data(messages, channel)
+
+    def _messages_to_file_data(
+        self,
+        messages: list[dict],
+        channel: str,
+    ) -> FileData:
+        ts_oldest = min((message["ts"] for message in messages), key=lambda m: float(m))
+        ts_newest = max((message["ts"] for message in messages), key=lambda m: float(m))
+
+        identifier_base = f"{channel}-{ts_oldest}-{ts_newest}"
+        identifier = hashlib.sha256(identifier_base.encode("utf-8")).hexdigest()
+        filename = identifier[:16]
+
+        return FileData(
+            identifier=identifier,
+            connector_type=CONNECTOR_TYPE,
+            source_identifiers=SourceIdentifiers(
+                filename=f"{filename}.xml", fullpath=f"{filename}.xml"
+            ),
+            metadata=FileDataSourceMetadata(
+                date_created=ts_oldest,
+                date_modified=ts_newest,
+                date_processed=str(time.time()),
+                record_locator={
+                    "channel": channel,
+                    "oldest": ts_oldest,
+                    "latest": ts_newest,
+                },
+            ),
+        )
+
+    @SourceConnectionError.wrap
+    def precheck(self) -> None:
+        client = self.connection_config.get_client()
+        for channel in self.index_config.channels:
+            # NOTE: Querying conversations history guarantees that the bot is in the channel
+            client.conversations_history(channel=channel, limit=1)
+
+
+class SlackDownloaderConfig(DownloaderConfig):
+    pass
+
+
+@dataclass
+class SlackDownloader(Downloader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: SlackConnectionConfig
+    download_config: SlackDownloaderConfig = field(default_factory=SlackDownloaderConfig)
+
+    def run(self, file_data, **kwargs):
+        raise NotImplementedError
+
+    async def run_async(self, file_data: FileData, **kwargs) -> download_responses:
+        # NOTE: Indexer should provide source identifiers required to generate the download path
+        download_path = self.get_download_path(file_data)
+        if download_path is None:
+            logger.error(
+                "Generated download path is None, source_identifiers might be missing"
+                "from FileData."
+            )
+            raise ValueError("Generated invalid download path.")
+
+        await self._download_conversation(file_data, download_path)
+        return self.generate_download_response(file_data, download_path)
+
+    def is_async(self):
+        return True
+
+    async def _download_conversation(self, file_data: FileData, download_path: Path) -> None:
+        # NOTE: Indexer should supply the record locator in metadata
+        if (
+            file_data.metadata.record_locator is None
+            or "channel" not in file_data.metadata.record_locator
+            or "oldest" not in file_data.metadata.record_locator
+            or "latest" not in file_data.metadata.record_locator
+        ):
+            logger.error(
+                f"Invalid record locator in metadata: {file_data.metadata.record_locator}."
+                "Keys 'channel', 'oldest' and 'latest' must be present."
+            )
+            raise ValueError("Invalid record locator.")
+
+        client = self.connection_config.get_async_client()
+        messages = []
+        async for conversation_history in await client.conversations_history(
+            channel=file_data.metadata.record_locator["channel"],
+            oldest=file_data.metadata.record_locator["oldest"],
+            latest=file_data.metadata.record_locator["latest"],
+            limit=PAGINATION_LIMIT,
+            # NOTE: In order to get the exact same range of messages as indexer, it provides
+            # timestamps of oldest and newest messages, inclusive=True is necessary to include them
+            inclusive=True,
+        ):
+            messages += conversation_history.get("messages", [])
+
+        conversation = []
+        for message in messages:
+            thread_messages = []
+            async for conversations_replies in await client.conversations_replies(
+                channel=file_data.metadata.record_locator["channel"],
+                ts=message["ts"],
+                limit=PAGINATION_LIMIT,
+            ):
+                thread_messages += conversations_replies.get("messages", [])
+
+            # NOTE: Replies contains the whole thread, including the message references by the `ts`
+            # parameter even if it's the only message (there were no replies).
+            # Reference: https://api.slack.com/methods/conversations.replies#markdown
+            conversation.append(thread_messages)
+
+        conversation_xml = self._conversation_to_xml(conversation)
+        download_path.parent.mkdir(exist_ok=True, parents=True)
+        conversation_xml.write(download_path, encoding="utf-8", xml_declaration=True)
+
+    def _conversation_to_xml(self, conversation: list[list[dict]]) -> ET.ElementTree:
+        root = ET.Element("messages")
+
+        for thread in conversation:
+            message, *replies = thread
+            message_elem = ET.SubElement(root, "message")
+            text_elem = ET.SubElement(message_elem, "text")
+            text_elem.text = message.get("text")
+
+            for reply in replies:
+                reply_msg = reply.get("text", "")
+                text_elem.text = "".join([str(text_elem.text), " <reply> ", reply_msg])
+
+        return ET.ElementTree(root)
+
+
+slack_source_entry = SourceRegistryEntry(
+    indexer=SlackIndexer,
+    indexer_config=SlackIndexerConfig,
+    downloader=SlackDownloader,
+    downloader_config=DownloaderConfig,
+    connection_config=SlackConnectionConfig,
+)

unstructured_ingest/v2/processes/connectors/sql/__init__.py

@@ -2,12 +2,20 @@ from __future__ import annotations
 
 from unstructured_ingest.v2.processes.connector_registry import (
     add_destination_entry,
+    add_source_entry,
 )
 
 from .postgres import CONNECTOR_TYPE as POSTGRES_CONNECTOR_TYPE
-from .postgres import postgres_destination_entry
+from .postgres import postgres_destination_entry, postgres_source_entry
+from .snowflake import CONNECTOR_TYPE as SNOWFLAKE_CONNECTOR_TYPE
+from .snowflake import snowflake_destination_entry, snowflake_source_entry
 from .sqlite import CONNECTOR_TYPE as SQLITE_CONNECTOR_TYPE
-from .sqlite import sqlite_destination_entry
+from .sqlite import sqlite_destination_entry, sqlite_source_entry
+
+add_source_entry(source_type=SQLITE_CONNECTOR_TYPE, entry=sqlite_source_entry)
+add_source_entry(source_type=POSTGRES_CONNECTOR_TYPE, entry=postgres_source_entry)
+add_source_entry(source_type=SNOWFLAKE_CONNECTOR_TYPE, entry=snowflake_source_entry)
 
 add_destination_entry(destination_type=SQLITE_CONNECTOR_TYPE, entry=sqlite_destination_entry)
 add_destination_entry(destination_type=POSTGRES_CONNECTOR_TYPE, entry=postgres_destination_entry)
+add_destination_entry(destination_type=SNOWFLAKE_CONNECTOR_TYPE, entry=snowflake_destination_entry)

unstructured_ingest/v2/processes/connectors/sql/postgres.py

@@ -1,18 +1,24 @@
+from contextlib import contextmanager
 from dataclasses import dataclass, field
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any, Generator, Optional
 
-import numpy as np
-import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import FileData
 from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
 from unstructured_ingest.v2.processes.connectors.sql.sql import (
     _DATE_COLUMNS,
     SQLAccessConfig,
     SQLConnectionConfig,
+    SQLDownloader,
+    SQLDownloaderConfig,
+    SQLIndexer,
+    SQLIndexerConfig,
     SQLUploader,
     SQLUploaderConfig,
     SQLUploadStager,
@@ -22,6 +28,7 @@ from unstructured_ingest.v2.processes.connectors.sql.sql import (
 
 if TYPE_CHECKING:
     from psycopg2.extensions import connection as PostgresConnection
+    from psycopg2.extensions import cursor as PostgresCursor
 
 CONNECTOR_TYPE = "postgres"
 
@@ -43,18 +50,73 @@ class PostgresConnectionConfig(SQLConnectionConfig):
     port: Optional[int] = Field(default=5432, description="DB host connection port")
     connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
+    @contextmanager
     @requires_dependencies(["psycopg2"], extras="postgres")
-    def get_connection(self) -> "PostgresConnection":
+    def get_connection(self) -> Generator["PostgresConnection", None, None]:
         from psycopg2 import connect
 
         access_config = self.access_config.get_secret_value()
-        return connect(
+        connection = connect(
             user=self.username,
             password=access_config.password,
             dbname=self.database,
             host=self.host,
             port=self.port,
         )
+        try:
+            yield connection
+        finally:
+            connection.commit()
+            connection.close()
+
+    @contextmanager
+    def get_cursor(self) -> Generator["PostgresCursor", None, None]:
+        with self.get_connection() as connection:
+            cursor = connection.cursor()
+            try:
+                yield cursor
+            finally:
+                cursor.close()
+
+
+class PostgresIndexerConfig(SQLIndexerConfig):
+    pass
+
+
+@dataclass
+class PostgresIndexer(SQLIndexer):
+    connection_config: PostgresConnectionConfig
+    index_config: PostgresIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+class PostgresDownloaderConfig(SQLDownloaderConfig):
+    pass
+
+
+@dataclass
+class PostgresDownloader(SQLDownloader):
+    connection_config: PostgresConnectionConfig
+    download_config: PostgresDownloaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def query_db(self, file_data: FileData) -> tuple[list[tuple], list[str]]:
+        table_name = file_data.additional_metadata["table_name"]
+        id_column = file_data.additional_metadata["id_column"]
+        ids = file_data.additional_metadata["ids"]
+        with self.connection_config.get_cursor() as cursor:
+            fields = ",".join(self.download_config.fields) if self.download_config.fields else "*"
+            query = "SELECT {fields} FROM {table_name} WHERE {id_column} in ({ids})".format(
+                fields=fields,
+                table_name=table_name,
+                id_column=id_column,
+                ids=",".join([str(i) for i in ids]),
+            )
+            logger.debug(f"running query: {query}")
+            cursor.execute(query)
+            rows = cursor.fetchall()
+            columns = [col[0] for col in cursor.description]
+            return rows, columns
 
 
 class PostgresUploadStagerConfig(SQLUploadStagerConfig):
@@ -74,6 +136,7 @@ class PostgresUploader(SQLUploader):
     upload_config: PostgresUploaderConfig = field(default_factory=PostgresUploaderConfig)
     connection_config: PostgresConnectionConfig
     connector_type: str = CONNECTOR_TYPE
+    values_delimiter: str = "%s"
 
     def prepare_data(
         self, columns: list[str], data: tuple[tuple[Any, ...], ...]
@@ -92,25 +155,14 @@ class PostgresUploader(SQLUploader):
             output.append(tuple(parsed))
         return output
 
-    def upload_contents(self, path: Path) -> None:
-        df = pd.read_json(path, orient="records", lines=True)
-        logger.debug(f"uploading {len(df)} entries to {self.connection_config.database} ")
-        df.replace({np.nan: None}, inplace=True)
-
-        columns = tuple(df.columns)
-        stmt = f"INSERT INTO {self.upload_config.table_name} ({','.join(columns)}) \
-                VALUES({','.join(['%s' for x in columns])})"  # noqa E501
-
-        for rows in pd.read_json(
-            path, orient="records", lines=True, chunksize=self.upload_config.batch_size
-        ):
-            with self.connection_config.get_connection() as conn:
-                values = self.prepare_data(columns, tuple(rows.itertuples(index=False, name=None)))
-                with conn.cursor() as cur:
-                    cur.executemany(stmt, values)
-
-                conn.commit()
 
+postgres_source_entry = SourceRegistryEntry(
+    connection_config=PostgresConnectionConfig,
+    indexer_config=PostgresIndexerConfig,
+    indexer=PostgresIndexer,
+    downloader_config=PostgresDownloaderConfig,
+    downloader=PostgresDownloader,
+)
 
 postgres_destination_entry = DestinationRegistryEntry(
     connection_config=PostgresConnectionConfig,

unstructured_ingest/v2/processes/connectors/sql/snowflake.py

@@ -0,0 +1,164 @@
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Generator, Optional
+
+import numpy as np
+import pandas as pd
+from pydantic import Field, Secret
+
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.sql.postgres import (
+    PostgresDownloader,
+    PostgresDownloaderConfig,
+    PostgresIndexer,
+    PostgresIndexerConfig,
+    PostgresUploader,
+    PostgresUploaderConfig,
+    PostgresUploadStager,
+    PostgresUploadStagerConfig,
+)
+from unstructured_ingest.v2.processes.connectors.sql.sql import SQLAccessConfig, SQLConnectionConfig
+
+if TYPE_CHECKING:
+    from snowflake.connector import SnowflakeConnection
+    from snowflake.connector.cursor import SnowflakeCursor
+
+CONNECTOR_TYPE = "snowflake"
+
+
+class SnowflakeAccessConfig(SQLAccessConfig):
+    password: Optional[str] = Field(default=None, description="DB password")
+
+
+class SnowflakeConnectionConfig(SQLConnectionConfig):
+    access_config: Secret[SnowflakeAccessConfig] = Field(
+        default=SnowflakeAccessConfig(), validate_default=True
+    )
+    account: str = Field(
+        default=None,
+        description="Your account identifier. The account identifier "
+        "does not include the snowflakecomputing.com suffix.",
+    )
+    user: Optional[str] = Field(default=None, description="DB username")
+    host: Optional[str] = Field(default=None, description="DB host")
+    port: Optional[int] = Field(default=443, description="DB host connection port")
+    database: str = Field(
+        default=None,
+        description="Database name.",
+    )
+    schema: str = Field(
+        default=None,
+        description="Database schema.",
+    )
+    role: str = Field(
+        default=None,
+        description="Database role.",
+    )
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+
+    @contextmanager
+    @requires_dependencies(["snowflake"], extras="snowflake")
+    def get_connection(self) -> Generator["SnowflakeConnection", None, None]:
+        # https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-api#label-snowflake-connector-methods-connect
+        from snowflake.connector import connect
+
+        connect_kwargs = self.model_dump()
+        connect_kwargs.pop("access_configs", None)
+        connect_kwargs["password"] = self.access_config.get_secret_value().password
+        # https://peps.python.org/pep-0249/#paramstyle
+        connect_kwargs["paramstyle"] = "qmark"
+        connection = connect(**connect_kwargs)
+        try:
+            yield connection
+        finally:
+            connection.commit()
+            connection.close()
+
+    @contextmanager
+    def get_cursor(self) -> Generator["SnowflakeCursor", None, None]:
+        with self.get_connection() as connection:
+            cursor = connection.cursor()
+            try:
+                yield cursor
+            finally:
+                cursor.close()
+
+
+class SnowflakeIndexerConfig(PostgresIndexerConfig):
+    pass
+
+
+@dataclass
+class SnowflakeIndexer(PostgresIndexer):
+    connection_config: SnowflakeConnectionConfig
+    index_config: SnowflakeIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+class SnowflakeDownloaderConfig(PostgresDownloaderConfig):
+    pass
+
+
+@dataclass
+class SnowflakeDownloader(PostgresDownloader):
+    connection_config: SnowflakeConnectionConfig
+    download_config: SnowflakeDownloaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+class SnowflakeUploadStagerConfig(PostgresUploadStagerConfig):
+    pass
+
+
+class SnowflakeUploadStager(PostgresUploadStager):
+    upload_stager_config: SnowflakeUploadStagerConfig
+
+
+class SnowflakeUploaderConfig(PostgresUploaderConfig):
+    pass
+
+
+@dataclass
+class SnowflakeUploader(PostgresUploader):
+    upload_config: SnowflakeUploaderConfig = field(default_factory=SnowflakeUploaderConfig)
+    connection_config: SnowflakeConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+    values_delimiter: str = "?"
+
+    def upload_contents(self, path: Path) -> None:
+        df = pd.read_json(path, orient="records", lines=True)
+        df.replace({np.nan: None}, inplace=True)
+
+        columns = list(df.columns)
+        stmt = f"INSERT INTO {self.upload_config.table_name} ({','.join(columns)}) VALUES({','.join([self.values_delimiter for x in columns])})"  # noqa E501
+
+        for rows in pd.read_json(
+            path, orient="records", lines=True, chunksize=self.upload_config.batch_size
+        ):
+            with self.connection_config.get_cursor() as cursor:
+                values = self.prepare_data(columns, tuple(rows.itertuples(index=False, name=None)))
+                # TODO: executemany break on 'Binding data in type (list) is not supported'
+                for val in values:
+                    cursor.execute(stmt, val)
+
+
+snowflake_source_entry = SourceRegistryEntry(
+    connection_config=SnowflakeConnectionConfig,
+    indexer_config=SnowflakeIndexerConfig,
+    indexer=SnowflakeIndexer,
+    downloader_config=SnowflakeDownloaderConfig,
+    downloader=SnowflakeDownloader,
+)
+
+snowflake_destination_entry = DestinationRegistryEntry(
+    connection_config=SnowflakeConnectionConfig,
+    uploader=SnowflakeUploader,
+    uploader_config=SnowflakeUploaderConfig,
+    upload_stager=SnowflakeUploadStager,
+    upload_stager_config=SnowflakeUploadStagerConfig,
+)