unstructured-ingest 1.2.32__py3-none-any.whl

unstructured_ingest/processes/connectors/slack.py

@@ -0,0 +1,249 @@
+import hashlib
+import time
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Generator, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.data_types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
+from unstructured_ingest.error import SourceConnectionError, ValueError
+from unstructured_ingest.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    DownloadResponse,
+    Indexer,
+    IndexerConfig,
+)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import SourceRegistryEntry
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from slack_sdk import WebClient
+    from slack_sdk.web.async_client import AsyncWebClient
+
+# NOTE: Pagination limit set to the upper end of the recommended range
+# https://api.slack.com/apis/pagination#facts
+PAGINATION_LIMIT = 200
+
+CONNECTOR_TYPE = "slack"
+
+
+class SlackAccessConfig(AccessConfig):
+    token: str = Field(
+        description="Bot token used to access Slack API, must have channels:history scope for the"
+        " bot user."
+    )
+
+
+class SlackConnectionConfig(ConnectionConfig):
+    access_config: Secret[SlackAccessConfig]
+
+    @requires_dependencies(["slack_sdk"], extras="slack")
+    @SourceConnectionError.wrap
+    def get_client(self) -> "WebClient":
+        from slack_sdk import WebClient
+
+        return WebClient(token=self.access_config.get_secret_value().token)
+
+    @requires_dependencies(["slack_sdk"], extras="slack")
+    @SourceConnectionError.wrap
+    def get_async_client(self) -> "AsyncWebClient":
+        from slack_sdk.web.async_client import AsyncWebClient
+
+        return AsyncWebClient(token=self.access_config.get_secret_value().token)
+
+
+class SlackIndexerConfig(IndexerConfig):
+    channels: list[str] = Field(
+        description="Comma-delimited list of Slack channel IDs to pull messages from, can be"
+        " both public or private channels."
+    )
+    start_date: Optional[datetime] = Field(
+        default=None,
+        description="Start date/time in formats YYYY-MM-DD[T]HH:MM[:SS[.ffffff]][Z or [±]HH[:]MM]"
+        " or YYYY-MM-DD",
+    )
+    end_date: Optional[datetime] = Field(
+        default=None,
+        description="End date/time in formats YYYY-MM-DD[T]HH:MM[:SS[.ffffff]][Z or [±]HH[:]MM]"
+        " or YYYY-MM-DD",
+    )
+
+
+@dataclass
+class SlackIndexer(Indexer):
+    index_config: SlackIndexerConfig
+    connection_config: SlackConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        client = self.connection_config.get_client()
+        for channel in self.index_config.channels:
+            messages = []
+            oldest = (
+                str(self.index_config.start_date.timestamp())
+                if self.index_config.start_date is not None
+                else None
+            )
+            latest = (
+                str(self.index_config.end_date.timestamp())
+                if self.index_config.end_date is not None
+                else None
+            )
+            for conversation_history in client.conversations_history(
+                channel=channel,
+                oldest=oldest,
+                latest=latest,
+                limit=PAGINATION_LIMIT,
+            ):
+                messages = conversation_history.get("messages", [])
+                if messages:
+                    yield self._messages_to_file_data(messages, channel)
+
+    def _messages_to_file_data(
+        self,
+        messages: list[dict],
+        channel: str,
+    ) -> FileData:
+        ts_oldest = min((message["ts"] for message in messages), key=lambda m: float(m))
+        ts_newest = max((message["ts"] for message in messages), key=lambda m: float(m))
+
+        identifier_base = f"{channel}-{ts_oldest}-{ts_newest}"
+        identifier = hashlib.sha256(identifier_base.encode("utf-8")).hexdigest()
+        filename = identifier[:16]
+
+        source_identifiers = SourceIdentifiers(
+            filename=f"{filename}.xml", fullpath=f"{filename}.xml"
+        )
+        return FileData(
+            identifier=identifier,
+            connector_type=CONNECTOR_TYPE,
+            source_identifiers=source_identifiers,
+            metadata=FileDataSourceMetadata(
+                date_created=ts_oldest,
+                date_modified=ts_newest,
+                date_processed=str(time.time()),
+                record_locator={
+                    "channel": channel,
+                    "oldest": ts_oldest,
+                    "latest": ts_newest,
+                },
+            ),
+            display_name=source_identifiers.fullpath,
+        )
+
+    @SourceConnectionError.wrap
+    def precheck(self) -> None:
+        client = self.connection_config.get_client()
+        for channel in self.index_config.channels:
+            # NOTE: Querying conversations history guarantees that the bot is in the channel
+            client.conversations_history(channel=channel, limit=1)
+
+
+class SlackDownloaderConfig(DownloaderConfig):
+    pass
+
+
+@dataclass
+class SlackDownloader(Downloader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: SlackConnectionConfig
+    download_config: SlackDownloaderConfig = field(default_factory=SlackDownloaderConfig)
+
+    def run(self, file_data, **kwargs):
+        raise NotImplementedError
+
+    async def run_async(self, file_data: FileData, **kwargs) -> DownloadResponse:
+        # NOTE: Indexer should provide source identifiers required to generate the download path
+        download_path = self.get_download_path(file_data)
+        if download_path is None:
+            logger.error(
+                "Generated download path is None, source_identifiers might be missingfrom FileData."
+            )
+            raise ValueError("Generated invalid download path.")
+
+        await self._download_conversation(file_data, download_path)
+        return self.generate_download_response(file_data, download_path)
+
+    def is_async(self):
+        return True
+
+    async def _download_conversation(self, file_data: FileData, download_path: Path) -> None:
+        # NOTE: Indexer should supply the record locator in metadata
+        if (
+            file_data.metadata.record_locator is None
+            or "channel" not in file_data.metadata.record_locator
+            or "oldest" not in file_data.metadata.record_locator
+            or "latest" not in file_data.metadata.record_locator
+        ):
+            logger.error(
+                f"Invalid record locator in metadata: {file_data.metadata.record_locator}."
+                "Keys 'channel', 'oldest' and 'latest' must be present."
+            )
+            raise ValueError("Invalid record locator.")
+
+        client = self.connection_config.get_async_client()
+        messages = []
+        async for conversation_history in await client.conversations_history(
+            channel=file_data.metadata.record_locator["channel"],
+            oldest=file_data.metadata.record_locator["oldest"],
+            latest=file_data.metadata.record_locator["latest"],
+            limit=PAGINATION_LIMIT,
+            # NOTE: In order to get the exact same range of messages as indexer, it provides
+            # timestamps of oldest and newest messages, inclusive=True is necessary to include them
+            inclusive=True,
+        ):
+            messages += conversation_history.get("messages", [])
+
+        conversation = []
+        for message in messages:
+            thread_messages = []
+            async for conversations_replies in await client.conversations_replies(
+                channel=file_data.metadata.record_locator["channel"],
+                ts=message["ts"],
+                limit=PAGINATION_LIMIT,
+            ):
+                thread_messages += conversations_replies.get("messages", [])
+
+            # NOTE: Replies contains the whole thread, including the message references by the `ts`
+            # parameter even if it's the only message (there were no replies).
+            # Reference: https://api.slack.com/methods/conversations.replies#markdown
+            conversation.append(thread_messages)
+
+        conversation_xml = self._conversation_to_xml(conversation)
+        download_path.parent.mkdir(exist_ok=True, parents=True)
+        conversation_xml.write(download_path, encoding="utf-8", xml_declaration=True)
+
+    def _conversation_to_xml(self, conversation: list[list[dict]]) -> ET.ElementTree:
+        root = ET.Element("messages")
+
+        for thread in conversation:
+            message, *replies = thread
+            message_elem = ET.SubElement(root, "message")
+            text_elem = ET.SubElement(message_elem, "text")
+            text_elem.text = message.get("text")
+
+            for reply in replies:
+                reply_msg = reply.get("text", "")
+                text_elem.text = "".join([str(text_elem.text), " <reply> ", reply_msg])
+
+        return ET.ElementTree(root)
+
+
+slack_source_entry = SourceRegistryEntry(
+    indexer=SlackIndexer,
+    indexer_config=SlackIndexerConfig,
+    downloader=SlackDownloader,
+    downloader_config=DownloaderConfig,
+    connection_config=SlackConnectionConfig,
+)

unstructured_ingest/processes/connectors/sql/__init__.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from unstructured_ingest.processes.connector_registry import (
+    add_destination_entry,
+    add_source_entry,
+)
+
+from .databricks_delta_tables import CONNECTOR_TYPE as DATABRICKS_DELTA_TABLES_CONNECTOR_TYPE
+from .databricks_delta_tables import databricks_delta_tables_destination_entry
+from .postgres import CONNECTOR_TYPE as POSTGRES_CONNECTOR_TYPE
+from .postgres import postgres_destination_entry, postgres_source_entry
+from .singlestore import CONNECTOR_TYPE as SINGLESTORE_CONNECTOR_TYPE
+from .singlestore import singlestore_destination_entry, singlestore_source_entry
+from .snowflake import CONNECTOR_TYPE as SNOWFLAKE_CONNECTOR_TYPE
+from .snowflake import snowflake_destination_entry, snowflake_source_entry
+from .sqlite import CONNECTOR_TYPE as SQLITE_CONNECTOR_TYPE
+from .sqlite import sqlite_destination_entry, sqlite_source_entry
+from .teradata import CONNECTOR_TYPE as TERADATA_CONNECTOR_TYPE
+from .teradata import teradata_destination_entry, teradata_source_entry
+from .vastdb import CONNECTOR_TYPE as VASTDB_CONNECTOR_TYPE
+from .vastdb import vastdb_destination_entry, vastdb_source_entry
+
+add_source_entry(source_type=SQLITE_CONNECTOR_TYPE, entry=sqlite_source_entry)
+add_source_entry(source_type=POSTGRES_CONNECTOR_TYPE, entry=postgres_source_entry)
+add_source_entry(source_type=SNOWFLAKE_CONNECTOR_TYPE, entry=snowflake_source_entry)
+add_source_entry(source_type=SINGLESTORE_CONNECTOR_TYPE, entry=singlestore_source_entry)
+add_source_entry(source_type=TERADATA_CONNECTOR_TYPE, entry=teradata_source_entry)
+add_source_entry(source_type=VASTDB_CONNECTOR_TYPE, entry=vastdb_source_entry)
+
+add_destination_entry(destination_type=SQLITE_CONNECTOR_TYPE, entry=sqlite_destination_entry)
+add_destination_entry(destination_type=POSTGRES_CONNECTOR_TYPE, entry=postgres_destination_entry)
+add_destination_entry(destination_type=SNOWFLAKE_CONNECTOR_TYPE, entry=snowflake_destination_entry)
+add_destination_entry(
+    destination_type=SINGLESTORE_CONNECTOR_TYPE, entry=singlestore_destination_entry
+)
+add_destination_entry(
+    destination_type=DATABRICKS_DELTA_TABLES_CONNECTOR_TYPE,
+    entry=databricks_delta_tables_destination_entry,
+)
+add_destination_entry(destination_type=TERADATA_CONNECTOR_TYPE, entry=teradata_destination_entry)
+add_destination_entry(destination_type=VASTDB_CONNECTOR_TYPE, entry=vastdb_destination_entry)

unstructured_ingest/processes/connectors/sql/databricks_delta_tables.py

@@ -0,0 +1,228 @@
+import json
+import os
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Generator, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.error import ValueError
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
+    DestinationRegistryEntry,
+)
+from unstructured_ingest.processes.connectors.sql.sql import (
+    SQLAccessConfig,
+    SQLConnectionConfig,
+    SQLUploader,
+    SQLUploaderConfig,
+    SQLUploadStager,
+    SQLUploadStagerConfig,
+)
+from unstructured_ingest.utils.data_prep import split_dataframe
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from databricks.sdk.core import oauth_service_principal
+    from databricks.sql.client import Connection as DeltaTableConnection
+    from databricks.sql.client import Cursor as DeltaTableCursor
+    from pandas import DataFrame
+
+CONNECTOR_TYPE = "databricks_delta_tables"
+
+
+class DatabricksDeltaTablesAccessConfig(SQLAccessConfig):
+    token: Optional[str] = Field(default=None, description="Databricks Personal Access Token")
+    client_id: Optional[str] = Field(default=None, description="Client ID of the OAuth app.")
+    client_secret: Optional[str] = Field(
+        default=None, description="Client Secret of the OAuth app."
+    )
+
+
+class DatabricksDeltaTablesConnectionConfig(SQLConnectionConfig):
+    access_config: Secret[DatabricksDeltaTablesAccessConfig]
+    server_hostname: str = Field(description="server hostname connection config value")
+    http_path: str = Field(description="http path connection config value")
+
+    @requires_dependencies(["databricks"], extras="databricks-delta-tables")
+    def get_credentials_provider(self) -> "oauth_service_principal":
+        from databricks.sdk.core import Config, oauth_service_principal
+
+        host = f"https://{self.server_hostname}"
+        access_configs = self.access_config.get_secret_value()
+        client_id = access_configs.client_id
+        client_secret = access_configs.client_secret
+
+        def _get_credentials_provider():
+            return oauth_service_principal(
+                Config(
+                    host=host,
+                    client_id=client_id,
+                    client_secret=client_secret,
+                )
+            )
+
+        if client_id and client_secret:
+            return _get_credentials_provider
+
+        return False
+
+    def model_post_init(self, __context: Any) -> None:
+        access_config = self.access_config.get_secret_value()
+        if access_config.token and access_config.client_secret and access_config.client_id:
+            raise ValueError(
+                "One one for of auth can be provided, either token or client id and secret"
+            )
+        if not access_config.token and not (
+            access_config.client_secret and access_config.client_id
+        ):
+            raise ValueError(
+                "One form of auth must be provided, either token or client id and secret"
+            )
+
+    @contextmanager
+    @requires_dependencies(["databricks"], extras="databricks-delta-tables")
+    def get_connection(self, **connect_kwargs) -> Generator["DeltaTableConnection", None, None]:
+        from databricks.sql import connect
+
+        connect_kwargs = connect_kwargs or {}
+        connect_kwargs["_user_agent_entry"] = os.getenv(
+            "UNSTRUCTURED_USER_AGENT", "unstructuredio_oss"
+        )
+        connect_kwargs["server_hostname"] = connect_kwargs.get(
+            "server_hostname", self.server_hostname
+        )
+        connect_kwargs["http_path"] = connect_kwargs.get("http_path", self.http_path)
+
+        if credential_provider := self.get_credentials_provider():
+            connect_kwargs["credentials_provider"] = credential_provider
+        else:
+            connect_kwargs["access_token"] = self.access_config.get_secret_value().token
+        with connect(**connect_kwargs) as connection:
+            yield connection
+
+    @contextmanager
+    def get_cursor(self, **connect_kwargs) -> Generator["DeltaTableCursor", None, None]:
+        with self.get_connection(**connect_kwargs) as connection:
+            cursor = connection.cursor()
+            yield cursor
+
+
+class DatabricksDeltaTablesUploadStagerConfig(SQLUploadStagerConfig):
+    pass
+
+
+class DatabricksDeltaTablesUploadStager(SQLUploadStager):
+    upload_stager_config: DatabricksDeltaTablesUploadStagerConfig
+
+
+class DatabricksDeltaTablesUploaderConfig(SQLUploaderConfig):
+    catalog: str = Field(description="Name of the catalog in the Databricks Unity Catalog service")
+    database: str = Field(description="Database name", default="default")
+    table_name: str = Field(description="Table name")
+
+
+@dataclass
+class DatabricksDeltaTablesUploader(SQLUploader):
+    upload_config: DatabricksDeltaTablesUploaderConfig
+    connection_config: DatabricksDeltaTablesConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    @requires_dependencies(["pandas"], extras="databricks-delta-tables")
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        super().run(path=path, file_data=file_data, **kwargs)
+
+    @contextmanager
+    def get_cursor(self) -> Generator[Any, None, None]:
+        with self.connection_config.get_cursor() as cursor:
+            cursor.execute(f"USE CATALOG '{self.upload_config.catalog}'")
+            yield cursor
+
+    def precheck(self) -> None:
+        with self.connection_config.get_cursor() as cursor:
+            cursor.execute("SHOW CATALOGS")
+            catalogs = [r[0] for r in cursor.fetchall()]
+            if self.upload_config.catalog not in catalogs:
+                raise ValueError(
+                    "Catalog {} not found in {}".format(
+                        self.upload_config.catalog, ", ".join(catalogs)
+                    )
+                )
+            cursor.execute(f"USE CATALOG '{self.upload_config.catalog}'")
+            cursor.execute("SHOW DATABASES")
+            databases = [r[0] for r in cursor.fetchall()]
+            if self.upload_config.database not in databases:
+                raise ValueError(
+                    "Database {} not found in {}".format(
+                        self.upload_config.database, ", ".join(databases)
+                    )
+                )
+            cursor.execute("SHOW TABLES")
+            table_names = [r[1] for r in cursor.fetchall()]
+            if self.upload_config.table_name not in table_names:
+                raise ValueError(
+                    "Table {} not found in {}".format(
+                        self.upload_config.table_name, ", ".join(table_names)
+                    )
+                )
+
+    def create_statement(self, columns: list[str], values: tuple[Any, ...]) -> str:
+        values_list = []
+        for v in values:
+            if isinstance(v, dict):
+                values_list.append(json.dumps(v))
+            elif isinstance(v, list):
+                if v and isinstance(v[0], (int, float)):
+                    values_list.append("ARRAY({})".format(", ".join([str(val) for val in v])))
+                else:
+                    values_list.append("ARRAY({})".format(", ".join([f"'{val}'" for val in v])))
+            else:
+                values_list.append(f"'{v}'")
+        statement = "INSERT INTO {table_name} ({columns}) VALUES({values})".format(
+            table_name=self.upload_config.table_name,
+            columns=", ".join(columns),
+            values=", ".join(values_list),
+        )
+        return statement
+
+    @requires_dependencies(["pandas"], extras="databricks-delta-tables")
+    def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
+        import numpy as np
+
+        if self.can_delete():
+            self.delete_by_record_id(file_data=file_data)
+        else:
+            logger.warning(
+                f"table doesn't contain expected "
+                f"record id column "
+                f"{self.upload_config.record_id_key}, skipping delete"
+            )
+        df.replace({np.nan: None}, inplace=True)
+        self._fit_to_schema(df=df)
+
+        columns = list(df.columns)
+        logger.info(
+            f"writing a total of {len(df)} elements via"
+            f" document batches to destination"
+            f" table named {self.upload_config.table_name}"
+            # f" with batch size {self.upload_config.batch_size}"
+        )
+        # TODO: currently variable binding not supporting for list data_types,
+        #  update once that gets resolved in SDK
+        for rows in split_dataframe(df=df, chunk_size=self.upload_config.batch_size):
+            with self.get_cursor() as cursor:
+                values = self.prepare_data(columns, tuple(rows.itertuples(index=False, name=None)))
+                for v in values:
+                    stmt = self.create_statement(columns=columns, values=v)
+                    cursor.execute(stmt)
+
+
+databricks_delta_tables_destination_entry = DestinationRegistryEntry(
+    connection_config=DatabricksDeltaTablesConnectionConfig,
+    uploader=DatabricksDeltaTablesUploader,
+    uploader_config=DatabricksDeltaTablesUploaderConfig,
+    upload_stager=DatabricksDeltaTablesUploadStager,
+    upload_stager_config=DatabricksDeltaTablesUploadStagerConfig,
+)

unstructured_ingest/processes/connectors/sql/postgres.py

@@ -0,0 +1,168 @@
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Generator, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.processes.connectors.sql.sql import (
+    SQLAccessConfig,
+    SqlBatchFileData,
+    SQLConnectionConfig,
+    SQLDownloader,
+    SQLDownloaderConfig,
+    SQLIndexer,
+    SQLIndexerConfig,
+    SQLUploader,
+    SQLUploaderConfig,
+    SQLUploadStager,
+    SQLUploadStagerConfig,
+)
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from psycopg2.extensions import connection as PostgresConnection
+    from psycopg2.extensions import cursor as PostgresCursor
+
+CONNECTOR_TYPE = "postgres"
+
+
+class PostgresAccessConfig(SQLAccessConfig):
+    password: Optional[str] = Field(default=None, description="DB password")
+
+
+class PostgresConnectionConfig(SQLConnectionConfig):
+    access_config: Secret[PostgresAccessConfig] = Field(
+        default=PostgresAccessConfig(), validate_default=True
+    )
+    database: Optional[str] = Field(
+        default=None,
+        description="Database name.",
+    )
+    username: Optional[str] = Field(default=None, description="DB username")
+    host: Optional[str] = Field(default=None, description="DB host")
+    port: Optional[int] = Field(default=5432, description="DB host connection port")
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+
+    @contextmanager
+    @requires_dependencies(["psycopg2"], extras="postgres")
+    def get_connection(self) -> Generator["PostgresConnection", None, None]:
+        from psycopg2 import connect
+
+        access_config = self.access_config.get_secret_value()
+        connection = connect(
+            user=self.username,
+            password=access_config.password,
+            dbname=self.database,
+            host=self.host,
+            port=self.port,
+        )
+        try:
+            yield connection
+        finally:
+            connection.commit()
+            connection.close()
+
+    @contextmanager
+    def get_cursor(self) -> Generator["PostgresCursor", None, None]:
+        with self.get_connection() as connection:
+            cursor = connection.cursor()
+            try:
+                yield cursor
+            finally:
+                cursor.close()
+
+
+class PostgresIndexerConfig(SQLIndexerConfig):
+    pass
+
+
+@dataclass
+class PostgresIndexer(SQLIndexer):
+    connection_config: PostgresConnectionConfig
+    index_config: PostgresIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+class PostgresDownloaderConfig(SQLDownloaderConfig):
+    pass
+
+
+@dataclass
+class PostgresDownloader(SQLDownloader):
+    connection_config: PostgresConnectionConfig
+    download_config: PostgresDownloaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    @requires_dependencies(["psycopg2"], extras="postgres")
+    def query_db(self, file_data: SqlBatchFileData) -> tuple[list[tuple], list[str]]:
+        from psycopg2 import sql
+
+        table_name = file_data.additional_metadata.table_name
+        id_column = file_data.additional_metadata.id_column
+        ids = tuple([item.identifier for item in file_data.batch_items])
+
+        with self.connection_config.get_cursor() as cursor:
+            fields = (
+                sql.SQL(",").join(sql.Identifier(field) for field in self.download_config.fields)
+                if self.download_config.fields
+                else sql.SQL("*")
+            )
+
+            query = sql.SQL("SELECT {fields} FROM {table_name} WHERE {id_column} IN %s").format(
+                fields=fields,
+                table_name=sql.Identifier(table_name),
+                id_column=sql.Identifier(id_column),
+            )
+            logger.debug(f"running query: {cursor.mogrify(query, (ids,))}")
+            cursor.execute(query, (ids,))
+            rows = cursor.fetchall()
+            columns = [col[0] for col in cursor.description]
+            return rows, columns
+
+
+class PostgresUploadStagerConfig(SQLUploadStagerConfig):
+    pass
+
+
+class PostgresUploadStager(SQLUploadStager):
+    upload_stager_config: PostgresUploadStagerConfig
+
+
+class PostgresUploaderConfig(SQLUploaderConfig):
+    pass
+
+
+@dataclass
+class PostgresUploader(SQLUploader):
+    upload_config: PostgresUploaderConfig = field(default_factory=PostgresUploaderConfig)
+    connection_config: PostgresConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+    values_delimiter: str = "%s"
+
+    @requires_dependencies(["pandas"], extras="postgres")
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        super().run(path=path, file_data=file_data, **kwargs)
+
+
+postgres_source_entry = SourceRegistryEntry(
+    connection_config=PostgresConnectionConfig,
+    indexer_config=PostgresIndexerConfig,
+    indexer=PostgresIndexer,
+    downloader_config=PostgresDownloaderConfig,
+    downloader=PostgresDownloader,
+)
+
+postgres_destination_entry = DestinationRegistryEntry(
+    connection_config=PostgresConnectionConfig,
+    uploader=PostgresUploader,
+    uploader_config=PostgresUploaderConfig,
+    upload_stager=PostgresUploadStager,
+    upload_stager_config=PostgresUploadStagerConfig,
+)