unstructured-ingest 0.7.2__py3-none-any.whl → 1.0.1__py3-none-any.whl

examples/singlestore.py

@@ -1,49 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.singlestore import (
-    CONNECTOR_TYPE,
-    SingleStoreAccessConfig,
-    SingleStoreConnectionConfig,
-    SingleStoreUploaderConfig,
-    SingleStoreUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
-        indexer_config=LocalIndexerConfig(
-            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=SingleStoreConnectionConfig(
-            access_config=SingleStoreAccessConfig(password="password"),
-            host="localhost",
-            port=3306,
-            database="ingest_test",
-            user="root",
-        ),
-        stager_config=SingleStoreUploadStagerConfig(),
-        uploader_config=SingleStoreUploaderConfig(table_name="elements"),
-    ).run()

examples/sql.py

@@ -1,90 +0,0 @@
-import os
-import sqlite3
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.sql import (
-    CONNECTOR_TYPE,
-    POSTGRESQL_DB,
-    SQLITE_DB,
-    SQLAccessConfig,
-    SQLConnectionConfig,
-    SQLUploaderConfig,
-    SQLUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-SQLITE_DB_PATH = "test-sql-db.sqlite"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-
-    configs = {
-        "context": ProcessorConfig(work_dir=str(work_dir.resolve())),
-        "indexer_config": LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
-        "downloader_config": LocalDownloaderConfig(download_dir=download_path),
-        "source_connection_config": LocalConnectionConfig(),
-        "partitioner_config": PartitionerConfig(strategy="fast"),
-        "chunker_config": ChunkerConfig(
-            chunking_strategy="by_title",
-            chunk_include_orig_elements=False,
-            chunk_max_characters=1500,
-            chunk_multipage_sections=True,
-        ),
-        "embedder_config": EmbedderConfig(embedding_provider="huggingface"),
-        "stager_config": SQLUploadStagerConfig(),
-        "uploader_config": SQLUploaderConfig(batch_size=10),
-    }
-
-    if os.path.exists(SQLITE_DB):
-        os.remove(SQLITE_DB)
-
-    connection = sqlite3.connect(database=SQLITE_DB)
-
-    query = None
-    script_path = (
-        Path(__file__).parent.parent.parent.parent.parent
-        / Path("test_e2e/env_setup/sql/sqlite-schema.sql")
-    ).resolve()
-    with open(script_path) as f:
-        query = f.read()
-    cursor = connection.cursor()
-    cursor.executescript(query)
-    connection.close()
-
-    # sqlite test first
-    Pipeline.from_configs(
-        destination_connection_config=SQLConnectionConfig(
-            db_type=SQLITE_DB,
-            database=SQLITE_DB_PATH,
-            access_config=SQLAccessConfig(),
-        ),
-        **configs,
-    ).run()
-
-    # now, pg with pgvector
-    Pipeline.from_configs(
-        destination_connection_config=SQLConnectionConfig(
-            db_type=POSTGRESQL_DB,
-            database="elements",
-            host="localhost",
-            port=5433,
-            access_config=SQLAccessConfig(username="unstructured", password="test"),
-        ),
-        **configs,
-    ).run()

examples/vectara.py

@@ -1,54 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.vectara import (
-    CONNECTOR_TYPE,
-    VectaraAccessConfig,
-    VectaraConnectionConfig,
-    VectaraUploaderConfig,
-    VectaraUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(
-            chunking_strategy="by_title",
-            chunk_include_orig_elements=False,
-            chunk_max_characters=1500,
-            chunk_multipage_sections=True,
-        ),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=VectaraConnectionConfig(
-            access_config=VectaraAccessConfig(
-                oauth_client_id="fill oauth_client_id", oauth_secret="fill oauth_secret"
-            ),
-            customer_id="fill customer_id",
-            corpus_name="fill corpus_name",
-            corpus_key="fill corpus_key",
-            token_url="fill token_url",
-        ),
-        stager_config=VectaraUploadStagerConfig(batch_size=10),
-        uploader_config=VectaraUploaderConfig(),
-    ).run()

examples/weaviate.py

@@ -1,44 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.weaviate.local import (
-    CONNECTOR_TYPE,
-    LocalWeaviateConnectionConfig,
-    LocalWeaviateUploaderConfig,
-    LocalWeaviateUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=LocalWeaviateConnectionConfig(
-            # Connects to http://localhost:8080
-        ),
-        stager_config=LocalWeaviateUploadStagerConfig(),
-        uploader_config=LocalWeaviateUploaderConfig(
-            collection="elements", batch_size=10, dynamic_batch=False
-        ),
-    ).run()

test/integration/chunkers/test_chunkers.py

@@ -1,31 +0,0 @@
-import os
-from pathlib import Path
-
-import pytest
-
-from test.integration.utils import requires_env
-from unstructured_ingest.processes.chunker import Chunker, ChunkerConfig
-
-int_test_dir = Path(__file__).parent
-assets_dir = int_test_dir / "assets"
-
-chunker_files = [path for path in assets_dir.iterdir() if path.is_file()]
-
-
-@pytest.mark.parametrize("chunker_file", chunker_files, ids=[path.name for path in chunker_files])
-@pytest.mark.parametrize("strategy", ["basic", "by_title", "by_similarity", "by_page"])
-@requires_env("UNSTRUCTURED_API_KEY", "UNSTRUCTURED_API_URL")
-@pytest.mark.asyncio
-async def test_chunker_api(chunker_file: Path, strategy: str):
-    api_key = os.getenv("UNSTRUCTURED_API_KEY")
-    api_url = os.getenv("UNSTRUCTURED_API_URL")
-
-    chunker_config = ChunkerConfig(
-        chunking_strategy=strategy,
-        chunk_by_api=True,
-        chunk_api_key=api_key,
-        chunking_endpoint=api_url,
-    )
-    chunker = Chunker(config=chunker_config)
-    results = await chunker.run_async(elements_filepath=chunker_file)
-    assert results

test/integration/connectors/conftest.py

@@ -1,38 +0,0 @@
-import tempfile
-from pathlib import Path
-from typing import Generator
-
-import pytest
-
-from unstructured_ingest.logger import logger
-
-FILENAME = Path("DA-1p-with-duplicate-pages.pdf.json")
-
-
-@pytest.fixture
-def upload_file() -> Path:
-    int_test_dir = Path(__file__).parent
-    assets_dir = int_test_dir / "assets"
-    upload_file = assets_dir / FILENAME
-    assert upload_file.exists()
-    assert upload_file.is_file()
-    return upload_file
-
-
-@pytest.fixture
-def upload_file_ndjson() -> Path:
-    int_test_dir = Path(__file__).parent
-    assets_dir = int_test_dir / "assets"
-    upload_file = assets_dir / FILENAME.with_suffix(".ndjson")
-    assert upload_file.exists()
-    assert upload_file.is_file()
-    return upload_file
-
-
-@pytest.fixture
-def temp_dir() -> Generator[Path, None, None]:
-    with tempfile.TemporaryDirectory() as temp_dir:
-        temp_path = Path(temp_dir)
-        logger.info(f"Created temp dir '{temp_path}'")
-        yield temp_path
-        logger.info(f"Removing temp dir '{temp_path}'")

test/integration/connectors/databricks/test_volumes_native.py

@@ -1,273 +0,0 @@
-import json
-import os
-import uuid
-from contextlib import contextmanager
-from dataclasses import dataclass
-from pathlib import Path
-from unittest import mock
-
-import pytest
-from databricks.sdk import WorkspaceClient
-from databricks.sdk.errors.platform import NotFound
-
-from test.integration.connectors.utils.constants import (
-    BLOB_STORAGE_TAG,
-    DESTINATION_TAG,
-    SOURCE_TAG,
-)
-from test.integration.connectors.utils.validation.source import (
-    SourceValidationConfigs,
-    source_connector_validation,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
-from unstructured_ingest.errors_v2 import UserAuthError, UserError
-from unstructured_ingest.processes.connectors.databricks.volumes_native import (
-    CONNECTOR_TYPE,
-    DatabricksNativeVolumesAccessConfig,
-    DatabricksNativeVolumesConnectionConfig,
-    DatabricksNativeVolumesDownloader,
-    DatabricksNativeVolumesDownloaderConfig,
-    DatabricksNativeVolumesIndexer,
-    DatabricksNativeVolumesIndexerConfig,
-    DatabricksNativeVolumesUploader,
-    DatabricksNativeVolumesUploaderConfig,
-)
-
-
-@dataclass
-class BaseEnvData:
-    host: str
-    catalog: str
-
-
-@dataclass
-class BasicAuthEnvData(BaseEnvData):
-    client_id: str
-    client_secret: str
-
-    def get_connection_config(self) -> DatabricksNativeVolumesConnectionConfig:
-        return DatabricksNativeVolumesConnectionConfig(
-            host=self.host,
-            access_config=DatabricksNativeVolumesAccessConfig(
-                client_id=self.client_id,
-                client_secret=self.client_secret,
-            ),
-        )
-
-
-@dataclass
-class PATEnvData(BaseEnvData):
-    token: str
-
-    def get_connection_config(self) -> DatabricksNativeVolumesConnectionConfig:
-        return DatabricksNativeVolumesConnectionConfig(
-            host=self.host,
-            access_config=DatabricksNativeVolumesAccessConfig(
-                token=self.token,
-            ),
-        )
-
-
-def get_basic_auth_env_data() -> BasicAuthEnvData:
-    return BasicAuthEnvData(
-        host=os.environ["DATABRICKS_HOST"],
-        client_id=os.environ["DATABRICKS_CLIENT_ID"],
-        client_secret=os.environ["DATABRICKS_CLIENT_SECRET"],
-        catalog=os.environ["DATABRICKS_CATALOG"],
-    )
-
-
-def get_pat_env_data() -> PATEnvData:
-    return PATEnvData(
-        host=os.environ["DATABRICKS_HOST"],
-        catalog=os.environ["DATABRICKS_CATALOG"],
-        token=os.environ["DATABRICKS_PAT"],
-    )
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
-@requires_env(
-    "DATABRICKS_HOST", "DATABRICKS_CLIENT_ID", "DATABRICKS_CLIENT_SECRET", "DATABRICKS_CATALOG"
-)
-async def test_volumes_native_source(tmp_path: Path):
-    env_data = get_basic_auth_env_data()
-    with mock.patch.dict(os.environ, clear=True):
-        indexer_config = DatabricksNativeVolumesIndexerConfig(
-            recursive=True,
-            volume="test-platform",
-            volume_path="databricks-volumes-test-input",
-            catalog=env_data.catalog,
-        )
-        connection_config = env_data.get_connection_config()
-        download_config = DatabricksNativeVolumesDownloaderConfig(download_dir=tmp_path)
-        indexer = DatabricksNativeVolumesIndexer(
-            connection_config=connection_config, index_config=indexer_config
-        )
-        downloader = DatabricksNativeVolumesDownloader(
-            connection_config=connection_config, download_config=download_config
-        )
-        await source_connector_validation(
-            indexer=indexer,
-            downloader=downloader,
-            configs=SourceValidationConfigs(
-                test_id="databricks_volumes_native",
-                expected_num_files=1,
-            ),
-        )
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
-@requires_env("DATABRICKS_HOST", "DATABRICKS_PAT", "DATABRICKS_CATALOG")
-async def test_volumes_native_source_pat(tmp_path: Path):
-    env_data = get_pat_env_data()
-    with mock.patch.dict(os.environ, clear=True):
-        indexer_config = DatabricksNativeVolumesIndexerConfig(
-            recursive=True,
-            volume="test-platform",
-            volume_path="databricks-volumes-test-input",
-            catalog=env_data.catalog,
-        )
-        connection_config = env_data.get_connection_config()
-        download_config = DatabricksNativeVolumesDownloaderConfig(download_dir=tmp_path)
-        indexer = DatabricksNativeVolumesIndexer(
-            connection_config=connection_config, index_config=indexer_config
-        )
-        downloader = DatabricksNativeVolumesDownloader(
-            connection_config=connection_config, download_config=download_config
-        )
-        await source_connector_validation(
-            indexer=indexer,
-            downloader=downloader,
-            configs=SourceValidationConfigs(
-                test_id="databricks_volumes_native_pat",
-                expected_num_files=1,
-            ),
-        )
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
-@requires_env("DATABRICKS_HOST", "DATABRICKS_PAT", "DATABRICKS_CATALOG")
-def test_volumes_native_source_pat_invalid_catalog():
-    env_data = get_pat_env_data()
-    with mock.patch.dict(os.environ, clear=True):
-        indexer_config = DatabricksNativeVolumesIndexerConfig(
-            recursive=True,
-            volume="test-platform",
-            volume_path="databricks-volumes-test-input",
-            catalog="fake_catalog",
-        )
-        indexer = DatabricksNativeVolumesIndexer(
-            connection_config=env_data.get_connection_config(), index_config=indexer_config
-        )
-        with pytest.raises(UserError):
-            _ = list(indexer.run())
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
-@requires_env("DATABRICKS_HOST")
-def test_volumes_native_source_pat_invalid_pat():
-    host = os.environ["DATABRICKS_HOST"]
-    with mock.patch.dict(os.environ, clear=True):
-        indexer_config = DatabricksNativeVolumesIndexerConfig(
-            recursive=True,
-            volume="test-platform",
-            volume_path="databricks-volumes-test-input",
-            catalog="fake_catalog",
-        )
-        connection_config = DatabricksNativeVolumesConnectionConfig(
-            host=host,
-            access_config=DatabricksNativeVolumesAccessConfig(
-                token="invalid-token",
-            ),
-        )
-        indexer = DatabricksNativeVolumesIndexer(
-            connection_config=connection_config, index_config=indexer_config
-        )
-        with pytest.raises(UserAuthError):
-            _ = list(indexer.run())
-
-
-def _get_volume_path(catalog: str, volume: str, volume_path: str):
-    return f"/Volumes/{catalog}/default/{volume}/{volume_path}"
-
-
-@contextmanager
-def databricks_destination_context(
-    env_data: BasicAuthEnvData, volume: str, volume_path
-) -> WorkspaceClient:
-    client = WorkspaceClient(
-        host=env_data.host, client_id=env_data.client_id, client_secret=env_data.client_secret
-    )
-    try:
-        yield client
-    finally:
-        # Cleanup
-        try:
-            for file in client.files.list_directory_contents(
-                directory_path=_get_volume_path(env_data.catalog, volume, volume_path)
-            ):
-                client.files.delete(file.path)
-            client.files.delete_directory(_get_volume_path(env_data.catalog, volume, volume_path))
-        except NotFound:
-            # Directory was never created, don't need to delete
-            pass
-
-
-def validate_upload(client: WorkspaceClient, catalog: str, volume: str, volume_path: str):
-    files = list(
-        client.files.list_directory_contents(
-            directory_path=_get_volume_path(catalog, volume, volume_path)
-        )
-    )
-
-    assert len(files) == 1
-
-    resp = client.files.download(files[0].path)
-    data = json.loads(resp.contents.read())
-
-    assert len(data) == 22
-    element_types = {v["type"] for v in data}
-    assert len(element_types) == 1
-    assert "CompositeElement" in element_types
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, BLOB_STORAGE_TAG)
-@requires_env(
-    "DATABRICKS_HOST", "DATABRICKS_CLIENT_ID", "DATABRICKS_CLIENT_SECRET", "DATABRICKS_CATALOG"
-)
-async def test_volumes_native_destination(upload_file: Path):
-    env_data = get_basic_auth_env_data()
-    volume_path = f"databricks-volumes-test-output-{uuid.uuid4()}"
-    file_data = FileData(
-        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
-        connector_type=CONNECTOR_TYPE,
-        identifier="mock file data",
-    )
-    with databricks_destination_context(
-        volume="test-platform", volume_path=volume_path, env_data=env_data
-    ) as workspace_client:
-        connection_config = env_data.get_connection_config()
-        uploader = DatabricksNativeVolumesUploader(
-            connection_config=connection_config,
-            upload_config=DatabricksNativeVolumesUploaderConfig(
-                volume="test-platform",
-                volume_path=volume_path,
-                catalog=env_data.catalog,
-            ),
-        )
-        uploader.precheck()
-        if uploader.is_async():
-            await uploader.run_async(path=upload_file, file_data=file_data)
-        else:
-            uploader.run(path=upload_file, file_data=file_data)
-
-        validate_upload(
-            client=workspace_client,
-            catalog=env_data.catalog,
-            volume="test-platform",
-            volume_path=volume_path,
-        )

test/integration/connectors/discord/test_discord.py

@@ -1,90 +0,0 @@
-import os
-import tempfile
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Optional
-
-import pytest
-
-from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
-from test.integration.connectors.utils.validation.source import (
-    SourceValidationConfigs,
-    source_connector_validation,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.error import SourceConnectionError
-from unstructured_ingest.processes.connectors.discord import (
-    CONNECTOR_TYPE,
-    DiscordAccessConfig,
-    DiscordConnectionConfig,
-    DiscordDownloader,
-    DiscordDownloaderConfig,
-    DiscordIndexer,
-    DiscordIndexerConfig,
-)
-
-
-@dataclass(frozen=True)
-class EnvData:
-    token: Optional[str]
-    channels: Optional[list[str]]
-
-
-def get_env_data() -> EnvData:
-    return EnvData(
-        token=os.getenv("DISCORD_TOKEN"),
-        channels=os.getenv("DISCORD_CHANNELS", default=[]).split(","),
-    )
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
-@requires_env("DISCORD_TOKEN", "DISCORD_CHANNELS")
-async def test_discord_source():
-    env = get_env_data()
-    indexer_config = DiscordIndexerConfig(channels=env.channels)
-    with tempfile.TemporaryDirectory() as tempdir:
-        tempdir_path = Path(tempdir)
-        connection_config = DiscordConnectionConfig(
-            access_config=DiscordAccessConfig(token=env.token)
-        )
-        download_config = DiscordDownloaderConfig(download_dir=tempdir_path)
-        indexer = DiscordIndexer(connection_config=connection_config, index_config=indexer_config)
-        downloader = DiscordDownloader(
-            connection_config=connection_config, download_config=download_config
-        )
-        expected_num_files = len(env.channels)
-        await source_connector_validation(
-            indexer=indexer,
-            downloader=downloader,
-            configs=SourceValidationConfigs(
-                test_id=CONNECTOR_TYPE,
-                expected_num_files=expected_num_files,
-                expected_number_indexed_file_data=expected_num_files,
-                validate_downloaded_files=True,
-            ),
-        )
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
-@requires_env("DISCORD_CHANNELS")
-def test_discord_source_precheck_fail_no_token():
-    indexer_config = DiscordIndexerConfig(channels=get_env_data().channels)
-
-    connection_config = DiscordConnectionConfig(access_config=DiscordAccessConfig(token=""))
-    indexer = DiscordIndexer(connection_config=connection_config, index_config=indexer_config)
-    with pytest.raises(SourceConnectionError):
-        indexer.precheck()
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
-@requires_env("DISCORD_TOKEN")
-def test_discord_source_precheck_fail_no_channels():
-    indexer_config = DiscordIndexerConfig(channels=[])
-
-    connection_config = DiscordConnectionConfig(
-        access_config=DiscordAccessConfig(token=get_env_data().token)
-    )
-    indexer = DiscordIndexer(connection_config=connection_config, index_config=indexer_config)
-    with pytest.raises(SourceConnectionError):
-        indexer.precheck()