unstructured-ingest 0.2.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

test/integration/connectors/test_onedrive.py

@@ -0,0 +1,112 @@
+import os
+import uuid
+from pathlib import Path
+
+import pytest
+from office365.graph_client import GraphClient
+
+from test.integration.connectors.utils.constants import (
+    DESTINATION_TAG,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.processes.connectors.onedrive import (
+    CONNECTOR_TYPE,
+    OnedriveAccessConfig,
+    OnedriveConnectionConfig,
+    OnedriveUploader,
+    OnedriveUploaderConfig,
+)
+
+
+@pytest.fixture
+def onedrive_test_folder() -> str:
+    """
+    Pytest fixture that creates a test folder in OneDrive and deletes it after test run.
+    """
+    connection_config = get_connection_config()
+    user_pname = connection_config.user_pname
+
+    # Get the OneDrive client
+    client: GraphClient = connection_config.get_client()
+    drive = client.users[user_pname].drive
+
+    # Generate a unique test folder path
+    test_folder_path = f"utic-test-output-{uuid.uuid4()}"
+
+    # Create the test folder
+    root = drive.root
+    folder = root.create_folder(test_folder_path).execute_query()
+    print(f"created folder: {folder.name}")
+    try:
+        yield test_folder_path
+    finally:
+        # Teardown: delete the test folder and its contents
+        folder.delete_object().execute_query()
+        print(f"successfully deleted folder: {folder.name}")
+
+
+def get_connection_config():
+    """
+    Pytest fixture that provides the OnedriveConnectionConfig for tests.
+    """
+    client_id = os.getenv("MS_CLIENT_ID")
+    client_secret = os.getenv("MS_CLIENT_CRED")
+    tenant_id = os.getenv("MS_TENANT_ID")
+    user_pname = os.getenv("MS_USER_PNAME")
+
+    connection_config = OnedriveConnectionConfig(
+        client_id=client_id,
+        tenant=tenant_id,
+        user_pname=user_pname,
+        access_config=OnedriveAccessConfig(client_cred=client_secret),
+    )
+    return connection_config
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@requires_env("MS_CLIENT_CRED", "MS_CLIENT_ID", "MS_TENANT_ID", "MS_USER_PNAME")
+def test_onedrive_destination(upload_file: Path, onedrive_test_folder: str):
+    """
+    Integration test for the OneDrive destination connector.
+
+    This test uploads a file to OneDrive and verifies that it exists.
+    """
+    connection_config = get_connection_config()
+    # Retrieve user principal name from the connection config
+    user_pname = connection_config.user_pname
+
+    # The test folder is provided by the fixture
+    destination_folder = onedrive_test_folder
+    destination_fullpath = f"{destination_folder}/{upload_file.name}"
+
+    # Configure the uploader with remote_url
+    upload_config = OnedriveUploaderConfig(remote_url=f"onedrive://{destination_folder}")
+
+    uploader = OnedriveUploader(
+        connection_config=connection_config,
+        upload_config=upload_config,
+    )
+
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(
+            fullpath=destination_fullpath,
+            filename=upload_file.name,
+        ),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mock_file_data",
+    )
+    uploader.precheck()
+    uploader.run(path=upload_file, file_data=file_data)
+
+    # Verify that the file was uploaded
+    client = connection_config.get_client()
+    drive = client.users[user_pname].drive
+
+    uploaded_file = (
+        drive.root.get_by_path(destination_fullpath).select(["id", "name"]).get().execute_query()
+    )
+
+    # Check if the file exists
+    assert uploaded_file is not None
+    assert uploaded_file.name == upload_file.name

test/integration/connectors/test_pinecone.py

@@ -0,0 +1,161 @@
+import json
+import os
+import time
+from pathlib import Path
+from uuid import uuid4
+
+import pytest
+from pinecone import Pinecone, ServerlessSpec
+from pinecone.core.openapi.shared.exceptions import NotFoundException
+
+from test.integration.connectors.utils.constants import (
+    DESTINATION_TAG,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connectors.pinecone import (
+    CONNECTOR_TYPE,
+    PineconeAccessConfig,
+    PineconeConnectionConfig,
+    PineconeUploader,
+    PineconeUploaderConfig,
+    PineconeUploadStager,
+    PineconeUploadStagerConfig,
+)
+
+API_KEY = "PINECONE_API_KEY"
+
+
+def get_api_key() -> str:
+    api_key = os.getenv(API_KEY, None)
+    assert api_key
+    return api_key
+
+
+def wait_for_delete(client: Pinecone, index_name: str, timeout=60, interval=1) -> None:
+    start = time.time()
+    while True and time.time() - start < timeout:
+        try:
+            description = client.describe_index(name=index_name)
+            logger.info(f"current index status: {description}")
+        except NotFoundException:
+            return
+        time.sleep(interval)
+
+    raise TimeoutError("time out waiting for index to delete")
+
+
+def wait_for_ready(client: Pinecone, index_name: str, timeout=60, interval=1) -> None:
+    def is_ready_status():
+        description = client.describe_index(name=index_name)
+        status = description["status"]
+        return status["ready"]
+
+    start = time.time()
+    is_ready = is_ready_status()
+    while not is_ready and time.time() - start < timeout:
+        time.sleep(interval)
+        is_ready = is_ready_status()
+    if not is_ready:
+        raise TimeoutError("time out waiting for index to be ready")
+
+
+@pytest.fixture
+def pinecone_index() -> str:
+    pinecone = Pinecone(api_key=get_api_key())
+    random_id = str(uuid4()).split("-")[0]
+    index_name = f"ingest-test-{random_id}"
+    assert len(index_name) < 45
+    logger.info(f"Creating index: {index_name}")
+    try:
+        pinecone.create_index(
+            name=index_name,
+            dimension=384,
+            metric="cosine",
+            spec=ServerlessSpec(
+                cloud="aws",
+                region="us-east-1",
+            ),
+            deletion_protection="disabled",
+        )
+        wait_for_ready(client=pinecone, index_name=index_name)
+        yield index_name
+    except Exception as e:
+        logger.error(f"failed to create index {index_name}: {e}")
+    finally:
+        try:
+            logger.info(f"deleting index: {index_name}")
+            pinecone.delete_index(name=index_name)
+            wait_for_delete(client=pinecone, index_name=index_name)
+        except NotFoundException:
+            return
+
+
+def validate_pinecone_index(
+    index_name: str, expected_num_of_vectors: int, retries=30, interval=1
+) -> None:
+    # Because there's a delay for the index to catch up to the recent writes, add in a retry
+    pinecone = Pinecone(api_key=get_api_key())
+    index = pinecone.Index(name=index_name)
+    vector_count = -1
+    for i in range(retries):
+        index_stats = index.describe_index_stats()
+        vector_count = index_stats["total_vector_count"]
+        if vector_count == expected_num_of_vectors:
+            logger.info(f"expected {expected_num_of_vectors} == vector count {vector_count}")
+            break
+        logger.info(
+            f"retry attempt {i}: expected {expected_num_of_vectors} != vector count {vector_count}"
+        )
+        time.sleep(interval)
+    assert vector_count == expected_num_of_vectors
+
+
+@requires_env(API_KEY)
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+async def test_pinecone_destination(pinecone_index: str, upload_file: Path, temp_dir: Path):
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=CONNECTOR_TYPE,
+        identifier="pinecone_mock_id",
+    )
+    connection_config = PineconeConnectionConfig(
+        index_name=pinecone_index,
+        access_config=PineconeAccessConfig(api_key=get_api_key()),
+    )
+    stager_config = PineconeUploadStagerConfig()
+    stager = PineconeUploadStager(upload_stager_config=stager_config)
+    new_upload_file = stager.run(
+        elements_filepath=upload_file,
+        output_dir=temp_dir,
+        output_filename=upload_file.name,
+        file_data=file_data,
+    )
+
+    upload_config = PineconeUploaderConfig()
+    uploader = PineconeUploader(connection_config=connection_config, upload_config=upload_config)
+    uploader.precheck()
+
+    if uploader.is_async():
+        await uploader.run_async(path=new_upload_file, file_data=file_data)
+    else:
+        uploader.run(path=new_upload_file, file_data=file_data)
+    with new_upload_file.open() as f:
+        staged_content = json.load(f)
+    expected_num_of_vectors = len(staged_content)
+    logger.info("validating first upload")
+    validate_pinecone_index(
+        index_name=pinecone_index, expected_num_of_vectors=expected_num_of_vectors
+    )
+
+    # Rerun uploader and make sure no duplicates exist
+    if uploader.is_async():
+        await uploader.run_async(path=new_upload_file, file_data=file_data)
+    else:
+        uploader.run(path=new_upload_file, file_data=file_data)
+    logger.info("validating second upload")
+    validate_pinecone_index(
+        index_name=pinecone_index, expected_num_of_vectors=expected_num_of_vectors
+    )

test/integration/connectors/test_qdrant.py

@@ -0,0 +1,137 @@
+import json
+import uuid
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import AsyncGenerator
+
+import pytest
+from qdrant_client import AsyncQdrantClient
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.docker import container_context
+from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.v2.processes.connectors.qdrant.local import (
+    CONNECTOR_TYPE as LOCAL_CONNECTOR_TYPE,
+)
+from unstructured_ingest.v2.processes.connectors.qdrant.local import (
+    LocalQdrantConnectionConfig,
+    LocalQdrantUploader,
+    LocalQdrantUploaderConfig,
+    LocalQdrantUploadStager,
+    LocalQdrantUploadStagerConfig,
+)
+from unstructured_ingest.v2.processes.connectors.qdrant.server import (
+    CONNECTOR_TYPE as SERVER_CONNECTOR_TYPE,
+)
+from unstructured_ingest.v2.processes.connectors.qdrant.server import (
+    ServerQdrantConnectionConfig,
+    ServerQdrantUploader,
+    ServerQdrantUploaderConfig,
+    ServerQdrantUploadStager,
+    ServerQdrantUploadStagerConfig,
+)
+
+COLLECTION_NAME = f"test-coll-{uuid.uuid4().hex[:12]}"
+VECTORS_CONFIG = {"size": 384, "distance": "Cosine"}
+
+
+@asynccontextmanager
+async def qdrant_client(client_params: dict) -> AsyncGenerator[AsyncQdrantClient, None]:
+    client = AsyncQdrantClient(**client_params)
+    try:
+        yield client
+    finally:
+        await client.close()
+
+
+async def validate_upload(client: AsyncQdrantClient, upload_file: Path):
+    with upload_file.open() as upload_fp:
+        elements = json.load(upload_fp)
+    expected_point_count = len(elements)
+    first_element = elements[0]
+    expected_text = first_element["text"]
+    embeddings = first_element["embeddings"]
+    collection = await client.get_collection(COLLECTION_NAME)
+    assert collection.points_count == expected_point_count
+
+    response = await client.query_points(COLLECTION_NAME, query=embeddings, limit=1)
+    assert response.points[0].payload is not None
+    assert response.points[0].payload["text"] == expected_text
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(LOCAL_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant")
+async def test_qdrant_destination_local(upload_file: Path, tmp_path: Path):
+    connection_kwargs = {"path": str(tmp_path / "qdrant")}
+    async with qdrant_client(connection_kwargs) as client:
+        await client.create_collection(COLLECTION_NAME, vectors_config=VECTORS_CONFIG)
+    AsyncQdrantClient(**connection_kwargs)
+    stager = LocalQdrantUploadStager(
+        upload_stager_config=LocalQdrantUploadStagerConfig(),
+    )
+    uploader = LocalQdrantUploader(
+        connection_config=LocalQdrantConnectionConfig(**connection_kwargs),
+        upload_config=LocalQdrantUploaderConfig(collection_name=COLLECTION_NAME),
+    )
+
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=LOCAL_CONNECTOR_TYPE,
+        identifier="mock-file-data",
+    )
+
+    staged_upload_file = stager.run(
+        elements_filepath=upload_file,
+        file_data=file_data,
+        output_dir=tmp_path,
+        output_filename=upload_file.name,
+    )
+
+    if uploader.is_async():
+        await uploader.run_async(path=staged_upload_file, file_data=file_data)
+    else:
+        uploader.run(path=upload_file, file_data=file_data)
+    async with qdrant_client(connection_kwargs) as client:
+        await validate_upload(client=client, upload_file=upload_file)
+
+
+@pytest.fixture
+def docker_context():
+    with container_context(image="qdrant/qdrant:latest", ports={"6333": "6333"}) as container:
+        yield container
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(SERVER_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant")
+async def test_qdrant_destination_server(upload_file: Path, tmp_path: Path, docker_context):
+    connection_kwargs = {"location": "http://localhost:6333"}
+    async with qdrant_client(connection_kwargs) as client:
+        await client.create_collection(COLLECTION_NAME, vectors_config=VECTORS_CONFIG)
+    AsyncQdrantClient(**connection_kwargs)
+    stager = ServerQdrantUploadStager(
+        upload_stager_config=ServerQdrantUploadStagerConfig(),
+    )
+    uploader = ServerQdrantUploader(
+        connection_config=ServerQdrantConnectionConfig(**connection_kwargs),
+        upload_config=ServerQdrantUploaderConfig(collection_name=COLLECTION_NAME),
+    )
+
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=SERVER_CONNECTOR_TYPE,
+        identifier="mock-file-data",
+    )
+
+    staged_upload_file = stager.run(
+        elements_filepath=upload_file,
+        file_data=file_data,
+        output_dir=tmp_path,
+        output_filename=upload_file.name,
+    )
+
+    if uploader.is_async():
+        await uploader.run_async(path=staged_upload_file, file_data=file_data)
+    else:
+        uploader.run(path=upload_file, file_data=file_data)
+    async with qdrant_client(connection_kwargs) as client:
+        await validate_upload(client=client, upload_file=upload_file)

test/integration/connectors/test_s3.py

@@ -71,6 +71,29 @@ async def test_s3_source(anon_connection_config: S3ConnectionConfig):
         )
 
 
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+async def test_s3_source_special_char(anon_connection_config: S3ConnectionConfig):
+    indexer_config = S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/special-characters/")
+    with tempfile.TemporaryDirectory() as tempdir:
+        tempdir_path = Path(tempdir)
+        download_config = S3DownloaderConfig(download_dir=tempdir_path)
+        indexer = S3Indexer(connection_config=anon_connection_config, index_config=indexer_config)
+        downloader = S3Downloader(
+            connection_config=anon_connection_config, download_config=download_config
+        )
+        await source_connector_validation(
+            indexer=indexer,
+            downloader=downloader,
+            configs=ValidationConfigs(
+                test_id="s3-specialchar",
+                predownload_file_data_check=validate_predownload_file_data,
+                postdownload_file_data_check=validate_postdownload_file_data,
+                expected_num_files=1,
+            ),
+        )
+
+
 @pytest.mark.asyncio
 @pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
 async def test_s3_source_no_access(anon_connection_config: S3ConnectionConfig):

test/integration/connectors/utils/docker.py

@@ -47,14 +47,15 @@ def healthcheck_wait(container: Container, timeout: int = 10) -> None:
 
 @contextmanager
 def container_context(
-    docker_client: docker.DockerClient,
     image: str,
     ports: dict,
     environment: Optional[dict] = None,
     volumes: Optional[dict] = None,
     healthcheck: Optional[dict] = None,
     healthcheck_timeout: int = 10,
+    docker_client: Optional[docker.DockerClient] = None,
 ):
+    docker_client = docker_client or docker.from_env()
     container: Optional[Container] = None
     try:
         container = get_container(

test/integration/connectors/utils/validation.py

@@ -7,13 +7,14 @@ from pathlib import Path
 from typing import Callable, Optional
 
 import pandas as pd
+from bs4 import BeautifulSoup
 from deepdiff import DeepDiff
 
 from test.integration.connectors.utils.constants import expected_results_path
 from unstructured_ingest.v2.interfaces import Downloader, FileData, Indexer
 
 
-def pandas_df_equality_check(expected_filepath: Path, current_filepath: Path) -> bool:
+def json_equality_check(expected_filepath: Path, current_filepath: Path) -> bool:
     expected_df = pd.read_csv(expected_filepath)
     current_df = pd.read_csv(current_filepath)
     if expected_df.equals(current_df):
@@ -27,6 +28,42 @@ def pandas_df_equality_check(expected_filepath: Path, current_filepath: Path) ->
     return False
 
 
+def html_equality_check(expected_filepath: Path, current_filepath: Path) -> bool:
+    with expected_filepath.open() as expected_f:
+        expected_soup = BeautifulSoup(expected_f, "html.parser")
+    with current_filepath.open() as current_f:
+        current_soup = BeautifulSoup(current_f, "html.parser")
+    return expected_soup.text == current_soup.text
+
+
+def txt_equality_check(expected_filepath: Path, current_filepath: Path) -> bool:
+    with expected_filepath.open() as expected_f:
+        expected_text_lines = expected_f.readlines()
+    with current_filepath.open() as current_f:
+        current_text_lines = current_f.readlines()
+    if len(expected_text_lines) != len(current_text_lines):
+        print(
+            f"Lines in expected text file ({len(expected_text_lines)}) "
+            f"don't match current text file ({len(current_text_lines)})"
+        )
+        return False
+    expected_text = "\n".join(expected_text_lines)
+    current_text = "\n".join(current_text_lines)
+    if expected_text == current_text:
+        return True
+    print("txt content don't match:")
+    print(f"expected: {expected_text}")
+    print(f"current: {current_text}")
+    return False
+
+
+file_type_equality_check = {
+    ".json": json_equality_check,
+    ".html": html_equality_check,
+    ".txt": txt_equality_check,
+}
+
+
 @dataclass
 class ValidationConfigs:
     test_id: str
@@ -39,6 +76,7 @@ class ValidationConfigs:
     )
     exclude_fields_extend: list[str] = field(default_factory=list)
     validate_downloaded_files: bool = False
+    validate_file_data: bool = True
     downloaded_file_equality_check: Optional[Callable[[Path, Path], bool]] = None
 
     def get_exclude_fields(self) -> list[str]:
@@ -86,7 +124,7 @@ class ValidationConfigs:
 
 def get_files(dir_path: Path) -> list[str]:
     return [
-        str(f).replace(str(dir_path), "").lstrip("/") for f in dir_path.iterdir() if f.is_file()
+        str(f).replace(str(dir_path), "").lstrip("/") for f in dir_path.rglob("*") if f.is_file()
     ]
 
 
@@ -122,6 +160,23 @@ def check_contents(
     assert not found_diff, f"Diffs found between files: {found_diff}"
 
 
+def detect_diff(
+    configs: ValidationConfigs, expected_filepath: Path, current_filepath: Path
+) -> bool:
+    if expected_filepath.suffix != current_filepath.suffix:
+        return True
+    if downloaded_file_equality_check := configs.downloaded_file_equality_check:
+        return not downloaded_file_equality_check(expected_filepath, current_filepath)
+    current_suffix = expected_filepath.suffix
+    if current_suffix in file_type_equality_check:
+        equality_check_callable = file_type_equality_check[current_suffix]
+        return not equality_check_callable(
+            expected_filepath=expected_filepath, current_filepath=current_filepath
+        )
+    # Fallback is using filecmp.cmp to compare the files
+    return not filecmp.cmp(expected_filepath, current_filepath, shallow=False)
+
+
 def check_raw_file_contents(
     expected_output_dir: Path,
     current_output_dir: Path,
@@ -133,15 +188,7 @@ def check_raw_file_contents(
     for current_file in current_files:
         current_file_path = current_output_dir / current_file
         expected_file_path = expected_output_dir / current_file
-        if downloaded_file_equality_check := configs.downloaded_file_equality_check:
-            is_different = downloaded_file_equality_check(expected_file_path, current_file_path)
-        elif expected_file_path.suffix == ".csv" and current_file_path.suffix == ".csv":
-            is_different = not pandas_df_equality_check(
-                expected_filepath=expected_file_path, current_filepath=current_file_path
-            )
-        else:
-            is_different = not filecmp.cmp(expected_file_path, current_file_path, shallow=False)
-        if is_different:
+        if detect_diff(configs, expected_file_path, current_file_path):
             found_diff = True
             files.append(str(expected_file_path))
             print(f"diffs between files {expected_file_path} and {current_file_path}")
@@ -185,17 +232,19 @@ def update_fixtures(
     download_dir: Path,
     all_file_data: list[FileData],
     save_downloads: bool = False,
+    save_filedata: bool = True,
 ):
     # Delete current files
     shutil.rmtree(path=output_dir, ignore_errors=True)
     output_dir.mkdir(parents=True)
     # Rewrite the current file data
-    file_data_output_path = output_dir / "file_data"
-    file_data_output_path.mkdir(parents=True, exist_ok=True)
-    for file_data in all_file_data:
-        file_data_path = file_data_output_path / f"{file_data.identifier}.json"
-        with file_data_path.open(mode="w") as f:
-            json.dump(file_data.to_dict(), f, indent=2)
+    if save_filedata:
+        file_data_output_path = output_dir / "file_data"
+        file_data_output_path.mkdir(parents=True, exist_ok=True)
+        for file_data in all_file_data:
+            file_data_path = file_data_output_path / f"{file_data.identifier}.json"
+            with file_data_path.open(mode="w") as f:
+                json.dump(file_data.to_dict(), f, indent=2)
 
     # Record file structure of download directory
     download_files = get_files(dir_path=download_dir)
@@ -229,11 +278,12 @@ def run_all_validations(
             predownload_file_data=pre_data, postdownload_file_data=post_data
         )
     configs.run_download_dir_validation(download_dir=download_dir)
-    run_expected_results_validation(
-        expected_output_dir=test_output_dir / "file_data",
-        all_file_data=postdownload_file_data,
-        configs=configs,
-    )
+    if configs.validate_file_data:
+        run_expected_results_validation(
+            expected_output_dir=test_output_dir / "file_data",
+            all_file_data=postdownload_file_data,
+            configs=configs,
+        )
     download_files = get_files(dir_path=download_dir)
     download_files.sort()
     run_directory_structure_validation(
@@ -291,4 +341,5 @@ async def source_connector_validation(
             download_dir=download_dir,
             all_file_data=all_postdownload_file_data,
             save_downloads=configs.validate_downloaded_files,
+            save_filedata=configs.validate_file_data,
         )

test/unit/v2/chunkers/test_chunkers.py

@@ -0,0 +1,49 @@
+import random
+
+import faker
+import pytest
+
+from unstructured_ingest.v2.processes.chunker import Chunker, ChunkerConfig
+
+fake = faker.Faker()
+
+
+def generate_chunker_config_params() -> dict:
+    params = {}
+    random_val = random.random()
+    if random_val < 0.5:
+        params["chunking_strategy"] = fake.word() if random.random() < 0.5 else None
+        params["chunk_combine_text_under_n_chars"] = (
+            fake.random_int() if random.random() < 0.5 else None
+        )
+        params["chunk_include_orig_elements"] = fake.boolean() if random.random() < 0.5 else None
+        params["chunk_max_characters"] = fake.random_int()
+        params["chunk_multipage_sections"] = fake.boolean()
+        params["chunk_new_after_n_chars"] = fake.random_int() if random.random() < 0.5 else None
+        params["chunk_overlap"] = fake.random_int() if random.random() < 0.5 else None
+        params["chunk_overlap_all"] = fake.boolean() if random.random() < 0.5 else None
+    if random_val < 0.5:
+        params["chunk_by_api"] = True
+        params["chunking_endpoint"] = fake.url()
+        params["chunk_api_key"] = fake.password()
+    else:
+        params["chunk_by_api"] = False
+
+    return params
+
+
+@pytest.mark.parametrize(
+    "partition_config_params", [generate_chunker_config_params() for i in range(10)]
+)
+def test_chunker_config(partition_config_params: dict):
+    chunker_config = ChunkerConfig.model_validate(partition_config_params)
+    assert chunker_config
+
+
+@pytest.mark.parametrize(
+    "partition_config_params", [generate_chunker_config_params() for i in range(10)]
+)
+def test_chunker(partition_config_params: dict):
+    chunker_config = ChunkerConfig.model_validate(partition_config_params)
+    chunker = Chunker(config=chunker_config)
+    assert chunker