unstructured-ingest 0.3.13__py3-none-any.whl

test/integration/connectors/utils/validation/destination.py

@@ -0,0 +1,75 @@
+import os
+import shutil
+from pathlib import Path
+
+from test.integration.connectors.utils.validation.utils import ValidationConfig
+from unstructured_ingest.utils.data_prep import get_data
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers, UploadStager
+
+
+class StagerValidationConfigs(ValidationConfig):
+    expected_count: int
+
+    def stager_output_dir(self) -> Path:
+        dir = self.test_output_dir() / "stager"
+        dir.mkdir(exist_ok=True, parents=True)
+        return dir
+
+    def stager_output_path(self, input_path: Path) -> Path:
+        return self.stager_output_dir() / input_path.name
+
+
+def run_all_stager_validations(
+    configs: StagerValidationConfigs, input_file: Path, staged_filepath: Path
+):
+    # Validate matching extensions
+    assert input_file.suffix == staged_filepath.suffix
+
+    # Validate length
+    staged_data = get_data(path=staged_filepath)
+    assert len(staged_data) == configs.expected_count
+
+    # Validate file
+    expected_filepath = configs.stager_output_path(input_path=input_file)
+    assert expected_filepath.exists(), f"{expected_filepath} does not exist"
+    assert expected_filepath.is_file(), f"{expected_filepath} is not a file"
+    if configs.detect_diff(expected_filepath=expected_filepath, current_filepath=staged_filepath):
+        raise AssertionError(
+            f"Current file ({staged_filepath}) does not match expected file: {expected_filepath}"
+        )
+
+
+def update_stager_fixtures(stager_output_path: Path, staged_filepath: Path):
+    copied_filepath = stager_output_path / staged_filepath.name
+    shutil.copy(staged_filepath, copied_filepath)
+
+
+def stager_validation(
+    stager: UploadStager,
+    tmp_dir: Path,
+    input_file: Path,
+    configs: StagerValidationConfigs,
+    overwrite_fixtures: bool = os.getenv("OVERWRITE_FIXTURES", "False").lower() == "true",
+) -> None:
+    # Run stager
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=input_file.name, filename=input_file.name),
+        connector_type=configs.test_id,
+        identifier="mock file data",
+    )
+    staged_filepath = stager.run(
+        elements_filepath=input_file,
+        file_data=file_data,
+        output_dir=tmp_dir,
+        output_filename=input_file.name,
+    )
+    if not overwrite_fixtures:
+        print("Running validation")
+        run_all_stager_validations(
+            configs=configs, input_file=input_file, staged_filepath=staged_filepath
+        )
+    else:
+        print("Running fixtures update")
+        update_stager_fixtures(
+            stager_output_path=configs.stager_output_dir(), staged_filepath=staged_filepath
+        )

test/integration/connectors/utils/validation/equality.py

@@ -0,0 +1,75 @@
+import json
+from pathlib import Path
+
+import ndjson
+from bs4 import BeautifulSoup
+from deepdiff import DeepDiff
+
+
+def json_equality_check(expected_filepath: Path, current_filepath: Path) -> bool:
+    with expected_filepath.open() as f:
+        expected_data = json.load(f)
+    with current_filepath.open() as f:
+        current_data = json.load(f)
+    diff = DeepDiff(expected_data, current_data)
+    if diff:
+        print("diff between expected and current json")
+        print(diff.to_json(indent=2))
+        return False
+    return True
+
+
+def ndjson_equality_check(expected_filepath: Path, current_filepath: Path) -> bool:
+    with expected_filepath.open() as f:
+        expected_data = ndjson.load(f)
+    with current_filepath.open() as f:
+        current_data = ndjson.load(f)
+    if len(current_data) != len(expected_data):
+        print(
+            f"expected data length {len(expected_data)} "
+            f"didn't match current results: {len(current_data)}"
+        )
+    for i in range(len(expected_data)):
+        e = expected_data[i]
+        r = current_data[i]
+        if e != r:
+            print(f"{i}th element doesn't match:\nexpected {e}\ncurrent {r}")
+            return False
+    return True
+
+
+def html_equality_check(expected_filepath: Path, current_filepath: Path) -> bool:
+    with expected_filepath.open() as expected_f:
+        expected_soup = BeautifulSoup(expected_f, "html.parser")
+    with current_filepath.open() as current_f:
+        current_soup = BeautifulSoup(current_f, "html.parser")
+    return expected_soup.text == current_soup.text
+
+
+def txt_equality_check(expected_filepath: Path, current_filepath: Path) -> bool:
+    with expected_filepath.open() as expected_f:
+        expected_text_lines = expected_f.readlines()
+    with current_filepath.open() as current_f:
+        current_text_lines = current_f.readlines()
+    if len(expected_text_lines) != len(current_text_lines):
+        print(
+            f"Lines in expected text file ({len(expected_text_lines)}) "
+            f"don't match current text file ({len(current_text_lines)})"
+        )
+        return False
+    expected_text = "\n".join(expected_text_lines)
+    current_text = "\n".join(current_text_lines)
+    if expected_text == current_text:
+        return True
+    print("txt content don't match:")
+    print(f"expected: {expected_text}")
+    print(f"current: {current_text}")
+    return False
+
+
+file_type_equality_check = {
+    ".json": json_equality_check,
+    ".ndjson": ndjson_equality_check,
+    ".html": html_equality_check,
+    ".txt": txt_equality_check,
+}

test/integration/connectors/utils/validation/source.py

@@ -0,0 +1,299 @@
+import json
+import os
+import shutil
+from pathlib import Path
+from typing import Callable, Optional
+
+from deepdiff import DeepDiff
+from pydantic import Field
+
+from test.integration.connectors.utils.validation.utils import ValidationConfig
+from unstructured_ingest.v2.interfaces import Downloader, FileData, Indexer
+
+
+class SourceValidationConfigs(ValidationConfig):
+    expected_number_indexed_file_data: Optional[int] = None
+    expected_num_files: Optional[int] = None
+    predownload_file_data_check: Optional[Callable[[FileData], None]] = None
+    postdownload_file_data_check: Optional[Callable[[FileData], None]] = None
+    exclude_fields: list[str] = Field(
+        default_factory=lambda: ["local_download_path", "metadata.date_processed"]
+    )
+    exclude_fields_extend: list[str] = Field(default_factory=list)
+    validate_downloaded_files: bool = False
+    validate_file_data: bool = True
+
+    def get_exclude_fields(self) -> list[str]:
+        exclude_fields = self.exclude_fields
+        exclude_fields.extend(self.exclude_fields_extend)
+        return exclude_fields
+
+    def run_file_data_validation(
+        self, predownload_file_data: FileData, postdownload_file_data: FileData
+    ):
+        if predownload_file_data_check := self.predownload_file_data_check:
+            predownload_file_data_check(predownload_file_data)
+        if postdownload_file_data_check := self.postdownload_file_data_check:
+            postdownload_file_data_check(postdownload_file_data)
+
+    def run_download_dir_validation(self, download_dir: Path):
+        if expected_num_files := self.expected_num_files:
+            downloaded_files = [p for p in download_dir.rglob("*") if p.is_file()]
+            assert len(downloaded_files) == expected_num_files
+
+    def omit_ignored_fields(self, data: dict) -> dict:
+        exclude_fields = self.get_exclude_fields()
+        # Ignore fields that dynamically change every time the tests run
+        copied_data = data.copy()
+        for exclude_field in exclude_fields:
+            exclude_field_vals = exclude_field.split(".")
+            if len(exclude_field_vals) == 1:
+                current_val = copied_data
+                drop_field = exclude_field_vals[0]
+                copied_data.pop(exclude_field_vals[0], None)
+            else:
+                current_val = copied_data
+                for val in exclude_field_vals[:-1]:
+                    current_val = current_val.get(val, {})
+                drop_field = exclude_field_vals[-1]
+            if drop_field == "*":
+                current_val.clear()
+            else:
+                current_val.pop(drop_field, None)
+        return copied_data
+
+
+def get_files(dir_path: Path) -> list[str]:
+    return [
+        str(f).replace(str(dir_path), "").lstrip("/") for f in dir_path.rglob("*") if f.is_file()
+    ]
+
+
+def check_files(expected_output_dir: Path, all_file_data: list[FileData]):
+    expected_files = get_files(dir_path=expected_output_dir)
+    current_files = [f"{file_data.identifier}.json" for file_data in all_file_data]
+    diff = set(expected_files) ^ set(current_files)
+    assert not diff, "diff in files that exist: {}".format(", ".join(diff))
+
+
+def check_files_in_paths(expected_output_dir: Path, current_output_dir: Path):
+    expected_files = get_files(dir_path=expected_output_dir)
+    current_files = get_files(dir_path=current_output_dir)
+    diff = set(expected_files) ^ set(current_files)
+    assert not diff, "diff in files that exist: {}".format(", ".join(diff))
+
+
+def check_contents(
+    expected_output_dir: Path, all_file_data: list[FileData], configs: SourceValidationConfigs
+):
+    found_diff = False
+    for file_data in all_file_data:
+        file_data_path = expected_output_dir / f"{file_data.identifier}.json"
+        with file_data_path.open("r") as file:
+            expected_file_data_contents = json.load(file)
+        current_file_data_contents = file_data.model_dump()
+        expected_file_data_contents = configs.omit_ignored_fields(expected_file_data_contents)
+        current_file_data_contents = configs.omit_ignored_fields(current_file_data_contents)
+        diff = DeepDiff(expected_file_data_contents, current_file_data_contents)
+        if diff:
+            found_diff = True
+            print(diff.to_json(indent=2))
+    assert not found_diff, f"Diffs found between files: {found_diff}"
+
+
+def check_raw_file_contents(
+    expected_output_dir: Path,
+    current_output_dir: Path,
+    configs: SourceValidationConfigs,
+):
+    current_files = get_files(dir_path=current_output_dir)
+    found_diff = False
+    files = []
+    for current_file in current_files:
+        current_file_path = current_output_dir / current_file
+        expected_file_path = expected_output_dir / current_file
+        if configs.detect_diff(expected_file_path, current_file_path):
+            found_diff = True
+            files.append(str(expected_file_path))
+            print(f"diffs between files {expected_file_path} and {current_file_path}")
+    assert not found_diff, "Diffs found between files: {}".format(", ".join(files))
+
+
+def run_expected_results_validation(
+    expected_output_dir: Path, all_file_data: list[FileData], configs: SourceValidationConfigs
+):
+    check_files(expected_output_dir=expected_output_dir, all_file_data=all_file_data)
+    check_contents(
+        expected_output_dir=expected_output_dir, all_file_data=all_file_data, configs=configs
+    )
+
+
+def run_expected_download_files_validation(
+    expected_output_dir: Path,
+    current_download_dir: Path,
+    configs: SourceValidationConfigs,
+):
+    check_files_in_paths(
+        expected_output_dir=expected_output_dir, current_output_dir=current_download_dir
+    )
+    check_raw_file_contents(
+        expected_output_dir=expected_output_dir,
+        current_output_dir=current_download_dir,
+        configs=configs,
+    )
+
+
+def run_directory_structure_validation(expected_output_dir: Path, download_files: list[str]):
+    directory_record = expected_output_dir / "directory_structure.json"
+    with directory_record.open("r") as directory_file:
+        directory_file_contents = json.load(directory_file)
+    directory_structure = directory_file_contents["directory_structure"]
+    assert directory_structure == download_files
+
+
+def update_fixtures(
+    output_dir: Path,
+    download_dir: Path,
+    all_file_data: list[FileData],
+    save_downloads: bool = False,
+    save_filedata: bool = True,
+):
+    # Rewrite the current file data
+    if not output_dir.exists():
+        output_dir.mkdir(parents=True)
+    if save_filedata:
+        file_data_output_path = output_dir / "file_data"
+        shutil.rmtree(path=file_data_output_path, ignore_errors=True)
+        print(
+            f"Writing {len(all_file_data)} file data to "
+            f"saved fixture location {file_data_output_path}"
+        )
+        file_data_output_path.mkdir(parents=True, exist_ok=True)
+        for file_data in all_file_data:
+            file_data_path = file_data_output_path / f"{file_data.identifier}.json"
+            with file_data_path.open(mode="w") as f:
+                json.dump(file_data.model_dump(), f, indent=2)
+
+    # Record file structure of download directory
+    download_files = get_files(dir_path=download_dir)
+    download_files.sort()
+    download_dir_record = output_dir / "directory_structure.json"
+    with download_dir_record.open(mode="w") as f:
+        json.dump({"directory_structure": download_files}, f, indent=2)
+
+    # If applicable, save raw downloads
+    if save_downloads:
+        raw_download_output_path = output_dir / "downloads"
+        shutil.rmtree(path=raw_download_output_path, ignore_errors=True)
+        print(
+            f"Writing {len(download_files)} downloaded files to "
+            f"saved fixture location {raw_download_output_path}"
+        )
+        shutil.copytree(download_dir, raw_download_output_path)
+
+
+def run_all_validations(
+    configs: SourceValidationConfigs,
+    predownload_file_data: list[FileData],
+    postdownload_file_data: list[FileData],
+    download_dir: Path,
+    test_output_dir: Path,
+):
+    if expected_number_indexed_file_data := configs.expected_number_indexed_file_data:
+        assert (
+            len(predownload_file_data) == expected_number_indexed_file_data
+        ), f"expected {expected_number_indexed_file_data} but got {len(predownload_file_data)}"
+    if expected_num_files := configs.expected_num_files:
+        assert len(postdownload_file_data) == expected_num_files
+
+    for pre_data, post_data in zip(predownload_file_data, postdownload_file_data):
+        configs.run_file_data_validation(
+            predownload_file_data=pre_data, postdownload_file_data=post_data
+        )
+    configs.run_download_dir_validation(download_dir=download_dir)
+    if configs.validate_file_data:
+        run_expected_results_validation(
+            expected_output_dir=test_output_dir / "file_data",
+            all_file_data=get_all_file_data(
+                all_predownload_file_data=predownload_file_data,
+                all_postdownload_file_data=postdownload_file_data,
+            ),
+            configs=configs,
+        )
+    download_files = get_files(dir_path=download_dir)
+    download_files.sort()
+    run_directory_structure_validation(
+        expected_output_dir=configs.test_output_dir(), download_files=download_files
+    )
+    if configs.validate_downloaded_files:
+        run_expected_download_files_validation(
+            expected_output_dir=test_output_dir / "downloads",
+            current_download_dir=download_dir,
+            configs=configs,
+        )
+
+
+def get_all_file_data(
+    all_postdownload_file_data: list[FileData], all_predownload_file_data: list[FileData]
+) -> list[FileData]:
+    all_file_data = all_postdownload_file_data
+    indexed_file_data = [
+        fd
+        for fd in all_predownload_file_data
+        if fd.identifier not in [f.identifier for f in all_file_data]
+    ]
+    all_file_data += indexed_file_data
+    return all_file_data
+
+
+async def source_connector_validation(
+    indexer: Indexer,
+    downloader: Downloader,
+    configs: SourceValidationConfigs,
+    overwrite_fixtures: bool = os.getenv("OVERWRITE_FIXTURES", "False").lower() == "true",
+) -> None:
+    # Run common validations on the process of running a source connector, supporting dynamic
+    # validators that get passed in along with comparisons on the saved expected values.
+    # If overwrite_fixtures is st to True, will ignore all validators but instead overwrite the
+    # expected values with what gets generated by this test.
+    all_predownload_file_data = []
+    all_postdownload_file_data = []
+    indexer.precheck()
+    download_dir = downloader.download_config.download_dir
+    test_output_dir = configs.test_output_dir()
+    for file_data in indexer.run():
+        assert file_data
+        predownload_file_data = file_data.model_copy(deep=True)
+        all_predownload_file_data.append(predownload_file_data)
+        if downloader.is_async():
+            resp = await downloader.run_async(file_data=file_data)
+        else:
+            resp = downloader.run(file_data=file_data)
+        if isinstance(resp, list):
+            for r in resp:
+                postdownload_file_data = r["file_data"].model_copy(deep=True)
+                all_postdownload_file_data.append(postdownload_file_data)
+        else:
+            postdownload_file_data = resp["file_data"].model_copy(deep=True)
+            all_postdownload_file_data.append(postdownload_file_data)
+    if not overwrite_fixtures:
+        print("Running validation")
+        run_all_validations(
+            configs=configs,
+            predownload_file_data=all_predownload_file_data,
+            postdownload_file_data=all_postdownload_file_data,
+            download_dir=download_dir,
+            test_output_dir=test_output_dir,
+        )
+    else:
+        print("Running fixtures update")
+        update_fixtures(
+            output_dir=test_output_dir,
+            download_dir=download_dir,
+            all_file_data=get_all_file_data(
+                all_predownload_file_data=all_predownload_file_data,
+                all_postdownload_file_data=all_postdownload_file_data,
+            ),
+            save_downloads=configs.validate_downloaded_files,
+            save_filedata=configs.validate_file_data,
+        )

test/integration/connectors/utils/validation/utils.py

@@ -0,0 +1,36 @@
+import filecmp
+import shutil
+from pathlib import Path
+from typing import Callable, Optional
+
+from pydantic import BaseModel
+
+from test.integration.connectors.utils.constants import expected_results_path
+from test.integration.connectors.utils.validation.equality import file_type_equality_check
+
+
+class ValidationConfig(BaseModel):
+    test_id: str
+    file_equality_check: Optional[Callable[[Path, Path], bool]] = None
+
+    def test_output_dir(self) -> Path:
+        return expected_results_path / self.test_id
+
+    def detect_diff(self, expected_filepath: Path, current_filepath: Path) -> bool:
+        if expected_filepath.suffix != current_filepath.suffix:
+            return True
+        if file_equality_check := self.file_equality_check:
+            return not file_equality_check(expected_filepath, current_filepath)
+        current_suffix = expected_filepath.suffix
+        if current_suffix in file_type_equality_check:
+            equality_check_callable = file_type_equality_check[current_suffix]
+            return not equality_check_callable(
+                expected_filepath=expected_filepath, current_filepath=current_filepath
+            )
+        # Fallback is using filecmp.cmp to compare the files
+        return not filecmp.cmp(expected_filepath, current_filepath, shallow=False)
+
+
+def reset_dir(dir_path: Path) -> None:
+    shutil.rmtree(path=dir_path, ignore_errors=True)
+    dir_path.mkdir(parents=True)

test/integration/connectors/weaviate/conftest.py

@@ -0,0 +1,15 @@
+import json
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture
+def collections_schema_config() -> dict:
+    int_test_dir = Path(__file__).parent
+    assets_dir = int_test_dir / "assets"
+    config_file = assets_dir / "elements.json"
+    assert config_file.exists()
+    assert config_file.is_file()
+    with config_file.open() as config_data:
+        return json.load(config_data)

test/integration/connectors/weaviate/test_cloud.py

@@ -0,0 +1,34 @@
+import pytest
+from pydantic import ValidationError
+
+from unstructured_ingest.v2.processes.connectors.weaviate.cloud import (
+    CloudWeaviateAccessConfig,
+    CloudWeaviateConnectionConfig,
+)
+
+
+def test_weaviate_failing_connection_config():
+    with pytest.raises(ValidationError):
+        CloudWeaviateConnectionConfig(
+            access_config=CloudWeaviateAccessConfig(api_key="my key", password="password"),
+            username="username",
+            cluster_url="clusterurl",
+        )
+
+
+def test_weaviate_connection_config_happy_path():
+    CloudWeaviateConnectionConfig(
+        access_config=CloudWeaviateAccessConfig(
+            api_key="my key",
+        ),
+        cluster_url="clusterurl",
+    )
+
+
+def test_weaviate_connection_config_anonymous():
+    CloudWeaviateConnectionConfig(
+        access_config=CloudWeaviateAccessConfig(api_key="my key", password="password"),
+        username="username",
+        anonymous=True,
+        cluster_url="clusterurl",
+    )

test/integration/connectors/weaviate/test_local.py

@@ -0,0 +1,131 @@
+import json
+import time
+from pathlib import Path
+
+import pytest
+import requests
+import weaviate
+from weaviate.client import WeaviateClient
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.docker import container_context
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.processes.connectors.weaviate.local import (
+    CONNECTOR_TYPE,
+    LocalWeaviateConnectionConfig,
+    LocalWeaviateUploader,
+    LocalWeaviateUploaderConfig,
+    LocalWeaviateUploadStager,
+)
+
+COLLECTION_NAME = "elements"
+
+
+def wait_for_container(timeout: int = 10, interval: int = 1) -> None:
+    start_time = time.time()
+    while time.time() - start_time < timeout:
+        try:
+            requests.get("http://localhost:8080/v1/.well-known/read")
+            return
+        except Exception as e:
+            print(f"Failed to validate container healthy, sleeping for {interval} seconds: {e}")
+            time.sleep(interval)
+    raise TimeoutError("Docker container never came up healthy")
+
+
+@pytest.fixture
+def collection(collections_schema_config: dict) -> str:
+    with container_context(
+        image="semitechnologies/weaviate:1.27.3",
+        ports={8080: 8080, 50051: 50051},
+    ):
+        wait_for_container()
+        with weaviate.connect_to_local() as weaviate_client:
+            weaviate_client.collections.create_from_dict(config=collections_schema_config)
+        yield COLLECTION_NAME
+
+
+def get_count(client: WeaviateClient) -> int:
+    collection = client.collections.get(COLLECTION_NAME)
+    resp = collection.aggregate.over_all(total_count=True)
+    return resp.total_count
+
+
+def validate_count(expected_count: int, retries: int = 10, interval: int = 1) -> None:
+    with weaviate.connect_to_local() as weaviate_client:
+        current_count = get_count(client=weaviate_client)
+        retry_count = 0
+        while current_count != expected_count and retry_count < retries:
+            retry_count += 1
+            time.sleep(interval)
+            current_count = get_count(client=weaviate_client)
+        assert current_count == expected_count, (
+            f"Expected count ({expected_count}) doesn't match how "
+            f"much came back from collection: {current_count}"
+        )
+
+
+def run_uploader_and_validate(
+    uploader: LocalWeaviateUploader, path: Path, file_data: FileData, expected_count: int
+):
+    uploader.precheck()
+    uploader.run(path=path, file_data=file_data)
+    validate_count(expected_count=expected_count)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+def test_weaviate_local_destination(upload_file: Path, collection: str, tmp_path: Path):
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mock file data",
+    )
+    stager = LocalWeaviateUploadStager()
+
+    staged_filepath = stager.run(
+        elements_filepath=upload_file,
+        file_data=file_data,
+        output_dir=tmp_path,
+        output_filename=upload_file.name,
+    )
+    dynamic_uploader = LocalWeaviateUploader(
+        upload_config=LocalWeaviateUploaderConfig(
+            collection=COLLECTION_NAME,
+        ),
+        connection_config=LocalWeaviateConnectionConfig(),
+    )
+    fixed_size_uploader = LocalWeaviateUploader(
+        upload_config=LocalWeaviateUploaderConfig(
+            collection=COLLECTION_NAME, batch_size=10, dynamic_batch=False
+        ),
+        connection_config=LocalWeaviateConnectionConfig(),
+    )
+    rate_limited_uploader = LocalWeaviateUploader(
+        upload_config=LocalWeaviateUploaderConfig(
+            collection=COLLECTION_NAME, requests_per_minute=50, dynamic_batch=False
+        ),
+        connection_config=LocalWeaviateConnectionConfig(),
+    )
+    with staged_filepath.open() as f:
+        staged_elements = json.load(f)
+    expected_count = len(staged_elements)
+
+    run_uploader_and_validate(
+        uploader=dynamic_uploader,
+        path=staged_filepath,
+        file_data=file_data,
+        expected_count=expected_count,
+    )
+    run_uploader_and_validate(
+        uploader=fixed_size_uploader,
+        path=staged_filepath,
+        file_data=file_data,
+        expected_count=expected_count,
+    )
+    run_uploader_and_validate(
+        uploader=rate_limited_uploader,
+        path=staged_filepath,
+        file_data=file_data,
+        expected_count=expected_count,
+    )

test/integration/embedders/conftest.py

@@ -0,0 +1,13 @@
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture
+def embedder_file() -> Path:
+    int_test_dir = Path(__file__).parent
+    assets_dir = int_test_dir / "assets"
+    embedder_file = assets_dir / "DA-1p-with-duplicate-pages.pdf.json"
+    assert embedder_file.exists()
+    assert embedder_file.is_file()
+    return embedder_file