unstructured-ingest 1.2.32__py3-none-any.whl

unstructured_ingest/pipeline/steps/embed.py

@@ -0,0 +1,77 @@
+import asyncio
+import hashlib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, Optional, TypedDict
+
+from unstructured_ingest.data_types.file_data import FileData, file_data_from_file
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.interfaces import PipelineStep
+from unstructured_ingest.processes.embedder import Embedder
+from unstructured_ingest.utils.data_prep import write_data
+from unstructured_ingest.utils.pydantic_models import serialize_base_model_json
+
+STEP_ID = "embed"
+
+
+class EmbedStepResponse(TypedDict):
+    file_data_path: str
+    path: str
+
+
+@dataclass
+class EmbedStep(PipelineStep):
+    process: Embedder
+    identifier: str = STEP_ID
+
+    def __str__(self):
+        return f"{self.identifier} ({self.process.config.embedding_provider})"
+
+    def __post_init__(self):
+        config = self.process.config.model_dump_json() if self.process.config else None
+        logger.info(f"created {self.identifier} with configs: {config}")
+
+    def should_embed(self, filepath: Path, file_data: FileData) -> bool:
+        if self.context.reprocess or file_data.reprocess:
+            return True
+        return not filepath.exists()
+
+    def get_output_filepath(self, filename: Path) -> Path:
+        hashed_output_file = f"{self.get_hash(extras=[filename.name])}.ndjson"
+        filepath = (self.cache_dir / hashed_output_file).resolve()
+        filepath.parent.mkdir(parents=True, exist_ok=True)
+        return filepath
+
+    def _save_output(self, output_filepath: str, embedded_content: list[dict]):
+        logger.debug(f"writing embedded output to: {output_filepath}")
+        write_data(path=Path(output_filepath), data=embedded_content)
+
+    async def _run_async(self, fn: Callable, path: str, file_data_path: str) -> EmbedStepResponse:
+        path = Path(path)
+        file_data = file_data_from_file(path=file_data_path)
+        output_filepath = self.get_output_filepath(filename=path)
+        if not self.should_embed(filepath=output_filepath, file_data=file_data):
+            logger.debug(f"skipping embedding, output already exists: {output_filepath}")
+            return EmbedStepResponse(file_data_path=file_data_path, path=str(output_filepath))
+        fn_kwargs = {"elements_filepath": path}
+        if not asyncio.iscoroutinefunction(fn):
+            embed_content_raw = fn(**fn_kwargs)
+        elif semaphore := self.context.semaphore:
+            async with semaphore:
+                embed_content_raw = await fn(**fn_kwargs)
+        else:
+            embed_content_raw = await fn(**fn_kwargs)
+
+        self._save_output(
+            output_filepath=str(output_filepath),
+            embedded_content=embed_content_raw,
+        )
+        return EmbedStepResponse(file_data_path=file_data_path, path=str(output_filepath))
+
+    def get_hash(self, extras: Optional[list[str]]) -> str:
+        hashable_string = serialize_base_model_json(
+            model=self.process.config, sort_keys=True, ensure_ascii=True
+        )
+        if extras:
+            hashable_string += "".join(extras)
+        return hashlib.sha256(hashable_string.encode()).hexdigest()[:12]

unstructured_ingest/pipeline/steps/filter.py

@@ -0,0 +1,35 @@
+import asyncio
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+from unstructured_ingest.data_types.file_data import file_data_from_file
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.interfaces import PipelineStep
+from unstructured_ingest.processes.filter import Filterer
+
+STEP_ID = "filter"
+
+
+@dataclass
+class FilterStep(PipelineStep):
+    process: Filterer
+    identifier: str = STEP_ID
+
+    def __post_init__(self):
+        config = self.process.config.model_dump_json() if self.process.config else None
+        logger.info(f"created {self.identifier} with configs: {config}")
+
+    async def _run_async(self, fn: Callable, file_data_path: str, **kwargs) -> Optional[dict]:
+        file_data = file_data_from_file(path=file_data_path)
+        fn_kwargs = {"file_data": file_data}
+        if not asyncio.iscoroutinefunction(fn):
+            resp = fn(**fn_kwargs)
+        elif semaphore := self.context.semaphore:
+            async with semaphore:
+                resp = await fn(**fn_kwargs)
+        else:
+            resp = await fn(**fn_kwargs)
+
+        if resp:
+            return {"file_data_path": file_data_path}
+        return None

unstructured_ingest/pipeline/steps/index.py

@@ -0,0 +1,86 @@
+import hashlib
+import json
+from dataclasses import dataclass
+from typing import AsyncGenerator, Generator, Optional, TypeVar
+
+from unstructured_ingest.interfaces.indexer import Indexer
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.interfaces import PipelineStep
+from unstructured_ingest.pipeline.otel import instrument
+from unstructured_ingest.utils.pydantic_models import serialize_base_model_json
+
+IndexerT = TypeVar("IndexerT", bound=Indexer)
+
+STEP_ID = "indexer"
+
+
+@dataclass
+class IndexStep(PipelineStep):
+    process: IndexerT
+    identifier: str = STEP_ID
+
+    def __str__(self):
+        return f"{self.identifier} ({self.process.__class__.__name__})"
+
+    def __post_init__(self):
+        config = self.process.index_config.model_dump_json() if self.process.index_config else None
+        connection_config = (
+            self.process.connection_config.model_dump_json()
+            if self.process.connection_config
+            else None
+        )
+        logger.info(
+            f"created {self.identifier} with configs: {config}, "
+            f"connection configs: {connection_config}"
+        )
+
+    @instrument(span_name=STEP_ID)
+    def run(self) -> Generator[str, None, None]:
+        for file_data in self.process.run():
+            logger.debug(f"generated file data: {file_data.model_dump_json()}")
+            try:
+                record_hash = self.get_hash(extras=[file_data.identifier])
+                filename = f"{record_hash}.json"
+                filepath = (self.cache_dir / filename).resolve()
+                filepath.parent.mkdir(parents=True, exist_ok=True)
+                with open(str(filepath), "w") as f:
+                    f.write(file_data.model_dump_json(indent=2))
+                yield str(filepath)
+            except Exception as e:
+                logger.error(f"failed to create index for file data: {file_data}", exc_info=True)
+                if self.context.raise_on_error:
+                    raise e
+                continue
+
+    async def run_async(self) -> AsyncGenerator[str, None]:
+        async for file_data in self.process.run_async():
+            logger.debug(f"generated file data: {file_data.model_dump_json()}")
+            try:
+                record_hash = self.get_hash(extras=[file_data.identifier])
+                filename = f"{record_hash}.json"
+                filepath = (self.cache_dir / filename).resolve()
+                filepath.parent.mkdir(parents=True, exist_ok=True)
+                with open(str(filepath), "w") as f:
+                    f.write(file_data.model_dump_json(indent=2))
+                yield str(filepath)
+            except Exception as e:
+                logger.error(f"failed to create index for file data: {file_data}", exc_info=True)
+                if self.context.raise_on_error:
+                    raise e
+                continue
+
+    def get_hash(self, extras: Optional[list[str]]) -> str:
+        index_config_dict = json.loads(
+            serialize_base_model_json(model=self.process.index_config, sort_keys=True)
+        )
+        connection_config_dict = json.loads(
+            serialize_base_model_json(model=self.process.connection_config, sort_keys=True)
+        )
+        hashable_dict = {
+            "index_config": index_config_dict,
+            "connection_config": connection_config_dict,
+        }
+        hashable_string = json.dumps(hashable_dict, sort_keys=True)
+        if extras:
+            hashable_string += "".join(extras)
+        return hashlib.sha256(hashable_string.encode()).hexdigest()[:12]

unstructured_ingest/pipeline/steps/partition.py

@@ -0,0 +1,77 @@
+import asyncio
+import hashlib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, Optional, TypedDict
+
+from unstructured_ingest.data_types.file_data import FileData, file_data_from_file
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.interfaces import PipelineStep
+from unstructured_ingest.processes.partitioner import Partitioner
+from unstructured_ingest.utils.data_prep import write_data
+from unstructured_ingest.utils.pydantic_models import serialize_base_model_json
+
+STEP_ID = "partition"
+
+
+class PartitionStepResponse(TypedDict):
+    file_data_path: str
+    path: str
+
+
+@dataclass
+class PartitionStep(PipelineStep):
+    process: Partitioner
+    identifier: str = STEP_ID
+
+    def __str__(self):
+        return f"{self.identifier} ({self.process.config.strategy})"
+
+    def __post_init__(self):
+        config = self.process.config.model_dump_json()
+        logger.info(f"created {self.identifier} with configs: {config}")
+
+    def should_partition(self, filepath: Path, file_data: FileData) -> bool:
+        if self.context.reprocess or file_data.reprocess:
+            return True
+        return not filepath.exists()
+
+    def get_output_filepath(self, filename: Path) -> Path:
+        hashed_output_file = f"{self.get_hash(extras=[filename.name])}.ndjson"
+        filepath = (self.cache_dir / hashed_output_file).resolve()
+        filepath.parent.mkdir(parents=True, exist_ok=True)
+        return filepath
+
+    def _save_output(self, output_filepath: str, partitioned_content: list[dict]):
+        logger.debug(f"writing partitioned output to: {output_filepath}")
+        write_data(path=Path(output_filepath), data=partitioned_content)
+
+    async def _run_async(
+        self, fn: Callable, path: str, file_data_path: str
+    ) -> Optional[PartitionStepResponse]:
+        path = Path(path)
+        file_data = file_data_from_file(path=file_data_path)
+        output_filepath = self.get_output_filepath(filename=Path(file_data_path))
+        if not self.should_partition(filepath=output_filepath, file_data=file_data):
+            logger.debug(f"skipping partitioning, output already exists: {output_filepath}")
+            return PartitionStepResponse(file_data_path=file_data_path, path=str(output_filepath))
+        fn_kwargs = {"filename": path, "metadata": file_data.metadata.model_dump()}
+        if not asyncio.iscoroutinefunction(fn):
+            partitioned_content = fn(**fn_kwargs)
+        elif semaphore := self.context.semaphore:
+            async with semaphore:
+                partitioned_content = await fn(**fn_kwargs)
+        else:
+            partitioned_content = await fn(**fn_kwargs)
+        self._save_output(
+            output_filepath=str(output_filepath), partitioned_content=partitioned_content
+        )
+        return PartitionStepResponse(file_data_path=file_data_path, path=str(output_filepath))
+
+    def get_hash(self, extras: Optional[list[str]]) -> str:
+        hashable_string = serialize_base_model_json(
+            model=self.process.config, sort_keys=True, ensure_ascii=True
+        )
+        if extras:
+            hashable_string += "".join(extras)
+        return hashlib.sha256(hashable_string.encode()).hexdigest()[:12]

unstructured_ingest/pipeline/steps/stage.py

@@ -0,0 +1,65 @@
+import asyncio
+import hashlib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, Optional, TypedDict
+
+from unstructured_ingest.data_types.file_data import file_data_from_file
+from unstructured_ingest.interfaces import UploadStager
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.interfaces import PipelineStep
+from unstructured_ingest.utils.pydantic_models import serialize_base_model_json
+
+STEP_ID = "upload_stage"
+
+
+class UploadStageStepResponse(TypedDict):
+    file_data_path: str
+    path: str
+
+
+@dataclass
+class UploadStageStep(PipelineStep):
+    process: UploadStager
+    identifier: str = STEP_ID
+
+    def __str__(self):
+        return f"{self.identifier} ({self.process.__class__.__name__})"
+
+    def __post_init__(self):
+        config = (
+            self.process.upload_stager_config.model_dump_json()
+            if self.process.upload_stager_config
+            else None
+        )
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        logger.info(f"created {self.identifier} with configs: {config}")
+
+    async def _run_async(
+        self, fn: Callable, path: str, file_data_path: str
+    ) -> UploadStageStepResponse:
+        path = Path(path)
+        # Maintain extension
+        output_filename = f"{self.get_hash(extras=[path.name])}{path.suffix}"
+        fn_kwargs = {
+            "elements_filepath": path,
+            "file_data": file_data_from_file(path=file_data_path),
+            "output_dir": self.cache_dir,
+            "output_filename": output_filename,
+        }
+        if not asyncio.iscoroutinefunction(fn):
+            staged_output_path = fn(**fn_kwargs)
+        elif semaphore := self.context.semaphore:
+            async with semaphore:
+                staged_output_path = await fn(**fn_kwargs)
+        else:
+            staged_output_path = await fn(**fn_kwargs)
+        return UploadStageStepResponse(file_data_path=file_data_path, path=str(staged_output_path))
+
+    def get_hash(self, extras: Optional[list[str]]) -> str:
+        hashable_string = serialize_base_model_json(
+            model=self.process.upload_stager_config, sort_keys=True, ensure_ascii=True
+        )
+        if extras:
+            hashable_string += "".join(extras)
+        return hashlib.sha256(hashable_string.encode()).hexdigest()[:12]

unstructured_ingest/pipeline/steps/uncompress.py

@@ -0,0 +1,50 @@
+import asyncio
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, TypedDict
+
+from unstructured_ingest.data_types.file_data import file_data_from_file
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.interfaces import PipelineStep
+from unstructured_ingest.processes.uncompress import Uncompressor
+
+STEP_ID = "uncompress"
+
+
+class UncompressStepResponse(TypedDict):
+    file_data_path: str
+    path: str
+
+
+@dataclass
+class UncompressStep(PipelineStep):
+    process: Uncompressor
+    identifier: str = STEP_ID
+
+    def __post_init__(self):
+        config = self.process.config.model_dump_json() if self.process.config else None
+        logger.info(f"created {self.identifier} with configs: {config}")
+
+    async def _run_async(
+        self, fn: Callable, path: str, file_data_path: str
+    ) -> list[UncompressStepResponse]:
+        file_data = file_data_from_file(path=file_data_path)
+        fn_kwargs = {"file_data": file_data}
+        if not asyncio.iscoroutinefunction(fn):
+            new_file_data = fn(**fn_kwargs)
+        elif semaphore := self.context.semaphore:
+            async with semaphore:
+                new_file_data = await fn(**fn_kwargs)
+        else:
+            new_file_data = await fn(**fn_kwargs)
+        responses = []
+        for new_file in new_file_data:
+            new_file_data_path = Path(file_data_path).parent / f"{new_file.identifier}.json"
+            new_file.to_file(path=str(new_file_data_path.resolve()))
+            responses.append(
+                UncompressStepResponse(
+                    path=new_file.local_download_path,
+                    file_data_path=str(new_file_data_path),
+                )
+            )
+        return responses

unstructured_ingest/pipeline/steps/upload.py

@@ -0,0 +1,58 @@
+import asyncio
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, Optional, TypedDict
+
+from unstructured_ingest.data_types.file_data import file_data_from_file
+from unstructured_ingest.interfaces import UploadContent
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.interfaces import BatchPipelineStep
+from unstructured_ingest.pipeline.otel import instrument
+
+STEP_ID = "upload"
+
+
+class UploadStepContent(TypedDict):
+    path: str
+    file_data_path: str
+
+
+@dataclass
+class UploadStep(BatchPipelineStep):
+    identifier: str = STEP_ID
+
+    def __str__(self):
+        return f"{self.identifier} ({self.process.__class__.__name__})"
+
+    def __post_init__(self):
+        config = (
+            self.process.upload_config.model_dump_json() if self.process.upload_config else None
+        )
+        connection_config = (
+            self.process.connection_config.model_dump_json()
+            if self.process.connection_config
+            else None
+        )
+        logger.info(
+            f"Created {self.identifier} with configs: {config}, "
+            f"connection configs: {connection_config}"
+        )
+
+    @instrument(span_name=STEP_ID)
+    def _run_batch(self, contents: list[UploadStepContent]) -> None:
+        upload_contents = [
+            UploadContent(path=Path(c["path"]), file_data=file_data_from_file(c["file_data_path"]))
+            for c in contents
+        ]
+        self.process.run_batch(contents=upload_contents)
+
+    async def _run_async(self, path: str, file_data_path: str, fn: Optional[Callable] = None):
+        fn = fn or self.process.run_async
+        fn_kwargs = {"path": Path(path), "file_data": file_data_from_file(path=file_data_path)}
+        if not asyncio.iscoroutinefunction(fn):
+            fn(**fn_kwargs)
+        elif semaphore := self.context.semaphore:
+            async with semaphore:
+                await fn(**fn_kwargs)
+        else:
+            await fn(**fn_kwargs)

unstructured_ingest/processes/__init__.py

@@ -0,0 +1,18 @@
+from .chunker import Chunker, ChunkerConfig
+from .embedder import Embedder, EmbedderConfig
+from .filter import Filterer, FiltererConfig
+from .partitioner import Partitioner, PartitionerConfig
+from .uncompress import UncompressConfig, Uncompressor
+
+__all__ = [
+    "Chunker",
+    "ChunkerConfig",
+    "Embedder",
+    "EmbedderConfig",
+    "Filterer",
+    "FiltererConfig",
+    "Partitioner",
+    "PartitionerConfig",
+    "Uncompressor",
+    "UncompressConfig",
+]

unstructured_ingest/processes/chunker.py

@@ -0,0 +1,131 @@
+from abc import ABC
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field, SecretStr
+
+from unstructured_ingest.interfaces.process import BaseProcess
+from unstructured_ingest.logger import logger
+from unstructured_ingest.unstructured_api import call_api_async
+from unstructured_ingest.utils.chunking import assign_and_map_hash_ids
+from unstructured_ingest.utils.data_prep import get_json_data
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+CHUNK_MAX_CHARS_DEFAULT: int = 500
+CHUNK_MULTI_PAGE_DEFAULT: bool = True
+
+
+class ChunkerConfig(BaseModel):
+    chunking_strategy: Optional[str] = Field(
+        default=None, description="The rule-set to use to form chunks. Omit to disable chunking."
+    )
+    chunking_endpoint: Optional[str] = Field(
+        default="https://api.unstructuredapp.io/general/v0/general",
+        description="If chunking via api, use the following host.",
+    )
+    chunk_api_timeout_ms: Optional[int] = Field(
+        default=None, description="Timeout in milliseconds for all api call during chunking."
+    )
+    chunk_by_api: bool = Field(default=False, description="Flag to use api for chunking")
+    chunk_api_key: Optional[SecretStr] = Field(
+        default=None, description="API Key for chunking endpoint."
+    )
+
+    chunk_combine_text_under_n_chars: Optional[int] = Field(
+        default=None,
+        description="Combine consecutive chunks when the first does not exceed this length and"
+        " the second will fit without exceeding the hard-maximum length. Only"
+        " operative for 'by_title' chunking-strategy.",
+    )
+    chunk_include_orig_elements: Optional[bool] = Field(
+        default=None,
+        description="When chunking, add the original elements consolidated to form each chunk to"
+        " `.metadata.orig_elements` on that chunk.",
+    )
+    chunk_max_characters: int = Field(
+        default=CHUNK_MAX_CHARS_DEFAULT,
+        description="Hard maximum chunk length. No chunk will exceed this length. An oversized"
+        " element will be divided by text-splitting to fit this window.",
+    )
+    chunk_multipage_sections: bool = Field(
+        default=CHUNK_MULTI_PAGE_DEFAULT,
+        description="Ignore page boundaries when chunking such that elements from two different"
+        " pages can appear in the same chunk. Only operative for 'by_title'"
+        " chunking-strategy.",
+    )
+    chunk_new_after_n_chars: Optional[int] = Field(
+        default=None,
+        description="Soft-maximum chunk length. Another element will not be added to a chunk of"
+        " this length even when it would fit without exceeding the hard-maximum"
+        " length.",
+    )
+    chunk_overlap: Optional[int] = Field(
+        default=None,
+        description="Prefix chunk text with last overlap=N characters of prior chunk. Only"
+        " applies to oversized chunks divided by text-splitting. To apply overlap to"
+        " non-oversized chunks use the --overlap-all option.",
+    )
+    chunk_overlap_all: Optional[bool] = Field(
+        default=None,
+        description="Apply overlap to chunks formed from whole elements as well as those formed"
+        " by text-splitting oversized elements. Overlap length is take from --overlap"
+        " option value.",
+    )
+
+    def to_chunking_kwargs(self) -> dict[str, Any]:
+        return {
+            "chunking_strategy": self.chunking_strategy,
+            "combine_under_n_chars": self.chunk_combine_text_under_n_chars,
+            "max_characters": self.chunk_max_characters,
+            "include_orig_elements": self.chunk_include_orig_elements,
+            "multipage_sections": self.chunk_multipage_sections,
+            "new_after_n_chars": self.chunk_new_after_n_chars,
+            "overlap": self.chunk_overlap,
+            "overlap_all": self.chunk_overlap_all,
+        }
+
+
+@dataclass
+class Chunker(BaseProcess, ABC):
+    config: ChunkerConfig
+
+    def is_async(self) -> bool:
+        return self.config.chunk_by_api
+
+    @requires_dependencies(dependencies=["unstructured"])
+    def run(self, elements_filepath: Path, **kwargs: Any) -> list[dict]:
+        from unstructured.chunking import dispatch
+        from unstructured.staging.base import elements_from_dicts
+
+        element_dicts = get_json_data(elements_filepath)
+
+        elements = elements_from_dicts(element_dicts=element_dicts)
+        if not elements:
+            return [e.to_dict() for e in elements]
+        local_chunking_strategies = ("basic", "by_title")
+        if self.config.chunking_strategy not in local_chunking_strategies:
+            logger.warning(
+                "chunking strategy not supported for local chunking: {}, must be one of: {}".format(
+                    self.config.chunking_strategy, ", ".join(local_chunking_strategies)
+                )
+            )
+            return [e.to_dict() for e in elements]
+        chunked_elements = dispatch.chunk(elements=elements, **self.config.to_chunking_kwargs())
+        chunked_elements_dicts = [e.to_dict() for e in chunked_elements]
+        chunked_elements_dicts = assign_and_map_hash_ids(elements=chunked_elements_dicts)
+        return chunked_elements_dicts
+
+    @requires_dependencies(dependencies=["unstructured_client"], extras="remote")
+    async def run_async(self, elements_filepath: Path, **kwargs: Any) -> list[dict]:
+        elements = await call_api_async(
+            server_url=self.config.chunking_endpoint,
+            api_key=self.config.chunk_api_key.get_secret_value(),
+            filename=elements_filepath,
+            api_parameters=self.config.to_chunking_kwargs(),
+            timeout_ms=self.config.chunk_api_timeout_ms,
+        )
+
+        elements = assign_and_map_hash_ids(elements=elements)
+
+        return elements

unstructured_ingest/processes/connector_registry.py

@@ -0,0 +1,69 @@
+from abc import ABC
+from dataclasses import dataclass
+from typing import Optional, Type, TypeVar
+
+from unstructured_ingest.interfaces import (
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    Indexer,
+    IndexerConfig,
+    Uploader,
+    UploaderConfig,
+    UploadStager,
+    UploadStagerConfig,
+)
+
+IndexerT = TypeVar("IndexerT", bound=Indexer)
+IndexerConfigT = TypeVar("IndexerConfigT", bound=IndexerConfig)
+DownloaderT = TypeVar("DownloaderT", bound=Downloader)
+DownloaderConfigT = TypeVar("DownloaderConfigT", bound=DownloaderConfig)
+ConnectionConfigT = TypeVar("ConnectionConfigT", bound=ConnectionConfig)
+UploadStagerConfigT = TypeVar("UploadStagerConfigT", bound=UploadStagerConfig)
+UploadStagerT = TypeVar("UploadStagerT", bound=UploadStager)
+UploaderConfigT = TypeVar("UploaderConfigT", bound=UploaderConfig)
+UploaderT = TypeVar("UploaderT", bound=Uploader)
+
+
+@dataclass
+class RegistryEntry(ABC):
+    pass
+
+
+@dataclass
+class SourceRegistryEntry(RegistryEntry):
+    indexer: Type[IndexerT]
+    downloader: Type[DownloaderT]
+
+    downloader_config: Optional[Type[DownloaderConfigT]] = None
+    indexer_config: Optional[Type[IndexerConfigT]] = None
+    connection_config: Optional[Type[ConnectionConfigT]] = None
+
+
+source_registry: dict[str, SourceRegistryEntry] = {}
+
+
+def add_source_entry(source_type: str, entry: SourceRegistryEntry):
+    if source_type in source_registry:
+        raise ValueError(f"source {source_type} has already been registered")
+    source_registry[source_type] = entry
+
+
+@dataclass
+class DestinationRegistryEntry(RegistryEntry):
+    uploader: Type[UploaderT]
+    upload_stager: Optional[Type[UploadStagerT]] = None
+
+    upload_stager_config: Optional[Type[UploadStagerConfigT]] = None
+    uploader_config: Optional[Type[UploaderConfigT]] = None
+
+    connection_config: Optional[Type[ConnectionConfigT]] = None
+
+
+destination_registry: dict[str, DestinationRegistryEntry] = {}
+
+
+def add_destination_entry(destination_type: str, entry: DestinationRegistryEntry):
+    if destination_type in destination_registry:
+        raise ValueError(f"destination {destination_type} has already been registered")
+    destination_registry[destination_type] = entry