unstructured-ingest 0.0.3__py3-none-any.whl → 0.0.5__py3-none-any.whl

unstructured_ingest/v2/cli/{utils.py → utils/click.py}

@@ -1,16 +1,12 @@
 import json
 import os.path
-import sys
-from dataclasses import fields, is_dataclass
 from gettext import gettext, ngettext
 from gettext import gettext as _
 from pathlib import Path
-from typing import Any, ForwardRef, Optional, Type, TypeVar, Union, get_args, get_origin
+from typing import Any, Optional, Type, TypeVar
 
 import click
-
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
-from unstructured_ingest.v2.logger import logger
+from pydantic import BaseModel, ConfigDict, Secret
 
 
 def conform_click_options(options: dict):
@@ -30,7 +26,13 @@ class Dict(click.ParamType):
         ctx: Optional[click.Context] = None,
     ) -> Any:
         try:
-            return json.loads(value)
+            if isinstance(value, dict):
+                return value
+            if isinstance(value, Path) and value.is_file():
+                with value.open() as f:
+                    return json.load(f)
+            if isinstance(value, str):
+                return json.loads(value)
         except json.JSONDecodeError:
             self.fail(
                 gettext(
@@ -107,86 +109,33 @@ class DelimitedString(click.ParamType):
         return split
 
 
-EnhancedDataClassJsonMixinT = TypeVar(
-    "EnhancedDataClassJsonMixinT", bound=EnhancedDataClassJsonMixin
-)
-
-
-def extract_config(
-    flat_data: dict, config: Type[EnhancedDataClassJsonMixinT]
-) -> EnhancedDataClassJsonMixinT:
-    """
-    To be able to extract a nested dataclass from a flat dictionary (as in one coming
-    from a click-based options input), the config class is dynamically looked through for
-    nested dataclass fields and new nested dictionaries are created to conform to the
-    shape the overall class expects when parsing from a dict. During the process, this will create
-    copies of the original dictionary to avoid pruning fields but this isn't a
-    problem since the `from_dict()` method ignores unneeded values.
-
-    Not handling more complex edge cases for now such as nested types i.e Union[List[List[...]]]
-    """
-
-    def conform_dict(inner_d: dict, inner_config: Type[EnhancedDataClassJsonMixinT]):
-        # Catch edge cases (i.e. Dict[str, ...]) where underlying type is not a concrete Class,
-        # causing 'issubclass() arg 1 must be a class' errors, return False
-        def is_subclass(instance, class_type) -> bool:
-            try:
-                return issubclass(instance, class_type)
-            except Exception:
-                return False
-
-        dd = inner_d.copy()
-        for field in fields(inner_config):
-            f_type = field.type
-            # typing can be defined using a string, in which case it needs to be resolved
-            # to the actual type. following logic is cherry picked from the typing
-            # get_type_hints() since type resolution can be expensive, only do it
-            # when the type is a string
-            if isinstance(f_type, str):
-                try:
-                    base_globals = sys.modules[inner_config.__module__].__dict__
-                    for_ref = ForwardRef(f_type, is_argument=False, is_class=True)
-                    f_type = for_ref._evaluate(
-                        globalns=base_globals, localns=None, recursive_guard=frozenset()
-                    )
-                except NameError as e:
-                    logger.warning(f"couldn't resolve type {f_type}: {e}")
-            # Handle the case where the type of a value if a Union (possibly optional)
-            if get_origin(f_type) is Union:
-                union_values = get_args(f_type)
-                # handle List types
-                union_values = [
-                    get_args(u)[0] if get_origin(u) is list else u for u in union_values
-                ]
-                # Ignore injected NoneType when optional
-                concrete_union_values = [v for v in union_values if not is_subclass(v, type(None))]
-                dataclass_union_values = [v for v in concrete_union_values if is_dataclass(v)]
-                non_dataclass_union_values = [
-                    v for v in concrete_union_values if not is_dataclass(v)
-                ]
-                if not dataclass_union_values:
-                    continue
-                # Check if the key for this field already exists in the dictionary,
-                # if so it might map to one of these non dataclass fields and this
-                # can't be enforced
-                if non_dataclass_union_values and field.name in dd:
-                    continue
-                if len(dataclass_union_values) > 1:
-                    logger.warning(
-                        "more than one dataclass type possible for field {}, "
-                        "not extracting: {}".format(field.name, ", ".join(dataclass_union_values))
-                    )
-                    continue
-                f_type = dataclass_union_values[0]
-            origin = get_origin(f_type)
-            if origin:
-                f_type = origin
-            if is_subclass(f_type, EnhancedDataClassJsonMixin):
-                dd[field.name] = conform_dict(inner_d=dd, inner_config=f_type)
-        return dd
-
-    adjusted_dict = conform_dict(inner_d=flat_data, inner_config=config)
-    return config.from_dict(adjusted_dict, apply_name_overload=False)
+BaseModelT = TypeVar("BaseModelT", bound=BaseModel)
+
+
+def extract_config(flat_data: dict, config: Type[BaseModelT]) -> BaseModelT:
+    fields = config.model_fields
+    config.model_config = ConfigDict(extra="ignore")
+    field_names = [v.alias or k for k, v in fields.items()]
+    data = {k: v for k, v in flat_data.items() if k in field_names and v is not None}
+    if access_config := fields.get("access_config"):
+        access_config_type = access_config.annotation
+        # Check if raw type is wrapped by a secret
+        if (
+            hasattr(access_config_type, "__origin__")
+            and hasattr(access_config_type, "__args__")
+            and access_config_type.__origin__ is Secret
+        ):
+            ac_subtypes = access_config_type.__args__
+            ac_fields = ac_subtypes[0].model_fields
+        elif issubclass(access_config_type, BaseModel):
+            ac_fields = access_config_type.model_fields
+        else:
+            raise TypeError(f"Unrecognized access_config type: {access_config_type}")
+        ac_field_names = [v.alias or k for k, v in ac_fields.items()]
+        data["access_config"] = {
+            k: v for k, v in flat_data.items() if k in ac_field_names and v is not None
+        }
+    return config.model_validate(obj=data)
 
 
 class Group(click.Group):
@@ -195,13 +144,11 @@ class Group(click.Group):
         This allows for subcommands to be called with the --help flag without breaking
         if parent command is missing any of its required parameters
         """
-
         try:
             return super().parse_args(ctx, args)
         except click.MissingParameter:
             if "--help" not in args:
                 raise
-
             # remove the required params so that help can display
             for param in self.params:
                 param.required = False

unstructured_ingest/v2/cli/utils/model_conversion.py

@@ -0,0 +1,199 @@
+import contextlib
+import datetime
+from collections import Counter
+from enum import EnumMeta
+from pathlib import Path
+from typing import Any, Callable, Literal, Optional, Type, TypedDict, Union, get_args, get_origin
+from uuid import UUID
+
+import click
+from annotated_types import Ge, Gt, Le, Lt, SupportsGe, SupportsGt, SupportsLe, SupportsLt
+from click import Option
+from pydantic import BaseModel, Secret, SecretStr
+from pydantic.fields import FieldInfo
+from pydantic.types import _SecretBase
+from pydantic_core import PydanticUndefined
+
+from unstructured_ingest.v2.cli.utils.click import DelimitedString, Dict
+
+NoneType = type(None)
+
+
+class _RangeDict(TypedDict, total=False):
+    """Represent arguments to `click.IntRange` or `click.FloatRange`."""
+
+    max: Union[SupportsLt, SupportsLe]
+    min: Union[SupportsGt, SupportsGe]
+    max_open: bool
+    min_open: bool
+
+
+def get_range_from_metadata(metadata: list[Any]) -> _RangeDict:
+    range_args: _RangeDict = {}
+    for constraint in metadata:
+        if isinstance(constraint, Le):
+            range_args["max"] = constraint.le
+            range_args["max_open"] = False
+        if isinstance(constraint, Lt):
+            range_args["max"] = constraint.lt
+            range_args["max_open"] = True
+        if isinstance(constraint, Ge):
+            range_args["min"] = constraint.ge
+            range_args["min_open"] = False
+        if isinstance(constraint, Gt):
+            range_args["min"] = constraint.gt
+            range_args["min_open"] = True
+    return range_args
+
+
+def is_boolean_flag(field_info: FieldInfo) -> bool:
+    annotation = field_info.annotation
+    raw_annotation = get_raw_type(annotation)
+    return raw_annotation is bool
+
+
+def get_raw_type(val: Any) -> Any:
+    field_args = get_args(val)
+    field_origin = get_origin(val)
+    if field_origin is Union and len(field_args) == 2 and NoneType in field_args:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return field_type
+    if field_origin is Secret and len(field_args) == 1:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return field_type
+    if val is SecretStr:
+        return str
+    return val
+
+
+def get_default_value_from_field(field: FieldInfo) -> Optional[Union[Any, Callable[[], Any]]]:
+    if field.default is not PydanticUndefined:
+        return field.default
+    elif field.default_factory is not None:
+        return field.default_factory
+    return None
+
+
+def get_option_name(field_name: str, field_info: FieldInfo) -> str:
+    field_name = field_info.alias or field_name
+    if field_name.startswith("--"):
+        field_name = field_name[2:]
+    field_name = field_name.lower().replace("_", "-")
+    if is_boolean_flag(field_info):
+        return f"--{field_name}/--no-{field_name}"
+    return f"--{field_name}"
+
+
+def get_numerical_type(field: FieldInfo) -> click.ParamType:
+    range_args = get_range_from_metadata(field.metadata)
+    if field.annotation is int:
+        if range_args:
+            return click.IntRange(**range_args)  # type: ignore[arg-type]
+        return click.INT
+    # Non-integer numerical types default to float
+    if range_args:
+        return click.FloatRange(**range_args)  # type: ignore[arg-type]
+    return click.FLOAT
+
+
+def get_type_from_annotation(field_type: Any) -> click.ParamType:
+    field_origin = get_origin(field_type)
+    field_args = get_args(field_type)
+    if field_origin is Union and len(field_args) == 2 and NoneType in field_args:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return get_type_from_annotation(field_type=field_type)
+    if field_origin is Secret and len(field_args) == 1:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return get_type_from_annotation(field_type=field_type)
+    if field_origin is list and len(field_args) == 1 and field_args[0] is str:
+        return DelimitedString()
+    if field_type is SecretStr:
+        return click.STRING
+    if dict in [field_type, field_origin]:
+        return Dict()
+    if field_type is str:
+        return click.STRING
+    if field_type is bool:
+        return click.BOOL
+    if field_type is UUID:
+        return click.UUID
+    if field_type is Path:
+        return click.Path(path_type=Path)
+    if field_type in (datetime.datetime, datetime.date):
+        return click.DateTime()
+    if field_origin is Literal:
+        return click.Choice(field_args)
+    if isinstance(field_type, EnumMeta):
+        values = [i.value for i in field_type]
+        return click.Choice(values)
+    raise TypeError(f"Unexpected field type: {field_type}")
+
+
+def _get_type_from_field(field: FieldInfo) -> click.ParamType:
+    raw_field_type = get_raw_type(field.annotation)
+
+    if raw_field_type in (int, float):
+        return get_numerical_type(field)
+    return get_type_from_annotation(field_type=field.annotation)
+
+
+def get_option_from_field(option_name: str, field_info: FieldInfo) -> Option:
+    param_decls = [option_name]
+    help = field_info.description or ""
+    if examples := field_info.examples:
+        help += f" [Examples: {', '.join(examples)}]"
+    option_kwargs = {
+        "type": _get_type_from_field(field_info),
+        "default": get_default_value_from_field(field_info),
+        "required": field_info.is_required(),
+        "help": help,
+        "is_flag": is_boolean_flag(field_info),
+        "show_default": field_info.default is not PydanticUndefined,
+    }
+    return click.Option(param_decls=param_decls, **option_kwargs)
+
+
+def is_subclass(x: Any, y: Any) -> bool:
+    with contextlib.suppress(TypeError):
+        return issubclass(x, y)
+
+    return False
+
+
+def post_check(options: list[Option]):
+    option_names = [option.name for option in options]
+    duplicate_names = [name for name, count in Counter(option_names).items() if count > 1]
+    if duplicate_names:
+        raise ValueError(
+            "the following field name were reused, all must be unique: {}".format(
+                ", ".join(duplicate_names)
+            )
+        )
+
+
+def is_secret(value: Any) -> bool:
+    # Case Secret[int]
+    if hasattr(value, "__origin__") and hasattr(value, "__args__"):
+        origin = value.__origin__
+        return is_subclass(origin, _SecretBase)
+    # Case SecretStr
+    return is_subclass(value, _SecretBase)
+
+
+def options_from_base_model(model: Union[BaseModel, Type[BaseModel]]) -> list[Option]:
+    options = []
+    model_fields = model.model_fields
+    for field_name, field_info in model_fields.items():
+        if field_info.init is False:
+            continue
+        option_name = get_option_name(field_name=field_name, field_info=field_info)
+        raw_annotation = get_raw_type(field_info.annotation)
+        if is_subclass(raw_annotation, BaseModel):
+            options.extend(options_from_base_model(model=raw_annotation))
+        else:
+            if is_secret(field_info.annotation):
+                field_info.description = f"[sensitive] {field_info.description}"
+            options.append(get_option_from_field(option_name=option_name, field_info=field_info))
+
+    post_check(options=options)
+    return options

unstructured_ingest/v2/interfaces/connector.py

@@ -2,11 +2,10 @@ from abc import ABC
 from dataclasses import dataclass
 from typing import Any, TypeVar
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from pydantic import BaseModel, Secret
 
 
-@dataclass
-class AccessConfig(EnhancedDataClassJsonMixin):
+class AccessConfig(BaseModel):
     """Meant to designate holding any sensitive information associated with other configs
     and also for access specific configs."""
 
@@ -14,14 +13,13 @@ class AccessConfig(EnhancedDataClassJsonMixin):
 AccessConfigT = TypeVar("AccessConfigT", bound=AccessConfig)
 
 
-@dataclass
-class ConnectionConfig(EnhancedDataClassJsonMixin):
-    access_config: AccessConfigT
+class ConnectionConfig(BaseModel):
+    access_config: Secret[AccessConfigT]
 
     def get_access_config(self) -> dict[str, Any]:
         if not self.access_config:
             return {}
-        return self.access_config.to_dict(apply_name_overload=False)
+        return self.access_config.get_secret_value().dict()
 
 
 ConnectionConfigT = TypeVar("ConnectionConfigT", bound=ConnectionConfig)

unstructured_ingest/v2/interfaces/downloader.py

@@ -1,18 +1,21 @@
 import os
 from abc import ABC, abstractmethod
-from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Optional, TypedDict, TypeVar, Union
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from pydantic import BaseModel, Field
+
 from unstructured_ingest.v2.interfaces.connector import BaseConnector
 from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 
 
-@dataclass
-class DownloaderConfig(EnhancedDataClassJsonMixin):
-    download_dir: Optional[Path] = None
+class DownloaderConfig(BaseModel):
+    download_dir: Optional[Path] = Field(
+        default=None,
+        description="Where files are downloaded to, defaults to a location at"
+        "`$HOME/.cache/unstructured/ingest/<connector name>/<SHA256>`.",
+    )
 
 
 DownloaderConfigT = TypeVar("DownloaderConfigT", bound=DownloaderConfig)

unstructured_ingest/v2/interfaces/file_data.py

@@ -4,7 +4,6 @@ from pathlib import Path
 from typing import Any, Literal, Optional
 
 from dataclasses_json import DataClassJsonMixin
-from unstructured.documents.elements import DataSourceMetadata
 
 
 @dataclass
@@ -23,7 +22,14 @@ class SourceIdentifiers:
 
 
 @dataclass
-class FileDataSourceMetadata(DataSourceMetadata):
+class FileDataSourceMetadata(DataClassJsonMixin):
+    url: Optional[str] = None
+    version: Optional[str] = None
+    record_locator: Optional[dict[str, Any]] = None
+    date_created: Optional[str] = None
+    date_modified: Optional[str] = None
+    date_processed: Optional[str] = None
+    permissions_data: Optional[list[dict[str, Any]]] = None
     filesize_bytes: Optional[int] = None
 
 

unstructured_ingest/v2/interfaces/indexer.py

@@ -1,15 +1,14 @@
 from abc import ABC, abstractmethod
-from dataclasses import dataclass
 from typing import Any, Generator, Optional, TypeVar
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from pydantic import BaseModel
+
 from unstructured_ingest.v2.interfaces.connector import BaseConnector
 from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 
 
-@dataclass
-class IndexerConfig(EnhancedDataClassJsonMixin):
+class IndexerConfig(BaseModel):
     pass
 
 

unstructured_ingest/v2/interfaces/processor.py

@@ -1,24 +1,24 @@
 import os
 from asyncio import Semaphore
-from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Optional
+from typing import Any, Optional
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from pydantic import BaseModel, ConfigDict, Field
 
 DEFAULT_WORK_DIR = str((Path.home() / ".cache" / "unstructured" / "ingest" / "pipeline").resolve())
 
 
-@dataclass
-class ProcessorConfig(EnhancedDataClassJsonMixin):
+class ProcessorConfig(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     reprocess: bool = False
     verbose: bool = False
     tqdm: bool = False
-    work_dir: str = field(default_factory=lambda: DEFAULT_WORK_DIR)
+    work_dir: str = Field(default_factory=lambda: DEFAULT_WORK_DIR)
     num_processes: int = 2
     max_connections: Optional[int] = None
     raise_on_error: bool = False
-    disable_parallelism: bool = field(
+    disable_parallelism: bool = Field(
         default_factory=lambda: os.getenv("INGEST_DISABLE_PARALLELISM", "false").lower() == "true"
     )
     preserve_downloads: bool = False
@@ -28,10 +28,10 @@ class ProcessorConfig(EnhancedDataClassJsonMixin):
     uncompress: bool = False
 
     # Used to keep track of state in pipeline
-    status: dict = field(default_factory=dict)
-    semaphore: Optional[Semaphore] = field(init=False, default=None)
+    status: dict = Field(default_factory=dict)
+    semaphore: Optional[Semaphore] = Field(init=False, default=None)
 
-    def __post_init__(self):
+    def model_post_init(self, __context: Any) -> None:
         if self.max_connections is not None:
             self.semaphore = Semaphore(self.max_connections)
 

unstructured_ingest/v2/interfaces/upload_stager.py

@@ -3,13 +3,13 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, TypeVar
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from pydantic import BaseModel
+
 from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 
 
-@dataclass
-class UploadStagerConfig(EnhancedDataClassJsonMixin):
+class UploadStagerConfig(BaseModel):
     pass
 
 

unstructured_ingest/v2/interfaces/uploader.py

@@ -3,14 +3,14 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, TypeVar
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from pydantic import BaseModel
+
 from unstructured_ingest.v2.interfaces.connector import BaseConnector
 from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 
 
-@dataclass
-class UploaderConfig(EnhancedDataClassJsonMixin):
+class UploaderConfig(BaseModel):
     pass
 
 

unstructured_ingest/v2/pipeline/pipeline.py

@@ -15,7 +15,6 @@ from unstructured_ingest.v2.pipeline.steps.partition import Partitioner, Partiti
 from unstructured_ingest.v2.pipeline.steps.stage import UploadStager, UploadStageStep
 from unstructured_ingest.v2.pipeline.steps.uncompress import Uncompressor, UncompressStep
 from unstructured_ingest.v2.pipeline.steps.upload import Uploader, UploadStep
-from unstructured_ingest.v2.pipeline.utils import sterilize_dict
 from unstructured_ingest.v2.processes.chunker import ChunkerConfig
 from unstructured_ingest.v2.processes.connector_registry import (
     ConnectionConfig,
@@ -178,10 +177,7 @@ class Pipeline:
         return filtered_records
 
     def _run(self):
-        logger.info(
-            f"Running local pipline: {self} with configs: "
-            f"{sterilize_dict(self.context.to_dict(redact_sensitive=True))}"
-        )
+        logger.info(f"Running local pipline: {self} with configs: " f"{self.context.json()}")
         if self.context.mp_supported:
             manager = mp.Manager()
             self.context.status = manager.dict()
@@ -192,22 +188,26 @@ class Pipeline:
         indices = self.indexer_step.run()
         indices_inputs = [{"file_data_path": i} for i in indices]
         if not indices_inputs:
+            logger.info("No files to process after indexer, exiting")
             return
 
         # Initial filtering on indexed content
         indices_inputs = self.apply_filter(records=indices_inputs)
         if not indices_inputs:
+            logger.info("No files to process after filtering indexed content, exiting")
             return
 
         # Download associated content to local file system
         downloaded_data = self.downloader_step(indices_inputs)
         downloaded_data = self.clean_results(results=downloaded_data)
         if not downloaded_data:
+            logger.info("No files to process after downloader, exiting")
             return
 
         # Post download filtering
         downloaded_data = self.apply_filter(records=downloaded_data)
         if not downloaded_data:
+            logger.info("No files to process after filtering downloaded content, exiting")
             return
 
         # Run uncompress if available
@@ -219,6 +219,7 @@ class Pipeline:
             # Post uncompress filtering
             downloaded_data = self.apply_filter(records=downloaded_data)
             if not downloaded_data:
+                logger.info("No files to process after filtering uncompressed content, exiting")
                 return
 
         if not downloaded_data:
@@ -228,6 +229,7 @@ class Pipeline:
         elements = self.partitioner_step(downloaded_data)
         elements = self.clean_results(results=elements)
         if not elements:
+            logger.info("No files to process after partitioning, exiting")
             return
 
         # Run element specific modifiers
@@ -235,6 +237,7 @@ class Pipeline:
             elements = step(elements) if step else elements
             elements = self.clean_results(results=elements)
             if not elements:
+                logger.info(f"No files to process after {step.__class__.__name__}, exiting")
                 return
 
         # Upload the final result
@@ -337,7 +340,7 @@ class Pipeline:
             )
         if len(destination_entry) != 1:
             raise ValueError(
-                "no entry found in source registry with matching uploader, "
+                "no entry found in destination registry with matching uploader, "
                 "stager and connection configs"
             )
 

unstructured_ingest/v2/pipeline/steps/chunk.py

@@ -5,13 +5,11 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
-from unstructured.staging.base import elements_to_dicts
-
 from unstructured_ingest.v2.interfaces import FileData
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
-from unstructured_ingest.v2.pipeline.utils import sterilize_dict
 from unstructured_ingest.v2.processes.chunker import Chunker
+from unstructured_ingest.v2.utils import serialize_base_model_json
 
 STEP_ID = "chunk"
 
@@ -30,11 +28,7 @@ class ChunkStep(PipelineStep):
         return f"{self.identifier} ({self.process.config.chunking_strategy})"
 
     def __post_init__(self):
-        config = (
-            sterilize_dict(self.process.config.to_dict(redact_sensitive=True))
-            if self.process.config
-            else None
-        )
+        config = self.process.config.json() if self.process.config else None
         logger.info(f"Created {self.identifier} with configs: {config}")
 
     def should_chunk(self, filepath: Path, file_data: FileData) -> bool:
@@ -72,13 +66,13 @@ class ChunkStep(PipelineStep):
             chunked_content_raw = await fn(**fn_kwargs)
         self._save_output(
             output_filepath=str(output_filepath),
-            chunked_content=elements_to_dicts(chunked_content_raw),
+            chunked_content=chunked_content_raw,
         )
         return ChunkStepResponse(file_data_path=file_data_path, path=str(output_filepath))
 
     def get_hash(self, extras: Optional[list[str]]) -> str:
-        hashable_string = json.dumps(
-            self.process.config.to_dict(), sort_keys=True, ensure_ascii=True
+        hashable_string = serialize_base_model_json(
+            model=self.process.config, sort_keys=True, ensure_ascii=True
         )
         if extras:
             hashable_string += "".join(extras)