dlt-filesystem 0.18.0__py3-none-any.whl

dlt_filesystem/error.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+
+class MissingConnectorOption(Exception):
+    def __init__(self, option, connector):
+        super().__init__(f"{option} is required to connect to {connector}")
+
+
+class InvalidBlobTableError(Exception):
+    def __init__(self, source):
+        super().__init__(
+            f"Invalid source table for: {source}. "
+            "Ensure that the table is in the format {bucket-name}/{file glob}"
+        )

dlt_filesystem/source/adapter.py

@@ -0,0 +1,180 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Reads files in s3, gs or azure buckets using fsspec and provides convenience resources for chunked reading of various file formats"""
+
+from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, Union
+
+import dlt
+from dlt.sources import DltResource
+from dlt.sources.credentials import FileSystemCredentials
+from dlt.sources.filesystem import FileItem, FileItemDict, fsspec_filesystem
+from fsspec import AbstractFileSystem
+
+from dlt_filesystem.source.error import NoFilesFoundError
+from dlt_filesystem.source.format import readers as reader_functions
+from dlt_filesystem.source.format.readers import ReadersSource
+from dlt_filesystem.source.format.registry import (
+    READER_REGISTRATIONS,
+    ReaderRegistration,
+)
+from dlt_filesystem.source.lister import glob_files
+
+from .model import FilesystemConfigurationResource
+
+
+def _resolve_reader(registration: ReaderRegistration):
+    """Resolve a registered reader name without making the registry import reader code."""
+    reader = getattr(reader_functions, registration.reader_name, None)
+    if not callable(reader):
+        raise ValueError(
+            f"Reader function {registration.reader_name!r} is not defined in "
+            "dlt_filesystem.source.format.readers"
+        )
+    return reader
+
+
+@dlt.source(_impl_cls=ReadersSource, spec=FilesystemConfigurationResource)
+def readers(
+    bucket_url: str = dlt.secrets.value,
+    credentials: Union[FileSystemCredentials, AbstractFileSystem] = dlt.secrets.value,
+    file_glob: Optional[str] = "*",
+    *,
+    kwargs: Optional[Dict[str, Any]] = None,
+    client_kwargs: Optional[Dict[str, Any]] = None,
+    incremental: Optional[dlt.sources.incremental[Any]] = None,
+) -> Tuple[DltResource, ...]:
+    """This source provides a few resources that are chunked file readers. Readers can be further parametrized before use
+       read_csv(chunksize, **pandas_kwargs)
+       read_json(chunksize)
+       read_jsonl(chunksize)
+       read_parquet(chunksize)
+
+    Args:
+        bucket_url (str): The url to the bucket.
+        credentials (FileSystemCredentials | AbstractFilesystem): The credentials to the filesystem of fsspec `AbstractFilesystem` instance.
+        file_glob (str, optional): The filter to apply to the files in glob format. by default lists all files in bucket_url non-recursively
+        kwargs (Optional[Dict[str, Any]]): Additional arguments passed to the fsspec constructor, ie. dict(use_ssl=True) for s3fs
+        client_kwargs (Optional[Dict[str, Any]]): Additional arguments passed to the underlying fsspec native client, ie. dict(verify="public.crt") for botocore
+        incremental (Optional[dlt.sources.incremental[Any]]): Defines an incremental cursor on the listed files, with `modification_date`
+            being the most common choice, which returns only files created since the previous run.
+    """
+    filesystem_resource = filesystem(
+        bucket_url,
+        credentials,
+        file_glob=file_glob,
+        kwargs=kwargs,
+        client_kwargs=client_kwargs,
+        incremental=incremental,
+    )
+
+    return tuple(
+        filesystem_resource
+        | dlt.transformer(
+            name=registration.reader_name,
+            max_table_nesting=registration.max_table_nesting,
+        )(_resolve_reader(registration))
+        for registration in READER_REGISTRATIONS
+    )
+
+
+@dlt.resource(
+    primary_key="file_url", spec=FilesystemConfigurationResource, standalone=True
+)
+def filesystem(
+    bucket_url: str = dlt.secrets.value,
+    credentials: Union[FileSystemCredentials, AbstractFileSystem] = dlt.secrets.value,
+    file_glob: Optional[str] = "*",
+    files_per_page: int = 100,
+    extract_content: bool = False,
+    require_file_match: bool = False,
+    filesystem_incremental: bool = False,
+    *,
+    kwargs: Optional[Dict[str, Any]] = None,
+    client_kwargs: Optional[Dict[str, Any]] = None,
+    incremental: Optional[dlt.sources.incremental[Any]] = None,
+) -> Iterator[List[FileItem]]:
+    """This resource lists files in `bucket_url` using `file_glob` pattern. The files are yielded as FileItem which also
+    provide methods to open and read file data. It should be combined with transformers that further process (ie. load files)
+
+    Args:
+        bucket_url (str): The url to the bucket.
+        credentials (FileSystemCredentials | AbstractFilesystem): The credentials to the filesystem of fsspec `AbstractFilesystem` instance.
+        file_glob (str, optional): The filter to apply to the files in glob format. by default lists all files in bucket_url non-recursively
+        files_per_page (int, optional): The number of files to process at once, defaults to 100.
+        extract_content (bool, optional): If true, the content of the file will be extracted if
+            false it will return a fsspec file, defaults to False.
+        require_file_match (bool, optional): Raise when the concrete source selection
+            matches no file. Defaults to False for direct uses of this resource.
+        filesystem_incremental (bool, optional): Resolve trustworthy modification
+            times when the listing itself does not carry one. Defaults to False.
+        kwargs (Optional[Dict[str, Any]]): Additional arguments passed to the fsspec constructor, ie. dict(use_ssl=True) for s3fs
+        client_kwargs (Optional[Dict[str, Any]]): Additional arguments passed to the underlying fsspec native client, ie. dict(verify="public.crt") for botocore
+        incremental (Optional[dlt.sources.incremental[Any]]): Defines an incremental cursor on the listed files, with `modification_date`
+            being the most common choice, which returns only files created since the previous run.
+            A cursor carrying `row_order` also orders the listing by its cursor field.
+
+    Returns:
+        Iterator[List[FileItem]]: The list of files.
+    """
+
+    fs_client: AbstractFileSystem
+    if isinstance(credentials, AbstractFileSystem):
+        # A caller who hands over a constructed filesystem has already spent
+        # `kwargs` and `client_kwargs` on building it, so both are ignored here,
+        # exactly as they are in dlt's own resource.
+        fs_client = credentials
+    else:
+        fs_client = fsspec_filesystem(
+            bucket_url, credentials, kwargs=kwargs, client_kwargs=client_kwargs
+        )[0]
+
+    file_models: Iterable[FileItem] = glob_files(
+        fs_client,
+        bucket_url,
+        file_glob or "**",
+        filesystem_incremental=filesystem_incremental,
+    )
+    if incremental and incremental.row_order:
+        # `row_order` is ascending or descending *in the direction `last_value_func`
+        # advances*, so it maps onto a raw sort only through that function: `max`
+        # advances upwards and `min` advances downwards, which inverts the
+        # comparison for `min`. Mirrors dlt's own expression.
+        reverse = (
+            incremental.row_order == "asc" and incremental.last_value_func is min
+        ) or (incremental.row_order == "desc" and incremental.last_value_func is max)
+        # The listing has to be materialised to be ordered. Only this branch pays
+        # for it; the default stays lazy.
+        file_models = sorted(
+            file_models,
+            key=lambda listed: listed[incremental.cursor_path],  # ty: ignore[invalid-key]
+            reverse=reverse,
+        )
+
+    matched_files = 0
+    files_chunk: List[FileItem] = []
+    for file_model in file_models:
+        matched_files += 1
+        file_dict = FileItemDict(file_model, fs_client)
+        if extract_content:
+            file_dict["file_content"] = file_dict.read_bytes()
+        files_chunk.append(file_dict)  # ty: ignore[invalid-argument-type]
+        # wait for the chunk to be full
+        if len(files_chunk) >= files_per_page:
+            yield files_chunk
+            files_chunk = []
+    if require_file_match and matched_files == 0:
+        raise NoFilesFoundError(bucket_url, file_glob or "**")
+    if files_chunk:
+        yield files_chunk

dlt_filesystem/source/base.py

@@ -0,0 +1,89 @@
+from typing import Union
+from urllib.parse import urlparse
+
+
+class FilesystemSource:
+    """Shared capabilities for the filesystem-family sources.
+
+    Covers the local ``file://`` source and every remote transport
+    (``s3://``, ``gs://``, ``az://`` / ``adls://`` / ``abfss://``, ``sftp://``),
+    which all converge on the same reader after URI parsing.
+
+    Filesystem sources manage their own incremental behaviour
+    (``handles_incrementality`` is ``True``) and support opt-in file selection by
+    modification time (``supports_filesystem_incremental`` is ``True``). They
+    carry no resource-level write disposition, so a run-level disposition is safe
+    to apply: ``run_ingest`` honours an explicit ``--incremental-strategy append``
+    / ``replace`` for them (``honours_run_disposition`` is ``True``). Sources that
+    set their own resource-level disposition leave this ``False`` (the default)
+    so the run-level value never overrides theirs.
+    """
+
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def honours_run_disposition(self) -> bool:
+        return True
+
+    def consumed_run_options(self) -> frozenset:
+        """Return the run options this source's ``dlt_source`` accepts by name.
+
+        Every other name in omniload's run vocabulary is filtered out before the
+        call, so it never reaches an fsspec or Arrow constructor as a stray
+        keyword. The two named here are resource options, not connector ones:
+        they configure how the reader resource is built (``FilesystemReference``)
+        and every ``dlt_source`` implementation in this family declares them
+        explicitly rather than reading them out of ``**kwargs``.
+        """
+        return frozenset({"filesystem_incremental", "column_types"})
+
+    def supports_filesystem_incremental(self) -> bool:
+        """Return whether the source supports file-level mtime selection."""
+        return True
+
+    def produces_multiple_tables(self, uri: str, table: str) -> bool:
+        """Return whether a workbook selection dispatches worksheet tables."""
+        from dlt_filesystem.source.error import UnsupportedEndpointError
+        from dlt_filesystem.source.format.readers import (
+            spreadsheet_selection_is_plural,
+        )
+        from dlt_filesystem.source.router import (
+            blob_hints,
+            determine_endpoint,
+            parse_uri,
+        )
+
+        parsed_uri = urlparse(uri)
+        _, path = parse_uri(parsed_uri, table)
+        try:
+            endpoint = determine_endpoint(table, path)
+        except (UnsupportedEndpointError, ValueError):
+            return False
+        return endpoint in {
+            "read_excel",
+            "read_ods",
+        } and spreadsheet_selection_is_plural(blob_hints(parsed_uri, table))
+
+    @staticmethod
+    def endpoint_namespace(endpoint: Union[str, None], default: str) -> str:
+        """
+        Return a normalized endpoint identity without credentials or query values.
+        It is used for incremental loading based on file modification times.
+
+        # TODO: Remove `default` argument again?
+        """
+        if not endpoint:
+            return default
+
+        parsed = urlparse(endpoint if "://" in endpoint else f"//{endpoint}")
+        host = parsed.hostname
+        if not host:
+            return default
+
+        host = host.lower()
+        if ":" in host:
+            host = f"[{host}]"
+        if parsed.port is not None:
+            host = f"{host}:{parsed.port}"
+
+        return f"{host}{parsed.path.rstrip('/')}"

dlt_filesystem/source/core.py

@@ -0,0 +1,132 @@
+from typing import Any, Optional, Union
+
+import dlt
+from dlt.extract import DltResource, DltSource
+from fsspec import AbstractFileSystem
+
+from dlt_filesystem.source.adapter import filesystem, readers
+from dlt_filesystem.source.error import UnsupportedEndpointError
+from dlt_filesystem.source.format.registry import (
+    reader_for_format,
+    supported_file_format_message,
+)
+from dlt_filesystem.source.model import (
+    FilesystemLocator,
+    FilesystemReference,
+    ResourceOptions,
+)
+from dlt_filesystem.source.router import determine_endpoint
+
+
+def resource_for_reader(ref: FilesystemReference) -> Union[DltSource, DltResource]:
+    """Build the filesystem reader resource named by ``ref.reader_name``.
+
+    Threads ``column_types`` into ``read_csv_headless`` and per-URI reader hints (e.g. XML's
+    ``#tagname``) into a hint-consuming reader; every other reader is selected as-is.
+    """
+
+    # Enforce concrete selections on this outer lister only. Piping it into the
+    # selected reader replaces the reader source's inner parent, so discovery runs
+    # exactly once and the check happens before any downstream incremental filter.
+    filesystem_resource = filesystem(
+        ref.bucket_url,
+        ref.fs,
+        file_glob=ref.file_glob,
+        extract_content=False,
+        require_file_match=ref.require_file_match,
+        filesystem_incremental=ref.filesystem_incremental,
+    )
+    if ref.filesystem_incremental:
+        filesystem_resource = filesystem_resource.with_name(
+            ref.incremental_resource_name
+        )
+        filesystem_resource.apply_hints(
+            incremental=dlt.sources.incremental("modification_date")
+        )
+    all_readers = readers(
+        ref.bucket_url, ref.fs, file_glob=ref.file_glob
+    ).with_resources(ref.reader_name)
+    reader = all_readers.selected_resources[ref.reader_name]
+
+    # Apply parameter bindings for certain readers.
+    # TODO: Can this be generalized? Why not always loop in column_names into reader hints?
+    reader_kwargs: dict[str, Any] = dict(ref.hints)
+    if ref.reader_name in {"read_excel", "read_ods"}:
+        # The filesystem lister yields pages of files. Keep one collision registry
+        # bound to the reader so distinct worksheet names cannot normalize to the
+        # same table even when the workbooks occur in different pages.
+        reader_kwargs["worksheet_names"] = {}
+
+    if ref.reader_name == "read_csv_headless":
+        column_names = list(ref.column_types.keys()) if ref.column_types else None
+        reader = reader.bind(column_names=column_names, **reader_kwargs)
+    else:
+        reader = reader.bind(**reader_kwargs)
+
+    # Connect and propagate elements.
+    return filesystem_resource | reader
+
+
+def infer_resource(
+    fs: AbstractFileSystem,
+    locator: FilesystemLocator,
+    options: Optional[ResourceOptions] = None,
+) -> Union[DltSource, DltResource]:
+    """
+    Infer dlt resource from fsspec filesystem, with reader.
+
+    Args:
+        fs: The filesystem the connector built from its own connection arguments.
+        locator: The parsed source URI.
+        options: The resource options a `dlt_source` implementation declares by
+            name in its own signature (`filesystem_incremental`, `column_types`,
+            `reader_hints`). Omitted by callers that have none, which reads the
+            same as a run that enabled nothing.
+    """
+
+    options = options or ResourceOptions()
+
+    # Decode into base url and url path / file glob, and apply sanity checks.
+    locator.validate()
+
+    # TODO: Naming things: Rename `determine_endpoint` to `infer_reader`.
+    try:
+        # A `#format` the selection names wins over the file extension, whichever
+        # carrier it rode in on. `determine_endpoint` reads it from the table form
+        # itself; this also honours it on the URI, which for an HTTP URL is the
+        # only place it can be written.
+        endpoint = (
+            reader_for_format(locator.format_hint)
+            if locator.format_hint
+            else determine_endpoint(locator.path, locator.file_glob)
+        )
+    except UnsupportedEndpointError:
+        raise ValueError(supported_file_format_message(locator.name)) from None
+
+    # TODO: FilesystemLocator and FilesystemReference are somewhat redundant now. Refactor!
+    #       => Bundle fs, locator and reader into another data class , then feed that to
+    #       `resource_for_reader`.
+    return resource_for_reader(
+        FilesystemReference(
+            fs=fs,
+            bucket_url=locator.bucket_url,
+            file_glob=locator.file_glob,
+            reader_name=endpoint,
+            # Require a match only when the locator's unparsed carrier names one
+            # concrete file. This keeps wildcard discovery empty-safe.
+            require_file_match=locator.require_file_match,
+            # A URI fragment addresses one file, so it wins over a hint the source
+            # derived for the whole run.
+            hints={**(options.reader_hints or {}), **locator.hints},
+            filesystem_incremental=options.filesystem_incremental,
+            # TODO: Can `column_types` be looped into reader|writer hints instead?
+            #       We believe it represents a special case handling for `csv_headless`.
+            # The run value (`--columns`) wins; the URI query parameter remains the
+            # fallback for callers that address it there.
+            column_types=(
+                options.column_types
+                if options.column_types is not None
+                else locator.options.params.get("column_types")
+            ),
+        )
+    )

dlt_filesystem/source/error.py

@@ -0,0 +1,76 @@
+from urllib.parse import urlsplit, urlunsplit
+
+
+def _safe_location(bucket_url: str) -> str:
+    """Remove query and fragment data that may contain filesystem credentials."""
+    parsed = urlsplit(bucket_url)
+    netloc = parsed.netloc.rpartition("@")[2]
+    location = urlunsplit((parsed.scheme, netloc, parsed.path, "", ""))
+    if parsed.scheme and not netloc and not parsed.path and "://" in bucket_url:
+        return f"{parsed.scheme}://"
+    return location
+
+
+class NoFilesFoundError(FileNotFoundError):
+    """A concrete filesystem source selection matched no file."""
+
+    def __init__(self, bucket_url: str, file_glob: str):
+        super().__init__(
+            f"No files found at {_safe_location(bucket_url)!r} "
+            f"for concrete source selection {file_glob!r}"
+        )
+
+
+class UnsupportedEndpointError(Exception):
+    pass
+
+
+class WorksheetNameCollisionError(ValueError):
+    """Two distinct worksheet names resolve to the same destination table."""
+
+    def __init__(
+        self,
+        *,
+        table_name: str,
+        first_sheet: str,
+        first_file: str,
+        second_sheet: str,
+        second_file: str,
+    ) -> None:
+        super().__init__(
+            f"Worksheet names {first_sheet!r} in {first_file!r} and "
+            f"{second_sheet!r} in {second_file!r} both resolve to destination "
+            f"table {table_name!r} under the active schema naming convention"
+        )
+
+
+class MissingDecoderError(UnsupportedEndpointError):
+    """A routable format resolved to a reader or writer whose package is not installed.
+
+    Raised (instead of a bare ``ImportError``) when an iterable-backed format such as
+    ``msgpack`` is requested but the optional ``iterable`` extra / its per-format decoder is
+    absent. Carries the exact ``pip install`` target so the message is actionable.
+
+    The write side raises it too (``target.writer.write_yaml``): the missing package is the
+    format's, not the direction's, so both halves owe the same install hint rather than one
+    of them a bare ``ImportError``.
+    """
+
+    pass
+
+
+class MissingReaderOptionError(ValueError):
+    """A file reader needs a per-URI ``#key=value`` hint that was not supplied.
+
+    Raised (instead of the connection-shaped ``MissingValueError`` or a bare ``AttributeError``
+    from the decoder) when a format requires a reader option that must arrive via the
+    ``#key=value`` fragment. XML, for instance, needs ``#tagname=<row-tag>`` to know which
+    repeated element is a row. Subclasses ``ValueError`` so it reads as a bad-argument error.
+    """
+
+    def __init__(self, option: str, file_format: str, example: str):
+        super().__init__(
+            f"The {file_format} reader requires a '{option}' hint naming the repeated row "
+            f"element. Append it to the source URI as a #{option}=<value> fragment, "
+            f"e.g. {example}"
+        )

dlt_filesystem/source/format/bson_codec.py

@@ -0,0 +1,99 @@
+"""BSON extended-type normalization for the filesystem BSON reader.
+
+Self-contained mirror of omniload's ``convert_mongo_objs`` MongoDB helper that
+deliberately does *not* import the Mongo source: ``mongodb/helpers.py`` imports the
+``pymongo`` client classes (``MongoClient``/``Collection``/``Cursor``) at module top,
+which would couple this filesystem reader to the Mongo driver. ``bson`` itself ships
+with ``pymongo`` (a hard dependency), so decoding needs no extra package.
+
+This module imports the ``bson`` submodules at its own top level rather than lazily,
+because it is itself imported lazily (only from ``_read_bson``), so the cost is paid
+only when a BSON file is actually read, never at CLI startup or for other formats.
+
+It goes beyond the Mongo helper's coverage: besides the shared ObjectId / Decimal128 /
+datetime / Regex / Timestamp cases, it converts ``Binary`` (base64 str) and the extended
+types the Mongo helper leaves raw (``DBRef``, ``MinKey``, ``MaxKey``, ``Code``). Those
+raw BSON objects are not JSON-serializable, so without conversion a dump containing one
+crashes the load (``TypeError: Type is not JSON serializable``); here they become
+portable Extended-JSON-shaped values.
+"""
+
+import base64
+import datetime as _datetime
+from typing import Any
+
+from bson.code import Code
+from bson.dbref import DBRef
+from bson.decimal128 import Decimal128
+from bson.max_key import MaxKey
+from bson.min_key import MinKey
+from bson.objectid import ObjectId
+from bson.regex import Regex
+from bson.timestamp import Timestamp
+from dlt.common.time import ensure_pendulum_datetime_utc
+from dlt.common.utils import map_nested_values_in_place
+
+
+def convert_bson_objs(value: Any) -> Any:
+    """Convert a single BSON extended value to a dlt-serializable Python type.
+
+    Applied to leaf values only: ``map_nested_values_in_place`` recurses into nested
+    dicts and lists and calls this on the scalars, so this never sees a plain container
+    (BSON extended types like ``DBRef`` are objects, not dict/list, so they do reach
+    here). Conversions:
+
+    - ``Binary`` (and any raw ``bytes``) -> base64 ``str``
+    - ``ObjectId`` / ``Decimal128`` -> ``str``
+    - ``datetime`` -> pendulum UTC ``datetime``
+    - ``Regex`` -> pattern ``str``
+    - ``Timestamp`` -> pendulum UTC ``datetime``
+    - ``DBRef`` -> ``{"$ref", "$id" (normalized), "$db"?}``
+    - ``MinKey`` / ``MaxKey`` -> ``{"$minKey": 1}`` / ``{"$maxKey": 1}``
+    - ``Code`` -> code ``str`` (or ``{"$code", "$scope" (normalized)}`` when it has scope)
+    - anything else -> unchanged
+    """
+    # Binary subclasses bytes, so the bytes branch must come first and covers both. Raw
+    # bytes are emitted as base64 str because dlt's CSV/text writer UTF-8-decodes binary
+    # values (raises on arbitrary bytes); base64 str is portable across jsonl, file,
+    # parquet and warehouse targets. See PLAN Decisions §1.
+    if isinstance(value, (bytes, bytearray)):
+        return base64.b64encode(bytes(value)).decode("ascii")
+    if isinstance(value, (ObjectId, Decimal128)):
+        return str(value)
+    if isinstance(value, _datetime.datetime):
+        return ensure_pendulum_datetime_utc(value)
+    if isinstance(value, Regex):
+        # value.pattern is the raw pattern string; do NOT try_compile() it (BSON regexes
+        # can carry PCRE-only syntax that Python's re rejects, which would crash the read
+        # of an otherwise valid dump).
+        return value.pattern
+    if isinstance(value, Timestamp):
+        return ensure_pendulum_datetime_utc(value.as_datetime())
+    if isinstance(value, DBRef):
+        # $id is normalized recursively (it is typically an ObjectId). The convert()
+        # return is used as-is by the caller (map_nested does not re-descend into it), so
+        # the id must be converted here rather than relying on outer recursion.
+        ref: dict[str, Any] = {
+            "$ref": value.collection,
+            "$id": convert_bson_objs(value.id),
+        }
+        if value.database is not None:
+            ref["$db"] = value.database
+        return ref
+    if isinstance(value, MinKey):
+        return {"$minKey": 1}
+    if isinstance(value, MaxKey):
+        return {"$maxKey": 1}
+    if isinstance(value, Code):
+        # Code subclasses str, so str(value) is the code text. Scope (when present) is a
+        # nested doc that needs its own normalization pass.
+        if value.scope:
+            return {
+                "$code": str(value),
+                "$scope": map_nested_values_in_place(
+                    convert_bson_objs, dict(value.scope)
+                ),
+            }
+        return str(value)
+
+    return value

dlt_filesystem/source/format/helpers.py

@@ -0,0 +1,74 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for the filesystem resource."""
+
+from typing import Any, Dict, Generator, Iterable, List, Optional
+
+from dlt.common.typing import TDataItem
+
+from dlt_filesystem.source.format.settings import DEFAULT_CHUNK_SIZE
+
+
+def add_columns(columns: List[str], rows: List[List[Any]]) -> List[Dict[str, Any]]:
+    """Adds column names to the given rows.
+
+    Args:
+        columns (List[str]): The column names.
+        rows (List[List[Any]]): The rows.
+
+    Returns:
+        List[Dict[str, Any]]: The rows with column names.
+    """
+    result = []
+    for row in rows:
+        result.append(dict(zip(columns, row)))
+
+    return result
+
+
+def fetch_arrow(file_data, chunk_size: Optional[int] = None) -> Iterable[TDataItem]:
+    """Fetches data from the given CSV file.
+
+    Args:
+        file_data (DuckDBPyRelation): The CSV file data.
+        chunk_size (int): The number of rows to read at once.
+
+    Yields:
+        Iterable[TDataItem]: Data items, read from the given CSV file.
+    """
+    chunk_size = chunk_size or DEFAULT_CHUNK_SIZE
+    batcher = file_data.fetch_arrow_reader(batch_size=chunk_size)
+    yield from batcher
+
+
+def fetch_json(
+    file_data, chunk_size: Optional[int] = None
+) -> Generator[List[Dict[str, Any]], None, None]:
+    """Fetches data from the given CSV file.
+
+    Args:
+        file_data (DuckDBPyRelation): The CSV file data.
+        chunk_size (int): The number of rows to read at once.
+
+    Yields:
+        Iterable[TDataItem]: Data items, read from the given CSV file.
+    """
+    chunk_size = chunk_size or DEFAULT_CHUNK_SIZE
+    while True:
+        batch = file_data.fetchmany(chunk_size)
+        if not batch:
+            break
+
+        yield add_columns(file_data.columns, batch)