omniload 0.9.0__tar.gz → 0.9.2__tar.gz

{omniload-0.9.0 → omniload-0.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omniload
-Version: 0.9.0
+Version: 0.9.2
 Summary: omniload is a polyglot data loader based on dlt. Copy data between any source and any destination.
 Author: Mustafa Ersan, Sanju Shah Gupta, Saravjeet 'Aman' Singh
 Author-email: Burak Karakan <burak.karakan@getbruin.com>
@@ -74,12 +74,12 @@ Requires-Dist: couchbase<5,>=4
 Requires-Dist: databricks-sql-connector<5
 Requires-Dist: databricks-sqlalchemy<3
 Requires-Dist: dataclasses-json<0.7
-Requires-Dist: dlt<1.30,>=1.22
+Requires-Dist: dlt<1.31,>=1.22
 Requires-Dist: dlt-cratedb<0.2
 Requires-Dist: duckdb<1.6
 Requires-Dist: duckdb-engine<0.18
 Requires-Dist: elasticsearch<10
-Requires-Dist: facebook-business<26
+Requires-Dist: facebook-business<27
 Requires-Dist: flatten-json<0.2
 Requires-Dist: gitpython>=3.1.55
 Requires-Dist: google-ads<32
@@ -132,7 +132,7 @@ Requires-Dist: poethepoet<1; extra == "develop"
 Requires-Dist: pyproject-fmt<3; extra == "develop"
 Requires-Dist: pytest-repeat<1; extra == "develop"
 Requires-Dist: ruff==0.16.*; extra == "develop"
-Requires-Dist: ty==0.0.65; extra == "develop"
+Requires-Dist: ty==0.0.70; extra == "develop"
 Requires-Dist: validate-pyproject<1; extra == "develop"
 Provides-Extra: filesystem
 Requires-Dist: adlfs; extra == "filesystem"

{omniload-0.9.0 → omniload-0.9.2}/pyproject.toml

@@ -83,12 +83,12 @@ dependencies = [
   "databricks-sql-connector<5",
   "databricks-sqlalchemy<3",
   "dataclasses-json<0.7",
-  "dlt>=1.22,<1.30",
+  "dlt>=1.22,<1.31",
   "dlt-cratedb<0.2",
   "duckdb<1.6",
   "duckdb-engine<0.18",
   "elasticsearch<10",
-  "facebook-business<26",
+  "facebook-business<27",
   "flatten-json<0.2",
   "gitpython>=3.1.55",
   "google-ads<32",
@@ -142,7 +142,7 @@ optional-dependencies.develop = [
   "pyproject-fmt<3",
   "pytest-repeat<1",
   "ruff==0.16.*",
-  "ty==0.0.65",
+  "ty==0.0.70",
   "validate-pyproject<1",
 ]
 optional-dependencies.filesystem = [

{omniload-0.9.0 → omniload-0.9.2}/src/dlt_filesystem/source/adapter.py

@@ -19,7 +19,7 @@ from typing import Iterator, List, Optional, Tuple, Union
 import dlt
 from dlt.sources import DltResource
 from dlt.sources.credentials import FileSystemCredentials
-from dlt.sources.filesystem import FileItem, FileItemDict, fsspec_filesystem, glob_files
+from dlt.sources.filesystem import FileItem, FileItemDict, fsspec_filesystem
 from fsspec import AbstractFileSystem
 
 from dlt_filesystem.source.error import NoFilesFoundError
@@ -38,6 +38,7 @@ from dlt_filesystem.source.format.readers import (
     read_xml,
     read_yaml,
 )
+from dlt_filesystem.source.lister import glob_files
 
 from .model import FilesystemConfigurationResource
 

{omniload-0.9.0 → omniload-0.9.2}/src/dlt_filesystem/source/core.py

@@ -1,4 +1,4 @@
-from typing import Any, Union
+from typing import Any, Optional, Union
 
 import dlt
 from dlt.extract import DltResource, DltSource
@@ -7,7 +7,11 @@ from fsspec import AbstractFileSystem
 from dlt_filesystem.source.adapter import filesystem, readers
 from dlt_filesystem.source.error import UnsupportedEndpointError
 from dlt_filesystem.source.format.registry import supported_file_format_message
-from dlt_filesystem.source.model import FilesystemLocator, FilesystemReference
+from dlt_filesystem.source.model import (
+    FilesystemLocator,
+    FilesystemReference,
+    ResourceOptions,
+)
 from dlt_filesystem.source.router import determine_endpoint
 
 
@@ -60,12 +64,23 @@ def resource_for_reader(ref: FilesystemReference) -> Union[DltSource, DltResourc
 
 
 def infer_resource(
-    fs: AbstractFileSystem, locator: FilesystemLocator
+    fs: AbstractFileSystem,
+    locator: FilesystemLocator,
+    options: Optional[ResourceOptions] = None,
 ) -> Union[DltSource, DltResource]:
     """
     Infer dlt resource from fsspec filesystem, with reader.
+
+    Args:
+        fs: The filesystem the connector built from its own connection arguments.
+        locator: The parsed source URI.
+        options: The resource options omniload's run contributes, as split out by
+            `split_run_options`. Omitted by callers that have none, which reads the
+            same as a run that enabled nothing.
     """
 
+    options = options or ResourceOptions()
+
     # Decode into base url and url path / file glob, and apply sanity checks.
     locator.validate()
 
@@ -88,8 +103,15 @@ def infer_resource(
             # concrete file. This keeps wildcard discovery empty-safe.
             require_file_match=locator.require_file_match,
             hints=locator.hints,
+            filesystem_incremental=options.filesystem_incremental,
             # TODO: Can `column_types` be looped into reader|writer hints instead?
             #       We believe it represents a special case handling for `csv_headless`.
-            column_types=locator.options.params.get("column_types"),
+            # The run value (`--columns`) wins; the URI query parameter remains the
+            # fallback for callers that address it there.
+            column_types=(
+                options.column_types
+                if options.column_types is not None
+                else locator.options.params.get("column_types")
+            ),
         )
     )

{omniload-0.9.0 → omniload-0.9.2}/src/dlt_filesystem/source/fsspec/databricks.py

@@ -4,7 +4,7 @@ from fsspec import AbstractFileSystem
 
 from dlt_filesystem.source.base import FilesystemSource
 from dlt_filesystem.source.core import infer_resource
-from dlt_filesystem.source.model import FilesystemLocator
+from dlt_filesystem.source.model import FilesystemLocator, split_run_options
 from dlt_filesystem.util.python import (
     cast_to_bool,
     cast_to_int,
@@ -43,8 +43,9 @@ class DatabricksSource(FilesystemSource):
         )
 
         # Decode individual options (type casting, default values, sanity checks).
+        resource_options, connector_kwargs = split_run_options(kwargs)
         fs_kwargs = locator.options.fs_kwargs
-        fs_kwargs.update(kwargs)
+        fs_kwargs.update(connector_kwargs)
         cast_to_int(
             fs_kwargs,
             [
@@ -71,4 +72,4 @@ class DatabricksSource(FilesystemSource):
 
         # Create filesystem and dlt resource wrapper.
         fs = self.fs_class(**fs_kwargs)
-        return infer_resource(fs=fs, locator=locator)
+        return infer_resource(fs=fs, locator=locator, options=resource_options)

{omniload-0.9.0 → omniload-0.9.2}/src/dlt_filesystem/source/fsspec/dropbox.py

@@ -4,7 +4,7 @@ from fsspec import AbstractFileSystem
 
 from dlt_filesystem.source.base import FilesystemSource
 from dlt_filesystem.source.core import infer_resource
-from dlt_filesystem.source.model import FilesystemLocator
+from dlt_filesystem.source.model import FilesystemLocator, split_run_options
 
 
 class DropboxSource(FilesystemSource):
@@ -34,9 +34,10 @@ class DropboxSource(FilesystemSource):
         )
 
         # Decode individual options (type casting, default values, sanity checks). Schema:
+        resource_options, connector_kwargs = split_run_options(kwargs)
         fs_kwargs = locator.options.fs_kwargs
-        fs_kwargs.update(kwargs)
+        fs_kwargs.update(connector_kwargs)
 
         # Create filesystem and dlt resource wrapper.
         fs = self.fs_class(**fs_kwargs)
-        return infer_resource(fs=fs, locator=locator)
+        return infer_resource(fs=fs, locator=locator, options=resource_options)

{omniload-0.9.0 → omniload-0.9.2}/src/dlt_filesystem/source/fsspec/ftp.py

@@ -1,7 +1,7 @@
 from dlt_filesystem.error import MissingConnectorOption
 from dlt_filesystem.source.base import FilesystemSource
 from dlt_filesystem.source.core import infer_resource
-from dlt_filesystem.source.model import FilesystemLocator
+from dlt_filesystem.source.model import FilesystemLocator, split_run_options
 from dlt_filesystem.util.python import asbool, cast_to_int
 
 
@@ -23,8 +23,9 @@ class FTPSource(FilesystemSource):
         )
 
         # Decode individual options (type casting, default values, sanity checks).
+        resource_options, connector_kwargs = split_run_options(kwargs)
         fs_kwargs = locator.options.fs_kwargs
-        fs_kwargs.update(kwargs)
+        fs_kwargs.update(connector_kwargs)
         fs_kwargs["port"] = fs_kwargs.get("port", locator.default_port)
         # Cast values to `int`.
         cast_to_int(fs_kwargs, ["block_size", "port", "timeout"])
@@ -41,4 +42,4 @@ class FTPSource(FilesystemSource):
 
         # Create filesystem and dlt resource wrapper.
         fs = FTPFileSystem(**fs_kwargs)
-        return infer_resource(fs=fs, locator=locator)
+        return infer_resource(fs=fs, locator=locator, options=resource_options)

{omniload-0.9.0 → omniload-0.9.2}/src/dlt_filesystem/source/fsspec/gdrive.py

@@ -4,7 +4,7 @@ from fsspec import AbstractFileSystem
 
 from dlt_filesystem.source.base import FilesystemSource
 from dlt_filesystem.source.core import infer_resource
-from dlt_filesystem.source.model import FilesystemLocator
+from dlt_filesystem.source.model import FilesystemLocator, split_run_options
 from dlt_filesystem.util.python import apply_alias, cast_to_bool, cast_to_dict
 
 
@@ -37,8 +37,9 @@ class GoogleDriveSource(FilesystemSource):
         )
 
         # Decode individual options (type casting, default values, sanity checks).
+        resource_options, connector_kwargs = split_run_options(kwargs)
         fs_kwargs = locator.options.fs_kwargs
-        fs_kwargs.update(kwargs)
+        fs_kwargs.update(connector_kwargs)
         apply_alias(fs_kwargs, "auth", "auth_kwargs")
         cast_to_dict(fs_kwargs, ["creds", "auth_kwargs"])
         if "auth_kwargs" in fs_kwargs:
@@ -46,4 +47,4 @@ class GoogleDriveSource(FilesystemSource):
 
         # Create filesystem and dlt resource wrapper.
         fs = self.fs_class(**fs_kwargs)
-        return infer_resource(fs=fs, locator=locator)
+        return infer_resource(fs=fs, locator=locator, options=resource_options)

{omniload-0.9.0 → omniload-0.9.2}/src/dlt_filesystem/source/fsspec/hdfs.py

@@ -5,7 +5,7 @@ from fsspec import AbstractFileSystem
 from dlt_filesystem.error import MissingConnectorOption
 from dlt_filesystem.source.base import FilesystemSource
 from dlt_filesystem.source.core import infer_resource
-from dlt_filesystem.source.model import FilesystemLocator
+from dlt_filesystem.source.model import FilesystemLocator, split_run_options
 from dlt_filesystem.util.python import apply_alias, cast_to_dict, cast_to_int
 
 
@@ -33,8 +33,9 @@ class HDFSSource(FilesystemSource):
         )
 
         # Decode individual options (type casting, default values, sanity checks).
+        resource_options, connector_kwargs = split_run_options(kwargs)
         fs_kwargs = locator.options.fs_kwargs
-        fs_kwargs.update(kwargs)
+        fs_kwargs.update(connector_kwargs)
         fs_kwargs["port"] = fs_kwargs.get("port", locator.default_port)
         apply_alias(fs_kwargs, "block_size", "default_block_size")
         cast_to_int(
@@ -48,4 +49,4 @@ class HDFSSource(FilesystemSource):
 
         # Create filesystem and dlt resource wrapper.
         fs = self.fs_class(**fs_kwargs)
-        return infer_resource(fs=fs, locator=locator)
+        return infer_resource(fs=fs, locator=locator, options=resource_options)

{omniload-0.9.0 → omniload-0.9.2}/src/dlt_filesystem/source/fsspec/oci.py

@@ -4,7 +4,7 @@ from fsspec import AbstractFileSystem
 
 from dlt_filesystem.source.base import FilesystemSource
 from dlt_filesystem.source.core import infer_resource
-from dlt_filesystem.source.model import FilesystemLocator
+from dlt_filesystem.source.model import FilesystemLocator, split_run_options
 from dlt_filesystem.util.python import apply_alias, cast_to_dict, cast_to_int
 
 
@@ -36,8 +36,9 @@ class OCISource(FilesystemSource):
 
         # Decode individual options (type casting, default values, sanity checks). Schema:
         # {"default_block_size": int, "config": dict, "config_kwargs": dict, "oci_additional_kwargs": dict}
+        resource_options, connector_kwargs = split_run_options(kwargs)
         fs_kwargs = locator.options.fs_kwargs
-        fs_kwargs.update(kwargs)
+        fs_kwargs.update(connector_kwargs)
 
         # Decode dict-typed `config`, `config_kwargs`, `oci_additional_kwargs` from JSON.
         cast_to_dict(fs_kwargs, ["config"], on_error="ignore")
@@ -49,4 +50,4 @@ class OCISource(FilesystemSource):
 
         # Create filesystem and dlt resource wrapper.
         fs = self.fs_class(**fs_kwargs)
-        return infer_resource(fs=fs, locator=locator)
+        return infer_resource(fs=fs, locator=locator, options=resource_options)

{omniload-0.9.0 → omniload-0.9.2}/src/dlt_filesystem/source/fsspec/oss.py

@@ -4,7 +4,7 @@ from fsspec import AbstractFileSystem
 
 from dlt_filesystem.source.base import FilesystemSource
 from dlt_filesystem.source.core import infer_resource
-from dlt_filesystem.source.model import FilesystemLocator
+from dlt_filesystem.source.model import FilesystemLocator, split_run_options
 from dlt_filesystem.util.python import apply_alias, cast_to_int
 
 
@@ -35,8 +35,9 @@ class OSSSource(FilesystemSource):
         )
 
         # Decode individual options (type casting, default values, sanity checks).
+        resource_options, connector_kwargs = split_run_options(kwargs)
         fs_kwargs = locator.options.fs_kwargs
-        fs_kwargs.update(kwargs)
+        fs_kwargs.update(connector_kwargs)
         apply_alias(fs_kwargs, "block_size", "default_block_size")
         apply_alias(fs_kwargs, "cache_type", "default_cache_type")
         cast_to_int(fs_kwargs, ["default_block_size"])
@@ -50,4 +51,4 @@ class OSSSource(FilesystemSource):
 
         # Create filesystem and dlt resource wrapper.
         fs = self.fs_class(**fs_kwargs)
-        return infer_resource(fs=fs, locator=locator)
+        return infer_resource(fs=fs, locator=locator, options=resource_options)

{omniload-0.9.0 → omniload-0.9.2}/src/dlt_filesystem/source/fsspec/sharepoint.py

@@ -4,7 +4,7 @@ from fsspec import AbstractFileSystem
 
 from dlt_filesystem.source.base import FilesystemSource
 from dlt_filesystem.source.core import infer_resource
-from dlt_filesystem.source.model import FilesystemLocator
+from dlt_filesystem.source.model import FilesystemLocator, split_run_options
 from dlt_filesystem.util.python import cast_to_bool, cast_to_dict
 
 
@@ -39,11 +39,12 @@ class SharePointSource(FilesystemSource):
         )
 
         # Decode individual options (type casting, default values, sanity checks). Schema:
+        resource_options, connector_kwargs = split_run_options(kwargs)
         fs_kwargs = locator.options.fs_kwargs
-        fs_kwargs.update(kwargs)
+        fs_kwargs.update(connector_kwargs)
         cast_to_dict(fs_kwargs, ["oauth2_client_params"])
         cast_to_bool(fs_kwargs, ["use_recycle_bin"])
 
         # Create filesystem and dlt resource wrapper.
         fs = self.fs_class(**fs_kwargs)
-        return infer_resource(fs=fs, locator=locator)
+        return infer_resource(fs=fs, locator=locator, options=resource_options)

{omniload-0.9.0 → omniload-0.9.2}/src/dlt_filesystem/source/fsspec/smb.py

@@ -5,7 +5,7 @@ from fsspec import AbstractFileSystem
 from dlt_filesystem.error import MissingConnectorOption
 from dlt_filesystem.source.base import FilesystemSource
 from dlt_filesystem.source.core import infer_resource
-from dlt_filesystem.source.model import FilesystemLocator
+from dlt_filesystem.source.model import FilesystemLocator, split_run_options
 from dlt_filesystem.util.python import cast_to_bool, cast_to_int
 
 
@@ -36,8 +36,9 @@ class SMBSource(FilesystemSource):
         )
 
         # Decode individual options (type casting, default values, sanity checks). Schema:
+        resource_options, connector_kwargs = split_run_options(kwargs)
         fs_kwargs = locator.options.fs_kwargs
-        fs_kwargs.update(kwargs)
+        fs_kwargs.update(connector_kwargs)
         cast_to_int(
             fs_kwargs,
             [
@@ -56,4 +57,4 @@ class SMBSource(FilesystemSource):
 
         # Create filesystem and dlt resource wrapper.
         fs = self.fs_class(**fs_kwargs)
-        return infer_resource(fs=fs, locator=locator)
+        return infer_resource(fs=fs, locator=locator, options=resource_options)

{omniload-0.9.0 → omniload-0.9.2}/src/dlt_filesystem/source/fsspec/webdav.py

@@ -6,7 +6,7 @@ from dlt_filesystem.error import MissingConnectorOption
 from dlt_filesystem.source.base import FilesystemSource
 from dlt_filesystem.source.core import infer_resource
 from dlt_filesystem.source.impl.util import strip_protocol_suffix
-from dlt_filesystem.source.model import FilesystemLocator
+from dlt_filesystem.source.model import FilesystemLocator, split_run_options
 from dlt_filesystem.util.python import asbool, cast_to_bool, cast_to_dict, cast_to_int
 
 
@@ -42,8 +42,9 @@ class WebdavSource(FilesystemSource):
         )
 
         # Decode individual arguments.
+        resource_options, connector_kwargs = split_run_options(kwargs)
         fs_kwargs = locator.options.fs_kwargs
-        fs_kwargs.update(kwargs)
+        fs_kwargs.update(connector_kwargs)
         cast_to_bool(fs_kwargs, ["retry", "trust_env"])
         cast_to_dict(fs_kwargs, ["headers", "cookies", "proxies"])
         cast_to_int(fs_kwargs, ["chunk_size", "max_redirects"])
@@ -68,4 +69,4 @@ class WebdavSource(FilesystemSource):
 
         # Create filesystem and dlt resource wrapper.
         fs = self.fs_class(uri, auth=auth, **fs_kwargs)
-        return infer_resource(fs=fs, locator=locator)
+        return infer_resource(fs=fs, locator=locator, options=resource_options)

{omniload-0.9.0 → omniload-0.9.2}/src/dlt_filesystem/source/fsspec/webhdfs.py

@@ -5,7 +5,7 @@ from fsspec import AbstractFileSystem
 from dlt_filesystem.error import MissingConnectorOption
 from dlt_filesystem.source.base import FilesystemSource
 from dlt_filesystem.source.core import infer_resource
-from dlt_filesystem.source.model import FilesystemLocator
+from dlt_filesystem.source.model import FilesystemLocator, split_run_options
 from dlt_filesystem.util.python import (
     apply_alias,
     cast_to_bool,
@@ -44,8 +44,9 @@ class WebHDFSSource(FilesystemSource):
         )
 
         # Decode individual options (type casting, default values, sanity checks).
+        resource_options, connector_kwargs = split_run_options(kwargs)
         fs_kwargs = locator.options.fs_kwargs
-        fs_kwargs.update(kwargs)
+        fs_kwargs.update(connector_kwargs)
         apply_alias(fs_kwargs, "kerberos_options", "kerb_kwargs")
         cast_to_int(fs_kwargs, ["port"])
         cast_to_bool(fs_kwargs, ["kerberos", "use_https", "session_verify"])
@@ -57,4 +58,4 @@ class WebHDFSSource(FilesystemSource):
 
         # Create filesystem and dlt resource wrapper.
         fs = self.fs_class(**fs_kwargs)
-        return infer_resource(fs=fs, locator=locator)
+        return infer_resource(fs=fs, locator=locator, options=resource_options)

{omniload-0.9.0 → omniload-0.9.2}/src/dlt_filesystem/source/impl/remote.py

@@ -8,6 +8,7 @@ from dlt_filesystem.error import InvalidBlobTableError, MissingConnectorOption
 from dlt_filesystem.source.base import FilesystemSource
 from dlt_filesystem.source.error import UnsupportedEndpointError
 from dlt_filesystem.source.format.registry import supported_file_format_message
+from dlt_filesystem.source.model import split_run_options, strip_run_options
 from dlt_filesystem.source.router import (
     blob_hints,
     determine_endpoint,
@@ -55,7 +56,13 @@ class GCSSource(FilesystemSource):
 
         bucket_url = f"gs://{bucket_name}"
 
-        fs = self.fs_class(**gcs_filesystem_kwargs(params, kwargs))
+        # gcsfs takes the caller's own keyword arguments as the baseline it merges the
+        # URI parameters into, and this connector merges the query string wholesale, so
+        # both carriers get filtered before they reach the constructor.
+        resource_options, connector_kwargs = split_run_options(kwargs)
+        fs = self.fs_class(
+            **gcs_filesystem_kwargs(strip_run_options(params), connector_kwargs)
+        )
 
         try:
             endpoint: str = determine_endpoint(table, path_to_file)
@@ -76,10 +83,10 @@ class GCSSource(FilesystemSource):
                 file_glob=path_to_file,
                 reader_name=endpoint,
                 storage_namespace="gcs",
-                filesystem_incremental=kwargs.get("filesystem_incremental", False),
+                filesystem_incremental=resource_options.filesystem_incremental,
                 require_file_match=source_selects_single_file(uri, table),
                 hints=blob_hints(parsed_uri, table),
-                column_types=kwargs.get("column_types"),
+                column_types=resource_options.column_types,
             )
         )
 
@@ -199,8 +206,9 @@ class AzureSource(FilesystemSource):
 
         bucket_url = f"az://{bucket_name}"
 
-        kwargs.update(azure_blob_filesystem_kwargs(auth))
-        fs = self.fs_class(**kwargs)
+        resource_options, connector_kwargs = split_run_options(kwargs)
+        connector_kwargs.update(azure_blob_filesystem_kwargs(auth))
+        fs = self.fs_class(**connector_kwargs)
 
         try:
             endpoint: str = determine_endpoint(table, path_to_file)
@@ -224,10 +232,10 @@ class AzureSource(FilesystemSource):
                     f"azure:{(auth.account_name or '').lower()}:"
                     f"{self.endpoint_namespace(auth.account_host, 'azure-public')}"
                 ),
-                filesystem_incremental=kwargs.get("filesystem_incremental", False),
+                filesystem_incremental=resource_options.filesystem_incremental,
                 require_file_match=source_selects_single_file(uri, table),
                 hints=blob_hints(parsed_uri, table),
-                column_types=kwargs.get("column_types"),
+                column_types=resource_options.column_types,
             )
         )
 

omniload-0.9.2/src/dlt_filesystem/source/lister.py

@@ -0,0 +1,178 @@
+"""File discovery for the filesystem sources.
+
+``glob_files`` is vendored from dlt (Apache-2.0,
+``dlt.common.storages.fsspec_filesystem``) so a file's modification date is
+resolved from the shape the filesystem client actually returns.
+
+dlt selects that value through a table keyed by URL scheme, where each entry
+reads one backend-specific key out of the listing (``LastModified`` for ``s3``,
+``updated`` for ``gs``, ``last_modified`` for ``az``). Both halves of that
+coupling break here:
+
+- A scheme dlt does not know raises ``KeyError`` on the scheme itself, so
+  listing crashes on the first matched file for ``r2://``, ``oss://``,
+  ``hdfs://``, ``smb://``, ``ftp://``, ``dbfs://``, ``oci://`` and
+  ``webhdfs://``.
+- A client whose listing uses a different key than its scheme is mapped to
+  raises ``KeyError`` on the key, which is what any ``pyarrow.fs``-backed
+  client does (its entries carry ``mtime`` whatever the scheme).
+
+Resolving from the listing keeps every scheme dlt does know byte-identical,
+because its own table is consulted first.
+"""
+
+import glob
+import pathlib
+import posixpath
+from datetime import datetime, timezone
+from typing import Any, Callable, Iterator, Mapping, Tuple
+from urllib.parse import urlparse
+
+from dlt.common.storages import FilesystemConfiguration
+from dlt.common.storages.configuration import make_fsspec_url
+from dlt.common.storages.fsspec_filesystem import (
+    MTIME_DISPATCH,
+    FileItem,
+    guess_mime_type,
+)
+from dlt.common.time import ensure_pendulum_datetime_utc
+from fsspec import AbstractFileSystem
+
+
+def _from_epoch_millis(value: Any) -> Any:
+    """Read a timestamp in milliseconds, as WebHDFS reports one."""
+    return ensure_pendulum_datetime_utc(float(value) / 1000)
+
+
+def _from_mlsd_timestamp(value: Any) -> Any:
+    """Read an RFC 3659 ``modify`` fact, ``YYYYMMDDHHMMSS[.sss]``, as FTP reports one.
+
+    fsspec falls back to parsing ``dir`` output on servers without MLSD, which
+    yields a year-less ``Aug 4 09:30``. That cannot name an instant, so it is
+    rejected here and surfaces as the "no usable modification date" error.
+    """
+    stamp = datetime.strptime(str(value)[:14], "%Y%m%d%H%M%S")
+    return ensure_pendulum_datetime_utc(stamp.replace(tzinfo=timezone.utc))
+
+
+# The key each backend's listing carries a file's last-modified time under,
+# paired with how that backend encodes it. Every entry is taken from the
+# backend's own `modified()` implementation, which reads the same key.
+MODIFICATION_DATE_KEYS: Tuple[Tuple[str, Callable[[Any], Any]], ...] = (
+    ("LastModified", ensure_pendulum_datetime_utc),  # s3fs, ossfs
+    ("last_modified", ensure_pendulum_datetime_utc),  # adlfs
+    ("updated", ensure_pendulum_datetime_utc),  # gcsfs
+    ("modifiedTime", ensure_pendulum_datetime_utc),  # Google Drive
+    ("modificationTime", _from_epoch_millis),  # WebHDFS
+    ("timeModified", ensure_pendulum_datetime_utc),  # ocifs
+    ("modified", ensure_pendulum_datetime_utc),  # fsspec-databricks
+    ("modify", _from_mlsd_timestamp),  # FTP
+    # `mtime` last: SMB carries both it and `time`, where `time` is the access
+    # time, so a backend that offers a more specific key is preferred first.
+    ("mtime", ensure_pendulum_datetime_utc),  # local, SMB, pyarrow.fs clients
+)
+
+
+def resolve_modification_date(scheme: str, file_info: Mapping[str, Any]) -> Any:
+    """Return a file's modification date from one entry of a filesystem listing.
+
+    Prefers dlt's own per-scheme extractor so known schemes keep their exact
+    behaviour, then reads the key the backend emits and decodes it the way that
+    backend encodes it.
+
+    Raises:
+        ValueError: when the listing carries no usable modification date, naming
+            the scheme, the keys present, and why any candidate was rejected.
+    """
+    extractor = MTIME_DISPATCH.get(scheme)
+    if extractor is not None:
+        try:
+            return extractor(file_info)
+        except KeyError:
+            # The scheme is known but this client reports a different key, e.g.
+            # any pyarrow.fs filesystem addressed as `s3://`.
+            pass
+
+    rejected = []
+    for key, decode in MODIFICATION_DATE_KEYS:
+        if key not in file_info:
+            continue
+        try:
+            return decode(file_info[key])
+        except (ValueError, TypeError, OverflowError) as ex:
+            rejected.append(f"{key}={file_info[key]!r} ({ex})")
+
+    detail = f" Rejected: {'; '.join(rejected)}." if rejected else ""
+    raise ValueError(
+        f"Filesystem listing for scheme '{scheme}' carries no usable modification "
+        f"date. Keys present: {sorted(file_info)}.{detail}"
+    )
+
+
+def glob_files(
+    fs_client: AbstractFileSystem, bucket_url: str, file_glob: str = "**"
+) -> Iterator[FileItem]:
+    """Get the files from the filesystem client.
+
+    Args:
+        fs_client (AbstractFileSystem): The filesystem client.
+        bucket_url (str): The url to the bucket.
+        file_glob (str): A glob for the filename filter.
+
+    Returns:
+        Iterable[FileItem]: The list of files.
+    """
+    is_local_fs = "file" in fs_client.protocol
+    if is_local_fs and FilesystemConfiguration.is_local_path(bucket_url):
+        bucket_url = FilesystemConfiguration.make_file_url(bucket_url)
+    bucket_url_parsed = urlparse(bucket_url)
+
+    if is_local_fs:
+        root_dir = FilesystemConfiguration.make_local_path(bucket_url)
+        # use a Python glob to get files
+        files = glob.glob(
+            str(pathlib.Path(root_dir).joinpath(file_glob)), recursive=True
+        )
+        glob_result = {file: fs_client.info(file) for file in files}
+    else:
+        # convert to fs_path
+        root_dir = fs_client._strip_protocol(bucket_url)
+        filter_url = posixpath.join(root_dir, file_glob)
+        # dlt's copy invalidates the listing cache for the `hf` protocol here.
+        # No HuggingFace scheme is registered, so that branch is left out.
+        glob_result = fs_client.glob(filter_url, detail=True)
+        if isinstance(glob_result, list):
+            raise NotImplementedError(
+                "Cannot request details when using fsspec.glob. For adlfs (Azure)"
+                " please use version 2023.9.0 or later"
+            )
+
+    for file, md in glob_result.items():
+        if md["type"] != "file":
+            continue
+        scheme = bucket_url_parsed.scheme
+
+        # relative paths are always POSIX
+        if is_local_fs:
+            # use OS pathlib for local paths
+            loc_path = pathlib.Path(file)
+            file_name = loc_path.name
+            rel_path = loc_path.relative_to(root_dir).as_posix()
+            file_url = FilesystemConfiguration.make_file_url(file)
+        else:
+            file_name = posixpath.basename(file)
+            rel_path = posixpath.relpath(file, root_dir)
+            file_url = make_fsspec_url(scheme, file, bucket_url)
+
+        mime_type, encoding = guess_mime_type(rel_path)
+        file_item = FileItem(
+            file_name=file_name,
+            relative_path=rel_path,
+            file_url=file_url,
+            mime_type=mime_type,
+            modification_date=resolve_modification_date(scheme, md),
+            size_in_bytes=int(md["size"]),
+        )
+        if encoding is not None:
+            file_item["encoding"] = encoding
+        yield file_item