fpathlib 0.1.3.dev0__py3-none-any.whl → 0.1.4.dev16__py3-none-any.whl

fpathlib/__init__.py

@@ -2,6 +2,7 @@ from fpathlib.path import Path
 from fpathlib.fpath import FPath, ExpandedFPath
 from fpathlib.expand import (
     expand_fpath,
+    iexpand_fpath,
     expand_fpath_decorator,
     is_expandable,
 )
@@ -11,6 +12,7 @@ __all__ = [
     "FPath",
     "ExpandedFPath",
     "expand_fpath",
+    "iexpand_fpath",
     "expand_fpath_decorator",
     "is_expandable",
 ]

fpathlib/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.3.dev0'
-__version_tuple__ = version_tuple = (0, 1, 3, 'dev0')
+__version__ = version = '0.1.4.dev16'
+__version_tuple__ = version_tuple = (0, 1, 4, 'dev16')
 
 __commit_id__ = commit_id = None

fpathlib/expand.py

@@ -31,6 +31,38 @@ def expand_fpath(fpath, *, exclude_path_patterns=None, require_metadata=True):
     )
 
 
+def iexpand_fpath(fpath, *, exclude_path_patterns=None, require_metadata=True, errors="raise"):
+    """
+    Generator equivalent of :func:`.expand_fpath` -- lazily yields each
+    matching :obj:`.Path` instead of building the whole
+    :obj:`.ExpandedFPath` up front. This is a convenience function that
+    simply creates an :obj:`FPath` and calls its :meth:`.FPath.iexpand`
+    method.
+
+    Parameters
+    ----------
+    fpath : :obj:`str`
+        An f-string path, where the variables are captured and stored along the path
+        name.
+    exclude_path_patterns : :obj:`str` or :obj:`Iterable`[:obj:`str`]
+        Exclude paths that match the supplied pattern. (Default: None).
+    require_metadata : :obj:`bool`
+        Require that all paths identified must have found metadata. (Default: True).
+    errors : :obj:`str`
+        How to handle the "no matches" case. See :meth:`.FPath.iexpand`. (Default: "raise").
+
+    Yields
+    ------
+    :obj:`.Path`
+    """
+
+    return FPath(fpath).iexpand(
+        exclude_path_patterns=exclude_path_patterns,
+        require_metadata=require_metadata,
+        errors=errors,
+    )
+
+
 def expand_fpath_decorator(f=None, require_expandable=True, post_process=None):
     """
     Decorator for :func:`.expand_fpath`.
@@ -88,7 +120,8 @@ def expand_fpath_decorator(f=None, require_expandable=True, post_process=None):
 
 def is_expandable(fpath):
     """
-    Check if expandable.
+    Check whether `fpath` has fpathlib's own named f-string captures (e.g.
+    `{name}`).
 
     Parameters
     ----------

fpathlib/ext/polars.py

@@ -169,6 +169,7 @@ def scan_txt(
     has_header=False,
     keep_line=False,
     usecols=None,
+    validate_schema=True,
     *args,
     **kwargs,
 ):
@@ -198,6 +199,17 @@ def scan_txt(
         Whether to keep the original line as a column in the output.
     usecols : :obj:`list`[:obj:`int`], optional
         Indexes of columns to keep in the output. If not provided, all columns are kept. Only applicable if `separator` is provided.
+    validate_schema : :obj:`bool`
+        When the field count is inferred from a single representative file
+        (the fast path for a multi-file glob/pattern), also check that no
+        *other* matched file has *more* fields than that sample. A file with
+        fewer fields than the sample already raises a clear polars error
+        (out-of-bounds list access); a file with more fields would otherwise
+        have its extra columns silently dropped instead of erroring. This
+        check reads every matched file's line count, which costs an extra
+        full pass over the data on top of the fast path -- pass False if
+        you've already confirmed your files are consistent and want to
+        skip it for speed on very large globs. (Default: True)
     *args
         Positional arguments to pass to :meth:`polars.scan_csv`.
     **kwargs
@@ -282,6 +294,26 @@ def scan_txt(
             # Initial field names, may be renamed later from header or by `new_columns`
             fields = {i: f"field_{i}" for i in range(n_fields)}
 
+            # n_fields came from a single sample file (see above), so a file
+            # with *fewer* fields than the sample will already raise a clear
+            # polars error below (list.get() on an out-of-bounds index). A
+            # file with *more* fields would not -- its extra columns would
+            # just be silently dropped -- so check for that explicitly if
+            # requested.
+            if validate_schema and sample_schema is not None:
+                max_fields = (
+                    lf.select(_polars.col("fields").list.len().max())
+                    .collect()
+                    .item()
+                )
+                if max_fields is not None and max_fields > n_fields:
+                    msg = (
+                        f"field count mismatch: schema was inferred from a "
+                        f"single sample file with {n_fields} fields, but at "
+                        f"least one matched file has {max_fields} fields"
+                    )
+                    raise ValueError(msg)
+
         # Add each field as a separate column
         for i, field in fields.items():
             lf = lf.with_columns(_polars.col("fields").list.get(i).alias(field))

fpathlib/fpath.py

@@ -1,5 +1,5 @@
 from collections.abc import Sequence
-from glob import glob
+from glob import glob, iglob
 import parse
 import re
 
@@ -31,10 +31,19 @@ class FPath:
     def __repr__(self):
         return "FPath({!r})".format(self.fpath)
 
-    def expand(self, exclude_path_patterns=None, require_metadata=True, errors="raise"):
+    def iexpand(self, exclude_path_patterns=None, require_metadata=True, errors="raise"):
         """
-        Use an f-string to extract out a collection of paths, where the f-string
-        variables are captured and stored along the path name.
+        Lazily yield each :obj:`.Path` matching the f-string pattern, one at
+        a time, instead of building the whole :obj:`.ExpandedFPath` up
+        front. Useful when a pattern could match a very large number of
+        files and you don't want them all held in memory at once, or want
+        to start processing before the full glob finishes walking the
+        filesystem. :meth:`.expand` is built on top of this generator.
+
+        Like any generator function, nothing in the body runs -- including
+        argument validation and the wildcard/{} capture check -- until the
+        result is first iterated (a `for` loop, `list(...)`, etc.), not at
+        the moment :meth:`.iexpand` is called.
 
         Parameters
         ----------
@@ -43,13 +52,13 @@ class FPath:
         require_metadata : :obj:`bool`
             Require that all paths identified must have found metadata. (Default: True).
         errors : :obj:`str`
-            How to handle errors. If "raise", then raise an error. If "warn", then warn
-            and return an empty collection. If "ignore", then ignore the error and
-            return an empty collection. (Default: "raise").
+            How to handle the "no matches" case, checked once the pattern is
+            fully exhausted. If "raise", then raise an error. If "warn",
+            then warn. If "ignore", then do nothing. (Default: "raise").
 
-        Returns
-        -------
-        :obj:`.ExpandedFPath`
+        Yields
+        ------
+        :obj:`.Path`
         """
 
         if errors not in {"raise", "warn", "ignore"}:
@@ -58,8 +67,34 @@ class FPath:
 
         parser = parse.compile(self.fpath)
 
-        paths = []
-        for fname in glob(re.sub(r"\{.*?\}", "*", self.fpath)):
+        # glob() (used below to find files) and `parse` (used above to
+        # extract {name} values) disagree about what a bare '*', '?', or
+        # '[...]' means outside a {} capture: glob treats it as a wildcard,
+        # but parse's format-string language only special-cases {...} and
+        # reads everything else -- including '*' -- as literal text to
+        # match. A pattern like "{a}/*" would find files fine via glob()
+        # but then fail to parse against that same string, since real
+        # filenames don't literally contain "/*" -- silently producing
+        # metadata=None for every match, or raising a confusing "metadata
+        # not found" error, with no indication that '*' was the actual
+        # problem. Fail fast and explain it instead. Only applies when
+        # there's a {} capture to begin with -- a plain glob with none is a
+        # separate, unaffected case (no metadata is expected from it).
+        if parser.named_fields:
+            literal_fpath = re.sub(r"\{.*?\}", "", self.fpath)
+            for c in "*?[":
+                if c in literal_fpath:
+                    msg = (
+                        f"glob wildcard {c!r} outside a {{}} capture is not "
+                        f"supported in {self.fpath!r} -- 'parse' treats it "
+                        "as a literal character, not a wildcard. Use a "
+                        "named capture (e.g. '{a}/{b}') if you want that "
+                        "segment captured too."
+                    )
+                    raise ValueError(msg)
+
+        n = 0
+        for fname in iglob(re.sub(r"\{.*?\}", "*", self.fpath)):
             path = Path(fname)
             if exclude_path_patterns and path.match_any(exclude_path_patterns):
                 continue
@@ -67,9 +102,10 @@ class FPath:
             if require_metadata and path.metadata is None:
                 msg = f"metadata not found for '{fname}' with '{self.fpath}'"
                 raise AttributeError(msg)
-            paths.append(path)
+            n += 1
+            yield path
 
-        if len(paths) == 0 and errors != "ignore":
+        if n == 0 and errors != "ignore":
             msg = f"no paths found for {self.fpath.__repr__()}"
             if errors == "raise":
                 raise IOError(msg)
@@ -78,6 +114,35 @@ class FPath:
 
                 warnings.warn(msg)
 
+    def expand(self, exclude_path_patterns=None, require_metadata=True, errors="raise"):
+        """
+        Use an f-string to extract out a collection of paths, where the f-string
+        variables are captured and stored along the path name.
+
+        Parameters
+        ----------
+        exclude_path_patterns : :obj:`str` or :obj:`Iterable`[:obj:`str`]
+            Exclude paths that match the supplied pattern. (Default: None).
+        require_metadata : :obj:`bool`
+            Require that all paths identified must have found metadata. (Default: True).
+        errors : :obj:`str`
+            How to handle errors. If "raise", then raise an error. If "warn", then warn
+            and return an empty collection. If "ignore", then ignore the error and
+            return an empty collection. (Default: "raise").
+
+        Returns
+        -------
+        :obj:`.ExpandedFPath`
+        """
+
+        paths = list(
+            self.iexpand(
+                exclude_path_patterns=exclude_path_patterns,
+                require_metadata=require_metadata,
+                errors=errors,
+            )
+        )
+
         return ExpandedFPath(paths=paths, fpath=self)
 
 
@@ -153,7 +218,13 @@ class ExpandedFPath(Sequence):
 
         pl = import_optional_dependency("polars")
 
-        data = [{"fname": str(key), **value} for key, value in self.metadata.items()]
+        # `value` is None whenever this ExpandedFPath was built with
+        # require_metadata=False and a path's metadata simply wasn't found
+        # (e.g. a plain glob with no named captures at all) -- treat that as
+        # "no metadata columns" rather than crashing on `**None`.
+        data = [
+            {"fname": str(key), **(value or {})} for key, value in self.metadata.items()
+        ]
         df = pl.DataFrame(data)
         if lazy:
             df = df.lazy()

{fpathlib-0.1.3.dev0.dist-info → fpathlib-0.1.4.dev16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fpathlib
-Version: 0.1.3.dev0
+Version: 0.1.4.dev16
 Summary: A package to combine paths with metadata
 Author-email: "C. Lockhart" <clockha2@gmu.edu>
 Requires-Python: >=3.12

fpathlib-0.1.4.dev16.dist-info/RECORD

@@ -0,0 +1,13 @@
+fpathlib/__init__.py,sha256=6HdQu7fISBIc6osLZp7CMJ65DW2aGNsvFCLKbUXgE10,348
+fpathlib/_version.py,sha256=28sM26orIv-swz6NJDAQttdFakxmunN6hkOeiiQ6fYM,535
+fpathlib/expand.py,sha256=31dMQxizf7SdZzDSv7UOfLkfwqsbzl_5JP1QbEeDR_0,4587
+fpathlib/fpath.py,sha256=H7MnxyqE5N2wGu13ReBUjNg9WbSuV6PqEgtzV4uL3GU,8406
+fpathlib/path.py,sha256=ME2jfMBCK1Emu35mu8wN-KvmPE4p8qQqX8q-uO5cDsI,2629
+fpathlib/utils.py,sha256=os9ggFllqOZXYcjfKcxPIy92LuRfvt6jWL0gNfet1N0,545
+fpathlib/ext/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fpathlib/ext/polars.py,sha256=lamkEI1CKW0g1n6jvj0aJug-ypyNqa_H3vdTb6YfmIo,12819
+fpathlib-0.1.4.dev16.dist-info/licenses/LICENSE,sha256=76rZ9i4ZumZ1M4jlYM9ZDOTYEeyzuxdsTKjF2qha3Lc,1069
+fpathlib-0.1.4.dev16.dist-info/METADATA,sha256=lEBsRE0z0zJqVyXkgx5kM4257JIJAh91UCc8UZNH-qc,2142
+fpathlib-0.1.4.dev16.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+fpathlib-0.1.4.dev16.dist-info/top_level.txt,sha256=QT5pDgO4--AJn7X4RB51xpNTt0pD5_U2mCjsydpQAQw,9
+fpathlib-0.1.4.dev16.dist-info/RECORD,,

fpathlib-0.1.3.dev0.dist-info/RECORD

@@ -1,18 +0,0 @@
-fpathlib/__init__.py,sha256=VniZjNqT5pxzj6ymcpAlhNoWC44MwkQLu9jssc7o13Q,308
-fpathlib/_version.py,sha256=dxsNQZw4RtnKvRTjSi1emxt4lZsN7c7hbUngeAYGVZY,533
-fpathlib/expand.py,sha256=M_K1nfRAD9c983d68kCRuzo7-zNaOY_3R9gn8aBritI,3393
-fpathlib/fpath.py,sha256=okEgqpWQjearSyDTp6RdIWbFolijI3BrXR8F5t-9Was,4877
-fpathlib/path.py,sha256=ME2jfMBCK1Emu35mu8wN-KvmPE4p8qQqX8q-uO5cDsI,2629
-fpathlib/utils.py,sha256=os9ggFllqOZXYcjfKcxPIy92LuRfvt6jWL0gNfet1N0,545
-fpathlib/__pycache__/__init__.cpython-313.pyc,sha256=clLXRuVcvX9dJt4CTvaUBR6vhAzg67Ex7gKRwtumJFY,428
-fpathlib/__pycache__/path.cpython-313.pyc,sha256=mQVWw1qjmvUp3NflXzhC_Q654H4hFW29gLYHP9nxRPQ,12257
-fpathlib/__pycache__/utils.cpython-313.pyc,sha256=V4KSHMjm3ePQfaOKWwAkmaocvYb4VOul6Zooh0EvLa0,772
-fpathlib/ext/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fpathlib/ext/polars.py,sha256=r3-MoGfbB11cOrECGCEyFzdKXGGEI9DSV6Xfc8vHeJo,10994
-fpathlib/ext/__pycache__/__init__.cpython-313.pyc,sha256=lrgZbujuH_tlWGfe_Huo46srJdYJd1NQ6ZQFWamc6tc,153
-fpathlib/ext/__pycache__/polars.cpython-313.pyc,sha256=S0epPPABmHRj0IrCxyOC8hD2_g0hBKggFFWfsmXw4m0,7740
-fpathlib-0.1.3.dev0.dist-info/licenses/LICENSE,sha256=76rZ9i4ZumZ1M4jlYM9ZDOTYEeyzuxdsTKjF2qha3Lc,1069
-fpathlib-0.1.3.dev0.dist-info/METADATA,sha256=EqyLMoPqqdMHwjd0To4_dc-o-wW2B2t6HmHel926ueM,2141
-fpathlib-0.1.3.dev0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-fpathlib-0.1.3.dev0.dist-info/top_level.txt,sha256=QT5pDgO4--AJn7X4RB51xpNTt0pD5_U2mCjsydpQAQw,9
-fpathlib-0.1.3.dev0.dist-info/RECORD,,