splitlog 4.1.6__tar.gz → 5.0.0__tar.gz

{splitlog-4.1.6 → splitlog-5.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splitlog
-Version: 4.1.6
+Version: 5.0.0
 Summary: Utility to split aggregated logs from Apache Hadoop Yarn applications into a folder hierarchy
 License-Expression: MIT
 License-File: LICENSE
@@ -15,8 +15,9 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Topic :: System :: Distributed Computing
 Classifier: Topic :: System :: Logging
 Classifier: Topic :: Utilities
+Requires-Dist: fsspec (>=2026.2.0)
 Requires-Dist: python-dateutil (>=2.9.0,<3.0.0)
-Requires-Dist: pytz (>=2025.2)
+Requires-Dist: pytz (>=2026.1)
 Project-URL: Bug Tracker, https://github.com/splitlog/splitlog/issues
 Project-URL: Repository, https://github.com/splitlog/splitlog.git
 Description-Content-Type: text/markdown

{splitlog-4.1.6 → splitlog-5.0.0}/pyproject.toml

@@ -24,9 +24,10 @@ classifiers = [
 ]
 dependencies = [
     "python-dateutil (>=2.9.0,<3.0.0)",
-    "pytz (>=2025.2)",
+    "pytz (>=2026.1)",
+    "fsspec (>=2026.2.0)",
 ]
-version = "4.1.6"
+version = "5.0.0"
 
 
 [project.urls]
@@ -38,11 +39,11 @@ splitlog = 'splitlog.__main__:main'
 
 [dependency-groups]
 dev = [
-    "mypy (>=1.18.2,<2.0.0)",
+    "mypy (>=1.19.1,<2.0.0)",
     "black (>=26.3.1,<27.0.0)",
     "types-python-dateutil (>=2.9.0.20251108,<3.0.0.0)",
-    "pytest (>=9.0.0,<10.0.0)",
-    "pytest-cov (>=7.0.0,<8.0.0)",
+    "pytest (>=9.0.2,<10.0.0)",
+    "pytest-cov (>=7.1.0,<8.0.0)",
 ]
 
 [tool.poetry]

{splitlog-4.1.6 → splitlog-5.0.0}/splitlog/__init__.py

@@ -6,8 +6,7 @@ from pathlib import Path
 from typing import BinaryIO, Optional, Set
 
 from dateutil.parser import parse as parse_date
-
-from splitlog.outputfolder import OutputFolder
+from fsspec import AbstractFileSystem  # type: ignore
 
 logger = logging.getLogger(__name__)
 
@@ -45,7 +44,7 @@ class _Splitter(object):
     )
 
     def __init__(
-        self: "_Splitter", infile: BinaryIO, output_folder: OutputFolder
+        self: "_Splitter", infile: BinaryIO, output_folder: AbstractFileSystem
     ) -> None:
         self.container: Optional[str] = None
         self.host: Optional[str] = None
@@ -55,7 +54,7 @@ class _Splitter(object):
         self.length: Optional[int] = None
         self.dirs_created: Set[Path] = set()
         self.infile: BinaryIO = infile
-        self.output_folder: OutputFolder = output_folder
+        self.output_folder: AbstractFileSystem = output_folder
         self.offset: int = 0
         self.line: Optional[str] = None
         self.eof: bool = False
@@ -247,14 +246,14 @@ class _Splitter(object):
 
     def _create_hierarchy(self: "_Splitter") -> Path:
         assert self.host is not None, "host must be present"
-        host_dir = self.output_folder.root / self.host
+        host_dir = Path(self.host)
         if host_dir not in self.dirs_created:
-            self.output_folder.mkdir(host_dir)
+            self.output_folder.makedirs(host_dir.as_posix(), exist_ok=True)
             self.dirs_created.add(host_dir)
         assert self.container is not None, "container must be present"
         container_dir = host_dir / self.container
         if container_dir not in self.dirs_created:
-            self.output_folder.mkdir(container_dir)
+            self.output_folder.makedirs(container_dir.as_posix(), exist_ok=True)
             self.dirs_created.add(container_dir)
         return container_dir
 
@@ -263,14 +262,14 @@ class _Splitter(object):
         assert self.filename, "filename must be present"
         log_path = container_dir / self.filename
 
-        with self.output_folder.create(log_path) as outfile:
+        with self.output_folder.open(log_path.as_posix(), "wb") as outfile:
             logger.debug("Created empty log file %s", log_path)
 
     def _copy(self: "_Splitter") -> None:
         container_dir = self._create_hierarchy()
         assert self.filename, "filename must be present"
         log_path = container_dir / self.filename
-        with self.output_folder.create(log_path) as outfile:
+        with self.output_folder.open(log_path.as_posix(), "wb") as outfile:
             assert self.length is not None, "length must be present"
             logger.debug("Created log file %s, size: %d", log_path, self.length)
             remaining = self.length
@@ -321,7 +320,7 @@ class _Splitter(object):
         return parse_error
 
 
-def split(infile: BinaryIO, output_folder: OutputFolder) -> None:
+def split(infile: BinaryIO, output_folder: AbstractFileSystem) -> None:
     splitter = _Splitter(infile=infile, output_folder=output_folder)
 
     splitter.split()

{splitlog-4.1.6 → splitlog-5.0.0}/splitlog/__main__.py

@@ -126,12 +126,17 @@ def main(cli_args: Optional[List[str]] = None) -> None:
     assert args.output_folder is not None, "output_folder argument must be present"
 
     from splitlog import split, ParseError
-    from splitlog.outputfolder import new_output_folder
+    from fsspec.implementations.dirfs import DirFileSystem  # type: ignore
 
     with _open_input(args.input_file) as infile:
         try:
-            with new_output_folder(args.output_folder) as output_folder:
-                split(infile, output_folder)
+            # Atomic directory creation. Fails if it already exists or if parent is missing.
+            args.output_folder.mkdir()
+            # use fsspec DirFileSystem to handle output folder
+            output_folder = DirFileSystem(
+                path=str(args.output_folder), auto_mkdir=False
+            )
+            split(infile, output_folder)
         except FileNotFoundError as e:
             _error_exit(e)
         except FileExistsError as e:

splitlog-4.1.6/splitlog/outputfolder.py

@@ -1,262 +0,0 @@
-import abc
-import contextlib
-import logging
-import os
-import stat
-import types
-import typing as t
-from pathlib import Path
-
-_logger = logging.getLogger(__name__)
-
-
-class BinWriter(contextlib.AbstractContextManager, metaclass=abc.ABCMeta):
-    """Abstract base class for writing binary data into an output file."""
-
-    @abc.abstractmethod
-    def write(self, b: bytes) -> int:
-        """Writes bytes into a file and returns how many bytes were written successfully.
-
-        :returns: number of bytes written successfully
-        """
-        raise NotImplementedError()
-
-
-class OutputFolder(contextlib.AbstractContextManager, metaclass=abc.ABCMeta):
-    """Abstract base class for output folder IO."""
-
-    @property
-    @abc.abstractmethod
-    def root(self) -> Path:
-        """Root path of the output folder to construct paths inside the output folder.
-
-        :returns: root path
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def mkdir(self, path: Path) -> None:
-        """Creates a subdirectory."""
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def create(self, path: Path) -> BinWriter:
-        """Creates a new file and returns a writer to write to it.
-
-        :returns: writer to write binary data into the file
-        """
-        raise NotImplementedError()
-
-
-class FileWrapper(BinWriter):
-    def __init__(self, file: t.BinaryIO):
-        self._file = file
-
-    def write(self, b: bytes) -> int:
-        return self._file.write(b)
-
-    def __enter__(self) -> "FileWrapper":
-        return self
-
-    def __exit__(
-        self,
-        exc: t.Union[t.Type[BaseException], None],
-        value: t.Union[BaseException, None],
-        tb: t.Union[types.TracebackType, None],
-    ) -> t.Union[bool, None]:
-        return self._file.__exit__(exc, value, tb)
-
-
-def _is_relative_to(path: Path, parent: Path) -> bool:
-    try:
-        path.relative_to(parent)
-    except ValueError:
-        return False
-    else:
-        return True
-
-
-class DefaultLocalFilesystemOutputFolder(OutputFolder):
-    """Encapsulates filesystem IO on output folders.
-
-    This implementation is portable but unsafe to use when invoking splitlog with privileges.
-    """
-
-    FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
-    DIR_MODE = (
-        stat.S_IRUSR
-        | stat.S_IWUSR
-        | stat.S_IXUSR
-        | stat.S_IRGRP
-        | stat.S_IXGRP
-        | stat.S_IROTH
-        | stat.S_IXOTH
-    )
-
-    def __init__(self, path: Path):
-        self._path: Path = path.resolve()
-
-    def __enter__(self) -> OutputFolder:
-        self._path.mkdir(mode=self.DIR_MODE, exist_ok=False)
-        return self
-
-    def __exit__(
-        self,
-        exc: t.Union[t.Type[BaseException], None],
-        value: t.Union[BaseException, None],
-        tb: t.Union[types.TracebackType, None],
-    ) -> t.Union[bool, None]:
-        return None
-
-    def mkdir(self, path: Path) -> None:
-        real_path = self._check_paths(path)
-        os.mkdir(real_path, mode=self.DIR_MODE)
-
-    def _check_paths(self, path: Path) -> Path:
-        if path.is_absolute():
-            raise ValueError(f"Path {path} must be relative")
-        real_path = Path(os.path.normpath(self._path / path))
-        if not _is_relative_to(real_path, self._path):
-            raise ValueError(f"Path {path} outside {self._path}")
-        return real_path
-
-    def create(self, path: Path) -> BinWriter:
-        real_path = self._check_paths(path)
-        f = open(real_path, "xb")
-        try:
-            return FileWrapper(f)
-        except Exception:
-            f.close()
-            raise
-
-    @property
-    def root(self) -> Path:
-        return Path()
-
-
-class LinuxLocalFilesystemOutputFolder(OutputFolder):
-    """Encapsulates filesystem IO on output folders.
-
-    This implementation avoids TOCTTOU attacks using modern Linux APIs.
-    """
-
-    DIR_MODE = (
-        stat.S_IRUSR
-        | stat.S_IWUSR
-        | stat.S_IXUSR
-        | stat.S_IRGRP
-        | stat.S_IXGRP
-        | stat.S_IROTH
-        | stat.S_IXOTH
-    )
-    FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
-
-    @staticmethod
-    def is_supported() -> bool:
-        for flag in ("O_PATH", "O_DIRECTORY", "O_NOFOLLOW", "O_CLOEXEC"):
-            if not hasattr(os, flag):
-                _logger.debug(f"os.{flag} not supported")
-                return False
-
-        for needs_dir_fd_support in (os.open, os.mkdir):
-            if needs_dir_fd_support not in os.supports_dir_fd:
-                _logger.debug(f"{needs_dir_fd_support} does not support dir fds")
-                return False
-
-        return True
-
-    def __init__(self, path: Path):
-        if not self.is_supported():
-            raise RuntimeError(
-                "File system semantics are not supported by runtime environment"
-            )
-        self._path: Path = path.resolve()
-        self._dir_fd: t.Union[int, None] = None
-
-    def __enter__(
-        self,
-    ) -> OutputFolder:
-        # split path
-        parent = self._path.parent
-        name = self._path.name
-
-        # allow parent folder to be a symlink
-        parent_dir_fd = self._open_dir_fd(parent, no_follow=False)
-
-        try:
-            os.mkdir(name, mode=self.DIR_MODE, dir_fd=parent_dir_fd)
-            self._dir_fd = self._open_dir_fd(name, no_follow=True, dir_fd=parent_dir_fd)
-            return self
-        finally:
-            os.close(parent_dir_fd)
-
-    def __exit__(
-        self,
-        exc: t.Union[t.Type[BaseException], None],
-        value: t.Union[BaseException, None],
-        tb: t.Union[types.TracebackType, None],
-    ) -> t.Union[bool, None]:
-        if self._dir_fd is not None:
-            saved, self._dir_fd = self._dir_fd, None
-            os.close(saved)
-        return None
-
-    @property
-    def root(self) -> Path:
-        return Path()
-
-    def mkdir(self, path: Path) -> None:
-        real_path = self._ensure_path_under_root(path)
-        os.mkdir(real_path, mode=self.DIR_MODE, dir_fd=self._dir_fd)
-
-    def _opener(self, path: str, flags: int) -> int:
-        return os.open(
-            path,
-            flags | getattr(os, "O_NOFOLLOW") | getattr(os, "O_CLOEXEC"),
-            mode=self.FILE_MODE,
-            dir_fd=self._dir_fd,
-        )
-
-    def create(self, path: Path) -> BinWriter:
-        real_path = self._ensure_path_under_root(path)
-        f = open(real_path, "xb", opener=self._opener)
-        try:
-            return FileWrapper(f)
-        except Exception:
-            f.close()
-            raise
-
-    def _ensure_path_under_root(self, path: Path) -> Path:
-        if path.is_absolute():
-            raise ValueError(f"Path {path} must be relative")
-
-        # remove all ".." and "." components
-        real_path = Path(os.path.normpath(self._path / path))
-
-        # ensure resulting absolute path is still inside self._path
-        if not _is_relative_to(real_path, self._path):
-            raise ValueError(f"Path {path} outside {self._path}")
-        return real_path.relative_to(self._path)
-
-    @staticmethod
-    def _open_dir_fd(
-        path: t.Union[Path, str], no_follow: bool, dir_fd: t.Optional[int] = None
-    ) -> int:
-        flags = (
-            getattr(os, "O_PATH")
-            | getattr(os, "O_DIRECTORY")
-            | getattr(os, "O_CLOEXEC")
-        )
-
-        if no_follow:
-            flags |= getattr(os, "O_NOFOLLOW")
-
-        return os.open(path, flags, dir_fd=dir_fd)
-
-
-def new_output_folder(path: Path) -> OutputFolder:
-    """Chooses an output folder implementation and creates an instance."""
-    if LinuxLocalFilesystemOutputFolder.is_supported():
-        return LinuxLocalFilesystemOutputFolder(path=path)
-    else:
-        return DefaultLocalFilesystemOutputFolder(path=path)