pathlib-next 0.1.0__py3-none-any.whl

pathlib_next/uri/schemes/http.py

@@ -0,0 +1,136 @@
+from ...utils.stat import FileStat
+from .. import UriPath
+from ... import utils as _utils
+import io as _io
+import requests as _req
+import typing as _ty
+import bs4 as _bs4
+import time as _time
+
+from htmllistparse import parse as _htmlparse
+
+
+class _FileEntry(_ty.NamedTuple):
+    name: str
+    modified: _ty.Optional[_time.struct_time]
+    size: _ty.Optional[int]
+    description: _ty.Optional[str]
+
+
+class HttpBackend(_ty.NamedTuple):
+    session: _req.Session
+    requests_args: dict
+
+    def request(self, method, uri: "HttpPath|str", **kwargs):
+        return self.session.request(
+            **self.requests_args,
+            **kwargs,
+            method=method,
+            url=uri if isinstance(uri, str) else uri.as_uri(False),
+        )
+
+
+class HttpPath(UriPath):
+    __SCHEMES = ("http", "https")
+    __slots__ = ("_isdir", "_session", "_requests_args")
+    _isdir: bool
+
+    if _ty.TYPE_CHECKING:
+        backend: HttpBackend
+
+    def _initbackend(self):
+        return HttpBackend(_req.Session(), {})
+
+    def _listdir(self) -> list[_FileEntry]:
+        req = self.backend.session.request("GET", self)
+        req.raise_for_status()
+        soup = _bs4.BeautifulSoup(req.content, "html5lib")
+        _, listing = _htmlparse(soup)
+        return listing
+
+    def iterdir(self):
+        _self = self.path.removesuffix("/")
+        cls = type(self)
+        for path in self._listdir():
+            inst = type(self).__new__(cls, backend=self.backend)
+            inst._init(self.source, f"{_self}/{path.name}", "", "")
+            yield inst
+
+    def _is_dir(self, resp: _req.Response):
+        return (
+            resp.is_redirect
+            or resp.url.endswith("/")
+            or resp.url.endswith("/..")
+            or resp.url.endswith("/.")
+        )
+
+    def stat(self):
+        check = (
+            [self.with_path(self.path.removesuffix("/")), self]
+            if self.path.endswith("/")
+            else [self]
+        )
+        for uri in check:
+            resp = self.backend.request("HEAD", uri, allow_redirects=False)
+            resp.close()
+            if resp.status_code < 400:
+                break
+
+        if self._isdir is None:
+            self._isdir = self._is_dir(resp)
+
+        if resp.is_redirect:
+            resp = self.backend.request("HEAD", uri)
+        if resp.status_code == 404:
+            raise FileNotFoundError(self)
+        elif resp.status_code == 403:
+            raise PermissionError(self)
+        else:
+            resp.raise_for_status()
+
+        st_size = 0 if self._isdir else int(resp.headers.get("Content-Length", 0))
+        lm = resp.headers.get("Last-Modified")
+        if lm is None:
+            parent = self.parent
+            if self != parent:
+                try:
+                    entry = next(
+                        filter(
+                            lambda p: p.name.removesuffix("/") == self.name,
+                            parent._listdir(),
+                        )
+                    )
+                    if entry and entry.modified:
+                        lm = entry.modified
+                except:
+                    pass
+
+        return FileStat(
+            st_size=st_size, st_mtime=_utils.parsedate(lm), is_dir=self._isdir
+        )
+
+    def _open(
+        self,
+        mode="r",
+        buffering=-1,
+    ):
+        if mode != "r":
+            raise NotImplementedError(mode)
+        buffer_size = _io.DEFAULT_BUFFER_SIZE if buffering < 0 else buffering
+        req = self.backend.request("GET", self.as_uri(), stream=True)
+        return (
+            req.raw
+            if buffer_size == 0
+            else _io.BufferedReader(req.raw, buffer_size=buffer_size)
+        )
+
+    def is_dir(self):
+        if self._isdir is None:
+            self.stat()
+        return self._is_dir is not None and self._isdir
+
+    def is_file(self):
+        return self.is_dir is not None and not self.is_dir()
+
+    def with_session(self, session: _req.Session, **requests_args):
+        return type(self)(self, backend=HttpBackend(session, requests_args))

pathlib_next/uri/schemes/sftp.py

@@ -0,0 +1,101 @@
+import typing as _ty
+from .. import UriPath, Source
+from ... import utils as _utils
+import threading as _thread
+import paramiko as _paramiko
+
+
+class BaseSftpBackend(object):
+    __slots__ = ()
+
+    @_utils.notimplemented
+    def client(self, source: Source) -> _paramiko.SFTPClient: ...
+
+
+class SftpBackend(BaseSftpBackend):
+    __slots__ = ("connect_opts", "hostkeypolicy")
+    connect_opts: dict[str, str]
+    hostkeypolicy: _paramiko.MissingHostKeyPolicy
+
+    def __init__(self, connect_opts, hostkeypolicy) -> None:
+        self.connect_opts = connect_opts
+        self.hostkeypolicy = hostkeypolicy
+
+    def opts(self, source: Source):
+        connect_ops = {
+            **self.connect_opts,
+            "hostname": str(source.host),
+            "port": source.port or 22,
+        }
+        user, password = source.parsed_userinfo()
+        if user:
+            connect_ops["username"] = user
+        if password:
+            connect_ops["password"] = password
+        return connect_ops
+
+    def transport(self, source: Source) -> _paramiko.Transport:
+        client = _paramiko.SSHClient()
+        client.set_missing_host_key_policy(self.hostkeypolicy)
+        client.connect(**self.opts(source))
+        transport = client.get_transport()
+        if not transport:
+            raise Exception()
+        return transport
+
+    def client(self, source: Source):
+        return self.transport(source).open_sftp_client()
+
+
+def _create_sftpclient(backend: BaseSftpBackend, source: Source, thread_id: int):
+    return backend.client(source)
+
+
+_CACHED_CLIENTS = _utils.LRU(_create_sftpclient, maxsize=128)
+
+
+class SftpPath(UriPath):
+
+    __SCHEMES = ("sftp",)
+    __slots__ = ()
+
+    if _ty.TYPE_CHECKING:
+        backend: BaseSftpBackend
+
+    def _initbackend(self):
+        return SftpBackend({}, _paramiko.MissingHostKeyPolicy)
+
+    @property
+    def _sftpclient(self):
+        thead_id = _thread.get_ident()
+        client = _CACHED_CLIENTS(self.backend, self.source, thead_id)
+        if client is None or not client.sock.active:
+            client = _CACHED_CLIENTS.invalidate(self.backend, self.source, thead_id)
+        return client
+
+    def _listdir(self):
+        for path in self._sftpclient.listdir(self.path):
+            yield path
+
+    def stat(self):
+        return self._sftpclient.stat(self.path)
+
+    def _open(self, mode="r", buffering=-1):
+        return self._sftpclient.open(self.path, mode, buffering)
+
+    def _mkdir(self, mode):
+        return self._sftpclient.mkdir(self.path, mode)
+
+    def chmod(self, mode):
+        return self._sftpclient.chmod(self.path, mode)
+
+    def unlink(self, missing_ok=False):
+        if missing_ok and not self.exists():
+            return
+        return self._sftpclient.remove(self.path)
+
+    def rmdir(self):
+        return self._sftpclient.rmdir(self.path)
+
+    def _rename(self, target):
+        return self._sftpclient.rename(self.path, target.as_posix())

pathlib_next/uri/source.py

@@ -0,0 +1,73 @@
+import typing as _ty
+import uritools as _uritools
+import socket as _socket
+import ipaddress as _ip
+
+from .. import utils as _utils
+
+_IPAddress = _ip.IPv4Address | _ip.IPv6Address
+
+if _ty.TYPE_CHECKING:
+    from . import UriPath
+
+
+class Source(_ty.NamedTuple):
+    scheme: str | None
+    userinfo: str | None
+    host: str | _IPAddress | None
+    port: int | None
+
+    def __bool__(self):
+        for val in self:
+            if val == '' or val is None:
+                continue
+            return True
+        return False
+
+    def __str__(self) -> str:
+        return _uritools.uricompose(
+            scheme=self.scheme, userinfo=self.userinfo, host=self.host, port=self.port
+        )
+
+    @classmethod
+    def from_str(cls, source: str, strict=True):
+        uri = _uritools.urisplit(source)
+        if strict and (uri.path or uri.fragment or uri.query):
+            raise ValueError(source)
+        return cls(uri.getscheme(), uri.getuserinfo(), uri.gethost(), uri.getport())
+
+    def keys(self):
+        return self._asdict().keys()
+
+    def __getitem__(self, key: int | str):
+        if not isinstance(key, str):
+            key = self._fields[key]
+        return getattr(self, key)
+
+    def parsed_userinfo(self):
+        parts = []
+        if self.userinfo:
+            parts = self.userinfo.split(":", maxsplit=1)
+        parts = parts + ["", ""]
+        return parts[0], parts[1]
+
+    def get_scheme_cls(self, schemesmap: _ty.Mapping[str, type["UriPath"]] = None):
+        from . import UriPath
+
+        if self.scheme:
+            if schemesmap is None:
+                schemesmap = UriPath._schemesmap()
+            _cls = schemesmap.get(self.scheme, None)
+            return _cls if _cls else UriPath
+        return UriPath
+
+    def is_local(self):
+        host = self.host
+        if not host or host == "localhost":
+            return True
+        if isinstance(host, str):
+            host = _ip.ip_address(_socket.gethostbyname(host))
+        return host.is_loopback or host in _utils.get_machine_ips()
+
+
+_NOSOURCE = Source(None, None, None, None)

pathlib_next/utils/__init__.py

@@ -0,0 +1,93 @@
+import functools as _functools
+from threading import RLock
+import typing as _ty
+import time as _time
+from email.utils import parsedate as _parsedate
+
+
+import collections
+
+K = _ty.ParamSpec("K")
+V = _ty.TypeVar("V")
+
+
+class LRU(_ty.Generic[K, V]):
+
+    def __init__(self, func: _ty.Callable[K, V], maxsize=128):
+        self.cache = collections.OrderedDict()
+        self.func = func
+        self._maxsize = maxsize
+        self.lock = RLock()
+
+    @property
+    def maxsize(self):
+        return self._maxsize
+    
+    @maxsize.setter
+    def maxsize(self, maxsize: int):
+        cache = self.cache
+        with self.lock:
+            self._maxsize = maxsize
+            while len(cache) > maxsize:
+                cache.pop(last=False)
+
+    def __call__(self, *args: K.args) -> V:
+        cache = self.cache
+        with self.lock:
+            if args in cache:
+                cache.move_to_end(args)
+                return cache[args]
+        result = self.func(*args)
+        with self.lock:
+            cache[args] = result
+            if len(cache) > self._maxsize:
+                cache.popitem(last=False)
+        return result
+
+    def invalidate(self, *args: K.args) -> V:
+        with self.lock():
+            if args in self.cache:
+                self.cache.pop(args, None)
+
+        return self(*args)
+
+
+def parsedate(date: _ty.Union[str, _time.struct_time, tuple, float]):
+    if date is None:
+        return _time.time()
+    if isinstance(date, str):
+        date = _parsedate(date)
+    return _time.mktime(date)
+
+
+def sizeof_fmt(num: _ty.Union[int, float]):
+    for unit in ("", "K", "M", "G", "T", "P", "E", "Z"):
+        if abs(num) < 1024:
+            if unit:
+                return "%3.1f%s" % (num, unit)
+            else:
+                return int(num)
+        num /= 1024.0
+    return "%.1f%s" % (num, "Y")
+
+
+def notimplemented(method):
+    @_functools.wraps(method)
+    def _notimplemented(*args, **kwargs):
+        raise NotImplementedError(f"Not implement method  {method.__name__}")
+
+    return _notimplemented
+
+import socket as _socket
+import ipaddress as _ip
+
+def get_machine_ips():
+    ips:list[_ip.IPv4Address|_ip.IPv6Address] = list()
+    for item in _socket.getaddrinfo(_socket.gethostname(), None):
+        protocol, *_, (ip, *_) = item
+        if protocol == _socket.AddressFamily.AF_INET:
+            ips.append(_ip.ip_address(ip))
+        elif protocol == _socket.AddressFamily.AF_INET6:
+            ips.append(_ip.ip_address(ip))
+
+    return ips

pathlib_next/utils/glob.py

@@ -0,0 +1,145 @@
+# Based on glob built-in on python modified to work with Uri/anything that implemetns fspath/iterdir that is similar to pathlib.Path
+
+
+"""Filename globbing utility."""
+
+import fnmatch as _fnmatch
+import typing as _ty
+import functools as _func
+import re as _re
+
+
+RECURSIVE = "**"
+ANY_PATTERN = _re.compile(_fnmatch.translate("*"))
+WILCARD_PATTERN = _re.compile("([*?[])")
+
+if _ty.TYPE_CHECKING:
+    from ..path import P as _Globable
+else:
+
+    class _Globable(_ty.Protocol): ...
+
+
+@_func.lru_cache(maxsize=256, typed=True)
+def compile_pattern(pat: str, case_sensitive: bool):
+    flags = _re.NOFLAG if case_sensitive else _re.IGNORECASE
+    return _re.compile(_fnmatch.translate(pat), flags)
+
+
+def glob(
+    path: _Globable,
+    *,
+    dironly: bool = False,
+    root_dir: _Globable | None = None,
+    recursive: bool = False,
+    include_hidden: bool = False,
+    case_sensitive: bool | None = None
+) -> _ty.Iterable[_Globable]:
+    """Return an iterator which yields the paths matching a pathname pattern.
+
+    The pattern may contain simple shell-style wildcards a la
+    fnmatch. However, unlike fnmatch, filenames starting with a
+    dot are special cases that are not matched by '*' and '?'
+    patterns.
+
+    If recursive is true, the pattern '**' will match any files and
+    zero or more directories and subdirectories.
+    """
+    if case_sensitive is None:
+        case_sensitive = path._is_case_sensitive
+
+    include_hidden = include_hidden or path.is_hidden()
+    pattern = compile_pattern(path.name, case_sensitive) if path.name else ANY_PATTERN
+
+    name_is_pattern = WILCARD_PATTERN.match(path.name) != None
+    wildcard_in_path = name_is_pattern
+    parent = next(iter(path.parents), None)
+    for segment in path.segments:
+        if WILCARD_PATTERN.match(segment) != None:
+            wildcard_in_path = True
+            break
+
+    root: _Globable = (
+        (root_dir or parent) if not root_dir or not parent else (root_dir / parent)
+    )
+
+    if recursive and path.name == RECURSIVE:
+        globber = _glob_recursive
+    else:
+        globber = _glob_with_pattern
+
+    if not parent or not wildcard_in_path:
+        yield from globber(
+            root or path,
+            pattern,
+            dironly,
+            include_hidden=include_hidden,
+        )
+        return
+
+    if parent and name_is_pattern:
+        dirs = glob(
+            parent,
+            root_dir=root_dir,
+            recursive=recursive,
+            dironly=True,
+            include_hidden=include_hidden,
+            case_sensitive=case_sensitive,
+        )
+    else:
+        dirs = [parent]
+
+    for parent in dirs:
+        yield from globber(parent, pattern, dironly, include_hidden)
+
+
+def _glob_with_pattern(
+    parent: _Globable, pattern: _re.Pattern, dironly: bool, include_hidden=False
+) -> _ty.Iterable[_Globable]:
+    if not include_hidden:
+
+        def _filter(p: _Globable):
+            return not p.is_hidden()
+
+    else:
+
+        def _filter(p: _Globable):
+            return True
+
+    for path in _iterdir(parent, dironly):
+        if _filter(path) and pattern.match(path.name):
+            yield path
+
+
+# This helper function recursively yields relative pathnames inside a literal
+# directory.
+
+
+def _glob_recursive(
+    parent: _Globable, pattern: _re.Pattern, dironly: bool, include_hidden=False
+):
+    if parent and parent.is_dir():
+        yield parent
+    yield from _rlistdir(parent, dironly, include_hidden=include_hidden)
+
+
+# If dironly is false, yields all file names inside a directory.
+# If dironly is true, yields only directory names.
+def _iterdir(path: _Globable, dironly: bool):
+    for entry in path.iterdir():
+        try:
+            if not dironly or entry.is_dir():
+                yield entry
+        except OSError:
+            pass
+
+
+# Recursively yields relative pathnames inside a literal directory.
+def _rlistdir(
+    dirname: _Globable, dironly: bool, include_hidden=False
+) -> _ty.Iterable[_Globable]:
+    for path in _iterdir(dirname, dironly):
+        if include_hidden or not path.is_hidden():
+            yield path
+            for y in _rlistdir(path, dironly, include_hidden):
+                yield y

pathlib_next/utils/stat.py

@@ -0,0 +1,77 @@
+import stat as _stat
+import typing as _ty
+import abc as _abc
+from .. import utils as _utils
+
+class FileStatLike(_ty.Protocol):
+    __slots__ = ()
+
+    @property
+    @_abc.abstractmethod
+    def st_mode(self) -> int: ...
+    @property
+    @_abc.abstractmethod
+    def st_size(self) -> int: ...
+    @property
+    @_abc.abstractmethod
+    def st_mtime(self) -> int: ...
+
+class FileStat(FileStatLike):
+    __slots__ = (
+        "st_mode",
+        "st_nlink",
+        "st_uid",
+        "st_gid",
+        "st_size",
+        "st_atime",
+        "st_mtime",
+        "st_ctime",
+    )
+
+    def __init__(
+        self,
+        st_mode: int = None,
+        st_size: int = 0,
+        st_mtime: int = 0,
+        is_dir: bool = False,
+    ):
+        self.st_mode = st_mode or (
+            _stat.S_IFDIR | 0o555 if is_dir else _stat.S_IFREG | 0o444
+        )
+        self.st_nlink = 1
+        self.st_uid = 0
+        self.st_gid = 0
+        self.st_size = st_size
+        self.st_atime = 0
+        self.st_mtime = st_mtime
+        self.st_ctime = 0
+
+    def settime(self, value):
+        self.st_atime = self.st_mtime = self.st_ctime = value
+
+    def setmode(self, value, isdir=None):
+        if isdir is None:
+            isdir = self.st_mode & _stat.S_IFDIR
+        if isdir:
+            self.st_mode = _stat.S_IFDIR | value
+        else:
+            self.st_mode = _stat.S_IFREG | value
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def items(self):
+        for key in self.__slots__:
+            yield key, getattr(self, key)
+
+    def __repr__(self):
+        return "<%s mode=%o, size=%s, mtime=%d>" % (
+            type(self).__name__,
+            self.st_mode,
+            _utils.sizeof_fmt(self.st_size),
+            self.st_mtime,
+        )
+
+    def __str__(self):
+        props = [f"{k}={v}" for k, v in self.items()]
+        return "<%s %s>" % (type(self).__name__, ", ".join(props))