python-misc-utils 0.2__py3-none-any.whl

py_misc_utils/fs/ftp_fs.py

@@ -0,0 +1,242 @@
+import ftplib
+import functools
+import hashlib
+import io
+import os
+import tempfile
+import urllib.parse as uparse
+
+import ftputil
+
+from .. import alog
+from .. import assert_checks as tas
+from .. import context_managers as cm
+from .. import fs_base as fsb
+from .. import fs_utils as fsu
+from .. import cached_file as chf
+from .. import object_cache as objc
+from .. import writeback_file as wbf
+
+
+class CacheHandler(objc.Handler):
+
+  def __init__(self, *args, **kwargs):
+    super().__init__()
+    self._args = args
+    self._kwargs = kwargs
+
+  def create(self):
+    return ftputil.FTPHost(*self._args, **self._kwargs)
+
+  def is_alive(self, obj):
+    try:
+      obj.keep_alive()
+
+      return True
+    except:
+      return False
+
+  def close(self, obj):
+    obj.close()
+
+
+class FtpReader:
+
+  def __init__(self, conn, path):
+    self._conn = conn
+    self._path = path
+
+  @classmethod
+  def tag(cls, sres):
+    return chf.make_tag(size=sres.st_size, mtime=sres.st_mtime)
+
+  def support_blocks(self):
+    return False
+
+  def read_block(self, bpath, offset, size):
+    tas.check_eq(offset, chf.CachedBlockFile.WHOLE_OFFSET,
+                 msg=f'Wrong offset for whole content read: {offset}')
+
+    bfd = os.open(bpath, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, mode=0o440)
+    with open(bfd, mode='wb') as wfd:
+      with self._conn.open(self._path, mode='rb') as rfd:
+        self._conn.copyfileobj(rfd, wfd)
+
+    return os.path.getsize(bpath)
+
+
+# https://docs.python.org/3/library/ftplib.html
+# https://ftputil.sschwarzer.net/
+class FtpSession(ftplib.FTP):
+
+  def __init__(self, host, userid, passwd, port):
+    super().__init__()
+    self.connect(host, port=port)
+    self.login(userid, passwd)
+
+
+class FtpFs(fsb.FsBase):
+
+  ID = 'ftp'
+  IDS = (ID,)
+
+  def __init__(self, cache_iface=None, **kwargs):
+    super().__init__(cache_iface=cache_iface, **kwargs)
+
+  def _get_connection(self, host, port, user, passwd):
+    handler = CacheHandler(host, user, passwd,
+                           port=port,
+                           session_factory=FtpSession)
+    name = ('FTPFS', host, port, user)
+
+    return objc.cache().get(name, handler)
+
+  def _netloc(self, purl):
+    return (purl.hostname.lower(), purl.port or 21)
+
+  def _parse_url(self, url):
+    purl = uparse.urlparse(url)
+
+    host, port = self._netloc(purl)
+    user = purl.username or 'anonymous'
+    passwd = purl.password or ''
+
+    conn = self._get_connection(host, port, user, passwd)
+
+    return conn, purl
+
+  def _make_reader(self, conn, purl):
+    sres = self._stat(conn, purl.path)
+
+    tag = FtpReader.tag(sres)
+    meta = chf.Meta(size=sres.st_size, mtime=sres.st_mtime, tag=tag)
+    reader = FtpReader(conn, purl.path)
+
+    return reader, meta
+
+  def remove(self, url):
+    conn, purl = self._parse_url(url)
+    conn.remove(purl.path)
+
+  def rename(self, src_url, dest_url):
+    src_conn, src_purl = self._parse_url(src_url)
+    dest_conn, dest_purl = self._parse_url(dest_url)
+
+    src_netloc, dest_netloc = self._netloc(src_purl), self._netloc(dest_purl)
+
+    tas.check_eq(src_netloc, dest_netloc,
+                 msg=f'Source and destination URL must be on the same host: ' \
+                 f'{src_netloc} vs. {dest_netloc}')
+
+    src_conn.rename(src_purl.path, dest_purl.path)
+
+  def mkdir(self, url, mode=None):
+    conn, purl = self._parse_url(url)
+    conn.mkdir(purl.path)
+
+  def makedirs(self, url, mode=None, exist_ok=None):
+    conn, purl = self._parse_url(url)
+
+    conn.makedirs(purl.path, exist_ok=exist_ok or False)
+
+  def rmdir(self, url):
+    conn, purl = self._parse_url(url)
+    conn.rmdir(purl.path)
+
+  def rmtree(self, url, ignore_errors=None):
+    conn, purl = self._parse_url(url)
+
+    conn.rmtree(purl.path, ignore_errors=ignore_errors or False)
+
+  def _stat(self, conn, path):
+    sres = conn.stat(path)
+
+    tag = FtpReader.tag(sres)
+
+    return fsb.DirEntry(name=os.path.basename(path),
+                        path=path,
+                        etag=tag,
+                        st_mode=sres.st_mode,
+                        st_size=sres.st_size,
+                        st_ctime=sres.st_ctime or sres.st_mtime,
+                        st_mtime=sres.st_mtime)
+
+  def stat(self, url):
+    conn, purl = self._parse_url(url)
+
+    return self._stat(conn, purl.path)
+
+  def list(self, url):
+    conn, purl = self._parse_url(url)
+
+    for name in conn.listdir(purl.path):
+      path = os.path.join(purl.path, name)
+
+      yield self._stat(conn, path)
+
+  def open(self, url, mode, **kwargs):
+    conn, purl = self._parse_url(url)
+
+    if self.read_mode(mode):
+      reader, meta = self._make_reader(conn, purl)
+      cfile = self._cache_iface.open(url, meta, reader, **kwargs)
+
+      return io.TextIOWrapper(cfile) if self.text_mode(mode) else cfile
+    else:
+      writeback_fn = functools.partial(self._upload_file, url)
+      if not self.truncate_mode(mode) and conn.path.exists(purl.path):
+        url_file = self._download_file(url)
+        self.seek_stream(mode, url_file)
+      else:
+        url_file = tempfile.TemporaryFile()
+
+      wbfile = wbf.WritebackFile(url_file, writeback_fn)
+
+      return io.TextIOWrapper(wbfile) if self.text_mode(mode) else wbfile
+
+  def _upload_file(self, url, stream):
+    conn, purl = self._parse_url(url)
+
+    stream.seek(0)
+    with conn.open(purl.path, mode='wb') as dest_fd:
+      conn.copyfileobj(stream, dest_fd)
+
+  def _download_file(self, url):
+    conn, purl = self._parse_url(url)
+
+    with cm.Wrapper(tempfile.TemporaryFile()) as ftmp:
+      with conn.open(purl.path, mode='rb') as src_fd:
+        conn.copyfileobj(src_fd, ftmp.v)
+
+      return ftmp.detach()
+
+  def put_file(self, url, data_gen):
+    conn, purl = self._parse_url(url)
+
+    with conn.open(purl.path, mode='wb') as fd:
+      for data in data_gen:
+        fd.write(data)
+
+  def get_file(self, url):
+    conn, purl = self._parse_url(url)
+
+    with conn.open(purl.path, mode='rb') as fd:
+      for data in fsu.enum_chunks(fd):
+        yield data
+
+  def as_local(self, url, **kwargs):
+    conn, purl = self._parse_url(url)
+    reader, meta = self._make_reader(conn, purl)
+
+    return self._cache_iface.as_local(url, meta, reader, **kwargs)
+
+  def link(self, src_url, dest_url):
+    # There is not link support in FTP.
+    self.copyfile(src_url, dest_url)
+
+  def symlink(self, src_url, dest_url):
+    self.link(src_url, dest_url)
+
+
+FILE_SYSTEMS = (FtpFs,)
+

py_misc_utils/fs/gcs_fs.py

@@ -0,0 +1,196 @@
+import functools
+import hashlib
+import io
+import os
+import tempfile
+import urllib.parse as uparse
+
+from .. import alog
+from .. import assert_checks as tas
+from .. import context_managers as cm
+from .. import fs_base as fsb
+from .. import fs_utils as fsu
+from .. import cached_file as chf
+from .. import gcs_fs as gcs
+from .. import object_cache as objc
+from .. import osfd
+from .. import writeback_file as wbf
+
+
+class CacheHandler(objc.Handler):
+
+  def __init__(self, *args, **kwargs):
+    super().__init__()
+    self._args = args
+    self._kwargs = kwargs
+
+  def create(self):
+    return gcs.GcsFs(*self._args, **self._kwargs)
+
+
+class GcsReader:
+
+  def __init__(self, fs, path, sres):
+    self._fs = fs
+    self._path = path
+    self._sres = sres
+
+  @classmethod
+  def tag(cls, sres):
+    return sres.etag or chf.make_tag(size=sres.st_size, mtime=sres.st_mtime)
+
+  def support_blocks(self):
+    return True
+
+  def read_block(self, bpath, offset, size):
+    if offset != chf.CachedBlockFile.WHOLE_OFFSET:
+      size = min(size, self._sres.st_size - offset)
+      data = self._fs.pread(self._path, offset, size)
+
+      with osfd.OsFd(bpath, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, mode=0o440) as wfd:
+        os.write(wfd, data)
+
+      return len(data)
+    else:
+      with osfd.OsFd(bpath, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, mode=0o440) as wfd:
+        for data in self._fs.download(self._path):
+          os.write(wfd, data)
+
+      return os.path.getsize(bpath)
+
+
+class GcsFs(fsb.FsBase):
+
+  ID = 'gcs'
+  IDS = (ID,)
+
+  def __init__(self, cache_iface=None, **kwargs):
+    super().__init__(cache_iface=cache_iface, **kwargs)
+
+  def _get_fs(self, bucket):
+    handler = CacheHandler(bucket)
+    name = ('GCSFS', bucket)
+
+    return objc.cache().get(name, handler)
+
+  def _parse_url(self, url):
+    purl = uparse.urlparse(url)
+    purl = purl._replace(path=purl.path.lstrip('/'))
+    fs = self._get_fs(purl.hostname)
+
+    return fs, purl
+
+  def _make_reader(self, fs, purl):
+    sres = fs.stat(purl.path)
+    tas.check_is_not_none(sres, msg=f'File does not exist: {purl.geturl()}')
+
+    tag = GcsReader.tag(sres)
+    meta = chf.Meta(size=sres.st_size, mtime=sres.st_mtime, tag=tag)
+    reader = GcsReader(fs, purl.path, sres)
+
+    return reader, meta
+
+  def _parse_samefs(self, src_url, dest_url):
+    src_fs, src_purl = self._parse_url(src_url)
+    dest_fs, dest_purl = self._parse_url(dest_url)
+
+    tas.check_eq(src_fs.bucket, dest_fs.bucket,
+                 msg=f'Source and destination URL must be on the same bucket: ' \
+                 f'{src_url} vs. {dest_url}')
+
+    return (src_fs, src_purl), (dest_fs, dest_purl)
+
+  def _copy(self, src_url, dest_url):
+    (src_fs, src_purl), (dest_fs, dest_purl) = self._parse_samefs(src_url, dest_url)
+
+    src_fs.copy(src_purl.path, dest_purl.path)
+
+  def remove(self, url):
+    fs, purl = self._parse_url(url)
+    fs.remove(purl.path)
+
+  def rename(self, src_url, dest_url):
+    (src_fs, src_purl), (dest_fs, dest_purl) = self._parse_samefs(src_url, dest_url)
+
+    src_fs.rename(src_purl.path, dest_purl.path)
+
+  def mkdir(self, url, mode=None):
+    pass
+
+  def makedirs(self, url, mode=None, exist_ok=None):
+    pass
+
+  def rmdir(self, url):
+    pass
+
+  def rmtree(self, url, ignore_errors=None):
+    fs, purl = self._parse_url(url)
+
+    fs.rmtree(purl.path, ignore_errors=ignore_errors or False)
+
+  def stat(self, url):
+    fs, purl = self._parse_url(url)
+
+    return fs.stat(purl.path)
+
+  def list(self, url):
+    fs, purl = self._parse_url(url)
+
+    return fs.listdir(purl.path)
+
+  def open(self, url, mode, **kwargs):
+    fs, purl = self._parse_url(url)
+
+    if self.read_mode(mode):
+      reader, meta = self._make_reader(fs, purl)
+      cfile = self._cache_iface.open(url, meta, reader, **kwargs)
+
+      return io.TextIOWrapper(cfile) if self.text_mode(mode) else cfile
+    else:
+      writeback_fn = functools.partial(self._upload_file, url)
+      if not self.truncate_mode(mode) and fs.exists(purl.path):
+        url_file = self._download_file(url)
+        self.seek_stream(mode, url_file)
+      else:
+        url_file = tempfile.TemporaryFile()
+
+      wbfile = wbf.WritebackFile(url_file, writeback_fn)
+
+      return io.TextIOWrapper(wbfile) if self.text_mode(mode) else wbfile
+
+  def _upload_file(self, url, stream):
+    stream.seek(0)
+    self.put_file(url, fsu.enum_chunks(stream))
+
+  def _download_file(self, url):
+    with cm.Wrapper(tempfile.TemporaryFile()) as ftmp:
+      for data in self.get_file(url):
+        ftmp.v.write(data)
+
+      return ftmp.detach()
+
+  def put_file(self, url, data_gen):
+    fs, purl = self._parse_url(url)
+
+    fs.upload(purl.path, data_gen)
+
+  def get_file(self, url):
+    fs, purl = self._parse_url(url)
+
+    return fs.download(purl.path)
+
+  def as_local(self, url, **kwargs):
+    fs, purl = self._parse_url(url)
+    reader, meta = self._make_reader(fs, purl)
+
+    return self._cache_iface.as_local(url, meta, reader, **kwargs)
+
+  def link(self, src_url, dest_url):
+    self._copy(src_url, dest_url)
+
+  def symlink(self, src_url, dest_url):
+    self.link(src_url, dest_url)
+
+
+FILE_SYSTEMS = (GcsFs,)
+

py_misc_utils/fs/http_fs.py

@@ -0,0 +1,241 @@
+import functools
+import hashlib
+import io
+import mimetypes
+import os
+import re
+import requests
+import stat as st
+import tempfile
+
+import bs4
+
+from .. import alog
+from .. import assert_checks as tas
+from .. import context_managers as cm
+from .. import fs_base as fsb
+from .. import fs_utils as fsu
+from .. import cached_file as chf
+from .. import http_utils as hu
+from .. import osfd
+from .. import writeback_file as wbf
+
+
+class HttpReader:
+
+  def __init__(self, url, session=None, head=None, req_kwargs=None, chunk_size=None):
+    session = session if session is not None else requests.Session()
+    req_kwargs = req_kwargs or dict()
+    if head is None:
+      head = hu.info(url, mod=session, **req_kwargs)
+
+    allow_ranges = hu.support_ranges(head.headers)
+
+    self._url = url
+    self._session = session
+    self._req_kwargs = req_kwargs
+    self._chunk_size = chunk_size or 16 * 1024**2
+    self._size = hu.content_length(head.headers)
+    self._support_blocks = self._size is not None and allow_ranges
+
+  @classmethod
+  def tag(cls, head):
+    tag = hu.etag(head.headers)
+    if tag is None:
+      mtime = hu.last_modified(head.headers)
+      length = hu.content_length(head.headers)
+      tag = chf.make_tag(size=length, mtime=mtime)
+
+    return tag
+
+  def support_blocks(self):
+    return self._support_blocks
+
+  def read_block(self, bpath, offset, size):
+    if self._support_blocks and offset != chf.CachedBlockFile.WHOLE_OFFSET:
+      with osfd.OsFd(bpath, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, mode=0o440) as wfd:
+        size = min(size, self._size - offset)
+
+        headers = self._req_kwargs.get('headers', dict()).copy()
+        hu.add_range(headers, offset, offset + size)
+
+        resp = self._session.get(self._url, headers=headers)
+        resp.raise_for_status()
+        data = hu.range_data(offset, offset + size - 1, resp.headers, resp.content)
+
+        os.write(wfd, data)
+
+        return len(data)
+    else:
+      resp = self._session.get(self._url, stream=True, **self._req_kwargs)
+      resp.raise_for_status()
+      with osfd.OsFd(bpath, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, mode=0o440) as wfd:
+        for data in resp.iter_content(chunk_size=self._chunk_size):
+          os.write(wfd, data)
+
+      return os.path.getsize(bpath)
+
+
+class HttpFs(fsb.FsBase):
+
+  mimetypes.init()
+
+  ID = 'http'
+  IDS = (ID, 'https')
+
+  def __init__(self, cache_iface=None, **kwargs):
+    super().__init__(cache_iface=cache_iface, **kwargs)
+    self._req_kwargs = hu.filter_request_args(kwargs)
+    self._session = requests.Session()
+
+  def _exists(self, url):
+    try:
+      hu.info(url, mod=self._session, **self._req_kwargs)
+
+      return True
+    except requests.exceptions.HTTPError:
+      return False
+
+  def _make_reader(self, url):
+    head = hu.info(url, mod=self._session, **self._req_kwargs)
+
+    tag = HttpReader.tag(head)
+    size = hu.content_length(head.headers)
+    mtime = hu.last_modified(head.headers)
+    meta = chf.Meta(size=size, mtime=mtime, tag=tag)
+    reader = HttpReader(url,
+                        session=self._session,
+                        head=head,
+                        req_kwargs=self._req_kwargs)
+
+    return reader, meta
+
+  def stat(self, url):
+    head = hu.info(url, mod=self._session, **self._req_kwargs)
+
+    length = hu.content_length(head.headers)
+    mtime = hu.last_modified(head.headers)
+    tag = hu.etag(head.headers) or chf.make_tag(size=length, mtime=mtime)
+
+    # HTML pages have content, but can also be listed (for HREF linked from it).
+    # hence the weird st.S_IFREG | st.S_IFDIR.
+    return fsb.DirEntry(name=os.path.basename(url.rstrip('/')),
+                        path=url,
+                        etag=tag,
+                        st_mode=st.S_IFREG | st.S_IFDIR,
+                        st_size=length,
+                        st_ctime=mtime,
+                        st_mtime=mtime)
+
+  def open(self, url, mode, **kwargs):
+    if self.read_mode(mode):
+      reader, meta = self._make_reader(url)
+      cfile = self._cache_iface.open(url, meta, reader, **kwargs)
+
+      return io.TextIOWrapper(cfile) if self.text_mode(mode) else cfile
+    else:
+      writeback_fn = functools.partial(self._upload_file, url)
+      if not self.truncate_mode(mode) and self._exists(url):
+        url_file = self._download_file(url)
+        self.seek_stream(mode, url_file)
+      else:
+        url_file = tempfile.TemporaryFile()
+
+      wbfile = wbf.WritebackFile(url_file, writeback_fn)
+
+      return io.TextIOWrapper(wbfile) if self.text_mode(mode) else wbfile
+
+  def remove(self, url):
+    self._session.delete(url, **self._req_kwargs)
+
+  def rename(self, src_url, dest_url):
+    # There is no "rename" in HTTP ...
+    with self._download_file(src_url) as fd:
+      self._upload_file(dest_url, fd)
+
+    self.remove(src_url)
+
+  def mkdir(self, url, mode=None):
+    pass
+
+  def makedirs(self, url, mode=None, exist_ok=None):
+    pass
+
+  def rmdir(self, url):
+    pass
+
+  def rmtree(self, url, ignore_errors=None):
+    pass
+
+  def list(self, url):
+    resp = self._session.get(url, **self._req_kwargs)
+    resp.raise_for_status()
+
+    html_parser = bs4.BeautifulSoup(resp.text, 'html.parser')
+
+    for link in html_parser.find_all('a'):
+      href = link.get('href')
+      if href and not re.match(r'[a-zA-Z]+://', href):
+        lurl = os.path.join(url, href)
+        try:
+          de = self.stat(lurl)
+
+          yield de
+        except GeneratorExit:
+          raise
+        except Exception as ex:
+          alog.debug(f'Unable to stat URL {lurl}: {ex}')
+
+  def _upload_data_gen(self, url, data_gen):
+    ctype, cencoding = mimetypes.guess_type(url, strict=False)
+
+    headers = self._req_kwargs.get('headers', dict()).copy()
+    if ctype is not None:
+      headers[hu.CONTENT_TYPE] = ctype
+    if cencoding is not None:
+      headers[hu.CONTENT_ENCODING] = cencoding
+
+    self._session.put(url, headers=headers, data=data_gen)
+
+  def _upload_file(self, url, stream):
+    stream.seek(0)
+    self._upload_data_gen(url, fsu.enum_chunks(stream))
+
+  def _iterate_chunks(self, url, chunk_size=None):
+    chunk_size = chunk_size or 16 * 1024**2
+
+    resp = self._session.get(url, stream=True, **self._req_kwargs)
+    resp.raise_for_status()
+
+    for data in resp.iter_content(chunk_size=chunk_size):
+      yield data
+
+  def _download_file(self, url, chunk_size=None):
+    with cm.Wrapper(tempfile.TemporaryFile()) as ftmp:
+      for data in self._iterate_chunks(url, chunk_size=chunk_size):
+        ftmp.v.write(data)
+
+      return ftmp.detach()
+
+  def put_file(self, url, data_gen):
+    self._upload_data_gen(url, data_gen)
+
+  def get_file(self, url):
+    for data in self._iterate_chunks(url):
+      yield data
+
+  def as_local(self, url, **kwargs):
+    reader, meta = self._make_reader(url)
+
+    return self._cache_iface.as_local(url, meta, reader, **kwargs)
+
+  def link(self, src_url, dest_url):
+    # There is not link support in HTTP.
+    self.copyfile(src_url, dest_url)
+
+  def symlink(self, src_url, dest_url):
+    self.link(src_url, dest_url)
+
+
+FILE_SYSTEMS = (HttpFs,)
+