fsspec 2024.5.0__py3-none-any.whl → 2024.6.1__py3-none-any.whl

fsspec/tests/test_async.py

@@ -1,230 +0,0 @@
-import asyncio
-import inspect
-import io
-import os
-import time
-
-import pytest
-
-import fsspec
-import fsspec.asyn
-from fsspec.asyn import _run_coros_in_chunks
-
-
-def test_sync_methods():
-    inst = fsspec.asyn.AsyncFileSystem()
-    assert inspect.iscoroutinefunction(inst._info)
-    assert hasattr(inst, "info")
-    assert inst.info.__qualname__ == "AsyncFileSystem._info"
-    assert not inspect.iscoroutinefunction(inst.info)
-
-
-def test_when_sync_methods_are_disabled():
-    class TestFS(fsspec.asyn.AsyncFileSystem):
-        mirror_sync_methods = False
-
-    inst = TestFS()
-    assert inspect.iscoroutinefunction(inst._info)
-    assert not inspect.iscoroutinefunction(inst.info)
-    assert inst.info.__qualname__ == "AbstractFileSystem.info"
-
-
-def test_interrupt():
-    loop = fsspec.asyn.get_loop()
-
-    async def f():
-        await asyncio.sleep(1000000)
-        return True
-
-    fut = asyncio.run_coroutine_threadsafe(f(), loop)
-    time.sleep(0.01)  # task launches
-    out = fsspec.asyn._dump_running_tasks(with_task=True)
-    task = out[0]["task"]
-    assert task.done() and fut.done()
-    assert isinstance(fut.exception(), fsspec.asyn.FSSpecCoroutineCancel)
-
-
-class _DummyAsyncKlass:
-    def __init__(self):
-        self.loop = fsspec.asyn.get_loop()
-
-    async def _dummy_async_func(self):
-        # Sleep 1 second function to test timeout
-        await asyncio.sleep(1)
-        return True
-
-    async def _bad_multiple_sync(self):
-        fsspec.asyn.sync_wrapper(_DummyAsyncKlass._dummy_async_func)(self)
-        return True
-
-    dummy_func = fsspec.asyn.sync_wrapper(_dummy_async_func)
-    bad_multiple_sync_func = fsspec.asyn.sync_wrapper(_bad_multiple_sync)
-
-
-def test_sync_wrapper_timeout_on_less_than_expected_wait_time_not_finish_function():
-    test_obj = _DummyAsyncKlass()
-    with pytest.raises(fsspec.FSTimeoutError):
-        test_obj.dummy_func(timeout=0.1)
-
-
-def test_sync_wrapper_timeout_on_more_than_expected_wait_time_will_finish_function():
-    test_obj = _DummyAsyncKlass()
-    assert test_obj.dummy_func(timeout=5)
-
-
-def test_sync_wrapper_timeout_none_will_wait_func_finished():
-    test_obj = _DummyAsyncKlass()
-    assert test_obj.dummy_func(timeout=None)
-
-
-def test_sync_wrapper_treat_timeout_0_as_none():
-    test_obj = _DummyAsyncKlass()
-    assert test_obj.dummy_func(timeout=0)
-
-
-def test_sync_wrapper_bad_multiple_sync():
-    test_obj = _DummyAsyncKlass()
-    with pytest.raises(NotImplementedError):
-        test_obj.bad_multiple_sync_func(timeout=5)
-
-
-def test_run_coros_in_chunks(monkeypatch):
-    total_running = 0
-
-    async def runner():
-        nonlocal total_running
-
-        total_running += 1
-        await asyncio.sleep(0)
-        if total_running > 4:
-            raise ValueError("More than 4 coroutines are running together")
-        total_running -= 1
-        return 1
-
-    async def main(**kwargs):
-        nonlocal total_running
-
-        total_running = 0
-        coros = [runner() for _ in range(32)]
-        results = await _run_coros_in_chunks(coros, **kwargs)
-        for result in results:
-            if isinstance(result, Exception):
-                raise result
-        return results
-
-    assert sum(asyncio.run(main(batch_size=4))) == 32
-
-    with pytest.raises(ValueError):
-        asyncio.run(main(batch_size=5))
-
-    with pytest.raises(ValueError):
-        asyncio.run(main(batch_size=-1))
-
-    assert sum(asyncio.run(main(batch_size=4))) == 32
-
-    monkeypatch.setitem(fsspec.config.conf, "gather_batch_size", 5)
-    with pytest.raises(ValueError):
-        asyncio.run(main())
-    assert sum(asyncio.run(main(batch_size=4))) == 32  # override
-
-    monkeypatch.setitem(fsspec.config.conf, "gather_batch_size", 4)
-    assert sum(asyncio.run(main())) == 32  # override
-
-
-@pytest.mark.skipif(os.name != "nt", reason="only for windows")
-def test_windows_policy():
-    from asyncio.windows_events import SelectorEventLoop
-
-    loop = fsspec.asyn.get_loop()
-    policy = asyncio.get_event_loop_policy()
-
-    # Ensure that the created loop always uses selector policy
-    assert isinstance(loop, SelectorEventLoop)
-
-    # Ensure that the global policy is not changed and it is
-    # set to the default one. This is important since the
-    # get_loop() method will temporarily override the policy
-    # with the one which uses selectors on windows, so this
-    # check ensures that we are restoring the old policy back
-    # after our change.
-    assert isinstance(policy, asyncio.DefaultEventLoopPolicy)
-
-
-def test_running_async():
-    assert not fsspec.asyn.running_async()
-
-    async def go():
-        assert fsspec.asyn.running_async()
-
-    asyncio.run(go())
-
-
-class DummyAsyncFS(fsspec.asyn.AsyncFileSystem):
-    _file_class = fsspec.asyn.AbstractAsyncStreamedFile
-
-    async def _info(self, path, **kwargs):
-        return {"name": "misc/foo.txt", "type": "file", "size": 100}
-
-    async def open_async(
-        self,
-        path,
-        mode="rb",
-        block_size=None,
-        autocommit=True,
-        cache_options=None,
-        **kwargs,
-    ):
-        return DummyAsyncStreamedFile(
-            self,
-            path,
-            mode,
-            block_size,
-            autocommit,
-            cache_options=cache_options,
-            **kwargs,
-        )
-
-
-class DummyAsyncStreamedFile(fsspec.asyn.AbstractAsyncStreamedFile):
-    def __init__(self, fs, path, mode, block_size, autocommit, **kwargs):
-        super().__init__(fs, path, mode, block_size, autocommit, **kwargs)
-        self.temp_buffer = io.BytesIO(b"foo-bar" * 20)
-
-    async def _fetch_range(self, start, end):
-        return self.temp_buffer.read(end - start)
-
-    async def _initiate_upload(self):
-        # Reinitialize for new uploads.
-        self.temp_buffer = io.BytesIO()
-
-    async def _upload_chunk(self, final=False):
-        self.temp_buffer.write(self.buffer.getbuffer())
-
-    async def get_data(self):
-        return self.temp_buffer.getbuffer().tobytes()
-
-    async def get_data(self):
-        return self.temp_buffer.getbuffer().tobytes()
-
-
-@pytest.mark.asyncio
-async def test_async_streamed_file_write():
-    test_fs = DummyAsyncFS()
-    streamed_file = await test_fs.open_async("misc/foo.txt", mode="wb")
-    inp_data = "foo-bar".encode("utf8") * streamed_file.blocksize * 2
-    await streamed_file.write(inp_data)
-    assert streamed_file.loc == len(inp_data)
-    await streamed_file.close()
-    out_data = await streamed_file.get_data()
-    assert out_data.count(b"foo-bar") == streamed_file.blocksize * 2
-
-
-@pytest.mark.asyncio
-async def test_async_streamed_file_read():
-    test_fs = DummyAsyncFS()
-    streamed_file = await test_fs.open_async("misc/foo.txt", mode="rb")
-    assert (
-        await streamed_file.read(7 * 3) + await streamed_file.read(7 * 18)
-        == b"foo-bar" * 20
-    )
-    await streamed_file.close()

fsspec/tests/test_caches.py

@@ -1,255 +0,0 @@
-import pickle
-import string
-
-import pytest
-
-from fsspec.caching import (
-    BlockCache,
-    FirstChunkCache,
-    ReadAheadCache,
-    caches,
-    register_cache,
-)
-from fsspec.implementations.cached import WholeFileCacheFileSystem
-
-
-def test_cache_getitem(Cache_imp):
-    cacher = Cache_imp(4, letters_fetcher, len(string.ascii_letters))
-    assert cacher._fetch(0, 4) == b"abcd"
-    assert cacher._fetch(None, 4) == b"abcd"
-    assert cacher._fetch(2, 4) == b"cd"
-
-
-def test_block_cache_lru():
-    # BlockCache is a cache that stores blocks of data and uses LRU to evict
-    block_size = 4
-    cache = BlockCache(
-        block_size, letters_fetcher, len(string.ascii_letters), maxblocks=2
-    )
-    # miss
-    cache._fetch(0, 2)
-    assert cache.cache_info().misses == 1
-    assert cache.cache_info().currsize == 1
-    assert cache.total_requested_bytes == block_size * cache.miss_count
-    assert cache.size == 52
-
-    # hit
-    cache._fetch(0, 2)
-    assert cache.cache_info().misses == 1
-    assert cache.cache_info().currsize == 1
-    assert cache.total_requested_bytes == block_size * cache.miss_count
-
-    # hit
-    cache._fetch(0, 2)
-    assert cache.cache_info().misses == 1
-    assert cache.cache_info().currsize == 1
-    # this works as a counter since all the reads are from the cache
-    assert cache.hit_count == 3
-    assert cache.miss_count == 1
-    # so far only 4 bytes have been read using range requests
-    assert cache.total_requested_bytes == block_size * cache.miss_count
-
-    # miss
-    cache._fetch(4, 6)
-    assert cache.cache_info().misses == 2
-    assert cache.cache_info().currsize == 2
-    assert cache.total_requested_bytes == block_size * cache.miss_count
-
-    # miss & evict
-    cache._fetch(12, 13)
-    assert cache.cache_info().misses == 3
-    assert cache.cache_info().currsize == 2
-    assert cache.hit_count == 5
-    assert cache.miss_count == 3
-    assert cache.total_requested_bytes == block_size * cache.miss_count
-
-
-def test_first_cache():
-    """
-    FirstChunkCache is a cache that only caches the first chunk of data
-    when some of that first block is requested.
-    """
-    block_size = 5
-    cache = FirstChunkCache(block_size, letters_fetcher, len(string.ascii_letters))
-    assert cache.cache is None
-    assert cache._fetch(12, 15) == letters_fetcher(12, 15)
-    assert cache.miss_count == 1
-    assert cache.hit_count == 0
-    assert cache.cache is None
-    total_requested_bytes = 15 - 12
-    assert cache.total_requested_bytes == total_requested_bytes
-
-    # because we overlap with the cache range, it will be cached
-    assert cache._fetch(3, 10) == letters_fetcher(3, 10)
-    assert cache.miss_count == 2
-    assert cache.hit_count == 0
-    # we'll read the first 5 and then the rest
-    total_requested_bytes += block_size + 5
-    assert cache.total_requested_bytes == total_requested_bytes
-
-    # partial hit again
-    assert cache._fetch(3, 10) == letters_fetcher(3, 10)
-    assert cache.miss_count == 2
-    assert cache.hit_count == 1
-    # we have the first 5 bytes cached
-    total_requested_bytes += 10 - 5
-    assert cache.total_requested_bytes == total_requested_bytes
-
-    assert cache.cache == letters_fetcher(0, 5)
-    assert cache._fetch(0, 4) == letters_fetcher(0, 4)
-    assert cache.hit_count == 2
-    assert cache.miss_count == 2
-    assert cache.total_requested_bytes == 18
-
-
-def test_readahead_cache():
-    """
-    ReadAheadCache is a cache that reads ahead of the requested range.
-    If the access pattern is not sequential it will be very inefficient.
-    """
-    block_size = 5
-    cache = ReadAheadCache(block_size, letters_fetcher, len(string.ascii_letters))
-    assert cache._fetch(12, 15) == letters_fetcher(12, 15)
-    assert cache.miss_count == 1
-    assert cache.hit_count == 0
-    total_requested_bytes = 15 - 12 + block_size
-    assert cache.total_requested_bytes == total_requested_bytes
-
-    assert cache._fetch(3, 10) == letters_fetcher(3, 10)
-    assert cache.miss_count == 2
-    assert cache.hit_count == 0
-    assert len(cache.cache) == 12
-    total_requested_bytes += (10 - 3) + block_size
-    assert cache.total_requested_bytes == total_requested_bytes
-
-    # caache hit again
-    assert cache._fetch(3, 10) == letters_fetcher(3, 10)
-    assert cache.miss_count == 2
-    assert cache.hit_count == 1
-    assert len(cache.cache) == 12
-    assert cache.total_requested_bytes == total_requested_bytes
-    assert cache.cache == letters_fetcher(3, 15)
-
-    # cache miss
-    assert cache._fetch(0, 4) == letters_fetcher(0, 4)
-    assert cache.hit_count == 1
-    assert cache.miss_count == 3
-    assert len(cache.cache) == 9
-    total_requested_bytes += (4 - 0) + block_size
-    assert cache.total_requested_bytes == total_requested_bytes
-
-
-def _fetcher(start, end):
-    return b"0" * (end - start)
-
-
-def letters_fetcher(start, end):
-    return string.ascii_letters[start:end].encode()
-
-
-not_parts_caches = {k: v for k, v in caches.items() if k != "parts"}
-
-
-@pytest.fixture(params=not_parts_caches.values(), ids=list(not_parts_caches))
-def Cache_imp(request):
-    return request.param
-
-
-def test_cache_empty_file(Cache_imp):
-    blocksize = 5
-    size = 0
-    cache = Cache_imp(blocksize, _fetcher, size)
-    assert cache._fetch(0, 0) == b""
-
-
-def test_cache_pickleable(Cache_imp):
-    blocksize = 5
-    size = 100
-    cache = Cache_imp(blocksize, _fetcher, size)
-    cache._fetch(0, 5)  # fill in cache
-    unpickled = pickle.loads(pickle.dumps(cache))
-    assert isinstance(unpickled, Cache_imp)
-    assert unpickled.blocksize == blocksize
-    assert unpickled.size == size
-    assert unpickled._fetch(0, 10) == b"0" * 10
-
-
-@pytest.mark.parametrize(
-    "size_requests",
-    [[(0, 30), (0, 35), (51, 52)], [(0, 1), (1, 11), (1, 52)], [(0, 52), (11, 15)]],
-)
-@pytest.mark.parametrize("blocksize", [1, 10, 52, 100])
-def test_cache_basic(Cache_imp, blocksize, size_requests):
-    cache = Cache_imp(blocksize, letters_fetcher, len(string.ascii_letters))
-
-    for start, end in size_requests:
-        result = cache._fetch(start, end)
-        expected = string.ascii_letters[start:end].encode()
-        assert result == expected
-
-
-@pytest.mark.parametrize("strict", [True, False])
-@pytest.mark.parametrize("sort", [True, False])
-def test_known(sort, strict):
-    parts = {(10, 20): b"1" * 10, (20, 30): b"2" * 10, (0, 10): b"0" * 10}
-    if sort:
-        parts = dict(sorted(parts.items()))
-    c = caches["parts"](None, None, 100, parts, strict=strict)
-    assert (0, 30) in c.data  # got consolidated
-    assert c._fetch(5, 15) == b"0" * 5 + b"1" * 5
-    assert c._fetch(15, 25) == b"1" * 5 + b"2" * 5
-    if strict:
-        # Over-read will raise error
-        with pytest.raises(ValueError):
-            # tries to call None fetcher
-            c._fetch(25, 35)
-    else:
-        # Over-read will be zero-padded
-        assert c._fetch(25, 35) == b"2" * 5 + b"\x00" * 5
-
-
-def test_background(server, monkeypatch):
-    import threading
-    import time
-
-    import fsspec
-
-    head = {"head_ok": "true", "head_give_length": "true"}
-    urla = server + "/index/realfile"
-    h = fsspec.filesystem("http", headers=head)
-    thread_ids = {threading.current_thread().ident}
-    f = h.open(urla, block_size=5, cache_type="background")
-    orig = f.cache._fetch_block
-
-    def wrapped(*a, **kw):
-        thread_ids.add(threading.current_thread().ident)
-        return orig(*a, **kw)
-
-    f.cache._fetch_block = wrapped
-    assert len(thread_ids) == 1
-    f.read(1)
-    time.sleep(0.1)  # second block is loading
-    assert len(thread_ids) == 2
-
-
-def test_register_cache():
-    # just test that we have them populated and fail to re-add again unless overload
-    with pytest.raises(ValueError):
-        register_cache(BlockCache)
-    register_cache(BlockCache, clobber=True)
-
-
-def test_cache_kwargs(mocker):
-    # test that kwargs are passed to the underlying filesystem after cache commit
-
-    fs = WholeFileCacheFileSystem(target_protocol="memory")
-    fs.touch("test")
-    fs.fs.put = mocker.MagicMock()
-
-    with fs.open("test", "wb", overwrite=True) as file_handle:
-        file_handle.write(b"foo")
-
-    # We don't care about the first parameter, just retrieve its expected value.
-    # It is a random location that cannot be predicted.
-    # The important thing is the 'overwrite' kwarg
-    fs.fs.put.assert_called_with(fs.fs.put.call_args[0][0], "/test", overwrite=True)

fsspec/tests/test_callbacks.py

@@ -1,89 +0,0 @@
-import pytest
-
-from fsspec.callbacks import Callback, TqdmCallback
-
-
-def test_callbacks():
-    empty_callback = Callback()
-    assert empty_callback.call("something", somearg=None) is None
-
-    hooks = {"something": lambda *_, arg=None: arg + 2}
-    simple_callback = Callback(hooks=hooks)
-    assert simple_callback.call("something", arg=2) == 4
-
-    hooks = {"something": lambda *_, arg1=None, arg2=None: arg1 + arg2}
-    multi_arg_callback = Callback(hooks=hooks)
-    assert multi_arg_callback.call("something", arg1=2, arg2=2) == 4
-
-
-def test_callbacks_as_callback():
-    empty_callback = Callback.as_callback(None)
-    assert empty_callback.call("something", arg="somearg") is None
-    assert Callback.as_callback(None) is Callback.as_callback(None)
-
-    hooks = {"something": lambda *_, arg=None: arg + 2}
-    real_callback = Callback.as_callback(Callback(hooks=hooks))
-    assert real_callback.call("something", arg=2) == 4
-
-
-def test_callbacks_as_context_manager(mocker):
-    spy_close = mocker.spy(Callback, "close")
-
-    with Callback() as cb:
-        assert isinstance(cb, Callback)
-
-    spy_close.assert_called_once()
-
-
-def test_callbacks_branched():
-    callback = Callback()
-
-    branch = callback.branched("path_1", "path_2")
-
-    assert branch is not callback
-    assert isinstance(branch, Callback)
-
-
-@pytest.mark.asyncio
-async def test_callbacks_branch_coro(mocker):
-    async_fn = mocker.AsyncMock(return_value=10)
-    callback = Callback()
-    wrapped_fn = callback.branch_coro(async_fn)
-    spy = mocker.spy(callback, "branched")
-
-    assert await wrapped_fn("path_1", "path_2", key="value") == 10
-
-    spy.assert_called_once_with("path_1", "path_2", key="value")
-    async_fn.assert_called_once_with(
-        "path_1", "path_2", callback=spy.spy_return, key="value"
-    )
-
-
-def test_callbacks_wrap():
-    events = []
-
-    class TestCallback(Callback):
-        def relative_update(self, inc=1):
-            events.append(inc)
-
-    callback = TestCallback()
-    for _ in callback.wrap(range(10)):
-        ...
-
-    assert events == [1] * 10
-
-
-@pytest.mark.parametrize("tqdm_kwargs", [{}, {"desc": "A custom desc"}])
-def test_tqdm_callback(tqdm_kwargs, mocker):
-    pytest.importorskip("tqdm")
-    callback = TqdmCallback(tqdm_kwargs=tqdm_kwargs)
-    mocker.patch.object(callback, "_tqdm_cls")
-    callback.set_size(10)
-    for _ in callback.wrap(range(10)):
-        ...
-
-    assert callback.tqdm.update.call_count == 11
-    if not tqdm_kwargs:
-        callback._tqdm_cls.assert_called_with(total=10)
-    else:
-        callback._tqdm_cls.assert_called_with(total=10, **tqdm_kwargs)