fsspec 2024.5.0__py3-none-any.whl → 2024.6.1__py3-none-any.whl

fsspec/tests/test_compression.py

@@ -1,164 +0,0 @@
-import pathlib
-
-import pytest
-
-import fsspec.core
-from fsspec.compression import compr, register_compression
-from fsspec.utils import compressions, infer_compression
-
-
-def test_infer_custom_compression():
-    """Inferred compression gets values from fsspec.compression.compr."""
-    assert infer_compression("fn.zip") == "zip"
-    assert infer_compression("fn.gz") == "gzip"
-    assert infer_compression("fn.unknown") is None
-    assert infer_compression("fn.test_custom") is None
-    assert infer_compression("fn.tst") is None
-
-    register_compression("test_custom", lambda f, **kwargs: f, "tst")
-
-    try:
-        assert infer_compression("fn.zip") == "zip"
-        assert infer_compression("fn.gz") == "gzip"
-        assert infer_compression("fn.unknown") is None
-        assert infer_compression("fn.test_custom") is None
-        assert infer_compression("fn.tst") == "test_custom"
-
-        # Duplicate registration in name or extension raises a value error.
-        with pytest.raises(ValueError):
-            register_compression("test_custom", lambda f, **kwargs: f, "tst")
-
-        with pytest.raises(ValueError):
-            register_compression("test_conflicting", lambda f, **kwargs: f, "tst")
-        assert "test_conflicting" not in compr
-
-        # ...but can be forced.
-        register_compression(
-            "test_conflicting", lambda f, **kwargs: f, "tst", force=True
-        )
-        assert infer_compression("fn.zip") == "zip"
-        assert infer_compression("fn.gz") == "gzip"
-        assert infer_compression("fn.unknown") is None
-        assert infer_compression("fn.test_custom") is None
-        assert infer_compression("fn.tst") == "test_conflicting"
-
-    finally:
-        del compr["test_custom"]
-        del compr["test_conflicting"]
-        del compressions["tst"]
-
-
-def test_infer_uppercase_compression():
-    assert infer_compression("fn.ZIP") == "zip"
-    assert infer_compression("fn.GZ") == "gzip"
-    assert infer_compression("fn.UNKNOWN") is None
-    assert infer_compression("fn.TEST_UPPERCASE") is None
-    assert infer_compression("fn.TEST") is None
-
-
-def test_lzma_compression_name():
-    pytest.importorskip("lzma")
-    assert infer_compression("fn.xz") == "xz"
-    assert infer_compression("fn.lzma") == "lzma"
-
-
-def test_lz4_compression(tmpdir):
-    """Infer lz4 compression for .lz4 files if lz4 is available."""
-    tmp_path = pathlib.Path(str(tmpdir))
-
-    lz4 = pytest.importorskip("lz4")
-
-    tmp_path.mkdir(exist_ok=True)
-
-    tdat = "foobar" * 100
-
-    with fsspec.core.open(
-        str(tmp_path / "out.lz4"), mode="wt", compression="infer"
-    ) as outfile:
-        outfile.write(tdat)
-
-    compressed = (tmp_path / "out.lz4").open("rb").read()
-    assert lz4.frame.decompress(compressed).decode() == tdat
-
-    with fsspec.core.open(
-        str(tmp_path / "out.lz4"), mode="rt", compression="infer"
-    ) as infile:
-        assert infile.read() == tdat
-
-    with fsspec.core.open(
-        str(tmp_path / "out.lz4"), mode="rt", compression="lz4"
-    ) as infile:
-        assert infile.read() == tdat
-
-
-def test_zstd_compression(tmpdir):
-    """Infer zstd compression for .zst files if zstandard is available."""
-    tmp_path = pathlib.Path(str(tmpdir))
-
-    zstd = pytest.importorskip("zstandard")
-
-    tmp_path.mkdir(exist_ok=True)
-
-    tdat = "foobar" * 100
-
-    with fsspec.core.open(
-        str(tmp_path / "out.zst"), mode="wt", compression="infer"
-    ) as outfile:
-        outfile.write(tdat)
-
-    compressed = (tmp_path / "out.zst").open("rb").read()
-    assert zstd.ZstdDecompressor().decompress(compressed, len(tdat)).decode() == tdat
-
-    with fsspec.core.open(
-        str(tmp_path / "out.zst"), mode="rt", compression="infer"
-    ) as infile:
-        assert infile.read() == tdat
-
-    with fsspec.core.open(
-        str(tmp_path / "out.zst"), mode="rt", compression="zstd"
-    ) as infile:
-        assert infile.read() == tdat
-
-    # fails in https://github.com/fsspec/filesystem_spec/issues/725
-    infile = fsspec.core.open(
-        str(tmp_path / "out.zst"), mode="rb", compression="infer"
-    ).open()
-
-    infile.close()
-
-
-def test_snappy_compression(tmpdir):
-    """No registered compression for snappy, but can be specified."""
-    tmp_path = pathlib.Path(str(tmpdir))
-
-    snappy = pytest.importorskip("snappy")
-
-    tmp_path.mkdir(exist_ok=True)
-
-    tdat = "foobar" * 100
-
-    # Snappy isn't inferred.
-    with fsspec.core.open(
-        str(tmp_path / "out.snappy"), mode="wt", compression="infer"
-    ) as outfile:
-        outfile.write(tdat)
-    assert (tmp_path / "out.snappy").open("rb").read().decode() == tdat
-
-    # but can be specified.
-    with fsspec.core.open(
-        str(tmp_path / "out.snappy"), mode="wt", compression="snappy"
-    ) as outfile:
-        outfile.write(tdat)
-
-    compressed = (tmp_path / "out.snappy").open("rb").read()
-    assert snappy.StreamDecompressor().decompress(compressed).decode() == tdat
-
-    with fsspec.core.open(
-        str(tmp_path / "out.snappy"), mode="rb", compression="infer"
-    ) as infile:
-        assert infile.read() == compressed
-
-    with fsspec.core.open(
-        str(tmp_path / "out.snappy"), mode="rt", compression="snappy"
-    ) as infile:
-        assert infile.read() == tdat

fsspec/tests/test_config.py

@@ -1,129 +0,0 @@
-import os
-from warnings import catch_warnings
-
-import pytest
-
-import fsspec
-from fsspec.config import conf, set_conf_env, set_conf_files
-
-
-@pytest.fixture
-def clean_conf():
-    """Tests should start and end with clean config dict"""
-    conf.clear()
-    yield
-    conf.clear()
-
-
-def test_from_env_ignored(clean_conf):
-    env = {
-        "FSSPEC": "missing_protocol",
-        "FSSPEC_": "missing_protocol",
-        "FSSPEC__INVALID_KEY": "invalid_protocol",
-        "FSSPEC_INVALID1": "not_json_dict",
-        "FSSPEC_INVALID2": '["not_json_dict"]',
-    }
-    cd = {}
-    with catch_warnings(record=True) as w:
-        set_conf_env(conf_dict=cd, envdict=env)
-        assert len(w) == 5
-        assert "unexpected name" in str(w[0].message)
-        assert "unexpected name" in str(w[1].message)
-        assert "unexpected name" in str(w[2].message)
-        assert "parse failure" in str(w[3].message)
-        assert "not being a dict" in str(w[4].message)
-    assert cd == {}
-
-
-def test_from_env_kwargs(clean_conf):
-    env = {
-        "FSSPEC_PROTO_KEY": "value",
-        "FSSPEC_PROTO_LONG_KEY": "othervalue",
-        "FSSPEC_MALFORMED": "novalue",
-    }
-    cd = {}
-    with catch_warnings(record=True) as w:
-        set_conf_env(conf_dict=cd, envdict=env)
-        assert len(w) == 1
-        assert "parse failure" in str(w[0].message)
-    assert cd == {"proto": {"key": "value", "long_key": "othervalue"}}
-
-
-def test_from_env_protocol_dict(clean_conf):
-    env = {
-        "FSSPEC_PROTO": '{"int": 1, "float": 2.3, "bool": true, "dict": {"key": "val"}}'
-    }
-    cd = {}
-    set_conf_env(conf_dict=cd, envdict=env)
-    assert cd == {
-        "proto": {"int": 1, "float": 2.3, "bool": True, "dict": {"key": "val"}}
-    }
-
-
-def test_from_env_kwargs_override_protocol_dict(clean_conf):
-    env = {
-        "FSSPEC_PROTO_LONG_KEY": "override1",
-        "FSSPEC_PROTO": '{"key": "value1", "long_key": "value2", "otherkey": "value3"}',
-        "FSSPEC_PROTO_KEY": "override2",
-    }
-    cd = {}
-    set_conf_env(conf_dict=cd, envdict=env)
-    assert cd == {
-        "proto": {"key": "override2", "long_key": "override1", "otherkey": "value3"}
-    }
-
-
-def test_from_file_ini(clean_conf, tmpdir):
-    file1 = os.path.join(tmpdir, "1.ini")
-    file2 = os.path.join(tmpdir, "2.ini")
-    with open(file1, "w") as f:
-        f.write(
-            """[proto]
-key=value
-other_key:othervalue
-overwritten=dont_see
-        """
-        )
-    with open(file2, "w") as f:
-        f.write(
-            """[proto]
-overwritten=see
-        """
-        )
-    cd = {}
-    set_conf_files(tmpdir, cd)
-    assert cd == {
-        "proto": {"key": "value", "other_key": "othervalue", "overwritten": "see"}
-    }
-
-
-def test_from_file_json(clean_conf, tmpdir):
-    file1 = os.path.join(tmpdir, "1.json")
-    file2 = os.path.join(tmpdir, "2.json")
-    with open(file1, "w") as f:
-        f.write(
-            """{"proto":
-{"key": "value",
-"other_key": "othervalue",
-"overwritten": false}}
-        """
-        )
-    with open(file2, "w") as f:
-        f.write(
-            """{"proto":
-{"overwritten": true}}
-        """
-        )
-    cd = {}
-    set_conf_files(tmpdir, cd)
-    assert cd == {
-        "proto": {"key": "value", "other_key": "othervalue", "overwritten": True}
-    }
-
-
-def test_apply(clean_conf):
-    conf["file"] = {"auto_mkdir": "test"}
-    fs = fsspec.filesystem("file")
-    assert fs.auto_mkdir == "test"
-    fs = fsspec.filesystem("file", auto_mkdir=True)
-    assert fs.auto_mkdir is True