bw-processing 0.9.6__tar.gz → 1.0__tar.gz

{bw_processing-0.9.6/bw_processing.egg-info → bw_processing-1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bw_processing
-Version: 0.9.6
+Version: 1.0
 Summary: Foo
 Author-email: Chris Mutel <cmutel@gmail.com>
 Maintainer-email: Chris Mutel <cmutel@gmail.com>

{bw_processing-0.9.6 → bw_processing-1.0}/bw_processing/__init__.py

@@ -26,7 +26,7 @@ __all__ = (
     "UndefinedInterface",
 )
 
-__version__ = "0.9.6"
+__version__ = "1.0"
 
 
 from .array_creation import create_array, create_structured_array

{bw_processing-0.9.6 → bw_processing-1.0}/bw_processing/constants.py

@@ -4,6 +4,7 @@ from enum import Enum
 import numpy as np
 
 MAX_SIGNED_32BIT_INT = 2147483647
+MAX_SIGNED_64BIT_INT = 9223372036854775807
 
 # We could try to save space by not storing the columns
 # `row_index` and `col_index`, and add them after loading from
@@ -20,7 +21,7 @@ UNCERTAINTY_DTYPE = [
     ("maximum", np.float32),
     ("negative", bool),
 ]
-INDICES_DTYPE = [("row", np.int32), ("col", np.int32)]
+INDICES_DTYPE = [("row", np.int64), ("col", np.int64)]
 
 NAME_RE = re.compile(r"^[\w\-\.]*$")
 

{bw_processing-0.9.6 → bw_processing-1.0}/bw_processing/datapackage.py

@@ -14,6 +14,8 @@ from morefs.dict import DictFS
 from .constants import (
     DEFAULT_LICENSES,
     INDICES_DTYPE,
+    MAX_SIGNED_32BIT_INT,
+    MAX_SIGNED_64BIT_INT,
     NUMPY_SERIALIZE_FORMAT_EXTENSION,
     NUMPY_SERIALIZE_FORMAT_NAME,
     PARQUET_SERIALIZE_FORMAT_EXTENSION,
@@ -257,6 +259,12 @@ class DatapackageBase(ABC):
 
         self.data[index] = resource
 
+    def get_max_index_value(self) -> int:
+        """Get maximum index value (max signed 32 or 64 bit integer) for this datapackage"""
+        if self.metadata.get("64_bit_indices"):
+            return MAX_SIGNED_64BIT_INT
+        return MAX_SIGNED_32BIT_INT
+
 
 class FilteredDatapackage(DatapackageBase):
     """A subset of a datapackage. Used in matrix construction or other data manipulation operations.
@@ -360,6 +368,7 @@ class Datapackage(DatapackageBase):
             "combinatorial": combinatorial,
             "sequential": sequential,
             "seed": seed,
+            "64_bit_indices": True,
             "sum_intra_duplicates": sum_intra_duplicates,
             "sum_inter_duplicates": sum_inter_duplicates,
             "matrix_serialize_format_type": matrix_serialize_format_type.value,

{bw_processing-0.9.6 → bw_processing-1.0/bw_processing.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bw_processing
-Version: 0.9.6
+Version: 1.0
 Summary: Foo
 Author-email: Chris Mutel <cmutel@gmail.com>
 Maintainer-email: Chris Mutel <cmutel@gmail.com>

{bw_processing-0.9.6 → bw_processing-1.0}/bw_processing.egg-info/SOURCES.txt

@@ -2,26 +2,6 @@ LICENSE
 MANIFEST.in
 README.md
 pyproject.toml
-./bw_processing/__init__.py
-./bw_processing/array_creation.py
-./bw_processing/constants.py
-./bw_processing/datapackage.py
-./bw_processing/errors.py
-./bw_processing/filesystem.py
-./bw_processing/indexing.py
-./bw_processing/io_helpers.py
-./bw_processing/merging.py
-./bw_processing/proxies.py
-./bw_processing/unique_fields.py
-./bw_processing/utils.py
-./bw_processing/examples/__init__.py
-./bw_processing/examples/interfaces.py
-./dev/calculation_package.py
-./dev/loading.py
-./dev/processed_package.py
-./dev/resources.py
-./dev/speed_tests.py
-./docs/conf.py
 bw_processing/__init__.py
 bw_processing/array_creation.py
 bw_processing/constants.py
@@ -54,4 +34,17 @@ bw_processing/examples/datapackage_1/sa-data-vector-from-dict.indices.parquet
 bw_processing/examples/datapackage_1/some name.data.parquet
 bw_processing/examples/datapackage_1/some name.flip.parquet
 bw_processing/examples/datapackage_1/some name.indices.parquet
+tests/test_array_creation.py
+tests/test_datapackage.py
+tests/test_filesystem.py
+tests/test_filtered_datapackage.py
+tests/test_indexing.py
+tests/test_integration.py
+tests/test_interfaces.py
+tests/test_io_parquet_helpers.py
+tests/test_io_pyarrow_helpers.py
+tests/test_loading.py
+tests/test_merging.py
+tests/test_proxies.py
+tests/test_unique_fields.py
 tests/test_utils.py

bw_processing-1.0/tests/test_array_creation.py

@@ -0,0 +1,14 @@
+from bw_processing.array_creation import chunked
+
+
+def test_chunked():
+    c = chunked(range(600), 250)
+    for x in next(c):
+        pass
+    assert x == 249
+    for x in next(c):
+        pass
+    assert x == 499
+    for x in next(c):
+        pass
+    assert x == 599

bw_processing-1.0/tests/test_datapackage.py

@@ -0,0 +1,499 @@
+import shutil
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+from fsspec.implementations.zip import ZipFileSystem
+from morefs.dict import DictFS
+
+from bw_processing import create_datapackage, load_datapackage, simple_graph
+from bw_processing.constants import INDICES_DTYPE, UNCERTAINTY_DTYPE, MAX_SIGNED_64BIT_INT, MAX_SIGNED_32BIT_INT
+from bw_processing.errors import NonUnique, PotentialInconsistency, ShapeMismatch, WrongDatatype
+from bw_processing.io_helpers import generic_directory_filesystem
+
+dirpath = Path(__file__).parent.resolve() / "fixtures"
+
+
+class Dummy:
+    pass
+
+
+def add_data(dp):
+    data_array = np.array([2, 7, 12])
+    indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=INDICES_DTYPE)
+    flip_array = np.array([1, 0, 0], dtype=bool)
+    dp.add_persistent_vector(
+        matrix="sa_matrix",
+        data_array=data_array,
+        name="sa-data-vector",
+        indices_array=indices_array,
+        nrows=2,
+        flip_array=flip_array,
+    )
+
+    json_data = [{"a": "b"}, 1, True]
+    df = pd.DataFrame(
+        [
+            {"id": 1, "a": 1, "c": 3, "d": 11},
+            {"id": 2, "a": 2, "c": 4, "d": 11},
+            {"id": 3, "a": 1, "c": 4, "d": 11},
+        ]
+    ).set_index(["id"])
+
+    dp.add_persistent_array(
+        matrix="sa_matrix",
+        data_array=np.arange(12).reshape((3, 4)),
+        indices_array=indices_array,
+        name="sa-data-array",
+        flip_array=flip_array,
+    )
+
+    dp.add_dynamic_vector(
+        interface=Dummy(),
+        indices_array=indices_array,
+        matrix="sa_matrix",
+        name="sa-vector-interface",
+    )
+
+    dp.add_dynamic_array(
+        interface=Dummy(),
+        matrix="sa_matrix",
+        name="sa-array-interface",
+        indices_array=indices_array,
+    )
+    dp.add_csv_metadata(
+        dataframe=df,
+        valid_for=[("sa-data-vector", "rows")],
+        name="sa-data-vector-csv-metadata",
+    )
+    dp.add_json_metadata(
+        data=json_data, valid_for="sa-data-array", name="sa-data-array-json-metadata"
+    )
+
+
+def copy_fixture(fixture_name, dest):
+    source = Path(__file__).parent.resolve() / "fixtures" / fixture_name
+    for fp in source.iterdir():
+        shutil.copy(fp, dest / fp.name)
+
+
+def test_group_ordering_consistent():
+    dp = load_datapackage(ZipFileSystem(dirpath / "test-fixture.zip"))
+    assert list(dp.groups) == [
+        "sa-data-vector-from-dict",
+        "sa-data-vector",
+        "sa-data-array",
+        "sa-vector-interface",
+        "sa-array-interface",
+    ]
+
+
+def test_get_max_index_value():
+    dp = load_datapackage(ZipFileSystem(dirpath / "test-fixture.zip"))
+    assert dp.metadata.get("64_bit_indices")
+    assert dp.get_max_index_value() == MAX_SIGNED_64BIT_INT
+
+    dp = load_datapackage(ZipFileSystem(dirpath / "test-fixture-32bit.zip"))
+    assert not dp.metadata.get("64_bit_indices")
+    assert dp.get_max_index_value() == MAX_SIGNED_32BIT_INT
+
+
+def test_add_resource_with_same_name():
+    dp = create_datapackage()
+    add_data(dp)
+
+    data_array = np.array([2, 7, 12])
+    indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=INDICES_DTYPE)
+    with pytest.raises(NonUnique):
+        dp.add_persistent_vector(
+            matrix="sa_matrix",
+            data_array=data_array,
+            name="sa-data-vector",
+            indices_array=indices_array,
+        )
+
+
+def test_save_modifications(tmp_path):
+    copy_fixture("tfd", tmp_path)
+    dp = load_datapackage(generic_directory_filesystem(dirpath=tmp_path))
+
+    assert dp.resources[1]["name"] == "sa-data-vector-from-dict.data"
+    assert np.allclose(dp.data[1], [3.3, 8.3])
+
+    dp.data[1][:] = 42
+    dp._modified = [1]
+    dp.write_modified()
+
+    assert np.allclose(dp.data[1], 42)
+    assert not dp._modified
+
+    dp = load_datapackage(generic_directory_filesystem(dirpath=tmp_path))
+    assert np.allclose(dp.data[1], 42)
+
+
+def test_del_resource_filesystem(tmp_path):
+    copy_fixture("tfd", tmp_path)
+    dp = load_datapackage(generic_directory_filesystem(dirpath=tmp_path))
+    reference_length = len(dp)
+    assert "sa-vector-interface.indices.npy" in [o.name for o in tmp_path.iterdir()]
+    dp.del_resource("sa-vector-interface.indices")
+    assert "sa-vector-interface.indices.npy" not in [o.name for o in tmp_path.iterdir()]
+    assert len(dp) == reference_length - 1
+    assert len(dp.data) == reference_length - 1
+    assert len(dp.metadata["resources"]) == reference_length - 1
+    assert len(dp.resources) == reference_length - 1
+
+
+def test_del_resource_in_memory():
+    dp = create_datapackage()
+    add_data(dp)
+    assert isinstance(dp.fs, DictFS)
+
+    reference_length = len(dp)
+    assert "sa-vector-interface.indices" in [o["name"] for o in dp.resources]
+    dp.del_resource("sa-vector-interface.indices")
+    assert "sa-vector-interface.indices" not in [o["name"] for o in dp.resources]
+    assert len(dp) == reference_length - 1
+    assert len(dp.data) == reference_length - 1
+    assert len(dp.metadata["resources"]) == reference_length - 1
+    assert len(dp.resources) == reference_length - 1
+
+
+def test_del_resource_error_modifications(tmp_path):
+    copy_fixture("tfd", tmp_path)
+    dp = load_datapackage(generic_directory_filesystem(dirpath=tmp_path))
+    dp._modified = [1]
+    with pytest.raises(PotentialInconsistency):
+        dp.del_resource(1)
+
+
+def test_del_resource_group_filesystem(tmp_path):
+    copy_fixture("tfd", tmp_path)
+    dp = load_datapackage(generic_directory_filesystem(dirpath=tmp_path))
+
+    reference_length = len(dp)
+    assert "sa-data-vector.indices.npy" in [o.name for o in tmp_path.iterdir()]
+    dp.del_resource_group("sa-data-vector")
+    assert "sa-data-vector.indices.npy" not in [o.name for o in tmp_path.iterdir()]
+    assert len(dp) == reference_length - 3
+    assert len(dp.data) == reference_length - 3
+    assert len(dp.metadata["resources"]) == reference_length - 3
+    assert len(dp.resources) == reference_length - 3
+
+
+def test_del_resource_group_in_memory():
+    dp = create_datapackage()
+    add_data(dp)
+    assert isinstance(dp.fs, DictFS)
+
+    reference_length = len(dp)
+    assert "sa-data-vector.indices" in [o["name"] for o in dp.resources]
+    dp.del_resource_group("sa-data-vector")
+    assert "sa-data-vector.indices" not in [o["name"] for o in dp.resources]
+    assert len(dp) == reference_length - 3
+    assert len(dp.data) == reference_length - 3
+    assert len(dp.metadata["resources"]) == reference_length - 3
+    assert len(dp.resources) == reference_length - 3
+
+
+def test_del_resource_group_error_modifications(tmp_path):
+    copy_fixture("tfd", tmp_path)
+    dp = load_datapackage(generic_directory_filesystem(dirpath=tmp_path))
+    dp._modified = [1]
+    with pytest.raises(PotentialInconsistency):
+        dp.del_resource_group("sa-vector-interface")
+
+
+def test_exclude_basic():
+    dp = create_datapackage()
+    add_data(dp)
+    assert isinstance(dp.fs, DictFS)
+
+    reference_length = len(dp)
+    assert "sa-data-vector.indices" in [o["name"] for o in dp.resources]
+    ndp = dp.exclude({"group": "sa-data-vector"})
+    assert ndp is not dp
+    assert "sa-data-vector.indices" in [o["name"] for o in dp.resources]
+    assert "sa-data-vector.indices" not in [o["name"] for o in ndp.resources]
+    assert len(ndp) == reference_length - 3
+    assert len(ndp.data) == reference_length - 3
+    assert len(ndp.metadata["resources"]) == reference_length - 3
+    assert len(ndp.resources) == reference_length - 3
+
+
+def test_exclude_no_match():
+    dp = create_datapackage()
+    add_data(dp)
+    assert isinstance(dp.fs, DictFS)
+
+    reference_length = len(dp)
+    assert "sa-data-vector.indices" in [o["name"] for o in dp.resources]
+    ndp = dp.exclude({"foo": "bar"})
+    assert ndp is not dp
+    assert "sa-data-vector.indices" in [o["name"] for o in dp.resources]
+    assert "sa-data-vector.indices" in [o["name"] for o in ndp.resources]
+    assert len(ndp) == reference_length
+
+
+def test_exclude_multiple_filters():
+    dp = create_datapackage()
+    add_data(dp)
+    assert isinstance(dp.fs, DictFS)
+
+    reference_length = len(dp)
+    assert "sa-array-interface.indices" in [o["name"] for o in dp.resources]
+    ndp = dp.exclude({"group": "sa-array-interface", "matrix": "sa_matrix"})
+    assert ndp is not dp
+    assert "sa-array-interface.indices" in [o["name"] for o in dp.resources]
+    assert "sa-array-interface.indices" not in [o["name"] for o in ndp.resources]
+    assert len(ndp) == reference_length - 2
+    assert len(ndp.data) == reference_length - 2
+    assert len(ndp.metadata["resources"]) == reference_length - 2
+    assert len(ndp.resources) == reference_length - 2
+
+
+def test_exclude_multiple_matrix():
+    dp = create_datapackage()
+    add_data(dp)
+    assert isinstance(dp.fs, DictFS)
+
+    assert "sa-data-vector.indices" in [o["name"] for o in dp.resources]
+    ndp = dp.exclude({"matrix": "sa_matrix"})
+    assert ndp is not dp
+    assert "sa-data-vector.indices" in [o["name"] for o in dp.resources]
+    assert "sa-data-vector.indices" not in [o["name"] for o in ndp.resources]
+    assert len(ndp) == 2
+    assert len(ndp.data) == 2
+    assert len(ndp.metadata["resources"]) == 2
+    assert len(ndp.resources) == 2
+
+
+def test_add_persistent_vector_data_shapemismatch_ndimensions():
+    dp = create_datapackage()
+    data_array = np.array([[2, 7, 12], [4, 5, 15]])
+    indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=INDICES_DTYPE)
+    with pytest.raises(ShapeMismatch):
+        dp.add_persistent_vector(
+            matrix="sa_matrix",
+            data_array=data_array,
+            name="sa-data-vector",
+            indices_array=indices_array,
+        )
+
+
+def test_add_persistent_vector_data_shapemismatch_nrows():
+    dp = create_datapackage()
+    data_array = np.array([2, 7, 12, 15])
+    indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=INDICES_DTYPE)
+    with pytest.raises(ShapeMismatch):
+        dp.add_persistent_vector(
+            matrix="sa_matrix",
+            data_array=data_array,
+            name="sa-data-vector",
+            indices_array=indices_array,
+        )
+
+
+def test_add_persistent_vector_distributions_shapemismatch():
+    dp = create_datapackage()
+    distributions_array = np.array(
+        [
+            (3, 1.3, 2.5, np.nan, np.nan, np.nan, False),
+            (0, 1.3, 2.5, np.nan, np.nan, np.nan, False),
+        ],
+        dtype=UNCERTAINTY_DTYPE,
+    )
+    indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=INDICES_DTYPE)
+    with pytest.raises(ShapeMismatch):
+        dp.add_persistent_vector(
+            matrix="sa_matrix",
+            distributions_array=distributions_array,
+            name="sa-data-vector",
+            indices_array=indices_array,
+        )
+
+
+def test_add_persistent_vector_flip_dtype():
+    dp = create_datapackage()
+    data_array = np.array([2, 7, 12])
+    flip_array = np.array([0, 1, 0])
+    indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=INDICES_DTYPE)
+    with pytest.raises(WrongDatatype):
+        dp.add_persistent_vector(
+            matrix="sa_matrix",
+            data_array=data_array,
+            name="sa-data-vector",
+            flip_array=flip_array,
+            indices_array=indices_array,
+        )
+
+
+def test_add_persistent_vector_flip_shapemistmatch():
+    dp = create_datapackage()
+    data_array = np.array([2, 7, 12])
+    flip_array = np.array([0, 1, 0, 1], dtype=bool)
+    indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=ShapeMismatch)
+    with pytest.raises(ShapeMismatch):
+        dp.add_persistent_vector(
+            matrix="sa_matrix",
+            data_array=data_array,
+            name="sa-data-vector",
+            flip_array=flip_array,
+            indices_array=indices_array,
+        )
+
+
+def test_add_persistent_array_data_shapemismatch_ndimensions():
+    dp = create_datapackage()
+    data_array = np.array([2, 7, 12])
+    indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=INDICES_DTYPE)
+    with pytest.raises(ShapeMismatch):
+        dp.add_persistent_array(
+            matrix="sa_matrix",
+            data_array=data_array,
+            name="sa-data-vector",
+            indices_array=indices_array,
+        )
+
+
+def test_add_persistent_array_data_shapemismatch_nrows():
+    dp = create_datapackage()
+    data_array = np.arange(12).reshape(4, 3)
+    indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=INDICES_DTYPE)
+    with pytest.raises(ShapeMismatch):
+        dp.add_persistent_array(
+            matrix="sa_matrix",
+            data_array=data_array,
+            name="sa-data-vector",
+            indices_array=indices_array,
+        )
+
+
+def test_add_persistent_array_flip_dtype():
+    dp = create_datapackage()
+    data_array = np.arange(12).reshape(3, 4)
+    flip_array = np.array([0, 1, 0])
+    indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=INDICES_DTYPE)
+    with pytest.raises(WrongDatatype):
+        dp.add_persistent_array(
+            matrix="sa_matrix",
+            data_array=data_array,
+            name="sa-data-vector",
+            flip_array=flip_array,
+            indices_array=indices_array,
+        )
+
+
+def test_add_persistent_array_flip_shapemistmatch():
+    dp = create_datapackage()
+    data_array = np.arange(12).reshape(3, 4)
+    flip_array = np.array([0, 1, 0, 1], dtype=bool)
+    indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=ShapeMismatch)
+    with pytest.raises(ShapeMismatch):
+        dp.add_persistent_array(
+            matrix="sa_matrix",
+            data_array=data_array,
+            name="sa-data-vector",
+            flip_array=flip_array,
+            indices_array=indices_array,
+        )
+
+
+def test_add_dynamic_array_flip_dtype():
+    dp = create_datapackage()
+    flip_array = np.array([0, 1, 0])
+    indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=INDICES_DTYPE)
+    with pytest.raises(WrongDatatype):
+        dp.add_dynamic_array(
+            matrix="sa_matrix",
+            interface=Dummy(),
+            name="sa-data-vector",
+            flip_array=flip_array,
+            indices_array=indices_array,
+        )
+
+
+def test_add_dynamic_array_flip_shapemistmatch():
+    dp = create_datapackage()
+    flip_array = np.array([0, 1, 0, 1], dtype=bool)
+    indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=ShapeMismatch)
+    with pytest.raises(ShapeMismatch):
+        dp.add_dynamic_array(
+            matrix="sa_matrix",
+            interface=Dummy(),
+            name="sa-data-vector",
+            flip_array=flip_array,
+            indices_array=indices_array,
+        )
+
+
+def test_add_dynamic_vector_flip_dtype():
+    dp = create_datapackage()
+    flip_array = np.array([0, 1, 0])
+    indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=INDICES_DTYPE)
+    with pytest.raises(WrongDatatype):
+        dp.add_dynamic_vector(
+            matrix="sa_matrix",
+            interface=Dummy(),
+            name="sa-data-vector",
+            flip_array=flip_array,
+            indices_array=indices_array,
+        )
+
+
+def test_add_dynamic_vector_flip_shapemistmatch():
+    dp = create_datapackage()
+    flip_array = np.array([0, 1, 0, 1], dtype=bool)
+    indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=ShapeMismatch)
+    with pytest.raises(ShapeMismatch):
+        dp.add_dynamic_vector(
+            matrix="sa_matrix",
+            interface=Dummy(),
+            name="sa-data-vector",
+            flip_array=flip_array,
+            indices_array=indices_array,
+        )
+
+
+def test_simple_graph():
+    data = {
+        "some_matrix": [(1, 2, 3.14), (4, 5, 17, True), (8, 9, 11.11, False)],
+        "a different life": [
+            (99, 100, 101.234, False, "ignored"),
+            (1001, 1002, 777, 1),
+        ],
+    }
+    expected_resource = {
+        "category": "vector",
+        "profile": "data-resource",
+        "format": "npy",
+        "mediatype": "application/octet-stream",
+        "name": "some_matrix-data.flip",
+        "matrix": "some_matrix",
+        "kind": "flip",
+        "nrows": 3,
+        "path": "some_matrix-data.flip.npy",
+        "group": "some_matrix-data",
+    }
+    dp = simple_graph(data, name="foo", sequential=True)
+    assert dp.metadata["name"] == "foo"
+    assert dp.metadata["sequential"]
+    assert len(dp.metadata["resources"]) == 6
+    assert expected_resource in dp.metadata["resources"]
+
+    d, _ = dp.get_resource("some_matrix-data.flip")
+    assert np.allclose(d, np.array([False, True, False]))
+    d, _ = dp.get_resource("some_matrix-data.indices")
+    assert d["row"].sum() == 13
+    assert d["col"].sum() == 16
+    d, _ = dp.get_resource("some_matrix-data.data")
+    assert np.allclose(d, np.array([3.14, 17, 11.11]))
+
+    d, _ = dp.get_resource("a different life-data.flip")
+    assert np.allclose(d, np.array([False, True]))
+    d, _ = dp.get_resource("a different life-data.indices")
+    assert d["row"].sum() == 1100
+    assert d["col"].sum() == 1102
+    d, _ = dp.get_resource("a different life-data.data")
+    assert np.allclose(d, np.array([101.234, 777]))

bw_processing-1.0/tests/test_filesystem.py

@@ -0,0 +1,33 @@
+import platform
+from pathlib import Path
+
+import pytest
+
+from bw_processing.filesystem import clean_datapackage_name, md5, safe_filename
+
+fixtures_dir = Path(__file__, "..").resolve() / "fixtures"
+_windows = platform.system() == "Windows"
+
+
+@pytest.mark.skipif(_windows, reason="Different line encodings on Windows")
+def test_md5_text():
+    assert md5(fixtures_dir / "lorem.txt") == "edc715389af2498a623134608ba0a55b"
+
+
+def test_md5_binary():
+    assert md5(fixtures_dir / "array.npy") == "bbadddf09cf6b1e36d8333f474e36cee"
+
+
+def test_safe_filename():
+    assert safe_filename("Wave your hand yeah 🙋!") == "Wave-your-hand-yeah.f7952a3d"
+    assert (
+        safe_filename("Wave your hand yeah 🙋!", full=True)
+        == "Wave-your-hand-yeah.f7952a3d4b0534cdac0e0cbbf66aac73"
+    )
+    assert safe_filename("Wave your hand yeah 🙋!", add_hash=False) == "Wave-your-hand-yeah"
+
+
+def test_clean_datapackage_name():
+    a = "('IPCC', 'simple') processed arrays"
+    b = "IPCC_simple_processed_arrays"
+    assert clean_datapackage_name(a) == b