torchness 2.1.0__tar.gz → 2.2.0__tar.gz

{torchness-2.1.0 → torchness-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: torchness
-Version: 2.1.0
+Version: 2.2.0
 Summary: PyTorch tools
 Home-page: https://github.com/piteren/torchness.git
 Author: Piotr Niewinski

{torchness-2.1.0 → torchness-2.2.0}/setup.py

@@ -5,7 +5,7 @@ from pypaq.lipytools.files import get_requirements
 
 setup(
     name=               'torchness',
-    version=            'v2.1.0',
+    version=            'v2.2.0',
     url=                'https://github.com/piteren/torchness.git',
     author=             'Piotr Niewinski',
     author_email=       'pioniewinski@gmail.com',

torchness-2.2.0/tests/test_batcher.py

@@ -0,0 +1,174 @@
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import pytest
+from ompr.runner import RunningWorker
+from pypaq.lipytools.files import prep_folder, w_pickle, r_pickle, get_files
+from pypaq.lipytools.stats import msmx
+
+from torchness.batcher import DataBatcher, FilesBatcher, FilesBatcherMP, BATCHING_TYPES
+
+TMP_DIR = Path(__file__).parent / '_tmp_batcher'
+DATA_DIR = TMP_DIR / 'datafiles'
+
+
+@pytest.fixture(autouse=True, scope='module')
+def tmp_dir():
+    prep_folder(TMP_DIR, flush_non_empty=True)
+
+
+def test_base_init():
+    data = {'input': np.random.rand(1000, 3)}
+    batcher = DataBatcher(data_TR=data)  # type: ignore
+    nTR, nTS = batcher.get_data_size()
+    assert (nTR, nTS) == (1000, 0)
+    assert batcher.keys == ['input']
+
+    batcher = DataBatcher(data_TR=data, split_factor=0.2)  # type: ignore
+    nTR, nTS = batcher.get_data_size()
+    assert (nTR, nTS) == (800, 200)
+
+    data_TS = {'input': np.random.rand(300, 3)}
+    batcher = DataBatcher(data_TR=data, data_TS=data_TS)  # type: ignore
+    nTR, nTS = batcher.get_data_size()
+    assert (nTR, nTS) == (1000, 300)
+
+
+def test_TS_batches():
+    data = {'input': np.random.rand(1000, 3)}
+    data_TS = {'input': np.random.rand(300, 3)}
+    batcher = DataBatcher(data_TR=data, data_TS=data_TS, batch_size=15, batch_size_TS_mul=2)  # type: ignore
+    batches_TS = batcher.get_TS_batches()
+    assert len(batches_TS) == 10
+
+    data = {'input': np.random.rand(1000, 3)}
+    data_TS_named = {
+        'test_A': {'input': np.random.rand(300, 3)},
+        'test_B': {'input': np.random.rand(200, 3)},
+    }
+    batcher = DataBatcher(data_TR=data, data_TS=data_TS_named, batch_size=10, batch_size_TS_mul=2)  # type: ignore
+    nTR, nTS = batcher.get_data_size()
+    assert (nTR, nTS) == (1000, 500)
+    batches_TS = batcher.get_TS_batches('test_B')
+    assert len(batches_TS) == 10
+
+
+def test_coverage(
+        num_samples: int = 1000,
+        batch_size: int = 64,
+        num_batches: int = 1000):
+
+    for btype in BATCHING_TYPES:
+        print(f'\nstarts coverage tests of {btype}')
+
+        samples = np.arange(num_samples)
+        np.random.shuffle(samples)
+        data = {'samples': samples}
+        batcher = DataBatcher(data_TR=data, batch_size=batch_size, batching_type=btype)
+
+        sL = []
+        n_b = 0
+        s_counter = {s: 0 for s in range(num_samples)}
+        for _ in range(num_batches):
+            sL += batcher.get_batch()['samples'].tolist()
+            n_b += 1
+            if len(set(sL)) == num_samples:
+                print(f'got full coverage with {n_b} batches')
+                for s in sL: s_counter[s] += 1
+                sL = []
+                n_b = 0
+
+        print(msmx(list(s_counter.values()))['string'])
+    print('*** finished coverage tests')
+
+
+def test_seed():
+    c_size = 1000
+    b_size = 64
+
+    samples = np.arange(c_size)
+    np.random.shuffle(samples)
+    data = {'samples': samples}
+
+    batcher = DataBatcher(data, batch_size=b_size, batching_type='random')
+    sA = []
+    while len(sA) < 10000:
+        sA += batcher.get_batch()['samples'].tolist()
+        np.random.seed(len(sA))
+
+    batcher = DataBatcher(data, batch_size=b_size, batching_type='random')
+    sB = []
+    while len(sB) < 10000:
+        sB += batcher.get_batch()['samples'].tolist()
+        np.random.seed(10000000 - len(sB))
+
+    seed_is_fixed = all(sA[ix] == sB[ix] for ix in range(len(sA)))
+    print(f'seed is fixed: {seed_is_fixed}!')
+    assert seed_is_fixed
+
+
+def _prepare_data_files(n_files: int = 10, nf_samples: int = 10_000):
+    prep_folder(DATA_DIR, flush_non_empty=True)
+    for n in range(n_files):
+        data = {
+            'x': np.random.rand(nf_samples, 1000),
+            'y': np.arange(nf_samples) + n * nf_samples}
+        w_pickle(data, f'{DATA_DIR}/f{n}.npp')
+
+
+def test_FilesBatcher():
+    n_files = 10
+    nf_samples = 10_000
+    batch_size = 1_000
+    n_epochs = 5
+
+    def chunk_builder(file: str):
+        return r_pickle(file)
+
+    print('Preparing data files for FilesBatcher ..')
+    _prepare_data_files(n_files, nf_samples)
+
+    fb = FilesBatcher(
+        data_TR_chunk_fp=   get_files(str(DATA_DIR)),
+        chunk_builder=      chunk_builder,
+        batch_size=         batch_size)
+
+    ys = []
+    for _ in range(int(n_files * nf_samples / batch_size * n_epochs)):
+        batch = fb.get_batch()
+        ys += batch['y'].tolist()
+
+    print(len(ys))
+    assert len(ys) == n_files * nf_samples * n_epochs
+    fb.exit()
+
+
+def test_FilesBatcherMP():
+    n_files = 10
+    nf_samples = 10_000
+    batch_size = 1_000
+    n_epochs = 5
+
+    class CB(RunningWorker):
+        def process(self, file) -> Any:
+            return r_pickle(file)
+
+    print('Preparing data files for FilesBatcherMP ..')
+    _prepare_data_files(n_files, nf_samples)
+
+    fb = FilesBatcherMP(
+        data_TR_chunk_fp=       get_files(str(DATA_DIR)),
+        data_TS_chunk_fp=       None,
+        chunk_processor_class=  CB,
+        n_workers=              10,
+        batch_size=             batch_size)
+
+    ys = []
+    for _ in range(int(n_files * nf_samples / batch_size * n_epochs)):
+        batch = fb.get_batch()
+        ys += batch['y'].tolist()
+
+    print(len(ys))
+    assert len(ys) == n_files * nf_samples * n_epochs
+    fb.exit()

torchness-2.2.0/tests/test_ckpt.py

@@ -0,0 +1,63 @@
+from pathlib import Path
+import pytest
+import torch
+from pypaq.lipytools.files import prep_folder
+
+from torchness.ckpt import mrg_ckpts, ckpt_nfo
+from torchness.layers import LayDense
+from torchness.motorch import Module, MOTorch
+
+TMP_DIR = Path(__file__).parent / '_tmp_ckpt'
+MOTORCH_DIR = TMP_DIR / 'motorch'
+
+
+@pytest.fixture(autouse=True)
+def tmp_dir():
+    prep_folder(MOTORCH_DIR, flush_non_empty=True)
+    MOTorch.SAVE_TOPDIR = str(MOTORCH_DIR)
+
+
+class LinModel(Module):
+
+    def __init__(
+            self,
+            in_drop: float,
+            in_shape=   784,
+            out_shape=  10,
+            loss_func=  torch.nn.functional.cross_entropy,
+            device=     None,
+            dtype=      None,
+            seed=       121,
+            **kwargs,
+    ):
+        Module.__init__(self, **kwargs)
+        self.in_drop_lay = torch.nn.Dropout(p=in_drop) if in_drop > 0 else None
+        self.lin = LayDense(in_features=in_shape, out_features=out_shape)
+        self.loss_func = loss_func
+
+    def forward(self, inp) -> dict:
+        if self.in_drop_lay is not None: inp = self.in_drop_lay(inp)
+        logits = self.lin(inp)
+        return {'logits': logits}
+
+    def loss(self, inp, lbl) -> dict:
+        out = self(inp)
+        out['loss'] = self.loss_func(out['logits'], lbl)
+        out['acc'] = self.accuracy(out['logits'], lbl)
+        return out
+
+
+def test_mrg_ckpts():
+    model = MOTorch(name='modA', module_type=LinModel, in_drop=0.1, device=None)
+    model.save()
+    model = MOTorch(name='modB', module_type=LinModel, in_drop=0.1, device=-1)
+    model.save()
+
+    mrg_ckpts(
+        ckptA=  f'{MOTORCH_DIR}/modA/modA.pt',
+        ckptB=  f'{MOTORCH_DIR}/modB/modB.pt',
+        ckptM=  f'{MOTORCH_DIR}/ckptM.pt',
+        ratio=  0.4,
+        noise=  0.1)
+
+    ckpt_nfo(f'{MOTORCH_DIR}/ckptM.pt')

torchness-2.2.0/tests/test_devices.py

@@ -0,0 +1,166 @@
+import pytest
+import torch
+
+from torchness.devices import get_cuda_mem, get_available_cuda_id, report_cuda, _get_devices_torchness, get_devices, mask_cuda, mask_cuda_devices
+
+
+def test_get_cuda_mem():
+    mem = get_cuda_mem()
+    print(mem)
+
+
+def test_get_available_cuda_id():
+    av_cuda = get_available_cuda_id()
+    print(av_cuda)
+    av_cuda = get_available_cuda_id(mem_free=16000)
+    print(av_cuda)
+    av_cuda = get_available_cuda_id(load_max=0.5)
+    print(av_cuda)
+
+
+def test_report_cuda():
+    print(report_cuda())
+
+
+def test_get_devices_torchness():
+
+    d = _get_devices_torchness(0)
+    print(d)
+    assert d == [0]
+
+    d = _get_devices_torchness(1)
+    print(d)
+    assert d == [1]
+
+    d = _get_devices_torchness(13)
+    print(d)
+    assert d == [13]
+
+    d = _get_devices_torchness(-1)
+    print(d)
+    assert type(d) is list and (not d or len(d) == 1 and ((type(d[0]) is int and d[0] >= 0) or d[0] is None))
+
+    d = _get_devices_torchness([])
+    print(d)
+    assert d == [] or list(set([type(e) for e in d]))[0] in [int, None]
+
+    d = _get_devices_torchness(None)
+    print(d)
+    assert d == [None]
+
+    d = _get_devices_torchness(0.7)
+    print(len(d), d)
+    assert type(d) is list and len(d) >= 1 and d[0] is None
+
+    d = _get_devices_torchness(-0.5)
+    print(d)
+    assert d == [None]
+
+    d = _get_devices_torchness('all')
+    print(len(d), d)
+    assert type(d) is list and len(d) >= 1 and d[0] is None
+
+    d = _get_devices_torchness('cuda')
+    print(d)
+    assert d == [0]
+
+    d = _get_devices_torchness('cuda:0')
+    print(d)
+    assert d == [0]
+
+    d = _get_devices_torchness('cuda:1')
+    print(d)
+    assert d == [1]
+
+    d = torch.device('cpu')
+    d = _get_devices_torchness(d)
+    print(d)
+    assert d == [None]
+
+    d = torch.device('cuda')
+    d = _get_devices_torchness(d)
+    print(d)
+    assert d == [0]
+
+    d = torch.device('cuda:1')
+    d = _get_devices_torchness(d)
+    print(d)
+    assert d == [1]
+
+    d = torch.device('cuda:8')
+    d = _get_devices_torchness(d)
+    print(d)
+    assert d == [8]
+
+    d = _get_devices_torchness([0, 'all'])
+    print(d)
+    assert None in d and 0 in d and len(d) > 1
+
+    d = _get_devices_torchness([0, [], None])
+    print(len(d), d)
+    assert None in d and 0 in d and len(d) > 1
+
+    d = _get_devices_torchness([0, 2, [], None, -1])
+    print(len(d), d)
+    assert None in d and 0 in d and -1 not in d and len(d) > 1
+
+    d = _get_devices_torchness([1, 2, [], None, -1, 'all', 'cuda:0'])
+    print(len(d), d)
+    assert None in d and 0 in d and -1 not in d and len(d) > 3
+
+
+def test_get_devices():
+    devices = get_devices()
+    print(devices)
+    devices = get_devices(devices=[])
+    print(devices)
+    devices = get_devices(devices=[], eventually_cpu=True)
+    print(devices)
+    devices = get_devices(devices=[0, 1])
+    print(devices)
+    devices = get_devices(devices=-1, eventually_cpu=True)
+    print(devices)
+    devices = get_devices(devices=[], mem_free=24000)
+    print(devices)
+
+
+def test_get_devices_exceptions():
+    with pytest.raises(Exception):
+        get_devices('alll')
+    with pytest.raises(Exception):
+        get_devices((0, 1))  # type: ignore
+
+
+def test_get_devices_torch():
+
+    d = get_devices(0)
+    print(d)
+    assert d == [] or 'cuda' in d[0]
+
+    d = get_devices(-1)
+    print(d)
+    assert d == [] or 'cuda' in d[0]
+
+    d = get_devices([])
+    print(d)
+    assert type(d) is list and (not d or type(d[0]) is str)
+
+    d = get_devices(None)
+    print(d)
+    assert d == ['cpu']
+
+    d = get_devices([0, 1, 'cuda:0', None])
+    print(d)
+    assert d == ['cuda:0', 'cuda:1', 'cuda:0', 'cpu']
+
+
+def test_mask_cuda():
+    av_cuda = get_available_cuda_id()
+    print(av_cuda)
+    mask_cuda(av_cuda)
+
+
+def test_mask_cuda_devices():
+    d = get_devices([None, 0, 0, 'cuda', None], torch_namespace=True)
+    print(d)
+    mask_cuda_devices(d)