klongpy 0.6.8__py3-none-any.whl → 0.7.0__py3-none-any.whl

tests/perf_sys_fn_db.py

@@ -1,263 +0,0 @@
-import time
-import unittest
-
-from utils import *
-
-from klongpy import KlongInterpreter
-
-
-def run_lines(s, klong=None):
-    klong = klong or KlongInterpreter()
-    for line in s.splitlines():
-        r = klong(line)
-    return r
-
-
-def time_sql(klong, sql):
-    start_t = time.time()
-    try:
-        return klong(sql)
-    finally:
-        print(sql, " : ", time.time() - start_t)
-
-
-def perf_single_col_no_pre_alloc():
-    s = """
-.py("klongpy.db")
-a::[]
-
-e::[]
-e::e,,"a",,a
-t::.table(e)
-
-q:::{}
-q,"T",,t
-db::.db(q)
-"""
-    klong = KlongInterpreter()
-    klong(s)
-    klong("tfn::{[t0];t0::.pc();x@[];.pc()-t0}")
-    klong("afn::{{.insert(t; x)}'!10000}")
-    r = klong("tfn(afn)")
-    pr = r / 10000
-    print("single col (no index) (no pre alloc)", r, pr, int(1/pr))
-    r = time_sql(klong, 'db("select count(*) from T")')
-    assert r == 10000
-
-
-def perf_single_index_no_pre_alloc():
-    s = """
-.py("klongpy.db")
-a::[]
-
-e::[]
-e::e,,"a",,a
-t::.table(e)
-
-.index(t;["a"])
-
-q:::{}
-q,"T",,t
-db::.db(q)
-"""
-    klong = KlongInterpreter()
-    klong(s)
-    klong("tfn::{[t0];t0::.pc();x@[];.pc()-t0}")
-    klong("afn::{{.insert(t; x)}'!10000}")
-    r = klong("tfn(afn)")
-    pr = r / 10000
-    print("single col (index) (no pre alloc)", r, pr, int(1/pr))
-    r = time_sql(klong, 'db("select count(*) from T")')
-    assert r == 10000
-
-
-def perf_single_col_index_pre_alloc():
-    s = """
-.py("klongpy.db")
-a::!10000
-
-e::[]
-e::e,,"a",,a
-t::.table(e)
-
-.index(t;["a"])
-
-q:::{}
-q,"T",,t
-db::.db(q)
-"""
-    klong = KlongInterpreter()
-    klong(s)
-    klong("tfn::{[t0];t0::.pc();x@[];.pc()-t0}")
-    klong("afn::{{.insert(t; x)}'!10000}")
-    r = klong("tfn(afn)")
-    pr = r / 10000
-    print("single col (index) (pre alloc)", r, pr, int(1/pr))
-    r = time_sql(klong, 'db("select count(*) from T")')
-    assert r == 10000
-
-
-def perf_multi_col():
-    s = """
-.py("klongpy.db")
-a::[]
-b::[]
-c::[]
-
-e::[]
-e::e,,"a",,a
-e::e,,"b",,b
-e::e,,"c",,c
-t::.table(e)
-
-q:::{}
-q,"T",,t
-db::.db(q)
-"""
-    klong = KlongInterpreter()
-    klong(s)
-    klong("tfn::{[t0];t0::.pc();x@[];.pc()-t0}")
-    klong("afn::{{.insert(t; x,x,x)}'!10000}")
-    r = klong("tfn(afn)")
-    pr = r / 10000
-    print("multi col (no index) (no pre alloc)", r, pr, int(1/pr))
-    r = time_sql(klong, 'db("select count(*) from T")')
-    assert r == 10000
-
-
-def perf_multi_col_single_index():
-    s = """
-.py("klongpy.db")
-a::[]
-b::[]
-c::[]
-
-e::[]
-e::e,,"a",,a
-e::e,,"b",,b
-e::e,,"c",,c
-t::.table(e)
-
-.index(t;["a"])
-
-q:::{}
-q,"T",,t
-db::.db(q)
-"""
-    klong = KlongInterpreter()
-    klong(s)
-    klong("tfn::{[t0];t0::.pc();x@[];.pc()-t0}")
-    klong("afn::{{.insert(t; x,x,x)}'!10000}")
-    klong("bfn::{{.insert(t; x,x,x)}'10000+!10}")
-    r = klong("tfn(afn)")
-    pr = r / 10000
-    print("multi col (single index) (no pre alloc)", r, pr, int(1/pr))
-    r = time_sql(klong, 'db("select count(*) from T")')
-    assert r == 10000
-    r = klong("tfn(bfn)")
-    pr = r / 10
-    print("multi col (single index) (no pre alloc) [10]", r, pr, int(1/pr))
-    r = time_sql(klong, 'db("select count(*) from T")')
-    assert r == 10010
-
-
-
-def perf_multi_col_single_index_pre_alloc():
-    s = """
-.py("klongpy.db")
-a::!10000
-b::!10000
-c::!10000
-
-e::[]
-e::e,,"a",,a
-e::e,,"b",,b
-e::e,,"c",,c
-t::.table(e)
-
-.index(t;["a"])
-
-q:::{}
-q,"T",,t
-db::.db(q)
-"""
-    klong = KlongInterpreter()
-    klong(s)
-    klong("tfn::{[t0];t0::.pc();x@[];.pc()-t0}")
-    klong("afn::{{.insert(t; x,x,x)}'!10000}")
-    r = klong("tfn(afn)")
-    pr = r / 10000
-    print("multi col (single index) (pre alloc)", r, pr, int(1/pr))
-    r = time_sql(klong, 'db("select count(*) from T")')
-    assert r == 10000
-
-
-def perf_multi_col_multi_index():
-    s = """
-.py("klongpy.db")
-a::[]
-b::[]
-c::[]
-
-e::[]
-e::e,,"a",,a
-e::e,,"b",,b
-e::e,,"c",,c
-t::.table(e)
-
-.index(t;["a" "b"])
-
-q:::{}
-q,"T",,t
-db::.db(q)
-"""
-    klong = KlongInterpreter()
-    klong(s)
-    klong("tfn::{[t0];t0::.pc();x@[];.pc()-t0}")
-    klong("afn::{{.insert(t; x,x,x)}'!10000}")
-    r = klong("tfn(afn)")
-    pr = r / 10000
-    print("multi col (multi index) (no pre alloc)", r, pr, int(1/pr))
-    r = time_sql(klong, 'db("select count(*) from T")')
-    assert r == 10000
-
-
-def perf_multi_col_multi_index_pre_alloc():
-    s = """
-.py("klongpy.db")
-a::!10000
-b::!10000
-c::!10000
-
-e::[]
-e::e,,"a",,a
-e::e,,"b",,b
-e::e,,"c",,c
-t::.table(e)
-
-.index(t;["a" "b"])
-
-q:::{}
-q,"T",,t
-db::.db(q)
-"""
-    klong = KlongInterpreter()
-    klong(s)
-    klong("tfn::{[t0];t0::.pc();x@[];.pc()-t0}")
-    klong("afn::{{.insert(t; x,x,x)}'!10000}")
-    r = klong("tfn(afn)")
-    pr = r / 10000
-    print("multi col (multi index) (pre alloc)", r, pr, int(1/pr))
-    r = time_sql(klong, 'db("select count(*) from T")')
-    assert r == 10000
-
-
-if __name__ == "__main__":
-    perf_single_col_no_pre_alloc()
-    perf_single_index_no_pre_alloc()
-    perf_single_col_index_pre_alloc()
-    perf_multi_col()
-    perf_multi_col_single_index()
-    perf_multi_col_single_index_pre_alloc()
-    perf_multi_col_multi_index()
-    perf_multi_col_multi_index_pre_alloc()

tests/perf_vector.py

@@ -1,40 +0,0 @@
-import os
-import timeit
-
-import numpy as np
-
-from klongpy import KlongInterpreter
-
-
-def numpy_vec(number=100):
-    r = timeit.timeit(lambda: np.multiply(np.add(np.arange(10000000), 1), 2), number=number)
-    return r/number
-
-
-def klong_vec(number=100):
-    klong = KlongInterpreter()
-    r = timeit.timeit(lambda: klong("2*1+!10000000"), number=number)
-    return r/number
-
-
-def python_vec(number=100):
-    r = timeit.timeit(lambda: [2 * (1 + x) for x in range(10000000)], number=number)
-    return r/number
-
-
-if __name__ == "__main__":
-    number = 1000
-    print("Python: ", end='')
-    pr = python_vec(number=number)
-    print(f"{round(pr,6)}s")
-
-    print(f"KlongPy USE_GPU={os.environ.get('USE_GPU')}: ", end='')
-    kr = klong_vec(number=number)
-    print(f"{round(kr,6)}s")
-
-    print("Numpy: ", end='')
-    nr = numpy_vec(number=number)
-    print(f"{round(nr,6)}s")
-
-    print(f"Python / KlongPy => {round(pr/kr,6)}")
-    print(f"Numpy / KlongPy => {round(nr/kr,6)}")

tests/test_accel.py

@@ -1,227 +0,0 @@
-import unittest
-from klongpy import KlongInterpreter
-from utils import *
-from klongpy.backend import np
-import numpy
-
-
-class Executed:
-    def __init__(self, fn):
-        self.fn = fn
-        self.executed = False
-
-    def __call__(self, *args, **kwargs):
-        self.executed = True
-        return self.fn(*args, **kwargs)
-
-
-class ExecutedReduce:
-    def __init__(self, fn):
-        self.fn = fn
-        self.executed = False
-
-    def reduce(self, *args, **kwargs):
-        self.executed = True
-        return self.fn.reduce(*args, **kwargs)
-
-
-def get_rnd_nested_array():
-    return np.asarray([np.random.rand(100), np.random.rand(100)])
-
-
-def get_rnd_array():
-    return np.random.rand(100)
-
-
-def get_reduce_data(data):
-    return data if isinstance(data, numpy.ndarray) else data.get()
-
-
-# This approach isn't great because any usage of the thunked method will pass
-# We need a way to intercept the real call
-class TestAccelerate(unittest.TestCase):
-    """
-    Verify that we are actually running the adverb_over accelerated paths for cases that we can.
-    """
-
-    # TODO: this is not parallel test safe
-    #       add ability to intercept calls in interpeter
-    def test_over_add_nested_array(self):
-        klong = KlongInterpreter()
-        e = ExecutedReduce(np.add)
-        data = get_rnd_nested_array()
-        try:
-            np.add = e
-            klong['data'] = data
-            r = klong('+/data')
-        finally:
-            np.add = e.fn
-        self.assertTrue(kg_equal(r, numpy.add.reduce(get_reduce_data(data))))
-        self.assertTrue(e.executed)
-
-    def test_over_add_array(self):
-        klong = KlongInterpreter()
-        e = ExecutedReduce(np.add)
-        data = get_rnd_array()
-        try:
-            np.add = e
-            klong['data'] = data
-            r = klong('+/data')
-        finally:
-            np.add = e.fn
-        self.assertTrue(np.isclose(r, numpy.add.reduce(get_reduce_data(data))))
-        self.assertTrue(e.executed)
-
-    ####### Subtract
-
-    def test_over_subtract_nested_array(self):
-        klong = KlongInterpreter()
-        e = ExecutedReduce(np.subtract)
-        data = get_rnd_nested_array()
-        try:
-            np.subtract = e
-            klong['data'] = data
-            r = klong('-/data')
-        finally:
-            np.subtract = e.fn
-        self.assertTrue(kg_equal(r, numpy.subtract.reduce(get_reduce_data(data))))
-        self.assertTrue(e.executed)
-
-    def test_over_subtract(self):
-        klong = KlongInterpreter()
-        e = ExecutedReduce(np.subtract)
-        data = get_rnd_array()
-        try:
-            np.subtract = e
-            klong['data'] = data
-            r = klong('-/data')
-        finally:
-            np.subtract = e.fn
-        self.assertTrue(numpy.isclose(r, numpy.subtract.reduce(get_reduce_data(data))))
-        self.assertTrue(e.executed)
-
-    ####### Multiply
-
-    def test_over_multiply_nested_array(self):
-        klong = KlongInterpreter()
-        e = ExecutedReduce(np.multiply)
-        data = get_rnd_nested_array()
-        try:
-            np.multiply = e
-            klong['data'] = data
-            r = klong('*/data')
-        finally:
-            np.multiply = e.fn
-        self.assertTrue(kg_equal(r, numpy.multiply.reduce(get_reduce_data(data))))
-        self.assertTrue(e.executed)
-
-    def test_over_multipy(self):
-        klong = KlongInterpreter()
-        e = ExecutedReduce(np.multiply)
-        data = get_rnd_array()
-        try:
-            np.multiply = e
-            klong['data'] = data
-            r = klong('*/data')
-        finally:
-            np.multiply = e.fn
-        self.assertTrue(numpy.isclose(r, numpy.multiply.reduce(get_reduce_data(data))))
-        self.assertTrue(e.executed)
-
-    ####### Divide
-
-    def test_over_divide_nested_array(self):
-        klong = KlongInterpreter()
-        e = ExecutedReduce(np.divide)
-        data = get_rnd_nested_array()
-        try:
-            np.divide = e
-            klong['data'] = data
-            r = klong('%/data')
-        finally:
-            np.divide = e.fn
-        self.assertTrue(kg_equal(r, numpy.divide.reduce(get_reduce_data(data))))
-        self.assertTrue(e.executed)
-
-    def test_over_divide(self):
-        if np != numpy:
-            return
-        if not hasattr(np.divide, "reduce"):
-            return
-        klong = KlongInterpreter()
-        e = ExecutedReduce(np.divide)
-        data = get_rnd_array()
-        try:
-            np.divide = e
-            klong['data'] = data
-            r = klong('%/data')
-        finally:
-            np.divide = e.fn
-        self.assertTrue(numpy.isclose(r, numpy.divide.reduce(get_reduce_data(data))))
-        self.assertTrue(e.executed)
-
-    ####### Min
-
-    def test_over_min_nested_arrays(self):
-        klong = KlongInterpreter()
-        e = Executed(np.min)
-        try:
-            np.min = e
-            r = klong('&/[[1 2 3] [4 5 6]]')
-        finally:
-            np.min = e.fn
-        self.assertTrue(kg_equal(r, [1,2,3]))
-        self.assertFalse(e.executed)
-
-    def test_over_min(self):
-        klong = KlongInterpreter()
-        e = Executed(np.min)
-        try:
-            np.min = e
-            r = klong('&/[1 2 3 4]')
-        finally:
-            np.min = e.fn
-        self.assertEqual(r, 1)
-        self.assertTrue(e.executed)
-
-    ####### Max
-
-    def test_over_max_nested_arrays(self):
-        klong = KlongInterpreter()
-        e = Executed(np.max)
-        try:
-            np.max = e
-            r = klong('|/[[1 2 3] [4 5 6]]')
-        finally:
-            np.max = e.fn
-        self.assertTrue(kg_equal(r, [4,5,6]))
-        self.assertFalse(e.executed)
-
-    def test_over_max(self):
-        klong = KlongInterpreter()
-        e = Executed(np.max)
-        try:
-            np.max = e
-            r = klong('|/[1 2 3 4]')
-        finally:
-            np.max = e.fn
-        self.assertEqual(r, 4)
-        self.assertTrue(e.executed)
-
-    ####### Join
-
-    @unittest.skip
-    def test_over_join(self):
-        if np != numpy:
-            return
-        klong = KlongInterpreter()
-        e = Executed(np.concatenate)
-        try:
-            np.concatenate = e
-            r = klong(',/:~[[1] [2] [3]]')
-        finally:
-            np.concatenate = e.fn
-        self.assertTrue(kg_equal(r, [1,2,3]))
-        self.assertTrue(e.executed)
-
-

tests/test_df_cache.py

@@ -1,85 +0,0 @@
-import os
-import platform
-import tempfile
-import threading
-import unittest
-
-import pandas as pd
-
-from klongpy.db.df_cache import PandasDataFrameCache
-
-# TODO: add MacOS RAM disk
-# hdiutil attach -nomount ram://$((2 * 1024 * 100))
-# diskutil eraseVolume HFS+ RAMDisk /dev/disk3
-# https://stackoverflow.com/questions/1854/how-to-identify-which-os-python-is-running-on
-tmp_dir = "/dev/shm" if platform.system() == "Linux" else None
-
-class TestPandasDataFrameCache(unittest.TestCase):
-    def setUp(self):
-        self.test_file_1 = tempfile.NamedTemporaryFile(delete=False, dir=tmp_dir)
-        self.df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]}, index=[1, 2, 3])
-        self.cache = PandasDataFrameCache(max_memory=1024)
-        self.cache.update(self.test_file_1.name, self.df)
-
-    def test_get_dataframe(self):
-        start = 2
-        end = 3
-        result = self.cache.get_dataframe(self.test_file_1.name, start, end)
-        expected = self.df.loc[start:end]
-        pd.testing.assert_frame_equal(result, expected)
-
-    def test_append(self):
-        new_df = pd.DataFrame({'A': [4, 5, 6], 'B': [7, 8, 9]}, index=[4, 5, 6])
-        result = self.cache.update(self.test_file_1.name, new_df)
-        expected = pd.concat([self.df, new_df]).sort_index().drop_duplicates()
-        pd.testing.assert_frame_equal(result, expected)
-
-    def test_file_not_found(self):
-        with self.assertRaises(FileNotFoundError):
-            self.cache.get_file('not_found.pickle')
-
-    def test_memory_error(self):
-        with self.assertRaises(MemoryError):
-            self.cache = PandasDataFrameCache(max_memory=1)
-            self.cache.update(self.test_file_1.name, self.df)
-
-    def tearDown(self):
-        os.unlink(self.test_file_1.name)
-
-
-class TestMultithreadedPandasDataFrameCache(unittest.TestCase):
-    def setUp(self):
-        self.test_file_1 = tempfile.NamedTemporaryFile(delete=False, dir=tmp_dir)
-        self.test_file_2 = tempfile.NamedTemporaryFile(delete=False, dir=tmp_dir)
-        self.cache = PandasDataFrameCache(max_memory=2**30)
-
-    def test_append_concurrently(self):
-        def run_append_1(cache, start_i):
-            values = list(range(start_i, start_i+100))
-            new_df = pd.DataFrame({'A': values, 'B': values}, index=values)
-            cache.update(self.test_file_1.name, new_df)
-        def run_append_2(cache, start_i):
-            values = list(range(start_i, start_i+100))
-            new_df = pd.DataFrame({'C': values, 'D': values}, index=values)
-            cache.update(self.test_file_2.name, new_df)
-
-        threads = []
-        for i in range(100):
-            thread = threading.Thread(target=run_append_1, args=(self.cache, i*100))
-            thread.start()
-            threads.append(thread)
-        for i in range(150):
-            thread = threading.Thread(target=run_append_2, args=(self.cache, i*100))
-            thread.start()
-            threads.append(thread)
-        for thread in threads:
-            thread.join()
-
-        result = self.cache.get_file(self.test_file_1.name)
-        self.assertEqual(len(result), 10000)
-        result = self.cache.get_file(self.test_file_2.name)
-        self.assertEqual(len(result), 15000)
-
-    def tearDown(self):
-        os.remove(self.test_file_1.name)
-        os.remove(self.test_file_2.name)

tests/test_examples.py

@@ -1,64 +0,0 @@
-import glob
-import math
-import os
-import unittest
-
-from utils import *
-from unittest.mock import MagicMock
-
-
-class TestExamples(unittest.TestCase):
-
-    def test_load_lib(self):
-        """
-        Verify we can at least load the library.
-        """
-        ran_nstat = False
-        try:
-            cwd = os.getcwd()
-            os.chdir(os.path.join("klongpy","lib"))
-            for x in glob.glob("./**/*.kg", recursive=True):
-                fname = os.path.basename(x)
-                print(f"loading {fname}")
-                klong = None
-                if fname != "help.kg":
-                    klong, _ = run_file("help.kg")
-                klong, _ = run_file(x, klong=klong)
-                if fname == "nstat.kg":
-                    ran_nstat = True
-                    r = klong("sqr2pi(2)")
-                    self.assertTrue(np.isclose(r, math.sqrt(math.pi*2)))
-        finally:
-            os.chdir(cwd)
-        self.assertTrue(ran_nstat)
-
-    @unittest.skip("TODO: need to mock out the web server")
-    def test_load_examples(self):
-        """
-        Verify we can at least load the example examples.
-        """
-        ran_dfs = False
-        try:
-            cwd = os.getcwd()
-            os.chdir("examples")
-            for x in glob.glob("./**/*.kg", recursive=True):
-                fname = os.path.basename(x)
-                if fname == "update_data.kg":
-                    # requires polygon module
-                    continue
-                print(f"loading example {fname}")
-                klong = KlongInterpreter()
-                klong['.system'] = {}
-                klong['.system']['ioloop'] = MagicMock()
-                klong['.system']['klongloop'] = MagicMock()
-                klong['.system']['closeEvent'] = MagicMock()
-                try:
-                    run_file(x, klong=klong)
-                except Exception as e:
-                    print(f"failed to load example {fname} with error {e}")
-                    raise e
-                if fname == "dfs.kg":
-                    ran_dfs = True
-        finally:
-            os.chdir(cwd)
-        self.assertTrue(ran_dfs)