remote-run-everything 2.0.2__py3-none-any.whl

remote_run_everything/nosql/no_sql.py

@@ -0,0 +1,73 @@
+import sqlite3, os, json
+
+from remote_run_everything.nosql.no_sql_tool import NosqlTool
+
+from remote_run_everything.db.crude_duck import CrudeDuck
+
+
+class Nosql:
+    def __init__(self, dbpath=None):
+        self.t = NosqlTool()
+        self.cd = CrudeDuck()
+        self.db_path = self.t.default_db_path(dbpath)
+        self.conn = self.cd.install_sql_ext(self.db_path)
+        self.conn.create_function("my_q", self.t.query)
+        self.table = None
+
+    def drop_db(self):
+        os.remove(self.db_path)
+
+    def drop_table(self):
+        assert self.table is not None
+        self.cd.drop_table(self.conn, self.table)
+        self.table=None
+
+    def __getitem__(self, table):
+        self.table = table
+        conn = sqlite3.connect(self.db_path, isolation_level=None)
+        conn.execute('pragma journal_mode=wal')
+        sql = f'''CREATE TABLE IF NOT EXISTS {table} (id  INTEGER PRIMARY KEY AUTOINCREMENT, data text)'''
+        conn.execute(sql)
+        return self
+
+    def insert_one(self, dic):
+        assert self.table is not None
+        value = json.dumps(dic)
+        sql=f"insert INTO {self.table} (data) VALUES ('{value}')"
+        self.conn.execute(sql)
+        self.conn.commit()
+
+    def find(self, query={}):
+        return self.t.find(self.conn, self.table, query)
+
+    def delete(self, query={}):
+        return self.t.delete(self.conn, self.table, query)
+
+    # 保证queery唯一,最后会隐含删除
+    def upsert_one(self, query, dic):
+        assert self.table is not None
+        l = self.find(query)
+        if len(l) == 0:
+            return self.insert_one(dic)
+        retain = l[0]
+        self.t.revise_one(self.conn, self.table, retain, dic)
+        rest_ids = [i['id'] for i in l if i['id'] != retain["id"]]
+        if len(rest_ids) > 0:
+            self.cd.delete_by_ids(self.conn, self.table, rest_ids)
+            self.conn.commit()
+
+
+if __name__ == '__main__':
+    db = Nosql()
+    db.drop_db()
+    t = "test"
+    col = db['test']
+    dic = {"a": 2, "b": 456, 'c': "adf", "d": "2020-02-02"}
+    col.insert_one(dic)
+    dic = {"a": 56, "b": 456, 'c': "adf", "d": "2020-07-02"}
+    col.insert_one(dic)
+    q = {"a": 2}
+    print(111, col.find(q))
+    db.upsert_one(q, {"b": 999})
+    q = {"a": 2}
+    print(222, col.find(q))

remote_run_everything/nosql/no_sql_mysql.py

@@ -0,0 +1,67 @@
+import json
+
+from remote_run_everything.nosql.no_sql_tool import NosqlTool
+from remote_run_everything.tools.common import Common
+
+from remote_run_everything.db.crude_duck import CrudeDuck
+
+
+class NosqlMysql:
+    def __init__(self, user, pwd, host, port, dbname):
+        self.t = NosqlTool()
+        self.cd = CrudeDuck()
+        self.conn = self.cd.install_mysql_ext(user, pwd, host, port, dbname)
+        self.conn.create_function("my_q", self.t.query)
+        self.table = None
+
+    def drop_table(self):
+        assert self.table is not None
+        self.cd.drop_table(self.conn, self.table)
+        self.table = None
+
+    def __getitem__(self, table):
+        self.table = table
+        sql = f'''CREATE TABLE IF NOT EXISTS {table} (
+        id bigint NOT NULL  primary key ,
+        data text
+        )'''
+        self.conn.execute(sql)
+        self.conn.commit()
+        return self
+
+    def insert_one(self, dic):
+        assert self.table is not None
+        mid = self.cd.max_id(self.conn, self.table)
+        value = json.dumps(dic)
+        sql = f"insert INTO {self.table} (id,data) VALUES ({mid},'{value}')"
+        self.conn.execute(sql)
+        self.conn.commit()
+
+    def find(self, query={}):
+        return self.t.find(self.conn, self.table, query)
+
+    def delete(self, query={}):
+        return self.t.delete(self.conn, self.table, query)
+
+    # 保证queery唯一,最后会隐含删除
+    def upsert_one(self, query, dic):
+        assert self.table is not None
+        l = self.find(query)
+        if len(l) == 0:
+            return self.insert_one(dic)
+        retain = l[0]
+        self.t.revise_one(self.conn, self.table, retain, dic)
+        rest_ids = [i['id'] for i in l if i['id'] != retain["id"]]
+        if len(rest_ids) > 0:
+            self.cd.delete_by_ids(self.conn, self.table, rest_ids)
+            self.conn.commit()
+
+
+if __name__ == '__main__':
+    conf = Common().read_conf()['mysql']['test']
+    dbname = Common().read_conf()['mysql']['price']['test']
+    t = 'nosql'
+    no = NosqlMysql(**{**conf, "dbname": dbname})
+    col = no[t]
+    col.insert_one({"a": 1})
+    print(col.find({}))

remote_run_everything/nosql/no_sql_pg.py

@@ -0,0 +1,69 @@
+import json
+
+from remote_run_everything.nosql.no_sql_tool import NosqlTool
+from remote_run_everything.tools.common import Common
+
+from remote_run_everything.db.crude_duck import CrudeDuck
+
+
+class NosqlPg:
+    def __init__(self, user, pwd, host, port, dbname):
+        self.t = NosqlTool()
+        self.cd = CrudeDuck()
+        self.conn = self.cd.install_pg_ext(user, pwd, host, port, dbname)
+        self.conn.create_function("my_q", self.t.query)
+        self.table = None
+
+    def drop_table(self):
+        assert self.table is not None
+        self.cd.drop_table(self.conn, self.table)
+        self.table=None
+
+    def __getitem__(self, table):
+        self.table = table
+        sql = f'''CREATE TABLE IF NOT EXISTS {table} (
+        id bigint NOT NULL  primary key ,
+        data text
+        )'''
+        self.conn.execute(sql)
+        self.conn.commit()
+        return self
+
+    def insert_one(self, dic):
+        assert self.table is not None
+        mid = self.cd.max_id(self.conn, self.table)
+        value = json.dumps(dic)
+        sql = f"insert INTO {self.table} (id,data) VALUES ({mid},'{value}')"
+        self.conn.execute(sql)
+        self.conn.commit()
+
+    def find(self, query={}):
+        return self.t.find(self.conn, self.table, query)
+
+    def delete(self, query={}):
+        return self.t.delete(self.conn, self.table, query)
+
+    # 保证queery唯一,最后会隐含删除
+    def upsert_one(self, query, dic):
+        assert self.table is not None
+        l = self.find(query)
+        if len(l) == 0:
+            return self.insert_one(dic)
+        retain = l[0]
+        self.t.revise_one(self.conn, self.table, retain, dic)
+        rest_ids = [i['id'] for i in l if i['id'] != retain["id"]]
+        if len(rest_ids) > 0:
+            self.cd.delete_by_ids(self.conn, self.table, rest_ids)
+            self.conn.commit()
+
+
+if __name__ == '__main__':
+    conf = Common().read_conf()['pg']
+    t = 'nosql'
+    no = NosqlPg(**{**conf, "dbname": "projects"})
+    col = no[t]
+    # no.drop_table()
+    col.insert_one({"a": 1})
+    print(col.find({}))
+    # col.upsert_one({"a": 1}, {"b": 44})
+    # print(col.find({}))

remote_run_everything/nosql/no_sql_tool.py

@@ -0,0 +1,81 @@
+import sqlite3, os, json, arrow
+from remote_run_everything.db.crude_duck import CrudeDuck
+
+
+class NosqlTool:
+    def __init__(self):
+        self.cd = CrudeDuck()
+
+    def default_db_path(self, db_path):
+        if db_path is None:
+            db_path = "D://wq/temp/emongo.db" if os.name == 'nt' else "/data/temp/emongo.db"
+        os.makedirs(os.path.dirname(db_path), exist_ok=True)
+        return db_path
+
+    def op_match(self, opdic, dvalue):
+        for op, qv in opdic.items():
+            if op == "$gt":
+                return dvalue > qv
+            elif op == "$gte":
+                return dvalue >= qv
+            elif op == "$lt":
+                return dvalue < qv
+            elif op == "$lte":
+                return dvalue < qv
+            elif op == "$ne":
+                return dvalue != qv
+            elif op == "$in":
+                return dvalue in qv
+            elif op == "$between":
+                return (dvalue >= qv[0]) and (dvalue <= qv[1])
+        return False
+
+    def query(self, data: str, query: str) -> int:
+        d = json.loads(data)
+        q = json.loads(query)
+        if len(q) == 0: return 1
+        for qk, qv in q.items():
+            # 查询的key在数据不存在
+            if qk not in d.keys(): return 0
+            if isinstance(qv, dict):
+                if not self.op_match(qv, d[qk]):
+                    return 0
+            elif d[qk] != qv:
+                return 0
+        return 1
+
+    def add_id(self, l):
+        res = []
+        for i in l:
+            dic = json.loads(i['data'])
+            dic['id'] = i['id']
+            res.append(dic)
+        return res
+
+    def find(self, conn, table, query={}):
+        assert table is not None
+        qs = json.dumps(query)
+        sql = f"select * from {table} where my_q(data,'{qs}') = 1 "
+        df = conn.sql(sql).df()
+        if len(df) == 0: return []
+        return self.add_id(df.to_dict("records"))
+
+    def delete(self, conn, table, query={}):
+        assert table is not None
+        l = self.find(conn, table, query)
+        if len(l) == 0: return
+        ids = [i['id'] for i in l]
+        self.cd.delete_by_ids(conn, table, ids)
+        conn.commit()
+
+        # 保证queery唯一,最后会隐含删除
+
+    def revise_one(self, conn, table, ex, dic):
+        assert table is not None
+        id = ex["id"]
+        v = {**ex, **dic}
+        s = json.dumps(v)
+        sql = f'''update {table} set data='{s}' where id={id}
+    '''
+        conn.execute(sql)
+        conn.commit()

remote_run_everything/tools/common.py

@@ -0,0 +1,56 @@
+import jinja2, requests, os
+import pandas as pd
+import socket, os, tomllib
+import base64
+import os, signal
+import subprocess, sys
+from remote_run_everything.tools.common1 import Common1
+
+
+class Common(Common1):
+    @property
+    def local_ip(self):
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        s.connect(("8.8.8.8", 80))
+        ip = s.getsockname()[0]
+        s.close()
+        return ip
+
+    def read_conf(self, path=None):
+        if path is None:
+            d = "D://mypy/conf" if os.name == 'nt' else "/data/mypy/conf"
+            n = "win" if os.name == "nt" else self.local_ip
+            path = f"{d}/{n}.toml"
+        with open(path, "rb") as f:
+            data = tomllib.load(f)
+        return data
+
+    def kill_by_pidfile(self, pidfile):
+        if os.path.exists(pidfile):
+            with open(pidfile, "rb") as f:
+                pid = f.read().decode()
+            print("exist pid===", pid)
+        try:
+            os.kill(int(pid), signal.SIGTERM)
+        except Exception as e:
+            print("kill err==", e)
+
+    def start_with_pidfile(self, workdir, pidfile, app):
+        os.chdir(workdir)
+        process = subprocess.Popen(app, creationflags=subprocess.CREATE_NEW_CONSOLE)
+        print("new pid====", process.pid)
+        with open(pidfile, "wb") as f:
+            s = str(process.pid).encode()
+            f.write(s)
+        sys.exit()
+
+    def supervise(self, pidfile, app, workdir=None):
+        if workdir is None:
+            workdir = os.path.dirname(pidfile)
+        self.kill_by_pidfile(pidfile)
+        self.start_with_pidfile(workdir, pidfile, app)
+
+
+if __name__ == '__main__':
+    g = Common()
+    a = g.prefix_zero(5, 111)

remote_run_everything/tools/common1.py

@@ -0,0 +1,100 @@
+import jinja2, requests, os
+import pandas as pd
+import base64
+import os, signal, glob, arrow
+
+
+class Common1:
+
+    def table_search(self, data, search):
+        res = []
+        for i in data:
+            if search == "":
+                res.append(i)
+            elif search != "" and search in str(i):
+                res.append(i)
+        return res
+
+    def split_page(self, numPerPage, cur, search, data):
+        data = self.table_search(data, search)
+        if data is None or len(data) == 0:
+            return {"total": 0, "data": []}
+        cur = int(cur)
+        res = {"total": len(data)}
+        n = int(numPerPage)
+        total = len(data)
+        start = (cur - 1) * n
+        end = cur * n
+        if start > total - 1:
+            res['data'] = []
+        elif end > total - 1:
+            remainder = total % numPerPage
+            if remainder == 0:
+                res['data'] = data[-numPerPage:]
+            else:
+                res['data'] = data[-remainder:]
+        else:
+            res['data'] = data[start:end]
+        return res
+
+    def render(self, dir, html, data):
+        env = jinja2.Environment(loader=jinja2.FileSystemLoader(f'{dir}/templates'))
+        template = env.get_template(html)
+        outputText = template.render(data=data)  # this is where to put args
+        return outputText
+
+    def inven_everyday(self, df, begin, end):
+        df1 = df.groupby(by=['id', 'date']).sum()
+        # 可以把复合index重新拆分成columns
+        trades = df1.reset_index()
+        trades['id'] = trades['id']
+        trades['date'] = trades['date']
+        # create a range of dates for the merged dataframe pd.datetime.today()
+        if not begin:
+            begin = df['date'].min()
+        if not end:
+            end = df['date'].max()
+        index_of_dates = pd.date_range(begin, end).to_frame().reset_index(drop=True).rename(columns={0: 'date'}).astype(
+            str)
+        # create a merged dataframe with columns date / stock / stock_Tr.
+        merged = pd.merge(index_of_dates, trades, how='left', on='date')
+        # create a pivottable showing the shares_TR of each stock for each date
+        shares_tr = merged.pivot(index='date', columns='id', values='q')
+        shares_tr = shares_tr.dropna(axis=1, how='all').fillna(0)
+        cumShares = shares_tr.cumsum()
+        cumShares.index = cumShares.index.astype(str)
+        return cumShares
+
+    def writeb64(self, path, b64):
+        dir = os.path.dirname(path)
+        if not os.path.exists(dir):
+            os.makedirs(dir, exist_ok=True)
+        content = base64.b64decode(b64)
+        with open(path, "wb") as f:
+            f.write(content)
+            os.chmod(path, 0o777)
+
+    def readb64(self, f):
+        with open(f, "rb") as file:
+            encoded_string = base64.b64encode(file.read())
+            return encoded_string.decode()
+
+    def prefix_zero(self, n, d):
+        if len(str(d)) >= n: return str(d)[-n:]
+        l = ["0"] * (n - len(str(d)))
+        zeros = "".join(l)
+        return f"{zeros}{d}"
+
+    def clear_by_days(self, root, n):
+        files = glob.glob(f"{root}/*/*.*", recursive=True)
+        now = arrow.now()
+        for f in files:
+            info = os.stat(f).st_mtime
+            dif = now - arrow.get(info)
+            if dif.days > n:
+                os.remove(f)
+
+
+if __name__ == '__main__':
+    g = Common1()
+    a = g.prefix_zero(5, 111)

remote_run_everything/tools/decorators.py

@@ -0,0 +1,81 @@
+from functools import wraps
+from remote_run_everything.db.kv_store import KvStore
+
+
+def cache_by_name(k, ts):
+    def _wrapper(f):
+        wraps(f)
+
+        def _wrapped(*args, **kwargs):
+            mykv = KvStore()
+            res = mykv.read_with_ex(k, ts)
+            if res is None:
+                res = f(*args, **kwargs)
+                if res is not None:
+                    mykv.write_with_ex(k, res)
+                return res
+            return res
+
+        return _wrapped
+
+    return _wrapper
+
+
+def cache_by_1starg(sub, ts):
+    def _wrapper(f):
+        wraps(f)
+
+        def _wrapped(*args, **kwargs):
+            mykv = KvStore()
+            k = f"{sub}{args[0]}"
+            res = mykv.read_with_ex(k, ts)
+            if res is None:
+                res = f(*args, **kwargs)
+                if res is not None:
+                    mykv.write_with_ex(k, res)
+                return res
+            return res
+
+        return _wrapped
+
+    return _wrapper
+
+
+def cache_by_nth_arg(sub, index, ts):
+    def _wrapper(f):
+        wraps(f)
+
+        def _wrapped(*args, **kwargs):
+            mykv = KvStore()
+            k = f"{sub}{args[index]}"
+            res = mykv.read_with_ex(k, ts)
+            if res is None:
+                res = f(*args, **kwargs)
+                if res is not None:
+                    mykv.write_with_ex(k, res)
+                return res
+            return res
+
+        return _wrapped
+
+    return _wrapper
+
+
+def cache_by_rkey(ts):
+    def _wrapper(f):
+        wraps(f)
+
+        def _wrapped(*args, **kwargs):
+            mykv = KvStore()
+            k = str(kwargs['rkey'])
+            res = mykv.read_with_ex(k, ts)
+            if res is None:
+                res = f(*args, **kwargs)
+                if res is not None:
+                    mykv.write_with_ex(k, res)
+                return res
+            return res
+
+        return _wrapped
+
+    return _wrapper

remote_run_everything/tools/sqlacodegen_go_struct.py

@@ -0,0 +1,109 @@
+import os
+from dataclasses import dataclass
+
+
+@dataclass
+class Tab:
+    stname: str
+    tbname: str
+    fields: list
+
+
+class Sql2go:
+    def __init__(self, path):
+        self.path = path
+
+    def read_lines(self):
+        with open(self.path, 'r') as f:
+            content = f.readlines()
+            return content
+
+    def replace(self, s, l):
+        for i in l:
+            s = s.replace(i, "")
+        return s
+
+    def split_cls(self):
+        res = []
+        content = self.read_lines()
+        cur_cls = ""
+        t = Tab("", "", [])
+        for s in content:
+            if s.startswith("class "):
+                if cur_cls != "":
+                    res.append(t)
+                t = Tab("", "", [])
+                stname = self.replace(s, [" ", "\n", "class", "(Base)", ":"])
+                cur_cls = stname
+                t.stname = cur_cls
+            elif "__tablename__" in s:
+                tbname = self.replace(s, [" ", "__tablename__", "\n", "="])
+                t.tbname = tbname
+            elif "= Column" in s:
+                t.fields.append(s)
+        for t in res:
+            ll = []
+            for s in t.fields:
+                ll.append({
+                    "field": self.parse_name(s),
+                    "tag": self.parse_tag(s),
+                    "ty": self.parse_ty(s)
+                })
+            t.fields = ll
+        print(res)
+        return res
+
+    def parse_name(self, s):
+        name = s.split("=")[0].replace(" ", "")
+        return name
+
+    def parse_ty(self, s):
+        if "DateTime" in s:
+            ty = "time.time"
+        elif "String" in s:
+            ty = "string"
+        elif "BigInteger" in s:
+            ty = "int64"
+        elif "Integer" in s or "TINYINT" in s:
+            ty = "int32"
+        elif "CHAR" in s:
+            ty = "string"
+        elif "Float" in s or "MONEY" in s:
+            ty = "float32"
+        elif "DECIMAL" in s:
+            ty = "float64"
+        else:
+            ty = "unk"
+        return ty
+
+    def parse_tag(self, s):
+        name = s.split("=")[0].replace(" ", "").lower()
+        tag = ""
+        if "primary_key=True" in s:
+            tag = f'''`xorm:"pk not null '{name}'" json:"{name}"`'''
+        elif "nullable=False" in s:
+            tag = f'''`xorm:"not null '{name}'" json:"{name}"`'''
+        else:
+            tag = f'''`xorm:"'{name}'" json:"{name}"`'''
+        return tag
+
+    def write_go(self):
+        res = self.split_cls()
+        lines = []
+        for tab in res:
+            header = f"type {tab.stname} struct " + "{"
+            st = [header]
+            for f in tab.fields:
+                ss = f"{f['field']} {f['ty']} {f['tag']}"
+                st.append(ss)
+            st.append("}" + "\r\n")
+            lines.append("\n".join(st))
+        ss = "package mod" + "\r\n" + "".join(lines)
+        with open("./mod.go", 'w') as f:
+            f.write(ss)
+        os.system("go fmt mod.go")
+
+
+if __name__ == '__main__':
+    s = SqlRead("./pstarback.py")
+    s.write_go()