kaqing 2.0.14__py3-none-any.whl → 2.0.189__py3-none-any.whl

adam/utils_repl/repl_completer.py

@@ -0,0 +1,172 @@
+from concurrent.futures import ThreadPoolExecutor
+import copy
+import inspect
+import re
+import threading
+import traceback
+from typing import Iterable, TypeVar, cast
+from prompt_toolkit.completion import CompleteEvent, Completer, Completion, NestedCompleter, WordCompleter
+from prompt_toolkit.document import Document
+
+from adam.utils import debug, debug_complete, log2, log_timing
+from adam.utils_repl.appendable_completer import AppendableCompleter
+
+import nest_asyncio
+nest_asyncio.apply()
+
+import asyncio
+
+__all__ = [
+    "ReplCompleter",
+]
+
+T = TypeVar('T')
+
+def merge_completions(dict1, dict2):
+    if isinstance(dict1, dict):
+        target = dict1.copy()
+    else:
+        target = copy.copy(dict1)
+
+    try:
+        for key, value in dict2.items():
+            if key in target:
+                debug_complete(f'[{key}] {type(dict2)} is being merged to {type(target[key])} completions')
+                if isinstance(value, dict):
+                    if isinstance(target[key], dict):
+                        target[key] = merge_completions(target[key], value)
+                    elif isinstance(target[key], AppendableCompleter):
+                        cast(AppendableCompleter, target[key]).append_completions(key, value)
+                elif isinstance(value, AppendableCompleter):
+                    if isinstance(target[key], dict):
+                        cast(AppendableCompleter, value).append_completions(key, target[key])
+                        target[key] = value
+                    else:
+                        log2(f'* {key} of {type(value)} is overriding existing {type(target[key])} completions')
+                else:
+                    target[key] = value
+            else:
+                target[key] = value
+
+    except Exception as e:
+        traceback.print_exc()
+
+    return target
+
+class ReplCompleter(NestedCompleter):
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        # Split document.
+        text = document.text_before_cursor.lstrip()
+        stripped_len = len(document.text_before_cursor) - len(text)
+
+        # If there is a space, check for the first term, and use a
+        # subcompleter.
+        if " " in text:
+            first_term = text.split()[0]
+            completer = self.options.get(first_term)
+
+            # If we have a sub completer, use this for the completions.
+            if completer is not None:
+                remaining_text = text[len(first_term) :].lstrip()
+                move_cursor = len(text) - len(remaining_text) + stripped_len
+
+                new_document = Document(
+                    remaining_text,
+                    cursor_position=document.cursor_position - move_cursor,
+                )
+
+                for c in completer.get_completions(new_document, complete_event):
+                    yield c
+
+        # No space in the input: behave exactly like `WordCompleter`.
+        else:
+            completer = WordCompleter(
+                # Allow dot in the middle or a word
+                list(self.options.keys()), ignore_case=self.ignore_case, pattern=re.compile(r"([a-zA-Z0-9_\.\@\&]+|[^a-zA-Z0-9_\.\@\&\s]+)")
+            )
+            for c in completer.get_completions(document, complete_event):
+                yield c
+
+lock = threading.Lock()
+in_queue = set()
+
+def preload(action: callable, log_key: str = None):
+    with lock:
+        if not LazyNestedCompleter.loop:
+            LazyNestedCompleter.loop = asyncio.new_event_loop()
+            LazyNestedCompleter.async_exec = ThreadPoolExecutor(max_workers=6, thread_name_prefix='async')
+            LazyNestedCompleter.loop.set_default_executor(LazyNestedCompleter.async_exec)
+
+        # some lib does not handle asyncio loop properly, as sync exec submit does not work, use another async loop
+        async def a():
+            try:
+                arg_needed = len(action.__code__.co_varnames)
+
+                if log_key:
+                    with log_timing(log_key):
+                        r = action(None) if arg_needed else action()
+                else:
+                    r = action(None) if arg_needed else action()
+                if inspect.isawaitable(r):
+                    await r
+
+                in_queue.remove(log_key)
+            except Exception as e:
+                log2('preloading error', e, inspect.getsourcelines(action)[0][0])
+                traceback.print_exc()
+
+        if log_key not in in_queue:
+            in_queue.add(log_key)
+            LazyNestedCompleter.async_exec.submit(lambda: LazyNestedCompleter.loop.run_until_complete(a()))
+
+class LazyNestedCompleter(NestedCompleter):
+    loop: asyncio.AbstractEventLoop = None
+    async_exec: ThreadPoolExecutor = None
+
+    def __init__(self, name: str, options_lambda: callable, ignore_case: bool = True, auto: str = 'lazy') -> None:
+        super().__init__(None, ignore_case)
+        self.options_lambda = options_lambda
+        if auto == 'lazy':
+            preload(options_lambda, log_key=name)
+
+    def __repr__(self) -> str:
+        return "LazyNestedCompleter(%r, ignore_case=%r)" % (self.options, self.ignore_case)
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        if not self.options:
+            self.options = self.options_lambda()
+
+        # Split document.
+        text = document.text_before_cursor.lstrip()
+        stripped_len = len(document.text_before_cursor) - len(text)
+
+        # If there is a space, check for the first term, and use a
+        # subcompleter.
+        if " " in text:
+            first_term = text.split()[0]
+            completer = self.options.get(first_term)
+
+            # If we have a sub completer, use this for the completions.
+            if completer is not None:
+                remaining_text = text[len(first_term) :].lstrip()
+                move_cursor = len(text) - len(remaining_text) + stripped_len
+
+                new_document = Document(
+                    remaining_text,
+                    cursor_position=document.cursor_position - move_cursor,
+                )
+
+                for c in completer.get_completions(new_document, complete_event):
+                    yield c
+
+        # No space in the input: behave exactly like `WordCompleter`.
+        else:
+            completer = WordCompleter(
+                list(self.options.keys()), ignore_case=self.ignore_case
+            )
+            for c in completer.get_completions(document, complete_event):
+                yield c

adam/utils_repl/state_machine.py

@@ -0,0 +1,173 @@
+from abc import abstractmethod
+from typing import Generic, TypeVar
+
+from adam.utils import log_exc
+
+__all__ = [
+    'State',
+    'StateMachine',
+]
+
+T = TypeVar('T')
+
+class State:
+    def __init__(self, state: str, comeback_token: str = None, comeback_state: str = None):
+        self.state = state
+        self.comeback_token = comeback_token
+        self.comeback_state = comeback_state
+        self.context: dict[str, str] = {}
+
+    def __str__(self):
+        return f'{self.state if self.state else None} comeback[{self.comeback_token} {self.comeback_state}]'
+
+class StateMachine(Generic[T]):
+    @abstractmethod
+    def spec(self) -> str:
+        return None
+
+    @abstractmethod
+    def keywords(self) -> list[str]:
+        return None
+
+    def expandable_names(self):
+        return []
+
+    def incomplete_name_transition_condition(self, from_s: str, token: str, to_s: str, suggestions: str) -> list[str]:
+        if not suggestions:
+            return None
+
+        tokens = [token]
+        if '|' in token:
+            tokens = token.split('|')
+
+        if 'name' not in tokens:
+            return None
+
+        return tokens
+
+    def __init__(self, indent=0, push_level = 0, debug = False):
+        self.states: dict[str, State] = {}
+        self.suggestions: dict[str, str] = {}
+
+        self.indent = indent
+        self.push_level = push_level
+        self.comebacks: dict[int, str] = {}
+        self.debug = debug
+
+        from_ss_to_add = []
+        from_ss = ['']
+        words: str = None
+        for l in self.spec():
+            t_and_w = l.split('^')
+            if len(t_and_w) > 1:
+                words = t_and_w[1].strip()
+            else:
+                words = None
+
+            tks = t_and_w[0].strip(' ').split('>')
+            if not l.startswith('-'):
+                if words:
+                    self.suggestions[tks[0].strip(' ')] = words
+
+                if len(tks) == 1:
+                    from_ss_to_add.append(tks[0].strip(' '))
+                    continue
+
+                from_ss = []
+                from_ss.extend(from_ss_to_add)
+                from_ss_to_add = []
+                from_ss.append(tks[0].strip(' '))
+
+            self.add_transitions(from_ss, tks, words)
+
+    def add_transitions(self, from_ss: list[str], tks: list[str], words: str):
+        token = tks[1].strip(' ')
+        if len(tks) > 2:
+            to_s = tks[2].strip(' ')
+            for from_s in from_ss:
+                self.add_whitespace_transition(from_s, to_s)
+                self.add_transition(from_s, token, to_s)
+                self.add_incomplete_name_transition(from_s, token, to_s, words)
+        elif '<' in tks[0]:
+            from_and_token = tks[0].split('<')
+            if len(from_and_token) > 1:
+                for from_s in from_ss:
+                    self.add_comeback_transition(from_s, from_and_token[1], tks[1].strip(' '))
+
+    def add_whitespace_transition(self, from_s: str, to_s: str):
+        if self.witespace_transition_condition(from_s, to_s):
+            if self.debug:
+                print(f'{from_s[:-1]} > _ = {to_s}')
+            self.states[f'{from_s[:-1]} > _'] = State(from_s)
+
+    def witespace_transition_condition(self, from_s: str, to_s: str):
+        return from_s.endswith('_')
+
+    def add_incomplete_name_transition(self, from_s: str, token: str, to_s: str, words: str):
+        if tokens := self.incomplete_name_transition_condition(from_s, token, to_s, words):
+            self.suggestions[to_s] = words
+            for token in tokens:
+                if self.debug:
+                    print(f'{to_s} > {token} = {to_s}')
+                self.states[f'{to_s} > {token}'] = State(to_s)
+
+    def add_transition(self, from_s: str, token: str, to_s: str):
+        tokens = [token]
+        if '|' in token:
+            tokens = token.split('|')
+
+        for t in tokens:
+            if t == '_or_':
+                t = '||'
+            elif t == 'pipe':
+                t = '|'
+            elif t == '_rdr0_':
+                t = '<'
+            elif t == '_rdr1_':
+                t = '>'
+            elif t == '_rdr2_':
+                t = '2>'
+
+            if self.debug:
+                print(f'{from_s} > {t} = {to_s}')
+            self.states[f'{from_s} > {t}'] = State(to_s)
+
+    def add_comeback_transition(self, from_s: str, token: str, to_s: str):
+        key = f'{from_s} > ('
+        orig = self.states[key]
+        if not orig:
+            raise Exception(f'from state not found for {key}')
+
+        orig.comeback_token = token
+        orig.comeback_state = to_s
+        if self.debug:
+            print(f'{from_s} > ) = {to_s}')
+        self.states[key] = orig
+
+    def traverse_tokens(self, tokens: list[str], state: State = State('')):
+        for token in tokens[:-1]:
+            if not token:
+                continue
+
+            if self.debug:
+                print(f'{token} ', end='')
+
+            last_name = None
+
+            if (t := token.lower()) in self.keywords():
+                token = t
+            elif token in ['*', ',', '.']:
+                pass
+            else:
+                last_name = token
+                token = 'word'
+
+            with log_exc():
+                context = state.context
+                state = self.states[f'{state.state} > {token}']
+                state.context = context
+
+                if last_name:
+                    state.context['last_name'] = last_name
+
+        return state

adam/utils_sqlite.py

@@ -0,0 +1,137 @@
+from collections.abc import Callable
+
+import functools
+import glob
+import os
+import sqlite3
+import pandas
+
+from adam.config import Config
+from adam.utils import tabulize, log, wait_log
+
+class CursorHandler:
+    def __init__(self, conn: sqlite3.Connection):
+        self.conn = conn
+        self.cursor = None
+
+    def __enter__(self):
+        self.cursor = self.conn.cursor()
+
+        return self.cursor
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.cursor:
+            self.cursor.close()
+
+        return False
+
+class SQLiteConnectionHandler:
+    def __init__(self, database: str, keyspace = None):
+        self.database = database
+        self.keyspace = keyspace
+        self.conn = None
+
+    def __enter__(self) -> sqlite3.Connection:
+        self.conn = SQLite.connect(self.database, self.keyspace)
+
+        self.conn.__enter__()
+
+        return self.conn
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.conn:
+            self.conn.__exit__(exc_type, exc_val, exc_tb)
+
+        return False
+
+def sqlite(database: str, keyspace = None):
+    return SQLiteConnectionHandler(database, keyspace=keyspace)
+
+# no state utility class
+class SQLite:
+    def cursor(conn: sqlite3.Connection):
+        return CursorHandler(conn)
+
+    def local_db_dir():
+        return Config().get('export.sqlite.local-db-dir', '/tmp/qing-db')
+
+    def keyspace(database: str):
+        return '_'.join(database.replace(".db", "").split('_')[1:])
+
+    @functools.lru_cache()
+    def database_names(prefix: str = None):
+        wait_log('Inspecting export databases...')
+
+        pattern = f'{SQLite.local_db_dir()}/s*.db'
+        if prefix:
+            pattern = f'{SQLite.local_db_dir()}/{prefix}*'
+        return [os.path.basename(f) for f in glob.glob(pattern)]
+
+    def clear_cache(cache: str = None):
+        SQLite.database_names.cache_clear()
+        SQLite.table_names.cache_clear()
+
+    @functools.lru_cache()
+    def table_names(database: str):
+      tokens = database.replace('.db', '').split('_')
+      ts_prefix = tokens[0]
+      keyspace = '_'.join(tokens[1:])
+
+      conn = None
+      tables = []
+      try:
+         conn = sqlite3.connect(f'{SQLite.local_db_dir()}/{ts_prefix}_{keyspace}.db')
+         with SQLite.cursor(conn) as cursor:
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;")
+
+            tables = [row[0] for row in cursor.fetchall() if row[0] != "sqlite_sequence"]
+
+         return tables
+      except sqlite3.Error as e:
+         print(f"Error connecting to or querying the database: {e}")
+         return []
+      finally:
+         if conn:
+               conn.close()
+
+    def connect(database: str, keyspace: str = None):
+        os.makedirs(SQLite.local_db_dir(), exist_ok=True)
+
+        if keyspace:
+            return sqlite3.connect(f'{SQLite.local_db_dir()}/{database}_{keyspace}.db')
+        else:
+            conn = sqlite3.connect(f'{SQLite.local_db_dir()}/{database}_root.db')
+            with SQLite.cursor(conn) as cursor:
+                for d in SQLite.database_names(database):
+                    if d != f'{database}.db':
+                        q = f"ATTACH DATABASE '{SQLite.local_db_dir()}/{d}' AS {SQLite.keyspace(d)}"
+                        cursor.execute(q)
+
+            return conn
+
+    @functools.lru_cache()
+    def column_names(tables: list[str] = [], database: str = None, function: str = 'audit', partition_cols_only = False):
+        pass
+
+    def run_query(query: str, database: str = None, output: Callable[[str], str] = None) -> tuple[int, str]:
+        with sqlite(database) as conn:
+            return SQLite.run_query_with_conn(conn, query, output=output)
+
+    def run_query_with_conn(conn, query: str, output: Callable[[str], str] = None) -> tuple[int, str]:
+        log_file = None
+
+        df = SQLite.query(conn, query)
+        lines = ['\t'.join(map(str, line)) for line in df.values.tolist()]
+        out = tabulize(lines, header='\t'.join(df.columns.tolist()), separator='\t', to=0)
+        if output:
+            log_file = output(out)
+        else:
+            log(out)
+
+        return len(lines), log_file
+
+    def query(conn, sql: str) -> tuple[str, str, list]:
+        return pandas.read_sql_query(sql, conn)
+
+def log_prefix():
+    return Config().get('export.log-prefix', '/tmp/qing')

adam/version.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-__version__ = "2.0.14"  #: the working version
+__version__ = "2.0.189"  #: the working version
 __release__ = "1.0.0"  #: the release version

{kaqing-2.0.14.dist-info → kaqing-2.0.189.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kaqing
-Version: 2.0.14
+Version: 2.0.189
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN