kaqing 2.0.110__py3-none-any.whl → 2.0.214__py3-none-any.whl

adam/utils_local.py

@@ -0,0 +1,80 @@
+import os
+import subprocess
+import traceback
+
+from adam.config import Config
+from adam.utils import ExecResult, creating_dir, debug
+
+def local_qing_dir():
+    return creating_dir(Config().get('local-qing-dir', '/tmp/qing-db/q'))
+
+def local_downloads_dir():
+    return creating_dir(Config().get('local-downloads-dir', '/tmp/qing-db/q/downloads'))
+
+class LocalExecResult(ExecResult):
+    def __init__(self, stdout: str, stderr: str, command: str = None, code = 0, log_file: str = None):
+        self.stdout: str = stdout
+        self.stderr: str = stderr
+        self.command: str = command
+        self.code = code
+        self.pod = 'local'
+        self.log_file = log_file
+
+    def exit_code(self) -> int:
+        return self.code
+
+    def cat_log_file_cmd(self):
+        if self.log_file:
+            return f':cat {self.log_file}'
+
+        return None
+
+    def __str__(self):
+        return f'{"OK" if self.exit_code() == 0 else self.exit_code()} {self.command}'
+
+    def __audit_extra__(self):
+        return self.log_file if self.log_file else None
+
+def local_exec(cmd: list[str], shell=False, show_out=False):
+    stdout = ''
+    stderr = ''
+    returncode = 0
+
+    try:
+        if show_out:
+            debug(' '.join(cmd))
+
+        r = subprocess.run(cmd, capture_output=True, text=True, shell=shell)
+        stdout = r.stdout
+        stderr = r.stderr
+        returncode = r.returncode
+    except FileNotFoundError as e:
+        pass
+
+    return LocalExecResult(stdout, stderr, ' '.join(cmd), returncode)
+
+def find_local_files(pattern: str = f'{local_qing_dir()}/*', file_type: str = None, max_depth = 0, mmin: int = 0):
+    # find . -maxdepth 1 -type f -name '*'
+    log_files = []
+    try:
+        dir = os.path.dirname(pattern)
+        base = os.path.basename(pattern)
+        cmd = ['find', dir]
+        if file_type:
+            cmd += ['-type', file_type]
+        if max_depth:
+            cmd += ['-maxdepth', str(max_depth)]
+        if mmin:
+                cmd += ['-mmin',  f'-{mmin}']
+        cmd += ['-name', base]
+
+        stdout = local_exec(cmd, show_out=Config().is_debug()).stdout
+
+        for line in stdout.split('\n'):
+            line = line.strip(' \r')
+            if line:
+                log_files.append(line)
+    except:
+        traceback.print_exc()
+
+    return log_files

adam/utils_net.py

@@ -18,7 +18,7 @@ def get_my_host():
     return MY_HOST
 
 def get_ip_from_hostname(hostname):
-  try:
-    return socket.gethostbyname(hostname)
-  except socket.gaierror:
-    return None
+    try:
+        return socket.gethostbyname(hostname)
+    except socket.gaierror:
+        return None

adam/utils_repl/appendable_completer.py

@@ -0,0 +1,6 @@
+from abc import abstractmethod
+
+class AppendableCompleter:
+    @abstractmethod
+    def append_completions(self, key: str, value: dict[str, any]):
+        pass

adam/utils_repl/automata_completer.py

@@ -0,0 +1,48 @@
+from typing import Generic, Iterable, TypeVar
+from prompt_toolkit.completion import CompleteEvent, Completer, Completion, WordCompleter
+from prompt_toolkit.document import Document
+
+from adam.utils_repl.state_machine import StateMachine, State
+
+__all__ = [
+    "AutomataCompleter",
+]
+
+T = TypeVar('T')
+
+class AutomataCompleter(Completer, Generic[T]):
+    def __init__(self,
+                 state_machine: StateMachine,
+                 first_term: str = '',
+                 debug = False):
+        super().__init__()
+        self.machine = state_machine
+        self.first_term = first_term
+        self.debug = debug
+
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        text = document.text_before_cursor.lstrip()
+        state = ''
+        if self.first_term:
+            text = f'{self.first_term} {text}'
+
+        completer: Completer = None
+        state: State = self.machine.traverse_tokens(self.tokens(text), State(state))
+        if self.debug:
+            print('\n  =>', state.state if isinstance(state, State) else '')
+
+        if state.state in self.machine.suggestions:
+            if completer := self.suggestions_completer(state, self.machine.suggestions[state.state].strip(' ')):
+                for c in completer.get_completions(document, complete_event):
+                    yield c
+
+    def tokens(self, text: str) -> list[T]:
+        return text.split(' ')
+
+    def suggestions_completer(self, _: State, suggestions: str) -> list[str]:
+        if not suggestions:
+            return None
+
+        return WordCompleter(suggestions.split(','))

adam/utils_repl/repl_completer.py

@@ -0,0 +1,93 @@
+import copy
+import re
+import traceback
+from typing import Iterable, TypeVar, cast
+from prompt_toolkit.completion import CompleteEvent, Completion, NestedCompleter, WordCompleter
+from prompt_toolkit.document import Document
+
+from adam.utils import debug_complete, log2
+from adam.utils_repl.appendable_completer import AppendableCompleter
+
+import nest_asyncio
+nest_asyncio.apply()
+
+import asyncio
+
+__all__ = [
+    "ReplCompleter",
+]
+
+T = TypeVar('T')
+
+def merge_completions(dict1, dict2):
+    if isinstance(dict1, dict):
+        target = dict1.copy()
+    else:
+        target = copy.copy(dict1)
+
+    try:
+        for key, value in dict2.items():
+            if key in target:
+                debug_complete(f'[{key}] {type(dict2)} is being merged to {type(target[key])} completions')
+                if isinstance(value, dict):
+                    if isinstance(target[key], dict):
+                        target[key] = merge_completions(target[key], value)
+                    elif isinstance(target[key], AppendableCompleter):
+                        cast(AppendableCompleter, target[key]).append_completions(key, value)
+                    elif isinstance(target[key], NestedCompleter):
+                        cast(NestedCompleter, target[key]).options = merge_completions(cast(NestedCompleter, target[key]).options, value)
+                elif isinstance(value, AppendableCompleter):
+                    if isinstance(target[key], dict):
+                        cast(AppendableCompleter, value).append_completions(key, target[key])
+                        target[key] = value
+                    else:
+                        log2(f'* {key} of {type(value)} is overriding existing {type(target[key])} completions')
+                else:
+                    target[key] = value
+            else:
+                target[key] = value
+
+    except Exception as e:
+        traceback.print_exc()
+
+    return target
+
+class ReplCompleter(NestedCompleter):
+    def get_completions(
+        self, document: Document, complete_event: CompleteEvent
+    ) -> Iterable[Completion]:
+        # Split document.
+        text = document.text_before_cursor.lstrip()
+        stripped_len = len(document.text_before_cursor) - len(text)
+
+        # If there is a space, check for the first term, and use a
+        # subcompleter.
+        if " " in text:
+            first_term = text.split()[0]
+            completer = self.options.get(first_term)
+
+            # If we have a sub completer, use this for the completions.
+            if completer is not None:
+                remaining_text = text[len(first_term) :].lstrip()
+                move_cursor = len(text) - len(remaining_text) + stripped_len
+
+                new_document = Document(
+                    remaining_text,
+                    cursor_position=document.cursor_position - move_cursor,
+                )
+
+                try:
+                    # potential thread racing
+                    for c in completer.get_completions(new_document, complete_event):
+                        yield c
+                except:
+                    pass
+
+        # No space in the input: behave exactly like `WordCompleter`.
+        else:
+            completer = WordCompleter(
+                # Allow dot in the middle or a word
+                list(self.options.keys()), ignore_case=self.ignore_case, pattern=re.compile(r"([a-zA-Z0-9_\.\@\&]+|[^a-zA-Z0-9_\.\@\&\s]+)")
+            )
+            for c in completer.get_completions(document, complete_event):
+                yield c

adam/utils_repl/state_machine.py

@@ -0,0 +1,173 @@
+from abc import abstractmethod
+from typing import Generic, TypeVar
+
+from adam.utils import log_exc
+
+__all__ = [
+    'State',
+    'StateMachine',
+]
+
+T = TypeVar('T')
+
+class State:
+    def __init__(self, state: str, comeback_token: str = None, comeback_state: str = None):
+        self.state = state
+        self.comeback_token = comeback_token
+        self.comeback_state = comeback_state
+        self.context: dict[str, str] = {}
+
+    def __str__(self):
+        return f'{self.state if self.state else None} comeback[{self.comeback_token} {self.comeback_state}]'
+
+class StateMachine(Generic[T]):
+    @abstractmethod
+    def spec(self) -> str:
+        return None
+
+    @abstractmethod
+    def keywords(self) -> list[str]:
+        return None
+
+    def expandable_names(self):
+        return []
+
+    def incomplete_name_transition_condition(self, from_s: str, token: str, to_s: str, suggestions: str) -> list[str]:
+        if not suggestions:
+            return None
+
+        tokens = [token]
+        if '|' in token:
+            tokens = token.split('|')
+
+        if 'name' not in tokens:
+            return None
+
+        return tokens
+
+    def __init__(self, indent=0, push_level = 0, debug = False):
+        self.states: dict[str, State] = {}
+        self.suggestions: dict[str, str] = {}
+
+        self.indent = indent
+        self.push_level = push_level
+        self.comebacks: dict[int, str] = {}
+        self.debug = debug
+
+        from_ss_to_add = []
+        from_ss = ['']
+        words: str = None
+        for l in self.spec():
+            t_and_w = l.split('^')
+            if len(t_and_w) > 1:
+                words = t_and_w[1].strip()
+            else:
+                words = None
+
+            tks = t_and_w[0].strip(' ').split('>')
+            if not l.startswith('-'):
+                if words:
+                    self.suggestions[tks[0].strip(' ')] = words
+
+                if len(tks) == 1:
+                    from_ss_to_add.append(tks[0].strip(' '))
+                    continue
+
+                from_ss = []
+                from_ss.extend(from_ss_to_add)
+                from_ss_to_add = []
+                from_ss.append(tks[0].strip(' '))
+
+            self.add_transitions(from_ss, tks, words)
+
+    def add_transitions(self, from_ss: list[str], tks: list[str], words: str):
+        token = tks[1].strip(' ')
+        if len(tks) > 2:
+            to_s = tks[2].strip(' ')
+            for from_s in from_ss:
+                self.add_whitespace_transition(from_s, to_s)
+                self.add_transition(from_s, token, to_s)
+                self.add_incomplete_name_transition(from_s, token, to_s, words)
+        elif '<' in tks[0]:
+            from_and_token = tks[0].split('<')
+            if len(from_and_token) > 1:
+                for from_s in from_ss:
+                    self.add_comeback_transition(from_s, from_and_token[1], tks[1].strip(' '))
+
+    def add_whitespace_transition(self, from_s: str, to_s: str):
+        if self.witespace_transition_condition(from_s, to_s):
+            if self.debug:
+                print(f'{from_s[:-1]} > _ = {to_s}')
+            self.states[f'{from_s[:-1]} > _'] = State(from_s)
+
+    def witespace_transition_condition(self, from_s: str, to_s: str):
+        return from_s.endswith('_')
+
+    def add_incomplete_name_transition(self, from_s: str, token: str, to_s: str, words: str):
+        if tokens := self.incomplete_name_transition_condition(from_s, token, to_s, words):
+            self.suggestions[to_s] = words
+            for token in tokens:
+                if self.debug:
+                    print(f'{to_s} > {token} = {to_s}')
+                self.states[f'{to_s} > {token}'] = State(to_s)
+
+    def add_transition(self, from_s: str, token: str, to_s: str):
+        tokens = [token]
+        if '|' in token:
+            tokens = token.split('|')
+
+        for t in tokens:
+            if t == '_or_':
+                t = '||'
+            elif t == 'pipe':
+                t = '|'
+            elif t == '_rdr0_':
+                t = '<'
+            elif t == '_rdr1_':
+                t = '>'
+            elif t == '_rdr2_':
+                t = '2>'
+
+            if self.debug:
+                print(f'{from_s} > {t} = {to_s}')
+            self.states[f'{from_s} > {t}'] = State(to_s)
+
+    def add_comeback_transition(self, from_s: str, token: str, to_s: str):
+        key = f'{from_s} > ('
+        orig = self.states[key]
+        if not orig:
+            raise Exception(f'from state not found for {key}')
+
+        orig.comeback_token = token
+        orig.comeback_state = to_s
+        if self.debug:
+            print(f'{from_s} > ) = {to_s}')
+        self.states[key] = orig
+
+    def traverse_tokens(self, tokens: list[str], state: State = State('')):
+        for token in tokens[:-1]:
+            if not token:
+                continue
+
+            if self.debug:
+                print(f'{token} ', end='')
+
+            last_name = None
+
+            if (t := token.lower()) in self.keywords():
+                token = t
+            elif token in ['*', ',', '.']:
+                pass
+            else:
+                last_name = token
+                token = 'word'
+
+            with log_exc():
+                context = state.context
+                state = self.states[f'{state.state} > {token}']
+                state.context = context
+
+                if last_name:
+                    state.context['last_name'] = last_name
+
+        return state

adam/utils_sqlite.py

@@ -0,0 +1,132 @@
+from collections.abc import Callable
+
+import functools
+import glob
+import os
+import sqlite3
+import pandas
+
+from adam.config import Config
+from adam.utils import creating_dir, tabulize, log, wait_log
+
+class CursorHandler:
+    def __init__(self, conn: sqlite3.Connection):
+        self.conn = conn
+        self.cursor = None
+
+    def __enter__(self):
+        self.cursor = self.conn.cursor()
+
+        return self.cursor
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.cursor:
+            self.cursor.close()
+
+        return False
+
+class SQLiteConnectionHandler:
+    def __init__(self, database: str, keyspace = None):
+        self.database = database
+        self.keyspace = keyspace
+        self.conn = None
+
+    def __enter__(self) -> sqlite3.Connection:
+        self.conn = SQLite.connect(self.database, self.keyspace)
+
+        self.conn.__enter__()
+
+        return self.conn
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.conn:
+            self.conn.__exit__(exc_type, exc_val, exc_tb)
+
+        return False
+
+def sqlite(database: str, keyspace = None):
+    return SQLiteConnectionHandler(database, keyspace=keyspace)
+
+# no state utility class
+class SQLite:
+    def cursor(conn: sqlite3.Connection):
+        return CursorHandler(conn)
+
+    def local_db_dir():
+        return creating_dir(Config().get('export.sqlite.local-db-dir', '/tmp/qing-db/q/export/db'))
+
+    def keyspace(database: str):
+        return '_'.join(database.replace(".db", "").split('_')[1:])
+
+    @functools.lru_cache()
+    def database_names(prefix: str = None):
+        wait_log('Inspecting export databases...')
+
+        pattern = f'{SQLite.local_db_dir()}/s*.db'
+        if prefix:
+            pattern = f'{SQLite.local_db_dir()}/{prefix}*'
+        return [os.path.basename(f) for f in glob.glob(pattern)]
+
+    def clear_cache(cache: str = None):
+        SQLite.database_names.cache_clear()
+        SQLite.table_names.cache_clear()
+
+    @functools.lru_cache()
+    def table_names(database: str):
+      tokens = database.replace('.db', '').split('_')
+      ts_prefix = tokens[0]
+      keyspace = '_'.join(tokens[1:])
+
+      conn = None
+      tables = []
+      try:
+         conn = sqlite3.connect(f'{SQLite.local_db_dir()}/{ts_prefix}_{keyspace}.db')
+         with SQLite.cursor(conn) as cursor:
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;")
+
+            tables = [row[0] for row in cursor.fetchall() if row[0] != "sqlite_sequence"]
+
+         return tables
+      except sqlite3.Error as e:
+         print(f"Error connecting to or querying the database: {e}")
+         return []
+      finally:
+         if conn:
+               conn.close()
+
+    def connect(database: str, keyspace: str = None):
+        if keyspace:
+            return sqlite3.connect(f'{SQLite.local_db_dir()}/{database}_{keyspace}.db')
+        else:
+            conn = sqlite3.connect(f'{SQLite.local_db_dir()}/{database}_root.db')
+            with SQLite.cursor(conn) as cursor:
+                for d in SQLite.database_names(database):
+                    if d != f'{database}.db':
+                        q = f"ATTACH DATABASE '{SQLite.local_db_dir()}/{d}' AS {SQLite.keyspace(d)}"
+                        cursor.execute(q)
+
+            return conn
+
+    @functools.lru_cache()
+    def column_names(tables: list[str] = [], database: str = None, function: str = 'audit', partition_cols_only = False):
+        pass
+
+    def run_query(query: str, database: str = None, output: Callable[[str], str] = None) -> tuple[int, str]:
+        with sqlite(database) as conn:
+            return SQLite.run_query_with_conn(conn, query, output=output)
+
+    def run_query_with_conn(conn, query: str, output: Callable[[str], str] = None) -> tuple[int, str]:
+        log_file = None
+
+        df = SQLite.query(conn, query)
+        lines = ['\t'.join(map(str, line)) for line in df.values.tolist()]
+        out = tabulize(lines, header='\t'.join(df.columns.tolist()), separator='\t', to=0)
+        if output:
+            log_file = output(out)
+        else:
+            log(out)
+
+        return len(lines), log_file
+
+    def query(conn, sql: str) -> tuple[str, str, list]:
+        return pandas.read_sql_query(sql, conn)

adam/version.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-__version__ = "2.0.110"  #: the working version
+__version__ = "2.0.214"  #: the working version
 __release__ = "1.0.0"  #: the release version

{kaqing-2.0.110.dist-info → kaqing-2.0.214.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kaqing
-Version: 2.0.110
+Version: 2.0.214
 Summary: UNKNOWN
 Home-page: UNKNOWN
 License: UNKNOWN