pykx 4.2.0__cp315-cp315-win_amd64.whl

pykx/__init__.py

@@ -0,0 +1,586 @@
+"""
+An interface between Python and q.
+"""
+
+import base64
+import logging
+import os
+import sys
+
+if os.getenv('PYKX_LOADING_Q', '') == '':
+    os.environ['PYKX_LOADING_PYTHON'] = 'true'
+
+if os.getenv('PYKX_OLD_QHOME', '') != '' and os.getenv('PYKX_LOADING_PYTHON', '') != '':
+    os.environ['QHOME'] = os.getenv('PYKX_OLD_QHOME', "")
+if os.getenv('PYKX_OLD_QPATH', '') != '' and os.getenv('PYKX_LOADING_PYTHON', '') != '':
+    os.environ['QPATH'] = os.getenv('PYKX_OLD_QPATH', "")
+
+if os.getenv('PYKX_LOADED_UNDER_Q') == 'True':
+    sys.exit(0)
+
+# Attempt to import PyArrow early to get ahead of others (e.g. Pandas) who would try to import it
+# without guarding against segfaults. Skip this if we're just doing a qinit check.
+if os.getenv('PYKX_QINIT_CHECK', '') == '':
+    try:
+        from ._pyarrow import pyarrow
+    except ImportError: # nocov
+        pass
+else: # nocov
+    pass
+
+
+# List of beta features available in the current KDB-X Python version
+beta_features = []
+
+
+from . import reimporter
+# Importing core initializes q if in licensed mode, and loads the q C API symbols. This should
+# happen early on so that if the qinit check is currently happening then no time is wasted.
+# The current process will exit while the core module is loading if the qinit check is happening.
+from . import core
+
+from abc import ABCMeta, abstractmethod
+import itertools as it
+from pathlib import Path
+import platform
+import shutil
+import signal
+from typing import Any, List, Optional, Union
+from warnings import warn
+from weakref import proxy
+
+from .config import k_allocator, licensed, no_pykx_signal, pykx_lib_dir, pykx_platlib_dir, under_q
+from . import util
+
+if platform.system() == 'Windows': # nocov
+    os.environ['PATH'] += f';{pykx_platlib_dir}'
+    if platform.python_version_tuple()[:2] >= ('3', '8'):
+        os.add_dll_directory(pykx_platlib_dir)
+
+# Cache initialised signal values prior to KDB-X Python loading
+_signal_list = [
+    'signal.SIGINT',
+    'signal.SIGTERM',
+]
+
+_signal_dict = {}
+
+for i in _signal_list:
+    _signal_dict[i] = signal.getsignal(eval(i))
+
+
+def _first_resolved_path(possible_paths: List[Union[str, Path]]) -> Path:
+    """Returns the resolved version of the first path that exists."""
+    for path in possible_paths:
+        try:
+            return Path(path).resolve(strict=True)
+        except FileNotFoundError:
+            pass
+    unfound_paths = '\n'.join(str(path) for path in possible_paths)
+    raise FileNotFoundError(f'Could not find any of the following files:\n{unfound_paths}')
+
+
+class Q(metaclass=ABCMeta):
+    """Abstract base class for all interfaces between Python and q.
+
+    See Also:
+        - [`pykx.EmbeddedQ`][pykx.EmbeddedQ]
+        - [`pykx.QConnection`][pykx.QConnection]
+    """
+    reserved_words = {
+        'abs', 'acos', 'aj', 'aj0', 'ajf', 'ajf0', 'all', 'and', 'any', 'asc',
+        'asin', 'asof', 'atan', 'attr', 'avg', 'avgs', 'bin',
+        'binr', 'ceiling', 'cols', 'cor', 'cos', 'count', 'cov', 'cross',
+        'csv', 'cut', 'delete', 'deltas', 'desc', 'dev', 'differ', 'distinct',
+        'div', 'do', 'dsave', 'each', 'ej', 'ema', 'enlist', 'eval',
+        'except', 'exec', 'exit', 'exp', 'fby', 'fills', 'first', 'fkeys',
+        'flip', 'floor', 'get', 'getenv', 'group', 'gtime', 'hclose', 'hcount', 'hdel',
+        'hopen', 'hsym', 'iasc', 'idesc', 'if', 'ij', 'ijf', 'in',
+        'inter', 'inv', 'key', 'keys', 'last', 'like', 'lj', 'ljf',
+        'load', 'log', 'lower', 'lsq', 'ltime', 'ltrim', 'mavg', 'max', 'maxs',
+        'mcount', 'md5', 'mdev', 'med', 'meta', 'min', 'mins', 'mmax', 'mmin', 'mmu', 'mod',
+        'msum', 'neg', 'next', 'not', 'null', 'or', 'over', 'parse', 'peach', 'pj', 'prd',
+        'prds', 'prev', 'prior', 'rand', 'rank', 'ratios', 'raze', 'read0', 'read1', 'reciprocal',
+        'reval', 'reverse', 'rload', 'rotate', 'rsave', 'rtrim', 'save', 'scan', 'scov', 'sdev',
+        'select', 'set', 'setenv', 'show', 'signum', 'sin',
+        'sqrt', 'ss', 'ssr', 'string', 'sublist', 'sum', 'sums', 'sv', 'svar',
+        'system', 'tables', 'tan', 'til', 'trim', 'type', 'uj', 'ujf', 'ungroup',
+        'union', 'update', 'upper', 'value', 'var', 'view', 'views',
+        'vs', 'wavg', 'where', 'while', 'within', 'wj', 'wj1',
+        'wsum', 'xasc', 'xbar', 'xcol', 'xcols', 'xdesc', 'xexp', 'xgroup', 'xkey', 'xlog',
+        'xprev', 'xrank'
+    }
+
+    operators = {
+        'drop': '_',
+        'coalesce': '^', 'fill': '^',
+        'take': '#', 'set_attribute': '#',
+        'join': ',',
+        'find': '?', 'enum_extend': '?', 'roll': '?', 'deal': '?',
+        'dict': '!', 'enkey': '!', 'unkey': '!', 'enumeration': '!',
+        'enumerate': '$', 'pad': '$', 'cast': '$', 'tok': '$',
+        'compose': '\''
+    }
+
+    def __init__(self):
+        self.paths = default_paths
+        # Start with an empty set to avoid errors during initialization.
+        object.__setattr__(self, '_q_ctx_keys', set())
+        # HACK: `'_connection_info' in self.__dict__` is used as a proxy for a type-check to avoid a
+        # cyclic import.
+        if licensed or '_connection_info' in self.__dict__:
+            if '_connection_info' in self.__dict__ and self._connection_info['no_ctx']:
+                object.__setattr__(self, 'ctx', QContext(proxy(self), '', None, no_ctx=True))
+            else:
+                object.__setattr__(self, 'ctx', QContext(proxy(self), '', None))
+                object.__setattr__(self, '_q_ctx_keys', {
+                    *self.ctx.q._context_keys,
+                    *self.reserved_words,
+                })
+        object.__setattr__(self, 'console', QConsole(proxy(self)))
+        object.__setattr__(self, 'insert', Insert(proxy(self)))
+        object.__setattr__(self, 'upsert', Upsert(proxy(self)))
+        object.__setattr__(self, 'query', None) # placeholder
+        object.__setattr__(self, 'qsql', QSQL(proxy(self)))
+        object.__setattr__(self, 'sql', SQL(proxy(self)))
+        object.__setattr__(self, 'read', QReader(proxy(self)))
+        object.__setattr__(self, 'write', QWriter(proxy(self)))
+        object.__setattr__(self, 'system', SystemCommands(proxy(self)))
+
+    @abstractmethod
+    def __call__(self, query: str, *args, wait: Optional[bool] = None):
+        pass # nocov
+
+    def __getattr__(self, key):
+        if key == "__objclass__":
+            raise AttributeError
+        # Elevate the q context to the global context, as is done in q normally
+        try:
+            ctx = self.__getattribute__('ctx')
+        except BaseException:
+            raise exceptions.QError('Cannot load requested context object in unlicensed mode')
+        if key in self.__getattribute__('_q_ctx_keys'):
+            # if-statement used instead of try-block for performance
+            return ctx.q.__getattr__(key)
+        try:
+            return ctx.__getattr__(key)
+        except AttributeError as attribute_error:
+            try:
+                self.__getattribute__('_register')(name=key)
+                ctx._invalidate_cache()
+                return ctx.__getattr__(key)
+            except Exception as inner_error:
+                raise attribute_error from inner_error
+
+    # __setattr__ takes precedence over data descriptors, so we implement a custom one that uses
+    # the default behavior for select keys
+    def __setattr__(self, key, value):
+        if key in self.__dict__ or key in Q.__dict__:
+            object.__setattr__(self, key, value)
+        else:
+            if key in self.reserved_words or key in self.q:
+                raise exceptions.PyKXException(
+                    'Cannot assign to reserved word or overwrite q namespace.'
+                )
+            self._call('set', f'.{key}', value, wait=True)
+
+    def __delattr__(self, key):
+        self.ctx.__delattr__(key)
+
+    def __getitem__(self, key):
+        return self._call('get', key, wait=True)
+
+    def __setitem__(self, key, value):
+        if key in self.reserved_words or key in self.q:
+            raise exceptions.PyKXException(
+                'Cannot assign to reserved word or overwrite q namespace.'
+            )
+        self._call('set', key, value, wait=True)
+
+    def __delitem__(self, key):
+        key = str(key)
+        if key.startswith('.'):
+            raise exceptions.PyKXException('Cannot delete from the global context.')
+        elif key not in self._call('key `.', wait=True).py():
+            raise KeyError(key)
+        self._call('{![`.;();0b;enlist x]}', key, wait=True) # delete key from `.
+
+    def __dir__(self):
+        if hasattr(self, 'ctx'): # Temporary workaround to disable the context interface over IPC.
+            return sorted({
+                *super().__dir__(),
+                *dir(self.ctx),
+                *dir(self.ctx.__getattr__('q')),
+            })
+        else:
+            return super().__dir__()
+
+    def _register(self,
+                  name: Optional[str] = None,
+                  path: Optional[Union[Path, str]] = None,
+    ) -> str:
+        """Switch to a named q context (read [Q for Mortals
+        Chapter 12](https://code.kx.com/q4m3/12_Workspace_Organization/#122-contexts))
+        and load variable definitions into that context from a q/k script. Once the script is
+        loaded this function switches back to the previous q context.
+
+        Parameters:
+            name: Name to assign the context being loaded. If no argument is provided the
+                assigned name of the context is set to the name of the file without filetype
+                extension.
+            path: Path to the script to load. If no argument is provided this function
+                searches for a file matching the given name, loading it if found.
+
+        Returns:
+            The attribute name for the newly loaded module.
+        """
+        if name is None and path is None:
+            raise ValueError('Module name or path must be provided.')
+        if path is not None:
+            path = Path(path).resolve(strict=True)
+        else:
+            path = _first_resolved_path([''.join(x) for x in it.product(
+                # `str(Path/@)[:-1]` gets the path with a trailing path separator
+                (str(x/Path('@'))[:-1] for x in self.paths),
+                ('.', ''),
+                (name,),
+                ('.q', '.k'),
+                ('', '_')
+            )])
+        if " " in str(path) and not suppress_warnings:
+            warn("""Space detected in path being loaded.
+                     KDB-X Python will change directory to the path parent and then load directly before returning to current working directory.
+                     To turn off this warning set PYKX_SUPPRESS_WARNINGS to True.""", exceptions.PyKXWarning) # noqa: E501
+        if name is None: # defaults to filename at end of path sans extension
+            name = path.stem
+        prev_ctx = self._call('string system"d"', wait=True)
+        try:
+            self._call('''{[name;folder;file]
+                            name set (enlist`)!enlist(::);
+                            system "d ",string name;
+                            $[@[{get x;1b};`.pykx.util.loadfile;{0b}];
+                              .pykx.util.loadfile[folder;file];
+                              system"l ",$[.z.o like "w*";"\\\\";"/"] sv ((),folder;(),file)]}
+                       ''', "" if name[0] == "." else "."+name, str(path.parent).encode(),
+                       path.name.encode(), wait=True,
+            )
+            return name[1:] if name[0] == '.' else name
+        finally:
+            self._call('{system"d ",x}', prev_ctx, wait=True)
+
+    @property
+    def paths(self):
+        """List of locations for the context interface to find q scripts in.
+
+        Defaults to the current working directory and `#!bash $QHOME`.
+
+        If you change directories, the current working directory stored in this list
+        automatically reflects that change.
+        """
+        return object.__getattribute__(self, '_paths')
+
+    @paths.setter
+    def paths(self, paths: List[Union[str, Path]]):
+        resolved = []
+        for path in paths:
+            if not isinstance(path, Path):
+                path = Path(path)
+            try:
+                resolved.append(path.resolve(strict=True))
+            except FileNotFoundError:
+                warn(f"Module path '{path!r}' not found", RuntimeWarning)
+        object.__setattr__(self, '_paths', resolved)
+
+
+# Import order matters here, so the imports are not ordered conventionally.
+from .serialize import deserialize, serialize
+from .console import QConsole
+from .ctx import default_paths, QContext
+from .query import Insert, QSQL, SQL, Upsert
+from .read import QReader
+from .system import SystemCommands
+from .write import QWriter
+from .reimporter import PyKXReimport
+from .module import QModule, QNamespaceProxy, use
+
+from . import config
+from . import console
+from . import exceptions
+from . import wrappers
+from . import schema
+from . import streamlit
+from . import random
+from . import help
+
+from ._wrappers import _init as _wrappers_init
+_wrappers_init(wrappers)
+
+from .embedded_q import EmbeddedQ, EmbeddedQFuture, q
+from ._version import version as __version__
+from .exceptions import *
+
+from .util import _init as _util_init
+_util_init(q)
+
+from ._ipc import _init as _ipc_init
+_ipc_init(q)
+
+from .compress_encrypt import Compress, CompressionAlgorithm, Encrypt
+from .db import DB
+from .tick import TICK
+from .ipc import AsyncQConnection, QConnection, QFuture, RawQConnection, SecureQConnection, SyncQConnection # noqa
+from .config import qargs, qhome, qlic, suppress_warnings
+from .wrappers import *
+from .wrappers import CharVector, K
+
+from ._ipc import ssl_info
+
+from .schema import _init as _schema_init
+_schema_init(q)
+
+from .register import _init as _register_init
+_register_init(q)
+
+from .license import _init as _license_init
+_license_init(q)
+
+from .random import _init as _random_init
+_random_init(q)
+
+from .db import _init as _db_init
+_db_init(q)
+
+from .tick import _init as _tick_init
+_tick_init(q)
+
+from .remote import _init as _remote_init
+_remote_init(q)
+
+from .compress_encrypt import _init as _compress_init
+_compress_init(q)
+
+from .help import _init as _help_init
+_help_init(q)
+qhelp = help.qhelp
+
+from .module import _init as _module_init
+_module_init(q)
+
+if k_allocator:
+    from . import _numpy as _pykx_numpy_cext
+
+
+def merge_asof(left, *args, **kwargs):
+    return left.merge_asof(*args, **kwargs)
+
+
+if sys.version_info[1] < 9:
+    warn(
+        'Python 3.9 is the minimum support version. Please upgrade.',
+        exceptions.PyKXWarning
+    )
+
+
+def install_into_QHOME(overwrite_embedpy=False,
+                       to_local_folder=False,
+                       cloud_libraries=False) -> None:
+    """Copies the embedded Python functionality of KDB-X Python into `$QHOME`.
+
+        Parameters:
+            overwrite_embedpy: If embedPy had previously been installed replace it otherwise
+                save functionality as pykx.q.
+            to_local_folder: Copy the files to your local folder using True, or specify '/some/path'
+              rather than `#!bash QHOME`.
+            cloud_libraries: Copy cloud libraries to `#!bash QHOME`.
+
+        Returns:
+            None
+    """
+    if isinstance(to_local_folder, (str, Path)):
+        dest = Path(to_local_folder)
+    elif to_local_folder:
+        dest = Path('.')
+    else:
+        dest = qhome
+    if dest == pykx_lib_dir:
+        warn(f"""
+             The destination {dest} matches pykx.config.pykx_lib_dir which already contains pykx.q,
+             no files will be copied.
+             If your environment does not have QHOME set or you wish to control where pykx.q
+             is installed use to_local_folder:
+                 to_local_folder = True will install pykx.q to your current working directory
+                 to_local_folder = 'path/to/folder' will install pykx.q to the specified path
+            """)
+        return
+
+    p = Path(dest)/'p.k'
+    c_files = ['kurl.q_', 'kurl.sidecar.q_', 'objstor.q_', 'qlog.q_', 'rest.q_', 'bq.q_']
+
+    if not p.exists() or overwrite_embedpy:
+        shutil.copy(Path(__file__).parent/'p.k', p)
+    if overwrite_embedpy:
+        shutil.copy(Path(__file__).parent/'p.k', Path(dest)/'p.q')
+    if cloud_libraries:
+        for i in c_files:
+            shutil.copy(Path(__file__).parent/'lib'/i, Path(dest)/i)
+        shutil.copy(Path(__file__).parent/'lib'/'sql.k_', Path(dest)/'s.k_')
+    shutil.copy(Path(__file__).parent/'pykx.q', dest)
+    shutil.copy(Path(__file__).parent/'pykx_init.q_', dest)
+
+
+def activate_numpy_allocator() -> None:
+    """Sets the allocator used for Numpy array data to one optimized for use with KDB-X Python.
+
+    This will only change the default allocator if the environment variable `PYKX_NO_ALLOCATOR`
+    is not set to 1 or if the flag `--pykxnoalloc` is not present in the QARGS environment variable.
+
+    The name of the allocator set by this function is `'pykx_allocator'`.
+
+    The name of the active Numpy allocator can be retrieved by running
+    `numpy.core.multiarray.get_handler_name()`.
+
+    Refer to NEP 49 for details about custom Numpy allocators: https://numpy.org/neps/nep-0049.html
+
+    The custom allocator set by KDB-X Python allocates Numpy array data into the embedded q memory
+    space.
+    Numpy arrays created with this allocator can be converted into a q vector without copying the
+    data.
+
+    Because q objects must have their metadata immediately preceding the data, only a single
+    q vector can be created using this approach. Repeated conversions of the Numpy array into a q
+    vector will yield the same q vector with its reference count incremented by 1 each time.
+
+    Since Numpy stores its metadata separately from its data, multiple views can be taken over the
+    data without incurring a copy. The same is not true for q vectors. If a Numpy array is
+    converted into a q vector, then a view is taken of the Numpy array, this results in the view
+    being converted into a q vector and a copy of the data being incurred. Regardless of whether
+    this custom Numpy allocator is activated.
+
+    The Numpy array data object holds a reference to the q vector. When the Numpy array data object
+    is deallocated, it decrements the refcount of the q vector.
+
+    If the installed version of Numpy support custom allocators (i.e. is greater than or equal to
+    version `1.22.0`), and the active Numpy allocator when KDB-X Python is imported is the default
+    allocator, KDB-X Python will automatically call this function.
+
+    Note that Numpy arrays created with an allocator other than `'pykx_allocator'` (e.g. before
+    KDB-X Python has been imported) will not gain the benefits enabled by this allocator.
+    The name of the allocator used by a given array `x` can be retrieved by running
+    `numpy.core.multiarray.get_handler_name(x)`.
+    """
+    if k_allocator:
+        _pykx_numpy_cext.activate_pykx_allocators()
+
+
+import numpy as np
+if k_allocator:
+    _pykx_numpy_cext.init_numpy_ctx(core._r0_ptr, core._k_ptr, core._ktn_ptr)
+    if np.__version__[0] == '1':
+        handler = np.core.multiarray.get_handler_name()
+    else:
+        handler = np._core.multiarray.get_handler_name()
+    if handler == 'default_allocator':
+        activate_numpy_allocator()
+
+
+def deactivate_numpy_allocator():
+    """Deactivate the KDB-X Python Numpy allocator.
+
+    If the KDB-X Python Numpy allocator has not been activated then this function has no effect.
+    Otherwise, it replaces the KDB-X Python Numpy allocator with the allocator that was in use
+    when it was first activated.
+    """
+    if k_allocator:
+        _pykx_numpy_cext.deactivate_pykx_allocators()
+
+
+try:
+    # If we are running under IPython/Jupyter...
+    ipython = get_ipython()  # noqa
+    # Load the KDB-X Python extension for Jupyter Notebook.
+    ipython.extension_manager.load_extension('pykx.nbextension')
+
+    if config.jupyterq:
+        util.jupyter_qfirst_enable()
+except NameError:
+    # Not running under IPython/Jupyter...
+    pass
+
+shutdown_thread = core.shutdown_thread
+
+if licensed:
+    days_to_expiry = q('"D"$', q.z.l[1]) - q.z.D
+    if days_to_expiry < 10:
+        logging.warning(f'KDB-X Python license set to expire in {int(days_to_expiry)} days, '
+                        'please consider installing an updated license')
+
+__all__ = sorted([
+    'AsyncQConnection',
+    'deserialize',
+    'EmbeddedQ',
+    'EmbeddedQFuture',
+    'Q',
+    'qargs',
+    'QConnection',
+    'QConsole',
+    'QContext',
+    'QFuture',
+    'qhome',
+    'QReader',
+    'random',
+    'remote',
+    'QWriter',
+    'qlic',
+    'serialize',
+    'SyncQConnection',
+    'RawQConnection',
+    'activate_numpy_allocator',
+    'console',
+    'ctx',
+    'deactivate_numpy_allocator',
+    'exceptions',
+    'install_into_QHOME',
+    'licensed',
+    'toq',
+    'schema',
+    'config',
+    'util',
+    'q',
+    'QModule',
+    'QNamespaceProxy',
+    'use',
+    'shutdown_thread',
+    'PyKXReimport',
+    'help',
+    'qhelp',
+    *exceptions.__all__,
+    *wrappers.__all__,
+])
+
+if not no_pykx_signal:
+    for k, v in _signal_dict.items():
+        try:
+            signal.signal(eval(k), v)
+        except Exception:
+            pass
+
+
+def __dir__():
+    return __all__
+
+
+envlist = (
+    'PYKX_UNDER_Q',
+    'PYKX_UNDER_PYTHON',
+    'PYKX_Q_LOADED_MARKER',
+    'PYKX_LOADED_UNDER_Q',
+    'PYKX_LOADING_PYTHON',
+    'PYKX_LOADING_Q',
+    'PYKX_QINIT_CHECK',
+)
+if os.getenv('PYKX_LOADING_PYTHON', '') != '':
+    for x in envlist:
+        os.environ[x] = ''
+        del os.environ[x]
+        if licensed:
+            q(f'setenv[`{x}; ""]')