qwak-sdk 0.5.102__py3-none-any.whl

qwak_sdk/tools/files.py

@@ -0,0 +1,63 @@
+import os
+import shutil
+
+
+def copytree(
+    src,
+    dst,
+    symlinks=False,
+    ignore=None,
+    ignore_dangling_symlinks=False,
+    dirs_exist_ok=False,
+):
+    names = os.listdir(src)
+    if ignore is not None:
+        ignored_names = ignore(src, names)
+    else:
+        ignored_names = set()
+
+    os.makedirs(dst, exist_ok=dirs_exist_ok)
+    errors = []
+    for name in names:
+        if name in ignored_names:
+            continue
+        srcname = os.path.join(src, name)
+        dstname = os.path.join(dst, name)
+        try:
+            if os.path.islink(srcname):
+                linkto = os.readlink(srcname)
+                if symlinks:
+                    # We can't just leave it to `copy_function` because legacy
+                    # code with a custom `copy_function` may rely on copytree
+                    # doing the right thing.
+                    os.symlink(linkto, dstname)
+                    shutil.copystat(srcname, dstname, follow_symlinks=not symlinks)
+                else:
+                    # ignore dangling symlink if the flag is on
+                    if not os.path.exists(linkto) and ignore_dangling_symlinks:
+                        continue
+                    # otherwise let the copy occurs. copy2 will raise an error
+                    if os.path.isdir(srcname):
+                        copytree(srcname, dstname, symlinks, ignore)
+                    else:
+                        shutil.copy2(srcname, dstname)
+            elif os.path.isdir(srcname):
+                copytree(srcname, dstname, symlinks, ignore)
+            else:
+                # Will raise a SpecialFileError for unsupported file types
+                shutil.copy2(srcname, dstname)
+        # catch the Error from the recursive copytree so that we can
+        # continue with other files
+        except shutil.Error as err:
+            errors.extend(err.args[0])
+        except OSError as why:
+            errors.append((srcname, dstname, str(why)))
+    try:
+        shutil.copystat(src, dst)
+    except OSError as why:
+        # Copying file access times may fail on Windows
+        if getattr(why, "winerror", None) is None:
+            errors.append((src, dst, str(why)))
+    if errors:
+        raise shutil.Error(errors)
+    return dst

qwak_sdk/tools/log_handling.py

@@ -0,0 +1,159 @@
+import time
+from datetime import datetime, timedelta, timezone
+
+from _qwak_proto.qwak.build.v1.build_pb2 import BuildStatus
+from qwak.clients.build_orchestrator import BuildOrchestratorClient
+from qwak.clients.logging_client import LoggingClient
+from qwak.exceptions import QwakException
+
+from qwak_sdk.tools.colors import Color
+
+INGESTED_TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S.%f"
+# Ingested timestamp arrives with format "1900-01-01T00:00:00.000000000Z" but python doesn't support
+# nanoseconds or Zulu
+INGESTED_TIMESTAMP_MAXSIZE = 26
+# Because the source of the timestamp truncates trailing zeros, we need to
+# support two formats for the timestamp that is exactly on the second
+ALTERNATIVE_INGESTED_TIMESTAMP_FORMAT_WITHOUT_DECIMALS = "%Y-%m-%dT%H:%M:%S"
+
+
+class QwakLogHandling:
+    TIME_UNITS = ["seconds", "minutes", "hours", "days"]
+    FOLLOW_SLEEP_IF_NO_RESULTS_SECS = 5
+    TIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
+    BUILD_PENDING_STATUS = ["IN_PROGRESS", "REMOTE_BUILD_INITIALIZING"]
+    BUILD_FINISHED_STATUS = [
+        "INVALID",
+        "SUCCESSFUL",
+        "FAILED",
+        "REMOTE_BUILD_CANCELLED",
+        "REMOTE_BUILD_TIMED_OUT",
+        "REMOTE_BUILD_UNKNOWN",
+    ]
+
+    def __init__(self):
+        self.log_formatter = self.LogFormatting()
+
+    @staticmethod
+    def get_build_status_name(build_client, build_id: str):
+        return BuildStatus.Name(build_client.get_build(build_id).build.build_status)
+
+    def get_logs(
+        self, follow, since, number_of_results, grep, source_params, source_name
+    ):
+        after = self.parse_since(since)
+        after = after.strftime(self.TIME_FORMAT) if after is not None else after
+
+        logging_client = LoggingClient()
+        if source_name == "build":
+            build_client = BuildOrchestratorClient()
+        while True:
+            params = {
+                **source_params,
+                "before_offset": None,
+                "after_offset": after,
+                "max_number_of_results": number_of_results,
+                "log_text_filter": grep,
+            }
+            if source_name == "runtime_model":
+                response = logging_client.read_model_runtime_logs(**params)
+            elif source_name == "build":
+                response = logging_client.read_build_logs(**params)
+            else:
+                raise QwakException(
+                    f"Provided a non existing source name [{source_name}]"
+                )
+
+            if response.log_line:
+                formatted_log_lines = self.log_formatter.format_logs(
+                    response.log_line, without_instance_id=True
+                )
+                print("\n".join(formatted_log_lines), end="")
+                after = response.last_offset
+            elif follow:
+                if source_name == "build":
+                    build_status = self.get_build_status_name(
+                        build_client, source_params["build_id"]
+                    )
+                    if build_status in self.BUILD_FINISHED_STATUS:
+                        break
+                time.sleep(self.FOLLOW_SLEEP_IF_NO_RESULTS_SECS)
+            else:
+                print("No logs returned")
+                return
+
+            if not follow:
+                break
+
+    def parse_since(self, since_sentence):
+        if since_sentence is None:
+            return None
+
+        timedelta_params = {}
+        split_line = since_sentence.split(" ")
+
+        for base_index in range(0, len(split_line) - 1, 2):
+            amount_string = split_line[base_index]
+            if not amount_string.isdigit():
+                raise QwakException(
+                    "First word of [--since] must be a number representing "
+                    "the amount of the time unit. (i.e. 3 minutes ago)"
+                )
+            time_unit = split_line[base_index + 1].lower()
+            if time_unit not in self.TIME_UNITS:
+                raise QwakException(
+                    f"Must provide one of: {self.TIME_UNITS} to [--since] paramater. (i.e. 2 hours ago)"
+                )
+            timedelta_params[time_unit] = int(amount_string)
+
+        return datetime.now(timezone.utc) - timedelta(**timedelta_params)
+
+    class LogFormatting:
+        def __init__(self):
+            self.instance_color_map = {}
+            self.colors = [
+                Color.PURPLE,
+                Color.CYAN,
+                Color.DARKCYAN,
+                Color.BLUE,
+                Color.GREEN,
+                Color.YELLOW,
+                Color.WHITE,
+                Color.GREY,
+            ]
+
+        def __format_datetime(self, timestamp_string):
+            if timestamp_string.endswith("Z"):
+                timestamp_string = timestamp_string[:-1]
+            try:
+                return f"{datetime.strptime(timestamp_string[:INGESTED_TIMESTAMP_MAXSIZE], INGESTED_TIMESTAMP_FORMAT)}\t"
+            except ValueError:
+                return f"{datetime.strptime(timestamp_string, ALTERNATIVE_INGESTED_TIMESTAMP_FORMAT_WITHOUT_DECIMALS)}\t"
+
+        def format_logs(self, log_lines, without_instance_id: bool = False):
+            colors_lines = []
+
+            for line in log_lines:
+                if line.source_instance_id not in self.instance_color_map:
+                    source_id = len(self.instance_color_map) + 1
+                    self.instance_color_map[line.source_instance_id] = {
+                        "id": source_id,
+                        "color": self.colors[source_id % len(self.colors)],
+                    }
+
+                instance_params = self.instance_color_map[line.source_instance_id]
+                datetime_prefix = self.__format_datetime(line.ingested_iso_timestamp)
+                prefix = ""
+                if "level" in line.metadata:
+                    prefix = f"{prefix}{line.metadata['level']} - "
+
+                if "phase" in line.metadata:
+                    prefix = f"{prefix}{line.metadata['phase']} - "
+
+                if without_instance_id:
+                    formatted_line = f'{instance_params["color"]}{datetime_prefix}{prefix}{line.text}{Color.END}'
+                else:
+                    formatted_line = f'{datetime_prefix}{instance_params["id"]} - {instance_params["color"]}{prefix}{line.text}{Color.END}'
+                colors_lines.append(formatted_line)
+
+            return colors_lines

qwak_sdk/tools/utils.py

@@ -0,0 +1,42 @@
+import logging
+import sys
+from contextlib import contextmanager
+from typing import Callable, Optional
+
+from qwak.tools.logger.logger import get_qwak_logger, get_qwak_logger_verbosity_level
+from yaspin import yaspin
+
+logger = get_qwak_logger()
+
+
+@contextmanager
+def qwak_spinner(
+    begin_text: Optional[str],
+    end_text: Optional[str] = "",
+    print_callback: Callable[[str], None] = logger.info,
+):
+    """Qwak spinner.
+
+    Args:
+        begin_text: Text to shown when spinner starts.
+        end_text: Text to shown when spinner ends.
+        print_callback: Callback used to print the output.
+    """
+    if (
+        logging.getLevelName(get_qwak_logger_verbosity_level()) < logging.WARNING
+        if print_callback != print
+        else False
+    ) or not sys.stdout.isatty():
+        print_callback(begin_text)
+        yield
+        print_callback(end_text)
+    else:
+        with yaspin(text=begin_text, color="blue", timer=True) as sp:
+            try:
+                yield sp
+            except Exception as e:
+                sp.fail("💥")
+                raise e
+            if end_text:
+                sp.text = end_text
+            sp.ok("✅")

qwak_sdk-0.5.102.dist-info/METADATA

@@ -0,0 +1,51 @@
+Metadata-Version: 2.3
+Name: qwak-sdk
+Version: 0.5.102
+Summary: Qwak SDK and CLI for qwak models
+License: Apache-2.0
+Keywords: mlops,ml,deployment,serving,model
+Author: Qwak
+Author-email: info@qwak.com
+Requires-Python: >=3.9,<3.12
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Provides-Extra: batch
+Provides-Extra: feature-store
+Provides-Extra: feedback
+Provides-Extra: local-deployment
+Requires-Dist: boto3 (>=1.24.116,<2.0.0) ; extra == "batch" or extra == "feedback" or extra == "local-deployment"
+Requires-Dist: click (==8.1.8)
+Requires-Dist: cloudpickle (==2.2.1) ; extra == "feature-store"
+Requires-Dist: cookiecutter
+Requires-Dist: croniter (==1.4.1)
+Requires-Dist: gitpython (>=2.1.0)
+Requires-Dist: joblib (>=1.1.0,<2.0.0) ; extra == "batch" or extra == "feedback"
+Requires-Dist: numpy (<2.0.0)
+Requires-Dist: pandas (>1.1.3) ; (python_version == "3.9") and (extra == "batch" or extra == "feedback" or extra == "feature-store")
+Requires-Dist: pandas (>=1.4.0) ; (python_version >= "3.10" and python_version < "3.12") and (extra == "batch" or extra == "feedback" or extra == "feature-store")
+Requires-Dist: pyarrow (>=6.0.0) ; extra == "batch" or extra == "feature-store"
+Requires-Dist: pyathena (>=2.2.0,!=2.18.0) ; extra == "feature-store"
+Requires-Dist: pyspark (==3.4.2) ; extra == "feature-store"
+Requires-Dist: python-json-logger (>=3.2.1,<4.0.0)
+Requires-Dist: qwak-core (>=0.5.7)
+Requires-Dist: qwak-inference (>=0.1.25,<0.2.0)
+Requires-Dist: rich (>=13.0.0)
+Requires-Dist: tabulate (>=0.8.0)
+Requires-Dist: yaspin (>=2.0.0)
+Project-URL: Home page, https://www.qwak.com/
+Description-Content-Type: text/markdown
+
+# Qwak SDK
+
+Qwak is an end-to-end production ML platform designed to allow data scientists to build, deploy, and monitor their models in production with minimal engineering friction.
+
+### Qwak SDK
+