tensorwatch-api 0.1.0__py3-none-any.whl

tensorwatch_api/__init__.py

@@ -0,0 +1,11 @@
+import warnings
+import logging
+
+warnings.filterwarnings("ignore", category=UserWarning, module="pykafka")
+warnings.filterwarnings("ignore", category=UserWarning, module='pkg_resources')
+warnings.filterwarnings("ignore",message=".*cache_frame_data.*",category=UserWarning  )
+
+# Suppress "No partitions assigned" warnings from pykafka.balancedconsumer, which can be noisy during rebalancing.
+logging.getLogger('pykafka.balancedconsumer').setLevel(logging.ERROR)
+
+from .twapi import twapi

tensorwatch_api/twapi.py

@@ -0,0 +1,191 @@
+import tensorwatchext as tw
+from tensorwatchext import kafka_connector as kc
+from tensorwatchext import pykafka_connector as pyc
+from IPython.display import display
+from ipywidgets import widgets
+import time
+import logging
+import matplotlib.pyplot as plt
+
+class twapi:
+    """TensorWatch API Wrapper for Kafka Streaming and Visualization"""
+
+    def __init__(self):
+        """Initializes the twapi class, setting up the UI widgets and event handlers."""
+        self.default_value = 10
+        self.visualizer = None  # Initialize visualizer as None
+        self.client = tw.WatcherClient()
+        self.out = widgets.Output(layout={})
+
+        # Initialize UI widgets
+        self.update_interval = 0.5  # Delay in seconds
+        self.my_slider = widgets.IntSlider(value=self.default_value, min=1, max=100, step=1, description="Window Size:")
+        self.my_slider2 = widgets.IntSlider(value=self.default_value, min=1, max=100, step=1, description="Window Width:")
+        self.datebutton = widgets.Checkbox(value=False, description="Date")
+        self.offsetbutton = widgets.Checkbox(value=False, description="Use Offset")
+        self.dimhistorybutton = widgets.Checkbox(value=True, description="Dim History")
+        self.colorpicker = widgets.ColorPicker(value="blue", description="Pick a Color")
+        
+        self.button_reset = widgets.Button(description="Reset", tooltip="Reset stream settings")        
+        self.button_apply = widgets.Button(description="Please wait", tooltip="Apply changes to the visualization", disabled=True)
+
+        # Group widgets for a cleaner UI
+        left_box = widgets.VBox([self.my_slider, self.my_slider2, self.colorpicker])
+        right_box = widgets.VBox([self.offsetbutton, self.dimhistorybutton, self.datebutton])
+        self.options_box = widgets.HBox([left_box, right_box])
+        self.accordion = widgets.Accordion(children=[self.options_box])
+        self.accordion.set_title(0, 'Visualization Options')
+
+        # Event handlers
+        self._last_update = time.time()
+        self.button_reset.on_click(self.reset)
+        self.button_apply.on_click(self.apply_with_debounce)
+        self.metrics_label = widgets.Label(value="")
+
+        # Observe widget changes directly
+        self.my_slider.observe(self.apply_with_debounce, names='value')
+        self.my_slider2.observe(self.apply_with_debounce, names='value')
+        self.colorpicker.observe(self.apply_with_debounce, names='value')
+
+    def stream(self, expr):
+        """Creates a TensorWatch stream from an expression."""
+        self.expr = expr
+        try:
+            self.streamdata = self.client.create_stream(expr=expr)
+            logging.debug("Stream created successfully")
+        except Exception as e:
+            logging.error(f"Error creating stream: {e}")
+            print(f"Error creating stream: {e}")
+        return self
+
+    def apply_with_debounce(self, _=None):
+        """Debounced apply function to prevent too frequent updates."""
+        now = time.time()
+        if now - self._last_update > self.update_interval:
+            self.update_visualizer()
+            self._last_update = now
+            if self.button_apply.description == "Start":
+                self.button_apply.description = "Apply Changes"
+
+    def update_visualizer(self, _=None):
+        """Updates the TensorWatch visualizer with the latest widget values."""
+        if not hasattr(self, 'streamdata') or not self.streamdata:
+            self.out.clear_output(wait=True)
+            with self.out:
+                print("Stream data not available or empty yet. Please wait for data.")
+            return
+
+        try:
+            # Always clear output before drawing
+            self.out.clear_output(wait=True)
+
+            # Close previous visualizer if it exists to free resources
+            if self.visualizer:
+                # self.visualizer.close()
+                try:    
+                    plt.pause(0.05)
+                    plt.close('all') # Also close any lingering matplotlib figures
+                except Exception:
+                    pass
+            # Create a new visualizer with the current settings
+            self.visualizer = tw.Visualizer(
+                self.streamdata,
+                vis_type="line",
+                window_width=self.my_slider2.value,
+                window_size=self.my_slider.value,
+                Date=self.datebutton.value,
+                useOffset=self.offsetbutton.value,
+                dim_history=self.dimhistorybutton.value,
+                color=self.colorpicker.value,
+            )
+            with self.out:
+                self.visualizer.show()
+                
+        except Exception as e:
+            self.out.clear_output(wait=True)
+            with self.out:
+                print(f"Error updating visualizer: {e}")
+
+    def enable_apply_button(self):
+        """Enables the apply button and changes its description to 'Start'."""
+        logging.debug("Enabling apply button.")
+        self.button_apply.disabled = False
+        self.button_apply.description = "Start"
+
+    def reset(self, _=None):
+        """Resets all widget values to their defaults and clears the visualization."""
+        self.my_slider.value = self.default_value
+        self.my_slider2.value = self.default_value
+        self.datebutton.value = False
+        self.offsetbutton.value = False
+        self.dimhistorybutton.value = True
+        self.colorpicker.value = "blue"
+        
+        # Clear the output and close the visualizer
+        self.out.clear_output()
+        plt.close('all')
+        self.visualizer = None
+
+    def draw(self):
+        """Displays the UI for controlling the visualization."""
+        ui = widgets.VBox([
+            widgets.HBox([self.button_reset, self.button_apply]),
+            self.accordion,
+            self.out
+        ])
+        display(ui)
+
+    def draw_with_metrics(self):
+        """Displays the UI for controlling the visualization with a metrics label."""
+        ui = widgets.VBox([
+            self.metrics_label,
+            widgets.HBox([self.button_reset, self.button_apply]),
+            self.accordion,
+            self.out
+        ])
+        display(ui)
+
+    def update_metrics(self, metrics):
+        """Updates the metrics label with the provided text."""
+        self.metrics_label.value = metrics
+
+    def connector(self, topic, host, parsetype="json", cluster_size=1, conn_type="kafka", queue_length=50000,
+                  group_id="mygroup", schema_path=None, protobuf_message=None, parser_extra=None,
+                  random_sampling=None, countmin_width=None, countmin_depth=None, ordering_field=None):
+        """
+        Creates and returns a Kafka or PyKafka connector.
+
+        Args:
+            topic (str): The Kafka topic to consume from.
+            host (str): The Kafka broker host.
+            parsetype (str): The message format (e.g., 'json', 'pickle', 'avro').
+            cluster_size (int): The number of consumer threads.
+            conn_type (str): The type of connector to use ('kafka' or 'pykafka').
+            queue_length (int): The maximum size of the message queue.
+            group_id (str): The Kafka consumer group ID.
+            schema_path (str): The path to the schema file.
+            protobuf_message (str): The name of the Protobuf message class.
+            parser_extra (str): Extra data for the parser (e.g., Avro schema for 'pykafka').
+            random_sampling (int): The percentage of messages to sample.
+            countmin_width (int): The width of the Count-Min Sketch.
+            countmin_depth (int): The depth of the Count-Min Sketch.
+
+        Returns:
+            A KafkaConnector or pykafka_connector instance.
+        """
+        if conn_type == "kafka":
+            return kc(
+                topic=topic, hosts=host, parsetype=parsetype, cluster_size=cluster_size, queue_length=queue_length, group_id=group_id,
+                 schema_path=schema_path, protobuf_message=protobuf_message,parser_extra=parser_extra,
+                random_sampling=random_sampling, countmin_width=countmin_width,ordering_field=ordering_field,
+                countmin_depth=countmin_depth,
+                twapi_instance=self)
+        elif conn_type == "pykafka":
+            return pyc(
+                topic=topic, hosts=host, parsetype=parsetype, cluster_size=cluster_size,twapi_instance=self, 
+                queue_length=queue_length, consumer_group=bytes(group_id, 'utf-8'),
+                parser_extra=parser_extra, schema_path=schema_path, protobuf_message=protobuf_message,
+                random_sampling=random_sampling, countmin_width=countmin_width,ordering_field=ordering_field,
+                countmin_depth=countmin_depth)
+        else:
+            raise ValueError("Invalid connector type. Choose 'kafka' or 'pykafka'.")

tensorwatch_api-0.1.0.dist-info/METADATA

@@ -0,0 +1,141 @@
+Metadata-Version: 2.4
+Name: tensorwatch-api
+Version: 0.1.0
+Summary: Interactive Kafka stream visualization API for Jupyter using TensorWatch.
+Home-page: https://github.com/costasrevi/jupyterstreamvis
+Author: Konstantinos Revythis
+Author-email: Konstantinos Revythis <krevythis@tuc.gr>
+License: MIT
+Project-URL: Homepage, https://github.com/costasrevi/jupyterstreamvis
+Project-URL: Source, https://github.com/costasrevi/jupyterstreamvis
+Keywords: kafka,tensorwatch,jupyter,streaming,visualization
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: tensorwatch
+Requires-Dist: ipywidgets
+Requires-Dist: matplotlib
+Requires-Dist: confluent-kafka; extra == "kafka"
+Requires-Dist: pykafka; extra == "pykafka"
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+
+# TensorWatch API (`tensorwatch-api`)
+
+`tensorwatch-api` is a Python library designed to simplify **real-time streaming data visualization from Kafka** directly inside Jupyter notebooks.  
+It wraps **TensorWatch** and provides an interactive, high-level API with **ipywidgets** for controlling visualization parameters on the fly.
+
+This package is ideal for data scientists, engineers, and analysts who want to inspect, monitor, and visualize streaming data **without leaving JupyterLab**.
+
+---
+
+## Features
+
+- **Interactive UI**: Control plot parameters like window size, colors, and history in real-time.  
+- **Chainable API**: Fluent API for connecting to Kafka, defining streams, and drawing visualizations.  
+- **Dual Kafka Backends**: Supports `confluent-kafka` (default) and `pykafka`.  
+- **Live Metrics**: Display real-time throughput and latency metrics alongside visualizations.  
+- **Custom Processing**: Apply Python lambda expressions for aggregation or preprocessing.  
+- **Efficient Updates**: Debouncing ensures smooth UI updates without overloading the kernel.  
+
+---
+
+## Installation
+
+Install the package and its dependencies:
+
+```bash
+pip install tensorwatch-api
+pip install tensorwatch ipywidgets matplotlib
+```
+Install a Kafka client:
+```bash
+# Default Kafka connector
+pip install confluent-kafka
+
+# Or pykafka connector
+pip install pykafka
+```
+##Quick Start
+# Enable interactive matplotlib backend
+%matplotlib widget
+
+# Import the library
+from tensorwatch_api import twapi as tw
+
+# Initialize the API
+test = tw()
+
+# Create a connector to a Kafka topic
+test.connector(
+    topic='mytopic',
+    host='localhost:9092',
+    cluster_size=5,
+    queue_length=100000,
+    parsetype="protobuf",               # optional: json, pickle, avro, protobuf
+    parser_extra="benchmark_pb2",       # module name for protobuf
+    protobuf_message="BenchmarkMessage",# class name for protobuf
+    schema_path=r"C:\path\to\protobuf" # path to protobuf module
+)
+
+# Define the stream processing logic
+test.stream(expr='lambda d: sum(msg["seq"] for msg in d.data) if d.data else 0')
+
+# Draw the UI with metrics
+test.draw_with_metrics()
+
+
+## API Reference
+
+### `twapi()`
+Initializes the API wrapper and UI components.
+
+### `.connector(...)`
+Creates a Kafka consumer in a background thread, fetching data and reporting metrics.
+
+**Arguments:**
+
+- `topic` (str): Kafka topic to consume.  
+- `host` (str): Broker host (`localhost:9092`).  
+- `conn_type` (str): Connector type (`'kafka'` or `'pykafka'`). Default is `'kafka'`.  
+- `parsetype` (str): Message format (`'json'`, `'pickle'`, `'avro'`, `'protobuf'`). Default `'json'`.  
+- `cluster_size` (int): Number of consumer threads. Default 1.  
+- `queue_length` (int): Max messages in memory. Default 50000.  
+- Other parameters: `schema_path`, `protobuf_message`, `parser_extra`, `random_sampling`, `countmin_width`, `countmin_depth`, `ordering_field`.
+
+### `.stream(expr)`
+Defines the data processing logic for the stream.
+
+- `expr` (str): A Python lambda expression. Receives `d` (connector instance) and returns a single numerical value to plot.
+
+### `.draw()` / `.draw_with_metrics()`
+Renders interactive UI widgets and plot area.
+
+- `.draw()`: Standard UI.  
+- `.draw_with_metrics()`: Includes real-time benchmark metrics.
+
+---
+
+## UI Controls
+
+- **Reset Button**: Reset all options and clear the plot.  
+- **Start / Apply Changes Button**: Initially `"Start"`. Activates on first Kafka message, then applies UI changes.  
+- **Accordion Options**:  
+  - **Window Size**: Number of points shown.  
+  - **Window Width**: Width of plot.  
+  - **Pick a Color**: Plot line color.  
+  - **Date / Use Offset / Dim History**: TensorWatch visualization options.
+
+---
+
+## License
+
+This project is licensed under the **MIT License**. See [LICENSE](LICENSE) for details.

tensorwatch_api-0.1.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+tensorwatch_api/__init__.py,sha256=RE2b0TnopwHIV_NcvhQA3M8Q4fEYItq-HJsPwdpzZWA,485
+tensorwatch_api/twapi.py,sha256=WyISrhNbMwVt659oej8VA4gNZbzPs56i7DjZl8aC9O0,9016
+tensorwatch_api-0.1.0.dist-info/licenses/LICENSE.txt,sha256=oHOsDZJ_S9nBtvRFJZv-9L31kmphhq2zf48PGQ-JHwU,1091
+tensorwatch_api-0.1.0.dist-info/METADATA,sha256=99FoB0FFr9wC_58Avl2gwkT0yKIjmlmjOKDbzDaRrOg,5100
+tensorwatch_api-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+tensorwatch_api-0.1.0.dist-info/top_level.txt,sha256=wL9uSNrri-CFUXFN-uolrvW4Bfg8_hHXhjH1uhaO7dk,16
+tensorwatch_api-0.1.0.dist-info/RECORD,,

tensorwatch_api-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

tensorwatch_api-0.1.0.dist-info/licenses/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Costa S. Revi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tensorwatch_api-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+tensorwatch_api