isage-middleware 0.1.4__py3-none-any.whl → 0.1.5.1__py3-none-any.whl

sage/middleware/components/sage_flow/data/generate_vectors.py

@@ -0,0 +1,34 @@
+import csv
+import random
+
+# Parameters
+output_file = "vectors_input.csv"
+num_vectors = 100  # Number of vectors to generate
+num_dimensions = 5  # Number of dimensions per vector
+
+
+# Generate fake vector data
+def generate_vectors(num_vectors, num_dimensions):
+    data = []
+    for i in range(num_vectors):
+        vector = [i]  # Start with an ID
+        vector.extend(
+            [round(random.uniform(0.0, 1.0), 2) for _ in range(num_dimensions)]
+        )
+        data.append(vector)
+    return data
+
+
+# Write data to CSV
+def write_csv(file_name, data, num_dimensions):
+    header = ["id"] + [f"dim{d+1}" for d in range(num_dimensions)]
+    with open(file_name, mode="w", newline="") as file:
+        writer = csv.writer(file)
+        writer.writerow(header)
+        writer.writerows(data)
+
+
+if __name__ == "__main__":
+    vectors = generate_vectors(num_vectors, num_dimensions)
+    write_csv(output_file, vectors, num_dimensions)
+    print(f"Fake vector data written to {output_file}")

sage/middleware/components/sage_flow/python/__init__.py

@@ -0,0 +1,4 @@
+"""Python bindings and wrappers for SAGE-Flow live here.
+
+This package is intended to house all Python-side modules for the component.
+"""

sage/middleware/components/sage_flow/python/micro_service/__init__.py

@@ -0,0 +1,4 @@
+# Expose SageFlowService from submodule
+from .sage_flow_service import SageFlowService
+
+__all__ = ["SageFlowService"]

sage/middleware/components/sage_flow/python/micro_service/sage_flow_service.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+import queue
+import threading
+import time
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    pass
+
+import numpy as np
+from sage.middleware.components.sage_flow.python.sage_flow import (
+    SimpleStreamSource,
+    StreamEnvironment,
+)
+
+
+@dataclass
+class _Record:
+    uid: int
+    vec: np.ndarray
+
+
+class SageFlowService:
+    """
+    A minimal micro-service wrapper for SAGE-Flow used by examples.
+
+    - push(uid, vec): enqueue vector for processing
+    - run(): drain queue, feed to flow, and execute once
+    """
+
+    def __init__(self, dim: int = 4, dtype: str = "Float32") -> None:
+        self.dim = dim
+        self.dtype = dtype
+        self._q: "queue.Queue[_Record]" = queue.Queue()
+        self._env = StreamEnvironment()
+        self._source = SimpleStreamSource("sage_flow_service_source")
+        self._lock = threading.Lock()
+        self._added_to_env = False
+        # Note: don't add to env yet; defer until a sink is attached
+
+    # API expected by examples
+    def push(self, uid: int, vec: np.ndarray) -> None:
+        if not isinstance(vec, np.ndarray):
+            vec = np.asarray(vec, dtype=np.float32)
+        vec = vec.astype(np.float32, copy=False)
+        if vec.ndim != 1 or vec.shape[0] != self.dim:
+            raise ValueError(f"vector shape must be ({self.dim},)")
+        self._q.put(_Record(uid=int(uid), vec=vec))
+
+    def run(self) -> None:
+        # Drain queue into source, then execute once
+        drained = 0
+        with self._lock:
+            while True:
+                try:
+                    rec = self._q.get_nowait()
+                except queue.Empty:
+                    break
+                ts = int(time.time() * 1000)
+                self._source.addRecord(rec.uid, ts, rec.vec)
+                drained += 1
+        if drained:
+            # If user hasn't attached sinks, add source to env once so execution proceeds
+            if not self._added_to_env:
+                # Attach a default printing sink for visibility
+                self._source.write_sink_py(
+                    "default_print_sink",
+                    lambda uid, ts: print(f"[svc sink] uid={uid}, ts={ts}", flush=True),
+                )
+                self._env.addStream(self._source)
+                self._added_to_env = True
+            self._env.execute()
+
+    def set_sink(self, callback, name: str = "py_sink") -> None:
+        """Attach a Python sink callback for visible outputs.
+
+        Args:
+            callback: Callable taking (uid: int, ts: int)
+            name: Sink name, defaults to 'py_sink'.
+        """
+        self._source.write_sink_py(name, callback)
+        if not self._added_to_env:
+            self._env.addStream(self._source)
+            self._added_to_env = True
+
+    # Optional: expose environment for advanced integrations
+    @property
+    def env(self) -> StreamEnvironment:
+        return self._env

sage/middleware/components/sage_flow/python/sage_flow.py

@@ -0,0 +1,118 @@
+"""
+SAGE Flow - High-performance vector stream processing engine (Python side)
+
+All Python-facing APIs for SAGE-Flow live under this module.
+"""
+
+from typing import Any, Dict, Optional
+
+import numpy as np
+
+try:
+    from . import _sage_flow
+except ImportError as e:
+    import importlib
+    import sys
+    from pathlib import Path
+
+    here = Path(__file__).resolve().parent
+    candidate_paths = [
+        here,  # same directory as this file (editable install case)
+        here / "build" / "lib",  # standard local build
+        here.parent / "build" / "lib",  # component-level build
+        here.parent / "build",  # build directory
+        here.parent / "install",  # install directory
+    ]
+
+    # Add paths to sys.path
+    for p in candidate_paths:
+        if p.exists() and str(p) not in sys.path:
+            sys.path.insert(0, str(p))
+
+    # Try to find the .so file directly
+    found_so = False
+    for p in candidate_paths:
+        if p.exists():
+            # Look for _sage_flow.*.so files
+            so_files = list(p.glob("_sage_flow*.so"))
+            if so_files:
+                found_so = True
+                # Add this directory to sys.path if not already there
+                if str(p) not in sys.path:
+                    sys.path.insert(0, str(p))
+                break
+
+    try:
+        _sage_flow = importlib.import_module("_sage_flow")
+    except Exception:
+        raise ImportError(
+            f"_sage_flow native module not found. Please build the extension by running 'sage extensions install sage_flow' or executing the build.sh under packages/sage-middleware/src/sage/middleware/components/sage_flow. "
+            f"Searched in: {[str(p) for p in candidate_paths if p.exists()]}, Found .so files: {found_so}"
+        ) from e
+
+DataType = _sage_flow.DataType
+VectorData = _sage_flow.VectorData
+VectorRecord = _sage_flow.VectorRecord
+Stream = _sage_flow.Stream
+StreamEnvironment = _sage_flow.StreamEnvironment
+SimpleStreamSource = _sage_flow.SimpleStreamSource
+
+
+class SageFlow:
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        self.env = StreamEnvironment()
+        self.streams = []
+        self.config = config or {}
+
+    def create_stream(self, name: str):
+        return Stream(name)
+
+    def create_simple_source(self, name: str):
+        return SimpleStreamSource(name)
+
+    def add_vector_record(self, source, uid: int, timestamp: int, vector):
+        if isinstance(vector, np.ndarray):
+            vector = vector.astype(np.float32, copy=False)
+        else:
+            vector = np.asarray(vector, dtype=np.float32)
+        source.addRecord(uid, timestamp, vector)
+
+    def add_stream(self, stream):
+        self.streams.append(stream)
+        self.env.addStream(stream)
+
+    def execute(self):
+        self.env.execute()
+
+    def get_stream_snapshot(self) -> Dict[str, Any]:
+        return {
+            "streams_count": len(self.streams),
+            "config": self.config,
+            "status": "active",
+        }
+
+
+def create_stream_engine(config: Optional[Dict[str, Any]] = None) -> SageFlow:
+    return SageFlow(config)
+
+
+def create_vector_stream(name: str):
+    return Stream(name)
+
+
+def create_simple_data_source(name: str):
+    return SimpleStreamSource(name)
+
+
+__all__ = [
+    "SageFlow",
+    "create_stream_engine",
+    "create_vector_stream",
+    "create_simple_data_source",
+    "DataType",
+    "VectorData",
+    "VectorRecord",
+    "Stream",
+    "StreamEnvironment",
+    "SimpleStreamSource",
+]

sage/middleware/utils/__init__.py

@@ -2,36 +2,11 @@
 SAGE - Streaming-Augmented Generative Execution
 """
 
-# 动态版本加载
-def _load_version():
-    """从项目根目录动态加载版本信息"""
-    from pathlib import Path
-    
-    # 获取项目根目录
-    current_file = Path(__file__).resolve()
-    root_dir = current_file.parent.parent.parent.parent.parent.parent
-    version_file = root_dir / "_version.py"
-    
-    # 加载版本信息
-    if version_file.exists():
-        version_globals = {}
-        with open(version_file, 'r', encoding='utf-8') as f:
-            exec(f.read(), version_globals)
-        return {
-            'version': version_globals.get('__version__', '0.1.4'),
-            'author': version_globals.get('__author__', 'SAGE Team'),
-            'email': version_globals.get('__email__', 'shuhao_zhang@hust.edu.cn')
-        }
-    
-    # 默认值
-    return {
-        'version': '0.1.4',
-        'author': 'SAGE Team', 
-        'email': 'shuhao_zhang@hust.edu.cn'
-    }
-
-# 加载信息
-_info = _load_version()
-__version__ = _info['version']
-__author__ = _info['author']
-__email__ = _info['email']
+# 直接从本包的_version模块加载版本信息
+try:
+    from sage.middleware._version import __author__, __email__, __version__
+except ImportError:
+    # 备用硬编码版本
+    __version__ = "0.1.4"
+    __author__ = "IntelliStream Team"
+    __email__ = "shuhao_zhang@hust.edu.cn"

sage/middleware/utils/embedding/__init__.py

@@ -2,36 +2,11 @@
 SAGE - Streaming-Augmented Generative Execution
 """
 
-# 动态版本加载
-def _load_version():
-    """从项目根目录动态加载版本信息"""
-    from pathlib import Path
-    
-    # 获取项目根目录
-    current_file = Path(__file__).resolve()
-    root_dir = current_file.parent.parent.parent.parent.parent.parent.parent
-    version_file = root_dir / "_version.py"
-    
-    # 加载版本信息
-    if version_file.exists():
-        version_globals = {}
-        with open(version_file, 'r', encoding='utf-8') as f:
-            exec(f.read(), version_globals)
-        return {
-            'version': version_globals.get('__version__', '0.1.4'),
-            'author': version_globals.get('__author__', 'SAGE Team'),
-            'email': version_globals.get('__email__', 'shuhao_zhang@hust.edu.cn')
-        }
-    
-    # 默认值
-    return {
-        'version': '0.1.4',
-        'author': 'SAGE Team', 
-        'email': 'shuhao_zhang@hust.edu.cn'
-    }
-
-# 加载信息
-_info = _load_version()
-__version__ = _info['version']
-__author__ = _info['author']
-__email__ = _info['email']
+# 直接从本包的_version模块加载版本信息
+try:
+    from sage.middleware._version import __author__, __email__, __version__
+except ImportError:
+    # 备用硬编码版本
+    __version__ = "0.1.4"
+    __author__ = "IntelliStream Team"
+    __email__ = "shuhao_zhang@hust.edu.cn"

sage/middleware/utils/embedding/_cohere.py

@@ -1,13 +1,14 @@
 import os
-import cohere
-import asyncio
 
-import numpy as np
+import cohere
 
 
 async def cohere_embed(
-        texts: list[str], api_key: str, model: str = "embed-multilingual-v3.0", input_type: str = "classification",
-        embedding_types: list[str] = ["float"]
+    texts: list[str],
+    api_key: str,
+    model: str = "embed-multilingual-v3.0",
+    input_type: str = "classification",
+    embedding_types: list[str] = ["float"],
 ) -> list[float]:
     if api_key is None:
         api_key = os.environ.get("COHERE_API_KEY")
@@ -22,15 +23,13 @@ async def cohere_embed(
     )
     return response.embeddings
 
-import os
-import cohere
 
 def cohere_embed_sync(
     texts: list[str],
     api_key: str = None,
     model: str = "embed-multilingual-v3.0",
     input_type: str = "classification",
-    embedding_types: list[str] = ["float"]
+    embedding_types: list[str] = ["float"],
 ) -> list[list[float]]:
     """
     同步版本：使用 Cohere 同步客户端生成文本 embeddings。
@@ -59,10 +58,3 @@ def cohere_embed_sync(
         embedding_types=embedding_types,
     )
     return response.embeddings
-
-
-
-
-
-
-

sage/middleware/utils/embedding/bedrock.py

@@ -1,8 +1,7 @@
-import copy
-import os
 import json
+import os
 
-import pipmaster as pm  # Pipmaster for dynamic library install
+import boto3
 
 # Dependencies should be installed via requirements.txt
 # aioboto3 and tenacity are required for this module
@@ -16,27 +15,21 @@ except ImportError:
     )
 
 try:
-    from tenacity import (
-        retry,
-        stop_after_attempt,
-        wait_exponential,
-        retry_if_exception_type,
-    )
+    from tenacity import retry  # noqa: F401
+    from tenacity import retry_if_exception_type  # noqa: F401
+    from tenacity import stop_after_attempt  # noqa: F401
+    from tenacity import wait_exponential  # noqa: F401
 except ImportError:
     raise ImportError(
         "tenacity package is required for AWS Bedrock embedding functionality. "
         "Please install it via: pip install tenacity"
     )
 
-import numpy as np
-
-
 
 class BedrockError(Exception):
     """Generic error for issues related to Amazon Bedrock"""
 
 
-
 async def bedrock_embed(
     text: str,
     model: str = "amazon.titan-embed-text-v2:0",
@@ -60,10 +53,12 @@ async def bedrock_embed(
 
         if model_provider == "amazon":
             if "v2" in model:
-                body = json.dumps({
-                    "inputText": text,
-                    "embeddingTypes": ["float"],
-                })
+                body = json.dumps(
+                    {
+                        "inputText": text,
+                        "embeddingTypes": ["float"],
+                    }
+                )
             elif "v1" in model:
                 body = json.dumps({"inputText": text})
             else:
@@ -80,11 +75,13 @@ async def bedrock_embed(
             return response_body["embedding"]
 
         elif model_provider == "cohere":
-            body = json.dumps({
-                "texts": [text],
-                "input_type": "search_document",
-                "truncate": "NONE",
-            })
+            body = json.dumps(
+                {
+                    "texts": [text],
+                    "input_type": "search_document",
+                    "truncate": "NONE",
+                }
+            )
 
             response = await bedrock_async_client.invoke_model(
                 model=model,
@@ -100,10 +97,6 @@ async def bedrock_embed(
             raise ValueError(f"Model provider '{model_provider}' is not supported!")
 
 
-import os
-import json
-import boto3
-
 def bedrock_embed_sync(
     text: str,
     model: str = "amazon.titan-embed-text-v2:0",
@@ -136,10 +129,12 @@ def bedrock_embed_sync(
 
     if model_provider == "amazon":
         if "v2" in model:
-            body = json.dumps({
-                "inputText": text,
-                "embeddingTypes": ["float"],
-            })
+            body = json.dumps(
+                {
+                    "inputText": text,
+                    "embeddingTypes": ["float"],
+                }
+            )
         elif "v1" in model:
             body = json.dumps({"inputText": text})
         else:
@@ -155,11 +150,13 @@ def bedrock_embed_sync(
         return response_body["embedding"]
 
     elif model_provider == "cohere":
-        body = json.dumps({
-            "texts": [text],
-            "input_type": "search_document",
-            "truncate": "NONE",
-        })
+        body = json.dumps(
+            {
+                "texts": [text],
+                "input_type": "search_document",
+                "truncate": "NONE",
+            }
+        )
 
         response = bedrock_client.invoke_model(
             modelId=model,

sage/middleware/utils/embedding/embedding_api.py

@@ -1,7 +1,11 @@
+import os
+
+# flake8: noqa: E402
+os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
 from sage.middleware.utils.embedding.embedding_model import EmbeddingModel
 
 
-def apply_embedding_model(name: str = "default",**kwargs) -> EmbeddingModel:
+def apply_embedding_model(name: str = "default", **kwargs) -> EmbeddingModel:
     """
     usage  参见sage/api/model/operator_test.py
     while name(method) = "hf", please set the param:model;
@@ -9,4 +13,4 @@ def apply_embedding_model(name: str = "default",**kwargs) -> EmbeddingModel:
     while name(method) = "jina/siliconcloud/cohere",please set the params:api_key,model;
     Example:operator_test.py
     """
-    return EmbeddingModel(method=name,**kwargs)
+    return EmbeddingModel(method=name, **kwargs)