flowllm 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

llmflow/vector_store/file_vector_store.py → flowllm/storage/vector_store/local_vector_store.py

@@ -1,18 +1,20 @@
+import fcntl
+import json
 import math
 from pathlib import Path
 from typing import List, Iterable
 
 from loguru import logger
 from pydantic import Field, model_validator
+from tqdm import tqdm
 
-from llmflow.embedding_model.openai_compatible_embedding_model import OpenAICompatibleEmbeddingModel
-from llmflow.schema.vector_node import VectorNode
-from llmflow.vector_store import VECTOR_STORE_REGISTRY
-from llmflow.vector_store.base_vector_store import BaseVectorStore
+from flowllm.context.service_context import C
+from flowllm.schema.vector_node import VectorNode
+from flowllm.storage.vector_store.base_vector_store import BaseVectorStore
 
 
-@VECTOR_STORE_REGISTRY.register("local_file")
-class FileVectorStore(BaseVectorStore):
+@C.register_vector_store("local")
+class LocalVectorStore(BaseVectorStore):
     store_dir: str = Field(default="./file_vector_store")
 
     @model_validator(mode="after")
@@ -21,6 +23,55 @@ class FileVectorStore(BaseVectorStore):
         store_path.mkdir(parents=True, exist_ok=True)
         return self
 
+    @staticmethod
+    def _load_from_path(workspace_id: str, path: str | Path, callback_fn=None, **kwargs) -> Iterable[VectorNode]:
+        workspace_path = Path(path) / f"{workspace_id}.jsonl"
+        if not workspace_path.exists():
+            logger.warning(f"workspace_path={workspace_path} is not exists!")
+            return
+
+        with workspace_path.open() as f:
+            fcntl.flock(f, fcntl.LOCK_SH)
+            try:
+                for line in tqdm(f, desc="load from path"):
+                    if line.strip():
+                        node_dict = json.loads(line.strip())
+                        if callback_fn:
+                            node = callback_fn(node_dict)
+                        else:
+                            node = VectorNode(**node_dict, **kwargs)
+                        node.workspace_id = workspace_id
+                        yield node
+
+            finally:
+                fcntl.flock(f, fcntl.LOCK_UN)
+
+    @staticmethod
+    def _dump_to_path(nodes: Iterable[VectorNode], workspace_id: str, path: str | Path = "", callback_fn=None,
+                      ensure_ascii: bool = False, **kwargs):
+        dump_path: Path = Path(path)
+        dump_path.mkdir(parents=True, exist_ok=True)
+        dump_file = dump_path / f"{workspace_id}.jsonl"
+
+        count = 0
+        with dump_file.open("w") as f:
+            fcntl.flock(f, fcntl.LOCK_EX)
+            try:
+                for node in tqdm(nodes, desc="dump to path"):
+                    node.workspace_id = workspace_id
+                    if callback_fn:
+                        node_dict = callback_fn(node)
+                    else:
+                        node_dict = node.model_dump()
+                    assert isinstance(node_dict, dict)
+                    f.write(json.dumps(node_dict, ensure_ascii=ensure_ascii, **kwargs))
+                    f.write("\n")
+                    count += 1
+
+                return {"size": count}
+            finally:
+                fcntl.flock(f, fcntl.LOCK_UN)
+
     @property
     def store_path(self) -> Path:
         return Path(self.store_dir)
@@ -41,6 +92,54 @@ class FileVectorStore(BaseVectorStore):
         for i, node in enumerate(self._load_from_path(path=self.store_path, workspace_id=workspace_id, **kwargs)):
             yield node
 
+    def dump_workspace(self, workspace_id: str, path: str | Path = "", callback_fn=None, **kwargs):
+        if not self.exist_workspace(workspace_id=workspace_id, **kwargs):
+            logger.warning(f"workspace_id={workspace_id} is not exist!")
+            return {}
+
+        return self._dump_to_path(nodes=self._iter_workspace_nodes(workspace_id=workspace_id, **kwargs),
+                                  workspace_id=workspace_id,
+                                  path=path,
+                                  callback_fn=callback_fn,
+                                  **kwargs)
+
+    def load_workspace(self, workspace_id: str, path: str | Path = "", nodes: List[VectorNode] = None, callback_fn=None,
+                       **kwargs):
+        if self.exist_workspace(workspace_id, **kwargs):
+            self.delete_workspace(workspace_id=workspace_id, **kwargs)
+            logger.info(f"delete workspace_id={workspace_id}")
+
+        self.create_workspace(workspace_id=workspace_id, **kwargs)
+
+        all_nodes: List[VectorNode] = []
+        if nodes:
+            all_nodes.extend(nodes)
+        for node in self._load_from_path(path=path, workspace_id=workspace_id, callback_fn=callback_fn, **kwargs):
+            all_nodes.append(node)
+        self.insert(nodes=all_nodes, workspace_id=workspace_id, **kwargs)
+        return {"size": len(all_nodes)}
+
+    def copy_workspace(self, src_workspace_id: str, dest_workspace_id: str, **kwargs):
+        if not self.exist_workspace(workspace_id=src_workspace_id, **kwargs):
+            logger.warning(f"src_workspace_id={src_workspace_id} is not exist!")
+            return {}
+
+        if not self.exist_workspace(dest_workspace_id, **kwargs):
+            self.create_workspace(workspace_id=dest_workspace_id, **kwargs)
+
+        nodes = []
+        node_size = 0
+        for node in self._iter_workspace_nodes(workspace_id=src_workspace_id, **kwargs):
+            nodes.append(node)
+            node_size += 1
+            if len(nodes) >= self.batch_size:
+                self.insert(nodes=nodes, workspace_id=dest_workspace_id, **kwargs)
+                nodes.clear()
+
+        if nodes:
+            self.insert(nodes=nodes, workspace_id=dest_workspace_id, **kwargs)
+        return {"size": node_size}
+
     @staticmethod
     def calculate_similarity(query_vector: List[float], node_vector: List[float]):
         assert query_vector, f"query_vector is empty!"
@@ -104,14 +203,15 @@ class FileVectorStore(BaseVectorStore):
         self._dump_to_path(nodes=all_nodes, workspace_id=workspace_id, path=self.store_path, **kwargs)
         logger.info(f"delete workspace_id={workspace_id} before_size={before_size} after_size={after_size}")
 
-
 def main():
-    from dotenv import load_dotenv
-    load_dotenv()
+    from flowllm.utils.common_utils import load_env
+    from flowllm.embedding_model import OpenAICompatibleEmbeddingModel
+
+    load_env()
 
     embedding_model = OpenAICompatibleEmbeddingModel(dimensions=64, model_name="text-embedding-v4")
     workspace_id = "rag_nodes_index"
-    client = FileVectorStore(embedding_model=embedding_model)
+    client = LocalVectorStore(embedding_model=embedding_model)
     client.delete_workspace(workspace_id)
     client.create_workspace(workspace_id)
 
@@ -160,4 +260,3 @@ def main():
 
 if __name__ == "__main__":
     main()
-    # launch with: python -m llmflow.storage.file_vector_store

flowllm/utils/common_utils.py

@@ -0,0 +1,52 @@
+import os
+import re
+from pathlib import Path
+
+from loguru import logger
+
+
+def camel_to_snake(content: str) -> str:
+    """
+    BaseWorker -> base_worker
+    """
+    snake_str = re.sub(r'(?<!^)(?=[A-Z])', '_', content).lower()
+    return snake_str
+
+
+def snake_to_camel(content: str) -> str:
+    """
+    base_worker -> BaseWorker
+    """
+    camel_str = "".join(x.capitalize() for x in content.split("_"))
+    return camel_str
+
+
+def _load_env(path: Path):
+    with path.open() as f:
+        for line in f:
+            line = line.strip()
+            if line.startswith("#"):
+                continue
+
+            line_split = line.strip().split("=", 1)
+            if len(line_split) >= 2:
+                key = line_split[0].strip()
+                value = line_split[1].strip()
+                os.environ[key] = value
+
+
+def load_env(path: str | Path = None):
+    if path is not None:
+        path = Path(path)
+        if path.exists():
+            _load_env(path)
+
+    else:
+        for i in range(5):
+            path = Path("../" * i + ".env")
+            if path.exists():
+                logger.info(f"using path={path}")
+                _load_env(path)
+                return
+
+        raise FileNotFoundError(".env not found")

flowllm/utils/fetch_url.py

@@ -0,0 +1,117 @@
+import random
+import time
+
+import requests
+import urllib3
+from bs4 import BeautifulSoup
+
+# Disable SSL warnings (optional, for handling insecure HTTPS)
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+def get_random_headers():
+    """Generate random headers to avoid detection"""
+    user_agents = [
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15',
+        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:89.0) Gecko/20100101 Firefox/89.0',
+        'Mozilla/5.0 (X11; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.59'
+    ]
+
+    accept_languages = [
+        'en-US,en;q=0.9',
+        'zh-CN,zh;q=0.9,en;q=0.8',
+        'en-GB,en;q=0.9',
+        'fr-FR,fr;q=0.9,en;q=0.8',
+        'de-DE,de;q=0.9,en;q=0.8'
+    ]
+
+    accept_encodings = [
+        'gzip, deflate, br',
+        'gzip, deflate',
+        'br, gzip, deflate'
+    ]
+
+    headers = {
+        'User-Agent': random.choice(user_agents),
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+        'Accept-Language': random.choice(accept_languages),
+        'Accept-Encoding': random.choice(accept_encodings),
+        'Connection': 'keep-alive',
+        'Upgrade-Insecure-Requests': '1',
+    }
+
+    # Randomly add some optional headers
+    if random.random() > 0.5:
+        headers['DNT'] = '1'
+    if random.random() > 0.7:
+        headers['Cache-Control'] = 'max-age=0'
+    if random.random() > 0.6:
+        headers['Sec-Fetch-Dest'] = 'document'
+        headers['Sec-Fetch-Mode'] = 'navigate'
+        headers['Sec-Fetch-Site'] = 'none'
+
+    return headers
+
+
+def fetch_webpage_text(url, min_delay=1, max_delay=3):
+    """
+    Fetch and extract text content from a webpage with randomization
+    
+    Args:
+        url (str): The URL to fetch
+        min_delay (int): Minimum delay before request (seconds)
+        max_delay (int): Maximum delay before request (seconds)
+    
+    Returns:
+        str: Extracted text content or error message
+    """
+    # Add random delay to avoid being detected as bot
+    delay = random.uniform(min_delay, max_delay)
+    time.sleep(delay)
+
+    headers = get_random_headers()
+
+    # Random timeout between 8-15 seconds
+    timeout = random.randint(8, 15)
+
+    try:
+        # Send request with random headers and timeout
+        response = requests.get(url, headers=headers, timeout=timeout, verify=False)
+        response.raise_for_status()  # Check if request was successful
+        response.encoding = response.apparent_encoding  # Auto-detect encoding
+
+        # Parse HTML using BeautifulSoup
+        soup = BeautifulSoup(response.text, 'html.parser')
+
+        # Remove script, style and navigation elements to avoid interference
+        for script_or_style in soup(['script', 'style', 'nav', 'footer', 'header']):
+            script_or_style.decompose()
+
+        # Extract text content
+        text = soup.get_text()
+
+        # Clean whitespace: remove extra blank lines and spaces
+        lines = (line.strip() for line in text.splitlines())
+        chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
+        text = ' '.join(chunk for chunk in chunks if chunk)
+
+        return text
+
+    except requests.exceptions.RequestException as e:
+        return f"Request failed: {e}"
+    except Exception as e:
+        return f"Parsing failed: {e}"
+
+
+# Example usage
+if __name__ == "__main__":
+    url = "http://finance.eastmoney.com/a/202508133482756869.html"
+    text = fetch_webpage_text(url)
+    print(text)

flowllm/utils/llm_utils.py

@@ -0,0 +1,28 @@
+from typing import List
+
+from flowllm.enumeration.role import Role
+from flowllm.schema.message import Message
+
+
+def merge_messages_content(messages: List[Message | dict]) -> str:
+    content_collector = []
+    for i, message in enumerate(messages):
+        if isinstance(message, dict):
+            message = Message(**message)
+
+        if message.role is Role.ASSISTANT:
+            line = f"### step.{i} role={message.role.value} content=\n{message.reasoning_content}\n\n{message.content}\n"
+            if message.tool_calls:
+                for tool_call in message.tool_calls:
+                    line += f" - tool call={tool_call.name}\n   params={tool_call.arguments}\n"
+            content_collector.append(line)
+
+        elif message.role is Role.USER:
+            line = f"### step.{i} role={message.role.value} content=\n{message.content}\n"
+            content_collector.append(line)
+
+        elif message.role is Role.TOOL:
+            line = f"### step.{i} role={message.role.value} tool call result=\n{message.content}\n"
+            content_collector.append(line)
+
+    return "\n".join(content_collector)

flowllm/utils/ridge_v2.py

@@ -0,0 +1,54 @@
+import numpy as np
+from sklearn.linear_model import Ridge
+from sklearn.preprocessing import StandardScaler
+
+
+class RidgeV2:
+
+    def __init__(self, bound: int = 4.6, use_ridge_v2=True, **kwargs):
+        self.bound: int = bound
+        self.use_ridge_v2: bool = use_ridge_v2
+        self.kwargs: dict = kwargs
+        self.model = Ridge(**self.kwargs)
+
+    def clear(self):
+        self.model = Ridge(**self.kwargs)
+        return self
+
+    def sigmoid(self, x):
+        x = np.asarray(x, dtype=float)
+        x = np.clip(x, -self.bound, self.bound)
+        return 1 / (1 + np.exp(-x))
+
+    def inv_sigmoid(self, p):
+        p = np.asarray(p, dtype=float)
+        p = np.clip(p, self.sigmoid(-self.bound), self.sigmoid(self.bound))
+        return np.log(p / (1 - p))
+
+    def fit(self, x, y, sample_weight=None):
+        if self.use_ridge_v2:
+            return self.model.fit(x, self.inv_sigmoid(y), sample_weight=sample_weight)
+        else:
+            return self.model.fit(x, y, sample_weight=sample_weight)
+
+    def predict(self, x):
+        if self.use_ridge_v2:
+            return self.sigmoid(self.model.predict(x))
+        else:
+            return self.model.predict(x)
+
+    def fit_and_predict(self,
+                        train_x_nd: np.ndarray,
+                        train_y_nd: np.ndarray,
+                        test_x_nd: np.ndarray,
+                        check_y: bool = True):
+        if check_y:
+            assert np.all((train_y_nd >= 0) & (train_y_nd <= 1))
+
+        scaler = StandardScaler()
+        scaler.fit(train_x_nd)
+        train_x_nd = scaler.transform(train_x_nd)
+        test_x_nd = scaler.transform(test_x_nd)
+        self.model.fit(train_x_nd, train_y_nd)
+        pred_y_nd = self.model.predict(test_x_nd)
+        return np.minimum(np.maximum(pred_y_nd, 0), 1)

{llmflow → flowllm}/utils/timer.py

@@ -1,9 +1,10 @@
 import time
+from typing import Optional
 
 from loguru import logger
 
 
-class Timer(object):
+class Timer:
     def __init__(self, name: str, use_ms: bool = False, stack_level: int = 2):
         self.name: str = name
         self.use_ms: bool = use_ms
@@ -15,7 +16,7 @@ class Timer(object):
 
     def __enter__(self, *args, **kwargs):
         self.time_start = time.time()
-        logger.info(f"---------- enter {self.name} ----------", stacklevel=self.stack_level)
+        logger.info(f"========== timer.{self.name} start ==========", stacklevel=self.stack_level)
         return self
 
     def __exit__(self, *args):
@@ -26,10 +27,10 @@ class Timer(object):
         else:
             time_str = f"{self.time_cost:.3f}s"
 
-        logger.info(f"---------- leave {self.name} [{time_str}] ----------", stacklevel=self.stack_level)
+        logger.info(f"========== timer.{self.name} end, time_cost={time_str} ==========", stacklevel=self.stack_level)
 
 
-def timer(name: str = None, use_ms: bool = False, stack_level: int = 2):
+def timer(name: Optional[str] = None, use_ms: bool = False, stack_level: int = 2):
     def decorator(func):
         def wrapper(*args, **kwargs):
             with Timer(name=name or func.__name__, use_ms=use_ms, stack_level=stack_level + 1):