maque 0.2.1__py3-none-any.whl

maque/ai_platform/crawl.py

@@ -0,0 +1,45 @@
+import asyncio
+
+from aiolimiter.compat import wait_for
+from crawl4ai import AsyncWebCrawler, CacheMode
+
+
+async def capture_and_save_screenshot(url: str, output_path: str):
+    async with AsyncWebCrawler(verbose=True, headless=False) as crawler:
+        result = await crawler.arun(
+            url=url,
+            # screenshot=True,
+            cache_mode=CacheMode.BYPASS,
+
+            # magic=True, # 当使用magic模式时,就不用设置下面两行
+
+            simulate_user=True,  # Causes random mouse movements and clicks
+            override_navigator=True,  # Makes the browser appear more like a real user
+
+            # include_links_on_markdown=False,
+
+            remove_overlay_elements=True,  # Remove popups/modals
+            page_timeout=60000,  # Increased timeout for protection checks
+            # wait_for="css:.content-loaded",
+
+
+            excluded_tags=['nav', 'footer'],
+
+        )
+        print(result.markdown)
+        # print(result.fit_markdown)
+
+        if result.success and result.screenshot:
+            import base64
+            screenshot_data = base64.b64decode(result.screenshot)
+            with open(output_path, "wb") as f:
+                f.write(screenshot_data)
+            print(f"Screenshot saved successfully to {output_path}")
+        else:
+            print("Failed to capture screenshot")
+
+
+if __name__ == "__main__":
+    # asyncio.run(capture_and_save_screenshot("https://www.gradio.app/guides/object-detection-from-video", "screenshot.png"))
+    # asyncio.run(capture_and_save_screenshot("https://www.autohome.com.cn", "screenshot.png"))
+    asyncio.run(capture_and_save_screenshot("https://www.zhihu.com/question/654186093/answer/3483543427", "screenshot.png"))

maque/ai_platform/metrics.py

@@ -0,0 +1,258 @@
+import os
+from tabulate import tabulate
+from datetime import datetime
+from maque.io import yaml_dump
+from rich.console import Console
+from rich.markdown import Markdown
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pandas import DataFrame
+
+
+def truncate_labels(labels, max_length=50):
+    """截断长标签，确保每个标签的长度不超过max_length"""
+    truncated_labels = []
+    for label in labels:
+        if len(label) > max_length:
+            truncated_label = label[:max_length] + "..."  # 截断并加上省略号
+        else:
+            truncated_label = label
+        truncated_labels.append(truncated_label)
+    return truncated_labels
+
+
+class MetricsCalculator:
+    def __init__(self, df: "DataFrame", pred_col: str = 'predict', label_col: str = 'label',
+                 include_macro_micro_avg=False,
+                 remove_matrix_zero_row=False,
+                 ):
+        self.df = df
+        self.y_pred = df[pred_col]
+        self.y_true = df[label_col]
+        self.all_labels = sorted(list(set(self.y_true.unique()).union(set(self.y_pred.unique()))))
+        self.needed_labels = None
+        self.remove_matrix_zero_row = remove_matrix_zero_row
+        self.include_macro_micro_avg = include_macro_micro_avg
+        self.metrics = self._calculate_metrics()
+
+    def plot_confusion_matrix(self, save_path: str = None, figsize=(12, 10), font_scale=1.2, font_path=None,
+                              x_rotation=45, y_rotation=0):
+        import matplotlib.pyplot as plt
+        import seaborn as sns
+        from matplotlib.font_manager import FontProperties
+        import matplotlib
+        import warnings
+        import warnings
+
+        # 全局设置默认字体，避免警告
+        matplotlib.rcParams['font.sans-serif'] = ['SimHei']  # 设置中文字体
+        matplotlib.rcParams['axes.unicode_minus'] = False  # 解决负号显示问题
+
+        # 计算混淆矩阵
+        conf_matrix = self.metrics['confusion_matrix']
+
+        # 截断长标签
+        all_labels = truncate_labels(self.all_labels, max_length=6)
+        num_classes = len(all_labels)  # 获取类的数量
+
+        # 指定中文字体路径
+        if font_path:
+            font_prop = FontProperties(fname=font_path)
+        else:
+            font_prop = None
+
+        # 设置动态字体大小，字体大小随着类的数量增加而减小
+        dynamic_font_size = max(8, 20 - num_classes)  # 例如：最小字体为8，随类别数增多字体减小
+        tick_font_prop = FontProperties(fname=font_path, size=dynamic_font_size) if font_path else None
+
+        # 设置绘图的大小和风格
+        plt.figure(figsize=figsize)
+        sns.set_theme(font_scale=font_scale)
+
+        # 忽略所有 UserWarning
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", category=UserWarning)
+
+            # 绘制热力图
+            sns.heatmap(conf_matrix, annot=True, fmt="d", cmap="Blues", xticklabels=all_labels,
+                        yticklabels=all_labels)
+
+            # 设置标题和轴标签
+            plt.title("Confusion Matrix", fontproperties=font_prop)
+            plt.xlabel("Predicted Labels", fontproperties=font_prop)
+            plt.ylabel("True Labels", fontproperties=font_prop)
+
+            # 设置轴标签的字体属性和角度
+            plt.xticks(ha="right", fontproperties=tick_font_prop, rotation=x_rotation)
+            plt.yticks(fontproperties=tick_font_prop, rotation=y_rotation)
+
+            plt.tight_layout()
+
+            # 保存或显示图表
+            if save_path:
+                plt.savefig(save_path)
+            plt.show()
+
+    def _calculate_metrics(self):
+        from sklearn.metrics import precision_score, recall_score, accuracy_score, confusion_matrix, \
+            classification_report
+        # 计算准确率
+        accuracy = accuracy_score(self.y_true, self.y_pred)
+
+        # 计算每个类别的精确率和召回率
+        precision = precision_score(self.y_true, self.y_pred, labels=self.all_labels, average='weighted',
+                                    zero_division=0)
+        recall = recall_score(self.y_true, self.y_pred, labels=self.all_labels, average='weighted', zero_division=0)
+
+        # 计算混淆矩阵
+        conf_matrix = confusion_matrix(self.y_true, self.y_pred, labels=self.all_labels)
+
+        # 计算每个类别的精确率、召回率、F1分数等
+        report = classification_report(self.y_true, self.y_pred, labels=self.all_labels, output_dict=True,
+                                       zero_division=0)
+        # 移除宏平均和微平均，默认只保留加权平均
+        if not self.include_macro_micro_avg:
+            report = {label: metrics for label, metrics in report.items() if
+                      label in self.all_labels or label == 'weighted avg'}
+
+        # 从report中移除不需要的类别，具体来说，去除support为0的类别
+        report = {label: metrics for label, metrics in report.items()
+                  if metrics['support'] > 0}
+
+        self.needed_labels = [label for label in report.keys() if label in self.all_labels]
+
+        # 移除matrix中不需要的行
+        needed_idx_list = [self.all_labels.index(label) for label in self.needed_labels]
+
+        if self.remove_matrix_zero_row:
+            conf_matrix = conf_matrix[needed_idx_list]
+
+        # 返回结果
+        return {
+            'accuracy': accuracy,
+            'precision': precision,
+            'recall': recall,
+            'confusion_matrix': conf_matrix,
+            'classification_report': report
+        }
+
+    def get_metrics(self):
+        return self.metrics
+
+    def format_classification_report_as_markdown(self):
+        report = self.metrics['classification_report']
+        header = "| Label | Precision | Recall | F1-score | Support |\n"
+        separator = "|-------|-----------|--------|----------|---------|\n"
+        rows = []
+        for label, metrics in report.items():
+            if isinstance(metrics, dict):
+                rows.append(
+                    f"| {label} | {metrics['precision']:.2f} | {metrics['recall']:.2f} | {metrics['f1-score']:.2f} | {metrics['support']:.0f} |")
+        return header + separator + "\n".join(rows)
+
+    def clean_label_for_markdown(self, label, max_length=20):
+        """清理标签文本，使其适合在markdown表格中显示"""
+        # 转换为字符串并替换换行符
+        label = str(label).replace('\n', ' ')
+        
+        # 移除或替换可能破坏markdown格式的字符
+        label = label.replace("|", "\\|")
+        label = label.replace("-", "\\-")
+        label = label.replace("<", "&lt;")
+        label = label.replace(">", "&gt;")
+        
+        # 截断长文本
+        if len(label) > max_length:
+            label = label[:max_length] + "..."
+            
+        # 确保标签至少有一个可见字符
+        label = label.strip()
+        if not label:
+            label = "(empty)"
+            
+        return label
+
+    def format_confusion_matrix_as_markdown(self, max_label_length=20):
+        matrix = self.metrics['confusion_matrix']
+        
+        # 处理标签
+        if self.remove_matrix_zero_row:
+            labels = self.needed_labels
+        else:
+            labels = self.all_labels
+            
+        # 处理所有标签
+        processed_labels = [self.clean_label_for_markdown(label, max_label_length) for label in labels]
+            
+        # 构建表头，确保第一列也有标题
+        header = "| 真实值/预测值 | " + " | ".join(processed_labels) + " |\n"
+        
+        # 修复分隔符，确保每列都有正确的分隔符
+        separator_parts = [":---:"] * (len(processed_labels) + 1)  # +1 是为了第一列
+        separator = "| " + " | ".join(separator_parts) + " |\n"
+        
+        rows = []
+        for i, row in enumerate(matrix):
+            # 处理行标签
+            row_label = self.clean_label_for_markdown(labels[i], max_label_length)
+            # 格式化数字
+            formatted_row = [f"{num:,}" for num in row]
+            rows.append(f"| {row_label} | " + " | ".join(formatted_row) + " |")
+
+        return header + separator + "\n".join(rows)
+
+
+def save_pred_metrics(df: "DataFrame", pred_col: str, label_col: str,
+                      record_folder='record', config=None, prompt=None, font_path=None,
+                      plot_confusion_matrix=False,
+                      ):
+    """ 保存预测结果的指标概览和分类报告 """
+    metrics_calculator = MetricsCalculator(df, pred_col=pred_col, label_col=label_col)
+    metrics = metrics_calculator.get_metrics()
+
+    table = [["指标概览", "Accuracy", "Precision", "Recall"],
+             ["值", metrics['accuracy'], metrics['precision'], metrics['recall']]]
+    md = f"\n\n### 指标概览\n\n{tabulate(table, headers='firstrow', tablefmt='github')}"
+    metrics_md = metrics_calculator.format_classification_report_as_markdown()
+    confusion_matrix_md = metrics_calculator.format_confusion_matrix_as_markdown()
+    md += (f"\n\n### Classification Report\n{metrics_md}\n"
+           f"\n### Confusion Matrix\n{confusion_matrix_md}")
+    now = datetime.now().strftime("%m月%d日%H时%M分%S秒")
+    record_folder = Path(record_folder)
+    record_folder = record_folder / f'记录时间-{now}'
+    record_folder.mkdir(parents=True, exist_ok=True)
+    console = Console()
+    console.print(Markdown(md))
+
+    # save files:
+    with open(os.path.join(record_folder, 'metrics.md'), 'w', encoding='utf-8') as f:
+        f.write(md)
+
+    if plot_confusion_matrix:
+        try:
+            metrics_calculator.plot_confusion_matrix(
+                save_path=os.path.join(record_folder, 'confusion_matrix.png'),
+                font_path=font_path,
+                x_rotation=45,
+                y_rotation=0,
+            )
+        except Exception as e:
+            print(f"warning: Failed to plot confusion matrix: {e}")
+
+    if prompt:
+        yaml_dump(os.path.join(record_folder, 'prompt.yaml'), prompt)
+    if config:
+        yaml_dump(os.path.join(record_folder, 'config.yaml'), config)
+
+    bad_case_df = df[df[pred_col] != df[label_col]]
+    try:
+        df.to_excel(os.path.join(record_folder, 'result.xlsx'), index=False, engine='openpyxl')
+        bad_case_df.to_excel(os.path.join(record_folder, 'bad_case.xlsx'), index=False, engine='openpyxl')
+    except Exception:
+        print("No module named 'openpyxl'. Please install it with 'pip install openpyxl'.\n"
+              "Save result.csv and bad_case.csv instead.")
+        df.to_csv(os.path.join(record_folder, 'result.csv'), index=False)
+        bad_case_df.to_csv(os.path.join(record_folder, 'bad_case.csv'), index=False)
+

maque/ai_platform/nlp_preprocess.py

@@ -0,0 +1,67 @@
+import streamlit as st
+import pandas as pd
+import io
+import jionlp as jio
+from maque.nlp.deduplicate import EditSimilarity
+
+# streamlit run main.py --server.maxUploadSize=1024
+simi = EditSimilarity()
+
+
+def load_data(file):
+    return pd.read_excel(file)
+
+
+def save_df_to_excel(df):
+    output = io.BytesIO()
+    with pd.ExcelWriter(output, engine='openpyxl') as writer:
+        df.to_excel(writer, index=False)
+    output.seek(0)
+    return output
+
+
+st.title('文本预处理工具')
+
+uploaded_file = st.file_uploader("选择一个 Excel 文件", type=['xlsx'])
+if 'replay' not in st.session_state:
+    st.session_state['replay'] = False
+
+if uploaded_file is not None:
+    df = load_data(uploaded_file)
+
+    st.write("数据预览：")
+    st.dataframe(df.head(10))
+    # 获取 Excel 文件中的列名
+    columns = df.columns.tolist()
+
+    # 让用户选择要清理的多个列
+    selected_columns = st.multiselect("选择要清理的列", columns)
+    # 选择操作
+    operation = st.multiselect("选择操作", [ "去重", "清洗文本", ])
+    dedup_threshold = 0.7
+    if "去重" in operation:
+        dedup_threshold = st.slider("选择去重阈值（越小越严格）", 0.0, 1.0, 0.7)
+
+    if st.button('执行'):
+        for column in selected_columns:
+            if "去重" in operation:
+                simi.load_from_df(df, target_col=column)
+                df = simi.deduplicate(threshold=dedup_threshold)
+            if "删除文本中的冗余字符" in operation:
+                df[column] = df[column].apply(lambda x: jio.remove_redundant_char(x))
+
+            if "清洗文本" in operation:
+                df[column] = df[column].apply(lambda x: jio.clean_text(x))
+
+        st.write("清理后的数据预览：")
+        st.dataframe(df.head(10))
+
+        cleaned_data = save_df_to_excel(df)
+        st.download_button(
+            label="下载处理后的数据",
+            data=cleaned_data,
+            file_name="cleaned_data.xlsx",
+            mime="application/vnd.ms-excel",
+        )
+else:
+    st.info("请上传一个 Excel 文件。")

maque/ai_platform/webpage_screen_shot.py

@@ -0,0 +1,195 @@
+# pip install playwright
+# playwright install
+
+import asyncio
+from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeoutError
+from PIL import Image
+from io import BytesIO
+import traceback
+from loguru import logger
+
+
+class ScreenshotTaker:
+    def __init__(self, viewport_width=1920, viewport_height=1080, max_concurrent_tasks=16, retry_limit=3):
+        self.viewport_width = viewport_width
+        self.viewport_height = viewport_height
+        self.retry_limit = retry_limit  # Maximum retry attempts for loading a page
+        self.browser = None
+        self.semaphore = asyncio.Semaphore(max_concurrent_tasks)  # Limit concurrent tasks
+
+    async def _initialize_browser(self):
+        """Initialize the browser if not already done."""
+        if self.browser is None:
+            try:
+                self.playwright = await async_playwright().start()
+                self.browser = await self.playwright.chromium.launch(headless=True,
+                                                                     args=['--disable-web-security'],  # Handle CORS
+               )
+            except Exception as e:
+                logger.error(f"Failed to initialize browser: {e}")
+                raise
+
+    async def _load_page_with_retries(self, page, url):
+        """Attempt to load a page with retries if it fails to load."""
+        for attempt in range(self.retry_limit):
+            try:
+                await page.goto(url, wait_until="networkidle", timeout=15000)  # Short timeout for quicker retries
+                return True
+            except (PlaywrightTimeoutError, Exception) as e:
+                logger.warning(f"Attempt {attempt + 1} to load {url} failed: {e}")
+                await asyncio.sleep(1.1)  # Wait briefly before retrying
+
+        logger.error(f"Failed to load {url} after {self.retry_limit} attempts.")
+        return False
+
+    async def capture_screenshot_in_context(self, context, url, screenshot_path=None):
+        """Capture a full-page screenshot of a single URL within a shared context."""
+        async with self.semaphore:  # Limit concurrency
+            try:
+                # Open a new page in the shared context
+                page = await context.new_page()
+
+                # Attempt to load the page with retries
+                if not await self._load_page_with_retries(page, url):
+                    await page.close()
+                    return None  # Skip screenshot if page load fails
+
+                logger.debug(f"Loading {url}")
+                # Scroll progressively to ensure lazy-loaded content is displayed
+                # 滚动次数
+                # scroll_height = await page.evaluate("document.body.scrollHeight")
+                # viewport_height = await page.evaluate("window.innerHeight")
+                # scroll_times = max(1, scroll_height // viewport_height)
+                # logger.info(f"Scrolling {scroll_times} times")
+                # for _ in range(scroll_times):
+                #     await page.evaluate("""() => { window.scrollBy(0, window.innerHeight); }""")
+                #     await asyncio.sleep(1/scroll_times)
+                await page.evaluate("""() => { window.scrollTo(0, document.body.scrollHeight); }""")
+                await asyncio.sleep(1.1)  # Allow time for content to load
+
+                # Adjust iframe heights if present
+                for iframe_element in await page.query_selector_all("iframe"):
+                    try:
+                        frame = await iframe_element.content_frame()
+                        if frame:
+                            logger.debug(f"Handling iframe in {url}")
+                            frame_height = await frame.evaluate("document.body.scrollHeight")
+                            await iframe_element.evaluate(f"el => el.style.height = '{frame_height}px'")
+                            await asyncio.sleep(1.1)
+                    except Exception as iframe_error:
+                        logger.warning(f"Error handling iframe in {url}: {iframe_error}")
+
+                # Capture and store screenshot as bytes
+                screenshot_bytes = await page.screenshot(full_page=True)
+                image = Image.open(BytesIO(screenshot_bytes))
+
+                # Save screenshot if a path is specified
+                if screenshot_path:
+                    image.save(screenshot_path)
+                    logger.info(f"Screenshot saved at {screenshot_path}")
+
+                await page.close()  # Close page to release resources
+                return image
+
+            except Exception as e:
+                logger.error(f"Unexpected error capturing screenshot for {url}: {e}")
+                traceback.print_exc()
+                return None  # Return None if an error occurs
+
+    async def _capture_batch_screenshots(self, urls, screenshot_paths=None):
+        """
+        Capture screenshots for a batch of URLs within a single context.
+
+        Parameters:
+        - urls: list of URLs to capture
+        - screenshot_paths: optional list of paths to save each screenshot.
+                            If None, screenshots will not be saved to files.
+
+        Returns:
+        - List of PIL image objects for each URL.
+        """
+        await self._initialize_browser()
+        screenshot_paths = screenshot_paths or [None] * len(urls)
+
+        # Create a new context for this batch
+        context = await self.browser.new_context(
+            viewport={"width": self.viewport_width, "height": self.viewport_height}
+        )
+
+        # Execute all screenshot tasks within this context
+        tasks = [self.capture_screenshot_in_context(context, url, path) for url, path in zip(urls, screenshot_paths)]
+        images = await asyncio.gather(*tasks, return_exceptions=True)
+
+        # Close context after batch is completed
+        await context.close()
+
+        # Filter out None or exceptions from results
+        images = [img for img in images if isinstance(img, Image.Image)]
+        return images
+
+    async def capture_multi_screenshots(self, urls, batch_size=3, max_batches=3, screenshot_paths=None):
+        """
+        Capture screenshots for multiple URLs in multiple batches, with specified batch size and concurrency.
+
+        Parameters:
+        - urls: list of URLs to capture
+        - batch_size: number of URLs per batch
+        - max_batches: maximum number of concurrent batches
+        - screenshot_paths: optional list of paths to save each screenshot.
+                            If None, screenshots will not be saved to files.
+
+        Returns:
+        - List of PIL image objects for each URL.
+        """
+        await self._initialize_browser()
+        screenshot_paths = screenshot_paths or [None] * len(urls)
+
+        # Split URLs into batches
+        url_batches = [urls[i:i + batch_size] for i in range(0, len(urls), batch_size)]
+        path_batches = [screenshot_paths[i:i + batch_size] for i in range(0, len(screenshot_paths), batch_size)]
+
+        # Limit concurrent batch execution
+        batch_semaphore = asyncio.Semaphore(max_batches)
+
+        async def run_batch(urls, paths):
+            async with batch_semaphore:
+                return await self._capture_batch_screenshots(urls, paths)
+
+        # Schedule all batch tasks
+        tasks = [run_batch(url_batch, path_batch) for url_batch, path_batch in zip(url_batches, path_batches)]
+        batch_images = await asyncio.gather(*tasks, return_exceptions=True)
+
+        # Flatten the list of images from all batches and filter out None or exceptions
+        images = [img for batch in batch_images if isinstance(batch, list) for img in batch]
+        return images
+
+    async def close(self):
+        """Close the browser and Playwright if they are open."""
+        if self.browser:
+            await self.browser.close()
+            self.browser = None
+        if hasattr(self, 'playwright'):
+            await self.playwright.stop()
+
+
+if __name__ == "__main__":
+    async def main():
+        screenshot_taker = ScreenshotTaker()  # Limit concurrent requests
+        url1 = "https://baidu.com"
+        urls = [url1] * 20  # Example: large batch of URLs
+        paths = [f"screenshot_{i}.png" for i in range(len(urls))]
+
+        # Batch screenshot and get a list of PIL image objects
+        images = await screenshot_taker.capture_multi_screenshots(urls,
+                                                                  batch_size=3,
+                                                                  max_batches=3,
+                                                                  screenshot_paths=paths)
+
+        # Close the browser
+        await screenshot_taker.close()
+        return images
+
+
+    asyncio.run(main())
+
+

maque/algorithms/__init__.py

@@ -0,0 +1,78 @@
+"""
+算法模块 - 数据结构和算法实现
+
+包含各种算法实现：Trie树、变换算法、去重算法、数学函数等
+"""
+
+# 数据结构
+from .trie import *
+from .bktree import *
+
+# 变换算法
+from .hilbert import *
+from .transform import *
+
+# 去重算法
+from .video import *  # video deduplication
+
+# 数学函数
+from .bezier import *
+from .core import *  # functions core
+from .rate_function import *
+from .utils import *  # functions utils
+
+__all__ = [
+    # Data structures
+    "BKTree",
+    "brute_query",
+    "levenshtein",
+    "Trie",
+    "PyTrie",
+    "HatTrie",
+    "MarisaTrie",
+    "DaTrie",
+    "AutomatonTrie",
+    "Benchmark",
+    # Transforms
+    "get_hilbert_1d_array",
+    "repeat",
+    # Video deduplication
+    "VideoFrameDeduplicator",
+    # Math functions
+    "bezier",
+    "dict_topk",
+    "topk",
+    "random_idx",
+    "clamp",
+    "choose_using_cache",
+    "choose",
+    "get_num_args",
+    "get_parameters",
+    # Rate functions
+    "linear",
+    "smooth",
+    "rush_into",
+    "rush_from",
+    "slow_into",
+    "double_smooth",
+    "there_and_back",
+    "there_and_back_with_pause",
+    "running_start",
+    "not_quite_there",
+    "wiggle",
+    "squish_rate_func",
+    "lingering",
+    "exponential_decay",
+    # Utils
+    "exists",
+    "default",
+    "cast_tuple",
+    "null_context",
+    "pick_and_pop",
+    "group_dict_by_key",
+    "string_begins_with",
+    "group_by_key_prefix",
+    "groupby_prefix_and_trim",
+    "num_to_groups",
+    "find_first",
+]

maque/algorithms/bezier.py

@@ -0,0 +1,15 @@
+from .core import choose
+
+
+def bezier(points):
+    n = len(points) - 1
+
+    def result(t):
+        return sum(
+            [
+                ((1 - t) ** (n - k)) * (t**k) * choose(n, k) * point
+                for k, point in enumerate(points)
+            ]
+        )
+
+    return result