maque 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- maque/__init__.py +30 -0
- maque/__main__.py +926 -0
- maque/ai_platform/__init__.py +0 -0
- maque/ai_platform/crawl.py +45 -0
- maque/ai_platform/metrics.py +258 -0
- maque/ai_platform/nlp_preprocess.py +67 -0
- maque/ai_platform/webpage_screen_shot.py +195 -0
- maque/algorithms/__init__.py +78 -0
- maque/algorithms/bezier.py +15 -0
- maque/algorithms/bktree.py +117 -0
- maque/algorithms/core.py +104 -0
- maque/algorithms/hilbert.py +16 -0
- maque/algorithms/rate_function.py +92 -0
- maque/algorithms/transform.py +27 -0
- maque/algorithms/trie.py +272 -0
- maque/algorithms/utils.py +63 -0
- maque/algorithms/video.py +587 -0
- maque/api/__init__.py +1 -0
- maque/api/common.py +110 -0
- maque/api/fetch.py +26 -0
- maque/api/static/icon.png +0 -0
- maque/api/static/redoc.standalone.js +1782 -0
- maque/api/static/swagger-ui-bundle.js +3 -0
- maque/api/static/swagger-ui.css +3 -0
- maque/cli/__init__.py +1 -0
- maque/cli/clean_invisible_chars.py +324 -0
- maque/cli/core.py +34 -0
- maque/cli/groups/__init__.py +26 -0
- maque/cli/groups/config.py +205 -0
- maque/cli/groups/data.py +615 -0
- maque/cli/groups/doctor.py +259 -0
- maque/cli/groups/embedding.py +222 -0
- maque/cli/groups/git.py +29 -0
- maque/cli/groups/help.py +410 -0
- maque/cli/groups/llm.py +223 -0
- maque/cli/groups/mcp.py +241 -0
- maque/cli/groups/mllm.py +1795 -0
- maque/cli/groups/mllm_simple.py +60 -0
- maque/cli/groups/quant.py +210 -0
- maque/cli/groups/service.py +490 -0
- maque/cli/groups/system.py +570 -0
- maque/cli/mllm_run.py +1451 -0
- maque/cli/script.py +52 -0
- maque/cli/tree.py +49 -0
- maque/clustering/__init__.py +52 -0
- maque/clustering/analyzer.py +347 -0
- maque/clustering/clusterers.py +464 -0
- maque/clustering/sampler.py +134 -0
- maque/clustering/visualizer.py +205 -0
- maque/constant.py +13 -0
- maque/core.py +133 -0
- maque/cv/__init__.py +1 -0
- maque/cv/image.py +219 -0
- maque/cv/utils.py +68 -0
- maque/cv/video/__init__.py +3 -0
- maque/cv/video/keyframe_extractor.py +368 -0
- maque/embedding/__init__.py +43 -0
- maque/embedding/base.py +56 -0
- maque/embedding/multimodal.py +308 -0
- maque/embedding/server.py +523 -0
- maque/embedding/text.py +311 -0
- maque/git/__init__.py +24 -0
- maque/git/pure_git.py +912 -0
- maque/io/__init__.py +29 -0
- maque/io/core.py +38 -0
- maque/io/ops.py +194 -0
- maque/llm/__init__.py +111 -0
- maque/llm/backend.py +416 -0
- maque/llm/base.py +411 -0
- maque/llm/server.py +366 -0
- maque/mcp_server.py +1096 -0
- maque/mllm_data_processor_pipeline/__init__.py +17 -0
- maque/mllm_data_processor_pipeline/core.py +341 -0
- maque/mllm_data_processor_pipeline/example.py +291 -0
- maque/mllm_data_processor_pipeline/steps/__init__.py +56 -0
- maque/mllm_data_processor_pipeline/steps/data_alignment.py +267 -0
- maque/mllm_data_processor_pipeline/steps/data_loader.py +172 -0
- maque/mllm_data_processor_pipeline/steps/data_validation.py +304 -0
- maque/mllm_data_processor_pipeline/steps/format_conversion.py +411 -0
- maque/mllm_data_processor_pipeline/steps/mllm_annotation.py +331 -0
- maque/mllm_data_processor_pipeline/steps/mllm_refinement.py +446 -0
- maque/mllm_data_processor_pipeline/steps/result_validation.py +501 -0
- maque/mllm_data_processor_pipeline/web_app.py +317 -0
- maque/nlp/__init__.py +14 -0
- maque/nlp/ngram.py +9 -0
- maque/nlp/parser.py +63 -0
- maque/nlp/risk_matcher.py +543 -0
- maque/nlp/sentence_splitter.py +202 -0
- maque/nlp/simple_tradition_cvt.py +31 -0
- maque/performance/__init__.py +21 -0
- maque/performance/_measure_time.py +70 -0
- maque/performance/_profiler.py +367 -0
- maque/performance/_stat_memory.py +51 -0
- maque/pipelines/__init__.py +15 -0
- maque/pipelines/clustering.py +252 -0
- maque/quantization/__init__.py +42 -0
- maque/quantization/auto_round.py +120 -0
- maque/quantization/base.py +145 -0
- maque/quantization/bitsandbytes.py +127 -0
- maque/quantization/llm_compressor.py +102 -0
- maque/retriever/__init__.py +35 -0
- maque/retriever/chroma.py +654 -0
- maque/retriever/document.py +140 -0
- maque/retriever/milvus.py +1140 -0
- maque/table_ops/__init__.py +1 -0
- maque/table_ops/core.py +133 -0
- maque/table_viewer/__init__.py +4 -0
- maque/table_viewer/download_assets.py +57 -0
- maque/table_viewer/server.py +698 -0
- maque/table_viewer/static/element-plus-icons.js +5791 -0
- maque/table_viewer/static/element-plus.css +1 -0
- maque/table_viewer/static/element-plus.js +65236 -0
- maque/table_viewer/static/main.css +268 -0
- maque/table_viewer/static/main.js +669 -0
- maque/table_viewer/static/vue.global.js +18227 -0
- maque/table_viewer/templates/index.html +401 -0
- maque/utils/__init__.py +56 -0
- maque/utils/color.py +68 -0
- maque/utils/color_string.py +45 -0
- maque/utils/compress.py +66 -0
- maque/utils/constant.py +183 -0
- maque/utils/core.py +261 -0
- maque/utils/cursor.py +143 -0
- maque/utils/distance.py +58 -0
- maque/utils/docker.py +96 -0
- maque/utils/downloads.py +51 -0
- maque/utils/excel_helper.py +542 -0
- maque/utils/helper_metrics.py +121 -0
- maque/utils/helper_parser.py +168 -0
- maque/utils/net.py +64 -0
- maque/utils/nvidia_stat.py +140 -0
- maque/utils/ops.py +53 -0
- maque/utils/packages.py +31 -0
- maque/utils/path.py +57 -0
- maque/utils/tar.py +260 -0
- maque/utils/untar.py +129 -0
- maque/web/__init__.py +0 -0
- maque/web/image_downloader.py +1410 -0
- maque-0.2.1.dist-info/METADATA +450 -0
- maque-0.2.1.dist-info/RECORD +143 -0
- maque-0.2.1.dist-info/WHEEL +4 -0
- maque-0.2.1.dist-info/entry_points.txt +3 -0
- maque-0.2.1.dist-info/licenses/LICENSE +21 -0
maque/api/common.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from fastapi import FastAPI, status
|
|
3
|
+
from fastapi.openapi.docs import (
|
|
4
|
+
get_redoc_html,
|
|
5
|
+
get_swagger_ui_html,
|
|
6
|
+
get_swagger_ui_oauth2_redirect_html,
|
|
7
|
+
)
|
|
8
|
+
from fastapi.middleware.cors import CORSMiddleware
|
|
9
|
+
from fastapi import APIRouter
|
|
10
|
+
from fastapi.staticfiles import StaticFiles
|
|
11
|
+
from fastapi.responses import JSONResponse
|
|
12
|
+
from pydantic import BaseModel
|
|
13
|
+
import orjson
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def add_healthcheck(app: FastAPI) -> None:
|
|
17
|
+
class HealthCheck(BaseModel):
|
|
18
|
+
status: str = "OK"
|
|
19
|
+
|
|
20
|
+
@app.get(
|
|
21
|
+
"/healthz",
|
|
22
|
+
summary="Perform a Health Check",
|
|
23
|
+
response_description="Return HTTP Status Code 200 (OK)",
|
|
24
|
+
status_code=status.HTTP_200_OK,
|
|
25
|
+
response_model=HealthCheck,
|
|
26
|
+
)
|
|
27
|
+
def get_health() -> HealthCheck:
|
|
28
|
+
"""
|
|
29
|
+
Perform a Health Check
|
|
30
|
+
Endpoint to perform a healthcheck on. This endpoint can primarily be used Docker
|
|
31
|
+
to ensure a robust container orchestration and management is in place. Other
|
|
32
|
+
services which rely on proper functioning of the API service will not deploy if this
|
|
33
|
+
endpoint returns any other HTTP status code except 200 (OK).
|
|
34
|
+
Returns:
|
|
35
|
+
HealthCheck: Returns a JSON response with the health status
|
|
36
|
+
"""
|
|
37
|
+
return HealthCheck(status="OK")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class ORJSONResponse(JSONResponse):
|
|
41
|
+
media_type = "application/json"
|
|
42
|
+
|
|
43
|
+
def render(self, content) -> bytes:
|
|
44
|
+
return orjson.dumps(content, option=orjson.OPT_SERIALIZE_NUMPY)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def set_app_doc_static_file(app, favicon_url="/static/icon.png"):
|
|
48
|
+
root = os.path.dirname(os.path.realpath(__file__))
|
|
49
|
+
app.mount("/static", StaticFiles(directory=f"{root}/static"), name="static")
|
|
50
|
+
|
|
51
|
+
@app.get("/docs", include_in_schema=False)
|
|
52
|
+
async def custom_swagger_ui_html():
|
|
53
|
+
return get_swagger_ui_html(
|
|
54
|
+
openapi_url=app.openapi_url,
|
|
55
|
+
title=app.title + " - Swagger UI",
|
|
56
|
+
oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url,
|
|
57
|
+
swagger_js_url="/static/swagger-ui-bundle.js",
|
|
58
|
+
swagger_css_url="/static/swagger-ui.css",
|
|
59
|
+
swagger_favicon_url=f"{favicon_url}",
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
@app.get(app.swagger_ui_oauth2_redirect_url, include_in_schema=False)
|
|
63
|
+
async def swagger_ui_redirect():
|
|
64
|
+
return get_swagger_ui_oauth2_redirect_html()
|
|
65
|
+
|
|
66
|
+
@app.get("/redoc", include_in_schema=False)
|
|
67
|
+
async def redoc_html():
|
|
68
|
+
return get_redoc_html(
|
|
69
|
+
openapi_url=app.openapi_url,
|
|
70
|
+
title=app.title + " - ReDoc",
|
|
71
|
+
redoc_js_url="/static/redoc.standalone.js",
|
|
72
|
+
redoc_favicon_url=f"{favicon_url}",
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def create_app(title, version='1.0', add_health_route=False, openapi_tags=None, set_static_file=True,
|
|
77
|
+
favicon_url='/static/icon.png') -> FastAPI:
|
|
78
|
+
if openapi_tags is None:
|
|
79
|
+
openapi_tags = [
|
|
80
|
+
{"name": "private",
|
|
81
|
+
"description": "私有接口"},
|
|
82
|
+
{"name": "public",
|
|
83
|
+
"description": "公有接口"},
|
|
84
|
+
]
|
|
85
|
+
app = FastAPI(openapi_tags=openapi_tags,
|
|
86
|
+
title=title,
|
|
87
|
+
version=version,
|
|
88
|
+
description="",
|
|
89
|
+
docs_url=None,
|
|
90
|
+
redoc_url=None,
|
|
91
|
+
default_response_class=ORJSONResponse,
|
|
92
|
+
)
|
|
93
|
+
if set_static_file:
|
|
94
|
+
set_app_doc_static_file(app, favicon_url=favicon_url)
|
|
95
|
+
app.add_middleware(
|
|
96
|
+
CORSMiddleware,
|
|
97
|
+
allow_origins=["*"],
|
|
98
|
+
allow_credentials=True,
|
|
99
|
+
allow_methods=["*"],
|
|
100
|
+
allow_headers=["*"],
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
if add_health_route:
|
|
104
|
+
add_healthcheck(app)
|
|
105
|
+
|
|
106
|
+
return app
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def add_router(app: FastAPI, router: APIRouter):
|
|
110
|
+
app.include_router(router)
|
maque/api/fetch.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import aiohttp
|
|
2
|
+
import asyncio
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
async def fetch(session, url, data, method):
|
|
6
|
+
if method == "POST":
|
|
7
|
+
async with session.post(url, json=data) as response:
|
|
8
|
+
return await response.text()
|
|
9
|
+
elif method == "GET":
|
|
10
|
+
async with session.get(url, params=data) as response:
|
|
11
|
+
return await response.text()
|
|
12
|
+
else:
|
|
13
|
+
raise ValueError("Invalid method specified")
|
|
14
|
+
|
|
15
|
+
async def run(url, data: dict, method: str = "POST", concurrent: int = 10):
|
|
16
|
+
if method not in ["POST", "GET"]:
|
|
17
|
+
raise ValueError("Method should be either 'POST' or 'GET'")
|
|
18
|
+
|
|
19
|
+
tasks = []
|
|
20
|
+
async with aiohttp.ClientSession() as session:
|
|
21
|
+
for _ in range(concurrent):
|
|
22
|
+
task = asyncio.create_task(fetch(session, url, data, method))
|
|
23
|
+
tasks.append(task)
|
|
24
|
+
responses = await asyncio.gather(*tasks)
|
|
25
|
+
for response in responses:
|
|
26
|
+
print(response)
|
|
Binary file
|