maque 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. maque/__init__.py +30 -0
  2. maque/__main__.py +926 -0
  3. maque/ai_platform/__init__.py +0 -0
  4. maque/ai_platform/crawl.py +45 -0
  5. maque/ai_platform/metrics.py +258 -0
  6. maque/ai_platform/nlp_preprocess.py +67 -0
  7. maque/ai_platform/webpage_screen_shot.py +195 -0
  8. maque/algorithms/__init__.py +78 -0
  9. maque/algorithms/bezier.py +15 -0
  10. maque/algorithms/bktree.py +117 -0
  11. maque/algorithms/core.py +104 -0
  12. maque/algorithms/hilbert.py +16 -0
  13. maque/algorithms/rate_function.py +92 -0
  14. maque/algorithms/transform.py +27 -0
  15. maque/algorithms/trie.py +272 -0
  16. maque/algorithms/utils.py +63 -0
  17. maque/algorithms/video.py +587 -0
  18. maque/api/__init__.py +1 -0
  19. maque/api/common.py +110 -0
  20. maque/api/fetch.py +26 -0
  21. maque/api/static/icon.png +0 -0
  22. maque/api/static/redoc.standalone.js +1782 -0
  23. maque/api/static/swagger-ui-bundle.js +3 -0
  24. maque/api/static/swagger-ui.css +3 -0
  25. maque/cli/__init__.py +1 -0
  26. maque/cli/clean_invisible_chars.py +324 -0
  27. maque/cli/core.py +34 -0
  28. maque/cli/groups/__init__.py +26 -0
  29. maque/cli/groups/config.py +205 -0
  30. maque/cli/groups/data.py +615 -0
  31. maque/cli/groups/doctor.py +259 -0
  32. maque/cli/groups/embedding.py +222 -0
  33. maque/cli/groups/git.py +29 -0
  34. maque/cli/groups/help.py +410 -0
  35. maque/cli/groups/llm.py +223 -0
  36. maque/cli/groups/mcp.py +241 -0
  37. maque/cli/groups/mllm.py +1795 -0
  38. maque/cli/groups/mllm_simple.py +60 -0
  39. maque/cli/groups/quant.py +210 -0
  40. maque/cli/groups/service.py +490 -0
  41. maque/cli/groups/system.py +570 -0
  42. maque/cli/mllm_run.py +1451 -0
  43. maque/cli/script.py +52 -0
  44. maque/cli/tree.py +49 -0
  45. maque/clustering/__init__.py +52 -0
  46. maque/clustering/analyzer.py +347 -0
  47. maque/clustering/clusterers.py +464 -0
  48. maque/clustering/sampler.py +134 -0
  49. maque/clustering/visualizer.py +205 -0
  50. maque/constant.py +13 -0
  51. maque/core.py +133 -0
  52. maque/cv/__init__.py +1 -0
  53. maque/cv/image.py +219 -0
  54. maque/cv/utils.py +68 -0
  55. maque/cv/video/__init__.py +3 -0
  56. maque/cv/video/keyframe_extractor.py +368 -0
  57. maque/embedding/__init__.py +43 -0
  58. maque/embedding/base.py +56 -0
  59. maque/embedding/multimodal.py +308 -0
  60. maque/embedding/server.py +523 -0
  61. maque/embedding/text.py +311 -0
  62. maque/git/__init__.py +24 -0
  63. maque/git/pure_git.py +912 -0
  64. maque/io/__init__.py +29 -0
  65. maque/io/core.py +38 -0
  66. maque/io/ops.py +194 -0
  67. maque/llm/__init__.py +111 -0
  68. maque/llm/backend.py +416 -0
  69. maque/llm/base.py +411 -0
  70. maque/llm/server.py +366 -0
  71. maque/mcp_server.py +1096 -0
  72. maque/mllm_data_processor_pipeline/__init__.py +17 -0
  73. maque/mllm_data_processor_pipeline/core.py +341 -0
  74. maque/mllm_data_processor_pipeline/example.py +291 -0
  75. maque/mllm_data_processor_pipeline/steps/__init__.py +56 -0
  76. maque/mllm_data_processor_pipeline/steps/data_alignment.py +267 -0
  77. maque/mllm_data_processor_pipeline/steps/data_loader.py +172 -0
  78. maque/mllm_data_processor_pipeline/steps/data_validation.py +304 -0
  79. maque/mllm_data_processor_pipeline/steps/format_conversion.py +411 -0
  80. maque/mllm_data_processor_pipeline/steps/mllm_annotation.py +331 -0
  81. maque/mllm_data_processor_pipeline/steps/mllm_refinement.py +446 -0
  82. maque/mllm_data_processor_pipeline/steps/result_validation.py +501 -0
  83. maque/mllm_data_processor_pipeline/web_app.py +317 -0
  84. maque/nlp/__init__.py +14 -0
  85. maque/nlp/ngram.py +9 -0
  86. maque/nlp/parser.py +63 -0
  87. maque/nlp/risk_matcher.py +543 -0
  88. maque/nlp/sentence_splitter.py +202 -0
  89. maque/nlp/simple_tradition_cvt.py +31 -0
  90. maque/performance/__init__.py +21 -0
  91. maque/performance/_measure_time.py +70 -0
  92. maque/performance/_profiler.py +367 -0
  93. maque/performance/_stat_memory.py +51 -0
  94. maque/pipelines/__init__.py +15 -0
  95. maque/pipelines/clustering.py +252 -0
  96. maque/quantization/__init__.py +42 -0
  97. maque/quantization/auto_round.py +120 -0
  98. maque/quantization/base.py +145 -0
  99. maque/quantization/bitsandbytes.py +127 -0
  100. maque/quantization/llm_compressor.py +102 -0
  101. maque/retriever/__init__.py +35 -0
  102. maque/retriever/chroma.py +654 -0
  103. maque/retriever/document.py +140 -0
  104. maque/retriever/milvus.py +1140 -0
  105. maque/table_ops/__init__.py +1 -0
  106. maque/table_ops/core.py +133 -0
  107. maque/table_viewer/__init__.py +4 -0
  108. maque/table_viewer/download_assets.py +57 -0
  109. maque/table_viewer/server.py +698 -0
  110. maque/table_viewer/static/element-plus-icons.js +5791 -0
  111. maque/table_viewer/static/element-plus.css +1 -0
  112. maque/table_viewer/static/element-plus.js +65236 -0
  113. maque/table_viewer/static/main.css +268 -0
  114. maque/table_viewer/static/main.js +669 -0
  115. maque/table_viewer/static/vue.global.js +18227 -0
  116. maque/table_viewer/templates/index.html +401 -0
  117. maque/utils/__init__.py +56 -0
  118. maque/utils/color.py +68 -0
  119. maque/utils/color_string.py +45 -0
  120. maque/utils/compress.py +66 -0
  121. maque/utils/constant.py +183 -0
  122. maque/utils/core.py +261 -0
  123. maque/utils/cursor.py +143 -0
  124. maque/utils/distance.py +58 -0
  125. maque/utils/docker.py +96 -0
  126. maque/utils/downloads.py +51 -0
  127. maque/utils/excel_helper.py +542 -0
  128. maque/utils/helper_metrics.py +121 -0
  129. maque/utils/helper_parser.py +168 -0
  130. maque/utils/net.py +64 -0
  131. maque/utils/nvidia_stat.py +140 -0
  132. maque/utils/ops.py +53 -0
  133. maque/utils/packages.py +31 -0
  134. maque/utils/path.py +57 -0
  135. maque/utils/tar.py +260 -0
  136. maque/utils/untar.py +129 -0
  137. maque/web/__init__.py +0 -0
  138. maque/web/image_downloader.py +1410 -0
  139. maque-0.2.1.dist-info/METADATA +450 -0
  140. maque-0.2.1.dist-info/RECORD +143 -0
  141. maque-0.2.1.dist-info/WHEEL +4 -0
  142. maque-0.2.1.dist-info/entry_points.txt +3 -0
  143. maque-0.2.1.dist-info/licenses/LICENSE +21 -0
maque/api/common.py ADDED
@@ -0,0 +1,110 @@
1
+ import os
2
+ from fastapi import FastAPI, status
3
+ from fastapi.openapi.docs import (
4
+ get_redoc_html,
5
+ get_swagger_ui_html,
6
+ get_swagger_ui_oauth2_redirect_html,
7
+ )
8
+ from fastapi.middleware.cors import CORSMiddleware
9
+ from fastapi import APIRouter
10
+ from fastapi.staticfiles import StaticFiles
11
+ from fastapi.responses import JSONResponse
12
+ from pydantic import BaseModel
13
+ import orjson
14
+
15
+
16
+ def add_healthcheck(app: FastAPI) -> None:
17
+ class HealthCheck(BaseModel):
18
+ status: str = "OK"
19
+
20
+ @app.get(
21
+ "/healthz",
22
+ summary="Perform a Health Check",
23
+ response_description="Return HTTP Status Code 200 (OK)",
24
+ status_code=status.HTTP_200_OK,
25
+ response_model=HealthCheck,
26
+ )
27
+ def get_health() -> HealthCheck:
28
+ """
29
+ Perform a Health Check
30
+ Endpoint to perform a healthcheck on. This endpoint can primarily be used Docker
31
+ to ensure a robust container orchestration and management is in place. Other
32
+ services which rely on proper functioning of the API service will not deploy if this
33
+ endpoint returns any other HTTP status code except 200 (OK).
34
+ Returns:
35
+ HealthCheck: Returns a JSON response with the health status
36
+ """
37
+ return HealthCheck(status="OK")
38
+
39
+
40
+ class ORJSONResponse(JSONResponse):
41
+ media_type = "application/json"
42
+
43
+ def render(self, content) -> bytes:
44
+ return orjson.dumps(content, option=orjson.OPT_SERIALIZE_NUMPY)
45
+
46
+
47
+ def set_app_doc_static_file(app, favicon_url="/static/icon.png"):
48
+ root = os.path.dirname(os.path.realpath(__file__))
49
+ app.mount("/static", StaticFiles(directory=f"{root}/static"), name="static")
50
+
51
+ @app.get("/docs", include_in_schema=False)
52
+ async def custom_swagger_ui_html():
53
+ return get_swagger_ui_html(
54
+ openapi_url=app.openapi_url,
55
+ title=app.title + " - Swagger UI",
56
+ oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url,
57
+ swagger_js_url="/static/swagger-ui-bundle.js",
58
+ swagger_css_url="/static/swagger-ui.css",
59
+ swagger_favicon_url=f"{favicon_url}",
60
+ )
61
+
62
+ @app.get(app.swagger_ui_oauth2_redirect_url, include_in_schema=False)
63
+ async def swagger_ui_redirect():
64
+ return get_swagger_ui_oauth2_redirect_html()
65
+
66
+ @app.get("/redoc", include_in_schema=False)
67
+ async def redoc_html():
68
+ return get_redoc_html(
69
+ openapi_url=app.openapi_url,
70
+ title=app.title + " - ReDoc",
71
+ redoc_js_url="/static/redoc.standalone.js",
72
+ redoc_favicon_url=f"{favicon_url}",
73
+ )
74
+
75
+
76
+ def create_app(title, version='1.0', add_health_route=False, openapi_tags=None, set_static_file=True,
77
+ favicon_url='/static/icon.png') -> FastAPI:
78
+ if openapi_tags is None:
79
+ openapi_tags = [
80
+ {"name": "private",
81
+ "description": "私有接口"},
82
+ {"name": "public",
83
+ "description": "公有接口"},
84
+ ]
85
+ app = FastAPI(openapi_tags=openapi_tags,
86
+ title=title,
87
+ version=version,
88
+ description="",
89
+ docs_url=None,
90
+ redoc_url=None,
91
+ default_response_class=ORJSONResponse,
92
+ )
93
+ if set_static_file:
94
+ set_app_doc_static_file(app, favicon_url=favicon_url)
95
+ app.add_middleware(
96
+ CORSMiddleware,
97
+ allow_origins=["*"],
98
+ allow_credentials=True,
99
+ allow_methods=["*"],
100
+ allow_headers=["*"],
101
+ )
102
+
103
+ if add_health_route:
104
+ add_healthcheck(app)
105
+
106
+ return app
107
+
108
+
109
+ def add_router(app: FastAPI, router: APIRouter):
110
+ app.include_router(router)
maque/api/fetch.py ADDED
@@ -0,0 +1,26 @@
1
+ import aiohttp
2
+ import asyncio
3
+
4
+
5
+ async def fetch(session, url, data, method):
6
+ if method == "POST":
7
+ async with session.post(url, json=data) as response:
8
+ return await response.text()
9
+ elif method == "GET":
10
+ async with session.get(url, params=data) as response:
11
+ return await response.text()
12
+ else:
13
+ raise ValueError("Invalid method specified")
14
+
15
+ async def run(url, data: dict, method: str = "POST", concurrent: int = 10):
16
+ if method not in ["POST", "GET"]:
17
+ raise ValueError("Method should be either 'POST' or 'GET'")
18
+
19
+ tasks = []
20
+ async with aiohttp.ClientSession() as session:
21
+ for _ in range(concurrent):
22
+ task = asyncio.create_task(fetch(session, url, data, method))
23
+ tasks.append(task)
24
+ responses = await asyncio.gather(*tasks)
25
+ for response in responses:
26
+ print(response)
Binary file