xlite 0.1.0rc12__tar.gz

xlite-0.1.0rc12/CMakeLists.txt

@@ -0,0 +1,221 @@
+cmake_minimum_required(VERSION 3.16)
+project(xlite)
+
+set(RUN_MODE "npu" CACHE STRING "sim/npu")
+set(SOC_VERSION "Ascend910B3" CACHE STRING "system on chip type")
+set(ASCEND_CANN_PACKAGE_PATH "/usr/local/Ascend/ascend-toolkit/latest"
+    CACHE STRING "ASCEND CANN package installation directory"
+)
+message(STATUS "RUN_MODE: ${RUN_MODE}")
+message(STATUS "SOC_VERSION: ${SOC_VERSION}")
+message(STATUS "ASCEND_CANN_PACKAGE_PATH: ${ASCEND_CANN_PACKAGE_PATH}")
+
+find_program(CCACHE_FOUND ccache)
+if (CCACHE_FOUND)
+    set(CMAKE_CXX_COMPILER_LAUNCHER ccache)
+endif()
+
+if(NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type Release/Debug (default Release)" FORCE)
+endif()
+if(CMAKE_INSTALL_PREFIX STREQUAL /usr/local)
+    set(CMAKE_INSTALL_PATH ${CMAKE_CURRENT_LIST_DIR})
+    set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/out" CACHE STRING "path for install()" FORCE)
+else()
+    set(CMAKE_INSTALL_PATH ${CMAKE_INSTALL_PREFIX})
+endif()
+message(STATUS "CMAKE_INSTALL_PATH: ${CMAKE_INSTALL_PATH}")
+
+# xlite_kernels_common_${RUN_MODE}
+file(GLOB KERNEL_FILES_COMMON
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/add_bias_float.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/group_matmul_float.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/matmul_float.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/silu_and_mul_float.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/softmax_topk_float.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/sigmoid_topk_float.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/topk_float.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/cast_bfloat16_t_float.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/softmax.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/permutation.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/unpermutation_float.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/all_reduce.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/reduce_scatter.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/all_gather.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/matmul_int8_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/group_matmul_int8_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/experts_counts_sum.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/reorder_moe.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/conv1d_and_silu_float.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/beta_decay_float.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/transpose_1_2_float.cpp
+)
+
+# xlite_kernels_bf16_${RUN_MODE}
+file(GLOB KERNEL_FILES_BF16
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/add_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/add_bias_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/cast_float_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/embed_kernel_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/group_matmul_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/matmul_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/rope_and_cache_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/silu_and_mul_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/softmax_topk_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/sigmoid_topk_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/topk_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/unpermutation_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/attention_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/rope_complex_and_cache_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/flash_attention_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/quant_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/quant_dyn_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/flash_mla_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/norm_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/mla_prepare_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/indexer_scores_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/muls_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/mla_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/conv1d_and_silu_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/beta_decay_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/transpose_1_2_bfloat16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/indexer_topk_bfloat16_t.cpp
+)
+
+# xlite_kernels_f16_${RUN_MODE}
+file(GLOB KERNEL_FILES_F16
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/add_bias_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/add_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/embed_kernel_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/group_matmul_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/matmul_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/rope_and_cache_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/silu_and_mul_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/attention_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/rope_complex_and_cache_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/flash_attention_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/dequant_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/norm_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/mla_prepare_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/indexer_scores_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/muls_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/conv1d_and_silu_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/beta_decay_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/transpose_1_2_float16_t.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/kernels/indexer_topk_float16_t.cpp
+)
+
+if(EXISTS ${ASCEND_CANN_PACKAGE_PATH}/compiler/tikcpp/ascendc_kernel_cmake)
+    set(ASCENDC_CMAKE_DIR ${ASCEND_CANN_PACKAGE_PATH}/compiler/tikcpp/ascendc_kernel_cmake)
+elseif(EXISTS ${ASCEND_CANN_PACKAGE_PATH}/tools/tikcpp/ascendc_kernel_cmake)
+    set(ASCENDC_CMAKE_DIR ${ASCEND_CANN_PACKAGE_PATH}/tools/tikcpp/ascendc_kernel_cmake)
+else()
+    message(FATAL_ERROR "ascendc_kernel_cmake does not exist ,please check whether the cann package is installed")
+endif()
+include(${ASCENDC_CMAKE_DIR}/ascendc.cmake)
+
+# ascendc_library use to add kernel file to generate ascendc library
+ascendc_library(xlite_kernels_common_${RUN_MODE} SHARED ${KERNEL_FILES_COMMON})
+ascendc_library(xlite_kernels_bf16_${RUN_MODE} SHARED ${KERNEL_FILES_BF16})
+ascendc_library(xlite_kernels_f16_${RUN_MODE} SHARED ${KERNEL_FILES_F16})
+
+option(XLITE_DEBUG "Enable xlite debug mode" OFF)
+
+# xlite
+add_library(xlite SHARED
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/base.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/sock.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/swizzle.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/op.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/runtime.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/model.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/core_assigner.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/ccl.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/csrc/auto_tuner.cpp
+)
+
+target_compile_options(xlite PRIVATE
+    -std=c++17 -O3 -Wall -Werror
+)
+
+target_link_libraries(xlite PRIVATE
+    xlite_kernels_common_${RUN_MODE}
+    xlite_kernels_bf16_${RUN_MODE}
+    xlite_kernels_f16_${RUN_MODE}
+    hccl
+    torch
+)
+
+if(XLITE_DEBUG)
+    target_compile_definitions(xlite PRIVATE XLITE_DEBUG_ON)
+endif()
+
+# _C
+find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED)
+
+execute_process(
+    COMMAND ${Python3_EXECUTABLE} -c "import sysconfig; print(sysconfig.get_path('purelib'))"
+    OUTPUT_VARIABLE PYTHON_SYS_DIR
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+if(PYTHON_SYS_DIR)
+  # prepend so Python sys dir is searched first; use APPEND if you want it last
+  list(PREPEND CMAKE_PREFIX_PATH "${PYTHON_SYS_DIR}")
+endif()
+
+find_package(pybind11 REQUIRED)
+find_package(Torch COMPONENTS Development REQUIRED)
+
+message(STATUS "Python3 executable: ${Python3_EXECUTABLE}")
+message(STATUS "pybind11 include: ${pybind11_INCLUDE_DIRS}")
+message(STATUS "torch cpp_extension include: ${TORCH_INCLUDE_DIRS}")
+message(STATUS "torch cpp_extension lib: ${TORCH_INSTALL_PREFIX}/lib")
+
+pybind11_add_module(
+    _C
+    csrc/_C.cpp
+)
+
+target_compile_options(_C PRIVATE
+    -std=c++17 -O3 -Wall -Werror -fPIC
+)
+
+target_include_directories(_C PRIVATE
+    ${pybind11_INCLUDE_DIRS}
+    ${TORCH_INCLUDE_DIRS}
+)
+
+target_link_directories(_C PRIVATE
+    ${TORCH_INSTALL_PREFIX}/lib
+)
+
+target_link_libraries(_C PRIVATE
+    xlite
+    c10
+    torch
+    torch_python
+    torch_cpu
+)
+
+set(XLITE_EDITABLE_BUILD "OFF" CACHE STRING "Enable editable build and include torch from source path")
+if (XLITE_EDITABLE_BUILD STREQUAL "ON")
+    set(TORCH_RPATH "\$ORIGIN/../torch/lib:${TORCH_INSTALL_PREFIX}/lib")
+else()
+    set(TORCH_RPATH "\$ORIGIN/../torch/lib")
+endif()
+message(STATUS "TORCH_RPATH: ${TORCH_RPATH}")
+
+set_target_properties(xlite PROPERTIES
+    LINK_FLAGS "-Wl,-rpath,\$ORIGIN:\$ORIGIN/lib:${TORCH_RPATH}"
+)
+set_target_properties(_C PROPERTIES
+    LINK_FLAGS "-Wl,-rpath,\$ORIGIN:\$ORIGIN/lib:${TORCH_RPATH}"
+)
+
+install(TARGETS xlite_kernels_common_${RUN_MODE} xlite_kernels_bf16_${RUN_MODE} xlite_kernels_f16_${RUN_MODE} xlite
+    LIBRARY DESTINATION ${CMAKE_INSTALL_PATH}/xlite/lib
+)
+
+install(TARGETS _C
+    LIBRARY DESTINATION ${CMAKE_INSTALL_PATH}/xlite
+)

xlite-0.1.0rc12/LICENSE

@@ -0,0 +1,194 @@
+木兰宽松许可证，第2版
+
+木兰宽松许可证，第2版
+
+2020年1月 http://license.coscl.org.cn/MulanPSL2
+
+您对“软件”的复制、使用、修改及分发受木兰宽松许可证，第2版（“本许可证”）的如下条款的约束：
+
+0.   定义
+
+“软件” 是指由“贡献”构成的许可在“本许可证”下的程序和相关文档的集合。
+
+“贡献” 是指由任一“贡献者”许可在“本许可证”下的受版权法保护的作品。
+
+“贡献者” 是指将受版权法保护的作品许可在“本许可证”下的自然人或“法人实体”。
+
+“法人实体” 是指提交贡献的机构及其“关联实体”。
+
+“关联实体” 是指，对“本许可证”下的行为方而言，控制、受控制或与其共同受控制的机构，此处的控制是
+指有受控方或共同受控方至少50%直接或间接的投票权、资金或其他有价证券。
+
+1.   授予版权许可
+
+每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的版权许可，您可
+以复制、使用、修改、分发其“贡献”，不论修改与否。
+
+2.   授予专利许可
+
+每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的（根据本条规定
+撤销除外）专利许可，供您制造、委托制造、使用、许诺销售、销售、进口其“贡献”或以其他方式转移其“贡
+献”。前述专利许可仅限于“贡献者”现在或将来拥有或控制的其“贡献”本身或其“贡献”与许可“贡献”时的“软
+件”结合而将必然会侵犯的专利权利要求，不包括对“贡献”的修改或包含“贡献”的其他结合。如果您或您的“
+关联实体”直接或间接地，就“软件”或其中的“贡献”对任何人发起专利侵权诉讼（包括反诉或交叉诉讼）或
+其他专利维权行动，指控其侵犯专利权，则“本许可证”授予您对“软件”的专利许可自您提起诉讼或发起维权
+行动之日终止。
+
+3.   无商标许可
+
+“本许可证”不提供对“贡献者”的商品名称、商标、服务标志或产品名称的商标许可，但您为满足第4条规定
+的声明义务而必须使用除外。
+
+4.   分发限制
+
+您可以在任何媒介中将“软件”以源程序形式或可执行形式重新分发，不论修改与否，但您必须向接收者提供“
+本许可证”的副本，并保留“软件”中的版权、商标、专利及免责声明。
+
+5.   免责声明与责任限制
+
+“软件”及其中的“贡献”在提供时不带任何明示或默示的担保。在任何情况下，“贡献者”或版权所有者不对
+任何人因使用“软件”或其中的“贡献”而引发的任何直接或间接损失承担责任，不论因何种原因导致或者基于
+何种法律理论，即使其曾被建议有此种损失的可能性。
+
+6.   语言
+
+“本许可证”以中英文双语表述，中英文版本具有同等法律效力。如果中英文版本存在任何冲突不一致，以中文
+版为准。
+
+条款结束
+
+如何将木兰宽松许可证，第2版，应用到您的软件
+
+如果您希望将木兰宽松许可证，第2版，应用到您的新软件，为了方便接收者查阅，建议您完成如下三步：
+
+1， 请您补充如下声明中的空白，包括软件名、软件的首次发表年份以及您作为版权人的名字；
+
+2， 请您在软件包的一级目录下创建以“LICENSE”为名的文件，将整个许可证文本放入该文件中；
+
+3， 请将如下声明文本放入每个源文件的头部注释中。
+
+Copyright (c) [Year] [name of copyright holder]
+[Software Name] is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan
+PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY
+KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+
+Mulan Permissive Software License，Version 2
+
+Mulan Permissive Software License，Version 2 (Mulan PSL v2)
+
+January 2020 http://license.coscl.org.cn/MulanPSL2
+
+Your reproduction, use, modification and distribution of the Software shall
+be subject to Mulan PSL v2 (this License) with the following terms and
+conditions:
+
+0. Definition
+
+Software means the program and related documents which are licensed under
+this License and comprise all Contribution(s).
+
+Contribution means the copyrightable work licensed by a particular
+Contributor under this License.
+
+Contributor means the Individual or Legal Entity who licenses its
+copyrightable work under this License.
+
+Legal Entity means the entity making a Contribution and all its
+Affiliates.
+
+Affiliates means entities that control, are controlled by, or are under
+common control with the acting entity under this License, ‘control’ means
+direct or indirect ownership of at least fifty percent (50%) of the voting
+power, capital or other securities of controlled or commonly controlled
+entity.
+
+1. Grant of Copyright License
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to you a perpetual, worldwide, royalty-free, non-exclusive,
+irrevocable copyright license to reproduce, use, modify, or distribute its
+Contribution, with modification or not.
+
+2. Grant of Patent License
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to you a perpetual, worldwide, royalty-free, non-exclusive,
+irrevocable (except for revocation under this Section) patent license to
+make, have made, use, offer for sale, sell, import or otherwise transfer its
+Contribution, where such patent license is only limited to the patent claims
+owned or controlled by such Contributor now or in future which will be
+necessarily infringed by its Contribution alone, or by combination of the
+Contribution with the Software to which the Contribution was contributed.
+The patent license shall not apply to any modification of the Contribution,
+and any other combination which includes the Contribution. If you or your
+Affiliates directly or indirectly institute patent litigation (including a
+cross claim or counterclaim in a litigation) or other patent enforcement
+activities against any individual or entity by alleging that the Software or
+any Contribution in it infringes patents, then any patent license granted to
+you under this License for the Software shall terminate as of the date such
+litigation or activity is filed or taken.
+
+3. No Trademark License
+
+No trademark license is granted to use the trade names, trademarks, service
+marks, or product names of Contributor, except as required to fulfill notice
+requirements in section 4.
+
+4. Distribution Restriction
+
+You may distribute the Software in any medium with or without modification,
+whether in source or executable forms, provided that you provide recipients
+with a copy of this License and retain copyright, patent, trademark and
+disclaimer statements in the Software.
+
+5. Disclaimer of Warranty and Limitation of Liability
+
+THE SOFTWARE AND CONTRIBUTION IN IT ARE PROVIDED WITHOUT WARRANTIES OF ANY
+KIND, EITHER EXPRESS OR IMPLIED. IN NO EVENT SHALL ANY CONTRIBUTOR OR
+COPYRIGHT HOLDER BE LIABLE TO YOU FOR ANY DAMAGES, INCLUDING, BUT NOT
+LIMITED TO ANY DIRECT, OR INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING
+FROM YOUR USE OR INABILITY TO USE THE SOFTWARE OR THE CONTRIBUTION IN IT, NO
+MATTER HOW IT’S CAUSED OR BASED ON WHICH LEGAL THEORY, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGES.
+
+6. Language
+
+THIS LICENSE IS WRITTEN IN BOTH CHINESE AND ENGLISH, AND THE CHINESE VERSION
+AND ENGLISH VERSION SHALL HAVE THE SAME LEGAL EFFECT. IN THE CASE OF
+DIVERGENCE BETWEEN THE CHINESE AND ENGLISH VERSIONS, THE CHINESE VERSION
+SHALL PREVAIL.
+
+END OF THE TERMS AND CONDITIONS
+
+How to Apply the Mulan Permissive Software License，Version 2
+(Mulan PSL v2) to Your Software
+
+To apply the Mulan PSL v2 to your work, for easy identification by
+recipients, you are suggested to complete following three steps:
+
+i. Fill in the blanks in following statement, including insert your software
+name, the year of the first publication of your software, and your name
+identified as the copyright owner;
+
+ii. Create a file named "LICENSE" which contains the whole context of this
+License in the first directory of your software package;
+
+iii. Attach the statement to the appropriate annotated syntax at the
+beginning of each source file.
+
+Copyright (c) [Year] [name of copyright holder]
+[Software Name] is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan
+PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY
+KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.

xlite-0.1.0rc12/MANIFEST.in

@@ -0,0 +1,8 @@
+include pyproject.toml
+include setup.py
+include CMakeLists.txt
+include LICENSE
+include README.md
+include requirements*.txt
+
+recursive-include csrc *

xlite-0.1.0rc12/PKG-INFO

@@ -0,0 +1,131 @@
+Metadata-Version: 2.4
+Name: xlite
+Version: 0.1.0rc12
+Summary: A lightweight, effective and easy-to-extend inference runtime
+Author: Huawei Euler OS Team
+License-Expression: MulanPSL-2.0
+Project-URL: Homepage, https://atomgit.com/openeuler/GVirt
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch>=2.5.1
+Provides-Extra: dev
+Requires-Dist: setuptools<82.0.0,>=64.0.0; extra == "dev"
+Requires-Dist: cmake>=3.22.0; extra == "dev"
+Requires-Dist: numpy<2.0.0; extra == "dev"
+Requires-Dist: tqdm; extra == "dev"
+Requires-Dist: safetensors; extra == "dev"
+Requires-Dist: tiktoken; extra == "dev"
+Requires-Dist: sentencepiece; extra == "dev"
+Requires-Dist: protobuf; extra == "dev"
+Requires-Dist: transformers; extra == "dev"
+Requires-Dist: pyyaml>=6.0; extra == "dev"
+Requires-Dist: torch-npu>=2.5.1; extra == "dev"
+Requires-Dist: clang-format==22.1.0; extra == "dev"
+Requires-Dist: clang-tidy==22.1.0; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Dynamic: license-file
+
+<div align="center">
+  <img src="doc/images/xlite_logo.png" alt="xlite logo">
+</div>
+
+<h1 align="center">Xlite轻量化推理运行时</h1>
+
+## 介绍
+
+xlite (GVirt前端)：轻量级Transformer模型运行时，支持多样性算力协同，当前支持在昇腾硬件上高效运行。
+xlite公开了Transformer运行所需的模型构图以及算子，所有算子基于昇腾AscendC/CCE开发。
+
+
+## 背景与动机
+
+在大模型推理场景中，传统的单流串行执行模式存在以下问题：
+
+- **核间负载不均**：不同AICORE之间的任务分配不均衡，部分核心闲置
+- **资源浪费多**：计算资源和传输资源利用率低，存在显著浪费
+- **执行时间长**：Host CPU下发算子开销大，造成严重的host bond问题
+
+xlite通过以下技术手段解决上述问题：
+
+- **多流并行**：充分利用卡内资源，将单流串行改为多流并行执行
+- **核间负载均衡**：核间负载均衡，提升资源利用率
+- **CPU NPU协同**：C++侧完全消除Python的GC、线程等干扰，简化Host tiling计算，去除小块内存申请释放及拷贝，消除Host bond
+
+性能效果显著：
+
+在GLM-4.7双机推理场景（40K输入、1K输出、prefix cache命中率约90%）下：
+
+- TPOT时延降低17%~30%
+- 吞吐提升13%~41%
+
+详细性能数据参考 [PR #7935](https://github.com/vllm-project/vllm-ascend/pull/7935)。
+
+
+
+## 软件架构
+
+![image](doc/images/architecture.png)
+
+xlite已适配vllm_ascend，可通过xlite_graph_config配置快速使能xlite加速效果，使用方法参考[官方指导文档](https://docs.vllm.ai/projects/ascend/en/latest/user_guide/feature_guide/graph_mode.html)。
+
+xlite支持模型见[模型列表](doc/models.md)。
+
+
+## 快速开始
+
+### 安装
+
+```bash
+# 安装vllm_ascend, 可参考https://github.com/vllm-project/vllm-ascend/blob/main/README.md
+# 安装xlite
+pip install xlite --extra-index https://download.pytorch.org/whl/cpu/
+```
+
+### 离线推理示例
+
+```python
+import os
+from vllm import LLM
+
+# xlite默认支持decode-only模式, 可通过设置 "full_mode": True 使能full模式
+model = LLM(model="path/to/Qwen3-32B", tensor_parallel_size=8, additional_config={"xlite_graph_config": {"enabled": True, "full_mode": True}})
+outputs = model.generate("Hello, how are you?")
+```
+
+### 在线服务示例
+
+```bash
+vllm serve path/to/Qwen3-32B --tensor-parallel-size 8 --additional-config='{"xlite_graph_config": {"enabled": true, "full_mode": true}}'
+```
+
+## 开发指南
+
+编译构建、容器镜像、源码安装等开发相关内容，请参考 [开发指南](doc/contributing.md)
+
+## 环境变量
+
+xlite使用的环境变量及其配置方法，请参考 [环境变量](doc/ENVIRONMENT_VARIABLES.md)
+
+## 目录结构
+
+```
+.
+├── csrc/    - 轻量化运行时的核心代码
+├── xlite/   - python代码，包括tools等
+├── doc/     - 相关文档介绍
+├── docker/  - 容器镜像Dockerfile
+└── tests/   - 测试用例
+```
+
+## 致谢
+
+本项目的核心算子初始版本和思路来自于华为终端小艺AI Infra团队的贡献，相关优化实现可参考论文：
+
+[《XY-Serve: End-to-End Versatile Production Serving for Dynamic LLM Workloads》 [ASPLOS 2026]](https://arxiv.org/abs/2412.18106)

xlite-0.1.0rc12/README.md

@@ -0,0 +1,97 @@
+<div align="center">
+  <img src="doc/images/xlite_logo.png" alt="xlite logo">
+</div>
+
+<h1 align="center">Xlite轻量化推理运行时</h1>
+
+## 介绍
+
+xlite (GVirt前端)：轻量级Transformer模型运行时，支持多样性算力协同，当前支持在昇腾硬件上高效运行。
+xlite公开了Transformer运行所需的模型构图以及算子，所有算子基于昇腾AscendC/CCE开发。
+
+
+## 背景与动机
+
+在大模型推理场景中，传统的单流串行执行模式存在以下问题：
+
+- **核间负载不均**：不同AICORE之间的任务分配不均衡，部分核心闲置
+- **资源浪费多**：计算资源和传输资源利用率低，存在显著浪费
+- **执行时间长**：Host CPU下发算子开销大，造成严重的host bond问题
+
+xlite通过以下技术手段解决上述问题：
+
+- **多流并行**：充分利用卡内资源，将单流串行改为多流并行执行
+- **核间负载均衡**：核间负载均衡，提升资源利用率
+- **CPU NPU协同**：C++侧完全消除Python的GC、线程等干扰，简化Host tiling计算，去除小块内存申请释放及拷贝，消除Host bond
+
+性能效果显著：
+
+在GLM-4.7双机推理场景（40K输入、1K输出、prefix cache命中率约90%）下：
+
+- TPOT时延降低17%~30%
+- 吞吐提升13%~41%
+
+详细性能数据参考 [PR #7935](https://github.com/vllm-project/vllm-ascend/pull/7935)。
+
+
+
+## 软件架构
+
+![image](doc/images/architecture.png)
+
+xlite已适配vllm_ascend，可通过xlite_graph_config配置快速使能xlite加速效果，使用方法参考[官方指导文档](https://docs.vllm.ai/projects/ascend/en/latest/user_guide/feature_guide/graph_mode.html)。
+
+xlite支持模型见[模型列表](doc/models.md)。
+
+
+## 快速开始
+
+### 安装
+
+```bash
+# 安装vllm_ascend, 可参考https://github.com/vllm-project/vllm-ascend/blob/main/README.md
+# 安装xlite
+pip install xlite --extra-index https://download.pytorch.org/whl/cpu/
+```
+
+### 离线推理示例
+
+```python
+import os
+from vllm import LLM
+
+# xlite默认支持decode-only模式, 可通过设置 "full_mode": True 使能full模式
+model = LLM(model="path/to/Qwen3-32B", tensor_parallel_size=8, additional_config={"xlite_graph_config": {"enabled": True, "full_mode": True}})
+outputs = model.generate("Hello, how are you?")
+```
+
+### 在线服务示例
+
+```bash
+vllm serve path/to/Qwen3-32B --tensor-parallel-size 8 --additional-config='{"xlite_graph_config": {"enabled": true, "full_mode": true}}'
+```
+
+## 开发指南
+
+编译构建、容器镜像、源码安装等开发相关内容，请参考 [开发指南](doc/contributing.md)
+
+## 环境变量
+
+xlite使用的环境变量及其配置方法，请参考 [环境变量](doc/ENVIRONMENT_VARIABLES.md)
+
+## 目录结构
+
+```
+.
+├── csrc/    - 轻量化运行时的核心代码
+├── xlite/   - python代码，包括tools等
+├── doc/     - 相关文档介绍
+├── docker/  - 容器镜像Dockerfile
+└── tests/   - 测试用例
+```
+
+## 致谢
+
+本项目的核心算子初始版本和思路来自于华为终端小艺AI Infra团队的贡献，相关优化实现可参考论文：
+
+[《XY-Serve: End-to-End Versatile Production Serving for Dynamic LLM Workloads》 [ASPLOS 2026]](https://arxiv.org/abs/2412.18106)