union_kb_ingest 1.0.1 → 1.0.2

package/ArkKickidcService.java

@@ -0,0 +1,578 @@
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.TypeReference;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by liuhonghui on 2019/10/7.
+ */
+@Service
+@Slf4j
+@Data
+public class ArkKickidcService {
+
+    Logger securityLogger = LoggerFactory.getLogger(LogFileType.ARK_SECURITY.getLogFileType());
+
+    @Value("${db.name}")
+    private String dbName;
+
+    @Value("${db.key}")
+    private String dbKey;
+
+    @Value("${application.name}")
+    private String applicationName;
+
+    @Value("${exchange.db.url}")
+    private String dbUrl;
+
+    @Value("${exchange.db.type}")
+    private String dbType;
+
+    @Value("${exchange.db.golden.url}")
+    private String goldenDbUrl;
+
+    @Autowired
+    private TaskServiceClient taskServiceClient;
+
+    @Autowired
+    private ArkControlBaseFacade arkControlBaseFacade;
+
+    @Autowired
+    private DictCacheService dictCacheService;
+
+    private static final String UPDATEDBY = "kickidc";
+    private static final String GOLDEN_DB_TYPE = "goldenDb";
+    private static final String GOLDEN_DB_SUCCESS_CODE = "20000";
+    private static final String GOLDEN_DB_PARAM_ERROR_CODE = "10001";
+    private static final String GOLDEN_DB_DUPLICATE_REQUEST_CODE = "10002";
+    private static final String GOLDEN_DB_SWITCH_FAIL_CODE = "10003";
+    private static final String GOLDEN_DB_INTERNAL_ERROR_CODE = "10004";
+    private static final String GOLDEN_DB_PLAN_SWITCH = "planswitch";
+
+    /**
+     * 提机房步骤
+     */
+    public List<String> arkKickidc(String idc, String requestTime) {
+        List<String> list = new ArrayList<>();
+        createMDCParam(idc, requestTime);
+
+        // 1. 确定在制定idc是否有部署服务
+        // 有部署服务
+        AnnounceIDCInfoEnum kickIdc = AnnounceIDCInfoEnum.explainIdc(idc);
+
+        // KickIdcParamEnum kickIdc = KickIdcParamEnum.explain(idc);
+        if (null == kickIdc) {
+            MDC.put("idc", idc);
+            MDC.put("requestType", "2");
+            MDC.put("ip", "-");
+            MDC.put("errCode", "联合运维团队-" + idc + "无业务部署");
+            log.error("arkKickidc ark-web idc error,idc={}", idc);
+            securityLogger.info("arkKickidc ark-web idc error,idc={}", idc);
+            list.add("联合运维团队-" + idc + "无业务部署");
+            return list;
+        }
+
+        String targetIdc = kickIdc.getTargetIdc();
+        try {
+            List<AnnounceDataDictionaryDTO> targetIdcList = dictCacheService.getDictModelList("targetIdc");
+            targetIdc = targetIdcList.stream()
+                    .filter(c -> kickIdc.getKickIdc().equals(c.getDataName()))
+                    .findFirst()
+                    .get()
+                    .getDataValue();
+        } catch (Exception e) {
+            log.error("获取切换目标IDC失败，使用默认配置");
+        }
+
+        return arkChangeIdc(targetIdc, idc);
+    }
+
+    public List<String> arkRecoveridc(String targetIdc, String requestTime) {
+        List<String> list = new ArrayList<>();
+        createMDCParam(targetIdc, requestTime);
+
+        // 1. 确定在目标idc是否有部署服务
+        AnnounceIDCInfoEnum targetIdcEnum = AnnounceIDCInfoEnum.explainIdc(targetIdc);
+        // KickIdcParamEnum targetIdcEnum = KickIdcParamEnum.explain(targetIdc);
+        if (null == targetIdcEnum) {
+            MDC.put("requestType", "2");
+            MDC.put("ip", "-");
+            MDC.put("errCode", "联合运维团队-" + targetIdc + "无业务部署，无法切换");
+            log.error("联合运维团队{}机房无业务部署", targetIdc);
+            list.add("联合运维团队-" + targetIdc + "无业务部署");
+            return list;
+        }
+
+        return arkChangeIdc(targetIdc, null);
+    }
+
+    /**
+     * 切机房步骤
+     */
+    public List<String> arkChangeIdc(String targetIdc, String kickIdc) {
+        // 踢机房流程开始，需要区分mysql和goldenDb
+        // 如果能读取到开关就用开关控制，不能就配置文件控制
+        boolean useGoldenDb = GOLDEN_DB_TYPE.equals(dbType);
+        try {
+            useGoldenDb = dictCacheService.getDictModelListMap()
+                    .getOrDefault("kickGoldenDb", new ArrayList<>())
+                    .stream()
+                    .anyMatch(c -> c.getDataValue().equals("1"));
+        } catch (Exception e) {
+            log.info("读取数据字典dbType配置失败，使用配置文件dbType, dbType={}", dbType);
+        }
+
+        if (useGoldenDb) {
+            return kickGoldenDb(targetIdc, kickIdc);
+        }
+        return kickMySql(targetIdc, kickIdc);
+    }
+
+    private List<String> kickGoldenDb(String targetIdc, String kickIdc) {
+        List<String> list = new ArrayList<>();
+        try {
+            String aliveIdc = getGoldenDbMasterIdcFlag();
+            MDC.put("idc", aliveIdc);
+
+            if (kickIdc != null && !aliveIdc.equals(kickIdc)) {
+                MDC.put("idc", kickIdc);
+                MDC.put("requestType", "2");
+                MDC.put("errCode", "联合运维团队-" + kickIdc + "不是主机房，无须切换");
+                log.error("arkKickidc ark-web goldenDb idc error,idc={}", kickIdc);
+                securityLogger.info("arkKickidc ark-web goldenDb idc error,idc={}", kickIdc);
+                list.add("联合运维团队-" + kickIdc + "不是主机房，无须切换");
+                return list;
+            }
+
+            if (aliveIdc.equals(targetIdc)) {
+                MDC.put("requestType", "2");
+                MDC.put("errCode", "联合运维团队-" + targetIdc + "已是主机房，无须切换");
+                log.error("arkChangeidc ark-web goldenDb idc error,idc={}", targetIdc);
+                securityLogger.info("arkChangeidc ark-web goldenDb idc error,idc={}", targetIdc);
+                list.add("联合运维团队-" + targetIdc + "已是主机房，无须切换");
+                return list;
+            }
+
+            MDC.put("ip", targetIdc);
+            MDC.put("errCode", "联合运维团队开始执行goldenDb切机房操作,当前机房为" + aliveIdc + ",目标机房为" + targetIdc);
+            log.error("arkChangeidc ark-web goldenDb idc start,idc={}", aliveIdc);
+            securityLogger.info("arkChangeidc ark-web goldenDb idc start,idc={}", aliveIdc);
+            list.add("联合运维团队开始执行goldenDb切机房操作,所属机房为" + aliveIdc + ",目标机房为" + targetIdc);
+
+            exchangeGoldenDb(aliveIdc, targetIdc);
+            MDC.put("errCode", "联合运维团队-执行goldenDb切换操作");
+            log.error("arkChangeidc ark-web goldenDb切换完成,idc={}", aliveIdc);
+            securityLogger.info("arkChangeidc ark-web goldenDb切换完成,idc={}", aliveIdc);
+            list.add("联合运维团队-执行goldenDb切换操作成功");
+
+            this.exchangeSchedule(aliveIdc, targetIdc);
+            MDC.put("errCode", "联合运维团队-执行定时任务起停操作");
+            log.error("arkChangeidc ark-web goldenDb切换应用完成,idc={}", aliveIdc);
+            securityLogger.info("arkChangeidc ark-web goldenDb切换应用完成,idc={}", aliveIdc);
+            list.add("联合运维团队-执行定时任务起停操作成功");
+            addChangeIdcSuccessResult(list, targetIdc);
+            return list;
+        } catch (BusinessException e) {
+            MDC.put("requestType", "0");
+            MDC.put("errCode", e.getMessage());
+            log.info("arkChangeidc ark-web goldenDb idc error,idc={}", targetIdc, e);
+            log.error("arkChangeidc ark-web goldenDb idc error,idc={}", targetIdc);
+            securityLogger.info("arkChangeidc ark-web goldenDb idc error,idc={}", targetIdc);
+            list.add("联合运维团队执行goldenDb踢机房失败:" + e.getMessage());
+            return list;
+        }
+    }
+
+    /**
+     * 隔离mysql主备，用老的流程
+     */
+    private List<String> kickMySql(String targetIdc, String kickIdc) {
+        List<String> list = new ArrayList<>();
+        try {
+            String aliveIdc = getMasterIdcFlag();
+            MDC.put("idc", aliveIdc);
+
+            if (kickIdc != null && !aliveIdc.equals(kickIdc)) {
+                MDC.put("idc", kickIdc);
+                MDC.put("requestType", "2");
+                MDC.put("errCode", "联合运维团队-" + kickIdc + "不是主机房，无须切换");
+                log.error("arkKickidc ark-web idc error,idc={}", kickIdc);
+                securityLogger.info("arkKickidc ark-web idc error,idc={}", kickIdc);
+                list.add("联合运维团队-" + kickIdc + "不是主机房，无须切换");
+                return list;
+            }
+
+            // 当前机房既是目标机房
+            if (aliveIdc.equals(targetIdc)) {
+                MDC.put("requestType", "2");
+                MDC.put("errCode", "联合运维团队-" + targetIdc + "已是主机房，无须切换");
+                log.error("arkChangeidc ark-web idc error,idc={}", targetIdc);
+                securityLogger.info("arkChangeidc ark-web idc error,idc={}", targetIdc);
+                list.add("联合运维团队-" + targetIdc + "已是主机房，无须切换");
+                return list;
+            }
+
+            MDC.put("ip", targetIdc);
+            MDC.put("errCode", "联合运维团队开始执行切机房操作,当前机房为" + aliveIdc + ",目标机房为" + targetIdc);
+            log.error("arkChangeidc ark-web idc start,idc={}", aliveIdc);
+            securityLogger.info("arkKickidc ark-web idc start,idc={}", aliveIdc);
+            list.add("联合运维团队开始执行切机房操作,所属机房为" + aliveIdc + ",目标机房为" + targetIdc);
+
+            // 1) 调用db接口
+            String failcode = null;
+            failcode = this.exchangeDB(aliveIdc, targetIdc);
+            if ("11112".equals(failcode)) {
+                MDC.put("errCode", "联合运维团队-已在目标主库" + targetIdc + "无须切换");
+                log.error("arkChangeidc ark-web 切换DB完成,idc={}", aliveIdc);
+                securityLogger.info("arkChangeidc ark-web 切换DB完成,idc={}", aliveIdc);
+                MDC.put("requestType", "2");
+                MDC.put("errCode", "联合运维团队-所有切机房操作执行成功");
+                log.error("lhyw arkChangeidc success,idc={}", aliveIdc);
+                securityLogger.info("lhyw arkChangeidc success,idc={}", aliveIdc);
+                list.add("联合运维团队-已在目标主库" + aliveIdc + "无须切换");
+                list.add("联合运维团队-所有踢机房操作执行成功");
+                return list;
+            }
+
+            MDC.put("errCode", "联合运维团队-执行DB切换操作");
+            log.error("arkChangeidc ark-web 切换DB完成,idc={}", aliveIdc);
+            securityLogger.info("arkChangeidc ark-web 切换DB完成,idc={}", aliveIdc);
+            list.add("联合运维团队-执行DB切换操作切换成功");
+
+            // 2) 切换定时任务
+            this.exchangeSchedule(aliveIdc, targetIdc);
+            MDC.put("errCode", "联合运维团队-执行定时任务起停操作");
+            log.error("arkChangeidc ark-web 切换应用完成,idc={}", aliveIdc);
+            securityLogger.info("arkChangeidc ark-web 切换应用完成,idc={}", aliveIdc);
+            list.add("联合运维团队-执行定时任务起停操作成功");
+            addChangeIdcSuccessResult(list, targetIdc);
+            return list;
+        } catch (BusinessException e) {
+            MDC.put("requestType", "0");
+            MDC.put("errCode", e.getMessage());
+            log.info("arkChangeidc ark-web idc error,idc={}", targetIdc, e);
+            log.error("arkChangeidc ark-web idc error,idc={}", targetIdc);
+            securityLogger.info("arkChangeidc ark-web idc error,idc={}", targetIdc);
+            list.add("联合运维团队执行踢机房失败:" + e.getMessage());
+            return list;
+        }
+    }
+
+    private void addChangeIdcSuccessResult(List<String> list, String targetIdc) {
+        MDC.put("requestType", "2");
+        MDC.put("errCode", "联合运维团队-所有踢机房操作执行成功");
+        log.error("lhyw arkChangeidc success,idc={}", targetIdc);
+        securityLogger.info("lhyw arkChangeidc success,idc={}", targetIdc);
+        list.add("联合运维团队-所有踢机房操作执行成功");
+    }
+
+    /**
+     * 查询主机房
+     *
+     * @return
+     */
+    private String getGoldenDbMasterIdcFlag() throws BusinessException {
+        log.info("开始查询goldenDb主机房");
+        JSONObject jsonObject = new JSONObject();
+        jsonObject.put("db_name", dbName);
+
+        String response = postGoldenDb(goldenDbUrl, jsonObject, "query goldenDb masterIdc flag");
+        log.info("query goldenDb masterIdc response={}", response);
+        JSONObject resJson = JSON.parseObject(response);
+        String errCode = resJson.getString("errorCode");
+        String errMsg = resJson.getString("errorMsg");
+        Boolean checkState = resJson.getBoolean("check_state");
+        String masterIdc = resJson.getString("idc_no");
+
+        if (Boolean.TRUE.equals(checkState) && !isBlank(masterIdc)
+                && (isBlank(errCode) || "ES000000".equals(errCode))) {
+            return masterIdc;
+        }
+
+        log.error("query goldenDb masterIdc flag error,errorCode={},errorMsg={},checkState={},idcNo={}",
+                errCode, errMsg, checkState, masterIdc);
+        throw new BusinessException("查询goldenDb主机房失败:" + buildGoldenDbErrorMessage(errCode, errMsg));
+    }
+
+    private void exchangeGoldenDb(String fromIdc, String targetIdc) throws BusinessException {
+        JSONObject jsonObject = new JSONObject();
+        jsonObject.put("db_name", dbName);
+        jsonObject.put("key", dbKey);
+        jsonObject.put("to_idc", targetIdc);
+        jsonObject.put("switch_type", GOLDEN_DB_PLAN_SWITCH);
+        log.info("exchangeGoldenDb param jsonObject={}", jsonObject);
+
+        String response = postGoldenDb(goldenDbUrl, jsonObject, "exchange goldenDb");
+        log.info("exchangeGoldenDb response={}", response);
+        JSONObject resJson = JSON.parseObject(response);
+        String switchCode = resJson.getString("switch_code");
+        String errorMsg = resJson.getString("error_msg");
+        Boolean switchState = resJson.getBoolean("switch_state");
+
+        if (GOLDEN_DB_SUCCESS_CODE.equals(switchCode) && Boolean.TRUE.equals(switchState)) {
+            updateIDCStatus(fromIdc, targetIdc);
+            return;
+        }
+
+        String errorDetail = buildGoldenDbSwitchErrorMessage(switchCode, errorMsg, switchState);
+        MDC.put("errCode", errorDetail);
+        log.error("exchangeGoldenDb error idc={},targetIdc={},switchType={},switchCode={},errorMsg={},switchState={},errorDetail={}",
+                fromIdc, targetIdc, GOLDEN_DB_PLAN_SWITCH, switchCode, errorMsg, switchState, errorDetail);
+        throw new BusinessException(errorDetail);
+    }
+
+    private String postGoldenDb(String urlConfig, JSONObject jsonObject, String logPrefix) throws BusinessException {
+        if (isBlank(urlConfig)) {
+            log.error("{} url empty", logPrefix);
+            throw new BusinessException(logPrefix + "接口地址未配置");
+        }
+
+        String[] addresses = urlConfig.split(";");
+        Exception lastException = null;
+        for (int i = 0; i < addresses.length; i++) {
+            String address = addresses[i];
+            if (isBlank(address)) {
+                continue;
+            }
+            try {
+                return HttpUtils.httpPostReq(address.trim(), jsonObject, 180000);
+            } catch (Exception e) {
+                lastException = e;
+                log.error("{} address[{}] error,address={},e={}", logPrefix, i, address, e);
+            }
+        }
+
+        throw new BusinessException(logPrefix + "接口调用失败" +
+                (lastException == null ? "" : ":" + lastException.getMessage()));
+    }
+
+    private String buildGoldenDbErrorMessage(String code, String message) {
+        if (isBlank(code) && isBlank(message)) {
+            return "接口返回异常";
+        }
+        if (isBlank(message)) {
+            return code;
+        }
+        if (isBlank(code)) {
+            return message;
+        }
+        return code + "-" + message;
+    }
+
+    private String buildGoldenDbSwitchErrorMessage(String switchCode, String errorMsg, Boolean switchState) {
+        String reason;
+        if (GOLDEN_DB_PARAM_ERROR_CODE.equals(switchCode)) {
+            reason = "参数错误";
+        } else if (GOLDEN_DB_DUPLICATE_REQUEST_CODE.equals(switchCode)) {
+            reason = "重复请求";
+        } else if (GOLDEN_DB_SWITCH_FAIL_CODE.equals(switchCode)) {
+            reason = "切换失败";
+        } else if (GOLDEN_DB_INTERNAL_ERROR_CODE.equals(switchCode)) {
+            reason = "内部错误";
+        } else if (GOLDEN_DB_SUCCESS_CODE.equals(switchCode)) {
+            reason = "接口返回成功码但切换状态异常";
+        } else {
+            reason = "接口返回异常";
+        }
+
+        String detail = buildGoldenDbErrorMessage(switchCode, errorMsg);
+        return "联合运维goldenDb切换失败:" + reason + ",switchState=" + switchState + ",detail=" + detail;
+    }
+
+    private boolean isBlank(String value) {
+        return value == null || value.trim().isEmpty();
+    }
+
+    private String getMasterIdcFlag() throws BusinessException {
+        String masterIdc = null;
+        Request request = Request.builder()
+                .traceId(LoggerUtils.getTraceNo())
+                .bizType(BaseDataBizTypeConstants.ANNOUNCE + AnnounceBizTypeConstants.IDC_INFO)
+                .operateType(AnnounceOperateTypeConstants.IdcInfoConstants.GET_IDC_FLAG)
+                .systemId(SystemIdConstants.ARK_ID)
+                .requestTime(new Date())
+                .paramJson("getIdcFlag")
+                .build();
+        com.epcc.dubbo.result.Result<String> result = null;
+        try {
+            result = arkControlBaseFacade.processBiz(request);
+            masterIdc = JSON.parseObject(result.getResult(), new TypeReference<String>() {
+            });
+            log.info("current master idc={}", masterIdc);
+        } catch (Exception e1) {
+            log.error("kickidc getIDCFlag error,e1={}", e1);
+            try {
+                result = arkControlBaseFacade.processBiz(request);
+                masterIdc = JSON.parseObject(result.getResult(), new TypeReference<String>() {
+                });
+            } catch (Exception e2) {
+                log.error("kickidc updateIDCStatus error,e2={}", e2);
+                throw new BusinessException("远程获取数据库IDC有误");
+            }
+        }
+
+        return masterIdc;
+    }
+
+    private void createMDCParam(String idc, String requestTime) {
+        MDC.put("idc", idc);
+        MDC.put("requestTime", requestTime);
+        MDC.put("applicationName", applicationName);
+        MDC.put("updatedBy", UPDATEDBY);
+    }
+
+    protected void exchangeSchedule(String idc, String excIdc) throws BusinessException {
+        // 停止定时任务
+        com.epcc.dubbo.result.Result<JSONArray> resultArray = null;
+        JSONArray jsonArray = null;
+        JobProperties jobProperties = new JobProperties();
+
+        try {
+            resultArray = taskServiceClient.findJobs(idc);
+            jsonArray = resultArray.getResult();
+            for (int i = 0; i < jsonArray.size(); i++) {
+                JSONObject jsonObject = (JSONObject) jsonArray.get(i);
+                jobProperties.setName(jsonObject.getString("name"));
+                taskServiceClient.jobOperate(idc, JobOperateStatus.PAUSE, jobProperties);
+            }
+        } catch (Exception e1) {
+            log.info("exchangeSchedule PAUSE1 errorCode={}", e1);
+            try {
+                resultArray = taskServiceClient.findJobs(idc);
+                jsonArray = resultArray.getResult();
+                for (int i = 0; i < jsonArray.size(); i++) {
+                    JSONObject jsonObject = (JSONObject) jsonArray.get(i);
+                    jobProperties.setName(jsonObject.getString("name"));
+                    taskServiceClient.jobOperate(idc, JobOperateStatus.PAUSE, jobProperties);
+                }
+            } catch (Exception e2) {
+                log.info("exchangeSchedule PAUSE2 errorCode={}", e2);
+                throw new BusinessException("lhyw 定时任务禁用失败");
+            }
+        }
+
+        // 启动定时任务
+        try {
+            resultArray = taskServiceClient.findJobs(excIdc);
+            jsonArray = resultArray.getResult();
+            for (int i = 0; i < jsonArray.size(); i++) {
+                JSONObject jsonObject = (JSONObject) jsonArray.get(i);
+                jobProperties.setName(jsonObject.getString("name"));
+                taskServiceClient.jobOperate(excIdc, JobOperateStatus.RESUME, jobProperties);
+            }
+        } catch (Exception e3) {
+            log.info("exchangeSchedule RESUME e3={}", e3);
+            try {
+                resultArray = taskServiceClient.findJobs(excIdc);
+                jsonArray = resultArray.getResult();
+                for (int i = 0; i < jsonArray.size(); i++) {
+                    JSONObject jsonObject = (JSONObject) jsonArray.get(i);
+                    jobProperties.setName(jsonObject.getString("name"));
+                    taskServiceClient.jobOperate(excIdc, JobOperateStatus.RESUME, jobProperties);
+                }
+            } catch (Exception e4) {
+                log.info("exchangeSchedule RESUME2 e4={}", e4);
+                throw new BusinessException("lhyw 定时任务启用失败");
+            }
+        }
+    }
+
+    protected String exchangeDB(String kickIdc, String targetIdc) throws BusinessException {
+        JSONObject jsonObject = new JSONObject();
+        jsonObject.put("db_name", dbName);
+        jsonObject.put("key", dbKey);
+        jsonObject.put("ts", Long.parseLong(DateTimeUtil.format(new Date())));
+        jsonObject.put("to_idc", targetIdc);
+        jsonObject.put("allow_data_loss", 0);
+        log.info("exchangeDB parma jsonObject={},failover_code={}", jsonObject);
+        String response = null;
+
+        String[] address = dbUrl.split(";");
+        try {
+            response = HttpUtils.httpPostReq(address[0], jsonObject, 180000);
+        } catch (Exception e1) {
+            log.error("db exchange address[0] error,e={}", e1);
+            try {
+                response = HttpUtils.httpPostReq(address[1], jsonObject, 180000);
+            } catch (Exception e2) {
+                log.error("db exchange address[1] error,e={}", e2);
+                try {
+                    response = HttpUtils.httpPostReq(address[2], jsonObject, 180000);
+                } catch (Exception e3) {
+                    log.error("db exchange address[2] error,e={}", e3);
+                    throw new BusinessException("联合运维DB接口调用失败");
+                }
+            }
+        }
+
+        log.info("exchangeDB response={}", response);
+        JSONObject resJson = JSON.parseObject(response);
+        String failCode = resJson.getString("failover_code");
+
+        if ("20000".equals(failCode)) {
+            // 修改数据库标识位
+            updateIDCStatus(kickIdc, targetIdc);
+            return failCode;
+        }
+
+        if ("11112".equals(failCode)) {
+            return failCode;
+        }
+
+        log.info("exchangeDB error idc={},failover_code={}", kickIdc, failCode);
+        throw new BusinessException("联合运维DB切换失败");
+    }
+
+    private void updateIDCStatus(String fromIdc, String targetIdc) throws BusinessException {
+        this.updateIDCStatus(fromIdc, targetIdc, UPDATEDBY);
+    }
+
+    public void updateIDCStatus(String fromIdc, String targetIdc, String updatedBy) throws BusinessException {
+        Map<String, Object> map = new HashMap<>();
+        map.put("fromIdc", fromIdc);
+        map.put("toIdc", targetIdc);
+        map.put("updatedBy", updatedBy);
+        map.put("status", 1);
+
+        Request request = Request.builder()
+                .traceId(LoggerUtils.getTraceNo())
+                .bizType(BaseDataBizTypeConstants.ANNOUNCE + AnnounceBizTypeConstants.IDC_INFO)
+                .operateType(AnnounceOperateTypeConstants.IdcInfoConstants.UPDATE_IDC_STATUS)
+                .systemId(SystemIdConstants.ARK_ID)
+                .requestTime(new Date())
+                .paramJson(JSON.toJSONString(map))
+                .build();
+
+        com.epcc.dubbo.result.Result<String> result = null;
+        try {
+            result = arkControlBaseFacade.processBiz(request);
+        } catch (Exception e1) {
+            log.error("kickidc updateIDCStatus error,e1={}", e1);
+            try {
+                result = arkControlBaseFacade.processBiz(request);
+            } catch (Exception e2) {
+                log.error("kickidc updateIDCStatus error,e2={}", e2);
+                throw new BusinessException("远程更新数据库更新IDC有误");
+            }
+        }
+
+        if (!result.isSuccess()) {
+            log.error("更新IDC信息表失败，errCode： [{}],errMsg: [{}]", result.getErrorCode(), result.getErrorMsg());
+            throw new BusinessException("数据库更新IDC出错");
+        }
+    }
+}

package/README.md

@@ -65,6 +65,8 @@ python ingest.py promote
 
 默认目录为 `input/`、`parsed/`、`drafts/`、`approved/` 和 `result/`。只有需要处理其他目录时，才使用 `--input`、`--output` 或 `--result-dir` 覆盖。
 
+`draft` 默认按 `config/config.yaml` 的 `draft.max_chars` 控制单次送入模型的原文长度，并额外提供文档目录和相邻片段摘要作为辅助上下文。这样可以降低私有模型单轮负载，同时尽量保留前后章节关系。命令行仍可用 `--max-chars` 临时覆盖。
+
 ## 大模型配置
 
 默认不强制调用大模型，会使用启发式模板生成 `draft` 文件。
@@ -74,24 +76,29 @@ python ingest.py promote
 ```yaml
 llm:
   enabled: true
-  base_url: "https://your-model-endpoint"
-  api_key: "your-api-key"
-  model: "your-model"
+  base_url: "https://open.bigmodel.cn/api/paas/v4/"
+  api_key: "your-zhipu-api-key"
+  model: "glm-4.7"
   timeout_seconds: 120
-  max_tokens: 4096
+  max_tokens: 8192
   temperature: 0.1
+
+draft:
+  max_chars: 3600
+  context_chars: 800
+  outline_max_sections: 40
 ```
 
 也可以继续使用环境变量覆盖配置文件：
 
 ```bash
 export KB_LLM_ENABLED=true
-export KB_LLM_BASE_URL="https://your-model-endpoint"
-export KB_LLM_API_KEY="your-api-key"
-export KB_LLM_MODEL="your-model"
+export KB_LLM_BASE_URL="https://open.bigmodel.cn/api/paas/v4/"
+export KB_LLM_API_KEY="your-zhipu-api-key"
+export KB_LLM_MODEL="glm-4.7"
 ```
 
-`base_url` 必须填写完整的大模型调用地址，工具不会自动拼接任何路径。工具不 import 项目 `src` 代码。
+工具通过 ZhipuAI SDK 调用 GLM，不再包含 OpenAI 调用路径。`base_url` 使用 ZhipuAI SDK 的服务根地址即可。工具不 import 项目 `src` 代码。
 
 ## 与线上项目的关系
 

package/app_config.py

@@ -24,6 +24,13 @@ class LlmConfig:
     temperature: float = 0.1
 
 
+@dataclass(frozen=True)
+class DraftConfig:
+    max_chars: int = 3600
+    context_chars: int = 800
+    outline_max_sections: int = 40
+
+
 @lru_cache(maxsize=1)
 def get_llm_config() -> LlmConfig:
     raw = _read_config().get("llm", {})
@@ -41,6 +48,19 @@ def get_llm_config() -> LlmConfig:
     )
 
 
+@lru_cache(maxsize=1)
+def get_draft_config() -> DraftConfig:
+    raw = _read_config().get("draft", {})
+    if not isinstance(raw, dict):
+        raw = {}
+
+    return DraftConfig(
+        max_chars=_env_int("KB_DRAFT_MAX_CHARS", raw.get("max_chars"), 3600),
+        context_chars=_env_int("KB_DRAFT_CONTEXT_CHARS", raw.get("context_chars"), 800),
+        outline_max_sections=_env_int("KB_DRAFT_OUTLINE_MAX_SECTIONS", raw.get("outline_max_sections"), 40),
+    )
+
+
 def _read_config() -> Dict[str, Any]:
     path = Path(os.environ.get("KB_INGEST_CONFIG", DEFAULT_CONFIG_PATH))
     if not path.exists():

package/config/config.yaml

@@ -1,8 +1,13 @@
 llm:
   enabled: true
   timeout_seconds: 120
-  max_tokens: 4096
+  max_tokens: 8192
   temperature: 0.1
-  api_key: "15f066c4509845038027ea5746524af5.w4CLSC6ODiKVC1wK"          # GLM API Key
-  model: "glm-4.7-flash"                 # 模型名称，可选: glm-4, glm-4-plus, glm-4-flash 等
-  base_url: "https://open.bigmodel.cn/api/paas/v4/"  # 模型服务地址
+  api_key: "15f066c4509845038027ea5746524af5.w4CLSC6ODiKVC1wK"
+  model: "GLM-4.7-Flash"
+  base_url: "https://open.bigmodel.cn/api/paas/v4/"
+
+draft:
+  max_chars: 3600
+  context_chars: 800
+  outline_max_sections: 40

package/ingest.py

@@ -5,13 +5,16 @@ import argparse
 import shutil
 import sys
 from pathlib import Path
+from typing import List
 
 CURRENT_DIR = Path(__file__).resolve().parent
 if str(CURRENT_DIR) not in sys.path:
     sys.path.insert(0, str(CURRENT_DIR))
 
+from app_config import get_draft_config
 from normalizer import normalize_block
 from parser import iter_input_files, parse_document
+from schemas import ParsedBlock
 from splitter import split_blocks
 from validator import validate_dir
 from writer import write_item
@@ -52,10 +55,17 @@ def cmd_draft(args) -> int:
     output_dir.mkdir(parents=True, exist_ok=True)
 
     total_items = 0
+    draft_config = get_draft_config()
+    max_chars = args.max_chars or draft_config.max_chars
     files = iter_input_files(input_path)
     for path in files:
         parsed = parse_document(path)
-        blocks = split_blocks(parsed.blocks, max_chars=args.max_chars)
+        blocks = split_blocks(parsed.blocks, max_chars=max_chars)
+        blocks = _attach_block_context(
+            blocks,
+            context_chars=draft_config.context_chars,
+            outline_max_sections=draft_config.outline_max_sections,
+        )
         for block in blocks:
             for item in normalize_block(block, status="draft"):
                 write_item(item, output_dir)
@@ -96,6 +106,67 @@ def _clear_generated_files(*dirs: Path) -> None:
             print(f"deleted: {path}")
 
 
+def _attach_block_context(
+    blocks: List[ParsedBlock],
+    context_chars: int,
+    outline_max_sections: int,
+) -> List[ParsedBlock]:
+    if context_chars <= 0:
+        return blocks
+
+    outline = _document_outline(blocks, outline_max_sections)
+    output: List[ParsedBlock] = []
+    for idx, block in enumerate(blocks):
+        parts = []
+        if outline:
+            parts.append(f"文档章节目录：\n{outline}")
+        if idx > 0:
+            parts.append(
+                "上一片段摘要：\n"
+                f"章节：{blocks[idx - 1].source_section}\n"
+                f"{_compact_context_text(blocks[idx - 1].content, context_chars // 2)}"
+            )
+        if idx + 1 < len(blocks):
+            parts.append(
+                "下一片段摘要：\n"
+                f"章节：{blocks[idx + 1].source_section}\n"
+                f"{_compact_context_text(blocks[idx + 1].content, context_chars // 2)}"
+            )
+        output.append(ParsedBlock(
+            source_doc=block.source_doc,
+            source_section=block.source_section,
+            content=block.content,
+            pages=block.pages,
+            order=block.order,
+            context="\n\n".join(parts),
+        ))
+    return output
+
+
+def _document_outline(blocks: List[ParsedBlock], max_sections: int) -> str:
+    sections = []
+    seen = set()
+    for block in blocks:
+        section = block.source_section.strip()
+        if not section or section in seen:
+            continue
+        seen.add(section)
+        sections.append(f"- {section}")
+        if len(sections) >= max_sections:
+            remaining = len(blocks) - len(sections)
+            if remaining > 0:
+                sections.append(f"- ... 其余 {remaining} 个片段")
+            break
+    return "\n".join(sections)
+
+
+def _compact_context_text(text: str, limit: int) -> str:
+    compact = " ".join(text.split())
+    if limit <= 0 or len(compact) <= limit:
+        return compact
+    return compact[:limit].rstrip() + "..."
+
+
 def cmd_validate(args) -> int:
     issues = validate_dir(Path(args.input))
     for issue in issues:
@@ -132,7 +203,7 @@ def build_parser() -> argparse.ArgumentParser:
     draft_cmd.add_argument("--output", default=str(CURRENT_DIR / "drafts"))
     draft_cmd.add_argument("--approved-dir", default=str(CURRENT_DIR / "approved"))
     draft_cmd.add_argument("--result-dir", default=str(CURRENT_DIR / "result"))
-    draft_cmd.add_argument("--max-chars", type=int, default=8000)
+    draft_cmd.add_argument("--max-chars", type=int, default=None)
     draft_cmd.set_defaults(func=cmd_draft)
 
     validate_cmd = sub.add_parser("validate", help="Validate generated Markdown files.")

package/normalizer.py

@@ -33,7 +33,7 @@ def _normalize_with_llm(block: ParsedBlock, status: str) -> List[KnowledgeItem]:
     if not (config.base_url and config.api_key and config.model):
         _abort_llm("missing base_url, api_key, or model", block)
 
-    prompt = _build_prompt(block, status)
+    base_prompt = _build_prompt(block, status)
     started_at = time.monotonic()
     print(
         "llm start: "
@@ -45,34 +45,24 @@ def _normalize_with_llm(block: ParsedBlock, status: str) -> List[KnowledgeItem]:
     except ImportError as exc:
         print(f"llm error: cannot load ZhipuAI SDK ({exc})")
         return []
-
     client = client_cls(api_key=config.api_key, base_url=config.base_url)
+
+    compact_retry = False
     for attempt in range(1, LLM_MAX_RETRIES + 1):
+        prompt = _compact_retry_prompt(base_prompt) if compact_retry else base_prompt
         try:
-            print(f"llm request: base_url={config.base_url} attempt={attempt}/{LLM_MAX_RETRIES}")
-            response = client.chat.completions.create(
-                model=config.model,
-                messages=[
-                    {
-                        "role": "system",
-                        "content": (
-                            "你是严谨的运维知识库整理助手，只能依据输入原文生成结构化知识条目。"
-                            "你必须只返回一个 JSON object，根节点必须只有 items 字段，"
-                            "且 items 必须是数组。不要返回 Markdown、解释文字或其他根字段。"
-                        ),
-                    },
-                    {"role": "user", "content": prompt},
-                ],
-                stream=False,
-                max_tokens=config.max_tokens,
-                temperature=config.temperature,
-                do_sample=False,
-                response_format={"type": "json_object"},
-                thinking={"type": "disabled", "clear_thinking": True},
+            print(
+                "llm request: "
+                f"provider=zhipu base_url={config.base_url} attempt={attempt}/{LLM_MAX_RETRIES}"
             )
+            response = _create_zhipu_completion(client, config, prompt)
         except Exception as exc:
             elapsed = time.monotonic() - started_at
-            print(f"llm error: {type(exc).__name__} attempt={attempt}/{LLM_MAX_RETRIES} after {elapsed:.1f}s")
+            print(
+                "llm error: "
+                f"{type(exc).__name__} attempt={attempt}/{LLM_MAX_RETRIES} "
+                f"after {elapsed:.1f}s detail={exc}"
+            )
             if attempt >= LLM_MAX_RETRIES:
                 _abort_llm(f"request failed after {LLM_MAX_RETRIES} attempts: {type(exc).__name__}", block)
             time.sleep(min(2 ** (attempt - 1), 30))
@@ -80,12 +70,19 @@ def _normalize_with_llm(block: ParsedBlock, status: str) -> List[KnowledgeItem]:
 
         content = _extract_response_content(response)
         elapsed = time.monotonic() - started_at
-        print(f"llm response: {len(content)} chars in {elapsed:.1f}s attempt={attempt}/{LLM_MAX_RETRIES}")
+        finish_reason = _finish_reason(response)
+        print(
+            "llm response: "
+            f"{len(content)} chars in {elapsed:.1f}s attempt={attempt}/{LLM_MAX_RETRIES} "
+            f"finish_reason={finish_reason or 'unknown'}"
+        )
+        if content:
+            print(f"llm response content:\n{_preview(content)}")
         if not content.strip():
             reasoning = _extract_reasoning_content(response)
             print(
                 "llm parse failed: empty response content "
-                f"finish_reason={_finish_reason(response)} reasoning_chars={len(reasoning)} "
+                f"finish_reason={finish_reason} reasoning_chars={len(reasoning)} "
                 f"response={_response_debug(response)}"
             )
             if attempt >= LLM_MAX_RETRIES:
@@ -95,16 +92,25 @@ def _normalize_with_llm(block: ParsedBlock, status: str) -> List[KnowledgeItem]:
 
         parsed = _extract_json(content)
         if not parsed:
-            print(f"llm parse failed: response is not valid JSON preview={_preview(content)}")
+            compact_retry = compact_retry or _looks_truncated(content, finish_reason)
+            print(
+                "llm parse failed: response is not valid JSON "
+                f"finish_reason={finish_reason or 'unknown'} "
+                f"truncated={_looks_truncated(content, finish_reason)} "
+                f"preview={_preview(content)}"
+            )
             if attempt >= LLM_MAX_RETRIES:
                 _abort_llm("response is not valid JSON after 10 attempts", block)
             time.sleep(min(2 ** (attempt - 1), 30))
             continue
 
-        raw_items = parsed.get("items") if isinstance(parsed, dict) else parsed
+        raw_items = _coerce_raw_items(parsed)
         if not isinstance(raw_items, list):
+            compact_retry = compact_retry or _looks_truncated(content, finish_reason)
             print(
                 "llm parse failed: JSON does not contain an items list "
+                f"finish_reason={finish_reason or 'unknown'} "
+                f"truncated={_looks_truncated(content, finish_reason)} "
                 f"top_level={_json_shape(parsed)} preview={_preview(content)}"
             )
             if attempt >= LLM_MAX_RETRIES:
@@ -127,6 +133,43 @@ def _normalize_with_llm(block: ParsedBlock, status: str) -> List[KnowledgeItem]:
     _abort_llm("model call failed", block)
 
 
+def _system_message() -> str:
+    return (
+        "你是严谨的运维知识库整理助手，只能依据输入原文生成结构化知识条目。"
+        "你必须只返回一个 JSON object，根节点必须只有 items 字段，"
+        "且 items 必须是数组。不要返回 Markdown、解释文字或其他根字段。"
+    )
+
+
+def _messages(prompt: str) -> List[Dict[str, str]]:
+    return [
+        {"role": "system", "content": _system_message()},
+        {"role": "user", "content": prompt},
+    ]
+
+
+def _compact_retry_prompt(base_prompt: str) -> str:
+    return (
+        base_prompt
+        + "\n\n重试补充要求：上一次输出疑似过长或结构不完整。"
+        "本次必须只生成 1 个 item，保留规范要求的正文 1-7 节，但每节只写当前原文中最必要、最确定的信息。"
+        "不要省略 JSON 外层 items，不要输出多个条目，不要输出解释文字。"
+    )
+
+
+def _create_zhipu_completion(client, config, prompt: str):
+    return client.chat.completions.create(
+        model=config.model,
+        messages=_messages(prompt),
+        stream=False,
+        max_tokens=config.max_tokens,
+        temperature=config.temperature,
+        do_sample=False,
+        response_format={"type": "json_object"},
+        thinking={"type": "disabled", "clear_thinking": True},
+    )
+
+
 def _get_zhipu_client_class():
     try:
         from zai import ZaiClient
@@ -207,6 +250,59 @@ def _json_shape(value) -> str:
     return type(value).__name__
 
 
+def _coerce_raw_items(parsed):
+    if isinstance(parsed, dict):
+        items = parsed.get("items")
+        if isinstance(items, list):
+            return items
+
+        for key in ("knowledge_items", "records", "data", "result", "results"):
+            value = parsed.get(key)
+            if isinstance(value, list):
+                print(f"llm parse notice: using non-standard list field '{key}' as items")
+                return value
+            if isinstance(value, dict):
+                nested = _coerce_raw_items(value)
+                if isinstance(nested, list):
+                    print(f"llm parse notice: using nested field '{key}' as items")
+                    return nested
+
+        if _looks_like_single_item(parsed):
+            print("llm parse notice: wrapping single item object as items[0]")
+            return [parsed]
+
+    if isinstance(parsed, list):
+        print("llm parse notice: wrapping root array as items")
+        return parsed
+
+    return None
+
+
+def _looks_like_single_item(value: Dict) -> bool:
+    required_signal = {"title", "body"}
+    item_fields = {
+        "title",
+        "doc_type",
+        "business_modules",
+        "source_version",
+        "risk_level",
+        "applicable_roles",
+        "tags",
+        "body",
+        "split_reason",
+    }
+    return required_signal.issubset(value.keys()) and len(item_fields.intersection(value.keys())) >= 4
+
+
+def _looks_truncated(content: str, finish_reason: str) -> bool:
+    if finish_reason == "length":
+        return True
+    stripped = content.strip()
+    if not stripped:
+        return False
+    return stripped.count("{") > stripped.count("}") or stripped.count("[") > stripped.count("]")
+
+
 def _abort_llm(message: str, block: ParsedBlock) -> None:
     print(
         "ALERT: llm draft failed; aborting. "
@@ -230,6 +326,8 @@ def _build_prompt(block: ParsedBlock, status: str) -> str:
 7. 输出严格 JSON 对象，不要 Markdown 代码围栏，不要解释文字，不要在 JSON 前后添加任何内容。
 8. status 固定为 "{status}"。
 9. JSON 根节点必须严格为一个对象：{{"items": [...]}}。禁止返回单个 item 对象、禁止返回纯数组、禁止返回 result/data/records/knowledge_items 等其他根字段。
+10. 为避免输出被截断，优先生成 1 个覆盖本片段核心内容的综合 item；只有原文明确包含多个相互独立主题时，才拆分为多个 items。
+11. “辅助上下文”只用于理解当前片段在全文中的位置、术语和前后关系；不要把辅助上下文中独有而当前原文没有的事实写成正文依据。
 
 doc_type 只能取：
 {", ".join(sorted(DOC_TYPES))}
@@ -258,8 +356,11 @@ doc_type 只能取：
 来源章节：{block.source_section}
 来源页码：{",".join(map(str, block.pages)) if block.pages else ""}
 
+辅助上下文：
+{block.context or "无"}
+
 原文：
-{block.content[:12000]}
+{block.content}
 """.strip()
 
 
@@ -307,7 +408,7 @@ def _json_candidates(text: str) -> List[str]:
 
 
 def _preview(text: str) -> str:
-    return re.sub(r"\s+", " ", text).strip()
+    return text.strip()
 
 
 def _normalize_heuristically(block: ParsedBlock, status: str) -> KnowledgeItem:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "union_kb_ingest",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "Offline knowledge-base ingest helper for PDF, Word, Markdown and TXT documents.",
   "bin": {
     "union_kb_ingest": "bin/union_kb_ingest"
@@ -9,6 +9,7 @@
     "README.md",
     "requirements.txt",
     "bin/union_kb_ingest",
+    "ArkKickidcService.java",
     "*.py",
     "config/config.yaml",
     "prompts/",

package/requirements.txt

@@ -3,9 +3,6 @@ pyyaml>=6.0.1
 zhipuai>=2.1.0
 sniffio>=1.3.0
 
-# Pin numpy to avoid incompatible x86-64-v2 wheels on older offline machines.
-numpy==1.26.4
-
 # Docling slim plus only file-format backends used by this offline tool.
 # Do not install `docling` or `docling-slim[standard]` here: those pull OCR,
 # layout/table ML models, torch/onnxruntime, and may try to download artifacts.

package/schemas.py

@@ -24,6 +24,7 @@ class ParsedBlock:
     content: str
     pages: List[int] = field(default_factory=list)
     order: int = 0
+    context: str = ""
 
 
 @dataclass
@@ -82,4 +83,3 @@ class ValidationIssue:
     path: Path
     level: str
     message: str
-

package/splitter.py

@@ -34,6 +34,7 @@ def _merge_parent_child_blocks(blocks: List[ParsedBlock]) -> List[ParsedBlock]:
                 content=f"{current.content.rstrip()}\n\n{block.content.strip()}",
                 pages=sorted(set(current.pages + block.pages)),
                 order=current.order,
+                context=current.context,
             )
             continue
 
@@ -79,6 +80,7 @@ def _split_one(block: ParsedBlock, max_chars: int) -> List[ParsedBlock]:
             content=text,
             pages=block.pages,
             order=block.order * 100 + idx,
+            context=block.context,
         ))
     return output