deepdoc-lib 0.2.0__py3-none-any.whl

deepdoc/parser/resume/step_two.py

@@ -0,0 +1,692 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import logging
+import re
+import copy
+import time
+import datetime
+import demjson3
+import traceback
+import signal
+import numpy as np
+from .entities import degrees, schools, corporations
+from ...depend import rag_tokenizer
+from ...depend.surname import surname
+from xpinyin import Pinyin
+from contextlib import contextmanager
+
+
+class TimeoutException(Exception):
+    pass
+
+
+@contextmanager
+def time_limit(seconds):
+    def signal_handler(signum, frame):
+        raise TimeoutException("Timed out!")
+
+    signal.signal(signal.SIGALRM, signal_handler)
+    signal.alarm(seconds)
+    try:
+        yield
+    finally:
+        signal.alarm(0)
+
+
+ENV = None
+PY = Pinyin()
+
+
+def rmHtmlTag(line):
+    return re.sub(r"<[a-z0-9.\"=';,:\+_/ -]+>", " ", line, count=100000, flags=re.IGNORECASE)
+
+
+def highest_degree(dg):
+    if not dg:
+        return ""
+    if isinstance(dg, str):
+        dg = [dg]
+    m = {"初中": 0, "高中": 1, "中专": 2, "大专": 3, "专升本": 4, "本科": 5, "硕士": 6, "博士": 7, "博士后": 8}
+    return sorted([(d, m.get(d, -1)) for d in dg], key=lambda x: x[1] * -1)[0][0]
+
+
+def forEdu(cv):
+    if not cv.get("education_obj"):
+        cv["integerity_flt"] *= 0.8
+        return cv
+
+    first_fea, fea, maj, fmaj, deg, fdeg, sch, fsch, st_dt, ed_dt = [], [], [], [], [], [], [], [], [], []
+    edu_nst = []
+    edu_end_dt = ""
+    cv["school_rank_int"] = 1000000
+    for ii, n in enumerate(sorted(cv["education_obj"], key=lambda x: x.get("start_time", "3"))):
+        e = {}
+        if n.get("end_time"):
+            if n["end_time"] > edu_end_dt:
+                edu_end_dt = n["end_time"]
+            try:
+                dt = n["end_time"]
+                if re.match(r"[0-9]{9,}", dt):
+                    dt = turnTm2Dt(dt)
+                y, m, d = getYMD(dt)
+                ed_dt.append(str(y))
+                e["end_dt_kwd"] = str(y)
+            except Exception as e:
+                pass
+        if n.get("start_time"):
+            try:
+                dt = n["start_time"]
+                if re.match(r"[0-9]{9,}", dt):
+                    dt = turnTm2Dt(dt)
+                y, m, d = getYMD(dt)
+                st_dt.append(str(y))
+                e["start_dt_kwd"] = str(y)
+            except Exception:
+                pass
+
+        r = schools.select(n.get("school_name", ""))
+        if r:
+            if str(r.get("type", "")) == "1":
+                fea.append("211")
+            if str(r.get("type", "")) == "2":
+                fea.append("211")
+            if str(r.get("is_abroad", "")) == "1":
+                fea.append("留学")
+            if str(r.get("is_double_first", "")) == "1":
+                fea.append("双一流")
+            if str(r.get("is_985", "")) == "1":
+                fea.append("985")
+            if str(r.get("is_world_known", "")) == "1":
+                fea.append("海外知名")
+            if r.get("rank") and cv["school_rank_int"] > r["rank"]:
+                cv["school_rank_int"] = r["rank"]
+
+        if n.get("school_name") and isinstance(n["school_name"], str):
+            sch.append(re.sub(r"(211|985|重点大学|[,&;；-])", "", n["school_name"]))
+            e["sch_nm_kwd"] = sch[-1]
+        fea.append(rag_tokenizer.fine_grained_tokenize(rag_tokenizer.tokenize(n.get("school_name", ""))).split()[-1])
+
+        if n.get("discipline_name") and isinstance(n["discipline_name"], str):
+            maj.append(n["discipline_name"])
+            e["major_kwd"] = n["discipline_name"]
+
+        if not n.get("degree") and "985" in fea and not first_fea:
+            n["degree"] = "1"
+
+        if n.get("degree"):
+            d = degrees.get_name(n["degree"])
+            if d:
+                e["degree_kwd"] = d
+            if d == "本科" and ("专科" in deg or "专升本" in deg or "中专" in deg or "大专" in deg or re.search(r"(成人|自考|自学考试)", n.get("school_name",""))):
+                d = "专升本"
+            if d:
+                deg.append(d)
+
+            # for first degree
+            if not fdeg and d in ["中专", "专升本", "专科", "本科", "大专"]:
+                fdeg = [d]
+                if n.get("school_name"):
+                    fsch = [n["school_name"]]
+                if n.get("discipline_name"):
+                    fmaj = [n["discipline_name"]]
+                first_fea = copy.deepcopy(fea)
+
+        edu_nst.append(e)
+
+    cv["sch_rank_kwd"] = []
+    if cv["school_rank_int"] <= 20 or ("海外名校" in fea and cv["school_rank_int"] <= 200):
+        cv["sch_rank_kwd"].append("顶尖学校")
+    elif 50 >= cv["school_rank_int"] > 20 or ("海外名校" in fea and 500 >= cv["school_rank_int"] > 200):
+        cv["sch_rank_kwd"].append("精英学校")
+    elif cv["school_rank_int"] > 50 and ("985" in fea or "211" in fea) or ("海外名校" in fea and cv["school_rank_int"] > 500):
+        cv["sch_rank_kwd"].append("优质学校")
+    else:
+        cv["sch_rank_kwd"].append("一般学校")
+
+    if edu_nst:
+        cv["edu_nst"] = edu_nst
+    if fea:
+        cv["edu_fea_kwd"] = list(set(fea))
+    if first_fea:
+        cv["edu_first_fea_kwd"] = list(set(first_fea))
+    if maj:
+        cv["major_kwd"] = maj
+    if fsch:
+        cv["first_school_name_kwd"] = fsch
+    if fdeg:
+        cv["first_degree_kwd"] = fdeg
+    if fmaj:
+        cv["first_major_kwd"] = fmaj
+    if st_dt:
+        cv["edu_start_kwd"] = st_dt
+    if ed_dt:
+        cv["edu_end_kwd"] = ed_dt
+    if ed_dt:
+        cv["edu_end_int"] = max([int(t) for t in ed_dt])
+    if deg:
+        if "本科" in deg and "专科" in deg:
+            deg.append("专升本")
+            deg = [d for d in deg if d != '本科']
+        cv["degree_kwd"] = deg
+        cv["highest_degree_kwd"] = highest_degree(deg)
+    if edu_end_dt:
+        try:
+            if re.match(r"[0-9]{9,}", edu_end_dt):
+                edu_end_dt = turnTm2Dt(edu_end_dt)
+            if edu_end_dt.strip("\n") == "至今":
+                edu_end_dt = cv.get("updated_at_dt", str(datetime.date.today()))
+            y, m, d = getYMD(edu_end_dt)
+            cv["work_exp_flt"] = min(int(str(datetime.date.today())[0:4]) - int(y), cv.get("work_exp_flt", 1000))
+        except Exception as e:
+            logging.exception("forEdu {} {} {}".format(e, edu_end_dt, cv.get("work_exp_flt")))
+    if sch:
+        cv["school_name_kwd"] = sch
+        if (len(cv.get("degree_kwd", [])) >= 1 and "本科" in cv["degree_kwd"]) \
+                or all([c.lower() in ["硕士", "博士", "mba", "博士后"] for c in cv.get("degree_kwd", [])]) \
+                or not cv.get("degree_kwd"):
+            for c in sch:
+                if schools.is_good(c):
+                    if "tag_kwd" not in cv:
+                        cv["tag_kwd"] = []
+                    cv["tag_kwd"].append("好学校")
+                    cv["tag_kwd"].append("好学历")
+                    break
+        if (len(cv.get("degree_kwd", [])) >= 1 and "本科" in cv["degree_kwd"] and
+            any([d.lower() in ["硕士", "博士", "mba", "博士"] for d in cv.get("degree_kwd", [])])) \
+                or all([d.lower() in ["硕士", "博士", "mba", "博士后"] for d in cv.get("degree_kwd", [])]) \
+                or any([d in ["mba", "emba", "博士后"] for d in cv.get("degree_kwd", [])]):
+            if "tag_kwd" not in cv:
+                cv["tag_kwd"] = []
+            if "好学历" not in cv["tag_kwd"]:
+                cv["tag_kwd"].append("好学历")
+
+    if cv.get("major_kwd"):
+        cv["major_tks"] = rag_tokenizer.tokenize(" ".join(maj))
+    if cv.get("school_name_kwd"):
+        cv["school_name_tks"] = rag_tokenizer.tokenize(" ".join(sch))
+    if cv.get("first_school_name_kwd"):
+        cv["first_school_name_tks"] = rag_tokenizer.tokenize(" ".join(fsch))
+    if cv.get("first_major_kwd"):
+        cv["first_major_tks"] = rag_tokenizer.tokenize(" ".join(fmaj))
+
+    return cv
+
+
+def forProj(cv):
+    if not cv.get("project_obj"):
+        return cv
+
+    pro_nms, desc = [], []
+    for i, n in enumerate(
+            sorted(cv.get("project_obj", []), key=lambda x: str(x.get("updated_at", "")) if isinstance(x, dict) else "",
+                   reverse=True)):
+        if n.get("name"):
+            pro_nms.append(n["name"])
+        if n.get("describe"):
+            desc.append(str(n["describe"]))
+        if n.get("responsibilities"):
+            desc.append(str(n["responsibilities"]))
+        if n.get("achivement"):
+            desc.append(str(n["achivement"]))
+
+    if pro_nms:
+        # cv["pro_nms_tks"] = rag_tokenizer.tokenize(" ".join(pro_nms))
+        cv["project_name_tks"] = rag_tokenizer.tokenize(pro_nms[0])
+    if desc:
+        cv["pro_desc_ltks"] = rag_tokenizer.tokenize(rmHtmlTag(" ".join(desc)))
+        cv["project_desc_ltks"] = rag_tokenizer.tokenize(rmHtmlTag(desc[0]))
+
+    return cv
+
+
+def json_loads(line):
+    return demjson3.decode(re.sub(r": *(True|False)", r": '\1'", line))
+
+
+def forWork(cv):
+    if not cv.get("work_obj"):
+        cv["integerity_flt"] *= 0.7
+        return cv
+
+    flds = ["position_name", "corporation_name", "corporation_id", "responsibilities",
+            "industry_name", "subordinates_count"]
+    duas = []
+    scales = []
+    fea = {c: [] for c in flds}
+    latest_job_tm = ""
+    goodcorp = False
+    goodcorp_ = False
+    work_st_tm = ""
+    corp_tags = []
+    for i, n in enumerate(
+            sorted(cv.get("work_obj", []), key=lambda x: str(x.get("start_time", "")) if isinstance(x, dict) else "",
+                   reverse=True)):
+        if isinstance(n, str):
+            try:
+                n = json_loads(n)
+            except Exception:
+                continue
+
+        if n.get("start_time") and (not work_st_tm or n["start_time"] < work_st_tm):
+            work_st_tm = n["start_time"]
+        for c in flds:
+            if not n.get(c) or str(n[c]) == '0':
+                fea[c].append("")
+                continue
+            if c == "corporation_name":
+                n[c] = corporations.corpNorm(n[c], False)
+                if corporations.is_good(n[c]):
+                    if i == 0:
+                        goodcorp = True
+                    else:
+                        goodcorp_ = True
+                ct = corporations.corp_tag(n[c])
+                if i == 0:
+                    corp_tags.extend(ct)
+                elif ct and ct[0] != "软外":
+                    corp_tags.extend([f"{t}(曾)" for t in ct])
+
+            fea[c].append(rmHtmlTag(str(n[c]).lower()))
+
+        y, m, d = getYMD(n.get("start_time"))
+        if not y or not m:
+            continue
+        st = "%s-%02d-%02d" % (y, int(m), int(d))
+        latest_job_tm = st
+
+        y, m, d = getYMD(n.get("end_time"))
+        if (not y or not m) and i > 0:
+            continue
+        if not y or not m or int(y) > 2022:
+            y, m, d = getYMD(str(n.get("updated_at", "")))
+        if not y or not m:
+            continue
+        ed = "%s-%02d-%02d" % (y, int(m), int(d))
+
+        try:
+            duas.append((datetime.datetime.strptime(ed, "%Y-%m-%d") - datetime.datetime.strptime(st, "%Y-%m-%d")).days)
+        except Exception:
+            logging.exception("forWork {} {}".format(n.get("start_time"), n.get("end_time")))
+
+        if n.get("scale"):
+            r = re.search(r"^([0-9]+)", str(n["scale"]))
+            if r:
+                scales.append(int(r.group(1)))
+
+    if goodcorp:
+        if "tag_kwd" not in cv:
+            cv["tag_kwd"] = []
+        cv["tag_kwd"].append("好公司")
+    if goodcorp_:
+        if "tag_kwd" not in cv:
+            cv["tag_kwd"] = []
+        cv["tag_kwd"].append("好公司(曾)")
+
+    if corp_tags:
+        if "tag_kwd" not in cv:
+            cv["tag_kwd"] = []
+        cv["tag_kwd"].extend(corp_tags)
+        cv["corp_tag_kwd"] = [c for c in corp_tags if re.match(r"(综合|行业)", c)]
+
+    if latest_job_tm:
+        cv["latest_job_dt"] = latest_job_tm
+    if fea["corporation_id"]:
+        cv["corporation_id"] = fea["corporation_id"]
+
+    if fea["position_name"]:
+        cv["position_name_tks"] = rag_tokenizer.tokenize(fea["position_name"][0])
+        cv["position_name_sm_tks"] = rag_tokenizer.fine_grained_tokenize(cv["position_name_tks"])
+        cv["pos_nm_tks"] = rag_tokenizer.tokenize(" ".join(fea["position_name"][1:]))
+
+    if fea["industry_name"]:
+        cv["industry_name_tks"] = rag_tokenizer.tokenize(fea["industry_name"][0])
+        cv["industry_name_sm_tks"] = rag_tokenizer.fine_grained_tokenize(cv["industry_name_tks"])
+        cv["indu_nm_tks"] = rag_tokenizer.tokenize(" ".join(fea["industry_name"][1:]))
+
+    if fea["corporation_name"]:
+        cv["corporation_name_kwd"] = fea["corporation_name"][0]
+        cv["corp_nm_kwd"] = fea["corporation_name"]
+        cv["corporation_name_tks"] = rag_tokenizer.tokenize(fea["corporation_name"][0])
+        cv["corporation_name_sm_tks"] = rag_tokenizer.fine_grained_tokenize(cv["corporation_name_tks"])
+        cv["corp_nm_tks"] = rag_tokenizer.tokenize(" ".join(fea["corporation_name"][1:]))
+
+    if fea["responsibilities"]:
+        cv["responsibilities_ltks"] = rag_tokenizer.tokenize(fea["responsibilities"][0])
+        cv["resp_ltks"] = rag_tokenizer.tokenize(" ".join(fea["responsibilities"][1:]))
+
+    if fea["subordinates_count"]:
+        fea["subordinates_count"] = [int(i) for i in fea["subordinates_count"] if
+                                                               re.match(r"[^0-9]+$", str(i))]
+    if fea["subordinates_count"]:
+        cv["max_sub_cnt_int"] = np.max(fea["subordinates_count"])
+
+    if isinstance(cv.get("corporation_id"), int):
+        cv["corporation_id"] = [str(cv["corporation_id"])]
+    if not cv.get("corporation_id"):
+        cv["corporation_id"] = []
+    for i in cv.get("corporation_id", []):
+        cv["baike_flt"] = max(corporations.baike(i), cv["baike_flt"] if "baike_flt" in cv else 0)
+
+    if work_st_tm:
+        try:
+            if re.match(r"[0-9]{9,}", work_st_tm):
+                work_st_tm = turnTm2Dt(work_st_tm)
+            y, m, d = getYMD(work_st_tm)
+            cv["work_exp_flt"] = min(int(str(datetime.date.today())[0:4]) - int(y), cv.get("work_exp_flt", 1000))
+        except Exception as e:
+            logging.exception("forWork {} {} {}".format(e, work_st_tm, cv.get("work_exp_flt")))
+
+    cv["job_num_int"] = 0
+    if duas:
+        cv["dua_flt"] = np.mean(duas)
+        cv["cur_dua_int"] = duas[0]
+        cv["job_num_int"] = len(duas)
+    if scales:
+        cv["scale_flt"] = np.max(scales)
+    return cv
+
+
+def turnTm2Dt(b):
+    if not b:
+        return None
+    b = str(b).strip()
+    if re.match(r"[0-9]{10,}", b):
+        b = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(b[:10])))
+    return b
+
+
+def getYMD(b):
+    y, m, d = "", "", "01"
+    if not b:
+        return y, m, d
+    b = turnTm2Dt(b)
+    if re.match(r"[0-9]{4}", b):
+        y = int(b[:4])
+    r = re.search(r"[0-9]{4}.?([0-9]{1,2})", b)
+    if r:
+        m = r.group(1)
+    r = re.search(r"[0-9]{4}.?[0-9]{,2}.?([0-9]{1,2})", b)
+    if r:
+        d = r.group(1)
+    if not d or int(d) == 0 or int(d) > 31:
+        d = "1"
+    if not m or int(m) > 12 or int(m) < 1:
+        m = "1"
+    return y, m, d
+
+
+def birth(cv):
+    if not cv.get("birth"):
+        cv["integerity_flt"] *= 0.9
+        return cv
+    y, m, d = getYMD(cv["birth"])
+    if not m or not y:
+        return cv
+    b = "%s-%02d-%02d" % (y, int(m), int(d))
+    cv["birth_dt"] = b
+    cv["birthday_kwd"] = "%02d%02d" % (int(m), int(d))
+
+    cv["age_int"] = datetime.datetime.now().year - int(y)
+    return cv
+
+
+def parse(cv):
+    for k in cv.keys():
+        if cv[k] == '\\N':
+            cv[k] = ''
+    # cv = cv.asDict()
+    tks_fld = ["address", "corporation_name", "discipline_name", "email", "expect_city_names",
+               "expect_industry_name", "expect_position_name", "industry_name", "industry_names", "name",
+               "position_name", "school_name", "self_remark", "title_name"]
+    small_tks_fld = ["corporation_name", "expect_position_name", "position_name", "school_name", "title_name"]
+    kwd_fld = ["address", "city", "corporation_type", "degree", "discipline_name", "expect_city_names", "email",
+               "expect_industry_name", "expect_position_name", "expect_type", "gender", "industry_name",
+               "industry_names", "political_status", "position_name", "scale", "school_name", "phone", "tel"]
+    num_fld = ["annual_salary", "annual_salary_from", "annual_salary_to", "expect_annual_salary", "expect_salary_from",
+               "expect_salary_to", "salary_month"]
+
+    is_fld = [
+        ("is_fertility", "已育", "未育"),
+        ("is_house", "有房", "没房"),
+        ("is_management_experience", "有管理经验", "无管理经验"),
+        ("is_marital", "已婚", "未婚"),
+        ("is_oversea", "有海外经验", "无海外经验")
+    ]
+
+    rmkeys = []
+    for k in cv.keys():
+        if cv[k] is None:
+            rmkeys.append(k)
+        if (isinstance(cv[k], list) or isinstance(cv[k], str)) and len(cv[k]) == 0:
+            rmkeys.append(k)
+    for k in rmkeys:
+        del cv[k]
+
+    integrity = 0.
+    flds_num = 0.
+
+    def hasValues(flds):
+        nonlocal integrity, flds_num
+        flds_num += len(flds)
+        for f in flds:
+            v = str(cv.get(f, ""))
+            if len(v) > 0 and v != '0' and v != '[]':
+                integrity += 1
+
+    hasValues(tks_fld)
+    hasValues(small_tks_fld)
+    hasValues(kwd_fld)
+    hasValues(num_fld)
+    cv["integerity_flt"] = integrity / flds_num
+
+    if cv.get("corporation_type"):
+        for p, r in [(r"(公司|企业|其它|其他|Others*|\n|未填写|Enterprises|Company|companies)", ""),
+                     (r"[／/．·　<\(（]+.*", ""),
+                     (r".*(合资|民企|股份制|中外|私营|个体|Private|创业|Owned|投资).*", "民营"),
+                     (r".*(机关|事业).*", "机关"),
+                     (r".*(非盈利|Non-profit).*", "非盈利"),
+                     (r".*(外企|外商|欧美|foreign|Institution|Australia|港资).*", "外企"),
+                     (r".*国有.*", "国企"),
+                     (r"[ （）\(\)人/·0-9-]+", ""),
+                     (r".*(元|规模|于|=|北京|上海|至今|中国|工资|州|shanghai|强|餐饮|融资|职).*", "")]:
+            cv["corporation_type"] = re.sub(p, r, cv["corporation_type"], count=1000, flags=re.IGNORECASE)
+        if len(cv["corporation_type"]) < 2:
+            del cv["corporation_type"]
+
+    if cv.get("political_status"):
+        for p, r in [
+            (r".*党员.*", "党员"),
+            (r".*(无党派|公民).*", "群众"),
+            (r".*团员.*", "团员")]:
+            cv["political_status"] = re.sub(p, r, cv["political_status"])
+        if not re.search(r"[党团群]", cv["political_status"]):
+            del cv["political_status"]
+
+    if cv.get("phone"):
+        cv["phone"] = re.sub(r"^0*86([0-9]{11})", r"\1", re.sub(r"[^0-9]+", "", cv["phone"]))
+
+    keys = list(cv.keys())
+    for k in keys:
+        # deal with json objects
+        if k.find("_obj") > 0:
+            try:
+                cv[k] = json_loads(cv[k])
+                cv[k] = [a for _, a in cv[k].items()]
+                nms = []
+                for n in cv[k]:
+                    if not isinstance(n, dict) or "name" not in n or not n.get("name"):
+                        continue
+                    n["name"] = re.sub(r"(（442）|\t )", "", n["name"]).strip().lower()
+                    if not n["name"]:
+                        continue
+                    nms.append(n["name"])
+                if nms:
+                    t = k[:-4]
+                    cv[f"{t}_kwd"] = nms
+                    cv[f"{t}_tks"] = rag_tokenizer.tokenize(" ".join(nms))
+            except Exception:
+                logging.exception("parse {} {}".format(str(traceback.format_exc()), cv[k]))
+                cv[k] = []
+
+        # tokenize fields
+        if k in tks_fld:
+            cv[f"{k}_tks"] = rag_tokenizer.tokenize(cv[k])
+            if k in small_tks_fld:
+                cv[f"{k}_sm_tks"] = rag_tokenizer.tokenize(cv[f"{k}_tks"])
+
+        # keyword fields
+        if k in kwd_fld:
+            cv[f"{k}_kwd"] = [n.lower()
+                                           for n in re.split(r"[\t,，；;. ]",
+                                                             re.sub(r"([^a-zA-Z])[ ]+([^a-zA-Z ])", r"\1，\2", cv[k])
+                                                             ) if n]
+
+        if k in num_fld and cv.get(k):
+            cv[f"{k}_int"] = cv[k]
+
+    cv["email_kwd"] = cv.get("email_tks", "").replace(" ", "")
+    # for name field
+    if cv.get("name"):
+        nm = re.sub(r"[\n——\-\(（\+].*", "", cv["name"].strip())
+        nm = re.sub(r"[ \t　]+", " ", nm)
+        if re.match(r"[a-zA-Z ]+$", nm):
+            if len(nm.split()) > 1:
+                cv["name"] = nm
+            else:
+                nm = ""
+        elif nm and (surname.isit(nm[0]) or surname.isit(nm[:2])):
+            nm = re.sub(r"[a-zA-Z]+.*", "", nm[:5])
+        else:
+            nm = ""
+        cv["name"] = nm.strip()
+        name = cv["name"]
+
+        # name pingyin and its prefix
+        cv["name_py_tks"] = " ".join(PY.get_pinyins(nm[:20], '')) + " " + " ".join(PY.get_pinyins(nm[:20], ' '))
+        cv["name_py_pref0_tks"] = ""
+        cv["name_py_pref_tks"] = ""
+        for py in PY.get_pinyins(nm[:20], ''):
+            for i in range(2, len(py) + 1):
+                cv["name_py_pref_tks"] += " " + py[:i]
+        for py in PY.get_pinyins(nm[:20], ' '):
+            py = py.split()
+            for i in range(1, len(py) + 1):
+                cv["name_py_pref0_tks"] += " " + "".join(py[:i])
+
+        cv["name_kwd"] = name
+        cv["name_pinyin_kwd"] = PY.get_pinyins(nm[:20], ' ')[:3]
+        cv["name_tks"] = (
+                rag_tokenizer.tokenize(name) + " " + (" ".join(list(name)) if not re.match(r"[a-zA-Z ]+$", name) else "")
+        ) if name else ""
+    else:
+        cv["integerity_flt"] /= 2.
+
+    if cv.get("phone"):
+        r = re.search(r"(1[3456789][0-9]{9})", cv["phone"])
+        if not r:
+            cv["phone"] = ""
+        else:
+            cv["phone"] = r.group(1)
+
+    # deal with date  fields
+    if cv.get("updated_at") and isinstance(cv["updated_at"], datetime.datetime):
+        cv["updated_at_dt"] = cv["updated_at"].strftime('%Y-%m-%d %H:%M:%S')
+    else:
+        y, m, d = getYMD(str(cv.get("updated_at", "")))
+        if not y:
+            y = "2012"
+        if not m:
+            m = "01"
+        if not d:
+            d = "01"
+        cv["updated_at_dt"] = "%s-%02d-%02d 00:00:00" % (y, int(m), int(d))
+        # long text tokenize
+
+    if cv.get("responsibilities"):
+        cv["responsibilities_ltks"] = rag_tokenizer.tokenize(rmHtmlTag(cv["responsibilities"]))
+
+    # for yes or no field
+    fea = []
+    for f, y, n in is_fld:
+        if f not in cv:
+            continue
+        if cv[f] == '是':
+            fea.append(y)
+        if cv[f] == '否':
+            fea.append(n)
+
+    if fea:
+        cv["tag_kwd"] = fea
+
+    cv = forEdu(cv)
+    cv = forProj(cv)
+    cv = forWork(cv)
+    cv = birth(cv)
+
+    cv["corp_proj_sch_deg_kwd"] = [c for c in cv.get("corp_tag_kwd", [])]
+    for i in range(len(cv["corp_proj_sch_deg_kwd"])):
+        for j in cv.get("sch_rank_kwd", []):
+            cv["corp_proj_sch_deg_kwd"][i] += "+" + j
+    for i in range(len(cv["corp_proj_sch_deg_kwd"])):
+        if cv.get("highest_degree_kwd"):
+            cv["corp_proj_sch_deg_kwd"][i] += "+" + cv["highest_degree_kwd"]
+
+    try:
+        if not cv.get("work_exp_flt") and cv.get("work_start_time"):
+            if re.match(r"[0-9]{9,}", str(cv["work_start_time"])):
+                cv["work_start_dt"] = turnTm2Dt(cv["work_start_time"])
+                cv["work_exp_flt"] = (time.time() - int(int(cv["work_start_time"]) / 1000)) / 3600. / 24. / 365.
+            elif re.match(r"[0-9]{4}[^0-9]", str(cv["work_start_time"])):
+                y, m, d = getYMD(str(cv["work_start_time"]))
+                cv["work_start_dt"] = "%s-%02d-%02d 00:00:00" % (y, int(m), int(d))
+                cv["work_exp_flt"] = int(str(datetime.date.today())[0:4]) - int(y)
+    except Exception as e:
+        logging.exception("parse {} ==> {}".format(e, cv.get("work_start_time")))
+    if "work_exp_flt" not in cv and cv.get("work_experience", 0):
+        cv["work_exp_flt"] = int(cv["work_experience"]) / 12.
+
+    keys = list(cv.keys())
+    for k in keys:
+        if not re.search(r"_(fea|tks|nst|dt|int|flt|ltks|kwd|id)$", k):
+            del cv[k]
+    for k in cv.keys():
+        if not re.search("_(kwd|id)$", k) or not isinstance(cv[k], list):
+            continue
+        cv[k] = list(set([re.sub("(市)$", "", str(n)) for n in cv[k] if n not in ['中国', '0']]))
+    keys = [k for k in cv.keys() if re.search(r"_feas*$", k)]
+    for k in keys:
+        if cv[k] <= 0:
+            del cv[k]
+
+    cv["tob_resume_id"] = str(cv["tob_resume_id"])
+    cv["id"] = cv["tob_resume_id"]
+    logging.debug("CCCCCCCCCCCCCCC")
+
+    return dealWithInt64(cv)
+
+
+def dealWithInt64(d):
+    if isinstance(d, dict):
+        for n, v in d.items():
+            d[n] = dealWithInt64(v)
+
+    if isinstance(d, list):
+        d = [dealWithInt64(t) for t in d]
+
+    if isinstance(d, np.integer):
+        d = int(d)
+    return d