from hanziconv import HanziConv
import pandas as pd
import jieba

df = pd.read_excel('../../data-dev/消化内科对话_已解决问题0510.xlsx', sheet_name='对话数据+')
text = df['医生回答'].tolist()
stopwords = [line.strip() for line in open('../Data/hit_stopwords.txt', 'r', encoding='utf-8').readlines()]
result = []
for line in text:
    # 精确模式，繁体转换
    seg_list = jieba.cut(HanziConv.toSimplified(line))
    # print(" ".join(seg_list))

    seg_list = [i for i in seg_list if i not in stopwords]
    result.append(' '.join(seg_list))

df_result = pd.DataFrame({'分词': result, '医生回答': df.医生回答})
df_result.to_excel('../../data-dev/医生回答分词0510.xlsx', sheet_name='回答分词', index=False)