weiweiw
/
Langchain-Chatchat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix merging issue

This commit is contained in:
wvivi2023 2023-12-15 09:48:22 +08:00
parent 77bc5891c8
commit 2ac52147d3
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
text_splitter/chinese_recursive_text_splitter.py
View File

@ -101,11 +101,12 @@ class ChineseRecursiveTextSplitter(RecursiveCharacterTextSplitter):
final_chunks = [re.sub(r"\n{2,}", "\n", chunk.strip()) for chunk in final_chunks if chunk.strip()!=""] final_chunks = [re.sub(r"\n{2,}", "\n", chunk.strip()) for chunk in final_chunks if chunk.strip()!=""]
#将单行和两行的和下面的分块合并 #将单行和两行的和下面的分块合并
#将单行并且字数小于25，和下面的分块合并
return_chunks = [] return_chunks = []
temp_sencond = "" temp_sencond = ""
for chunk in final_chunks: for chunk in final_chunks:
if temp_sencond =="": if temp_sencond =="":
if len(chunk.splitlines()) <= 1: if len(chunk.splitlines()) <= 1 and len(chunk) <= 25:
temp_sencond = chunk temp_sencond = chunk
else: else:
return_chunks.append(chunk) return_chunks.append(chunk)