weiweiw
/
Langchain-Chatchat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

commit log

This commit is contained in:
wvivi2023 2023-12-15 09:48:42 +08:00
parent 2ac52147d3
commit 33dc60df5e
1 changed files with 0 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
text_splitter/chinese_recursive_text_splitter.py
View File

@ -100,7 +100,6 @@ class ChineseRecursiveTextSplitter(RecursiveCharacterTextSplitter):
final_chunks.extend(merged_text) final_chunks.extend(merged_text)
final_chunks = [re.sub(r"\n{2,}", "\n", chunk.strip()) for chunk in final_chunks if chunk.strip()!=""] final_chunks = [re.sub(r"\n{2,}", "\n", chunk.strip()) for chunk in final_chunks if chunk.strip()!=""]
#将单行和两行的和下面的分块合并
#将单行并且字数小于25，和下面的分块合并 #将单行并且字数小于25，和下面的分块合并
return_chunks = [] return_chunks = []
temp_sencond = "" temp_sencond = ""