26 lines
848 B
Python
26 lines
848 B
Python
from langchain.text_splitter import CharacterTextSplitter
|
||
import re
|
||
from typing import List
|
||
|
||
|
||
class ChineseTextSplitter(CharacterTextSplitter):
|
||
def __init__(self, pdf: bool = False, **kwargs):
|
||
super().__init__(**kwargs)
|
||
self.pdf = pdf
|
||
|
||
def split_text(self, text: str) -> List[str]:
|
||
if self.pdf:
|
||
text = re.sub(r"\n{3,}", "\n", text)
|
||
text = re.sub('\s', ' ', text)
|
||
text = text.replace("\n\n", "")
|
||
sent_sep_pattern = re.compile('([﹒﹔﹖﹗．。！？]["’”」』]{0,2}|(?=["‘“「『]{1,2}|$))') # del ：；
|
||
sent_list = []
|
||
for ele in sent_sep_pattern.split(text):
|
||
if sent_sep_pattern.match(ele) and sent_list:
|
||
sent_list[-1] += ele
|
||
elif ele:
|
||
sent_list.append(ele)
|
||
return sent_list
|
||
|
||
|