From 77bc5891c8379034ae2f37858f7ab43d83bf9012 Mon Sep 17 00:00:00 2001
From: wvivi2023 <wang272160587@gmail.com>
Date: Fri, 15 Dec 2023 08:59:13 +0800
Subject: [PATCH] manually split

---
 .DS_Store                                     | Bin 6148 -> 6148 bytes
 server/chat/knowledge_base_chat.py            |   2 +-
 .../chinese_recursive_text_splitter.py        |   7 ++++---
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.DS_Store b/.DS_Store
index e9f6b052135d03daaae787798e9cead96ad6c76c..23badb7dba054fdf05b0fea42dccabb9962d5bd0 100644
GIT binary patch
delta 214
zcmZoMXfc=|#>B!ku~2NHo}wrV0|Nsi1A_nqLncEyLvd1haY0hfW<}=Z%)%fkc7|kz
ze1<%RG>G(MEv5z$LqiK41yl1{9ffK`Q$qtC1q%!F$@`d$rO_0Z1sCPz<maV>Y?%Cx
zNe-x)n{oSOGp0f{@#<<LGaUsJQ$vs`mPSBBOpMKHYdJYYmG!NI;<Iyd^YS|<pJS3|
n?ArW}NsVbUI|n}p(A7YfeP^D`FXG4nbTkvtWQNTVB5Rldev~pc

delta 122
zcmZoMXfc=|#>B)qu~2NHo}wrd0|Nsi1A_nqLp(z<Ljgk$LncGX#=_-{lMO^zCO=`7
zo-D`8Jz0~bRK&=@Ku5vE$fQ<Bq1w_2#5OjYyotql^AlEOri~4snKrX?@N)o7-Ym%R
Woq009h$9EkY>?qBn<GTlFarR-6&x-A

diff --git a/server/chat/knowledge_base_chat.py b/server/chat/knowledge_base_chat.py
index f7e9b3a..67445be 100644
--- a/server/chat/knowledge_base_chat.py
+++ b/server/chat/knowledge_base_chat.py
@@ -39,7 +39,7 @@ async def knowledge_base_chat(query: str = Body(..., description="用户输入",
         return BaseResponse(code=404, msg=f"未找到知识库 {knowledge_base_name}")
 
     history = [History.from_data(h) for h in history]
-    print(f"server/chat/knowledge_base_chat function, history:{history}")
+    print(f"******server/chat/knowledge_base_chat function, history:{history}")
     async def knowledge_base_chat_iterator(query: str,
                                            top_k: int,
                                            history: Optional[List[History]],
diff --git a/text_splitter/chinese_recursive_text_splitter.py b/text_splitter/chinese_recursive_text_splitter.py
index ac2448b..57ec53f 100644
--- a/text_splitter/chinese_recursive_text_splitter.py
+++ b/text_splitter/chinese_recursive_text_splitter.py
@@ -57,12 +57,13 @@ class ChineseRecursiveTextSplitter(RecursiveCharacterTextSplitter):
         new_separators = []
         text = re.sub(r'(\n+前\s+言\n+)',  r"\n\n\n\n\n\n\n\n\n\n\1", text) #通过前言分块
         text = re.sub(r'(\n+\d+[^\S\n]+[^\s\.]+)', r"\n\n\n\n\n\n\n\n\n\n\1", text) #通过1 这样的
-        text = re.sub(r'(\n+[a-zA-Z1-9]+\s*\.\s*[a-zA-Z1-9]+\s+(?!\.|[a-zA-Z1-9]))', r"\n\n\n\n\n\n\n\n\n\n\1", text)  # 通过1.2 这样的章和节来分块
+        text = re.sub(r'(\n+[a-zA-Z0-9]+\s*\.\s*[a-zA-Z0-9]+\s+(?!\.|[a-zA-Z0-9]))', r"\n\n\n\n\n\n\n\n\n\n\1", text)  # 通过\n1.2 这样的章和节来分块
         text = re.sub(r'(\n+表\s*[A-Za-z0-9]+(\.[A-Za-z0-9]+)+\s+)', r"\n\n\n\n\n\n\n\n\n\n\1", text)  # 通过表  A.4.a 
         text = re.sub(r'(\n+第\s*\S+\s*条\s+)', r"\n\n\n\n\n\n\n\n\n\n\1", text)  # 通过第 条
         text = re.sub(r'(\n+第\s*\S+\s*章\s+)', r"\n\n\n\n\n\n\n\n\n\n\1", text)  # 通过第 条
         text = re.sub(r'(\n+(一、|二、|三、|四、|五、|六、|七、|八、|九、|十、|十一、|十二、|十三、|十四、|十五、|十六、|十七、|十八、|十九、|二十、))', r"\n\n\n\n\n\n\n\n\n\n\1", text)  # 通过第 条
-        text = re.sub(r'(\n+[a-zA-Z1-9]+\s*\.\s*[a-zA-Z1-9]+\s+)', r"\n\n\n\n\n\n\n\n\n\n\1", text)  # 再通过 1.2 来分块
+        text = re.sub(r'(手工分段\*\*\s+)', r"\n\n\n\n\n\n\n\n\n\n", text)  # 通过“手工分段**”
+        #text = re.sub(r'(\n+[a-zA-Z0-9]+\s*\.\s*[a-zA-Z0-9]+\s+)', r"\n\n\n\n\n\n\n\n\n\n\1", text)  # 再通过 1.2 来分块
         text = text.rstrip()  # 段尾如果有多余的\n就去掉它
         for i, _s in enumerate(separators):
             _separator = _s if self._is_separator_regex else re.escape(_s)
@@ -91,7 +92,7 @@ class ChineseRecursiveTextSplitter(RecursiveCharacterTextSplitter):
                 if not new_separators:
                     final_chunks.append(s)
                 else:
-                    s = re.sub(r'(\n+[a-zA-Z1-9]+\s*\.\s*[a-zA-Z1-9]+\s*\.\s*[a-zA-Z1-9]+\s+)', r"\n\n\n\n\n\n\n\n\n\n\1", s)  # 再通过 1.2.3 来分块
+                    s = re.sub(r'(\n+[a-zA-Z0-9]+\s*\.\s*[a-zA-Z0-9]+\s*\.\s*[a-zA-Z0-9]+\s+)', r"\n\n\n\n\n\n\n\n\n\n\1", s)  # 再通过 1.2.3 来分块
                     other_info = self._split_text(s, new_separators)
                     final_chunks.extend(other_info)
         if _good_splits: