BNSTechnologyDevelopment
/
Langchain-Chatchat
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

增加日志

This commit is contained in:
weiweiw 2025-01-14 12:41:19 +08:00
parent 32202f3334
commit ae7e658c1a
1 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
libs/chatchat-server/chatchat/server/knowledge_base/utils.py
View File

@ -22,6 +22,7 @@ from chatchat.server.file_rag.text_splitter import (
from chatchat.server.utils import run_in_process_pool, run_in_thread_pool from chatchat.server.utils import run_in_process_pool, run_in_thread_pool
from chatchat.utils import build_logger from chatchat.utils import build_logger
import re import re
import threading
logger = build_logger() logger = build_logger()
@ -228,7 +229,7 @@ def make_text_splitter(splitter_name, chunk_size, chunk_overlap):
""" """
根据参数获取特定的分词器 根据参数获取特定的分词器
""" """
logger.info(f"make_text_splitter start....splitter_name:{splitter_name}") logger.info(f"threadid:{threading.get_ident()}, make_text_splitter start....splitter_name:{splitter_name}")
splitter_name = splitter_name or "SpacyTextSplitter" splitter_name = splitter_name or "SpacyTextSplitter"
try: try:
if ( if (
@ -378,7 +379,7 @@ class KnowledgeFile:
else: else:
print("文件不存在") print("文件不存在")
logger.info(f"********docs2texts") logger.info(f"threadid:{threading.get_ident()},********docs2texts")
docs = docs or self.file2docs(refresh=refresh) docs = docs or self.file2docs(refresh=refresh)
#remove the redundant line break after loading, by weiweiwang 2025/1/13 #remove the redundant line break after loading, by weiweiwang 2025/1/13
for doc in docs: for doc in docs:
@ -388,9 +389,9 @@ class KnowledgeFile:
if not docs: if not docs:
return [] return []
if self.ext not in [".csv"]: if self.ext not in [".csv"]:
logger.info(f"self.ext not in csv") logger.info(f"threadid:{threading.get_ident()}, self.ext not in csv")
if text_splitter is None: if text_splitter is None:
logger.info(f"text_splitter is None") logger.info(f" threadid:{threading.get_ident()}, text_splitter is None")
text_splitter = make_text_splitter( text_splitter = make_text_splitter(
splitter_name=self.text_splitter_name, splitter_name=self.text_splitter_name,
chunk_size=chunk_size, chunk_size=chunk_size,
@ -451,6 +452,7 @@ def files2docs_in_thread_file2docs(
*, file: KnowledgeFile, **kwargs *, file: KnowledgeFile, **kwargs
) -> Tuple[bool, Tuple[str, str, List[Document]]]: ) -> Tuple[bool, Tuple[str, str, List[Document]]]:
try: try:
logger.info(f"file2docs 从文件 {file.kb_name}/{file.filename}， threadid:{threading.get_ident()}")
return True, (file.kb_name, file.filename, file.file2text(**kwargs)) return True, (file.kb_name, file.filename, file.file2text(**kwargs))
except Exception as e: except Exception as e:
msg = f"从文件 {file.kb_name}/{file.filename} 加载文档时出错：{e}" msg = f"从文件 {file.kb_name}/{file.filename} 加载文档时出错：{e}"