支持软连接的知识库 (#2167)

* GLM3支持传入提示词

支持传入提示词的Agent ChatGLM3-6B

* 软连接处理

1. 有效的软连接处理，参考了 #2125
2. 更新了依赖，现在fschat原生支持GLM3的全功能

* 更新了支持的模型列表和依赖
1. 推荐用户使用GLM3-6B取代2代GLM模型
2. 推荐用户使用性能更强的bge模型
3 推荐使用vllm和torch的最新稳定版本

configs/model_config.py.example

@@ -1,13 +1,12 @@
 import os
 
-
 # 可以指定一个绝对路径，统一存放所有的Embedding和LLM模型。
 # 每个模型可以是一个单独的目录，也可以是某个目录下的二级子目录。
 # 如果模型目录名称和 MODEL_PATH 中的 key 或 value 相同，程序会自动检测加载，无需修改 MODEL_PATH 中的路径。
 MODEL_ROOT_PATH = ""
 
 # 选用的 Embedding 名称
-EMBEDDING_MODEL = "m3e-base" # bge-large-zh
+EMBEDDING_MODEL = "bge-large-zh"
 
 # Embedding 模型运行设备。设为"auto"会自动检测，也可手动设定为"cuda","mps","cpu"其中之一。
 EMBEDDING_DEVICE = "auto"
@@ -18,7 +17,7 @@ EMBEDDING_MODEL_OUTPUT_PATH = "output"
 
 # 要运行的 LLM 名称，可以包括本地模型和在线模型。
 # 第一个将作为 API 和 WEBUI 的默认模型
-LLM_MODELS = ["chatglm2-6b", "zhipu-api", "openai-api"]
+LLM_MODELS = ["chatglm3-6b", "zhipu-api", "openai-api"]
 
 # AgentLM模型的名称 (可以不指定，指定之后就锁定进入Agent之后的Chain的模型，不指定就是LLM_MODELS[0])
 Agent_MODEL = None
@@ -153,6 +152,7 @@ MODEL_PATH = {
         "bge-large-zh-v1.5": "BAAI/bge-large-zh-v1.5",
         "piccolo-base-zh": "sensenova/piccolo-base-zh",
         "piccolo-large-zh": "sensenova/piccolo-large-zh",
+        "nlp_gte_sentence-embedding_chinese-large": "damo/nlp_gte_sentence-embedding_chinese-large",
         "text-embedding-ada-002": "your OPENAI_API_KEY",
     },
 
@@ -169,7 +169,7 @@ MODEL_PATH = {
 
         "baichuan-7b": "baichuan-inc/Baichuan-7B",
         "baichuan-13b": "baichuan-inc/Baichuan-13B",
-        'baichuan-13b-chat': 'baichuan-inc/Baichuan-13B-Chat',
+        "baichuan-13b-chat": "baichuan-inc/Baichuan-13B-Chat",
 
         "aquila-7b": "BAAI/Aquila-7B",
         "aquilachat-7b": "BAAI/AquilaChat-7B",
@@ -210,6 +210,12 @@ MODEL_PATH = {
         "Qwen-14B-Chat": "Qwen/Qwen-14B-Chat",
         "Qwen-14B-Chat-Int8": "Qwen/Qwen-14B-Chat-Int8",  # 确保已经安装了auto-gptq optimum flash-attn
         "Qwen-14B-Chat-Int4": "Qwen/Qwen-14B-Chat-Int4",  # 确保已经安装了auto-gptq optimum flash-attn
+
+        "agentlm-7b": "THUDM/agentlm-7b",
+        "agentlm-13b": "THUDM/agentlm-13b",
+        "agentlm-70b": "THUDM/agentlm-70b",
+
+        "Yi-34B-Chat": "https://huggingface.co/01-ai/Yi-34B-Chat", # 更多01-ai模型尚未进行测试。如果需要使用，请自行测试。
     },
 }
 
@@ -225,7 +231,16 @@ VLLM_MODEL_DICT = {
 
     "baichuan-7b": "baichuan-inc/Baichuan-7B",
     "baichuan-13b": "baichuan-inc/Baichuan-13B",
-    'baichuan-13b-chat': 'baichuan-inc/Baichuan-13B-Chat',
+    "baichuan-13b-chat": "baichuan-inc/Baichuan-13B-Chat",
+
+    "chatglm2-6b": "THUDM/chatglm2-6b",
+    "chatglm2-6b-32k": "THUDM/chatglm2-6b-32k",
+    "chatglm3-6b": "THUDM/chatglm3-6b",
+    "chatglm3-6b-32k": "THUDM/chatglm3-6b-32k",
+
+    "BlueLM-7B-Chat": "vivo-ai/BlueLM-7B-Chat",
+    "BlueLM-7B-Chat-32k": "vivo-ai/BlueLM-7B-Chat-32k",
+
     # 注意：bloom系列的tokenizer与model是分离的，因此虽然vllm支持，但与fschat框架不兼容
     # "bloom":"bigscience/bloom",
     # "bloomz":"bigscience/bloomz",

configs/server_config.py.example

@@ -91,9 +91,12 @@ FSCHAT_MODEL_WORKERS = {
 
     },
     # 可以如下示例方式更改默认配置
-    # "baichuan-7b": { # 使用default中的IP和端口
+    # "Qwen-7B-Chat": { # 使用default中的IP和端口
     #    "device": "cpu",
     # },
+    "chatglm3-6b": { # 使用default中的IP和端口
+        "device": "cuda",
+    },
 
     #以下配置可以不用修改，在model_config中设置启动的模型
     "zhipu-api": {

requirements.txt

@@ -1,21 +1,21 @@
 # API requirements
 
 langchain>=0.0.334 
-langchain-experimental>=0.0.30
-fschat[model_worker]==0.2.32
-xformers>=0.0.22.post4
-openai>=0.28.1
+langchain-experimental>=0.0.42
+fschat[model_worker]>=0.2.33
+xformers>=0.0.22.post7
+openai~=0.28.1
 sentence_transformers
-transformers>=4.34
-torch>=2.0.1 # suggest version 2.1
+transformers>=4.35.2
+torch==2.1.0
 torchvision
 torchaudio
 fastapi>=0.104
 nltk>=3.8.1
 uvicorn~=0.23.1
 starlette~=0.27.0
-pydantic~=1.10.11
-unstructured[all-docs]>=0.10.12
+pydantic<2
+unstructured[all-docs]>=0.11.0
 python-magic-bin; sys_platform == 'win32'
 SQLAlchemy==2.0.19
 faiss-cpu
@@ -38,13 +38,14 @@ pandas~=2.0.3
 einops
 transformers_stream_generator==0.0.4
 
-vllm>=0.2.0; sys_platform == "linux"
+vllm==0.2.2; sys_platform == "linux"
 
-# online api libs
-# zhipuai
-# dashscope>=1.10.0 # qwen
-# qianfan
-# volcengine>=1.0.106 # fangzhou
+# online api libs dependencies
+
+# zhipuai>=1.0.7
+# dashscope>=1.10.0
+# qianfan>=0.2.0
+# volcengine>=1.0.106
 
 # uncomment libs if you want to use corresponding vector store
 # pymilvus==2.1.3 # requires milvus==2.1.3
@@ -53,10 +54,10 @@ vllm>=0.2.0; sys_platform == "linux"
 
 # WebUI requirements
 
-streamlit~=1.28.1
+streamlit~=1.28.2
 streamlit-option-menu>=0.3.6
-streamlit-antd-components>=0.1.11
+streamlit-antd-components>=0.2.3
 streamlit-chatbox>=1.1.11
 streamlit-aggrid>=0.3.4.post3
-httpx~=0.24.0
+httpx[brotli,http2,socks]~=0.24.1
 watchdog

requirements_api.txt

@@ -1,25 +1,25 @@
 # API requirements
 
-langchain>=0.0.334 
-langchain-experimental>=0.0.30
-fschat[model_worker]==0.2.32
-xformers>=0.0.22.post4
-openai>=0.28.1
+langchain>=0.0.334
+langchain-experimental>=0.0.42
+fschat[model_worker]>=0.2.33
+xformers>=0.0.22.post7
+openai~=0.28.1
 sentence_transformers
-transformers>=4.34
-torch>=2.0.1 # suggest version 2.1
+transformers>=4.35.2
+torch==2.1.0
 torchvision
 torchaudio
 fastapi>=0.104
 nltk>=3.8.1
 uvicorn~=0.23.1
 starlette~=0.27.0
-pydantic~=1.10.11
-unstructured[all-docs]>=0.10.12
+pydantic<2
+unstructured[all-docs]>=0.11.0
 python-magic-bin; sys_platform == 'win32'
 SQLAlchemy==2.0.19
 faiss-cpu
-accelerate
+accelerate>=0.24.1
 spacy
 PyMuPDF
 rapidocr_onnxruntime
@@ -36,7 +36,7 @@ websockets
 numpy~=1.24.4
 pandas~=2.0.3
 einops
-transformers_stream_generator==0.0.4
+transformers_stream_generator>=0.0.4
 
 vllm>=0.2.0; sys_platform == "linux"
 

requirements_webui.txt

@@ -1,9 +1,10 @@
 # WebUI requirements
 
-streamlit~=1.28.1
+streamlit~=1.28.2
 streamlit-option-menu>=0.3.6
-streamlit-antd-components>=0.1.11
+streamlit-antd-components>=0.2.3
 streamlit-chatbox>=1.1.11
 streamlit-aggrid>=0.3.4.post3
-httpx[brotli,http2,socks]>=0.25.0
+httpx[brotli,http2,socks]~=0.24.1
 watchdog
+

server/knowledge_base/utils.py

@@ -51,25 +51,35 @@ def list_kbs_from_folder():
 
 
 def list_files_from_folder(kb_name: str):
-    def is_skiped_path(path: str): # 跳过 [temp, tmp, ., ~$] 开头的目录和文件
-        tail = os.path.basename(path).lower()
-        flag = False
-        for x in ["temp", "tmp", ".", "~$"]:
-            if tail.startswith(x):
-                flag = True
-                break
-        return flag
-
     doc_path = get_doc_path(kb_name)
     result = []
-    for root, _, files in os.walk(doc_path):
-        if is_skiped_path(root):
-            continue
-        for file in files:
-            if is_skiped_path(file):
-                continue
-            path = Path(doc_path) / root / file
-            result.append(path.resolve().relative_to(doc_path).as_posix())
+
+    def is_skiped_path(path: str):
+        tail = os.path.basename(path).lower()
+        for x in ["temp", "tmp", ".", "~$"]:
+            if tail.startswith(x):
+                return True
+        return False
+
+    def process_entry(entry):
+        if is_skiped_path(entry.path):
+            return
+
+        if entry.is_symlink():
+            target_path = os.path.realpath(entry.path)
+            with os.scandir(target_path) as target_it:
+                for target_entry in target_it:
+                    process_entry(target_entry)
+        elif entry.is_file():
+            result.append(entry.path)
+        elif entry.is_dir():
+            with os.scandir(entry.path) as it:
+                for sub_entry in it:
+                    process_entry(sub_entry)
+
+    with os.scandir(doc_path) as it:
+        for entry in it:
+            process_entry(entry)
 
     return result