From 9f4567865cd834ad3710657dc8f3a473f83533db Mon Sep 17 00:00:00 2001 From: imClumsyPanda Date: Tue, 1 Aug 2023 14:12:28 +0800 Subject: [PATCH 1/9] add chatglm2-6b-32k and make m3e default embedding model --- configs/model_config.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/configs/model_config.py b/configs/model_config.py index f00b646..eb4c4a9 100644 --- a/configs/model_config.py +++ b/configs/model_config.py @@ -26,7 +26,7 @@ embedding_model_dict = { } # 选用的 Embedding 名称 -EMBEDDING_MODEL = "text2vec" +EMBEDDING_MODEL = "m3e-base" # Embedding 模型运行设备 EMBEDDING_DEVICE = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu" @@ -51,6 +51,12 @@ llm_model_dict = { "api_key": "EMPTY" }, + "chatglm2-6b-32k": { + "local_model_path": "THUDM/chatglm2-6b-32k", # "THUDM/chatglm2-6b-32k", + "api_base_url": "http://localhost:8888/v1", # "name"修改为fastchat服务中的"api_base_url" + "api_key": "EMPTY" + }, + "vicuna-13b-hf": { "local_model_path": "", "api_base_url": "http://localhost:8000/v1", # "name"修改为fastchat服务中的"api_base_url" From a1a7484ef4d61e398fc54530ac4ba3eccf4bd35b Mon Sep 17 00:00:00 2001 From: liunux4odoo Date: Tue, 1 Aug 2023 14:15:42 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=E5=A2=9E=E5=8A=A0webui=5Futils.py=EF=BC=8C?= =?UTF-8?q?=E5=8C=85=E6=8B=AC=E5=88=B6=E4=BD=9Cwebui=E9=80=9A=E7=94=A8?= =?UTF-8?q?=E7=9A=84=E5=B7=A5=E5=85=B7=EF=BC=8C=E6=96=B9=E4=BE=BF=E4=BB=A5?= =?UTF-8?q?=E5=90=8E=E5=BC=80=E5=8F=91=E5=85=B6=E4=BB=96webui?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- webui_utils.py | 247 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 webui_utils.py diff --git a/webui_utils.py b/webui_utils.py new file mode 100644 index 0000000..12dab3d --- /dev/null +++ b/webui_utils.py @@ -0,0 +1,247 @@ +# 该文件包含webui通用工具，可以被不同的webui使用 + +from typing import * +from pathlib import Path +import os +from configs.model_config import ( + KB_ROOT_PATH, + LLM_MODEL, + llm_model_dict, +) +import httpx +import asyncio +from server.chat.openai_chat import OpenAiChatMsgIn +from fastapi.responses import StreamingResponse + + +def set_httpx_timeout(timeout=60.0): + ''' + 设置httpx默认timeout到60秒。 + httpx默认timeout是5秒，在请求LLM回答时不够用。 + ''' + httpx._config.DEFAULT_TIMEOUT_CONFIG.connect = timeout + httpx._config.DEFAULT_TIMEOUT_CONFIG.read = timeout + httpx._config.DEFAULT_TIMEOUT_CONFIG.write = timeout + + +KB_ROOT_PATH = Path(KB_ROOT_PATH) +set_httpx_timeout() + + +def get_kb_list() -> List[str]: + ''' + 获取知识库列表 + ''' + kb_list = os.listdir(KB_ROOT_PATH) + return [x for x in kb_list if (KB_ROOT_PATH / x).is_dir()] + + +def get_kb_files(kb: str) -> List[str]: + ''' + 获取某个知识库下包含的所有文件（只包括根目录一级） + ''' + kb = KB_ROOT_PATH / kb / "content" + if kb.is_dir(): + kb_files = os.listdir(kb) + return kb_files + else: + return [] + + +def run_async(cor): + ''' + 在同步环境中运行异步代码. + ''' + try: + loop = asyncio.get_event_loop() + except: + loop = asyncio.new_event_loop() + return loop.run_until_complete(cor) + + +def iter_over_async(ait, loop): + ''' + 将异步生成器封装成同步生成器. + ''' + ait = ait.__aiter__() + async def get_next(): + try: + obj = await ait.__anext__() + return False, obj + except StopAsyncIteration: + return True, None + while True: + done, obj = loop.run_until_complete(get_next()) + if done: + break + yield obj + + +class ApiRequest: + ''' + api.py调用的封装,主要实现: + 1. 简化api调用方式 + 2. 实现无api调用(直接运行server.chat.*中的视图函数获取结果),无需启动api.py + ''' + def __init__( + self, + base_url: str = "http://127.0.0.1:7861", + timeout: float = 60.0, + ): + self.base_url = base_url + self.timeout = timeout + + def _parse_url(self, url: str) -> str: + if (not url.startswith("http") + and self.base_url + ): + part1 = self.base_url.strip(" /") + part2 = url.strip(" /") + return f"{part1}/{part2}" + else: + return url + + def get( + self, + url: str, + params: Union[Dict, List[Tuple], bytes] = None, + retry: int = 3, + **kwargs: Any, + ) -> Union[httpx.Response, None]: + url = self._parse_url(url) + kwargs.setdefault("timeout", self.timeout) + while retry > 0: + try: + return httpx.get(url, params, **kwargs) + except: + retry -= 1 + + async def aget( + self, + url: str, + params: Union[Dict, List[Tuple], bytes] = None, + retry: int = 3, + **kwargs: Any, + ) -> Union[httpx.Response, None]: + rl = self._parse_url(url) + kwargs.setdefault("timeout", self.timeout) + async with httpx.AsyncClient() as client: + while retry > 0: + try: + return await client.get(url, params, **kwargs) + except: + retry -= 1 + + def post( + self, + url: str, + data: Dict = None, + json: Dict = None, + retry: int = 3, + stream: bool = False, + **kwargs: Any + ) -> Union[httpx.Response, None]: + url = self._parse_url(url) + kwargs.setdefault("timeout", self.timeout) + while retry > 0: + try: + # return requests.post(url, data=data, json=json, stream=stream, **kwargs) + if stream: + return httpx.stream("POST", url, data=data, json=json, **kwargs) + else: + return httpx.post(url, data=data, json=json, **kwargs) + except: + retry -= 1 + + async def apost( + self, + url: str, + data: Dict = None, + json: Dict = None, + retry: int = 3, + **kwargs: Any + ) -> Union[httpx.Response, None]: + rl = self._parse_url(url) + kwargs.setdefault("timeout", self.timeout) + async with httpx.AsyncClient() as client: + while retry > 0: + try: + return await client.post(url, data=data, json=json, **kwargs) + except: + retry -= 1 + + def _stream2generator(self, response: StreamingResponse): + ''' + 将api.py中视图函数返回的StreamingResponse转化为同步生成器 + ''' + try: + loop = asyncio.get_event_loop() + except: + loop = asyncio.new_event_loop() + return iter_over_async(response.body_iterator, loop) + + def chat_fastchat( + self, + messages: List[Dict], + stream: bool = True, + model: str = LLM_MODEL, + temperature: float = 0.7, + max_tokens: int = 1024, # todo:根据message内容自动计算max_tokens + no_remote_api=False, # all api view function directly + **kwargs: Any, + ): + ''' + 对应api.py/chat/fastchat接口 + ''' + msg = OpenAiChatMsgIn(**{ + "messages": messages, + "stream": stream, + "model": model, + "temperature": temperature, + "max_tokens": max_tokens, + **kwargs, + }) + + if no_remote_api: + from server.chat.openai_chat import openai_chat + response = openai_chat(msg) + return self._stream2generator(response) + else: + data = msg.dict(exclude_unset=True, exclude_none=True) + response = self.post( + "/chat/fastchat", + json=data, + stream=stream, + ) + return response + + def chat_chat( + self, + query: str, + no_remote_api: bool = False, + ): + ''' + 对应api.py/chat/chat接口 + ''' + if no_remote_api: + from server.chat.chat import chat + response = chat(query) + return self._stream2generator(response) + else: + response = self.post("/chat/chat", json=f"{query}", stream=True) + return response + + +if __name__ == "__main__": + api = ApiRequest() + # print(api.chat_fastchat( + # messages=[{"role": "user", "content": "hello"}] + # )) + + with api.chat_chat("你好") as r: + for t in r.iter_text(None): + print(t) + + r = api.chat_chat("你好", no_remote_api=True) + for t in r: + print(t) \ No newline at end of file From 2c5b6bb0adaf0380447beca4634daa19743418f0 Mon Sep 17 00:00:00 2001 From: liunux4odoo Date: Tue, 1 Aug 2023 14:18:30 +0800 Subject: [PATCH 3/9] =?UTF-8?q?streamlit=20ui=20=E5=AE=9E=E7=8E=B0LLM?= =?UTF-8?q?=E6=B5=81=E5=BC=8F=E5=AF=B9=E8=AF=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 3 ++- webui.py | 52 ++++++++++++++++++++++-------------------------- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2c57bad..89eef3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,4 +17,5 @@ pydantic~=1.10.11 unstructured[local-inference] streamlit>=1.25.0 -streamlit-option-menu \ No newline at end of file +streamlit-option-menu +streamlit-chatbox>=1.1.0 diff --git a/webui.py b/webui.py index 08d2d25..1976a7f 100644 --- a/webui.py +++ b/webui.py @@ -1,6 +1,10 @@ import streamlit as st +from streamlit_chatbox import * +from webui_utils import * from streamlit_option_menu import option_menu -import openai + + +api = ApiRequest() def dialogue_page(): with st.sidebar: @@ -8,37 +12,30 @@ def dialogue_page(): ["LLM 对话", "知识库问答", "Bing 搜索问答"]) + history_len = st.slider("历史对话轮数：", 1, 10, 1) if dialogue_mode == "知识库问答": - selected_kb = st.selectbox("请选择知识库：", ["知识库1", "知识库2"]) + selected_kb = st.selectbox("请选择知识库：", get_kb_list()) with st.expander(f"{selected_kb} 中已存储文件"): - st.write("123") + st.write(get_kb_files(selected_kb)) # Display chat messages from history on app rerun - for message in st.session_state.messages: - with st.chat_message(message["role"]): - st.markdown(message["content"]) + chat_box.output_messages() if prompt := st.chat_input("What is up?"): - st.session_state.messages.append({"role": "user", "content": prompt}) - with st.chat_message("user"): - st.markdown(prompt) - - with st.chat_message("assistant"): - message_placeholder = st.empty() - full_response = "" - for response in openai.ChatCompletion.create( - model=OPENAI_MODEL, - messages=[ - {"role": m["role"], "content": m["content"]} - for m in st.session_state.messages - ], - stream=True, - ): - full_response += response.choices[0].delta.get("content", "") - message_placeholder.markdown(full_response + "▌") - message_placeholder.markdown(full_response) - st.session_state.messages.append({"role": "assistant", "content": full_response}) - + chat_box.user_say(prompt) + chat_box.ai_say("正在思考...") + # with api.chat_fastchat([{"role": "user", "content": "prompt"}], stream=streaming) as r: # todo: support history len + text = "" + r = api.chat_chat(prompt, no_remote_api=True) + for t in r: + text += t + chat_box.update_msg(text) + chat_box.update_msg(text, streaming=False) + # with api.chat_chat(prompt) as r: + # for t in r.iter_text(None): + # text += t + # chat_box.update_msg(text) + # chat_box.update_msg(text, streaming=False) def knowledge_base_edit_page(): pass @@ -51,8 +48,7 @@ def config_page(): if __name__ == "__main__": st.set_page_config("langchain-chatglm WebUI") - if "messages" not in st.session_state: - st.session_state.messages = [] + chat_box = ChatBox() pages = {"对话": {"icon": "chat", "func": dialogue_page, From c8a75ab11f9e0453e43883f47e27024d38442115 Mon Sep 17 00:00:00 2001 From: imClumsyPanda Date: Tue, 1 Aug 2023 14:33:18 +0800 Subject: [PATCH 4/9] update llm_api.py and webui.py --- server/llm_api.py | 8 ++------ webui.py | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/server/llm_api.py b/server/llm_api.py index e8e5452..60227cf 100644 --- a/server/llm_api.py +++ b/server/llm_api.py @@ -1,7 +1,7 @@ - from multiprocessing import Process, Queue import sys import os + sys.path.append(os.path.dirname(os.path.dirname(__file__))) from configs.model_config import llm_model_dict, LLM_MODEL, LLM_DEVICE, LOG_PATH, logger import asyncio @@ -31,7 +31,7 @@ def create_controller_app( controller = Controller(dispatch_method) sys.modules["fastchat.serve.controller"].controller = controller - #todo 替换fastchat的日志文件 + # todo 替换fastchat的日志文件 sys.modules["fastchat.serve.controller"].logger = logger logger.info(f"controller dispatch method: {dispatch_method}") return app @@ -199,9 +199,6 @@ def run_openai_api(q): uvicorn.run(app, host=host_ip, port=openai_api_port) - - - if __name__ == "__main__": logger.info(llm_model_dict[LLM_MODEL]) model_path = llm_model_dict[LLM_MODEL]["local_model_path"] @@ -243,7 +240,6 @@ if __name__ == "__main__": # model_worker_process.join() openai_api_process.join() - # 服务启动后接口调用示例： # import openai # openai.api_key = "EMPTY" # Not support yet diff --git a/webui.py b/webui.py index 1976a7f..a79f4c0 100644 --- a/webui.py +++ b/webui.py @@ -21,7 +21,7 @@ def dialogue_page(): # Display chat messages from history on app rerun chat_box.output_messages() - if prompt := st.chat_input("What is up?"): + if prompt := st.chat_input("请输入对话内容，换行请使用Ctrl+Enter"): chat_box.user_say(prompt) chat_box.ai_say("正在思考...") # with api.chat_fastchat([{"role": "user", "content": "prompt"}], stream=streaming) as r: # todo: support history len From bcfd3f5af54eccc00ff7f8d600788c188205d0a3 Mon Sep 17 00:00:00 2001 From: imClumsyPanda Date: Tue, 1 Aug 2023 14:47:38 +0800 Subject: [PATCH 5/9] add webui_pages --- configs/model_config.py | 4 +- webui.py | 50 ++------------------ webui_pages/__init__.py | 3 ++ webui_pages/dialogue/__init__.py | 1 + webui_pages/dialogue/dialogue.py | 36 ++++++++++++++ webui_pages/knowledge_base/__init__.py | 1 + webui_pages/knowledge_base/knowledge_base.py | 6 +++ webui_pages/model_config/__init__.py | 1 + webui_pages/model_config/model_config.py | 5 ++ webui_utils.py => webui_pages/utils.py | 0 10 files changed, 59 insertions(+), 48 deletions(-) create mode 100644 webui_pages/__init__.py create mode 100644 webui_pages/dialogue/__init__.py create mode 100644 webui_pages/dialogue/dialogue.py create mode 100644 webui_pages/knowledge_base/__init__.py create mode 100644 webui_pages/knowledge_base/knowledge_base.py create mode 100644 webui_pages/model_config/__init__.py create mode 100644 webui_pages/model_config/model_config.py rename webui_utils.py => webui_pages/utils.py (100%) diff --git a/configs/model_config.py b/configs/model_config.py index eb4c4a9..b9e5206 100644 --- a/configs/model_config.py +++ b/configs/model_config.py @@ -21,7 +21,7 @@ embedding_model_dict = { "text2vec-sentence": "shibing624/text2vec-base-chinese-sentence", "text2vec-multilingual": "shibing624/text2vec-base-multilingual", "m3e-small": "moka-ai/m3e-small", - "m3e-base": "moka-ai/m3e-base", + "m3e-base": "/Users/liuqian/Downloads/ChatGLM-6B/m3e-base", "m3e-large": "moka-ai/m3e-large", } @@ -46,7 +46,7 @@ llm_model_dict = { }, "chatglm2-6b": { - "local_model_path": "THUDM/chatglm2-6b", + "local_model_path": "/Users/liuqian/Downloads/ChatGLM-6B/chatglm2-6b", # "THUDM/chatglm2-6b", "api_base_url": "http://localhost:8888/v1", # "name"修改为fastchat服务中的"api_base_url" "api_key": "EMPTY" }, diff --git a/webui.py b/webui.py index a79f4c0..c06646e 100644 --- a/webui.py +++ b/webui.py @@ -1,63 +1,21 @@ import streamlit as st -from streamlit_chatbox import * -from webui_utils import * +from webui_pages.utils import * from streamlit_option_menu import option_menu - +from webui_pages import * api = ApiRequest() -def dialogue_page(): - with st.sidebar: - dialogue_mode = st.radio("请选择对话模式", - ["LLM 对话", - "知识库问答", - "Bing 搜索问答"]) - history_len = st.slider("历史对话轮数：", 1, 10, 1) - if dialogue_mode == "知识库问答": - selected_kb = st.selectbox("请选择知识库：", get_kb_list()) - with st.expander(f"{selected_kb} 中已存储文件"): - st.write(get_kb_files(selected_kb)) - - # Display chat messages from history on app rerun - chat_box.output_messages() - - if prompt := st.chat_input("请输入对话内容，换行请使用Ctrl+Enter"): - chat_box.user_say(prompt) - chat_box.ai_say("正在思考...") - # with api.chat_fastchat([{"role": "user", "content": "prompt"}], stream=streaming) as r: # todo: support history len - text = "" - r = api.chat_chat(prompt, no_remote_api=True) - for t in r: - text += t - chat_box.update_msg(text) - chat_box.update_msg(text, streaming=False) - # with api.chat_chat(prompt) as r: - # for t in r.iter_text(None): - # text += t - # chat_box.update_msg(text) - # chat_box.update_msg(text, streaming=False) - -def knowledge_base_edit_page(): - pass - - -def config_page(): - pass - - if __name__ == "__main__": st.set_page_config("langchain-chatglm WebUI") - chat_box = ChatBox() - pages = {"对话": {"icon": "chat", "func": dialogue_page, }, "知识库管理": {"icon": "database-fill-gear", - "func": knowledge_base_edit_page, + "func": knowledge_base_page, }, "模型配置": {"icon": "gear", - "func": config_page, + "func": model_config_page, } } diff --git a/webui_pages/__init__.py b/webui_pages/__init__.py new file mode 100644 index 0000000..8d3ae10 --- /dev/null +++ b/webui_pages/__init__.py @@ -0,0 +1,3 @@ +from .dialogue import dialogue_page +from .knowledge_base import knowledge_base_page +from .model_config import model_config_page \ No newline at end of file diff --git a/webui_pages/dialogue/__init__.py b/webui_pages/dialogue/__init__.py new file mode 100644 index 0000000..b3aad16 --- /dev/null +++ b/webui_pages/dialogue/__init__.py @@ -0,0 +1 @@ +from .dialogue import dialogue_page \ No newline at end of file diff --git a/webui_pages/dialogue/dialogue.py b/webui_pages/dialogue/dialogue.py new file mode 100644 index 0000000..08b348c --- /dev/null +++ b/webui_pages/dialogue/dialogue.py @@ -0,0 +1,36 @@ +import streamlit as st +from webui_pages.utils import * +from streamlit_chatbox import * + +chat_box = ChatBox() + +def dialogue_page(): + with st.sidebar: + dialogue_mode = st.radio("请选择对话模式", + ["LLM 对话", + "知识库问答", + "Bing 搜索问答"]) + history_len = st.slider("历史对话轮数：", 1, 10, 1) + if dialogue_mode == "知识库问答": + selected_kb = st.selectbox("请选择知识库：", get_kb_list()) + with st.expander(f"{selected_kb} 中已存储文件"): + st.write(get_kb_files(selected_kb)) + + # Display chat messages from history on app rerun + chat_box.output_messages() + + if prompt := st.chat_input("请输入对话内容，换行请使用Ctrl+Enter"): + chat_box.user_say(prompt) + chat_box.ai_say("正在思考...") + # with api.chat_fastchat([{"role": "user", "content": "prompt"}], stream=streaming) as r: # todo: support history len + text = "" + r = api.chat_chat(prompt, no_remote_api=True) + for t in r: + text += t + chat_box.update_msg(text) + chat_box.update_msg(text, streaming=False) + # with api.chat_chat(prompt) as r: + # for t in r.iter_text(None): + # text += t + # chat_box.update_msg(text) + # chat_box.update_msg(text, streaming=False) \ No newline at end of file diff --git a/webui_pages/knowledge_base/__init__.py b/webui_pages/knowledge_base/__init__.py new file mode 100644 index 0000000..b7b37a0 --- /dev/null +++ b/webui_pages/knowledge_base/__init__.py @@ -0,0 +1 @@ +from .knowledge_base import knowledge_base_page \ No newline at end of file diff --git a/webui_pages/knowledge_base/knowledge_base.py b/webui_pages/knowledge_base/knowledge_base.py new file mode 100644 index 0000000..8515b62 --- /dev/null +++ b/webui_pages/knowledge_base/knowledge_base.py @@ -0,0 +1,6 @@ +import streamlit as st +from webui_pages.utils import * + +def knowledge_base_page(): + st.write(123) + pass \ No newline at end of file diff --git a/webui_pages/model_config/__init__.py b/webui_pages/model_config/__init__.py new file mode 100644 index 0000000..3cfc701 --- /dev/null +++ b/webui_pages/model_config/__init__.py @@ -0,0 +1 @@ +from .model_config import model_config_page \ No newline at end of file diff --git a/webui_pages/model_config/model_config.py b/webui_pages/model_config/model_config.py new file mode 100644 index 0000000..cc19d93 --- /dev/null +++ b/webui_pages/model_config/model_config.py @@ -0,0 +1,5 @@ +import streamlit as st +from webui_pages.utils import * + +def model_config_page(): + pass \ No newline at end of file diff --git a/webui_utils.py b/webui_pages/utils.py similarity index 100% rename from webui_utils.py rename to webui_pages/utils.py From 8261deb99d7fe01354c8314486cad95ea7eeeee6 Mon Sep 17 00:00:00 2001 From: imClumsyPanda Date: Tue, 1 Aug 2023 14:49:49 +0800 Subject: [PATCH 6/9] fix model_config path --- configs/model_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/model_config.py b/configs/model_config.py index b9e5206..eb4c4a9 100644 --- a/configs/model_config.py +++ b/configs/model_config.py @@ -21,7 +21,7 @@ embedding_model_dict = { "text2vec-sentence": "shibing624/text2vec-base-chinese-sentence", "text2vec-multilingual": "shibing624/text2vec-base-multilingual", "m3e-small": "moka-ai/m3e-small", - "m3e-base": "/Users/liuqian/Downloads/ChatGLM-6B/m3e-base", + "m3e-base": "moka-ai/m3e-base", "m3e-large": "moka-ai/m3e-large", } @@ -46,7 +46,7 @@ llm_model_dict = { }, "chatglm2-6b": { - "local_model_path": "/Users/liuqian/Downloads/ChatGLM-6B/chatglm2-6b", # "THUDM/chatglm2-6b", + "local_model_path": "THUDM/chatglm2-6b", "api_base_url": "http://localhost:8888/v1", # "name"修改为fastchat服务中的"api_base_url" "api_key": "EMPTY" }, From 7d79b676d51b4721dfed6c5def5c952cdb971448 Mon Sep 17 00:00:00 2001 From: imClumsyPanda Date: Tue, 1 Aug 2023 14:55:00 +0800 Subject: [PATCH 7/9] add model_config.py.example instead of model_config.py --- .gitignore | 1 + configs/{model_config.py => model_config.py.example} | 0 2 files changed, 1 insertion(+) rename configs/{model_config.py => model_config.py.example} (100%) diff --git a/.gitignore b/.gitignore index 124c04e..af50500 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ logs .idea/ __pycache__/ knowledge_base/ +configs/model_config.py \ No newline at end of file diff --git a/configs/model_config.py b/configs/model_config.py.example similarity index 100% rename from configs/model_config.py rename to configs/model_config.py.example From 5ce2484af086d763962dd369c9fd03f87abe852f Mon Sep 17 00:00:00 2001 From: imClumsyPanda Date: Tue, 1 Aug 2023 15:08:19 +0800 Subject: [PATCH 8/9] update webui.py --- webui.py | 2 +- webui_pages/dialogue/dialogue.py | 8 +++++--- webui_pages/knowledge_base/knowledge_base.py | 2 +- webui_pages/model_config/model_config.py | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/webui.py b/webui.py index c06646e..bd02735 100644 --- a/webui.py +++ b/webui.py @@ -26,4 +26,4 @@ if __name__ == "__main__": menu_icon="chat-quote", default_index=0) - pages[selected_page]["func"]() + pages[selected_page]["func"](api) diff --git a/webui_pages/dialogue/dialogue.py b/webui_pages/dialogue/dialogue.py index 08b348c..f01c0ac 100644 --- a/webui_pages/dialogue/dialogue.py +++ b/webui_pages/dialogue/dialogue.py @@ -4,7 +4,8 @@ from streamlit_chatbox import * chat_box = ChatBox() -def dialogue_page(): + +def dialogue_page(api: ApiRequest): with st.sidebar: dialogue_mode = st.radio("请选择对话模式", ["LLM 对话", @@ -22,7 +23,8 @@ def dialogue_page(): if prompt := st.chat_input("请输入对话内容，换行请使用Ctrl+Enter"): chat_box.user_say(prompt) chat_box.ai_say("正在思考...") - # with api.chat_fastchat([{"role": "user", "content": "prompt"}], stream=streaming) as r: # todo: support history len + # with api.chat_fastchat([{"role": "user", "content": "prompt"}], stream=streaming) as r: + # todo: support history len text = "" r = api.chat_chat(prompt, no_remote_api=True) for t in r: @@ -33,4 +35,4 @@ def dialogue_page(): # for t in r.iter_text(None): # text += t # chat_box.update_msg(text) - # chat_box.update_msg(text, streaming=False) \ No newline at end of file + # chat_box.update_msg(text, streaming=False) diff --git a/webui_pages/knowledge_base/knowledge_base.py b/webui_pages/knowledge_base/knowledge_base.py index 8515b62..119a4a2 100644 --- a/webui_pages/knowledge_base/knowledge_base.py +++ b/webui_pages/knowledge_base/knowledge_base.py @@ -1,6 +1,6 @@ import streamlit as st from webui_pages.utils import * -def knowledge_base_page(): +def knowledge_base_page(api: ApiRequest): st.write(123) pass \ No newline at end of file diff --git a/webui_pages/model_config/model_config.py b/webui_pages/model_config/model_config.py index cc19d93..119a6ca 100644 --- a/webui_pages/model_config/model_config.py +++ b/webui_pages/model_config/model_config.py @@ -1,5 +1,5 @@ import streamlit as st from webui_pages.utils import * -def model_config_page(): +def model_config_page(api: ApiRequest): pass \ No newline at end of file From 7c01a2a25313a8f2540ca753d887384f6dbf6ddb Mon Sep 17 00:00:00 2001 From: imClumsyPanda Date: Tue, 1 Aug 2023 16:39:17 +0800 Subject: [PATCH 9/9] add bing_search_chat.py and duckduckgo_search_chat.py --- configs/model_config.py.example | 15 +++++- server/api.py | 13 ++++-- server/chat/__init__.py | 2 + server/chat/bing_chat.py | 3 -- server/chat/bing_search_chat.py | 67 +++++++++++++++++++++++++++ server/chat/duckduckgo_search_chat.py | 62 +++++++++++++++++++++++++ 6 files changed, 155 insertions(+), 7 deletions(-) delete mode 100644 server/chat/bing_chat.py create mode 100644 server/chat/bing_search_chat.py create mode 100644 server/chat/duckduckgo_search_chat.py diff --git a/configs/model_config.py.example b/configs/model_config.py.example index eb4c4a9..7751bb4 100644 --- a/configs/model_config.py.example +++ b/configs/model_config.py.example @@ -106,4 +106,17 @@ PROMPT_TEMPLATE = """【指令】根据已知信息，简洁和专业的来回 # API 是否开启跨域，默认为False，如果需要开启，请设置为True # is open cross domain -OPEN_CROSS_DOMAIN = False \ No newline at end of file +OPEN_CROSS_DOMAIN = False + +# Bing 搜索必备变量 +# 使用 Bing 搜索需要使用 Bing Subscription Key,需要在azure port中申请试用bing search +# 具体申请方式请见 +# https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/create-bing-search-service-resource +# 使用python创建bing api 搜索实例详见: +# https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/quickstarts/rest/python +BING_SEARCH_URL = "https://api.bing.microsoft.com/v7.0/search" +# 注意不是bing Webmaster Tools的api key， + +# 此外，如果是在服务器上，报Failed to establish a new connection: [Errno 110] Connection timed out +# 是因为服务器加了防火墙，需要联系管理员加白名单，如果公司的服务器的话，就别想了GG +BING_SUBSCRIPTION_KEY = "" \ No newline at end of file diff --git a/server/api.py b/server/api.py index d58336c..9bbbbce 100644 --- a/server/api.py +++ b/server/api.py @@ -7,8 +7,9 @@ import argparse import uvicorn from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from starlette.responses import RedirectResponse, StreamingResponse -from server.chat import chat, knowledge_base_chat, openai_chat +from starlette.responses import RedirectResponse +from server.chat import (chat, knowledge_base_chat, openai_chat, + bing_search_chat, duckduckgo_search_chat) from server.knowledge_base import (list_kbs, create_kb, delete_kb, list_docs, upload_doc, delete_doc, update_doc) from server.utils import BaseResponse, ListResponse @@ -50,7 +51,13 @@ def create_app(): tags=["Chat"], summary="与知识库对话")(knowledge_base_chat) - # app.post("/chat/bing_search_chat", tags=["Chat"], summary="与Bing搜索对话")(bing_search_chat) + app.post("/chat/bing_search_chat", + tags=["Chat"], + summary="与Bing搜索对话")(bing_search_chat) + + app.post("/chat/duckduckgo_search_chat", + tags=["Chat"], + summary="与DuckDuckGo搜索对话")(duckduckgo_search_chat) app.get("/knowledge_base/list_knowledge_bases", tags=["Knowledge Base Management"], diff --git a/server/chat/__init__.py b/server/chat/__init__.py index 9fef958..728db91 100644 --- a/server/chat/__init__.py +++ b/server/chat/__init__.py @@ -1,3 +1,5 @@ from .chat import chat from .knowledge_base_chat import knowledge_base_chat from .openai_chat import openai_chat +from .duckduckgo_search_chat import duckduckgo_search_chat +from .bing_search_chat import bing_search_chat diff --git a/server/chat/bing_chat.py b/server/chat/bing_chat.py deleted file mode 100644 index 71453d2..0000000 --- a/server/chat/bing_chat.py +++ /dev/null @@ -1,3 +0,0 @@ -# TODO: 完成 bing_chat agent 接口实现 -def bing_chat(): - pass \ No newline at end of file diff --git a/server/chat/bing_search_chat.py b/server/chat/bing_search_chat.py new file mode 100644 index 0000000..bbb9089 --- /dev/null +++ b/server/chat/bing_search_chat.py @@ -0,0 +1,67 @@ +from langchain.utilities import BingSearchAPIWrapper +from configs.model_config import BING_SEARCH_URL, BING_SUBSCRIPTION_KEY +from fastapi import Body +from fastapi.responses import StreamingResponse +from configs.model_config import (llm_model_dict, LLM_MODEL, PROMPT_TEMPLATE) +from server.chat.utils import wrap_done +from langchain.chat_models import ChatOpenAI +from langchain import LLMChain +from langchain.callbacks import AsyncIteratorCallbackHandler +from typing import AsyncIterable +import asyncio +from langchain.prompts import PromptTemplate +from langchain.docstore.document import Document + +def bing_search(text, result_len=3): + if not (BING_SEARCH_URL and BING_SUBSCRIPTION_KEY): + return [{"snippet": "please set BING_SUBSCRIPTION_KEY and BING_SEARCH_URL in os ENV", + "title": "env info is not found", + "link": "https://python.langchain.com/en/latest/modules/agents/tools/examples/bing_search.html"}] + search = BingSearchAPIWrapper(bing_subscription_key=BING_SUBSCRIPTION_KEY, + bing_search_url=BING_SEARCH_URL) + return search.results(text, result_len) + + +def search_result2docs(search_results): + docs = [] + for result in search_results: + doc = Document(page_content=result["snippet"] if "snippet" in result.keys() else "", + metadata={"source": result["link"] if "link" in result.keys() else "", + "filename": result["title"] if "title" in result.keys() else ""}) + docs.append(doc) + return docs + + +def bing_search_chat(query: str = Body(..., description="用户输入", example="你好"), + ): + async def bing_search_chat_iterator(query: str, + ) -> AsyncIterable[str]: + callback = AsyncIteratorCallbackHandler() + model = ChatOpenAI( + streaming=True, + verbose=True, + callbacks=[callback], + openai_api_key=llm_model_dict[LLM_MODEL]["api_key"], + openai_api_base=llm_model_dict[LLM_MODEL]["api_base_url"], + model_name=LLM_MODEL + ) + + results = bing_search(query, result_len=3) + docs = search_result2docs(results) + context = "\n".join([doc.page_content for doc in docs]) + prompt = PromptTemplate(template=PROMPT_TEMPLATE, input_variables=["context", "question"]) + + chain = LLMChain(prompt=prompt, llm=model) + + # Begin a task that runs in the background. + task = asyncio.create_task(wrap_done( + chain.acall({"context": context, "question": query}), + callback.done), + ) + + async for token in callback.aiter(): + # Use server-sent-events to stream the response + yield token + await task + + return StreamingResponse(bing_search_chat_iterator(query), media_type="text/event-stream") diff --git a/server/chat/duckduckgo_search_chat.py b/server/chat/duckduckgo_search_chat.py new file mode 100644 index 0000000..c10e065 --- /dev/null +++ b/server/chat/duckduckgo_search_chat.py @@ -0,0 +1,62 @@ +from langchain.utilities import DuckDuckGoSearchAPIWrapper +from fastapi import Body +from fastapi.responses import StreamingResponse +from configs.model_config import (llm_model_dict, LLM_MODEL, PROMPT_TEMPLATE) +from server.chat.utils import wrap_done +from langchain.chat_models import ChatOpenAI +from langchain import LLMChain +from langchain.callbacks import AsyncIteratorCallbackHandler +from typing import AsyncIterable +import asyncio +from langchain.prompts import PromptTemplate +from langchain.docstore.document import Document + + +def duckduckgo_search(text, result_len=3): + search = DuckDuckGoSearchAPIWrapper() + return search.results(text, result_len) + + +def search_result2docs(search_results): + docs = [] + for result in search_results: + doc = Document(page_content=result["snippet"] if "snippet" in result.keys() else "", + metadata={"source": result["link"] if "link" in result.keys() else "", + "filename": result["title"] if "title" in result.keys() else ""}) + docs.append(doc) + return docs + + +def duckduckgo_search_chat(query: str = Body(..., description="用户输入", example="你好"), + ): + async def duckduckgo_search_chat_iterator(query: str, + ) -> AsyncIterable[str]: + callback = AsyncIteratorCallbackHandler() + model = ChatOpenAI( + streaming=True, + verbose=True, + callbacks=[callback], + openai_api_key=llm_model_dict[LLM_MODEL]["api_key"], + openai_api_base=llm_model_dict[LLM_MODEL]["api_base_url"], + model_name=LLM_MODEL + ) + + results = duckduckgo_search(query, result_len=3) + docs = search_result2docs(results) + context = "\n".join([doc.page_content for doc in docs]) + prompt = PromptTemplate(template=PROMPT_TEMPLATE, input_variables=["context", "question"]) + + chain = LLMChain(prompt=prompt, llm=model) + + # Begin a task that runs in the background. + task = asyncio.create_task(wrap_done( + chain.acall({"context": context, "question": query}), + callback.done), + ) + + async for token in callback.aiter(): + # Use server-sent-events to stream the response + yield token + await task + + return StreamingResponse(duckduckgo_search_chat_iterator(query), media_type="text/event-stream")