Update server_config.py.example

针对fschat=0.2.33关于vllm启动的bug给出修补方案
This commit is contained in:
Zhi-guo Huang 2023-11-25 19:41:21 +08:00 committed by GitHub
parent 63bdca1343
commit a83e03011d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 0 deletions

View File

@ -40,6 +40,8 @@ FSCHAT_MODEL_WORKERS = {
"device": LLM_DEVICE,
# False,'vllm',使用的推理加速框架,使用vllm如果出现HuggingFace通信问题参见doc/FAQ
# vllm对一些模型支持还不成熟暂时默认关闭
# fschat=0.2.33的代码有bug, 如需使用源码修改fastchat.server.vllm_worker
# 将103行中sampling_params = SamplingParams的参数stop=list(stop)修改为stop= [i for i in stop if i!=""]
"infer_turbo": False,
# model_worker多卡加载需要配置的参数