From abb32722fe1b92a13eaa9db4681112ee39f53fbd Mon Sep 17 00:00:00 2001 From: Zhi-guo Huang Date: Sat, 25 Nov 2023 19:44:33 +0800 Subject: [PATCH] Update startup.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复vllm=0.2.2版本中vllm_worker启动失败的bug --- startup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/startup.py b/startup.py index fab87c2..587564f 100644 --- a/startup.py +++ b/startup.py @@ -137,7 +137,9 @@ def create_model_worker_app(log_level: str = "INFO", **kwargs) -> FastAPI: args.quantization = None args.max_log_len = None args.tokenizer_revision = None - + # 0.2.2 vllm需要新加的参数 + args.max_paddings = 256 + if args.model_path: args.model = args.model_path if args.num_gpus > 1: @@ -161,7 +163,7 @@ def create_model_worker_app(log_level: str = "INFO", **kwargs) -> FastAPI: conv_template = args.conv_template, ) sys.modules["fastchat.serve.vllm_worker"].engine = engine - # sys.modules["fastchat.serve.vllm_worker"].worker = worker + sys.modules["fastchat.serve.vllm_worker"].worker = worker sys.modules["fastchat.serve.vllm_worker"].logger.setLevel(log_level) else: