diff --git a/configs/model_config.py b/configs/model_config.py index e7c661f..11aaf88 100644 --- a/configs/model_config.py +++ b/configs/model_config.py @@ -34,32 +34,24 @@ EMBEDDING_DEVICE = "cuda" if torch.cuda.is_available() else "mps" if torch.backe llm_model_dict = { "chatglm-6b": { - "name": "chatglm-6b", # "name"修改为fastchat服务中的"model_name" - "pretrained_model_name": "chatglm-6b", - "local_model_path": "", + "local_model_path": "THUDM/chatglm-6b", "api_base_url": "http://localhost:8888/v1", # "name"修改为fastchat服务中的"api_base_url" "api_key": "EMPTY" }, "chatglm-6b-int4": { - "name": "chatglm-6b-int4", # "name"修改为fastchat服务中的"model_name" - "pretrained_model_name": "chatglm-6b-int4", - "local_model_path": "", + "local_model_path": "THUDM/chatglm-6b-int4", "api_base_url": "http://localhost:8001/v1", # "name"修改为fastchat服务中的"api_base_url" "api_key": "EMPTY" }, "chatglm2-6b": { - "name": "chatglm2-6b", # "name"修改为fastchat服务中的"model_name" - "pretrained_model_name": "chatglm2-6b", - "local_model_path": "", + "local_model_path": "THUDM/chatglm2-6b", "api_base_url": "http://localhost:8888/v1", # "name"修改为fastchat服务中的"api_base_url" "api_key": "EMPTY" }, "vicuna-13b-hf": { - "name": "vicuna-13b-hf", # "name"修改为fastchat服务中的"model_name" - "pretrained_model_name": "vicuna-13b-hf", "local_model_path": "", "api_base_url": "http://localhost:8000/v1", # "name"修改为fastchat服务中的"api_base_url" "api_key": "EMPTY" @@ -76,11 +68,9 @@ llm_model_dict = { # Failed to establish a new connection: [WinError 10060] # 则是因为内地和香港的IP都被OPENAI封了,需要切换为日本、新加坡等地 "openai-chatgpt-3.5": { - "name": "gpt-3.5-turbo", - "pretrained_model_name": "gpt-3.5-turbo", - "local_model_path": "", + "local_model_path": "gpt-3.5-turbo", "api_base_url": "https://api.openapi.com/v1", - "api_key": "" + "api_key": os.environ["OPENAI_API_KEY"] }, } @@ -92,6 +82,8 @@ LLM_DEVICE = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mp # 日志存储路径 LOG_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "logs") +if not os.path.exists(LOG_PATH): + os.mkdir(LOG_PATH) # 知识库默认存储路径 KB_ROOT_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "knowledge_base") diff --git a/server/llm_api.py b/server/llm_api.py index b68a863..4f64908 100644 --- a/server/llm_api.py +++ b/server/llm_api.py @@ -11,29 +11,42 @@ def execute_command(command): host_ip = "0.0.0.0" -port = 8888 +port = 8887 +# 1. llm_model_dict精简; +# 2. 不同任务的日志还是分开; + +# 3. 在此处定义args,可选fastchat为每个服务所提供的命令行参数,model_path除外; + +# 4. 用logger.removeHandler把它添加的handler删掉,添加我们自己的handler; + +# 5. 用watchdog监控第二步的执行情况; + +# 6. requirements指定fastchat版本号。 + +print(llm_model_dict[LLM_MODEL]) model_path = llm_model_dict[LLM_MODEL]["local_model_path"] if not model_path: logger.error("local_model_path 不能为空") else: # 启动任务 - command1 = f'nohup python -m fastchat.serve.controller >> {LOG_PATH}/fastchat_log.txt 2>&1 &' + command1 = f'nohup python -m fastchat.serve.controller >> {LOG_PATH}/controller_log.txt 2>&1 &' process1 = execute_command(command1) logger.info(f"已执行 {command1}") logger.info(f"Process 1 started with PID: {process1}") - command2 = f'nohup python -m fastchat.serve.model_worker --model-path "{model_path}" --device mps >> {LOG_PATH}/fastchat_log.txt 2>&1 &' + command2 = f'nohup python -m fastchat.serve.model_worker --model-path "{model_path}" --device cuda >> {LOG_PATH}/worker_log.txt 2>&1 &' process2 = execute_command(command2) logger.info(f"已执行 {command2}") logger.info(f"Process 2 started with PID: {process2}") - command3 = f'nohup python -m fastchat.serve.openai_api_server --host "{host_ip}" --port {port} >> {LOG_PATH}/fastchat_log.txt 2>&1 &' + command3 = f'nohup python -m fastchat.serve.openai_api_server --host "{host_ip}" --port {port} >> {LOG_PATH}/api_log.txt 2>&1 &' process3 = execute_command(command3) logger.info(f"已执行 {command3}") logger.info(f"Process 3 started with PID: {process3}") - # TODO: model_worker.log 与 controller.log 存储位置未指定为 LOG_PATH + # TODO: model_worker.log 与 controller.log 存储位置未指定为 LOG_PATH --> (hzg0601)model_worker.py,controller.py自行指定的文件写入路径, + # TODO(hzg0601): -->而且是写死的,如果想修改路径必须修改fastchat的代码 logger.info(f"如需查看 llm_api 日志,请前往 {LOG_PATH}") # 服务启动后接口调用示例: diff --git a/setup_server.sh b/setup_server.sh new file mode 100644 index 0000000..3fd62c4 --- /dev/null +++ b/setup_server.sh @@ -0,0 +1,30 @@ +[ -d "./logs/" ] && echo "log dir exists" || mkdir "./logs/" +# controller +nohup python3 -m fastchat.serve.controller >./logs/controller.log 2>&1 & +while [ `grep -c "Uvicorn running on" ./logs/controller.log` -eq '0' ];do + sleep 1s; + echo "wait controller running" +done +echo "controller running" + +# worker +nohup python3 -m fastchat.serve.model_worker \ +--model-name 'chatglm2-6b' \ +--model-path THUDM/chatglm2-6b \ +--num-gpus 2 \ +>> ./logs/worker.log 2>&1 & + +while [ `grep -c "Uvicorn running on" ./logs/worker.log` -eq '0' ];do + sleep 3s; + echo "wait worker running" +done +echo "worker running" + +# webui +nohup python3 -m fastchat.serve.openai_api_server >> "./logs/openai_server.log" 2>&1 & + +while [ `grep -c "Uvicorn running on" ./logs/openai_server.log` -eq '0' ];do + sleep 3s; + echo "wait openai_server running" +done +echo "openai_server running" \ No newline at end of file diff --git a/shutdown_server.sh b/shutdown_server.sh new file mode 100644 index 0000000..8b93d15 --- /dev/null +++ b/shutdown_server.sh @@ -0,0 +1 @@ +ps -eo user,pid,cmd|grep fastchat|grep -v grep|awk '{print $2}'|xargs kill -9 \ No newline at end of file