From 9e2b411b01666ca1125feb2025b1f11d16767623 Mon Sep 17 00:00:00 2001
From: liunux4odoo <liunux@qq.com>
Date: Mon, 31 Jul 2023 11:18:57 +0800
Subject: [PATCH] cuda error with multiprocessing, change model_worker to main
 process

---
 server/llm_api.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/server/llm_api.py b/server/llm_api.py
index caa5127..e8e5452 100644
--- a/server/llm_api.py
+++ b/server/llm_api.py
@@ -219,14 +219,15 @@ if __name__ == "__main__":
         )
         controller_process.start()
 
-        model_worker_process = Process(
-            target=run_model_worker,
-            name=f"model_worker({os.getpid()})",
-            args=(queue,),
-            # kwargs={"load_8bit": True},
-            daemon=True,
-        )
-        model_worker_process.start()
+        # cuda 没办法用在fork的多进程中
+        # model_worker_process = Process(
+        #     target=run_model_worker,
+        #     name=f"model_worker({os.getpid()})",
+        #     args=(queue,),
+        #     # kwargs={"load_8bit": True},
+        #     daemon=True,
+        # )
+        # model_worker_process.start()
 
         openai_api_process = Process(
             target=run_openai_api,
@@ -236,10 +237,13 @@ if __name__ == "__main__":
         )
         openai_api_process.start()
 
+        run_model_worker(queue)
+
         controller_process.join()
-        model_worker_process.join()
+        # model_worker_process.join()
         openai_api_process.join()
 
+
 # 服务启动后接口调用示例：
 # import openai
 # openai.api_key = "EMPTY" # Not support yet