update loaders

api.py

@@ -16,9 +16,8 @@ from starlette.responses import RedirectResponse
 
 from chains.local_doc_qa import LocalDocQA
 from configs.model_config import (VS_ROOT_PATH, UPLOAD_ROOT_PATH, EMBEDDING_DEVICE,
-                                  EMBEDDING_MODEL, LLM_MODEL, NLTK_DATA_PATH,
+                                  EMBEDDING_MODEL, NLTK_DATA_PATH,
                                   VECTOR_SEARCH_TOP_K, LLM_HISTORY_LEN, OPEN_CROSS_DOMAIN)
-from agent import bing_search
 import models.shared as shared
 from models.loader.args import parser
 from models.loader import LoaderCheckPoint

loader/image_loader.py

@@ -4,7 +4,10 @@ from typing import List
 from langchain.document_loaders.unstructured import UnstructuredFileLoader
 from paddleocr import PaddleOCR
 import os
+import nltk
+from configs.model_config import NLTK_DATA_PATH
 
+nltk.data.path = [NLTK_DATA_PATH] + nltk.data.path
 
 class UnstructuredPaddleImageLoader(UnstructuredFileLoader):
     """Loader that uses unstructured to load image files, such as PNGs and JPGs."""

loader/pdf_loader.py

@@ -5,7 +5,10 @@ from langchain.document_loaders.unstructured import UnstructuredFileLoader
 from paddleocr import PaddleOCR
 import os
 import fitz
+import nltk
+from configs.model_config import NLTK_DATA_PATH
 
+nltk.data.path = [NLTK_DATA_PATH] + nltk.data.path
 
 class UnstructuredPaddlePDFLoader(UnstructuredFileLoader):
     """Loader that uses unstructured to load image files, such as PNGs and JPGs."""