update requirements: 统一各文件依赖顺序，便于比对；移出streamlit-antd-components; jq默认安装; numexpr 限定 2.8.6 以兼容 py38

requirements.txt

@@ -4,23 +4,24 @@
 torch~=2.1.2
 torchvision~=0.16.2
 torchaudio~=2.1.2
+xformers==0.0.23.post1
+transformers==4.36.2
+sentence_transformers==2.2.2
 
 langchain==0.0.352
 langchain-experimental==0.0.47
 pydantic==1.10.13
 fschat==0.2.34
-xformers==0.0.23.post1
 openai~=1.6.0
-sentence_transformers==2.2.2
-transformers==4.36.2
 fastapi>=0.105
+sse_starlette
 nltk>=3.8.1
 uvicorn>=0.24.0.post1
 starlette~=0.27.0
 unstructured[all-docs]==0.11.0
 python-magic-bin; sys_platform == 'win32'
 SQLAlchemy==2.0.19
-faiss-cpu~=1.7.4
+faiss-cpu~=1.7.4 # `conda install faiss-gpu -c conda-forge` if you want to accelerate with gpus
 accelerate==0.24.1
 spacy~=3.7.2
 PyMuPDF~=1.23.8
@@ -28,10 +29,10 @@ rapidocr_onnxruntime~=1.3.8
 requests>=2.31.0
 pathlib>=1.0.1
 pytest>=7.4.3
-numexpr>=2.8.8
+numexpr>=2.8.6 # max version for py38
 strsimpy>=0.2.1
 markdownify>=0.11.6
-tiktoken>=0.5.2
+tiktoken~=0.5.2
 tqdm>=4.66.1
 websockets>=12.0
 numpy~=1.26.2
@@ -39,21 +40,24 @@ pandas~=2.1.4
 einops>=0.7.0
 transformers_stream_generator==0.0.4
 vllm==0.2.6; sys_platform == "linux"
+httpx[brotli,http2,socks]~=0.25.2
 
 # optional document loaders
 
-rapidocr_paddle[gpu]>=1.3.0.post5
-jq>=1.6.0
-html2text
-beautifulsoup4~=4.12.2
+rapidocr_paddle[gpu]>=1.3.0.post5 # gpu accelleration for ocr of pdf and image files
+jq>=1.6.0 # for .json and .jsonl files. suggest `conda install jq` on windows
+# html2text # for .enex files
+beautifulsoup4~=4.12.2 # for .mhtml files
 pysrt~=1.1.2
 
 # Online api libs dependencies
 
-zhipuai>=1.0.7，<=2.0.0
-dashscope>=1.13.6
-volcengine>=1.0.119
-pymilvus>=2.3.4
+zhipuai>=1.0.7，<=2.0.0 # zhipu
+dashscope>=1.13.6 # qwen
+# volcengine>=1.0.119 # fangzhou
+
+# uncomment libs if you want to use corresponding vector store
+# pymilvus>=2.3.4
 # psycopg2==2.9.9
 # pgvector>=0.2.4
 
@@ -66,7 +70,7 @@ metaphor-python>=0.1.23
 
 # WebUI requirements
 
-streamlit~=1.29.0
+streamlit~=1.29.0 # do remember to add streamlit to environment variables if you use windows
 streamlit-option-menu>=0.3.6
 streamlit-chatbox==1.1.11
 streamlit-modal>=0.1.0

requirements_api.txt

@@ -4,23 +4,24 @@
 torch~=2.1.2
 torchvision~=0.16.2
 torchaudio~=2.1.2
+xformers==0.0.23.post1
+transformers==4.36.2
+sentence_transformers==2.2.2
 
 langchain==0.0.352
 langchain-experimental==0.0.47
 pydantic==1.10.13
 fschat==0.2.34
-xformers==0.0.23.post1
 openai~=1.6.0
-sentence_transformers==2.2.2
-transformers==4.36.2
 fastapi>=0.105
+sse_starlette
 nltk>=3.8.1
 uvicorn>=0.24.0.post1
 starlette~=0.27.0
 unstructured[all-docs]==0.11.0
 python-magic-bin; sys_platform == 'win32'
 SQLAlchemy==2.0.19
-faiss-cpu~=1.7.4
+faiss-cpu~=1.7.4 # `conda install faiss-gpu -c conda-forge` if you want to accelerate with gpus
 accelerate==0.24.1
 spacy~=3.7.2
 PyMuPDF~=1.23.8
@@ -28,10 +29,10 @@ rapidocr_onnxruntime~=1.3.8
 requests>=2.31.0
 pathlib>=1.0.1
 pytest>=7.4.3
-numexpr>=2.8.8
+numexpr>=2.8.6 # max version for py38
 strsimpy>=0.2.1
 markdownify>=0.11.6
-tiktoken>=0.5.2
+tiktoken~=0.5.2
 tqdm>=4.66.1
 websockets>=12.0
 numpy~=1.26.2
@@ -39,21 +40,24 @@ pandas~=2.1.4
 einops>=0.7.0
 transformers_stream_generator==0.0.4
 vllm==0.2.6; sys_platform == "linux"
+httpx[brotli,http2,socks]~=0.25.2
 
 # optional document loaders
 
-rapidocr_paddle[gpu]>=1.3.0.post5
-jq>=1.6.0
-html2text
-beautifulsoup4~=4.12.2
+rapidocr_paddle[gpu]>=1.3.0.post5 # gpu accelleration for ocr of pdf and image files
+jq>=1.6.0 # for .json and .jsonl files. suggest `conda install jq` on windows
+# html2text # for .enex files
+beautifulsoup4~=4.12.2 # for .mhtml files
 pysrt~=1.1.2
 
 # Online api libs dependencies
 
-zhipuai>=1.0.7，<=2.0.0
-dashscope>=1.13.6
-volcengine>=1.0.119
-pymilvus>=2.3.4
+zhipuai>=1.0.7，<=2.0.0 # zhipu
+dashscope>=1.13.6 # qwen
+# volcengine>=1.0.119 # fangzhou
+
+# uncomment libs if you want to use corresponding vector store
+# pymilvus>=2.3.4
 # psycopg2==2.9.9
 # pgvector>=0.2.4
 

requirements_lite.txt

@@ -1,10 +1,18 @@
 # API requirements
 
-langchain==0.0.344
-langchain-experimental>=0.0.42
+# On Windows system, install the cuda version manually from https://pytorch.org/
+# torch~=2.1.2
+# torchvision~=0.16.2
+# torchaudio~=2.1.2
+# xformers==0.0.23.post1
+# transformers==4.36.2
+# sentence_transformers==2.2.2
+
+langchain==0.0.352
+langchain-experimental==0.0.47
 pydantic==1.10.13
-fschat>=0.2.34
-openai>=1.6.0
+fschat==0.2.34
+openai~=1.6.0
 fastapi>=0.105
 sse_starlette
 nltk>=3.8.1
@@ -13,34 +21,44 @@ starlette~=0.27.0
 unstructured[docx,csv]==0.11.0 # add pdf if need
 python-magic-bin; sys_platform == 'win32'
 SQLAlchemy==2.0.19
-faiss-cpu # `conda install faiss-gpu -c conda-forge` if you want to accelerate with gpus
+faiss-cpu~=1.7.4 # `conda install faiss-gpu -c conda-forge` if you want to accelerate with gpus
+# accelerate==0.24.1
+# spacy~=3.7.2
+# PyMuPDF~=1.23.8
+# rapidocr_onnxruntime~=1.3.8
 requests>=2.31.0
 pathlib>=1.0.1
 pytest>=7.4.3
-numexpr>=2.8.7
+numexpr>=2.8.6 # max version for py38
 strsimpy>=0.2.1
 markdownify>=0.11.6
+# tiktoken~=0.5.2
 tqdm>=4.66.1
-websockets
-numpy~=1.24.4
-pandas~=2.0.3
+websockets>=12.0
+numpy~=1.26.2
+pandas~=2.1.4
+# einops>=0.7.0
+# transformers_stream_generator==0.0.4
+# vllm==0.2.6; sys_platform == "linux"
+httpx[brotli,http2,socks]~=0.25.2
 
 # optional document loaders
-# rapidocr_paddle[gpu] # gpu accelleration for ocr of pdf and image files
-jq # for .json and .jsonl files. suggest `conda install jq` on windows
+
+rapidocr_paddle[gpu]>=1.3.0.post5 # gpu accelleration for ocr of pdf and image files
+jq>=1.6.0 # for .json and .jsonl files. suggest `conda install jq` on windows
 # html2text # for .enex files
-# beautifulsoup4 # for .mhtml files
-# pysrt # for .srt files
+beautifulsoup4~=4.12.2 # for .mhtml files
+pysrt~=1.1.2
 
 # Online api libs dependencies
 
 zhipuai>=1.0.7，<=2.0.0 # zhipu
-dashscope>=1.10.0 # qwen
-# volcengine>=1.0.106 # fangzhou
+dashscope>=1.13.6 # qwen
+# volcengine>=1.0.119 # fangzhou
 
 # uncomment libs if you want to use corresponding vector store
-# pymilvus==2.1.3 # requires milvus==2.1.3
-# psycopg2
+# pymilvus>=2.3.4
+# psycopg2==2.9.9
 # pgvector>=0.2.4
 
 # Agent and Search Tools
@@ -52,10 +70,10 @@ metaphor-python>=0.1.23
 
 # WebUI requirements
 
-streamlit>=1.29.0 # do remember to add streamlit to environment variables if you use windows
+streamlit~=1.29.0 # do remember to add streamlit to environment variables if you use windows
 streamlit-option-menu>=0.3.6
-streamlit-chatbox>=1.1.11
+streamlit-chatbox==1.1.11
 streamlit-modal>=0.1.0
 streamlit-aggrid>=0.3.4.post3
+httpx[brotli,http2,socks]>=0.25.2
 watchdog>=3.0.0
-

requirements_webui.txt

@@ -1,6 +1,6 @@
 # WebUI requirements
 
-streamlit~=1.29.0
+streamlit~=1.29.0 # do remember to add streamlit to environment variables if you use windows
 streamlit-option-menu>=0.3.6
 streamlit-chatbox==1.1.11
 streamlit-modal>=0.1.0