{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d2ff171c-f5f8-4590-9ce0-21c87e3d5b39",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO  2023-06-01 20:26:48,576-1d: \n",
      "loading model config\n",
      "llm device: cuda\n",
      "embedding device: cuda\n",
      "dir: /media/gpt4-pdf-chatbot-langchain/dev-langchain-ChatGLM\n",
      "flagging username: 7daba79785044bceb6896b9e6f8f9894\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append('/media/gpt4-pdf-chatbot-langchain/dev-langchain-ChatGLM/')\n",
    "from langchain.llms.base import LLM\n",
    "import torch\n",
    "import transformers \n",
    "import models.shared as shared \n",
    "from abc import ABC\n",
    "\n",
    "from langchain.llms.base import LLM\n",
    "import random\n",
    "from transformers.generation.logits_process import LogitsProcessor\n",
    "from transformers.generation.utils import LogitsProcessorList, StoppingCriteriaList\n",
    "from typing import Optional, List, Dict, Any\n",
    "from models.loader import LoaderCheckPoint \n",
    "from models.base import (BaseAnswer,\n",
    "                         AnswerResult)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "68978c38-c0e9-4ae9-ba90-9c02aca335be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading vicuna-7b-hf...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Overriding torch_dtype=None with `torch_dtype=torch.float16` due to requirements of `bitsandbytes` to enable model loading in mixed int8. Either pass torch_dtype=torch.float16 or don't pass this argument at all to remove this warning.\n",
      "/media/gpt4-pdf-chatbot-langchain/pyenv-langchain/lib/python3.10/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /media/gpt4-pdf-chatbot-langchain/pyenv-langchain did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...\n",
      "  warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please run\n",
      "\n",
      "python -m bitsandbytes\n",
      "\n",
      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "bin /media/gpt4-pdf-chatbot-langchain/pyenv-langchain/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda118.so\n",
      "CUDA SETUP: CUDA runtime path found: /opt/cuda/lib64/libcudart.so.11.0\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.6\n",
      "CUDA SETUP: Detected CUDA version 118\n",
      "CUDA SETUP: Loading binary /media/gpt4-pdf-chatbot-langchain/pyenv-langchain/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda118.so...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b61d05e18044b009c72b862c84ab5cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded the model in 6.39 seconds.\n"
     ]
    }
   ],
   "source": [
    "import asyncio\n",
    "from argparse import Namespace\n",
    "from models.loader.args import parser\n",
    "from langchain.agents import initialize_agent, Tool\n",
    "from langchain.agents import AgentType\n",
    " \n",
    "args = parser.parse_args(args=['--model-dir', '/media/checkpoint/',  '--model', 'vicuna-7b-hf', '--no-remote-model', '--load-in-8bit'])\n",
    "\n",
    "args_dict = vars(args)\n",
    "\n",
    "shared.loaderCheckPoint = LoaderCheckPoint(args_dict)\n",
    "torch.cuda.empty_cache()\n",
    "shared.loaderCheckPoint.unload_model()\n",
    "shared.loaderCheckPoint.reload_model() \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c8e4a58d-1a3a-484a-8417-bcec0eb7170e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'action': 'State of Dialogue History System', 'action_input': '露ᥫᩣ,'}\n"
     ]
    }
   ],
   "source": [
    "from jsonformer import Jsonformer\n",
    "json_schema = {\n",
    "    \"type\": \"object\",\n",
    "    \"properties\": {\n",
    "        \"action\": {\"type\": \"string\"},\n",
    "        \"action_input\": {\"type\": \"string\"}\n",
    "    }\n",
    "}\n",
    "\n",
    "prompt = \"\"\" Respond to the human as helpfully and accurately as possible. You have access to the following tools:\n",
    "\n",
    "State of Dialogue History System: 露ᥫᩣ, 张毛峰的对话 - 当需要查找露ᥫᩣ, 张毛峰之间的聊天内容时，这里面的回答是很有用的。输入应该是一个完整的问题。\n",
    "Summary: useful for when you summarize a conversation. The input to this tool should be a string, representing who will read this summary.\n",
    "\n",
    "The way you use the tools is by specifying a json blob.\n",
    "Specifically, this json should have a `action` key (with the name of the tool to use) and a `action_input` key (with the input to the tool going here).\n",
    "\n",
    "The only values that should be in the \"action\" field are: {tool_names}\n",
    "\n",
    "The $JSON_BLOB should only contain a SINGLE action, do NOT return a list of multiple actions. Here is an example of a valid $JSON_BLOB:\n",
    "\n",
    "```\n",
    "{{{{\n",
    "  \"action\": $TOOL_NAME,\n",
    "  \"action_input\": $INPUT\n",
    "}}}}\n",
    "```\n",
    "\n",
    "ALWAYS use the following format:\n",
    "\n",
    "Question: the input question you must answer\n",
    "Thought: you should always think about what to do\n",
    "Action:\n",
    "```\n",
    "$JSON_BLOB\n",
    "```\n",
    "Observation: the result of the action\n",
    "... (this Thought/Action/Observation can repeat N times)\n",
    "Thought: I now know the final answer\n",
    "Final Answer: the final answer to the original input question\n",
    "\n",
    "Begin! Reminder to always use the exact characters `Final Answer` when responding.\n",
    "\n",
    "Question: 我想查看关于露露的摘要信息\n",
    "\"\"\"\n",
    "jsonformer = Jsonformer(shared.loaderCheckPoint.model, shared.loaderCheckPoint.tokenizer, json_schema, prompt)\n",
    "generated_data = jsonformer()\n",
    "\n",
    "print(generated_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e089a828-b662-4d9a-8d88-4bf95ccadbab",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}