From 607be337816009b7d44160e643e1500a6537b155 Mon Sep 17 00:00:00 2001 From: "yanyuxiyangzk@126.com" Date: Wed, 3 Apr 2024 19:39:16 +0800 Subject: [PATCH] =?UTF-8?q?vllm=E4=BB=A3=E7=A0=81=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm/README.md | 4 ++ llm/VllmGPT.py | 105 ++++++++++++++++++++++++++++++++++--------------- 2 files changed, 78 insertions(+), 31 deletions(-) diff --git a/llm/README.md b/llm/README.md index 3518a6d..7528405 100644 --- a/llm/README.md +++ b/llm/README.md @@ -25,6 +25,10 @@ curl -X POST "http://127.0.0.1:8101/v1/completions" \ -H "Content-Type: application/json" \ -d "{\"model\": \"THUDM/chatglm3-6b\",\"prompt\": \"你叫什么名字\", \"history\": [{\"role\": \"user\", \"content\": \"你出生在哪里.\"}, {\"role\": \"assistant\", \"content\": \"出生在北京\"}]}" +多轮对话 +curl -X POST "http://127.0.0.1:8101/v1/chat/completions" \ +-H "Content-Type: application/json" \ +-d "{\"model\": \"THUDM/chatglm3-6b\", \"messages\": [{\"role\": \"system\", \"content\": \"You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.\"}, {\"role\": \"user\", \"content\": \"你好，给我讲一个故事，大概100字\"}], \"stream\": false, \"max_tokens\": 100, \"temperature\": 0.8, \"top_p\": 0.8}" diff --git a/llm/VllmGPT.py b/llm/VllmGPT.py index 6d3b37c..6e70096 100644 --- a/llm/VllmGPT.py +++ b/llm/VllmGPT.py @@ -2,37 +2,80 @@ import json import requests # from core import content_db +class VllmGPT: -def question(cont): - chat_list = [] - # contentdb = content_db.new_instance() - # list = contentdb.get_list('all','desc',11) - # answer_info = dict() - # chat_list = [] - # i = len(list)-1 - # while i >= 0: - # answer_info = dict() - # if list[i][0] == "member": - # answer_info["role"] = "user" - # answer_info["content"] = list[i][2] - # elif list[i][0] == "fay": - # answer_info["role"] = "bot" - # answer_info["content"] = list[i][2] - # chat_list.append(answer_info) - # i -= 1 - content = { - "model": "THUDM/chatglm3-6b", - "prompt":"请简单回复我。" + cont, - "history":chat_list} - url = "http://192.168.1.3:8101/v1/completions" - req = json.dumps(content) - - headers = {'content-type': 'application/json'} - r = requests.post(url, headers=headers, data=req) - res = json.loads(r.text) - - return res['choices'][0]['text'] + def __init__(self, host="127.0.0.1", + port="8000", + model="THUDM/chatglm3-6b", + max_tokens="1024"): + self.host = host + self.port = port + self.model=model + self.max_tokens=max_tokens + self.__URL = "http://{}:{}/v1/completions".format(self.host, self.port) + self.__URL2 = "http://{}:{}/v1/chat/completions".format(self.host, self.port) + def question(self,cont): + chat_list = [] + # contentdb = content_db.new_instance() + # list = contentdb.get_list('all','desc',11) + # answer_info = dict() + # chat_list = [] + # i = len(list)-1 + # while i >= 0: + # answer_info = dict() + # if list[i][0] == "member": + # answer_info["role"] = "user" + # answer_info["content"] = list[i][2] + # elif list[i][0] == "fay": + # answer_info["role"] = "bot" + # answer_info["content"] = list[i][2] + # chat_list.append(answer_info) + # i -= 1 + content = { + "model": self.model, + "prompt":"请简单回复我。" + cont, + "history":chat_list} + url = self.__URL + req = json.dumps(content) + + headers = {'content-type': 'application/json'} + r = requests.post(url, headers=headers, data=req) + res = json.loads(r.text) + + return res['choices'][0]['text'] + + def question2(self,cont): + chat_list = [] + # contentdb = content_db.new_instance() + # list = contentdb.get_list('all','desc',11) + # answer_info = dict() + # chat_list = [] + # i = len(list)-1 + # while i >= 0: + # answer_info = dict() + # if list[i][0] == "member": + # answer_info["role"] = "user" + # answer_info["content"] = list[i][2] + # elif list[i][0] == "fay": + # answer_info["role"] = "bot" + # answer_info["content"] = list[i][2] + # chat_list.append(answer_info) + # i -= 1 + content = { + "model": self.model, + "prompt":"请简单回复我。" + cont, + "history":chat_list} + url = self.__URL2 + req = json.dumps(content) + + headers = {'content-type': 'application/json'} + r = requests.post(url, headers=headers, data=req) + res = json.loads(r.text) + + return res['choices'][0]['message']['content'] + if __name__ == "__main__": - req = question("你叫什么名字啊今年多大了") - print(req) \ No newline at end of file + vllm = VllmGPT('192.168.1.3','8101') + req = vllm.question("你叫什么名字啊今年多大了") + print(req)