From 54fcbb8cc7515acc43eb1641446aa74474c8bdd2 Mon Sep 17 00:00:00 2001 From: waani Date: Wed, 17 Apr 2024 10:11:47 +0800 Subject: [PATCH 1/3] fix some problem for qwen --- llm/LLM.py | 2 +- llm/Qwen.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/llm/LLM.py b/llm/LLM.py index 3157e8e..e279d26 100644 --- a/llm/LLM.py +++ b/llm/LLM.py @@ -26,7 +26,7 @@ class LLM: elif model_name == 'ChatGPT': llm = ChatGPT(model_path, api_key=api_key) elif model_name == 'Qwen': - llm = Qwen(self.mode, model_path) + llm = Qwen(model_path) elif model_name == 'VllmGPT': llm = VllmGPT() return llm diff --git a/llm/Qwen.py b/llm/Qwen.py index 6c79b31..ab2341b 100644 --- a/llm/Qwen.py +++ b/llm/Qwen.py @@ -8,18 +8,19 @@ class Qwen: def __init__(self, model_path="Qwen/Qwen-1_8B-Chat") -> None: '''暂时不写api版本,与Linly-api相类似,感兴趣可以实现一下''' self.model, self.tokenizer = self.init_model(model_path) - - def init_model(self, path = "Qwen/Qwen-1_8B-Chat"): - model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen-1_8B-Chat", - device_map="auto", + self.data = {} + + def init_model(self, path="Qwen/Qwen-1_8B-Chat"): + model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen-1_8B-Chat", + device_map="auto", trust_remote_code=True).eval() tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True) - return model, tokenizer - + return model, tokenizer + def chat(self, question): - self.data["question"] = f"{self.prompt} ### Instruction:{question} ### Response:" + self.data["question"] = f"{question} ### Instruction:{question} ### Response:" try: response, history = self.model.chat(self.tokenizer, self.data["question"], history=None) print(history) @@ -30,7 +31,7 @@ class Qwen: def test(): llm = Qwen(model_path="Qwen/Qwen-1_8B-Chat") - answer = llm.generate("如何应对压力？") + answer = llm.chat(question="如何应对压力？") print(answer) if __name__ == '__main__': From 3674875095f6c184676066eb2f4d07f27abb95d0 Mon Sep 17 00:00:00 2001 From: waani Date: Fri, 19 Apr 2024 10:15:55 +0800 Subject: [PATCH 2/3] add qwen openapi --- llm/LLM.py | 2 +- llm/Qwen.py | 41 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/llm/LLM.py b/llm/LLM.py index e279d26..ff27736 100644 --- a/llm/LLM.py +++ b/llm/LLM.py @@ -26,7 +26,7 @@ class LLM: elif model_name == 'ChatGPT': llm = ChatGPT(model_path, api_key=api_key) elif model_name == 'Qwen': - llm = Qwen(model_path) + llm = Qwen(model_path=model_path, api_key=api_key, api_base=proxy_url) elif model_name == 'VllmGPT': llm = VllmGPT() return llm diff --git a/llm/Qwen.py b/llm/Qwen.py index ab2341b..6275b68 100644 --- a/llm/Qwen.py +++ b/llm/Qwen.py @@ -1,12 +1,29 @@ import os -import torch -import requests -from transformers import AutoModelForCausalLM, AutoTokenizer +import openai + +''' +`huggingface`连接不上可以使用 `modelscope` +`pip install modelscope` +''' +from modelscope import AutoModelForCausalLM, AutoTokenizer +#from transformers import AutoModelForCausalLM, AutoTokenizer + os.environ['CUDA_LAUNCH_BLOCKING'] = '1' + class Qwen: - def __init__(self, model_path="Qwen/Qwen-1_8B-Chat") -> None: + def __init__(self, model_path="Qwen/Qwen-1_8B-Chat", api_base=None, api_key=None) -> None: '''暂时不写api版本,与Linly-api相类似,感兴趣可以实现一下''' + # 默认本地推理 + self.local = True + + # api_base和api_key不为空时使用openapi的方式 + if api_key is not None and base_url is not None: + openai.api_base = api_base + openai.api_key = api_key + self.local = False + return + self.model, self.tokenizer = self.init_model(model_path) self.data = {} @@ -19,7 +36,20 @@ class Qwen: return model, tokenizer def chat(self, question): + # 优先调用qwen openapi的方式 + if not self.local: + # 不使用流式回复的请求 + response = openai.ChatCompletion.create( + model="Qwen", + messages=[ + {"role": "user", "content": question} + ], + stream=False, + stop=[] + ) + return response.choices[0].message.content + # 默认本地推理 self.data["question"] = f"{question} ### Instruction:{question} ### Response:" try: response, history = self.model.chat(self.tokenizer, self.data["question"], history=None) @@ -28,11 +58,12 @@ class Qwen: except: return "对不起，你的请求出错了，请再次尝试。\nSorry, your request has encountered an error. Please try again.\n" - + def test(): llm = Qwen(model_path="Qwen/Qwen-1_8B-Chat") answer = llm.chat(question="如何应对压力？") print(answer) + if __name__ == '__main__': test() From 2389c97fb48be01c28b7ab48943f6a8b9a12ad4c Mon Sep 17 00:00:00 2001 From: waani Date: Fri, 19 Apr 2024 16:05:55 +0800 Subject: [PATCH 3/3] add qwen openapi --- llm/LLM.py | 26 ++++++++++++++------------ llm/Qwen.py | 2 +- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/llm/LLM.py b/llm/LLM.py index ff27736..d749648 100644 --- a/llm/LLM.py +++ b/llm/LLM.py @@ -2,25 +2,25 @@ from llm.Qwen import Qwen from llm.Gemini import Gemini from llm.ChatGPT import ChatGPT from llm.VllmGPT import VllmGPT - + def test_Qwen(question = "如何应对压力？", mode='offline', model_path="Qwen/Qwen-1_8B-Chat"): llm = Qwen(mode, model_path) answer = llm.generate(question) print(answer) - + def test_Gemini(question = "如何应对压力？", model_path='gemini-pro', api_key=None, proxy_url=None): llm = Gemini(model_path, api_key, proxy_url) answer = llm.generate(question) print(answer) - + class LLM: def __init__(self, mode='offline'): self.mode = mode - + def init_model(self, model_name, model_path, api_key=None, proxy_url=None): if model_name not in ['Qwen', 'Gemini', 'ChatGPT', 'VllmGPT']: raise ValueError("model_name must be 'ChatGPT', 'VllmGPT', 'Qwen', or 'Gemini'(其他模型还未集成)") - + if model_name == 'Gemini': llm = Gemini(model_path, api_key, proxy_url) elif model_name == 'ChatGPT': @@ -32,9 +32,9 @@ class LLM: return llm - def test_Qwen(self, question="如何应对压力？", model_path="Qwen/Qwen-1_8B-Chat"): - llm = Qwen(self.mode, model_path) - answer = llm.generate(question) + def test_Qwen(self, question="如何应对压力？", model_path="Qwen/Qwen-1_8B-Chat", api_key=None, proxy_url=None): + llm = Qwen(model_path=model_path, api_key=api_key, api_base=proxy_url) + answer = llm.chat(question) print(answer) def test_Gemini(self, question="如何应对压力？", model_path='gemini-pro', api_key=None, proxy_url=None): @@ -43,10 +43,12 @@ class LLM: print(answer) if __name__ == '__main__': - # llm = LLM() + llm = LLM() # llm.test_Gemini(api_key='你的API Key', proxy_url=None) # llm = LLM().init_model('Gemini', model_path= 'gemini-pro',api_key='AIzaSyBWAWfT8zsyAZcRIXLS5Vzlw8KKCN9qsAg', proxy_url='http://172.31.71.58:7890') # response = llm.chat("如何应对压力？") - llm = LLM().init_model('VllmGPT', model_path= 'THUDM/chatglm3-6b') - response = llm.chat("如何应对压力？") - # print(response) \ No newline at end of file + # llm = LLM().init_model('VllmGPT', model_path= 'THUDM/chatglm3-6b') + # response = llm.chat("如何应对压力？") + # print(response) + + llm.test_Qwen(api_key="none", proxy_url="http://10.1.1.113:18000/v1") diff --git a/llm/Qwen.py b/llm/Qwen.py index 6275b68..21dd2e7 100644 --- a/llm/Qwen.py +++ b/llm/Qwen.py @@ -18,7 +18,7 @@ class Qwen: self.local = True # api_base和api_key不为空时使用openapi的方式 - if api_key is not None and base_url is not None: + if api_key is not None and api_base is not None: openai.api_base = api_base openai.api_key = api_key self.local = False