From f83d29620b93f8891a0f6a626fa21c0afe3be89a Mon Sep 17 00:00:00 2001 From: "yanyuxiyangzk@126.com" Date: Thu, 4 Apr 2024 13:38:44 +0800 Subject: [PATCH] =?UTF-8?q?vllm=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm/README.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/llm/README.md b/llm/README.md index 7528405..6e92bbf 100644 --- a/llm/README.md +++ b/llm/README.md @@ -1,16 +1,16 @@ -1、推理加速 - +1、利用vllm可以显著推理加速大模型 conda create -n vllm python=3.10 +conda activate vllm conda install pytorch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0 pytorch-cuda=12.1 -c pytorch -c nvidia -启动 +2、启动推理 python -m vllm.entrypoints.openai.api_server --tensor-parallel-size=1 --trust-remote-code --max-model-len 1024 --model THUDM/chatglm3-6b +指定ip和端口：--host 127.0.0.1 --port 8101 -python -m vllm.entrypoints.openai.api_server --host 127.0.0.1 --port 8101 --tensor-parallel-size=1 --trust-remote-code --max-model-len 1024 --model THUDM/chatglm3-6b +python -m vllm.entrypoints.openai.api_server --port 8101 --tensor-parallel-size=1 --trust-remote-code --max-model-len 1024 --model THUDM/chatglm3-6b - -测试 +3、测试 curl http://127.0.0.1:8101/v1/completions \ -H "Content-Type: application/json" \ -d '{ @@ -31,8 +31,16 @@ curl -X POST "http://127.0.0.1:8101/v1/chat/completions" \ -d "{\"model\": \"THUDM/chatglm3-6b\", \"messages\": [{\"role\": \"system\", \"content\": \"You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.\"}, {\"role\": \"user\", \"content\": \"你好，给我讲一个故事，大概100字\"}], \"stream\": false, \"max_tokens\": 100, \"temperature\": 0.8, \"top_p\": 0.8}" - - +4、启动前端访问 +docker run -d \ +--network=host \ +--name nginx2 --restart=always \ +-v $PWD/nginx/conf/nginx.conf:/etc/nginx/nginx.conf \ +-v $PWD/nginx/html:/usr/share/nginx/html \ +-v $PWD/nginx/logs:/var/log/nginx \ +--privileged=true \ +--restart=always \ +nginx 参考文档：https://docs.vllm.ai/en/latest/ \ No newline at end of file