mkdir llama3-edge-inference && cd llama3-edge-inference

# ============================================================
# Dockerfile for Llama3.2-3B Model with vLLM 0.6.3.post1
# ============================================================

# 使用 vLLM 官方 OpenAI 兼容镜像作为基础镜像
FROM vllm/vllm-openai:v0.6.3.post1

# 设置环境变量，跳过 HuggingFace Hub 在线验证（离线模式）
ENV HF_HUB_OFFLINE=1
ENV HF_HOME=/data/models
ENV TRANSFORMERS_CACHE=/data/models

# 安装 modelscope 用于下载模型（使用国内镜像源加速）
RUN pip3 install --no-cache-dir modelscope -i https://pypi.tuna.tsinghua.edu.cn/simple

# 创建模型目录并下载模型文件
RUN mkdir -p /data/models/LLM-Research/Llama-3.2-3B-Instruct && \\
    python3 -c "from modelscope import snapshot_download; snapshot_download('LLM-Research/Llama-3.2-3B-Instruct', local_dir='/data/models/LLM-Research/Llama-3.2-3B-Instruct')"

# 暴露推理服务端口
EXPOSE 8000

# 启动参数说明：
# --host 0.0.0.0      监听所有网络接口
# --port 8000          服务监听端口
# --model              模型文件路径
# --trust-remote-code  信任远程代码（部分模型需要）
# --dtype half         使用半精度推理，降低显存占用
# --max-model-len 8192 最大上下文长度
CMD ["--host", "0.0.0.0", \\
     "--port", "8000", \\
     "--model", "/data/models/LLM-Research/Llama-3.2-3B-Instruct", \\
     "--trust-remote-code", \\
     "--dtype", "half", \\
     "--max-model-len", "8192"]

docker build -t llama3-3b-vllm:v1.0 .

docker images | grep llama3-3b-vllm

llama3-3b-vllm    v1.0    xxxxxxxxxxxx    xx minutes ago    约15GB

docker run --gpus all -p 8000:8000 llama3-3b-vllm:v1.0

curl http://localhost:8000/v1/chat/completions \\
  -H "Content-Type: application/json" \\
  -d '{
    "model": "/data/models/LLM-Research/Llama-3.2-3B-Instruct",
    "messages": [{"role": "user", "content": "Hello!"}],
    "max_tokens": 50
  }'

# 个人版 TCR 登录
docker login ccr.ccs.tencentyun.com --username=<腾讯云账号ID>

# 为镜像打标签（请替换为您的实际仓库地址）
docker tag llama3-3b-vllm:v1.0 ccr.ccs.tencentyun.com/edge-inference/llama3-3b-vllm:v1.0

# 推送镜像到 TCR
docker push ccr.ccs.tencentyun.com/edge-inference/llama3-3b-vllm:v1.0

python3 -m vllm.entrypoints.openai.api_server --host 0.0.0.0 --port 8000 --model /data/models/LLM-Research/Llama-3.2-3B-Instruct --trust-remote-code --dtype half --max-model-len 8192

curl https://YOUR_SERVICE_URL/v1/chat/completions \\
  -H "Authorization: Bearer YOUR_BEARER_TOKEN" \\
  -H "Content-Type: application/json" \\
  -d '{
    "model": "/data/models/LLM-Research/Llama-3.2-3B-Instruct",
    "messages": [
      {"role": "system", "content": "You are a helpful assistant."},
      {"role": "user", "content": "Hello, who are you?"}
    ],
    "max_tokens": 256,
    "temperature": 0.7
  }'


{
  "id": "cmpl-xxxxxxxx",
  "object": "chat.completion",
  "created": 1739260800,
  "model": "/data/models/LLM-Research/Llama-3.2-3B-Instruct",
  "choices": [
    {
      "index": 0,
      "message": {
        "role": "assistant",
        "content": "Hello! I'm Llama, a helpful AI assistant developed by Meta. I'm here to help you with questions, provide information, and assist with various tasks. How can I help you today?"
      },
      "finish_reason": "stop"
    }
  ],
  "usage": {
    "prompt_tokens": 25,
    "completion_tokens": 42,
    "total_tokens": 67
  }
}

curl -sS -X POST \\
  "https://YOUR_SERVICE_URL/v1/chat/completions" \\
  -H "Authorization: Bearer YOUR_BEARER_TOKEN" \\
  -H "Content-Type: application/json" \\
  -H "Work-Mode: async" \\
  -d '{
    "model": "/data/models/LLM-Research/Llama-3.2-3B-Instruct",
    "messages": [
      {"role": "system", "content": "You are a helpful assistant."},
      {"role": "user", "content": "Hello, who are you?"}
    ],
    "max_tokens": 256,
    "temperature": 0.7
  }'


{
  "TaskId": "YOUR_TASK_ID",
  "Status": "PENDING"
}


curl -sS -X GET \\
  "https://YOUR_SERVICE_URL/query/YOUR_TASK_ID" \\
  -H "Authorization: Bearer YOUR_BEARER_TOKEN" \\
  -H "Work-Infer-Action: query"


{
  "TaskId": "YOUR_TASK_ID",
  "Status": "SUCCEEDED",
  "Response": {
    "id": "chatcmpl-abc123",
    "object": "chat.completion",
    "created": 1746789012,
    "model": "/data/models/LLM-Research/Llama-3.2-3B-Instruct",
    "choices": [
      {
        "index": 0,
        "message": {
          "role": "assistant",
          "content": "Hello! I'm Llama, a helpful AI assistant developed by Meta."
        },
        "finish_reason": "stop"
      }
    ],
    "usage": {
      "prompt_tokens": 25,
      "completion_tokens": 42,
      "total_tokens": 67
    }
  }
}


{
  "TaskId": "YOUR_TASK_ID",
  "Status": "FAILED",
  "Error": "错误原因描述"
}


import time
import requests
BASE = "https://YOUR_SERVICE_URL"
TOKEN = "YOUR_BEARER_TOKEN"
MODEL = "/data/models/LLM-Research/Llama-3.2-3B-Instruct"
def submit_async():
    url = f"{BASE}/v1/chat/completions"
    headers = {
        "Authorization": f"Bearer {TOKEN}",
        "Content-Type": "application/json",
        "Work-Mode": "async",
    }
    payload = {
        "model": MODEL,
        "messages": [
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "Hello, who are you?"},
        ],
        "max_tokens": 256,
        "temperature": 0.7,
    }
    r = requests.post(url, headers=headers, json=payload, timeout=30)
    r.raise_for_status()
    return r.json()["TaskId"]
def query_task(task_id, timeout=120, interval=1.0):
    url = f"{BASE}/query/{task_id}"
    headers = {
        "Authorization": f"Bearer {TOKEN}",
        "Work-Infer-Action": "query",
    }
    deadline = time.time() + timeout
    while time.time() < deadline:
        r = requests.get(url, headers=headers, timeout=15)
        r.raise_for_status()
        data = r.json()
        status = data.get("Status")
        if status == "SUCCEEDED":
            return data
        if status == "FAILED":
            raise RuntimeError(f"task failed: {data}")
        time.sleep(interval)
    raise TimeoutError(f"task {task_id} timeout")
if __name__ == "__main__":
    task_id = submit_async()
    print("submitted:", task_id)
    result = query_task(task_id)
    print(result["Response"]["choices"][0]["message"]["content"])