docker run -it \\
    --privileged \\
    --net=host \\
    --ipc=host \\
    --shm-size=16g \\
    --name=taco_llm \\
    --gpus all \\
    -v /home/workspace:/home/workspace \\
    ccr.ccs.tencentyun.com/taco/tacollm-dev:latest /bin/bash

taco_llm serve facebook/opt-125m --api-key taco-llm-test

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="taco-llm-test",
)

completion = client.chat.completions.create(
    model="facebook/opt-125m",
    messages=[
        {"role": "user", "content": "Hello!"}
    ]
)

print(completion.choices[0].message)

import requests

api_key = "taco-llm-test"

headers = {
    "Authorization": f"Bearer {api_key}"
}

pload = {
    "prompt": "Hello!",
    "stream": True,
    "max_tokens": 128,
}

response = requests.post("http://localhost:8000/v1/completions",
                         headers=headers,
                         json=pload,
                         stream=True)

for chunk in response.iter_lines(chunk_size=8192,
                                 decode_unicode=False,
                                 delimiter=b"\\0"):
    if chunk:
        data = json.loads(chunk.decode("utf-8"))
        output = data["text"][0]
        print(output)