vllm and sglang integration

from vllm import LLM, SamplingParams

# explicitly use transformers backend
llm = LLM(model="new-model", model_impl="transformers")

# with custom models
llm = LLM(model="custom-model", model_impl="transformers", trust_remote_code=True)

# generate
sampling_params = SamplingParams(max_tokens=100)
outputs = llm.generate(["Hello"], sampling_params)

vllm serve llava-hf/llava-onevision-qwen2-0.5b-ov-hf --model_impl transformers

# model_impl options:
# "auto" - try native first, fallback to transformers
# "vllm" - native only
# "transformers" - transformers backend only

from vllm import LLM, SamplingParams
from PIL import Image
from transformers import AutoProcessor

model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
processor = AutoProcessor.from_pretrained(model_id)

messages = [{
    "role": "user",
    "content": [
        {"type": "image", "url": "image.jpg"},
        {"type": "text", "text": "Describe this image."},
    ],
}]

prompt = processor.apply_chat_template(
    messages, tokenize=False, add_generation_prompt=True
)

vlm = LLM(model=model_id, model_impl="transformers")
outputs = vlm.generate(
    {"prompt": prompt, "multi_modal_data": {"image": image}},
    sampling_params=SamplingParams(max_tokens=100)
)

# transformers backend is official way for encoder models in vllm
llm = LLM(model="bert-base-uncased", model_impl="transformers")
llm = LLM(model="answerdotai/ModernBERT-base", model_impl="transformers")

import sglang as sgl

# use transformers backend
llm = sgl.Engine("meta-llama/Llama-3.2-1B-Instruct", impl="transformers")

# generate
output = llm.generate(["Hello"], {"max_new_tokens": 20})

python3 -m sglang.launch_server \
    --model-path meta-llama/Llama-3.2-1B-Instruct \
    --impl transformers \
    --host 0.0.0.0 \
    --port 30000

python -m sglang.launch_server \
    --model-path meta-llama/Llama-3.1-8B-Instruct \
    --impl transformers \
    --tp 2

# torchao quantization
python -m sglang.launch_server \
    --model-path meta-llama/Llama-3.1-8B \
    --impl transformers \
    --torchao-config int8dq

from openai import OpenAI

client = OpenAI(
    api_key="EMPTY",
    base_url="http://localhost:8000/v1",  # vllm
    # base_url="http://localhost:30000/v1",  # sglang
)

response = client.chat.completions.create(
    model="meta-llama/Llama-3.2-1B-Instruct",
    messages=[{"role": "user", "content": "Hello!"}]
)

from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS

class MyAttention(nn.Module):
    _supports_attention_backend = True

    def forward(self, query, key, value, **kwargs):
        attention_fn = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
        return attention_fn(self, query, key, value, **kwargs)

class MyModelConfig(PretrainedConfig):
    base_model_tp_plan = {
        "model.layers.*.self_attn.q_proj": "colwise",
        "model.layers.*.self_attn.k_proj": "colwise",
        "model.layers.*.self_attn.v_proj": "colwise",
        "model.layers.*.self_attn.o_proj": "rowwise",
        "model.layers.*.mlp.gate_proj": "colwise",
        "model.layers.*.mlp.up_proj": "colwise",
        "model.layers.*.mlp.down_proj": "rowwise",
    }