SGLANG_URL := "http://localhost:8000"
MODEL := "Qwen/Qwen3-0.6B"

run:
    docker run \
        --gpus=all \
        --shm-size=32g \
        --volume="$HOME/.cache/huggingface:/root/.cache/huggingface" \
        --publish=0.0.0.0:8000:8000 \
        --ipc=host \
        docker.io/lmsysorg/sglang:v0.5.6.post2 \
            python3 -m sglang.launch_server \
                --host=0.0.0.0 \
                --port=8000 \
                --tp-size=2 \
                --mem-fraction-static=0.9 \
                --model-path={{ MODEL }} \
                --max-running-requests=128 \
                --context-length=8192 \
                --attention-backend=flashinfer

query:
    curl {{ SGLANG_URL }}/v1/chat/completions \
        --header "Content-Type: application/json" \
        --data '{ \
            "model": "{{ MODEL }}", \
            "messages": [ \
                {"role": "system", "content": "You are a helpful assistant."}, \
                {"role": "user", "content": "What is gravity?"} \
            ] \
        }'
