Last active
June 13, 2025 12:03
-
-
Save kiya00/a86590e305ab63e3fb2964b1db9a7bb1 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 1. build docker on my server | |
| 2004 git clone https://github.com/nv-auto-deploy/TensorRT-LLM | |
| 2005 git checkout user/sg/trtllm-bench-fix | |
| 2006 cd TensorRT-LLM/ | |
| 2007 git checkout user/sg/trtllm-bench-fix | |
| 2008 git branch | |
| 2009 docker build --pull --target devel --file docker/Dockerfile.multi --tag tensorrt_llm/devel:latest . | |
| 2010 docker images | |
| 2011 docker save -o tensorrt_llm_latest tensorrt_llm/devel:latest | |
| 2. scp to H100 | |
| srun -A pjnl -p viking-prod-pjnl --pty -t 8:00:00 /bin/bash (viking-prod-229) | |
| 505 scp [email protected]:/home/wayan/nv_autodeploy/TensorRT-LLM/tensorrt_llm_latest tensorrt_llm_latest.tar | |
| 509 docker load -i tensorrt_llm_latest.tar | |
| 510 docker images -a | |
| 511 docker run --rm -it --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 --gpus=all --workdir /app/tensorrt_llm --tmpfs /tmp:exec tensorrt_llm/devel | |
| 3. inside docker | |
| 1 git clone https://github.com/nv-auto-deploy/TensorRT-LLM | |
| 2 git checkout user/sg/trtllm-bench-fix | |
| 3 cd TensorRT-LLM/ | |
| 4 git checkout user/sg/trtllm-bench-fix | |
| 5 git branch | |
| 6 python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt --cuda_architectures "90-real" --clean | |
| 7 pip install -e . | |
| 8 huggingface-cli login | |
| 9 git config --global credential.helper store | |
| 10 export MODEL_ID="meta-llama/Llama-3.1-8B" | |
| 13 cd TensorRT-LLM/ | |
| 14 python benchmarks/cpp/prepare_dataset.py --stdout --tokenizer $MODEL_ID token-norm-dist --input-mean 128 --output-mean 128 --input-stdev 0 --output-stdev 0 --num-requests 3000 > /tmp/synthetic_128_128.txt | |
| 15 trtllm-bench --model $MODEL_ID throughput --dataset /tmp/synthetic_128_128.txt --backend autodeploy | |
| 16 cd .. | |
| 17 ls | |
| 18 cd TensorRT-LLM/ | |
| 19 trtllm-bench --model $MODEL_ID throughput --dataset /tmp/synthetic_128_128.txt --backend _autodeploy | |
| 20 git status -uno | |
| 21 git branch | |
| 22 history | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment