我们现在使用的图片token数量远没有达到32768个，如何能够降低这个数量
1、环境
将模型部署在四卡4090上，启动参数为
--limit-mm-per-prompt image=2,video=0 --tensor-parallel-size 4 --gpu_memory_utilization 0.95 --max-model-len 32768 --enable-prefix-caching --max-num-batched-tokens 32768
--enable-chunked-prefill --trust-remote-code --max-num-seqs 1 --enforce-eager
2、当我把--max-num-seqs增大为1以上的时候，一旦并发会出现报错：
ValueError: Attempted to assign 1230 + 1230 = 2460 multimodal tokens to 0 placeholders
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] Exception in worker VllmWorkerProcess while processing method start_worker_execution_loop.
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] Traceback (most recent call last):
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/executor/multiproc_worker_utils.py", line 236, in _run_worker_process
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] output = run_method(worker, method, args, kwargs)
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/utils.py", line 2220, in run_method
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] return func(*args, **kwargs)
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/worker/worker_base.py", line 93, in start_worker_execution_loop
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] output = self.execute_model(execute_model_req=None)
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/worker/worker_base.py", line 413, in execute_model
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] output = self.model_runner.execute_model(
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] return func(*args, **kwargs)
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/worker/model_runner.py", line 1719, in execute_model
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] hidden_or_intermediate_states = model_executable(
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] return self._call_impl(*args, **kwargs)
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] return forward_call(*args, **kwargs)
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/qwen2_vl.py", line 1347, in forward
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] inputs_embeds = self.get_input_embeddings_v0(
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/qwen2_vl.py", line 1283, in get_input_embeddings_v0
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] inputs_embeds = merge_multimodal_embeddings(
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py", line 455, in merge_multimodal_embeddings
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] return _merge_multimodal_embeddings(
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py", line 371, in _merge_multimodal_embeddings
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] raise ValueError(
(VllmWorkerProcess pid=5911) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ValueError: Attempted to assign 1230 + 1230 = 2460 multimodal tokens to 0 placeholders
CRITICAL 03-03 19:17:58 launcher.py:101] MQLLMEngine is already dead, terminating server process
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] Exception in worker VllmWorkerProcess while processing method start_worker_execution_loop.
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] Traceback (most recent call last):
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/executor/multiproc_worker_utils.py", line 236, in _run_worker_process
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] output = run_method(worker, method, args, kwargs)
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/utils.py", line 2220, in run_method
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] return func(*args, **kwargs)
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/worker/worker_base.py", line 93, in start_worker_execution_loop
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] output = self.execute_model(execute_model_req=None)
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/worker/worker_base.py", line 413, in execute_model
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] output = self.model_runner.execute_model(
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] return func(*args, **kwargs)
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/worker/model_runner.py", line 1719, in execute_model
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] hidden_or_intermediate_states = model_executable(
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] return self._call_impl(*args, **kwargs)
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1747, in _call_impl
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] return forward_call(*args, **kwargs)
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/qwen2_vl.py", line 1347, in forward
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] inputs_embeds = self.get_input_embeddings_v0(
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/qwen2_vl.py", line 1283, in get_input_embeddings_v0
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] inputs_embeds = merge_multimodal_embeddings(
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py", line 455, in merge_multimodal_embeddings
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] return _merge_multimodal_embeddings(
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py", line 371, in _merge_multimodal_embeddings
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] raise ValueError(
(VllmWorkerProcess pid=5910) ERROR 03-03 19:17:58 multiproc_worker_utils.py:242] ValueError: Attempted to assign 1230 + 1230 = 2460 multimodal tokens to 0 placeholders
INFO: 14.104.129.40:8374 - "POST /v1/chat/completions HTTP/1.1" 500 Internal Server Error
INFO: Shutting down
INFO: Waiting for application shutdown.
INFO: Application shutdown complete.
INFO: Finished server process [5560]
Process SpawnProcess-1:
Traceback (most recent call last):
File "/usr/lib/python3.12/multiprocessing/process.py", line 317, in _bootstrap
util._exit_function()
File "/usr/lib/python3.12/multiprocessing/util.py", line 360, in _exit_function
p.join()
File "/usr/lib/python3.12/multiprocessing/process.py", line 149, in join
res = self._popen.wait(timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/multiprocessing/popen_fork.py", line 43, in wait
return self.poll(os.WNOHANG if timeout == 0.0 else 0)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/multiprocessing/popen_fork.py", line 27, in poll
pid, sts = os.waitpid(self.pid, flag)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/engine.py", line 374, in signal_handler
raise KeyboardInterrupt("MQLLMEngine terminated")
所以这是因为显存不够了？