解决方法

经查资料，我的系统是ubuntu18.04,不支持最新的qwen2-vl模型，似乎至少要20.04，于是重装系统升级到24.04解决了.注意vllm也需要最新版

启动命令如下

sudo docker run -it \
      --gpus '"device=0,1"' \
-v $HOME/.cache/huggingface:/root/.cache/huggingface \
      -e https_proxy=http://192.168.1.213:7890 \
     -p 11434:8000 \
     --ipc=host \
     vllm/vllm-openai:v0.6.4.post1 \
     --model Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int4  --gpu_memory_utilization 0.95

报错内容如下

OpenBLAS blas_thread_init: RLIMIT_NPROC -1 current, -1 max
INFO 11-24 23:37:42 config.py:350] This model supports multiple tasks: {'generate', 'embedding'}. Defaulting to 'generate'.
WARNING 11-24 23:37:42 arg_utils.py:1075] [DEPRECATED] Block manager v1 has been removed, and setting --use-v2-block-manager to True or False has no effect on vLLM behavior. Please remove --use-v2-block-manager in your engine argument. If your use case is not supported by SelfAttnBlockSpaceManager (i.e. block manager v2), please file an issue with detailed information.
Traceback (most recent call last):
  File "/usr/lib/python3.12/asyncio/runners.py", line 194, in run
    return runner.run(main)
           ^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
    return self._loop.run_until_complete(task)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
  File "/usr/local/lib/python3.12/dist-packages/uvloop/__init__.py", line 61, in wrapper
    return await main
           ^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/openai/api_server.py", line 609, in run_server
    async with build_async_engine_client(args) as engine_client:
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/contextlib.py", line 210, in __aenter__
    return await anext(self.gen)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/openai/api_server.py", line 113, in build_async_engine_client
    async with build_async_engine_client_from_engine_args(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/contextlib.py", line 210, in __aenter__
    return await anext(self.gen)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/openai/api_server.py", line 200, in build_async_engine_client_from_engine_args
    mq_engine_client = await asyncio.get_running_loop().run_in_executor(
                             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "uvloop/loop.pyx", line 2747, in uvloop.loop.Loop.run_in_executor
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 179, in submit
    self._adjust_thread_count()
  File "/usr/lib/python3.12/concurrent/futures/thread.py", line 202, in _adjust_thread_count
    t.start()
  File "/usr/lib/python3.12/threading.py", line 994, in start
    _start_new_thread(self._bootstrap, ())
RuntimeError: can't start new thread

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/openai/api_server.py", line 643, in <module>
    uvloop.run(run_server(args))
  File "/usr/local/lib/python3.12/dist-packages/uvloop/__init__.py", line 109, in run
    return __asyncio.run(
           ^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/asyncio/runners.py", line 193, in run
    with Runner(debug=debug, loop_factory=loop_factory) as runner:
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.12/asyncio/runners.py", line 62, in __exit__
    self.close()
  File "/usr/lib/python3.12/asyncio/runners.py", line 72, in close
    loop.run_until_complete(
  File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
  File "uvloop/loop.pyx", line 3236, in shutdown_default_executor
  File "/usr/lib/python3.12/threading.py", line 994, in start
    _start_new_thread(self._bootstrap, ())
RuntimeError: can't start new thread
INFO 11-24 23:37:47 config.py:350] This model supports multiple tasks: {'generate', 'embedding'}. Defaulting to 'generate'.
WARNING 11-24 23:37:47 arg_utils.py:1075] [DEPRECATED] Block manager v1 has been removed, and setting --use-v2-block-manager to True or False has no effect on vLLM behavior. Please remove --use-v2-block-manager in your engine argument. If your use case is not supported by SelfAttnBlockSpaceManager (i.e. block manager v2), please file an issue with detailed information.
INFO 11-24 23:37:47 llm_engine.py:249] Initializing an LLM engine (v0.6.4.post1) with config: model='shibing624/chinese-text-correction-7b', speculative_config=None, tokenizer='shibing624/chinese-text-correction-7b', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=shibing624/chinese-text-correction-7b, num_scheduler_steps=1, chunked_prefill_enabled=False multi_step_stream_outputs=True, enable_prefix_caching=False, use_async_output_proc=True, use_cached_outputs=True, chat_template_text_format=string, mm_processor_kwargs=None, pooler_config=None)
INFO 11-24 23:37:48 selector.py:135] Using Flash Attention backend.
ERROR 11-24 23:37:48 engine.py:366] Unexpected error from cudaGetDeviceCount(). Did you run some cuda functions before calling NumCudaDevices() that might have already set an error? Error 304: OS call failed or operation not supported on this OS
ERROR 11-24 23:37:48 engine.py:366] Traceback (most recent call last):
ERROR 11-24 23:37:48 engine.py:366]   File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/engine.py", line 357, in run_mp_engine
ERROR 11-24 23:37:48 engine.py:366]     engine = MQLLMEngine.from_engine_args(engine_args=engine_args,
ERROR 11-24 23:37:48 engine.py:366]              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 11-24 23:37:48 engine.py:366]   File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/engine.py", line 119, in from_engine_args
ERROR 11-24 23:37:48 engine.py:366]     return cls(ipc_path=ipc_path,
ERROR 11-24 23:37:48 engine.py:366]            ^^^^^^^^^^^^^^^^^^^^^^
ERROR 11-24 23:37:48 engine.py:366]   File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/engine.py", line 71, in __init__
ERROR 11-24 23:37:48 engine.py:366]     self.engine = LLMEngine(*args, **kwargs)
ERROR 11-24 23:37:48 engine.py:366]                   ^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 11-24 23:37:48 engine.py:366]   File "/usr/local/lib/python3.12/dist-packages/vllm/engine/llm_engine.py", line 347, in __init__
ERROR 11-24 23:37:48 engine.py:366]     self.model_executor = executor_class(vllm_config=vllm_config, )
ERROR 11-24 23:37:48 engine.py:366]                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 11-24 23:37:48 engine.py:366]   File "/usr/local/lib/python3.12/dist-packages/vllm/executor/executor_base.py", line 36, in __init__
ERROR 11-24 23:37:48 engine.py:366]     self._init_executor()
ERROR 11-24 23:37:48 engine.py:366]   File "/usr/local/lib/python3.12/dist-packages/vllm/executor/gpu_executor.py", line 39, in _init_executor
ERROR 11-24 23:37:48 engine.py:366]     self.driver_worker.init_device()
ERROR 11-24 23:37:48 engine.py:366]   File "/usr/local/lib/python3.12/dist-packages/vllm/worker/worker.py", line 135, in init_device
ERROR 11-24 23:37:48 engine.py:366]     torch.cuda.set_device(self.device)
ERROR 11-24 23:37:48 engine.py:366]   File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 478, in set_device
ERROR 11-24 23:37:48 engine.py:366]     torch._C._cuda_setDevice(device)
ERROR 11-24 23:37:48 engine.py:366]   File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 319, in _lazy_init
ERROR 11-24 23:37:48 engine.py:366]     torch._C._cuda_init()
ERROR 11-24 23:37:48 engine.py:366] RuntimeError: Unexpected error from cudaGetDeviceCount(). Did you run some cuda functions before calling NumCudaDevices() that might have already set an error? Error 304: OS call failed or operation not supported on this OS
Process SpawnProcess-1:
Traceback (most recent call last):
  File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/engine.py", line 368, in run_mp_engine
    raise e
  File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/engine.py", line 357, in run_mp_engine
    engine = MQLLMEngine.from_engine_args(engine_args=engine_args,
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/engine.py", line 119, in from_engine_args
    return cls(ipc_path=ipc_path,
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/vllm/engine/multiprocessing/engine.py", line 71, in __init__
    self.engine = LLMEngine(*args, **kwargs)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/vllm/engine/llm_engine.py", line 347, in __init__
    self.model_executor = executor_class(vllm_config=vllm_config, )
                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/dist-packages/vllm/executor/executor_base.py", line 36, in __init__
    self._init_executor()
  File "/usr/local/lib/python3.12/dist-packages/vllm/executor/gpu_executor.py", line 39, in _init_executor
    self.driver_worker.init_device()
  File "/usr/local/lib/python3.12/dist-packages/vllm/worker/worker.py", line 135, in init_device
    torch.cuda.set_device(self.device)
  File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 478, in set_device
    torch._C._cuda_setDevice(device)
  File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 319, in _lazy_init
    torch._C._cuda_init()
RuntimeError: Unexpected error from cudaGetDeviceCount(). Did you run some cuda functions before calling NumCudaDevices() that might have already set an error? Error 304: OS call failed or operation not supported on this OS