Skip to content

vllm.platforms.hpu

logger module-attribute 

logger = init_logger(__name__)

HpuPlatform

Bases: Platform

Source code in vllm/platforms/hpu.py 
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
class HpuPlatform(Platform):
    _enum = PlatformEnum.HPU
    device_name: str = "hpu"
    device_type: str = "hpu"
    dispatch_key: str = "HPU"
    ray_device_key: str = "HPU"
    dist_backend: str = "hccl"
    device_control_env_var: str = "HABANA_VISIBLE_MODULES"

    @classmethod
    def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
                             dtype: torch.dtype, kv_cache_dtype: Optional[str],
                             block_size: int, use_v1: bool,
                             use_mla: bool) -> str:
        logger.info("Using HPUAttention backend.")
        return "vllm.attention.backends.hpu_attn.HPUAttentionBackend"

    @classmethod
    def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
        return True

    @classmethod
    def inference_mode(cls):
        return torch.no_grad()

    @classmethod
    def set_device(cls, device: torch.device) -> None:
        """
        Set the device for the current platform.
        """
        torch.hpu.set_device(device)

    @classmethod
    def check_and_update_config(cls, vllm_config: VllmConfig) -> None:

        scheduler_config = vllm_config.scheduler_config
        parallel_config = vllm_config.parallel_config
        if scheduler_config.is_multi_step:
            parallel_config.worker_cls = \
                "vllm.worker.multi_step_hpu_worker.MultiStepHPUWorker"

        if vllm_config.speculative_config is not None:
            raise NotImplementedError(
                "Speculative decoding is not implemented for HPU")

        if parallel_config.worker_cls == "auto":
            parallel_config.worker_cls = "vllm.worker.hpu_worker.HPUWorker"

        # NOTE(kzawora): default block size for Gaudi should be 128
        # smaller sizes still work, but very inefficiently
        cache_config = vllm_config.cache_config
        if cache_config and cache_config.block_size is None:
            cache_config.block_size = 128
        if (parallel_config.distributed_executor_backend == 'mp'
                and envs.VLLM_WORKER_MULTIPROC_METHOD == 'fork'):
            if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD",
                              None) is not None:
                logger.warning("On HPU, VLLM_WORKER_MULTIPROC_METHOD=fork "
                               "might cause application hangs on exit. Using "
                               "VLLM_WORKER_MULTIPROC_METHOD=fork anyway, "
                               "as it was explicitly requested.")
            else:
                logger.warning(
                    "On HPU, VLLM_WORKER_MULTIPROC_METHOD=fork "
                    "might cause application hangs on exit. Setting "
                    "VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. "
                    "To override that behavior, please set "
                    "VLLM_WORKER_MULTIPROC_METHOD=fork explicitly.")
                os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"

        if vllm_config.model_config and vllm_config.model_config.use_mla:
            logger.info(
                "MLA is enabled on a non-GPU platform; forcing chunked "
                "prefill and prefix caching to be disabled.")
            vllm_config.scheduler_config.enable_chunked_prefill = False
            vllm_config.scheduler_config.chunked_prefill_enabled = False
            vllm_config.scheduler_config.max_num_batched_tokens = max(
                vllm_config.scheduler_config.max_model_len,
                DEFAULT_MAX_NUM_BATCHED_TOKENS)

    @classmethod
    def is_pin_memory_available(cls):
        logger.warning("Pin memory is not supported on HPU.")
        return False

    @classmethod
    def get_punica_wrapper(cls) -> str:
        return "vllm.lora.punica_wrapper.punica_hpu.PunicaWrapperHPU"

    @classmethod
    def get_device_communicator_cls(cls) -> str:
        return "vllm.distributed.device_communicators.hpu_communicator.HpuCommunicator"  # noqa

_enum class-attribute instance-attribute 

_enum = HPU

device_control_env_var class-attribute instance-attribute 

device_control_env_var: str = 'HABANA_VISIBLE_MODULES'

device_name class-attribute instance-attribute 

device_name: str = 'hpu'

device_type class-attribute instance-attribute 

device_type: str = 'hpu'

dispatch_key class-attribute instance-attribute 

dispatch_key: str = 'HPU'

dist_backend class-attribute instance-attribute 

dist_backend: str = 'hccl'

ray_device_key class-attribute instance-attribute 

ray_device_key: str = 'HPU'

check_and_update_config classmethod 

check_and_update_config(vllm_config: VllmConfig) -> None
Source code in vllm/platforms/hpu.py 
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
@classmethod
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:

    scheduler_config = vllm_config.scheduler_config
    parallel_config = vllm_config.parallel_config
    if scheduler_config.is_multi_step:
        parallel_config.worker_cls = \
            "vllm.worker.multi_step_hpu_worker.MultiStepHPUWorker"

    if vllm_config.speculative_config is not None:
        raise NotImplementedError(
            "Speculative decoding is not implemented for HPU")

    if parallel_config.worker_cls == "auto":
        parallel_config.worker_cls = "vllm.worker.hpu_worker.HPUWorker"

    # NOTE(kzawora): default block size for Gaudi should be 128
    # smaller sizes still work, but very inefficiently
    cache_config = vllm_config.cache_config
    if cache_config and cache_config.block_size is None:
        cache_config.block_size = 128
    if (parallel_config.distributed_executor_backend == 'mp'
            and envs.VLLM_WORKER_MULTIPROC_METHOD == 'fork'):
        if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD",
                          None) is not None:
            logger.warning("On HPU, VLLM_WORKER_MULTIPROC_METHOD=fork "
                           "might cause application hangs on exit. Using "
                           "VLLM_WORKER_MULTIPROC_METHOD=fork anyway, "
                           "as it was explicitly requested.")
        else:
            logger.warning(
                "On HPU, VLLM_WORKER_MULTIPROC_METHOD=fork "
                "might cause application hangs on exit. Setting "
                "VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. "
                "To override that behavior, please set "
                "VLLM_WORKER_MULTIPROC_METHOD=fork explicitly.")
            os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"

    if vllm_config.model_config and vllm_config.model_config.use_mla:
        logger.info(
            "MLA is enabled on a non-GPU platform; forcing chunked "
            "prefill and prefix caching to be disabled.")
        vllm_config.scheduler_config.enable_chunked_prefill = False
        vllm_config.scheduler_config.chunked_prefill_enabled = False
        vllm_config.scheduler_config.max_num_batched_tokens = max(
            vllm_config.scheduler_config.max_model_len,
            DEFAULT_MAX_NUM_BATCHED_TOKENS)

get_attn_backend_cls classmethod 

get_attn_backend_cls(
    selected_backend: _Backend,
    head_size: int,
    dtype: dtype,
    kv_cache_dtype: Optional[str],
    block_size: int,
    use_v1: bool,
    use_mla: bool,
) -> str
Source code in vllm/platforms/hpu.py 
32
33
34
35
36
37
38
@classmethod
def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
                         dtype: torch.dtype, kv_cache_dtype: Optional[str],
                         block_size: int, use_v1: bool,
                         use_mla: bool) -> str:
    logger.info("Using HPUAttention backend.")
    return "vllm.attention.backends.hpu_attn.HPUAttentionBackend"

get_device_communicator_cls classmethod 

get_device_communicator_cls() -> str
Source code in vllm/platforms/hpu.py 
112
113
114
@classmethod
def get_device_communicator_cls(cls) -> str:
    return "vllm.distributed.device_communicators.hpu_communicator.HpuCommunicator"  # noqa

get_punica_wrapper classmethod 

get_punica_wrapper() -> str
Source code in vllm/platforms/hpu.py 
108
109
110
@classmethod
def get_punica_wrapper(cls) -> str:
    return "vllm.lora.punica_wrapper.punica_hpu.PunicaWrapperHPU"

inference_mode classmethod 

inference_mode()
Source code in vllm/platforms/hpu.py 
44
45
46
@classmethod
def inference_mode(cls):
    return torch.no_grad()

is_async_output_supported classmethod 

is_async_output_supported(
    enforce_eager: Optional[bool],
) -> bool
Source code in vllm/platforms/hpu.py 
40
41
42
@classmethod
def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
    return True

is_pin_memory_available classmethod 

is_pin_memory_available()
Source code in vllm/platforms/hpu.py 
103
104
105
106
@classmethod
def is_pin_memory_available(cls):
    logger.warning("Pin memory is not supported on HPU.")
    return False

set_device classmethod 

set_device(device: device) -> None

Set the device for the current platform.

Source code in vllm/platforms/hpu.py 
48
49
50
51
52
53
@classmethod
def set_device(cls, device: torch.device) -> None:
    """
    Set the device for the current platform.
    """
    torch.hpu.set_device(device)