rlightning.utils.inference_engine.vllm_engine

class rlightning.utils.inference_engine.vllm_engine.VLLMEngineAsync(*args, **kwargs)

Bases: object

async generate_async(queries, sampling_params, request_id=None)

get_hidden_size()

get_tokenizer()

rlightning.utils.inference_engine.vllm_engine.create_vllm_engine(model_config, rollout_mode='sync')

class rlightning.utils.inference_engine.vllm_engine.vLLMEngine(*args, **kwargs)

Bases: object

generate(queries, sampling_params)

Process requests from rank0 and generate responses. Since only rank0 will send requests, we don’t need to track actor ranks.

get_hidden_size()

get_tokenizer()

init_process_group(master_address, master_port, rank_offset, world_size, group_name, backend, use_ray)

reset_prefix_cache()

sleep(level=1)

update_weight(name, dtype, shape, empty_cache=False)

update_weight_cuda_ipc(name, dtype, shape, ipc_handles, empty_cache=False)

wake_up()