frozenleaves/vllm
1
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2025-10-20 14:53:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

[model] make llama4 compatible with pure dense layers (#17315)

Browse Source 
Signed-off-by: Lucia Fang <fanglu@fb.com>
This commit is contained in:
Lucia Fang
2025-04-28 19:22:22 -07:00
committed by GitHub
parent e136000595
commit b4ac4fa04d
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/model_executor/models/llama4.py
View File

@ -273,8 +273,8 @@ class Llama4DecoderLayer(nn.Module):
cache_config=cache_config,
prefix=f"{prefix}.self_attn",
)
is_moe_layer = (self.layer_idx +
1) % config.interleave_moe_layer_step == 0
is_moe_layer = config.interleave_moe_layer_step > 0 and (
self.layer_idx + 1) % config.interleave_moe_layer_step == 0
if is_moe_layer:
self.feed_forward = Llama4MoE(
config=config,