frozenleaves/vllm
1
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2025-10-20 14:53:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Fix dynamic FP8 quantization for Mixtral (#4793)

Browse Source
This commit is contained in:
Philipp Moritz
2024-05-13 16:00:27 -07:00
committed by GitHub
parent 1356df53bd
commit 33d3914b1e
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
vllm/model_executor/models/mixtral.py
View File

@ -95,7 +95,7 @@ class MixtralMoE(nn.Module):
params_dtype=self.params_dtype,
quant_config=None)
if self.use_fp8:
if self.use_fp8 and self.quant_config.is_checkpoint_fp8_serialized:
params_dtype = torch.float8_e4m3fn
self.w13_weight = nn.Parameter(