frozenleaves/vllm
1
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2025-10-20 14:53:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Bugfix] Correct LayerNorm epsilon parameter in modernbert.py (#27008)

Browse Source 
Signed-off-by: bogdanm <152898065+bogdan01m@users.noreply.github.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
bogdanm
2025-10-16 17:27:44 +05:00
committed by GitHub
parent 00417f4e44
commit dcbb3f1871
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
vllm/model_executor/models/modernbert.py
View File

@ -39,9 +39,12 @@ class ModernBertEmbeddings(nn.Module):
self.tok_embeddings = VocabParallelEmbedding(
config.vocab_size, config.hidden_size
)
self.norm = nn.LayerNorm(
config.hidden_size, eps=config.layer_norm_eps, bias=config.norm_bias
eps = (
getattr(config, "norm_eps", None)
or getattr(config, "layer_norm_eps", None)
or 1e-5
)
self.norm = nn.LayerNorm(config.hidden_size, eps=eps, bias=config.norm_bias)
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.tok_embeddings(input_ids)