frozenleaves/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-10-20 17:13:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Double router compute? (#41653)

Browse Source 
* weird double router compute?

* flip it
This commit is contained in:
Pablo Montalvo
2025-10-16 15:17:21 +02:00
committed by GitHub
parent 503c933f36
commit 9176af574a
2 changed files with 0 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/transformers/models/deepseek_v2/modeling_deepseek_v2.py
View File

@ -117,7 +117,6 @@ class DeepseekV2Moe(nn.Module):
residuals = hidden_states residuals = hidden_states
orig_shape = hidden_states.shape orig_shape = hidden_states.shape
router_logits = nn.functional.linear(hidden_states.type(torch.float32), self.gate.weight.type(torch.float32)) router_logits = nn.functional.linear(hidden_states.type(torch.float32), self.gate.weight.type(torch.float32))
router_logits = self.gate(hidden_states)
topk_indices, topk_weights = self.route_tokens_to_experts(router_logits) topk_indices, topk_weights = self.route_tokens_to_experts(router_logits)
hidden_states = hidden_states.view(-1, hidden_states.shape[-1]) hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
hidden_states = self.experts(hidden_states, topk_indices, topk_weights).view(*orig_shape) hidden_states = self.experts(hidden_states, topk_indices, topk_weights).view(*orig_shape)

1
src/transformers/models/deepseek_v2/modular_deepseek_v2.py
View File

@ -275,7 +275,6 @@ class DeepseekV2Moe(nn.Module):
residuals = hidden_states residuals = hidden_states
orig_shape = hidden_states.shape orig_shape = hidden_states.shape
router_logits = nn.functional.linear(hidden_states.type(torch.float32), self.gate.weight.type(torch.float32)) router_logits = nn.functional.linear(hidden_states.type(torch.float32), self.gate.weight.type(torch.float32))
router_logits = self.gate(hidden_states)
topk_indices, topk_weights = self.route_tokens_to_experts(router_logits) topk_indices, topk_weights = self.route_tokens_to_experts(router_logits)
hidden_states = hidden_states.view(-1, hidden_states.shape[-1]) hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
hidden_states = self.experts(hidden_states, topk_indices, topk_weights).view(*orig_shape) hidden_states = self.experts(hidden_states, topk_indices, topk_weights).view(*orig_shape)