Update myr1/modeling_deepseek.py

myr1/modeling_deepseek.py

@@ -54,17 +54,19 @@ logger = logging.get_logger(__name__)
 
 # If flash-attn is available
 if is_flash_attn_2_available():
-    from flash_attn import flash_attn_func, flash_attn_varlen_func, flash_attn_with_paged_kv
+    from flash_attn import flash_attn_func, flash_attn_varlen_func
     from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input  # noqa
 
 # This helps make `_prepare_4d_causal_attention_mask` a leaf function in the FX graph.
 if is_torch_fx_available():
     if not is_torch_greater_or_equal_than_1_13:
         import torch.fx
+
     _prepare_4d_causal_attention_mask = torch.fx.wrap(_prepare_4d_causal_attention_mask)
 
 _CONFIG_FOR_DOC = "DeepseekV3Config"
 
+
 # ==============================================================================
 # Rotary Embedding Helpers
 # ==============================================================================
@@ -80,9 +82,11 @@ def _get_unpad_data(attention_mask: torch.Tensor) -> Tuple[torch.Tensor, torch.T
     seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
     indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
     max_seqlen_in_batch = seqlens_in_batch.max().item()
+    # Build prefix sums
     cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0))
     return indices, cu_seqlens, max_seqlen_in_batch
 
+
 # ==============================================================================
 # Normalization Layers
 # ==============================================================================
@@ -98,13 +102,16 @@ class DeepseekV3RMSNorm(nn.Module):
         self.variance_epsilon = eps
 
     def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        # IMPROVEMENT: Provide type-safety & potential in-place usage
         input_dtype = hidden_states.dtype
         variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)
         hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
         return (self.weight * hidden_states).to(input_dtype)
 
+
 ALL_LAYERNORM_LAYERS.append(DeepseekV3RMSNorm)
 
+
 # ==============================================================================
 # Rotary Embeddings
 # ==============================================================================
@@ -125,20 +132,25 @@ class DeepseekV3RotaryEmbedding(nn.Module):
         self.max_position_embeddings = max_position_embeddings
         self.base = base
 
-        inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))
+        inv_freq = 1.0 / (
+            self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)
+        )
         self.register_buffer("inv_freq", inv_freq, persistent=False)
-        # Build here to make torch.jit.trace work.
+
+        # Build here to make `torch.jit.trace` work.
         self._set_cos_sin_cache(
             seq_len=max_position_embeddings,
             device=self.inv_freq.device,
             dtype=torch.get_default_dtype(),
         )
-        self.max_seq_len_cached = max_position_embeddings
+        self.max_seq_len_cached = None
 
     def _set_cos_sin_cache(self, seq_len: int, device: torch.device, dtype: torch.dtype):
         self.max_seq_len_cached = seq_len
-        t = torch.arange(seq_len, device=device, dtype=self.inv_freq.dtype)
-        freqs = torch.outer(t, self.inv_freq.to(device))
+        t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)
+
+        freqs = torch.outer(t, self.inv_freq.to(t.device))
+        # Different from paper, but uses a different permutation to achieve the same effect
         emb = torch.cat((freqs, freqs), dim=-1)
         self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
         self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
@@ -147,17 +159,16 @@ class DeepseekV3RotaryEmbedding(nn.Module):
         """
         x: [batch_size, num_heads, seq_len, head_size]
         """
-        if seq_len is None:
-            seq_len = x.shape[-2]
-        if seq_len > self.max_seq_len_cached:
-            self._set_cos_sin_cache(seq_len, device=x.device, dtype=x.dtype)
-        return (self.cos_cached[:seq_len].to(x.dtype),
-                self.sin_cached[:seq_len].to(x.dtype))
+        if (self.max_seq_len_cached is None) or (seq_len and seq_len > self.max_seq_len_cached):
+            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype)
+
+        return (self.cos_cached[:seq_len].to(dtype=x.dtype),
+                self.sin_cached[:seq_len].to(dtype=x.dtype))
 
 
 class DeepseekV3LinearScalingRotaryEmbedding(DeepseekV3RotaryEmbedding):
     """
-    RoPE extended with linear scaling. Credits to the Reddit user /u/kaiokendev.
+    RoPE extended with linear scaling. Credits to the Reddit user /u/kaiokendev
     """
     def __init__(
         self,
@@ -172,7 +183,8 @@ class DeepseekV3LinearScalingRotaryEmbedding(DeepseekV3RotaryEmbedding):
 
     def _set_cos_sin_cache(self, seq_len: int, device: torch.device, dtype: torch.dtype):
         self.max_seq_len_cached = seq_len
-        t = torch.arange(seq_len, device=device, dtype=self.inv_freq.dtype) / self.scaling_factor
+        t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)
+        t = t / self.scaling_factor
         freqs = torch.outer(t, self.inv_freq)
         emb = torch.cat((freqs, freqs), dim=-1)
         self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
@@ -182,7 +194,7 @@ class DeepseekV3LinearScalingRotaryEmbedding(DeepseekV3RotaryEmbedding):
 class DeepseekV3DynamicNTKScalingRotaryEmbedding(DeepseekV3RotaryEmbedding):
     """
     RoPE extended with Dynamic NTK scaling.
-    Credits to the Reddit users /u/bloc97 and /u/emozilla.
+    Credits to the Reddit users /u/bloc97 and /u/emozilla
     """
     def __init__(
         self,
@@ -197,28 +209,34 @@ class DeepseekV3DynamicNTKScalingRotaryEmbedding(DeepseekV3RotaryEmbedding):
 
     def _set_cos_sin_cache(self, seq_len: int, device: torch.device, dtype: torch.dtype):
         self.max_seq_len_cached = seq_len
+
         if seq_len > self.max_position_embeddings:
             base = self.base * (
                 (self.scaling_factor * seq_len / self.max_position_embeddings)
                 - (self.scaling_factor - 1)
             ) ** (self.dim / (self.dim - 2))
-            inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))
+            inv_freq = 1.0 / (
+                base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)
+            )
             self.register_buffer("inv_freq", inv_freq, persistent=False)
-        t = torch.arange(seq_len, device=device, dtype=self.inv_freq.dtype)
+
+        t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)
         freqs = torch.outer(t, self.inv_freq)
         emb = torch.cat((freqs, freqs), dim=-1)
         self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
         self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
 
 
-# Extra Yarn-based formulas from your original code
+# Extra Yarn-based formulas, as in your original code
 def yarn_find_correction_dim(
     num_rotations: float,
     dim: int,
     base: int = 10000,
     max_position_embeddings: int = 2048
 ):
-    return (dim * math.log(max_position_embeddings / (num_rotations * 2 * math.pi))) / (2 * math.log(base))
+    return (dim * math.log(max_position_embeddings / (num_rotations * 2 * math.pi))) / (
+        2 * math.log(base)
+    )
 
 
 def yarn_find_correction_range(
@@ -228,8 +246,13 @@ def yarn_find_correction_range(
     base: int = 10000,
     max_position_embeddings: int = 2048
 ):
-    low = math.floor(yarn_find_correction_dim(low_rot, dim, base, max_position_embeddings))
-    high = math.ceil(yarn_find_correction_dim(high_rot, dim, base, max_position_embeddings))
+    low = math.floor(
+        yarn_find_correction_dim(low_rot, dim, base, max_position_embeddings)
+    )
+    high = math.ceil(
+        yarn_find_correction_dim(high_rot, dim, base, max_position_embeddings)
+    )
+    # Clamped range
     return max(low, 0), min(high, dim - 1)
 
 
@@ -275,21 +298,39 @@ class DeepseekV3YarnRotaryEmbedding(DeepseekV3RotaryEmbedding):
     def _set_cos_sin_cache(self, seq_len: int, device: torch.device, dtype: torch.dtype):
         self.max_seq_len_cached = seq_len
         dim = self.dim
-        freq_extra = 1.0 / (self.base ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim))
-        freq_inter = 1.0 / (self.scaling_factor * self.base ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim))
-        low, high = yarn_find_correction_range(self.beta_fast, self.beta_slow, dim, self.base, self.original_max_position_embeddings)
+
+        freq_extra = 1.0 / (
+            self.base ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim)
+        )
+        freq_inter = 1.0 / (
+            self.scaling_factor
+            * self.base ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim)
+        )
+
+        low, high = yarn_find_correction_range(
+            self.beta_fast,
+            self.beta_slow,
+            dim,
+            self.base,
+            self.original_max_position_embeddings,
+        )
         inv_freq_mask = 1.0 - yarn_linear_ramp_mask(low, high, dim // 2).to(device=device, dtype=torch.float32)
         inv_freq = freq_inter * (1 - inv_freq_mask) + freq_extra * inv_freq_mask
         self.register_buffer("inv_freq", inv_freq, persistent=False)
+
         t = torch.arange(seq_len, device=device, dtype=torch.float32)
         freqs = torch.outer(t, inv_freq)
-        _mscale = float(yarn_get_mscale(self.scaling_factor, self.mscale) / yarn_get_mscale(self.scaling_factor, self.mscale_all_dim))
+        _mscale = float(
+            yarn_get_mscale(self.scaling_factor, self.mscale)
+            / yarn_get_mscale(self.scaling_factor, self.mscale_all_dim)
+        )
+
         emb = torch.cat((freqs, freqs), dim=-1)
         self.register_buffer("cos_cached", (emb.cos() * _mscale).to(dtype), persistent=False)
         self.register_buffer("sin_cached", (emb.sin() * _mscale).to(dtype), persistent=False)
 
 
-# ==============================================================================
+# ============================================================================== 
 # General Rotary helper functions
 # ==============================================================================
 
@@ -339,7 +380,9 @@ class DeepseekV3MLP(nn.Module):
         super().__init__()
         self.config = config
         self.hidden_size = config.hidden_size if hidden_size is None else hidden_size
-        self.intermediate_size = config.intermediate_size if intermediate_size is None else intermediate_size
+        self.intermediate_size = (
+            config.intermediate_size if intermediate_size is None else intermediate_size
+        )
 
         self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
         self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
@@ -370,9 +413,14 @@ class MoEGate(nn.Module):
         self.norm_topk_prob = config.norm_topk_prob
         self.gating_dim = config.hidden_size
 
+        # Gating weight
         self.weight = nn.Parameter(torch.empty((self.n_routed_experts, self.gating_dim)))
+
         if self.topk_method == "noaux_tc":
-            self.e_score_correction_bias = nn.Parameter(torch.empty((self.n_routed_experts)))
+            self.e_score_correction_bias = nn.Parameter(
+                torch.empty((self.n_routed_experts))
+            )
+
         self.reset_parameters()
 
     def reset_parameters(self):
@@ -385,29 +433,46 @@ class MoEGate(nn.Module):
         Compute gating scores and select top-k experts.
         """
         bsz, seq_len, h = hidden_states.shape
+
+        # 1) Compute gating scores
         logits = F.linear(hidden_states.float(), self.weight.float(), None)
         if self.scoring_func == "sigmoid":
             scores = logits.sigmoid()
         else:
-            raise NotImplementedError(f"Unsupported gating scoring function: {self.scoring_func}")
+            raise NotImplementedError(
+                f"Unsupported gating scoring function: {self.scoring_func}"
+            )
 
+        # 2) TopK selection
         if self.topk_method == "noaux_tc":
+            # This is a specialized approach from your original code
+            # IMPROVEMENT: Could consider generalizing to top2 gating or other advanced techniques
             scores_for_choice = scores.view(bsz * seq_len, -1) + self.e_score_correction_bias.unsqueeze(0)
-            group_scores = (scores_for_choice.view(bsz * seq_len, self.n_group, -1).topk(2, dim=-1)[0].sum(dim=-1))
-            group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]
+            group_scores = (
+                scores_for_choice.view(bsz * seq_len, self.n_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
+            )
+            group_idx = torch.topk(
+                group_scores, k=self.topk_group, dim=-1, sorted=False
+            )[1]  # [n, top_k_group]
             group_mask = torch.zeros_like(group_scores)
             group_mask.scatter_(1, group_idx, 1)
-            score_mask = group_mask.unsqueeze(-1).expand(bsz * seq_len, self.n_group, self.n_routed_experts // self.n_group).reshape(bsz * seq_len, -1)
+            score_mask = group_mask.unsqueeze(-1).expand(
+                bsz * seq_len, self.n_group, self.n_routed_experts // self.n_group
+            ).reshape(bsz * seq_len, -1)
             tmp_scores = scores_for_choice.masked_fill(~score_mask.bool(), 0.0)
             _, topk_idx = torch.topk(tmp_scores, k=self.top_k, dim=-1, sorted=False)
             topk_weight = scores_for_choice.gather(1, topk_idx)
         else:
-            raise NotImplementedError(f"Unsupported topk_method: {self.topk_method}")
+            raise NotImplementedError(
+                f"Unsupported topk_method: {self.topk_method}"
+            )
 
+        # 3) Norm gate to sum to 1 if top_k > 1
         if self.top_k > 1 and self.norm_topk_prob:
             denominator = topk_weight.sum(dim=-1, keepdim=True) + 1e-20
             topk_weight = topk_weight / denominator
 
+        # 4) Multiply scaling factor
         topk_weight = topk_weight * self.routed_scaling_factor
 
         return topk_idx, topk_weight
@@ -430,43 +495,62 @@ class DeepseekV3MoE(nn.Module):
             self.experts_per_rank = config.n_routed_experts // config.ep_size
             self.ep_rank = dist.get_rank()
 
+        # Build experts
         experts_list = []
         for i in range(config.n_routed_experts):
+            # only build if belongs to current rank
             if self.ep_size > 1:
                 if i >= self.ep_rank * self.experts_per_rank and i < (self.ep_rank + 1) * self.experts_per_rank:
-                    experts_list.append(DeepseekV3MLP(config, intermediate_size=config.moe_intermediate_size))
+                    experts_list.append(
+                        DeepseekV3MLP(config, intermediate_size=config.moe_intermediate_size)
+                    )
                 else:
                     experts_list.append(None)
             else:
-                experts_list.append(DeepseekV3MLP(config, intermediate_size=config.moe_intermediate_size))
+                experts_list.append(
+                    DeepseekV3MLP(config, intermediate_size=config.moe_intermediate_size)
+                )
         self.experts = nn.ModuleList(experts_list)
 
+        # Gate
         self.gate = MoEGate(config)
 
+        # Optionally shared experts
         if config.n_shared_experts is not None:
             intermediate_size = config.moe_intermediate_size * config.n_shared_experts
-            self.shared_experts = DeepseekV3MLP(config=config, intermediate_size=intermediate_size)
+            self.shared_experts = DeepseekV3MLP(
+                config=config, intermediate_size=intermediate_size
+            )
         else:
             self.shared_experts = None
 
     def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
         identity = hidden_states
         orig_shape = hidden_states.shape
+
         topk_idx, topk_weight = self.gate(hidden_states)
+        # Flatten
         hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
+
+        # Inference
         if not self.training:
             y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)
         else:
+            # For training, you’d typically do a distributed MoE approach
+            # or a specialized approach from your original code.
+            # This placeholder just calls `moe_infer` for demonstration.
             y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(*orig_shape)
+
+        # Add shared experts if present
         if self.shared_experts is not None:
             y = y + self.shared_experts(identity)
+
         return y
 
     @torch.no_grad()
     def moe_infer(self, x: torch.Tensor, topk_ids: torch.Tensor, topk_weight: torch.Tensor) -> torch.Tensor:
         """
-        MoE inference path for each token. Processes experts in parallel and combines
-        results via an efficient scatter-add.
+        MoE inference path for each token. This code can be parallelized or distributed for better performance.
         """
         cnts = topk_ids.new_zeros((topk_ids.shape[0], len(self.experts)))
         cnts.scatter_(1, topk_ids, 1)
@@ -475,16 +559,30 @@ class DeepseekV3MoE(nn.Module):
         sorted_tokens = x[idxs // topk_ids.shape[1]]
         sorted_tokens_shape = sorted_tokens.shape
 
+        # Handle distribution if ep_size>1
         if self.ep_size > 1:
             tokens_per_ep_rank = tokens_per_expert.view(self.ep_size, -1).sum(dim=1)
             tokens_per_expert_group = tokens_per_expert.new_empty(tokens_per_expert.shape[0])
             dist.all_to_all_single(tokens_per_expert_group, tokens_per_expert)
-            output_splits = tokens_per_expert_group.view(self.ep_size, self.experts_per_rank).sum(1).cpu().numpy().tolist()
-            gathered_tokens = sorted_tokens.new_empty(tokens_per_expert_group.sum(dim=0).cpu().item(), sorted_tokens.shape[1])
+            output_splits = (
+                tokens_per_expert_group.view(self.ep_size, self.experts_per_rank)
+                .sum(1)
+                .cpu()
+                .numpy()
+                .tolist()
+            )
+            gathered_tokens = sorted_tokens.new_empty(
+                tokens_per_expert_group.sum(dim=0).cpu().item(), sorted_tokens.shape[1]
+            )
             input_split_sizes = tokens_per_ep_rank.cpu().numpy().tolist()
-            dist.all_to_all(list(gathered_tokens.split(input_split_sizes)), list(sorted_tokens.split(input_split_sizes)))
-            tokens_per_expert_post_gather = tokens_per_expert_group.view(self.ep_size, self.experts_per_rank).sum(dim=0)
-            gatherd_idxs = np.zeros((gathered_tokens.shape[0],), dtype=np.int32)
+            dist.all_to_all(
+                list(gathered_tokens.split(input_split_sizes)),
+                list(sorted_tokens.split(input_split_sizes)),
+            )
+            tokens_per_expert_post_gather = tokens_per_expert_group.view(
+                self.ep_size, self.experts_per_rank
+            ).sum(dim=0)
+            gatherd_idxs = np.zeros(shape=(gathered_tokens.shape[0],), dtype=np.int32)
             s = 0
             for i, k in enumerate(tokens_per_expert_group.cpu().numpy()):
                 gatherd_idxs[s : s + k] = i % self.experts_per_rank
@@ -494,8 +592,10 @@ class DeepseekV3MoE(nn.Module):
             tokens_per_expert = tokens_per_expert_post_gather
 
         tokens_per_expert = tokens_per_expert.cpu().numpy()
+
         outputs = []
         start_idx = 0
+        # Forward pass for each expert’s assigned tokens
         for i, num_tokens in enumerate(tokens_per_expert):
             end_idx = start_idx + num_tokens
             if num_tokens == 0:
@@ -505,19 +605,32 @@ class DeepseekV3MoE(nn.Module):
             expert_out = expert(tokens_for_this_expert) if expert else tokens_for_this_expert
             outputs.append(expert_out)
             start_idx = end_idx
-        outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0)
+
+        outs = (
+            torch.cat(outputs, dim=0)
+            if len(outputs)
+            else sorted_tokens.new_empty(0)
+        )
+
         if self.ep_size > 1:
             new_x = torch.empty_like(outs)
             new_x[gatherd_idxs] = outs
             gathered_tokens = new_x.new_empty(*sorted_tokens_shape)
-            dist.all_to_all(list(gathered_tokens.split(input_split_sizes)), list(new_x.split(output_splits)))
+            dist.all_to_all(
+                list(gathered_tokens.split(input_split_sizes)),
+                list(new_x.split(output_splits)),
+            )
             outs = gathered_tokens
+
         new_x = torch.empty_like(outs)
         new_x[idxs] = outs
-        final_out = (new_x.view(*topk_ids.shape, -1).type(topk_weight.dtype)
-                     .mul_(topk_weight.unsqueeze(dim=-1))
-                     .sum(dim=1)
-                     .type(new_x.dtype))
+        final_out = (
+            new_x.view(*topk_ids.shape, -1)
+            .type(topk_weight.dtype)
+            .mul_(topk_weight.unsqueeze(dim=-1))
+            .sum(dim=1)
+            .type(new_x.dtype)
+        )
         return final_out
 
 
@@ -530,7 +643,9 @@ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
     batch, num_key_value_heads, slen, head_dim = hidden_states.shape
     if n_rep == 1:
         return hidden_states
-    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    hidden_states = hidden_states[:, :, None, :, :].expand(
+        batch, num_key_value_heads, n_rep, slen, head_dim
+    )
     return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
 
 
@@ -562,59 +677,96 @@ class DeepseekV3Attention(nn.Module):
 
         self.is_causal = True
 
+        # Q-proj
         if self.q_lora_rank is None:
-            self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.q_head_dim, bias=False)
+            self.q_proj = nn.Linear(
+                self.hidden_size, self.num_heads * self.q_head_dim, bias=False
+            )
         else:
-            self.q_a_proj = nn.Linear(self.hidden_size, config.q_lora_rank, bias=config.attention_bias)
+            self.q_a_proj = nn.Linear(
+                self.hidden_size, config.q_lora_rank, bias=config.attention_bias
+            )
             self.q_a_layernorm = DeepseekV3RMSNorm(config.q_lora_rank)
-            self.q_b_proj = nn.Linear(config.q_lora_rank, self.num_heads * self.q_head_dim, bias=False)
+            self.q_b_proj = nn.Linear(
+                config.q_lora_rank, self.num_heads * self.q_head_dim, bias=False
+            )
 
-        self.kv_a_proj_with_mqa = nn.Linear(self.hidden_size,
-                                              config.kv_lora_rank + config.qk_rope_head_dim,
-                                              bias=config.attention_bias)
+        # K,V-proj (MQA style)
+        self.kv_a_proj_with_mqa = nn.Linear(
+            self.hidden_size,
+            config.kv_lora_rank + config.qk_rope_head_dim,
+            bias=config.attention_bias,
+        )
         self.kv_a_layernorm = DeepseekV3RMSNorm(config.kv_lora_rank)
-        self.kv_b_proj = nn.Linear(config.kv_lora_rank,
-                                   self.num_heads * (self.q_head_dim - self.qk_rope_head_dim + self.v_head_dim),
-                                   bias=False)
+        self.kv_b_proj = nn.Linear(
+            config.kv_lora_rank,
+            self.num_heads * (self.q_head_dim - self.qk_rope_head_dim + self.v_head_dim),
+            bias=False,
+        )
 
+        # Out proj
         self.o_proj = nn.Linear(self.num_heads * self.v_head_dim, self.hidden_size, bias=config.attention_bias)
 
+        # Build the rotary embedding
         self._init_rope()
 
+        # IMPROVEMENT: Custom softmax scaling, adapt for Yarn scaling
         self.softmax_scale = self.q_head_dim ** (-0.5)
         if self.config.rope_scaling is not None:
+            # E.g. yarn-based scaling can factor in additional multipliers
             mscale_all_dim = self.config.rope_scaling.get("mscale_all_dim", 0)
             scaling_factor = self.config.rope_scaling["factor"]
             if mscale_all_dim:
+                # Simple example using the Yarn approach
                 self.softmax_scale *= yarn_get_mscale(scaling_factor, mscale_all_dim) ** 2
 
     def _init_rope(self):
+        """
+        Initializes RoPE depending on scaling type: linear, dynamic, yarn, etc.
+        """
         if self.config.rope_scaling is None:
-            self.rotary_emb = DeepseekV3RotaryEmbedding(self.qk_rope_head_dim,
-                                                        max_position_embeddings=self.max_position_embeddings,
-                                                        base=self.rope_theta)
+            self.rotary_emb = DeepseekV3RotaryEmbedding(
+                self.qk_rope_head_dim,
+                max_position_embeddings=self.max_position_embeddings,
+                base=self.rope_theta,
+            )
         else:
             scaling_type = self.config.rope_scaling["type"]
             scaling_factor = self.config.rope_scaling["factor"]
+
             if scaling_type == "linear":
-                self.rotary_emb = DeepseekV3LinearScalingRotaryEmbedding(self.qk_rope_head_dim,
-                                                                         max_position_embeddings=self.max_position_embeddings,
-                                                                         scaling_factor=scaling_factor,
-                                                                         base=self.rope_theta)
+                self.rotary_emb = DeepseekV3LinearScalingRotaryEmbedding(
+                    self.qk_rope_head_dim,
+                    max_position_embeddings=self.max_position_embeddings,
+                    scaling_factor=scaling_factor,
+                    base=self.rope_theta,
+                )
             elif scaling_type == "dynamic":
-                self.rotary_emb = DeepseekV3DynamicNTKScalingRotaryEmbedding(self.qk_rope_head_dim,
-                                                                             max_position_embeddings=self.max_position_embeddings,
-                                                                             scaling_factor=scaling_factor,
-                                                                             base=self.rope_theta)
+                self.rotary_emb = DeepseekV3DynamicNTKScalingRotaryEmbedding(
+                    self.qk_rope_head_dim,
+                    max_position_embeddings=self.max_position_embeddings,
+                    scaling_factor=scaling_factor,
+                    base=self.rope_theta,
+                )
             elif scaling_type == "yarn":
-                kwargs = {key: self.config.rope_scaling[key]
-                          for key in ["original_max_position_embeddings", "beta_fast", "beta_slow", "mscale", "mscale_all_dim"]
-                          if key in self.config.rope_scaling}
-                self.rotary_emb = DeepseekV3YarnRotaryEmbedding(self.qk_rope_head_dim,
-                                                                max_position_embeddings=self.max_position_embeddings,
-                                                                scaling_factor=scaling_factor,
-                                                                base=self.rope_theta,
-                                                                **kwargs)
+                kwargs = {
+                    key: self.config.rope_scaling[key]
+                    for key in [
+                        "original_max_position_embeddings",
+                        "beta_fast",
+                        "beta_slow",
+                        "mscale",
+                        "mscale_all_dim",
+                    ]
+                    if key in self.config.rope_scaling
+                }
+                self.rotary_emb = DeepseekV3YarnRotaryEmbedding(
+                    self.qk_rope_head_dim,
+                    max_position_embeddings=self.max_position_embeddings,
+                    scaling_factor=scaling_factor,
+                    base=self.rope_theta,
+                    **kwargs,
+                )
             else:
                 raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
 
@@ -628,10 +780,17 @@ class DeepseekV3Attention(nn.Module):
         use_cache: bool = False,
         **kwargs,
     ):
+        """
+        Standard forward pass for multi-headed self-attention.
+        """
         if "padding_mask" in kwargs:
-            warnings.warn("Passing `padding_mask` is deprecated. Use `attention_mask` instead.")
+            warnings.warn(
+                "Passing `padding_mask` is deprecated. Use `attention_mask` instead."
+            )
+
         bsz, q_len, _ = hidden_states.size()
 
+        # Q projection
         if self.q_lora_rank is None:
             q = self.q_proj(hidden_states)
         else:
@@ -639,51 +798,75 @@ class DeepseekV3Attention(nn.Module):
         q = q.view(bsz, q_len, self.num_heads, self.q_head_dim).transpose(1, 2)
         q_nope, q_pe = torch.split(q, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
 
+        # MQA: K,V from single projection
         compressed_kv = self.kv_a_proj_with_mqa(hidden_states)
-        compressed_kv, k_pe = torch.split(compressed_kv, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
+        compressed_kv, k_pe = torch.split(
+            compressed_kv, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
+        )
         k_pe = k_pe.view(bsz, q_len, 1, self.qk_rope_head_dim).transpose(1, 2)
-        kv = (self.kv_b_proj(self.kv_a_layernorm(compressed_kv))
-              .view(bsz, q_len, self.num_heads, self.qk_nope_head_dim + self.v_head_dim)
-              .transpose(1, 2))
-        k_nope, value_states = torch.split(kv, [self.qk_nope_head_dim, self.v_head_dim], dim=-1)
+        kv = (
+            self.kv_b_proj(self.kv_a_layernorm(compressed_kv))
+            .view(bsz, q_len, self.num_heads, self.qk_nope_head_dim + self.v_head_dim)
+            .transpose(1, 2)
+        )
+        k_nope, value_states = torch.split(
+            kv, [self.qk_nope_head_dim, self.v_head_dim], dim=-1
+        )
         kv_seq_len = value_states.shape[-2]
         if past_key_value is not None:
             if self.layer_idx is None:
-                raise ValueError(f"Missing `layer_idx` for caching in {self.__class__.__name__}.")
+                raise ValueError(
+                    f"Missing `layer_idx` for caching. Provide layer_idx in {self.__class__.__name__}."
+                )
             kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
 
         cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+
+        # Apply rotary to query and key
         q_pe, k_pe = apply_rotary_pos_emb(q_pe, k_pe, cos, sin, position_ids)
 
         query_states = k_pe.new_empty(bsz, self.num_heads, q_len, self.q_head_dim)
-        query_states[:, :, :, :self.qk_nope_head_dim] = q_nope
+        query_states[:, :, :, : self.qk_nope_head_dim] = q_nope
         query_states[:, :, :, self.qk_nope_head_dim :] = q_pe
 
         key_states = k_pe.new_empty(bsz, self.num_heads, q_len, self.q_head_dim)
-        key_states[:, :, :, :self.qk_nope_head_dim] = k_nope
+        key_states[:, :, :, : self.qk_nope_head_dim] = k_nope
         key_states[:, :, :, self.qk_nope_head_dim :] = k_pe
 
         if past_key_value is not None:
-            cache_kwargs = {"sin": sin, "cos": cos}
-            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+            cache_kwargs = {"sin": sin, "cos": cos}  # for RoPE
+            key_states, value_states = past_key_value.update(
+                key_states, value_states, self.layer_idx, cache_kwargs
+            )
+
+        attn_weights = (torch.matmul(query_states, key_states.transpose(2, 3))
+                        * self.softmax_scale)
 
-        attn_weights = (torch.matmul(query_states, key_states.transpose(2, 3)) * self.softmax_scale)
         if attention_mask is not None:
             attn_weights = attn_weights + attention_mask
-        attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
-        attn_weights = F.dropout(attn_weights, p=self.attention_dropout, training=self.training)
+
+        # Use float32 for more stable softmax
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_weights = nn.functional.dropout(
+            attn_weights, p=self.attention_dropout, training=self.training
+        )
         attn_output = torch.matmul(attn_weights, value_states)
-        attn_output = attn_output.transpose(1, 2).contiguous().view(bsz, q_len, self.num_heads * self.v_head_dim)
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.v_head_dim)
+
         attn_output = self.o_proj(attn_output)
 
         if not output_attentions:
             attn_weights = None
+
         return attn_output, attn_weights, past_key_value
 
 
 class DeepseekV3FlashAttention2(DeepseekV3Attention):
     """
-    DeepseekV3 flash attention module using flash_attn APIs.
+    DeepseekV3 flash attention module. Inherits the same Q/K/V projections from DeepseekV3Attention.
+    Only the forward pass changes to use flash_attn APIs.
     """
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -699,11 +882,14 @@ class DeepseekV3FlashAttention2(DeepseekV3Attention):
         use_cache: bool = False,
         **kwargs,
     ):
+        # Overridden forward logic using flash attention
         if "padding_mask" in kwargs:
-            warnings.warn("Passing `padding_mask` is deprecated. Use `attention_mask` instead.")
+            warnings.warn(
+                "Passing `padding_mask` is deprecated. Use `attention_mask` instead."
+            )
             attention_mask = kwargs.pop("padding_mask")
 
-        output_attentions = False  # flash_attn2 does not expose attention probs
+        output_attentions = False  # flash attn 2 doesn't expose attention probs
 
         bsz, q_len, _ = hidden_states.shape
         if self.q_lora_rank is None:
@@ -714,45 +900,64 @@ class DeepseekV3FlashAttention2(DeepseekV3Attention):
         q_nope, q_pe = torch.split(q, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
 
         compressed_kv = self.kv_a_proj_with_mqa(hidden_states)
-        compressed_kv, k_pe = torch.split(compressed_kv, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
+        compressed_kv, k_pe = torch.split(
+            compressed_kv, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
+        )
         k_pe = k_pe.view(bsz, q_len, 1, self.qk_rope_head_dim).transpose(1, 2)
-        kv = (self.kv_b_proj(self.kv_a_layernorm(compressed_kv))
-              .view(bsz, q_len, self.num_heads, self.qk_nope_head_dim + self.v_head_dim)
-              .transpose(1, 2))
-        k_nope, value_states = torch.split(kv, [self.qk_nope_head_dim, self.v_head_dim], dim=-1)
+        kv = (
+            self.kv_b_proj(self.kv_a_layernorm(compressed_kv))
+            .view(bsz, q_len, self.num_heads, self.qk_nope_head_dim + self.v_head_dim)
+            .transpose(1, 2)
+        )
+        k_nope, value_states = torch.split(
+            kv, [self.qk_nope_head_dim, self.v_head_dim], dim=-1
+        )
         kv_seq_len = value_states.shape[-2]
         if past_key_value is not None:
             kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
+
         cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
         q_pe, k_pe = apply_rotary_pos_emb(q_pe, k_pe, cos, sin, position_ids)
 
         query_states = k_pe.new_empty(bsz, self.num_heads, q_len, self.q_head_dim)
-        query_states[:, :, :, :self.qk_nope_head_dim] = q_nope
+        query_states[:, :, :, : self.qk_nope_head_dim] = q_nope
         query_states[:, :, :, self.qk_nope_head_dim :] = q_pe
 
         key_states = k_pe.new_empty(bsz, self.num_heads, q_len, self.q_head_dim)
-        key_states[:, :, :, :self.qk_nope_head_dim] = k_nope
+        key_states[:, :, :, : self.qk_nope_head_dim] = k_nope
         key_states[:, :, :, self.qk_nope_head_dim :] = k_pe
 
         if self.q_head_dim != self.v_head_dim:
+            # Pad if needed
             value_states = F.pad(value_states, [0, self.q_head_dim - self.v_head_dim])
 
         if past_key_value is not None:
-            cache_kwargs = {"sin": sin, "cos": sin}
-            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+            cache_kwargs = {"sin": sin, "cos": cos}
+            key_states, value_states = past_key_value.update(
+                key_states, value_states, self.layer_idx, cache_kwargs
+            )
 
+        # Prepare for flash-attn which needs [bsz, seqlen, n_heads, head_dim]
         query_states = query_states.transpose(1, 2)
         key_states = key_states.transpose(1, 2)
         value_states = value_states.transpose(1, 2)
 
         dropout_rate = self.attention_dropout if self.training else 0.0
+
+        # Possibly revert to original Q,K,V dtype if upcast to float32
         input_dtype = query_states.dtype
         if input_dtype == torch.float32:
-            target_dtype = self.q_proj.weight.dtype if self.q_lora_rank is None else self.q_a_proj.weight.dtype
+            # Attempt to revert to original param dtype if different
+            target_dtype = (
+                self.q_proj.weight.dtype
+                if self.q_lora_rank is None
+                else self.q_a_proj.weight.dtype
+            )
             query_states = query_states.to(target_dtype)
             key_states = key_states.to(target_dtype)
             value_states = value_states.to(target_dtype)
 
+        # Flash attention pass
         attn_output = self._flash_attention_forward(
             query_states,
             key_states,
@@ -764,10 +969,12 @@ class DeepseekV3FlashAttention2(DeepseekV3Attention):
         )
 
         if self.q_head_dim != self.v_head_dim:
-            attn_output = attn_output[:, :, :, :self.v_head_dim]
+            attn_output = attn_output[:, :, :, : self.v_head_dim]
 
+        # [bsz, seqlen, n_heads, head_dim]
         attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.v_head_dim)
         attn_output = self.o_proj(attn_output)
+
         return attn_output, None, past_key_value
 
     def _flash_attention_forward(
@@ -780,9 +987,13 @@ class DeepseekV3FlashAttention2(DeepseekV3Attention):
         dropout: float = 0.0,
         softmax_scale: Optional[float] = None,
     ) -> torch.Tensor:
+        """
+        Wraps the flash-attn calls. If attention_mask has padding, we unpad first.
+        """
         if not self._flash_attn_uses_top_left_mask:
             causal = self.is_causal
         else:
+            # For flash_attn<2.1.0
             causal = self.is_causal and query_length != 1
 
         if attention_mask is not None:
@@ -792,7 +1003,9 @@ class DeepseekV3FlashAttention2(DeepseekV3Attention):
              value_states,
              indices_q,
              (cu_seqlens_q, cu_seqlens_k),
-             (max_seqlen_in_batch_q, max_seqlen_in_batch_k)) = self._upad_input(query_states, key_states, value_states, attention_mask, query_length)
+             (max_seqlen_in_batch_q, max_seqlen_in_batch_k)) = self._upad_input(
+                query_states, key_states, value_states, attention_mask, query_length
+            )
             attn_output_unpad = flash_attn_varlen_func(
                 query_states,
                 key_states,
@@ -805,7 +1018,9 @@ class DeepseekV3FlashAttention2(DeepseekV3Attention):
                 softmax_scale=softmax_scale,
                 causal=causal,
             )
-            attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
+            attn_output = pad_input(
+                attn_output_unpad, indices_q, batch_size, query_length
+            )
         else:
             attn_output = flash_attn_func(
                 query_states,
@@ -815,6 +1030,7 @@ class DeepseekV3FlashAttention2(DeepseekV3Attention):
                 softmax_scale=softmax_scale,
                 causal=causal,
             )
+
         return attn_output
 
     def _upad_input(
@@ -825,29 +1041,53 @@ class DeepseekV3FlashAttention2(DeepseekV3Attention):
         attention_mask: torch.Tensor,
         query_length: int,
     ):
+        """
+        Unpads the Q, K, and V for FlashAttention in variable-length mode.
+        """
         indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
         batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
 
-        key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k)
-        value_layer = index_first_axis(value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k)
+        key_layer = index_first_axis(
+            key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim),
+            indices_k,
+        )
+        value_layer = index_first_axis(
+            value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim),
+            indices_k,
+        )
         if query_length == kv_seq_len:
-            query_layer = index_first_axis(query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k)
+            query_layer = index_first_axis(
+                query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim),
+                indices_k,
+            )
             cu_seqlens_q = cu_seqlens_k
             max_seqlen_in_batch_q = max_seqlen_in_batch_k
             indices_q = indices_k
         elif query_length == 1:
             max_seqlen_in_batch_q = 1
-            cu_seqlens_q = torch.arange(batch_size + 1, dtype=torch.int32, device=query_layer.device)
+            cu_seqlens_q = torch.arange(
+                batch_size + 1, dtype=torch.int32, device=query_layer.device
+            )
             indices_q = cu_seqlens_q[:-1]
             query_layer = query_layer.squeeze(1)
         else:
+            # handle partial left padding
             attention_mask = attention_mask[:, -query_length:]
-            query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
-        return (query_layer, key_layer, value_layer, indices_q, (cu_seqlens_q, cu_seqlens_k),
-                (max_seqlen_in_batch_q, max_seqlen_in_batch_k))
+            query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(
+                query_layer, attention_mask
+            )
+
+        return (
+            query_layer,
+            key_layer,
+            value_layer,
+            indices_q,
+            (cu_seqlens_q, cu_seqlens_k),
+            (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+        )
 
 
-# Attach attention classes in a dictionary for easy selection.
+# Attach the attention classes in a dictionary for easy selection
 ATTENTION_CLASSES = {
     "eager": DeepseekV3Attention,
     "flash_attention_2": DeepseekV3FlashAttention2,
@@ -865,15 +1105,27 @@ class DeepseekV3DecoderLayer(nn.Module):
     def __init__(self, config: DeepseekV3Config, layer_idx: int):
         super().__init__()
         self.hidden_size = config.hidden_size
-        self.self_attn = ATTENTION_CLASSES[config._attn_implementation](config=config, layer_idx=layer_idx)
-        if (config.n_routed_experts is not None and
-            layer_idx >= config.first_k_dense_replace and
-            layer_idx % config.moe_layer_freq == 0):
+
+        self.self_attn = ATTENTION_CLASSES[config._attn_implementation](
+            config=config, layer_idx=layer_idx
+        )
+
+        # Optionally use MoE
+        if (
+            config.n_routed_experts is not None
+            and layer_idx >= config.first_k_dense_replace
+            and layer_idx % config.moe_layer_freq == 0
+        ):
             self.mlp = DeepseekV3MoE(config)
         else:
             self.mlp = DeepseekV3MLP(config)
-        self.input_layernorm = DeepseekV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-        self.post_attention_layernorm = DeepseekV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+        self.input_layernorm = DeepseekV3RMSNorm(
+            config.hidden_size, eps=config.rms_norm_eps
+        )
+        self.post_attention_layernorm = DeepseekV3RMSNorm(
+            config.hidden_size, eps=config.rms_norm_eps
+        )
 
     def forward(
         self,
@@ -886,10 +1138,14 @@ class DeepseekV3DecoderLayer(nn.Module):
         **kwargs
     ):
         """
-        Forward pass for one Deepseek decoder layer.
+        Forward pass for one Deepseek decoder layer. 
         """
         residual = hidden_states
+
+        # Pre-attention norm
         hidden_states = self.input_layernorm(hidden_states)
+
+        # Self-attention
         hidden_states, self_attn_weights, present_key_value = self.self_attn(
             hidden_states=hidden_states,
             attention_mask=attention_mask,
@@ -901,21 +1157,21 @@ class DeepseekV3DecoderLayer(nn.Module):
         )
         hidden_states = residual + hidden_states
 
+        # Post-attention norm
         residual = hidden_states
         hidden_states = self.post_attention_layernorm(hidden_states)
-        # Dynamic Token Dropping
-        importance = torch.sigmoid(nn.Linear(self.hidden_size, 1).to(hidden_states.device)(hidden_states))
-        mask = (importance > (0.2 if self.training else 0.5)).float()
-        hidden_states = hidden_states * mask + (1 - mask) * hidden_states.detach()
 
+        # MLP or MoE
         hidden_states = self.mlp(hidden_states)
         hidden_states = residual + hidden_states
 
         outputs = (hidden_states,)
         if output_attentions:
             outputs += (self_attn_weights,)
+
         if use_cache:
             outputs += (present_key_value,)
+
         return outputs
 
 
@@ -925,7 +1181,7 @@ class DeepseekV3DecoderLayer(nn.Module):
 
 DeepseekV3_START_DOCSTRING = r"""
     This model inherits from `PreTrainedModel`. Check the superclass documentation
-    for the generic methods the library implements for all its models (such as loading or saving, etc.)
+    for the generic methods the library implements for all its model (such as loading or saving, etc.)
 """
 
 class DeepseekV3PreTrainedModel(PreTrainedModel):
@@ -938,6 +1194,7 @@ class DeepseekV3PreTrainedModel(PreTrainedModel):
     _supports_cache_class = True
 
     def _init_weights(self, module):
+        # IMPROVEMENT: Could add more robust initialization or variants (e.g., Xavier)
         std = self.config.initializer_range
         if isinstance(module, nn.Linear):
             module.weight.data.normal_(mean=0.0, std=std)
@@ -960,32 +1217,41 @@ DeepseekV3_INPUTS_DOCSTRING = r"""
         input_ids (torch.LongTensor): shape `(batch_size, sequence_length)`
         attention_mask (torch.Tensor): shape `(batch_size, sequence_length)` or `(batch_size, 1, seq_len, seq_len)`, optional.
         position_ids (torch.LongTensor): shape `(batch_size, sequence_length)`, optional.
-        past_key_values (Cache or tuple(tuple(torch.FloatTensor))): optional pre-computed key/value hidden-states.
+        past_key_values (Cache or tuple(tuple(torch.FloatTensor))), optional:
+            Pre-computed hidden-states (key and values) that can be used to speed up sequential decoding.
         inputs_embeds (torch.FloatTensor): shape `(batch_size, sequence_length, hidden_size)`, optional.
-        use_cache (bool), optional.
-        output_attentions (bool), optional.
-        output_hidden_states (bool), optional.
-        return_dict (bool), optional.
+        use_cache (bool), optional
+        output_attentions (bool), optional
+        output_hidden_states (bool), optional
+        return_dict (bool), optional
 """
 
-@add_start_docstrings("The bare DeepseekV3 Model outputting raw hidden-states without any specific head on top.", DeepseekV3_START_DOCSTRING)
+@add_start_docstrings(
+    "The bare DeepseekV3 Model outputting raw hidden-states without any specific head on top.",
+    DeepseekV3_START_DOCSTRING,
+)
 class DeepseekV3Model(DeepseekV3PreTrainedModel):
     """
-    Transformer decoder consisting of config.num_hidden_layers layers. Each layer is a DeepseekV3DecoderLayer.
+    Transformer decoder consisting of config.num_hidden_layers layers. Each layer is a `DeepseekV3DecoderLayer`.
     """
     def __init__(self, config: DeepseekV3Config):
         super().__init__(config)
         self.padding_idx = config.pad_token_id
         self.vocab_size = config.vocab_size
+
         self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
-        self.layers = nn.ModuleList([DeepseekV3DecoderLayer(config, layer_idx)
-                                     for layer_idx in range(config.num_hidden_layers)])
-        self._use_flash_attention_2 = (config._attn_implementation == "flash_attention_2")
+
+        # Build decoder layers
+        self.layers = nn.ModuleList([
+            DeepseekV3DecoderLayer(config, layer_idx)
+            for layer_idx in range(config.num_hidden_layers)
+        ])
+
+        self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
         self.norm = DeepseekV3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
         self.gradient_checkpointing = False
         self.post_init()
-        # Enable Torch 2.x compile for the forward pass.
-        self.forward = torch.compile(self.forward, dynamic=True)
 
     def get_input_embeddings(self) -> nn.Embedding:
         return self.embed_tokens
@@ -1006,8 +1272,10 @@ class DeepseekV3Model(DeepseekV3PreTrainedModel):
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
     ) -> Union[Tuple, BaseModelOutputWithPast]:
+
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
-        output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        output_hidden_states = (output_hidden_states if output_hidden_states is not None
+                                else self.config.output_hidden_states)
         use_cache = use_cache if use_cache is not None else self.config.use_cache
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
@@ -1029,17 +1297,29 @@ class DeepseekV3Model(DeepseekV3PreTrainedModel):
             past_key_values_length = past_key_values.get_usable_length(seq_length)
 
         if position_ids is None:
-            device = input_ids.device if input_ids is not None else inputs_embeds.device
-            position_ids = torch.arange(past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device)
+            device = (input_ids.device if input_ids is not None else inputs_embeds.device)
+            position_ids = torch.arange(
+                past_key_values_length,
+                seq_length + past_key_values_length,
+                dtype=torch.long,
+                device=device
+            )
             position_ids = position_ids.unsqueeze(0)
 
         if inputs_embeds is None:
             inputs_embeds = self.embed_tokens(input_ids)
 
+        # If flash attention is used, we pass 2D mask to the layers
         if self._use_flash_attention_2:
             attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
         else:
-            attention_mask = _prepare_4d_causal_attention_mask(attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length)
+            # standard 4D mask
+            attention_mask = _prepare_4d_causal_attention_mask(
+                attention_mask,
+                (batch_size, seq_length),
+                inputs_embeds,
+                past_key_values_length,
+            )
 
         hidden_states = inputs_embeds
 
@@ -1050,33 +1330,57 @@ class DeepseekV3Model(DeepseekV3PreTrainedModel):
         for idx, decoder_layer in enumerate(self.layers):
             if output_hidden_states:
                 all_hidden_states += (hidden_states,)
+
+            # Potential gradient checkpointing
             if self.gradient_checkpointing and self.training:
                 def create_custom_forward(module):
                     def custom_forward(*inputs):
                         return module(*inputs, output_attentions=output_attentions, use_cache=use_cache)
                     return custom_forward
-                layer_outputs = torch.utils.checkpoint.checkpoint(create_custom_forward(decoder_layer),
-                                                                  hidden_states, attention_mask, position_ids, past_key_values)
+                layer_outputs = torch.utils.checkpoint.checkpoint(
+                    create_custom_forward(decoder_layer),
+                    hidden_states,
+                    attention_mask,
+                    position_ids,
+                    past_key_values
+                )
             else:
-                layer_outputs = decoder_layer(hidden_states,
-                                              attention_mask=attention_mask,
-                                              position_ids=position_ids,
-                                              past_key_value=past_key_values,
-                                              output_attentions=output_attentions,
-                                              use_cache=use_cache)
+                layer_outputs = decoder_layer(
+                    hidden_states,
+                    attention_mask=attention_mask,
+                    position_ids=position_ids,
+                    past_key_value=past_key_values,
+                    output_attentions=output_attentions,
+                    use_cache=use_cache,
+                )
+
             hidden_states = layer_outputs[0]
             if use_cache:
                 next_decoder_cache = layer_outputs[2 if output_attentions else 1]
+
             if output_attentions:
                 all_self_attns += (layer_outputs[1],)
+
         hidden_states = self.norm(hidden_states)
         if output_hidden_states:
             all_hidden_states += (hidden_states,)
 
-        next_cache = next_decoder_cache.to_legacy_cache() if (use_cache and use_legacy_cache) else next_decoder_cache
+        # Prepare next_cache
+        next_cache = None
+        if use_cache:
+            next_cache = (
+                next_decoder_cache.to_legacy_cache()
+                if use_legacy_cache
+                else next_decoder_cache
+            )
 
         if not return_dict:
-            return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
+            return tuple(
+                v
+                for v in [hidden_states, next_cache, all_hidden_states, all_self_attns]
+                if v is not None
+            )
+
         return BaseModelOutputWithPast(
             last_hidden_state=hidden_states,
             past_key_values=next_cache,
@@ -1097,7 +1401,7 @@ class DeepseekV3ForCausalLM(DeepseekV3PreTrainedModel):
         self.model = DeepseekV3Model(config)
         self.vocab_size = config.vocab_size
         self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
-        # The Knowledge Distillation head can be added here if needed.
+
         self.post_init()
 
     def get_input_embeddings(self) -> nn.Embedding:
@@ -1119,7 +1423,9 @@ class DeepseekV3ForCausalLM(DeepseekV3PreTrainedModel):
         return self.model
 
     @add_start_docstrings_to_model_forward(DeepseekV3_INPUTS_DOCSTRING)
-    @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
+    @replace_return_docstrings(
+        output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC
+    )
     def forward(
         self,
         input_ids: Optional[torch.LongTensor] = None,
@@ -1133,39 +1439,43 @@ class DeepseekV3ForCausalLM(DeepseekV3PreTrainedModel):
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
     ) -> Union[Tuple, CausalLMOutputWithPast]:
+        """
+        Args:
+            labels (torch.LongTensor of shape (batch_size, sequence_length), optional):
+                For computing the language modeling loss. Indices in [0, config.vocab_size] or -100.
+        """
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
-        output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        output_hidden_states = (output_hidden_states if output_hidden_states is not None
+                                else self.config.output_hidden_states)
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
-        # FP8 mixed precision support if configured.
-        if self.config.mixed_precision == "fp8":
-            autocast_context = torch.autocast(device_type='cuda', dtype=torch.float8)
-        else:
-            autocast_context = torch.no_grad()  # no-op context
-
-        with autocast_context:
-            outputs = self.model(
-                input_ids=input_ids,
-                attention_mask=attention_mask,
-                position_ids=position_ids,
-                past_key_values=past_key_values,
-                inputs_embeds=inputs_embeds,
-                use_cache=use_cache,
-                output_attentions=output_attentions,
-                output_hidden_states=output_hidden_states,
-                return_dict=return_dict,
-            )
+        # Decoder forward
+        outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
         hidden_states = outputs[0]
-        logits = self.lm_head(hidden_states).float()
-        # Optionally, if you want to compute distillation logits:
-        # distill_logits = self.distill_head(hidden_states)
+        logits = self.lm_head(hidden_states)
+        logits = logits.float()  # IMPROVEMENT: Could keep FP16 if stable
+
         loss = None
         if labels is not None:
+            # SHIFT
             shift_logits = logits[..., :-1, :].contiguous()
             shift_labels = labels[..., 1:].contiguous()
+
             loss_fct = CrossEntropyLoss()
             shift_logits = shift_logits.view(-1, self.config.vocab_size)
-            shift_labels = shift_labels.view(-1).to(shift_logits.device)
+            shift_labels = shift_labels.view(-1)
+            shift_labels = shift_labels.to(shift_logits.device)
             loss = loss_fct(shift_logits, shift_labels)
 
         if not return_dict:
@@ -1188,6 +1498,9 @@ class DeepseekV3ForCausalLM(DeepseekV3PreTrainedModel):
         inputs_embeds: Optional[torch.FloatTensor] = None,
         **kwargs
     ):
+        """
+        Prepare inputs during generation loops.
+        """
         if past_key_values is not None:
             if isinstance(past_key_values, Cache):
                 cache_length = past_key_values.get_seq_length()
@@ -1196,37 +1509,50 @@ class DeepseekV3ForCausalLM(DeepseekV3PreTrainedModel):
             else:
                 cache_length = past_length = past_key_values[0][0].shape[2]
                 max_cache_length = None
+
             if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
-                input_ids = input_ids[:, -(attention_mask.shape[1] - past_length):]
+                # match up with the unprocessed tokens
+                input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
             elif past_length < input_ids.shape[1]:
                 input_ids = input_ids[:, past_length:]
+
             if max_cache_length is not None and attention_mask is not None:
                 if cache_length + input_ids.shape[1] > max_cache_length:
                     attention_mask = attention_mask[:, -max_cache_length:]
+
         position_ids = kwargs.get("position_ids", None)
         if attention_mask is not None and position_ids is None:
             position_ids = attention_mask.long().cumsum(-1) - 1
             position_ids.masked_fill_(attention_mask == 0, 1)
             if past_key_values:
-                position_ids = position_ids[:, -input_ids.shape[1]:]
+                position_ids = position_ids[:, -input_ids.shape[1] :]
+
+        # If we have inputs_embeds only for the first token
         if inputs_embeds is not None and past_key_values is None:
             model_inputs = {"inputs_embeds": inputs_embeds}
         else:
             model_inputs = {"input_ids": input_ids}
-        model_inputs.update({
-            "position_ids": position_ids,
-            "past_key_values": past_key_values,
-            "use_cache": kwargs.get("use_cache"),
-            "attention_mask": attention_mask,
-        })
+
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+            }
+        )
         return model_inputs
 
     @staticmethod
     def _reorder_cache(past_key_values: Tuple, beam_idx: torch.Tensor) -> Tuple:
         reordered_past = ()
         for layer_past in past_key_values:
-            reordered_past += (tuple(past_state.index_select(0, beam_idx.to(past_state.device))
-                                       for past_state in layer_past),)
+            reordered_past += (
+                tuple(
+                    past_state.index_select(0, beam_idx.to(past_state.device))
+                    for past_state in layer_past
+                ),
+            )
         return reordered_past
 
 
@@ -1247,6 +1573,7 @@ class DeepseekV3ForSequenceClassification(DeepseekV3PreTrainedModel):
         self.num_labels = config.num_labels
         self.model = DeepseekV3Model(config)
         self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
+
         self.post_init()
 
     def get_input_embeddings(self) -> nn.Embedding:
@@ -1269,6 +1596,7 @@ class DeepseekV3ForSequenceClassification(DeepseekV3PreTrainedModel):
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
     ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
+
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
         transformer_outputs = self.model(
             input_ids,
@@ -1289,17 +1617,24 @@ class DeepseekV3ForSequenceClassification(DeepseekV3PreTrainedModel):
         else:
             batch_size = inputs_embeds.shape[0]
 
+        # If no pad_token_id, assume last token for each sample
         if self.config.pad_token_id is None and batch_size != 1:
-            raise ValueError("Cannot handle batch sizes > 1 if no pad token is defined.")
+            raise ValueError(
+                "Cannot handle batch sizes > 1 if no pad token is defined."
+            )
         if self.config.pad_token_id is None:
             sequence_lengths = -1
         else:
             if input_ids is not None:
-                sequence_lengths = (torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1).to(logits.device)
+                sequence_lengths = (
+                    torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1
+                ).to(logits.device)
             else:
                 sequence_lengths = -1
 
-        pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
+        pooled_logits = logits[
+            torch.arange(batch_size, device=logits.device), sequence_lengths
+        ]
 
         loss = None
         if labels is not None:
@@ -1311,12 +1646,18 @@ class DeepseekV3ForSequenceClassification(DeepseekV3PreTrainedModel):
                     self.config.problem_type = "single_label_classification"
                 else:
                     self.config.problem_type = "multi_label_classification"
+
             if self.config.problem_type == "regression":
                 loss_fct = MSELoss()
-                loss = loss_fct(pooled_logits.squeeze(), labels.squeeze()) if self.num_labels == 1 else loss_fct(pooled_logits, labels)
+                if self.num_labels == 1:
+                    loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
+                else:
+                    loss = loss_fct(pooled_logits, labels)
             elif self.config.problem_type == "single_label_classification":
                 loss_fct = CrossEntropyLoss()
-                loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
+                loss = loss_fct(
+                    pooled_logits.view(-1, self.num_labels), labels.view(-1)
+                )
             elif self.config.problem_type == "multi_label_classification":
                 loss_fct = BCEWithLogitsLoss()
                 loss = loss_fct(pooled_logits, labels)