update modeling_qwen.py
Browse files- modeling_qwen.py +1 -1
modeling_qwen.py
CHANGED
|
@@ -535,7 +535,7 @@ class QWenAttention(nn.Module):
|
|
| 535 |
else:
|
| 536 |
seq_start = key.size(1) - query.size(1)
|
| 537 |
seq_end = key.size(1)
|
| 538 |
-
logn_tensor = self.logn_tensor[:, seq_start:seq_end, :, :]
|
| 539 |
query = query * logn_tensor.expand_as(query)
|
| 540 |
|
| 541 |
if (
|
|
|
|
| 535 |
else:
|
| 536 |
seq_start = key.size(1) - query.size(1)
|
| 537 |
seq_end = key.size(1)
|
| 538 |
+
logn_tensor = self.logn_tensor[:, seq_start:seq_end, :, :].type_as(query)
|
| 539 |
query = query * logn_tensor.expand_as(query)
|
| 540 |
|
| 541 |
if (
|