traintogpb
commited on
Commit
•
0d2c0ea
1
Parent(s):
171cd58
chore: fix model card
Browse files
README.md
CHANGED
@@ -54,44 +54,44 @@ pipeline_tag: translation
|
|
54 |
- Should remove the EOS token (`<|endoftext|>`, id=46332) at the end of the prompt.
|
55 |
|
56 |
```python
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
|
66 |
-
|
67 |
-
|
68 |
-
|
69 |
-
|
70 |
-
|
71 |
-
|
72 |
|
73 |
-
|
74 |
-
|
75 |
-
|
76 |
-
|
77 |
-
|
78 |
-
|
79 |
-
|
80 |
-
|
81 |
|
82 |
-
|
83 |
-
|
84 |
-
|
85 |
|
86 |
-
|
87 |
-
|
88 |
-
|
89 |
-
|
90 |
|
91 |
-
|
92 |
|
93 |
-
|
94 |
-
|
95 |
-
|
96 |
-
|
97 |
```
|
|
|
54 |
- Should remove the EOS token (`<|endoftext|>`, id=46332) at the end of the prompt.
|
55 |
|
56 |
```python
|
57 |
+
# MODEL
|
58 |
+
plm_name = 'beomi/open-llama-2-ko-7b'
|
59 |
+
adapter_name = 'traintogpb/llama-2-enko-translator-7b-qlora-adapter'
|
60 |
+
model = LlamaForCausalLM.from_pretrained(
|
61 |
+
plm_name,
|
62 |
+
max_length=768,
|
63 |
+
quantization_config=bnb_config, # Use the QLoRA config above
|
64 |
+
attn_implementation='flash_attention_2',
|
65 |
+
torch_dtype=torch.bfloat16
|
66 |
+
)
|
67 |
+
model = PeftModel.from_pretrained(
|
68 |
+
model,
|
69 |
+
adapter_name,
|
70 |
+
torch_dtype=torch.bfloat16
|
71 |
+
)
|
72 |
|
73 |
+
# TOKENIZER
|
74 |
+
tokenizer = LlamaTokenizer.from_pretrained(plm_name)
|
75 |
+
tokenizer.pad_token = "</s>"
|
76 |
+
tokenizer.pad_token_id = 2
|
77 |
+
tokenizer.eos_token = "<|endoftext|>" # Must be differentiated from the PAD token
|
78 |
+
tokenizer.eos_token_id = 46332
|
79 |
+
tokenizer.add_eos_token = True
|
80 |
+
tokenizer.model_max_length = 768
|
81 |
|
82 |
+
# INFERENCE
|
83 |
+
text = "NMIXX is the world-best female idol group, who came back with the new song 'DASH'."
|
84 |
+
prompt = f"Translate this from {src_lang} to {tgt_lang}\n### {src_lang}: {src_text}\n### {tgt_lang}:"
|
85 |
|
86 |
+
inputs = tokenizer(prompt, return_tensors="pt", max_length=max_length, truncation=True)
|
87 |
+
# REMOVE EOS TOKEN IN THE PROMPT
|
88 |
+
inputs['input_ids'] = inputs['input_ids'][0][:-1].unsqueeze(dim=0)
|
89 |
+
inputs['attention_mask'] = inputs['attention_mask'][0][:-1].unsqueeze(dim=0)
|
90 |
|
91 |
+
outputs = model.generate(**inputs, max_length=max_length, eos_token_id=46332)
|
92 |
|
93 |
+
input_len = len(inputs['input_ids'].squeeze())
|
94 |
+
|
95 |
+
translated_text = tokenizer.decode(outputs[0][input_len:], skip_special_tokens=True)
|
96 |
+
print(translated_text)
|
97 |
```
|