Infilling update (untested)
Browse filesFollowing this code: https://github.com/facebookresearch/codellama/blob/427d6ac90f0b7db206bc4c62f4c5d38f92ca4d10/llama/generation.py#L380
app.py
CHANGED
@@ -13,13 +13,14 @@ HF_TOKEN = os.environ.get("HF_TOKEN", None)
|
|
13 |
|
14 |
API_URL = "https://api-inference.huggingface.co/models/codellama/CodeLlama-7b-hf"
|
15 |
|
16 |
-
FIM_PREFIX = "<
|
17 |
-
FIM_MIDDLE = "<
|
18 |
-
FIM_SUFFIX = "<
|
19 |
|
20 |
FIM_INDICATOR = "<FILL_HERE>"
|
21 |
|
22 |
EOS_STRING = "</s>"
|
|
|
23 |
|
24 |
theme = gr.themes.Monochrome(
|
25 |
primary_hue="indigo",
|
@@ -78,7 +79,7 @@ def generate(
|
|
78 |
|
79 |
previous_token = ""
|
80 |
for response in stream:
|
81 |
-
if response.token.text
|
82 |
if fim_mode:
|
83 |
output += suffix
|
84 |
else:
|
|
|
13 |
|
14 |
API_URL = "https://api-inference.huggingface.co/models/codellama/CodeLlama-7b-hf"
|
15 |
|
16 |
+
FIM_PREFIX = "<PRE> "
|
17 |
+
FIM_MIDDLE = " <MID>"
|
18 |
+
FIM_SUFFIX = "<SUF>"
|
19 |
|
20 |
FIM_INDICATOR = "<FILL_HERE>"
|
21 |
|
22 |
EOS_STRING = "</s>"
|
23 |
+
EOT_STRING = "<EOT>"
|
24 |
|
25 |
theme = gr.themes.Monochrome(
|
26 |
primary_hue="indigo",
|
|
|
79 |
|
80 |
previous_token = ""
|
81 |
for response in stream:
|
82 |
+
if response.token.text in [EOS_STRING, EOT_STRING]:
|
83 |
if fim_mode:
|
84 |
output += suffix
|
85 |
else:
|