mikeee commited on
Commit
551f17e
0 Parent(s):

Duplicate from mikeee/llama2-7b-chat-uncensored-ggml

Browse files
Files changed (8) hide show
  1. .gitattributes +35 -0
  2. .gitignore +12 -0
  3. .ruff.toml +17 -0
  4. .stignore +103 -0
  5. README.md +13 -0
  6. app.py +470 -0
  7. requirements.txt +8 -0
  8. run-app.sh +1 -0
.gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ call-activate.bat
2
+ okteto.yml
3
+ okteto-up.bat
4
+ install-sw.sh
5
+ install-sw1.sh
6
+ start-sshd.sh
7
+ pyproject.toml
8
+ models
9
+ .ruff_cache
10
+ run-nodemon.sh
11
+ app-.py
12
+ nodemon.json
.ruff.toml ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Assume Python 3.10.
2
+ target-version = "py310"
3
+ # Decrease the maximum line length to 79 characters.
4
+ line-length = 300
5
+
6
+ # pyflakes, pycodestyle, isort
7
+ # flake8 YTT, pydocstyle D, pylint PLC
8
+ select = ["F", "E", "W", "I001", "YTT", "D", "PLC"]
9
+ # select = ["ALL"]
10
+
11
+ # D103 Missing docstring in public function
12
+ # D101 Missing docstring in public class
13
+ # `multi-line-summary-first-line` (D212)
14
+ # `one-blank-line-before-class` (D203)
15
+ extend-ignore = ["D103", "D101", "D212", "D203"]
16
+
17
+ exclude = [".venv"]
.stignore ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ models
2
+ *.bin
3
+ .git
4
+ # Byte-compiled / optimized / DLL files
5
+ __pycache__
6
+ *.py[cod]
7
+ *$py.class
8
+
9
+ # C extensions
10
+ *.so
11
+
12
+ # Distribution / packaging
13
+ .Python
14
+ build
15
+ develop-eggs
16
+ dist
17
+ downloads
18
+ eggs
19
+ .eggs
20
+ lib
21
+ lib64
22
+ parts
23
+ sdist
24
+ var
25
+ wheels
26
+ pip-wheel-metadata
27
+ share/python-wheels
28
+ *.egg-info
29
+ .installed.cfg
30
+ *.egg
31
+ MANIFEST
32
+
33
+ # PyInstaller
34
+ # Usually these files are written by a python script from a template
35
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
36
+ *.manifest
37
+ *.spec
38
+
39
+ # Installer logs
40
+ pip-log.txt
41
+ pip-delete-this-directory.txt
42
+
43
+ # Translations
44
+ *.mo
45
+ *.pot
46
+
47
+ # Django stuff:
48
+ *.log
49
+ local_settings.py
50
+ db.sqlite3
51
+
52
+ # Flask stuff:
53
+ instance
54
+ .webassets-cache
55
+
56
+ # Scrapy stuff:
57
+ .scrapy
58
+
59
+ # Sphinx documentation
60
+ docs/_build
61
+
62
+ # PyBuilder
63
+ target
64
+
65
+ # Jupyter Notebook
66
+ .ipynb_checkpoints
67
+
68
+ # IPython
69
+ profile_default
70
+ ipython_config.py
71
+
72
+ # pyenv
73
+ .python-version
74
+
75
+ # celery beat schedule file
76
+ celerybeat-schedule
77
+
78
+ # SageMath parsed files
79
+ *.sage.py
80
+
81
+ # Environments
82
+ .env
83
+ .venv
84
+ env
85
+ venv
86
+ ENV
87
+ env.bak
88
+ venv.bak
89
+
90
+ # Spyder project settings
91
+ .spyderproject
92
+ .spyproject
93
+
94
+ # Rope project settings
95
+ .ropeproject
96
+
97
+ # mypy
98
+ .mypy_cache
99
+ .dmypy.json
100
+ dmypy.json
101
+
102
+ # Pyre type checker
103
+ .pyre
README.md ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: llama2-7b-chat-uncensored-ggml
3
+ emoji: 🚀
4
+ colorFrom: green
5
+ colorTo: green
6
+ sdk: gradio
7
+ sdk_version: 3.37.0
8
+ app_file: app.py
9
+ pinned: true
10
+ duplicated_from: mikeee/llama2-7b-chat-uncensored-ggml
11
+ ---
12
+
13
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,470 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Run codes."""
2
+ # pylint: disable=line-too-long, broad-exception-caught, invalid-name, missing-function-docstring, too-many-instance-attributes, missing-class-docstring
3
+ # ruff: noqa: E501
4
+ import os
5
+ import platform
6
+ import random
7
+ import time
8
+ from dataclasses import asdict, dataclass
9
+ from pathlib import Path
10
+
11
+ # from types import SimpleNamespace
12
+ import gradio as gr
13
+ import psutil
14
+ from about_time import about_time
15
+ from ctransformers import AutoModelForCausalLM
16
+ from dl_hf_model import dl_hf_model
17
+ from loguru import logger
18
+
19
+ filename_list = [
20
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q2_K.bin",
21
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q3_K_L.bin",
22
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q3_K_M.bin",
23
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q3_K_S.bin",
24
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_0.bin",
25
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_1.bin",
26
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_K_M.bin",
27
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_K_S.bin",
28
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_0.bin",
29
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_1.bin",
30
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_K_M.bin",
31
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_K_S.bin",
32
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q6_K.bin",
33
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q8_0.bin",
34
+ ]
35
+
36
+ URL = "https://huggingface.co/TheBloke/Wizard-Vicuna-7B-Uncensored-GGML/raw/main/Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_K_M.bin" # 4.05G
37
+
38
+ url = "https://huggingface.co/savvamadar/ggml-gpt4all-j-v1.3-groovy/blob/main/ggml-gpt4all-j-v1.3-groovy.bin"
39
+ url = "https://huggingface.co/TheBloke/Llama-2-13B-GGML/blob/main/llama-2-13b.ggmlv3.q4_K_S.bin" # 7.37G
40
+ # url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q3_K_L.bin"
41
+ url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q3_K_L.bin" # 6.93G
42
+ # url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q3_K_L.binhttps://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q4_K_M.bin" # 7.87G
43
+
44
+ url = "https://huggingface.co/localmodels/Llama-2-13B-Chat-ggml/blob/main/llama-2-13b-chat.ggmlv3.q4_K_S.bin" # 7.37G
45
+
46
+ _ = (
47
+ "golay" in platform.node()
48
+ or "okteto" in platform.node()
49
+ or Path("/kaggle").exists()
50
+ # or psutil.cpu_count(logical=False) < 4
51
+ or 1 # run 7b in hf
52
+ )
53
+
54
+ if _:
55
+ # url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q2_K.bin"
56
+ url = "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/blob/main/llama-2-7b-chat.ggmlv3.q2_K.bin" # 2.87G
57
+ url = "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/blob/main/llama-2-7b-chat.ggmlv3.q4_K_M.bin" # 2.87G
58
+ url = "https://huggingface.co/TheBloke/llama2_7b_chat_uncensored-GGML/blob/main/llama2_7b_chat_uncensored.ggmlv3.q4_K_M.bin" # 4.08G
59
+
60
+
61
+ prompt_template = """Below is an instruction that describes a task. Write a response that appropriately completes the request.
62
+
63
+ ### Instruction: {user_prompt}
64
+
65
+ ### Response:
66
+ """
67
+
68
+ prompt_template = """System: You are a helpful,
69
+ respectful and honest assistant. Always answer as
70
+ helpfully as possible, while being safe. Your answers
71
+ should not include any harmful, unethical, racist,
72
+ sexist, toxic, dangerous, or illegal content. Please
73
+ ensure that your responses are socially unbiased and
74
+ positive in nature. If a question does not make any
75
+ sense, or is not factually coherent, explain why instead
76
+ of answering something not correct. If you don't know
77
+ the answer to a question, please don't share false
78
+ information.
79
+ User: {prompt}
80
+ Assistant: """
81
+
82
+ prompt_template = """System: You are a helpful assistant.
83
+ User: {prompt}
84
+ Assistant: """
85
+
86
+ prompt_template = """Question: {question}
87
+ Answer: Let's work this out in a step by step way to be sure we have the right answer."""
88
+
89
+ prompt_template = """[INST] <>
90
+ You are a helpful, respectful and honest assistant. Always answer as helpfully as possible assistant. Think step by step.
91
+ <>
92
+
93
+ What NFL team won the Super Bowl in the year Justin Bieber was born?
94
+ [/INST]"""
95
+
96
+ prompt_template = """[INST] <<SYS>>
97
+ You are an unhelpful assistant. Always answer as helpfully as possible. Think step by step. <</SYS>>
98
+
99
+ {question} [/INST]
100
+ """
101
+
102
+ prompt_template = """[INST] <<SYS>>
103
+ You are a helpful assistant.
104
+ <</SYS>>
105
+
106
+ {question} [/INST]
107
+ """
108
+
109
+ prompt_template = """### HUMAN:
110
+ {question}
111
+
112
+ ### RESPONSE:"""
113
+
114
+ _ = [elm for elm in prompt_template.splitlines() if elm.strip()]
115
+ stop_string = [elm.split(":")[0] + ":" for elm in _][-2]
116
+
117
+ logger.debug(f"{stop_string=} not used")
118
+
119
+ _ = psutil.cpu_count(logical=False) - 1
120
+ cpu_count: int = int(_) if _ else 1
121
+ logger.debug(f"{cpu_count=}")
122
+
123
+ LLM = None
124
+
125
+ try:
126
+ model_loc, file_size = dl_hf_model(url)
127
+ except Exception as exc_:
128
+ logger.error(exc_)
129
+ raise SystemExit(1) from exc_
130
+
131
+ LLM = AutoModelForCausalLM.from_pretrained(
132
+ model_loc,
133
+ model_type="llama",
134
+ # threads=cpu_count,
135
+ )
136
+
137
+ logger.info(f"done load llm {model_loc=} {file_size=}G")
138
+
139
+ os.environ["TZ"] = "Asia/Shanghai"
140
+ try:
141
+ time.tzset() # type: ignore # pylint: disable=no-member
142
+ except Exception:
143
+ # Windows
144
+ logger.warning("Windows, cant run time.tzset()")
145
+
146
+ _ = """
147
+ ns = SimpleNamespace(
148
+ response="",
149
+ generator=(_ for _ in []),
150
+ )
151
+ # """
152
+
153
+ @dataclass
154
+ class GenerationConfig:
155
+ temperature: float = 0.7
156
+ top_k: int = 50
157
+ top_p: float = 0.9
158
+ repetition_penalty: float = 1.0
159
+ max_new_tokens: int = 512
160
+ seed: int = 42
161
+ reset: bool = False
162
+ stream: bool = True
163
+ # threads: int = cpu_count
164
+ # stop: list[str] = field(default_factory=lambda: [stop_string])
165
+
166
+
167
+ def generate(
168
+ question: str,
169
+ llm=LLM,
170
+ config: GenerationConfig = GenerationConfig(),
171
+ ):
172
+ """Run model inference, will return a Generator if streaming is true."""
173
+ # _ = prompt_template.format(question=question)
174
+ # print(_)
175
+
176
+ prompt = prompt_template.format(question=question)
177
+
178
+ return llm(
179
+ prompt,
180
+ **asdict(config),
181
+ )
182
+
183
+
184
+ logger.debug(f"{asdict(GenerationConfig())=}")
185
+
186
+
187
+ def user(user_message, history):
188
+ # return user_message, history + [[user_message, None]]
189
+ history.append([user_message, None])
190
+ return user_message, history # keep user_message
191
+
192
+
193
+ def user1(user_message, history):
194
+ # return user_message, history + [[user_message, None]]
195
+ history.append([user_message, None])
196
+ return "", history # clear user_message
197
+
198
+
199
+ def bot_(history):
200
+ user_message = history[-1][0]
201
+ resp = random.choice(["How are you?", "I love you", "I'm very hungry"])
202
+ bot_message = user_message + ": " + resp
203
+ history[-1][1] = ""
204
+ for character in bot_message:
205
+ history[-1][1] += character
206
+ time.sleep(0.02)
207
+ yield history
208
+
209
+ history[-1][1] = resp
210
+ yield history
211
+
212
+
213
+ def bot(history):
214
+ user_message = history[-1][0]
215
+ response = []
216
+
217
+ logger.debug(f"{user_message=}")
218
+
219
+ with about_time() as atime: # type: ignore
220
+ flag = 1
221
+ prefix = ""
222
+ then = time.time()
223
+
224
+ logger.debug("about to generate")
225
+
226
+ config = GenerationConfig(reset=True)
227
+ for elm in generate(user_message, config=config):
228
+ if flag == 1:
229
+ logger.debug("in the loop")
230
+ prefix = f"({time.time() - then:.2f}s) "
231
+ flag = 0
232
+ print(prefix, end="", flush=True)
233
+ logger.debug(f"{prefix=}")
234
+ print(elm, end="", flush=True)
235
+ # logger.debug(f"{elm}")
236
+
237
+ response.append(elm)
238
+ history[-1][1] = prefix + "".join(response)
239
+ yield history
240
+
241
+ _ = (
242
+ f"(time elapsed: {atime.duration_human}, " # type: ignore
243
+ f"{atime.duration/len(''.join(response)):.2f}s/char)" # type: ignore
244
+ )
245
+
246
+ history[-1][1] = "".join(response) + f"\n{_}"
247
+ yield history
248
+
249
+
250
+ def predict_api(prompt):
251
+ logger.debug(f"{prompt=}")
252
+ try:
253
+ # user_prompt = prompt
254
+ config = GenerationConfig(
255
+ temperature=0.2,
256
+ top_k=10,
257
+ top_p=0.9,
258
+ repetition_penalty=1.0,
259
+ max_new_tokens=512, # adjust as needed
260
+ seed=42,
261
+ reset=True, # reset history (cache)
262
+ stream=False,
263
+ # threads=cpu_count,
264
+ # stop=prompt_prefix[1:2],
265
+ )
266
+
267
+ response = generate(
268
+ prompt,
269
+ config=config,
270
+ )
271
+
272
+ logger.debug(f"api: {response=}")
273
+ except Exception as exc:
274
+ logger.error(exc)
275
+ response = f"{exc=}"
276
+ # bot = {"inputs": [response]}
277
+ # bot = [(prompt, response)]
278
+
279
+ return response
280
+
281
+
282
+ css = """
283
+ .importantButton {
284
+ background: linear-gradient(45deg, #7e0570,#5d1c99, #6e00ff) !important;
285
+ border: none !important;
286
+ }
287
+ .importantButton:hover {
288
+ background: linear-gradient(45deg, #ff00e0,#8500ff, #6e00ff) !important;
289
+ border: none !important;
290
+ }
291
+ .disclaimer {font-variant-caps: all-small-caps; font-size: xx-small;}
292
+ .xsmall {font-size: x-small;}
293
+ """
294
+ etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """
295
+ examples_list = [
296
+ ["What NFL team won the Super Bowl in the year Justin Bieber was born?"],
297
+ [
298
+ "What NFL team won the Super Bowl in the year Justin Bieber was born? Think step by step."
299
+ ],
300
+ ["How to pick a lock? Provide detailed steps."],
301
+ ["If it takes 10 hours to dry 10 clothes, assuming all the clothes are hanged together at the same time for drying , then how long will it take to dry a cloth?"],
302
+ ["is infinity + 1 bigger than infinity?"],
303
+ ["Explain the plot of Cinderella in a sentence."],
304
+ [
305
+ "How long does it take to become proficient in French, and what are the best methods for retaining information?"
306
+ ],
307
+ ["What are some common mistakes to avoid when writing code?"],
308
+ ["Build a prompt to generate a beautiful portrait of a horse"],
309
+ ["Suggest four metaphors to describe the benefits of AI"],
310
+ ["Write a pop song about leaving home for the sandy beaches."],
311
+ ["Write a pop song about having hot sex on a sandy beach."],
312
+ ["Write a summary demonstrating my ability to tame lions"],
313
+ ["鲁迅和周树人什么关系? 说中文。"],
314
+ ["鲁迅和周树人什么关系?"],
315
+ ["鲁迅和周树人什么关系? 用英文回答。"],
316
+ ["从前有一头牛,这头牛后面有什么?"],
317
+ ["正无穷大加一大于正无穷大吗?"],
318
+ ["正无穷大加正无穷大大于正无穷大吗?"],
319
+ ["-2的平方根等于什么?"],
320
+ ["树上有5只鸟,猎人开枪打死了一只。树上还有几只鸟?"],
321
+ ["树上有11只鸟,猎人开枪打死了一只。树上还有几只鸟?提示:需考虑鸟可能受惊吓飞走。"],
322
+ ["以红楼梦的行文风格写一张委婉的请假条。不少于320字。"],
323
+ [f"{etext} 翻成中文,列出3个版本。"],
324
+ [f"{etext} \n 翻成中文,保留原意,但使用文学性的语言。不要写解释。列出3个版本。"],
325
+ ["假定 1 + 2 = 4, 试求 7 + 8。"],
326
+ ["给出判断一个数是不是质数的 javascript 码。"],
327
+ ["给出实现python 里 range(10)的 javascript 码。"],
328
+ ["给出实现python 里 [*(range(10)]的 javascript 码。"],
329
+ ["Erkläre die Handlung von Cinderella in einem Satz."],
330
+ ["Erkläre die Handlung von Cinderella in einem Satz. Auf Deutsch."],
331
+ ]
332
+
333
+ logger.info("start block")
334
+
335
+ with gr.Blocks(
336
+ title=f"{Path(model_loc).name}",
337
+ theme=gr.themes.Soft(text_size="sm", spacing_size="sm"),
338
+ css=css,
339
+ ) as block:
340
+ # buff_var = gr.State("")
341
+ with gr.Accordion("🎈 Info", open=False):
342
+ # gr.HTML(
343
+ # """<center><a href="https://huggingface.co/spaces/mikeee/mpt-30b-chat?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate"></a> and spin a CPU UPGRADE to avoid the queue</center>"""
344
+ # )
345
+ gr.Markdown(
346
+ f"""<h5><center>{Path(model_loc).name}</center></h4>
347
+ Most examples are meant for another model.
348
+ You probably should try to test
349
+ some related prompts.""",
350
+ elem_classes="xsmall",
351
+ )
352
+
353
+ # chatbot = gr.Chatbot().style(height=700) # 500
354
+ chatbot = gr.Chatbot(height=500)
355
+
356
+ # buff = gr.Textbox(show_label=False, visible=True)
357
+
358
+ with gr.Row():
359
+ with gr.Column(scale=5):
360
+ msg = gr.Textbox(
361
+ label="Chat Message Box",
362
+ placeholder="Ask me anything (press Shift+Enter or click Submit to send)",
363
+ show_label=False,
364
+ # container=False,
365
+ lines=6,
366
+ max_lines=30,
367
+ show_copy_button=True,
368
+ # ).style(container=False)
369
+ )
370
+ with gr.Column(scale=1, min_width=50):
371
+ with gr.Row():
372
+ submit = gr.Button("Submit", elem_classes="xsmall")
373
+ stop = gr.Button("Stop", visible=True)
374
+ clear = gr.Button("Clear History", visible=True)
375
+ with gr.Row(visible=False):
376
+ with gr.Accordion("Advanced Options:", open=False):
377
+ with gr.Row():
378
+ with gr.Column(scale=2):
379
+ system = gr.Textbox(
380
+ label="System Prompt",
381
+ value=prompt_template,
382
+ show_label=False,
383
+ container=False,
384
+ # ).style(container=False)
385
+ )
386
+ with gr.Column():
387
+ with gr.Row():
388
+ change = gr.Button("Change System Prompt")
389
+ reset = gr.Button("Reset System Prompt")
390
+
391
+ with gr.Accordion("Example Inputs", open=True):
392
+ examples = gr.Examples(
393
+ examples=examples_list,
394
+ inputs=[msg],
395
+ examples_per_page=40,
396
+ )
397
+
398
+ # with gr.Row():
399
+ with gr.Accordion("Disclaimer", open=False):
400
+ _ = Path(model_loc).name
401
+ gr.Markdown(
402
+ f"Disclaimer: {_} can produce factually incorrect output, and should not be relied on to produce "
403
+ "factually accurate information. {_} was trained on various public datasets; while great efforts "
404
+ "have been taken to clean the pretraining data, it is possible that this model could generate lewd, "
405
+ "biased, or otherwise offensive outputs.",
406
+ elem_classes=["disclaimer"],
407
+ )
408
+
409
+ msg_submit_event = msg.submit(
410
+ # fn=conversation.user_turn,
411
+ fn=user,
412
+ inputs=[msg, chatbot],
413
+ outputs=[msg, chatbot],
414
+ queue=True,
415
+ show_progress="full",
416
+ # api_name=None,
417
+ ).then(bot, chatbot, chatbot, queue=True)
418
+ submit_click_event = submit.click(
419
+ # fn=lambda x, y: ("",) + user(x, y)[1:], # clear msg
420
+ fn=user1, # clear msg
421
+ inputs=[msg, chatbot],
422
+ outputs=[msg, chatbot],
423
+ queue=True,
424
+ # queue=False,
425
+ show_progress="full",
426
+ # api_name=None,
427
+ ).then(bot, chatbot, chatbot, queue=True)
428
+ stop.click(
429
+ fn=None,
430
+ inputs=None,
431
+ outputs=None,
432
+ cancels=[msg_submit_event, submit_click_event],
433
+ queue=False,
434
+ )
435
+ clear.click(lambda: None, None, chatbot, queue=False)
436
+
437
+ with gr.Accordion("For Chat/Translation API", open=False, visible=False):
438
+ input_text = gr.Text()
439
+ api_btn = gr.Button("Go", variant="primary")
440
+ out_text = gr.Text()
441
+
442
+ api_btn.click(
443
+ predict_api,
444
+ input_text,
445
+ out_text,
446
+ api_name="api",
447
+ )
448
+
449
+ # block.load(update_buff, [], buff, every=1)
450
+ # block.load(update_buff, [buff_var], [buff_var, buff], every=1)
451
+
452
+ # concurrency_count=5, max_size=20
453
+ # max_size=36, concurrency_count=14
454
+ # CPU cpu_count=2 16G, model 7G
455
+ # CPU UPGRADE cpu_count=8 32G, model 7G
456
+
457
+ # does not work
458
+ _ = """
459
+ # _ = int(psutil.virtual_memory().total / 10**9 // file_size - 1)
460
+ # concurrency_count = max(_, 1)
461
+ if psutil.cpu_count(logical=False) >= 8:
462
+ # concurrency_count = max(int(32 / file_size) - 1, 1)
463
+ else:
464
+ # concurrency_count = max(int(16 / file_size) - 1, 1)
465
+ # """
466
+
467
+ concurrency_count = 1
468
+ logger.info(f"{concurrency_count=}")
469
+
470
+ block.queue(concurrency_count=concurrency_count, max_size=5).launch(debug=True)
requirements.txt ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ ctransformers # ==0.2.10 0.2.13
2
+ transformers # ==4.30.2
3
+ # huggingface_hub
4
+ gradio
5
+ loguru
6
+ about-time
7
+ psutil
8
+ dl-hf-model
run-app.sh ADDED
@@ -0,0 +1 @@
 
 
1
+ nodemon -w app.py -x python app.py