405b now support 128k ctx

llm.py

@@ -2,7 +2,7 @@
 import utils; from utils import *
 import os, sys, lzma, json, pprint, time, subprocess
 
-thinker = os.getenv("thinker", "gemini")
+thinker = os.getenv("thinker", "405b")
 TEMPERATURE = float(os.getenv("temperature", 0.1)) # 0.0 conservative (good for coding and correct syntax)
 
 LLM_HOST = "gemini"
@@ -78,7 +78,7 @@ elif thinker in "70b|405b":
 
     # https://docs.together.ai/docs/chat-models#hosted-models
     model = {
-        "405b": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo    8k 3k 1.2", # $5.00 / 1m tokens(*)
+        "405b": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo  128k 4k 1.2", # $4.00 / 1m tokens(*)
          "70b": "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo   128k 4k 1.2", # $0.88 / 1m tokens(*)
     }[thinker]
 
@@ -89,12 +89,12 @@ elif thinker in "70b|405b":
     TKNZ_RATIO = float(TKNZ_RATIO)
 
     CTXLEN = int(CTXLEN[:-1])
-    if CTXLEN > 32: CTXLEN = 32 # max 32k ctxlen
+    if CTXLEN > 64: CTXLEN = 64 # max 32k ctxlen
     CTXLEN = CTXLEN*1024 - MAX_TOKENS
     # print(model, CTXLEN, MAX_TOKENS, TKNZ_RATIO); input(); # DEBUG
 
     from together import Together
-    together_client = Together(api_key=os.environ.get('TOGETHER_API_KEY'))
+    together_client = Together(api_key='adc0db56b77fe6508bdeadb4d8253771750a50639f8e87313153e49d4599f6ea')
     ###
     stops = ["<|eot_id|>","<|eom_id|>","</answer>","</output>"]
     def thinker_chat(prompt, history=[], stream=False, use_cache=True, testing=False):
@@ -163,51 +163,6 @@ elif thinker in "70b|405b":
         return messages
 
 
-elif thinker in "gemma2:27b|commandr:35b|llama3.1:70b":
-    #################
-    ## Ollama connect
-    import subprocess, ollama # pip install ollama
-    try: ollama.list()
-    except: subprocess.run('nohup ssh -N -L 11434:localhost:11434 -p 22021 [email protected] &', shell=True)
-    subprocess.run('nohup ssh -N -L 9999:localhost:11434 -p 17340 [email protected] &', shell=True)
-    #################
-    OLLAMA_CLIENT = ollama.Client(host='http://localhost:11434')
-    machine = "RTX-4090-24G"
-
-    ## ~30b models
-    if   thinker in    "gemma2:27b": OLLAMA_MODEL =     "gemma2:27b-instruct-q5_K_M" ; CTXLEN = 512*14 # fit 24G
-    elif thinker in  "commandr:35b": OLLAMA_MODEL =  "command-r:35b-08-2024-q4_K_M"  ; CTXLEN = 512*18 # fit 24G
-    else: OLLAMA_MODEL = "not found"
-
-    try:    connect_to_4090 = OLLAMA_MODEL in str(ollama.list())
-    except: connect_to_4090 = False
-
-    if not connect_to_4090: # switch to A100
-        OLLAMA_CLIENT = ollama.Client(host='http://localhost:9999')
-        machine = "A100-PCIE-40GB"
-        ## ~30b to ~70b models
-        if   thinker in   "gemma2:27b": OLLAMA_MODEL =    "gemma2:27b-instruct-q8_0"   ; CTXLEN = 1024*24
-        elif thinker in "commandr:35b": OLLAMA_MODEL = "command-r:35b-08-2024-q8_0"    ; CTXLEN = 1024*32
-        elif thinker in "llama3.1:70b": OLLAMA_MODEL =  "llama3.1:70b-instruct-q3_K_M" ; CTXLEN = 1024*12#fit 40G
-    LLM_HOST = f"{machine}__{OLLAMA_MODEL}"
-    
-    def thinker_chat(prompt, history=[],  stream=False, use_cache=False):
-        if stream:
-            with open(llm_log_filename,"at") as f: f.write(f"\n- - - [ {LLM_HOST} ] - - -\n\nPROMPT:\n{prompt}\n") 
-            return OLLAMA_CLIENT.chat(model=OLLAMA_MODEL, messages=[{"role": "user", "content": prompt}], \
-                stream=True, options={'num_ctx': CTXLEN, 'temperature': TEMPERATURE})
-
-        messages = history + [{"role": "user", "content": prompt}]
-        with open(llm_log_filename,"at") as f: f.write(f"\n- - - [ {LLM_HOST} ] - - -\n\nPROMPT:\n{prompt}\n")
-        res = OLLAMA_CLIENT.chat(model=OLLAMA_MODEL, messages=messages, options={'temperature': TEMPERATURE})
-        content = res["message"]["content"]
-        with open(llm_log_filename,"at") as f: f.write(f"\nCONTENT:\n{content}\n")
-        messages += [{"role": "assistant", "content": content}]
-        return messages
-
-    ## To make it's 100% local llm, normal chat can also use thinker
-    # chat = thinker_chat
-
 LLM_HOST += f"__{round(CTXLEN/1024)}k_ctxlen"
 who_are_you()