import { prebuiltAppConfig, CreateMLCEngine } from "@charliefruan/web-llm"; import hljs from "highlight.js"; import ace from "ace-builds"; // Required for ace to resolve the module correctly require("ace-builds/src-noconflict/mode-javascript"); require("ace-builds/webpack-resolver"); // DO NOT REMOVE // Required for user input type definition to be eval const { Type } = require("@sinclair/typebox"); let engine = null; let useCustomGrammar = false; document.addEventListener("DOMContentLoaded", () => { // Ensure elements are loaded before using them const grammarSelection = document.getElementById("grammar-selection"); const ebnfContainer = document.getElementById("ebnf-grammar-container"); const schemaContainer = document.getElementById("schema-container"); const modelSelection = document.getElementById("model-selection"); const ebnfTextarea = document.getElementById("ebnf-grammar"); const promptTextarea = document.getElementById("prompt"); const outputDiv = document.getElementById("output"); const statsParagraph = document.getElementById("stats"); // Handle grammar selection changes grammarSelection.onchange = (ev) => { console.log("Grammar selection changed:", ev.target.value); if (ev.target.value === "json") { ebnfContainer.classList.add("hidden"); schemaContainer.classList.remove("hidden"); useCustomGrammar = false; } else { ebnfContainer.classList.remove("hidden"); schemaContainer.classList.add("hidden"); useCustomGrammar = true; } }; // Populate model selection dropdown const availableModels = prebuiltAppConfig.model_list .filter( (m) => m.model_id.startsWith("Llama-3") || m.model_id.startsWith("Hermes-2") || m.model_id.startsWith("Hermes-3") || m.model_id.startsWith("Phi-3") ) .map((m) => m.model_id); let selectedModel = availableModels[0]; availableModels.forEach((modelId) => { const option = document.createElement("option"); option.value = modelId; option.textContent = modelId; modelSelection.appendChild(option); }); modelSelection.value = selectedModel; modelSelection.onchange = (e) => { selectedModel = e.target.value; engine = null; // Reset the engine when the model changes }; // Editors setup with Ace const jsonSchemaEditor = ace.edit("schema", { mode: "ace/mode/javascript", theme: "ace/theme/github", wrap: true, }); jsonSchemaEditor.setTheme("ace/theme/github"); jsonSchemaEditor.setValue(`Type.Object({ "name": Type.String(), "house": Type.Enum({ "Gryffindor": "Gryffindor", "Hufflepuff": "Hufflepuff", "Ravenclaw": "Ravenclaw", "Slytherin": "Slytherin", }), "blood_status": Type.Enum({ "Pure-blood": "Pure-blood", "Half-blood": "Half-blood", "Muggle-born": "Muggle-born", }), "occupation": Type.Enum({ "Student": "Student", "Professor": "Professor", "Ministry of Magic": "Ministry of Magic", "Other": "Other", }), "wand": Type.Object({ "wood": Type.String(), "core": Type.String(), "length": Type.Number(), }), "alive": Type.Boolean(), "patronus": Type.String(), })`); const grammarEditor = ace.edit("ebnf-grammar", { theme: "ace/theme/github", wrap: true, }); grammarEditor.setTheme("ace/theme/github"); grammarEditor.setValue(String.raw`main ::= basic_array | basic_object basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) ".0"? basic_number ::= ("0" | "-"? [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= (([\"] basic_string_1 [\"])) basic_string_1 ::= "" | [^"\\\x00-\x1F] basic_string_1 | "\\" escape basic_string_1 escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= "[" ("" | ws basic_any (ws "," ws basic_any)*) ws "]" basic_object ::= "{" ("" | ws basic_string ws ":" ws basic_any ( ws "," ws basic_string ws ":" ws basic_any)*) ws "}" ws ::= [\n\t]*`); // Set initial prompt promptTextarea.value = `Hermione Granger is a character in Harry Potter. Please fill in the following information about this character in JSON format. Name is a string of character name. House is one of Gryffindor, Hufflepuff, Ravenclaw, Slytherin. Blood status is one of Pure-blood, Half-blood, Muggle-born. Occupation is one of Student, Professor, Ministry of Magic, Other. Wand is an object with wood, core, and length. Alive is a boolean. Patronus is a string. `; // Generate button click handler document.getElementById("generate").onclick = async () => { if (!engine) { engine = await CreateMLCEngine(selectedModel, { initProgressCallback: (progress) => { console.log(progress); outputDiv.textContent = progress.text; }, }); } let response_format = { type: "grammar", grammar: grammarEditor.getValue() }; if (!useCustomGrammar) { const schemaInput = jsonSchemaEditor.getValue(); let T; try { T = eval(schemaInput); } catch (e) { console.error("Invalid schema", e); return; } const schema = JSON.stringify(T); response_format = { type: "json_object", schema } } console.log(response_format); const request = { stream: true, stream_options: { include_usage: true }, messages: [{ role: "user", content: promptTextarea.value }], max_tokens: 512, response_format, }; let curMessage = ""; let usage = null; const generator = await engine.chatCompletion(request); for await (const chunk of generator) { const curDelta = chunk.choices[0]?.delta.content; if (curDelta) curMessage += curDelta; if (chunk.usage) usage = chunk.usage; outputDiv.textContent = curMessage; } const finalMessage = await engine.getMessage(); outputDiv.innerHTML = hljs.highlight(finalMessage, { language: "json", }).value; if (usage) { const statsTextParts = []; console.log(usage); if (usage.extra.prefill_tokens_per_s) { statsTextParts.push(`Prefill Speed: ${usage.extra.prefill_tokens_per_s.toFixed( 1 )} tok/s`); } if (usage.extra.decode_tokens_per_s) { statsTextParts.push(`Decode Speed: ${usage.extra.decode_tokens_per_s.toFixed( 1 )} tok/s`); } if (usage.extra.time_per_output_token_s) { statsTextParts.push(`Time Per Output Token: ${(1000 * usage.extra.time_per_output_token_s).toFixed( 0 )} ms`); } if (usage.extra.time_to_first_token_s) { statsTextParts.push(`Time to First Token: ${(1000 * usage.extra.time_to_first_token_s).toFixed( 0 )} ms`); } if (usage.extra.grammar_init_s) { statsTextParts.push(`Grammar Init Overhead: ${(1000 * usage.extra.grammar_init_s).toFixed( 0 )} ms`); } if (usage.extra.grammar_per_token_s) { statsTextParts.push(`Grammar Per-token Overhead: ${(1000 * usage.extra.grammar_per_token_s).toFixed( 2 )} ms`); } statsParagraph.textContent = statsTextParts.join(", "); statsParagraph.classList.remove("hidden"); } }; });