Spaces:
Sleeping
Sleeping
Luke Stanley
commited on
Commit
•
a0f49a0
1
Parent(s):
ddb0d91
Make llm_stream_sans_network actually stream to stdout
Browse files
utils.py
CHANGED
@@ -95,14 +95,21 @@ def llm_stream_sans_network(
|
|
95 |
json_schema = json.dumps(schema)
|
96 |
grammar = LlamaGrammar.from_json_schema(json_schema)
|
97 |
|
98 |
-
|
99 |
prompt,
|
100 |
max_tokens=1000,
|
101 |
temperature=0.7,
|
102 |
grammar=grammar,
|
103 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
104 |
|
105 |
-
print(
|
106 |
|
107 |
if return_pydantic_object:
|
108 |
model_object = pydantic_model_class.model_validate_json(output_text)
|
|
|
95 |
json_schema = json.dumps(schema)
|
96 |
grammar = LlamaGrammar.from_json_schema(json_schema)
|
97 |
|
98 |
+
stream = in_memory_llm(
|
99 |
prompt,
|
100 |
max_tokens=1000,
|
101 |
temperature=0.7,
|
102 |
grammar=grammar,
|
103 |
+
stream=True
|
104 |
+
)
|
105 |
+
|
106 |
+
output_text = ""
|
107 |
+
for chunk in stream:
|
108 |
+
result = chunk["choices"][0]
|
109 |
+
print(result["text"], end='', flush=True)
|
110 |
+
output_text = output_text + result["text"]
|
111 |
|
112 |
+
print('\n')
|
113 |
|
114 |
if return_pydantic_object:
|
115 |
model_object = pydantic_model_class.model_validate_json(output_text)
|