Leetmonkey In Action via Inference

app.py

@@ -39,7 +39,6 @@ generation_kwargs = {
     "repeat_penalty": 1.1
 }
 
-@spaces.GPU
 def download_model(model_name: str) -> str:
     logger.info(f"Downloading model: {model_name}")
     model_path = hf_hub_download(
@@ -136,7 +135,6 @@ def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)):
         raise HTTPException(status_code=401, detail="Invalid token")
 
 @app.post("/generate_solution")
-@spaces.GPU
 async def generate_solution_api(request: ProblemRequest, authorized: bool = Depends(verify_token)):
     logger.info("Generating solution")
     generated_output = generate_solution(request.instruction)
@@ -145,7 +143,6 @@ async def generate_solution_api(request: ProblemRequest, authorized: bool = Depe
     return {"solution": formatted_code}
 
 @app.post("/stream_solution")
-@spaces.GPU
 async def stream_solution_api(request: ProblemRequest, authorized: bool = Depends(verify_token)):
     async def generate():
         logger.info("Streaming solution")
@@ -177,7 +174,6 @@ Here's the complete Python function implementation:
 
 # Gradio wrapper for FastAPI
 def gradio_wrapper(app):
-    @spaces.GPU
     def inference(instruction, token):
         import requests
         url = "http://localhost:8000/generate_solution"
@@ -200,17 +196,24 @@ def gradio_wrapper(app):
     )
     return iface
 
-if __name__ == "__main__":
-    import uvicorn
-    from threading import Thread
-
+@spaces.GPU
+def main():
     # Verify GPU availability
     zero = torch.Tensor().cuda()
     print(f"GPU availability: {zero.device}")
 
+    # Download and load the model
+    model_path = download_model(MODEL_NAME)
+    global llm
+    llm = load_model(model_path)
+    logger.info("8-bit model loaded successfully")
+
     # Start FastAPI in a separate thread
     Thread(target=lambda: uvicorn.run(app, host="0.0.0.0", port=8000)).start()
 
     # Launch Gradio interface
     iface = gradio_wrapper(app)
     iface.launch(share=True)
+
+if __name__ == "__main__":
+    main()