Update app.py

app.py

@@ -2,6 +2,7 @@ import os
 import logging
 import time
 import threading
+import asyncio
 from io import BytesIO
 from typing import Union
 import requests
@@ -98,7 +99,8 @@ class S3ModelLoader:
 
     async def load_model_and_tokenizer(self, model_name):
         try:
-            model_dir = await self._download_from_s3(model_name)
+            # Offload the blocking I/O to a separate thread
+            model_dir = await asyncio.to_thread(self._download_from_s3, model_name)
             config = AutoConfig.from_pretrained(model_dir)
             tokenizer = AutoTokenizer.from_pretrained(model_dir, config=config)
             model = AutoModelForCausalLM.from_pretrained(model_dir, config=config)
@@ -141,8 +143,6 @@ class S3ModelLoader:
 async def startup_event():
     model_loader.run_in_background()
 
-app = FastAPI()
-
 s3_client = boto3.client('s3', aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, region_name=AWS_REGION)
 model_loader = S3ModelLoader(S3_BUCKET_NAME, s3_client)