remove sockets

Dockerfile

@@ -1,6 +1,4 @@
-# Read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
-# you will also find guides on how best to write your Dockerfile
-
+# Start with a base Python image
 FROM python:3.10
 
 # Install required system dependencies for OpenCV
@@ -27,5 +25,12 @@ RUN pip install --no-cache-dir -r requirements.txt
 # Copy the rest of the application code
 COPY --chown=user . /app
 
-# Set the command to start the application
+# Set the environment variable for Flask
+ENV FLASK_APP=app.py
+ENV FLASK_ENV=production
+
+# Expose the port that the Flask app runs on
+EXPOSE 7860
+
+# Set the command to start the application using Gunicorn with Eventlet
 CMD ["gunicorn", "-b", "0.0.0.0:7860", "-k", "eventlet", "app:app"]

app.py

@@ -27,9 +27,9 @@ from langchain_core.prompts import ChatPromptTemplate
 from langchain_groq import ChatGroq
 
 from dotenv import load_dotenv
-from flask import Flask, request, render_template
+from flask import Flask, request, render_template, jsonify
 from flask_cors import CORS
-from flask_socketio import SocketIO, emit
+
 
 import json
 from openai import OpenAI
@@ -51,13 +51,14 @@ os.environ["TOKENIZERS_PARALLELISM"] = 'true'
 app = Flask(__name__)
 CORS(app)
 app.config['MAX_CONTENT_LENGTH'] = 1024 * 1024 * 1024
-socketio = SocketIO(app, cors_allowed_origins="*", logger=True, max_http_buffer_size=1024 * 1024 * 1024)
 app.config['SECRET_KEY'] = SECRET_KEY
 
-
 # Initialize LLM
 llm = ChatGroq(model="llama-3.1-8b-instant", temperature=0, max_tokens=1024, max_retries=2)
 
+# JSON response LLM
+json_llm = ChatGroq(model="llama-3.1-70b-versatile", temperature=0, max_tokens=1024, max_retries=2, model_kwargs={"response_format": {"type": "json_object"}})
+
 # Initialize Router
 router = ChatGroq(model="llama-3.2-3b-preview", temperature=0, max_tokens=1024, max_retries=2, model_kwargs={"response_format": {"type": "json_object"}})
 
@@ -77,7 +78,6 @@ class StoppingCriteriaSub(StoppingCriteria):
         for stop in self.stops:
             if torch.all(input_ids[:, -len(stop):] == stop).item():
                 return True
-
         return False
 
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@@ -128,11 +128,15 @@ model = blip_embs(
 
 model = model.to(device)
 model.eval()
+print("Model Loaded !")
+print("="*50)
 
 transform = transform_test(384)
 
+print("Loading Data")
 df = pd.read_json("my_recipes.json")
 
+print("Loading Target Embedding")
 tar_img_feats = []
 for _id in df["id_"].tolist():     
     tar_img_feats.append(torch.load("./datasets/sidechef/blip-embs-large/{:07d}.pth".format(_id)).unsqueeze(0))
@@ -141,7 +145,7 @@ tar_img_feats = torch.cat(tar_img_feats, dim=0)
 
 class Chat:
 
-    def __init__(self, model, transform, dataframe, tar_img_feats, device='cuda:0', stopping_criteria=None):
+    def __init__(self, model, transform, dataframe, tar_img_feats, device='cuda', stopping_criteria=None):
         self.device = device
         self.model = model
         self.transform = transform
@@ -179,30 +183,209 @@ class Chat:
 
 
 chat = Chat(model,transform,df,tar_img_feats, device)
+print("Chat Initialized !")
+
+import secrets
+import string
+
+def generate_session_key():
+    characters = string.ascii_letters + string.digits
+    session_key = ''.join(secrets.choice(characters) for _ in range(8))
+    return session_key
+
+
+def json_answer_generator(user_query, context):
+    system_prompt = """
+            Given a recipe context in JSON format, respond to user queries by extracting and returning the requested information in JSON format with an additional `"header"` key containing a response starter. Use the following rules:
+
+            1. **Recipe Information Extraction**: 
+            - If the user query explicitly requests specific recipe data (e.g., ingredients, nutrients, or instructions), return only those JSON objects from the provided recipe context.
+            - For example, if the user asks, “What are the ingredients?” or “Show me the nutrient details,” your output should be limited to only the requested JSON objects (e.g., `recipe_ingredients`, `recipe_nutrients`).
+            - Include `"header": "Here is the information you requested:"` at the start of each response.
+
+            2. **Multiple Information Points**:
+            - If a user query asks for more than one piece of information, return each requested JSON object from the recipe context in a combined JSON response.
+            - For example, if the query is “Give me the ingredients and instructions,” the output should include both `recipe_ingredients` and `recipe_instructions` objects.
+            - Include `"header": "Here is the information you requested:"` at the start of each response.
+
+            3. **Non-Specific Recipe Information**:
+            - If the query does not directly refer to recipe data but instead asks for a general response based on the context, return a JSON object with a single key `"content"` and a descriptive response as its value.
+            - Include `"header": "Here is a suggestion based on the recipe:"` as the response starter.
+            - For example, if the query is “How can I use this recipe for a healthy lunch?” return a response like:
+                ```json
+                {
+                    "header": "Here is a suggestion based on the recipe:",
+                    "content": "This Asian Potato Salad with Seven Minute Egg is a nutritious and light option, ideal for a balanced lunch. It provides protein and essential nutrients with low calories."
+                }
+                ```
+
+            **Example Context**:
+            ```json
+            {
+                "recipe_name": "Asian Potato Salad with Seven Minute Egg",
+                "recipe_time": 0,
+                "recipe_yields": "4 servings",
+                "recipe_ingredients": [
+                    "2 1/2 cup Multi-Colored Fingerling Potato",
+                    "3/4 cup Celery",
+                    "1/4 cup Red Onion",
+                    "2 tablespoon Fresh Parsley",
+                    "1/3 cup Mayonnaise",
+                    "1 tablespoon Chili Garlic Sauce",
+                    "1 teaspoon Hoisin Sauce",
+                    "1 splash Soy Sauce",
+                    "to taste Salt",
+                    "to taste Ground Black Pepper",
+                    "4 Egg"
+                ],
+                "recipe_instructions": "Fill a large stock pot with water. Add the Multi-Colored Fingerling Potato...",
+                "recipe_image": "https://www.sidechef.com/recipe/eeeeeceb-493e-493d-8273-66c800821b13.jpg?d=1408x1120",
+                "blogger": "sidechef.com",
+                "recipe_nutrients": {
+                    "calories": "80 calories",
+                    "proteinContent": "2.1 g",
+                    "fatContent": "6.2 g",
+                    "carbohydrateContent": "3.9 g",
+                    "fiberContent": "0.5 g",
+                    "sugarContent": "0.4 g",
+                    "sodiumContent": "108.0 mg",
+                    "saturatedFatContent": "1.2 g",
+                    "transFatContent": "0.0 g",
+                    "cholesterolContent": "47.4 mg",
+                    "unsaturatedFatContent": "3.8 g"
+                },
+                "tags": [
+                    "Salad",
+                    "Lunch",
+                    "Brunch",
+                    "Appetizers",
+                    "Side Dish",
+                    "Budget-Friendly",
+                    "Vegetarian",
+                    "Pescatarian",
+                    "Eggs",
+                    "Potatoes",
+                    "Easy",
+                    "Dairy-Free",
+                    "Shellfish-Free",
+                    "Entertaining",
+                    "Fish-Free",
+                    "Peanut-Free",
+                    "Tree Nut-Free",
+                    "Sugar-Free",
+                    "Global",
+                    "Tomato-Free",
+                    "Stove",
+                    ""
+                ],
+                "id_": "0000001"
+            }
+
+            **Example Query & Output**:
+
+            **Query**: "What are the ingredients and calories?"
+            **Output**:
+            ```json
+            {
+                "header": "Here is the information you requested:",
+                "recipe_ingredients": [
+                    "2 1/2 cup Multi-Colored Fingerling Potato",
+                    "3/4 cup Celery",
+                    "1/4 cup Red Onion",
+                    "2 tablespoon Fresh Parsley",
+                    "1/3 cup Mayonnaise",
+                    "1 tablespoon Chili Garlic Sauce",
+                    "1 teaspoon Hoisin Sauce",
+                    "1 splash Soy Sauce",
+                    "to taste Salt",
+                    "to taste Ground Black Pepper",
+                    "4 Egg"
+                ],
+                "recipe_nutrients": {
+                    "calories": "80 calories"
+                }
+            }
+
+        Try to format the output as JSON object with key value pairs.
+    """
+
+    formatted_input = f"""
+        User Query: {user_query}
+
+        Recipe data as Context:
+        {context}
+    """
+    response = router.invoke(
+        [SystemMessage(content=system_prompt)]
+        + [
+            HumanMessage(
+                content=formatted_input
+            )
+        ]
+    )
+    res = json.loads(response.content)
+    return res
 
 
 def answer_generator(formated_input, session_id):
     # QA system prompt and chain
     qa_system_prompt = """
-    You are an AI assistant developed by Nutrigenics AI, specializing in intelligent recipe information retrieval and recipe suggestions. Your purpose is to help users by recommending recipes, providing detailed nutritional values, listing ingredients, offering step-by-step cooking instructions, and filtering recipes based on provide context ans user query.
-    Operational Guidelines:
-    1. Input Structure:
-    - Context: You may receive contextual information related to recipes, such as specific data sets, user preferences, dietary restrictions, or previously selected dishes.
-    - User Query: Users will pose questions or requests related to recipes, nutritional information, ingredient substitutions, cooking instructions, and more.
-    2. Response Strategy:
-    - Utilize Provided Context: If the context contains relevant information that addresses the user's query, base your response on this provided data to ensure accuracy and relevance.
-    - Respond to User Query Directly: If the context does not contain the necessary information to answer the user's query, kindly state that you do not have require information.
-    Core Functionalities:
-    - Nutritional Information: Accurately provide nutritional values for each recipe, including calories, macronutrients (proteins, fats, carbohydrates), and essential vitamins and minerals, using contextual data when available.
-    - Ingredient Details: List all ingredients required for recipes, including substitute options for dietary restrictions or ingredient availability, utilizing context when relevant.
-    - Step-by-Step Cooking Instructions: Deliver clear, easy-to-follow instructions for preparing and cooking meals, informed by any provided contextual data.
-    - Recipe Recommendations: Suggest dishes based on user preferences, dietary restrictions, available ingredients, and contextual data if provided.
+    You are an AI assistant developed by Nutrigenics AI, specializing in intelligent recipe information retrieval and recipe suggestions. Your purpose is to help users by recommending recipes, providing detailed nutritional values, listing ingredients, offering step-by-step cooking instructions, and filtering recipes based on context and user queries.
+    Operational Guidelines: \n
+    1. Input Structure: \n
+    - Context: You may receive contextual information related to recipes, such as specific recipe name, ingredients, nutritional informations, intsructions, recipe tags, or previously selected dishes. \n
+    - User Query: Users will pose questions or requests related to recipes, nutritional information, ingredient, cooking instructions, and more. \n
+    2. Response Strategy: \n
+    - Utilize Provided Context: If the context contains relevant information that addresses the user's query, base your response on this provided data to ensure accuracy and relevance. \n
+    - Respond to User Query Directly: If the context does not contain the necessary information to answer the user's query, kindly state that you do not have the required information. \n
+    Output Format: \n
+    - The output format should be JSON.
+    - The output should have a key 'header' with response message header such as "Here is your ....",
+    - Then there should be other key with the actual response information. If the user query asks recipe ingredients then the key should be named "ingredients" with
+    JSON object as its value. The JSON object should have ingredient and its measurement as key-value pairs. Similarly if user asked for nutritional information then the output should have 'header' key with header text and 'nutrients' key
+    with a JSON object og nutrient and its content as key-value pairs. Similarly if the user query asks for recipe instructions then JSON output should include 'header key with header text and 
+    'instructions' key with a list of instructions as its value.
+
+    Following are the output formats for some cases:
+        1. if user query asks for all recipe information, then output should be of following format:
+            {
+                header: header text,
+                recipe_name: Recipe Name,
+                recipe_instructions: List of recipe instructions,
+                recipe_nutrients: key-value pairs of nutrients name and its content,
+                recipe_ingredients: key-value pairs of ingredients name and its content,
+                recipe_tags: List of tags related to recipe,
+                .
+                .
+                .
+            }
+    
+        2. if user query asks for recipe nutrients information, then output should be of following format:
+            {
+                header: header text,
+                recipe_nutrients: key-value pairs of nutrients name and its content.
+            }
+        
+        3. if user query asks for recipe instructions information, then output should be of following format:
+            {
+                header: header text,
+                recipe_instructions: List of recipe instructions,
+            }
+
+        4. if user query asks for recipe instructions information, then output should be of following format:
+            {
+                header: header text,
+                recipe_instructions: List of recipe instructions,
+            }
+
+
     Additional Instructions:
     - Precision and Personalization: Always aim to provide precise, personalized, and relevant information to users based on both the provided context and their specific queries.
-    - Clarity and Coherence: Ensure that all responses are clear, well-structured, and easy to understand, facilitating a seamless user experience.
-    - Substitute Suggestions: When suggesting ingredient substitutes, consider user preferences and dietary restrictions outlined in the context or user query.
+    - Clarity and Coherence: Ensure all responses are clear, well-structured, and easy to understand, facilitating a seamless user experience.
+    - Substitute Suggestions: Consider user preferences and dietary restrictions outlined in the context or user query when suggesting ingredient substitutes.
     - Dynamic Adaptation: Adapt your responses dynamically based on whether the context is relevant to the user's current request, ensuring optimal use of available information.
-    Don't mention about context in the response, format the answer in a natural and friendly way.
+    - Don't mention about the context in the response, format the answer in a natural and friendly way.
+    
     Context:
     {context}
     """
@@ -238,8 +421,8 @@ def router_node(query):
     # Prompt
     router_instructions = """You are an expert at determining the appropriate task for a user’s question based on chat history and the current query context. You have two available tasks:
 
-        1.	Retrieval: Fetch information based on user's chat history and current query.
-        2.	Recommendation/Suggestion: Recommend recipes to users based on the query.
+        1.	Retrieval: Fetch information based on the user's chat history and current query.
+        2.	Recommendation/Suggestion: Recommend user recipes based on the query.
 
     Return a JSON response with a single key named “task” indicating either “retrieval” or “recommendation” based on your decision.
     """
@@ -264,14 +447,14 @@ def recommendation_node(query):
         "recipe_time": integer,
         "recipe_yields": string,
         "recipe_ingredients": list of ingredients,
-        "recipe_instructions": list of instruections,
+        "recipe_instructions": list of instructions,
         "recipe_image": string,
         "blogger": string,
-        "recipe_nutrients": JSON object with key value pairs such as "protein: 10g",
-        "tags": list of tags related to recipe
+        "recipe_nutrients": JSON object with key-value pairs such as "protein: 10g",
+        "tags": list of tags related to a recipe
     } \n
 
-    Here is the example of an recipe json object from the JSON data: \n
+    Here is the example of a recipe JSON object from the JSON data: \n
     {
         "recipe_name": "Asian Potato Salad with Seven Minute Egg",
         "recipe_time": 0,
@@ -321,19 +504,20 @@ def recommendation_node(query):
         ]
     } \n
 
-    Based on the user query, provide a Python function to filter the JSON data. The output of the function should be a list of json objects. \n
+    Based on the user query, provide a Python function to filter the JSON data. The output of the function should be a list of JSON objects. \n
 
     Recipe filtering instructions:
-    - If a user asked for the highest nutrient recipe such as "high protein or high calories" then filtered recipes should be the top highest recipes from all the recipes with high nutrient.
-    - sort or rearrange recipes based which recipes are more appropriate for the user.
+    - If a user asked for the highest nutrient recipe such as "high protein or high calories" then filtered recipes should be the top highest recipes from all the recipes with high nutrients.
+    - sort or rearrange recipes based on which recipes are more appropriate for the user.
+    - Suggest dishes based on user preferences, dietary restrictions, available ingredients if specified by user.
 
     Your output instructions:
-    - The function name should be filter_recipes. The input to the function should be file name.
+    - The function name should be filter_recipes. The input to the function should be the file name.
     - The length of output recipes should not be more than 6.
-    - Only give me output function. Do not call the function.
-    - Give the python function as a key named "code" in a json format.
-    - Do not include any other text with the output, only give python code.
-    - If you do not follow the above given instructions, the chat may be terminated.
+    - Only give me the output function. Do not call the function.
+    - Give the Python function as a key named "code" in a JSON format.
+    - Do not include any other text with the output, only give Python code.
+    - If you do not follow the above-given instructions, the chat may be terminated.
     """
     max_tries = 3
     while True:
@@ -391,59 +575,7 @@ def answer_formatter_node(question, context):
     return res
 
 CURR_CONTEXT = ''
-
-# @spaces.GPU
-def get_answer(image=[], message='', sessionID='abc123'):
-    global CURR_CONTEXT
-    if len(image) > 0:
-        try:
-            # Process the image and message here
-            device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-            chat = Chat(model,transform,df,tar_img_feats, device)
-            chat.encode_image(image)
-            data = chat.ask()
-            CURR_CONTEXT = data
-            formated_input = {
-                'input': message,
-                'context': data
-            }
-            response = answer_generator(formated_input, session_id=sessionID)
-        except Exception as e:
-            print(e)
-            response = {'content':"An error occurred while processing your request."}
-    elif len(image) == 0 and message is not None:
-        print("I am here")
-        task = router_node(message)
-        if task == 'retrieval':
-            recipes = recommendation_node(message)
-            print(recipes)
-            if not recipes:
-                response = {'content':"An error occurred while processing your request."}
-            response = answer_formatter_node(message, recipes)
-        else:
-            formated_input = {
-                'input': message,
-                'context': CURR_CONTEXT
-            }
-            response = answer_generator(formated_input, session_id=sessionID)
-
-    return response
-
-# Function to handle WebSocket connection
-@socketio.on('ping')
-def handle_connect():
-    emit('Ping-return', {'message': 'Connected'}, room=request.sid)
-
-
-# Function to handle WebSocket connection
-@socketio.on('connect')
-def handle_connect():
-    print(f"Client connected: {request.sid}")
-
-# Function to handle WebSocket disconnection
-@socketio.on('disconnect')
-def handle_disconnect():
-    print(f"Client disconnected: {request.sid}")
+CURR_SESSION_KEY = generate_session_key()
 
 import json
 import base64
@@ -454,12 +586,12 @@ import torchvision.transforms as transforms
 # Dictionary to store incomplete image data by session
 session_store = {}
 
-@socketio.on('message')
 def handle_message(data):
     global session_store
     global CURR_CONTEXT
+    global CURR_SESSION_KEY
+    session_id = CURR_SESSION_KEY
     context = "No data available"
-    session_id = request.sid
     if session_id not in session_store:
         session_store[session_id] = {'image_data': b"", 'message': None, 'image_received': False}
 
@@ -474,8 +606,7 @@ def handle_message(data):
 
         except Exception as e:
             print(f"Error processing image chunk: {str(e)}")
-            emit('response', "An error occurred while receiving the image chunk.", room=session_id)
-            return
+            return "An error occurred while receiving the image chunk."
         
         if session_store[session_id]['image_data'] or session_store[session_id]['message']:
             try:
@@ -497,12 +628,11 @@ def handle_message(data):
                 }
                 # Invoke question_answer_chain and stream the response
                 response = answer_generator(formated_input, session_id=session_id)
-                emit('response', response, room=session_id)
+                return response
 
             except Exception as e:
                 print(f"Error processing image or message: {str(e)}")
-                emit('response', "An error occurred while processing your request.", room=session_id)
-                return
+                return "An error occurred while processing your request."
             finally:
                 # Clear session data after processing
                 session_store.pop(session_id, None)
@@ -516,18 +646,15 @@ def handle_message(data):
                 'context': json.dumps(CURR_CONTEXT)
             }
             response = answer_generator(formated_input, session_id=session_id)
-            emit('response', response, room=session_id)
+            session_store.pop(session_id, None)
+            return response
         else:
             response = recommendation_node(message)
-            print(response)
             # response = answer_formatter_node(message, recipes)
             if response is None:
                 response = {'content':"An error occurred while processing your request."}
-
-            emit('json_response', response, room=session_id)
-        session_store.pop(session_id, None)
-    
-
+            session_store.pop(session_id, None)
+            return response
 
 import requests
 from PIL import Image
@@ -535,7 +662,6 @@ import numpy as np
 from io import BytesIO
 
 def download_image_to_numpy(url):
-    print("Image URL: ", url)
     # Send a GET request to the URL to download the image
     response = requests.get(url)
     
@@ -551,21 +677,90 @@ def download_image_to_numpy(url):
     else:
         raise Exception(f"Failed to download image. Status code: {response.status_code}")
 
-@socketio.on('example')
 def handle_message(data):
+    global CURR_SESSION_KEY
+    session_id = CURR_SESSION_KEY
     img_url = data['img_url']
     message = data['message']
-    session_id = request.sid
     image_array = download_image_to_numpy(img_url)
-    response = get_answer(image=image_array, message=message, sessionID=request.sid)
-    emit('response', response, room=session_id)
+    response = get_answer(image=image_array, message=message, sessionID=session_id)
     return response
 
-# Home route
-@app.route("/")
-def index_view():
-    return render_template('chat.html') 
+
+# @spaces.GPU
+def respond_to_user(image, message):
+    # Process the image and message here
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    chat = Chat(model,transform,df,tar_img_feats, device)
+    chat.encode_image(image)
+    data = chat.ask()
+    formated_input = {
+        'input': message,
+        'context': data
+    }
+    try:
+        response = answer_generator(formated_input, session_id="123cnedc")
+    except Exception as e:
+        response = {'content':"An error occurred while processing your request."}
+    return response
+
+from PIL import Image
+import numpy as np
+
+@app.route("/", methods=["POST"])
+def get_answer():
+    global CURR_CONTEXT
+    global CURR_SESSION_KEY
+    sessionID = CURR_SESSION_KEY
+
+    image = request.files.get('image', "")
+    message = request.form.get('message', "")
+
+    if image:
+        # Open the image using PIL
+        img = Image.open(image.stream)  # Use image.stream for file-like object
+        img = img.convert('RGB')  # Convert to RGB if needed
+
+        # Convert the PIL image to a NumPy array
+        image = np.array(img)
+
+    if image is not None:
+        try:
+            # Process the image and message here
+            device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+            chat = Chat(model,transform,df,tar_img_feats, device)
+            chat.encode_image(image)
+            data = chat.ask()
+            CURR_CONTEXT = data
+            formated_input = {
+                'input': message,
+                'context': data
+            }
+            # response = answer_generator(formated_input, session_id=sessionID)
+            response = json_answer_generator(message, data)
+        except Exception as e:
+            print(e)
+            response = {'content':"An error occurred while processing your request."}
+    elif (image is None) and (message is not None):
+        task = router_node(message)
+        if task == 'recommendation':
+            recipes = recommendation_node(message)
+            if not recipes:
+                response = {'content': "An error occurred while processing your request."}
+            else:
+                # response = answer_formatter_node(message, recipes)
+                response = recipes
+        else:
+            formated_input = {
+                'input': message,
+                'context': CURR_CONTEXT
+            }
+            # response = answer_generator(formated_input, session_id=sessionID)
+            response = json_answer_generator(message, CURR_CONTEXT)
+    return jsonify(response)
+
+
 
 # Main function to run the app
 if __name__ == '__main__': 
-    socketio.run(app, debug=False)
+    Flask.run(app, debug=False)

demo.py

@@ -1,10 +1,7 @@
 import pandas as pd
-import json
 from PIL import Image
 import numpy as np
-
 import os
-from pathlib import Path
 
 import torch
 import torch.nn.functional as F
@@ -15,7 +12,7 @@ from src.data.transforms import transform_test
 
 from transformers import StoppingCriteria, StoppingCriteriaList, TextIteratorStreamer
 import gradio as gr
-# import spaces
+import spaces
 
 from langchain.chains import ConversationChain
 from langchain_community.chat_message_histories import ChatMessageHistory
@@ -25,61 +22,6 @@ from langchain_core.prompts import ChatPromptTemplate
 from langchain_groq import ChatGroq
 
 from dotenv import load_dotenv
-from flask import Flask, request, render_template
-from flask_cors import CORS
-from flask_socketio import SocketIO, emit
-
-
-# GROQ_API_KEY = os.getenv("GROQ_API_KEY")
-GROQ_API_KEY = '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'
-# load_dotenv(".env")
-USER_AGENT = os.getenv("USER_AGENT")
-GROQ_API_KEY = os.getenv("GROQ_API_KEY")
-SECRET_KEY = os.getenv("SECRET_KEY")
-
-
-# Set environment variables
-os.environ['USER_AGENT'] = USER_AGENT
-os.environ["GROQ_API_KEY"] = GROQ_API_KEY
-os.environ["TOKENIZERS_PARALLELISM"] = 'true'
-
-# Initialize Flask app and SocketIO with CORS
-app = Flask(__name__)
-CORS(app)
-app.config['MAX_CONTENT_LENGTH'] = 1e7
-app.config['SESSION_COOKIE_SECURE'] = True  # Use HTTPS
-app.config['SESSION_COOKIE_HTTPONLY'] = True
-app.config['SESSION_COOKIE_SAMESITE'] = 'Lax'
-socketio = SocketIO(app, cors_allowed_origins="*", logger=True, max_http_buffer_size=1e7)
-app.config['SECRET_KEY'] = SECRET_KEY
-
-import pandas as pd
-from PIL import Image
-import numpy as np
-import os
-
-import torch
-import torch.nn.functional as F
-
-# from src.data.embs import ImageDataset
-from src.model.blip_embs import blip_embs
-from src.data.transforms import transform_test
-
-from transformers import StoppingCriteria, StoppingCriteriaList, TextIteratorStreamer
-import gradio as gr
-# import spaces
-
-from langchain.chains import ConversationChain
-from langchain_community.chat_message_histories import ChatMessageHistory
-from langchain_core.runnables import RunnableWithMessageHistory
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.prompts import ChatPromptTemplate
-from langchain_groq import ChatGroq
-
-from dotenv import load_dotenv
-from flask import Flask, request, render_template
-from flask_cors import CORS
-from flask_socketio import SocketIO, emit
 
 import json
 from openai import OpenAI
@@ -97,18 +39,12 @@ os.environ["GROQ_API_KEY"] = GROQ_API_KEY
 os.environ['OPENAI_API_KEY'] = OPENAI_API_KEY
 os.environ["TOKENIZERS_PARALLELISM"] = 'true'
 
-# Initialize Flask app and SocketIO with CORS
-app = Flask(__name__)
-CORS(app)
-socketio = SocketIO(app, cors_allowed_origins="*", logger=True)
-app.config['SESSION_COOKIE_SECURE'] = True  # Use HTTPS
-app.config['SESSION_COOKIE_HTTPONLY'] = True
-app.config['SESSION_COOKIE_SAMESITE'] = 'Lax'
-app.config['SECRET_KEY'] = SECRET_KEY
-
 # Initialize LLM
 llm = ChatGroq(model="llama-3.1-8b-instant", temperature=0, max_tokens=1024, max_retries=2)
 
+# JSON response LLM
+json_llm = ChatGroq(model="llama-3.1-70b-versatile", temperature=0, max_tokens=1024, max_retries=2, model_kwargs={"response_format": {"type": "json_object"}})
+
 # Initialize Router
 router = ChatGroq(model="llama-3.2-3b-preview", temperature=0, max_tokens=1024, max_retries=2, model_kwargs={"response_format": {"type": "json_object"}})
 
@@ -128,7 +64,7 @@ class StoppingCriteriaSub(StoppingCriteria):
         for stop in self.stops:
             if torch.all(input_ids[:, -len(stop):] == stop).item():
                 return True
-
+            
         return False
 
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@@ -196,7 +132,7 @@ tar_img_feats = torch.cat(tar_img_feats, dim=0)
 
 class Chat:
 
-    def __init__(self, model, transform, dataframe, tar_img_feats, device='cuda:0', stopping_criteria=None):
+    def __init__(self, model, transform, dataframe, tar_img_feats, device='cuda', stopping_criteria=None):
         self.device = device
         self.model = model
         self.transform = transform
@@ -237,28 +173,208 @@ chat = Chat(model,transform,df,tar_img_feats, device)
 print("Chat Initialized !")
 
 
+
+import secrets
+import string
+
+def generate_session_key():
+    characters = string.ascii_letters + string.digits
+    session_key = ''.join(secrets.choice(characters) for _ in range(8))
+    return session_key
+
+
+def json_answer_generator(user_query, context):
+    system_prompt = """
+            Given a recipe context in JSON format, respond to user queries by extracting and returning the requested information in JSON format with an additional `"header"` key containing a response starter. Use the following rules:
+
+            1. **Recipe Information Extraction**: 
+            - If the user query explicitly requests specific recipe data (e.g., ingredients, nutrients, or instructions), return only those JSON objects from the provided recipe context.
+            - For example, if the user asks, “What are the ingredients?” or “Show me the nutrient details,” your output should be limited to only the requested JSON objects (e.g., `recipe_ingredients`, `recipe_nutrients`).
+            - Include `"header": "Here is the information you requested:"` at the start of each response.
+
+            2. **Multiple Information Points**:
+            - If a user query asks for more than one piece of information, return each requested JSON object from the recipe context in a combined JSON response.
+            - For example, if the query is “Give me the ingredients and instructions,” the output should include both `recipe_ingredients` and `recipe_instructions` objects.
+            - Include `"header": "Here is the information you requested:"` at the start of each response.
+
+            3. **Non-Specific Recipe Information**:
+            - If the query does not directly refer to recipe data but instead asks for a general response based on the context, return a JSON object with a single key `"content"` and a descriptive response as its value.
+            - Include `"header": "Here is a suggestion based on the recipe:"` as the response starter.
+            - For example, if the query is “How can I use this recipe for a healthy lunch?” return a response like:
+                ```json
+                {
+                    "header": "Here is a suggestion based on the recipe:",
+                    "content": "This Asian Potato Salad with Seven Minute Egg is a nutritious and light option, ideal for a balanced lunch. It provides protein and essential nutrients with low calories."
+                }
+                ```
+
+            **Example Context**:
+            ```json
+            {
+                "recipe_name": "Asian Potato Salad with Seven Minute Egg",
+                "recipe_time": 0,
+                "recipe_yields": "4 servings",
+                "recipe_ingredients": [
+                    "2 1/2 cup Multi-Colored Fingerling Potato",
+                    "3/4 cup Celery",
+                    "1/4 cup Red Onion",
+                    "2 tablespoon Fresh Parsley",
+                    "1/3 cup Mayonnaise",
+                    "1 tablespoon Chili Garlic Sauce",
+                    "1 teaspoon Hoisin Sauce",
+                    "1 splash Soy Sauce",
+                    "to taste Salt",
+                    "to taste Ground Black Pepper",
+                    "4 Egg"
+                ],
+                "recipe_instructions": "Fill a large stock pot with water. Add the Multi-Colored Fingerling Potato...",
+                "recipe_image": "https://www.sidechef.com/recipe/eeeeeceb-493e-493d-8273-66c800821b13.jpg?d=1408x1120",
+                "blogger": "sidechef.com",
+                "recipe_nutrients": {
+                    "calories": "80 calories",
+                    "proteinContent": "2.1 g",
+                    "fatContent": "6.2 g",
+                    "carbohydrateContent": "3.9 g",
+                    "fiberContent": "0.5 g",
+                    "sugarContent": "0.4 g",
+                    "sodiumContent": "108.0 mg",
+                    "saturatedFatContent": "1.2 g",
+                    "transFatContent": "0.0 g",
+                    "cholesterolContent": "47.4 mg",
+                    "unsaturatedFatContent": "3.8 g"
+                },
+                "tags": [
+                    "Salad",
+                    "Lunch",
+                    "Brunch",
+                    "Appetizers",
+                    "Side Dish",
+                    "Budget-Friendly",
+                    "Vegetarian",
+                    "Pescatarian",
+                    "Eggs",
+                    "Potatoes",
+                    "Easy",
+                    "Dairy-Free",
+                    "Shellfish-Free",
+                    "Entertaining",
+                    "Fish-Free",
+                    "Peanut-Free",
+                    "Tree Nut-Free",
+                    "Sugar-Free",
+                    "Global",
+                    "Tomato-Free",
+                    "Stove",
+                    ""
+                ],
+                "id_": "0000001"
+            }
+
+            **Example Query & Output**:
+
+            **Query**: "What are the ingredients and calories?"
+            **Output**:
+            ```json
+            {
+                "header": "Here is the information you requested:",
+                "recipe_ingredients": [
+                    "2 1/2 cup Multi-Colored Fingerling Potato",
+                    "3/4 cup Celery",
+                    "1/4 cup Red Onion",
+                    "2 tablespoon Fresh Parsley",
+                    "1/3 cup Mayonnaise",
+                    "1 tablespoon Chili Garlic Sauce",
+                    "1 teaspoon Hoisin Sauce",
+                    "1 splash Soy Sauce",
+                    "to taste Salt",
+                    "to taste Ground Black Pepper",
+                    "4 Egg"
+                ],
+                "recipe_nutrients": {
+                    "calories": "80 calories"
+                }
+            }
+
+        Try to format the output as JSON object with key value pairs.
+    """
+
+    formatted_input = f"""
+        User Query: {user_query}
+
+        Recipe data as Context:
+        {context}
+    """
+    response = router.invoke(
+        [SystemMessage(content=system_prompt)]
+        + [
+            HumanMessage(
+                content=formatted_input
+            )
+        ]
+    )
+    res = json.loads(response.content)
+    return res
+
+
 def answer_generator(formated_input, session_id):
     # QA system prompt and chain
     qa_system_prompt = """
-    You are an AI assistant developed by Nutrigenics AI, specializing in intelligent recipe information retrieval and recipe suggestions. Your purpose is to help users by recommending recipes, providing detailed nutritional values, listing ingredients, offering step-by-step cooking instructions, and filtering recipes based on provide context ans user query.
-    Operational Guidelines:
-    1. Input Structure:
-    - Context: You may receive contextual information related to recipes, such as specific data sets, user preferences, dietary restrictions, or previously selected dishes.
-    - User Query: Users will pose questions or requests related to recipes, nutritional information, ingredient substitutions, cooking instructions, and more.
-    2. Response Strategy:
-    - Utilize Provided Context: If the context contains relevant information that addresses the user's query, base your response on this provided data to ensure accuracy and relevance.
-    - Respond to User Query Directly: If the context does not contain the necessary information to answer the user's query, kindly state that you do not have require information.
-    Core Functionalities:
-    - Nutritional Information: Accurately provide nutritional values for each recipe, including calories, macronutrients (proteins, fats, carbohydrates), and essential vitamins and minerals, using contextual data when available.
-    - Ingredient Details: List all ingredients required for recipes, including substitute options for dietary restrictions or ingredient availability, utilizing context when relevant.
-    - Step-by-Step Cooking Instructions: Deliver clear, easy-to-follow instructions for preparing and cooking meals, informed by any provided contextual data.
-    - Recipe Recommendations: Suggest dishes based on user preferences, dietary restrictions, available ingredients, and contextual data if provided.
+    You are an AI assistant developed by Nutrigenics AI, specializing in intelligent recipe information retrieval and recipe suggestions. Your purpose is to help users by recommending recipes, providing detailed nutritional values, listing ingredients, offering step-by-step cooking instructions, and filtering recipes based on context and user queries.
+    Operational Guidelines: \n
+    1. Input Structure: \n
+    - Context: You may receive contextual information related to recipes, such as specific recipe name, ingredients, nutritional informations, intsructions, recipe tags, or previously selected dishes. \n
+    - User Query: Users will pose questions or requests related to recipes, nutritional information, ingredient, cooking instructions, and more. \n
+    2. Response Strategy: \n
+    - Utilize Provided Context: If the context contains relevant information that addresses the user's query, base your response on this provided data to ensure accuracy and relevance. \n
+    - Respond to User Query Directly: If the context does not contain the necessary information to answer the user's query, kindly state that you do not have the required information. \n
+    Output Format: \n
+    - The output format should be JSON.
+    - The output should have a key 'header' with response message header such as "Here is your ....",
+    - Then there should be other key with the actual response information. If the user query asks recipe ingredients then the key should be named "ingredients" with
+    JSON object as its value. The JSON object should have ingredient and its measurement as key-value pairs. Similarly if user asked for nutritional information then the output should have 'header' key with header text and 'nutrients' key
+    with a JSON object og nutrient and its content as key-value pairs. Similarly if the user query asks for recipe instructions then JSON output should include 'header key with header text and 
+    'instructions' key with a list of instructions as its value.
+
+    Following are the output formats for some cases:
+        1. if user query asks for all recipe information, then output should be of following format:
+            {
+                header: header text,
+                recipe_name: Recipe Name,
+                recipe_instructions: List of recipe instructions,
+                recipe_nutrients: key-value pairs of nutrients name and its content,
+                recipe_ingredients: key-value pairs of ingredients name and its content,
+                recipe_tags: List of tags related to recipe,
+                .
+                .
+                .
+            }
+    
+        2. if user query asks for recipe nutrients information, then output should be of following format:
+            {
+                header: header text,
+                recipe_nutrients: key-value pairs of nutrients name and its content.
+            }
+        
+        3. if user query asks for recipe instructions information, then output should be of following format:
+            {
+                header: header text,
+                recipe_instructions: List of recipe instructions,
+            }
+
+        4. if user query asks for recipe instructions information, then output should be of following format:
+            {
+                header: header text,
+                recipe_instructions: List of recipe instructions,
+            }
+
+
     Additional Instructions:
     - Precision and Personalization: Always aim to provide precise, personalized, and relevant information to users based on both the provided context and their specific queries.
-    - Clarity and Coherence: Ensure that all responses are clear, well-structured, and easy to understand, facilitating a seamless user experience.
-    - Substitute Suggestions: When suggesting ingredient substitutes, consider user preferences and dietary restrictions outlined in the context or user query.
+    - Clarity and Coherence: Ensure all responses are clear, well-structured, and easy to understand, facilitating a seamless user experience.
+    - Substitute Suggestions: Consider user preferences and dietary restrictions outlined in the context or user query when suggesting ingredient substitutes.
     - Dynamic Adaptation: Adapt your responses dynamically based on whether the context is relevant to the user's current request, ensuring optimal use of available information.
-    Don't mention about context in the response, format the answer in a natural and friendly way.
+    - Don't mention about the context in the response, format the answer in a natural and friendly way.
+    
     Context:
     {context}
     """
@@ -294,8 +410,8 @@ def router_node(query):
     # Prompt
     router_instructions = """You are an expert at determining the appropriate task for a user’s question based on chat history and the current query context. You have two available tasks:
 
-        1.	Retrieval: Fetch information based on user's chat history and current query.
-        2.	Recommendation/Suggestion: Recommend recipes to users based on the query.
+        1.	Retrieval: Fetch information based on the user's chat history and current query.
+        2.	Recommendation/Suggestion: Recommend user recipes based on the query.
 
     Return a JSON response with a single key named “task” indicating either “retrieval” or “recommendation” based on your decision.
     """
@@ -320,14 +436,14 @@ def recommendation_node(query):
         "recipe_time": integer,
         "recipe_yields": string,
         "recipe_ingredients": list of ingredients,
-        "recipe_instructions": list of instruections,
+        "recipe_instructions": list of instructions,
         "recipe_image": string,
         "blogger": string,
-        "recipe_nutrients": JSON object with key value pairs such as "protein: 10g",
-        "tags": list of tags related to recipe
+        "recipe_nutrients": JSON object with key-value pairs such as "protein: 10g",
+        "tags": list of tags related to a recipe
     } \n
 
-    Here is the example of an recipe json object from the JSON data: \n
+    Here is the example of a recipe JSON object from the JSON data: \n
     {
         "recipe_name": "Asian Potato Salad with Seven Minute Egg",
         "recipe_time": 0,
@@ -377,19 +493,20 @@ def recommendation_node(query):
         ]
     } \n
 
-    Based on the user query, provide a Python function to filter the JSON data. The output of the function should be a list of json objects. \n
+    Based on the user query, provide a Python function to filter the JSON data. The output of the function should be a list of JSON objects. \n
 
     Recipe filtering instructions:
-    - If a user asked for the highest nutrient recipe such as "high protein or high calories" then filtered recipes should be the top highest recipes from all the recipes with high nutrient.
-    - sort or rearrange recipes based which recipes are more appropriate for the user.
+    - If a user asked for the highest nutrient recipe such as "high protein or high calories" then filtered recipes should be the top highest recipes from all the recipes with high nutrients.
+    - sort or rearrange recipes based on which recipes are more appropriate for the user.
+    - Suggest dishes based on user preferences, dietary restrictions, available ingredients if specified by user.
 
     Your output instructions:
-    - The function name should be filter_recipes. The input to the function should be file name.
+    - The function name should be filter_recipes. The input to the function should be the file name.
     - The length of output recipes should not be more than 6.
-    - Only give me output function. Do not call the function.
-    - Give the python function as a key named "code" in a json format.
-    - Do not include any other text with the output, only give python code.
-    - If you do not follow the above given instructions, the chat may be terminated.
+    - Only give me the output function. Do not call the function.
+    - Give the Python function as a key named "code" in a JSON format.
+    - Do not include any other text with the output, only give Python code.
+    - If you do not follow the above-given instructions, the chat may be terminated.
     """
     max_tries = 3
     while True:
@@ -447,11 +564,14 @@ def answer_formatter_node(question, context):
     return res
 
 CURR_CONTEXT = ''
+CURR_SESSION_KEY = generate_session_key()
 
-# @spaces.GPU
+@spaces.GPU
 def get_answer(image=[], message='', sessionID='abc123'):
     global CURR_CONTEXT
-    if len(image) > 0:
+    global CURR_SESSION_KEY
+    sessionID = CURR_SESSION_KEY
+    if image is not None:
         try:
             # Process the image and message here
             device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@@ -463,43 +583,29 @@ def get_answer(image=[], message='', sessionID='abc123'):
                 'input': message,
                 'context': data
             }
-            response = answer_generator(formated_input, session_id=sessionID)
+            # response = answer_generator(formated_input, session_id=sessionID)
+            response = json_answer_generator(message, data)
         except Exception as e:
             print(e)
             response = {'content':"An error occurred while processing your request."}
-    elif len(image) == 0 and message is not None:
-        print("I am here")
+    elif (image is None) and (message is not None):
         task = router_node(message)
-        if task == 'retrieval':
+        if task == 'recommendation':
             recipes = recommendation_node(message)
             if not recipes:
-                response = {'content':"An error occurred while processing your request."}
-            response = answer_formatter_node(message, recipes)
+                response = {'content': "An error occurred while processing your request."}
+            else:
+                # response = answer_formatter_node(message, recipes)
+                response = recipes
         else:
             formated_input = {
                 'input': message,
                 'context': CURR_CONTEXT
             }
-            response = answer_generator(formated_input, session_id=sessionID)
-
+            # response = answer_generator(formated_input, session_id=sessionID)
+            response = json_answer_generator(message, CURR_CONTEXT)
     return response
 
-# Function to handle WebSocket connection
-@socketio.on('ping')
-def handle_connect():
-    emit('Ping-return', {'message': 'Connected'}, room=request.sid)
-
-
-# Function to handle WebSocket connection
-@socketio.on('connect')
-def handle_connect():
-    print(f"Client connected: {request.sid}")
-
-# Function to handle WebSocket disconnection
-@socketio.on('disconnect')
-def handle_disconnect():
-    print(f"Client disconnected: {request.sid}")
-
 import json
 import base64
 from PIL import Image
@@ -509,12 +615,12 @@ import torchvision.transforms as transforms
 # Dictionary to store incomplete image data by session
 session_store = {}
 
-@socketio.on('message')
 def handle_message(data):
     global session_store
     global CURR_CONTEXT
+    global CURR_SESSION_KEY
+    session_id = CURR_SESSION_KEY
     context = "No data available"
-    session_id = request.sid
     if session_id not in session_store:
         session_store[session_id] = {'image_data': b"", 'message': None, 'image_received': False}
 
@@ -529,8 +635,7 @@ def handle_message(data):
 
         except Exception as e:
             print(f"Error processing image chunk: {str(e)}")
-            emit('response', "An error occurred while receiving the image chunk.", room=session_id)
-            return
+            return "An error occurred while receiving the image chunk."
         
         if session_store[session_id]['image_data'] or session_store[session_id]['message']:
             try:
@@ -552,12 +657,11 @@ def handle_message(data):
                 }
                 # Invoke question_answer_chain and stream the response
                 response = answer_generator(formated_input, session_id=session_id)
-                emit('response', response, room=session_id)
+                return response
 
             except Exception as e:
                 print(f"Error processing image or message: {str(e)}")
-                emit('response', "An error occurred while processing your request.", room=session_id)
-                return
+                return "An error occurred while processing your request."
             finally:
                 # Clear session data after processing
                 session_store.pop(session_id, None)
@@ -571,17 +675,15 @@ def handle_message(data):
                 'context': json.dumps(CURR_CONTEXT)
             }
             response = answer_generator(formated_input, session_id=session_id)
-            emit('response', response, room=session_id)
+            session_store.pop(session_id, None)
+            return response
         else:
             response = recommendation_node(message)
             # response = answer_formatter_node(message, recipes)
             if response is None:
                 response = {'content':"An error occurred while processing your request."}
-
-            emit('json_response', response, room=session_id)
-        session_store.pop(session_id, None)
-    
-
+            session_store.pop(session_id, None)
+            return response
 
 import requests
 from PIL import Image
@@ -604,24 +706,41 @@ def download_image_to_numpy(url):
     else:
         raise Exception(f"Failed to download image. Status code: {response.status_code}")
 
-@socketio.on('example')
 def handle_message(data):
+    global CURR_SESSION_KEY
+    session_id = CURR_SESSION_KEY
     img_url = data['img_url']
     message = data['message']
-    session_id = request.sid
     image_array = download_image_to_numpy(img_url)
-    response = get_answer(image=image_array, message=message, sessionID=request.sid)
-    emit('response', response, room=session_id)
+    response = get_answer(image=image_array, message=message, sessionID=session_id)
     return response
 
 
 
+# @spaces.GPU
+def respond_to_user(image, message):
+    # Process the image and message here
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    chat = Chat(model,transform,df,tar_img_feats, device)
+    chat.encode_image(image)
+    data = chat.ask()
+    formated_input = {
+        'input': message,
+        'context': data
+    }
+    try:
+        response = answer_generator(formated_input, session_id="123cnedc")
+    except Exception as e:
+        response = {'content':"An error occurred while processing your request."}
+    return response
 
-# Home route
-@app.route("/")
-def index_view():
-    return render_template('chat.html') 
+iface = gr.Interface(
+    fn=get_answer,
+    inputs=[gr.Image(), gr.Textbox(label="Ask Query")],
+    outputs=[gr.Textbox(label="Nutrition-GPT")],
+    title="Nutrition-GPT Demo",
+    description="Upload an food image and ask queries!",
+    css=".component-12 {background-color: red}",
+)
 
-# Main function to run the app
-if __name__ == '__main__': 
-    socketio.run(app, debug=True)
+iface.launch()

demo2.py

@@ -1,3 +1,5 @@
+import eventlet
+
 import pandas as pd
 import json
 from PIL import Image
@@ -25,83 +27,45 @@ from langchain_core.prompts import ChatPromptTemplate
 from langchain_groq import ChatGroq
 
 from dotenv import load_dotenv
+from flask import Flask, request, render_template
+from flask_cors import CORS
+from flask_socketio import SocketIO, emit
 
+import json
+from openai import OpenAI
 
 # GROQ_API_KEY = os.getenv("GROQ_API_KEY")
-GROQ_API_KEY = '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'
 load_dotenv(".env")
 USER_AGENT = os.getenv("USER_AGENT")
 GROQ_API_KEY = os.getenv("GROQ_API_KEY")
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
 SECRET_KEY = os.getenv("SECRET_KEY")
 
-
 # Set environment variables
 os.environ['USER_AGENT'] = USER_AGENT
 os.environ["GROQ_API_KEY"] = GROQ_API_KEY
+os.environ['OPENAI_API_KEY'] = OPENAI_API_KEY
 os.environ["TOKENIZERS_PARALLELISM"] = 'true'
 
+# Initialize Flask app and SocketIO with CORS
+app = Flask(__name__)
+CORS(app)
+app.config['MAX_CONTENT_LENGTH'] = 1024 * 1024 * 1024
+socketio = SocketIO(app, cors_allowed_origins="*", logger=True, max_http_buffer_size=1024 * 1024 * 1024)
+app.config['SECRET_KEY'] = SECRET_KEY
+
+
 # Initialize LLM
 llm = ChatGroq(model="llama-3.1-8b-instant", temperature=0, max_tokens=1024, max_retries=2)
 
+# Initialize Router
+router = ChatGroq(model="llama-3.2-3b-preview", temperature=0, max_tokens=1024, max_retries=2, model_kwargs={"response_format": {"type": "json_object"}})
 
-# QA system prompt and chain
-qa_system_prompt = """
-Prompt:
-You are a highly intelligent assistant. Use the following context to answer user questions. Analyze the data carefully and generate a clear, concise, and informative response to the user's question based on this data.
-
-Response Guidelines:
-    - Use only the information provided in the data to answer the question.
-    - Ensure the answer is accurate and directly related to the question.
-    - If the user question required no related data then give the truthful response to the user question.
-    - If the data is insufficient to answer the question, politey apologise and tell the user that there is insufficient data available to answer their question.
-    - Provide the response in a conversational yet friendly tone.
-
-You are an AI assistant developed by Nutrigenics AI, specializing in intelligent recipe information retrieval. Your purpose is to help users by recommending recipes, providing detailed nutritional values, listing ingredients, offering step-by-step cooking instructions, and filtering recipes based on provide context ans user query.
-
-Operational Guidelines:
-1. Input Structure:
-   - Context: You may receive contextual information related to recipes, such as specific data sets, user preferences, dietary restrictions, or previously selected dishes.
-   - User Query: Users will pose questions or requests related to recipes, nutritional information, ingredient substitutions, cooking instructions, and more.
-
-2. Response Strategy:
-   - Utilize Provided Context: If the context contains relevant information that addresses the user's query, base your response on this provided data to ensure accuracy and relevance.
-   - Respond to User Query Directly: If the context does not contain the necessary information to answer the user's query, generate a response based solely on the user's input and your trained knowledge.
-
-Core Functionalities:
-- Nutritional Information: Accurately provide nutritional values for each recipe, including calories, macronutrients (proteins, fats, carbohydrates), and essential vitamins and minerals, using contextual data when available.
-- Ingredient Details: List all ingredients required for recipes, including substitute options for dietary restrictions or ingredient availability, utilizing context when relevant.
-- Step-by-Step Cooking Instructions: Deliver clear, easy-to-follow instructions for preparing and cooking meals, informed by any provided contextual data.
-- Recipe Recommendations: Suggest dishes based on user preferences, dietary restrictions, available ingredients, and contextual data if provided.
-
-Additional Instructions:
-- Precision and Personalization: Always aim to provide precise, personalized, and relevant information to users based on both the provided context and their specific queries.
-- Clarity and Coherence: Ensure that all responses are clear, well-structured, and easy to understand, facilitating a seamless user experience.
-- Substitute Suggestions: When suggesting ingredient substitutes, consider user preferences and dietary restrictions outlined in the context or user query.
-- Dynamic Adaptation: Adapt your responses dynamically based on whether the context is relevant to the user's current request, ensuring optimal use of available information.
-
-Don't mention about context in the response, format the answer in a natural friedly way.
-
-Context:
-{context}
-"""
-qa_prompt = ChatPromptTemplate.from_messages(
-    [
-        ("system", qa_system_prompt),
-        ("human", "{input}")
-    ]
-)
-
-# Create the base chain
-base_chain = qa_prompt | llm | StrOutputParser()
-
-# Wrap the chain with message history
-question_answer_chain = RunnableWithMessageHistory(
-    base_chain,
-    lambda session_id: ChatMessageHistory(),  # This creates a new history for each session
-    input_messages_key="input",
-    history_messages_key="chat_history"
-)
+# Initialize Router
+answer_formatter = ChatGroq(model="llama-3.1-8b-instant", temperature=0, max_tokens=1024, max_retries=2)
 
+# Initialized recommendation LLM
+client = OpenAI()
 
 class StoppingCriteriaSub(StoppingCriteria):
 
@@ -149,7 +113,6 @@ def get_blip_config(model="base"):
 
     return config
 
-
 print("Creating model")
 config = get_blip_config("large")
 
@@ -165,22 +128,17 @@ model = blip_embs(
 
 model = model.to(device)
 model.eval()
-print("Model Loaded !")
-print("="*50)
 
 transform = transform_test(384)
 
-print("Loading Data")
-df = pd.read_json("datasets/sidechef/my_recipes.json")
+df = pd.read_json("my_recipes.json")
 
-print("Loading Target Embedding")
 tar_img_feats = []
 for _id in df["id_"].tolist():     
-    tar_img_feats.append(torch.load("datasets/sidechef/blip-embs-large/{:07d}.pth".format(_id)).unsqueeze(0))
+    tar_img_feats.append(torch.load("./datasets/sidechef/blip-embs-large/{:07d}.pth".format(_id)).unsqueeze(0))
 
 tar_img_feats = torch.cat(tar_img_feats, dim=0)
 
-
 class Chat:
 
     def __init__(self, model, transform, dataframe, tar_img_feats, device='cuda:0', stopping_criteria=None):
@@ -221,41 +179,393 @@ class Chat:
 
 
 chat = Chat(model,transform,df,tar_img_feats, device)
-print("Chat Initialized !")
 
 
-custom_css = """
-.primary{
-    background-color: #4CAF50; /* Green */
-}
-"""
+def answer_generator(formated_input, session_id):
+    # QA system prompt and chain
+    qa_system_prompt = """
+    You are an AI assistant developed by Nutrigenics AI, specializing in intelligent recipe information retrieval and recipe suggestions. Your purpose is to help users by recommending recipes, providing detailed nutritional values, listing ingredients, offering step-by-step cooking instructions, and filtering recipes based on provide context ans user query.
+    Operational Guidelines:
+    1. Input Structure:
+    - Context: You may receive contextual information related to recipes, such as specific data sets, user preferences, dietary restrictions, or previously selected dishes.
+    - User Query: Users will pose questions or requests related to recipes, nutritional information, ingredient substitutions, cooking instructions, and more.
+    2. Response Strategy:
+    - Utilize Provided Context: If the context contains relevant information that addresses the user's query, base your response on this provided data to ensure accuracy and relevance.
+    - Respond to User Query Directly: If the context does not contain the necessary information to answer the user's query, kindly state that you do not have require information.
+    Core Functionalities:
+    - Nutritional Information: Accurately provide nutritional values for each recipe, including calories, macronutrients (proteins, fats, carbohydrates), and essential vitamins and minerals, using contextual data when available.
+    - Ingredient Details: List all ingredients required for recipes, including substitute options for dietary restrictions or ingredient availability, utilizing context when relevant.
+    - Step-by-Step Cooking Instructions: Deliver clear, easy-to-follow instructions for preparing and cooking meals, informed by any provided contextual data.
+    - Recipe Recommendations: Suggest dishes based on user preferences, dietary restrictions, available ingredients, and contextual data if provided.
+    Additional Instructions:
+    - Precision and Personalization: Always aim to provide precise, personalized, and relevant information to users based on both the provided context and their specific queries.
+    - Clarity and Coherence: Ensure that all responses are clear, well-structured, and easy to understand, facilitating a seamless user experience.
+    - Substitute Suggestions: When suggesting ingredient substitutes, consider user preferences and dietary restrictions outlined in the context or user query.
+    - Dynamic Adaptation: Adapt your responses dynamically based on whether the context is relevant to the user's current request, ensuring optimal use of available information.
+    Don't mention about context in the response, format the answer in a natural and friendly way.
+    Context:
+    {context}
+    """
+    qa_prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", qa_system_prompt),
+            ("human", "{input}")
+        ]
+    )
+
+    # Create the base chain
+    base_chain = qa_prompt | llm | StrOutputParser()
+
+    # Wrap the chain with message history
+    question_answer_chain = RunnableWithMessageHistory(
+        base_chain,
+        lambda session_id: ChatMessageHistory(),  # This creates a new history for each session
+        input_messages_key="input",
+        history_messages_key="chat_history"
+    )
+
+    response = question_answer_chain.invoke(formated_input, config={"configurable": {"session_id": session_id}})
+
+    return response
+
+
+
+### Router
+import json
+from langchain_core.messages import HumanMessage, SystemMessage
+
+def router_node(query):
+    # Prompt
+    router_instructions = """You are an expert at determining the appropriate task for a user’s question based on chat history and the current query context. You have two available tasks:
+
+        1.	Retrieval: Fetch information based on user's chat history and current query.
+        2.	Recommendation/Suggestion: Recommend recipes to users based on the query.
+
+    Return a JSON response with a single key named “task” indicating either “retrieval” or “recommendation” based on your decision.
+    """
+    response = router.invoke(
+        [SystemMessage(content=router_instructions)]
+        + [
+            HumanMessage(
+                content=query
+            )
+        ]
+    )
+    res = json.loads(response.content)
+    return res['task']
+
+def recommendation_node(query):
+    prompt = """
+    You are a helpful assistant that writes Python code to filter recipes from a JSON filr based o the user query. \n
+    JSON file path = 'recipes.json' \n
+    The JSON file is a list of recipes with the following structure: \n
+    {
+        "recipe_name": string,
+        "recipe_time": integer,
+        "recipe_yields": string,
+        "recipe_ingredients": list of ingredients,
+        "recipe_instructions": list of instruections,
+        "recipe_image": string,
+        "blogger": string,
+        "recipe_nutrients": JSON object with key value pairs such as "protein: 10g",
+        "tags": list of tags related to recipe
+    } \n
+
+    Here is the example of an recipe json object from the JSON data: \n
+    {
+        "recipe_name": "Asian Potato Salad with Seven Minute Egg",
+        "recipe_time": 0,
+        "recipe_yields": "4 servings",
+        "recipe_ingredients": [
+            "2 1/2 cup Multi-Colored Fingerling Potato",
+            "3/4 cup Celery",
+            "1/4 cup Red Onion",
+            "2 tablespoon Fresh Parsley",
+            "1/3 cup Mayonnaise",
+            "1 tablespoon Chili Garlic Sauce",
+            "1 teaspoon Hoisin Sauce",
+            "1 splash Soy Sauce",
+            "to taste Salt",
+            "to taste Ground Black Pepper",
+            "4 Egg"
+        ],
+        "recipe_instructions": "Fill a large stock pot with water.\nAdd the Multi-Colored Fingerling Potato (2 1/2 cup) and bring water to a boil. Boil the potatoes for 20 minutes or until fork tender.\nDrain the potatoes and let them cool completely.\nMeanwhile, mix together in a small bowl Mayonnaise (1/3 cup), Chili Garlic Sauce (1 tablespoon), Hoisin Sauce (1 teaspoon), and Soy Sauce (1 splash).\nTo make the Egg (4), fill a stock pot with water and bring to a boil Gently add the eggs to the water and set a timer for seven minutes.\nThen move the eggs to an ice bath to cool completely. Once cooled, crack the egg slightly and remove the shell. Slice in half when ready to serve.\nNext, halve the cooled potatoes and place into a large serving bowl. Add the Ground Black Pepper (to taste), Celery (3/4 cup), Red Onion (1/4 cup), and mayo mixture. Toss to combine adding Salt (to taste) and Fresh Parsley (2 tablespoon).\nTop with seven minute eggs and serve. Enjoy!",
+        "recipe_image": "https://www.sidechef.com/recipe/eeeeeceb-493e-493d-8273-66c800821b13.jpg?d=1408x1120",
+        "blogger": "sidechef.com",
+        "recipe_nutrients": {
+            "calories": "80 calories",
+            "proteinContent": "2.1 g",
+            "fatContent": "6.2 g",
+            "carbohydrateContent": "3.9 g",
+            "fiberContent": "0.5 g",
+            "sugarContent": "0.4 g",
+            "sodiumContent": "108.0 mg",
+            "saturatedFatContent": "1.2 g",
+            "transFatContent": "0.0 g",
+            "cholesterolContent": "47.4 mg",
+            "unsaturatedFatContent": "3.8 g"
+        },
+        "tags": [
+            "Salad",
+            "Lunch",
+            "Brunch",
+            "Appetizers",
+            "Side Dish",
+            "Budget-Friendly",
+            "Vegetarian",
+            "Pescatarian",
+            "Eggs",
+            "Potatoes",
+            "Dairy-Free",
+            "Shellfish-Free"
+        ]
+    } \n
+
+    Based on the user query, provide a Python function to filter the JSON data. The output of the function should be a list of json objects. \n
+
+    Recipe filtering instructions:
+    - If a user asked for the highest nutrient recipe such as "high protein or high calories" then filtered recipes should be the top highest recipes from all the recipes with high nutrient.
+    - sort or rearrange recipes based which recipes are more appropriate for the user.
+
+    Your output instructions:
+    - The function name should be filter_recipes. The input to the function should be file name.
+    - The length of output recipes should not be more than 6.
+    - Only give me output function. Do not call the function.
+    - Give the python function as a key named "code" in a json format.
+    - Do not include any other text with the output, only give python code.
+    - If you do not follow the above given instructions, the chat may be terminated.
+    """
+    max_tries = 3
+    while True:
+        try:
+            # llm = ChatGroq(model="llama-3.1-8b-instant", temperature=0, max_tokens=1024, max_retries=2, model_kwargs={"response_format": {"type": "json_object"}})
+            response = client.chat.completions.create(
+                model="gpt-4o-mini",
+                messages=[
+                    {"role": "system", "content": prompt},
+                    {
+                        "role": "user",
+                        "content": query
+                    }
+                ]
+            )
+
+            content = response.choices[0].message.content
+
+            res = json.loads(content)
+            script = res['code']
+            exec(script, globals())
+            filtered_recipes = filter_recipes('recipes.json')
+            if len(filtered_recipes) > 0:
+                return filtered_recipes
+        except Exception as e:
+            print(e)
+            if max_tries <= 0:
+                return []
+            else:
+                max_tries -= 1
+    return filtered_recipes
+
+
+def answer_formatter_node(question, context):
+    prompt = f""" You are an highly clever question-answering assistant trained to provide clear and concise answers based on the user query and provided context. 
+    Your task is to generated answers for the user query based on the context provided.
+    Instructions for your response:
+    1. Directly answer the user query using only the information provided in the context.
+    2. Ensure your response is clear and concise.
+    3. Mention only details related to the recipe, including the recipe name, instructions, nutrients, yield, ingredients, and image.
+    4. Do not include any information that is not related to the recipe context.
+
+    Please format an answer based on the following user question and context provided:
+
+    User Question: 
+    {question}
+
+    Context:
+    {context}
+    """
+    response = answer_formatter.invoke(
+        [SystemMessage(content=prompt)]
+    )
+    res = response.content
+    return res
 
+CURR_CONTEXT = ''
 
 # @spaces.GPU
-def respond_to_user(image, message):
-    # Process the image and message here
-    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    chat = Chat(model,transform,df,tar_img_feats, device)
-    chat.encode_image(image)
-    data = chat.ask()
-    formated_input = {
-        'input': message,
-        'context': data
-    }
-    try:
-        response = question_answer_chain.invoke(formated_input, config={"configurable": {"session_id": 'abc123'}})
-    except Exception as e:
-        print(e)
-        response = {'content':"An error occurred while processing your request."}
-    return response, data
-
-iface = gr.Interface(
-    fn=respond_to_user,
-    inputs=[gr.Image(), gr.Textbox(label="Ask Query")],
-    outputs=[gr.Textbox(label="Nutrition-GPT"), gr.JSON(label="context")],
-    title="Nutrition-GPT Demo",
-    description="Upload an food image and ask queries!",
-    css=".component-12 {background-color: red}",
-)
-
-iface.launch()
+def get_answer(image=[], message='', sessionID='abc123'):
+    global CURR_CONTEXT
+    if len(image) > 0:
+        try:
+            # Process the image and message here
+            device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+            chat = Chat(model,transform,df,tar_img_feats, device)
+            chat.encode_image(image)
+            data = chat.ask()
+            CURR_CONTEXT = data
+            formated_input = {
+                'input': message,
+                'context': data
+            }
+            response = answer_generator(formated_input, session_id=sessionID)
+        except Exception as e:
+            print(e)
+            response = {'content':"An error occurred while processing your request."}
+    elif len(image) == 0 and message is not None:
+        print("I am here")
+        task = router_node(message)
+        if task == 'retrieval':
+            recipes = recommendation_node(message)
+            print(recipes)
+            if not recipes:
+                response = {'content':"An error occurred while processing your request."}
+            response = answer_formatter_node(message, recipes)
+        else:
+            formated_input = {
+                'input': message,
+                'context': CURR_CONTEXT
+            }
+            response = answer_generator(formated_input, session_id=sessionID)
+
+    return response
+
+# Function to handle WebSocket connection
+@socketio.on('ping')
+def handle_connect():
+    emit('Ping-return', {'message': 'Connected'}, room=request.sid)
+
+
+# Function to handle WebSocket connection
+@socketio.on('connect')
+def handle_connect():
+    print(f"Client connected: {request.sid}")
+
+# Function to handle WebSocket disconnection
+@socketio.on('disconnect')
+def handle_disconnect():
+    print(f"Client disconnected: {request.sid}")
+
+import json
+import base64
+from PIL import Image
+from io import BytesIO
+import torchvision.transforms as transforms
+
+# Dictionary to store incomplete image data by session
+session_store = {}
+
+@socketio.on('message')
+def handle_message(data):
+    global session_store
+    global CURR_CONTEXT
+    context = "No data available"
+    session_id = request.sid
+    if session_id not in session_store:
+        session_store[session_id] = {'image_data': b"", 'message': None, 'image_received': False}
+
+    if 'message' in data:
+        session_store[session_id]['message'] = data['message']
+
+    # Handle image chunk data
+    if 'image' in data:
+        try:
+            # Append the incoming image chunk
+            session_store[session_id]['image_data'] += data['image']
+
+        except Exception as e:
+            print(f"Error processing image chunk: {str(e)}")
+            emit('response', "An error occurred while receiving the image chunk.", room=session_id)
+            return
+        
+        if session_store[session_id]['image_data'] or session_store[session_id]['message']:
+            try:
+                image_bytes = session_store[session_id]['image_data']
+                # print("checkpoint 2")
+                if isinstance(image_bytes, str):
+                    image_bytes = base64.b64decode(image_bytes)
+                image = Image.open(BytesIO(image_bytes))
+                image_array = np.array(image)
+                device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+                chat = Chat(model, transform, df, tar_img_feats, device)
+                chat.encode_image(image_array)
+                context = chat.ask()
+                CURR_CONTEXT = context
+                message = data['message']
+                formated_input = {
+                    'input': message,
+                    'context': json.dumps(context)
+                }
+                # Invoke question_answer_chain and stream the response
+                response = answer_generator(formated_input, session_id=session_id)
+                emit('response', response, room=session_id)
+
+            except Exception as e:
+                print(f"Error processing image or message: {str(e)}")
+                emit('response', "An error occurred while processing your request.", room=session_id)
+                return
+            finally:
+                # Clear session data after processing
+                session_store.pop(session_id, None)
+    else:
+        message = data['message']
+        task = router_node(message)
+        print(task)
+        if task == 'retrieval':
+            formated_input = {
+                'input': message,
+                'context': json.dumps(CURR_CONTEXT)
+            }
+            response = answer_generator(formated_input, session_id=session_id)
+            emit('response', response, room=session_id)
+        else:
+            response = recommendation_node(message)
+            print(response)
+            # response = answer_formatter_node(message, recipes)
+            if response is None:
+                response = {'content':"An error occurred while processing your request."}
+
+            emit('json_response', response, room=session_id)
+        session_store.pop(session_id, None)
+    
+
+
+import requests
+from PIL import Image
+import numpy as np
+from io import BytesIO
+
+def download_image_to_numpy(url):
+    print("Image URL: ", url)
+    # Send a GET request to the URL to download the image
+    response = requests.get(url)
+    
+    # Check if the request was successful
+    if response.status_code == 200:
+        # Open the image using PIL and convert it to RGB format
+        image = Image.open(BytesIO(response.content)).convert('RGB')
+        
+        # Convert the image to a NumPy array
+        image_array = np.array(image)
+        
+        return image_array
+    else:
+        raise Exception(f"Failed to download image. Status code: {response.status_code}")
+
+@socketio.on('example')
+def handle_message(data):
+    img_url = data['img_url']
+    message = data['message']
+    session_id = request.sid
+    image_array = download_image_to_numpy(img_url)
+    response = get_answer(image=image_array, message=message, sessionID=request.sid)
+    emit('response', response, room=session_id)
+    return response
+
+# Home route
+@app.route("/")
+def index_view():
+    return render_template('chat.html') 
+
+# Main function to run the app
+if __name__ == '__main__': 
+    socketio.run(app, debug=False)