get_keywords.py

from openai import OpenAI
from rake_nltk import Rake
import nltk
nltk.download('stopwords')
nltk.download('punkt')

# Initialize OpenAI client
client = OpenAI()

"""
This function takes in user query and returns keywords
Input:
    user_query: str
    keyword_type: str (openai, rake, or na)
        If the keyword type is na, then user query is returned.
Output: keywords: str
"""
def get_keywords(user_query: str, keyword_type: str) -> str:
    if keyword_type == "openai":
        return get_keywords_openai(user_query)
    if keyword_type == "rake":
        return get_keywords_rake(user_query)
    else:
        return user_query


"""
This function takes user query and returns keywords using rake_nltk
rake_nltk actually returns keyphrases, not keywords. Since using keyphrases did not show improvement, we are using keywords
to match the output type of the other keyword functions.
Input:
    user_query: str
Output: keywords: str
"""
def get_keywords_rake(user_query: str) -> str:
    r = Rake()
    r.extract_keywords_from_text(user_query)
    keyphrases = r.get_ranked_phrases()

    # If we want to get keyphrases, return keyphrases but should do keywords
    out = ""
    for phrase in keyphrases:
        out += phrase + " "
    return out


"""
This function takes user query and returns keywords using openai
Input:
    user_query: str
Output: keywords: str
"""
def get_keywords_openai(user_query: str) -> str:
    messages = [
        {"role": "system", "content": "return the keywords of the following query. response should be words separated by commas."},
        {"role": "user", "content": user_query}
    ]
    
    completion = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=messages,
        temperature=0
    )
    response = completion.choices[0].message.content
    res = response.replace(",", "")
    return res