File size: 13,075 Bytes
28183db 164cb45 a746976 28183db 51082bd 164cb45 28183db 164cb45 28183db a746976 28183db 51082bd 28183db a746976 28183db 228ea6c 28183db 228ea6c 28183db 4ebac20 28183db 62f4be8 28183db |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 |
import time
import random
import logging
import streamlit as st
import pandas as pd
from dotenv import load_dotenv
import utils
import db
import modeling
import plots
def set_if_not_in_session_state(key, value):
"""Helper function to initialize a session state variable if it doesn't exist."""
if key not in st.session_state:
st.session_state[key] = value
def initialize():
"""Initialization function to set up logging, load environment variables, and initialize session state variables."""
load_dotenv()
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO)
keys = ['selected_rating', 'collect_data', 'gender_value', 'expert_value', 'show_launch', 'user_id', 'statements', 'current_statement', 'db']
values = [0, None, None, None, True, random.randint(1, 999_999_999), None, None, None]
for key, value in zip(keys, values):
set_if_not_in_session_state(key, value)
connect_to_database()
def connect_to_database():
"""Establishes a connection to the database."""
if st.session_state.db is None:
credentials_dict = db.load_credentials()
connection_attempts = 0
while st.session_state.db is None and connection_attempts < 3:
st.session_state.db = db.connect_to_db(credentials_dict)
if st.session_state.db is None:
logging.info('Retrying to connect to db...')
connection_attempts += 1
time.sleep(1)
else:
retrieve_statements()
def retrieve_statements():
"""Retrieves statements from the database."""
retrieval_attempts = 0
while st.session_state.statements is None and retrieval_attempts < 3:
st.session_state.statements = db.get_statements_from_db(st.session_state.db)
st.session_state.current_statement = db.pick_random(st.session_state.statements)
if st.session_state.statements is None:
logging.info('Retrying to retrieve statements from db...')
retrieval_attempts += 1
time.sleep(1)
def get_user_consent():
st.markdown("""
### Support Future Research
Additionally, we kindly ask for your agreement to collect anonymous data from your app usage in order to improve future research.
You may choose to agree or decline this data collection.
""")
collect_data_options = ['Yes, I agree and want to support and help improve this research', 'No']
collect_data_input = st.radio(
label='You may choose to agree or decline this data collection.',
options=collect_data_options,
horizontal=True,
label_visibility='collapsed'
)
return collect_data_options.index(collect_data_input) == 0
def get_user_info():
gender_options = ['[Please select]', 'Female', 'Male', 'Other']
gender_input = st.selectbox(
label='Please select your gender',
options=gender_options,
)
gender_value = gender_options.index(gender_input)
expert_options = [
'[PLEASE SELECT]',
'No, I do not have a background in social or behavioral sciences',
'Yes, I have either studied social or behavioral sciences or I am currently a student in this field',
'Yes, I have either worked as a researcher in the field of social or behavioral sciences or I have had past experience as a researcher in this area'
]
expert_input = st.selectbox(
label='Please indicate whether you have any experience or educational background in social or behavioral sciences (e.g., psychology)',
options=expert_options,
)
expert_value = expert_options.index(expert_input)
return expert_value, gender_value
def get_user_rating(placeholder):
with placeholder:
with st.container():
st.markdown(f"""
### How desirable is the following statement?
To support future research, rate the following statement according to whether it is socially desirable or undesirable.
Is it socially desirable or undesirable to endorse the following statement?
#### <center>\"{st.session_state.current_statement.capitalize()}\"</center>
""", unsafe_allow_html=True)
rating_options = ['[Please select]', 'Very undesirable', 'Undesirable', 'Neutral', 'Desirable', 'Very desirable']
selected_rating = st.selectbox(
label='Rate the statement above according to whether it is socially desirable or undesirable.',
options=rating_options,
key='selection'
)
suitability_options = ['No, I\'m just playing around', 'Yes, my input can help improve this research']
research_suitability = st.radio(
label='Is your input suitable for research purposes?',
options=suitability_options,
horizontal=True
)
st.session_state.collect_data_optout = st.checkbox(
label='Don\'t ask me to rate further statements.',
value=False
)
st.session_state.item_rating = rating_options.index(selected_rating)
st.session_state.suitability_rating = suitability_options.index(research_suitability)
def handle_acceptance(collect_data_value, expert_value, gender_value, message):
if st.button(label='Accept Disclaimer', type='primary', use_container_width=True):
if collect_data_value and not (expert_value > 0 and gender_value > 0):
message.error('Please answer the questions above!')
else:
st.session_state.expert_value = expert_value
st.session_state.gender_value = gender_value
st.session_state.show_launch = False
st.session_state.collect_data = collect_data_value
st.experimental_rerun()
def show_launch(placeholder):
with placeholder:
with st.container():
st.divider()
st.markdown("""
## Before Using the App
### Disclaimer
This application is provided as-is, without any warranty or guarantee of any kind, expressed or implied. It is intended for educational, non-commercial use only.
The developers of this app shall not be held liable for any damages or losses incurred from its use. By using this application, you agree to the terms and conditions
outlined herein and acknowledge that any commercial use or reliance on its functionality is strictly prohibited.
""")
collect_data_value = False
if st.session_state.db:
collect_data_value = get_user_consent()
expert_value, gender_value = (0, 0)
if collect_data_value:
expert_value, gender_value = get_user_info()
message = st.empty()
handle_acceptance(collect_data_value, expert_value, gender_value, message)
def show_summary(placeholder):
with placeholder:
with st.container():
st.markdown("""
## What is the focus of this research?
Certain biases can affect how people respond to surveys and psychological questionnaires.
For example, survey respondents may attempt to conceal socially undesirable traits (e.g.,
being ill-tempered) and endorse statements that cast them in a favorable manner (e.g.,
being cooperative).
Developers of psychological questionnaires hence sometimes aim to ensure that questions
are neutral, or that a subset of questions is equally (un)desirable. In the past, human
judges have been tasked with quantifying item desirability. In contrast, the research
underlying this web application demonstrates that large language models (LLMs) can
achieve this too!
""")
def handle_demo_input():
if st.session_state.collect_data:
if st.session_state.item_rating > 0:
st.session_state.sentiment, st.session_state.desirability = modeling.score_text(st.session_state.input_text)
payload = {
'user_id': st.session_state.user_id,
'gender_value': st.session_state.gender_value,
'expert_value': st.session_state.expert_value,
'statement': st.session_state.current_statement,
'rating': st.session_state.item_rating,
'suitability': st.session_state.suitability_rating,
'input_text': st.session_state.input_text,
'sentiment': st.session_state.sentiment,
'desirability': st.session_state.desirability,
}
write_to_db_success = db.write_to_db(st.session_state.db, payload)
if st.session_state.collect_data_optout:
st.session_state.collect_data = False
if write_to_db_success:
st.session_state.current_statement = db.pick_random(st.session_state.statements)
st.session_state.selection = '[Please select]'
else:
return None
else:
st.session_state.sentiment, st.session_state.desirability = modeling.score_text(st.session_state.input_text)
def show_demo(placeholder):
with placeholder:
with st.container():
st.divider()
st.markdown("""
## Try it yourself!
Use the text field below to enter a statement that might be part of a psychological
questionnaire (e.g., "I love a good fight."). Your input will be processed by
language models, returning a machine-based estimate of item sentiment (i.e., valence)
and desirability.
""")
modeling.load_model()
if 'sentiment' in st.session_state and 'desirability' in st.session_state:
plots.show_scores(
sentiment=st.session_state.sentiment,
desirability=st.session_state.desirability,
input_text=st.session_state.input_text
)
st.session_state.input_text = st.text_input(
label='Item text/statement:',
value='I love a good fight.',
placeholder='Enter item text'
)
user_rating_placeholder = st.empty()
if st.session_state.collect_data:
get_user_rating(user_rating_placeholder)
if st.button(
label='Evaluate Item Text',
on_click=handle_demo_input,
type='primary',
use_container_width=True
):
if st.session_state.collect_data and st.session_state.item_rating == 0:
st.error('Please rate the statement presented above!')
def show_data(placeholder):
with placeholder:
with st.container():
st.divider()
st.markdown("""
## Explore the data
Figures show the accuarcy in precitions of human-rated item desirability by the sentiment model (left) and the desirability model (right), using `test`-partition data only.
""")
show_covariates = st.checkbox('Show covariates', value=True)
if show_covariates:
option = st.selectbox('Group by', options=list(utils.covariate_columns.values()))
else:
option = None
if 'df' in st.session_state:
plot = plots.scatter_plot(st.session_state.df, option)
st.plotly_chart(plot, theme=None, use_container_width=True)
def main():
st.markdown("""
# Machine-Based Item Desirability Ratings
This web application demonstrates how item desirability ratings can be obtained with natural language processing ("AI") and accompanies the paper "*Expanding the Methodological Toolbox: Machine-Based Item Desirability Ratings as an Alternative to Human-Based Ratings*".
*Hommel, B. E. (2023). Expanding the methodological toolbox: Machine-based item desirability ratings as an alternative to human-based ratings. Personality and Individual Differences, 213, 112307. https://doi.org/10.1016/j.paid.2023.112307*
<small>https://www.magnolia-psychometrics.com/</small>
""", unsafe_allow_html=True)
placeholder_launch = st.empty()
placeholder_summary = st.empty()
placeholder_demo = st.empty()
placeholder_data = st.empty()
if st.session_state.show_launch is True:
show_launch(placeholder_launch)
else:
placeholder_launch = st.empty()
show_summary(placeholder_summary)
show_demo(placeholder_demo)
show_data(placeholder_data)
if __name__ == '__main__':
initialize()
main() |