Upload 5 files

Untitled.ipynb

@@ -0,0 +1,1220 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "c6c16352",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "9364c142",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_csv('stress.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "d74d44fa",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>subreddit</th>\n",
+       "      <th>post_id</th>\n",
+       "      <th>sentence_range</th>\n",
+       "      <th>text</th>\n",
+       "      <th>label</th>\n",
+       "      <th>confidence</th>\n",
+       "      <th>social_timestamp</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ptsd</td>\n",
+       "      <td>8601tu</td>\n",
+       "      <td>(15, 20)</td>\n",
+       "      <td>He said he had not felt that way before, sugge...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.8</td>\n",
+       "      <td>1521614353</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>assistance</td>\n",
+       "      <td>8lbrx9</td>\n",
+       "      <td>(0, 5)</td>\n",
+       "      <td>Hey there r/assistance, Not sure if this is th...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>1527009817</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>ptsd</td>\n",
+       "      <td>9ch1zh</td>\n",
+       "      <td>(15, 20)</td>\n",
+       "      <td>My mom then hit me with the newspaper and it s...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.8</td>\n",
+       "      <td>1535935605</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "    subreddit post_id sentence_range   \n",
+       "0        ptsd  8601tu       (15, 20)  \\\n",
+       "1  assistance  8lbrx9         (0, 5)   \n",
+       "2        ptsd  9ch1zh       (15, 20)   \n",
+       "\n",
+       "                                                text  label  confidence   \n",
+       "0  He said he had not felt that way before, sugge...      1         0.8  \\\n",
+       "1  Hey there r/assistance, Not sure if this is th...      0         1.0   \n",
+       "2  My mom then hit me with the newspaper and it s...      1         0.8   \n",
+       "\n",
+       "   social_timestamp  \n",
+       "0        1521614353  \n",
+       "1        1527009817  \n",
+       "2        1535935605  "
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.head(3)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "a9ab0f47",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'He said he had not felt that way before, suggeted I go rest and so ..TRIGGER AHEAD IF YOUI\\'RE A HYPOCONDRIAC LIKE ME: i decide to look up \"feelings of doom\" in hopes of maybe getting sucked into some rabbit hole of ludicrous conspiracy, a stupid \"are you psychic\" test or new age b.s., something I could even laugh at down the road. No, I ended up reading that this sense of doom can be indicative of various health ailments; one of which I am prone to.. So on top of my \"doom\" to my gloom..I am now f\\'n worried about my heart. I do happen to have a physical in 48 hours.'"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df['text'][0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "cddf65aa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import nltk\n",
+    "import re\n",
+    "from urllib.parse import urlparse\n",
+    "from spacy import load\n",
+    "from nltk.stem import WordNetLemmatizer\n",
+    "from nltk.corpus import stopwords\n",
+    "from nltk.tokenize import word_tokenize"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "183c5b14",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "cp: /usr/share/nltk_data/corpora/wordnet2022: No such file or directory\r\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[nltk_data] Downloading package omw-1.4 to\n",
+      "[nltk_data]     /Users/nileshpal/nltk_data...\n",
+      "[nltk_data]   Package omw-1.4 is already up-to-date!\n",
+      "[nltk_data] Downloading package wordnet to\n",
+      "[nltk_data]     /Users/nileshpal/nltk_data...\n",
+      "[nltk_data]   Package wordnet is already up-to-date!\n",
+      "[nltk_data] Downloading package wordnet2022 to\n",
+      "[nltk_data]     /Users/nileshpal/nltk_data...\n",
+      "[nltk_data]   Package wordnet2022 is already up-to-date!\n",
+      "[nltk_data] Downloading package punkt to /Users/nileshpal/nltk_data...\n",
+      "[nltk_data]   Package punkt is already up-to-date!\n",
+      "[nltk_data] Downloading package stopwords to\n",
+      "[nltk_data]     /Users/nileshpal/nltk_data...\n",
+      "[nltk_data]   Package stopwords is already up-to-date!\n"
+     ]
+    }
+   ],
+   "source": [
+    "nltk.download('omw-1.4')\n",
+    "nltk.download('wordnet') \n",
+    "nltk.download('wordnet2022')\n",
+    "nltk.download('punkt')\n",
+    "nltk.download('stopwords')\n",
+    "! cp -rf /usr/share/nltk_data/corpora/wordnet2022 /usr/share/nltk_data/corpora/wordnet"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "473ea714",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn', \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn', \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \"won't\", 'wouldn', \"wouldn't\"]\n"
+     ]
+    }
+   ],
+   "source": [
+    "lemmatizer = WordNetLemmatizer()\n",
+    "stop_words = list(stopwords.words('english'))\n",
+    "print(stop_words)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "d119482b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def textPocess(sent):\n",
+    "    try:\n",
+    "        sent = re.sub('[][)(]',' ',sent)\n",
+    "\n",
+    "        sent = [word for word in sent.split() if not urlparse(word).scheme]\n",
+    "        sent = ' '.join(sent)\n",
+    "\n",
+    "\n",
+    "        sent = re.sub(r'\\@\\w+','',sent)\n",
+    "\n",
+    "\n",
+    "        sent = re.sub(re.compile(\"<.*?>\"),'',sent)\n",
+    "\n",
+    "        sent = re.sub(\"[^A-Za-z0-9]\",' ',sent)\n",
+    "\n",
+    "        sent = sent.lower()\n",
+    "        \n",
+    "        sent = [word.strip() for word in sent.split()]\n",
+    "        sent = ' '.join(sent)\n",
+    "\n",
+    "        tokens = word_tokenize(sent)\n",
+    "        \n",
+    "        for word in tokens:\n",
+    "            if word in stop_words:\n",
+    "                tokens.remove(word)\n",
+    "        \n",
+    "        sent = [lemmatizer.lemmatize(word) for word in tokens]\n",
+    "        sent = ' '.join(sent)\n",
+    "        return sent\n",
+    "    \n",
+    "    except Exception as ex:\n",
+    "        print(sent,\"\\n\")\n",
+    "        print(\"Error \",ex)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "f2b42f59",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df['processed_text'] = df['text'].apply(lambda text: textPocess(text))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "074cb312",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+    "MIN_DF = 1 "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "39f0bd84",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([[0, 0, 0, ..., 0, 0, 0],\n",
+       "       [0, 0, 0, ..., 0, 0, 0],\n",
+       "       [0, 0, 0, ..., 0, 0, 0],\n",
+       "       ...,\n",
+       "       [0, 0, 0, ..., 0, 0, 0],\n",
+       "       [0, 0, 0, ..., 0, 0, 0],\n",
+       "       [0, 0, 0, ..., 0, 0, 0]])"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from sklearn.feature_extraction.text import CountVectorizer\n",
+    "cv = CountVectorizer(min_df=MIN_DF)\n",
+    "cv_df = cv.fit_transform(df['processed_text'])\n",
+    "cv_df.toarray()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "3c43aa15",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>00</th>\n",
+       "      <th>000</th>\n",
+       "      <th>02</th>\n",
+       "      <th>06</th>\n",
+       "      <th>10</th>\n",
+       "      <th>100</th>\n",
+       "      <th>1000</th>\n",
+       "      <th>100kg</th>\n",
+       "      <th>100mg</th>\n",
+       "      <th>100x</th>\n",
+       "      <th>...</th>\n",
+       "      <th>zines</th>\n",
+       "      <th>zinsser</th>\n",
+       "      <th>zip</th>\n",
+       "      <th>zofran</th>\n",
+       "      <th>zoloft</th>\n",
+       "      <th>zombie</th>\n",
+       "      <th>zone</th>\n",
+       "      <th>zoo</th>\n",
+       "      <th>zuko</th>\n",
+       "      <th>zumba</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>3 rows × 10267 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   00  000  02  06  10  100  1000  100kg  100mg  100x  ...  zines  zinsser   \n",
+       "0   0    0   0   0   0    0     0      0      0     0  ...      0        0  \\\n",
+       "1   0    0   0   0   0    0     0      0      0     0  ...      0        0   \n",
+       "2   0    0   0   0   0    0     0      0      0     0  ...      0        0   \n",
+       "\n",
+       "   zip  zofran  zoloft  zombie  zone  zoo  zuko  zumba  \n",
+       "0    0       0       0       0     0    0     0      0  \n",
+       "1    0       0       0       0     0    0     0      0  \n",
+       "2    0       0       0       0     0    0     0      0  \n",
+       "\n",
+       "[3 rows x 10267 columns]"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "cv_df = pd.DataFrame(cv_df.toarray(),columns=cv.get_feature_names_out())\n",
+    "cv_df.head(3)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "10526ca5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
+       "       [0., 0., 0., ..., 0., 0., 0.],\n",
+       "       [0., 0., 0., ..., 0., 0., 0.],\n",
+       "       ...,\n",
+       "       [0., 0., 0., ..., 0., 0., 0.],\n",
+       "       [0., 0., 0., ..., 0., 0., 0.],\n",
+       "       [0., 0., 0., ..., 0., 0., 0.]])"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tf = TfidfVectorizer(min_df=MIN_DF)\n",
+    "tf_df = tf.fit_transform(df['processed_text'])\n",
+    "tf_df.toarray()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "2de0d172",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['tfidf_vectorizer.joblib']"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import joblib\n",
+    "joblib.dump(tf, 'tfidf_vectorizer.joblib')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "869aefb8",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>00</th>\n",
+       "      <th>000</th>\n",
+       "      <th>02</th>\n",
+       "      <th>06</th>\n",
+       "      <th>10</th>\n",
+       "      <th>100</th>\n",
+       "      <th>1000</th>\n",
+       "      <th>100kg</th>\n",
+       "      <th>100mg</th>\n",
+       "      <th>100x</th>\n",
+       "      <th>...</th>\n",
+       "      <th>zines</th>\n",
+       "      <th>zinsser</th>\n",
+       "      <th>zip</th>\n",
+       "      <th>zofran</th>\n",
+       "      <th>zoloft</th>\n",
+       "      <th>zombie</th>\n",
+       "      <th>zone</th>\n",
+       "      <th>zoo</th>\n",
+       "      <th>zuko</th>\n",
+       "      <th>zumba</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>3 rows × 10267 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "    00  000   02   06   10  100  1000  100kg  100mg  100x  ...  zines   \n",
+       "0  0.0  0.0  0.0  0.0  0.0  0.0   0.0    0.0    0.0   0.0  ...    0.0  \\\n",
+       "1  0.0  0.0  0.0  0.0  0.0  0.0   0.0    0.0    0.0   0.0  ...    0.0   \n",
+       "2  0.0  0.0  0.0  0.0  0.0  0.0   0.0    0.0    0.0   0.0  ...    0.0   \n",
+       "\n",
+       "   zinsser  zip  zofran  zoloft  zombie  zone  zoo  zuko  zumba  \n",
+       "0      0.0  0.0     0.0     0.0     0.0   0.0  0.0   0.0    0.0  \n",
+       "1      0.0  0.0     0.0     0.0     0.0   0.0  0.0   0.0    0.0  \n",
+       "2      0.0  0.0     0.0     0.0     0.0   0.0  0.0   0.0    0.0  \n",
+       "\n",
+       "[3 rows x 10267 columns]"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tf_df = pd.DataFrame(tf_df.toarray(),columns=tf.get_feature_names_out())\n",
+    "tf_df.head(3)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "95bddea7",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>00</th>\n",
+       "      <th>000</th>\n",
+       "      <th>02</th>\n",
+       "      <th>06</th>\n",
+       "      <th>10</th>\n",
+       "      <th>100</th>\n",
+       "      <th>1000</th>\n",
+       "      <th>100kg</th>\n",
+       "      <th>100mg</th>\n",
+       "      <th>100x</th>\n",
+       "      <th>...</th>\n",
+       "      <th>zines</th>\n",
+       "      <th>zinsser</th>\n",
+       "      <th>zip</th>\n",
+       "      <th>zofran</th>\n",
+       "      <th>zoloft</th>\n",
+       "      <th>zombie</th>\n",
+       "      <th>zone</th>\n",
+       "      <th>zoo</th>\n",
+       "      <th>zuko</th>\n",
+       "      <th>zumba</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>count</th>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>...</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "      <td>2838.000000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>mean</th>\n",
+       "      <td>0.000452</td>\n",
+       "      <td>0.000548</td>\n",
+       "      <td>0.000109</td>\n",
+       "      <td>0.000069</td>\n",
+       "      <td>0.003342</td>\n",
+       "      <td>0.001784</td>\n",
+       "      <td>0.000576</td>\n",
+       "      <td>0.000106</td>\n",
+       "      <td>0.000115</td>\n",
+       "      <td>0.000067</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.000078</td>\n",
+       "      <td>0.000072</td>\n",
+       "      <td>0.000204</td>\n",
+       "      <td>0.000078</td>\n",
+       "      <td>0.000715</td>\n",
+       "      <td>0.000126</td>\n",
+       "      <td>0.000245</td>\n",
+       "      <td>0.000089</td>\n",
+       "      <td>0.000054</td>\n",
+       "      <td>0.000040</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>std</th>\n",
+       "      <td>0.011158</td>\n",
+       "      <td>0.009998</td>\n",
+       "      <td>0.005801</td>\n",
+       "      <td>0.003671</td>\n",
+       "      <td>0.021379</td>\n",
+       "      <td>0.017215</td>\n",
+       "      <td>0.011156</td>\n",
+       "      <td>0.005624</td>\n",
+       "      <td>0.004534</td>\n",
+       "      <td>0.003573</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.004145</td>\n",
+       "      <td>0.003841</td>\n",
+       "      <td>0.007786</td>\n",
+       "      <td>0.004157</td>\n",
+       "      <td>0.011797</td>\n",
+       "      <td>0.004733</td>\n",
+       "      <td>0.006851</td>\n",
+       "      <td>0.004754</td>\n",
+       "      <td>0.002873</td>\n",
+       "      <td>0.002105</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>min</th>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25%</th>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>50%</th>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>75%</th>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>max</th>\n",
+       "      <td>0.348349</td>\n",
+       "      <td>0.327600</td>\n",
+       "      <td>0.309059</td>\n",
+       "      <td>0.195550</td>\n",
+       "      <td>0.259369</td>\n",
+       "      <td>0.267281</td>\n",
+       "      <td>0.310333</td>\n",
+       "      <td>0.299611</td>\n",
+       "      <td>0.215011</td>\n",
+       "      <td>0.190366</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.220793</td>\n",
+       "      <td>0.204637</td>\n",
+       "      <td>0.336077</td>\n",
+       "      <td>0.221471</td>\n",
+       "      <td>0.306537</td>\n",
+       "      <td>0.183347</td>\n",
+       "      <td>0.268149</td>\n",
+       "      <td>0.253283</td>\n",
+       "      <td>0.153067</td>\n",
+       "      <td>0.112136</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>8 rows × 10267 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                00          000           02           06           10   \n",
+       "count  2838.000000  2838.000000  2838.000000  2838.000000  2838.000000  \\\n",
+       "mean      0.000452     0.000548     0.000109     0.000069     0.003342   \n",
+       "std       0.011158     0.009998     0.005801     0.003671     0.021379   \n",
+       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
+       "25%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
+       "50%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
+       "75%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
+       "max       0.348349     0.327600     0.309059     0.195550     0.259369   \n",
+       "\n",
+       "               100         1000        100kg        100mg         100x  ...   \n",
+       "count  2838.000000  2838.000000  2838.000000  2838.000000  2838.000000  ...  \\\n",
+       "mean      0.001784     0.000576     0.000106     0.000115     0.000067  ...   \n",
+       "std       0.017215     0.011156     0.005624     0.004534     0.003573  ...   \n",
+       "min       0.000000     0.000000     0.000000     0.000000     0.000000  ...   \n",
+       "25%       0.000000     0.000000     0.000000     0.000000     0.000000  ...   \n",
+       "50%       0.000000     0.000000     0.000000     0.000000     0.000000  ...   \n",
+       "75%       0.000000     0.000000     0.000000     0.000000     0.000000  ...   \n",
+       "max       0.267281     0.310333     0.299611     0.215011     0.190366  ...   \n",
+       "\n",
+       "             zines      zinsser          zip       zofran       zoloft   \n",
+       "count  2838.000000  2838.000000  2838.000000  2838.000000  2838.000000  \\\n",
+       "mean      0.000078     0.000072     0.000204     0.000078     0.000715   \n",
+       "std       0.004145     0.003841     0.007786     0.004157     0.011797   \n",
+       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
+       "25%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
+       "50%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
+       "75%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
+       "max       0.220793     0.204637     0.336077     0.221471     0.306537   \n",
+       "\n",
+       "            zombie         zone          zoo         zuko        zumba  \n",
+       "count  2838.000000  2838.000000  2838.000000  2838.000000  2838.000000  \n",
+       "mean      0.000126     0.000245     0.000089     0.000054     0.000040  \n",
+       "std       0.004733     0.006851     0.004754     0.002873     0.002105  \n",
+       "min       0.000000     0.000000     0.000000     0.000000     0.000000  \n",
+       "25%       0.000000     0.000000     0.000000     0.000000     0.000000  \n",
+       "50%       0.000000     0.000000     0.000000     0.000000     0.000000  \n",
+       "75%       0.000000     0.000000     0.000000     0.000000     0.000000  \n",
+       "max       0.183347     0.268149     0.253283     0.153067     0.112136  \n",
+       "\n",
+       "[8 rows x 10267 columns]"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tf_df.describe()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "b31e122a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(2838, 10267)"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tf_df.shape\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "3fabc741",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.linear_model import LogisticRegression"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "2fce3cf9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import warnings\n",
+    "warnings.filterwarnings('ignore')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "a40e9acb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "((2128, 10267), (710,))"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "X_train,X_test,y_train,y_test = train_test_split(cv_df,df['label'],stratify=df['label'])\n",
+    "X_train.shape,y_test.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "3b57e047",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(0.9976503759398496, 0.7619718309859155)"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model_lr = LogisticRegression().fit(X_train,y_train)\n",
+    "model_lr.score(X_train,y_train),model_lr.score(X_test,y_test)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "999b308a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "((2128, 10267), (710,))"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "X_train1,X_test1,y_train1,y_test1 = train_test_split(tf_df,df['label'],stratify=df['label'])\n",
+    "X_train1.shape,y_test1.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "0017098c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(0.9060150375939849, 0.7788732394366197)"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model_lr = LogisticRegression().fit(X_train1,y_train1)\n",
+    "model_lr.score(X_train1,y_train1),model_lr.score(X_test1,y_test1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "12ec4b9d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.9002818886539817"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model = LogisticRegression().fit(tf_df,df['label'])\n",
+    "model.score(tf_df,df['label'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "0555b0d5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def predictor(text):\n",
+    "    processed = textPocess(text)\n",
+    "    embedded_words = tf.transform([text])\n",
+    "    res = model.predict(embedded_words)\n",
+    "    if res[0] == 1:\n",
+    "        res = \"this person is in stress\"\n",
+    "    else:\n",
+    "        res = \"this person is not in stress\"\n",
+    "    return res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "2ad39b0f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['Stress identification NLP']"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import joblib\n",
+    "joblib.dump(model,\"Stress identification NLP\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "24f77186",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text = \"feeling wonderful\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "9d8c2171",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "this person is not in stress\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(predictor(text))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5b3b9972",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

app.py

@@ -0,0 +1,104 @@
+import streamlit as st
+import joblib
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+# Import necessary libraries
+import re
+from urllib.parse import urlparse
+from nltk.tokenize import word_tokenize
+from nltk.corpus import stopwords
+from nltk.stem import WordNetLemmatizer
+
+# Initialize NLTK resources
+stop_words = set(stopwords.words("english"))  # Create a set of English stopwords
+lemmatizer = WordNetLemmatizer()  # Initialize the WordNet Lemmatizer
+
+# Define a function for text processing
+def textProcess(sent):
+    try:
+        if sent is None:  # Check if the input is None
+            return ""  # Return an empty string if input is None
+
+        # Remove square brackets, parentheses, and other special characters
+        sent = re.sub('[][)(]', ' ', sent)
+
+        # Tokenize the text into words
+        sent = [word for word in sent.split() if not urlparse(word).scheme]
+
+        # Join the words back into a sentence
+        sent = ' '.join(sent)
+
+        # Remove Twitter usernames (words starting with @)
+        sent = re.sub(r'\@\w+', '', sent)
+
+        # Remove HTML tags using regular expression
+        sent = re.sub(re.compile("<.*?>"), '', sent)
+
+        # Remove non-alphanumeric characters (keep only letters and numbers)
+        sent = re.sub("[^A-Za-z0-9]", ' ', sent)
+
+        # Convert text to lowercase
+        sent = sent.lower()
+
+        # Split the text into words, strip whitespace, and join them back into a sentence
+        sent = [word.strip() for word in sent.split()]
+        sent = ' '.join(sent)
+
+        # Tokenize the text again
+        tokens = word_tokenize(sent)
+
+        # Remove stop words
+        for word in tokens.copy():
+            if word in stop_words:
+                tokens.remove(word)
+
+        # Lemmatize the remaining words
+        sent = [lemmatizer.lemmatize(word) for word in tokens]
+
+        # Join the lemmatized words back into a sentence
+        sent = ' '.join(sent)
+
+        # Return the processed text
+        return sent
+
+    except Exception as ex:
+        print(sent, "\n")
+        print("Error ", ex)
+        return ""  # Return an empty string in case of an error
+
+# Rest of your code...
+
+# Load the pre-trained model from joblib
+model = joblib.load('Stress identification NLP')
+
+# Load the TF-IDF vectorizer used during training
+tfidf_vectorizer = joblib.load('tfidf_vectorizer.joblib')
+
+# Define the Streamlit web app
+def main():
+    st.title("Stress Predictor Web App")
+    st.write("Enter some text to predict if the person is in stress or not.")
+
+    # Input text box
+    user_input = st.text_area("Enter text here:")
+
+    if st.button("Predict"):
+        if user_input:
+            # Process the input text
+            processed_text = textProcess(user_input)
+
+            # Use the same TF-IDF vectorizer to transform the input text
+            tfidf_text = tfidf_vectorizer.transform([processed_text])
+
+            # Make predictions using the loaded model
+            prediction = model.predict(tfidf_text)[0]
+
+            if prediction == 1:
+                result = "This person is in stress."
+            else:
+                result = "This person is not in stress."
+
+            st.write(result)
+
+if __name__ == '__main__':
+    main()

tfidf_vectorizer.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1411ab31d137fe43a4bda6ea175f16f706714b0ee14ea779455d00bd2f7df5d
+size 298535