Update app.py

app.py

@@ -5,31 +5,39 @@ import random
 from bs4 import BeautifulSoup
 import trafilatura
 
+USER_AGENTS = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+    'Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1'
+]
+
+def get_random_user_agent():
+    return random.choice(USER_AGENTS)
+
 def extract_content_bs4(url):
     try:
-        response = requests.get(url, timeout=10)
+        response = requests.get(url, headers={'User-Agent': get_random_user_agent()}, timeout=10)
         soup = BeautifulSoup(response.content, 'html.parser')
-        
-        # This is a simple extraction and might need to be adjusted based on the structure of the websites you're scraping
         paragraphs = soup.find_all('p')
         content = ' '.join([p.text for p in paragraphs])
-        
         return content[:1000] + "..." if len(content) > 1000 else content
     except Exception as e:
         return f"Error extracting content: {str(e)}"
 
 def extract_content_trafilatura(url):
     try:
-        downloaded = trafilatura.fetch_url(url)
+        downloaded = trafilatura.fetch_url(url, headers={'User-Agent': get_random_user_agent()})
         content = trafilatura.extract(downloaded, include_comments=False, include_tables=False)
-        
         return content[:1000] + "..." if content and len(content) > 1000 else content
     except Exception as e:
         return f"Error extracting content: {str(e)}"
 
-def search_searx(query, instance_url='https://searx.org', categories='general', max_retries=3, num_results=10, use_trafilatura=False):
+def search_searx(query, instance_url='https://searx.org', categories='general', max_retries=3, num_results=10, 
+                 use_trafilatura=False, time_range='', language='en', safesearch=0):
     """
-    Perform a search using the Searx API with error handling, retry logic, limited results, and content extraction.
+    Perform a search using the SearXNG API with advanced options.
     """
     search_endpoint = f"{instance_url}/search"
     params = {
@@ -37,14 +45,14 @@ def search_searx(query, instance_url='https://searx.org', categories='general',
         'format': 'json',
         'categories': categories,
         'pageno': 1,
-        'time_range': '',
-        'engines': '',
-        'safesearch': '0',
+        'time_range': time_range,
+        'language': language,
+        'safesearch': safesearch,
         'results': str(num_results)
     }
     
     headers = {
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        'User-Agent': get_random_user_agent(),
         'Accept': 'application/json, text/javascript, */*; q=0.01',
         'Accept-Language': 'en-US,en;q=0.5',
         'Referer': instance_url,
@@ -67,7 +75,6 @@ def search_searx(query, instance_url='https://searx.org', categories='general',
                 title = result.get('title', 'No Title')
                 url = result.get('url', 'No URL')
                 
-                # Extract content using the selected method
                 if use_trafilatura:
                     content = extract_content_trafilatura(url)
                 else:
@@ -87,12 +94,12 @@ def search_searx(query, instance_url='https://searx.org', categories='general',
 
 def create_gradio_interface():
     """
-    Creates and returns the Gradio interface.
+    Creates and returns the Gradio interface with advanced SearXNG options.
     """
     with gr.Blocks() as demo:
-        gr.Markdown("# 🕵️‍♂️ Private Search with Searx and Content Extraction")
+        gr.Markdown("# 🕵️‍♂️ Advanced SearXNG Search with Content Extraction")
         gr.Markdown(
-            "This application allows you to perform private searches using the [Searx](https://searx.org/) metasearch engine and extract content from the results."
+            "This application allows you to perform private searches using SearXNG with advanced options and content extraction."
         )
         with gr.Row():
             with gr.Column():
@@ -102,7 +109,7 @@ def create_gradio_interface():
                     lines=1
                 )
                 instance_url = gr.Textbox(
-                    label="Searx Instance URL",
+                    label="SearXNG Instance URL",
                     value="https://searx.org",
                     placeholder="https://searx.instance.url",
                     lines=1
@@ -110,7 +117,7 @@ def create_gradio_interface():
                 categories = gr.Textbox(
                     label="Categories",
                     value="general",
-                    placeholder="e.g., general, images, videos",
+                    placeholder="e.g., general, news, science",
                     lines=1
                 )
                 num_results = gr.Slider(
@@ -121,23 +128,42 @@ def create_gradio_interface():
                     label="Number of Results"
                 )
                 use_trafilatura = gr.Checkbox(label="Use Trafilatura for extraction (instead of BeautifulSoup)")
+                time_range = gr.Dropdown(
+                    choices=["", "day", "week", "month", "year"],
+                    value="",
+                    label="Time Range"
+                )
+                language = gr.Textbox(
+                    label="Language",
+                    value="en",
+                    placeholder="e.g., en, fr, de",
+                    lines=1
+                )
+                safesearch = gr.Slider(
+                    minimum=0,
+                    maximum=2,
+                    value=0,
+                    step=1,
+                    label="SafeSearch (0: Off, 1: Moderate, 2: Strict)"
+                )
                 search_button = gr.Button("Search")
             with gr.Column():
                 results = gr.Markdown("### Search Results will appear here...")
         
-        def perform_search(q, url, cats, num, use_traf):
-            return search_searx(q, instance_url=url, categories=cats, num_results=int(num), use_trafilatura=use_traf)
+        def perform_search(q, url, cats, num, use_traf, t_range, lang, safe):
+            return search_searx(q, instance_url=url, categories=cats, num_results=int(num), 
+                                use_trafilatura=use_traf, time_range=t_range, language=lang, safesearch=int(safe))
         
         search_button.click(
             perform_search,
-            inputs=[query, instance_url, categories, num_results, use_trafilatura],
+            inputs=[query, instance_url, categories, num_results, use_trafilatura, time_range, language, safesearch],
             outputs=results
         )
         
         gr.Markdown(
             """
             ---
-            **Note:** This application uses the Searx metasearch engine to fetch results from multiple sources while preserving your privacy. 
+            **Note:** This application uses SearXNG to fetch results from multiple sources while preserving your privacy. 
             It then attempts to extract content from the original sources, which may be subject to the terms of service of those websites.
             """
         )