Spaces:
Running
Running
File size: 2,221 Bytes
1151f26 d70a98e 1151f26 d70a98e 1151f26 d70a98e 1151f26 d70a98e 1151f26 d70a98e 1151f26 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 |
import gradio as gr
from rag_scraper.scraper import Scraper
from rag_scraper.converter import Converter
from rag_scraper.link_extractor import LinkExtractor, LinkType
from rag_scraper.utils import URLUtils
def scrape_and_convert(url, depth):
"""Fetch HTML content, extract links recursively (up to given depth), and convert to Markdown."""
try:
visited_urls = set()
def recursive_scrape(url, current_depth):
"""Recursively scrape and convert pages up to the given depth."""
if url in visited_urls or current_depth < 0:
return ""
visited_urls.add(url)
# Fetch HTML content
try:
html_content = Scraper.fetch_html(url)
except Exception as e:
return f"Error fetching {url}: {str(e)}\n"
# Convert to Markdown
markdown_content = Converter.html_to_markdown(
html=html_content,
base_url=url,
parser_features='html.parser',
ignore_links=True
)
# If depth > 0, extract links and process them
if current_depth > 0:
links = LinkExtractor.scrape_url(url, link_type=LinkType.INTERNAL)
for link in links:
markdown_content += f"\n\n## Extracted from: {link}\n"
markdown_content += recursive_scrape(link, current_depth - 1)
return markdown_content
# Start the recursive scraping process
result = recursive_scrape(url, depth)
return result
except Exception as e:
return f"Error: {str(e)}"
# Define Gradio interface
iface = gr.Interface(
fn=scrape_and_convert,
inputs=[
gr.Textbox(label="Enter URL"),
gr.Slider(minimum=0, maximum=3, step=1, label="Search Depth (0 = Only main page)")
],
outputs=gr.Code(label="Markdown Output", language="markdown"),
title="RAGScraper with Recursive Depth",
description="Enter a URL and specify the search depth. The app will fetch, extract links, and convert HTML to Markdown."
)
# Launch the Gradio app
if __name__ == "__main__":
iface.launch()
|