pdf_route.py

import os
from io import BytesIO
import pandas as pd
from fastapi import APIRouter, UploadFile, File, HTTPException
from fastapi.responses import StreamingResponse, JSONResponse
from azure.core.credentials import AzureKeyCredential
from azure.ai.formrecognizer import DocumentAnalysisClient
from dotenv import load_dotenv
from docx import Document
import re

# Load environment variables
load_dotenv()

router = APIRouter()

@router.post("/convert-to-markdown")
async def convert_to_markdown(file: UploadFile = File(...)):
    """
    Convert a PDF file to markdown format.
    Args:
        file: The PDF file to convert
    Returns:
        StreamingResponse: Markdown file
    """
    try:
        # Read the uploaded file content
        content = await file.read()
        
        # Save the content to a temporary file
        temp_pdf_path = "temp.pdf"
        with open(temp_pdf_path, "wb") as f:
            f.write(content)
        
        # Analyze the document
        result = analyze_document(temp_pdf_path)
        
        # Create markdown file
        temp_md_path = "temp.md"
        create_markdown_file(result, temp_md_path)
        
        # Read the markdown file
        with open(temp_md_path, "rb") as f:
            markdown_content = f.read()
        
        # Clean up temporary files
        os.remove(temp_pdf_path)
        os.remove(temp_md_path)
        
        # Return the markdown file as a download
        return StreamingResponse(
            BytesIO(markdown_content),
            media_type="text/markdown",
            headers={
                "Content-Disposition": f"attachment; filename={file.filename.rsplit('.', 1)[0]}.md"
            }
        )
        
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@router.post("/convert-to-excel")
async def convert_to_excel(file: UploadFile = File(...)):
    """
    Convert tables from markdown to Excel format.
    Args:
        file: The markdown file to convert
    Returns:
        StreamingResponse: Excel file containing all tables
    """
    try:
        # Read the markdown content
        content = await file.read()
        markdown_text = content.decode('utf-8')
        
        # Extract tables from markdown
        tables = extract_tables_from_markdown(markdown_text)
        
        if not tables:
            raise HTTPException(status_code=400, detail="No tables found in the markdown content")
        
        # Create Excel file
        excel_buffer = create_excel_from_markdown_tables(tables)
        
        # Return the Excel file as a download
        return StreamingResponse(
            excel_buffer,
            media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
            headers={
                "Content-Disposition": f"attachment; filename={file.filename.rsplit('.', 1)[0]}_tables.xlsx"
            }
        )
        
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@router.post("/convert-to-word")
async def convert_to_word(file: UploadFile = File(...)):
    """
    Convert markdown to Word document format.
    Args:
        file: The markdown file to convert
    Returns:
        StreamingResponse: Word document file
    """
    try:
        # Read the markdown content
        content = await file.read()
        markdown_text = content.decode('utf-8')
        
        # Create Word file
        temp_docx_path = "temp.docx"
        create_word_from_markdown(markdown_text, temp_docx_path)
        
        # Read the Word file
        with open(temp_docx_path, "rb") as f:
            word_content = f.read()
        
        # Clean up temporary file
        os.remove(temp_docx_path)
        
        # Return the Word file as a download
        return StreamingResponse(
            BytesIO(word_content),
            media_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
            headers={
                "Content-Disposition": f"attachment; filename={file.filename.rsplit('.', 1)[0]}.docx"
            }
        )
        
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

def analyze_document(file_path):
    """Analyze document using Azure Form Recognizer"""
    endpoint = "https://aal-ocr-ai-azureapi.cognitiveservices.azure.com/"
    key = os.getenv("AZURE_FORM_RECOGNIZER_KEY")
    
    document_analysis_client = DocumentAnalysisClient(
        endpoint=endpoint, credential=AzureKeyCredential(key)
    )

    with open(file_path, "rb") as f:
        poller = document_analysis_client.begin_analyze_document(
            "prebuilt-layout", document=f
        )
    
    result = poller.result()
    return result

def extract_tables_from_markdown(markdown_text):
    """Extract tables from markdown text"""
    tables = []
    current_table = []
    
    lines = markdown_text.split('\n')
    in_table = False
    
    for line in lines:
        if '|' in line:
            # Skip separator lines (e.g., |---|---|)
            if re.match(r'^[\s|:-]+$', line):
                continue
            
            # Process table row
            cells = [cell.strip() for cell in line.split('|')[1:-1]]
            if cells:
                if not in_table:
                    in_table = True
                current_table.append(cells)
        else:
            if in_table:
                if current_table:
                    tables.append(current_table)
                current_table = []
                in_table = False
    
    # Add the last table if exists
    if current_table:
        tables.append(current_table)
    
    return tables

def create_excel_from_markdown_tables(tables):
    """Create Excel file from markdown tables"""
    excel_buffer = BytesIO()
    
    with pd.ExcelWriter(excel_buffer, engine='openpyxl') as writer:
        for i, table in enumerate(tables):
            if table:
                # Convert table to DataFrame
                df = pd.DataFrame(table[1:], columns=table[0])
                
                # Save to Excel sheet
                sheet_name = f"Table_{i+1}"
                df.to_excel(writer, sheet_name=sheet_name, index=False)
    
    excel_buffer.seek(0)
    return excel_buffer

def create_word_from_markdown(markdown_text, output_file):
    """Create Word document from markdown text"""
    doc = Document()
    
    lines = markdown_text.split('\n')
    current_table = []
    in_table = False
    
    for line in lines:
        # Handle headers
        if line.startswith('#'):
            level = len(line.split()[0])  # Count the number of '#'
            text = line.lstrip('#').strip()
            doc.add_heading(text, level=min(level, 9))
        
        # Handle tables
        elif '|' in line:
            # Skip separator lines
            if re.match(r'^[\s|:-]+$', line):
                continue
                
            # Process table row
            cells = [cell.strip() for cell in line.split('|')[1:-1]]
            if cells:
                if not in_table:
                    in_table = True
                    current_table = []
                current_table.append(cells)
        
        # Handle end of table
        elif in_table:
            if current_table:
                table = doc.add_table(rows=len(current_table), cols=len(current_table[0]))
                table.style = 'Table Grid'
                
                for i, row in enumerate(current_table):
                    for j, cell in enumerate(row):
                        table.cell(i, j).text = cell
                
                doc.add_paragraph()  # Add space after table
            current_table = []
            in_table = False
        
        # Handle checkbox lists
        elif line.strip().startswith('- ['):
            p = doc.add_paragraph()
            run = p.add_run()
            if 'x' in line or 'X' in line:
                run.add_text("☑ " + line[5:].strip())
            else:
                run.add_text("☐ " + line[5:].strip())
        
        # Handle regular paragraphs
        elif line.strip():
            doc.add_paragraph(line.strip())
    
    # Handle the last table if exists
    if in_table and current_table:
        table = doc.add_table(rows=len(current_table), cols=len(current_table[0]))
        table.style = 'Table Grid'
        
        for i, row in enumerate(current_table):
            for j, cell in enumerate(row):
                table.cell(i, j).text = cell
    
    doc.save(output_file)

def create_markdown_file(result, output_file):
    """Create markdown file from analysis result"""
    with open(output_file, 'w', encoding='utf-8') as md_file:
        for page in result.pages:
            # md_file.write(f"### Page {page.page_number}\n\n")
            
            elements = []
            elements.extend([(paragraph.bounding_regions[0].polygon[0].y + paragraph.bounding_regions[0].polygon[0].x*0.05, 'paragraph', paragraph) 
                           for paragraph in result.paragraphs if paragraph.bounding_regions[0].page_number == page.page_number])
            elements.sort(key=lambda x: x[0])
            
            page_width = page.width / 2
            min_distance = float('inf')
            title_paragraph = None
            
            for element in elements[:5]:
                if element[1] == 'paragraph':
                    paragraph = element[2]
                    midpoint_x = (paragraph.bounding_regions[0].polygon[0].x + paragraph.bounding_regions[0].polygon[1].x) / 2
                    midpoint_y = paragraph.bounding_regions[0].polygon[0].y
                    distance = ((midpoint_x - page_width) ** 2 + midpoint_y ** 2) ** 0.5
                    if distance < min_distance:
                        min_distance = distance
                        title_paragraph = paragraph
            
            if title_paragraph:
                elements = [element for element in elements if element[2] != title_paragraph]
                md_file.write(f"# {title_paragraph.content}\n\n")
            
            elements.extend([(table.bounding_regions[0].polygon[0].y + table.bounding_regions[0].polygon[0].x*0.05, 'table', table) 
                           for table in result.tables if table.bounding_regions[0].page_number == page.page_number])
            elements.extend([(mark.polygon[0].y + mark.polygon[0].x*0.05, 'selection_mark', mark) for mark in page.selection_marks])

            elements.sort(key=lambda x: x[0])

            table_cells = set()
            for _, element_type, element in elements:
                if element_type == 'paragraph':
                    if any(is_element_inside_table(element, get_table_max_polygon(table)) for table in result.tables):
                        continue
                    md_file.write(f"{element.content}\n\n")
                
                elif element_type == 'table':
                    for row_idx in range(element.row_count):
                        row_content = "| "
                        for col_idx in range(element.column_count):
                            cell_content = ""
                            for cell in element.cells:
                                if cell.row_index == row_idx and cell.column_index == col_idx:
                                    cell_content = cell.content
                                    table_cells.add((cell.bounding_regions[0].polygon[0].x, cell.bounding_regions[0].polygon[0].y))
                                    break
                            row_content += f"{cell_content} | "
                        md_file.write(row_content + "\n")
                    md_file.write("\n")
                    
                elif element_type == 'selection_mark':
                    if element.state == "selected":
                        md_file.write("- [x] \n\n")
                    else:
                        md_file.write("- [ ] \n\n")

def create_word_file(result, output_file):
    """Create Word document from analysis result"""
    # Create a new Word document
    doc = Document()

    # Analyze pages
    for page in result.pages:
        # Combine paragraphs, tables, and selection marks in the order they appear on the page
        elements = []
        elements.extend([(paragraph.bounding_regions[0].polygon[0].y + paragraph.bounding_regions[0].polygon[0].x*0.01, 'paragraph', paragraph) 
                        for paragraph in result.paragraphs if paragraph.bounding_regions[0].page_number == page.page_number])
        elements.sort(key=lambda x: x[0])
        
        # Find the paragraph which is possible to be document title
        page_width = page.width / 2
        min_distance = float('inf')
        title_paragraph = None
        
        for element in elements[:5]:
            if element[1] == 'paragraph':
                paragraph = element[2]
                midpoint_x = (paragraph.bounding_regions[0].polygon[0].x + paragraph.bounding_regions[0].polygon[1].x) / 2
                midpoint_y = paragraph.bounding_regions[0].polygon[0].y
                distance = ((midpoint_x - page_width) ** 2 + midpoint_y ** 2) ** 0.5
                if distance < min_distance:
                    min_distance = distance
                    title_paragraph = paragraph
        
        if title_paragraph:
            elements = [element for element in elements if element[2] != title_paragraph]
            doc.add_heading(title_paragraph.content, level=1)
        
        # Continuous combine paragraphs, tables, and selection marks in the order they appear on the page
        elements.extend([(table.bounding_regions[0].polygon[0].y + table.bounding_regions[0].polygon[0].x*0.01, 'table', table) 
                        for table in result.tables if table.bounding_regions[0].page_number == page.page_number])
        elements.extend([(mark.polygon[0].y + mark.polygon[0].x*0.01, 'selection_mark', mark) 
                        for mark in page.selection_marks])

        # Sort elements by the sum of their horizontal and vertical positions on the page
        elements.sort(key=lambda x: x[0])

        # Track table cells to avoid duplicating content
        table_cells = set()
        for _, element_type, element in elements:
            if element_type == 'paragraph':
                # Skip lines that are part of a table
                if any(is_element_inside_table(element, get_table_max_polygon(table)) for table in result.tables):
                    continue
                doc.add_paragraph(element.content)
            elif element_type == 'table':
                table = doc.add_table(rows=element.row_count, cols=element.column_count)
                table.style = 'Table Grid' 
                for row_idx in range(element.row_count):
                    row_cells = table.rows[row_idx].cells
                    for col_idx in range(element.column_count):
                        cell_content = ""
                        for cell in element.cells:
                            if cell.row_index == row_idx and cell.column_index == col_idx:
                                cell_content = cell.content
                                table_cells.add((cell.bounding_regions[0].polygon[0].x, cell.bounding_regions[0].polygon[0].y))
                                break
                        row_cells[col_idx].text = cell_content
            elif element_type == 'selection_mark':
                p = doc.add_paragraph()
                run = p.add_run()
                if element.state == "selected":
                    run.add_text("☑ ")
                else:
                    run.add_text("☐ ")

    # Save Word document
    doc.save(output_file)

def format_polygon(polygon):
    """Format polygon coordinates to string"""
    if not polygon:
        return "N/A"
    return ", ".join([f"[{p.x}, {p.y}]" for p in polygon])

def get_table_max_polygon(table):
    """Get the maximum polygon coordinates for a table"""
    first_cell = table.cells[0]
    first_coordinate = first_cell.bounding_regions[0].polygon[0]
    last_cell = table.cells[-1]
    last_coordinate = last_cell.bounding_regions[0].polygon[-1]
    return [first_coordinate, last_coordinate]

def is_element_inside_table(element, table_max_polygon):
    """Check if an element is inside a table"""
    element_x = element.bounding_regions[0].polygon[0].x
    element_y = element.bounding_regions[0].polygon[0].y
    first_coordinate = table_max_polygon[0]
    last_coordinate = table_max_polygon[1]

    return (first_coordinate.x <= element_x <= last_coordinate.x and
            first_coordinate.y <= element_y <= last_coordinate.y)