Update market_analysis_crew.py

market_analysis_crew.py

@@ -2,49 +2,26 @@ from crewai import Agent, Task, Crew, Process
 from langchain_openai import ChatOpenAI
 from langchain.tools import Tool
 from langchain_community.tools import WriteFileTool
-from crewai_tools import SerperDevTool, FirecrawlCrawlWebsiteTool
+from crewai_tools import SerperDevTool, ScrapeWebsiteTool
 import os
 import time
 from pathlib import Path
 import json
-from dotenv import load_dotenv
-
-# Load environment variables from .env
-load_dotenv()
-
-# Set OpenAI API key
-os.environ["OPENAI_API_KEY"] = "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"
-os.environ["FIRECRAWL_API_KEY"] = "fc-042d8dd61b82422888b7c23b7bc9e369"  # Added Firecrawl API key
 
 # Initialize tools and models
 openai_model = ChatOpenAI(
     model_name="gpt-4o-mini",
-    temperature=0.7,
-    api_key=os.getenv("OPENAI_API_KEY")
+    temperature=0.7
 )
 
 class ReportGenerator:
     def __init__(self):
-        # Initialize SerperDev with API key from .env
         self.search_tool = SerperDevTool(
-            serper_api_key=os.getenv("SERPER_API_KEY"),
             search_url="https://google.serper.dev/search",
             n_results=10,
         )
         
-        self.scrape_tool = FirecrawlCrawlWebsiteTool(
-            api_key=os.environ["FIRECRAWL_API_KEY"],
-            crawler_options={
-                "maxDepth": 2,
-                "mode": "fast",
-                "generateImgAltText": True,
-                "limit": 10
-            },
-            page_options={
-                "onlyMainContent": True,
-                "includeHtml": False
-            }
-        )
+        self.scrape_tool = ScrapeWebsiteTool()
         
         self.write_file_tool = Tool(
             name="Write File",
@@ -52,11 +29,10 @@ class ReportGenerator:
             func=self.write_file_tool_wrapper
         )
         
-        # Initialize ChatOpenAI with API key
+        # Initialize ChatOpenAI with higher temperature for more creative inference
         self.question_generator = ChatOpenAI(
-            model_name="gpt-4o-mini",
-            temperature=0.7,
-            api_key=os.environ["OPENAI_API_KEY"]  # Explicitly pass API key
+            model_name="gpt-4-turbo-preview",
+            temperature=0.7
         )
         
         # Add default values for common fields
@@ -1162,25 +1138,11 @@ Format the report in clear, professional markdown with appropriate headers, bull
             return default_questions
 
     def scrape_company_website(self, url):
-        """Scrape company website using Firecrawl"""
+        """Scrape company website using existing tools"""
         try:
-            print(f"Scraping website: {url}")
-            
-            # Initialize crawler with proper configuration
-            crawler = FirecrawlCrawlWebsiteTool(
-                api_key=os.environ["FIRECRAWL_API_KEY"]
-            )
-            
-            # Run the crawler with URL parameter
-            content = crawler.run(url)
-            
-            # Convert content to string if needed
-            if content and not isinstance(content, str):
-                content = str(content)
-            
-            print("Website scraping completed successfully")
+            scraper = ScrapeWebsiteTool(website_url=url)
+            content = scraper.run()
             return content
-            
         except Exception as e:
             print(f"Error scraping website: {e}")
             return None
@@ -1189,18 +1151,20 @@ Format the report in clear, professional markdown with appropriate headers, bull
         """Analyze website content using GPT to detect industry and other details"""
         try:
             if not website_data:
-                return self.get_default_analysis()
+                return {
+                    "industry": "Technology",
+                    "business_model": "B2B",
+                    "target_market": "General",
+                    "products": ["Unknown"],
+                    "market_focus": "Global"
+                }
 
             print("Analyzing website content with AI...")
             
-            # Convert website_data to string and safely truncate
-            website_content = str(website_data) if website_data else ''
-            truncated_content = website_content[:2000] if len(website_content) > 2000 else website_content
-            
             prompt = f"""Analyze this website content for {company_name} and provide key business information.
             
             Website Content:
-            {truncated_content}
+            {website_data[:2000]}
             
             Return ONLY a JSON object with this exact format:
             {{
@@ -1212,30 +1176,32 @@ Format the report in clear, professional markdown with appropriate headers, bull
             }}
             """
             
+            # Use the question generator (ChatGPT) to analyze
             response = self.question_generator.invoke(prompt).content
+            
+            # Clean the response
             response = response.strip()
             if response.startswith('```json'):
                 response = response[7:]
             if response.endswith('```'):
                 response = response[:-3]
             
+            # Parse JSON response
             analysis = json.loads(response.strip())
             print("Website analysis completed successfully")
+            
             return analysis
             
         except Exception as e:
             print(f"Error analyzing website: {str(e)}")
-            return self.get_default_analysis()
-
-    def get_default_analysis(self):
-        """Return default analysis values"""
-        return {
-            "industry": "Technology",
-            "business_model": "B2B",
-            "target_market": "General",
-            "products": ["Unknown"],
-            "market_focus": "Global"
-        }
+            # Return default values if analysis fails
+            return {
+                "industry": "Technology",
+                "business_model": "B2B",
+                "target_market": "General",
+                "products": ["Unknown"],
+                "market_focus": "Global"
+            }
 
     def generate_questions(self, context):
         """Generate questions based on company context and detail level"""
@@ -1248,23 +1214,19 @@ Format the report in clear, professional markdown with appropriate headers, bull
 
             print(f"Generating {detail_level} questions for {report_type}...")
 
-            # Convert website_data to string and safely truncate
-            website_content = str(website_data) if website_data else ''
-            truncated_content = website_content[:1000] if len(website_content) > 1000 else website_content
-
             if detail_level == 'quick':
                 prompt = f"""Generate 2-3 brief but specific questions about {company_name} in the {industry} industry.
                 Focus on core business metrics and market position.
                 
                 Context:
-                {truncated_content}
+                {website_data[:1000]}
                 
                 Return ONLY a JSON object with this exact format:
                 {{
                     "questions": [
                         {{"id": 1, "question": "Brief, specific question about core metrics?"}},
                         {{"id": 2, "question": "Brief question about market position?"}},
-                        {{"id": 3, "question": "Brief question about growth/strategy?"}}
+                        {{"id": 3, "question": "Brief question about growth/strategy?"}},
                     ]
                 }}
                 """
@@ -1278,7 +1240,7 @@ Format the report in clear, professional markdown with appropriate headers, bull
                 - Business model
                 
                 Context:
-                {truncated_content}
+                {website_data[:1500]}
                 
                 Return ONLY a JSON object with this exact format:
                 {{
@@ -1316,48 +1278,45 @@ Format the report in clear, professional markdown with appropriate headers, bull
 
         except Exception as e:
             print(f"Error generating questions: {str(e)}")
-            return self.get_default_questions(company_name, industry, detail_level)
-
-    def get_default_questions(self, company_name, industry, detail_level):
-        """Return default questions based on company name, industry, and detail level"""
-        if detail_level == 'quick':
-            return [
-                {
-                    "id": 1,
-                    "question": f"What is {company_name}'s main competitive advantage in the {industry} market?"
-                },
-                {
-                    "id": 2,
-                    "question": "Who are your top 2-3 direct competitors?"
-                },
-                {
-                    "id": 3,
-                    "question": "What is your primary revenue stream?"
-                }
-            ]
-        else:
-            return [
-                {
-                    "id": 1,
-                    "question": f"What unique value proposition does {company_name} offer in the {industry} space?"
-                },
-                {
-                    "id": 2,
-                    "question": "Who are your main competitors and how do you differentiate?"
-                },
-                {
-                    "id": 3,
-                    "question": "What are your key growth metrics and targets?"
-                },
-                {
-                    "id": 4,
-                    "question": "What market opportunities are you targeting?"
-                },
-                {
-                    "id": 5,
-                    "question": "What are your main customer acquisition channels?"
-                }
-            ]
+            # Return default questions based on detail level
+            if detail_level == 'quick':
+                return [
+                    {
+                        "id": 1,
+                        "question": f"What is {company_name}'s main competitive advantage in the {industry} market?"
+                    },
+                    {
+                        "id": 2,
+                        "question": "Who are your top 2-3 direct competitors?"
+                    },
+                    {
+                        "id": 3,
+                        "question": "What is your primary revenue stream?"
+                    }
+                ]
+            else:
+                return [
+                    {
+                        "id": 1,
+                        "question": f"What unique value proposition does {company_name} offer in the {industry} space?"
+                    },
+                    {
+                        "id": 2,
+                        "question": "Who are your main competitors and how do you differentiate?"
+                    },
+                    {
+                        "id": 3,
+                        "question": "What are your key growth metrics and targets?"
+                    },
+                    {
+                        "id": 4,
+                        "question": "What market opportunities are you targeting?"
+                    },
+                    {
+                        "id": 5,
+                        "question": "What are your main customer acquisition channels?"
+                    }
+                ]
 
 def create_reports(result, context, report_type):
     """Create validation and report files"""
@@ -1414,4 +1373,4 @@ def get_report_generator():
 def get_market_analysis_crew(user_inputs):
     """Backward compatibility function for existing code"""
     generator = ReportGenerator()
-    return generator.create_market_analysis_crew(user_inputs)
+    return generator.create_market_analysis_crew(user_inputs)