continue re-design

app/api/scrape-website/route.ts

@@ -0,0 +1,106 @@
+import { NextRequest, NextResponse } from "next/server";
+import FirecrawlApp from '@mendable/firecrawl-js';
+
+export async function POST(request: NextRequest) {
+  try {
+    const { url, formats = ['markdown', 'html'], options = {} } = await request.json();
+    
+    if (!url) {
+      return NextResponse.json(
+        { error: "URL is required" },
+        { status: 400 }
+      );
+    }
+    
+    // Initialize Firecrawl with API key from environment
+    const apiKey = process.env.FIRECRAWL_API_KEY;
+    
+    if (!apiKey) {
+      console.error("FIRECRAWL_API_KEY not configured");
+      // For demo purposes, return mock data if API key is not set
+      return NextResponse.json({
+        success: true,
+        data: {
+          title: "Example Website",
+          content: `This is a mock response for ${url}. Configure FIRECRAWL_API_KEY to enable real scraping.`,
+          description: "A sample website",
+          markdown: `# Example Website\n\nThis is mock content for demonstration purposes.`,
+          html: `<h1>Example Website</h1><p>This is mock content for demonstration purposes.</p>`,
+          metadata: {
+            title: "Example Website",
+            description: "A sample website",
+            sourceURL: url,
+            statusCode: 200
+          }
+        }
+      });
+    }
+    
+    const app = new FirecrawlApp({ apiKey });
+    
+    // Scrape the website using the latest SDK patterns
+    // Include screenshot if requested in formats
+    const scrapeResult = await app.scrapeUrl(url, {
+      formats: formats,
+      onlyMainContent: options.onlyMainContent !== false, // Default to true for cleaner content
+      waitFor: options.waitFor || 2000, // Wait for dynamic content
+      timeout: options.timeout || 30000,
+      ...options // Pass through any additional options
+    });
+    
+    // Handle the response according to the latest SDK structure
+    if (!scrapeResult.success) {
+      throw new Error(scrapeResult.error || "Failed to scrape website");
+    }
+    
+    return NextResponse.json({
+      success: true,
+      data: {
+        title: scrapeResult.data?.metadata?.title || "Untitled",
+        content: scrapeResult.data?.markdown || scrapeResult.data?.html || "",
+        description: scrapeResult.data?.metadata?.description || "",
+        markdown: scrapeResult.data?.markdown || "",
+        html: scrapeResult.data?.html || "",
+        metadata: scrapeResult.data?.metadata || {},
+        screenshot: scrapeResult.data?.screenshot || null,
+        links: scrapeResult.data?.links || [],
+        // Include raw data for flexibility
+        raw: scrapeResult.data
+      }
+    });
+    
+  } catch (error) {
+    console.error("Error scraping website:", error);
+    
+    // Return a more detailed error response
+    return NextResponse.json({
+      success: false,
+      error: error instanceof Error ? error.message : "Failed to scrape website",
+      // Provide mock data as fallback for development
+      data: {
+        title: "Example Website",
+        content: "This is fallback content due to an error. Please check your configuration.",
+        description: "Error occurred while scraping",
+        markdown: `# Error\n\n${error instanceof Error ? error.message : 'Unknown error occurred'}`,
+        html: `<h1>Error</h1><p>${error instanceof Error ? error.message : 'Unknown error occurred'}</p>`,
+        metadata: {
+          title: "Error",
+          description: "Failed to scrape website",
+          statusCode: 500
+        }
+      }
+    }, { status: 500 });
+  }
+}
+
+// Optional: Add OPTIONS handler for CORS if needed
+export async function OPTIONS(request: NextRequest) {
+  return new NextResponse(null, {
+    status: 200,
+    headers: {
+      'Access-Control-Allow-Origin': '*',
+      'Access-Control-Allow-Methods': 'POST, OPTIONS',
+      'Access-Control-Allow-Headers': 'Content-Type',
+    },
+  });
+}