const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  
  let allData = [];
  let currentPage = 1;
  
  while (true) {
    console.log(`Scraping page ${currentPage}...`);
    
    await page.goto(`https://example.com/products?page=${currentPage}`);
    
    // Check if page has content
    const hasContent = await page.$('.product-item') !== null;
    if (!hasContent) {
      console.log('No more pages to scrape');
      break;
    }
    
    // Extract page data
    const pageData = await page.$$eval('.product-item', elements => {
      return elements.map(element => ({
        name: element.querySelector('.product-name').textContent,
        price: element.querySelector('.price').textContent,
        image: element.querySelector('img').src
      }));
    });
    
    allData.push(...pageData);
    
    // Check for next page button
    const nextButton = await page.$('.pagination .next');
    if (!nextButton) {
      console.log('No next page button found');
      break;
    }
    
    const isDisabled = await page.evaluate(btn => 
      btn.classList.contains('disabled'), nextButton
    );
    
    if (isDisabled) {
      console.log('Next button is disabled');
      break;
    }
    
    currentPage++;
  }
  
  console.log(`Total products scraped: ${allData.length}`);
  
  await browser.close();
})();

const puppeteer = require('puppeteer');

async function scrapeWithRetry(url, maxRetries = 3) {
  const browser = await puppeteer.launch();
  
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      const page = await browser.newPage();
      
      // Set timeout
      await page.setDefaultTimeout(30000);
      
      // Navigate with retry logic
      await page.goto(url, { 
        waitUntil: 'networkidle2',
        timeout: 30000 
      });
      
      // Wait for content to load
      await page.waitForSelector('.content', { timeout: 10000 });
      
      // Extract data
      const data = await page.evaluate(() => {
        const elements = document.querySelectorAll('.item');
        if (elements.length === 0) {
          throw new Error('No items found on page');
        }
        
        return Array.from(elements).map(element => ({
          title: element.querySelector('.title')?.textContent || '',
          description: element.querySelector('.description')?.textContent || ''
        }));
      });
      
      await page.close();
      await browser.close();
      
      return data;
      
    } catch (error) {
      console.log(`Attempt ${attempt} failed:`, error.message);
      
      if (attempt === maxRetries) {
        await browser.close();
        throw new Error(`Scraping failed after ${maxRetries} attempts`);
      }
      
      // Wait before retry
      await new Promise(resolve => setTimeout(resolve, 2000 * attempt));
    }
  }
}

// Usage
(async () => {
  try {
    const data = await scrapeWithRetry('https://example.com/data');
    console.log('Scraped data:', data);
  } catch (error) {
    console.error('Scraping failed:', error.message);
  }
})();

const puppeteer = require('puppeteer');
const fs = require('fs');
const path = require('path');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  
  await page.goto('https://example.com/data');
  
  // Extract data
  const scrapedData = await page.$$eval('.data-item', elements => {
    return elements.map(element => ({
      id: element.dataset.id,
      title: element.querySelector('.title').textContent,
      description: element.querySelector('.description').textContent,
      price: element.querySelector('.price').textContent,
      scrapedAt: new Date().toISOString()
    }));
  });
  
  // Save as JSON
  const jsonData = JSON.stringify(scrapedData, null, 2);
  fs.writeFileSync('scraped-data.json', jsonData);
  
  // Save as CSV
  const csvHeaders = Object.keys(scrapedData[0]).join(',');
  const csvRows = scrapedData.map(item => 
    Object.values(item).map(value => 
      `"${value}"`
    ).join(',')
  );
  const csvContent = [csvHeaders, ...csvRows].join('\n');
  fs.writeFileSync('scraped-data.csv', csvContent);
  
  // Save as HTML report
  const htmlReport = `
    
    
    
      
      
    
    
      
Scraping Report
      
Total items: ${scrapedData.length}
      
Generated: ${new Date().toLocaleString()}
      
      
        ${scrapedData.map(item => `
          
        `).join('')}
      

        

          
ID
          
Title
          
Description
          
Price
        

            
${item.id}
            
${item.title}
            
${item.description}
            
${item.price}
          
    
    
  `;
  
  fs.writeFileSync('report.html', htmlReport);
  
  console.log('Data exported to:');
  console.log('- scraped-data.json');
  console.log('- scraped-data.csv');
  console.log('- report.html');
  
  await browser.close();
})();