Advanced Puppeteer Techniques

const puppeteer = require('puppeteer');

class MemoryEfficientScraper {
    constructor() {
        this.browser = null;
        this.pagePool = [];
        this.maxPages = 5;
    }
    
    async initialize() {
        this.browser = await puppeteer.launch({
            headless: true,
            args: ['--no-sandbox', '--disable-setuid-sandbox']
        });
        
        // Pre-create page pool
        for (let i = 0; i < this.maxPages; i++) {
            const page = await this.browser.newPage();
            this.pagePool.push(page);
        }
    }
    
    async getPage() {
        if (this.pagePool.length > 0) {
            return this.pagePool.pop();
        }
        
        // If no pages available, create a new one
        return await this.browser.newPage();
    }
    
    async releasePage(page) {
        try {
            // Clear page state
            await page.evaluate(() => {
                // Clear any intervals/timeouts
                const highestTimeoutId = setTimeout(() => {});
                for (let i = 0; i < highestTimeoutId; i++) {
                    clearTimeout(i);
                }
                
                // Clear any event listeners
                window.removeEventListener = () => {};
                
                // Clear localStorage and sessionStorage
                localStorage.clear();
                sessionStorage.clear();
            });
            
            // Navigate to about:blank to free memory
            await page.goto('about:blank');
            
            // Return to pool if under limit
            if (this.pagePool.length < this.maxPages) {
                this.pagePool.push(page);
            } else {
                await page.close();
            }
        } catch (error) {
            console.error('Error releasing page:', error);
            await page.close();
        }
    }
    
    async scrapeWithMemoryManagement(urls) {
        const results = [];
        
        for (const url of urls) {
            const page = await this.getPage();
            
            try {
                await page.goto(url, { waitUntil: 'networkidle2' });
                
                const data = await page.evaluate(() => ({
                    title: document.title,
                    url: window.location.href,
                    timestamp: new Date().toISOString()
                }));
                
                results.push(data);
                
                // Monitor memory usage
                const metrics = await page.metrics();
                console.log(`Memory usage for ${url}:`, {
                    JSHeapUsedSize: Math.round(metrics.JSHeapUsedSize / 1024 / 1024) + 'MB',
                    JSHeapTotalSize: Math.round(metrics.JSHeapTotalSize / 1024 / 1024) + 'MB'
                });
                
            } catch (error) {
                console.error(`Error scraping ${url}:`, error);
            } finally {
                await this.releasePage(page);
            }
        }
        
        return results;
    }
    
    async cleanup() {
        // Close all pages in pool
        for (const page of this.pagePool) {
            await page.close();
        }
        
        if (this.browser) {
            await this.browser.close();
        }
    }
}

// Usage
(async () => {
    const scraper = new MemoryEfficientScraper();
    
    try {
        await scraper.initialize();
        
        const urls = [
            'https://example1.com',
            'https://example2.com',
            'https://example3.com'
        ];
        
        const results = await scraper.scrapeWithMemoryManagement(urls);
        console.log('Results:', results);
        
    } finally {
        await scraper.cleanup();
    }
})();

const puppeteer = require('puppeteer');

class ResourceBlocker {
    constructor() {
        this.blockedResources = new Set([
            'image',
            'stylesheet',
            'font',
            'media'
        ]);
        
        this.blockedDomains = new Set([
            'google-analytics.com',
            'googletagmanager.com',
            'facebook.com',
            'twitter.com',
            'doubleclick.net',
            'googlesyndication.com'
        ]);
    }
    
    shouldBlockRequest(request) {
        const url = request.url();
        const resourceType = request.resourceType();
        
        // Block by resource type
        if (this.blockedResources.has(resourceType)) {
            return true;
        }
        
        // Block by domain
        for (const domain of this.blockedDomains) {
            if (url.includes(domain)) {
                return true;
            }
        }
        
        return false;
    }
    
    async setupRequestInterception(page) {
        await page.setRequestInterception(true);
        
        page.on('request', (request) => {
            if (this.shouldBlockRequest(request)) {
                console.log(`Blocked: ${request.resourceType()} - ${request.url()}`);
                request.abort();
            } else {
                request.continue();
            }
        });
        
        // Track blocked vs allowed requests
        let blockedCount = 0;
        let allowedCount = 0;
        
        page.on('requestfailed', (request) => {
            if (request.failure().errorText === 'net::ERR_BLOCKED_BY_CLIENT') {
                blockedCount++;
            }
        });
        
        page.on('response', () => {
            allowedCount++;
        });
        
        // Log statistics periodically
        setInterval(() => {
            console.log(`Requests - Blocked: ${blockedCount}, Allowed: ${allowedCount}`);
        }, 5000);
    }
    
    async optimizedPageLoad(url) {
        const browser = await puppeteer.launch({ headless: true });
        const page = await browser.newPage();
        
        await this.setupRequestInterception(page);
        
        // Set cache to disk to avoid re-downloading
        await page.setCacheEnabled(true);
        
        console.time('Page Load with Blocking');
        await page.goto(url, { waitUntil: 'networkidle2' });
        console.timeEnd('Page Load with Blocking');
        
        // Compare with normal page load
        const normalPage = await browser.newPage();
        console.time('Normal Page Load');
        await normalPage.goto(url, { waitUntil: 'networkidle2' });
        console.timeEnd('Normal Page Load');
        
        await browser.close();
    }
}

// Advanced resource blocking with custom rules
class SmartResourceBlocker extends ResourceBlocker {
    constructor() {
        super();
        this.allowedImageHosts = new Set([
            'cdn.example.com',
            'images.example.com'
        ]);
    }
    
    shouldBlockRequest(request) {
        const url = request.url();
        const resourceType = request.resourceType();
        
        // Allow images from specific hosts
        if (resourceType === 'image') {
            for (const host of this.allowedImageHosts) {
                if (url.includes(host)) {
                    return false;
                }
            }
        }
        
        return super.shouldBlockRequest(request);
    }
    
    async setupAdvancedBlocking(page) {
        await page.setRequestInterception(true);
        
        page.on('request', (request) => {
            const resourceType = request.resourceType();
            const url = request.url();
            
            // Block tracking scripts
            if (resourceType === 'script' && 
                (url.includes('analytics') || url.includes('tracking'))) {
                request.abort();
                return;
            }
            
            // Allow critical resources
            if (resourceType === 'document' || 
                resourceType === 'xhr' || 
                resourceType === 'fetch') {
                request.continue();
                return;
            }
            
            // Custom blocking logic
            if (this.shouldBlockRequest(request)) {
                request.abort();
            } else {
                request.continue();
            }
        });
    }
}

// Usage
(async () => {
    const blocker = new SmartResourceBlocker();
    await blocker.optimizedPageLoad('https://example.com');
})();

const puppeteer = require('puppeteer');
const { Worker, isMainThread, parentPort, workerData } = require('worker_threads');

// Main thread - Worker pool manager
class WorkerPool {
    constructor(maxWorkers = 4) {
        this.maxWorkers = maxWorkers;
        this.workers = [];
        this.taskQueue = [];
        this.activeWorkers = 0;
    }
    
    async initialize() {
        for (let i = 0; i < this.maxWorkers; i++) {
            const worker = new Worker(__filename, {
                workerData: { workerId: i }
            });
            
            worker.on('message', (result) => {
                this.handleWorkerResult(result);
            });
            
            worker.on('error', (error) => {
                console.error('Worker error:', error);
            });
            
            this.workers.push({
                worker,
                busy: false,
                id: i
            });
        }
    }
    
    async addTask(task) {
        return new Promise((resolve, reject) => {
            this.taskQueue.push({ task, resolve, reject });
            this.processQueue();
        });
    }
    
    processQueue() {
        if (this.taskQueue.length === 0) return;
        
        const availableWorker = this.workers.find(w => !w.busy);
        if (!availableWorker) return;
        
        const { task, resolve, reject } = this.taskQueue.shift();
        
        availableWorker.busy = true;
        availableWorker.resolve = resolve;
        availableWorker.reject = reject;
        
        availableWorker.worker.postMessage(task);
    }
    
    handleWorkerResult(result) {
        const worker = this.workers.find(w => w.id === result.workerId);
        
        if (worker) {
            worker.busy = false;
            
            if (result.error) {
                worker.reject(new Error(result.error));
            } else {
                worker.resolve(result.data);
            }
            
            // Process next task in queue
            this.processQueue();
        }
    }
    
    async scrapeUrls(urls) {
        const tasks = urls.map(url => ({ type: 'scrape', url }));
        const results = await Promise.all(
            tasks.map(task => this.addTask(task))
        );
        
        return results;
    }
    
    async cleanup() {
        await Promise.all(
            this.workers.map(({ worker }) => worker.terminate())
        );
    }
}

// Worker thread code
if (!isMainThread) {
    const workerId = workerData.workerId;
    let browser = null;
    
    const initializeBrowser = async () => {
        if (!browser) {
            browser = await puppeteer.launch({
                headless: true,
                args: ['--no-sandbox', '--disable-setuid-sandbox']
            });
        }
        return browser;
    };
    
    const scrapeUrl = async (url) => {
        const browser = await initializeBrowser();
        const page = await browser.newPage();
        
        try {
            await page.goto(url, { waitUntil: 'networkidle2' });
            
            const data = await page.evaluate(() => ({
                title: document.title,
                url: window.location.href,
                wordCount: document.body.innerText.split(/\s+/).length,
                timestamp: new Date().toISOString()
            }));
            
            return data;
            
        } finally {
            await page.close();
        }
    };
    
    parentPort.on('message', async (task) => {
        try {
            let result;
            
            switch (task.type) {
                case 'scrape':
                    result = await scrapeUrl(task.url);
                    break;
                default:
                    throw new Error('Unknown task type');
            }
            
            parentPort.postMessage({
                workerId,
                data: result
            });
            
        } catch (error) {
            parentPort.postMessage({
                workerId,
                error: error.message
            });
        }
    });
    
    // Cleanup on exit
    process.on('beforeExit', async () => {
        if (browser) {
            await browser.close();
        }
    });
}

// Usage (main thread only)
if (isMainThread) {
    (async () => {
        const pool = new WorkerPool(4);
        
        try {
            await pool.initialize();
            
            const urls = [
                'https://example1.com',
                'https://example2.com',
                'https://example3.com',
                'https://example4.com',
                'https://example5.com',
                'https://example6.com'
            ];
            
            console.time('Worker Pool Scraping');
            const results = await pool.scrapeUrls(urls);
            console.timeEnd('Worker Pool Scraping');
            
            console.log(`Processed ${results.length} URLs with worker pool`);
            
        } finally {
            await pool.cleanup();
        }
    })();
}

const { Cluster } = require('puppeteer-cluster');
const puppeteer = require('puppeteer');

class AdvancedClusterScraper {
    constructor(concurrency = 5) {
        this.concurrency = concurrency;
        this.cluster = null;
        this.results = [];
    }
    
    async initialize() {
        this.cluster = await Cluster.launch({
            concurrency: Cluster.CONCURRENCY_CONTEXT,
            maxConcurrency: this.concurrency,
            puppeteerOptions: {
                headless: true,
                args: ['--no-sandbox', '--disable-setuid-sandbox']
            },
            timeout: 30000,
            retryLimit: 3,
            retryDelay: 1000,
            monitor: true // Enable monitoring
        });
        
        // Global error handling
        this.cluster.on('taskerror', (err, data) => {
            console.error(`Task error for ${data.url}:`, err.message);
        });
        
        // Define the scraping task
        await this.cluster.task(async ({ page, data }) => {
            const { url, taskType } = data;
            
            try {
                await page.goto(url, { waitUntil: 'networkidle2' });
                
                switch (taskType) {
                    case 'basic':
                        return await this.basicScrape(page, url);
                    case 'detailed':
                        return await this.detailedScrape(page, url);
                    case 'screenshot':
                        return await this.screenshotTask(page, url);
                    default:
                        throw new Error('Unknown task type');
                }
                
            } catch (error) {
                console.error(`Error processing ${url}:`, error.message);
                throw error;
            }
        });
    }
    
    async basicScrape(page, url) {
        const data = await page.evaluate(() => ({
            title: document.title,
            url: window.location.href,
            description: document.querySelector('meta[name="description"]')?.content || '',
            links: Array.from(document.querySelectorAll('a')).length
        }));
        
        return { ...data, taskType: 'basic' };
    }
    
    async detailedScrape(page, url) {
        const data = await page.evaluate(() => {
            const getElementText = (selector) => {
                const element = document.querySelector(selector);
                return element ? element.textContent.trim() : '';
            };
            
            return {
                title: document.title,
                url: window.location.href,
                headings: Array.from(document.querySelectorAll('h1, h2, h3')).map(h => ({
                    tag: h.tagName.toLowerCase(),
                    text: h.textContent.trim()
                })),
                paragraphs: Array.from(document.querySelectorAll('p')).length,
                images: Array.from(document.querySelectorAll('img')).map(img => ({
                    src: img.src,
                    alt: img.alt || ''
                })),
                forms: Array.from(document.querySelectorAll('form')).length,
                wordCount: document.body.innerText.split(/\s+/).length
            };
        });
        
        return { ...data, taskType: 'detailed' };
    }
    
    async screenshotTask(page, url) {
        const screenshotBuffer = await page.screenshot({
            fullPage: true,
            type: 'png'
        });
        
        // In a real scenario, you'd save this to disk or upload to cloud storage
        return {
            url,
            screenshotSize: screenshotBuffer.length,
            taskType: 'screenshot',
            timestamp: new Date().toISOString()
        };
    }
    
    async processBatch(urls, taskType = 'basic') {
        const results = [];
        
        // Queue all tasks
        const promises = urls.map(url => 
            this.cluster.execute({ url, taskType })
                .then(result => {
                    results.push(result);
                    console.log(`Completed: ${url} (${taskType})`);
                })
                .catch(error => {
                    console.error(`Failed: ${url}`, error.message);
                    results.push({ url, error: error.message, taskType });
                })
        );
        
        // Wait for all tasks to complete
        await Promise.all(promises);
        
        return results;
    }
    
    async processWithDifferentTasks(urlsWithTasks) {
        const results = [];
        
        for (const { url, taskType } of urlsWithTasks) {
            try {
                const result = await this.cluster.execute({ url, taskType });
                results.push(result);
                console.log(`Completed: ${url} (${taskType})`);
            } catch (error) {
                console.error(`Failed: ${url}`, error.message);
                results.push({ url, error: error.message, taskType });
            }
        }
        
        return results;
    }
    
    async getClusterStats() {
        // Monitor cluster performance
        return {
            activeWorkers: this.cluster.workerCount,
            queueSize: this.cluster.queueSize,
            // Add custom metrics as needed
        };
    }
    
    async cleanup() {
        if (this.cluster) {
            await this.cluster.idle();
            await this.cluster.close();
        }
    }
}

// Usage examples
(async () => {
    const scraper = new AdvancedClusterScraper(4);
    
    try {
        await scraper.initialize();
        
        // Example 1: Basic scraping
        const basicUrls = [
            'https://example1.com',
            'https://example2.com',
            'https://example3.com'
        ];
        
        console.time('Basic Scraping');
        const basicResults = await scraper.processBatch(basicUrls, 'basic');
        console.timeEnd('Basic Scraping');
        
        // Example 2: Mixed task types
        const mixedTasks = [
            { url: 'https://example1.com', taskType: 'detailed' },
            { url: 'https://example2.com', taskType: 'screenshot' },
            { url: 'https://example3.com', taskType: 'basic' }
        ];
        
        console.time('Mixed Tasks');
        const mixedResults = await scraper.processWithDifferentTasks(mixedTasks);
        console.timeEnd('Mixed Tasks');
        
        console.log('Basic results:', basicResults.length);
        console.log('Mixed results:', mixedResults.length);
        
    } finally {
        await scraper.cleanup();
    }
})();

const puppeteer = require('puppeteer');

class ResponseModifier {
    constructor() {
        this.modifiedResponses = new Map();
        this.injectedScripts = new Set();
    }
    
    async setupResponseModification(page) {
        await page.setRequestInterception(true);
        
        page.on('request', async (request) => {
            const url = request.url();
            const resourceType = request.resourceType();
            
            // Modify HTML responses
            if (resourceType === 'document') {
                await this.modifyHtmlResponse(request);
            }
            // Modify JSON API responses
            else if (this.isJsonApiRequest(request)) {
                await this.modifyJsonResponse(request);
            }
            // Inject custom scripts
            else if (this.shouldInjectScript(request)) {
                await this.injectCustomScript(request);
            }
            else {
                request.continue();
            }
        });
    }
    
    async modifyHtmlResponse(request) {
        try {
            // Fetch the original response
            const response = await request.continue();
            
            // Get the response body
            const originalHtml = await response.text();
            
            // Modify the HTML
            const modifiedHtml = this.injectCustomElements(originalHtml);
            
            // Store modified response
            this.modifiedResponses.set(request.url(), modifiedHtml);
            
            // Respond with modified HTML
            await request.respond({
                status: 200,
                contentType: 'text/html',
                body: modifiedHtml
            });
            
        } catch (error) {
            console.error('Error modifying HTML:', error);
            request.continue();
        }
    }
    
    injectCustomElements(html) {
        // Add custom CSS
        const customCSS = `
            
        `;
        
        // Add custom elements
        const customElements = `
            

            
🤖 Puppeteer Active
        `;
        
        // Inject custom JavaScript
        const customJS = `
            
        `;
        
        // Inject into HTML
        let modifiedHtml = html;
        
        // Add CSS to head
        modifiedHtml = modifiedHtml.replace('', customCSS + '');
        
        // Add elements to body
        modifiedHtml = modifiedHtml.replace('', customElements + customJS + '');
        
        return modifiedHtml;
    }
    
    async modifyJsonResponse(request) {
        try {
            const url = request.url();
            
            // Define modifications for specific API endpoints
            const apiModifications = {
                '/api/users': (data) => {
                    if (Array.isArray(data)) {
                        return data.map(user => ({
                            ...user,
                            modified: true,
                            timestamp: new Date().toISOString()
                        }));
                    }
                    return data;
                },
                
                '/api/posts': (data) => {
                    if (Array.isArray(data)) {
                        return [
                            { id: 999, title: 'Injected Post', content: 'This was added by Puppeteer' },
                            ...data
                        ];
                    }
                    return data;
                }
            };
            
            // Check if we should modify this response
            const modifier = Object.entries(apiModifications).find(([pattern]) => 
                url.includes(pattern)
            );
            
            if (modifier) {
                // Fetch original response
                const response = await fetch(url);
                const originalData = await response.json();
                
                // Apply modification
                const modifiedData = modifier[1](originalData);
                
                console.log(`📝 Modified JSON response: ${url}`);
                
                // Respond with modified data
                await request.respond({
                    status: 200,
                    contentType: 'application/json',
                    body: JSON.stringify(modifiedData)
                });
            } else {
                request.continue();
            }
            
        } catch (error) {
            console.error('Error modifying JSON response:', error);
            request.continue();
        }
    }
    
    isJsonApiRequest(request) {
        const url = request.url();
        const headers = request.headers();
        
        return url.includes('/api/') || 
               url.includes('/graphql') ||
               headers.accept?.includes('application/json');
    }
    
    shouldInjectScript(request) {
        const url = request.url();
        return request.resourceType() === 'script' && 
               url.includes('jquery') || 
               url.includes('lodash');
    }
    
    async injectCustomScript(request) {
        try {
            const customScript = `
                // Custom utility functions
                window.PuppeteerExtensions = {
                    waitForElement: function(selector, timeout = 5000) {
                        return new Promise((resolve, reject) => {
                            const element = document.querySelector(selector);
                            if (element) {
                                resolve(element);
                                return;
                            }
                            
                            const observer = new MutationObserver(() => {
                                const element = document.querySelector(selector);
                                if (element) {
                                    observer.disconnect();
                                    resolve(element);
                                }
                            });
                            
                            observer.observe(document.body, {
                                childList: true,
                                subtree: true
                            });
                            
                            setTimeout(() => {
                                observer.disconnect();
                                reject(new Error('Element not found within timeout'));
                            }, timeout);
                        });
                    },
                    
                    simulateTyping: function(element, text, delay = 100) {
                        return new Promise((resolve) => {
                            let i = 0;
                            const type = () => {
                                if (i < text.length) {
                                    element.value += text[i];
                                    element.dispatchEvent(new Event('input'));
                                    i++;
                                    setTimeout(type, delay);
                                } else {
                                    resolve();
                                }
                            };
                            type();
                        });
                    }
                };
                
                console.log('🔧 Puppeteer extensions loaded');
            `;
            
            await request.respond({
                status: 200,
                contentType: 'application/javascript',
                body: customScript
            });
            
        } catch (error) {
            console.error('Error injecting script:', error);
            request.continue();
        }
    }
}

// Usage
(async () => {
    const browser = await puppeteer.launch({ headless: false });
    const page = await browser.newPage();
    
    const modifier = new ResponseModifier();
    await modifier.setupResponseModification(page);
    
    await page.goto('https://example.com');
    
    // Use injected utilities
    const formData = await page.evaluate(() => {
        return window.PuppeteerUtils?.collectFormData();
    });
    
    console.log('Form data:', formData);
    
    await browser.close();
})();

const puppeteer = require('puppeteer');
const fs = require('fs');

class NetworkMonitor {
    constructor() {
        this.networkLog = [];
        this.performanceMetrics = {
            totalRequests: 0,
            totalBytes: 0,
            averageResponseTime: 0,
            slowRequests: [],
            failedRequests: []
        };
        this.startTime = Date.now();
    }
    
    async startMonitoring(page) {
        // Enable network domain
        const client = await page.target().createCDPSession();
        await client.send('Network.enable');
        
        // Track all network activity
        client.on('Network.requestWillBeSent', (params) => {
            this.handleRequestStart(params);
        });
        
        client.on('Network.responseReceived', (params) => {
            this.handleResponseReceived(params);
        });
        
        client.on('Network.loadingFinished', (params) => {
            this.handleLoadingFinished(params);
        });
        
        client.on('Network.loadingFailed', (params) => {
            this.handleLoadingFailed(params);
        });
        
        // Track WebSocket connections
        client.on('Network.webSocketCreated', (params) => {
            this.handleWebSocketCreated(params);
        });
        
        // Also monitor using page events
        page.on('request', (request) => {
            this.logRequest(request);
        });
        
        page.on('response', (response) => {
            this.logResponse(response);
        });
        
        page.on('requestfailed', (request) => {
            this.logFailedRequest(request);
        });
        
        return client;
    }
    
    handleRequestStart(params) {
        const request = {
            requestId: params.requestId,
            url: params.request.url,
            method: params.request.method,
            headers: params.request.headers,
            timestamp: params.timestamp,
            initiator: params.initiator,
            resourceType: params.type
        };
        
        this.networkLog.push({
            type: 'request',
            timestamp: Date.now(),
            data: request
        });
        
        this.performanceMetrics.totalRequests++;
    }
    
    handleResponseReceived(params) {
        const response = {
            requestId: params.requestId,
            url: params.response.url,
            status: params.response.status,
            statusText: params.response.statusText,
            headers: params.response.headers,
            mimeType: params.response.mimeType,
            timestamp: params.timestamp,
            fromDiskCache: params.response.fromDiskCache,
            fromServiceWorker: params.response.fromServiceWorker,
            encodedDataLength: params.response.encodedDataLength
        };
        
        this.networkLog.push({
            type: 'response',
            timestamp: Date.now(),
            data: response
        });
        
        // Track slow requests (>2 seconds)
        const requestTime = this.calculateRequestTime(params.requestId);
        if (requestTime > 2000) {
            this.performanceMetrics.slowRequests.push({
                url: params.response.url,
                responseTime: requestTime,
                timestamp: params.timestamp
            });
        }
        
        this.performanceMetrics.totalBytes += params.response.encodedDataLength || 0;
    }
    
    handleLoadingFinished(params) {
        const finished = {
            requestId: params.requestId,
            timestamp: params.timestamp,
            encodedDataLength: params.encodedDataLength
        };
        
        this.networkLog.push({
            type: 'finished',
            timestamp: Date.now(),
            data: finished
        });
    }
    
    handleLoadingFailed(params) {
        const failed = {
            requestId: params.requestId,
            timestamp: params.timestamp,
            errorText: params.errorText,
            canceled: params.canceled
        };
        
        this.networkLog.push({
            type: 'failed',
            timestamp: Date.now(),
            data: failed
        });
        
        this.performanceMetrics.failedRequests.push(failed);
    }
    
    handleWebSocketCreated(params) {
        this.networkLog.push({
            type: 'websocket',
            timestamp: Date.now(),
            data: {
                requestId: params.requestId,
                url: params.url,
                initiator: params.initiator
            }
        });
    }
    
    logRequest(request) {
        console.log(`➡️  ${request.method()} ${request.url()}`);
        
        // Log headers for debugging
        if (request.headers()['authorization']) {
            console.log('   🔐 Authorization header present');
        }
        
        if (request.isNavigationRequest()) {
            console.log('   🧭 Navigation request');
        }
    }
    
    logResponse(response) {
        const status = response.status();
        const url = response.url();
        
        let statusIcon = '✅';
        if (status >= 400) statusIcon = '❌';
        else if (status >= 300) statusIcon = '➡️';
        
        console.log(`${statusIcon} ${status} ${url}`);
        
        // Log content type
        const contentType = response.headers()['content-type'];
        if (contentType) {
            console.log(`   📄 Content-Type: ${contentType}`);
        }
    }
    
    logFailedRequest(request) {
        console.log(`💥 FAILED: ${request.url()}`);
        if (request.failure()) {
            console.log(`   Error: ${request.failure().errorText}`);
        }
    }
    
    calculateRequestTime(requestId) {
        // Find request start and response received events
        const requestEvent = this.networkLog.find(log => 
            log.type === 'request' && log.data.requestId === requestId
        );
        
        const responseEvent = this.networkLog.find(log => 
            log.type === 'response' && log.data.requestId === requestId
        );
        
        if (requestEvent && responseEvent) {
            return responseEvent.timestamp - requestEvent.timestamp;
        }
        
        return 0;
    }
    
    generateReport() {
        const totalTime = Date.now() - this.startTime;
        
        // Calculate averages
        const responseTimes = this.networkLog
            .filter(log => log.type === 'response')
            .map(log => this.calculateRequestTime(log.data.requestId))
            .filter(time => time > 0);
        
        const averageResponseTime = responseTimes.length > 0 
            ? responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length
            : 0;
        
        // Group by resource type
        const resourceTypes = {};
        this.networkLog
            .filter(log => log.type === 'request')
            .forEach(log => {
                const type = log.data.resourceType || 'unknown';
                resourceTypes[type] = (resourceTypes[type] || 0) + 1;
            });
        
        // Group by domain
        const domains = {};
        this.networkLog
            .filter(log => log.type === 'request')
            .forEach(log => {
                try {
                    const domain = new URL(log.data.url).hostname;
                    domains[domain] = (domains[domain] || 0) + 1;
                } catch (e) {
                    // Invalid URL
                }
            });
        
        return {
            summary: {
                totalRequests: this.performanceMetrics.totalRequests,
                totalBytes: this.performanceMetrics.totalBytes,
                averageResponseTime: Math.round(averageResponseTime),
                sessionDuration: totalTime,
                failedRequests: this.performanceMetrics.failedRequests.length,
                slowRequests: this.performanceMetrics.slowRequests.length
            },
            breakdown: {
                resourceTypes,
                domains,
                slowRequests: this.performanceMetrics.slowRequests.slice(0, 10), // Top 10
                failedRequests: this.performanceMetrics.failedRequests.slice(0, 10)
            }
        };
    }
    
    saveLog(filename = 'network-log.json') {
        const report = this.generateReport();
        const logData = {
            report,
            fullLog: this.networkLog
        };
        
        fs.writeFileSync(filename, JSON.stringify(logData, null, 2));
        console.log(`📊 Network log saved to ${filename}`);
    }
    
    printSummary() {
        const report = this.generateReport();
        
        console.log('\n📊 Network Performance Report');
        console.log('================================');
        console.log(`Total Requests: ${report.summary.totalRequests}`);
        console.log(`Total Bytes: ${(report.summary.totalBytes / 1024 / 1024).toFixed(2)} MB`);
        console.log(`Average Response Time: ${report.summary.averageResponseTime}ms`);
        console.log(`Failed Requests: ${report.summary.failedRequests}`);
        console.log(`Slow Requests (>2s): ${report.summary.slowRequests}`);
        
        console.log('\n📈 Resource Types:');
        Object.entries(report.breakdown.resourceTypes).forEach(([type, count]) => {
            console.log(`  ${type}: ${count}`);
        });
        
        console.log('\n🌐 Top Domains:');
        Object.entries(report.breakdown.domains)
            .sort(([,a], [,b]) => b - a)
            .slice(0, 5)
            .forEach(([domain, count]) => {
                console.log(`  ${domain}: ${count}`);
            });
    }
}

// Usage
(async () => {
    const browser = await puppeteer.launch({ headless: false });
    const page = await browser.newPage();
    
    const monitor = new NetworkMonitor();
    await monitor.startMonitoring(page);
    
    // Navigate to a website
    await page.goto('https://example.com');
    
    // Wait for page to fully load
    await page.waitForLoadState('networkidle');
    
    // Generate and display report
    monitor.printSummary();
    monitor.saveLog();
    
    await browser.close();
})();