How to Generate Website Thumbnails at Scale

const API_KEY = process.env.SCREENSHOT_API_KEY;
const BASE_URL = 'https://screenshotapi-api-production.up.railway.app';

async function generateThumbnail(url, width = 400, height = 300) {
  // Capture at a standard viewport, the API handles the rest
  const params = new URLSearchParams({
    url,
    width: '1280',
    height: '800',
    format: 'webp',   // WebP for smallest file size
    quality: '75',    // Good enough for thumbnails
  });

  const response = await fetch(`${BASE_URL}/v1/screenshot?${params}`, {
    headers: { 'Authorization': `Bearer ${API_KEY}` }
  });

  if (!response.ok) throw new Error(`Failed: ${response.statusText}`);
  return Buffer.from(await response.arrayBuffer());
}

// Generate thumbnails for a list of URLs
const urls = [
  'https://github.com',
  'https://vercel.com',
  'https://stripe.com',
  'https://tailwindcss.com',
];

const thumbnails = await Promise.all(
  urls.map(async (url) => ({
    url,
    image: await generateThumbnail(url),
  }))
);

// Save or upload to your CDN
thumbnails.forEach(({ url, image }) => {
  const slug = new URL(url).hostname.replace(/\./g, '-');
  fs.writeFileSync(`thumbnails/${slug}.webp`, image);
});

const pLimit = require('p-limit');
const fs = require('fs');
const path = require('path');

const API_KEY = process.env.SCREENSHOT_API_KEY;
const BASE_URL = 'https://screenshotapi-api-production.up.railway.app';
const CACHE_DIR = './thumbnail-cache';

// Limit concurrent requests to avoid rate limits
const limit = pLimit(5);

async function generateWithRetry(url, retries = 3) {
  for (let attempt = 1; attempt <= retries; attempt++) {
    try {
      const params = new URLSearchParams({
        url,
        width: '1280',
        height: '800',
        format: 'webp',
        quality: '75',
      });

      const response = await fetch(`${BASE_URL}/v1/screenshot?${params}`, {
        headers: { 'Authorization': `Bearer ${API_KEY}` }
      });

      if (response.status === 429) {
        // Rate limited - wait and retry
        const retryAfter = parseInt(response.headers.get('retry-after') || '5');
        console.log(`  Rate limited, waiting ${retryAfter}s...`);
        await new Promise(r => setTimeout(r, retryAfter * 1000));
        continue;
      }

      if (!response.ok) throw new Error(`HTTP ${response.status}`);
      return Buffer.from(await response.arrayBuffer());
    } catch (err) {
      if (attempt === retries) throw err;
      await new Promise(r => setTimeout(r, 1000 * attempt));
    }
  }
}

async function processBatch(urls) {
  fs.mkdirSync(CACHE_DIR, { recursive: true });

  const results = await Promise.allSettled(
    urls.map(url => limit(async () => {
      const slug = new URL(url).hostname.replace(/[^a-z0-9]/gi, '-');
      const cachePath = path.join(CACHE_DIR, `${slug}.webp`);

      // Skip if cached
      if (fs.existsSync(cachePath)) {
        console.log(`  CACHED: ${url}`);
        return { url, status: 'cached', path: cachePath };
      }

      const image = await generateWithRetry(url);
      fs.writeFileSync(cachePath, image);
      console.log(`  OK: ${url} (${image.length} bytes)`);
      return { url, status: 'ok', path: cachePath };
    }))
  );

  const ok = results.filter(r => r.status === 'fulfilled').length;
  const failed = results.filter(r => r.status === 'rejected').length;
  console.log(`\nDone: ${ok} succeeded, ${failed} failed`);
  return results;
}

// Usage
const urls = fs.readFileSync('urls.txt', 'utf-8')
  .split('\n')
  .filter(Boolean);

processBatch(urls);

// Express endpoint that generates thumbnails on demand
const express = require('express');
const crypto = require('crypto');

const app = express();

app.get('/thumbnail', async (req, res) => {
  const { url } = req.query;
  if (!url) return res.status(400).json({ error: 'url required' });

  // Generate cache key from URL
  const cacheKey = crypto
    .createHash('md5')
    .update(url)
    .digest('hex');

  // Check memory/Redis cache first
  const cached = await cache.get(cacheKey);
  if (cached) {
    res.set('Content-Type', 'image/webp');
    res.set('Cache-Control', 'public, max-age=86400');
    return res.send(cached);
  }

  // Generate thumbnail via ScreenshotAPI
  const params = new URLSearchParams({
    url, format: 'webp', width: '1280', height: '800', quality: '75',
  });

  const response = await fetch(
    `https://screenshotapi-api-production.up.railway.app/v1/screenshot?${params}`,
    { headers: { 'Authorization': `Bearer ${process.env.API_KEY}` } }
  );

  const buffer = Buffer.from(await response.arrayBuffer());

  // Cache for 24 hours
  await cache.set(cacheKey, buffer, 86400);

  res.set('Content-Type', 'image/webp');
  res.set('Cache-Control', 'public, max-age=86400');
  res.send(buffer);
});