Web自动化最佳实践：从脚本到生产级解决方案#

引言#

Web自动化已成为现代软件开发中不可或缺的一部分。无论是数据采集、测试自动化还是业务流程自动化，都需要高效、可靠的技术方案。本文将分享从简单脚本到生产级Web自动化解决方案的完整实践指南。

架构设计原则#

1. 分层架构#

1
// 分层架构示例
2
const automationArchitecture = {
3
  // 控制层
4
  controller: {
5
    strategy: 'page-object',
6
    flow: 'sequential'
7
  },
8

9
  // 业务逻辑层
10
  services: {
11
    authentication: 'auth-service',
12
    dataProcessing: 'processor-service',
13
    errorHandling: 'error-handler'
14
  },
15

16
  // 基础设施层
17
  infrastructure: {
18
    browser: 'puppeteer',
19
    storage: 'file-system',
20
    logging: 'structured-logs'
21
  }
22
};

2. 模块化设计#

将功能拆分为独立的模块，便于维护和扩展：

1
modules/
2
├── auth/            // 认证模块
3
│   ├── login.js
4
│   └── session.js
5
├── navigation/      // 导航模块
6
│   ├── page-loader.js
7
│   └── url-manager.js
8
├── interaction/     // 交互模块
9
│   ├── form-handler.js
10
│   └── element-locator.js
11
└── data/            // 数据处理模块
12
    ├── extractor.js
13
    └── validator.js

核心技术实现#

1. 浏览器自动化#

Puppeteer高级配置#

1
const puppeteer = require('puppeteer');
2
const browser = await puppeteer.launch({
3
  headless: 'new',
4
  args: [
5
    '--no-sandbox',
6
    '--disable-setuid-sandbox',
7
    '--disable-dev-shm-usage',
8
    '--disable-web-security',
9
    '--disable-features=TranslateUI',
10
    '--disable-ipc-flooding-protection'
11
  ],
12
  ignoreHTTPSErrors: true,
13
  defaultViewport: null,
14
  userDataDir: './user_data'
15
});

多页面管理#

1
class PageManager {
2
  constructor(browser) {
3
    this.browser = browser;
4
    this.pages = new Map();
5
  }
6

7
  async createPage(name, options = {}) {
8
    const page = await this.browser.newPage();
9
    page.setDefaultTimeout(options.timeout || 30000);
10
    this.pages.set(name, page);
11
    return page;
12
  }
13

14
  getPage(name) {
15
    return this.pages.get(name);
16
  }
17

18
  async closeAll() {
19
    for (const [name, page] of this.pages) {
20
      try {
21
        await page.close();
22
      } catch (error) {
23
        console.warn(`关闭页面 ${name} 时出错:`, error.message);
24
      }
25
    }
26
    this.pages.clear();
27
  }
28
}

2. 智能元素定位#

多重定位策略#

1
class ElementLocator {
2
  constructor(page) {
3
    this.page = page;
4
    this.strategies = [
5
      'id',
6
      'css',
7
      'xpath',
8
      'text',
9
      'name',
10
      'label'
11
    ];
12
  }
13

14
  async findElement(selectors) {
15
    for (const strategy of this.strategies) {
16
      try {
17
        const element = await this.tryLocate(selectors, strategy);
18
        if (element) {
19
          return element;
20
        }
21
      } catch (error) {
22
        continue;
23
      }
24
    }
25
    throw new Error(`无法定位元素: ${JSON.stringify(selectors)}`);
26
  }
27

28
  async tryLocate(selectors, strategy) {
29
    const selector = this.buildSelector(selectors, strategy);
30
    switch (strategy) {
31
      case 'id':
32
        return this.page.$(`#${selector}`);
33
      case 'css':
34
        return this.page.$(selector);
35
      case 'xpath':
36
        return this.page.$(`xpath=${selector}`);
37
      case 'text':
38
        return this.page.$(`text=${selector}`);
39
      default:
40
        return null;
41
    }
42
  }
43

44
  buildSelector(selectors, strategy) {
45
    if (selectors[strategy]) {
46
      return selectors[strategy];
47
    }
48
    return selectors.text || selectors.name;
49
  }
50
}

3. 错误处理与恢复机制#

全局错误处理#

1
class ErrorHandler {
2
  constructor() {
3
    this.retryCount = 3;
4
    this.retryDelay = 2000;
5
    this.maxRetryCount = 5;
6
  }
7

8
  async executeWithRetry(fn, context = {}) {
9
    let attempts = 0;
10
    let lastError = null;
11

12
    while (attempts < this.retryCount) {
13
      try {
14
        return await fn();
15
      } catch (error) {
16
        attempts++;
17
        lastError = error;
18

19
        console.warn(`执行失败 (尝试 ${attempts}/${this.retryCount}):`, error.message);
20

21
        if (this.isRecoverable(error)) {
22
          await this.sleep(this.retryDelay);
23
          await this.recoveryAction(context);
24
        } else {
25
          throw error;
26
        }
27
      }
28
    }
29

30
    throw new Error(`达到最大重试次数 (${this.retryCount})`, { cause: lastError });
31
  }
32

33
  isRecoverable(error) {
34
    const recoverableErrors = [
35
      'TIMEOUT',
36
      'ELEMENT_NOT_INTERACTABLE',
37
      'NETWORK_ERROR',
38
      'CONNECTION_REFUSED'
39
    ];
40

41
    return recoverableErrors.some(type => error.message.includes(type));
42
  }
43

44
  async recoveryAction(context) {
45
    if (context.page) {
46
      await context.page.reload();
47
    }
48

49
    if (context.session) {
50
      await context.session.refresh();
51
    }
52
  }
53

54
  sleep(ms) {
55
    return new Promise(resolve => setTimeout(resolve, ms));
56
  }
57
}

4. 性能优化#

缓存机制#

1
class CacheManager {
2
  constructor(cacheTTL = 3600000) { // 默认1小时
3
    this.cache = new Map();
4
    this.ttl = cacheTTL;
5
  }
6

7
  get(key) {
8
    const item = this.cache.get(key);
9
    if (item && Date.now() - item.timestamp < this.ttl) {
10
      return item.data;
11
    }
12
    return null;
13
  }
14

15
  set(key, data) {
16
    this.cache.set(key, {
17
      data,
18
      timestamp: Date.now()
19
    });
20
  }
21

22
  clear() {
23
    this.cache.clear();
24
  }
25

26
  cleanup() {
27
    const now = Date.now();
28
    for (const [key, item] of this.cache.entries()) {
29
      if (now - item.timestamp > this.ttl) {
30
        this.cache.delete(key);
31
      }
32
    }
33
  }
34
}

并发控制#

1
class ConcurrentExecutor {
2
  constructor(maxConcurrency = 3) {
3
    this.maxConcurrency = maxConcurrency;
4
    this.running = new Set();
5
    this.queue = [];
6
  }
7

8
  async addTask(task) {
9
    return new Promise((resolve, reject) => {
10
      this.queue.push({ task, resolve, reject });
11
      this.processQueue();
12
    });
13
  }
14

15
  async processQueue() {
16
    while (this.running.size < this.maxConcurrency && this.queue.length > 0) {
17
      const { task, resolve, reject } = this.queue.shift();
18
      const promise = task();
19
      this.running.add(promise);
20

21
      promise
22
        .then(result => {
23
          this.running.delete(promise);
24
          resolve(result);
25
          this.processQueue();
26
        })
27
        .catch(error => {
28
          this.running.delete(promise);
29
          reject(error);
30
          this.processQueue();
31
        });
32
    }
33
  }
34
}

监控与日志#

1. 结构化日志#

1
class Logger {
2
  constructor() {
3
    this.levels = {
4
      error: 0,
5
      warn: 1,
6
      info: 2,
7
      debug: 3
8
    };
9
    this.currentLevel = 'info';
10
  }
11

12
  log(level, message, data = {}) {
13
    if (this.levels[level] <= this.levels[this.currentLevel]) {
14
      const logEntry = {
15
        timestamp: new Date().toISOString(),
16
        level,
17
        message,
18
        data
19
      };
20

21
      console.log(JSON.stringify(logEntry));
22

23
      // 也可写入文件或发送到监控系统
24
      if (process.env.NODE_ENV === 'production') {
25
        this.writeToFile(logEntry);
26
      }
27
    }
28
  }
29

30
  error(message, data) {
31
    this.log('error', message, data);
32
  }
33

34
  warn(message, data) {
35
    this.log('warn', message, data);
36
  }
37

38
  info(message, data) {
39
    this.log('info', message, data);
40
  }
41

42
  debug(message, data) {
43
    this.log('debug', message, data);
44
  }
45

46
  writeToFile(logEntry) {
47
    // 实现文件写入逻辑
48
  }
49
}

2. 性能监控#

1
class PerformanceMonitor {
2
  constructor() {
3
    this.metrics = {
4
      requestTimes: [],
5
      errorCounts: {},
6
      successRates: []
7
    };
8
  }
9

10
  startTimer(operation) {
11
    return {
12
      operation,
13
      startTime: Date.now()
14
    };
15
  }
16

17
  endTimer(timer) {
18
    const duration = Date.now() - timer.startTime;
19
    this.metrics.requestTimes.push({
20
      operation: timer.operation,
21
      duration,
22
      timestamp: Date.now()
23
    });
24

25
    return duration;
26
  }
27

28
  getMetrics() {
29
    return {
30
      ...this.metrics,
31
      averageResponseTime: this.calculateAverageTime(),
32
      errorRate: this.calculateErrorRate()
33
    };
34
  }
35

36
  calculateAverageTime() {
37
    if (this.metrics.requestTimes.length === 0) return 0;
38

39
    const totalTime = this.metrics.requestTimes.reduce(
40
      (sum, item) => sum + item.duration, 0
41
    );
42

43
    return totalTime / this.metrics.requestTimes.length;
44
  }
45

46
  calculateErrorRate() {
47
    // 计算错误率逻辑
48
    return 0; // 示例值
49
  }
50
}

部署策略#

1. Docker化部署#

1
FROM node:18-alpine
2

3
# 安装Chrome
4
RUN apk add --no-cache \
5
    chromium \
6
    nss \
7
    freetype \
8
    harfbuzz \
9
    ca-certificates \
10
    ttf-freefont
11

12
# 设置Chrome路径
13
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
14

15
# 设置工作目录
16
WORKDIR /app
17

18
# 复制package文件
19
COPY package*.json ./
20

21
# 安装依赖
22
RUN npm ci --only=production
23

24
# 复制应用代码
25
COPY . .
26

27
# 构建应用
28
RUN npm run build
29

30
# 暴露端口
31
EXPOSE 3000
32

33
# 启动应用
34
CMD ["node", "dist/app.js"]

2. 环境变量配置#

1
module.exports = {
2
  database: {
3
    host: process.env.DB_HOST || 'localhost',
4
    port: process.env.DB_PORT || 5432,
5
    username: process.env.DB_USERNAME,
6
    password: process.env.DB_PASSWORD
7
  },
8

9
  automation: {
10
    maxRetries: parseInt(process.env.MAX_RETRIES) || 3,
11
    timeout: parseInt(process.env.TIMEOUT) || 30000,
12
    headless: process.env.HEADLESS !== 'false'
13
  },
14

15
  logging: {
16
    level: process.env.LOG_LEVEL || 'info',
17
    file: process.env.LOG_FILE || 'automation.log'
18
  }
19
};

实际应用案例#

1. 电商价格监控#

1
class PriceMonitor {
2
  constructor(url, selectors) {
3
    this.url = url;
4
    this.selectors = selectors;
5
    this.priceHistory = [];
6
  }
7

8
  async getCurrentPrice() {
9
    const browser = await puppeteer.launch({ headless: 'new' });
10
    const page = await browser.newPage();
11

12
    try {
13
      await page.goto(this.url, { waitUntil: 'networkidle2' });
14
      await page.waitForSelector(this.selectors.price);
15

16
      const priceElement = await page.$(this.selectors.price);
17
      const priceText = await priceElement.evaluate(el => el.textContent);
18

19
      const price = this.extractPrice(priceText);
20

21
      this.priceHistory.push({
22
        timestamp: new Date(),
23
        price
24
      });
25

26
      return price;
27
    } finally {
28
      await browser.close();
29
    }
30
  }
31

32
  extractPrice(text) {
33
    // 价格提取逻辑
34
    const match = text.match(/(\d+\.?\d*)/);
35
    return match ? parseFloat(match[1]) : null;
36
  }
37

38
  getPriceHistory() {
39
    return this.priceHistory;
40
  }
41
}

2. 社交媒体数据采集#

1
class SocialMediaScraper {
2
  constructor(platform) {
3
    this.platform = platform;
4
    this.userAgent = this.getUserAgent();
5
  }
6

7
  async scrapeProfile(profileUrl) {
8
    const browser = await puppeteer.launch({
9
      headless: 'new',
10
      args: [`--user-agent=${this.userAgent}`]
11
    });
12

13
    const page = await browser.newPage();
14

15
    try {
16
      await page.goto(profileUrl, {
17
        waitUntil: 'networkidle2',
18
        timeout: 60000
19
      });
20

21
      const data = await this.extractProfileData(page);
22

23
      return {
24
        profile: data,
25
        timestamp: new Date()
26
      };
27
    } finally {
28
      await browser.close();
29
    }
30
  }
31

32
  async extractProfileData(page) {
33
    // 数据提取逻辑
34
    return {
35
      followers: await this.getFollowersCount(page),
36
      posts: await this.getPostsCount(page),
37
      bio: await this.getBio(page)
38
    };
39
  }
40

41
  getUserAgent() {
42
    const userAgents = [
43
      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
44
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
45
      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36'
46
    ];
47

48
    return userAgents[Math.floor(Math.random() * userAgents.length)];
49
  }
50
}

性能优化技巧#

1. 并发处理#

1
async function processMultipleUrls(urls, batchSize = 3) {
2
  const results = [];
3
  const batches = [];
4

5
  // 将URL分批
6
  for (let i = 0; i < urls.length; i += batchSize) {
7
    batches.push(urls.slice(i, i + batchSize));
8
  }
9

10
  // 并行处理每批
11
  for (const batch of batches) {
12
    const batchPromises = batch.map(async (url) => {
13
      return await processUrl(url);
14
    });
15

16
    const batchResults = await Promise.all(batchPromises);
17
    results.push(...batchResults);
18
  }
19

20
  return results;
21
}

2. 智能等待策略#

1
class SmartWaiter {
2
  constructor(page) {
3
    this.page = page;
4
    this.defaultTimeout = 30000;
5
    this.maxTimeout = 120000;
6
  }
7

8
  async waitForElement(selector, options = {}) {
9
    const startTime = Date.now();
10
    let timeout = options.timeout || this.defaultTimeout;
11

12
    return new Promise(async (resolve, reject) => {
13
      try {
14
        await this.page.waitForSelector(selector, {
15
          timeout,
16
          visible: options.visible || true
17
        });
18
        resolve(await this.page.$(selector));
19
      } catch (error) {
20
        if (error.name === 'TimeoutError') {
21
          const elapsed = Date.now() - startTime;
22
          if (elapsed < this.maxTimeout) {
23
            // 尝试恢复策略
24
            await this.recoveryStrategy();
25
            // 重试
26
            return this.waitForElement(selector, {
27
              timeout: Math.min(timeout * 1.5, this.maxTimeout),
28
              ...options
29
            }).then(resolve).catch(reject);
30
          }
31
        }
32
        reject(error);
33
      }
34
    });
35
  }
36

37
  async recoveryStrategy() {
38
    // 恢复策略：刷新页面、重新登录等
39
    try {
40
      await this.page.reload({ waitUntil: 'networkidle2' });
41
    } catch (error) {
42
      console.warn('恢复策略失败:', error.message);
43
    }
44
  }
45
}

总结#

构建生产级Web自动化解决方案需要综合考虑架构设计、错误处理、性能优化等多个方面。通过模块化设计、智能定位策略、完善的错误处理机制和性能优化手段，可以打造出稳定、高效的自动化解决方案。

最重要的是，持续的监控和改进是确保自动化系统长期稳定运行的关键。通过收集运行数据、分析性能指标，不断优化和完善系统，才能适应不断变化的环境需求。

本文首发于Fuwari博客，欢迎交流讨论