开源项目分享：Reader ， 专为 LLM 设计的生产级网页抓取引擎

npm install @vakra-dev/reader

# Debian/Ubuntu
sudo apt-get install -y libnspr4 libnss3 libatk1.0-0 libatk-bridge2.0-0 \
  libcups2 libxcb1 libatspi2.0-0 libx11-6 libxcomposite1 libxdamage1 \
  libxext6 libxfixes3 libxrandr2 libgbm1 libcairo2 libpango-1.0-0 libasound2

import { ReaderClient } from "@vakra-dev/reader";

const reader = new ReaderClient();

const result = await reader.scrape({
  urls: ["https://example.com"],
  formats: ["markdown", "html"],
});

console.log(result.data[0].markdown);
await reader.close();

const result = await reader.scrape({
  urls: ["https://example.com", "https://example.org"],
  formats: ["markdown"],
  batchConcurrency: 3,
  onProgress: (progress) => {
    console.log(`${progress.completed}/${progress.total}: ${progress.currentUrl}`);
  },
});

const result = await reader.crawl({
  url: "https://example.com",
  depth: 2,
  maxPages: 20,
  scrape: true,
});

console.log(`发现 ${result.urls.length} 个页面`);

const result = await reader.scrape({
  urls: ["https://example.com"],
  formats: ["markdown"],
  proxy: {
    type: "residential",
    host: "proxy.example.com",
    port: 8080,
    username: "username",
    password: "password",
    country: "us",
  },
});

# 启动守护进程
npx reader start --pool-size 5

# 后续命令自动连接守护进程
npx reader scrape https://example.com
npx reader crawl https://example.com -d 2

# 查看状态
npx reader status

# 停止守护进程
npx reader stop

# 抓取单个 URL
npx reader scrape https://example.com

# 多格式输出
npx reader scrape https://example.com -f markdown,html

# 并发抓取多个 URL
npx reader scrape https://example.com https://example.org -c 2

# 保存到文件
npx reader scrape https://example.com -o output.md

# 基础爬取
npx reader crawl https://example.com

# 深度爬取并抓取内容
npx reader crawl https://example.com -d 3 -m 50 --scrape

# URL 过滤
npx reader crawl https://example.com --include "blog/*" --exclude "admin/*"

const reader = new ReaderClient({
  browserPool: {
    size: 5,                    // 5 个浏览器实例
    retireAfterPages: 50,       // 50 页后回收
    retireAfterMinutes: 15,     // 15 分钟后回收
  },
  verbose: true,
});

const reader = new ReaderClient({
  proxies: [
    { host: "proxy1.example.com", port: 8080, username: "user", password: "pass" },
    { host: "proxy2.example.com", port: 8080, username: "user", password: "pass" },
  ],
  proxyRotation: "round-robin", // 或 "random"
});

import HeroCore from "@ulixee/hero-core";
import { TransportBridge } from "@ulixee/net";
import { ConnectionToHeroCore } from "@ulixee/hero";
import { scrape } from "@vakra-dev/reader";

// 启动时初始化一次
const heroCore = new HeroCore();
await heroCore.start();

// 每个请求创建连接
function createConnection() {
  const bridge = new TransportBridge();
  heroCore.addConnection(bridge.transportToClient);
  return new ConnectionToHeroCore(bridge.transportToCore);
}

// 在请求中使用
const result = await scrape({
  urls: ["https://example.com"],
  connectionToCore: createConnection(),
});