LLM Scraper 完整教程：从不同网页模板提取统一商品数据

mkdir product-extraction
cd product-extraction
npm init -y
npm install llm-scraper@2.0.0 ai@6 @ai-sdk/openai@3 zod@4 playwright
npx playwright install chromium

MODEL_BASE_URL=https://api.example.16yun.cn/v1
MODEL_API_KEY=replace-with-your-key
MODEL_NAME=replace-with-an-available-model

import { writeFile } from 'node:fs/promises';
import { chromium } from 'playwright';
import { createOpenAI } from '@ai-sdk/openai';
import { Output } from 'ai';
import { z } from 'zod';
import LLMScraper from 'llm-scraper';
 
const inputs = [
    { id: 'card', html: '<main><h1>山谷红茶 500g</h1><p>当前价 CNY 12.90</p><p>有货</p></main>' },
    { id: 'table', html: '<main><h1>山谷红茶 500g</h1><table><tr><th>原价</th><td>CNY 16.00</td></tr><tr><th>当前价</th><td>CNY 12.90</td></tr><tr><th>库存</th><td>有货</td></tr></table></main>' },
    { id: 'description', html: '<main><h1>山谷红茶 500g</h1><dl><dt>现售金额</dt><dd>12.90</dd><dt>币种</dt><dd>CNY</dd><dt>库存状态</dt><dd>有货</dd></dl></main>' },
];
const schema = z.object({
    name: z.string().min(1).describe('完整商品名称，不添加页面没有的型号'),
    price: z.string().regex(/^\d+\.\d{2}$/).nullable()
        .describe('当前公开售卖价格，只返回十进制金额；排除原价、运费和会员价，缺失时为 null'),
    currency: z.enum(['CNY', 'USD', 'GBP']).nullable()
        .describe('页面明确提供的币种，不根据语言或网站域名猜测'),
    availability: z.enum(['in_stock', 'out_of_stock', 'unknown'])
        .describe('由页面明确库存文案判断，无法判断时为 unknown'),
});

const { MODEL_BASE_URL, MODEL_API_KEY, MODEL_NAME } = process.env;
if (!MODEL_BASE_URL || !MODEL_API_KEY || !MODEL_NAME) {
    throw new Error('请配置 MODEL_BASE_URL、MODEL_API_KEY、MODEL_NAME');
}
const provider = createOpenAI({
    baseURL: MODEL_BASE_URL,
    apiKey: MODEL_API_KEY,
});
const scraper = new LLMScraper(provider.chat(MODEL_NAME));
const accepted: unknown[] = [];
const failed: unknown[] = [];
const browser = await chromium.launch({ headless: true });
 
try {
    for (const input of inputs) {
        const page = await browser.newPage();
        try {
            await page.setContent(input.html);
            const { data } = await scraper.run(
                page,
                Output.object({ schema }),
                { format: 'html' },
            );
            const product = schema.parse(data);
            if (product.price === null || product.currency === null) {
                throw new Error('当前价格或币种缺失');
            }
            accepted.push({
                source_id: input.id,
                source_url: null,
                ...product,
            });
        } catch (error) {
            failed.push({
                source_id: input.id,
                error: error instanceof Error ? error.name : 'UnknownError',
                stage: 'load_or_extract_or_validate',
            });
        } finally {
            await page.close();
        }
    }
} finally {
    await browser.close();
}
await writeFile('accepted.json', JSON.stringify(accepted, null, 2));
await writeFile('failed.json', JSON.stringify(failed, null, 2));
console.log({ accepted: accepted.length, failed: failed.length });
if (failed.length) process.exitCode = 1;

node --env-file=.env extract.mts

await page.goto(input.url, { waitUntil: 'domcontentloaded' });
await page.locator('main h1').waitFor();
const sourceUrl = page.url();

{
  "source_id": "table",
  "source_url": null,
  "name": "山谷红茶 500g",
  "price": "12.90",
  "currency": "CNY",
  "availability": "in_stock"
}