Firecrawl 使用教程：站点发现、批量提取与结果导出

export FIRECRAWL_API_BASE='<paste-the-actual-firecrawl-api-base>'
export TARGET_URL='<paste-the-actual-target-url>'

export FIRECRAWL_API_KEY='replace-with-your-key'
# TARGET_URL and FIRECRAWL_API_BASE are configured above
export TARGET_PATH_PATTERN='^/3/tutorial/.*'

import argparse
import json
import os
import time
from pathlib import Path
from urllib.parse import quote, urljoin, urlsplit
from urllib.request import Request, urlopen
 
BASE = os.environ["FIRECRAWL_API_BASE"].rstrip("/") + "/"
API_ORIGIN = urlsplit(BASE)
if API_ORIGIN.scheme != "https" or not API_ORIGIN.netloc:
    raise ValueError("FIRECRAWL_API_BASE must be an HTTPS API base")
TOKEN = os.environ["FIRECRAWL_API_KEY"]
OUT = Path("firecrawl-output")
OUT.mkdir(exist_ok=True)
 
def save(name, data):
    (OUT / name).write_text(
        json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8"
    )
 
def api(path, payload=None):
    url = urljoin(BASE, path)
    if urlsplit(url).scheme != "https" or urlsplit(url).netloc != API_ORIGIN.netloc:
        raise ValueError("结果分页地址不属于 Firecrawl API")
    request = Request(
        url,
        data=None if payload is None else json.dumps(payload).encode(),
        headers={"Authorization": f"Bearer {TOKEN}", "Content-Type": "application/json"},
        method="GET" if payload is None else "POST",
    )
    with urlopen(request, timeout=30) as response:
        result = json.load(response)
    if result.get("success") is False:
        raise RuntimeError("API 返回失败，请检查请求与账户状态")
    return result
 
parser = argparse.ArgumentParser()
parser.add_argument("action", choices=["start", "collect"])
args = parser.parse_args()
 
if args.action == "start":
    if (OUT / "job.json").exists():
        raise SystemExit("已有 job.json；请先 collect 或为新任务使用新目录")
    root = os.environ["TARGET_URL"]
    pattern = os.environ["TARGET_PATH_PATTERN"]
    save("map.json", api("map", {"url": root}))
    created = api("crawl", {
        "url": root, "limit": 10, "includePaths": [pattern],
        "scrapeOptions": {"formats": ["markdown"], "onlyMainContent": True},
    })
    if not created.get("id"):
        raise RuntimeError("创建响应缺少任务 ID")
    save("job.json", {"id": created["id"], "target": root, "path_pattern": pattern})
    print("任务 ID 已保存；执行 collect 查询")
else:
    job = json.loads((OUT / "job.json").read_text(encoding="utf-8"))
    path = "crawl/" + quote(job["id"], safe="")
    for _ in range(30):
        status = api(path)
        save("status.json", status)
        if status["status"] in {"completed", "failed", "cancelled"}:
            break
        time.sleep(5)
    else:
        raise SystemExit("本轮等待结束；保留任务 ID，稍后再次 collect")
    if status["status"] != "completed":
        raise SystemExit("任务未正常完成，请检查 status.json")
 
    documents = list(status.get("data", []))
    next_url = status.get("next")
    visited = set()
    while next_url:
        if next_url in visited:
            raise RuntimeError("结果分页地址重复，停止以避免循环")
        visited.add(next_url)
        part = api(next_url)
        documents.extend(part.get("data", []))
        next_url = part.get("next")
    save("documents.json", documents)
    index, rejected = [], []
    for number, document in enumerate(documents):
        meta = document.get("metadata") or {}
        url = meta.get("sourceURL") or meta.get("url")
        markdown = document.get("markdown")
        if not url or not isinstance(markdown, str) or not markdown.strip():
            rejected.append({"index": number, "metadata": meta})
            continue
        filename = f"page-{number:04d}.md"
        (OUT / filename).write_text(markdown, encoding="utf-8")
        index.append({"source_url": url, "title": meta.get("title"), "file": filename})
    save("index.json", index)
    save("rejected.json", rejected)
    print({"returned": len(documents), "exported": len(index), "rejected": len(rejected)})

python firecrawl_help.py start
python firecrawl_help.py collect

[
  {
    "source_url": "https://example.16yun.cn/3/tutorial/index.html",
    "title": "页面实际标题",
    "file": "page-0000.md"
  }
]