Python feapder & ScrapySplash Tunnel Proxy: Framework Extensions

import os, feapder
 
class DemoSpider(feapder.AirSpider):
    def start_requests(self):
        yield feapder.Request("https://httpbin.org/ip")
 
    def download_midware(self, request):
        host = os.getenv("PROXY_HOST", "t.16yun.cn")
        port = os.getenv("PROXY_PORT", "31111")
        user = os.getenv("PROXY_USERNAME", "user")
        pwd = os.getenv("PROXY_PASSWORD", "password")
        request.proxies = {
            "http": f"http://{user}:{pwd}@{host}:{port}",
            "https": f"http://{user}:{pwd}@{host}:{port}",
        }
        return request
 
    def parse(self, request, response):
        print(response.text)
 
if __name__ == "__main__":
    DemoSpider().start()

# settings.py
SPLASH_URL = "http://splash:8050"
 
SPLASH_PROXY = {
    "host": os.getenv("PROXY_HOST", "t.16yun.cn"),
    "port": int(os.getenv("PROXY_PORT", "31111")),
    "username": os.getenv("PROXY_USERNAME", "user"),
    "password": os.getenv("PROXY_PASSWORD", "password"),
}
 
DOWNLOADER_MIDDLEWARES = {
    "scrapy_splash.SplashDeduplicateArgsMiddleware": 100,
}
 
DUPEFILTER_CLASS = "scrapy_splash.SplashAwareDupeFilter"

Scrapy → Splash (JS rendering) → 16Yun Crawler Proxy → Target site