Python Selenium Chrome Tunnel Proxy: Browser Automation with Crawler Proxy

Chrome Extension Proxy Auth

Selenium Chrome requires a Chrome extension to handle proxy authentication — Chrome doesn't support passing proxy credentials via command-line arguments.

Chrome Extension Generator

import os, string, tempfile, zipfile
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

def create_proxy_extension(host, port, username, password):
    manifest_json = """
    {"version":"1.0.0","manifest_version":2,"name":"Chrome Proxy",
    "permissions":["proxy","tabs","unlimitedStorage","storage",
    "<all_urls>","webRequest","webRequestBlocking"],
    "background":{"scripts":["background.js"]}}
    """
    background_js = string.Template("""
    var config = {mode:"fixed_servers",rules:{singleProxy:{
        scheme:"http",host:"${host}",port:parseInt(${port})
    },bypassList:["<local>"]}};
    chrome.proxy.settings.set({value:config,scope:"regular"},function(){});
    function callbackFn(details){return {authCredentials:{
        username:"${username}",password:"${password}"}};}
    chrome.webRequest.onAuthRequired.addListener(
        callbackFn,{urls:["<all_urls>"]},["blocking"]);
    """).substitute(host=host, port=port, username=username, password=password)

    tmpdir = tempfile.mkdtemp(prefix="chrome-proxy-")
    path = os.path.join(tmpdir, "proxyauth_plugin.zip")
    with zipfile.ZipFile(path, "w") as zp:
        zp.writestr("manifest.json", manifest_json)
        zp.writestr("background.js", background_js)
    return path

Launch Chrome with Extension

host = os.getenv("PROXY_HOST", "t.16yun.cn")
port = os.getenv("PROXY_PORT", "31111")
user = os.getenv("PROXY_USERNAME", "user")
pwd = os.getenv("PROXY_PASSWORD", "password")

options = Options()
options.add_argument(f"--proxy-server=http://{host}:{port}")
options.add_argument("--headless=new")
options.add_argument("--no-sandbox")

extension_path = create_proxy_extension(host, port, user, pwd)
options.add_extension(extension_path)

driver = webdriver.Chrome(options=options)
driver.get("http://httpbin.org/ip")
print(driver.page_source[:500])
driver.quit()

Use HTTP targets to avoid certificate issues. Add --ignore-certificate-errors for HTTPS.

Scenario Reference

curl -x http://$PROXY_USERNAME:$PROXY_PASSWORD@$PROXY_HOST:$PROXY_PORT http://httpbin.org/ip