"""Scrapy 2.19 example with HTTP proxy assignment before HttpProxyMiddleware.

Set BP_PROXY_ENDPOINTS_JSON to a JSON array of http://host:port strings,
BP_PROXY_USER, BP_PROXY_PASSWORD, and BP_TARGET_URL to an endpoint you control.
--check validates configuration using only the standard library; no network.
Run with python -B scrapy-proxy-spider.py after installing Scrapy 2.19.
"""
import argparse
from itertools import cycle
import json
import os
from urllib.parse import quote, urlsplit


def configuration():
    endpoints = json.loads(os.environ["BP_PROXY_ENDPOINTS_JSON"])
    if not isinstance(endpoints, list) or not endpoints:
        raise ValueError("Proxy endpoint list must be nonempty")
    user, password = os.environ["BP_PROXY_USER"], os.environ["BP_PROXY_PASSWORD"]
    if ":" in user:
        raise ValueError("Basic authentication usernames cannot contain a colon")
    (user + ":" + password).encode("latin1")
    proxies = []
    for endpoint in endpoints:
        if not isinstance(endpoint, str) or any(c.isspace() for c in endpoint):
            raise ValueError("Each endpoint must be a URL string without whitespace")
        parsed = urlsplit(endpoint)
        if (parsed.scheme != "http" or not parsed.hostname or not parsed.port
                or parsed.username is not None or parsed.password is not None
                or parsed.path or parsed.query or parsed.fragment):
            raise ValueError("Each endpoint must be http://host:port without credentials")
        if not 1 <= parsed.port <= 65535:
            raise ValueError("Invalid proxy port")
        auth = f"{quote(user, safe='')}:{quote(password, safe='')}"
        proxies.append(f"http://{auth}@{parsed.netloc}")
    target = os.environ["BP_TARGET_URL"]
    parsed = urlsplit(target)
    if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username is not None:
        raise ValueError("Target must be an HTTP(S) URL without userinfo")
    return proxies, target


class ProxyPoolMiddleware:
    @classmethod
    def from_crawler(cls, crawler):
        middleware = cls()
        proxies, unused_target = configuration()
        middleware.pool = cycle(enumerate(proxies))
        return middleware

    def process_request(self, request):
        index, proxy = next(self.pool)
        request.meta["proxy"] = proxy
        request.meta["bp_proxy_id"] = f"pool-{index}"


def run(proxies, target):
    import scrapy
    from scrapy.crawler import CrawlerProcess

    class ProxyCheckSpider(scrapy.Spider):
        name = "bp_proxy_example"
        custom_settings = {
            "DOWNLOADER_MIDDLEWARES": {"__main__.ProxyPoolMiddleware": 740},
            "HTTPPROXY_ENABLED": True,
            "DOWNLOAD_VERIFY_CERTIFICATES": True,
            "DOWNLOAD_TIMEOUT": 20,
            "DOWNLOAD_MAXSIZE": 2_000_000,
            "CONCURRENT_REQUESTS": 4,
            "CONCURRENT_REQUESTS_PER_DOMAIN": 2,
            "DOWNLOAD_DELAY": 1,
            "AUTOTHROTTLE_ENABLED": True,
            "AUTOTHROTTLE_TARGET_CONCURRENCY": 1,
            "ROBOTSTXT_OBEY": True,
            "RETRY_TIMES": 1,
            "RETRY_HTTP_CODES": [408, 500, 502, 503, 504],
            "REDIRECT_ENABLED": False,
            "LOG_ENABLED": False,
        }

        async def start(self):
            yield scrapy.Request(target, callback=self.parse, errback=self.failed,
                                 meta={"handle_httpstatus_all": True})

        def parse(self, response):
            proxy_id = response.meta.get("bp_proxy_id", "unknown")
            print(f"response status={response.status} proxy={proxy_id} bytes={len(response.body)}")

        def failed(self, failure):
            proxy_id = failure.request.meta.get("bp_proxy_id", "unknown")
            print(f"request_failed proxy={proxy_id} type={failure.type.__name__}")

    process = CrawlerProcess()
    process.crawl(ProxyCheckSpider)
    process.start()


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--check", action="store_true")
    args = parser.parse_args()
    try:
        proxies, target = configuration()
    except (KeyError, TypeError, ValueError, UnicodeError):
        print("configuration_error: check environment variables and endpoint JSON")
        return 2
    if args.check:
        print(f"configuration_ok proxies={len(proxies)} credentials=redacted network=none")
        return 0
    run(proxies, target)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())