Bulk File Download

Download hundreds of files with concurrency control and retry:

import httpx
from pathlib import Path
from pyarallel import async_parallel_map, RateLimit, Retry

# ONE client for the whole run: connection pooling + TLS session reuse.
# A client-per-download renegotiates TLS on every file — the #1 async
# HTTP anti-pattern.
client = httpx.AsyncClient()

async def download(url):
    r = await client.get(url, timeout=30, follow_redirects=True)
    r.raise_for_status()
    filename = Path("downloads") / url.split("/")[-1]
    filename.write_bytes(r.content)
    return {"url": url, "size": len(r.content)}

urls = [...]  # hundreds of file URLs

result = await async_parallel_map(
    download, urls,
    concurrency=20,
    retry=Retry(attempts=3, backoff=2.0, on=(httpx.ConnectError, httpx.ReadTimeout)),
)

if not result.ok:
    for idx, exc in result.failures():
        print(f"Failed: {urls[idx]} — {exc}")