以文件下载为例,
1. 单线程
import requests
import uuid
url_list = ['url1', 'url2']
for url in url_list:
html = requests.get(url, stream=True)
file_name = uuid.uuid1()
open(f'{file_name}.json', 'wb').write(html.content)
或:
import requests
from time import time
url_list = [
"https://via.placeholder.com/400",
"https://via.placeholder.com/410",
"https://via.placeholder.com/420",
"https://via.placeholder.com/430",
"https://via.placeholder.com/440",
"https://via.placeholder.com/450",
"https://via.placeholder.com/460",
"https://via.placeholder.com/470",
"https://via.placeholder.com/480",
"https://via.placeholder.com/490",
"https://via.placeholder.com/500",
"https://via.placeholder.com/510",
"https://via.placeholder.com/520",
"https://via.placeholder.com/530",
]
def download_file(url):
html = requests.get(url, stream=True)
return html.status_code
start = time()
for url in url_list:
print(download_file(url))
print(f'Time taken: {time() - start}')
2. 多线程
import requests
import uuid
from concurrent.futures import ThreadPoolExecutor, as_completed
url_list = ['url1', 'url2']
def download_file(url, file_name):
try:
html = requests.get(url, stream=True)
open(f'{file_name}.json', 'wb').write(html.content)
return html.status_code
except requests.exceptions.RequestException as e:
return e
def runner():
threads= []
with ThreadPoolExecutor(max_workers=20) as executor:
for url in url_list:
file_name = uuid.uuid1()
threads.append(executor.submit(download_file, url, file_name))
for task in as_completed(threads):
print(task.result())
#
runner()
或,
import requests
from concurrent.futures import ThreadPoolExecutor, as_completed
from time import time
url_list = [
"https://via.placeholder.com/400",
"https://via.placeholder.com/410",
"https://via.placeholder.com/420",
"https://via.placeholder.com/430",
"https://via.placeholder.com/440",
"https://via.placeholder.com/450",
"https://via.placeholder.com/460",
"https://via.placeholder.com/470",
"https://via.placeholder.com/480",
"https://via.placeholder.com/490",
"https://via.placeholder.com/500",
"https://via.placeholder.com/510",
"https://via.placeholder.com/520",
"https://via.placeholder.com/530",
]
def download_file(url):
html = requests.get(url, stream=True)
return html.status_code
start = time()
processes = []
with ThreadPoolExecutor(max_workers=10) as executor:
for url in url_list:
processes.append(executor.submit(download_file, url))
for task in as_completed(processes):
print(task.result())
print(f'Time taken: {time() - start}')