python 爬取壁紙網(wǎng)站的示例

import os
import bs4
import re
import time
import requests
from bs4 import BeautifulSoup

def getHTMLText(url, headers):
  """向目標(biāo)服務(wù)器發(fā)起請求并返回響應(yīng)"""
  try:
    r = requests.get(url=url, headers=headers)
    r.encoding = r.apparent_encoding
    soup = BeautifulSoup(r.text, "html.parser")
    return soup
  except:
    return ""

def CreateFolder():
  """創(chuàng)建存儲(chǔ)數(shù)據(jù)文件夾"""
  flag = True
  while flag == 1:
    file = input("請輸入保存數(shù)據(jù)文件夾的名稱：")
    if not os.path.exists(file):
      os.mkdir(file)
      flag = False
    else:
      print('該文件已存在，請重新輸入')
      flag = True

  # os.path.abspath(file) 獲取文件夾的絕對路徑
  path = os.path.abspath(file) + "\\"
  return path

def fillUnivList(ulist, soup):
  """獲取每一張圖片的原圖頁面"""
  # [0]使得獲得的ul是 class 'bs4.BeautifulSoup'> 類型
  div = soup.find_all('div', 'list')[0]
  for a in div('a'):
    if isinstance(a, bs4.element.Tag):
      hr = a.attrs['href']
      href = re.findall(r'/desk/[1-9]\d{4}.htm', hr)
      if bool(href) == True:
        ulist.append(href[0])

  return ulist

def DownloadPicture(left_url,list,path):
  for right in list:
    url = left_url + right
    r = requests.get(url=url, timeout=10)
    r.encoding = r.apparent_encoding
    soup = BeautifulSoup(r.text,"html.parser")
    tag = soup.find_all("p")
    # 獲取img標(biāo)簽的alt屬性，給保存圖片命名
    name = tag[0].a.img.attrs['alt']
    img_name = name + ".jpg"
    # 獲取圖片的信息
    img_src = tag[0].a.img.attrs['src']
    try:
      img_data = requests.get(url=img_src)
    except:
      continue

    img_path = path + img_name
    with open(img_path,'wb') as fp:
      fp.write(img_data.content)
    print(img_name, "  ******下載完成！")

def PageNumurl(urls):
  num = int(input("請輸入爬取所到的頁碼數(shù)："))
  for i in range(2,num+1):
    u = "http://www.netbian.com/index_" + str(i) + ".htm"
    urls.append(u)

  return urls


if __name__ == "__main__":
  uinfo = []
  left_url = "http://www.netbian.com"
  urls = ["http://www.netbian.com/index.htm"]
  headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36"
  }
  start = time.time()
  # 1.創(chuàng)建保存數(shù)據(jù)的文件夾
  path = CreateFolder()
  # 2. 確定要爬取的頁面數(shù)并返回每一頁的鏈接
  PageNumurl(urls)
  n = int(input("訪問的起始頁面："))
  for i in urls[n-1:]:
    # 3.獲取每一個(gè)頁面的首頁數(shù)據(jù)文本
    soup = getHTMLText(i, headers)
    # 4.訪問原圖所在頁鏈接并返回圖片的鏈接
    page_list = fillUnivList(uinfo, soup)
    # 5.下載原圖
    DownloadPicture(left_url, page_list, path)

  print("全部下載完成！", "共" + str(len(os.listdir(path))) + "張圖片")
  end = time.time()
  print("共耗時(shí)" + str(end-start) + "秒")