Python爬虫完全指导

...
<header><h2><a href="https://www...">【易大师网络工具箱】项目解析</a></h2></header>
...
<header><h2><a href="https://www...">JavaScript纯符号输出文本</a></h2></header>
...
<header><h2><a href="https://www...">几道不太简单的C艹面试题</a></h2></header>
...

pip install mitmproxy
pip install bs4
pip install requests
pip install selenium

import requests
from bs4 import BeautifulSoup

# HTTP请求内容，根据需要可以修改，主要是为了模拟浏览器
_headers = {
	'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
	'Accept-Encoding': 'gzip, deflate, br',
	'Accept-Language': 'zh-CN,zh;q=0.9',
	'Cache-Control': 'max-age=0',
	'Cookie': '',
	'Upgrade-Insecure-Requests': '1',
	'User-Agent': 'Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3423.2 Mobile Safari/537.36'
	#'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3423.2 Safari/537.36'
}
# 执行请求
_data = requests.get ('https://www.fawdlstty.com/', headers = _headers).text

_soup = BeautifulSoup (_data, 'html.parser')
_items = [x.text for x in _soup.select ('header > h2 > a')]

# 引入库
from bs4 import BeautifulSoup
from selenium import webdriver
 
# 正常打开chrome
_driver = webdriver.Chrome ()
# 打开隐藏的chrome
_chrome_options = webdriver.chrome.options.Options ()
_chrome_options.add_argument('--headless')
_chrome_options.add_argument('--disable-gpu')
_driver = webdriver.Chrome (chrome_options = _chrome_options)
# 打开带有代理的chrome
_chrome_options = webdriver.chrome.options.Options ()
_chrome_options.add_argument ('--proxy-server= http://127.0.0.1:8001')
_driver = webdriver.Chrome (chrome_options = _chrome_options)
# 打开模拟手机浏览器的chrome
_chrome_options = webdriver.chrome.options.Options ()
_chrome_options.add_experimental_option ("mobileEmulation", {"deviceName": "Nokia Lumia 520"})#iPhone X
_driver = webdriver.Chrome (chrome_options = _chrome_options)
# 打开IE
_driver = webdriver.Ie ()
# 打开firefox
_driver = webdriver.Firefox ()
 
# 普遍需要加入的代码
_driver.set_page_load_timeout (60)
_driver.set_script_timeout (60)
_driver.implicitly_wait (30)
_driver.delete_all_cookies ()
 
# 关闭浏览器代码
_driver.quit ()

# 打开博客
_driver.get ('https://www.fawdlstty.com')
 
# 根据xpath查找元素
_elem = _driver.find_element_by_xpath ('body')
_elems = _driver.find_elements_by_xpath ('body')
 
# 根据id查找元素
_elem = _driver.find_element_by_id ('body')
 
# 根据类名查找元素
_elem = _driver.find_element_by_class_name ('cls_name')
_elems = _driver.find_elements_by_class_name ('cls_name')
 
# 显示元素内容
print (_elem.text)
 
# 显示当前地址
print (_driver.current_url)
 
# 获取当前cookie
_cookie = ';'.join ([item['name'] + '=' + item['value'] for item in _driver.get_cookies ()])
 
# 模拟输入
_elem.send_keys ('12345678')
 
# 执行js
_driver.execute_script ('alert ("hello")')
 
# 模拟点击
_elem.click ()
 
# 获取当前网页的BeautifulSoup对象，用于解析数据
# 注意此处源码非实际源码，比如table里面可能会被浏览器插入tbody
_bs = BeautifulSoup (_driver.page_source, 'html.parser')
 
# 其他的操作，使用 dir (_elem) 找到元素所有属性或方法，通过help指令来获取使用方式

import re
from mitmproxy import ctx
 
def response(flow):
    if '/scripts/' in flow.request.url:
        for webdriver_key in ['webdriver', '__driver_evaluate', '__webdriver_evaluate', '__selenium_evaluate', '__fxdriver_evaluate', '__driver_unwrapped', '__webdriver_unwrapped', '__selenium_unwrapped', '__fxdriver_unwrapped', '_Selenium_IDE_Recorder', '_selenium', 'calledSelenium', '_WEBDRIVER_ELEM_CACHE', 'ChromeDriverw', 'driver-evaluate', 'webdriver-evaluate', 'selenium-evaluate', 'webdriverCommand', 'webdriver-evaluate-response', '__webdriverFunc', '__webdriver_script_fn', '__$webdriverAsyncExecutor', '__lastWatirAlert', '__lastWatirConfirm', '__lastWatirPrompt', '$chrome_asyncScriptInfo', '$cdc_asdjflasutopfhvcZLmcfl_']:
            ctx.log.info('Remove "{}" from {}.'.format(webdriver_key, flow.request.url))
        flow.response.text = flow.response.text.replace('"{}"'.format(webdriver_key), '"NO-SUCH-ATTR"')
    flow.response.text = flow.response.text.replace('webdriver', '_webdriver_').replace('ChromeDriver', '_ChromeDriver_')