• 28 comments
• Andrew Coyne

10. Xpath元素定位

xpath 是一种从XML对象中定位元素的一种方式
在很多框架中广泛使用, 收到绝大多数浏览器的支持.

Xpath基本用法
 //li 后代元素中的li
 /li  子元素中的li
 div[@id='abc'] 一个id为abc的div元素
 div[@class='def'] class为def的所有div
 *[@class='def'] class为def的所有元素
提取文本  /text() 子元素文本    //text()所有后代元素的文本
提取属性  /@属性名

import requests
# 导入lxml包中的etree对象,提供了对xpath的支持
from lxml import etree
# 原始url
# http://news.hebtu.edu.cn/a/zhxw/index.html
# 通过点击翻页按钮抓包,post请求,同时找到真的请求地址.
url = "http://news.hebtu.edu.cn/dynamic/PageFront.jsp"
# 构建请求头
headers = {
    "Origin": "http://news.hebtu.edu.cn",
    "Referer": "http://news.hebtu.edu.cn/dynamic/PageFront.jsp",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36"
}
# 构建form data
data = {
    "parseId": "9c8c6f6731504f05862f68889b94c182",
    "categoryId": "a15b330c05744aaab39dd6c801ea01fc",
    "pageNo": 1
}
r = requests.post(url,data=data,headers=headers)
# 用etree对象的HTML方法,将返回的HTML文本(r.text)解析成etree对象
html = etree.HTML(r.text)
# etree对象的xpath方法来定位元素.
title = html.xpath('//div[@class="item"]/ul/li/a/text()')
title

抓取链接的href地址

import requests
from lxml import etree
url = "http://www.shixiaolei.com/posts/3/"
r = requests.get(url)
html = etree.HTML(r.text)
hrefs = html.xpath('//div[@class="title"]/a/@href')
hrefs

抓取图片的src地址

import requests
from lxml import etree
url = "https://www.woyaogexing.com/touxiang/nv/2022/1223978.html"
r = requests.get(url)
html = etree.HTML(r.text)
srcs = html.xpath('//img[@class="lazy"]/@src')
srcs

抓取元素所有的后代元素文本

import requests
from lxml import etree
url = "http://www.hebkx.cn/indexnews/html/?6732.html"
r = requests.get(url)
html = etree.HTML(r.text)
contents = html.xpath('//div[@class="detail"]//text()')
contents