1. Beautiful Soup的简介
Beautiful Soup 是一个可以从HTML或XML文件中提取数据的Python库.它能够通过你喜欢的转换器实现惯用的文档导航,查找,修改文档的方式.Beautiful Soup会帮你节省数小时甚至数天的工作时间.Beautiful Soup 3 目前已经停止开发,我们推荐在现在的项目中使用Beautiful Soup 4, 移植到BS4，出现的例子在Python2.7和Python3.2中的执行结果相同
2.使用的示例代码
安装代码
sudo easy_install -U beautifulsoup4
下面的一段HTML代码将作为例子被多次用到.这是 爱丽丝梦游仙境的 的一段内容
html_doc = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title"><b>The Dormouse's story</b></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1">Elsie</a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""
常见的元素输出
#第一个参数是html对象
soup=BeautifulSoup(html,"lxml")
#输出<title>的元素
print soup.title
#输出<head>的元素
print soup.head
#输出<a>的元素
print soup.a
#输出<p>的元素
print soup.p
#输出name的元素
print soup.name
#输出head的name属性
print soup.head.name
#输出p元素的属性和对印值得到字典
print soup.p.attrs
#输出p元素的里面的class属性
print soup.p['class']
print soup.p.get('class')
#修改class属性内容
soup.p['class']="newClass"
print soup.p
#获取标签内容
print soup.p.string
#输出所有head属性的一个数组
print soup.head.contents
#遍历所有body的子节点
for child in soup.body.children:
print child
for child in soup.descendants:
print child
string 、strings和stripped_strings的效果
print soup.head.string
#The Dormouse's story
print soup.title.string
#The Dormouse's story
#获取多个内容，遍历获取所有元素
for string in soup.strings:
print(repr(string))
# u"The Dormouse's story"
# u'\n\n'
# u"The Dormouse's story"
# u'\n\n'
# u'Once upon a time there were three little sisters; and their names were\n'
# u'Elsie'
# u',\n'
# u'Lacie'
# u' and\n'
# u'Tillie'
# u';\nand they lived at the bottom of a well.'
# u'\n\n'
# u'...'
# u'\n
#输出的字符串中可能包含了很多空格或空行,使用 .stripped_strings 可以去除多余空白内容
for string in soup.stripped_strings:
print(repr(string))
#u'Once upon a time there were three little sisters; and their names were'
#u','
#u'Lacie'
#u'and'
#u'Tillie'
#u';\nand they lived at the bottom of a well.'
输出父节点parent、parents
content=soup.head.title.string
print content.parent.name
#title
for parent in content.parents:
print parent.name
#title
#head
#html
#[document]
兄弟节点next_sibling和prev_sibling
print soup.p.next_sibling
#
#
#
print soup.p.prev_sibling
#None
print soup.p.next_sibling.next_sibling
#<p class="story">Once upon a time there were three little sisters; and their names were
#<a class="sister" href="http://example.com/elsie" id="link1"><!-- Elsie --></a>,
#<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a> and
#<a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
搜索文档树
函数原型
find_all( name , attrs , recursive , string , **kwargs )
find_parents( name , attrs , recursive , string , **kwargs )
find_parent( name , attrs , recursive , string , **kwargs )
。。。
1.name查找所有名为name的tag
soup=BeautifulSoup(html,"lxml")
print soup.find_all('p')
#[<p class="title" name="dromouse"><b>The Dormouse's story</b></p>, <p class="story">Once upon a time there were three little si
sters; and their names were\n<a class="sister" href="http://example.com/elsie" id="link1"><!-- Elsie --></a>,\n<a class="sister
" href="http://example.com/lacie" id="link2">Lacie</a> and\n<a class="sister" href="http://example.com/tillie" id="link3">Tilli
e</a>;\nand they lived at the bottom of a well.</p>]
2.传入正则表达
soup=BeautifulSoup(html,"lxml")
for tag in soup.find_all(re.compile("b.*?")):
print(tag.name)
#body
#b
3.传入数组
soup=BeautifulSoup(html,"lxml")
#包含a,b标签
print soup.find_all(["a","b"])
#[<b>The Dormouse's story</b>, <a class="sister" href="http://example.com/elsie" id="link1"><!-- Elsie --></a>, <a class="sister
" href="http://example.com/lacie" id="link2">Lacie</a>, <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a
>]
4.传入True
#所有节点
soup=BeautifulSoup(html,"lxml")
for tag in soup.find_all(True):
print tag.name
#html
#head
#title
#body
#p
#b
#p
#a
#a
#a
5.传入方法
#含有class而没有href的tag
def has_class_but_no_id(tag):
return tag.has_attr('class') and not tag.has_attr('href')
print soup.find_all(has_class_but_no_id)
#[<p class="title" name="dromouse"><b>The Dormouse's story</b></p>, <p class="story">Once upon a time there were three little si
sters; and their names were\n<a class="sister" href="http://example.com/elsie" id="link1"><!-- Elsie --></a>,\n<a class="sister
" href="http://example.com/lacie" id="link2">Lacie</a> and\n<a class="sister" href="http://example.com/tillie" id="link3">Tilli
e</a>;\nand they lived at the bottom of a well.</p>]
6.传入字典
soup=BeautifulSoup(html,"lxml")
#id含有link2项的
print soup.find_all(id='link2')
#[<a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>]
#href中含有elsie项
print soup.find_all(href=re.compile("elsie"))
#[<a class="sister" href="http://example.com/elsie" id="link1"><!-- Elsie --></a>]
#href含有elsie项并且id为link1项
print soup.find_all(href=re.compile("elsie"),id='link1')
#[<a class="sister" href="http://example.com/elsie" id="link1"><!-- Elsie --></a>]
中文文档
