获取网站相关信息
import dns.resolver
import urllib2
resp = urllib2.urlopen('http://www.www.com')
print type(resp)
print resp.headers
print resp.headers['Server']
print resp.getcode()
print resp.geturl()
获取title
import mongo
import header
import pymongo
import random
import requests
import urllib2
from bs4 import BeautifulSoup
#----------------------------------------------------------------------
def url_info(url):
""""""
data = urllib2.Request(url,headers=header.get_header())
html_url = requests.get(url, timeout=random.randint(5,10))
#print type(data.headers)
soup = BeautifulSoup(html_url.content, 'html.parser')
try:
print soup.title.string
encoding_url = html_url.encoding
title_url = soup.title.string
mongo.ls_Info.update({"URL":url},
{"$set":{'title':title_url,'encoding':encoding_url}},
upsert = True)
except Exception,e:
print str(e)
pass
