抓取页面编码格式

import requests
def monkey_patch():
    prop = requests.models.Response.content
    def content(self):
        _content = prop.fget(self)
        if self.encoding == 'ISO-8859-1':
            encodings = requests.utils.get_encodings_from_content(_content)
            if encodings:
                self.encoding = encodings[0]
            else:
                self.encoding = self.apparent_encoding
            _content = _content.decode(self.encoding, 'replace').encode('utf8', 'replace')
            self._content = _content
        return _content
    requests.models.Response.content = property(content)
monkey_patch()

批量处理url格式

import urlparse
f = open('new_url.txt','w+')
with open('b.txt') as b:
    for i in b.readlines():
        i = i.strip('\r').strip('\n')
        url = urlparse.urlparse(i)
        url = url.scheme + "://" + url.hostname
        print url
        f.write(url)
        f.write('\n')

查找是否存在某个字符串中

import urllib2
#----------------------------------------------------------------------
def waf_url(url):
    """"""
    
resp = urllib2.urlopen('http://www.xxxxx.cn')

print str(resp.headers)
print resp.headers['Server']

if str(resp.headers).upper().find('date'.upper()) == -1:
    print 'NO'
else:
    print 'yes'

wwwww = 'http://www.22222.com'
if "2" in wwwww:
print('111111')