- 背景：刚好公司有一个刷数据的任务，就上手了python，早该开始了。
- 小结知识点如下：
- 1 python的默认编码是unicode，如果在程序中输入中文就会有乱码，如果文件中中文是utf-8编码的，就可以用以下这种方式解码。
import os
import codecs
path = os.getcwd();
print(path);
os.chdir('/Users/liuchaoqun01/Desktop')
print(os.getcwd());
with codecs.open('to see you', encoding='utf-8') as f:
print(f.read())
- 2 cc助攻的解码问题：文本文件是latin1编码的中文，120w行。甚至蛋疼，一直不知道应该怎么转为utf-8。经cc点化：
with open('d:/res.txt','wb') as w:
with open(r'd:/entity.list.all.txt','rb') as f:
for x in f:
try:
w.write(x.decode('gb2312').encode('utf8'))
except Exception:
w.write(b'\n')
还是一知半解，等待cc指正！
- 3 这是最后成形的代码，本质上就是一个数据脚本，总共跑了1hour，120w数据，其实最后发现120w的数据查询太耗时了，一开始就应该用批量查询！！
import sys,re,os
import datetime
import MySQLdb
begin = datetime.datetime.now()
reload(sys)
sys.setdefaultencoding("utf-8")
path = os.getcwd();
os.chdir('/Users/liuchaoqun01/Documents/cloud/input')
predictFile = open('predict.trade2.entity.list.all', 'r')
count = 0
businessList = []
emptyList = []
predictFileList = []
resultList = []
emptyList = []
with open('res.txt','r') as w:
for business in w:
line = business.replace('\n', '')
businessList.append(line)
for predict in predictFile:
p = re.compile(r'\s|__')
ss = p.split(predict)
if len(ss) < 3:
count = count + 1
line = '-1'
else:
line = "select industryname from aodfeed.industry where industryid = " + (ss[2])
predictFileList.append(line)
line = ''
predictFile.close()
print 'The count of blank line is:' + str(count)
# mysql
outputResult = open('/Users/liuchaoqun01/Documents/cloud/output/result', 'w+')
outputEmpty = open('/Users/liuchaoqun01/Documents/cloud/output/empty', 'w+')
conn = MySQLdb.connect(host='10.99.196.227', port=8306, user='test', passwd='xxx', db='xxx', charset='utf8')
cursor = conn.cursor()
for i in range(len(predictFileList)):
predict = predictFileList[i]
business = businessList[i]
if predict == '-1':
emptyList.append(business + '\t' + '-1')
else:
sql = predict
cursor.execute(sql)
r = cursor.fetchall()
# print (business + '\t' + r[0][0].decode('utf-8'))
resultList.append(business + '\t' + r[0][0])
# write
for out in resultList:
outputResult.write(out + '\n')
outputResult.close()
for out in emptyList:
outputEmpty.write(out + '\n')
outputEmpty.close()
print 'process end!'
end = datetime.datetime.now()
print (end - begin)
