#-*-coding:utf-8-*-
import csv
import os
#记录已存在的date.csv
   
#将words写入date.csv文件最后一行，文件打开采用'a'模式，即在原文件后添加（add）
def writeByDate(words):
    file_name ="1111.csv"
    os.chdir('../data/')
    f = open(file_name,'a',newline='')
    write = csv.writer(f)
    write.writerow(words)
    f.close()

#主函数
def splitByDate():
    f = open("../data/yu.csv")
    rows = csv.reader(f)
    for row in rows:
        for i in range(len(row)):
             row[i] = int(row[i])
             if(row[i]<10):
                  print(row[i])
        
splitByDate()

#-*-coding:utf-8-*-
"""
将tianchi_mobile_recommend_train_user.csv按照日期分割为31份**.csv文件，放在'/data/date/'目录下。
生成的**.csv文件内容格式如下：

user_id, item_id, behavior_type,user_geohash,item_category,    hour
99512554,37320317,  3,            94gn6nd,    9232,             20

"""

import csv
import os


#记录已存在的date.csv
date_dictionary = {}
   
#将words写入date.csv文件最后一行，文件打开采用'a'模式，即在原文件后添加（add）
def writeByDate(date,words):
    file_name = date+".csv"
    os.chdir('../data/date/')
    if date in date_dictionary:
        f = open(file_name,'a',newline='')
        write = csv.writer(f)
        write.writerow(words)
        f.close()
    else:
        date_dictionary[date] = True
        f = open(file_name,'a',newline='')
        write = csv.writer(f)
        write.writerow(['user_id','item_id','behavior_type','user_geohash','item_category','hour'])
        write.writerow(words)
        f.close()
    os.chdir('../../data/') #回到本代码所在的上一级文件


#主函数
def splitByDate():
    os.mkdir('../data/date/')
    f = open("../data/tianchi_fresh_comp_train_user.csv")
    rows = csv.reader(f)
    header  = next(rows )
    for row in rows:
        date = row[-1].split(" ")[0]
        hour = row[-1].split(" ")[1]
        words = row[0:-1]
        words.append(hour)
        writeByDate(date,words)
        
splitByDate()

#-*-coding:utf-8-*-
"""
对于某一天的数据按四种操作切割：浏览，收藏，加入购物车，购买
"""

import csv
import os


#记录已存在的date.csv
behavior_dictionary = {}

#将words写入date.csv文件最后一行，文件打开采用'a'模式，即在原文件后添加（add）
def writeByDate(behavior,words):
    file_name = behavior+".csv"
    os.chdir('../data/behaviour_type/')
    if not behavior in behavior_dictionary:
        behavior_dictionary[behavior] = True
        f = open(file_name,'a')
        write = csv.writer(f)
        write.writerow(['user_id','item_id','behavior_type','user_geohash','item_category','date'])
        write.writerow(words)
        f.close()
    else:
        f = open(file_name,'a')
        write = csv.writer(f)
        write.writerow(words)
        f.close()
    os.chdir('../../data/')

#主函数
def splitByDate():
    os.mkdir('../data/behaviour_type/')
    f = open("../data/2014-12-18.csv")
    rows = csv.reader(f)
    for row in rows:
        behavior = row[2]
        words = row[0:-1]
        words.append(behavior)
        writeByDate(behavior,words)

splitByDate()

文章来自于[Merge CSV Files Into One Large CSV File In Windows 7](http://www.solveyourtech.com/merge-csv-files/)。经[测试](http://lib.csdn.net/base/softwaretest)，win7以上版本的windows都可以。步骤如下：

将所有的csv文件放到一个文件夹，位置任意。
打开cmd，切换到存放csv的文件夹，也可以在csv文件夹中，按住shift加鼠标右键，选择在此处打开命令窗口。
输入**copy *.csv all-groups.csv**，all-group的名字，可以任意。然后按enter，等待完成就可以了。
打开csv文件夹就可以看到all-group.csv

excel不适合操作大量的数据，建议将csv导入到[数据库](http://lib.csdn.net/base/mysql)中去。

怎么读取超大数据的csv文件呢？
注意：csv文件默认格式是gbk而不是utf-8.

import pandas as pd

df = pd.read_csv('JData_User.csv',encoding='gbk')
print(df)

pandas读取超大数据文件

import pandas as pd

df = pd.read_csv('JData_Action_201604.csv',encoding='gbk')
df2 = pd.read_csv('JData_Product.csv',encoding='gbk')
#==============================================================================
# df = df[df.type==2]
#==============================================================================
df = df[(df.type==3)|(df.type==4)]
#==============================================================================
# df.to_csv('wantpay_4.csv',encoding='utf-8')
#==============================================================================
result1 = pd.merge(df, df2, on=['sku_id'])
print(result1)
result1.to_csv('pay_or_cancel_4.csv',encoding='utf-8')
#==============================================================================
# result1 = pd.merge(df, df2, on=['sku_id'])
# print(result1)
# result1.to_csv('wantpay_product.csv',encoding='utf-8')
#==============================================================================

进行时间格式转换

import time
import datetime

time_original = '17/9/2012 11:40:00'
day = time_original.split(' ')[0]
time_format = datetime.datetime.strptime(time_original,'%d/%m/%Y %H:%M:%S')
#这里可以 print time_format 或者 直接 time_format 一下看看输出结果，默认存储为datetime格式
time_format = time_format.strftime('%Y%m%d%H%M%S')
print(time_format)

pandas中的删除重复数据duplicates的用法

import pandas as pd
import time
import datetime

df = pd.read_csv('all.csv',encoding='gbk')
df = df.drop_duplicates(['user_id','sku_id'], keep=False)
df.to_csv('will_pay_4.csv',encoding='utf-8')
print(df)

import pandas as pd
#按日期截取csv文件中的数据
actions = pd.read_csv("2m.csv",encoding='gbk')
actions = actions[(actions.time >= '2016-03-15') & (actions.time < '2016-04-16')]
actions.to_csv('between_days.csv',encoding='utf-8')

将list写入txt:

fl=open('list.txt', 'w')
    for i in lists:
         fl.write(i)
         fl.write("\n")
    fl.close()

df = df.groupby(['user_id'], as_index=False).sum()#获取所有对应的['area']的总数
df =df.groupby(['area'], as_index=False).size()#获取每个['area']的个数
df =df.groupby(['area'], as_index=False).mean()#获取每个['area']的均值

import pandas as pd

df = pd.read_csv('submission.csv')
df['sort_id'] = df['lab'].groupby(df['user_id']).rank()
print(df)

求日期的前一天

from datetime import timedelta, datetime
yesterday = (datetime.strptime(str(end_date), '%Y-%m-%d') - timedelta(days=1)).strftime('%Y-%m-%d')
print(yesterday)
#相隔的日期
a = (datetime.strptime(str('2016-04-06'), '%Y-%m-%d') - datetime.strptime(str('2016-04-01'), '%Y-%m-%d')).days

将任意格式的时间转化为指定日期

results = pd.read_csv("nba_data.csv", parse_dates=["Date"])

将以毫秒计数的时间转换为正常YMDHMS格式

all_ratings["Datetime"] = pd.to_datetime(all_ratings['Datetime'],unit='s')

简单小tips

print("There are {} movies with more than {} favorable reviews".format(3,4))
"""
There are 3 movies with more than 4 favorable reviews
"""

#除去‘user’在0-2之间的数据
df=  df[~df['user'].isin(range(3))]  

#显示出常见的参数
adult["Hours-per-week"].describe()
"""
count    19115.000000
mean        40.393408
std         12.253508
min          1.000000
25%         40.000000
50%         40.000000
75%         45.000000
max         99.000000
Name: Hours-per-week, dtype: float64
"""

怎么求pandas中的间隔日期呢？

def get_gap_days():
    ActData = pd.read_csv('../data/train.csv')
    ActData['time'] = pd.to_datetime(ActData['clickTime'],format='%d%H%M')
    time0 = ['1899-12-31 00:00:00']
    time0 = pd.to_datetime(pd.Series(time0))
    ActData['days'] = (ActData['time'] - time0.iat[0]).dropna()
    ActData['days'] = np.int32(ActData['days'] / np.timedelta64(24, 'h'))
    print(ActData)