如下所示：
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
|
import pandas as pdfrom numpy import *import matplotlib.pylab as pltimport copydef read(filename): dat=pd.read_csv(filename,iterator=True) loop = True chunkSize = 1000000 R=[] while loop: try: data = dat.get_chunk(chunkSize) data=data.loc[:,'B':'C'] # 切片 data=data[data.B==855] #条件选择 data=data.set_index(['C']) # 设置索引 data.loc[:,'D']=array([1]*len(data)) #增加一列 data=data.resample('D').sum() #按天求和 data=data.loc[:,'D'] #截取 data.fillna(0) #填充缺失值 R.append(data) except StopIteration: loop = False print ("Iteration is stopped.") R.to_csv('855_pay.csv') # 保存def read2(filename): reader=pd.read_csv(filename,iterator=True) loop = True chunkSize = 100000 chunks = [] while loop: try: chunk = reader.get_chunk(chunkSize) chunks.append(chunk) except StopIteration: loop = False print ("Iteration is stopped.") df = pd.concat(chunks, ignore_index=True) return dfdef read3save(filename): dat=pd.read_csv(filename) #data = dat.get_chunk(chunkSize) data=dat.loc[:,'B':'C'] # 切片 data=data[data.B==855]#条件选择 print(shape(data)) data['C']=pd.to_datetime(data['C']) # 转换成时间格式 data=data.set_index(['C'])# 设置索引 if len(data)==0: return data.loc[:,'D']=array([1]*len(data)) #增加一列 data=data.resample('D').sum() #按天求和 data=data.loc[:,'D'] #截取 data.fillna(0) #填充缺失值 data.to_csv('855_pay.csv',mode='a') # 保存def loadDataSet(fileName, delim='\t'): fr = open(fileName) stringArr = [line.strip().split(delim) for line in fr.readlines()] datArr = [list(map(float,line)) for line in stringArr] return mat(datArr)def getShopData(): fr = open('shopInfo.txt') shopID = [line.strip().split('\n') for line in fr.readlines()] # datArr = [list(map(float,line))for line in stringArr] for i in range(1,9): name="user_pay.001.00%d"%i dat=pd.read_csv(name) #data = dat.get_chunk(chunkSize) data=dat.loc[:,'B':'C'] # 切片 for factor in shopID: data=data[data.B==int(str(factor[0]))]#条件选择 print(shape(data)) if len(data)==0: continue data['C']=pd.to_datetime(data['C']) # 转换成时间格式 data=data.set_index(['C'])# 设置索引 data.loc[:,'D']=array([1]*len(data)) #增加一列 data=data.resample('D').sum() #按天求和 data=data.loc[:,'D'] #截取 data.fillna(0) #填充缺失值 s=str(factor[0]) data.to_csv(savename,mode='a') # 保存 del dat print("over")def tset(filename): dat=pd.read_csv(filename) #data = dat.get_chunk(chunkSize) data=dat.loc[:,'B':'C'] # 切片 data=data[data.B==855]#条件选择 print(shape(data)) data['C']=pd.to_datetime(data['C']) # 转换成时间格式 data=data.set_index(['C'])# 设置索引 if len(data)==0: return data.loc[:,'D']=array([1]*len(data)) #增加一列 data=data.resample('D').sum() #按天求和 data=data.loc[:,'D'] #截取 data.fillna(0) #填充缺失值 #data.to_csv('855_pay.csv',mode='a') # 保存 s='my' savename='D:\python\data\%s_pay.csv'%s data.to_csv(savename,mode='a') # 保存 def getShopData2(filename): import csv # fr = open('shopInfo.txt') # shopID = [line.strip().split('\n') for line in fr.readlines()] # datArr = [list(map(float,line))for line in stringArr] #for i in range(1,9): #name="user_pay.001.00%d"%i dat=pd.read_csv(filename) #data = dat.get_chunk(chunkSize) data=dat.loc[:,'B':'C'] # 切片 data['C']=pd.to_datetime(data['C']) # 转换成时间格式 data=data.set_index(['C'])# 设置索引 data.loc[:,'D']=array([1]*len(data)) #增加一列 for i in range(1,2001): d=copy.copy(data) d=d[data.B==i]#条件选择 #print(shape(d)) print(i) if len(d)==0: continue d=d.resample('D').sum() #按天求和 d=d.loc[:,'D'] #截取 d.fillna(0) #填充缺失值 s=str(i) #print(s) savename='D:\python\data2\%s_pay.csv'%s c=open(savename,'a') writer=csv.writer(c) writer.writerow(['C','D']) c.close() d.to_csv(savename,mode='a') # 保存 # del dat print("over")def formatData(): #fr = open('shopInfo.txt') #shopID = [line.strip().split('\n') for line in fr.readlines()] # datArr = [list(map(float,line))for line in stringArr] #data = dat.get_chunk(chunkSize) for i in range(1,2001): s=str(i) print(s) name='D:\python\data2\%s_pay.csv'%s dat=pd.read_csv(name) data['C']=pd.to_datetime(data['C']) # 转换成时间格式 data=data.set_index(['C'])# 设置索引 data=data.resample('D').sum() #按天求和 data.fillna(0) #填充缺失值 savename='D:\python\data3\%s_pay.csv'%s data.to_csv(savename,mode='w') # 保存 del dat print("over") |
以上这篇python pandas 对时间序列文件处理的实例就是小编分享给大家的全部内容了，希望能给大家一个参考，也希望大家多多支持服务器之家。
原文链接：https://blog.csdn.net/Faith_yu/article/details/56009125
