服务器之家

服务器之家 > 正文

python文件排序的方法总结

时间:2020-09-13 23:56     来源/作者:爱喝马黛茶的安东尼

在python环境中提供两种排序方案:用库函数sorted()对字符串排序,它的对象是字符;用函数sort()对数字排序,它的对象是数字,如果读取文件的话,需要进行处理(把文件后缀名‘屏蔽')。

(1)首先:我测试的文件夹是/img/,里面的文件都是图片,如下图所示:

python文件排序的方法总结

(2)测试库函数sorted(),直接贴出代码:

?
1
2
3
4
5
6
7
8
9
10
import numpy as np
import os
 
img_path='./img/'
 
img_list=sorted(os.listdir(img_path))#文件名按字母排序
img_nums=len(img_list)
for in range(img_nums):
    img_name=img_path+img_list[i]
    print(img_name)

运行效果如下:

python文件排序的方法总结

从图片可以清晰的看出,文件名是按字符排序的。

(3)测试函数sort(),代码:

?
1
2
3
4
5
6
7
8
9
10
11
import numpy as np
import os
img_path='./img/'
 
img_list=os.listdir(img_path)
img_list.sort()
img_list.sort(key = lambda x: int(x[:-4])) ##文件名按数字排序
img_nums=len(img_list)
for in range(img_nums):
    img_name=img_path+img_list[i]
    print(img_name)

运行效果如下:

python文件排序的方法总结

可以看出,文件名是按数字排序的;顺便提下,sort函数中用到了匿名函数(key = lambda x:int(x[:-4])),其作用是将后缀名'.jpg'“屏蔽”(因为‘.jpg'是4个字符,所以[:-4]的含义是从文件名开始到倒数第四个字符为止),具体看python的匿名函数和数组取值方式。

实例扩展:

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
import gzip
import os
from multiprocessing import Process, Queue, Pipe, current_process, freeze_support
from datetime import datetime
def sort_worker(input,output):
 while True:
 lines = input.get().splitlines()
 element_set = {}
 for line in lines:
  if line.strip() == 'STOP':
   return
  try:
   element = line.split(' ')[0]
   if not element_set.get(element): element_set[element] = ''
  except:
   pass
 sorted_element = sorted(element_set)
 #print sorted_element
 output.put('\n'.join(sorted_element))
def write_worker(input, pre):
 os.system('mkdir %s'%pre)
 i = 0
 while True:
  content = input.get()
  if content.strip() == 'STOP':
   return
  write_sorted_bulk(content, '%s/%s'%(pre, i))
  i += 1
def write_sorted_bulk(content, filename):
 f = file(filename, 'w')
 f.write(content)
 f.close()
def split_sort_file(filename, num_sort = 3, buf_size = 65536*64*4):
 t = datetime.now()
 pre, ext = os.path.splitext(filename)
 if ext == '.gz':
  file_file = gzip.open(filename, 'rb')
 else:
  file_file = open(filename)
 bulk_queue = Queue(10)
 sorted_queue = Queue(10)
 NUM_SORT = num_sort
 sort_worker_pool = []
 for i in range(NUM_SORT):
  sort_worker_pool.append( Process(target=sort_worker, args=(bulk_queue, sorted_queue)) )
  sort_worker_pool[i].start()
 NUM_WRITE = 1
 write_worker_pool = []
 for i in range(NUM_WRITE):
  write_worker_pool.append( Process(target=write_worker, args=(sorted_queue, pre)) )
  write_worker_pool[i].start()
 buf = file_file.read(buf_size)
 sorted_count = 0
 while len(buf):
  end_line = buf.rfind('\n')
  #print buf[:end_line+1]
  bulk_queue.put(buf[:end_line+1])
  sorted_count += 1
  if end_line != -1:
   buf = buf[end_line+1:] + file_file.read(buf_size)
  else:
   buf = file_file.read(buf_size)
 for i in range(NUM_SORT):
  bulk_queue.put('STOP')
 for i in range(NUM_SORT):
  sort_worker_pool[i].join()
  
 for i in range(NUM_WRITE):
  sorted_queue.put('STOP')
 for i in range(NUM_WRITE):
  write_worker_pool[i].join()
 print 'elasped ', datetime.now() - t
 return sorted_count
from heapq import heappush, heappop
from datetime import datetime
from multiprocessing import Process, Queue, Pipe, current_process, freeze_support
import os
class file_heap:
 def __init__(self, dir, idx = 0, count = 1):
  files = os.listdir(dir)
  self.heap = []
  self.files = {}
  self.bulks = {}
  self.pre_element = None
  for i in range(len(files)):
   file = files[i]
   if hash(file) % count != idx: continue
   input = open(os.path.join(dir, file))
   self.files[i] = input
   self.bulks[i] = ''
   heappush(self.heap, (self.get_next_element_buffered(i), i))
 def get_next_element_buffered(self, i):
  if len(self.bulks[i]) < 256:
   if self.files[i] is not None:
    buf = self.files[i].read(65536)
    if buf:
     self.bulks[i] += buf
    else:
     self.files[i].close()
     self.files[i] = None
  end_line = self.bulks[i].find('\n')
  if end_line == -1:
   end_line = len(self.bulks[i])
  element = self.bulks[i][:end_line]
  self.bulks[i] = self.bulks[i][end_line+1:]
  return element
 def poppush_uniq(self):
  while True:
   element = self.poppush()
   if element is None:
    return None
   if element != self.pre_element:
    self.pre_element = element
    return element
 def poppush(self):
  try:
   element, index = heappop(self.heap)
  except IndexError:
   return None
  new_element = self.get_next_element_buffered(index)
  if new_element:
   heappush(self.heap, (new_element, index))
  return element
def heappoppush(dir, queue, idx = 0, count = 1):
 heap = file_heap(dir, idx, count)
 while True:
  d = heap.poppush_uniq()
  queue.put(d)
  if d is None: return
def heappoppush2(dir, queue, count = 1):
 heap = []
 procs = []
 queues = []
 pre_element = None
 for i in range(count):
  q = Queue(1024)
  q_buf = queue_buffer(q)
  queues.append(q_buf)
  p = Process(target=heappoppush, args=(dir, q_buf, i, count))
  procs.append(p)
  p.start()
 queues = tuple(queues)
 for i in range(count):
  heappush(heap, (queues[i].get(), i))
 while True:
  try:
   d, i= heappop(heap)
  except IndexError:
   queue.put(None)
   for p in procs:
    p.join()
   return
  else:
   if d is not None:
    heappush(heap,(queues[i].get(), i))
    if d != pre_element:
     pre_element = d
     queue.put(d)
def merge_file(dir):
 heap = file_heap( dir )
 os.system('rm -f '+dir+'.merge')
 fmerge = open(dir+'.merge', 'a')
 element = heap.poppush_uniq()
 fmerge.write(element+'\n')
 while element is not None:
  element = heap.poppush_uniq()
  fmerge.write(element+'\n')
class queue_buffer:
 def __init__(self, queue):
  self.q = queue
  self.rbuf = []
  self.wbuf = []
 def get(self):
  if len(self.rbuf) == 0:
   self.rbuf = self.q.get()
  r = self.rbuf[0]
  del self.rbuf[0]
  return r
 def put(self, d):
  self.wbuf.append(d)
  if d is None or len(self.wbuf) > 1024:
   self.q.put(self.wbuf)
   self.wbuf = []
def diff_file(file_old, file_new, file_diff, buf = 268435456):
 print 'buffer size', buf
 from file_split import split_sort_file
 os.system('rm -rf '+ os.path.splitext(file_old)[0] )
 os.system('rm -rf '+ os.path.splitext(file_new)[0] )
 t = datetime.now()
 split_sort_file(file_old,5,buf)
 split_sort_file(file_new,5,buf)
 print 'split elasped ', datetime.now() - t
 os.system('cat %s/* | wc -l'%os.path.splitext(file_old)[0])
 os.system('cat %s/* | wc -l'%os.path.splitext(file_new)[0])
 os.system('rm -f '+file_diff)
 t = datetime.now()
 zdiff = open(file_diff, 'a')
 old_q = Queue(1024)
 new_q = Queue(1024)
 old_queue = queue_buffer(old_q)
 new_queue = queue_buffer(new_q)
 h1 = Process(target=heappoppush2, args=(os.path.splitext(file_old)[0], old_queue, 3))
 h2 = Process(target=heappoppush2, args=(os.path.splitext(file_new)[0], new_queue, 3))
 h1.start(), h2.start()
 old = old_queue.get()
 new = new_queue.get()
 old_count, new_count = 0, 0
 while old is not None or new is not None:
  if old > new or old is None:
   zdiff.write('< '+new+'\n')
   new = new_queue.get()
   new_count +=1
  elif old < new or new is None:
   zdiff.write('> '+old+'\n')
   old = old_queue.get()
   old_count +=1
  else:
   old = old_queue.get()
   new = new_queue.get()
 print 'new_count:', new_count
 print 'old_count:', old_count
 print 'diff elasped ', datetime.now() - t
 h1.join(), h2.join()

到此这篇关于python文件排序的方法总结的文章就介绍到这了,更多相关python文件排序都有哪些方法内容请搜索服务器之家以前的文章或继续浏览下面的相关文章希望大家以后多多支持服务器之家!

原文链接:https://www.py.cn/jishu/jichu/14723.html

标签:

相关文章

热门资讯

2020微信伤感网名听哭了 让对方看到心疼的伤感网名大全
2020微信伤感网名听哭了 让对方看到心疼的伤感网名大全 2019-12-26
Intellij idea2020永久破解,亲测可用!!!
Intellij idea2020永久破解,亲测可用!!! 2020-07-29
歪歪漫画vip账号共享2020_yy漫画免费账号密码共享
歪歪漫画vip账号共享2020_yy漫画免费账号密码共享 2020-04-07
电视剧《琉璃》全集在线观看 琉璃美人煞1-59集免费观看地址
电视剧《琉璃》全集在线观看 琉璃美人煞1-59集免费观看地址 2020-08-12
最新idea2020注册码永久激活(激活到2100年)
最新idea2020注册码永久激活(激活到2100年) 2020-07-29
返回顶部