我有一个包含传感器时间序列的二进制文件。数据格式如下:
#1(t0) #2(t0) #3(t0) ... #n(t0) #1(t1) #2(t1) #3(t1) ... #n(t1) ...
同时,来自n个传感器的测量数据以二进制格式存储在文件中。我想重建传感器的时序
#1(t0) #1(t1) #1(t2) ...
从#1(t0)到#1(t1)的距离是固定和已知的,传感器的数量也是已知的。下面的代码是我的实现。我的实现是尝试一次获取一个数据,而且速度不是那么快。有没有办法提高读取MPI中的集合io中的非连续数据的速度?
def collect_signal(fp, channel_no, stride, dtype):
byteSize = np.dtype(dtype).itemsize
fp.seek(0,2) # go to the file end
eof = fp.tell() # get the eof address
fp.seek(0,0) # rewind
fp.seek(0 + channel_no,0) # starting point per each channel
signal = []
while True:
start = fp.tell()
sample = np.frombuffer(fp.read(byteSize), dtype=dtype)
signal.append(sample[0])
if fp.tell() == eof or fp.tell() + stride > eof:
break;
else:
fp.seek(start + stride, 0)
return signal发布于 2018-07-26 21:47:13
这个更简单的代码可能会更快。您可能还希望考虑使用mmap将文件映射到进程的地址空间,这使您可以绕过一层内核I/O调用。
def collect_signal(fp, channel_no, stride, dtype):
byte_size = np.dtype(dtype).itemsize
fp.seek(channel_no, 0)
# Assuming that your read will always return an entire sample
# or an empty string.
for sample in iter(lambda: fp.read(byte_size), ''):
sample = np.frombuffer(sample, dtype=dtype)
signal.append(sample[0])
fp.seek(stride, 1)另一种选择可能是让frombuffer为您处理步幅,如果您知道有多少个通道。这涉及到在每一步将稍微多一点的数据读入内存,但是如果输入是缓冲的,那么您可能已经将比fp.read实际返回的数据更多的数据读入了缓冲区。
def collect_signal(fp, channel_no, stride, type):
byte_size = np.dtype(dtype).itemsize)
offset = channel_no * byte_size
while True:
sample = fp.read(byte_size * numChannels)
if not sample:
break
sample = np.frombuffer(sample, dtype=dtype, count=1, offset=offset)
signal.append(sample[0])发布于 2018-07-27 01:14:52
按照切普纳的建议,我做了一些测试。信号生成由以下代码完成:
#include <fstream>
#include <iostream>
#include <cmath>
int main(int argc, char** argv) {
std::ofstream of("signal.bin");
double omega = 10;
double delt = 0.1;
double phi = 1;
int number_of_channel = 4;
int nsize = 100000000;
for (auto i=0; i<nsize; ++i) {
for (auto j=0; j<number_of_channel; ++j) {
double data = sin(omega*delt*i + phi*j);
of.write(reinterpret_cast<char*>(&data), sizeof(double));
}
}
of.close();
return 0;
}信号生成了一个3 3GB的二进制文件。然后我编写了python代码,以非连续的方式读取数据。
import numpy as np
import time
import os
def collect_signal_rev0(fp, channel_no, stride, dtype):
byteSize = np.dtype(dtype).itemsize
fp.seek(0,2) # go to the file end
eof = fp.tell() # get the eof address
fp.seek(0,0) # rewind
fp.seek(0 + channel_no,0) # starting point per each channel
signal = []
while True:
start = fp.tell()
sample = np.frombuffer(fp.read(byteSize), dtype=dtype)
signal.append(sample[0])
if fp.tell() == eof or fp.tell() + stride > eof:
break;
else:
fp.seek(start + stride, 0)
return signal
def collect_signal_rev1(fp, channel_no, stride, dtype):
byte_size = np.dtype(dtype).itemsize
fp.seek(channel_no, 0)
signal = []
sentinel = bytes()
for sample in iter(lambda: fp.read(byte_size), sentinel):
sample = np.frombuffer(sample, dtype=dtype)
signal.append(sample[0])
# since the file pointer is pointing to the next item after reading
# it should stride with subtracted by 1 byte_size
fp.seek(stride-byte_size, 1)
return signal
def collect_signal_rev2(fp, num_channels, channel_no, stride, dtype):
byte_size = np.dtype(dtype).itemsize
offset = channel_no * byte_size
signal = []
while True:
sample = fp.read(byte_size * num_channels)
if not sample:
break
sample = np.frombuffer(sample, dtype=dtype, count=1, offset=offset)
signal.append(sample[0])
return signal
def collect_signal_rev3(fp, channel_no, stride, dtype):
return fp[channel_no::stride]
if __name__ == '__main__':
channels = 4
byte_size = np.dtype(np.float).itemsize
stride = channels*byte_size
file_size = os.path.getsize('signal.bin')
data_count = file_size//byte_size//channels
print('file size: {0} bytes'.format(file_size))
print('# of data count per channel: {0}'.format(data_count))
start = time.clock()
with open('signal.bin', 'rb') as fp:
signal0 = collect_signal_rev0(fp, 0, stride, np.float)
print('rev0 elapsed time = {0:.3f}sec'.format(time.clock() - start))
start = time.clock()
with open('signal.bin', 'rb') as fp:
signal1 = collect_signal_rev1(fp, 0, stride, np.float)
print('rev1 elapsed time = {0:.3f}sec'.format(time.clock() - start))
start = time.clock()
with open('signal.bin', 'rb') as fp:
signal2 = collect_signal_rev2(fp, channels, 0, stride, np.float)
print('rev2 elapsed time = {0:.3f}sec'.format(time.clock() - start))
start = time.clock()
fp = np.memmap('signal.bin', dtype=np.float, mode='r', shape=(data_count*channels,1))
signal3 = collect_signal_rev3(fp, 0, channels, np.float)
print('rev3 elapsed time = {0:.3f}sec'.format(time.clock() - start))上面的代码显示了以下结果:
file size: 3200000000 bytes
# of data count per channel: 100000000
rev0 elapsed time = 266.535sec
rev1 elapsed time = 116.474sec
rev2 elapsed time = 109.224sec
rev3 elapsed time = 0.001sec所以我发现numpy.mmap版本比其他3个函数要快得多。再次表示感谢。
https://stackoverflow.com/questions/51531224
复制相似问题