如何提高文件读取速度？

Question

我从一个大文件中读取数据切片。一张400的图片花了4秒。磁盘读取此文件只需1秒。这个程序计算量很小。如何提高速度？

from opentile import OpenTile
import time
import traceback
import os
os.environ.setdefault('TURBOJPEG', 'C:/lib/')
try:
    tiler = OpenTile.open('svs800.svs')
except:
    traceback.print_exc()

s=tiler.get_level(0)
tile_size=str(s.tiled_size).split("x")

time1=time.time()

from multiprocessing.pool import ThreadPool

def get_data(s):
    # This function reads a piece of binary data from a certain position of the image 
    # and then adds the header data
    return tiler.get_tile(0,0,0, (s[0], s[1]))
pool = ThreadPool(5)
y = pool.map(get_data, [(i,j) for i in range(int(tile_size[0])) for j in range(int(tile_size[1]))])

print("tiles",len(y))
time2=time.time()
print(time2)

Answer 1

简单的顺序方法：

from opentile import OpenTile
import os
import time

def timer(func):
    def wrapper(*args, **kwargs):
        start = time.perf_counter()
        result = func(*args, **kwargs)
        end = time.perf_counter()
        print(f'{func.__name__} {end-start:.4f}s')
        return result
    return wrapper

os.environ.setdefault('TURBOJPEG', '/opt/libjpeg-turbo')

@timer
def open_svs(filename):
    return OpenTile.open(filename)

@timer
def get_data(tiler, x, y):
    return [tiler.get_tile(0, 0, 0, (x_, y_)) for x_ in range(x) for y_ in range(y)]

tiler = open_svs('18959.svs')

x, y = map(int, str(tiler.get_level(0).tiled_size).split('x'))

data = get_data(tiler, x, y)

assert len(data) == x * y

输出：

open_svs 0.0082s
get_data 0.5843s

注：

x, y values for this file are 183, 114 respectively. The file size is 563,271,749 bytes