使用distutils加快构建过程

Question

我正在为Python编写一个C++扩展，并使用distutils来编译该项目。随着项目的发展，重建它需要的时间越来越长。有没有办法加快构建过程？

我读到并行构建(就像make -j一样)不可能使用distutils。有没有比distutils更快的更好的替代品？

我还注意到，每次我调用python setup.py build时，它都会重新编译所有的目标文件，即使我只更改了一个源文件。应该是这种情况，还是我做错了什么？

如果有帮助，下面是我尝试编译的一些文件：https://gist.github.com/2923577

谢谢!

Answer 1

1. 尝试使用环境变量CC="ccache gcc"进行构建，这将在源代码未更改的情况下显著加快构建速度。(奇怪的是，对于c++源文件，distutils也使用CC )。当然，安装ccache包。
2. 因为你有一个单扩展，它是由多个编译的目标文件组装而成的，你可以用-patch distutils来并行编译这些文件(它们是独立的)--把这个放到你的setup.py中(根据你的需要调整N=2 )：用于并行编译的-patch def parallelCCompile(self，sources，output_dir=None，macros=None，include_dirs=None，debug=0，extra_preargs=None，extra_postargs=None，depends=None)：#那些行是直接从distutils.ccompiler.CCompiler复制的宏，对象，extra_postargs，pp_opts，build = self._setup_compile(output_dir，macros，include_dirs，sources，depends，extra_postargs) cc_args = self._get_cc_args(pp_opts，debug，extra_preargs) # parallel code N=2 #并行编译数导入multiprocessing.pool定义_single_compile(obj)：try: src，ext = buildobj ext: KeyError: return self._compile(obj，src，ext，cc_args，extra_postargs，list(multiprocessing.pool.ThreadPool(N).imap(_single_compile，)# convert to list，imap is on-demand distutils.ccompiler.CCompiler.compile=parallelCCompile
3. For pp_opts) ) return objects导入distutils.ccompiler 为了完整起见，如果您有multiple extensions，可以使用以下解决方案：

import os import multiprocessing try: from concurrent.futures import ThreadPoolExecutor as Pool except : from multiprocessing.pool import ThreadPool as LegacyPool #以确保with语句有效。某些较早的2.7.x版本需要类池(LegacyPool)：def __enter__( self )：return self def __exit__(self，*args)：self.close() self.join() def build_extensions(self)：“给猴子打补丁的函数”“self.check_extensions_list(self.extensions) try: num_jobs = os.cpu_count() except : num_jobs = multiprocessing.cpu池(Num_jobs)作为池的_count()：pool.map(self.build_extension，self.extensions) def编译( self，sources，output_dir=None，macros=None，include_dirs=None，debug=0，extra_preargs=None，extra_postargs=None，depends=None，)：“Function to monkey-patch distutils.ccompiler.CCompiler”宏，对象，extra_postargs，pp_opts，build = self._setup_compile( output_dir，宏，include_dirs，sources，depends，extra_postargs ) cc_args = self._get_cc_args(pp_opts，debug，extra_preargs)对于对象中的obj : KeyError: src，ext = buildobj除了self._compile: continue self._compile(obj，src，ext，cc_args，extra_postargs，pp_opts) # Return *all*对象文件名，而不仅仅是我们刚才构建的文件名。从distutils.ccompiler返回对象从distutils.command.build_ext导入CCompiler导入build_ext build_ext.build_extensions = build_extensions CCompiler.compile =编译

Answer 2

我用clcache在Windows上实现了这一点，它源自eudoxos的回答：

# Python modules
import datetime
import distutils
import distutils.ccompiler
import distutils.sysconfig
import multiprocessing
import multiprocessing.pool
import os
import sys

from distutils.core import setup
from distutils.core import Extension
from distutils.errors import CompileError
from distutils.errors import DistutilsExecError

now = datetime.datetime.now

ON_LINUX = "linux" in sys.platform

N_JOBS = 4

#------------------------------------------------------------------------------
# Enable ccache to speed up builds

if ON_LINUX:
    os.environ['CC'] = 'ccache gcc'

# Windows
else:

    # Using clcache.exe, see: https://github.com/frerich/clcache

    # Insert path to clcache.exe into the path.

    prefix = os.path.dirname(os.path.abspath(__file__))
    path = os.path.join(prefix, "bin")

    print "Adding %s to the system path." % path
    os.environ['PATH'] = '%s;%s' % (path, os.environ['PATH'])

    clcache_exe = os.path.join(path, "clcache.exe")

#------------------------------------------------------------------------------
# Parallel Compile
#
# Reference:
#
# http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils
#

def linux_parallel_cpp_compile(
        self,
        sources,
        output_dir=None,
        macros=None,
        include_dirs=None,
        debug=0,
        extra_preargs=None,
        extra_postargs=None,
        depends=None):

    # Copied from distutils.ccompiler.CCompiler

    macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
        output_dir, macros, include_dirs, sources, depends, extra_postargs)

    cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)

    def _single_compile(obj):

        try:
            src, ext = build[obj]
        except KeyError:
            return

        self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)

    # convert to list, imap is evaluated on-demand

    list(multiprocessing.pool.ThreadPool(N_JOBS).imap(
        _single_compile, objects))

    return objects


def windows_parallel_cpp_compile(
        self,
        sources,
        output_dir=None,
        macros=None,
        include_dirs=None,
        debug=0,
        extra_preargs=None,
        extra_postargs=None,
        depends=None):

    # Copied from distutils.msvc9compiler.MSVCCompiler

    if not self.initialized:
        self.initialize()

    macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
        output_dir, macros, include_dirs, sources, depends, extra_postargs)

    compile_opts = extra_preargs or []
    compile_opts.append('/c')

    if debug:
        compile_opts.extend(self.compile_options_debug)
    else:
        compile_opts.extend(self.compile_options)

    def _single_compile(obj):

        try:
            src, ext = build[obj]
        except KeyError:
            return

        input_opt = "/Tp" + src
        output_opt = "/Fo" + obj
        try:
            self.spawn(
                [clcache_exe]
                + compile_opts
                + pp_opts
                + [input_opt, output_opt]
                + extra_postargs)

        except DistutilsExecError, msg:
            raise CompileError(msg)

    # convert to list, imap is evaluated on-demand

    list(multiprocessing.pool.ThreadPool(N_JOBS).imap(
        _single_compile, objects))

    return objects

#------------------------------------------------------------------------------
# Only enable parallel compile on 2.7 Python

if sys.version_info[1] == 7:

    if ON_LINUX:
        distutils.ccompiler.CCompiler.compile = linux_parallel_cpp_compile

    else:
        import distutils.msvccompiler
        import distutils.msvc9compiler

        distutils.msvccompiler.MSVCCompiler.compile = windows_parallel_cpp_compile
        distutils.msvc9compiler.MSVCCompiler.compile = windows_parallel_cpp_compile

# ... call setup() as usual

Answer 3

如果你有可用的Numpy 1.10，你可以很容易地做到这一点。只需添加：

 try:
     from numpy.distutils.ccompiler import CCompiler_compile
     import distutils.ccompiler
     distutils.ccompiler.CCompiler.compile = CCompiler_compile
 except ImportError:
     print("Numpy not found, parallel compile not available")

使用-j N或设置NPY_NUM_BUILD_JOBS。