用PortAudio使flite输出音频

Question

我正在努力让flite语音合成库在我的Mac上工作，但是在flite库中不支持我的声音架构。为了解决这个问题，我使用PortAudio来播放合成的音频；因此，我不得不在audio.c文件中做一些黑客操作，才能让flite使用这个库。在GNU AutoTools中混了一段时间之后，我设法将所有的编译工作都做得很好，但随后我运行了这个程序并获得了这个输出：

$ ./flite -t "test"
frameIndex: 0
maxFrameIndex: 0
numChannels: 1
numSamples: 7225
sampleRate: 8000
=== Now playing back. ===
Waiting for playback to finish.
frameIndex in callback: -2008986336
maxFrameIndex in callback: 32655
numChannels in callback: 152579008
numSamples in callback: 0
sampleRate in callback: 0
Segmentation fault: 11  

$ ./flite -t "test"
frameIndex: 0
maxFrameIndex: 0
numChannels: 1
numSamples: 7225
sampleRate: 8000
=== Now playing back. ===
Waiting for playback to finish.
frameIndex in callback: -71217888
maxFrameIndex in callback: 32712
numChannels in callback: 232979392
numSamples in callback: 0
sampleRate in callback: 0
Segmentation fault: 11

下面是来自audio.c文件的相关代码，当我提供命令行参数-t时会调用它。经过一些调试，我标记了playCallback()函数中发生分段错误的感兴趣区域。

static int playCallback( const void *inputBuffer, void *outputBuffer,
                        unsigned long framesPerBuffer,
                        const PaStreamCallbackTimeInfo* timeInfo,
                        PaStreamCallbackFlags statusFlags,
                        void *userData )
{
    cst_wave *data = (cst_wave*)userData;
    short *rptr = &data->samples[data->frameIndex * data->num_channels];
    short *wptr = (short*)outputBuffer;
    unsigned int i;
    int finished;
    unsigned int framesLeft = cst_wave_maxFrameIndex(data) - cst_wave_frameIndex(data);

    (void) inputBuffer; /* Prevent unused variable warnings. */
    (void) timeInfo;
    (void) statusFlags;
    (void) userData;

    printf("frameIndex in callback: %d\n", cst_wave_frameIndex(data));
    printf("maxFrameIndex in callback: %d\n", cst_wave_maxFrameIndex(data));
    printf("numChannels in callback: %d\n", cst_wave_num_channels(data));
    printf("numSamples in callback: %d\n", cst_wave_num_samples(data));
    printf("sampleRate in callback: %d\n\n", cst_wave_sample_rate(data));

    if( framesLeft < framesPerBuffer )
    {
        /* final buffer... */
        for( i=0; i<framesLeft; i++ )
        {
            *wptr++ = *rptr++;  /* left */
            if( cst_wave_num_channels(data) == 2 ) *wptr++ = *rptr++;  /* right */
        }
        for( ; i<framesPerBuffer; i++ )
        {
            *wptr++ = 0;  /* left */
            if( cst_wave_num_channels(data) == 2) *wptr++ = 0;  /* right */
        }
        data->frameIndex += framesLeft;
        finished = paComplete;
    }
    else
    {
        for( i=0; i<framesPerBuffer; i++ )
        {
            *wptr++ = *rptr++;  /* left */
            if( cst_wave_num_channels(data) == 2 ) *wptr++ = *rptr++;  /* right */
        }
        cst_wave_set_frameIndex(data, framesPerBuffer);
        finished = paContinue;
    }
    return finished;
}

int play_wave(cst_wave *w)
{
    PaStream* stream;
    PaStreamParameters outputParameters;
    cst_wave_set_frameIndex(w, 0);
    cst_wave_set_maxFrameIndex(w, (cst_wave_num_samples(w) / cst_wave_sample_rate(w)) * cst_wave_num_channels(w) * sizeof(short));
    int err = 0;
    err = Pa_Initialize();
    outputParameters.device = Pa_GetDefaultOutputDevice();
    if (outputParameters.device == paNoDevice)
    {
        fprintf(stderr,"Error: No default output device.\n");
        return -5;
    }
    printf("frameIndex: %d\n", cst_wave_frameIndex(w));
    printf("maxFrameIndex: %d\n", cst_wave_maxFrameIndex(w));
    printf("numChannels: %d\n", cst_wave_num_channels(w));
    printf("numSamples: %d\n", cst_wave_num_samples(w));
    printf("sampleRate: %d\n", cst_wave_sample_rate(w));

    outputParameters.channelCount = cst_wave_num_channels(w);
    outputParameters.sampleFormat = paInt16;
    outputParameters.suggestedLatency = Pa_GetDeviceInfo( outputParameters.device )->defaultLowOutputLatency;
    outputParameters.hostApiSpecificStreamInfo = NULL;
    puts("=== Now playing back. ===");
    err = Pa_OpenStream(&stream,
                        NULL, /* no input */
                        &outputParameters,
                        cst_wave_sample_rate(w),
                        512,
                        paClipOff,
                        playCallback,
                        &w);
    if( stream )
    {
        err = Pa_StartStream( stream );
        if( err != paNoError ) goto done;

        puts("Waiting for playback to finish.");

        while((err = Pa_IsStreamActive(stream)) == 1) Pa_Sleep(100);
        if( err < 0 ) goto done;

        err = Pa_CloseStream( stream );
        if( err != paNoError ) goto done;

        puts("Done.");
    }
done:
    Pa_Terminate();
    free(cst_wave_samples(w));
}

由于它是相关的，我还稍微修改了cst_wave.h中的cst_wave.h结构，使其包含了我必须存储的数据，并在已经存在的数据中添加了一些#defines：

typedef struct  cst_wave_struct {
    const char *type;
    int frameIndex;
    int maxFrameIndex;
    int sample_rate;
    int num_samples;
    int num_channels;
    short *samples;
} cst_wave;

#define cst_wave_num_samples(w) (w?w->num_samples:0)
#define cst_wave_num_channels(w) (w?w->num_channels:0)
#define cst_wave_sample_rate(w) (w?w->sample_rate:0)
#define cst_wave_samples(w) (w->samples)
#define cst_wave_frameIndex(w) (w->frameIndex)
#define cst_wave_maxFrameIndex(w) (w->maxFrameIndex)

#define cst_wave_set_num_samples(w,s) w->num_samples=s
#define cst_wave_set_num_channels(w,s) w->num_channels=s
#define cst_wave_set_sample_rate(w,s) w->sample_rate=s
#define cst_wave_set_frameIndex(w,s) w->frameIndex=s
#define cst_wave_set_maxFrameIndex(w,s) w->maxFrameIndex=s

更新1

按照@Rohan的建议，现在给我这个输出：

$ ./bin/flite -t "test"
frameIndex: 0
maxFrameIndex: 0
numChannels: 1
numSamples: 7225
sampleRate: 8000
=== Now playing back. ===
Waiting for playback to finish.
frameIndex in callback: 0
maxFrameIndex in callback: 0
numChannels in callback: 1
numSamples in callback: 7225
sampleRate in callback: 8000

Done.
flite(68929,0x7fff71c0d310) malloc: *** error for object 0x7fd6e2809800: pointer being freed was not allocated
*** set a breakpoint in malloc_error_break to debug
Abort trap: 6

为了解决这个问题，我删除了free(cst_wave_samples(w));。现在程序正常执行，没有明显的错误，但我的Mac上仍然没有音频输出。有什么建议吗？

Answer 1

你很幸运。我能够在我自己的Mac上编译PortAudio和flite，并解决您的问题。

除了前面提到的问题之外，您还有几个问题，我在下面的代码转储中已经讨论过所有这些问题。

• 小调:对于cst_wave，您没有一致地使用自己的API。
• 小调:我喜欢把我的while和if块和{}放在一起。这有一种防止神秘虫子的习惯。
• 最大帧被设置为零。这是因为在(cst_wave_num_samples(w) / cst_wave_sample_rate(w)) * cst_wave_num_channels(w) * sizeof(short)中，你除以样本率，它比你的样本数还要多。给定整数除法是左相联和截断，yadda零。
• 最大帧仍然是错误的，因为一个帧包含所有的信道样本。因此，帧的数目与信道的数目和样本本身的大小都是不相关的。允许我猜测flite误用示例来表示帧，您的最大帧索引只是cst_wave_num_samples(w)。否则就会是cst_wave_num_samples(w) / cst_wave_num_channels(w)。
• PortAudio的文档声明，在流变得不活跃时，应该调用Pa_StopStream(stream)，无论您是否一直等到它变得不活跃。
• 我简化了回调，并将其更正为
  • 小调:一致使用您的API
  • 专业:嗯..。cst_wave_set_frameIndex(data, framesPerBuffer);绝对是错的。您将自己固定在帧索引512，而不是递增！这是因为在打开流时，每个缓冲区需要512个帧，而不是通过framesPerBuffer递增帧索引，而是将帧索引设置为framesPerBuffer。你没有走那么远，因为你的maxFrameIndex是0，所以你退出了。我修复了它，这样框架索引就会增加-当然是用API。

​

下面是代码，我可以自由地记录和清理代码，直到它达到了我的优雅标准。享受吧！

#include <stdio.h>
#include <string.h>

/**
 * Audio play callback.
 * 
 * Follows the PaStreamCallback signature, wherein:
 * 
 * @param input   and
 * @param output  are either arrays of interleaved samples or; if
 *                non-interleaved samples were requested using the
 *                paNonInterleaved sample format flag, an array of buffer
 *                pointers, one non-interleaved buffer for each channel.
 * @param frameCount    The number of sample frames to be processed by the
 *                      stream callback.
 * @param timeInfo      Timestamps indicating the ADC capture time of the first
 *                      sample in the input buffer, the DAC output time of the
 *                      first sample in the output buffer and the time the
 *                      callback was invoked. See PaStreamCallbackTimeInfo and
 *                      Pa_GetStreamTime()
 * @param statusFlags   Flags indicating whether input and/or output buffers
 *                      have been inserted or will be dropped to overcome
 *                      underflow or overflow conditions.
 * @param userData      The value of a user supplied pointer passed to
 *                      Pa_OpenStream() intended for storing synthesis data
 *                      etc.
 */

static int  playCallback(const void*                     inputBuffer,
                         void*                           outputBuffer,
                         unsigned long                   framesPerBuffer,
                         const PaStreamCallbackTimeInfo* timeInfo,
                         PaStreamCallbackFlags           statusFlags,
                         void*                           userData){
    (void) inputBuffer; /* Prevent unused variable warnings. */
    (void) timeInfo;
    (void) statusFlags;
    (void) userData;


    /**
     * Compute current processing state.
     */

    cst_wave*    data;
    short*       rptr;
    short*       wptr;
    unsigned int framesLeft, /* Number of frames of data remaining within the stream ***as a whole*** */
                 frames,     /* Number of frames of data to be written for this buffer. */
                 framesPad,  /* Number of frames of padding required within the final buffer. */
                 samples,    /* Number of samples of data to be written for this buffer. */
                 samplesPad, /* Number of samples of padding required within the final buffer. */
                 numBytes,   /* Number of bytes of data to be written for this buffer. */
                 numBytesPad;/* Number of bytes of padding required within the final buffer. */
    int          finalBuffer;/* Stores whether or not this is the final buffer. */


    data         = (cst_wave*)userData;
    rptr         = &data->samples[cst_wave_frameIndex  (data) *
                                  cst_wave_num_channels(data)];
    wptr         = (short*)outputBuffer;
    framesLeft   = cst_wave_maxFrameIndex(data) - cst_wave_frameIndex(data);
    finalBuffer  = framesLeft      <= framesPerBuffer;
    frames       = finalBuffer     ?  framesLeft     : framesPerBuffer;
    framesPad    = framesPerBuffer -  frames;
    samples      = frames     * cst_wave_num_channels(data);
    samplesPad   = framesPad  * cst_wave_num_channels(data);
    numBytes     = samples    * sizeof(short);
    numBytesPad  = samplesPad * sizeof(short);


    /**
     * Debug code. Comment out in production.
     */

    printf("framesLeft in callback: %u\n", framesLeft);
    printf("framesPerBuffer in callback: %lu\n", framesPerBuffer);
    printf("frames in callback: %u\n", frames);

    printf("frameIndex in callback: %d\n", cst_wave_frameIndex(data));
    printf("maxFrameIndex in callback: %d\n", cst_wave_maxFrameIndex(data));
    printf("numChannels in callback: %d\n", cst_wave_num_channels(data));
    printf("numSamples in callback: %d\n", cst_wave_num_samples(data));
    printf("sampleRate in callback: %d\n\n", cst_wave_sample_rate(data));


    /**
     * Output data. We handle the final buffer specially, padding it with zeros.
     */

    memcpy(wptr, rptr, numBytes);
    wptr += samples;
    rptr += samples;
    cst_wave_set_frameIndex(data, cst_wave_frameIndex(data) + frames);
    memset(wptr, 0, numBytesPad);
    wptr += samplesPad;
    rptr += samplesPad;


    /**
     * Return a completion or continue code depending on whether this was the
     * final buffer or not respectively.
     */

    return finalBuffer ? paComplete : paContinue;
}

/**
 * Play wave function.
 * 
 * Plays the given cst_wave data as audio, blocking until this is done.
 */

int play_wave(cst_wave *w){
    PaStream*          stream;
    PaStreamParameters outputParameters;
    int                err;

    /**
     * Initialize custom fields in cst_wave struct.
     */

    cst_wave_set_frameIndex(w, 0);
    cst_wave_set_maxFrameIndex(w, (cst_wave_num_samples(w)));
    // / cst_wave_sample_rate(w)  * cst_wave_num_channels(w) * sizeof(short)


    /**
     * Initialize Port Audio device and stream parameters.
     */

    err = Pa_Initialize();
    outputParameters.device = Pa_GetDefaultOutputDevice();
    if (outputParameters.device == paNoDevice){
        fprintf(stderr,"Error: No default output device.\n");
        return -5;
    }

    printf("frameIndex: %d\n", cst_wave_frameIndex(w));
    printf("maxFrameIndex: %d\n", cst_wave_maxFrameIndex(w));
    printf("numChannels: %d\n", cst_wave_num_channels(w));
    printf("numSamples: %d\n", cst_wave_num_samples(w));
    printf("sampleRate: %d\n", cst_wave_sample_rate(w));

    outputParameters.channelCount = cst_wave_num_channels(w);
    outputParameters.sampleFormat = paInt16;
    outputParameters.suggestedLatency = Pa_GetDeviceInfo( outputParameters.device )->defaultLowOutputLatency;
    outputParameters.hostApiSpecificStreamInfo = NULL;


    /**
     * Open the stream for playback.
     */

    puts("=== Now playing back. ===");
    err = Pa_OpenStream(&stream,
                        NULL, /* no input */
                        &outputParameters,
                        cst_wave_sample_rate(w),
                        512,
                        paClipOff,
                        playCallback,
                        w);

    if(stream){
        /**
         * Start the stream.
         */

        err = Pa_StartStream(stream);
        if(err != paNoError){
            goto done;
        }

        /**
         * Block while it plays.
         */

        puts("Waiting for playback to finish.");
        while((err = Pa_IsStreamActive(stream)) == 1){
            Pa_Sleep(100);
        }
        if(err < 0){
            goto done;
        }


        /**
         * Stop and close the stream. Both are necessary.
         */

        Pa_StopStream(stream);
        err = Pa_CloseStream(stream);
        if(err != paNoError){
            goto done;
        }
        puts("Done.");
    }

    /**
     * Terminate and leave.
     */
done:
    Pa_Terminate();
    return 0;
}

Answer 2

在我看来，问题可能在别的地方。

添加注释的例程实际上非常琐碎，所有这些都已经说好了。它基本上只是将一个充满数据的缓冲区从一个地方复制到另一个地方，如果数据没有填充输入缓冲区，则为零填充剩余的数据。如果我是在编写代码，我可能会按照以下一般思路做一些更多的事情：

const unsigned frame_size = sizeof(short) * data->num_channels;    

char *source = &data->samples[data->frameIndex * data->num_channels];
char *dest = outputBuffer;

unsigned framesLeft = data->maxFrameIndex - data->frameIndex;
unsigned framesEmpty = framesPerBuffer - framesLeft;

memcpy(source, dest, framesLeft * frame_size);
memset(dest+framesLeft * frame_size, 0, framesEmpty * frame_size);

data->frameIndex += framesPerBuffer;

虽然编写起来有些笨拙，但如果需要填充的大小为零，问题中的if/else将跳过执行memset部分。

因此，这会将一个充满数据的缓冲区从一个地方复制到另一个地方，零填充任何剩余的数据。如果您得到了分段错误，那么任何分配目标缓冲区的内容显然都没有分配足够的空间。如果不仔细观察，就不可能猜测分配是发生在Pa_Initialize、Pa_OpenStream、Pa_StartStream还是其他地方--而且很可能您不太关心实际执行分配的代码，而只关心计算要分配多少空间的代码(这可能在上面的某个部分中，或者完全在其他地方)。

Answer 3

在您的play_wave函数中，您调用：

err = Pa_OpenStream(&stream,
                    NULL, /* no input */
                    &outputParameters,
                    cst_wave_sample_rate(w),
                    512,
                    paClipOff,
                    playCallback,
                    &w);

在这里，作为最后一个参数，您要传递&w，所以您要传递cst_wave **，因为w被定义为cst_wave *w。

但是在playCallback()中，您使用它作为

cst_wave *data = (cst_wave*)userData;

因此，在这个函数中，您不正确地以cst_wave *的形式访问cst_wave *。因此，在某个时候，您将在使用w的某个成员时访问无效内存。

此外，这也是您获得其他参数(如frameIndex, maxFrameIndex等)的不正确输出的原因，正如您的输出所示。

解决方案只是将w传递给Pa_OpenStream()函数，而不是&w。

您的下一个问题是您没有正确地设置maxFrameIndex。正如您在评论中所说的，这不应该是0。为了正确设置它，您应该有如下内容：

cst_wave_set_maxFrameIndex(w, cst_wave_num_samples(w) * cst_wave_num_channels(w));

最后，您的回调可能会使事情变得有些混乱。下面是一种更好、更有效的编写方法：

static int playCallback( const void *inputBuffer, void *outputBuffer,
                        unsigned long framesPerBuffer,
                        const PaStreamCallbackTimeInfo* timeInfo,
                        PaStreamCallbackFlags statusFlags,
                        void *userData )
{
    cst_wave *data = (cst_wave*)userData;
    short *rptr = &data->samples[data->frameIndex * data->num_channels];
    short *wptr = (short*)outputBuffer;
    int finished;
    unsigned int framesLeft = data->maxFrameIndex - data->frameIndex;

    (void) inputBuffer; /* Prevent unused variable warnings. */
    (void) timeInfo;
    (void) statusFlags;
    (void) userData;

    if( framesLeft < framesPerBuffer )
    {
        /* final buffer... */
        memcpy(wptr, rptr, sizeof(*wptr) * data->num_channels * framesLeft);
        memset(wptr, sizeof(*wptr) * data->num_channels * framesPerBuffer, 0);
        data->frameIndex += framesLeft;
        finished = paComplete;
    }
    else
    {
        memcpy(wptr, rptr, sizeof(*wptr) * data->num_channels * framesPerBuffer);
        data->frameIndex += framesPerBuffer;
        finished = paContinue;
    }
    return finished;
}