Nodejs > Gulp > through2 >16个文件的限制？

Question

更新 Ok，这似乎是链接到整个2的"highWaterMark“属性。基本上，它的意思是“不要缓冲超过x文件，等待某人使用它，然后才接受另一批文件”。因为它是这样设计的，所以这个问题中的片段是被审查的。必须有更好的方法来处理许多文件。

快速修复，允许8000个文件：

  through.obj({ highWaterMark: 8000 }, (file, enc, next) => { ... })

原始问题

我使用一个gulp任务来创建翻译文件。它扫描src文件夹以查找*.i18n.json文件，并在源文件中找到每种语言保存一个.json。

它可以正常工作，直到找到超过16个文件。它使用through2来处理每个文件。请参阅下面的源代码。方法processAll18nFiles()是一个自定义管道，它接收匹配的输入文件，读取每个文件的内容，动态构造结果字典，然后将其交给on('finish)处理程序编写字典。

在windows和mac上测试。我的方法似乎有一个限制，因为它在16个或更少的文件中工作得很好。

仍在寻找，线索欢迎:-)

源文件示例: signs.i18n.json

{
  "path": "profile.signs",
  "data": {
    "title": {
      "fr": "mes signes précurseurs",
      "en": "my warning signs"
    },
    "add": {
      "fr": "ajouter un nouveau signe",
      "en": "add a new warning sign"
    }
  }
}

输出文件示例: en.json

{"profile":{"signs":{"title":"my warning signs","add":"add a new warning sign"}}}

gulpfile.js

const fs = require('fs');
const path = require('path');
const gulp = require('gulp');
const watch = require('gulp-watch');
const through = require('through2');

const searchPatternFolder = 'src/app/**/*.i18n.json';
const outputFolder = path.join('src', 'assets', 'i18n');

gulp.task('default', () => {
  console.log('Ionosphere Gulp tasks');
  console.log(' > gulp i18n         builds the i18n file.');
  console.log(' > gulp i18n:watch   watches i18n file and trigger build.');
});

gulp.task('i18n:watch', () => watch(searchPatternFolder, { ignoreInitial: false }, () => gulp.start('i18n')));
gulp.task('i18n', done => processAll18nFiles(done));

function processAll18nFiles(done) {
  const dictionary = {};
  console.log('[i18n] Rebuilding...');
  gulp
    .src(searchPatternFolder)
    .pipe(
      through.obj((file, enc, next) => {
        console.log('doing ', file.path);
        const i18n = JSON.parse(file.contents.toString('utf8'));
        composeDictionary(dictionary, i18n.data, i18n.path.split('.'));
        next(null, file);
      })
    )
    .on('finish', () => {
      const writes = [];
      Object.keys(dictionary).forEach(langKey => {
        console.log('lang key ', langKey);
        writes.push(writeDictionary(langKey, dictionary[langKey]));
      });
      Promise.all(writes)
        .then(data => done())
        .catch(err => console.log('ERROR ', err));
    });
}

function composeDictionary(dictionary, data, path) {
  Object.keys(data)
    .map(key => ({ key, data: data[key] }))
    .forEach(({ key, data }) => {
      if (isString(data)) {
        setDictionaryEntry(dictionary, key, path, data);
      } else {
        composeDictionary(dictionary, data, [...path, key]);
      }
    });
}

function isString(x) {
  return Object.prototype.toString.call(x) === '[object String]';
}

function initDictionaryEntry(key, dictionary) {
  if (!dictionary[key]) {
    dictionary[key] = {};
  }
  return dictionary[key];
}

function setDictionaryEntry(dictionary, langKey, path, data) {
  initDictionaryEntry(langKey, dictionary);
  let subDict = dictionary[langKey];
  path.forEach(subKey => {
    isLastToken = path[path.length - 1] === subKey;
    if (isLastToken) {
      subDict[subKey] = data;
    } else {
      subDict = initDictionaryEntry(subKey, subDict);
    }
  });
}

function writeDictionary(lang, data) {
  return new Promise((resolve, reject) => {
    fs.writeFile(
      path.join(outputFolder, lang + '.json'),
      JSON.stringify(data),
      'utf8',
      err => (err ? reject(err) : resolve())
    );
  });
}

Answer 1

好的，正如解释过的here，一个必须使用来使用管道。这是通过添加“数据”事件的处理程序来完成的，例如：

  gulp
    .src(searchPatternFolder)
    .pipe(
      through.obj({ highWaterMark: 4, objectMode: true }, (file, enc, next) => {
        const { data, path } = JSON.parse(file.contents.toString('utf8'));
        next(null, { data, path });
      })
    )
    // The next line handles the "consumption" of upstream pipings
    .on('data', ({ data, path }) => ++count && composeDictionary(dictionary, data, path.split('.')))
    .on('end', () =>
      Promise.all(Object.keys(dictionary).map(langKey => writeDictionary(langKey, dictionary[langKey])))
        .then(() => {
          console.log(`[i18n] Done, ${count} files processed, language count: ${Object.keys(dictionary).length}`);
          done();
        })
        .catch(err => console.log('ERROR ', err))
    );