python:在不同的文件中搜索单词

Question

我正在编写一个脚本，它收集目录中所有文件的名称，然后搜索每个文件中的给定单词。每次找到单词时，都应该打印文件名和包含该单词的完整行。此外，在一个新文件中，我希望打印单词已被找到的次数。

到目前为止，这就是我所拥有的：

import os

print(os.listdir('./texts'), '\n\n\n')

suchwort ={"computational":0,"linguistics":0,"processing":0,"chunking":0,"coreference":0,"html":0,"machine":0}
hitlist = './hits.txt'


with open(hitlist, 'a+') as hits:
   for elem in os.listdir('./texts'):
      with open(os.path.join("./texts",elem)) as fh:
         for line in fh:
            words = line.split(' ')
            print(elem, " : ",line)
            for n in words:
               if n in suchwort:
                  if n in suchwort.keys():
                     suchwort[n]+=1
                  else:
                     suchwort[n]=1
   for k in suchwort:
      print(k,":",suchwort[k],file=hits)

新文件(hits.txt)的结果是：

chunking : 0
machine : 9
html : 0
processing : 4
linguistics : 12
coreference : 1
computational : 12

但是，这些值似乎是错误的，因为单词"html“在其中一个文件中。

Answer 1

这个问题是由文件逐行迭代的方式造成的.在下面的代码段中，每个“行”都有尾随的换行符。因此，使用尾换行符对最后一行进行拆分。

  with open(os.path.join("./texts",elem)) as fh:
     for line in fh:
        words = line.split(' ')

如果你打印单词的“回复”，

print repr(words)

你会发现最后一个字包含了后面的换行符

['other', 'word\n']

而不是预期的：

['other', 'word']

要解决这个问题，您可以在处理每一行之前使用“条带”：

line = line.strip() 

若要删除字符串的尾随和前导空格，请执行以下操作。

Answer 2

import itertools
import multiprocessing as mp
import glob

def filesearcher(qIn, qOut):
    for fpath in iter(qIn.get, None):
        keywords = {"computational":{'count':0, 'lines':[]},
                    "linguistics":{'count':0, 'lines':[]},
                    "processing":{'count':0, 'lines':[]},
                    "chunking":{'count':0, 'lines':[]},
                    "coreference":{'count':0, 'lines':[]},
                    "html":{'count':0, 'lines':[]},
                    "machine":{'count':0, 'lines':[]}}

        with open(fpath) as infile:
           for line in file:
               for word in line.split():
                   word = word.lower()
                   if word not in keywords: continue
                   keywords[word]['count'] += 1
                   keywords[word]['lines'].append(line)
        qOut.put(fpath, keywords)
    qOut.put(None)


def main():
    numProcs = 4  # fiddle to taste
    qIn, qOut = [mp.Queue() for _ in range(2)]
    procs = [mp.Process(target=filesearcher, args=(qIn, qOut)) for _ in range(numProcs)]
    for p in procs: p.start()
    for fpath in glob.glob('./texts/*'): qIn.put(fpath)
    for _ in procs: qIn.put(None)

    done = 0
    while done < numProcs:
        d = qOut.get()
        if d is None:
            done += 1
            continue
        fpath, stats = d
        print("showing results for", fpath)
        for word, d in stats.items():
            print(word, ":", d['count'])
            for line in d['lines']:
                print('\t', line)

    for p in procs: p.terminate()