如何提高空间相似度计算的速度？

Question

我目前有以下代码，用于处理搜索和候选字典之间的相似度计算。大约需要13秒才能得到4000名候选人的计算结果。我做了一些研究，认为使用nlp.pipe()可以对其进行改进。但是，我还是不明白我怎么能做到这一点？请给我建议。下面是我的python代码。

import os
import sys
from flask import Flask, request, jsonify

import spacy
nlp =  spacy.load("en_core_web_lg")
all_stopwords = nlp.Defaults.stop_words

app = Flask(__name__)

@app.route("/")
def index():
    return "Page does not exist"


@app.route('/calculate-matches', methods=['POST'])
def calculate_matches():
    data = request.get_json()
    candidates = data['candidates'] 
    cur_search = nlp('Looking for someone with experience in building vue frontend applications')

    tmp_search = ''
    for x in cur_search:
        if x.pos_ == "NOUN" or x.pos_ == "PROPN" or x.pos_=="PRON" or x.is_stop==False:
            tmp_search += str(x) + ' '
    cur_search = nlp(tmp_search)

    for member in candidates:
        member_bio = nlp(member['bio']+ ' ' + member['education']+ ' ' + member['experience'])
        
        #calculate similarity
        member['match_score'] = ( cur_search.similarity(member_bio) * 100 )

    #sort canidates' match_score from high to low
    results = sorted(candidates, key=lambda k: k['match_score'], reverse=True)
    return jsonify(results)


if __name__ == "__main__":
    currentdir = os.path.dirname(os.path.realpath(__file__))
    if currentdir not in sys.path:
        sys.path.insert(0, currentdir)
    app.run(host='0.0.0.0', port=5000)

Answer 1

您可以使用线性代数以广播的方式计算这种相似性：

import numpy as np

def cosine_similarity(v, A):
     return np.argmax(np.dot(v, A.T) / (np.linalg.norm(v, ord=2) * np.linalg.norm(A, axis=1, ord=2))

A = np.stack([member.vector for member in member_bio])
v = cur_search.vector
closest_idx = cosine_similarity(v, A)