用于vcf文件中的vcard重复删除的python代码与vobject一起工作，但仅用于“精确重复”。

Question

#!/usr/bin/env python2.7 

import vobject

abinfile='/foo/bar/dir/infile.vcf' #ab stands for address book  

aboutfile='/foo/bar/dir/outfile.vcf'  

def eliminate_vcard_duplicates (abinfile, aboutfile):

    #we first convert the Adrees Book IN FILE into a list

    with open(abinfile) as source_file:
        ablist = list(vobject.readComponents(source_file))

    #then add each vcard from that list in a new list unless it's already there

    ablist_norepeats=[]
    ablist_norepeats.append(ablist[0])

    for i in range(1, len(ablist)):
        jay=len(ablist_norepeats)
        for j in reversed(range(0, jay)): #we do reversed because usually cards have duplicates nearby
            if ablist_norepeats[j].serialize() == ablist[i].serialize():
                break
            else:
                jay += -1
        if jay == 0:
            ablist_norepeats.append(ablist[i])

    #and finally write the singularized list to the Adrees Book OUT FILE

    with open(aboutfile, 'w') as destination_file:
        for j in range(0, len(ablist_norepeats)):
            destination_file.write(ablist_norepeats[j].serialize)

eliminate_vcard_duplicates(abinfile, aboutfile)

上面的代码工作，并创建一个新的文件，其中没有确切的重复(重复与相同的奇异)。我知道这段代码存在一些效率问题:当它可能是n*log时，它是n平方的；我们只可以序列化每个空出一次；无效地使用for等。这里我想提供一个简短的代码来说明我不知道如何解决的问题之一。

我不知道该如何巧妙地解决这个问题:如果卡中的某些字段被置乱，它将无法检测到它们是否相等。是否有一种方法可以使用vobject、re或其他方法来检测这种重复？

测试中使用的文件内容有四个相等的vcards (手机扰乱了代码，而不是电子邮件加扰的想法)，是这样的吗：

BEGIN:VCARD
VERSION:3.0
FN:Foo_bar1
N:;Foo_bar1;;;
EMAIL;TYPE=INTERNET:foobar1@foo.bar.com
TEL;TYPE=CELL:123456789
TEL;TYPE=CELL:987654321
END:VCARD
BEGIN:VCARD
VERSION:3.0
FN:Foo_bar1
N:;Foo_bar1;;;
EMAIL;TYPE=INTERNET:foobar1@foo.bar.com
TEL;TYPE=CELL:123456789
TEL;TYPE=CELL:987654321
END:VCARD
BEGIN:VCARD
VERSION:3.0
FN:Foo_bar1
N:;Foo_bar1;;;
TEL;TYPE=CELL:123456789
TEL;TYPE=CELL:987654321
EMAIL;TYPE=INTERNET:foobar1@foo.bar.com
END:VCARD
BEGIN:VCARD
VERSION:3.0
FN:Foo_bar1
N:;Foo_bar1;;;
TEL;TYPE=CELL:987654321
TEL;TYPE=CELL:123456789
EMAIL;TYPE=INTERNET:foobar1@foo.bar.com
END:VCARD

上面的代码不会检测到四个都是相同的，因为最后一个是加扰的电话号码。

作为加分，如果有人有一个更快的算法，它将是伟大的，如果它可以共享。上面的30.000个Vcard文件需要几天时间.

Answer 1

您可能注意到的一件事是，如果调用.serialize()方法，那么EMAIL将在FN之前排序。但不幸的是，电视节目没有被分类。如果是，您可以将序列化的各个组件添加到一个集合中，并让唯一的散列排序多个出现的情况。

如果您研究从生成器vobject.readComponents() (例如使用type())得到了什么，您将看到这是来自模块vobject.base的Component，在实例中使用dir()可以看到方法getSortedChildren()。如果在源代码中查找这些信息，您会发现：

def getSortedChildren(self):
    return [obj for k in self.sortChildKeys() for obj in self.contents[k]]

而sortChildKeys()则直接高于这个值：

def sortChildKeys(self):
    try:
        first = [s for s in self.behavior.sortFirst if s in self.contents]
    except Exception:
        first = []
    return first + sorted(k for k in self.contents.keys() if k not in first)

在示例实例中调用sortChildKeys()将给出['version', 'email', 'fn', 'n', 'tel']，由此得出两个结论：

• sortFirst使version位于最前面
• 没有对for obj in self.contents[k]进行排序，因此没有对您的TEL条目进行排序。

解决方案似乎是将getSortedChildren()重新定义为：

    return [obj for k in self.sortChildKeys() for obj in sorted(self.contents[k])]

但这导致：

TypeError：“ContentLine”和“ContentLine”实例之间不支持<

因此，您需要为ContentLine提供一些基本的比较操作，这也是在vobject.base中定义的：

import vobject

from vobject.base import Component, ContentLine

def gsc(self):
    return [obj for k in self.sortChildKeys() for obj in sorted(self.contents[k])]

Component.getSortedChildren = gsc

def ltContentLine(self, other):
    return str(self) < str(other)

def eqContentLine(self, other):
    return str(self) == str(other)

ContentLine.__lt__ = ltContentLine
ContentLine.__eq__ = eqContentLine


addresses = set()
with open('infile.vcf') as fp:
  for vcard in vobject.readComponents(fp):
     # print(type(vcard))
     # print(dir(vcard))
     # print(vcard.sortChildKeys())
     # print(vcard.contents.keys())
     addresses.add(vcard.serialize())

with open('outfile.vcf', 'w') as fp:
    for a in addresses:
        fp.write(a)

# and check
with open('outfile.vcf') as fp:
    print(fp.read(), end="")

这意味着：

BEGIN:VCARD
VERSION:3.0
EMAIL;TYPE=INTERNET:foobar1@foo.bar.com
FN:Foo_bar1
N:;Foo_bar1;;;
TEL;TYPE=CELL:123456789
TEL;TYPE=CELL:987654321
END:VCARD

Answer 2

下面是一个更快的代码(大约三个数量级)，但仍然只删除精确的副本.

    #!/usr/bin/env python2.7 

    import vobject
    import datetime

    abinfile='/foo/bar/dir/infile.vcf' #ab stands for address book  

    aboutfile='/foo/bar/dir/outfile.vcf' 

    def eliminate_vcard_duplicatesv2(abinfile, aboutfile):

        #we first convert the Adrees Book IN FILE into a list
        ablist=[]
        with open(abinfile) as source_file:
            ablist = list(vobject.readComponents(source_file))

        #we then serialize the list to expedite comparison process
        ablist_serial=[]
        for i in range(0, len(ablist)):
            ablist_serial.append(ablist[i].serialize())

        #then add each unique vcard's position from that list in a new list unless it's already there
        ablist_singletons=[]
        duplicates=0
        for i in range(1, len(ablist_serial)):
            if i % 1000 == 0:
                print "COMPUTED CARD:", i, "Number of duplicates: ", duplicates, "Current time:", datetime.datetime.now().time()
            jay=len(ablist_singletons)
            for j in reversed(range(0, jay)): #we do reversed because usually cards have duplicates nearby
                if ablist_serial[ablist_singletons[j]] == ablist_serial[i]:
                    duplicates += 1
                    break
                else:
                    jay += -1
            if jay == 0:
                ablist_singletons.append(i)

        print "Length of Original Vcard File: ", len(ablist)
        print "Length of Singleton Vcard File: ", len(ablist_singletons)
        print "Generating Singleton Vcard file and storing it in: ", aboutfile

        #and finally write the singularized list to the Adrees Book OUT FILE
        with open(aboutfile, 'w') as destination_file:
            for k in range(0, len(ablist_singletons)):
                destination_file.write(ablist_serial[ablist_singletons[k]])

    eliminate_vcard_duplicatesv2(abinfile, aboutfile)

Answer 3

Anthon's answer上的一个变体，使用类装饰器。

import vobject
from vobject.base import Component, ContentLine


def sortedContents(cls):
    def getSortedChildren(self):
        return [obj for k in self.sortChildKeys() for obj in sorted(self.contents[k])]

    cls.getSortedChildren = getSortedChildren
    return cls


def sortableContent(cls):
    def __lt__(self, other):
        return str(self) < str(other)

    def __eq__(self, other):
        return str(self) == str(other)

    cls.__lt__ = __lt__
    cls.__eq__ = __eq__
    return cls


Component = sortedContents(Component)
ContentLine = sortableContent(ContentLine)


addresses = set()

with open('infile.vcf') as infile:
    for vcard in vobject.readComponents(infile):
        addresses.add(vcard.serialize())

with open('outfile.vcf', 'wb') as outfile:
    for address in addresses:
        outfile.write(bytes(address, 'UTF-8'))