调试pickle

Question

我正在尝试剔除相当复杂的对象层次结构，并获得异常：

pickle.PicklingError: Can't pickle <class 'function'>: attribute lookup builtins.function failed

有没有什么合理的方法可以用来测试对象层次结构的可拾取性？我的目标是找到问题函数的位置

Answer 1

为此，我将使用dill，它可以序列化python中的几乎任何内容。Dill还提供了some good tools，可以帮助您理解在代码失败时是什么原因导致酸洗失败。

>>> import dill
>>> dill.loads(dill.dumps(your_bad_object))
>>> ...
>>> # if you get a pickling error, use dill's tools to figure out a workaround
>>> dill.detect.badobjects(your_bad_object, depth=0)
>>> dill.detect.badobjects(your_bad_object, depth=1)
>>> ...

如果你确实想这样做，你可以使用dill的badobjects (或其他检测函数)递归地深入你的对象的引用链，并弹出不可拾取的对象，而不是像上面那样在每个深度调用它。

此外，objgraph也是对测试套件的一个非常方便的补充。

>>> # visualize the references in your bad objects
>>> objgraph.show_refs(your_bad_object, filename='your_bad_object.png')

Answer 2

我这么做了，很多时候它对我很有用...一旦我找到了万无一失的东西，我就会更新这个

它会生成一堆print，然后如果有要引发的异常，则会引发异常，这样您就可以看到对象层次结构的哪个部分导致了问题。

def test_pickle(xThing,lTested = []):
    import pickle
    if id(xThing) in lTested:
        return lTested
    sType = type(xThing).__name__
    print('Testing {0}...'.format(sType))

    if sType in ['type','int','str']:
        print('...too easy')
        return lTested
    if sType == 'dict':
        print('...testing members')
        for k in xThing:
            lTested = Pickalable.test_pickle(xThing[k],lTested)
        print('...tested members')
        return lTested
    if sType == 'list':
        print('...testing members')
        for x in xThing:
            lTested = Pickalable.test_pickle(x)
        print('...tested members')
        return lTested

    lTested.append(id(xThing))
    oClass = type(xThing)

    for s in dir(xThing):
        if s.startswith('_'):
            print('...skipping *private* thingy')
            continue
        #if it is an attribute: Skip it
        try:
            xClassAttribute = oClass.__getattribute__(oClass,s)
        except AttributeError:
            pass
        else:
            if type(xClassAttribute).__name__ == 'property':
                print('...skipping property')
                continue

        xAttribute = xThing.__getattribute__(s)
        print('Testing {0}.{1} of type {2}'.format(sType,s,type(xAttribute).__name__))
        #if it is a function make sure it is stuck to the class...
        if type(xAttribute).__name__ == 'function':
            raise Exception('ERROR: found a function')
        if type(xAttribute).__name__ == 'method':
            print('...skipping method')
            continue
        if type(xAttribute).__name__ == 'HtmlElement':
            continue
        if type(xAttribute) == dict:
            print('...testing dict values for {0}.{1}'.format(sType,s))
            for k in xAttribute:
                lTested = Pickalable.test_pickle(xAttribute[k])
                continue
            print('...finished testing dict values for {0}.{1}'.format(sType,s))

        try:
            oIter = xAttribute.__iter__()
        except AttributeError:
            pass
        except AssertionError:
            pass #lxml elements do this
        else:
            print('...testing iter values for {0}.{1} of type {2}'.format(sType,s,type(xAttribute).__name__))
            for x in xAttribute:   
                lTested = Pickalable.test_pickle(x,lTested)
            print('...finished testing iter values for {0}.{1}'.format(sType,s))

        try:
            xAttribute.__dict__
        except AttributeError:
            pass
        else:
            #this attribute should be explored seperately...
            lTested = Pickalable.test_pickle(xAttribute,lTested)
            continue
        pickle.dumps(xAttribute)


    print('Testing {0} as complete object'.format(sType))
    pickle.dumps(xThing)
    return lTested   

Answer 3

以下是@Sheena代码的一个略微调整的版本，它也适用于python 2和其他一些类型：

def test_pickle(xThing, lTested = []):
    import pickle
    if id(xThing) in lTested:
        return lTested
    sType = type(xThing).__name__
    print('Testing {0}...'.format(sType))

    if sType in ['type','int','str', 'bool', 'NoneType', 'unicode']:
        print('...too easy')
        return lTested
    if sType == 'dict':
        print('...testing members')
        for k in xThing:
            lTested = test_pickle(xThing[k],lTested)
        print('...tested members')
        return lTested
    if sType == 'list':
        print('...testing members')
        for x in xThing:
            lTested = test_pickle(x)
        print('...tested members')
        return lTested

    lTested.append(id(xThing))
    oClass = type(xThing)

    for s in dir(xThing):
        if s.startswith('_'):
            print('...skipping *private* thingy')
            continue
        #if it is an attribute: Skip it
        try:
            xClassAttribute = oClass.__getattribute__(oClass,s)
        except (AttributeError, TypeError):
            pass
        else:
            if type(xClassAttribute).__name__ == 'property':
                print('...skipping property')
                continue

        xAttribute = xThing.__getattribute__(s)
        print('Testing {0}.{1} of type {2}'.format(sType,s,type(xAttribute).__name__))
        if type(xAttribute).__name__ == 'function':
            print("...skipping function")
            continue
        if type(xAttribute).__name__ in ['method', 'instancemethod']:
            print('...skipping method')
            continue
        if type(xAttribute).__name__ == 'HtmlElement':
            continue
        if type(xAttribute) == dict:
            print('...testing dict values for {0}.{1}'.format(sType,s))
            for k in xAttribute:
                lTested = test_pickle(xAttribute[k])
                continue
            print('...finished testing dict values for {0}.{1}'.format(sType,s))

        try:
            oIter = xAttribute.__iter__()
        except (AttributeError, TypeError):
            pass
        except AssertionError:
            pass #lxml elements do this
        else:
            print('...testing iter values for {0}.{1} of type {2}'.format(sType,s,type(xAttribute).__name__))
            for x in xAttribute:
                lTested = test_pickle(x,lTested)
            print('...finished testing iter values for {0}.{1}'.format(sType,s))

        try:
            xAttribute.__dict__
        except AttributeError:
            pass
        else:
            #this attribute should be explored seperately...
            lTested = test_pickle(xAttribute,lTested)
            continue
        print(0, xThing, xAttribute)
        pickle.dumps(xAttribute)

    print('Testing {0} as complete object'.format(sType))
    pickle.dumps(xThing)
    return lTested

我发现这是最有用的选项(在pickle没有的地方，也有更宽宏大量的dill )。您可以简单地使用以下命令运行它

test_pickle(my_complex_object)
print("Done!")