在本例中，爬虫多线程是如何工作的？

Question

我正在尝试解决有关使用缓存并行获取URL的任务，以避免重复。我找到了正确的解决方案，并能理解它。我看到正确的答案包含通道，并通过chan在缓存中推送URL。但是为什么我的简单代码不能正常工作呢？我不知道哪里是错误。

package main

import (
    "fmt"
    "sync"
)

type Fetcher interface {
    // Fetch returns the body of URL and
    // a slice of URLs found on that page.
    Fetch(url string) (body string, urls []string, err error)
}

var cache = struct {
    cache map[string]int
    mux sync.Mutex
}{cache: make(map[string]int)}

// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(url string, depth int, fetcher Fetcher) {
    // TODO: Fetch URLs in parallel.
    // TODO: Don't fetch the same URL twice.
    // This implementation doesn't do either:

    if depth <= 0 {
        return
    }
    cache.mux.Lock()
    cache.cache[url] = 1 //put url in cache
    cache.mux.Unlock()
    body, urls, err := fetcher.Fetch(url)
    if err != nil {
        fmt.Println(err)
        return
    }

    fmt.Printf("found: %s %q\n", url, body)
    for _, u := range urls {
        cache.mux.Lock()
        if _, ok := cache.cache[u]; !ok { //check if url already in cache
            cache.mux.Unlock()
            go Crawl(u, depth-1, fetcher)
        } else {
            cache.mux.Unlock()
        }
    }
    return
}

func main() {
    Crawl("http://golang.org/", 4, fetcher)
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
    body string
    urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
    if res, ok := f[url]; ok {
        return res.body, res.urls, nil
    }
    return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
    "http://golang.org/": &fakeResult{
        "The Go Programming Language",
        []string{
            "http://golang.org/pkg/",
            "http://golang.org/cmd/",
        },
    },
    "http://golang.org/pkg/": &fakeResult{
        "Packages",
        []string{
            "http://golang.org/",
            "http://golang.org/cmd/",
            "http://golang.org/pkg/fmt/",
            "http://golang.org/pkg/os/",
        },
    },
    "http://golang.org/pkg/fmt/": &fakeResult{
        "Package fmt",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
    "http://golang.org/pkg/os/": &fakeResult{
        "Package os",
        []string{
            "http://golang.org/",
            "http://golang.org/pkg/",
        },
    },
}

UPD:执行结果：

found: http://golang.org/ "The Go Programming Language"

Process finished with exit code 0

当我启动goroutine的时候，我感觉没有递归。但是，如果让断点在线检查是否在cahce中的URL，我得到如下：

found: http://golang.org/ "The Go Programming Language"
found: http://golang.org/pkg/ "Packages"

Debugger finished with exit code 0

这就意味着递归可以工作，但是有些地方出错了，我猜是种族问题吧？当在线添加第二个断点时，会发生更有趣的事情，该断点将运行例程：

found: http://golang.org/ "The Go Programming Language"
found: http://golang.org/pkg/ "Packages"
fatal error: all goroutines are asleep - deadlock!

goroutine 1 [semacquire]:
sync.runtime_SemacquireMutex(0x58843c, 0x0, 0x1)
        /usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*Mutex).lockSlow(0x588438)
        /usr/local/go/src/sync/mutex.go:138 +0x295
sync.(*Mutex).Lock(0x588438)
        /usr/local/go/src/sync/mutex.go:81 +0x58
main.Crawl(0x4e9cf9, 0x12, 0x4, 0x4f7700, 0xc00008c180)
        /root/go/src/crwaler/main.go:38 +0x46c
main.main()
        /root/go/src/crwaler/main.go:48 +0x57

goroutine 18 [semacquire]:
sync.runtime_SemacquireMutex(0x58843c, 0x0, 0x1)
        /usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*Mutex).lockSlow(0x588438)
        /usr/local/go/src/sync/mutex.go:138 +0x295
sync.(*Mutex).Lock(0x588438)
        /usr/local/go/src/sync/mutex.go:81 +0x58
main.Crawl(0x4ea989, 0x16, 0x3, 0x4f7700, 0xc00008c180)
        /root/go/src/crwaler/main.go:38 +0x46c
created by main.Crawl
        /root/go/src/crwaler/main.go:41 +0x563

Debugger finished with exit code 0

Answer 1

您的main()在所有go Crawl()调用完成之前不会阻塞，因此退出。您可以使用一个sync.WaitGroup或一个通道来同步程序，结束于完成所有的goroutines。

我还看到了在goroutine中使用的变量u的问题；在执行goroutine时，u的值可能被range循环改变了，也可能没有改变。

Crawl的结束可以这样解决这两个问题；

wg := sync.WaitGroup{}

fmt.Printf("found: %s %q\n", url, body)
for _, u := range urls {
    cache.mux.Lock()
    if _, ok := cache.cache[u]; !ok { //check if url already in cache
        cache.mux.Unlock()
        wg.Add(1)
        go func(url string) {
            Crawl(url, depth-1, fetcher)
            wg.Done()
        }(u)
    } else {
        cache.mux.Unlock()
    }
}

// Block until all goroutines are done
wg.Wait()

return