有的时候运行会报错 invalid memory address or nil pointer dereference

来源：16-5 重构和总结

hehehei

2018-07-22

panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x40 pc=0x120c0dc]

goroutine 83 [running]:
learngo/crawler/fetcher.Fetch(0xc4202f14a0, 0x24, 0x0, 0x0, 0x0, 0x0, 0x0)
   /Users/wuyinhua/go/src/learngo/crawler/fetcher/fetcher.go:49 +0x2ec
learngo/crawler/engine.worker(0xc4202f14a0, 0x24, 0xc420506150, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
   /Users/wuyinhua/go/src/learngo/crawler/engine/simple.go:37 +0x68
learngo/crawler/engine.createWorker.func1(0x12fdda0, 0xc4200a0f00, 0xc4200979e0, 0xc420096120)
   /Users/wuyinhua/go/src/learngo/crawler/engine/concurruntengine.go:51 +0xcc
created by learngo/crawler/engine.createWorker
   /Users/wuyinhua/go/src/learngo/crawler/engine/concurruntengine.go:47 +0x76

fetch 代码我改了下header

func Fetch(url string) ([]byte, error) {
// <-rateLimiter
request, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
request.Header.Add("Accept", `text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8`)
request.Header.Add("Accept-Language", `gzh-CN,zh;q=0.9`)
request.Header.Add("Cache-Control", `max-age=0`)
request.Header.Add("Connection", `keep-alive`)
request.Header.Add("Cookie", `sid=kw4Q8rFPSB704BnO0CtJ; ipCityCode=10103000; ipOfflineCityCode=10103000; JSESSIONID=abccFnBOwlANG6LwxVYsw; CHANNEL=^~subid=2^~refererHost=^~channelId=903404^~refererHost=www.baidu.com^~; Hm_lvt_2c8ad67df9e787ad29dbd54ee608f5d2=1531749150,1532156726; clientp=38624; Hm_lpvt_2c8ad67df9e787ad29dbd54ee608f5d2=1532171973`)
// reqest.Header.Add("Accept-Encoding", "gzip, deflate")
request.Header.Add("Host", "album.zhenai.com")
request.Header.Add("Referer", "http://www.zhenai.com/zhenghun/shanghai/3")
request.Header.Add("Upgrade-Insecure-Requests", "1")
request.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36")
client := http.Client{}
resp, err := client.Do(request)
// resp, err := http.Get(url)
defer resp.Body.Close()
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("wrong http status code:%d", resp.StatusCode)
}
bodyReader := bufio.NewReader(resp.Body)
e := determineEncoding(bodyReader)
utf8Reader := transform.NewReader(bodyReader, e.NewDecoder())
return ioutil.ReadAll(utf8Reader)
}