go-colly爬取图片-以基恩士网站为例

爬取基恩士网站主页上的产品图，以下是一个简单的示例，注意要先创建对应的目录。

package main

import (
    "fmt"
    "github.com/gocolly/colly"
    "io"
    "net/http"
    "os"
)
// global variables
var destUrl string

/**
download images
 */
func downloadImages(_ int, e *colly.HTMLElement) {
    srcRef := e.Attr("src")
    fullurl := destUrl + srcRef
    res, _ := http.Get(fullurl)
    savedPath := "D:" + srcRef
    // TODO create dirs before create file
    f, err := os.Create(savedPath)
    if(err != nil) {
        panic(err)
    }
    io.Copy(f, res.Body)
    fmt.Println(destUrl  + srcRef)
}

func main()  {
    c := colly.NewCollector()
    
    c.OnHTML("ul[class]", func(e *colly.HTMLElement) {

        className := e.Attr("class")
        if (className == "ProductsList full") {
            e.ForEach("img[src]", downloadImages)
        }
    })
    
    c.OnRequest(func(r *colly.Request) {
        fmt.Println("Visiting", r.URL)
    })

    destUrl = "https://www.keyence.com.cn"
    c.Visit(destUrl)
}