scope/search/web/bing.go

116 lines
2.6 KiB
Go
Raw Normal View History

2021-12-08 17:24:05 +00:00
package web
import (
"net/http"
"strconv"
"github.com/PuerkitoBio/goquery"
)
var bingURL = urlMustParse("https://www.bing.com/search?count=10")
2021-12-08 21:18:14 +00:00
// Bing represents the Bing search engine
2021-12-08 17:24:05 +00:00
type Bing struct {
keyword string
userAgent string
first int
doc *goquery.Document
initDone bool
baseSel *goquery.Selection
}
2021-12-08 21:18:14 +00:00
// SetKeyword sets the keyword for searching
2021-12-08 17:24:05 +00:00
func (b *Bing) SetKeyword(keyword string) {
b.keyword = keyword
}
2021-12-08 21:18:14 +00:00
// SetPage sets the page number for searching
2021-12-08 17:24:05 +00:00
func (b *Bing) SetPage(page int) {
b.first = page * 10
}
2021-12-08 21:18:14 +00:00
// SetUserAgent sets the user agent to use for the request
2021-12-08 17:24:05 +00:00
func (b *Bing) SetUserAgent(ua string) {
b.userAgent = ua
}
2021-12-08 21:18:14 +00:00
// Init runs requests for Bing search engine
2021-12-08 17:24:05 +00:00
func (b *Bing) Init() error {
2021-12-08 21:18:14 +00:00
// Copy URL so it can be changed
2021-12-08 17:24:05 +00:00
initURL := copyURL(bingURL)
query := initURL.Query()
2021-12-08 21:18:14 +00:00
// Set query
2021-12-08 17:24:05 +00:00
query.Set("q", b.keyword)
if b.first > 0 {
query.Set("first", strconv.Itoa(b.first))
} else {
query.Set("first", "1")
}
2021-12-08 21:18:14 +00:00
// Update URL query parameters
2021-12-08 17:24:05 +00:00
initURL.RawQuery = query.Encode()
2021-12-08 21:18:14 +00:00
// Create new request for modified URL
2021-12-08 17:24:05 +00:00
req, err := http.NewRequest(
http.MethodGet,
initURL.String(),
nil,
)
if err != nil {
return err
}
2021-12-08 21:18:14 +00:00
// If no user agent, use default
2021-12-08 17:24:05 +00:00
if b.userAgent == "" {
b.userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
}
2021-12-08 21:18:14 +00:00
// Set request user agent
2021-12-08 17:24:05 +00:00
req.Header.Set("User-Agent", b.userAgent)
2021-12-08 21:18:14 +00:00
// Perform request
2021-12-08 17:24:05 +00:00
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
2021-12-08 21:18:14 +00:00
// Create new goquery document
2021-12-08 17:24:05 +00:00
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
return err
}
b.doc = doc
b.baseSel = doc.Find(`#b_results > li`)
b.initDone = true
return nil
}
2021-12-08 21:18:14 +00:00
// Each runs eachCb with the index of each search result
2021-12-08 17:24:05 +00:00
func (b *Bing) Each(eachCb func(int) error) error {
for i := 0; i < b.baseSel.Length(); i++ {
err := eachCb(i)
if err != nil {
return err
}
}
return nil
}
2021-12-08 21:18:14 +00:00
// Title returns the title of the search result corresponding to i
2021-12-08 17:24:05 +00:00
func (b *Bing) Title(i int) (string, error) {
return get(b.baseSel, i).ChildrenFiltered("h2").Children().First().Text(), nil
}
2021-12-08 21:18:14 +00:00
// Link returns the link to the search result corresponding to i
2021-12-08 17:24:05 +00:00
func (b *Bing) Link(i int) (string, error) {
return get(b.baseSel, i).ChildrenFiltered("h2").Children().First().AttrOr("href", ""), nil
}
2021-12-08 21:18:14 +00:00
// Desc returns the description of the search result corresponding to i
2021-12-08 17:24:05 +00:00
func (b *Bing) Desc(i int) (string, error) {
return get(b.baseSel, i).ChildrenFiltered(".b_caption").Children().Last().Text(), nil
}
2021-12-08 21:18:14 +00:00
// Name returns "bing"
2021-12-08 17:24:05 +00:00
func (b *Bing) Name() string {
return "bing"
}