scope/search/web/ddg.go

141 lines
3.6 KiB
Go

/*
* Scope - A simple and minimal metasearch engine
* Copyright (C) 2021 Arsen Musayelyan
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package web
import (
"net/http"
"strconv"
"strings"
"github.com/PuerkitoBio/goquery"
)
var ddgURL = urlMustParse("https://html.duckduckgo.com/html")
const uddgPrefix = "//duckduckgo.com/l/?uddg="
// DDG represents the DuckDuckGo search engine
type DDG struct {
keyword string
userAgent string
page int
doc *goquery.Document
initDone bool
baseSel *goquery.Selection
}
// SetKeyword sets the keyword for searching
func (d *DDG) SetKeyword(keyword string) {
d.keyword = keyword
}
// SetPage sets the page number for searching
func (d *DDG) SetPage(page int) {
d.page = page * 30
}
// SetUserAgent sets the user agent for the request
func (d *DDG) SetUserAgent(ua string) {
d.userAgent = ua
}
// Init runs requests for the DuckDuckGo search engine
func (d *DDG) Init() error {
// Copy URL so that it can be changed
initURL := copyURL(ddgURL)
// Get query parameters
query := initURL.Query()
// Set query
query.Set("q", d.keyword)
if d.page > 0 {
query.Set("s", strconv.Itoa(d.page))
query.Set("dc", strconv.Itoa(d.page+1))
}
// Update URL query
initURL.RawQuery = query.Encode()
// Create new request for modified URL
req, err := http.NewRequest(
http.MethodGet,
initURL.String(),
nil,
)
if err != nil {
return err
}
// If user agent empty, use default
if d.userAgent == "" {
d.userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
}
// Set user agent of request
req.Header.Set("User-Agent", d.userAgent)
// Perform request
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
// Create goquery document from reader
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
return err
}
d.doc = doc
d.baseSel = doc.Find(`#links > .result`)
d.initDone = true
return nil
}
// Each runs eachCb with the index of each search result
func (d *DDG) Each(eachCb func(int) error) error {
for i := 0; i < d.baseSel.Length(); i++ {
err := eachCb(i)
if err != nil {
return err
}
}
return nil
}
// Title returns the title of the search result corresponding to i
func (d *DDG) Title(i int) (string, error) {
return strings.TrimSpace(get(d.baseSel, i).Children().First().ChildrenFiltered("h2").Text()), nil
}
// Link returns the link to the search result corresponding to i
func (d *DDG) Link(i int) (string, error) {
link := get(d.baseSel, i).Children().First().ChildrenFiltered("a").AttrOr("href", "")
if strings.HasPrefix(link, uddgPrefix) {
link = urlMustParse(link).Query().Get("uddg")
}
return link, nil
}
// Desc returns the description of the search result corresponding to i
func (d *DDG) Desc(i int) (string, error) {
return get(d.baseSel, i).Children().First().ChildrenFiltered("a").Text(), nil
}
// Name returns "ddg"
func (d *DDG) Name() string {
return "ddg"
}