From 254155d44203cf8907ea5d0d5b30fe3c56cc605a Mon Sep 17 00:00:00 2001 From: Arsen Musayelyan Date: Wed, 8 Dec 2021 13:18:14 -0800 Subject: [PATCH] Fix golint errors and gofmt --- internal/cards/calc.go | 11 ++++++++++ internal/cards/cards.go | 5 +---- internal/cards/ddg.go | 16 +++++++++++--- internal/cards/metaweather.go | 11 +++++++++- internal/cards/plot.go | 14 +++++++----- main.go | 4 +++- search/web/bing.go | 18 +++++++++++++++ search/web/ddg.go | 21 +++++++++++++++++- search/web/google.go | 24 ++++++++++++++++++++ search/web/search.go | 41 ++++++++++++++++++++++++++++++----- 10 files changed, 144 insertions(+), 21 deletions(-) diff --git a/internal/cards/calc.go b/internal/cards/calc.go index 88c904d..b4275f0 100644 --- a/internal/cards/calc.go +++ b/internal/cards/calc.go @@ -44,6 +44,8 @@ func NewCalcCard(query, _ string) Card { return &CalcCard{query: query, useFunc: true} } +// Matches determines whether a query matches the requirements +// fot CalcCard and determines which function to run with what arguments func (cc *CalcCard) Matches() bool { // If query has solve prefix if strings.HasPrefix(cc.query, "solve") { @@ -106,6 +108,7 @@ func (cc *CalcCard) Matches() bool { return false } +// StripKey removes all key words related to CalcCard func (cc *CalcCard) StripKey() string { // Compile regex for words to be removed trimRgx := regexp.MustCompile(`^(.*?)solve|integrate|integral(?: of)?|diff|differentiate|derivative(?: of)?|derive|calculate(.*)$`) @@ -113,6 +116,7 @@ func (cc *CalcCard) StripKey() string { return trimRgx.ReplaceAllString(cc.query, "${1}${2}") } +// Content returns the solveRenderScript with the given input func (cc *CalcCard) Content() template.HTML { var input string // If function is being used @@ -130,22 +134,29 @@ func (cc *CalcCard) Content() template.HTML { )) } +// Footer returns empty string because CalcCard has no footer func (cc *CalcCard) Footer() template.HTML { return "" } +// Returned always returns true since CalcCard is a frontend +// card and it is thus impossible to determine whether +// the query gets an answer func (cc *CalcCard) Returned() bool { return true } +// RunQuery returns nil as CalcCard is a frontend card func (cc *CalcCard) RunQuery() error { return nil } +// Title for CalcCard is "Calculation" func (cc *CalcCard) Title() string { return "Calculation" } +// Head returns the extra head tags required for CalcCard func (cc *CalcCard) Head() template.HTML { return calcExtraHead } diff --git a/internal/cards/cards.go b/internal/cards/cards.go index 630e3e9..bf45b95 100644 --- a/internal/cards/cards.go +++ b/internal/cards/cards.go @@ -1,13 +1,10 @@ package cards import ( - "errors" "html/template" "sort" ) -var ErrCardRegistered = errors.New("card with that name has already been registered") - // cardRegistration represents a card that has been registered type cardRegistration struct { name string @@ -79,7 +76,7 @@ func GetCard(query, userAgent string) Card { for _, cardReg := range cards { // Create new card card := cardReg.newFn(query, userAgent) - // If card matches, return it + // If card matches, return it if card.Matches() { return card } diff --git a/internal/cards/ddg.go b/internal/cards/ddg.go index 4e4d17f..801f764 100644 --- a/internal/cards/ddg.go +++ b/internal/cards/ddg.go @@ -39,7 +39,7 @@ type DDGInstAns struct { AnswerType string } -// NewDDGCard isa NewCardFunc that creates a new DDGCard +// NewDDGCard is a NewCardFunc that creates a new DDGCard func NewDDGCard(query, userAgent string) Card { return &DDGCard{ query: url.QueryEscape(query), @@ -68,7 +68,7 @@ func (ddg *DDGCard) RunQuery() error { return err } - // Decode response into repsonse struct + // Decode response into response struct err = json.NewDecoder(res.Body).Decode(&ddg.resp) if err != nil { return err @@ -77,30 +77,38 @@ func (ddg *DDGCard) RunQuery() error { return nil } +// Returned checks if abstract is empty. +// If it is, query returned no result. func (ddg *DDGCard) Returned() bool { // Value was returned if abstract is not empty return ddg.resp.Abstract != "" } +// Matches always returns true as there are no keys func (ddg *DDGCard) Matches() bool { - // Everything matches since there are no keys return true } +// StripKey returns the query as there are no keys func (ddg *DDGCard) StripKey() string { // No key to strip, so return query return ddg.query } +// Title returns the instant answer heading func (ddg *DDGCard) Title() string { return ddg.resp.Heading } +// Content returns the instant answer abstract with +// DuckDuckGo attibution func (ddg *DDGCard) Content() template.HTML { // Return abstract with attribution return template.HTML(ddg.resp.Abstract + fmt.Sprintf(ddgAttribution, ddg.query)) } +// Footer returns a footer containing URL and name of source +// gotten from instant answer API func (ddg *DDGCard) Footer() template.HTML { // Return footer with abstract url and source return template.HTML(fmt.Sprintf( @@ -110,6 +118,8 @@ func (ddg *DDGCard) Footer() template.HTML { )) } +// Head returns an empty string as no extra tags are reuired +// for DDGCard func (ddg *DDGCard) Head() template.HTML { return "" } diff --git a/internal/cards/metaweather.go b/internal/cards/metaweather.go index 93731a3..8dc07d5 100644 --- a/internal/cards/metaweather.go +++ b/internal/cards/metaweather.go @@ -166,16 +166,21 @@ func (mc *MetaweatherCard) RunQuery() error { return nil } +// Returned checks whether no location was found or response +// said not found. func (mc *MetaweatherCard) Returned() bool { return len(mc.location) > 0 && mc.resp.Detail != "Not found." } +// Matches determines whether the query matches the keys for +// MetaweatherCard func (mc *MetaweatherCard) Matches() bool { return strings.HasPrefix(mc.query, "weather in") || strings.HasPrefix(mc.query, "weather") || strings.HasSuffix(mc.query, "weather") } +// StripKey removes keys related to MetaweatherCard func (mc *MetaweatherCard) StripKey() string { query := strings.TrimPrefix(mc.query, "weather in") query = strings.TrimPrefix(query, "weather") @@ -183,6 +188,7 @@ func (mc *MetaweatherCard) StripKey() string { return strings.TrimSpace(query) } +// Content returns metaweatherContent with all information added func (mc *MetaweatherCard) Content() template.HTML { return template.HTML(fmt.Sprintf( metaweatherContent, @@ -213,15 +219,18 @@ func (mc *MetaweatherCard) Content() template.HTML { )) } +// Title of MetaweatherCard is "Weather" func (mc *MetaweatherCard) Title() string { return "Weather" } +// Footer returns a footer with a link to metaweather func (mc *MetaweatherCard) Footer() template.HTML { - // Return footer with link to metaweather return `` } +// Head returns an empty string as no extra head tags +// are required for MetaweatherCard func (mc *MetaweatherCard) Head() template.HTML { return "" } diff --git a/internal/cards/plot.go b/internal/cards/plot.go index 9c49bd9..23cd530 100644 --- a/internal/cards/plot.go +++ b/internal/cards/plot.go @@ -43,12 +43,14 @@ func NewPlotCard(query, _ string) Card { return &PlotCard{query: query} } +// Matches checks if the query matches the rules for PlotCard func (pc *PlotCard) Matches() bool { return strings.HasPrefix(pc.query, "plot") || strings.HasPrefix(pc.query, "graph") || strings.HasPrefix(pc.query, "draw") } +// StripKey removes all keys related to PlotCard func (pc *PlotCard) StripKey() string { query := strings.TrimPrefix(pc.query, "plot") query = strings.TrimPrefix(query, "graph") @@ -56,6 +58,7 @@ func (pc *PlotCard) StripKey() string { return strings.TrimSpace(query) } +// Content returns plot script with given input func (pc *PlotCard) Content() template.HTML { return template.HTML(fmt.Sprintf( plotScript, @@ -63,27 +66,28 @@ func (pc *PlotCard) Content() template.HTML { )) } -// Since this card is frontend, this cannot be checked. -// Therefore, it will always return true. +// Returned will alwats return true because +// this card is frontend, and this cannot be checked. func (pc *PlotCard) Returned() bool { return true } +// Title generates a title formatted as "Plot ()" func (pc *PlotCard) Title() string { return "Plot (" + pc.StripKey() + ")" } +// Head returns extra head tags for PlotCard func (pc *PlotCard) Head() template.HTML { return plotExtraHead } +// Footer returns an empty string as PlotCard has no footer func (pc *PlotCard) Footer() template.HTML { return "" } - +// RunQuery returns nil as PlotCard is a frontend card func (pc *PlotCard) RunQuery() error { return nil } - - diff --git a/main.go b/main.go index c6e250d..c51c5e8 100644 --- a/main.go +++ b/main.go @@ -66,6 +66,7 @@ type TmplContext struct { Card cards.Card } +// ErrTmplContext represents context passed to an error template type ErrTmplContext struct { BaseContext Error string @@ -74,6 +75,7 @@ type ErrTmplContext struct { Status int } +// BaseContext is the common context between all templates type BaseContext struct { LoadTime time.Duration } @@ -252,7 +254,7 @@ func main() { // Join address and port from config addr := net.JoinHostPort(viper.GetString("server.addr"), viper.GetString("server.port")) - + // Log server starting with address log.Info().Str("address", addr).Msg("Starting HTTP server") // Start server diff --git a/search/web/bing.go b/search/web/bing.go index eb4f12d..2bbf90d 100644 --- a/search/web/bing.go +++ b/search/web/bing.go @@ -9,6 +9,7 @@ import ( var bingURL = urlMustParse("https://www.bing.com/search?count=10") +// Bing represents the Bing search engine type Bing struct { keyword string userAgent string @@ -18,29 +19,37 @@ type Bing struct { baseSel *goquery.Selection } +// SetKeyword sets the keyword for searching func (b *Bing) SetKeyword(keyword string) { b.keyword = keyword } +// SetPage sets the page number for searching func (b *Bing) SetPage(page int) { b.first = page * 10 } +// SetUserAgent sets the user agent to use for the request func (b *Bing) SetUserAgent(ua string) { b.userAgent = ua } +// Init runs requests for Bing search engine func (b *Bing) Init() error { + // Copy URL so it can be changed initURL := copyURL(bingURL) query := initURL.Query() + // Set query query.Set("q", b.keyword) if b.first > 0 { query.Set("first", strconv.Itoa(b.first)) } else { query.Set("first", "1") } + // Update URL query parameters initURL.RawQuery = query.Encode() + // Create new request for modified URL req, err := http.NewRequest( http.MethodGet, initURL.String(), @@ -49,17 +58,21 @@ func (b *Bing) Init() error { if err != nil { return err } + // If no user agent, use default if b.userAgent == "" { b.userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" } + // Set request user agent req.Header.Set("User-Agent", b.userAgent) + // Perform request res, err := http.DefaultClient.Do(req) if err != nil { return err } defer res.Body.Close() + // Create new goquery document doc, err := goquery.NewDocumentFromReader(res.Body) if err != nil { return err @@ -70,6 +83,7 @@ func (b *Bing) Init() error { return nil } +// Each runs eachCb with the index of each search result func (b *Bing) Each(eachCb func(int) error) error { for i := 0; i < b.baseSel.Length(); i++ { err := eachCb(i) @@ -80,18 +94,22 @@ func (b *Bing) Each(eachCb func(int) error) error { return nil } +// Title returns the title of the search result corresponding to i func (b *Bing) Title(i int) (string, error) { return get(b.baseSel, i).ChildrenFiltered("h2").Children().First().Text(), nil } +// Link returns the link to the search result corresponding to i func (b *Bing) Link(i int) (string, error) { return get(b.baseSel, i).ChildrenFiltered("h2").Children().First().AttrOr("href", ""), nil } +// Desc returns the description of the search result corresponding to i func (b *Bing) Desc(i int) (string, error) { return get(b.baseSel, i).ChildrenFiltered(".b_caption").Children().Last().Text(), nil } +// Name returns "bing" func (b *Bing) Name() string { return "bing" } diff --git a/search/web/ddg.go b/search/web/ddg.go index af540e5..4f9b53b 100644 --- a/search/web/ddg.go +++ b/search/web/ddg.go @@ -12,6 +12,7 @@ var ddgURL = urlMustParse("https://html.duckduckgo.com/html") const uddgPrefix = "//duckduckgo.com/l/?uddg=" +// DDG represents the DuckDuckGo search engine type DDG struct { keyword string userAgent string @@ -21,28 +22,37 @@ type DDG struct { baseSel *goquery.Selection } +// SetKeyword sets the keyword for searching func (d *DDG) SetKeyword(keyword string) { d.keyword = keyword } +// SetPage sets the page number for searching func (d *DDG) SetPage(page int) { d.page = page * 30 } +// SetUserAgent sets the user agent for the request func (d *DDG) SetUserAgent(ua string) { - d.userAgent = "Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10" //ua + d.userAgent = ua } +// Init runs requests for the DuckDuckGo search engine func (d *DDG) Init() error { + // Copy URL so that it can be changed initURL := copyURL(ddgURL) + // Get query parameters query := initURL.Query() + // Set query query.Set("q", d.keyword) if d.page > 0 { query.Set("s", strconv.Itoa(d.page)) query.Set("dc", strconv.Itoa(d.page+1)) } + // Update URL query initURL.RawQuery = query.Encode() + // Create new request for modified URL req, err := http.NewRequest( http.MethodGet, initURL.String(), @@ -51,17 +61,21 @@ func (d *DDG) Init() error { if err != nil { return err } + // If user agent empty, use default if d.userAgent == "" { d.userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" } + // Set user agent of request req.Header.Set("User-Agent", d.userAgent) + // Perform request res, err := http.DefaultClient.Do(req) if err != nil { return err } defer res.Body.Close() + // Create goquery document from reader doc, err := goquery.NewDocumentFromReader(res.Body) if err != nil { return err @@ -72,6 +86,7 @@ func (d *DDG) Init() error { return nil } +// Each runs eachCb with the index of each search result func (d *DDG) Each(eachCb func(int) error) error { for i := 0; i < d.baseSel.Length(); i++ { err := eachCb(i) @@ -82,10 +97,12 @@ func (d *DDG) Each(eachCb func(int) error) error { return nil } +// Title returns the title of the search result corresponding to i func (d *DDG) Title(i int) (string, error) { return strings.TrimSpace(get(d.baseSel, i).Children().First().ChildrenFiltered("h2").Text()), nil } +// Link returns the link to the search result corresponding to i func (d *DDG) Link(i int) (string, error) { link := get(d.baseSel, i).Children().First().ChildrenFiltered("a").AttrOr("href", "") if strings.HasPrefix(link, uddgPrefix) { @@ -94,10 +111,12 @@ func (d *DDG) Link(i int) (string, error) { return link, nil } +// Desc returns the description of the search result corresponding to i func (d *DDG) Desc(i int) (string, error) { return get(d.baseSel, i).Children().First().ChildrenFiltered("a").Text(), nil } +// Name returns "ddg" func (d *DDG) Name() string { return "ddg" } diff --git a/search/web/google.go b/search/web/google.go index 9302490..098da6c 100644 --- a/search/web/google.go +++ b/search/web/google.go @@ -10,6 +10,7 @@ import ( var googleURL = urlMustParse("https://www.google.com/search") +// Google represents the Google search engine type Google struct { keyword string userAgent string @@ -19,24 +20,35 @@ type Google struct { baseSel *goquery.Selection } +// SetKeyword sets the keyword for searching func (g *Google) SetKeyword(keyword string) { g.keyword = keyword } +// SetPage sets the page number for searching func (g *Google) SetPage(page int) { g.page = page * 10 } +// SetUserAgent sets the user agent for the request func (g *Google) SetUserAgent(ua string) { g.userAgent = ua } +// Init runs requests for the Google search engine func (g *Google) Init() error { + // Copy URL so that it can be changed initURL := copyURL(googleURL) + // Get query parameters query := initURL.Query() + // Set query query.Set("q", g.keyword) + // Set starting result (page number) query.Set("start", strconv.Itoa(g.page)) + // Update URL query initURL.RawQuery = query.Encode() + + // Create new request for modified URL req, err := http.NewRequest( http.MethodGet, initURL.String(), @@ -45,17 +57,21 @@ func (g *Google) Init() error { if err != nil { return err } + // If user agent empty, use default if g.userAgent == "" { g.userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" } + // Set user agent of request req.Header.Set("User-Agent", g.userAgent) + // Perform request res, err := http.DefaultClient.Do(req) if err != nil { return err } defer res.Body.Close() + // Create goquery document from reader doc, err := goquery.NewDocumentFromReader(res.Body) if err != nil { return err @@ -66,6 +82,7 @@ func (g *Google) Init() error { return nil } +// Each runs eachCb with the index of each search result func (g *Google) Each(eachCb func(int) error) error { for i := 0; i < g.baseSel.Length(); i++ { err := eachCb(i) @@ -76,31 +93,38 @@ func (g *Google) Each(eachCb func(int) error) error { return nil } +// Title returns the title of the search result corresponding to i func (g *Google) Title(i int) (string, error) { return get(g.baseSel, i).Text(), nil } +// Link returns the link to the search result corresponding to i func (g *Google) Link(i int) (string, error) { return get(g.baseSel, i).Parent().AttrOr("href", ""), nil } +// Desc returns the description of the search result corresponding to i func (g *Google) Desc(i int) (string, error) { return get(g.baseSel, i).Parent().Parent().Next().Text(), nil } +// Name returns "google" func (g *Google) Name() string { return "google" } +// get gets an element and given index from given selection func get(sel *goquery.Selection, i int) *goquery.Selection { return sel.Slice(i, i+1) } +// Parse url ignoring error func urlMustParse(urlStr string) *url.URL { out, _ := url.Parse(urlStr) return out } +// copyURL makes a copy of the url and returns it func copyURL(orig *url.URL) *url.URL { newURL := new(url.URL) *newURL = *orig diff --git a/search/web/search.go b/search/web/search.go index f1574d2..c4006af 100644 --- a/search/web/search.go +++ b/search/web/search.go @@ -13,6 +13,7 @@ func init() { http.DefaultClient.Timeout = 5 * time.Second } +// Result represents a search result type Result struct { Title string Link string @@ -21,6 +22,7 @@ type Result struct { Rank int } +// Engine represents a search engine for web results (not images, shopping, erc.) type Engine interface { // Set search keyword for engine SetKeyword(string) @@ -51,74 +53,95 @@ type Engine interface { Name() string } +// Options represents search options type Options struct { Keyword string UserAgent string Page int } +// Search searches the given engines concurrently and returns the results func Search(opts Options, engines ...Engine) ([]*Result, error) { var outMtx sync.Mutex var out []*Result + // Create new error group wg := errgroup.Group{} + // For every engine for index, engine := range engines { + // Copy index and engine (for goroutine) curIndex, curEngine := index, engine wg.Go(func() error { - + // Set options curEngine.SetKeyword(opts.Keyword) curEngine.SetUserAgent(opts.UserAgent) curEngine.SetPage(opts.Page) + // Attempt to init engine if err := curEngine.Init(); err != nil { return err } + // For each result err := curEngine.Each(func(i int) error { + // Get result link link, err := curEngine.Link(i) if err != nil { return err } + // Calculate result rank rank := (curIndex * 100) + i + // Check if result exists index, exists := linkExists(out, link) + // If result already exists if exists { + // Add engine to the existing result out[index].Engines = append(out[index].Engines, curEngine.Name()) + // If the rank is higher than the old one, update it if rank < out[index].Rank { out[index].Rank = rank } return nil } + // Get result title title, err := curEngine.Title(i) if err != nil { return err } + // Get result description desc, err := curEngine.Desc(i) if err != nil { return err } + // If title, link, or description empty, ignore if title == "" || link == "" || desc == "" { return nil } + // If length of description, truncate if len(desc) > 500 { desc = desc[:500] + "..." } + // Create result struct result := &Result{ - Title: title, - Link: link, - Desc: desc, - Rank: rank, + Title: title, + Link: link, + Desc: desc, + Rank: rank, + Engines: []string{curEngine.Name()}, } - result.Engines = append(result.Engines, curEngine.Name()) + // Lock out mutex outMtx.Lock() + // Add result to slice out = append(out, result) + // Unlock out mutex outMtx.Unlock() return nil @@ -127,6 +150,7 @@ func Search(opts Options, engines ...Engine) ([]*Result, error) { return err } + // Sort slice by rank sort.Slice(out, func(i, j int) bool { return out[i].Rank < out[j].Rank }) @@ -134,6 +158,7 @@ func Search(opts Options, engines ...Engine) ([]*Result, error) { }) } + // Wait for error group if err := wg.Wait(); err != nil { return out, err } @@ -141,9 +166,13 @@ func Search(opts Options, engines ...Engine) ([]*Result, error) { return out, nil } +// linkExists checks if a link exists in the results func linkExists(results []*Result, link string) (int, bool) { + // For every result for index, result := range results { + // If link is the same as provided if result.Link == link { + // Return index with true return index, true } }