|
|
|
@ -13,6 +13,7 @@ func init() { |
|
|
|
http.DefaultClient.Timeout = 5 * time.Second |
|
|
|
} |
|
|
|
|
|
|
|
// Result represents a search result
|
|
|
|
type Result struct { |
|
|
|
Title string |
|
|
|
Link string |
|
|
|
@ -21,6 +22,7 @@ type Result struct { |
|
|
|
Rank int |
|
|
|
} |
|
|
|
|
|
|
|
// Engine represents a search engine for web results (not images, shopping, erc.)
|
|
|
|
type Engine interface { |
|
|
|
// Set search keyword for engine
|
|
|
|
SetKeyword(string) |
|
|
|
@ -51,74 +53,95 @@ type Engine interface { |
|
|
|
Name() string |
|
|
|
} |
|
|
|
|
|
|
|
// Options represents search options
|
|
|
|
type Options struct { |
|
|
|
Keyword string |
|
|
|
UserAgent string |
|
|
|
Page int |
|
|
|
} |
|
|
|
|
|
|
|
// Search searches the given engines concurrently and returns the results
|
|
|
|
func Search(opts Options, engines ...Engine) ([]*Result, error) { |
|
|
|
var outMtx sync.Mutex |
|
|
|
var out []*Result |
|
|
|
|
|
|
|
// Create new error group
|
|
|
|
wg := errgroup.Group{} |
|
|
|
// For every engine
|
|
|
|
for index, engine := range engines { |
|
|
|
// Copy index and engine (for goroutine)
|
|
|
|
curIndex, curEngine := index, engine |
|
|
|
wg.Go(func() error { |
|
|
|
|
|
|
|
// Set options
|
|
|
|
curEngine.SetKeyword(opts.Keyword) |
|
|
|
curEngine.SetUserAgent(opts.UserAgent) |
|
|
|
curEngine.SetPage(opts.Page) |
|
|
|
|
|
|
|
// Attempt to init engine
|
|
|
|
if err := curEngine.Init(); err != nil { |
|
|
|
return err |
|
|
|
} |
|
|
|
|
|
|
|
// For each result
|
|
|
|
err := curEngine.Each(func(i int) error { |
|
|
|
// Get result link
|
|
|
|
link, err := curEngine.Link(i) |
|
|
|
if err != nil { |
|
|
|
return err |
|
|
|
} |
|
|
|
|
|
|
|
// Calculate result rank
|
|
|
|
rank := (curIndex * 100) + i |
|
|
|
|
|
|
|
// Check if result exists
|
|
|
|
index, exists := linkExists(out, link) |
|
|
|
// If result already exists
|
|
|
|
if exists { |
|
|
|
// Add engine to the existing result
|
|
|
|
out[index].Engines = append(out[index].Engines, curEngine.Name()) |
|
|
|
// If the rank is higher than the old one, update it
|
|
|
|
if rank < out[index].Rank { |
|
|
|
out[index].Rank = rank |
|
|
|
} |
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
// Get result title
|
|
|
|
title, err := curEngine.Title(i) |
|
|
|
if err != nil { |
|
|
|
return err |
|
|
|
} |
|
|
|
|
|
|
|
// Get result description
|
|
|
|
desc, err := curEngine.Desc(i) |
|
|
|
if err != nil { |
|
|
|
return err |
|
|
|
} |
|
|
|
|
|
|
|
// If title, link, or description empty, ignore
|
|
|
|
if title == "" || link == "" || desc == "" { |
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
// If length of description, truncate
|
|
|
|
if len(desc) > 500 { |
|
|
|
desc = desc[:500] + "..." |
|
|
|
} |
|
|
|
|
|
|
|
// Create result struct
|
|
|
|
result := &Result{ |
|
|
|
Title: title, |
|
|
|
Link: link, |
|
|
|
Desc: desc, |
|
|
|
Rank: rank, |
|
|
|
Title: title, |
|
|
|
Link: link, |
|
|
|
Desc: desc, |
|
|
|
Rank: rank, |
|
|
|
Engines: []string{curEngine.Name()}, |
|
|
|
} |
|
|
|
result.Engines = append(result.Engines, curEngine.Name()) |
|
|
|
|
|
|
|
// Lock out mutex
|
|
|
|
outMtx.Lock() |
|
|
|
// Add result to slice
|
|
|
|
out = append(out, result) |
|
|
|
// Unlock out mutex
|
|
|
|
outMtx.Unlock() |
|
|
|
|
|
|
|
return nil |
|
|
|
@ -127,6 +150,7 @@ func Search(opts Options, engines ...Engine) ([]*Result, error) { |
|
|
|
return err |
|
|
|
} |
|
|
|
|
|
|
|
// Sort slice by rank
|
|
|
|
sort.Slice(out, func(i, j int) bool { |
|
|
|
return out[i].Rank < out[j].Rank |
|
|
|
}) |
|
|
|
@ -134,6 +158,7 @@ func Search(opts Options, engines ...Engine) ([]*Result, error) { |
|
|
|
}) |
|
|
|
} |
|
|
|
|
|
|
|
// Wait for error group
|
|
|
|
if err := wg.Wait(); err != nil { |
|
|
|
return out, err |
|
|
|
} |
|
|
|
@ -141,9 +166,13 @@ func Search(opts Options, engines ...Engine) ([]*Result, error) { |
|
|
|
return out, nil |
|
|
|
} |
|
|
|
|
|
|
|
// linkExists checks if a link exists in the results
|
|
|
|
func linkExists(results []*Result, link string) (int, bool) { |
|
|
|
// For every result
|
|
|
|
for index, result := range results { |
|
|
|
// If link is the same as provided
|
|
|
|
if result.Link == link { |
|
|
|
// Return index with true
|
|
|
|
return index, true |
|
|
|
} |
|
|
|
} |
|
|
|
|
Arsen Musayelyan
6 months ago