lure
/
lure
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases 13 Activity

Normalize URL for caching
ci/woodpecker/push/woodpecker Pipeline was successful Details

This commit is contained in:
Elara 2023-01-29 13:42:23 -08:00
parent 192e3e8a4f
commit 0ece536810
3 changed files with 30 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -4,6 +4,7 @@ go 1.18
require (
github.com/AlecAivazis/survey/v2 v2.3.6
github.com/PuerkitoBio/purell v1.2.0
github.com/alecthomas/chroma/v2 v2.4.0
github.com/charmbracelet/bubbles v0.14.0
github.com/charmbracelet/bubbletea v0.23.1

2
go.sum
View File

@ -23,6 +23,8 @@ github.com/ProtonMail/go-crypto v0.0.0-20210512092938-c05353c2d58c h1:bNpaLLv2Y4
github.com/ProtonMail/go-crypto v0.0.0-20210512092938-c05353c2d58c/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
github.com/ProtonMail/go-mime v0.0.0-20220302105931-303f85f7fe0f h1:CGq7OieOz3wyQJ1fO8S0eO9TCW1JyvLrf8fhzz1i8ko=
github.com/ProtonMail/gopenpgp/v2 v2.2.2 h1:u2m7xt+CZWj88qK1UUNBoXeJCFJwJCZ/Ff4ymGoxEXs=
github.com/PuerkitoBio/purell v1.2.0 h1:/Jdm5QfyM8zdlqT6WVZU4cfP23sot6CEHA4CS49Ezig=
github.com/PuerkitoBio/purell v1.2.0/go.mod h1:OhLRTaaIzhvIyofkJfB24gokC7tM42Px5UhoT32THBk=
github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
github.com/alecthomas/assert/v2 v2.2.0 h1:f6L/b7KE2bfA+9O4FL3CM/xJccDEwPVYd5fALBiuwvw=

28
internal/dl/dl.go
View File

@ -9,6 +9,7 @@ import (
"os"
"path/filepath"
"github.com/PuerkitoBio/purell"
"github.com/vmihailenco/msgpack/v5"
"go.arsenm.dev/logger/log"
"go.arsenm.dev/lure/internal/dlcache"
@ -88,6 +89,12 @@ type UpdatingDownloader interface {
// it downloads the source to a new cache directory and links it
// to the destination.
func Download(ctx context.Context, opts Options) (err error) {
normalized, err := normalizeURL(opts.URL)
if err != nil {
return err
}
opts.URL = normalized
d := getDownloader(opts.URL)
if opts.CacheDisabled {
@ -232,7 +239,7 @@ func handleCache(cacheDir, dest string, t Type) (bool, error) {
// hard links for each file from the src directory to the
// dest directory. If it encounters a directory, it will
// create a directory with the same name and permissions
// in the dest directory, because hard links cannot be
// in the dest directory, because hard links cannot be
// created for directories.
func linkDir(src, dest string) error {
return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
@ -266,3 +273,22 @@ func getDownloader(u string) Downloader {
}
return nil
}
// normalizeURL normalizes a URL string, so that insignificant
// don't change the hash.
func normalizeURL(u string) (string, error) {
const normalizationFlags = purell.FlagRemoveTrailingSlash |
purell.FlagRemoveDefaultPort |
purell.FlagLowercaseHost |
purell.FlagLowercaseScheme |
purell.FlagRemoveDuplicateSlashes |
purell.FlagRemoveFragment |
purell.FlagRemoveUnnecessaryHostDots |
purell.FlagSortQuery |
purell.FlagDecodeHexHost |
purell.FlagDecodeOctalHost |
purell.FlagDecodeUnnecessaryEscapes |
purell.FlagRemoveEmptyPortSeparator
return purell.NormalizeURLString(u, normalizationFlags)
}