Normalize URL for caching
ci/woodpecker/push/woodpecker Pipeline was successful
Details
ci/woodpecker/push/woodpecker Pipeline was successful
Details
This commit is contained in:
parent
192e3e8a4f
commit
0ece536810
1
go.mod
1
go.mod
|
@ -4,6 +4,7 @@ go 1.18
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/AlecAivazis/survey/v2 v2.3.6
|
github.com/AlecAivazis/survey/v2 v2.3.6
|
||||||
|
github.com/PuerkitoBio/purell v1.2.0
|
||||||
github.com/alecthomas/chroma/v2 v2.4.0
|
github.com/alecthomas/chroma/v2 v2.4.0
|
||||||
github.com/charmbracelet/bubbles v0.14.0
|
github.com/charmbracelet/bubbles v0.14.0
|
||||||
github.com/charmbracelet/bubbletea v0.23.1
|
github.com/charmbracelet/bubbletea v0.23.1
|
||||||
|
|
2
go.sum
2
go.sum
|
@ -23,6 +23,8 @@ github.com/ProtonMail/go-crypto v0.0.0-20210512092938-c05353c2d58c h1:bNpaLLv2Y4
|
||||||
github.com/ProtonMail/go-crypto v0.0.0-20210512092938-c05353c2d58c/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
|
github.com/ProtonMail/go-crypto v0.0.0-20210512092938-c05353c2d58c/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo=
|
||||||
github.com/ProtonMail/go-mime v0.0.0-20220302105931-303f85f7fe0f h1:CGq7OieOz3wyQJ1fO8S0eO9TCW1JyvLrf8fhzz1i8ko=
|
github.com/ProtonMail/go-mime v0.0.0-20220302105931-303f85f7fe0f h1:CGq7OieOz3wyQJ1fO8S0eO9TCW1JyvLrf8fhzz1i8ko=
|
||||||
github.com/ProtonMail/gopenpgp/v2 v2.2.2 h1:u2m7xt+CZWj88qK1UUNBoXeJCFJwJCZ/Ff4ymGoxEXs=
|
github.com/ProtonMail/gopenpgp/v2 v2.2.2 h1:u2m7xt+CZWj88qK1UUNBoXeJCFJwJCZ/Ff4ymGoxEXs=
|
||||||
|
github.com/PuerkitoBio/purell v1.2.0 h1:/Jdm5QfyM8zdlqT6WVZU4cfP23sot6CEHA4CS49Ezig=
|
||||||
|
github.com/PuerkitoBio/purell v1.2.0/go.mod h1:OhLRTaaIzhvIyofkJfB24gokC7tM42Px5UhoT32THBk=
|
||||||
github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
|
github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
|
||||||
github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
|
github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
|
||||||
github.com/alecthomas/assert/v2 v2.2.0 h1:f6L/b7KE2bfA+9O4FL3CM/xJccDEwPVYd5fALBiuwvw=
|
github.com/alecthomas/assert/v2 v2.2.0 h1:f6L/b7KE2bfA+9O4FL3CM/xJccDEwPVYd5fALBiuwvw=
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
|
"github.com/PuerkitoBio/purell"
|
||||||
"github.com/vmihailenco/msgpack/v5"
|
"github.com/vmihailenco/msgpack/v5"
|
||||||
"go.arsenm.dev/logger/log"
|
"go.arsenm.dev/logger/log"
|
||||||
"go.arsenm.dev/lure/internal/dlcache"
|
"go.arsenm.dev/lure/internal/dlcache"
|
||||||
|
@ -88,6 +89,12 @@ type UpdatingDownloader interface {
|
||||||
// it downloads the source to a new cache directory and links it
|
// it downloads the source to a new cache directory and links it
|
||||||
// to the destination.
|
// to the destination.
|
||||||
func Download(ctx context.Context, opts Options) (err error) {
|
func Download(ctx context.Context, opts Options) (err error) {
|
||||||
|
normalized, err := normalizeURL(opts.URL)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
opts.URL = normalized
|
||||||
|
|
||||||
d := getDownloader(opts.URL)
|
d := getDownloader(opts.URL)
|
||||||
|
|
||||||
if opts.CacheDisabled {
|
if opts.CacheDisabled {
|
||||||
|
@ -232,7 +239,7 @@ func handleCache(cacheDir, dest string, t Type) (bool, error) {
|
||||||
// hard links for each file from the src directory to the
|
// hard links for each file from the src directory to the
|
||||||
// dest directory. If it encounters a directory, it will
|
// dest directory. If it encounters a directory, it will
|
||||||
// create a directory with the same name and permissions
|
// create a directory with the same name and permissions
|
||||||
// in the dest directory, because hard links cannot be
|
// in the dest directory, because hard links cannot be
|
||||||
// created for directories.
|
// created for directories.
|
||||||
func linkDir(src, dest string) error {
|
func linkDir(src, dest string) error {
|
||||||
return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
|
return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
|
||||||
|
@ -266,3 +273,22 @@ func getDownloader(u string) Downloader {
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// normalizeURL normalizes a URL string, so that insignificant
|
||||||
|
// don't change the hash.
|
||||||
|
func normalizeURL(u string) (string, error) {
|
||||||
|
const normalizationFlags = purell.FlagRemoveTrailingSlash |
|
||||||
|
purell.FlagRemoveDefaultPort |
|
||||||
|
purell.FlagLowercaseHost |
|
||||||
|
purell.FlagLowercaseScheme |
|
||||||
|
purell.FlagRemoveDuplicateSlashes |
|
||||||
|
purell.FlagRemoveFragment |
|
||||||
|
purell.FlagRemoveUnnecessaryHostDots |
|
||||||
|
purell.FlagSortQuery |
|
||||||
|
purell.FlagDecodeHexHost |
|
||||||
|
purell.FlagDecodeOctalHost |
|
||||||
|
purell.FlagDecodeUnnecessaryEscapes |
|
||||||
|
purell.FlagRemoveEmptyPortSeparator
|
||||||
|
|
||||||
|
return purell.NormalizeURLString(u, normalizationFlags)
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue