Add comments for internal/{dl,dlcache}
ci/woodpecker/push/woodpecker Pipeline was successful
Details
ci/woodpecker/push/woodpecker Pipeline was successful
Details
This commit is contained in:
parent
163ad12575
commit
fa4604c26c
|
@ -1,3 +1,5 @@
|
|||
// Package dl contains abstractions for downloadingfiles and directories
|
||||
// from various sources.
|
||||
package dl
|
||||
|
||||
import (
|
||||
|
@ -14,13 +16,18 @@ import (
|
|||
|
||||
const manifestFileName = ".lure_cache_manifest"
|
||||
|
||||
// ErrChecksumMismatch occurs when the checksum of a downloaded file
|
||||
// does not match the expected checksum provided in the Options struct.
|
||||
var ErrChecksumMismatch = errors.New("dl: checksums did not match")
|
||||
|
||||
// Downloaders contains all the downloaders in the order in which
|
||||
// they should be checked
|
||||
var Downloaders = []Downloader{
|
||||
GitDownloader{},
|
||||
FileDownloader{},
|
||||
}
|
||||
|
||||
// Type represents the type of download (file or directory)
|
||||
type Type uint8
|
||||
|
||||
const (
|
||||
|
@ -38,6 +45,8 @@ func (t Type) String() string {
|
|||
return "<unknown>"
|
||||
}
|
||||
|
||||
// Options contains the options for downloading
|
||||
// files and directories
|
||||
type Options struct {
|
||||
SHA256 []byte
|
||||
Name string
|
||||
|
@ -48,6 +57,9 @@ type Options struct {
|
|||
Progress io.Writer
|
||||
}
|
||||
|
||||
// Manifest holds information about the type and name
|
||||
// of a downloaded file or directory. It is stored inside
|
||||
// each cache directory for later use.
|
||||
type Manifest struct {
|
||||
Type Type
|
||||
Name string
|
||||
|
@ -59,11 +71,22 @@ type Downloader interface {
|
|||
Download(Options) (Type, string, error)
|
||||
}
|
||||
|
||||
// UpdatingDownloader extends the Downloader interface
|
||||
// with an Update method for protocols such as git, which
|
||||
// allow for incremental updates without changing the URL.
|
||||
type UpdatingDownloader interface {
|
||||
Downloader
|
||||
Update(Options) (bool, error)
|
||||
}
|
||||
|
||||
// Download downloads a file or directory using the specified options.
|
||||
// It first gets the appropriate downloader for the URL, then checks
|
||||
// if caching is enabled. If caching is enabled, it attempts to get
|
||||
// the cache directory for the URL and update it if necessary.
|
||||
// If the source is found in the cache, it links it to the destination
|
||||
// using hard links. If the source is not found in the cache,
|
||||
// it downloads the source to a new cache directory and links it
|
||||
// to the destination.
|
||||
func Download(ctx context.Context, opts Options) (err error) {
|
||||
d := getDownloader(opts.URL)
|
||||
|
||||
|
@ -144,6 +167,7 @@ func Download(ctx context.Context, opts Options) (err error) {
|
|||
return err
|
||||
}
|
||||
|
||||
// writeManifest writes the manifest to the specified cache directory.
|
||||
func writeManifest(cacheDir string, m Manifest) error {
|
||||
fl, err := os.Create(filepath.Join(cacheDir, manifestFileName))
|
||||
if err != nil {
|
||||
|
@ -153,6 +177,7 @@ func writeManifest(cacheDir string, m Manifest) error {
|
|||
return msgpack.NewEncoder(fl).Encode(m)
|
||||
}
|
||||
|
||||
// getManifest reads the manifest from the specified cache directory.
|
||||
func getManifest(cacheDir string) (m Manifest, err error) {
|
||||
fl, err := os.Open(filepath.Join(cacheDir, manifestFileName))
|
||||
if err != nil {
|
||||
|
@ -164,6 +189,7 @@ func getManifest(cacheDir string) (m Manifest, err error) {
|
|||
return
|
||||
}
|
||||
|
||||
// handleCache links the cache directory or a file within it to the destination
|
||||
func handleCache(cacheDir, dest string, t Type) (bool, error) {
|
||||
switch t {
|
||||
case TypeFile:
|
||||
|
@ -202,6 +228,12 @@ func handleCache(cacheDir, dest string, t Type) (bool, error) {
|
|||
return false, nil
|
||||
}
|
||||
|
||||
// linkDir recursively walks through a directory, creating
|
||||
// hard links for each file from the src directory to the
|
||||
// dest directory. If it encounters a directory, it will
|
||||
// create a directory with the same name and permissions
|
||||
// in the dest directory, because hard links cannot be
|
||||
// created for directories.
|
||||
func linkDir(src, dest string) error {
|
||||
return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
|
|
|
@ -18,20 +18,22 @@ import (
|
|||
"go.arsenm.dev/lure/internal/shutils"
|
||||
)
|
||||
|
||||
// FileDownloader downloads files using HTTP
|
||||
type FileDownloader struct{}
|
||||
|
||||
// Name always returns "file"
|
||||
func (FileDownloader) Name() string {
|
||||
return "file"
|
||||
}
|
||||
|
||||
func (FileDownloader) Type() Type {
|
||||
return TypeFile
|
||||
}
|
||||
|
||||
// MatchURL always returns true, as FileDownloader
|
||||
// is used as a fallback if nothing else matches
|
||||
func (FileDownloader) MatchURL(string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Download downloads a file using HTTP. If the file is
|
||||
// compressed using a supported format, it will be extracted
|
||||
func (FileDownloader) Download(opts Options) (Type, string, error) {
|
||||
res, err := http.Get(opts.URL)
|
||||
if err != nil {
|
||||
|
@ -115,6 +117,7 @@ func (FileDownloader) Download(opts Options) (Type, string, error) {
|
|||
return TypeDir, "", err
|
||||
}
|
||||
|
||||
// extractFile extracts an archive or decompresses a file
|
||||
func extractFile(r io.Reader, format archiver.Format, name string, opts Options) (err error) {
|
||||
fname := format.Name()
|
||||
|
||||
|
@ -185,6 +188,10 @@ func extractFile(r io.Reader, format archiver.Format, name string, opts Options)
|
|||
|
||||
var cdHeaderRgx = regexp.MustCompile(`filename="(.+)"`)
|
||||
|
||||
// getFilename attempts to parse the Content-Disposition
|
||||
// HTTP response header and extract a filename. If the
|
||||
// header does not exist, it will use the last element
|
||||
// of the path.
|
||||
func getFilename(res *http.Response) (name string) {
|
||||
cd := res.Header.Get("Content-Disposition")
|
||||
matches := cdHeaderRgx.FindStringSubmatch(cd)
|
||||
|
|
|
@ -11,20 +11,22 @@ import (
|
|||
"github.com/go-git/go-git/v5/plumbing"
|
||||
)
|
||||
|
||||
// GitDownloader downloads Git repositories
|
||||
type GitDownloader struct{}
|
||||
|
||||
// Name always returns "git"
|
||||
func (GitDownloader) Name() string {
|
||||
return "git"
|
||||
}
|
||||
|
||||
func (GitDownloader) Type() Type {
|
||||
return TypeDir
|
||||
}
|
||||
|
||||
// MatchURL matches any URLs that start with "git+"
|
||||
func (GitDownloader) MatchURL(u string) bool {
|
||||
return strings.HasPrefix(u, "git+")
|
||||
}
|
||||
|
||||
// Download uses git to clone the repository from the specified URL.
|
||||
// It allows specifying the revision, depth and recursion options
|
||||
// via query string
|
||||
func (GitDownloader) Download(opts Options) (Type, string, error) {
|
||||
u, err := url.Parse(opts.URL)
|
||||
if err != nil {
|
||||
|
@ -92,6 +94,11 @@ func (GitDownloader) Download(opts Options) (Type, string, error) {
|
|||
return TypeDir, name, nil
|
||||
}
|
||||
|
||||
// Update uses git to pull the repository and update it
|
||||
// to the latest revision. It allows specifying the depth
|
||||
// and recursion options via query string. It returns
|
||||
// true if update was successful and false if the
|
||||
// repository is already up-to-date
|
||||
func (GitDownloader) Update(opts Options) (bool, error) {
|
||||
u, err := url.Parse(opts.URL)
|
||||
if err != nil {
|
||||
|
|
|
@ -10,8 +10,12 @@ import (
|
|||
"go.arsenm.dev/lure/internal/config"
|
||||
)
|
||||
|
||||
// BasePath stores the base path to the download cache
|
||||
var BasePath = filepath.Join(config.CacheDir, "dl")
|
||||
|
||||
// New creates a new directory with the given ID in the cache.
|
||||
// If a directory with the same ID already exists,
|
||||
// it will be deleted before creating a new one.
|
||||
func New(id string) (string, error) {
|
||||
h, err := hashID(id)
|
||||
if err != nil {
|
||||
|
@ -35,6 +39,11 @@ func New(id string) (string, error) {
|
|||
return itemPath, nil
|
||||
}
|
||||
|
||||
// Get checks if an entry with the given ID
|
||||
// already exists in the cache, and if so,
|
||||
// returns the directory and true. If it
|
||||
// does not exist, it returns an empty string
|
||||
// and false.
|
||||
func Get(id string) (string, bool) {
|
||||
h, err := hashID(id)
|
||||
if err != nil {
|
||||
|
@ -50,6 +59,9 @@ func Get(id string) (string, bool) {
|
|||
return itemPath, true
|
||||
}
|
||||
|
||||
// hashID hashes the input ID with SHA1
|
||||
// and returns the hex string of the hashed
|
||||
// ID.
|
||||
func hashID(id string) (string, error) {
|
||||
h := sha1.New()
|
||||
_, err := io.WriteString(h, id)
|
||||
|
|
Loading…
Reference in New Issue