diff --git a/internal/dl/dl.go b/internal/dl/dl.go index 787f3ac..94f19f4 100644 --- a/internal/dl/dl.go +++ b/internal/dl/dl.go @@ -1,3 +1,5 @@ +// Package dl contains abstractions for downloadingfiles and directories +// from various sources. package dl import ( @@ -14,13 +16,18 @@ import ( const manifestFileName = ".lure_cache_manifest" +// ErrChecksumMismatch occurs when the checksum of a downloaded file +// does not match the expected checksum provided in the Options struct. var ErrChecksumMismatch = errors.New("dl: checksums did not match") +// Downloaders contains all the downloaders in the order in which +// they should be checked var Downloaders = []Downloader{ GitDownloader{}, FileDownloader{}, } +// Type represents the type of download (file or directory) type Type uint8 const ( @@ -38,6 +45,8 @@ func (t Type) String() string { return "" } +// Options contains the options for downloading +// files and directories type Options struct { SHA256 []byte Name string @@ -48,6 +57,9 @@ type Options struct { Progress io.Writer } +// Manifest holds information about the type and name +// of a downloaded file or directory. It is stored inside +// each cache directory for later use. type Manifest struct { Type Type Name string @@ -59,11 +71,22 @@ type Downloader interface { Download(Options) (Type, string, error) } +// UpdatingDownloader extends the Downloader interface +// with an Update method for protocols such as git, which +// allow for incremental updates without changing the URL. type UpdatingDownloader interface { Downloader Update(Options) (bool, error) } +// Download downloads a file or directory using the specified options. +// It first gets the appropriate downloader for the URL, then checks +// if caching is enabled. If caching is enabled, it attempts to get +// the cache directory for the URL and update it if necessary. +// If the source is found in the cache, it links it to the destination +// using hard links. If the source is not found in the cache, +// it downloads the source to a new cache directory and links it +// to the destination. func Download(ctx context.Context, opts Options) (err error) { d := getDownloader(opts.URL) @@ -144,6 +167,7 @@ func Download(ctx context.Context, opts Options) (err error) { return err } +// writeManifest writes the manifest to the specified cache directory. func writeManifest(cacheDir string, m Manifest) error { fl, err := os.Create(filepath.Join(cacheDir, manifestFileName)) if err != nil { @@ -153,6 +177,7 @@ func writeManifest(cacheDir string, m Manifest) error { return msgpack.NewEncoder(fl).Encode(m) } +// getManifest reads the manifest from the specified cache directory. func getManifest(cacheDir string) (m Manifest, err error) { fl, err := os.Open(filepath.Join(cacheDir, manifestFileName)) if err != nil { @@ -164,6 +189,7 @@ func getManifest(cacheDir string) (m Manifest, err error) { return } +// handleCache links the cache directory or a file within it to the destination func handleCache(cacheDir, dest string, t Type) (bool, error) { switch t { case TypeFile: @@ -202,6 +228,12 @@ func handleCache(cacheDir, dest string, t Type) (bool, error) { return false, nil } +// linkDir recursively walks through a directory, creating +// hard links for each file from the src directory to the +// dest directory. If it encounters a directory, it will +// create a directory with the same name and permissions +// in the dest directory, because hard links cannot be +// created for directories. func linkDir(src, dest string) error { return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { if err != nil { diff --git a/internal/dl/file.go b/internal/dl/file.go index 958a6f0..f154a57 100644 --- a/internal/dl/file.go +++ b/internal/dl/file.go @@ -18,20 +18,22 @@ import ( "go.arsenm.dev/lure/internal/shutils" ) +// FileDownloader downloads files using HTTP type FileDownloader struct{} +// Name always returns "file" func (FileDownloader) Name() string { return "file" } -func (FileDownloader) Type() Type { - return TypeFile -} - +// MatchURL always returns true, as FileDownloader +// is used as a fallback if nothing else matches func (FileDownloader) MatchURL(string) bool { return true } +// Download downloads a file using HTTP. If the file is +// compressed using a supported format, it will be extracted func (FileDownloader) Download(opts Options) (Type, string, error) { res, err := http.Get(opts.URL) if err != nil { @@ -115,6 +117,7 @@ func (FileDownloader) Download(opts Options) (Type, string, error) { return TypeDir, "", err } +// extractFile extracts an archive or decompresses a file func extractFile(r io.Reader, format archiver.Format, name string, opts Options) (err error) { fname := format.Name() @@ -185,6 +188,10 @@ func extractFile(r io.Reader, format archiver.Format, name string, opts Options) var cdHeaderRgx = regexp.MustCompile(`filename="(.+)"`) +// getFilename attempts to parse the Content-Disposition +// HTTP response header and extract a filename. If the +// header does not exist, it will use the last element +// of the path. func getFilename(res *http.Response) (name string) { cd := res.Header.Get("Content-Disposition") matches := cdHeaderRgx.FindStringSubmatch(cd) diff --git a/internal/dl/git.go b/internal/dl/git.go index fa54b3d..26a6016 100644 --- a/internal/dl/git.go +++ b/internal/dl/git.go @@ -11,20 +11,22 @@ import ( "github.com/go-git/go-git/v5/plumbing" ) +// GitDownloader downloads Git repositories type GitDownloader struct{} +// Name always returns "git" func (GitDownloader) Name() string { return "git" } -func (GitDownloader) Type() Type { - return TypeDir -} - +// MatchURL matches any URLs that start with "git+" func (GitDownloader) MatchURL(u string) bool { return strings.HasPrefix(u, "git+") } +// Download uses git to clone the repository from the specified URL. +// It allows specifying the revision, depth and recursion options +// via query string func (GitDownloader) Download(opts Options) (Type, string, error) { u, err := url.Parse(opts.URL) if err != nil { @@ -92,6 +94,11 @@ func (GitDownloader) Download(opts Options) (Type, string, error) { return TypeDir, name, nil } +// Update uses git to pull the repository and update it +// to the latest revision. It allows specifying the depth +// and recursion options via query string. It returns +// true if update was successful and false if the +// repository is already up-to-date func (GitDownloader) Update(opts Options) (bool, error) { u, err := url.Parse(opts.URL) if err != nil { diff --git a/internal/dlcache/dlcache.go b/internal/dlcache/dlcache.go index 96eb7a5..6fa0506 100644 --- a/internal/dlcache/dlcache.go +++ b/internal/dlcache/dlcache.go @@ -10,8 +10,12 @@ import ( "go.arsenm.dev/lure/internal/config" ) +// BasePath stores the base path to the download cache var BasePath = filepath.Join(config.CacheDir, "dl") +// New creates a new directory with the given ID in the cache. +// If a directory with the same ID already exists, +// it will be deleted before creating a new one. func New(id string) (string, error) { h, err := hashID(id) if err != nil { @@ -35,6 +39,11 @@ func New(id string) (string, error) { return itemPath, nil } +// Get checks if an entry with the given ID +// already exists in the cache, and if so, +// returns the directory and true. If it +// does not exist, it returns an empty string +// and false. func Get(id string) (string, bool) { h, err := hashID(id) if err != nil { @@ -50,6 +59,9 @@ func Get(id string) (string, bool) { return itemPath, true } +// hashID hashes the input ID with SHA1 +// and returns the hex string of the hashed +// ID. func hashID(id string) (string, error) { h := sha1.New() _, err := io.WriteString(h, id)