Add comments for internal/{dl,dlcache}
ci/woodpecker/push/woodpecker Pipeline was successful Details

This commit is contained in:
Arsen Musayelyan 2023-01-29 00:54:59 -08:00
parent 163ad12575
commit fa4604c26c
4 changed files with 66 additions and 8 deletions

View File

@ -1,3 +1,5 @@
// Package dl contains abstractions for downloadingfiles and directories
// from various sources.
package dl
import (
@ -14,13 +16,18 @@ import (
const manifestFileName = ".lure_cache_manifest"
// ErrChecksumMismatch occurs when the checksum of a downloaded file
// does not match the expected checksum provided in the Options struct.
var ErrChecksumMismatch = errors.New("dl: checksums did not match")
// Downloaders contains all the downloaders in the order in which
// they should be checked
var Downloaders = []Downloader{
GitDownloader{},
FileDownloader{},
}
// Type represents the type of download (file or directory)
type Type uint8
const (
@ -38,6 +45,8 @@ func (t Type) String() string {
return "<unknown>"
}
// Options contains the options for downloading
// files and directories
type Options struct {
SHA256 []byte
Name string
@ -48,6 +57,9 @@ type Options struct {
Progress io.Writer
}
// Manifest holds information about the type and name
// of a downloaded file or directory. It is stored inside
// each cache directory for later use.
type Manifest struct {
Type Type
Name string
@ -59,11 +71,22 @@ type Downloader interface {
Download(Options) (Type, string, error)
}
// UpdatingDownloader extends the Downloader interface
// with an Update method for protocols such as git, which
// allow for incremental updates without changing the URL.
type UpdatingDownloader interface {
Downloader
Update(Options) (bool, error)
}
// Download downloads a file or directory using the specified options.
// It first gets the appropriate downloader for the URL, then checks
// if caching is enabled. If caching is enabled, it attempts to get
// the cache directory for the URL and update it if necessary.
// If the source is found in the cache, it links it to the destination
// using hard links. If the source is not found in the cache,
// it downloads the source to a new cache directory and links it
// to the destination.
func Download(ctx context.Context, opts Options) (err error) {
d := getDownloader(opts.URL)
@ -144,6 +167,7 @@ func Download(ctx context.Context, opts Options) (err error) {
return err
}
// writeManifest writes the manifest to the specified cache directory.
func writeManifest(cacheDir string, m Manifest) error {
fl, err := os.Create(filepath.Join(cacheDir, manifestFileName))
if err != nil {
@ -153,6 +177,7 @@ func writeManifest(cacheDir string, m Manifest) error {
return msgpack.NewEncoder(fl).Encode(m)
}
// getManifest reads the manifest from the specified cache directory.
func getManifest(cacheDir string) (m Manifest, err error) {
fl, err := os.Open(filepath.Join(cacheDir, manifestFileName))
if err != nil {
@ -164,6 +189,7 @@ func getManifest(cacheDir string) (m Manifest, err error) {
return
}
// handleCache links the cache directory or a file within it to the destination
func handleCache(cacheDir, dest string, t Type) (bool, error) {
switch t {
case TypeFile:
@ -202,6 +228,12 @@ func handleCache(cacheDir, dest string, t Type) (bool, error) {
return false, nil
}
// linkDir recursively walks through a directory, creating
// hard links for each file from the src directory to the
// dest directory. If it encounters a directory, it will
// create a directory with the same name and permissions
// in the dest directory, because hard links cannot be
// created for directories.
func linkDir(src, dest string) error {
return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
if err != nil {

View File

@ -18,20 +18,22 @@ import (
"go.arsenm.dev/lure/internal/shutils"
)
// FileDownloader downloads files using HTTP
type FileDownloader struct{}
// Name always returns "file"
func (FileDownloader) Name() string {
return "file"
}
func (FileDownloader) Type() Type {
return TypeFile
}
// MatchURL always returns true, as FileDownloader
// is used as a fallback if nothing else matches
func (FileDownloader) MatchURL(string) bool {
return true
}
// Download downloads a file using HTTP. If the file is
// compressed using a supported format, it will be extracted
func (FileDownloader) Download(opts Options) (Type, string, error) {
res, err := http.Get(opts.URL)
if err != nil {
@ -115,6 +117,7 @@ func (FileDownloader) Download(opts Options) (Type, string, error) {
return TypeDir, "", err
}
// extractFile extracts an archive or decompresses a file
func extractFile(r io.Reader, format archiver.Format, name string, opts Options) (err error) {
fname := format.Name()
@ -185,6 +188,10 @@ func extractFile(r io.Reader, format archiver.Format, name string, opts Options)
var cdHeaderRgx = regexp.MustCompile(`filename="(.+)"`)
// getFilename attempts to parse the Content-Disposition
// HTTP response header and extract a filename. If the
// header does not exist, it will use the last element
// of the path.
func getFilename(res *http.Response) (name string) {
cd := res.Header.Get("Content-Disposition")
matches := cdHeaderRgx.FindStringSubmatch(cd)

View File

@ -11,20 +11,22 @@ import (
"github.com/go-git/go-git/v5/plumbing"
)
// GitDownloader downloads Git repositories
type GitDownloader struct{}
// Name always returns "git"
func (GitDownloader) Name() string {
return "git"
}
func (GitDownloader) Type() Type {
return TypeDir
}
// MatchURL matches any URLs that start with "git+"
func (GitDownloader) MatchURL(u string) bool {
return strings.HasPrefix(u, "git+")
}
// Download uses git to clone the repository from the specified URL.
// It allows specifying the revision, depth and recursion options
// via query string
func (GitDownloader) Download(opts Options) (Type, string, error) {
u, err := url.Parse(opts.URL)
if err != nil {
@ -92,6 +94,11 @@ func (GitDownloader) Download(opts Options) (Type, string, error) {
return TypeDir, name, nil
}
// Update uses git to pull the repository and update it
// to the latest revision. It allows specifying the depth
// and recursion options via query string. It returns
// true if update was successful and false if the
// repository is already up-to-date
func (GitDownloader) Update(opts Options) (bool, error) {
u, err := url.Parse(opts.URL)
if err != nil {

View File

@ -10,8 +10,12 @@ import (
"go.arsenm.dev/lure/internal/config"
)
// BasePath stores the base path to the download cache
var BasePath = filepath.Join(config.CacheDir, "dl")
// New creates a new directory with the given ID in the cache.
// If a directory with the same ID already exists,
// it will be deleted before creating a new one.
func New(id string) (string, error) {
h, err := hashID(id)
if err != nil {
@ -35,6 +39,11 @@ func New(id string) (string, error) {
return itemPath, nil
}
// Get checks if an entry with the given ID
// already exists in the cache, and if so,
// returns the directory and true. If it
// does not exist, it returns an empty string
// and false.
func Get(id string) (string, bool) {
h, err := hashID(id)
if err != nil {
@ -50,6 +59,9 @@ func Get(id string) (string, bool) {
return itemPath, true
}
// hashID hashes the input ID with SHA1
// and returns the hex string of the hashed
// ID.
func hashID(id string) (string, error) {
h := sha1.New()
_, err := io.WriteString(h, id)