/* * LURE - Linux User REpository * Copyright (C) 2023 Elara Musayelyan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ package build import ( "bytes" "context" "encoding/hex" "fmt" "io" "os" "path/filepath" "runtime" "slices" "strconv" "strings" _ "github.com/goreleaser/nfpm/v2/apk" _ "github.com/goreleaser/nfpm/v2/arch" _ "github.com/goreleaser/nfpm/v2/deb" _ "github.com/goreleaser/nfpm/v2/rpm" "github.com/goreleaser/nfpm/v2" "github.com/goreleaser/nfpm/v2/files" "lure.sh/lure/internal/cliutils" "lure.sh/lure/internal/config" "lure.sh/lure/internal/cpu" "lure.sh/lure/internal/db" "lure.sh/lure/internal/dl" "lure.sh/lure/internal/shutils" "lure.sh/lure/internal/shutils/decoder" "lure.sh/lure/internal/shutils/helpers" "lure.sh/lure/internal/types" "lure.sh/lure/pkg/distro" "lure.sh/lure/pkg/loggerctx" "lure.sh/lure/pkg/manager" "lure.sh/lure/pkg/repos" "mvdan.cc/sh/v3/expand" "mvdan.cc/sh/v3/interp" "mvdan.cc/sh/v3/syntax" ) // BuildPackage builds the script at the given path. It returns two slices. One contains the paths // to the built package(s), the other contains the names of the built package(s). func BuildPackage(ctx context.Context, opts types.BuildOpts) ([]string, []string, error) { log := loggerctx.From(ctx) info, err := distro.ParseOSRelease(ctx) if err != nil { return nil, nil, err } fl, err := parseScript(info, opts.Script) if err != nil { return nil, nil, err } // The first pass is just used to get variable values and runs before // the script is displayed, so it's restricted so as to prevent malicious // code from executing. vars, err := executeFirstPass(ctx, info, fl, opts.Script) if err != nil { return nil, nil, err } dirs := getDirs(ctx, vars, opts.Script) // If opts.Clean isn't set and we find the package already built, // just return it rather than rebuilding if !opts.Clean { builtPkgPath, ok, err := checkForBuiltPackage(opts.Manager, vars, getPkgFormat(opts.Manager), dirs.BaseDir) if err != nil { return nil, nil, err } if ok { return []string{builtPkgPath}, nil, err } } // Ask the user if they'd like to see the build script err = cliutils.PromptViewScript(ctx, opts.Script, vars.Name, config.Config(ctx).PagerStyle, opts.Interactive) if err != nil { log.Fatal("Failed to prompt user to view build script").Err(err).Send() } log.Info("Building package").Str("name", vars.Name).Str("version", vars.Version).Send() // The second pass will be used to execute the actual code, // so it's unrestricted. The script has already been displayed // to the user by this point, so it should be safe dec, err := executeSecondPass(ctx, info, fl, dirs) if err != nil { return nil, nil, err } // Get the installed packages on the system installed, err := opts.Manager.ListInstalled(nil) if err != nil { return nil, nil, err } cont, err := performChecks(ctx, vars, opts.Interactive, installed) if err != nil { return nil, nil, err } else if !cont { os.Exit(1) } // Prepare the directories for building err = prepareDirs(dirs) if err != nil { return nil, nil, err } buildDeps, err := installBuildDeps(ctx, vars, opts, installed) if err != nil { return nil, nil, err } err = installOptDeps(ctx, vars, opts, installed) if err != nil { return nil, nil, err } builtPaths, builtNames, repoDeps, err := buildLUREDeps(ctx, opts, vars) if err != nil { return nil, nil, err } log.Info("Downloading sources").Send() err = getSources(ctx, dirs, vars) if err != nil { return nil, nil, err } err = executeFunctions(ctx, dec, dirs, vars) if err != nil { return nil, nil, err } log.Info("Building package metadata").Str("name", vars.Name).Send() pkgInfo, err := buildPkgMetadata(vars, dirs, append(repoDeps, builtNames...)) if err != nil { return nil, nil, err } packager, err := nfpm.Get(getPkgFormat(opts.Manager)) if err != nil { return nil, nil, err } pkgName := packager.ConventionalFileName(pkgInfo) pkgPath := filepath.Join(dirs.BaseDir, pkgName) pkgFile, err := os.Create(pkgPath) if err != nil { return nil, nil, err } log.Info("Compressing package").Str("name", pkgName).Send() err = packager.Package(pkgInfo, pkgFile) if err != nil { return nil, nil, err } err = removeBuildDeps(ctx, buildDeps, opts) if err != nil { return nil, nil, err } // Add the path and name of the package we just built to the // appropriate slices pkgPaths := append(builtPaths, pkgPath) pkgNames := append(builtNames, vars.Name) // Remove any duplicates from the pkgPaths and pkgNames. // Duplicates can be introduced if several of the dependencies // depend on the same packages. pkgPaths = removeDuplicates(pkgPaths) pkgNames = removeDuplicates(pkgNames) return pkgPaths, pkgNames, nil } // parseScript parses the build script using the built-in bash implementation func parseScript(info *distro.OSRelease, script string) (*syntax.File, error) { fl, err := os.Open(script) if err != nil { return nil, err } defer fl.Close() file, err := syntax.NewParser().Parse(fl, "lure.sh") if err != nil { return nil, err } return file, nil } // executeFirstPass executes the parsed script in a restricted environment // to extract the build variables without executing any actual code. func executeFirstPass(ctx context.Context, info *distro.OSRelease, fl *syntax.File, script string) (*types.BuildVars, error) { scriptDir := filepath.Dir(script) env := createBuildEnvVars(info, types.Directories{ScriptDir: scriptDir}) runner, err := interp.New( interp.Env(expand.ListEnviron(env...)), interp.StdIO(os.Stdin, os.Stdout, os.Stderr), interp.ExecHandler(helpers.Restricted.ExecHandler(shutils.NopExec)), interp.ReadDirHandler(shutils.RestrictedReadDir(scriptDir)), interp.StatHandler(shutils.RestrictedStat(scriptDir)), interp.OpenHandler(shutils.RestrictedOpen(scriptDir)), ) if err != nil { return nil, err } err = runner.Run(ctx, fl) if err != nil { return nil, err } dec := decoder.New(info, runner) var vars types.BuildVars err = dec.DecodeVars(&vars) if err != nil { return nil, err } return &vars, nil } // getDirs returns the appropriate directories for the script func getDirs(ctx context.Context, vars *types.BuildVars, script string) types.Directories { baseDir := filepath.Join(config.GetPaths(ctx).PkgsDir, vars.Name) return types.Directories{ BaseDir: baseDir, SrcDir: filepath.Join(baseDir, "src"), PkgDir: filepath.Join(baseDir, "pkg"), ScriptDir: filepath.Dir(script), } } // executeSecondPass executes the build script for the second time, this time without any restrictions. // It returns a decoder that can be used to retrieve functions and variables from the script. func executeSecondPass(ctx context.Context, info *distro.OSRelease, fl *syntax.File, dirs types.Directories) (*decoder.Decoder, error) { env := createBuildEnvVars(info, dirs) runner, err := interp.New( interp.Env(expand.ListEnviron(env...)), interp.StdIO(os.Stdin, os.Stdout, os.Stderr), interp.ExecHandler(helpers.Helpers.ExecHandler(nil)), ) if err != nil { return nil, err } err = runner.Run(ctx, fl) if err != nil { return nil, err } return decoder.New(info, runner), nil } // prepareDirs prepares the directories for building. func prepareDirs(dirs types.Directories) error { err := os.RemoveAll(dirs.BaseDir) if err != nil { return err } err = os.MkdirAll(dirs.SrcDir, 0o755) if err != nil { return err } return os.MkdirAll(dirs.PkgDir, 0o755) } // performChecks checks various things on the system to ensure that the package can be installed. func performChecks(ctx context.Context, vars *types.BuildVars, interactive bool, installed map[string]string) (bool, error) { log := loggerctx.From(ctx) if !cpu.IsCompatibleWith(cpu.Arch(), vars.Architectures) { cont, err := cliutils.YesNoPrompt(ctx, "Your system's CPU architecture doesn't match this package. Do you want to build anyway?", interactive, true) if err != nil { return false, err } if !cont { return false, nil } } if instVer, ok := installed[vars.Name]; ok { log.Warn("This package is already installed"). Str("name", vars.Name). Str("version", instVer). Send() } return true, nil } // installBuildDeps installs any build dependencies that aren't already installed and returns // a slice containing the names of all the packages it installed. func installBuildDeps(ctx context.Context, vars *types.BuildVars, opts types.BuildOpts, installed map[string]string) ([]string, error) { log := loggerctx.From(ctx) var buildDeps []string if len(vars.BuildDepends) > 0 { found, notFound, err := repos.FindPkgs(ctx, vars.BuildDepends) if err != nil { return nil, err } found = removeAlreadyInstalled(found, installed) log.Info("Installing build dependencies").Send() flattened := cliutils.FlattenPkgs(ctx, found, "install", opts.Interactive) buildDeps = packageNames(flattened) InstallPkgs(ctx, flattened, notFound, opts) } return buildDeps, nil } // installOptDeps asks the user which, if any, optional dependencies they want to install. // If the user chooses to install any optional dependencies, it performs the installation. func installOptDeps(ctx context.Context, vars *types.BuildVars, opts types.BuildOpts, installed map[string]string) error { if len(vars.OptDepends) > 0 { optDeps, err := cliutils.ChooseOptDepends(ctx, vars.OptDepends, "install", opts.Interactive) if err != nil { return err } if len(optDeps) == 0 { return nil } found, notFound, err := repos.FindPkgs(ctx, optDeps) if err != nil { return err } found = removeAlreadyInstalled(found, installed) flattened := cliutils.FlattenPkgs(ctx, found, "install", opts.Interactive) InstallPkgs(ctx, flattened, notFound, opts) } return nil } // buildLUREDeps builds all the LURE dependencies of the package. It returns the paths and names // of the packages it built, as well as all the dependencies it didn't find in the LURE repo so // they can be installed from the system repos. func buildLUREDeps(ctx context.Context, opts types.BuildOpts, vars *types.BuildVars) (builtPaths, builtNames, repoDeps []string, err error) { log := loggerctx.From(ctx) if len(vars.Depends) > 0 { log.Info("Installing dependencies").Send() found, notFound, err := repos.FindPkgs(ctx, vars.Depends) if err != nil { return nil, nil, nil, err } repoDeps = notFound // If there are multiple options for some packages, flatten them all into a single slice pkgs := cliutils.FlattenPkgs(ctx, found, "install", opts.Interactive) scripts := GetScriptPaths(ctx, pkgs) for _, script := range scripts { newOpts := opts newOpts.Script = script // Build the dependency pkgPaths, pkgNames, err := BuildPackage(ctx, newOpts) if err != nil { return nil, nil, nil, err } // Append the paths of all the built packages to builtPaths builtPaths = append(builtPaths, pkgPaths...) // Append the names of all the built packages to builtNames builtNames = append(builtNames, pkgNames...) // Append the name of the current package to builtNames builtNames = append(builtNames, filepath.Base(filepath.Dir(script))) } } // Remove any potential duplicates, which can be introduced if // several of the dependencies depend on the same packages. repoDeps = removeDuplicates(repoDeps) builtPaths = removeDuplicates(builtPaths) builtNames = removeDuplicates(builtNames) return builtPaths, builtNames, repoDeps, nil } // executeFunctions executes the special LURE functions, such as version(), prepare(), etc. func executeFunctions(ctx context.Context, dec *decoder.Decoder, dirs types.Directories, vars *types.BuildVars) (err error) { log := loggerctx.From(ctx) version, ok := dec.GetFunc("version") if ok { log.Info("Executing version()").Send() buf := &bytes.Buffer{} err = version( ctx, interp.Dir(dirs.SrcDir), interp.StdIO(os.Stdin, buf, os.Stderr), ) if err != nil { return err } newVer := strings.TrimSpace(buf.String()) err = setVersion(ctx, dec.Runner, newVer) if err != nil { return err } vars.Version = newVer log.Info("Updating version").Str("new", newVer).Send() } prepare, ok := dec.GetFunc("prepare") if ok { log.Info("Executing prepare()").Send() err = prepare(ctx, interp.Dir(dirs.SrcDir)) if err != nil { return err } } build, ok := dec.GetFunc("build") if ok { log.Info("Executing build()").Send() err = build(ctx, interp.Dir(dirs.SrcDir)) if err != nil { return err } } packageFn, ok := dec.GetFunc("package") if ok { log.Info("Executing package()").Send() err = packageFn(ctx, interp.Dir(dirs.SrcDir)) if err != nil { return err } } else { log.Fatal("The package() function is required").Send() } return nil } // buildPkgMetadata builds the metadata for the package that's going to be built. func buildPkgMetadata(vars *types.BuildVars, dirs types.Directories, deps []string) (*nfpm.Info, error) { pkgInfo := &nfpm.Info{ Name: vars.Name, Description: vars.Description, Arch: cpu.Arch(), Platform: "linux", Version: vars.Version, Release: strconv.Itoa(vars.Release), Homepage: vars.Homepage, License: strings.Join(vars.Licenses, ", "), Maintainer: vars.Maintainer, Overridables: nfpm.Overridables{ Conflicts: vars.Conflicts, Replaces: vars.Replaces, Provides: vars.Provides, Depends: deps, }, } if vars.Epoch != 0 { pkgInfo.Epoch = strconv.FormatUint(uint64(vars.Epoch), 10) } setScripts(vars, pkgInfo, dirs.ScriptDir) if slices.Contains(vars.Architectures, "all") { pkgInfo.Arch = "all" } contents, err := buildContents(vars, dirs) if err != nil { return nil, err } pkgInfo.Overridables.Contents = contents return pkgInfo, nil } // buildContents builds the contents section of the package, which contains the files // that will be placed into the final package. func buildContents(vars *types.BuildVars, dirs types.Directories) ([]*files.Content, error) { contents := []*files.Content{} err := filepath.Walk(dirs.PkgDir, func(path string, fi os.FileInfo, err error) error { trimmed := strings.TrimPrefix(path, dirs.PkgDir) if fi.IsDir() { f, err := os.Open(path) if err != nil { return err } // If the directory is empty, skip it _, err = f.Readdirnames(1) if err != io.EOF { return nil } contents = append(contents, &files.Content{ Source: path, Destination: trimmed, Type: "dir", FileInfo: &files.ContentFileInfo{ MTime: fi.ModTime(), }, }) return f.Close() } if fi.Mode()&os.ModeSymlink != 0 { link, err := os.Readlink(path) if err != nil { return err } // Remove pkgdir from the symlink's path link = strings.TrimPrefix(link, dirs.PkgDir) contents = append(contents, &files.Content{ Source: link, Destination: trimmed, Type: "symlink", FileInfo: &files.ContentFileInfo{ MTime: fi.ModTime(), Mode: fi.Mode(), }, }) return nil } fileContent := &files.Content{ Source: path, Destination: trimmed, FileInfo: &files.ContentFileInfo{ MTime: fi.ModTime(), Mode: fi.Mode(), Size: fi.Size(), }, } // If the file is supposed to be backed up, set its type to config|noreplace if slices.Contains(vars.Backup, trimmed) { fileContent.Type = "config|noreplace" } contents = append(contents, fileContent) return nil }) return contents, err } // removeBuildDeps asks the user if they'd like to remove the build dependencies that were // installed by installBuildDeps. If so, it uses the package manager to do that. func removeBuildDeps(ctx context.Context, buildDeps []string, opts types.BuildOpts) error { if len(buildDeps) > 0 { remove, err := cliutils.YesNoPrompt(ctx, "Would you like to remove the build dependencies?", opts.Interactive, false) if err != nil { return err } if remove { err = opts.Manager.Remove( &manager.Opts{ AsRoot: true, NoConfirm: true, }, buildDeps..., ) if err != nil { return err } } } return nil } // checkForBuiltPackage tries to detect a previously-built package and returns its path // and true if it finds one. If it doesn't find it, it returns "", false, nil. func checkForBuiltPackage(mgr manager.Manager, vars *types.BuildVars, pkgFormat, baseDir string) (string, bool, error) { filename, err := pkgFileName(vars, pkgFormat) if err != nil { return "", false, err } pkgPath := filepath.Join(baseDir, filename) _, err = os.Stat(pkgPath) if err != nil { return "", false, nil } return pkgPath, true, nil } // pkgFileName returns the filename of the package if it were to be built. // This is used to check if the package has already been built. func pkgFileName(vars *types.BuildVars, pkgFormat string) (string, error) { pkgInfo := &nfpm.Info{ Name: vars.Name, Arch: cpu.Arch(), Version: vars.Version, Release: strconv.Itoa(vars.Release), Epoch: strconv.FormatUint(uint64(vars.Epoch), 10), } packager, err := nfpm.Get(pkgFormat) if err != nil { return "", err } return packager.ConventionalFileName(pkgInfo), nil } // getPkgFormat returns the package format of the package manager, // or LURE_PKG_FORMAT if that's set. func getPkgFormat(mgr manager.Manager) string { pkgFormat := mgr.Format() if format, ok := os.LookupEnv("LURE_PKG_FORMAT"); ok { pkgFormat = format } return pkgFormat } // createBuildEnvVars creates the environment variables that will be set in the // build script when it's executed. func createBuildEnvVars(info *distro.OSRelease, dirs types.Directories) []string { env := os.Environ() env = append( env, "DISTRO_NAME="+info.Name, "DISTRO_PRETTY_NAME="+info.PrettyName, "DISTRO_ID="+info.ID, "DISTRO_VERSION_ID="+info.VersionID, "DISTRO_ID_LIKE="+strings.Join(info.Like, " "), "ARCH="+cpu.Arch(), "NCPU="+strconv.Itoa(runtime.NumCPU()), ) if dirs.ScriptDir != "" { env = append(env, "scriptdir="+dirs.ScriptDir) } if dirs.PkgDir != "" { env = append(env, "pkgdir="+dirs.PkgDir) } if dirs.SrcDir != "" { env = append(env, "srcdir="+dirs.SrcDir) } return env } // getSources downloads the sources from the script. func getSources(ctx context.Context, dirs types.Directories, bv *types.BuildVars) error { log := loggerctx.From(ctx) if len(bv.Sources) != len(bv.Checksums) { log.Fatal("The checksums array must be the same length as sources").Send() } for i, src := range bv.Sources { opts := dl.Options{ Name: fmt.Sprintf("%s[%d]", bv.Name, i), URL: src, Destination: dirs.SrcDir, Progress: os.Stderr, LocalDir: dirs.ScriptDir, } if !strings.EqualFold(bv.Checksums[i], "SKIP") { // If the checksum contains a colon, use the part before the colon // as the algorithm and the part after as the actual checksum. // Otherwise, use the default sha256 with the whole string as the checksum. algo, hashData, ok := strings.Cut(bv.Checksums[i], ":") if ok { checksum, err := hex.DecodeString(hashData) if err != nil { return err } opts.Hash = checksum opts.HashAlgorithm = algo } else { checksum, err := hex.DecodeString(bv.Checksums[i]) if err != nil { return err } opts.Hash = checksum } } err := dl.Download(ctx, opts) if err != nil { return err } } return nil } // setScripts adds any hook scripts to the package metadata. func setScripts(vars *types.BuildVars, info *nfpm.Info, scriptDir string) { if vars.Scripts.PreInstall != "" { info.Scripts.PreInstall = filepath.Join(scriptDir, vars.Scripts.PreInstall) } if vars.Scripts.PostInstall != "" { info.Scripts.PostInstall = filepath.Join(scriptDir, vars.Scripts.PostInstall) } if vars.Scripts.PreRemove != "" { info.Scripts.PreRemove = filepath.Join(scriptDir, vars.Scripts.PreRemove) } if vars.Scripts.PostRemove != "" { info.Scripts.PostRemove = filepath.Join(scriptDir, vars.Scripts.PostRemove) } if vars.Scripts.PreUpgrade != "" { info.ArchLinux.Scripts.PreUpgrade = filepath.Join(scriptDir, vars.Scripts.PreUpgrade) info.APK.Scripts.PreUpgrade = filepath.Join(scriptDir, vars.Scripts.PreUpgrade) } if vars.Scripts.PostUpgrade != "" { info.ArchLinux.Scripts.PostUpgrade = filepath.Join(scriptDir, vars.Scripts.PostUpgrade) info.APK.Scripts.PostUpgrade = filepath.Join(scriptDir, vars.Scripts.PostUpgrade) } if vars.Scripts.PreTrans != "" { info.RPM.Scripts.PreTrans = filepath.Join(scriptDir, vars.Scripts.PreTrans) } if vars.Scripts.PostTrans != "" { info.RPM.Scripts.PostTrans = filepath.Join(scriptDir, vars.Scripts.PostTrans) } } // setVersion changes the version variable in the script runner. // It's used to set the version to the output of the version() function. func setVersion(ctx context.Context, r *interp.Runner, to string) error { fl, err := syntax.NewParser().Parse(strings.NewReader("version='"+to+"'"), "") if err != nil { return err } return r.Run(ctx, fl) } // removeAlreadyInstalled returns a map without any dependencies that are already installed func removeAlreadyInstalled(found map[string][]db.Package, installed map[string]string) map[string][]db.Package { filteredPackages := make(map[string][]db.Package) for name, pkgList := range found { filteredPkgList := []db.Package{} for _, pkg := range pkgList { if _, isInstalled := installed[pkg.Name]; !isInstalled { filteredPkgList = append(filteredPkgList, pkg) } } filteredPackages[name] = filteredPkgList } return filteredPackages } // packageNames returns the names of all the given packages func packageNames(pkgs []db.Package) []string { names := make([]string, len(pkgs)) for i, p := range pkgs { names[i] = p.Name } return names } // removeDuplicates removes any duplicates from the given slice func removeDuplicates(slice []string) []string { seen := map[string]struct{}{} result := []string{} for _, s := range slice { if _, ok := seen[s]; !ok { seen[s] = struct{}{} result = append(result, s) } } return result }