Install kernels in mounted volume instead of dockerfile layers

container.go

@@ -0,0 +1,93 @@
+// Copyright 2023 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"os/user"
+	"time"
+
+	"github.com/rs/zerolog/log"
+)
+
+type container struct {
+	name    string
+	timeout time.Duration
+	Volumes struct {
+		LibModules string
+		UsrSrc     string
+		Boot       string
+	}
+}
+
+func NewContainer(name string, timeout time.Duration) (c container, err error) {
+	c.name = name
+	c.timeout = timeout
+
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+
+	c.Volumes.LibModules = fmt.Sprintf(
+		"%s/.out-of-tree/volumes/%s/lib/modules", usr.HomeDir, name)
+	os.MkdirAll(c.Volumes.LibModules, 0777)
+
+	c.Volumes.UsrSrc = fmt.Sprintf(
+		"%s/.out-of-tree/volumes/%s/usr/src", usr.HomeDir, name)
+	os.MkdirAll(c.Volumes.UsrSrc, 0777)
+
+	c.Volumes.Boot = fmt.Sprintf(
+		"%s/.out-of-tree/volumes/%s/boot", usr.HomeDir, name)
+	os.MkdirAll(c.Volumes.Boot, 0777)
+
+	return
+}
+
+func (c container) Build(imagePath string) (output string, err error) {
+	args := []string{"build"}
+	args = append(args, "-t", c.name, imagePath)
+
+	cmd := exec.Command("docker", args...)
+	log.Debug().Msgf("%v", cmd)
+
+	rawOutput, err := cmd.CombinedOutput()
+	output = string(rawOutput)
+	return
+}
+
+func (c container) Run(workdir string, command string) (output string, err error) {
+	cmd := exec.Command("docker", "run", "--rm",
+		"-v", workdir+":/work",
+		"-v", c.Volumes.LibModules+":/lib/modules",
+		"-v", c.Volumes.UsrSrc+":/usr/src",
+		"-v", c.Volumes.Boot+":/boot",
+		c.name, "bash", "-c", "cd /work && "+command)
+
+	log.Debug().Msgf("%v", cmd)
+
+	timer := time.AfterFunc(c.timeout, func() {
+		log.Info().Str("container", c.name).
+			Str("workdir", workdir).
+			Str("command", command).
+			Msg("killing container by timeout")
+		cmd.Process.Kill()
+	})
+	defer timer.Stop()
+
+	raw, err := cmd.CombinedOutput()
+	if err != nil {
+		e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`",
+			err, command, string(raw))
+		err = errors.New(e)
+		return
+	}
+
+	output = string(raw)
+	return
+}

kernel.go

@@ -1,4 +1,4 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Copyright 2023 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"math"
 	"math/rand"
 	"os"
 	"os/exec"
@@ -27,10 +28,9 @@ type KernelCmd struct {
 	NoDownload bool `help:"do not download qemu image while kernel generation"`
 	UseHost    bool `help:"also use host kernels"`
 
-	List        KernelListCmd        `cmd:"" help:"list kernels"`
+	List    KernelListCmd    `cmd:"" help:"list kernels"`
-	Autogen     KernelAutogenCmd     `cmd:"" help:"generate kernels based on the current config"`
+	Autogen KernelAutogenCmd `cmd:"" help:"generate kernels based on the current config"`
-	DockerRegen KernelDockerRegenCmd `cmd:"" help:"regenerate kernels config from out_of_tree_* docker images"`
+	Genall  KernelGenallCmd  `cmd:"" help:"generate all kernels for distro"`
-	Genall      KernelGenallCmd      `cmd:"" help:"generate all kernels for distro"`
 }
 
 type KernelListCmd struct{}
@@ -38,7 +38,7 @@ type KernelListCmd struct{}
 func (cmd *KernelListCmd) Run(g *Globals) (err error) {
 	kcfg, err := config.ReadKernelConfig(g.Config.Kernels)
 	if err != nil {
-		return
+		log.Debug().Err(err).Msg("read kernel config")
 	}
 
 	if len(kcfg.Kernels) == 0 {
@@ -77,53 +77,6 @@ func (cmd KernelAutogenCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
 	return updateKernelsCfg(kernelCmd.UseHost, !kernelCmd.NoDownload)
 }
 
-type KernelDockerRegenCmd struct{}
-
-func (cmd *KernelDockerRegenCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
-	dockerImages, err := listDockerImages()
-	if err != nil {
-		return
-	}
-
-	for _, d := range dockerImages {
-		var imagePath string
-		imagePath, err = dockerImagePath(config.KernelMask{
-			DistroType:    d.DistroType,
-			DistroRelease: d.DistroRelease,
-		})
-		if err != nil {
-			return
-		}
-
-		args := []string{"build"}
-		args = append(args, "-t", d.ContainerName, imagePath)
-
-		cmd := exec.Command("docker", args...)
-		log.Debug().Msgf("%v", cmd)
-
-		var rawOutput []byte
-		rawOutput, err = cmd.CombinedOutput()
-		if err != nil {
-			log.Print("docker build:", string(rawOutput))
-			return
-		}
-
-		err = kickImage(d.ContainerName)
-		if err != nil {
-			log.Print("kick image", d.ContainerName, ":", err)
-			continue
-		}
-
-		err = copyKernels(d.ContainerName)
-		if err != nil {
-			log.Print("copy kernels", d.ContainerName, ":", err)
-			continue
-		}
-	}
-
-	return updateKernelsCfg(kernelCmd.UseHost, !kernelCmd.NoDownload)
-}
-
 type KernelGenallCmd struct {
 	Distro string `required:"" help:"distribution"`
 	Ver    string `required:"" help:"distro version"`
@@ -143,7 +96,7 @@ func (cmd *KernelGenallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
 	err = generateKernels(km,
 		g.Config.Docker.Registry,
 		g.Config.Docker.Commands,
-		100, // FIXME docker-related limit is 127 layers
+		math.MaxUint32,
 		!kernelCmd.NoDownload,
 	)
 	if err != nil {
@@ -157,7 +110,13 @@ func matchDebianHeadersPkg(container, mask string, generic bool) (
 	pkgs []string, err error) {
 
 	cmd := "apt-cache search linux-headers | cut -d ' ' -f 1"
-	output, err := dockerRun(time.Minute, container, "/tmp", cmd)
+
+	c, err := NewContainer(container, time.Minute)
+	if err != nil {
+		return
+	}
+
+	output, err := c.Run("/tmp", cmd)
 	if err != nil {
 		return
 	}
@@ -188,7 +147,13 @@ func matchCentOSDevelPkg(container, mask string, generic bool) (
 
 	cmd := "yum search kernel-devel --showduplicates | " +
 		"grep '^kernel-devel' | cut -d ' ' -f 1"
-	output, err := dockerRun(time.Minute, container, "/tmp", cmd)
+
+	c, err := NewContainer(container, time.Minute)
+	if err != nil {
+		return
+	}
+
+	output, err := c.Run("/tmp", cmd)
 	if err != nil {
 		return
 	}
@@ -211,7 +176,7 @@ func dockerImagePath(sk config.KernelMask) (path string, err error) {
 		return
 	}
 
-	path = usr.HomeDir + "/.out-of-tree/"
+	path = usr.HomeDir + "/.out-of-tree/containers/"
 	path += sk.DistroType.String() + "/" + sk.DistroRelease
 	return
 }
@@ -254,12 +219,12 @@ func generateBaseDockerImage(registry string, commands []config.DockerCommand,
 	}
 
 	if exists(dockerPath) && string(rawOutput) != "" {
-		log.Printf("Base image for %s:%s found",
+		log.Info().Msgf("Base image for %s:%s found",
 			sk.DistroType.String(), sk.DistroRelease)
 		return
 	}
 
-	log.Printf("Base image for %s:%s not found, start generating",
+	log.Info().Msgf("Base image for %s:%s not found, start generating",
 		sk.DistroType.String(), sk.DistroRelease)
 	os.MkdirAll(imagePath, os.ModePerm)
 
@@ -349,180 +314,84 @@ func generateBaseDockerImage(registry string, commands []config.DockerCommand,
 		return
 	}
 
-	args := []string{"build"}
+	c, err := NewContainer(sk.DockerName(), time.Hour)
-	args = append(args, "-t", sk.DockerName(), imagePath)
-
-	cmd = exec.Command("docker", args...)
-	log.Debug().Msgf("%v", cmd)
-
-	rawOutput, err = cmd.CombinedOutput()
 	if err != nil {
-		log.Printf("Base image for %s:%s generating error, see log",
-			sk.DistroType.String(), sk.DistroRelease)
-		log.Print(string(rawOutput))
 		return
 	}
 
-	log.Printf("Base image for %s:%s generating success",
+	output, err := c.Build(imagePath)
+	if err != nil {
+		log.Error().Err(err).Msgf("Base image for %s:%s generating error",
+			sk.DistroType.String(), sk.DistroRelease)
+		log.Fatal().Msg(output)
+		return
+	}
+
+	log.Info().Msgf("Base image for %s:%s generating success",
 		sk.DistroType.String(), sk.DistroRelease)
 
 	return
 }
 
-func dockerImageAppend(sk config.KernelMask, pkgname string) (err error) {
+func installKernel(sk config.KernelMask, pkgname string) (err error) {
-	imagePath, err := dockerImagePath(sk)
+	c, err := NewContainer(sk.DockerName(), time.Hour) // TODO conf
 	if err != nil {
 		return
 	}
 
-	raw, err := ioutil.ReadFile(imagePath + "/Dockerfile")
+	if exists(c.Volumes.LibModules + "/" + sk.ReleaseMask) {
-	if err != nil {
-		return
-	}
-
-	if strings.Contains(string(raw), pkgname) {
 		// already installed kernel
-		log.Printf("kernel %s for %s:%s is already exists",
+		log.Info().Msgf("kernel %s for %s:%s is already exists",
 			pkgname, sk.DistroType.String(), sk.DistroRelease)
 		return
 	}
 
-	var s string
-
 	switch sk.DistroType {
 	case config.Ubuntu:
 		imagepkg := strings.Replace(pkgname, "headers", "image", -1)
 
-		log.Printf("Start adding kernel %s for %s:%s",
+		log.Info().Msgf("Start adding kernel %s for %s:%s",
 			imagepkg, sk.DistroType.String(), sk.DistroRelease)
 
-		s = fmt.Sprintf("RUN apt-get install -y %s %s\n", imagepkg,
+		cmd := fmt.Sprintf("apt-get install -y %s %s", imagepkg,
 			pkgname)
+
+		_, err = c.Run("/tmp", cmd)
+		if err != nil {
+			return
+		}
 	case config.CentOS:
 		imagepkg := strings.Replace(pkgname, "-devel", "", -1)
 
-		log.Printf("Start adding kernel %s for %s:%s",
+		log.Info().Msgf("Start adding kernel %s for %s:%s",
 			imagepkg, sk.DistroType.String(), sk.DistroRelease)
 
 		version := strings.Replace(pkgname, "kernel-devel-", "", -1)
 
-		s = fmt.Sprintf("RUN yum -y install %s %s\n", imagepkg,
+		cmd := fmt.Sprintf("yum -y install %s %s\n", imagepkg,
 			pkgname)
-		s += fmt.Sprintf("RUN dracut --add-drivers 'e1000 ext4' -f "+
+		_, err = c.Run("/tmp", cmd)
+		if err != nil {
+			return
+		}
+
+		cmd = fmt.Sprintf("dracut --add-drivers 'e1000 ext4' -f "+
 			"/boot/initramfs-%s.img %s\n", version, version)
+		_, err = c.Run("/tmp", cmd)
+		if err != nil {
+			return
+		}
 	default:
 		err = fmt.Errorf("%s not yet supported", sk.DistroType.String())
 		return
 	}
 
-	err = ioutil.WriteFile(imagePath+"/Dockerfile",
+	log.Info().Msgf("Add kernel %s for %s:%s success",
-		append(raw, []byte(s)...), 0644)
-	if err != nil {
-		return
-	}
-
-	args := []string{"build"}
-	args = append(args, "-t", sk.DockerName(), imagePath)
-
-	cmd := exec.Command("docker", args...)
-	log.Debug().Msgf("%v", cmd)
-
-	rawOutput, err := cmd.CombinedOutput()
-	if err != nil {
-		// Fallback to previous state
-		werr := ioutil.WriteFile(imagePath+"/Dockerfile", raw, 0644)
-		if werr != nil {
-			return
-		}
-
-		log.Printf("Add kernel %s for %s:%s error, see log",
-			pkgname, sk.DistroType.String(), sk.DistroRelease)
-		log.Print(string(rawOutput))
-		return
-	}
-
-	log.Printf("Add kernel %s for %s:%s success",
 		pkgname, sk.DistroType.String(), sk.DistroRelease)
 
 	return
 }
 
-func kickImage(name string) (err error) {
-	cmd := exec.Command("docker", "run", name, "bash", "-c", "ls")
-	log.Debug().Msgf("%v", cmd)
-
-	_, err = cmd.CombinedOutput()
-	return
-}
-
-func copyKernels(name string) (err error) {
-	cmd := exec.Command("docker", "ps", "-a")
-	log.Debug().Msgf("%v", cmd)
-
-	rawOutput, err := cmd.CombinedOutput()
-	if err != nil {
-		log.Print(string(rawOutput))
-		return
-	}
-
-	r, err := regexp.Compile(".*" + name)
-	if err != nil {
-		return
-	}
-
-	var containerID string
-
-	what := r.FindAll(rawOutput, -1)
-	for _, w := range what {
-		containerID = strings.Fields(string(w))[0]
-		cmd = exec.Command("which", "podman")
-		log.Debug().Msgf("%v", cmd)
-		_, err = cmd.CombinedOutput()
-		if err != nil {
-			break
-		}
-	}
-
-	usr, err := user.Current()
-	if err != nil {
-		return
-	}
-
-	target := usr.HomeDir + "/.out-of-tree/kernels/" + name + "/"
-	if !exists(target) {
-		os.MkdirAll(target, os.ModePerm)
-	}
-
-	cmd = exec.Command("docker", "cp", containerID+":/boot/.", target)
-	log.Debug().Msgf("%v", cmd)
-
-	rawOutput, err = cmd.CombinedOutput()
-	if err != nil {
-		log.Print(string(rawOutput))
-		return
-	}
-
-	cmd = exec.Command("docker", "cp", containerID+":/lib/modules", target)
-	log.Debug().Msgf("%v", cmd)
-
-	rawOutput, err = cmd.CombinedOutput()
-	if err != nil {
-		log.Print(string(rawOutput))
-		return
-	}
-
-	cmd = exec.Command("find", target+"modules", "-type", "l", "-delete")
-	log.Debug().Msgf("%v", cmd)
-
-	rawOutput, err = cmd.CombinedOutput()
-	if err != nil {
-		log.Print(string(rawOutput))
-		return
-	}
-
-	return
-}
-
 func genKernelPath(files []os.FileInfo, kname string) string {
 	for _, file := range files {
 		if strings.HasPrefix(file.Name(), "vmlinuz") {
@@ -531,7 +400,9 @@ func genKernelPath(files []os.FileInfo, kname string) string {
 			}
 		}
 	}
-	return "unknown"
+
+	log.Fatal().Msgf("cannot find kernel %s", kname)
+	return ""
 }
 
 func genInitrdPath(files []os.FileInfo, kname string) string {
@@ -544,7 +415,9 @@ func genInitrdPath(files []os.FileInfo, kname string) string {
 			}
 		}
 	}
-	return "unknown"
+
+	log.Fatal().Msgf("cannot find initrd %s", kname)
+	return ""
 }
 
 func genRootfsImage(d dockerImageInfo, download bool) (rootfs string, err error) {
@@ -661,48 +534,45 @@ func updateKernelsCfg(host, download bool) (err error) {
 		return
 	}
 
-	log.Print(kernelsCfgPath, "is successfully updated")
+	log.Info().Msgf("%s is successfully updated", kernelsCfgPath)
 	return
 }
 
 func genDockerKernels(dii dockerImageInfo, newkcfg *config.KernelConfig,
 	download bool) (err error) {
 
-	name := dii.ContainerName
-	cmd := exec.Command("docker", "run", name, "ls", "/lib/modules")
-	log.Debug().Msgf("%v", cmd)
-
-	rawOutput, err := cmd.CombinedOutput()
-	if err != nil {
-		log.Print(string(rawOutput), err)
-		return
-	}
-
-	usr, err := user.Current()
-	if err != nil {
-		return
-	}
-	kernelsBase := usr.HomeDir + "/.out-of-tree/kernels/" + name + "/"
-	files, err := ioutil.ReadDir(kernelsBase)
-	if err != nil {
-		return
-	}
-
 	rootfs, err := genRootfsImage(dii, download)
 	if err != nil {
 		return
 	}
 
-	for _, k := range strings.Fields(string(rawOutput)) {
+	c, err := NewContainer(dii.ContainerName, time.Minute)
+	if err != nil {
+		return
+	}
+
+	moddirs, err := ioutil.ReadDir(c.Volumes.LibModules)
+	if err != nil {
+		return
+	}
+
+	bootfiles, err := ioutil.ReadDir(c.Volumes.Boot)
+	if err != nil {
+		return
+	}
+
+	for _, krel := range moddirs {
 		ki := config.KernelInfo{
 			DistroType:    dii.DistroType,
 			DistroRelease: dii.DistroRelease,
-			KernelRelease: k,
+			KernelRelease: krel.Name(),
-			ContainerName: name,
+			ContainerName: dii.ContainerName,
 
-			KernelPath:  kernelsBase + genKernelPath(files, k),
+			KernelPath: c.Volumes.Boot + "/" +
-			InitrdPath:  kernelsBase + genInitrdPath(files, k),
+				genKernelPath(bootfiles, krel.Name()),
-			ModulesPath: kernelsBase + "modules/" + k,
+			InitrdPath: c.Volumes.Boot + "/" +
+				genInitrdPath(bootfiles, krel.Name()),
+			ModulesPath: c.Volumes.LibModules + "/" + krel.Name(),
 
 			RootFS: rootfs,
 		}
@@ -734,7 +604,7 @@ func generateKernels(km config.KernelMask, registry string,
 	commands []config.DockerCommand, max int64,
 	download bool) (err error) {
 
-	log.Print("Generating for kernel mask", km)
+	log.Info().Msgf("Generating for kernel mask %v", km)
 
 	_, err = genRootfsImage(dockerImageInfo{ContainerName: km.DockerName()},
 		download)
@@ -768,26 +638,15 @@ func generateKernels(km config.KernelMask, registry string,
 			break
 		}
 
-		log.Print(i, "/", len(pkgs), pkg)
+		log.Info().Msgf("%d/%d %s", i, len(pkgs), pkg)
 
-		err = dockerImageAppend(km, pkg)
+		err = installKernel(km, pkg)
 		if err == nil {
 			max--
 		} else {
-			log.Print("dockerImageAppend", err)
+			log.Fatal().Err(err).Msg("install kernel")
 		}
 	}
 
-	err = kickImage(km.DockerName())
-	if err != nil {
-		log.Print("kick image", km.DockerName(), ":", err)
-		return
-	}
-
-	err = copyKernels(km.DockerName())
-	if err != nil {
-		log.Print("copy kernels", km.DockerName(), ":", err)
-		return
-	}
 	return
 }

pew.go

@@ -1,4 +1,4 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Copyright 2023 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
@@ -112,7 +112,7 @@ func (cmd PewCmd) Run(g *Globals) (err error) {
 	if cmd.Tag == "" {
 		cmd.Tag = fmt.Sprintf("%d", time.Now().Unix())
 	}
-	log.Info().Str("tag", cmd.Tag).Msg("")
+	log.Info().Str("tag", cmd.Tag).Msg("log")
 
 	err = performCI(ka, kcfg, cmd.Binary, cmd.Test, stop,
 		qemuTimeout, dockerTimeout,
@@ -144,35 +144,6 @@ func successRate(state runstate) float64 {
 
 const pathDevNull = "/dev/null"
 
-func dockerRun(timeout time.Duration, container, workdir, command string) (
-	output string, err error) {
-
-	cmd := exec.Command("docker", "run", "--rm", "-v", workdir+":/work",
-		container, "bash", "-c", "cd /work && "+command)
-
-	log.Debug().Msgf("%v", cmd)
-
-	timer := time.AfterFunc(timeout, func() {
-		log.Info().Str("container", container).
-			Str("workdir", workdir).
-			Str("command", command).
-			Msg("killing container by timeout")
-		cmd.Process.Kill()
-	})
-	defer timer.Stop()
-
-	raw, err := cmd.CombinedOutput()
-	if err != nil {
-		e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`",
-			err, command, string(raw))
-		err = errors.New(e)
-		return
-	}
-
-	output = string(raw)
-	return
-}
-
 func sh(workdir, cmd string) (output string, err error) {
 	command := exec.Command("sh", "-c", "cd "+workdir+" && "+cmd)
 
@@ -262,8 +233,13 @@ func build(tmp string, ka config.Artifact, ki config.KernelInfo,
 	}
 
 	if ki.ContainerName != "" {
-		output, err = dockerRun(dockerTimeout, ki.ContainerName,
+		var c container
-			outdir, buildCommand+" && chmod -R 777 /work")
+		c, err = NewContainer(ki.ContainerName, dockerTimeout)
+		if err != nil {
+			log.Fatal().Err(err).Msg("container creation failure")
+		}
+
+		output, err = c.Run(outdir, buildCommand+" && chmod -R 777 /work")
 	} else {
 		cmd := exec.Command("bash", "-c", "cd "+outdir+" && "+
 			buildCommand)
@@ -497,7 +473,25 @@ func copyStandardModules(q *qemu.System, ki config.KernelInfo) (err error) {
 		return
 	}
 
-	return q.CopyDirectory("root", ki.ModulesPath, "/lib/modules/")
+	files, err := ioutil.ReadDir(ki.ModulesPath)
+	if err != nil {
+		return
+	}
+
+	// FIXME scp cannot ignore symlinks
+	for _, f := range files {
+		if f.Mode()&os.ModeSymlink == os.ModeSymlink {
+			continue
+		}
+
+		path := ki.ModulesPath + "/" + f.Name()
+		err = q.CopyDirectory("root", path, "/lib/modules/"+ki.KernelRelease+"/")
+		if err != nil {
+			return
+		}
+	}
+
+	return
 }
 
 func testArtifact(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
@@ -505,13 +499,15 @@ func testArtifact(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
 	qemuTimeout, dockerTimeout time.Duration, dist, tag string,
 	db *sql.DB) {
 
+	defer swg.Done()
+
 	slog := log.With().
 		Str("distro_type", ki.DistroType.String()).
 		Str("distro_release", ki.DistroRelease).
 		Str("kernel", ki.KernelRelease).
 		Logger()
 
-	defer swg.Done()
+	slog.Info().Msg("start")
 
 	kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
 	q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)