From ebc597ff0bff4456f07b3a69c7fe4a4502ef269d Mon Sep 17 00:00:00 2001 From: Mikhail Klementev Date: Sat, 13 May 2023 10:14:45 +0000 Subject: [PATCH] refactor: move container functions to submodule --- container.go | 213 +-------------------------------------- container/container.go | 222 +++++++++++++++++++++++++++++++++++++++++ kernel.go | 27 ++--- kernel_linux.go | 3 +- main.go | 3 +- pew.go | 5 +- 6 files changed, 246 insertions(+), 227 deletions(-) create mode 100644 container/container.go diff --git a/container.go b/container.go index 87917c1..2058710 100644 --- a/container.go +++ b/container.go @@ -5,24 +5,15 @@ package main import ( - "bufio" - "errors" "fmt" - "os" "os/exec" - "os/user" - "regexp" "strings" - "time" - "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "code.dumpstack.io/tools/out-of-tree/config" + "code.dumpstack.io/tools/out-of-tree/container" ) -var containerRuntime = "docker" - type ContainerCmd struct { Filter string `help:"filter by name"` @@ -31,7 +22,7 @@ type ContainerCmd struct { } func (cmd ContainerCmd) Containers() (names []string) { - images, err := listContainerImages() + images, err := container.Images() if err != nil { log.Fatal().Err(err).Msg("") } @@ -59,7 +50,7 @@ type ContainerCleanupCmd struct{} func (cmd ContainerCleanupCmd) Run(containerCmd *ContainerCmd) (err error) { var output []byte for _, name := range containerCmd.Containers() { - output, err = exec.Command(containerRuntime, "image", "rm", name). + output, err = exec.Command(container.Runtime, "image", "rm", name). CombinedOutput() if err != nil { log.Error().Err(err).Str("output", string(output)).Msg("") @@ -68,201 +59,3 @@ func (cmd ContainerCleanupCmd) Run(containerCmd *ContainerCmd) (err error) { } return } - -type containerImageInfo struct { - Name string - DistroType config.DistroType - DistroRelease string // 18.04/7.4.1708/9.1 -} - -func listContainerImages() (diis []containerImageInfo, err error) { - cmd := exec.Command(containerRuntime, "images") - log.Debug().Msgf("%v", cmd) - - rawOutput, err := cmd.CombinedOutput() - if err != nil { - return - } - - r, err := regexp.Compile("out_of_tree_.*") - if err != nil { - return - } - - containers := r.FindAll(rawOutput, -1) - for _, c := range containers { - container := strings.Fields(string(c))[0] - - s := strings.Replace(container, "__", ".", -1) - values := strings.Split(s, "_") - distro, ver := values[3], values[4] - - dii := containerImageInfo{ - Name: container, - DistroRelease: ver, - } - - dii.DistroType, err = config.NewDistroType(distro) - if err != nil { - return - } - - diis = append(diis, dii) - } - return -} - -type container struct { - name string - - timeout time.Duration - - Volumes struct { - LibModules string - UsrSrc string - Boot string - } - - // Additional arguments - Args []string - - Log zerolog.Logger -} - -func NewContainer(name string, timeout time.Duration) (c container, err error) { - c.Log = log.With(). - Str("container", name). - Logger() - - c.name = name - c.timeout = timeout - - usr, err := user.Current() - if err != nil { - return - } - - c.Volumes.LibModules = fmt.Sprintf( - "%s/.out-of-tree/volumes/%s/lib/modules", usr.HomeDir, name) - os.MkdirAll(c.Volumes.LibModules, 0777) - - c.Volumes.UsrSrc = fmt.Sprintf( - "%s/.out-of-tree/volumes/%s/usr/src", usr.HomeDir, name) - os.MkdirAll(c.Volumes.UsrSrc, 0777) - - c.Volumes.Boot = fmt.Sprintf( - "%s/.out-of-tree/volumes/%s/boot", usr.HomeDir, name) - os.MkdirAll(c.Volumes.Boot, 0777) - - return -} - -func (c container) Build(imagePath string) (output string, err error) { - args := []string{"build"} - args = append(args, "-t", c.name, imagePath) - - cmd := exec.Command(containerRuntime, args...) - - flog := c.Log.With(). - Str("command", fmt.Sprintf("%v", cmd)). - Logger() - - stdout, err := cmd.StdoutPipe() - if err != nil { - return - } - cmd.Stderr = cmd.Stdout - - err = cmd.Start() - if err != nil { - return - } - - go func() { - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - m := scanner.Text() - output += m + "\n" - flog.Trace().Str("stdout", m).Msg("") - } - }() - - err = cmd.Wait() - return -} - -func (c container) Run(workdir string, command string) (output string, err error) { - flog := c.Log.With(). - Str("workdir", workdir). - Str("command", command). - Logger() - - var args []string - args = append(args, "run", "--rm") - args = append(args, c.Args...) - if workdir != "" { - args = append(args, "-v", workdir+":/work") - } - if c.Volumes.LibModules != "" { - args = append(args, "-v", c.Volumes.LibModules+":/lib/modules") - } - if c.Volumes.UsrSrc != "" { - args = append(args, "-v", c.Volumes.UsrSrc+":/usr/src") - } - if c.Volumes.Boot != "" { - args = append(args, "-v", c.Volumes.Boot+":/boot") - } - args = append(args, c.name, "bash", "-c") - if workdir != "" { - args = append(args, "cd /work && "+command) - } else { - args = append(args, command) - } - - cmd := exec.Command(containerRuntime, args...) - - flog.Debug().Msgf("%v", cmd) - - stdout, err := cmd.StdoutPipe() - if err != nil { - return - } - cmd.Stderr = cmd.Stdout - - timer := time.AfterFunc(c.timeout, func() { - flog.Info().Msg("killing container by timeout") - - flog.Debug().Msg("SIGINT") - cmd.Process.Signal(os.Interrupt) - - time.Sleep(time.Minute) - - flog.Debug().Msg("SIGKILL") - cmd.Process.Kill() - }) - defer timer.Stop() - - err = cmd.Start() - if err != nil { - return - } - - go func() { - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - m := scanner.Text() - output += m + "\n" - flog.Trace().Str("stdout", m).Msg("") - } - }() - - err = cmd.Wait() - if err != nil { - e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`", - err, command, output) - err = errors.New(e) - return - } - - return -} diff --git a/container/container.go b/container/container.go new file mode 100644 index 0000000..0d469b3 --- /dev/null +++ b/container/container.go @@ -0,0 +1,222 @@ +// Copyright 2023 Mikhail Klementev. All rights reserved. +// Use of this source code is governed by a AGPLv3 license +// (or later) that can be found in the LICENSE file. + +package container + +import ( + "bufio" + "errors" + "fmt" + "os" + "os/exec" + "os/user" + "regexp" + "strings" + "time" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + + "code.dumpstack.io/tools/out-of-tree/config" +) + +var Runtime = "docker" + +type Image struct { + Name string + DistroType config.DistroType + DistroRelease string // 18.04/7.4.1708/9.1 +} + +func Images() (diis []Image, err error) { + cmd := exec.Command(Runtime, "images") + log.Debug().Msgf("%v", cmd) + + rawOutput, err := cmd.CombinedOutput() + if err != nil { + return + } + + r, err := regexp.Compile("out_of_tree_.*") + if err != nil { + return + } + + containers := r.FindAll(rawOutput, -1) + for _, c := range containers { + containerName := strings.Fields(string(c))[0] + + s := strings.Replace(containerName, "__", ".", -1) + values := strings.Split(s, "_") + distro, ver := values[3], values[4] + + dii := Image{ + Name: containerName, + DistroRelease: ver, + } + + dii.DistroType, err = config.NewDistroType(distro) + if err != nil { + return + } + + diis = append(diis, dii) + } + return +} + +type Container struct { + name string + + timeout time.Duration + + Volumes struct { + LibModules string + UsrSrc string + Boot string + } + + // Additional arguments + Args []string + + Log zerolog.Logger +} + +func New(name string, timeout time.Duration) (c Container, err error) { + c.Log = log.With(). + Str("container", name). + Logger() + + c.name = name + c.timeout = timeout + + usr, err := user.Current() + if err != nil { + return + } + + c.Volumes.LibModules = fmt.Sprintf( + "%s/.out-of-tree/volumes/%s/lib/modules", usr.HomeDir, name) + os.MkdirAll(c.Volumes.LibModules, 0777) + + c.Volumes.UsrSrc = fmt.Sprintf( + "%s/.out-of-tree/volumes/%s/usr/src", usr.HomeDir, name) + os.MkdirAll(c.Volumes.UsrSrc, 0777) + + c.Volumes.Boot = fmt.Sprintf( + "%s/.out-of-tree/volumes/%s/boot", usr.HomeDir, name) + os.MkdirAll(c.Volumes.Boot, 0777) + + return +} + +func (c Container) Build(imagePath string) (output string, err error) { + args := []string{"build"} + args = append(args, "-t", c.name, imagePath) + + cmd := exec.Command(Runtime, args...) + + flog := c.Log.With(). + Str("command", fmt.Sprintf("%v", cmd)). + Logger() + + stdout, err := cmd.StdoutPipe() + if err != nil { + return + } + cmd.Stderr = cmd.Stdout + + err = cmd.Start() + if err != nil { + return + } + + go func() { + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + m := scanner.Text() + output += m + "\n" + flog.Trace().Str("stdout", m).Msg("") + } + }() + + err = cmd.Wait() + return +} + +func (c Container) Run(workdir string, command string) (output string, err error) { + flog := c.Log.With(). + Str("workdir", workdir). + Str("command", command). + Logger() + + var args []string + args = append(args, "run", "--rm") + args = append(args, c.Args...) + if workdir != "" { + args = append(args, "-v", workdir+":/work") + } + if c.Volumes.LibModules != "" { + args = append(args, "-v", c.Volumes.LibModules+":/lib/modules") + } + if c.Volumes.UsrSrc != "" { + args = append(args, "-v", c.Volumes.UsrSrc+":/usr/src") + } + if c.Volumes.Boot != "" { + args = append(args, "-v", c.Volumes.Boot+":/boot") + } + args = append(args, c.name, "bash", "-c") + if workdir != "" { + args = append(args, "cd /work && "+command) + } else { + args = append(args, command) + } + + cmd := exec.Command(Runtime, args...) + + flog.Debug().Msgf("%v", cmd) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return + } + cmd.Stderr = cmd.Stdout + + timer := time.AfterFunc(c.timeout, func() { + flog.Info().Msg("killing container by timeout") + + flog.Debug().Msg("SIGINT") + cmd.Process.Signal(os.Interrupt) + + time.Sleep(time.Minute) + + flog.Debug().Msg("SIGKILL") + cmd.Process.Kill() + }) + defer timer.Stop() + + err = cmd.Start() + if err != nil { + return + } + + go func() { + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + m := scanner.Text() + output += m + "\n" + flog.Trace().Str("stdout", m).Msg("") + } + }() + + err = cmd.Wait() + if err != nil { + e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`", + err, command, output) + err = errors.New(e) + return + } + + return +} diff --git a/kernel.go b/kernel.go index a6ba9de..7efe751 100644 --- a/kernel.go +++ b/kernel.go @@ -23,6 +23,7 @@ import ( "github.com/rs/zerolog/log" "code.dumpstack.io/tools/out-of-tree/config" + "code.dumpstack.io/tools/out-of-tree/container" "code.dumpstack.io/tools/out-of-tree/fs" ) @@ -79,7 +80,7 @@ func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error ReleaseMask: ".*", } - _, err = genRootfsImage(containerImageInfo{Name: km.DockerName()}, false) + _, err = genRootfsImage(container.Image{Name: km.DockerName()}, false) if err != nil { return } @@ -225,12 +226,12 @@ func (cmd *KernelConfigRegenCmd) Run(kernelCmd *KernelCmd, g *Globals) (err erro return updateKernelsCfg(kernelCmd.UseHost, !kernelCmd.NoDownload) } -func matchDebImagePkg(container, mask string) (pkgs []string, err error) { +func matchDebImagePkg(containerName, mask string) (pkgs []string, err error) { cmd := "apt-cache search --names-only '^linux-image-[0-9\\.\\-]*-generic' | awk '{ print $1 }'" // FIXME timeout should be in global out-of-tree config - c, err := NewContainer(container, time.Hour) + c, err := container.New(containerName, time.Hour) if err != nil { return } @@ -254,7 +255,7 @@ func matchDebImagePkg(container, mask string) (pkgs []string, err error) { return } -func matchOracleLinuxPkg(container, mask string) ( +func matchOracleLinuxPkg(containerName, mask string) ( pkgs []string, err error) { cmd := "yum search kernel --showduplicates " + @@ -263,7 +264,7 @@ func matchOracleLinuxPkg(container, mask string) ( "| cut -d ' ' -f 1" // FIXME timeout should be in global out-of-tree config - c, err := NewContainer(container, time.Hour) + c, err := container.New(containerName, time.Hour) if err != nil { return } @@ -345,7 +346,7 @@ func generateBaseDockerImage(registry string, commands []config.DockerCommand, d := "# BASE\n" // TODO move as function to container.go - cmd := exec.Command(containerRuntime, "images", "-q", sk.DockerName()) + cmd := exec.Command(container.Runtime, "images", "-q", sk.DockerName()) log.Debug().Msgf("run %v", cmd) rawOutput, err := cmd.CombinedOutput() @@ -495,7 +496,7 @@ func generateBaseDockerImage(registry string, commands []config.DockerCommand, return } - c, err := NewContainer(sk.DockerName(), time.Hour) + c, err := container.New(sk.DockerName(), time.Hour) if err != nil { return } @@ -521,7 +522,7 @@ func installKernel(sk config.KernelMask, pkgname string, force, headers bool) (e Str("pkg", pkgname). Logger() - c, err := NewContainer(sk.DockerName(), time.Hour) // TODO conf + c, err := container.New(sk.DockerName(), time.Hour) // TODO conf if err != nil { return } @@ -640,7 +641,7 @@ func findInitrdFile(files []os.FileInfo, kname string) (name string, err error) return } -func genRootfsImage(d containerImageInfo, download bool) (rootfs string, err error) { +func genRootfsImage(d container.Image, download bool) (rootfs string, err error) { usr, err := user.Current() if err != nil { return @@ -672,7 +673,7 @@ func updateKernelsCfg(host, download bool) (err error) { } // Get docker kernels - dockerImages, err := listContainerImages() + dockerImages, err := container.Images() if err != nil { return } @@ -715,7 +716,7 @@ func updateKernelsCfg(host, download bool) (err error) { return } -func listContainersKernels(dii containerImageInfo, newkcfg *config.KernelConfig, +func listContainersKernels(dii container.Image, newkcfg *config.KernelConfig, download bool) (err error) { rootfs, err := genRootfsImage(dii, download) @@ -723,7 +724,7 @@ func listContainersKernels(dii containerImageInfo, newkcfg *config.KernelConfig, return } - c, err := NewContainer(dii.Name, time.Hour) + c, err := container.New(dii.Name, time.Hour) if err != nil { return } @@ -827,7 +828,7 @@ func generateKernels(km config.KernelMask, registry string, log.Info().Msgf("Generating for kernel mask %v", km) - _, err = genRootfsImage(containerImageInfo{Name: km.DockerName()}, + _, err = genRootfsImage(container.Image{Name: km.DockerName()}, download) if err != nil || *shutdown { return diff --git a/kernel_linux.go b/kernel_linux.go index 36ea5a5..feca01f 100644 --- a/kernel_linux.go +++ b/kernel_linux.go @@ -16,6 +16,7 @@ import ( "github.com/zcalusic/sysinfo" "code.dumpstack.io/tools/out-of-tree/config" + "code.dumpstack.io/tools/out-of-tree/container" "code.dumpstack.io/tools/out-of-tree/fs" ) @@ -44,7 +45,7 @@ func genHostKernels(download bool) (kcfg config.KernelConfig, err error) { } // only for compatibility, docker is not really used - dii := containerImageInfo{ + dii := container.Image{ Name: config.KernelMask{ DistroType: distroType, DistroRelease: si.OS.Version, diff --git a/main.go b/main.go index 146521c..ffc8cb8 100644 --- a/main.go +++ b/main.go @@ -23,6 +23,7 @@ import ( "github.com/alecthomas/kong" "code.dumpstack.io/tools/out-of-tree/config" + "code.dumpstack.io/tools/out-of-tree/container" "code.dumpstack.io/tools/out-of-tree/fs" ) @@ -189,7 +190,7 @@ func main() { cli.ContainerRuntime) } } - containerRuntime = cli.ContainerRuntime + container.Runtime = cli.ContainerRuntime err = ctx.Run(&cli.Globals) ctx.FatalIfErrorf(err) diff --git a/pew.go b/pew.go index b4a2310..f895dc6 100644 --- a/pew.go +++ b/pew.go @@ -24,6 +24,7 @@ import ( "gopkg.in/logrusorgru/aurora.v2" "code.dumpstack.io/tools/out-of-tree/config" + "code.dumpstack.io/tools/out-of-tree/container" "code.dumpstack.io/tools/out-of-tree/fs" "code.dumpstack.io/tools/out-of-tree/qemu" ) @@ -268,8 +269,8 @@ func build(flog zerolog.Logger, tmp string, ka config.Artifact, } if ki.ContainerName != "" { - var c container - c, err = NewContainer(ki.ContainerName, dockerTimeout) + var c container.Container + c, err = container.New(ki.ContainerName, dockerTimeout) c.Log = flog if err != nil { log.Fatal().Err(err).Msg("container creation failure")