tools/out-of-tree
1
0
Fork 0
Code Releases Activity

refactor: move container functions to submodule

Browse Source
This commit is contained in:
dump_stack()
2023-05-13 10:14:45 +00:00
parent 2c2435a7a5
commit ebc597ff0b
6 changed files with 246 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

213
container.go
View File

@@ -5,24 +5,15 @@
package main
import (
"bufio"
"errors"
"fmt"
"os"
"os/exec"
"os/user"
"regexp"
"strings"
"time"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/container"
)
var containerRuntime = "docker"
type ContainerCmd struct {
Filter string `help:"filter by name"`
@@ -31,7 +22,7 @@ type ContainerCmd struct {
}
func (cmd ContainerCmd) Containers() (names []string) {
images, err := listContainerImages()
images, err := container.Images()
if err != nil {
log.Fatal().Err(err).Msg("")
}
@@ -59,7 +50,7 @@ type ContainerCleanupCmd struct{}
func (cmd ContainerCleanupCmd) Run(containerCmd *ContainerCmd) (err error) {
var output []byte
for _, name := range containerCmd.Containers() {
output, err = exec.Command(containerRuntime, "image", "rm", name).
output, err = exec.Command(container.Runtime, "image", "rm", name).
CombinedOutput()
if err != nil {
log.Error().Err(err).Str("output", string(output)).Msg("")
@@ -68,201 +59,3 @@ func (cmd ContainerCleanupCmd) Run(containerCmd *ContainerCmd) (err error) {
}
return
}
type containerImageInfo struct {
Name string
DistroType config.DistroType
DistroRelease string // 18.04/7.4.1708/9.1
}
func listContainerImages() (diis []containerImageInfo, err error) {
cmd := exec.Command(containerRuntime, "images")
log.Debug().Msgf("%v", cmd)
rawOutput, err := cmd.CombinedOutput()
if err != nil {
return
}
r, err := regexp.Compile("out_of_tree_.*")
if err != nil {
return
}
containers := r.FindAll(rawOutput, -1)
for _, c := range containers {
container := strings.Fields(string(c))[0]
s := strings.Replace(container, "__", ".", -1)
values := strings.Split(s, "_")
distro, ver := values[3], values[4]
dii := containerImageInfo{
Name: container,
DistroRelease: ver,
}
dii.DistroType, err = config.NewDistroType(distro)
if err != nil {
return
}
diis = append(diis, dii)
}
return
}
type container struct {
name string
timeout time.Duration
Volumes struct {
LibModules string
UsrSrc string
Boot string
}
// Additional arguments
Args []string
Log zerolog.Logger
}
func NewContainer(name string, timeout time.Duration) (c container, err error) {
c.Log = log.With().
Str("container", name).
Logger()
c.name = name
c.timeout = timeout
usr, err := user.Current()
if err != nil {
return
}
c.Volumes.LibModules = fmt.Sprintf(
"%s/.out-of-tree/volumes/%s/lib/modules", usr.HomeDir, name)
os.MkdirAll(c.Volumes.LibModules, 0777)
c.Volumes.UsrSrc = fmt.Sprintf(
"%s/.out-of-tree/volumes/%s/usr/src", usr.HomeDir, name)
os.MkdirAll(c.Volumes.UsrSrc, 0777)
c.Volumes.Boot = fmt.Sprintf(
"%s/.out-of-tree/volumes/%s/boot", usr.HomeDir, name)
os.MkdirAll(c.Volumes.Boot, 0777)
return
}
func (c container) Build(imagePath string) (output string, err error) {
args := []string{"build"}
args = append(args, "-t", c.name, imagePath)
cmd := exec.Command(containerRuntime, args...)
flog := c.Log.With().
Str("command", fmt.Sprintf("%v", cmd)).
Logger()
stdout, err := cmd.StdoutPipe()
if err != nil {
return
}
cmd.Stderr = cmd.Stdout
err = cmd.Start()
if err != nil {
return
}
go func() {
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
m := scanner.Text()
output += m + "\n"
flog.Trace().Str("stdout", m).Msg("")
}
}()
err = cmd.Wait()
return
}
func (c container) Run(workdir string, command string) (output string, err error) {
flog := c.Log.With().
Str("workdir", workdir).
Str("command", command).
Logger()
var args []string
args = append(args, "run", "--rm")
args = append(args, c.Args...)
if workdir != "" {
args = append(args, "-v", workdir+":/work")
}
if c.Volumes.LibModules != "" {
args = append(args, "-v", c.Volumes.LibModules+":/lib/modules")
}
if c.Volumes.UsrSrc != "" {
args = append(args, "-v", c.Volumes.UsrSrc+":/usr/src")
}
if c.Volumes.Boot != "" {
args = append(args, "-v", c.Volumes.Boot+":/boot")
}
args = append(args, c.name, "bash", "-c")
if workdir != "" {
args = append(args, "cd /work && "+command)
} else {
args = append(args, command)
}
cmd := exec.Command(containerRuntime, args...)
flog.Debug().Msgf("%v", cmd)
stdout, err := cmd.StdoutPipe()
if err != nil {
return
}
cmd.Stderr = cmd.Stdout
timer := time.AfterFunc(c.timeout, func() {
flog.Info().Msg("killing container by timeout")
flog.Debug().Msg("SIGINT")
cmd.Process.Signal(os.Interrupt)
time.Sleep(time.Minute)
flog.Debug().Msg("SIGKILL")
cmd.Process.Kill()
})
defer timer.Stop()
err = cmd.Start()
if err != nil {
return
}
go func() {
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
m := scanner.Text()
output += m + "\n"
flog.Trace().Str("stdout", m).Msg("")
}
}()
err = cmd.Wait()
if err != nil {
e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`",
err, command, output)
err = errors.New(e)
return
}
return
}

222
container/container.go Normal file
View File

@@ -0,0 +1,222 @@
// Copyright 2023 Mikhail Klementev. All rights reserved.
// Use of this source code is governed by a AGPLv3 license
// (or later) that can be found in the LICENSE file.
package container
import (
"bufio"
"errors"
"fmt"
"os"
"os/exec"
"os/user"
"regexp"
"strings"
"time"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"code.dumpstack.io/tools/out-of-tree/config"
)
var Runtime = "docker"
type Image struct {
Name string
DistroType config.DistroType
DistroRelease string // 18.04/7.4.1708/9.1
}
func Images() (diis []Image, err error) {
cmd := exec.Command(Runtime, "images")
log.Debug().Msgf("%v", cmd)
rawOutput, err := cmd.CombinedOutput()
if err != nil {
return
}
r, err := regexp.Compile("out_of_tree_.*")
if err != nil {
return
}
containers := r.FindAll(rawOutput, -1)
for _, c := range containers {
containerName := strings.Fields(string(c))[0]
s := strings.Replace(containerName, "__", ".", -1)
values := strings.Split(s, "_")
distro, ver := values[3], values[4]
dii := Image{
Name: containerName,
DistroRelease: ver,
}
dii.DistroType, err = config.NewDistroType(distro)
if err != nil {
return
}
diis = append(diis, dii)
}
return
}
type Container struct {
name string
timeout time.Duration
Volumes struct {
LibModules string
UsrSrc string
Boot string
}
// Additional arguments
Args []string
Log zerolog.Logger
}
func New(name string, timeout time.Duration) (c Container, err error) {
c.Log = log.With().
Str("container", name).
Logger()
c.name = name
c.timeout = timeout
usr, err := user.Current()
if err != nil {
return
}
c.Volumes.LibModules = fmt.Sprintf(
"%s/.out-of-tree/volumes/%s/lib/modules", usr.HomeDir, name)
os.MkdirAll(c.Volumes.LibModules, 0777)
c.Volumes.UsrSrc = fmt.Sprintf(
"%s/.out-of-tree/volumes/%s/usr/src", usr.HomeDir, name)
os.MkdirAll(c.Volumes.UsrSrc, 0777)
c.Volumes.Boot = fmt.Sprintf(
"%s/.out-of-tree/volumes/%s/boot", usr.HomeDir, name)
os.MkdirAll(c.Volumes.Boot, 0777)
return
}
func (c Container) Build(imagePath string) (output string, err error) {
args := []string{"build"}
args = append(args, "-t", c.name, imagePath)
cmd := exec.Command(Runtime, args...)
flog := c.Log.With().
Str("command", fmt.Sprintf("%v", cmd)).
Logger()
stdout, err := cmd.StdoutPipe()
if err != nil {
return
}
cmd.Stderr = cmd.Stdout
err = cmd.Start()
if err != nil {
return
}
go func() {
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
m := scanner.Text()
output += m + "\n"
flog.Trace().Str("stdout", m).Msg("")
}
}()
err = cmd.Wait()
return
}
func (c Container) Run(workdir string, command string) (output string, err error) {
flog := c.Log.With().
Str("workdir", workdir).
Str("command", command).
Logger()
var args []string
args = append(args, "run", "--rm")
args = append(args, c.Args...)
if workdir != "" {
args = append(args, "-v", workdir+":/work")
}
if c.Volumes.LibModules != "" {
args = append(args, "-v", c.Volumes.LibModules+":/lib/modules")
}
if c.Volumes.UsrSrc != "" {
args = append(args, "-v", c.Volumes.UsrSrc+":/usr/src")
}
if c.Volumes.Boot != "" {
args = append(args, "-v", c.Volumes.Boot+":/boot")
}
args = append(args, c.name, "bash", "-c")
if workdir != "" {
args = append(args, "cd /work && "+command)
} else {
args = append(args, command)
}
cmd := exec.Command(Runtime, args...)
flog.Debug().Msgf("%v", cmd)
stdout, err := cmd.StdoutPipe()
if err != nil {
return
}
cmd.Stderr = cmd.Stdout
timer := time.AfterFunc(c.timeout, func() {
flog.Info().Msg("killing container by timeout")
flog.Debug().Msg("SIGINT")
cmd.Process.Signal(os.Interrupt)
time.Sleep(time.Minute)
flog.Debug().Msg("SIGKILL")
cmd.Process.Kill()
})
defer timer.Stop()
err = cmd.Start()
if err != nil {
return
}
go func() {
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
m := scanner.Text()
output += m + "\n"
flog.Trace().Str("stdout", m).Msg("")
}
}()
err = cmd.Wait()
if err != nil {
e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`",
err, command, output)
err = errors.New(e)
return
}
return
}

27
kernel.go
View File

@@ -23,6 +23,7 @@ import (
"github.com/rs/zerolog/log"
"code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/container"
"code.dumpstack.io/tools/out-of-tree/fs"
)
@@ -79,7 +80,7 @@ func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error
ReleaseMask: ".*",
}
_, err = genRootfsImage(containerImageInfo{Name: km.DockerName()}, false)
_, err = genRootfsImage(container.Image{Name: km.DockerName()}, false)
if err != nil {
return
}
@@ -225,12 +226,12 @@ func (cmd *KernelConfigRegenCmd) Run(kernelCmd *KernelCmd, g *Globals) (err erro
return updateKernelsCfg(kernelCmd.UseHost, !kernelCmd.NoDownload)
}
func matchDebImagePkg(container, mask string) (pkgs []string, err error) {
func matchDebImagePkg(containerName, mask string) (pkgs []string, err error) {
cmd := "apt-cache search --names-only '^linux-image-[0-9\\.\\-]*-generic' | awk '{ print $1 }'"
// FIXME timeout should be in global out-of-tree config
c, err := NewContainer(container, time.Hour)
c, err := container.New(containerName, time.Hour)
if err != nil {
return
}
@@ -254,7 +255,7 @@ func matchDebImagePkg(container, mask string) (pkgs []string, err error) {
return
}
func matchOracleLinuxPkg(container, mask string) (
func matchOracleLinuxPkg(containerName, mask string) (
pkgs []string, err error) {
cmd := "yum search kernel --showduplicates " +
@@ -263,7 +264,7 @@ func matchOracleLinuxPkg(container, mask string) (
"| cut -d ' ' -f 1"
// FIXME timeout should be in global out-of-tree config
c, err := NewContainer(container, time.Hour)
c, err := container.New(containerName, time.Hour)
if err != nil {
return
}
@@ -345,7 +346,7 @@ func generateBaseDockerImage(registry string, commands []config.DockerCommand,
d := "# BASE\n"
// TODO move as function to container.go
cmd := exec.Command(containerRuntime, "images", "-q", sk.DockerName())
cmd := exec.Command(container.Runtime, "images", "-q", sk.DockerName())
log.Debug().Msgf("run %v", cmd)
rawOutput, err := cmd.CombinedOutput()
@@ -495,7 +496,7 @@ func generateBaseDockerImage(registry string, commands []config.DockerCommand,
return
}
c, err := NewContainer(sk.DockerName(), time.Hour)
c, err := container.New(sk.DockerName(), time.Hour)
if err != nil {
return
}
@@ -521,7 +522,7 @@ func installKernel(sk config.KernelMask, pkgname string, force, headers bool) (e
Str("pkg", pkgname).
Logger()
c, err := NewContainer(sk.DockerName(), time.Hour) // TODO conf
c, err := container.New(sk.DockerName(), time.Hour) // TODO conf
if err != nil {
return
}
@@ -640,7 +641,7 @@ func findInitrdFile(files []os.FileInfo, kname string) (name string, err error)
return
}
func genRootfsImage(d containerImageInfo, download bool) (rootfs string, err error) {
func genRootfsImage(d container.Image, download bool) (rootfs string, err error) {
usr, err := user.Current()
if err != nil {
return
@@ -672,7 +673,7 @@ func updateKernelsCfg(host, download bool) (err error) {
}
// Get docker kernels
dockerImages, err := listContainerImages()
dockerImages, err := container.Images()
if err != nil {
return
}
@@ -715,7 +716,7 @@ func updateKernelsCfg(host, download bool) (err error) {
return
}
func listContainersKernels(dii containerImageInfo, newkcfg *config.KernelConfig,
func listContainersKernels(dii container.Image, newkcfg *config.KernelConfig,
download bool) (err error) {
rootfs, err := genRootfsImage(dii, download)
@@ -723,7 +724,7 @@ func listContainersKernels(dii containerImageInfo, newkcfg *config.KernelConfig,
return
}
c, err := NewContainer(dii.Name, time.Hour)
c, err := container.New(dii.Name, time.Hour)
if err != nil {
return
}
@@ -827,7 +828,7 @@ func generateKernels(km config.KernelMask, registry string,
log.Info().Msgf("Generating for kernel mask %v", km)
_, err = genRootfsImage(containerImageInfo{Name: km.DockerName()},
_, err = genRootfsImage(container.Image{Name: km.DockerName()},
download)
if err != nil || *shutdown {
return

3
kernel_linux.go
View File

@@ -16,6 +16,7 @@ import (
"github.com/zcalusic/sysinfo"
"code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/container"
"code.dumpstack.io/tools/out-of-tree/fs"
)
@@ -44,7 +45,7 @@ func genHostKernels(download bool) (kcfg config.KernelConfig, err error) {
}
// only for compatibility, docker is not really used
dii := containerImageInfo{
dii := container.Image{
Name: config.KernelMask{
DistroType: distroType,
DistroRelease: si.OS.Version,

3
main.go
View File

@@ -23,6 +23,7 @@ import (
"github.com/alecthomas/kong"
"code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/container"
"code.dumpstack.io/tools/out-of-tree/fs"
)
@@ -189,7 +190,7 @@ func main() {
cli.ContainerRuntime)
}
}
containerRuntime = cli.ContainerRuntime
container.Runtime = cli.ContainerRuntime
err = ctx.Run(&cli.Globals)
ctx.FatalIfErrorf(err)

5
pew.go
View File

@@ -24,6 +24,7 @@ import (
"gopkg.in/logrusorgru/aurora.v2"
"code.dumpstack.io/tools/out-of-tree/config"
"code.dumpstack.io/tools/out-of-tree/container"
"code.dumpstack.io/tools/out-of-tree/fs"
"code.dumpstack.io/tools/out-of-tree/qemu"
)
@@ -268,8 +269,8 @@ func build(flog zerolog.Logger, tmp string, ka config.Artifact,
}
if ki.ContainerName != "" {
var c container
c, err = NewContainer(ki.ContainerName, dockerTimeout)
var c container.Container
c, err = container.New(ki.ContainerName, dockerTimeout)
c.Log = flog
if err != nil {
log.Fatal().Err(err).Msg("container creation failure")