Use single temp directory base

This reverts commit 5931c08de1.

.github/workflows/donate.yml

@@ -1,25 +0,0 @@
-name: donate
-
-on:
-  issues:
-    types: [opened, closed]
-  schedule:
-    - cron: '15 * * * *' # for updating balance
-
-jobs:
-  comment:
-    runs-on: ubuntu-latest
-    steps:
-      - env:
-          # https://github.com/jollheef/donate/blob/master/dashboard/whitelist.go
-          DASHBOARD_ACCESS_TOKEN: ${{ secrets.DONATE_DASHBOARD_ACCESS_TOKEN }}
-          # the scope is current repository only
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          TOOL: "https://github.com/jollheef/donate/archive/master.tar.gz"
-        run: |
-          curl https://nixos.org/nix/install | sh
-          . ~/.nix-profile/etc/profile.d/nix.sh
-          # Use latest stable nixpkgs channel
-          nix-channel --add https://nixos.org/channels/nixos-19.09 nixpkgs
-          nix-channel --update
-          nix run -f $TOOL -c donate-ci

.github/workflows/macos.yml

@@ -1,6 +1,6 @@
 name: macOS
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   build:

.github/workflows/ubuntu.yml

@@ -1,6 +1,6 @@
 name: Ubuntu
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   build:
@@ -21,7 +21,7 @@ jobs:
     - name: Install dependencies for tests
       run: |
         sudo apt-get update
-        sudo apt-get install qemu
+        sudo apt-get install qemu-system-x86
 
     - name: Bootstrap
       run: ./tools/qemu-debian-img/bootstrap.sh
@@ -41,16 +41,90 @@ jobs:
     - name: Install dependencies for tests
       run: |
         sudo apt-get update
-        sudo apt-get install qemu
+        sudo apt-get install qemu-system-x86
 
     - name: End-to-End Testing [Kernel Module]
       run: |
         cd examples/kernel-module
-        ../../out-of-tree kernel autogen --max=1
-        ../../out-of-tree pew --qemu-timeout=10m
+        ../../out-of-tree --log-level=debug kernel autogen --max=1
+        ../../out-of-tree --log-level=debug pew --qemu-timeout=10m
 
     - name: End-to-End Testing [Kernel Exploit]
       run: |
         cd examples/kernel-exploit
-        ../../out-of-tree kernel autogen --max=1
-        ../../out-of-tree pew --threshold=0 --qemu-timeout=10m
+        ../../out-of-tree --log-level=debug kernel autogen --max=1
+        ../../out-of-tree --log-level=debug pew --threshold=0 --qemu-timeout=10m
+
+    - name: Archive logs
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-end-to-end-logs
+        path: /home/runner/.out-of-tree/logs/out-of-tree.log
+
+  test-end-to-end-kernels:
+    name: End-to-End Testing (kernels)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build
+
+    - name: Install dependencies for tests
+      run: |
+        sudo apt-get update
+        sudo apt-get install qemu-system-x86
+
+    - name: End-to-End Testing [Install one Ubuntu 18.04 kernel]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=Ubuntu --ver=18.04 --kernel=4.15.0-70-generic
+
+    - name: End-to-End Testing [Reinstall one Ubuntu 18.04 kernel]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=Ubuntu --ver=18.04 --kernel=4.15.0-70-generic --force
+
+    - name: End-to-End Testing [Install one Ubuntu 22.04 kernel w/o headers]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=Ubuntu --ver=22.04 --kernel=5.19.0-28-generic --no-headers
+
+    - name: End-to-End Testing [Install one CentOS 7 kernel]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=CentOS --ver=7 --kernel=3.10.0-862.6.3
+
+    - name: End-to-End Testing [Install one CentOS 7 kernel w/o headers]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=CentOS --ver=7 --kernel=3.10.0-1160.71.1 --no-headers
+
+    - name: End-to-End Testing [Install one CentOS 8 kernel]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=CentOS --ver=8 --kernel=4.18.0-348.7.1
+
+    - name: Archive logs
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-end-to-end-kernels-log
+        path: /home/runner/.out-of-tree/logs/out-of-tree.log
+
+  test-end-to-end-genall:
+    name: End-to-End Testing (genall)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build
+
+    - name: Install dependencies for tests
+      run: |
+        sudo apt-get update
+        sudo apt-get install qemu-system-x86
+
+    - name: End-to-End Testing [Install all Ubuntu 22.04 kernels]
+      run: |
+        ./out-of-tree --log-level=debug kernel genall --distro=Ubuntu --ver=22.04
+
+    - name: Archive logs
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-end-to-end-genall-logs
+        path: /home/runner/.out-of-tree/logs/out-of-tree.log

.gitignore

@@ -10,3 +10,5 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+
+out-of-tree

CHANGELOG.md

@@ -4,6 +4,75 @@
 
 [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.0.0]
+
+### Breaking
+
+- Layers with kernels in containers have been abandoned in favor of
+  installation to mounted volumes.
+
+- Command line interface has been changed to alecthomas/kong.
+
+### Added
+
+- Command `kernel install` to install specific kernel.
+
+- Command `containers` to manage containers.
+
+- Command `image edit` to edit qemu image.
+
+- Flag `--force` to force reinstallation of the kernel.
+
+- Flag `--artifact-config` to specify the path to .out-of-tree.toml.
+
+- Flag `--no-headers` flag to install kernel and initrd only.
+
+- Flag `--shuffle` to randomize the order of kernels for
+  installation/testing.
+
+- Support make targets in artifact config.
+
+- Support patches in artifact config.
+
+- Support for copying standard modules to qemu.
+
+- Script artifact type for various automation and information gathering.
+
+- Add TestFiles to artifact config, transfers additional test files to VM.
+
+- Improved logging, with logfile at ~/.out-of-tree/logs/out-of-tree.log
+
+- Kernel installation will retry (10 times by default) in case of
+  network problems.
+
+- Stdout trace (with --log-level=trace, and always to logfile) for
+  qemu and container execution.
+
+- Compatibility with Podman.
+
+- Support for Ubuntu 22.04.
+
+## [1.4.0]
+
+### Added
+
+- Parameter `--docker-timeout` may also be set in the artifact
+  configuration file.
+
+- Preload modules before inserting module or run exploit. Modules can
+  be specified by git repository path in the `repo` parameter of
+  section `[[preload]]`. Also, there is a `path` parameter for local
+  projects. Note that `repo` is using a cache that uses last commit
+  hash to check is project needs to be rebuilt, so it's not suitable
+  for local development (except if you will commit each time before
+  run out-of-tree).
+
+- Flag `--disable-preload` to ignore `[[preload]]` section of
+  configuration file.
+
+- Now `out-of-tree log dump` will show the last log if no ID
+  specified.
+
 ## [1.3.0] 2020-05-30
 
 ### Added
@@ -162,7 +231,7 @@
 - Temporary files is moved to `~/.out-of-tree/tmp/` to avoid docker
   mounting issues on some systems.
 
-## [0.2.0] - 2019-12-01
+## [0.2.0] - 2018-12-01
 
 The main purpose of the release is to simplify installation.
 
@@ -184,7 +253,7 @@ The main purpose of the release is to simplify installation.
 
 - No warning anymore if test.sh is not exists.
 
-## [0.1.0] - 2019-11-20
+## [0.1.0] - 2018-11-20
 
 Initial release that was never tagged.
 

README.md

@@ -1,8 +1,6 @@
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/aba4aad2046b4d1a9a99cf98e22c018b)](https://app.codacy.com/app/jollheef/out-of-tree?utm_source=github.com&utm_medium=referral&utm_content=jollheef/out-of-tree&utm_campaign=Badge_Grade_Dashboard)
 [![Go Report Card](https://goreportcard.com/badge/code.dumpstack.io/tools/out-of-tree)](https://goreportcard.com/report/code.dumpstack.io/tools/out-of-tree)
 [![Documentation Status](https://readthedocs.org/projects/out-of-tree/badge/?version=latest)](https://out-of-tree.readthedocs.io/en/latest/?badge=latest)
-[![Donate](https://img.shields.io/badge/donate-paypal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R8W2UQPZ5X5JE&source=url)
-[![Donate](https://img.shields.io/badge/donate-bitcoin-green.svg)](https://blockchair.com/bitcoin/address/bc1q23fyuq7kmngrgqgp6yq9hk8a5q460f39m8nv87)
 
 # [out-of-tree](https://out-of-tree.io)
 
@@ -18,66 +16,34 @@ out-of-tree is for automating some routine actions for creating development envi
 
     $ curl -fsSL https://get.docker.com | sh
 	$ sudo usermod -aG docker user && newgrp docker
-    $ curl https://nixos.org/nix/install | sh
-    $ nix-env -iA nixpkgs.out-of-tree
+    $ curl -L https://nixos.org/nix/install | sh
+    $ nix-env -iA nixpkgs.out-of-tree # Note: may not be up to date immediately, in this case consider installing from source
 
 Note that adding a user to group *docker* has serious security implications. Check Docker documentation for more information.
 
 ### macOS
 
-    $ brew cask install docker
+    $ brew install --cask docker
     $ open --background -a Docker && sleep 1m
-    $ brew tap jollheef/repo
+    $ brew tap out-of-tree/repo
     $ brew install out-of-tree
 
 Read [documentation](https://out-of-tree.readthedocs.io) for further info.
 
 ## Examples
 
-Run by absolute path
+Generate all Ubuntu 22.04 kernels:
 
-    $ out-of-tree --path /path/to/exploit/directory pew
+    $ out-of-tree kernel genall --distro=Ubuntu --ver=22.04
 
-Test only with one kernel:
+Run tests based on .out-of-tree.toml definitions:
 
-    $ out-of-tree pew --kernel='Ubuntu:4.10.0-30-generic'
+	$ out-of-tree pew
+
+Test with a specific kernel:
+
+    $ out-of-tree pew --kernel='Ubuntu:5.4.0-29-generic'
 
 Run debug environment:
 
-    $ out-of-tree debug --kernel='Ubuntu:4.10.0-30-generic'
-
-Test binary module/exploit with implicit defined test ($BINARY_test)
-
-    $ out-of-tree pew --binary /path/to/exploit
-
-Test binary module/exploit with explicit defined test
-
-    $ out-of-tree pew --binary /path/to/exploit --test /path/to/exploit_test
-
-Guess work kernels:
-
-    $ out-of-tree pew --guess
-
-Use custom kernels config
-
-    $ out-of-tree --kernels /path/to/kernels.toml pew
-
-Generate all kernels
-
-    $ out-of-tree kernel genall --distro Ubuntu --ver 16.04
-
-## Troubleshooting
-
-If anything happens that you cannot solve -- just remove `$HOME/.out-of-tree`.
-
-But it'll be better if you'll write the bug report.
-
-## Development
-
-Read [Qemu API](qemu/README.md).
-
-### Generate images
-
-    $ cd $GOPATH/src/code.dumpstack.io/tools/out-of-tree/tools/qemu-debian-img/
-    $ docker run --privileged -v $(pwd):/shared -e IMAGE=/shared/ubuntu1404.img -e RELEASE=trusty -t gen-ubuntu1804-image
-    $ docker run --privileged -v $(pwd):/shared -e IMAGE=/shared/ubuntu1604.img -e RELEASE=xenial -t gen-ubuntu1804-image
+    $ out-of-tree debug --kernel='Ubuntu:5.4.0-29-generic'

config/config.go

@@ -49,10 +49,12 @@ const (
 	KernelModule ArtifactType = iota
 	// KernelExploit is the privilege escalation exploit
 	KernelExploit
+	// Script for information gathering or automation
+	Script
 )
 
 func (at ArtifactType) String() string {
-	return [...]string{"module", "exploit"}[at]
+	return [...]string{"module", "exploit", "script"}[at]
 }
 
 // UnmarshalTOML is for support github.com/naoina/toml
@@ -63,6 +65,8 @@ func (at *ArtifactType) UnmarshalTOML(data []byte) (err error) {
 		*at = KernelModule
 	} else if strings.Contains(stypelower, "exploit") {
 		*at = KernelExploit
+	} else if strings.Contains(stypelower, "script") {
+		*at = Script
 	} else {
 		err = fmt.Errorf("Type %s is unsupported", stype)
 	}
@@ -77,6 +81,8 @@ func (at ArtifactType) MarshalTOML() (data []byte, err error) {
 		s = "module"
 	case KernelExploit:
 		s = "exploit"
+	case Script:
+		s = "script"
 	default:
 		err = fmt.Errorf("Cannot marshal %d", at)
 	}
@@ -102,25 +108,61 @@ func (d Duration) MarshalTOML() (data []byte, err error) {
 	return
 }
 
+type PreloadModule struct {
+	Repo             string
+	Path             string
+	TimeoutAfterLoad Duration
+}
+
+// Extra test files to copy over
+type FileTransfer struct {
+	User   string
+	Local  string
+	Remote string
+}
+
+type Patch struct {
+	Path   string
+	Source string
+	Script string
+}
+
 // Artifact is for .out-of-tree.toml
 type Artifact struct {
 	Name             string
 	Type             ArtifactType
+	TestFiles        []FileTransfer
 	SourcePath       string
 	SupportedKernels []KernelMask
 
+	Script string
+
 	Qemu struct {
 		Cpus    int
 		Memory  int
 		Timeout Duration
 	}
 
+	Docker struct {
+		Timeout Duration
+	}
+
 	Mitigations struct {
 		DisableSmep  bool
 		DisableSmap  bool
 		DisableKaslr bool
 		DisableKpti  bool
 	}
+
+	Patches []Patch
+
+	Make struct {
+		Target string
+	}
+
+	StandardModules bool
+
+	Preload []PreloadModule
 }
 
 func (ka Artifact) checkSupport(ki KernelInfo, km KernelMask) (
@@ -230,9 +272,11 @@ type KernelInfo struct {
 	ContainerName string
 
 	// Runtime information
-	KernelPath string
-	InitrdPath string
-	RootFS     string
+	KernelPath  string
+	InitrdPath  string
+	ModulesPath string
+
+	RootFS string
 
 	// Debug symbols
 	VmlinuxPath string

config/out-of-tree.go

@@ -5,8 +5,13 @@
 package config
 
 import (
+	"errors"
+	"os"
 	"os/user"
+	"time"
 
+	"github.com/alecthomas/kong"
+	"github.com/mitchellh/go-homedir"
 	"github.com/naoina/toml"
 )
 
@@ -22,11 +27,11 @@ type OutOfTree struct {
 	Database string
 
 	Qemu struct {
-		Timeout string
+		Timeout Duration
 	}
 
 	Docker struct {
-		Timeout  string
+		Timeout  Duration
 		Registry string
 
 		// Commands that will be executed before
@@ -35,6 +40,30 @@ type OutOfTree struct {
 	}
 }
 
+func (c *OutOfTree) Decode(ctx *kong.DecodeContext) (err error) {
+	if ctx.Value.Set {
+		return
+	}
+
+	s, err := homedir.Expand(ctx.Scan.Pop().String())
+	if err != nil {
+		return
+	}
+
+	defaultValue, err := homedir.Expand(ctx.Value.Default)
+	if err != nil {
+		return
+	}
+
+	_, err = os.Stat(s)
+	if s != defaultValue && errors.Is(err, os.ErrNotExist) {
+		return errors.New("'" + s + "' does not exist")
+	}
+
+	*c, err = ReadOutOfTreeConf(s)
+	return
+}
+
 func ReadOutOfTreeConf(path string) (c OutOfTree, err error) {
 	buf, err := readFileAll(path)
 	if err == nil {
@@ -65,12 +94,12 @@ func ReadOutOfTreeConf(path string) (c OutOfTree, err error) {
 		c.Database = usr.HomeDir + "/.out-of-tree/db.sqlite"
 	}
 
-	if c.Qemu.Timeout == "" {
-		c.Qemu.Timeout = "1m"
+	if c.Qemu.Timeout.Duration == 0 {
+		c.Qemu.Timeout.Duration = time.Minute
 	}
 
-	if c.Docker.Timeout == "" {
-		c.Docker.Timeout = "1m"
+	if c.Docker.Timeout.Duration == 0 {
+		c.Docker.Timeout.Duration = time.Minute
 	}
 
 	return

container.go

@@ -0,0 +1,237 @@
+// Copyright 2023 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"os/user"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/rs/zerolog/log"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+)
+
+type ContainerCmd struct {
+	Filter string `help:"filter by name"`
+
+	List    ContainerListCmd    `cmd:"" help:"list containers"`
+	Cleanup ContainerCleanupCmd `cmd:"" help:"cleanup containers"`
+}
+
+func (cmd ContainerCmd) Containers() (names []string) {
+	images, err := listContainerImages()
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+
+	for _, img := range images {
+		if cmd.Filter != "" && !strings.Contains(img.Name, cmd.Filter) {
+			continue
+		}
+		names = append(names, img.Name)
+	}
+	return
+}
+
+type ContainerListCmd struct{}
+
+func (cmd ContainerListCmd) Run(containerCmd *ContainerCmd) (err error) {
+	for _, name := range containerCmd.Containers() {
+		fmt.Println(name)
+	}
+	return
+}
+
+type ContainerCleanupCmd struct{}
+
+func (cmd ContainerCleanupCmd) Run(containerCmd *ContainerCmd) (err error) {
+	var output []byte
+	for _, name := range containerCmd.Containers() {
+		output, err = exec.Command("docker", "image", "rm", name).CombinedOutput()
+		if err != nil {
+			log.Error().Err(err).Str("output", string(output)).Msg("")
+			return
+		}
+	}
+	return
+}
+
+type containerImageInfo struct {
+	Name          string
+	DistroType    config.DistroType
+	DistroRelease string // 18.04/7.4.1708/9.1
+}
+
+func listContainerImages() (diis []containerImageInfo, err error) {
+	cmd := exec.Command("docker", "images")
+	log.Debug().Msgf("%v", cmd)
+
+	rawOutput, err := cmd.CombinedOutput()
+	if err != nil {
+		return
+	}
+
+	r, err := regexp.Compile("out_of_tree_.*")
+	if err != nil {
+		return
+	}
+
+	containers := r.FindAll(rawOutput, -1)
+	for _, c := range containers {
+		container := strings.Fields(string(c))[0]
+
+		s := strings.Replace(container, "__", ".", -1)
+		values := strings.Split(s, "_")
+		distro, ver := values[3], values[4]
+
+		dii := containerImageInfo{
+			Name:          container,
+			DistroRelease: ver,
+		}
+
+		dii.DistroType, err = config.NewDistroType(distro)
+		if err != nil {
+			return
+		}
+
+		diis = append(diis, dii)
+	}
+	return
+}
+
+type container struct {
+	name    string
+	timeout time.Duration
+	Volumes struct {
+		LibModules string
+		UsrSrc     string
+		Boot       string
+	}
+	// Additional arguments
+	Args []string
+}
+
+func NewContainer(name string, timeout time.Duration) (c container, err error) {
+	c.name = name
+	c.timeout = timeout
+
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+
+	c.Volumes.LibModules = fmt.Sprintf(
+		"%s/.out-of-tree/volumes/%s/lib/modules", usr.HomeDir, name)
+	os.MkdirAll(c.Volumes.LibModules, 0777)
+
+	c.Volumes.UsrSrc = fmt.Sprintf(
+		"%s/.out-of-tree/volumes/%s/usr/src", usr.HomeDir, name)
+	os.MkdirAll(c.Volumes.UsrSrc, 0777)
+
+	c.Volumes.Boot = fmt.Sprintf(
+		"%s/.out-of-tree/volumes/%s/boot", usr.HomeDir, name)
+	os.MkdirAll(c.Volumes.Boot, 0777)
+
+	return
+}
+
+func (c container) Build(imagePath string) (output string, err error) {
+	args := []string{"build"}
+	args = append(args, "-t", c.name, imagePath)
+
+	cmd := exec.Command("docker", args...)
+
+	flog := log.With().
+		Str("command", fmt.Sprintf("%v", cmd)).
+		Logger()
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	cmd.Stderr = cmd.Stdout
+
+	err = cmd.Start()
+	if err != nil {
+		return
+	}
+
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			m := scanner.Text()
+			output += m + "\n"
+			flog.Trace().Str("stdout", m).Msg("")
+		}
+	}()
+
+	err = cmd.Wait()
+	return
+}
+
+func (c container) Run(workdir string, command string) (output string, err error) {
+	flog := log.With().
+		Str("container", c.name).
+		Str("workdir", workdir).
+		Str("command", command).
+		Logger()
+
+	var args []string
+	args = append(args, "run", "--rm")
+	args = append(args, c.Args...)
+	args = append(args,
+		"-v", workdir+":/work",
+		"-v", c.Volumes.LibModules+":/lib/modules",
+		"-v", c.Volumes.UsrSrc+":/usr/src",
+		"-v", c.Volumes.Boot+":/boot")
+	args = append(args, c.name, "bash", "-c", "cd /work && "+command)
+
+	cmd := exec.Command("docker", args...)
+
+	log.Debug().Msgf("%v", cmd)
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	cmd.Stderr = cmd.Stdout
+
+	timer := time.AfterFunc(c.timeout, func() {
+		flog.Info().Msg("killing container by timeout")
+		cmd.Process.Kill()
+	})
+	defer timer.Stop()
+
+	err = cmd.Start()
+	if err != nil {
+		return
+	}
+
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			m := scanner.Text()
+			output += m + "\n"
+			flog.Trace().Str("stdout", m).Msg("")
+		}
+	}()
+
+	err = cmd.Wait()
+	if err != nil {
+		e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`",
+			err, command, output)
+		err = errors.New(e)
+		return
+	}
+
+	return
+}

db.go

@@ -254,6 +254,24 @@ func getLogByID(db *sql.DB, id int) (le logEntry, err error) {
 	return
 }
 
+func getLastLog(db *sql.DB) (le logEntry, err error) {
+	err = db.QueryRow("SELECT MAX(id), time, name, type, tag, "+
+		"distro_type, distro_release, kernel_release, "+
+		"build_ok, run_ok, test_ok, "+
+		"build_output, run_output, test_output, "+
+		"qemu_stdout, qemu_stderr, "+
+		"kernel_panic, timeout_kill "+
+		"FROM log").Scan(&le.ID, &le.Timestamp,
+		&le.Name, &le.Type, &le.Tag,
+		&le.DistroType, &le.DistroRelease, &le.KernelRelease,
+		&le.Build.Ok, &le.Run.Ok, &le.Test.Ok,
+		&le.Build.Output, &le.Run.Output, &le.Test.Output,
+		&le.Stdout, &le.Stderr,
+		&le.KernelPanic, &le.KilledByTimeout,
+	)
+	return
+}
+
 func createSchema(db *sql.DB) (err error) {
 	err = createMetadataTable(db)
 	if err != nil {

debug.go

@@ -8,17 +8,217 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"os"
 	"strings"
 	"time"
 
+	"github.com/rs/zerolog/log"
 	"gopkg.in/logrusorgru/aurora.v2"
 
 	"code.dumpstack.io/tools/out-of-tree/config"
 	"code.dumpstack.io/tools/out-of-tree/qemu"
 )
 
+type DebugCmd struct {
+	Kernel string `help:"regexp (first match)" required:""`
+	Gdb    string `help:"gdb listen address" default:"tcp::1234"`
+
+	SshAddr string `help:"ssh address to listen" default:"127.0.0.1"`
+	SshPort int    `help:"ssh port to listen" default:"50022"`
+
+	ArtifactConfig string `help:"path to artifact config" type:"path"`
+
+	Kaslr bool `help:"Enable KASLR"`
+	Smep  bool `help:"Enable SMEP"`
+	Smap  bool `help:"Enable SMAP"`
+	Kpti  bool `help:"Enable KPTI"`
+
+	NoKaslr bool `help:"Disable KASLR"`
+	NoSmep  bool `help:"Disable SMEP"`
+	NoSmap  bool `help:"Disable SMAP"`
+	NoKpti  bool `help:"Disable KPTI"`
+}
+
+// TODO: merge with pew.go
+func (cmd *DebugCmd) Run(g *Globals) (err error) {
+	kcfg, err := config.ReadKernelConfig(g.Config.Kernels)
+	if err != nil {
+		log.Print(err)
+	}
+
+	var configPath string
+	if cmd.ArtifactConfig == "" {
+		configPath = g.WorkDir + "/.out-of-tree.toml"
+	} else {
+		configPath = cmd.ArtifactConfig
+	}
+	ka, err := config.ReadArtifactConfig(configPath)
+	if err != nil {
+		return
+	}
+
+	if ka.SourcePath == "" {
+		ka.SourcePath = g.WorkDir
+	}
+
+	ki, err := firstSupported(kcfg, ka, cmd.Kernel)
+	if err != nil {
+		return
+	}
+
+	kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
+	q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
+	if err != nil {
+		return
+	}
+
+	err = q.SetSSHAddrPort(cmd.SshAddr, cmd.SshPort)
+	if err != nil {
+		return
+	}
+
+	if ka.Qemu.Cpus != 0 {
+		q.Cpus = ka.Qemu.Cpus
+	}
+	if ka.Qemu.Memory != 0 {
+		q.Memory = ka.Qemu.Memory
+	}
+
+	if ka.Docker.Timeout.Duration != 0 {
+		g.Config.Docker.Timeout.Duration = ka.Docker.Timeout.Duration
+	}
+
+	q.SetKASLR(false) // set KASLR to false by default because of gdb
+	q.SetSMEP(!ka.Mitigations.DisableSmep)
+	q.SetSMAP(!ka.Mitigations.DisableSmap)
+	q.SetKPTI(!ka.Mitigations.DisableKpti)
+
+	if cmd.Kaslr {
+		q.SetKASLR(true)
+	} else if cmd.NoKaslr {
+		q.SetKASLR(false)
+	}
+
+	if cmd.Smep {
+		q.SetSMEP(true)
+	} else if cmd.NoSmep {
+		q.SetSMEP(false)
+	}
+
+	if cmd.Smap {
+		q.SetSMAP(true)
+	} else if cmd.NoSmap {
+		q.SetSMAP(false)
+	}
+
+	if cmd.Kpti {
+		q.SetKPTI(true)
+	} else if cmd.NoKpti {
+		q.SetKPTI(false)
+	}
+
+	redgreen := func(name string, enabled bool) aurora.Value {
+		if enabled {
+			return aurora.BgGreen(aurora.Black(name))
+		}
+
+		return aurora.BgRed(aurora.White(name))
+	}
+
+	fmt.Printf("[*] %s %s %s %s\n",
+		redgreen("KASLR", q.GetKASLR()),
+		redgreen("SMEP", q.GetSMEP()),
+		redgreen("SMAP", q.GetSMAP()),
+		redgreen("KPTI", q.GetKPTI()))
+
+	fmt.Printf("[*] SMP: %d CPUs\n", q.Cpus)
+	fmt.Printf("[*] Memory: %d MB\n", q.Memory)
+
+	q.Debug(cmd.Gdb)
+	coloredGdbAddress := aurora.BgGreen(aurora.Black(cmd.Gdb))
+	fmt.Printf("[*] gdb is listening on %s\n", coloredGdbAddress)
+
+	err = q.Start()
+	if err != nil {
+		return
+	}
+	defer q.Stop()
+
+	tmp, err := ioutil.TempDir(tempDirBase, "out-of-tree_")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(tmp)
+
+	err = q.WaitForSSH(time.Minute)
+	if err != nil {
+		return
+	}
+
+	if ka.StandardModules {
+		// Module depends on one of the standard modules
+		err = copyStandardModules(q, ki)
+		if err != nil {
+			log.Print(err)
+			return
+		}
+	}
+
+	err = preloadModules(q, ka, ki, g.Config.Docker.Timeout.Duration)
+	if err != nil {
+		log.Print(err)
+		return
+	}
+
+	var buildDir, outFile, output, remoteFile string
+
+	if ka.Type == config.Script {
+		err = q.CopyFile("root", ka.Script, ka.Script)
+		if err != nil {
+			return
+		}
+	} else {
+		buildDir, outFile, output, err = build(tmp, ka, ki, g.Config.Docker.Timeout.Duration)
+		if err != nil {
+			log.Print(err, output)
+			return
+		}
+
+		remoteFile = "/tmp/exploit"
+		if ka.Type == config.KernelModule {
+			remoteFile = "/tmp/module.ko"
+		}
+
+		err = q.CopyFile("user", outFile, remoteFile)
+		if err != nil {
+			return
+		}
+	}
+
+	// Copy all test files to the remote machine
+	for _, f := range ka.TestFiles {
+		if f.Local[0] != '/' {
+			f.Local = buildDir + "/" + f.Local
+		}
+		err = q.CopyFile(f.User, f.Local, f.Remote)
+		if err != nil {
+			log.Print("error copy err:", err, f.Local, f.Remote)
+			return
+		}
+	}
+
+	coloredRemoteFile := aurora.BgGreen(aurora.Black(remoteFile))
+	fmt.Printf("[*] build result copied to %s\n", coloredRemoteFile)
+
+	fmt.Printf("\n%s\n", q.GetSSHCommand())
+	fmt.Printf("gdb %s -ex 'target remote %s'\n\n", ki.VmlinuxPath, cmd.Gdb)
+
+	// TODO set substitute-path /build/.../linux-... /path/to/linux-source
+
+	err = interactive(q)
+	return
+}
+
 func firstSupported(kcfg config.KernelConfig, ka config.Artifact,
 	kernel string) (ki config.KernelInfo, err error) {
 
@@ -57,12 +257,12 @@ func handleLine(q *qemu.System) (err error) {
 		fmt.Printf("ssh\t: print arguments to ssh command\n")
 		fmt.Printf("quit\t: quit\n")
 	case "l", "log":
-		fmt.Println(string(q.Stdout))
+		fmt.Println(q.Stdout)
 	case "cl", "clog":
-		fmt.Println(string(q.Stdout))
-		q.Stdout = []byte{}
+		fmt.Println(q.Stdout)
+		q.Stdout = ""
 	case "c", "cleanup":
-		q.Stdout = []byte{}
+		q.Stdout = ""
 	case "s", "ssh":
 		fmt.Println(q.GetSSHCommand())
 	case "q", "quit":
@@ -81,124 +281,3 @@ func interactive(q *qemu.System) (err error) {
 		}
 	}
 }
-
-func debugHandler(kcfg config.KernelConfig, workPath, kernRegex, gdb string,
-	dockerTimeout time.Duration, yekaslr, yesmep, yesmap, yekpti,
-	nokaslr, nosmep, nosmap, nokpti bool) (err error) {
-
-	ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
-	if err != nil {
-		return
-	}
-
-	if ka.SourcePath == "" {
-		ka.SourcePath = workPath
-	}
-
-	ki, err := firstSupported(kcfg, ka, kernRegex)
-	if err != nil {
-		return
-	}
-
-	kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
-	q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
-	if err != nil {
-		return
-	}
-
-	if ka.Qemu.Cpus != 0 {
-		q.Cpus = ka.Qemu.Cpus
-	}
-	if ka.Qemu.Memory != 0 {
-		q.Memory = ka.Qemu.Memory
-	}
-
-	q.SetKASLR(false) // set KASLR to false by default because of gdb
-	q.SetSMEP(!ka.Mitigations.DisableSmep)
-	q.SetSMAP(!ka.Mitigations.DisableSmap)
-	q.SetKPTI(!ka.Mitigations.DisableKpti)
-
-	if yekaslr {
-		q.SetKASLR(true)
-	} else if nokaslr {
-		q.SetKASLR(false)
-	}
-
-	if yesmep {
-		q.SetSMEP(true)
-	} else if nosmep {
-		q.SetSMEP(false)
-	}
-
-	if yesmap {
-		q.SetSMAP(true)
-	} else if nosmap {
-		q.SetSMAP(false)
-	}
-
-	if yekpti {
-		q.SetKPTI(true)
-	} else if nokpti {
-		q.SetKPTI(false)
-	}
-
-	redgreen := func(name string, enabled bool) aurora.Value {
-		if enabled {
-			return aurora.BgGreen(aurora.Black(name))
-		}
-
-		return aurora.BgRed(aurora.White(name))
-	}
-
-	fmt.Printf("[*] %s %s %s %s\n",
-		redgreen("KASLR", q.GetKASLR()),
-		redgreen("SMEP", q.GetSMEP()),
-		redgreen("SMAP", q.GetSMAP()),
-		redgreen("KPTI", q.GetKPTI()))
-
-	fmt.Printf("[*] SMP: %d CPUs\n", q.Cpus)
-	fmt.Printf("[*] Memory: %d MB\n", q.Memory)
-
-	q.Debug(gdb)
-	coloredGdbAddress := aurora.BgGreen(aurora.Black(gdb))
-	fmt.Printf("[*] gdb is listening on %s\n", coloredGdbAddress)
-
-	err = q.Start()
-	if err != nil {
-		return
-	}
-	defer q.Stop()
-
-	tmp, err := ioutil.TempDir("/tmp/", "out-of-tree_")
-	if err != nil {
-		return
-	}
-	defer os.RemoveAll(tmp)
-
-	outFile, output, err := build(tmp, ka, ki, dockerTimeout)
-	if err != nil {
-		log.Println(err, output)
-		return
-	}
-
-	remoteFile := "/tmp/exploit"
-	if ka.Type == config.KernelModule {
-		remoteFile = "/tmp/module.ko"
-	}
-
-	err = q.CopyFile("user", outFile, remoteFile)
-	if err != nil {
-		return
-	}
-
-	coloredRemoteFile := aurora.BgGreen(aurora.Black(remoteFile))
-	fmt.Printf("[*] build result copied to %s\n", coloredRemoteFile)
-
-	fmt.Printf("\n%s\n", q.GetSSHCommand())
-	fmt.Printf("gdb %s -ex 'target remote %s'\n\n", ki.VmlinuxPath, gdb)
-
-	// TODO set substitute-path /build/.../linux-... /path/to/linux-source
-
-	err = interactive(q)
-	return
-}

docs/installation.rst

@@ -10,7 +10,7 @@ Ubuntu
 Install dependencies::
 
   $ sudo snap install go --classic
-  $ sudo snap install docker
+  $ # Install docker: https://docs.docker.com/engine/install/ubuntu/
   $ sudo apt install qemu-system-x86 build-essential gdb
 
 macOS
@@ -53,15 +53,16 @@ Install dependencies::
 Common
 ======
 
-Setup Go environment::
+Setup environment::
 
-  $ echo 'export GOPATH=$HOME' >> ~/.bashrc
   $ echo 'export PATH=$PATH:$HOME/bin' >> ~/.bashrc
   $ source ~/.bashrc
 
 Build *out-of-tree*::
 
-  $ go get -u code.dumpstack.io/tools/out-of-tree
+  $ git clone https://code.dumpstack.io/tools/out-of-tree
+  $ cd out-of-tree
+  $ CGO_ENABLED=1 go build -o ~/bin/out-of-tree
 
 .. note::
   On a GNU/Linux you need to add your user to docker group if you want
@@ -71,7 +72,7 @@ Build *out-of-tree*::
 
 Test that everything works::
 
-  $ cd $GOPATH/src/code.dumpstack.io/tools/out-of-tree/examples/kernel-exploit
+  $ cd out-of-tree/examples/kernel-exploit
   $ out-of-tree kernel autogen --max=1
   $ out-of-tree pew --max=1
 

examples/preload/.out-of-tree.toml

@@ -0,0 +1,12 @@
+name = "out-of-tree preload"
+type = "module"
+
+[[supported_kernels]]
+distro_type = "Ubuntu"
+distro_release = "18.04"
+release_mask = ".*"
+
+[[preload]]
+repo = "https://github.com/openwall/lkrg"
+#path = "/local/path/to/lkrg"
+timeout_after_load = "1s"

examples/preload/Makefile

@@ -0,0 +1,11 @@
+KERNEL := /lib/modules/$(shell uname -r)/build
+TARGET := module
+
+obj-m += $(TARGET).o
+$(TARGET)-objs = module.o
+
+all:
+	make -C $(KERNEL) M=$(PWD) modules
+
+clean:
+	make -C $(KERNEL) M=$(PWD) clean

examples/preload/README.md

@@ -0,0 +1,5 @@
+# out-of-tree kernel module preload example
+
+See .out-of-tree.toml
+
+Note that it should fail to insert module if lkrg is enabled in the preload list.

examples/preload/module.c

@@ -0,0 +1,17 @@
+#include <linux/module.h>
+#include <linux/slab.h>
+
+int init_module(void)
+{
+	char *argv[] = { "/bin/sh", "--help", NULL };
+	char *envp[] = { NULL };
+
+	/* trigger lkrg */
+	return call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC);
+}
+
+void cleanup_module(void)
+{
+}
+
+MODULE_LICENSE("GPL");

examples/script/.out-of-tree.toml

@@ -0,0 +1,11 @@
+# out-of-tree configuration file
+# docs at https://out-of-tree.io
+name = "out-of-tree script example"
+type = "script"
+
+script = "script.sh"
+
+[[supported_kernels]]
+distro_type = "Ubuntu"
+distro_release = "22.04"
+release_mask = ".*"

examples/script/README.md

@@ -0,0 +1,3 @@
+# out-of-tree script example
+
+See .out-of-tree.toml

examples/script/script.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+uname -a
+
+ls /proc | grep config

gen.go

@@ -12,6 +12,20 @@ import (
 	"code.dumpstack.io/tools/out-of-tree/config"
 )
 
+type GenCmd struct {
+	Type string `enum:"module,exploit" required:"" help:"module/exploit"`
+}
+
+func (cmd *GenCmd) Run(g *Globals) (err error) {
+	switch cmd.Type {
+	case "module":
+		err = genConfig(config.KernelModule)
+	case "exploit":
+		err = genConfig(config.KernelExploit)
+	}
+	return
+}
+
 func genConfig(at config.ArtifactType) (err error) {
 	a := config.Artifact{
 		Name: "Put name here",
@@ -22,6 +36,12 @@ func genConfig(at config.ArtifactType) (err error) {
 		DistroRelease: "18.04",
 		ReleaseMask:   ".*",
 	})
+	a.Preload = append(a.Preload, config.PreloadModule{
+		Repo: "Repo name (e.g. https://github.com/openwall/lkrg)",
+	})
+	a.Patches = append(a.Patches, config.Patch{
+		Path: "/path/to/profiling.patch",
+	})
 
 	buf, err := toml.Marshal(&a)
 	if err != nil {

go.mod

@@ -1,26 +1,50 @@
 module code.dumpstack.io/tools/out-of-tree
 
-go 1.14
+go 1.17
 
 replace code.dumpstack.io/tools/out-of-tree/qemu => ./qemu
 
 replace code.dumpstack.io/tools/out-of-tree/config => ./config
 
 require (
-	github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect
-	github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect
-	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/mattn/go-runewidth v0.0.4 // indirect
-	github.com/mattn/go-sqlite3 v1.11.0
-	github.com/naoina/go-stringutil v0.1.0 // indirect
+	github.com/alecthomas/kong v0.7.1
+	github.com/go-git/go-git/v5 v5.6.1
+	github.com/mattn/go-sqlite3 v1.14.16
+	github.com/mitchellh/go-homedir v1.1.0
 	github.com/naoina/toml v0.1.1
-	github.com/olekukonko/tablewriter v0.0.1
-	github.com/otiai10/copy v1.0.1
-	github.com/otiai10/curr v1.0.0 // indirect
-	github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce
-	github.com/stretchr/testify v1.5.1 // indirect
-	github.com/zcalusic/sysinfo v0.0.0-20190429151633-fbadb57345c2
-	golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5
-	gopkg.in/alecthomas/kingpin.v2 v2.2.6
-	gopkg.in/logrusorgru/aurora.v2 v2.0.0-20190417123914-21d75270181e
+	github.com/natefinch/lumberjack v2.0.0+incompatible
+	github.com/olekukonko/tablewriter v0.0.5
+	github.com/otiai10/copy v1.10.0
+	github.com/remeh/sizedwaitgroup v1.0.0
+	github.com/rs/zerolog v1.29.0
+	github.com/zcalusic/sysinfo v0.9.5
+	golang.org/x/crypto v0.7.0
+	gopkg.in/logrusorgru/aurora.v2 v2.0.3
+)
+
+require (
+	github.com/BurntSushi/toml v1.2.1 // indirect
+	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
+	github.com/acomagu/bufpipe v1.0.4 // indirect
+	github.com/cloudflare/circl v1.1.0 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/go-git/gcfg v1.5.0 // indirect
+	github.com/go-git/go-billy/v5 v5.4.1 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/naoina/go-stringutil v0.1.0 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
+	github.com/sergi/go-diff v1.1.0 // indirect
+	github.com/skeema/knownhosts v1.1.0 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 )

go.sum

@@ -1,54 +1,198 @@
-bou.ke/monkey v1.0.1 h1:zEMLInw9xvNakzUUPjfS4Ds6jYPqCFx3m7bRmG5NH2U=
-bou.ke/monkey v1.0.1/go.mod h1:FgHuK96Rv2Nlf+0u1OOVDpCMdsWyOFmeeketDHE7LIg=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
+github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
+github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA=
+github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
+github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
+github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
+github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0=
+github.com/alecthomas/assert/v2 v2.1.0/go.mod h1:b/+1DI2Q6NckYi+3mXyH3wFb8qG37K/DuK80n7WefXA=
+github.com/alecthomas/kong v0.7.1 h1:azoTh0IOfwlAX3qN9sHWTxACE2oV8Bg2gAwBsMwDQY4=
+github.com/alecthomas/kong v0.7.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U=
+github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
+github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
+github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY=
+github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
+github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
+github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
+github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
+github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
+github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
+github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
+github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
+github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
+github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ=
+github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
+github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk=
+github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
+github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
+github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
+github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
+github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
-github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
-github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
+github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
+github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
+github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM=
 github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
 github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
 github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
 github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
-github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
-github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
-github.com/otiai10/copy v1.0.1 h1:gtBjD8aq4nychvRZ2CyJvFWAw0aja+VHazDdruZKGZA=
-github.com/otiai10/copy v1.0.1/go.mod h1:8bMCJrAqOtN/d9oyh5HR7HhLQMvcGMpGdwRDYsfOCHc=
-github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
-github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI=
-github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
-github.com/otiai10/mint v1.2.3 h1:PsrRBmrxR68kyNu6YlqYHbNlItc5vOkuS6LBEsNttVA=
-github.com/otiai10/mint v1.2.3/go.mod h1:YnfyPNhBvnY8bW4SGQHCs/aAFhkgySlMZbrF5U0bOVw=
-github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc=
-github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
+github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/otiai10/copy v1.10.0 h1:znyI7l134wNg/wDktoVQPxPkgvhDfGCYUasey+h0rDQ=
+github.com/otiai10/copy v1.10.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww=
+github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
+github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
+github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
+github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce h1:aP+C+YbHZfOQlutA4p4soHi7rVUqHQdWEVMSkHfDTqY=
-github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
+github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
+github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w=
+github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0=
+github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXijpFO6roag=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/zcalusic/sysinfo v0.0.0-20190429151633-fbadb57345c2 h1:uMiaKNX5zFLOa6nNtun+d/lpV5bOBh7BvE4q9jfZacQ=
-github.com/zcalusic/sysinfo v0.0.0-20190429151633-fbadb57345c2/go.mod h1:zAn3FAIbgZPYnutDND49Ivf8sb/mXYk8UjZdqMswgHg=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
+github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/zcalusic/sysinfo v0.9.5 h1:ivoHyj9aIAYkwzo1+8QgJ5s4oeE6Etx9FmZtqa4wJjQ=
+github.com/zcalusic/sysinfo v0.9.5/go.mod h1:Z/gPVufBrFc8X5sef3m6kkw3r3nlNFp+I6bvASfvBZQ=
+golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
+golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
+golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
+golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
+golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
+golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
+golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/logrusorgru/aurora.v2 v2.0.0-20190417123914-21d75270181e h1:Wc0601/F/0TByNewL9UAKk18FfwumyYyT8pJMIHcolA=
-gopkg.in/logrusorgru/aurora.v2 v2.0.0-20190417123914-21d75270181e/go.mod h1:Wm+IEn1fgFp8E2paL93oFVrHZW4toMKARNE85fDY5w8=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/logrusorgru/aurora.v2 v2.0.3 h1:5Hr76hqgwx9PednedYf5Q1dBfiPMZ2IgExR7u3tNXIE=
+gopkg.in/logrusorgru/aurora.v2 v2.0.3/go.mod h1:Wm+IEn1fgFp8E2paL93oFVrHZW4toMKARNE85fDY5w8=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
+gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
+gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

images.go

@@ -5,14 +5,108 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"os/exec"
+	"os/user"
+	"time"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+	"code.dumpstack.io/tools/out-of-tree/qemu"
+	"github.com/rs/zerolog/log"
 )
 
+type ImageCmd struct {
+	List ImageListCmd `cmd:"" help:"list images"`
+	Edit ImageEditCmd `cmd:"" help:"edit image"`
+}
+
+type ImageListCmd struct{}
+
+func (cmd *ImageListCmd) Run(g *Globals) (err error) {
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+
+	entries, err := os.ReadDir(usr.HomeDir + "/.out-of-tree/images/")
+	if err != nil {
+		return
+	}
+
+	for _, e := range entries {
+		fmt.Println(e.Name())
+	}
+
+	return
+}
+
+type ImageEditCmd struct {
+	Name string `help:"image name" required:""`
+}
+
+func (cmd *ImageEditCmd) Run(g *Globals) (err error) {
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+
+	image := usr.HomeDir + "/.out-of-tree/images/" + cmd.Name
+	if !exists(image) {
+		fmt.Println("image does not exist")
+	}
+
+	kcfg, err := config.ReadKernelConfig(g.Config.Kernels)
+	if err != nil {
+		return
+	}
+
+	if len(kcfg.Kernels) == 0 {
+		return errors.New("No kernels found")
+	}
+
+	ki := config.KernelInfo{}
+	for _, k := range kcfg.Kernels {
+		if k.RootFS == image {
+			ki = k
+			break
+		}
+	}
+
+	kernel := qemu.Kernel{
+		KernelPath: ki.KernelPath,
+		InitrdPath: ki.InitrdPath,
+	}
+
+	q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
+
+	q.Mutable = true
+
+	err = q.Start()
+	if err != nil {
+		fmt.Println("Qemu start error:", err)
+		return
+	}
+	defer q.Stop()
+
+	fmt.Print("ssh command:\n\n\t")
+	fmt.Println(q.GetSSHCommand())
+
+	fmt.Print("\npress enter to stop")
+	fmt.Scanln()
+
+	q.Command("root", "poweroff")
+
+	for !q.Died {
+		time.Sleep(time.Second)
+	}
+	return
+}
+
 // inspired by Edd Turtle code
 func downloadFile(filepath string, url string) (err error) {
 	out, err := os.Create(filepath)
@@ -53,6 +147,8 @@ func unpackTar(archive, destination string) (err error) {
 	cmd := exec.Command("tar", "-Sxf", archive)
 	cmd.Dir = destination + "/"
 
+	log.Debug().Msgf("%v", cmd)
+
 	rawOutput, err := cmd.CombinedOutput()
 	if err != nil {
 		err = fmt.Errorf("%v: %s", err, rawOutput)
@@ -63,7 +159,7 @@ func unpackTar(archive, destination string) (err error) {
 }
 
 func downloadImage(path, file string) (err error) {
-	tmp, err := ioutil.TempDir("/tmp/", "out-of-tree_")
+	tmp, err := ioutil.TempDir(tempDirBase, "out-of-tree_")
 	if err != nil {
 		return
 	}

kernel.go

@@ -1,4 +1,4 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Copyright 2023 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
@@ -8,7 +8,6 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"math"
 	"math/rand"
 	"os"
@@ -20,47 +19,173 @@ import (
 	"time"
 
 	"github.com/naoina/toml"
+	"github.com/rs/zerolog/log"
 
 	"code.dumpstack.io/tools/out-of-tree/config"
 )
 
-const kernelsAll int64 = math.MaxInt64
+type KernelCmd struct {
+	NoDownload bool  `help:"do not download qemu image while kernel generation"`
+	UseHost    bool  `help:"also use host kernels"`
+	Force      bool  `help:"force reinstall kernel"`
+	NoHeaders  bool  `help:"do not install kernel headers"`
+	Shuffle    bool  `help:"randomize kernels installation order"`
+	Retries    int64 `help:"amount of tries for each kernel" default:"10"`
+
+	List        KernelListCmd        `cmd:"" help:"list kernels"`
+	Autogen     KernelAutogenCmd     `cmd:"" help:"generate kernels based on the current config"`
+	Genall      KernelGenallCmd      `cmd:"" help:"generate all kernels for distro"`
+	Install     KernelInstallCmd     `cmd:"" help:"install specific kernel"`
+	ConfigRegen KernelConfigRegenCmd `cmd:"" help:"regenerate config"`
+}
+
+type KernelListCmd struct{}
+
+func (cmd *KernelListCmd) Run(g *Globals) (err error) {
+	kcfg, err := config.ReadKernelConfig(g.Config.Kernels)
+	if err != nil {
+		log.Debug().Err(err).Msg("read kernel config")
+	}
 
-func kernelListHandler(kcfg config.KernelConfig) (err error) {
 	if len(kcfg.Kernels) == 0 {
 		return errors.New("No kernels found")
 	}
+
 	for _, k := range kcfg.Kernels {
 		fmt.Println(k.DistroType, k.DistroRelease, k.KernelRelease)
 	}
+
 	return
 }
 
-func matchDebianHeadersPkg(container, mask string, generic bool) (
-	pkgs []string, err error) {
+type KernelAutogenCmd struct {
+	Max int64 `help:"download kernels from set defined by regex in release_mask, but no more than X for each of release_mask" default:"100500"`
+}
 
-	cmd := "apt-cache search linux-headers | cut -d ' ' -f 1"
-	output, err := dockerRun(time.Minute, container, "/tmp", cmd)
+func (cmd KernelAutogenCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
+	ka, err := config.ReadArtifactConfig(g.WorkDir + "/.out-of-tree.toml")
 	if err != nil {
 		return
 	}
 
-	r, err := regexp.Compile("linux-headers-" + mask)
+	for _, sk := range ka.SupportedKernels {
+		if sk.DistroRelease == "" {
+			err = errors.New("Please set distro_release")
+			return
+		}
+
+		err = generateKernels(sk,
+			g.Config.Docker.Registry,
+			g.Config.Docker.Commands,
+			cmd.Max, kernelCmd.Retries,
+			!kernelCmd.NoDownload,
+			kernelCmd.Force,
+			!kernelCmd.NoHeaders,
+			kernelCmd.Shuffle,
+		)
+		if err != nil {
+			return
+		}
+	}
+
+	return updateKernelsCfg(kernelCmd.UseHost, !kernelCmd.NoDownload)
+}
+
+type KernelGenallCmd struct {
+	Distro string `required:"" help:"distribution"`
+	Ver    string `required:"" help:"distro version"`
+}
+
+func (cmd *KernelGenallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
+	distroType, err := config.NewDistroType(cmd.Distro)
 	if err != nil {
 		return
 	}
 
-	kernels := r.FindAll([]byte(output), -1)
+	km := config.KernelMask{
+		DistroType:    distroType,
+		DistroRelease: cmd.Ver,
+		ReleaseMask:   ".*",
+	}
+	err = generateKernels(km,
+		g.Config.Docker.Registry,
+		g.Config.Docker.Commands,
+		math.MaxUint32, kernelCmd.Retries,
+		!kernelCmd.NoDownload,
+		kernelCmd.Force,
+		!kernelCmd.NoHeaders,
+		kernelCmd.Shuffle,
+	)
+	if err != nil {
+		return
+	}
 
-	for _, k := range kernels {
-		pkg := string(k)
-		if generic && !strings.HasSuffix(pkg, "generic") {
-			continue
+	return updateKernelsCfg(kernelCmd.UseHost, !kernelCmd.NoDownload)
+}
+
+type KernelInstallCmd struct {
+	Distro string `required:"" help:"distribution"`
+	Ver    string `required:"" help:"distro version"`
+	Kernel string `required:"" help:"kernel release mask"`
+}
+
+func (cmd *KernelInstallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
+	distroType, err := config.NewDistroType(cmd.Distro)
+	if err != nil {
+		return
+	}
+
+	km := config.KernelMask{
+		DistroType:    distroType,
+		DistroRelease: cmd.Ver,
+		ReleaseMask:   cmd.Kernel,
+	}
+	err = generateKernels(km,
+		g.Config.Docker.Registry,
+		g.Config.Docker.Commands,
+		math.MaxUint32, kernelCmd.Retries,
+		!kernelCmd.NoDownload,
+		kernelCmd.Force,
+		!kernelCmd.NoHeaders,
+		kernelCmd.Shuffle,
+	)
+	if err != nil {
+		return
+	}
+
+	return updateKernelsCfg(kernelCmd.UseHost, !kernelCmd.NoDownload)
+}
+
+type KernelConfigRegenCmd struct{}
+
+func (cmd *KernelConfigRegenCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
+	return updateKernelsCfg(kernelCmd.UseHost, !kernelCmd.NoDownload)
+}
+
+func matchDebImagePkg(container, mask string) (pkgs []string, err error) {
+
+	cmd := "apt-cache search --names-only '^linux-image-[0-9\\.\\-]*-generic' | awk '{ print $1 }'"
+
+	// FIXME timeout should be in global out-of-tree config
+	c, err := NewContainer(container, time.Hour)
+	if err != nil {
+		return
+	}
+
+	output, err := c.Run(tempDirBase, cmd)
+	if err != nil {
+		return
+	}
+
+	r, err := regexp.Compile("linux-image-" + mask)
+	if err != nil {
+		return
+	}
+
+	for _, pkg := range strings.Fields(output) {
+		if r.MatchString(pkg) || strings.Contains(pkg, mask) {
+			pkgs = append(pkgs, pkg)
 		}
-		if pkg == "linux-headers-generic" {
-			continue
-		}
-		pkgs = append(pkgs, pkg)
 	}
 
 	return
@@ -71,7 +196,14 @@ func matchCentOSDevelPkg(container, mask string, generic bool) (
 
 	cmd := "yum search kernel-devel --showduplicates | " +
 		"grep '^kernel-devel' | cut -d ' ' -f 1"
-	output, err := dockerRun(time.Minute, container, "/tmp", cmd)
+
+	// FIXME timeout should be in global out-of-tree config
+	c, err := NewContainer(container, time.Hour)
+	if err != nil {
+		return
+	}
+
+	output, err := c.Run(tempDirBase, cmd)
 	if err != nil {
 		return
 	}
@@ -81,8 +213,10 @@ func matchCentOSDevelPkg(container, mask string, generic bool) (
 		return
 	}
 
-	for _, k := range r.FindAll([]byte(output), -1) {
-		pkgs = append(pkgs, string(k))
+	for _, pkg := range strings.Fields(output) {
+		if r.MatchString(pkg) || strings.Contains(pkg, mask) {
+			pkgs = append(pkgs, pkg)
+		}
 	}
 
 	return
@@ -94,7 +228,7 @@ func dockerImagePath(sk config.KernelMask) (path string, err error) {
 		return
 	}
 
-	path = usr.HomeDir + "/.out-of-tree/"
+	path = usr.HomeDir + "/.out-of-tree/containers/"
 	path += sk.DistroType.String() + "/" + sk.DistroRelease
 	return
 }
@@ -128,13 +262,21 @@ func generateBaseDockerImage(registry string, commands []config.DockerCommand,
 
 	d := "# BASE\n"
 
-	if exists(dockerPath) {
-		log.Printf("Base image for %s:%s found",
+	cmd := exec.Command("docker", "images", "-q", sk.DockerName())
+	log.Debug().Msgf("run %v", cmd)
+
+	rawOutput, err := cmd.CombinedOutput()
+	if err != nil {
+		return
+	}
+
+	if exists(dockerPath) && string(rawOutput) != "" {
+		log.Info().Msgf("Base image for %s:%s found",
 			sk.DistroType.String(), sk.DistroRelease)
 		return
 	}
 
-	log.Printf("Base image for %s:%s not found, start generating",
+	log.Info().Msgf("Base image for %s:%s not found, start generating",
 		sk.DistroType.String(), sk.DistroRelease)
 	os.MkdirAll(imagePath, os.ModePerm)
 
@@ -154,46 +296,41 @@ func generateBaseDockerImage(registry string, commands []config.DockerCommand,
 	}
 
 	for _, c := range commands {
-		switch c.DistroType {
-		case config.Ubuntu:
-			d += "RUN " + c.Command + "\n"
-		case config.CentOS:
-			d += "RUN " + c.Command + "\n"
-		case config.Debian:
-			d += "RUN " + c.Command + "\n"
-		default:
-			err = fmt.Errorf("%s not yet supported",
-				sk.DistroType.String())
-			return
-		}
+		d += "RUN " + c.Command + "\n"
 	}
 
 	switch sk.DistroType {
 	case config.Ubuntu:
 		d += "ENV DEBIAN_FRONTEND=noninteractive\n"
-		if sk.DistroRelease >= "16.04" {
-			from := "http://.*ubuntu/"
-			to := "mirror://mirrors.ubuntu.com/mirrors.txt"
-			file := "/etc/apt/sources.list"
-			s := fmt.Sprintf("sed -i 's;%s;%s;' %s", from, to, file)
-			d += "RUN " + s + "\n"
-		}
 		d += "RUN apt-get update\n"
 		d += "RUN apt-get install -y build-essential libelf-dev\n"
 		d += "RUN apt-get install -y wget git\n"
+		// Install a single kernel and headers to ensure all dependencies are cached
+		d += "RUN export PKGNAME=$(apt-cache search --names-only '^linux-headers-[0-9\\.\\-]*-generic' | awk '{ print $1 }' | head -n 1); " +
+			"apt-get install -y $PKGNAME $(echo $PKGNAME | sed 's/headers/image/'); " +
+			"apt-get remove -y $PKGNAME $(echo $PKGNAME | sed 's/headers/image/')\n"
 		if sk.DistroRelease >= "14.04" {
 			d += "RUN apt-get install -y libseccomp-dev\n"
 		}
 		d += "RUN mkdir -p /lib/modules\n"
 	case config.CentOS:
 		if sk.DistroRelease < "7" && !vsyscall {
-			log.Println("Old CentOS requires `vsyscall=emulate` " +
+			log.Print("Old CentOS requires `vsyscall=emulate` " +
 				"on the latest kernels")
-			log.Println("Check out `A note about vsyscall` " +
+			log.Print("Check out `A note about vsyscall` " +
 				"at https://hub.docker.com/_/centos")
-			log.Println("See also https://lwn.net/Articles/446528/")
+			log.Print("See also https://lwn.net/Articles/446528/")
 			err = fmt.Errorf("vsyscall is not available")
 			return
+		} else if sk.DistroRelease == "8" {
+			// CentOS 8 doesn't have Vault repos by default
+			for _, repover := range []string{
+				"8.0.1905", "8.1.1911", "8.2.2004", "8.3.2011", "8.4.2105", "8.5.2111",
+			} {
+				repo := fmt.Sprintf("[%s]\\nbaseurl=http://vault.centos.org/%s/BaseOS/$basearch/os/\\ngpgcheck=0", repover, repover)
+				d += fmt.Sprintf("RUN echo -e '%s' >> /etc/yum.repos.d/CentOS-Vault.repo\n", repo)
+			}
+			d += "RUN sed -i 's/enabled=1/enabled=0/' /etc/yum.repos.d/*\n"
 		}
 
 		// enable rpms from old minor releases
@@ -202,23 +339,24 @@ func generateBaseDockerImage(registry string, commands []config.DockerCommand,
 		d += "RUN sed -i 's;installonly_limit=;installonly_limit=100500;' /etc/yum.conf\n"
 		d += "RUN yum -y update\n"
 
-		if sk.DistroRelease == "8" {
-			// FIXME CentOS Vault repository list for 8 is empty
-			// at the time of this fix; check for it and use a
-			// workaround if it's still empty
-			d += `RUN grep enabled /etc/yum.repos.d/CentOS-Vault.repo` +
-				` || echo -e '[8.0.1905]\nbaseurl=http://vault.centos.org/8.0.1905/BaseOS/$basearch/os/'` +
-				` >> /etc/yum.repos.d/CentOS-Vault.repo` + "\n"
-		}
-
 		d += "RUN yum -y groupinstall 'Development Tools'\n"
 
 		if sk.DistroRelease < "8" {
 			d += "RUN yum -y install deltarpm\n"
 		} else {
-			d += "RUN yum -y install drpm grub2-tools-minimal " +
+			d += "RUN yum -y install grub2-tools-minimal " +
 				"elfutils-libelf-devel\n"
 		}
+
+		var flags string
+		if sk.DistroRelease >= "8" {
+			flags = "--noautoremove"
+		}
+
+		// Cache kernel package dependencies
+		d += "RUN export PKGNAME=$(yum search kernel-devel --showduplicates | grep '^kernel-devel' | cut -d ' ' -f 1 | head -n 1); " +
+			"yum -y install $PKGNAME $(echo $PKGNAME | sed 's/-devel//'); " +
+			fmt.Sprintf("yum -y remove $PKGNAME $(echo $PKGNAME | sed 's/-devel//') %s\n", flags)
 	default:
 		err = fmt.Errorf("%s not yet supported", sk.DistroType.String())
 		return
@@ -231,171 +369,187 @@ func generateBaseDockerImage(registry string, commands []config.DockerCommand,
 		return
 	}
 
-	cmd := exec.Command("docker", "build", "-t", sk.DockerName(), imagePath)
-	rawOutput, err := cmd.CombinedOutput()
+	c, err := NewContainer(sk.DockerName(), time.Hour)
 	if err != nil {
-		log.Printf("Base image for %s:%s generating error, see log",
-			sk.DistroType.String(), sk.DistroRelease)
-		log.Println(string(rawOutput))
 		return
 	}
 
-	log.Printf("Base image for %s:%s generating success",
+	output, err := c.Build(imagePath)
+	if err != nil {
+		log.Error().Err(err).Msgf("Base image for %s:%s generating error",
+			sk.DistroType.String(), sk.DistroRelease)
+		log.Fatal().Msg(output)
+		return
+	}
+
+	log.Info().Msgf("Base image for %s:%s generating success",
 		sk.DistroType.String(), sk.DistroRelease)
 
 	return
 }
 
-func dockerImageAppend(sk config.KernelMask, pkgname string) (err error) {
-	imagePath, err := dockerImagePath(sk)
+func installKernel(sk config.KernelMask, pkgname string, force, headers bool) (err error) {
+	tmpdir, err := os.MkdirTemp(tempDirBase, "out-of-tree-"+pkgname+"-")
+	if err != nil {
+		log.Fatal().Err(err).Msg("make tmp directory")
+	}
+	defer os.RemoveAll(tmpdir)
+
+	slog := log.With().
+		Str("distro_type", sk.DistroType.String()).
+		Str("distro_release", sk.DistroRelease).
+		Str("pkg", pkgname).
+		Str("tmpdir", tmpdir).
+		Logger()
+
+	c, err := NewContainer(sk.DockerName(), time.Hour) // TODO conf
 	if err != nil {
 		return
 	}
 
-	raw, err := ioutil.ReadFile(imagePath + "/Dockerfile")
+	moddirs, err := ioutil.ReadDir(c.Volumes.LibModules)
 	if err != nil {
 		return
 	}
 
-	if strings.Contains(string(raw), pkgname) {
-		// already installed kernel
-		log.Printf("kernel %s for %s:%s is already exists",
-			pkgname, sk.DistroType.String(), sk.DistroRelease)
-		return
+	for _, krel := range moddirs {
+		if strings.Contains(pkgname, krel.Name()) {
+			if force {
+				slog.Info().Msg("Reinstall")
+			} else {
+				slog.Info().Msg("Already installed")
+				return
+			}
+		}
 	}
 
-	var s string
+	volumes := c.Volumes
+
+	c.Volumes.LibModules = fmt.Sprintf("%s/libmodules", tmpdir)
+	os.MkdirAll(c.Volumes.LibModules, 0777)
+
+	c.Volumes.UsrSrc = fmt.Sprintf("%s/usrsrc", tmpdir)
+	os.MkdirAll(c.Volumes.UsrSrc, 0777)
+
+	c.Volumes.Boot = fmt.Sprintf("%s/boot", tmpdir)
+	os.MkdirAll(c.Volumes.Boot, 0777)
+
+	slog.Debug().Msgf("Installing kernel")
 
 	switch sk.DistroType {
 	case config.Ubuntu:
-		imagepkg := strings.Replace(pkgname, "headers", "image", -1)
+		var headerspkg string
+		if headers {
+			headerspkg = strings.Replace(pkgname, "image", "headers", -1)
+		}
 
-		log.Printf("Start adding kernel %s for %s:%s",
-			imagepkg, sk.DistroType.String(), sk.DistroRelease)
+		cmd := fmt.Sprintf("apt-get install -y %s %s", pkgname, headerspkg)
 
-		s = fmt.Sprintf("RUN apt-get install -y %s %s\n", imagepkg,
-			pkgname)
+		_, err = c.Run(tempDirBase, cmd)
+		if err != nil {
+			return
+		}
 	case config.CentOS:
 		imagepkg := strings.Replace(pkgname, "-devel", "", -1)
 
-		log.Printf("Start adding kernel %s for %s:%s",
-			imagepkg, sk.DistroType.String(), sk.DistroRelease)
-
 		version := strings.Replace(pkgname, "kernel-devel-", "", -1)
 
-		s = fmt.Sprintf("RUN yum -y install %s %s\n", imagepkg,
+		if !headers {
+			pkgname = ""
+		}
+		cmd := fmt.Sprintf("yum -y install %s %s\n", imagepkg,
 			pkgname)
-		s += fmt.Sprintf("RUN dracut --add-drivers 'e1000 ext4' -f "+
+		_, err = c.Run(tempDirBase, cmd)
+		if err != nil {
+			return
+		}
+
+		cmd = fmt.Sprintf("dracut --add-drivers 'e1000 ext4' -f "+
 			"/boot/initramfs-%s.img %s\n", version, version)
+		_, err = c.Run(tempDirBase, cmd)
+		if err != nil {
+			return
+		}
 	default:
 		err = fmt.Errorf("%s not yet supported", sk.DistroType.String())
 		return
 	}
 
-	err = ioutil.WriteFile(imagePath+"/Dockerfile",
-		append(raw, []byte(s)...), 0644)
+	c.Args = append(c.Args, "-v", volumes.LibModules+":/target/lib/modules")
+	c.Args = append(c.Args, "-v", volumes.UsrSrc+":/target/usr/src")
+	c.Args = append(c.Args, "-v", volumes.Boot+":/target/boot")
+
+	cmd := "true"
+
+	files, err := ioutil.ReadDir(c.Volumes.Boot)
+	if err != nil {
+		return
+	}
+	if len(files) != 0 {
+		cmd += " && cp -r /boot/* /target/boot/"
+	}
+
+	files, err = ioutil.ReadDir(c.Volumes.LibModules)
+	if err != nil {
+		return
+	}
+	if len(files) != 0 {
+		cmd += " && cp -r /lib/modules/* /target/lib/modules/"
+	}
+
+	files, err = ioutil.ReadDir(c.Volumes.UsrSrc)
+	if err != nil {
+		return
+	}
+	if len(files) != 0 {
+		cmd += " && cp -r /usr/src/* /target/usr/src/"
+	}
+
+	_, err = c.Run(tempDirBase, cmd)
 	if err != nil {
 		return
 	}
 
-	cmd := exec.Command("docker", "build", "-t", sk.DockerName(), imagePath)
-	rawOutput, err := cmd.CombinedOutput()
-	if err != nil {
-		// Fallback to previous state
-		werr := ioutil.WriteFile(imagePath+"/Dockerfile", raw, 0644)
-		if werr != nil {
-			return
-		}
-
-		log.Printf("Add kernel %s for %s:%s error, see log",
-			pkgname, sk.DistroType.String(), sk.DistroRelease)
-		log.Println(string(rawOutput))
-		return
-	}
-
-	log.Printf("Add kernel %s for %s:%s success",
-		pkgname, sk.DistroType.String(), sk.DistroRelease)
-
+	slog.Debug().Msgf("Success")
 	return
 }
 
-func kickImage(name string) (err error) {
-	cmd := exec.Command("docker", "run", name, "bash", "-c", "ls")
-	_, err = cmd.CombinedOutput()
-	return
-}
-
-func copyKernels(name string) (err error) {
-	cmd := exec.Command("docker", "ps", "-a")
-	rawOutput, err := cmd.CombinedOutput()
-	if err != nil {
-		log.Println(string(rawOutput))
-		return
-	}
-
-	r, err := regexp.Compile(".*" + name)
-	if err != nil {
-		return
-	}
-
-	var containerID string
-
-	what := r.FindAll(rawOutput, -1)
-	for _, w := range what {
-		containerID = strings.Fields(string(w))[0]
-		break
-	}
-
-	usr, err := user.Current()
-	if err != nil {
-		return
-	}
-
-	target := usr.HomeDir + "/.out-of-tree/kernels/"
-	if !exists(target) {
-		os.MkdirAll(target, os.ModePerm)
-	}
-
-	cmd = exec.Command("docker", "cp", containerID+":/boot/.", target)
-	rawOutput, err = cmd.CombinedOutput()
-	if err != nil {
-		log.Println(string(rawOutput))
-		return
-	}
-
-	return
-}
-
-func genKernelPath(files []os.FileInfo, kname string) string {
+func findKernelFile(files []os.FileInfo, kname string) (name string, err error) {
 	for _, file := range files {
 		if strings.HasPrefix(file.Name(), "vmlinuz") {
 			if strings.Contains(file.Name(), kname) {
-				return file.Name()
+				name = file.Name()
+				return
 			}
 		}
 	}
-	return "unknown"
+
+	err = errors.New("cannot find kernel")
+	return
 }
 
-func genInitrdPath(files []os.FileInfo, kname string) string {
+func findInitrdFile(files []os.FileInfo, kname string) (name string, err error) {
 	for _, file := range files {
 		if strings.HasPrefix(file.Name(), "initrd") ||
 			strings.HasPrefix(file.Name(), "initramfs") {
 
 			if strings.Contains(file.Name(), kname) {
-				return file.Name()
+				name = file.Name()
+				return
 			}
 		}
 	}
-	return "unknown"
+
+	err = errors.New("cannot find kernel")
+	return
 }
 
-func genRootfsImage(d dockerImageInfo, download bool) (rootfs string, err error) {
+func genRootfsImage(d containerImageInfo, download bool) (rootfs string, err error) {
 	usr, err := user.Current()
 	if err != nil {
 		return
 	}
-	imageFile := d.ContainerName + ".img"
+	imageFile := d.Name + ".img"
 
 	imagesPath := usr.HomeDir + "/.out-of-tree/images/"
 	os.MkdirAll(imagesPath, os.ModePerm)
@@ -403,54 +557,13 @@ func genRootfsImage(d dockerImageInfo, download bool) (rootfs string, err error)
 	rootfs = imagesPath + imageFile
 	if !exists(rootfs) {
 		if download {
-			log.Println(imageFile, "not exists, start downloading...")
+			log.Info().Msgf("%v not available, start download", imageFile)
 			err = downloadImage(imagesPath, imageFile)
 		}
 	}
 	return
 }
 
-type dockerImageInfo struct {
-	ContainerName string
-	DistroType    config.DistroType
-	DistroRelease string // 18.04/7.4.1708/9.1
-}
-
-func listDockerImages() (diis []dockerImageInfo, err error) {
-	cmd := exec.Command("docker", "images")
-	rawOutput, err := cmd.CombinedOutput()
-	if err != nil {
-		return
-	}
-
-	r, err := regexp.Compile("out_of_tree_.*")
-	if err != nil {
-		return
-	}
-
-	containers := r.FindAll(rawOutput, -1)
-	for _, c := range containers {
-		container := strings.Fields(string(c))[0]
-
-		s := strings.Replace(container, "__", ".", -1)
-		values := strings.Split(s, "_")
-		distro, ver := values[3], values[4]
-
-		dii := dockerImageInfo{
-			ContainerName: container,
-			DistroRelease: ver,
-		}
-
-		dii.DistroType, err = config.NewDistroType(distro)
-		if err != nil {
-			return
-		}
-
-		diis = append(diis, dii)
-	}
-	return
-}
-
 func updateKernelsCfg(host, download bool) (err error) {
 	newkcfg := config.KernelConfig{}
 
@@ -463,15 +576,15 @@ func updateKernelsCfg(host, download bool) (err error) {
 	}
 
 	// Get docker kernels
-	dockerImages, err := listDockerImages()
+	dockerImages, err := listContainerImages()
 	if err != nil {
 		return
 	}
 
 	for _, d := range dockerImages {
-		err = genDockerKernels(d, &newkcfg, download)
+		err = listContainersKernels(d, &newkcfg, download)
 		if err != nil {
-			log.Println("gen kernels", d.ContainerName, ":", err)
+			log.Print("gen kernels", d.Name, ":", err)
 			continue
 		}
 	}
@@ -502,50 +615,73 @@ func updateKernelsCfg(host, download bool) (err error) {
 		return
 	}
 
-	log.Println(kernelsCfgPath, "is successfully updated")
+	log.Info().Msgf("%s is successfully updated", kernelsCfgPath)
 	return
 }
 
-func genDockerKernels(dii dockerImageInfo, newkcfg *config.KernelConfig,
+func listContainersKernels(dii containerImageInfo, newkcfg *config.KernelConfig,
 	download bool) (err error) {
 
-	name := dii.ContainerName
-	cmd := exec.Command("docker", "run", name, "ls", "/lib/modules")
-	rawOutput, err := cmd.CombinedOutput()
-	if err != nil {
-		log.Println(string(rawOutput), err)
-		return
-	}
-
-	usr, err := user.Current()
-	if err != nil {
-		return
-	}
-	kernelsBase := usr.HomeDir + "/.out-of-tree/kernels/"
-	files, err := ioutil.ReadDir(kernelsBase)
-	if err != nil {
-		return
-	}
-
 	rootfs, err := genRootfsImage(dii, download)
 	if err != nil {
 		return
 	}
 
-	for _, k := range strings.Fields(string(rawOutput)) {
+	c, err := NewContainer(dii.Name, time.Hour)
+	if err != nil {
+		return
+	}
+
+	moddirs, err := ioutil.ReadDir(c.Volumes.LibModules)
+	if err != nil {
+		return
+	}
+
+	bootfiles, err := ioutil.ReadDir(c.Volumes.Boot)
+	if err != nil {
+		return
+	}
+
+	for _, krel := range moddirs {
+		log.Debug().Msgf("generate config entry for %s", krel.Name())
+
+		var kernelFile, initrdFile string
+		kernelFile, err = findKernelFile(bootfiles, krel.Name())
+		if err != nil {
+			log.Warn().Msgf("cannot find kernel %s", krel.Name())
+			continue
+		}
+
+		initrdFile, err = findInitrdFile(bootfiles, krel.Name())
+		if err != nil {
+			log.Warn().Msgf("cannot find initrd %s", krel.Name())
+			continue
+		}
+
 		ki := config.KernelInfo{
 			DistroType:    dii.DistroType,
 			DistroRelease: dii.DistroRelease,
-			KernelRelease: k,
-			ContainerName: name,
+			KernelRelease: krel.Name(),
+			ContainerName: dii.Name,
 
-			KernelPath: kernelsBase + genKernelPath(files, k),
-			InitrdPath: kernelsBase + genInitrdPath(files, k),
-			RootFS:     rootfs,
+			KernelPath:  c.Volumes.Boot + "/" + kernelFile,
+			InitrdPath:  c.Volumes.Boot + "/" + initrdFile,
+			ModulesPath: c.Volumes.LibModules + "/" + krel.Name(),
+
+			RootFS: rootfs,
 		}
 		newkcfg.Kernels = append(newkcfg.Kernels, ki)
 	}
 
+	for _, cmd := range []string{
+		"find /boot -type f -exec chmod a+r {} \\;",
+	} {
+		_, err = c.Run(tempDirBase, cmd)
+		if err != nil {
+			return
+		}
+	}
+
 	return
 }
 
@@ -558,7 +694,7 @@ func hasKernel(ki config.KernelInfo, kcfg config.KernelConfig) bool {
 	return false
 }
 
-func shuffle(a []string) []string {
+func shuffleStrings(a []string) []string {
 	// Fisher–Yates shuffle
 	for i := len(a) - 1; i > 0; i-- {
 		j := rand.Intn(i + 1)
@@ -568,12 +704,12 @@ func shuffle(a []string) []string {
 }
 
 func generateKernels(km config.KernelMask, registry string,
-	commands []config.DockerCommand, max int64,
-	download bool) (err error) {
+	commands []config.DockerCommand, max, retries int64,
+	download, force, headers, shuffle bool) (err error) {
 
-	log.Println("Generating for kernel mask", km)
+	log.Info().Msgf("Generating for kernel mask %v", km)
 
-	_, err = genRootfsImage(dockerImageInfo{ContainerName: km.DockerName()},
+	_, err = genRootfsImage(containerImageInfo{Name: km.DockerName()},
 		download)
 	if err != nil {
 		return
@@ -587,8 +723,7 @@ func generateKernels(km config.KernelMask, registry string,
 	var pkgs []string
 	switch km.DistroType {
 	case config.Ubuntu:
-		pkgs, err = matchDebianHeadersPkg(km.DockerName(),
-			km.ReleaseMask, true)
+		pkgs, err = matchDebImagePkg(km.DockerName(), km.ReleaseMask)
 	case config.CentOS:
 		pkgs, err = matchCentOSDevelPkg(km.DockerName(),
 			km.ReleaseMask, true)
@@ -599,119 +734,36 @@ func generateKernels(km config.KernelMask, registry string,
 		return
 	}
 
-	for i, pkg := range shuffle(pkgs) {
+	if shuffle {
+		pkgs = shuffleStrings(pkgs)
+	}
+	for i, pkg := range pkgs {
 		if max <= 0 {
-			log.Println("Max is reached")
+			log.Print("Max is reached")
 			break
 		}
 
-		log.Println(i, "/", len(pkgs), pkg)
+		log.Info().Msgf("%d/%d %s", i+1, len(pkgs), pkg)
 
-		err = dockerImageAppend(km, pkg)
-		if err == nil {
-			max--
-		} else {
-			log.Println("dockerImageAppend", err)
+		var attempt int64
+		for {
+			attempt++
+
+			err = installKernel(km, pkg, force, headers)
+			if err == nil {
+				max--
+				break
+			} else if attempt >= retries {
+				log.Error().Err(err).Msg("install kernel")
+				log.Debug().Msg("skip")
+				break
+			} else {
+				log.Warn().Err(err).Msg("install kernel")
+				time.Sleep(time.Second)
+				log.Info().Msg("retry")
+			}
 		}
 	}
 
-	err = kickImage(km.DockerName())
-	if err != nil {
-		log.Println("kick image", km.DockerName(), ":", err)
-		return
-	}
-
-	err = copyKernels(km.DockerName())
-	if err != nil {
-		log.Println("copy kernels", km.DockerName(), ":", err)
-		return
-	}
 	return
 }
-
-func kernelAutogenHandler(workPath, registry string,
-	commands []config.DockerCommand,
-	max int64, host, download bool) (err error) {
-
-	ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
-	if err != nil {
-		return
-	}
-
-	for _, sk := range ka.SupportedKernels {
-		if sk.DistroRelease == "" {
-			err = errors.New("Please set distro_release")
-			return
-		}
-
-		err = generateKernels(sk, registry, commands, max, download)
-		if err != nil {
-			return
-		}
-	}
-
-	err = updateKernelsCfg(host, download)
-	return
-}
-
-func kernelDockerRegenHandler(host, download bool) (err error) {
-	dockerImages, err := listDockerImages()
-	if err != nil {
-		return
-	}
-
-	for _, d := range dockerImages {
-		var imagePath string
-		imagePath, err = dockerImagePath(config.KernelMask{
-			DistroType:    d.DistroType,
-			DistroRelease: d.DistroRelease,
-		})
-		if err != nil {
-			return
-		}
-
-		cmd := exec.Command("docker", "build", "-t",
-			d.ContainerName, imagePath)
-		var rawOutput []byte
-		rawOutput, err = cmd.CombinedOutput()
-		if err != nil {
-			log.Println("docker build:", string(rawOutput))
-			return
-		}
-
-		err = kickImage(d.ContainerName)
-		if err != nil {
-			log.Println("kick image", d.ContainerName, ":", err)
-			continue
-		}
-
-		err = copyKernels(d.ContainerName)
-		if err != nil {
-			log.Println("copy kernels", d.ContainerName, ":", err)
-			continue
-		}
-	}
-
-	return updateKernelsCfg(host, download)
-}
-
-func kernelGenallHandler(distro, version, registry string,
-	commands []config.DockerCommand, host, download bool) (err error) {
-
-	distroType, err := config.NewDistroType(distro)
-	if err != nil {
-		return
-	}
-
-	km := config.KernelMask{
-		DistroType:    distroType,
-		DistroRelease: version,
-		ReleaseMask:   ".*",
-	}
-	err = generateKernels(km, registry, commands, kernelsAll, download)
-	if err != nil {
-		return
-	}
-
-	return updateKernelsCfg(host, download)
-}

kernel_linux.go

@@ -1,19 +1,21 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Copyright 2023 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
+//go:build linux
 // +build linux
 
 package main
 
 import (
 	"io/ioutil"
-	"log"
 	"os/exec"
 	"strings"
 
-	"code.dumpstack.io/tools/out-of-tree/config"
+	"github.com/rs/zerolog/log"
 	"github.com/zcalusic/sysinfo"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
 )
 
 func genHostKernels(download bool) (kcfg config.KernelConfig, err error) {
@@ -26,21 +28,23 @@ func genHostKernels(download bool) (kcfg config.KernelConfig, err error) {
 	}
 
 	cmd := exec.Command("ls", "/lib/modules")
+	log.Debug().Msgf("%v", cmd)
+
 	rawOutput, err := cmd.CombinedOutput()
 	if err != nil {
-		log.Println(string(rawOutput), err)
+		log.Print(string(rawOutput), err)
 		return
 	}
 
 	kernelsBase := "/boot/"
-	files, err := ioutil.ReadDir(kernelsBase)
+	bootfiles, err := ioutil.ReadDir(kernelsBase)
 	if err != nil {
 		return
 	}
 
 	// only for compatibility, docker is not really used
-	dii := dockerImageInfo{
-		ContainerName: config.KernelMask{
+	dii := containerImageInfo{
+		Name: config.KernelMask{
 			DistroType:    distroType,
 			DistroRelease: si.OS.Version,
 		}.DockerName(),
@@ -51,21 +55,36 @@ func genHostKernels(download bool) (kcfg config.KernelConfig, err error) {
 		return
 	}
 
-	for _, k := range strings.Fields(string(rawOutput)) {
+	for _, krel := range strings.Fields(string(rawOutput)) {
+		log.Debug().Msgf("generate config entry for %s", krel)
+
+		var kernelFile, initrdFile string
+		kernelFile, err = findKernelFile(bootfiles, krel)
+		if err != nil {
+			log.Warn().Msgf("cannot find kernel %s", krel)
+			continue
+		}
+
+		initrdFile, err = findInitrdFile(bootfiles, krel)
+		if err != nil {
+			log.Warn().Msgf("cannot find initrd %s", krel)
+			continue
+		}
+
 		ki := config.KernelInfo{
 			DistroType:    distroType,
 			DistroRelease: si.OS.Version,
-			KernelRelease: k,
+			KernelRelease: krel,
 
-			KernelSource: "/lib/modules/" + k + "/build",
+			KernelSource: "/lib/modules/" + krel + "/build",
 
-			KernelPath: kernelsBase + genKernelPath(files, k),
-			InitrdPath: kernelsBase + genInitrdPath(files, k),
+			KernelPath: kernelsBase + kernelFile,
+			InitrdPath: kernelsBase + initrdFile,
 			RootFS:     rootfs,
 		}
 
-		vmlinux := "/usr/lib/debug/boot/vmlinux-" + k
-		log.Println("vmlinux", vmlinux)
+		vmlinux := "/usr/lib/debug/boot/vmlinux-" + krel
+		log.Print("vmlinux", vmlinux)
 		if exists(vmlinux) {
 			ki.VmlinuxPath = vmlinux
 		}

log.go

@@ -1,4 +1,4 @@
-// Copyright 2019 Mikhail Klementev. All rights reserved.
+// Copyright 2023 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
@@ -8,66 +8,51 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
-	"log"
 	"math"
 	"os"
 
 	"github.com/olekukonko/tablewriter"
+	"github.com/rs/zerolog/log"
 	"gopkg.in/logrusorgru/aurora.v2"
 
 	"code.dumpstack.io/tools/out-of-tree/config"
 )
 
-func logLogEntry(l logEntry) {
-	distroInfo := fmt.Sprintf("%s-%s {%s}", l.DistroType,
-		l.DistroRelease, l.KernelRelease)
-
-	artifactInfo := fmt.Sprintf("{[%s] %s}", l.Type, l.Name)
-
-	colored := ""
-	if l.Type == config.KernelExploit {
-		colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s",
-			l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo,
-			genOkFail("BUILD", l.Build.Ok),
-			genOkFail("LPE", l.Test.Ok))
-	} else {
-		colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s %s",
-			l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo,
-			genOkFail("BUILD", l.Build.Ok),
-			genOkFail("INSMOD", l.Run.Ok),
-			genOkFail("TEST", l.Test.Ok))
-	}
-
-	additional := ""
-	if l.KernelPanic {
-		additional = "(panic)"
-	} else if l.KilledByTimeout {
-		additional = "(timeout)"
-	}
-
-	if additional != "" {
-		fmt.Println(colored, additional)
-	} else {
-		fmt.Println(colored)
-	}
+type LogCmd struct {
+	Query    LogQueryCmd    `cmd:"" help:"query logs"`
+	Dump     LogDumpCmd     `cmd:"" help:"show all info for log entry with ID"`
+	Json     LogJsonCmd     `cmd:"" help:"generate json statistics"`
+	Markdown LogMarkdownCmd `cmd:"" help:"generate markdown statistics"`
 }
 
-func logHandler(db *sql.DB, path, tag string, num int, rate bool) (err error) {
+type LogQueryCmd struct {
+	Num  int    `help:"how much lines" default:"50"`
+	Rate bool   `help:"show artifact success rate"`
+	Tag  string `help:"filter tag"`
+}
+
+func (cmd *LogQueryCmd) Run(g *Globals) (err error) {
+	db, err := openDatabase(g.Config.Database)
+	if err != nil {
+		panic(err)
+	}
+	defer db.Close()
+
 	var les []logEntry
 
-	ka, kaErr := config.ReadArtifactConfig(path + "/.out-of-tree.toml")
+	ka, kaErr := config.ReadArtifactConfig(g.WorkDir + "/.out-of-tree.toml")
 	if kaErr == nil {
-		log.Println(".out-of-tree.toml found, filter by artifact name")
-		les, err = getAllArtifactLogs(db, tag, num, ka)
+		log.Print(".out-of-tree.toml found, filter by artifact name")
+		les, err = getAllArtifactLogs(db, cmd.Tag, cmd.Num, ka)
 	} else {
-		les, err = getAllLogs(db, tag, num)
+		les, err = getAllLogs(db, cmd.Tag, cmd.Num)
 	}
 	if err != nil {
 		return
 	}
 
 	s := "\nS"
-	if rate {
+	if cmd.Rate {
 		if kaErr != nil {
 			err = kaErr
 			return
@@ -75,7 +60,7 @@ func logHandler(db *sql.DB, path, tag string, num int, rate bool) (err error) {
 
 		s = fmt.Sprintf("{[%s] %s} Overall s", ka.Type, ka.Name)
 
-		les, err = getAllArtifactLogs(db, tag, math.MaxInt64, ka)
+		les, err = getAllArtifactLogs(db, cmd.Tag, math.MaxInt64, ka)
 		if err != nil {
 			return
 		}
@@ -99,8 +84,23 @@ func logHandler(db *sql.DB, path, tag string, num int, rate bool) (err error) {
 	return
 }
 
-func logDumpHandler(db *sql.DB, id int) (err error) {
-	l, err := getLogByID(db, id)
+type LogDumpCmd struct {
+	ID int `help:"id" default:"-1"`
+}
+
+func (cmd *LogDumpCmd) Run(g *Globals) (err error) {
+	db, err := openDatabase(g.Config.Database)
+	if err != nil {
+		panic(err)
+	}
+	defer db.Close()
+
+	var l logEntry
+	if cmd.ID > 0 {
+		l, err = getLogByID(db, cmd.ID)
+	} else {
+		l, err = getLastLog(db)
+	}
 	if err != nil {
 		return
 	}
@@ -145,6 +145,102 @@ func logDumpHandler(db *sql.DB, id int) (err error) {
 	return
 }
 
+type LogJsonCmd struct {
+	Tag string `required:"" help:"filter tag"`
+}
+
+func (cmd *LogJsonCmd) Run(g *Globals) (err error) {
+	db, err := openDatabase(g.Config.Database)
+	if err != nil {
+		panic(err)
+	}
+	defer db.Close()
+
+	distros, err := getStats(db, g.WorkDir, cmd.Tag)
+	if err != nil {
+		return
+	}
+
+	bytes, err := json.Marshal(&distros)
+	if err != nil {
+		return
+	}
+
+	fmt.Println(string(bytes))
+	return
+}
+
+type LogMarkdownCmd struct {
+	Tag string `required:"" help:"filter tag"`
+}
+
+func (cmd *LogMarkdownCmd) Run(g *Globals) (err error) {
+	db, err := openDatabase(g.Config.Database)
+	if err != nil {
+		panic(err)
+	}
+	defer db.Close()
+
+	distros, err := getStats(db, g.WorkDir, cmd.Tag)
+	if err != nil {
+		return
+	}
+
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Distro", "Release", "Kernel", "Reliability"})
+	table.SetBorders(tablewriter.Border{
+		Left: true, Top: false, Right: true, Bottom: false})
+	table.SetCenterSeparator("|")
+
+	for distro, releases := range distros {
+		for release, kernels := range releases {
+			for kernel, stats := range kernels {
+				all := float64(stats.All)
+				ok := float64(stats.TestOK)
+				r := fmt.Sprintf("%6.02f%%", (ok/all)*100)
+				table.Append([]string{distro, release, kernel, r})
+			}
+		}
+	}
+
+	table.Render()
+	return
+}
+
+func logLogEntry(l logEntry) {
+	distroInfo := fmt.Sprintf("%s-%s {%s}", l.DistroType,
+		l.DistroRelease, l.KernelRelease)
+
+	artifactInfo := fmt.Sprintf("{[%s] %s}", l.Type, l.Name)
+
+	colored := ""
+	if l.Type == config.KernelExploit {
+		colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s",
+			l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo,
+			genOkFail("BUILD", l.Build.Ok),
+			genOkFail("LPE", l.Test.Ok))
+	} else {
+		colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s %s",
+			l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo,
+			genOkFail("BUILD", l.Build.Ok),
+			genOkFail("INSMOD", l.Run.Ok),
+			genOkFail("TEST", l.Test.Ok))
+	}
+
+	additional := ""
+	if l.KernelPanic {
+		additional = "(panic)"
+	} else if l.KilledByTimeout {
+		additional = "(timeout)"
+	}
+
+	if additional != "" {
+		fmt.Println(colored, additional)
+	} else {
+		fmt.Println(colored)
+	}
+}
+
 type runstat struct {
 	All, BuildOK, RunOK, TestOK, Timeout, Panic int
 }
@@ -201,45 +297,3 @@ func getStats(db *sql.DB, path, tag string) (
 
 	return
 }
-
-func logJSONHandler(db *sql.DB, path, tag string) (err error) {
-	distros, err := getStats(db, path, tag)
-	if err != nil {
-		return
-	}
-
-	bytes, err := json.Marshal(&distros)
-	if err != nil {
-		return
-	}
-
-	fmt.Println(string(bytes))
-	return
-}
-
-func logMarkdownHandler(db *sql.DB, path, tag string) (err error) {
-	distros, err := getStats(db, path, tag)
-	if err != nil {
-		return
-	}
-
-	table := tablewriter.NewWriter(os.Stdout)
-	table.SetHeader([]string{"Distro", "Release", "Kernel", "Reliability"})
-	table.SetBorders(tablewriter.Border{
-		Left: true, Top: false, Right: true, Bottom: false})
-	table.SetCenterSeparator("|")
-
-	for distro, releases := range distros {
-		for release, kernels := range releases {
-			for kernel, stats := range kernels {
-				all := float64(stats.All)
-				ok := float64(stats.TestOK)
-				r := fmt.Sprintf("%6.02f%%", (ok/all)*100)
-				table.Append([]string{distro, release, kernel, r})
-			}
-		}
-	}
-
-	table.Render()
-	return
-}

main.go

@@ -1,4 +1,4 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Copyright 2023 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
@@ -6,352 +6,153 @@ package main
 
 import (
 	"fmt"
-	"log"
+	"io"
 	"math/rand"
 	"os"
-	"os/exec"
 	"os/user"
-	"runtime"
-	"sort"
+	"runtime/debug"
 	"strconv"
 	"time"
 
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"github.com/natefinch/lumberjack"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+
+	"github.com/alecthomas/kong"
 
 	"code.dumpstack.io/tools/out-of-tree/config"
 )
 
-func findFallback(kcfg config.KernelConfig, ki config.KernelInfo) (rootfs string) {
-	for _, k := range kcfg.Kernels {
-		if !exists(k.RootFS) || k.DistroType != ki.DistroType {
-			continue
-		}
-		if k.RootFS < ki.RootFS {
-			rootfs = k.RootFS
-			return
-		}
-	}
-	return
+type Globals struct {
+	Config config.OutOfTree `help:"path to out-of-tree configuration" default:"~/.out-of-tree/out-of-tree.toml"`
+
+	WorkDir string `help:"path to work directory" default:"./" type:"path"`
 }
 
-func handleFallbacks(kcfg config.KernelConfig) {
-	sort.Sort(sort.Reverse(config.ByRootFS(kcfg.Kernels)))
+type CLI struct {
+	Globals
 
-	for i, k := range kcfg.Kernels {
-		if !exists(k.RootFS) {
-			newRootFS := findFallback(kcfg, k)
+	Pew       PewCmd       `cmd:"" help:"build, run, and test module/exploit"`
+	Kernel    KernelCmd    `cmd:"" help:"manipulate kernels"`
+	Debug     DebugCmd     `cmd:"" help:"debug environment"`
+	Log       LogCmd       `cmd:"" help:"query logs"`
+	Pack      PackCmd      `cmd:"" help:"exploit pack test"`
+	Gen       GenCmd       `cmd:"" help:"generate .out-of-tree.toml skeleton"`
+	Image     ImageCmd     `cmd:"" help:"manage images"`
+	Container ContainerCmd `cmd:"" help:"manage containers"`
 
-			s := k.RootFS + " does not exists "
-			if newRootFS != "" {
-				s += "(fallback to " + newRootFS + ")"
-			} else {
-				s += "(no fallback found)"
+	Version VersionFlag `name:"version" help:"print version information and quit"`
+
+	LogLevel LogLevelFlag `enum:"trace,debug,info,warn,error" default:"info"`
+}
+
+type LogLevelFlag string
+
+func (loglevel LogLevelFlag) AfterApply() error {
+	switch loglevel {
+	case "debug", "trace":
+		zerolog.CallerMarshalFunc = func(pc uintptr, file string, line int) string {
+			short := file
+			for i := len(file) - 1; i > 0; i-- {
+				if file[i] == '/' {
+					short = file[i+1:]
+					break
+				}
 			}
-
-			kcfg.Kernels[i].RootFS = newRootFS
-			log.Println(s)
+			file = short
+			return file + ":" + strconv.Itoa(line)
 		}
+		log.Logger = log.With().Caller().Logger()
 	}
+	return nil
 }
 
-func checkRequiredUtils() (err error) {
-	// Check for required commands
-	for _, cmd := range []string{"docker", "qemu-system-x86_64"} {
-		_, err := exec.Command("which", cmd).CombinedOutput()
-		if err != nil {
-			return fmt.Errorf("Command not found: %s", cmd)
-		}
-	}
-	return
+type VersionFlag string
+
+func (v VersionFlag) Decode(ctx *kong.DecodeContext) error { return nil }
+func (v VersionFlag) IsBool() bool                         { return true }
+func (v VersionFlag) BeforeApply(app *kong.Kong, vars kong.Vars) error {
+	fmt.Println(vars["version"])
+	app.Exit(0)
+	return nil
 }
 
-func checkDockerPermissions() (err error) {
-	output, err := exec.Command("docker", "ps").CombinedOutput()
-	if err != nil {
-		err = fmt.Errorf("%s", output)
-	}
-	return
+type LevelWriter struct {
+	io.Writer
+	Level zerolog.Level
 }
 
+func (lw *LevelWriter) WriteLevel(l zerolog.Level, p []byte) (n int, err error) {
+	if l >= lw.Level {
+		return lw.Writer.Write(p)
+	}
+	return len(p), nil
+}
+
+var tempDirBase string
+
 func main() {
-	log.SetFlags(log.Lshortfile)
-
 	rand.Seed(time.Now().UnixNano())
 
-	app := kingpin.New(
-		"out-of-tree",
-		"kernel {module, exploit} development tool",
+	cli := CLI{}
+	ctx := kong.Parse(&cli,
+		kong.Name("out-of-tree"),
+		kong.Description("kernel {module, exploit} development tool"),
+		kong.UsageOnError(),
+		kong.ConfigureHelp(kong.HelpOptions{
+			Compact: true,
+		}),
+		kong.Vars{
+			"version": "2.0.6",
+		},
 	)
 
-	app.Author("Mikhail Klementev <root@dumpstack.io>")
-	app.Version("1.3.0")
-
-	pathFlag := app.Flag("path", "Path to work directory")
-	path := pathFlag.Default(".").ExistingDir()
+	var loglevel zerolog.Level
+	switch cli.LogLevel {
+	case "trace":
+		loglevel = zerolog.TraceLevel
+	case "debug":
+		loglevel = zerolog.DebugLevel
+	case "info":
+		loglevel = zerolog.InfoLevel
+	case "warn":
+		loglevel = zerolog.WarnLevel
+	case "error":
+		loglevel = zerolog.ErrorLevel
+	}
 
 	usr, err := user.Current()
 	if err != nil {
 		return
 	}
-	os.MkdirAll(usr.HomeDir+"/.out-of-tree", os.ModePerm)
 
-	confPath := usr.HomeDir + "/.out-of-tree/out-of-tree.toml"
-	conf, err := config.ReadOutOfTreeConf(confPath)
-	if err != nil {
-		return
+	tempDirBase = usr.HomeDir + "/.out-of-tree/tmp/"
+	os.MkdirAll(tempDirBase, os.ModePerm)
+
+	log.Logger = log.Output(zerolog.MultiLevelWriter(
+		&LevelWriter{Writer: zerolog.NewConsoleWriter(
+			func(w *zerolog.ConsoleWriter) {
+				w.Out = os.Stderr
+			},
+		),
+			Level: loglevel,
+		},
+		&LevelWriter{Writer: &lumberjack.Logger{
+			Filename: usr.HomeDir + "/.out-of-tree/logs/out-of-tree.log",
+		},
+			Level: zerolog.TraceLevel,
+		},
+	))
+
+	log.Trace().Msg("start out-of-tree")
+	log.Debug().Msgf("%v", os.Args)
+	log.Debug().Msgf("%v", cli)
+
+	if buildInfo, ok := debug.ReadBuildInfo(); ok {
+		log.Debug().Msgf("%v", buildInfo.GoVersion)
+		log.Debug().Msgf("%v", buildInfo.Settings)
 	}
 
-	kcfgPathFlag := app.Flag("kernels", "Path to main kernels config")
-	kcfgPath := kcfgPathFlag.Default(conf.Kernels).String()
-
-	dbPathFlag := app.Flag("db", "Path to database")
-	dbPath := dbPathFlag.Default(conf.Database).String()
-
-	userKcfgPathFlag := app.Flag("user-kernels", "User kernels config")
-	userKcfgPathEnv := userKcfgPathFlag.Envar("OUT_OF_TREE_KCFG")
-	userKcfgPath := userKcfgPathEnv.Default(conf.UserKernels).String()
-
-	timeoutFlag := app.Flag("timeout", "Timeout after tool will not spawn new tests")
-	timeout := timeoutFlag.Duration()
-
-	qemuTimeoutFlag := app.Flag("qemu-timeout", "Timeout for qemu")
-	qemuTimeout := qemuTimeoutFlag.Default(conf.Qemu.Timeout).Duration()
-
-	dockerTimeoutFlag := app.Flag("docker-timeout", "Timeout for docker")
-	dockerTimeout := dockerTimeoutFlag.Default(conf.Docker.Timeout).Duration()
-
-	dockerRegistryFlag := app.Flag("docker-registry", "Registry for docker")
-	dockerRegistry := dockerRegistryFlag.Default(conf.Docker.Registry).String()
-
-	thresholdFlag := app.Flag("threshold", "Reliablity threshold for exit code")
-	threshold := thresholdFlag.Default("1.00").Float64()
-
-	pewCommand := app.Command("pew", "Build, run and test module/exploit")
-
-	pewMax := pewCommand.Flag("max", "Test no more than X kernels").
-		PlaceHolder("X").Default(fmt.Sprint(kernelsAll)).Int64()
-
-	pewRuns := pewCommand.Flag("runs", "Runs per each kernel").
-		Default("1").Int64()
-
-	pewKernelFlag := pewCommand.Flag("kernel", "Override kernel regex")
-	pewKernel := pewKernelFlag.String()
-
-	pewGuessFlag := pewCommand.Flag("guess", "Try all defined kernels")
-	pewGuess := pewGuessFlag.Bool()
-
-	pewBinaryFlag := pewCommand.Flag("binary", "Use binary, do not build")
-	pewBinary := pewBinaryFlag.String()
-
-	pewTestFlag := pewCommand.Flag("test", "Override path test")
-	pewTest := pewTestFlag.String()
-
-	pewDistFlag := pewCommand.Flag("dist", "Build result path")
-	pewDist := pewDistFlag.Default(pathDevNull).String()
-
-	pewThreadsFlag := pewCommand.Flag("threads", "Build result path")
-	pewThreads := pewThreadsFlag.Default(strconv.Itoa(runtime.NumCPU())).Int()
-
-	pewTagFlag := pewCommand.Flag("tag", "Log tagging")
-	pewTag := pewTagFlag.String()
-
-	pewVerboseFlag := pewCommand.Flag("verbose", "Show more information")
-	pewVerbose := pewVerboseFlag.Bool()
-
-	kernelCommand := app.Command("kernel", "Manipulate kernels")
-	kernelNoDownload := kernelCommand.Flag("no-download",
-		"Do not download qemu image while kernel generation").Bool()
-	kernelUseHost := kernelCommand.Flag("host", "Use also host kernels").Bool()
-	kernelListCommand := kernelCommand.Command("list", "List kernels")
-	kernelAutogenCommand := kernelCommand.Command("autogen",
-		"Generate kernels based on a current config")
-	kernelAutogenMax := kernelAutogenCommand.Flag("max",
-		"Download random kernels from set defined by regex in "+
-			"release_mask, but no more than X for each of "+
-			"release_mask").PlaceHolder("X").Default(
-		fmt.Sprint(kernelsAll)).Int64()
-	kernelDockerRegenCommand := kernelCommand.Command("docker-regen",
-		"Regenerate kernels config from out_of_tree_* docker images")
-	kernelGenallCommand := kernelCommand.Command("genall",
-		"Generate all kernels for distro")
-
-	genallDistroFlag := kernelGenallCommand.Flag("distro", "Distributive")
-	distro := genallDistroFlag.Required().String()
-
-	genallVerFlag := kernelGenallCommand.Flag("ver", "Distro version")
-	version := genallVerFlag.Required().String()
-
-	genCommand := app.Command("gen", "Generate .out-of-tree.toml skeleton")
-	genModuleCommand := genCommand.Command("module",
-		"Generate .out-of-tree.toml skeleton for kernel module")
-	genExploitCommand := genCommand.Command("exploit",
-		"Generate .out-of-tree.toml skeleton for kernel exploit")
-
-	debugCommand := app.Command("debug", "Kernel debug environment")
-	debugCommandFlag := debugCommand.Flag("kernel", "Regex (first match)")
-	debugKernel := debugCommandFlag.Required().String()
-	debugFlagGDB := debugCommand.Flag("gdb", "Set gdb listen address")
-	debugGDB := debugFlagGDB.Default("tcp::1234").String()
-
-	yekaslr := debugCommand.Flag("enable-kaslr", "Enable KASLR").Bool()
-	yesmep := debugCommand.Flag("enable-smep", "Enable SMEP").Bool()
-	yesmap := debugCommand.Flag("enable-smap", "Enable SMAP").Bool()
-	yekpti := debugCommand.Flag("enable-kpti", "Enable KPTI").Bool()
-
-	nokaslr := debugCommand.Flag("disable-kaslr", "Disable KASLR").Bool()
-	nosmep := debugCommand.Flag("disable-smep", "Disable SMEP").Bool()
-	nosmap := debugCommand.Flag("disable-smap", "Disable SMAP").Bool()
-	nokpti := debugCommand.Flag("disable-kpti", "Disable KPTI").Bool()
-
-	bootstrapCommand := app.Command("bootstrap", "Apparently nothing")
-
-	logCommand := app.Command("log", "Logs")
-
-	logQueryCommand := logCommand.Command("query", "Query logs")
-	logNum := logQueryCommand.Flag("num", "How much lines").Default("50").Int()
-	logRate := logQueryCommand.Flag("rate", "Show artifact success rate").Bool()
-	logTag := logQueryCommand.Flag("tag", "Filter tag").String()
-
-	logDumpCommand := logCommand.Command("dump",
-		"Show all info for log entry with ID")
-	logDumpID := logDumpCommand.Arg("ID", "").Required().Int()
-
-	logJSONCommand := logCommand.Command("json", "Generate json statistics")
-	logJSONTag := logJSONCommand.Flag("tag", "Filter tag").Required().String()
-
-	logMarkdownCommand := logCommand.Command("markdown", "Generate markdown statistics")
-	logMarkdownTag := logMarkdownCommand.Flag("tag", "Filter tag").Required().String()
-
-	packCommand := app.Command("pack", "Exploit pack test")
-	packAutogen := packCommand.Flag("autogen", "Kernel autogeneration").Bool()
-	packNoDownload := packCommand.Flag("no-download",
-		"Do not download qemu image while kernel generation").Bool()
-	packExploitRuns := packCommand.Flag("exploit-runs",
-		"Amount of runs of each exploit").Default("4").Int64()
-	packKernelRuns := packCommand.Flag("kernel-runs",
-		"Amount of runs of each kernel").Default("1").Int64()
-
-	err = checkRequiredUtils()
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	err = checkDockerPermissions()
-	if err != nil {
-		log.Println(err)
-		log.Println("You have two options:")
-		log.Println("\t1. Add user to group docker;")
-		log.Println("\t2. Run out-of-tree with sudo.")
-		os.Exit(1)
-	}
-
-	if !exists(usr.HomeDir + "/.out-of-tree/kernels.toml") {
-		log.Println("No ~/.out-of-tree/kernels.toml: Probably you " +
-			"need to run `out-of-tree kernel autogen` in " +
-			"directory that contains .out-of-tree.toml " +
-			"with defined kernel masks " +
-			"(see docs at https://out-of-tree.io)")
-	}
-
-	kingpin.MustParse(app.Parse(os.Args[1:]))
-
-	if *yekaslr && *nokaslr {
-		log.Fatalln("Only one of disable/enable can be used at once")
-	}
-
-	if *yesmep && *nosmep {
-		log.Fatalln("Only one of disable/enable can be used at once")
-	}
-
-	if *yesmap && *nosmap {
-		log.Fatalln("Only one of disable/enable can be used at once")
-	}
-
-	if *yekpti && *nokpti {
-		log.Fatalln("Only one of disable/enable can be used at once")
-	}
-
-	kcfg, err := config.ReadKernelConfig(*kcfgPath)
-	if err != nil {
-		log.Println(err)
-	}
-
-	if exists(*userKcfgPath) {
-		userKcfg, err := config.ReadKernelConfig(*userKcfgPath)
-		if err != nil {
-			log.Fatalln(err)
-		}
-
-		for _, nk := range userKcfg.Kernels {
-			if !hasKernel(nk, kcfg) {
-				kcfg.Kernels = append(kcfg.Kernels, nk)
-			}
-		}
-	}
-
-	handleFallbacks(kcfg)
-
-	db, err := openDatabase(*dbPath)
-	if err != nil {
-		log.Fatalln(err)
-	}
-	defer db.Close()
-
-	stop := time.Time{} // never stop
-	if *timeout != 0 {
-		stop = time.Now().Add(*timeout)
-	}
-
-	switch kingpin.MustParse(app.Parse(os.Args[1:])) {
-	case pewCommand.FullCommand():
-		err = pewHandler(kcfg, *path, *pewKernel, *pewBinary,
-			*pewTest, *pewGuess, stop, *qemuTimeout, *dockerTimeout,
-			*pewMax, *pewRuns, *pewDist, *pewTag, *pewThreads,
-			db, *pewVerbose)
-	case kernelListCommand.FullCommand():
-		err = kernelListHandler(kcfg)
-	case kernelAutogenCommand.FullCommand():
-		err = kernelAutogenHandler(*path, *dockerRegistry,
-			conf.Docker.Commands, *kernelAutogenMax,
-			*kernelUseHost, !*kernelNoDownload)
-	case kernelDockerRegenCommand.FullCommand():
-		err = kernelDockerRegenHandler(*kernelUseHost, !*kernelNoDownload)
-	case kernelGenallCommand.FullCommand():
-		err = kernelGenallHandler(*distro, *version,
-			*dockerRegistry, conf.Docker.Commands,
-			*kernelUseHost, !*kernelNoDownload)
-	case genModuleCommand.FullCommand():
-		err = genConfig(config.KernelModule)
-	case genExploitCommand.FullCommand():
-		err = genConfig(config.KernelExploit)
-	case debugCommand.FullCommand():
-		err = debugHandler(kcfg, *path, *debugKernel, *debugGDB,
-			*dockerTimeout, *yekaslr, *yesmep, *yesmap, *yekpti,
-			*nokaslr, *nosmep, *nosmap, *nokpti)
-	case bootstrapCommand.FullCommand():
-		fmt.Println("bootstrap is no more required, " +
-			"now images downloading on-demand")
-		fmt.Println("please, remove it from any automation scripts, " +
-			"because it'll be removed in the next release")
-	case logQueryCommand.FullCommand():
-		err = logHandler(db, *path, *logTag, *logNum, *logRate)
-	case logDumpCommand.FullCommand():
-		err = logDumpHandler(db, *logDumpID)
-	case logJSONCommand.FullCommand():
-		err = logJSONHandler(db, *path, *logJSONTag)
-	case logMarkdownCommand.FullCommand():
-		err = logMarkdownHandler(db, *path, *logMarkdownTag)
-	case packCommand.FullCommand():
-		err = packHandler(db, *path, *dockerRegistry, stop,
-			conf.Docker.Commands, kcfg, *packAutogen,
-			!*packNoDownload, *packExploitRuns, *packKernelRuns)
-	}
-
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	if successRate(state) < *threshold {
-		os.Exit(1)
-	}
+	err = ctx.Run(&cli.Globals)
+	ctx.FatalIfErrorf(err)
 }

pack.go

@@ -1,58 +1,83 @@
-// Copyright 2019 Mikhail Klementev. All rights reserved.
+// Copyright 2023 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
 package main
 
 import (
-	"database/sql"
 	"fmt"
 	"io/ioutil"
-	"log"
-	"runtime"
 	"time"
 
-	"code.dumpstack.io/tools/out-of-tree/config"
+	"github.com/rs/zerolog/log"
 )
 
-func packHandler(db *sql.DB, path, registry string, stop time.Time,
-	commands []config.DockerCommand, kcfg config.KernelConfig,
-	autogen, download bool, exploitRuns, kernelRuns int64) (err error) {
+type PackCmd struct {
+	Autogen     bool  `help:"kernel autogeneration"`
+	UseHost     bool  `help:"also use host kernels"`
+	NoDownload  bool  `help:"do not download qemu image while kernel generation"`
+	ExploitRuns int64 `default:"4" help:"amount of runs of each exploit"`
+	KernelRuns  int64 `default:"1" help:"amount of runs of each kernel"`
+	Max         int64 `help:"download random kernels from set defined by regex in release_mask, but no more than X for each of release_mask" default:"1"`
 
-	dockerTimeout := time.Minute
-	qemuTimeout := time.Minute
-	threads := runtime.NumCPU()
+	Threads int `help:"threads" default:"4"`
 
+	Tag string `help:"filter tag"`
+
+	Timeout       time.Duration `help:"timeout after tool will not spawn new tests"`
+	QemuTimeout   time.Duration `help:"timeout for qemu"`
+	DockerTimeout time.Duration `help:"timeout for docker"`
+}
+
+func (cmd *PackCmd) Run(g *Globals) (err error) {
 	tag := fmt.Sprintf("pack_run_%d", time.Now().Unix())
-	log.Println("Tag:", tag)
+	log.Print("Tag:", tag)
 
-	files, err := ioutil.ReadDir(path)
+	files, err := ioutil.ReadDir(g.WorkDir)
 	if err != nil {
 		return
 	}
 
 	for _, f := range files {
-		workPath := path + "/" + f.Name()
+		workPath := g.WorkDir + "/" + f.Name()
 
 		if !exists(workPath + "/.out-of-tree.toml") {
 			continue
 		}
 
-		if autogen {
-			var perRegex int64 = 1
-			err = kernelAutogenHandler(workPath, registry,
-				commands, perRegex, false, download)
+		if cmd.Autogen {
+			err = KernelAutogenCmd{Max: cmd.Max}.Run(
+				&KernelCmd{
+					NoDownload: cmd.NoDownload,
+					UseHost:    cmd.UseHost,
+				},
+				&Globals{
+					Config:  g.Config,
+					WorkDir: workPath,
+				},
+			)
 			if err != nil {
 				return
 			}
 		}
 
-		log.Println(f.Name())
+		log.Print(f.Name())
 
-		pewHandler(kcfg, workPath, "", "", "", false,
-			stop, dockerTimeout, qemuTimeout,
-			kernelRuns, exploitRuns, pathDevNull,
-			tag, threads, db, false)
+		pew := PewCmd{
+			Max:           cmd.KernelRuns,
+			Runs:          cmd.ExploitRuns,
+			Threads:       cmd.Threads,
+			Tag:           tag,
+			Timeout:       cmd.Timeout,
+			QemuTimeout:   cmd.QemuTimeout,
+			DockerTimeout: cmd.DockerTimeout,
+			Dist:          pathDevNull,
+		}
+
+		pew.Run(&Globals{
+			Config:  g.Config,
+			WorkDir: workPath,
+		})
 	}
 
 	return

pew.go

@@ -1,31 +1,137 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Copyright 2023 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
 package main
 
 import (
+	"bufio"
 	"database/sql"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"math/rand"
 	"os"
 	"os/exec"
-	"os/user"
 	"strings"
 	"time"
 
 	"github.com/otiai10/copy"
 	"github.com/remeh/sizedwaitgroup"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 	"gopkg.in/logrusorgru/aurora.v2"
 
 	"code.dumpstack.io/tools/out-of-tree/config"
 	"code.dumpstack.io/tools/out-of-tree/qemu"
 )
 
+type PewCmd struct {
+	Max     int64         `help:"test no more than X kernels" default:"100500"`
+	Runs    int64         `help:"runs per each kernel" default:"1"`
+	Kernel  string        `help:"override kernel regex"`
+	Guess   bool          `help:"try all defined kernels"`
+	Shuffle bool          `help:"randomize kernels test order"`
+	Binary  string        `help:"use binary, do not build"`
+	Test    string        `help:"override path for test"`
+	Dist    string        `help:"build result path" default:"/dev/null"`
+	Threads int           `help:"threads" default:"1"`
+	Tag     string        `help:"log tagging"`
+	Timeout time.Duration `help:"timeout after tool will not spawn new tests"`
+
+	ArtifactConfig string `help:"path to artifact config" type:"path"`
+
+	QemuTimeout   time.Duration `help:"timeout for qemu"`
+	DockerTimeout time.Duration `help:"timeout for docker"`
+
+	Threshold float64 `help:"reliablity threshold for exit code" default:"1.00"`
+
+	db              *sql.DB
+	kcfg            config.KernelConfig
+	timeoutDeadline time.Time
+}
+
+func (cmd *PewCmd) Run(g *Globals) (err error) {
+	cmd.kcfg, err = config.ReadKernelConfig(g.Config.Kernels)
+	if err != nil {
+		log.Fatal().Err(err).Msg("read kernels config")
+	}
+
+	if cmd.Timeout != 0 {
+		log.Info().Msgf("Set global timeout to %s", cmd.Timeout)
+		cmd.timeoutDeadline = time.Now().Add(cmd.Timeout)
+	}
+
+	cmd.db, err = openDatabase(g.Config.Database)
+	if err != nil {
+		log.Fatal().Err(err).
+			Msgf("Cannot open database %s", g.Config.Database)
+	}
+	defer cmd.db.Close()
+
+	var configPath string
+	if cmd.ArtifactConfig == "" {
+		configPath = g.WorkDir + "/.out-of-tree.toml"
+	} else {
+		configPath = cmd.ArtifactConfig
+	}
+	ka, err := config.ReadArtifactConfig(configPath)
+	if err != nil {
+		return
+	}
+
+	if ka.SourcePath == "" {
+		ka.SourcePath = g.WorkDir
+	}
+
+	if cmd.Kernel != "" {
+		var km config.KernelMask
+		km, err = kernelMask(cmd.Kernel)
+		if err != nil {
+			return
+		}
+
+		ka.SupportedKernels = []config.KernelMask{km}
+	}
+
+	if cmd.Guess {
+		ka.SupportedKernels, err = genAllKernels()
+		if err != nil {
+			return
+		}
+	}
+
+	if cmd.QemuTimeout != 0 {
+		log.Info().Msgf("Set qemu timeout to %s", cmd.QemuTimeout)
+	} else {
+		cmd.QemuTimeout = g.Config.Qemu.Timeout.Duration
+	}
+
+	if cmd.DockerTimeout != 0 {
+		log.Info().Msgf("Set docker timeout to %s", cmd.DockerTimeout)
+	} else {
+		cmd.DockerTimeout = g.Config.Docker.Timeout.Duration
+	}
+
+	if cmd.Tag == "" {
+		cmd.Tag = fmt.Sprintf("%d", time.Now().Unix())
+	}
+	log.Info().Str("tag", cmd.Tag).Msg("log")
+
+	err = cmd.performCI(ka)
+	if err != nil {
+		return
+	}
+
+	log.Info().Msgf("Success rate: %.02f, Threshold: %.02f",
+		successRate(state), cmd.Threshold)
+	if successRate(state) < cmd.Threshold {
+		err = errors.New("reliability threshold not met")
+	}
+	return
+}
+
 type runstate struct {
 	Overall, Success float64
 }
@@ -40,44 +146,107 @@ func successRate(state runstate) float64 {
 
 const pathDevNull = "/dev/null"
 
-func dockerRun(timeout time.Duration, container, workdir, command string) (
-	output string, err error) {
+func sh(workdir, command string) (output string, err error) {
+	flog := log.With().
+		Str("workdir", workdir).
+		Str("command", command).
+		Logger()
 
-	cmd := exec.Command("docker", "run", "-v", workdir+":/work",
-		container, "bash", "-c", "cd /work && "+command)
+	cmd := exec.Command("sh", "-c", "cd "+workdir+" && "+command)
 
-	timer := time.AfterFunc(timeout, func() {
-		cmd.Process.Kill()
-	})
-	defer timer.Stop()
+	flog.Debug().Msgf("%v", cmd)
 
-	raw, err := cmd.CombinedOutput()
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	cmd.Stderr = cmd.Stdout
+
+	err = cmd.Start()
 	if err != nil {
-		e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`",
-			err, command, string(raw))
-		err = errors.New(e)
 		return
 	}
 
-	output = string(raw)
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			m := scanner.Text()
+			output += m + "\n"
+			flog.Trace().Str("stdout", m).Msg("")
+		}
+	}()
+
+	err = cmd.Wait()
+
+	if err != nil {
+		e := fmt.Sprintf("%v %v output: %v", cmd, err, output)
+		err = errors.New(e)
+	}
+	return
+}
+
+func applyPatches(src string, ka config.Artifact) (err error) {
+	for i, patch := range ka.Patches {
+		name := fmt.Sprintf("patch_%02d", i)
+
+		path := src + "/" + name + ".diff"
+		if patch.Source != "" && patch.Path != "" {
+			err = errors.New("path and source are mutually exclusive")
+			return
+		} else if patch.Source != "" {
+			err = os.WriteFile(path, []byte(patch.Source), 0644)
+			if err != nil {
+				return
+			}
+		} else if patch.Path != "" {
+			err = copy.Copy(patch.Path, path)
+			if err != nil {
+				return
+			}
+		}
+
+		if patch.Source != "" || patch.Path != "" {
+			_, err = sh(src, "patch < "+path)
+			if err != nil {
+				return
+			}
+		}
+
+		if patch.Script != "" {
+			script := src + "/" + name + ".sh"
+			err = os.WriteFile(script, []byte(patch.Script), 0755)
+			if err != nil {
+				return
+			}
+			_, err = sh(src, script)
+			if err != nil {
+				return
+			}
+		}
+	}
 	return
 }
 
 func build(tmp string, ka config.Artifact, ki config.KernelInfo,
-	dockerTimeout time.Duration) (outPath, output string, err error) {
+	dockerTimeout time.Duration) (outdir, outpath, output string, err error) {
 
 	target := fmt.Sprintf("%d_%s", rand.Int(), ki.KernelRelease)
 
-	tmpSourcePath := tmp + "/source"
+	outdir = tmp + "/source"
 
-	err = copy.Copy(ka.SourcePath, tmpSourcePath)
+	err = copy.Copy(ka.SourcePath, outdir)
 	if err != nil {
 		return
 	}
 
-	outPath = tmpSourcePath + "/" + target
+	err = applyPatches(outdir, ka)
+	if err != nil {
+		return
+	}
+
+	outpath = outdir + "/" + target
 	if ka.Type == config.KernelModule {
-		outPath += ".ko"
+		outpath += ".ko"
 	}
 
 	kernel := "/lib/modules/" + ki.KernelRelease + "/build"
@@ -85,13 +254,25 @@ func build(tmp string, ka config.Artifact, ki config.KernelInfo,
 		kernel = ki.KernelSource
 	}
 
+	buildCommand := "make KERNEL=" + kernel + " TARGET=" + target
+	if ka.Make.Target != "" {
+		buildCommand += " " + ka.Make.Target
+	}
+
 	if ki.ContainerName != "" {
-		output, err = dockerRun(dockerTimeout, ki.ContainerName,
-			tmpSourcePath, "make KERNEL="+kernel+" TARGET="+target+
-				" && chmod -R 777 /work")
+		var c container
+		c, err = NewContainer(ki.ContainerName, dockerTimeout)
+		if err != nil {
+			log.Fatal().Err(err).Msg("container creation failure")
+		}
+
+		output, err = c.Run(outdir, buildCommand+" && chmod -R 777 /work")
 	} else {
-		command := "make KERNEL=" + kernel + " TARGET=" + target
-		cmd := exec.Command("bash", "-c", "cd "+tmpSourcePath+" && "+command)
+		cmd := exec.Command("bash", "-c", "cd "+outdir+" && "+
+			buildCommand)
+
+		log.Debug().Msgf("%v", cmd)
+
 		timer := time.AfterFunc(dockerTimeout, func() {
 			cmd.Process.Kill()
 		})
@@ -101,7 +282,7 @@ func build(tmp string, ka config.Artifact, ki config.KernelInfo,
 		raw, err = cmd.CombinedOutput()
 		if err != nil {
 			e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`",
-				err, command, string(raw))
+				err, buildCommand, string(raw))
 			err = errors.New(e)
 			return
 		}
@@ -111,6 +292,10 @@ func build(tmp string, ka config.Artifact, ki config.KernelInfo,
 	return
 }
 
+func runScript(q *qemu.System, script string) (output string, err error) {
+	return q.Command("root", script)
+}
+
 func testKernelModule(q *qemu.System, ka config.Artifact,
 	test string) (output string, err error) {
 
@@ -145,18 +330,23 @@ func testKernelExploit(q *qemu.System, ka config.Artifact,
 
 func genOkFail(name string, ok bool) (aurv aurora.Value) {
 	state.Overall += 1
+	s := " " + name
+	if name == "" {
+		s = ""
+	}
 	if ok {
 		state.Success += 1
-		s := " " + name + " SUCCESS "
+		s += " SUCCESS "
 		aurv = aurora.BgGreen(aurora.Black(s))
 	} else {
-		s := " " + name + " FAILURE "
+		s += " FAILURE "
 		aurv = aurora.BgRed(aurora.White(aurora.Bold(s)))
 	}
 	return
 }
 
 type phasesResult struct {
+	BuildDir         string
 	BuildArtifact    string
 	Build, Run, Test struct {
 		Output string
@@ -191,15 +381,19 @@ func dumpResult(q *qemu.System, ka config.Artifact, ki config.KernelInfo,
 		ki.DistroRelease, ki.KernelRelease)
 
 	colored := ""
-	if ka.Type == config.KernelExploit {
+	switch ka.Type {
+	case config.KernelExploit:
 		colored = aurora.Sprintf("[*] %40s: %s %s", distroInfo,
 			genOkFail("BUILD", res.Build.Ok),
 			genOkFail("LPE", res.Test.Ok))
-	} else {
+	case config.KernelModule:
 		colored = aurora.Sprintf("[*] %40s: %s %s %s", distroInfo,
 			genOkFail("BUILD", res.Build.Ok),
 			genOkFail("INSMOD", res.Run.Ok),
 			genOkFail("TEST", res.Test.Ok))
+	case config.Script:
+		colored = aurora.Sprintf("[*] %40s: %s", distroInfo,
+			genOkFail("", res.Test.Ok))
 	}
 
 	additional := ""
@@ -217,13 +411,13 @@ func dumpResult(q *qemu.System, ka config.Artifact, ki config.KernelInfo,
 
 	err := addToLog(db, q, ka, ki, res, tag)
 	if err != nil {
-		log.Println("[db] addToLog (", ka, ") error:", err)
+		log.Warn().Err(err).Msgf("[db] addToLog (%v)", ka)
 	}
 
 	if binary == "" && dist != pathDevNull {
 		err = os.MkdirAll(dist, os.ModePerm)
 		if err != nil {
-			log.Println("os.MkdirAll (", ka, ") error:", err)
+			log.Warn().Err(err).Msgf("os.MkdirAll (%v)", ka)
 		}
 
 		path := fmt.Sprintf("%s/%s-%s-%s", dist, ki.DistroType,
@@ -234,26 +428,40 @@ func dumpResult(q *qemu.System, ka config.Artifact, ki config.KernelInfo,
 
 		err = copyFile(res.BuildArtifact, path)
 		if err != nil {
-			log.Println("copyFile (", ka, ") error:", err)
+			log.Warn().Err(err).Msgf("copy file (%v)", ka)
 		}
 	}
 }
 
-func copyArtifactAndTest(q *qemu.System, ka config.Artifact,
+func copyArtifactAndTest(slog zerolog.Logger, q *qemu.System, ka config.Artifact,
 	res *phasesResult, remoteTest string) (err error) {
 
+	// Copy all test files to the remote machine
+	for _, f := range ka.TestFiles {
+		if f.Local[0] != '/' {
+			if res.BuildDir != "" {
+				f.Local = res.BuildDir + "/" + f.Local
+			}
+		}
+		err = q.CopyFile(f.User, f.Local, f.Remote)
+		if err != nil {
+			slog.Error().Err(err).Msg("copy test file")
+			return
+		}
+	}
+
 	switch ka.Type {
 	case config.KernelModule:
 		res.Run.Output, err = q.CopyAndInsmod(res.BuildArtifact)
 		if err != nil {
-			log.Println(res.Run.Output, err)
+			slog.Error().Err(err).Msg(res.Run.Output)
 			return
 		}
 		res.Run.Ok = true
 
 		res.Test.Output, err = testKernelModule(q, ka, remoteTest)
 		if err != nil {
-			log.Println(res.Test.Output, err)
+			slog.Error().Err(err).Msg(res.Test.Output)
 			return
 		}
 		res.Test.Ok = true
@@ -267,13 +475,22 @@ func copyArtifactAndTest(q *qemu.System, ka config.Artifact,
 		res.Test.Output, err = testKernelExploit(q, ka, remoteTest,
 			remoteExploit)
 		if err != nil {
-			log.Println(res.Test.Output)
+			slog.Error().Err(err).Msg(res.Test.Output)
 			return
 		}
 		res.Run.Ok = true // does not really used
 		res.Test.Ok = true
+	case config.Script:
+		res.Test.Output, err = runScript(q, remoteTest)
+		if err != nil {
+			slog.Error().Err(err).Msg(res.Test.Output)
+			return
+		}
+		slog.Info().Msg(res.Test.Output)
+		res.Run.Ok = true
+		res.Test.Ok = true
 	default:
-		log.Println("Unsupported artifact type")
+		slog.Fatal().Msg("Unsupported artifact type")
 	}
 
 	return
@@ -300,20 +517,53 @@ func copyTest(q *qemu.System, testPath string, ka config.Artifact) (
 	return
 }
 
-func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
-	ki config.KernelInfo, binaryPath, testPath string,
-	qemuTimeout, dockerTimeout time.Duration, dist, tag string,
-	db *sql.DB, verbose bool) {
+func copyStandardModules(q *qemu.System, ki config.KernelInfo) (err error) {
+	_, err = q.Command("root", "mkdir -p /lib/modules")
+	if err != nil {
+		return
+	}
+
+	files, err := ioutil.ReadDir(ki.ModulesPath)
+	if err != nil {
+		return
+	}
+
+	// FIXME scp cannot ignore symlinks
+	for _, f := range files {
+		if f.Mode()&os.ModeSymlink == os.ModeSymlink {
+			continue
+		}
+
+		path := ki.ModulesPath + "/" + f.Name()
+		err = q.CopyDirectory("root", path, "/lib/modules/"+ki.KernelRelease+"/")
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (cmd PewCmd) testArtifact(swg *sizedwaitgroup.SizedWaitGroup,
+	ka config.Artifact, ki config.KernelInfo) {
 
 	defer swg.Done()
 
+	slog := log.With().
+		Str("distro_type", ki.DistroType.String()).
+		Str("distro_release", ki.DistroRelease).
+		Str("kernel", ki.KernelRelease).
+		Logger()
+
+	slog.Info().Msg("start")
+
 	kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
 	q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
 	if err != nil {
-		log.Println("Qemu creation error:", err)
+		slog.Error().Err(err).Msg("qemu init")
 		return
 	}
-	q.Timeout = qemuTimeout
+	q.Timeout = cmd.QemuTimeout
 
 	if ka.Qemu.Timeout.Duration != 0 {
 		q.Timeout = ka.Qemu.Timeout.Duration
@@ -325,6 +575,10 @@ func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
 		q.Memory = ka.Qemu.Memory
 	}
 
+	if ka.Docker.Timeout.Duration != 0 {
+		cmd.DockerTimeout = ka.Docker.Timeout.Duration
+	}
+
 	q.SetKASLR(!ka.Mitigations.DisableKaslr)
 	q.SetSMEP(!ka.Mitigations.DisableSmep)
 	q.SetSMAP(!ka.Mitigations.DisableSmap)
@@ -332,66 +586,87 @@ func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
 
 	err = q.Start()
 	if err != nil {
-		log.Println("Qemu start error:", err)
+		slog.Error().Err(err).Msg("qemu start")
 		return
 	}
 	defer q.Stop()
 
-	if verbose {
-		go func() {
-			for !q.Died {
-				time.Sleep(time.Minute)
-				log.Println(ka.Name, ki.DistroType,
-					ki.DistroRelease, ki.KernelRelease,
-					"still alive")
+	go func() {
+		for !q.Died {
+			time.Sleep(time.Minute)
+			slog.Debug().Msg("still alive")
+		}
+	}()
 
-			}
-		}()
-	}
-
-	usr, err := user.Current()
+	tmp, err := ioutil.TempDir(tempDirBase, "out-of-tree_")
 	if err != nil {
-		return
-	}
-	tmpdir := usr.HomeDir + "/.out-of-tree/tmp"
-	os.MkdirAll(tmpdir, os.ModePerm)
-
-	tmp, err := ioutil.TempDir(tmpdir, "out-of-tree_")
-	if err != nil {
-		log.Println("Temporary directory creation error:", err)
+		slog.Error().Err(err).Msg("making tmp directory")
 		return
 	}
 	defer os.RemoveAll(tmp)
 
 	result := phasesResult{}
-	defer dumpResult(q, ka, ki, &result, dist, tag, binaryPath, db)
+	defer dumpResult(q, ka, ki, &result, cmd.Dist, cmd.Tag, cmd.Binary, cmd.db)
 
-	if binaryPath == "" {
-		result.BuildArtifact, result.Build.Output, err = build(tmp, ka,
-			ki, dockerTimeout)
+	if ka.Type == config.Script {
+		result.Build.Ok = true
+		cmd.Test = ka.Script
+	} else if cmd.Binary == "" {
+		// TODO: build should return structure
+		start := time.Now()
+		result.BuildDir, result.BuildArtifact, result.Build.Output, err =
+			build(tmp, ka, ki, cmd.DockerTimeout)
+		slog.Debug().Str("duration", time.Now().Sub(start).String()).
+			Msg("build done")
 		if err != nil {
-			log.Println(err)
+			log.Error().Err(err).Msg("build")
 			return
 		}
 		result.Build.Ok = true
 	} else {
-		result.BuildArtifact = binaryPath
+		result.BuildArtifact = cmd.Binary
 		result.Build.Ok = true
 	}
 
-	if testPath == "" {
-		testPath = result.BuildArtifact + "_test"
-		if !exists(testPath) {
-			testPath = tmp + "/" + "test.sh"
+	if cmd.Test == "" {
+		cmd.Test = result.BuildArtifact + "_test"
+		if !exists(cmd.Test) {
+			cmd.Test = tmp + "/source/" + "test.sh"
 		}
 	}
 
-	remoteTest, err := copyTest(q, testPath, ka)
+	err = q.WaitForSSH(cmd.QemuTimeout)
 	if err != nil {
 		return
 	}
 
-	copyArtifactAndTest(q, ka, &result, remoteTest)
+	remoteTest, err := copyTest(q, cmd.Test, ka)
+	if err != nil {
+		return
+	}
+
+	if ka.StandardModules {
+		// Module depends on one of the standard modules
+		start := time.Now()
+		err = copyStandardModules(q, ki)
+		if err != nil {
+			slog.Fatal().Err(err).Msg("copy standard modules")
+			return
+		}
+		slog.Debug().Str("duration", time.Now().Sub(start).String()).
+			Msg("copy standard modules")
+	}
+
+	err = preloadModules(q, ka, ki, cmd.DockerTimeout)
+	if err != nil {
+		slog.Error().Err(err).Msg("preload modules")
+		return
+	}
+
+	start := time.Now()
+	copyArtifactAndTest(slog, q, ka, &result, remoteTest)
+	slog.Debug().Str("duration", time.Now().Sub(start).String()).
+		Msg("test completed")
 }
 
 func shuffleKernels(a []config.KernelInfo) []config.KernelInfo {
@@ -403,16 +678,15 @@ func shuffleKernels(a []config.KernelInfo) []config.KernelInfo {
 	return a
 }
 
-func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath,
-	testPath string, stop time.Time,
-	qemuTimeout, dockerTimeout time.Duration,
-	max, runs int64, dist, tag string, threads int,
-	db *sql.DB, verbose bool) (err error) {
-
+func (cmd PewCmd) performCI(ka config.Artifact) (err error) {
 	found := false
+	max := cmd.Max
 
-	swg := sizedwaitgroup.New(threads)
-	for _, kernel := range shuffleKernels(kcfg.Kernels) {
+	swg := sizedwaitgroup.New(cmd.Threads)
+	if cmd.Shuffle {
+		cmd.kcfg.Kernels = shuffleKernels(cmd.kcfg.Kernels)
+	}
+	for _, kernel := range cmd.kcfg.Kernels {
 		if max <= 0 {
 			break
 		}
@@ -426,14 +700,14 @@ func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath,
 		if supported {
 			found = true
 			max--
-			for i := int64(0); i < runs; i++ {
-				if !stop.IsZero() && time.Now().After(stop) {
+			for i := int64(0); i < cmd.Runs; i++ {
+				if !cmd.timeoutDeadline.IsZero() &&
+					time.Now().After(cmd.timeoutDeadline) {
+
 					break
 				}
 				swg.Add()
-				go whatever(&swg, ka, kernel, binaryPath,
-					testPath, qemuTimeout, dockerTimeout,
-					dist, tag, db, verbose)
+				go cmd.testArtifact(&swg, ka, kernel)
 			}
 		}
 	}
@@ -448,8 +722,10 @@ func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath,
 
 func exists(path string) bool {
 	if _, err := os.Stat(path); err != nil {
+		log.Debug().Msgf("%s does not exist", path)
 		return false
 	}
+	log.Debug().Msgf("%s exist", path)
 	return true
 }
 
@@ -484,46 +760,3 @@ func genAllKernels() (sk []config.KernelMask, err error) {
 	}
 	return
 }
-
-// TODO: Now too many parameters, move all of them to some structure
-func pewHandler(kcfg config.KernelConfig,
-	workPath, ovrrdKrnl, binary, test string, guess bool,
-	stop time.Time, qemuTimeout, dockerTimeout time.Duration,
-	max, runs int64, dist, tag string, threads int,
-	db *sql.DB, verbose bool) (err error) {
-
-	ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
-	if err != nil {
-		return
-	}
-
-	if ka.SourcePath == "" {
-		ka.SourcePath = workPath
-	}
-
-	if ovrrdKrnl != "" {
-		var km config.KernelMask
-		km, err = kernelMask(ovrrdKrnl)
-		if err != nil {
-			return
-		}
-
-		ka.SupportedKernels = []config.KernelMask{km}
-	}
-
-	if guess {
-		ka.SupportedKernels, err = genAllKernels()
-		if err != nil {
-			return
-		}
-	}
-
-	err = performCI(ka, kcfg, binary, test,
-		stop, qemuTimeout, dockerTimeout,
-		max, runs, dist, tag, threads, db, verbose)
-	if err != nil {
-		return
-	}
-
-	return
-}

pew_test.go

@@ -1,44 +0,0 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
-// Use of this source code is governed by a AGPLv3 license
-// (or later) that can be found in the LICENSE file.
-
-package main
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"strings"
-	"testing"
-	"time"
-)
-
-func TestDockerRun(t *testing.T) {
-	tmp, err := ioutil.TempDir("/tmp/", "out-of-tree_test_")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmp)
-
-	start := time.Now()
-
-	timeout := time.Second
-
-	_, err = dockerRun(timeout, "ubuntu", tmp, "sleep 5s")
-	if err == nil {
-		t.Fatal("docker is not killed by timeout")
-	}
-	if time.Since(start) > 3*time.Second {
-		t.Fatal(fmt.Sprintf("timeout failed (%v instead of %v)",
-			time.Since(start), time.Second))
-	}
-
-	output, err := dockerRun(time.Minute, "ubuntu", tmp, "echo hello")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !strings.Contains(output, "hello") {
-		t.Fatal("wrong output (" + output + ")")
-	}
-}

preload.go

@@ -0,0 +1,166 @@
+// Copyright 2020 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package main
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"errors"
+	"io/ioutil"
+	"os"
+	"os/user"
+	"path/filepath"
+	"time"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/rs/zerolog/log"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+	"code.dumpstack.io/tools/out-of-tree/qemu"
+)
+
+func preloadModules(q *qemu.System, ka config.Artifact, ki config.KernelInfo,
+	dockerTimeout time.Duration) (err error) {
+
+	for _, pm := range ka.Preload {
+		err = preload(q, ki, pm, dockerTimeout)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func preload(q *qemu.System, ki config.KernelInfo, pm config.PreloadModule,
+	dockerTimeout time.Duration) (err error) {
+
+	var workPath, cache string
+	if pm.Path != "" {
+		log.Print("Use non-git path for preload module (no cache)")
+		workPath = pm.Path
+	} else if pm.Repo != "" {
+		workPath, cache, err = cloneOrPull(pm.Repo, ki)
+		if err != nil {
+			return
+		}
+	} else {
+		errors.New("No repo/path in preload entry")
+	}
+
+	err = buildAndInsmod(workPath, q, ki, dockerTimeout, cache)
+	if err != nil {
+		return
+	}
+
+	time.Sleep(pm.TimeoutAfterLoad.Duration)
+	return
+}
+
+func buildAndInsmod(workPath string, q *qemu.System, ki config.KernelInfo,
+	dockerTimeout time.Duration, cache string) (err error) {
+
+	tmp, err := ioutil.TempDir(tempDirBase, "out-of-tree_")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(tmp)
+
+	var artifact string
+	if exists(cache) {
+		artifact = cache
+	} else {
+		artifact, err = buildPreload(workPath, tmp, ki, dockerTimeout)
+		if err != nil {
+			return
+		}
+		if cache != "" {
+			err = copyFile(artifact, cache)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	output, err := q.CopyAndInsmod(artifact)
+	if err != nil {
+		log.Print(output)
+		return
+	}
+	return
+}
+
+func buildPreload(workPath, tmp string, ki config.KernelInfo,
+	dockerTimeout time.Duration) (artifact string, err error) {
+
+	ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
+	if err != nil {
+		return
+	}
+
+	ka.SourcePath = workPath
+
+	km := config.KernelMask{DistroType: ki.DistroType,
+		DistroRelease: ki.DistroRelease,
+		ReleaseMask:   ki.KernelRelease,
+	}
+	ka.SupportedKernels = []config.KernelMask{km}
+
+	if ka.Docker.Timeout.Duration != 0 {
+		dockerTimeout = ka.Docker.Timeout.Duration
+	}
+
+	_, artifact, _, err = build(tmp, ka, ki, dockerTimeout)
+	return
+}
+
+func cloneOrPull(repo string, ki config.KernelInfo) (workPath, cache string, err error) {
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+	base := filepath.Join(usr.HomeDir, "/.out-of-tree/preload/")
+	workPath = filepath.Join(base, "/repos/", sha1sum(repo))
+
+	var r *git.Repository
+	if exists(workPath) {
+		r, err = git.PlainOpen(workPath)
+		if err != nil {
+			return
+		}
+
+		var w *git.Worktree
+		w, err = r.Worktree()
+		if err != nil {
+			return
+		}
+
+		err = w.Pull(&git.PullOptions{})
+		if err != nil && err != git.NoErrAlreadyUpToDate {
+			log.Print(repo, "pull error:", err)
+		}
+	} else {
+		r, err = git.PlainClone(workPath, false, &git.CloneOptions{URL: repo})
+		if err != nil {
+			return
+		}
+	}
+
+	ref, err := r.Head()
+	if err != nil {
+		return
+	}
+
+	cachedir := filepath.Join(base, "/cache/")
+	os.MkdirAll(cachedir, 0700)
+
+	filename := sha1sum(repo + ki.KernelPath + ref.Hash().String())
+	cache = filepath.Join(cachedir, filename)
+	return
+}
+
+func sha1sum(data string) string {
+	h := sha1.Sum([]byte(data))
+	return hex.EncodeToString(h[:])
+}

qemu/qemu-kernel.go

@@ -5,7 +5,7 @@
 package qemu
 
 import (
-	"bytes"
+	"bufio"
 	"errors"
 	"fmt"
 	"io"
@@ -18,29 +18,11 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 	"golang.org/x/crypto/ssh"
 )
 
-func readUntilEOF(pipe io.ReadCloser, buf *[]byte) (err error) {
-	bufSize := 1024
-	for err != io.EOF {
-		stdout := make([]byte, bufSize)
-		var n int
-
-		n, err = pipe.Read(stdout)
-		if err != nil && err != io.EOF {
-			return
-		}
-
-		*buf = append(*buf, stdout[:n]...)
-	}
-
-	if err == io.EOF {
-		err = nil
-	}
-	return
-}
-
 type arch string
 
 const (
@@ -66,6 +48,8 @@ type System struct {
 	kernel    Kernel
 	drivePath string
 
+	Mutable bool
+
 	Cpus   int
 	Memory int
 
@@ -94,18 +78,25 @@ type System struct {
 		stdout io.ReadCloser
 	}
 
-	Stdout, Stderr []byte
+	Stdout, Stderr string
 
 	// accessible after qemu is closed
 	exitErr error
+
+	log zerolog.Logger
 }
 
 // NewSystem constructor
 func NewSystem(arch arch, kernel Kernel, drivePath string) (q *System, err error) {
+	q = &System{}
+	q.log = log.With().
+		Str("kernel", kernel.KernelPath).
+		Logger()
+
 	if _, err = exec.LookPath("qemu-system-" + string(arch)); err != nil {
 		return
 	}
-	q = &System{}
+
 	q.arch = arch
 
 	if _, err = os.Stat(kernel.KernelPath); err != nil {
@@ -125,6 +116,12 @@ func NewSystem(arch arch, kernel Kernel, drivePath string) (q *System, err error
 	return
 }
 
+func (q *System) SetSSHAddrPort(addr string, port int) (err error) {
+	// TODO validate
+	q.sshAddrPort = fmt.Sprintf("%s:%d", addr, port)
+	return
+}
+
 func getRandomAddrPort() (addr string) {
 	// 127.1-255.0-255.0-255:10000-50000
 	ip := fmt.Sprintf("127.%d.%d.%d",
@@ -178,11 +175,12 @@ func kvmExists() bool {
 func (q *System) panicWatcher() {
 	for {
 		time.Sleep(time.Second)
-		if bytes.Contains(q.Stdout, []byte("Kernel panic")) {
+		if strings.Contains(q.Stdout, "Kernel panic") {
+			q.KernelPanic = true
+			q.log.Debug().Msg("kernel panic")
 			time.Sleep(time.Second)
 			// There is no reason to stay alive after kernel panic
 			q.Stop()
-			q.KernelPanic = true
 			return
 		}
 	}
@@ -213,9 +211,11 @@ func (q System) cmdline() (s string) {
 // Start qemu process
 func (q *System) Start() (err error) {
 	rand.Seed(time.Now().UnixNano()) // Are you sure?
-	q.sshAddrPort = getFreeAddrPort()
+	if q.sshAddrPort == "" {
+		q.sshAddrPort = getFreeAddrPort()
+	}
 	hostfwd := fmt.Sprintf("hostfwd=tcp:%s-:22", q.sshAddrPort)
-	qemuArgs := []string{"-snapshot", "-nographic",
+	qemuArgs := []string{"-nographic",
 		"-hda", q.drivePath,
 		"-kernel", q.kernel.KernelPath,
 		"-smp", fmt.Sprintf("%d", q.Cpus),
@@ -224,6 +224,10 @@ func (q *System) Start() (err error) {
 		"-netdev", "user,id=n1," + hostfwd,
 	}
 
+	if !q.Mutable {
+		qemuArgs = append(qemuArgs, "-snapshot")
+	}
+
 	if q.debug {
 		qemuArgs = append(qemuArgs, "-gdb", q.gdb)
 	}
@@ -243,6 +247,7 @@ func (q *System) Start() (err error) {
 	qemuArgs = append(qemuArgs, "-append", q.cmdline())
 
 	q.cmd = exec.Command("qemu-system-"+string(q.arch), qemuArgs...)
+	q.log.Debug().Msgf("%v", q.cmd)
 
 	if q.pipe.stdin, err = q.cmd.StdinPipe(); err != nil {
 		return
@@ -261,8 +266,23 @@ func (q *System) Start() (err error) {
 		return
 	}
 
-	go readUntilEOF(q.pipe.stdout, &q.Stdout)
-	go readUntilEOF(q.pipe.stderr, &q.Stderr)
+	go func() {
+		scanner := bufio.NewScanner(q.pipe.stdout)
+		for scanner.Scan() {
+			m := scanner.Text()
+			q.Stdout += m + "\n"
+			q.log.Trace().Str("stdout", m).Msg("")
+		}
+	}()
+
+	go func() {
+		scanner := bufio.NewScanner(q.pipe.stderr)
+		for scanner.Scan() {
+			m := scanner.Text()
+			q.Stderr += m + "\n"
+			q.log.Trace().Str("stderr", m).Msg("")
+		}
+	}()
 
 	go func() {
 		q.exitErr = q.cmd.Wait()
@@ -301,6 +321,20 @@ func (q *System) Stop() {
 	}
 }
 
+func (q System) WaitForSSH(timeout time.Duration) error {
+	for start := time.Now(); time.Since(start) < timeout; {
+		client, err := q.ssh("root")
+		if err != nil {
+			time.Sleep(time.Second / 10)
+			continue
+		}
+		client.Close()
+		return nil
+	}
+
+	return errors.New("no ssh (timeout)")
+}
+
 func (q System) ssh(user string) (client *ssh.Client, err error) {
 	cfg := &ssh.ClientConfig{
 		User:            user,
@@ -313,6 +347,14 @@ func (q System) ssh(user string) (client *ssh.Client, err error) {
 
 // Command executes shell commands on qemu system
 func (q System) Command(user, cmd string) (output string, err error) {
+	flog := log.With().
+		Str("kernel", q.kernel.KernelPath).
+		Str("user", user).
+		Str("cmd", cmd).
+		Logger()
+
+	flog.Debug().Msg("qemu command")
+
 	client, err := q.ssh(user)
 	if err != nil {
 		return
@@ -324,8 +366,28 @@ func (q System) Command(user, cmd string) (output string, err error) {
 		return
 	}
 
-	bytesOutput, err := session.CombinedOutput(cmd)
-	output = string(bytesOutput)
+	stdout, err := session.StdoutPipe()
+	if err != nil {
+		return
+	}
+	session.Stderr = session.Stdout
+
+	err = session.Start(cmd)
+	if err != nil {
+		return
+	}
+
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			m := scanner.Text()
+			output += m + "\n"
+			flog.Trace().Str("stdout", m).Msg("")
+		}
+		output = strings.TrimSuffix(output, "\n")
+	}()
+
+	err = session.Wait()
 	return
 }
 
@@ -346,24 +408,70 @@ func (q System) AsyncCommand(user, cmd string) (err error) {
 		"nohup sh -c '%s' > /dev/null 2> /dev/null < /dev/null &", cmd))
 }
 
-// CopyFile is copy file from local machine to remote through ssh/scp
-func (q *System) CopyFile(user, localPath, remotePath string) (err error) {
+func (q System) scp(user, localPath, remotePath string, recursive bool) (err error) {
 	addrPort := strings.Split(q.sshAddrPort, ":")
 	addr := addrPort[0]
 	port := addrPort[1]
 
-	cmd := exec.Command("scp", "-P", port,
+	args := []string{
+		"-P", port,
 		"-o", "StrictHostKeyChecking=no",
+		"-o", "UserKnownHostsFile=/dev/null",
 		"-o", "LogLevel=error",
-		localPath, user+"@"+addr+":"+remotePath)
+	}
+
+	if recursive {
+		cmd := exec.Command("ssh", "-V")
+
+		log.Debug().Msgf("%v", cmd)
+
+		var output []byte
+		output, err = cmd.CombinedOutput()
+		if err != nil {
+			return
+		}
+		sshVersion := string(output)
+
+		log.Debug().Str("ssh version", sshVersion).Msg("")
+
+		if strings.Contains(sshVersion, "OpenSSH_9") {
+			// This release switches scp from using the
+			// legacy scp/rcp protocol to using the SFTP
+			// protocol by default.
+			//
+			// To keep compatibility with old distros,
+			// using -O flag to use the legacy scp/rcp.
+			//
+			// Note: old ssh doesn't support -O flag
+			args = append(args, "-O")
+		}
+
+		args = append(args, "-r")
+	}
+
+	args = append(args, localPath, user+"@"+addr+":"+remotePath)
+
+	cmd := exec.Command("scp", args...)
+	log.Debug().Msgf("%v", cmd)
+
 	output, err := cmd.CombinedOutput()
-	if err != nil {
+	if err != nil || string(output) != "" {
 		return errors.New(string(output))
 	}
 
 	return
 }
 
+// CopyFile from local machine to remote via scp
+func (q System) CopyFile(user, localPath, remotePath string) (err error) {
+	return q.scp(user, localPath, remotePath, false)
+}
+
+// CopyDirectory from local machine to remote via scp
+func (q System) CopyDirectory(user, localPath, remotePath string) (err error) {
+	return q.scp(user, localPath, remotePath, true)
+}
+
 // CopyAndInsmod copy kernel module to temporary file on qemu then insmod it
 func (q *System) CopyAndInsmod(localKoPath string) (output string, err error) {
 	remoteKoPath := fmt.Sprintf("/tmp/module_%d.ko", rand.Int())

qemu/test.config.go

@@ -4,7 +4,7 @@
 
 package qemu
 
-const testConfigVmlinuz = "../tools/qemu-debian-img/ubuntu2004.vmlinuz"
-const testConfigInitrd = "../tools/qemu-debian-img/ubuntu2004.initrd"
-const testConfigRootfs = "../tools/qemu-debian-img/ubuntu2004.img"
-const testConfigSampleKo = "../tools/qemu-debian-img/ubuntu2004.ko"
+const testConfigVmlinuz = "../tools/qemu-debian-img/ubuntu2204.vmlinuz"
+const testConfigInitrd = "../tools/qemu-debian-img/ubuntu2204.initrd"
+const testConfigRootfs = "../tools/qemu-debian-img/ubuntu2204.img"
+const testConfigSampleKo = "../tools/qemu-debian-img/ubuntu2204.ko"

shell.nix

@@ -0,0 +1,5 @@
+{ pkgs ? import <nixpkgs> {} }:
+
+with pkgs; mkShell {
+  packages = [ go gcc qemu ];
+}

tools/qemu-debian-img/Dockerfile

@@ -4,13 +4,13 @@
 #
 # Usage:
 #
-#     $ docker build -t gen-ubuntu2004-image .
-#     $ docker run --privileged -v $(pwd):/shared -t gen-ubuntu2004-image
+#     $ docker build -t gen-ubuntu2204-image .
+#     $ docker run --privileged -v $(pwd):/shared -t gen-ubuntu2204-image
 #
-# ubuntu2004.img will be created in current directory. You can change $(pwd) to
+# ubuntu2204.img will be created in current directory. You can change $(pwd) to
 # different directory to use different destination for image.
 #
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update
@@ -19,9 +19,9 @@ RUN apt install -y linux-image-generic
 
 ENV TMPDIR=/tmp/ubuntu
 ENV IMAGEDIR=/tmp/image
-ENV IMAGE=/shared/ubuntu2004.img
+ENV IMAGE=/shared/ubuntu2204.img
 ENV REPOSITORY=http://archive.ubuntu.com/ubuntu
-ENV RELEASE=focal
+ENV RELEASE=jammy
 
 RUN mkdir $IMAGEDIR
 

tools/qemu-debian-img/bootstrap.sh

@@ -1,9 +1,9 @@
 #!/bin/sh -eux
 cd $(dirname $(realpath $0))
 
-docker build -t gen-ubuntu2004-image .
-docker run --privileged -v $(pwd):/shared -t gen-ubuntu2004-image
-RUN="docker run -v $(pwd):/shared -t gen-ubuntu2004-image"
-$RUN sh -c 'chmod 644 /boot/vmlinuz && cp /boot/vmlinuz /shared/ubuntu2004.vmlinuz'
-$RUN sh -c 'cp /boot/initrd.img /shared/ubuntu2004.initrd'
-$RUN sh -c 'cp $(find /lib/modules -name test_bpf.ko) /shared/ubuntu2004.ko'
+docker build -t gen-ubuntu2204-image .
+docker run --privileged -v $(pwd):/shared -t gen-ubuntu2204-image
+RUN="docker run -v $(pwd):/shared -t gen-ubuntu2204-image"
+$RUN sh -c 'chmod 644 /boot/vmlinuz && cp /boot/vmlinuz /shared/ubuntu2204.vmlinuz'
+$RUN sh -c 'cp /boot/initrd.img /shared/ubuntu2204.initrd'
+$RUN sh -c 'cp $(find /lib/modules -name test_bpf.ko) /shared/ubuntu2204.ko'