Bump version

This reverts commit 5931c08de1.

.github/workflows/macos.yml

@@ -0,0 +1,13 @@
+name: macOS
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: Build on macOS
+    runs-on: macOS-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build

.github/workflows/ubuntu.yml

@@ -0,0 +1,130 @@
+name: Ubuntu
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: Build on Ubuntu
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build
+
+  test-unit:
+    name: Unit Testing
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Install dependencies for tests
+      run: |
+        sudo apt-get update
+        sudo apt-get install qemu-system-x86
+
+    - name: Bootstrap
+      run: ./tools/qemu-ubuntu-img/bootstrap.sh
+
+    - name: Unit Testing
+      run: go test -parallel 1 -v ./...
+
+  test-end-to-end:
+    name: End-to-End Testing
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build
+
+    - name: Install dependencies for tests
+      run: |
+        sudo apt-get update
+        sudo apt-get install qemu-system-x86
+
+    - name: End-to-End Testing [Kernel Module]
+      run: |
+        cd examples/kernel-module
+        ../../out-of-tree --log-level=debug kernel autogen --max=1
+        ../../out-of-tree --log-level=debug pew --qemu-timeout=10m
+
+    - name: End-to-End Testing [Kernel Exploit]
+      run: |
+        cd examples/kernel-exploit
+        ../../out-of-tree --log-level=debug kernel autogen --max=1
+        ../../out-of-tree --log-level=debug pew --threshold=0 --qemu-timeout=10m
+
+    - name: Archive logs
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-end-to-end-logs
+        path: /home/runner/.out-of-tree/logs/out-of-tree.log
+
+  test-end-to-end-kernels:
+    name: End-to-End Testing (kernels)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build
+
+    - name: Install dependencies for tests
+      run: |
+        sudo apt-get update
+        sudo apt-get install qemu-system-x86
+
+    - name: End-to-End Testing [Install one Ubuntu 18.04 kernel]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=Ubuntu --ver=18.04 --kernel=4.15.0-70-generic
+
+    - name: End-to-End Testing [Reinstall one Ubuntu 18.04 kernel]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=Ubuntu --ver=18.04 --kernel=4.15.0-70-generic --force
+
+    - name: End-to-End Testing [Install one Ubuntu 22.04 kernel w/o headers]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=Ubuntu --ver=22.04 --kernel=5.19.0-28-generic --no-headers
+
+    - name: End-to-End Testing [Install one CentOS 7 kernel]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=CentOS --ver=7 --kernel=3.10.0-862.6.3
+
+    - name: End-to-End Testing [Install one CentOS 7 kernel w/o headers]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=CentOS --ver=7 --kernel=3.10.0-1160.71.1 --no-headers
+
+    - name: End-to-End Testing [Install one CentOS 8 kernel]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=CentOS --ver=8 --kernel=4.18.0-348.7.1
+
+    - name: Archive logs
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-end-to-end-kernels-log
+        path: /home/runner/.out-of-tree/logs/out-of-tree.log
+
+  test-end-to-end-genall:
+    name: End-to-End Testing (genall)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build
+
+    - name: Install dependencies for tests
+      run: |
+        sudo apt-get update
+        sudo apt-get install qemu-system-x86
+
+    - name: End-to-End Testing [Install all Ubuntu 22.04 kernels]
+      run: |
+        ./out-of-tree --log-level=debug kernel genall --distro=Ubuntu --ver=22.04
+
+    - name: Archive logs
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-end-to-end-genall-logs
+        path: /home/runner/.out-of-tree/logs/out-of-tree.log

.gitignore

@@ -10,3 +10,5 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+
+out-of-tree

.travis.yml

@@ -1,30 +0,0 @@
-language: go
-
-go:
-  - 1.x
-  - master
-
-os:
-  - linux
-
-dist:
-  - bionic
-
-addons:
-  apt:
-    packages:
-    - qemu
-
-services:
-  - docker
-
-env:
-  - GO111MODULE=on
-
-install: true
-
-before_script:
-  - ./tools/qemu-debian-img/bootstrap.sh
-
-script:
-  - go test -parallel 1 -v ./...

CHANGELOG.md

@@ -4,7 +4,166 @@
 
 [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [2.1.0]
+
+### Added
+
+- Graceful shutdown on ^C while kernels generation.
+
+- Flag to set the container runtime command.
+
+- out-of-tree image --dry-run for printing full qemu command.
+
+### Changed
+
+- No exit at the end of the retries, will continue with the other
+  kernels.
+
+- All temporary files moved to ~/.out-of-tree/tmp/.
+
+### Fixed
+
+- Discrepancies between /lib/modules and /boot should no longer lead
+  to fatal errors.
+
+- Podman support on macOS.
+
+## [2.0.0]
+
+### Breaking
+
+- Layers with kernels in containers have been abandoned in favor of
+  installation to mounted volumes.
+
+- Command line interface has been changed to alecthomas/kong.
+
+### Added
+
+- Command `kernel install` to install specific kernel.
+
+- Command `containers` to manage containers.
+
+- Command `image edit` to edit qemu image.
+
+- Flag `--force` to force reinstallation of the kernel.
+
+- Flag `--artifact-config` to specify the path to .out-of-tree.toml.
+
+- Flag `--no-headers` flag to install kernel and initrd only.
+
+- Flag `--shuffle` to randomize the order of kernels for
+  installation/testing.
+
+- Support make targets in artifact config.
+
+- Support patches in artifact config.
+
+- Support for copying standard modules to qemu.
+
+- Script artifact type for various automation and information gathering.
+
+- Add TestFiles to artifact config, transfers additional test files to VM.
+
+- Improved logging, with logfile at ~/.out-of-tree/logs/out-of-tree.log
+
+- Kernel installation will retry (10 times by default) in case of
+  network problems.
+
+- Stdout trace (with --log-level=trace, and always to logfile) for
+  qemu and container execution.
+
+- Compatibility with Podman.
+
+- Support for Ubuntu 22.04.
+
+## [1.4.0]
+
+### Added
+
+- Parameter `--docker-timeout` may also be set in the artifact
+  configuration file.
+
+- Preload modules before inserting module or run exploit. Modules can
+  be specified by git repository path in the `repo` parameter of
+  section `[[preload]]`. Also, there is a `path` parameter for local
+  projects. Note that `repo` is using a cache that uses last commit
+  hash to check is project needs to be rebuilt, so it's not suitable
+  for local development (except if you will commit each time before
+  run out-of-tree).
+
+- Flag `--disable-preload` to ignore `[[preload]]` section of
+  configuration file.
+
+- Now `out-of-tree log dump` will show the last log if no ID
+  specified.
+
+## [1.3.0] 2020-05-30
+
+### Added
+
+- Support for Ubuntu 20.04 and CentOS 8.
+
+## [1.2.1] 2019-12-25
+
+### Fixed
+
+- macOS support.
+
+## [1.2.0] 2019-11-15
+
+### Added
+
+- Flag for Verbose output. Right now only qemu status messages is
+  implemented.
+
+### Fixed
+
+- Kpti settings was not affected for regular runs.
+
+## [1.1.2] 2019-09-05
+
+### Added
+
+- Added policykit-1 to rootfs for Ubuntu.
+
+### Fixed
+
+- Avoided slow mirrors with use of mirror://mirrors.ubuntu.com for
+  Ubuntu 16.04 and newer.
+
+## [1.1.1] 2019-08-31
+
+### Fixed
+
+- macOS support.
+
+## [1.1.0] 2019-08-30
+
+### Added
+
+- Global configuration file (~/.out-of-tree/out-of-tree.toml) allow to
+  set up default values for settings.
+
+- rootfs generator for Ubuntu 14.04.
+
+- Parameter for setting up docker registry server.
+
+- Support for (distro-specific) custom docker commands that will be
+  executed before the base template.
+
+- Parameter for setting up a reliability threshold for exit code.
+
+- Parameter for setting up global timeout, after which no new tasks
+  will be started.
+
+### Fixed
+
+- Spelling in output.
+
+- Now kernel generation will not fail if there are no directory
+  /lib/modules inside the container.
+
+## [1.0.0] 2019-08-20
 
 ### Added
 
@@ -96,7 +255,7 @@
 - Temporary files is moved to `~/.out-of-tree/tmp/` to avoid docker
   mounting issues on some systems.
 
-## [0.2.0] - 2019-12-01
+## [0.2.0] - 2018-12-01
 
 The main purpose of the release is to simplify installation.
 
@@ -118,7 +277,7 @@ The main purpose of the release is to simplify installation.
 
 - No warning anymore if test.sh is not exists.
 
-## [0.1.0] - 2019-11-20
+## [0.1.0] - 2018-11-20
 
 Initial release that was never tagged.
 

README.md

@@ -1,98 +1,54 @@
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/aba4aad2046b4d1a9a99cf98e22c018b)](https://app.codacy.com/app/jollheef/out-of-tree?utm_source=github.com&utm_medium=referral&utm_content=jollheef/out-of-tree&utm_campaign=Badge_Grade_Dashboard)
-[![Build Status](https://travis-ci.org/jollheef/out-of-tree.svg?branch=master)](https://travis-ci.org/jollheef/out-of-tree)
 [![Go Report Card](https://goreportcard.com/badge/code.dumpstack.io/tools/out-of-tree)](https://goreportcard.com/report/code.dumpstack.io/tools/out-of-tree)
 [![Documentation Status](https://readthedocs.org/projects/out-of-tree/badge/?version=latest)](https://out-of-tree.readthedocs.io/en/latest/?badge=latest)
-[![Donate](https://img.shields.io/badge/donate-paypal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R8W2UQPZ5X5JE&source=url)
-[![Donate](https://img.shields.io/badge/donate-bitcoin-green.svg)](https://blockchair.com/bitcoin/address/bc1q23fyuq7kmngrgqgp6yq9hk8a5q460f39m8nv87)
 
 # [out-of-tree](https://out-of-tree.io)
 
 out-of-tree kernel {module, exploit} development tool
 
+out-of-tree is for automating some routine actions for creating development environments for debugging kernel modules and exploits, generating reliability statistics for exploits, and also provides the ability to easily integrate into CI (Continuous Integration).
+
 ![Screenshot](https://cloudflare-ipfs.com/ipfs/Qmb88fgdDjbWkxz91sWsgmoZZNfVThnCtj37u3mF2s3T3T)
 
-## Requirements
+## Installation
 
-[Qemu](https://www.qemu.org), [docker](https://docker.com) and [golang](https://golang.org) is required.
+### GNU/Linux (with [Nix](https://nixos.org/nix/))
 
-Also do not forget to set GOPATH and PATH e.g.:
+    $ curl -fsSL https://get.docker.com | sh
+	$ sudo usermod -aG docker user && newgrp docker
+    $ curl -L https://nixos.org/nix/install | sh
+    $ nix-env -iA nixpkgs.out-of-tree # Note: may not be up to date immediately, in this case consider installing from source
 
-    $ echo 'export GOPATH=$HOME' >> ~/.bashrc
-    $ echo 'export PATH=$PATH:$HOME/bin' >> ~/.bashrc
-    $ source ~/.bashrc
-
-### Gentoo
-
-    # emerge app-emulation/qemu app-emulation/docker dev-lang/go
+Note that adding a user to group *docker* has serious security implications. Check Docker documentation for more information.
 
 ### macOS
 
-    $ brew install go qemu
-    $ brew cask install docker
+Note: case-sensitive FS is required for the ~/.out-of-tree directory.
 
-### Fedora
+    $ brew install podman
+    $ podman machine stop || true
+    $ podman machine rm || true
+    $ podman machine init --cpus=4 --memory=4096 -v $HOME:$HOME
+    $ podman machine start
+    $ brew tap out-of-tree/repo
+    $ brew install out-of-tree
 
-    $ sudo dnf install go qemu moby-engine
-
-Also check out [docker post-installation steps](https://docs.docker.com/install/linux/linux-postinstall/).
-
-## Build from source
-
-    $ go get -u code.dumpstack.io/tools/out-of-tree
-
-Then you can check it on kernel module example:
-
-    $ cd $GOPATH/src/code.dumpstack.io/tools/out-of-tree/examples/kernel-module
-    $ out-of-tree kernel autogen # generate kernels based on .out-of-tree.toml
-    $ out-of-tree pew
+Read [documentation](https://out-of-tree.readthedocs.io) for further info.
 
 ## Examples
 
-Run by absolute path
+Generate all Ubuntu 22.04 kernels:
 
-    $ out-of-tree --path /path/to/exploit/directory pew
+    $ out-of-tree kernel genall --distro=Ubuntu --ver=22.04
 
-Test only with one kernel:
+Run tests based on .out-of-tree.toml definitions:
 
-    $ out-of-tree pew --kernel='Ubuntu:4.10.0-30-generic'
+	$ out-of-tree pew
+
+Test with a specific kernel:
+
+    $ out-of-tree pew --kernel='Ubuntu:5.4.0-29-generic'
 
 Run debug environment:
 
-    $ out-of-tree debug --kernel='Ubuntu:4.10.0-30-generic'
-
-Test binary module/exploit with implicit defined test ($BINARY_test)
-
-    $ out-of-tree pew --binary /path/to/exploit
-
-Test binary module/exploit with explicit defined test
-
-    $ out-of-tree pew --binary /path/to/exploit --test /path/to/exploit_test
-
-Guess work kernels:
-
-    $ out-of-tree pew --guess
-
-Use custom kernels config
-
-    $ out-of-tree --kernels /path/to/kernels.toml pew
-
-Generate all kernels
-
-    $ out-of-tree kernel genall --distro Ubuntu --ver 16.04
-
-
-## Troubleshooting
-
-If anything happens that you cannot solve -- just remove `$HOME/.out-of-tree`.
-
-But it'll be better if you'll write the bug report.
-
-## Development
-
-Read [Qemu API](qemu/README.md).
-
-### Generate images
-
-    $ cd $GOPATH/src/code.dumpstack.io/tools/out-of-tree/tools/qemu-debian-img/
-    $ docker run --privileged -v $(pwd):/shared -e IMAGE=/shared/ubuntu1404.img -e RELEASE=trusty -t gen-ubuntu1804-image
-    $ docker run --privileged -v $(pwd):/shared -e IMAGE=/shared/ubuntu1604.img -e RELEASE=xenial -t gen-ubuntu1804-image
+    $ out-of-tree debug --kernel='Ubuntu:5.4.0-29-generic'

config/config.go

@@ -49,10 +49,12 @@ const (
 	KernelModule ArtifactType = iota
 	// KernelExploit is the privilege escalation exploit
 	KernelExploit
+	// Script for information gathering or automation
+	Script
 )
 
 func (at ArtifactType) String() string {
-	return [...]string{"module", "exploit"}[at]
+	return [...]string{"module", "exploit", "script"}[at]
 }
 
 // UnmarshalTOML is for support github.com/naoina/toml
@@ -63,6 +65,8 @@ func (at *ArtifactType) UnmarshalTOML(data []byte) (err error) {
 		*at = KernelModule
 	} else if strings.Contains(stypelower, "exploit") {
 		*at = KernelExploit
+	} else if strings.Contains(stypelower, "script") {
+		*at = Script
 	} else {
 		err = fmt.Errorf("Type %s is unsupported", stype)
 	}
@@ -77,6 +81,8 @@ func (at ArtifactType) MarshalTOML() (data []byte, err error) {
 		s = "module"
 	case KernelExploit:
 		s = "exploit"
+	case Script:
+		s = "script"
 	default:
 		err = fmt.Errorf("Cannot marshal %d", at)
 	}
@@ -102,25 +108,61 @@ func (d Duration) MarshalTOML() (data []byte, err error) {
 	return
 }
 
+type PreloadModule struct {
+	Repo             string
+	Path             string
+	TimeoutAfterLoad Duration
+}
+
+// Extra test files to copy over
+type FileTransfer struct {
+	User   string
+	Local  string
+	Remote string
+}
+
+type Patch struct {
+	Path   string
+	Source string
+	Script string
+}
+
 // Artifact is for .out-of-tree.toml
 type Artifact struct {
 	Name             string
 	Type             ArtifactType
+	TestFiles        []FileTransfer
 	SourcePath       string
 	SupportedKernels []KernelMask
 
+	Script string
+
 	Qemu struct {
 		Cpus    int
 		Memory  int
 		Timeout Duration
 	}
 
+	Docker struct {
+		Timeout Duration
+	}
+
 	Mitigations struct {
 		DisableSmep  bool
 		DisableSmap  bool
 		DisableKaslr bool
 		DisableKpti  bool
 	}
+
+	Patches []Patch
+
+	Make struct {
+		Target string
+	}
+
+	StandardModules bool
+
+	Preload []PreloadModule
 }
 
 func (ka Artifact) checkSupport(ki KernelInfo, km KernelMask) (
@@ -230,9 +272,11 @@ type KernelInfo struct {
 	ContainerName string
 
 	// Runtime information
-	KernelPath string
-	InitrdPath string
-	RootFS     string
+	KernelPath  string
+	InitrdPath  string
+	ModulesPath string
+
+	RootFS string
 
 	// Debug symbols
 	VmlinuxPath string

config/out-of-tree.go

@@ -0,0 +1,106 @@
+// Copyright 2019 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package config
+
+import (
+	"errors"
+	"os"
+	"os/user"
+	"time"
+
+	"github.com/alecthomas/kong"
+	"github.com/mitchellh/go-homedir"
+	"github.com/naoina/toml"
+)
+
+type DockerCommand struct {
+	DistroType DistroType
+	Command    string
+}
+
+type OutOfTree struct {
+	Kernels     string
+	UserKernels string
+
+	Database string
+
+	Qemu struct {
+		Timeout Duration
+	}
+
+	Docker struct {
+		Timeout  Duration
+		Registry string
+
+		// Commands that will be executed before
+		// the base layer of Dockerfile
+		Commands []DockerCommand
+	}
+}
+
+func (c *OutOfTree) Decode(ctx *kong.DecodeContext) (err error) {
+	if ctx.Value.Set {
+		return
+	}
+
+	s, err := homedir.Expand(ctx.Scan.Pop().String())
+	if err != nil {
+		return
+	}
+
+	defaultValue, err := homedir.Expand(ctx.Value.Default)
+	if err != nil {
+		return
+	}
+
+	_, err = os.Stat(s)
+	if s != defaultValue && errors.Is(err, os.ErrNotExist) {
+		return errors.New("'" + s + "' does not exist")
+	}
+
+	*c, err = ReadOutOfTreeConf(s)
+	return
+}
+
+func ReadOutOfTreeConf(path string) (c OutOfTree, err error) {
+	buf, err := readFileAll(path)
+	if err == nil {
+		err = toml.Unmarshal(buf, &c)
+		if err != nil {
+			return
+		}
+	} else {
+		// It's ok if there's no configuration
+		// then we'll just set default values
+		err = nil
+	}
+
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+
+	if c.Kernels == "" {
+		c.Kernels = usr.HomeDir + "/.out-of-tree/kernels.toml"
+	}
+
+	if c.UserKernels == "" {
+		c.UserKernels = usr.HomeDir + "/.out-of-tree/kernels.user.toml"
+	}
+
+	if c.Database == "" {
+		c.Database = usr.HomeDir + "/.out-of-tree/db.sqlite"
+	}
+
+	if c.Qemu.Timeout.Duration == 0 {
+		c.Qemu.Timeout.Duration = time.Minute
+	}
+
+	if c.Docker.Timeout.Duration == 0 {
+		c.Docker.Timeout.Duration = time.Minute
+	}
+
+	return
+}

container.go

@@ -0,0 +1,259 @@
+// Copyright 2023 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"os/user"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/rs/zerolog/log"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+)
+
+var containerRuntime = "docker"
+
+type ContainerCmd struct {
+	Filter string `help:"filter by name"`
+
+	List    ContainerListCmd    `cmd:"" help:"list containers"`
+	Cleanup ContainerCleanupCmd `cmd:"" help:"cleanup containers"`
+}
+
+func (cmd ContainerCmd) Containers() (names []string) {
+	images, err := listContainerImages()
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+
+	for _, img := range images {
+		if cmd.Filter != "" && !strings.Contains(img.Name, cmd.Filter) {
+			continue
+		}
+		names = append(names, img.Name)
+	}
+	return
+}
+
+type ContainerListCmd struct{}
+
+func (cmd ContainerListCmd) Run(containerCmd *ContainerCmd) (err error) {
+	for _, name := range containerCmd.Containers() {
+		fmt.Println(name)
+	}
+	return
+}
+
+type ContainerCleanupCmd struct{}
+
+func (cmd ContainerCleanupCmd) Run(containerCmd *ContainerCmd) (err error) {
+	var output []byte
+	for _, name := range containerCmd.Containers() {
+		output, err = exec.Command(containerRuntime, "image", "rm", name).
+			CombinedOutput()
+		if err != nil {
+			log.Error().Err(err).Str("output", string(output)).Msg("")
+			return
+		}
+	}
+	return
+}
+
+type containerImageInfo struct {
+	Name          string
+	DistroType    config.DistroType
+	DistroRelease string // 18.04/7.4.1708/9.1
+}
+
+func listContainerImages() (diis []containerImageInfo, err error) {
+	cmd := exec.Command(containerRuntime, "images")
+	log.Debug().Msgf("%v", cmd)
+
+	rawOutput, err := cmd.CombinedOutput()
+	if err != nil {
+		return
+	}
+
+	r, err := regexp.Compile("out_of_tree_.*")
+	if err != nil {
+		return
+	}
+
+	containers := r.FindAll(rawOutput, -1)
+	for _, c := range containers {
+		container := strings.Fields(string(c))[0]
+
+		s := strings.Replace(container, "__", ".", -1)
+		values := strings.Split(s, "_")
+		distro, ver := values[3], values[4]
+
+		dii := containerImageInfo{
+			Name:          container,
+			DistroRelease: ver,
+		}
+
+		dii.DistroType, err = config.NewDistroType(distro)
+		if err != nil {
+			return
+		}
+
+		diis = append(diis, dii)
+	}
+	return
+}
+
+type container struct {
+	name    string
+	timeout time.Duration
+	Volumes struct {
+		LibModules string
+		UsrSrc     string
+		Boot       string
+	}
+	// Additional arguments
+	Args []string
+}
+
+func NewContainer(name string, timeout time.Duration) (c container, err error) {
+	c.name = name
+	c.timeout = timeout
+
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+
+	c.Volumes.LibModules = fmt.Sprintf(
+		"%s/.out-of-tree/volumes/%s/lib/modules", usr.HomeDir, name)
+	os.MkdirAll(c.Volumes.LibModules, 0777)
+
+	c.Volumes.UsrSrc = fmt.Sprintf(
+		"%s/.out-of-tree/volumes/%s/usr/src", usr.HomeDir, name)
+	os.MkdirAll(c.Volumes.UsrSrc, 0777)
+
+	c.Volumes.Boot = fmt.Sprintf(
+		"%s/.out-of-tree/volumes/%s/boot", usr.HomeDir, name)
+	os.MkdirAll(c.Volumes.Boot, 0777)
+
+	return
+}
+
+func (c container) Build(imagePath string) (output string, err error) {
+	args := []string{"build"}
+	args = append(args, "-t", c.name, imagePath)
+
+	cmd := exec.Command(containerRuntime, args...)
+
+	flog := log.With().
+		Str("command", fmt.Sprintf("%v", cmd)).
+		Logger()
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	cmd.Stderr = cmd.Stdout
+
+	err = cmd.Start()
+	if err != nil {
+		return
+	}
+
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			m := scanner.Text()
+			output += m + "\n"
+			flog.Trace().Str("stdout", m).Msg("")
+		}
+	}()
+
+	err = cmd.Wait()
+	return
+}
+
+func (c container) Run(workdir string, command string) (output string, err error) {
+	flog := log.With().
+		Str("container", c.name).
+		Str("workdir", workdir).
+		Str("command", command).
+		Logger()
+
+	var args []string
+	args = append(args, "run", "--rm")
+	args = append(args, c.Args...)
+	if workdir != "" {
+		args = append(args, "-v", workdir+":/work")
+	}
+	if c.Volumes.LibModules != "" {
+		args = append(args, "-v", c.Volumes.LibModules+":/lib/modules")
+	}
+	if c.Volumes.UsrSrc != "" {
+		args = append(args, "-v", c.Volumes.UsrSrc+":/usr/src")
+	}
+	if c.Volumes.Boot != "" {
+		args = append(args, "-v", c.Volumes.Boot+":/boot")
+	}
+	args = append(args, c.name, "bash", "-c")
+	if workdir != "" {
+		args = append(args, "cd /work && "+command)
+	} else {
+		args = append(args, command)
+	}
+
+	cmd := exec.Command(containerRuntime, args...)
+
+	log.Debug().Msgf("%v", cmd)
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	cmd.Stderr = cmd.Stdout
+
+	timer := time.AfterFunc(c.timeout, func() {
+		flog.Info().Msg("killing container by timeout")
+
+		flog.Debug().Msg("SIGINT")
+		cmd.Process.Signal(os.Interrupt)
+
+		time.Sleep(time.Minute)
+
+		flog.Debug().Msg("SIGKILL")
+		cmd.Process.Kill()
+	})
+	defer timer.Stop()
+
+	err = cmd.Start()
+	if err != nil {
+		return
+	}
+
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			m := scanner.Text()
+			output += m + "\n"
+			flog.Trace().Str("stdout", m).Msg("")
+		}
+	}()
+
+	err = cmd.Wait()
+	if err != nil {
+		e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`",
+			err, command, output)
+		err = errors.New(e)
+		return
+	}
+
+	return
+}

db.go

@@ -254,6 +254,24 @@ func getLogByID(db *sql.DB, id int) (le logEntry, err error) {
 	return
 }
 
+func getLastLog(db *sql.DB) (le logEntry, err error) {
+	err = db.QueryRow("SELECT MAX(id), time, name, type, tag, "+
+		"distro_type, distro_release, kernel_release, "+
+		"build_ok, run_ok, test_ok, "+
+		"build_output, run_output, test_output, "+
+		"qemu_stdout, qemu_stderr, "+
+		"kernel_panic, timeout_kill "+
+		"FROM log").Scan(&le.ID, &le.Timestamp,
+		&le.Name, &le.Type, &le.Tag,
+		&le.DistroType, &le.DistroRelease, &le.KernelRelease,
+		&le.Build.Ok, &le.Run.Ok, &le.Test.Ok,
+		&le.Build.Output, &le.Run.Output, &le.Test.Output,
+		&le.Stdout, &le.Stderr,
+		&le.KernelPanic, &le.KilledByTimeout,
+	)
+	return
+}
+
 func createSchema(db *sql.DB) (err error) {
 	err = createMetadataTable(db)
 	if err != nil {

debug.go

@@ -8,17 +8,217 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"os"
 	"strings"
 	"time"
 
-	"gopkg.in/logrusorgru/aurora.v1"
+	"github.com/rs/zerolog/log"
+	"gopkg.in/logrusorgru/aurora.v2"
 
 	"code.dumpstack.io/tools/out-of-tree/config"
 	"code.dumpstack.io/tools/out-of-tree/qemu"
 )
 
+type DebugCmd struct {
+	Kernel string `help:"regexp (first match)" required:""`
+	Gdb    string `help:"gdb listen address" default:"tcp::1234"`
+
+	SshAddr string `help:"ssh address to listen" default:"127.0.0.1"`
+	SshPort int    `help:"ssh port to listen" default:"50022"`
+
+	ArtifactConfig string `help:"path to artifact config" type:"path"`
+
+	Kaslr bool `help:"Enable KASLR"`
+	Smep  bool `help:"Enable SMEP"`
+	Smap  bool `help:"Enable SMAP"`
+	Kpti  bool `help:"Enable KPTI"`
+
+	NoKaslr bool `help:"Disable KASLR"`
+	NoSmep  bool `help:"Disable SMEP"`
+	NoSmap  bool `help:"Disable SMAP"`
+	NoKpti  bool `help:"Disable KPTI"`
+}
+
+// TODO: merge with pew.go
+func (cmd *DebugCmd) Run(g *Globals) (err error) {
+	kcfg, err := config.ReadKernelConfig(g.Config.Kernels)
+	if err != nil {
+		log.Print(err)
+	}
+
+	var configPath string
+	if cmd.ArtifactConfig == "" {
+		configPath = g.WorkDir + "/.out-of-tree.toml"
+	} else {
+		configPath = cmd.ArtifactConfig
+	}
+	ka, err := config.ReadArtifactConfig(configPath)
+	if err != nil {
+		return
+	}
+
+	if ka.SourcePath == "" {
+		ka.SourcePath = g.WorkDir
+	}
+
+	ki, err := firstSupported(kcfg, ka, cmd.Kernel)
+	if err != nil {
+		return
+	}
+
+	kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
+	q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
+	if err != nil {
+		return
+	}
+
+	err = q.SetSSHAddrPort(cmd.SshAddr, cmd.SshPort)
+	if err != nil {
+		return
+	}
+
+	if ka.Qemu.Cpus != 0 {
+		q.Cpus = ka.Qemu.Cpus
+	}
+	if ka.Qemu.Memory != 0 {
+		q.Memory = ka.Qemu.Memory
+	}
+
+	if ka.Docker.Timeout.Duration != 0 {
+		g.Config.Docker.Timeout.Duration = ka.Docker.Timeout.Duration
+	}
+
+	q.SetKASLR(false) // set KASLR to false by default because of gdb
+	q.SetSMEP(!ka.Mitigations.DisableSmep)
+	q.SetSMAP(!ka.Mitigations.DisableSmap)
+	q.SetKPTI(!ka.Mitigations.DisableKpti)
+
+	if cmd.Kaslr {
+		q.SetKASLR(true)
+	} else if cmd.NoKaslr {
+		q.SetKASLR(false)
+	}
+
+	if cmd.Smep {
+		q.SetSMEP(true)
+	} else if cmd.NoSmep {
+		q.SetSMEP(false)
+	}
+
+	if cmd.Smap {
+		q.SetSMAP(true)
+	} else if cmd.NoSmap {
+		q.SetSMAP(false)
+	}
+
+	if cmd.Kpti {
+		q.SetKPTI(true)
+	} else if cmd.NoKpti {
+		q.SetKPTI(false)
+	}
+
+	redgreen := func(name string, enabled bool) aurora.Value {
+		if enabled {
+			return aurora.BgGreen(aurora.Black(name))
+		}
+
+		return aurora.BgRed(aurora.White(name))
+	}
+
+	fmt.Printf("[*] %s %s %s %s\n",
+		redgreen("KASLR", q.GetKASLR()),
+		redgreen("SMEP", q.GetSMEP()),
+		redgreen("SMAP", q.GetSMAP()),
+		redgreen("KPTI", q.GetKPTI()))
+
+	fmt.Printf("[*] SMP: %d CPUs\n", q.Cpus)
+	fmt.Printf("[*] Memory: %d MB\n", q.Memory)
+
+	q.Debug(cmd.Gdb)
+	coloredGdbAddress := aurora.BgGreen(aurora.Black(cmd.Gdb))
+	fmt.Printf("[*] gdb is listening on %s\n", coloredGdbAddress)
+
+	err = q.Start()
+	if err != nil {
+		return
+	}
+	defer q.Stop()
+
+	tmp, err := ioutil.TempDir(tempDirBase, "out-of-tree_")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(tmp)
+
+	err = q.WaitForSSH(time.Minute)
+	if err != nil {
+		return
+	}
+
+	if ka.StandardModules {
+		// Module depends on one of the standard modules
+		err = copyStandardModules(q, ki)
+		if err != nil {
+			log.Print(err)
+			return
+		}
+	}
+
+	err = preloadModules(q, ka, ki, g.Config.Docker.Timeout.Duration)
+	if err != nil {
+		log.Print(err)
+		return
+	}
+
+	var buildDir, outFile, output, remoteFile string
+
+	if ka.Type == config.Script {
+		err = q.CopyFile("root", ka.Script, ka.Script)
+		if err != nil {
+			return
+		}
+	} else {
+		buildDir, outFile, output, err = build(tmp, ka, ki, g.Config.Docker.Timeout.Duration)
+		if err != nil {
+			log.Print(err, output)
+			return
+		}
+
+		remoteFile = "/tmp/exploit"
+		if ka.Type == config.KernelModule {
+			remoteFile = "/tmp/module.ko"
+		}
+
+		err = q.CopyFile("user", outFile, remoteFile)
+		if err != nil {
+			return
+		}
+	}
+
+	// Copy all test files to the remote machine
+	for _, f := range ka.TestFiles {
+		if f.Local[0] != '/' {
+			f.Local = buildDir + "/" + f.Local
+		}
+		err = q.CopyFile(f.User, f.Local, f.Remote)
+		if err != nil {
+			log.Print("error copy err:", err, f.Local, f.Remote)
+			return
+		}
+	}
+
+	coloredRemoteFile := aurora.BgGreen(aurora.Black(remoteFile))
+	fmt.Printf("[*] build result copied to %s\n", coloredRemoteFile)
+
+	fmt.Printf("\n%s\n", q.GetSSHCommand())
+	fmt.Printf("gdb %s -ex 'target remote %s'\n\n", ki.VmlinuxPath, cmd.Gdb)
+
+	// TODO set substitute-path /build/.../linux-... /path/to/linux-source
+
+	err = interactive(q)
+	return
+}
+
 func firstSupported(kcfg config.KernelConfig, ka config.Artifact,
 	kernel string) (ki config.KernelInfo, err error) {
 
@@ -57,12 +257,12 @@ func handleLine(q *qemu.System) (err error) {
 		fmt.Printf("ssh\t: print arguments to ssh command\n")
 		fmt.Printf("quit\t: quit\n")
 	case "l", "log":
-		fmt.Println(string(q.Stdout))
+		fmt.Println(q.Stdout)
 	case "cl", "clog":
-		fmt.Println(string(q.Stdout))
-		q.Stdout = []byte{}
+		fmt.Println(q.Stdout)
+		q.Stdout = ""
 	case "c", "cleanup":
-		q.Stdout = []byte{}
+		q.Stdout = ""
 	case "s", "ssh":
 		fmt.Println(q.GetSSHCommand())
 	case "q", "quit":
@@ -81,124 +281,3 @@ func interactive(q *qemu.System) (err error) {
 		}
 	}
 }
-
-func debugHandler(kcfg config.KernelConfig, workPath, kernRegex, gdb string,
-	dockerTimeout time.Duration, yekaslr, yesmep, yesmap, yekpti,
-	nokaslr, nosmep, nosmap, nokpti bool) (err error) {
-
-	ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
-	if err != nil {
-		return
-	}
-
-	if ka.SourcePath == "" {
-		ka.SourcePath = workPath
-	}
-
-	ki, err := firstSupported(kcfg, ka, kernRegex)
-	if err != nil {
-		return
-	}
-
-	kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
-	q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
-	if err != nil {
-		return
-	}
-
-	if ka.Qemu.Cpus != 0 {
-		q.Cpus = ka.Qemu.Cpus
-	}
-	if ka.Qemu.Memory != 0 {
-		q.Memory = ka.Qemu.Memory
-	}
-
-	q.SetKASLR(false) // set KASLR to false by default because of gdb
-	q.SetSMEP(!ka.Mitigations.DisableSmep)
-	q.SetSMAP(!ka.Mitigations.DisableSmap)
-	q.SetKPTI(!ka.Mitigations.DisableKpti)
-
-	if yekaslr {
-		q.SetKASLR(true)
-	} else if nokaslr {
-		q.SetKASLR(false)
-	}
-
-	if yesmep {
-		q.SetSMEP(true)
-	} else if nosmep {
-		q.SetSMEP(false)
-	}
-
-	if yesmap {
-		q.SetSMAP(true)
-	} else if nosmap {
-		q.SetSMAP(false)
-	}
-
-	if yekpti {
-		q.SetKPTI(true)
-	} else if nokpti {
-		q.SetKPTI(false)
-	}
-
-	redgreen := func(name string, enabled bool) aurora.Value {
-		if enabled {
-			return aurora.BgGreen(aurora.Black(name))
-		}
-
-		return aurora.BgRed(aurora.Gray(name))
-	}
-
-	fmt.Printf("[*] %s %s %s %s\n",
-		redgreen("KASLR", q.GetKASLR()),
-		redgreen("SMEP", q.GetSMEP()),
-		redgreen("SMAP", q.GetSMAP()),
-		redgreen("KPTI", q.GetKPTI()))
-
-	fmt.Printf("[*] SMP: %d CPUs\n", q.Cpus)
-	fmt.Printf("[*] Memory: %d MB\n", q.Memory)
-
-	q.Debug(gdb)
-	coloredGdbAddress := aurora.BgGreen(aurora.Black(gdb))
-	fmt.Printf("[*] gdb runned on %s\n", coloredGdbAddress)
-
-	err = q.Start()
-	if err != nil {
-		return
-	}
-	defer q.Stop()
-
-	tmp, err := ioutil.TempDir("/tmp/", "out-of-tree_")
-	if err != nil {
-		return
-	}
-	defer os.RemoveAll(tmp)
-
-	outFile, output, err := build(tmp, ka, ki, dockerTimeout)
-	if err != nil {
-		log.Println(err, output)
-		return
-	}
-
-	remoteFile := "/tmp/exploit"
-	if ka.Type == config.KernelModule {
-		remoteFile = "/tmp/module.ko"
-	}
-
-	err = q.CopyFile("user", outFile, remoteFile)
-	if err != nil {
-		return
-	}
-
-	coloredRemoteFile := aurora.BgGreen(aurora.Black(remoteFile))
-	fmt.Printf("[*] build result copied to %s\n", coloredRemoteFile)
-
-	fmt.Printf("\n%s\n", q.GetSSHCommand())
-	fmt.Printf("gdb %s -ex 'target remote %s'\n\n", ki.VmlinuxPath, gdb)
-
-	// TODO set substitute-path /build/.../linux-... /path/to/linux-source
-
-	err = interactive(q)
-	return
-}

docs/installation.rst

@@ -1,4 +1,4 @@
-Installation
+Installation (from source)
 ============
 
 OS/Distro-specific
@@ -10,7 +10,7 @@ Ubuntu
 Install dependencies::
 
   $ sudo snap install go --classic
-  $ sudo snap install docker
+  $ # Install docker: https://docs.docker.com/engine/install/ubuntu/
   $ sudo apt install qemu-system-x86 build-essential gdb
 
 macOS
@@ -36,18 +36,33 @@ There's a minimal configuration that you need to apply::
     ];
   }
 
+Gentoo
+------
+
+Install dependencies::
+
+  $ sudo emerge app-emulation/qemu app-emulation/docker dev-lang/go
+
+Fedora
+------
+
+Install dependencies::
+
+  $ sudo dnf install go qemu moby-engine
+
 Common
 ======
 
-Setup Go environment::
+Setup environment::
 
-  $ echo 'export GOPATH=$HOME' >> ~/.bashrc
   $ echo 'export PATH=$PATH:$HOME/bin' >> ~/.bashrc
   $ source ~/.bashrc
 
 Build *out-of-tree*::
 
-  $ go get -u code.dumpstack.io/tools/out-of-tree
+  $ git clone https://code.dumpstack.io/tools/out-of-tree
+  $ cd out-of-tree
+  $ CGO_ENABLED=1 go build -o ~/bin/out-of-tree
 
 .. note::
   On a GNU/Linux you need to add your user to docker group if you want
@@ -57,7 +72,7 @@ Build *out-of-tree*::
 
 Test that everything works::
 
-  $ cd $GOPATH/src/code.dumpstack.io/tools/out-of-tree/examples/kernel-exploit
+  $ cd out-of-tree/examples/kernel-exploit
   $ out-of-tree kernel autogen --max=1
   $ out-of-tree pew --max=1
 

docs/introduction.rst

@@ -54,7 +54,7 @@ Overview
 
     $ out-of-tree debug --kernel 'Ubuntu:4.15.0-58-generic'
     [*] KASLR SMEP SMAP
-    [*] gdb runned on tcp::1234
+    [*] gdb is listening on tcp::1234
     [*] build result copied to /tmp/exploit
 
     ssh -o StrictHostKeyChecking=no -p 29308 root@127.133.45.236

examples/kernel-exploit/Makefile

@@ -2,7 +2,7 @@
 # - KERNEL: kernel headers path
 # - TARGET: name of exploit binary that MUST be produced by makefile.
 # - $(TARGET)_test: name of test binary that MUST be produced by makefile
-# and it's will be runned on a LPE stage. TARGET_TEST MUST accept two argument:
+# and it's will be executed on a LPE stage. TARGET_TEST MUST accept two argument:
 #   - Path to exploit binary
 #   - File that MUST be created with exploit. It uses for test that exploit works
 #     correctly.

examples/preload/.out-of-tree.toml

@@ -0,0 +1,12 @@
+name = "out-of-tree preload"
+type = "module"
+
+[[supported_kernels]]
+distro_type = "Ubuntu"
+distro_release = "18.04"
+release_mask = ".*"
+
+[[preload]]
+repo = "https://github.com/openwall/lkrg"
+#path = "/local/path/to/lkrg"
+timeout_after_load = "1s"

examples/preload/Makefile

@@ -0,0 +1,11 @@
+KERNEL := /lib/modules/$(shell uname -r)/build
+TARGET := module
+
+obj-m += $(TARGET).o
+$(TARGET)-objs = module.o
+
+all:
+	make -C $(KERNEL) M=$(PWD) modules
+
+clean:
+	make -C $(KERNEL) M=$(PWD) clean

examples/preload/README.md

@@ -0,0 +1,5 @@
+# out-of-tree kernel module preload example
+
+See .out-of-tree.toml
+
+Note that it should fail to insert module if lkrg is enabled in the preload list.

examples/preload/module.c

@@ -0,0 +1,17 @@
+#include <linux/module.h>
+#include <linux/slab.h>
+
+int init_module(void)
+{
+	char *argv[] = { "/bin/sh", "--help", NULL };
+	char *envp[] = { NULL };
+
+	/* trigger lkrg */
+	return call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC);
+}
+
+void cleanup_module(void)
+{
+}
+
+MODULE_LICENSE("GPL");

examples/script/.out-of-tree.toml

@@ -0,0 +1,11 @@
+# out-of-tree configuration file
+# docs at https://out-of-tree.io
+name = "out-of-tree script example"
+type = "script"
+
+script = "script.sh"
+
+[[supported_kernels]]
+distro_type = "Ubuntu"
+distro_release = "22.04"
+release_mask = ".*"

examples/script/README.md

@@ -0,0 +1,3 @@
+# out-of-tree script example
+
+See .out-of-tree.toml

examples/script/script.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+uname -a
+
+ls /proc | grep config

gen.go

@@ -12,6 +12,20 @@ import (
 	"code.dumpstack.io/tools/out-of-tree/config"
 )
 
+type GenCmd struct {
+	Type string `enum:"module,exploit" required:"" help:"module/exploit"`
+}
+
+func (cmd *GenCmd) Run(g *Globals) (err error) {
+	switch cmd.Type {
+	case "module":
+		err = genConfig(config.KernelModule)
+	case "exploit":
+		err = genConfig(config.KernelExploit)
+	}
+	return
+}
+
 func genConfig(at config.ArtifactType) (err error) {
 	a := config.Artifact{
 		Name: "Put name here",
@@ -22,6 +36,12 @@ func genConfig(at config.ArtifactType) (err error) {
 		DistroRelease: "18.04",
 		ReleaseMask:   ".*",
 	})
+	a.Preload = append(a.Preload, config.PreloadModule{
+		Repo: "Repo name (e.g. https://github.com/openwall/lkrg)",
+	})
+	a.Patches = append(a.Patches, config.Patch{
+		Path: "/path/to/profiling.patch",
+	})
 
 	buf, err := toml.Marshal(&a)
 	if err != nil {

go.mod

@@ -1,21 +1,50 @@
 module code.dumpstack.io/tools/out-of-tree
 
+go 1.17
+
 replace code.dumpstack.io/tools/out-of-tree/qemu => ./qemu
 
 replace code.dumpstack.io/tools/out-of-tree/config => ./config
 
 require (
-	github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect
-	github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect
-	github.com/mattn/go-runewidth v0.0.4 // indirect
-	github.com/mattn/go-sqlite3 v1.11.0
-	github.com/naoina/go-stringutil v0.1.0 // indirect
+	github.com/alecthomas/kong v0.7.1
+	github.com/go-git/go-git/v5 v5.6.1
+	github.com/mattn/go-sqlite3 v1.14.16
+	github.com/mitchellh/go-homedir v1.1.0
 	github.com/naoina/toml v0.1.1
-	github.com/olekukonko/tablewriter v0.0.1
-	github.com/otiai10/copy v1.0.1
-	github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce
-	github.com/zcalusic/sysinfo v0.0.0-20190429151633-fbadb57345c2
-	golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5
-	gopkg.in/alecthomas/kingpin.v2 v2.2.6
-	gopkg.in/logrusorgru/aurora.v1 v1.0.0-20181002194514-a7b3b318ed4e
+	github.com/natefinch/lumberjack v2.0.0+incompatible
+	github.com/olekukonko/tablewriter v0.0.5
+	github.com/otiai10/copy v1.10.0
+	github.com/remeh/sizedwaitgroup v1.0.0
+	github.com/rs/zerolog v1.29.0
+	github.com/zcalusic/sysinfo v0.9.5
+	golang.org/x/crypto v0.7.0
+	gopkg.in/logrusorgru/aurora.v2 v2.0.3
+)
+
+require (
+	github.com/BurntSushi/toml v1.2.1 // indirect
+	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
+	github.com/acomagu/bufpipe v1.0.4 // indirect
+	github.com/cloudflare/circl v1.1.0 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/go-git/gcfg v1.5.0 // indirect
+	github.com/go-git/go-billy/v5 v5.4.1 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/naoina/go-stringutil v0.1.0 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
+	github.com/sergi/go-diff v1.1.0 // indirect
+	github.com/skeema/knownhosts v1.1.0 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 )

go.sum

@@ -1,33 +1,198 @@
-bou.ke/monkey v1.0.1/go.mod h1:FgHuK96Rv2Nlf+0u1OOVDpCMdsWyOFmeeketDHE7LIg=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
-github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
-github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
+github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
+github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA=
+github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
+github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
+github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
+github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0=
+github.com/alecthomas/assert/v2 v2.1.0/go.mod h1:b/+1DI2Q6NckYi+3mXyH3wFb8qG37K/DuK80n7WefXA=
+github.com/alecthomas/kong v0.7.1 h1:azoTh0IOfwlAX3qN9sHWTxACE2oV8Bg2gAwBsMwDQY4=
+github.com/alecthomas/kong v0.7.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U=
+github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
+github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
+github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY=
+github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
+github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
+github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
+github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
+github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
+github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
+github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
+github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
+github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
+github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ=
+github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
+github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk=
+github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
+github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
+github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
+github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
+github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
+github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
+github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
+github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM=
 github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
 github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
 github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
 github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
-github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
-github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
-github.com/otiai10/copy v1.0.1 h1:gtBjD8aq4nychvRZ2CyJvFWAw0aja+VHazDdruZKGZA=
-github.com/otiai10/copy v1.0.1/go.mod h1:8bMCJrAqOtN/d9oyh5HR7HhLQMvcGMpGdwRDYsfOCHc=
-github.com/otiai10/mint v1.2.3/go.mod h1:YnfyPNhBvnY8bW4SGQHCs/aAFhkgySlMZbrF5U0bOVw=
-github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce h1:aP+C+YbHZfOQlutA4p4soHi7rVUqHQdWEVMSkHfDTqY=
-github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
-github.com/zcalusic/sysinfo v0.0.0-20190429151633-fbadb57345c2 h1:uMiaKNX5zFLOa6nNtun+d/lpV5bOBh7BvE4q9jfZacQ=
-github.com/zcalusic/sysinfo v0.0.0-20190429151633-fbadb57345c2/go.mod h1:zAn3FAIbgZPYnutDND49Ivf8sb/mXYk8UjZdqMswgHg=
+github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
+github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/otiai10/copy v1.10.0 h1:znyI7l134wNg/wDktoVQPxPkgvhDfGCYUasey+h0rDQ=
+github.com/otiai10/copy v1.10.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww=
+github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
+github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
+github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
+github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
+github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w=
+github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0=
+github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXijpFO6roag=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
+github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/zcalusic/sysinfo v0.9.5 h1:ivoHyj9aIAYkwzo1+8QgJ5s4oeE6Etx9FmZtqa4wJjQ=
+github.com/zcalusic/sysinfo v0.9.5/go.mod h1:Z/gPVufBrFc8X5sef3m6kkw3r3nlNFp+I6bvASfvBZQ=
+golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
+golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
+golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
+golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
+golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
+golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
-gopkg.in/logrusorgru/aurora.v1 v1.0.0-20181002194514-a7b3b318ed4e h1:uKdf1KQDFZDYqNzSDhxB5hFxj5Fq4e3/C/ejtRJxlY0=
-gopkg.in/logrusorgru/aurora.v1 v1.0.0-20181002194514-a7b3b318ed4e/go.mod h1:DGR33jeYG1jxERD2W4hGjuW94Pxf3mkUf/Ddhf5BskA=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
+golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/logrusorgru/aurora.v2 v2.0.3 h1:5Hr76hqgwx9PednedYf5Q1dBfiPMZ2IgExR7u3tNXIE=
+gopkg.in/logrusorgru/aurora.v2 v2.0.3/go.mod h1:Wm+IEn1fgFp8E2paL93oFVrHZW4toMKARNE85fDY5w8=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
+gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
+gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

images.go

@@ -5,14 +5,126 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"os/exec"
+	"os/user"
+	"strings"
+	"time"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+	"code.dumpstack.io/tools/out-of-tree/qemu"
+	"github.com/rs/zerolog/log"
 )
 
+type ImageCmd struct {
+	List ImageListCmd `cmd:"" help:"list images"`
+	Edit ImageEditCmd `cmd:"" help:"edit image"`
+}
+
+type ImageListCmd struct{}
+
+func (cmd *ImageListCmd) Run(g *Globals) (err error) {
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+
+	entries, err := os.ReadDir(usr.HomeDir + "/.out-of-tree/images/")
+	if err != nil {
+		return
+	}
+
+	for _, e := range entries {
+		fmt.Println(e.Name())
+	}
+
+	return
+}
+
+type ImageEditCmd struct {
+	Name   string `help:"image name" required:""`
+	DryRun bool   `help:"do nothing, just print commands"`
+}
+
+func (cmd *ImageEditCmd) Run(g *Globals) (err error) {
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+
+	image := usr.HomeDir + "/.out-of-tree/images/" + cmd.Name
+	if !exists(image) {
+		fmt.Println("image does not exist")
+	}
+
+	kcfg, err := config.ReadKernelConfig(g.Config.Kernels)
+	if err != nil {
+		return
+	}
+
+	if len(kcfg.Kernels) == 0 {
+		return errors.New("No kernels found")
+	}
+
+	ki := config.KernelInfo{}
+	for _, k := range kcfg.Kernels {
+		if k.RootFS == image {
+			ki = k
+			break
+		}
+	}
+
+	kernel := qemu.Kernel{
+		KernelPath: ki.KernelPath,
+		InitrdPath: ki.InitrdPath,
+	}
+
+	q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
+
+	q.Mutable = true
+
+	if cmd.DryRun {
+		s := q.Executable()
+		for _, arg := range q.Args() {
+			if strings.Contains(arg, " ") ||
+				strings.Contains(arg, ",") {
+
+				s += fmt.Sprintf(` "%s"`, arg)
+			} else {
+				s += fmt.Sprintf(" %s", arg)
+			}
+		}
+		fmt.Println(s)
+		fmt.Println(q.GetSSHCommand())
+		return
+	}
+
+	err = q.Start()
+	if err != nil {
+		fmt.Println("Qemu start error:", err)
+		return
+	}
+	defer q.Stop()
+
+	fmt.Print("ssh command:\n\n\t")
+	fmt.Println(q.GetSSHCommand())
+
+	fmt.Print("\npress enter to stop")
+	fmt.Scanln()
+
+	q.Command("root", "poweroff")
+
+	for !q.Died {
+		time.Sleep(time.Second)
+	}
+	return
+}
+
 // inspired by Edd Turtle code
 func downloadFile(filepath string, url string) (err error) {
 	out, err := os.Create(filepath)
@@ -53,6 +165,8 @@ func unpackTar(archive, destination string) (err error) {
 	cmd := exec.Command("tar", "-Sxf", archive)
 	cmd.Dir = destination + "/"
 
+	log.Debug().Msgf("%v", cmd)
+
 	rawOutput, err := cmd.CombinedOutput()
 	if err != nil {
 		err = fmt.Errorf("%v: %s", err, rawOutput)
@@ -63,7 +177,7 @@ func unpackTar(archive, destination string) (err error) {
 }
 
 func downloadImage(path, file string) (err error) {
-	tmp, err := ioutil.TempDir("/tmp/", "out-of-tree_")
+	tmp, err := ioutil.TempDir(tempDirBase, "out-of-tree_")
 	if err != nil {
 		return
 	}

kernel_linux.go

@@ -0,0 +1,96 @@
+// Copyright 2023 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+//go:build linux
+// +build linux
+
+package main
+
+import (
+	"io/ioutil"
+	"os/exec"
+	"strings"
+
+	"github.com/rs/zerolog/log"
+	"github.com/zcalusic/sysinfo"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+)
+
+func genHostKernels(download bool) (kcfg config.KernelConfig, err error) {
+	si := sysinfo.SysInfo{}
+	si.GetSysInfo()
+
+	distroType, err := config.NewDistroType(si.OS.Vendor)
+	if err != nil {
+		return
+	}
+
+	cmd := exec.Command("ls", "/lib/modules")
+	log.Debug().Msgf("%v", cmd)
+
+	rawOutput, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Print(string(rawOutput), err)
+		return
+	}
+
+	kernelsBase := "/boot/"
+	bootfiles, err := ioutil.ReadDir(kernelsBase)
+	if err != nil {
+		return
+	}
+
+	// only for compatibility, docker is not really used
+	dii := containerImageInfo{
+		Name: config.KernelMask{
+			DistroType:    distroType,
+			DistroRelease: si.OS.Version,
+		}.DockerName(),
+	}
+
+	rootfs, err := genRootfsImage(dii, download)
+	if err != nil {
+		return
+	}
+
+	for _, krel := range strings.Fields(string(rawOutput)) {
+		log.Debug().Msgf("generate config entry for %s", krel)
+
+		var kernelFile, initrdFile string
+		kernelFile, err = findKernelFile(bootfiles, krel)
+		if err != nil {
+			log.Warn().Msgf("cannot find kernel %s", krel)
+			continue
+		}
+
+		initrdFile, err = findInitrdFile(bootfiles, krel)
+		if err != nil {
+			log.Warn().Msgf("cannot find initrd %s", krel)
+			continue
+		}
+
+		ki := config.KernelInfo{
+			DistroType:    distroType,
+			DistroRelease: si.OS.Version,
+			KernelRelease: krel,
+
+			KernelSource: "/lib/modules/" + krel + "/build",
+
+			KernelPath: kernelsBase + kernelFile,
+			InitrdPath: kernelsBase + initrdFile,
+			RootFS:     rootfs,
+		}
+
+		vmlinux := "/usr/lib/debug/boot/vmlinux-" + krel
+		log.Print("vmlinux", vmlinux)
+		if exists(vmlinux) {
+			ki.VmlinuxPath = vmlinux
+		}
+
+		kcfg.Kernels = append(kcfg.Kernels, ki)
+	}
+
+	return
+}

kernel_macos.go

@@ -0,0 +1,18 @@
+// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+// +build darwin
+
+package main
+
+import (
+	"errors"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+)
+
+func genHostKernels(download bool) (kcfg config.KernelConfig, err error) {
+	err = errors.New("generate host kernels for macOS is not supported")
+	return
+}

log.go

@@ -1,4 +1,4 @@
-// Copyright 2019 Mikhail Klementev. All rights reserved.
+// Copyright 2023 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
@@ -8,66 +8,51 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
-	"log"
 	"math"
 	"os"
 
 	"github.com/olekukonko/tablewriter"
-	"gopkg.in/logrusorgru/aurora.v1"
+	"github.com/rs/zerolog/log"
+	"gopkg.in/logrusorgru/aurora.v2"
 
 	"code.dumpstack.io/tools/out-of-tree/config"
 )
 
-func logLogEntry(l logEntry) {
-	distroInfo := fmt.Sprintf("%s-%s {%s}", l.DistroType,
-		l.DistroRelease, l.KernelRelease)
-
-	artifactInfo := fmt.Sprintf("{[%s] %s}", l.Type, l.Name)
-
-	colored := ""
-	if l.Type == config.KernelExploit {
-		colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s",
-			l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo,
-			genOkFail("BUILD", l.Build.Ok),
-			genOkFail("LPE", l.Test.Ok))
-	} else {
-		colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s %s",
-			l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo,
-			genOkFail("BUILD", l.Build.Ok),
-			genOkFail("INSMOD", l.Run.Ok),
-			genOkFail("TEST", l.Test.Ok))
-	}
-
-	additional := ""
-	if l.KernelPanic {
-		additional = "(panic)"
-	} else if l.KilledByTimeout {
-		additional = "(timeout)"
-	}
-
-	if additional != "" {
-		fmt.Println(colored, additional)
-	} else {
-		fmt.Println(colored)
-	}
+type LogCmd struct {
+	Query    LogQueryCmd    `cmd:"" help:"query logs"`
+	Dump     LogDumpCmd     `cmd:"" help:"show all info for log entry with ID"`
+	Json     LogJsonCmd     `cmd:"" help:"generate json statistics"`
+	Markdown LogMarkdownCmd `cmd:"" help:"generate markdown statistics"`
 }
 
-func logHandler(db *sql.DB, path, tag string, num int, rate bool) (err error) {
+type LogQueryCmd struct {
+	Num  int    `help:"how much lines" default:"50"`
+	Rate bool   `help:"show artifact success rate"`
+	Tag  string `help:"filter tag"`
+}
+
+func (cmd *LogQueryCmd) Run(g *Globals) (err error) {
+	db, err := openDatabase(g.Config.Database)
+	if err != nil {
+		panic(err)
+	}
+	defer db.Close()
+
 	var les []logEntry
 
-	ka, kaErr := config.ReadArtifactConfig(path + "/.out-of-tree.toml")
+	ka, kaErr := config.ReadArtifactConfig(g.WorkDir + "/.out-of-tree.toml")
 	if kaErr == nil {
-		log.Println(".out-of-tree.toml found, filter by artifact name")
-		les, err = getAllArtifactLogs(db, tag, num, ka)
+		log.Print(".out-of-tree.toml found, filter by artifact name")
+		les, err = getAllArtifactLogs(db, cmd.Tag, cmd.Num, ka)
 	} else {
-		les, err = getAllLogs(db, tag, num)
+		les, err = getAllLogs(db, cmd.Tag, cmd.Num)
 	}
 	if err != nil {
 		return
 	}
 
 	s := "\nS"
-	if rate {
+	if cmd.Rate {
 		if kaErr != nil {
 			err = kaErr
 			return
@@ -75,7 +60,7 @@ func logHandler(db *sql.DB, path, tag string, num int, rate bool) (err error) {
 
 		s = fmt.Sprintf("{[%s] %s} Overall s", ka.Type, ka.Name)
 
-		les, err = getAllArtifactLogs(db, tag, math.MaxInt64, ka)
+		les, err = getAllArtifactLogs(db, cmd.Tag, math.MaxInt64, ka)
 		if err != nil {
 			return
 		}
@@ -99,8 +84,23 @@ func logHandler(db *sql.DB, path, tag string, num int, rate bool) (err error) {
 	return
 }
 
-func logDumpHandler(db *sql.DB, id int) (err error) {
-	l, err := getLogByID(db, id)
+type LogDumpCmd struct {
+	ID int `help:"id" default:"-1"`
+}
+
+func (cmd *LogDumpCmd) Run(g *Globals) (err error) {
+	db, err := openDatabase(g.Config.Database)
+	if err != nil {
+		panic(err)
+	}
+	defer db.Close()
+
+	var l logEntry
+	if cmd.ID > 0 {
+		l, err = getLogByID(db, cmd.ID)
+	} else {
+		l, err = getLastLog(db)
+	}
 	if err != nil {
 		return
 	}
@@ -145,6 +145,102 @@ func logDumpHandler(db *sql.DB, id int) (err error) {
 	return
 }
 
+type LogJsonCmd struct {
+	Tag string `required:"" help:"filter tag"`
+}
+
+func (cmd *LogJsonCmd) Run(g *Globals) (err error) {
+	db, err := openDatabase(g.Config.Database)
+	if err != nil {
+		panic(err)
+	}
+	defer db.Close()
+
+	distros, err := getStats(db, g.WorkDir, cmd.Tag)
+	if err != nil {
+		return
+	}
+
+	bytes, err := json.Marshal(&distros)
+	if err != nil {
+		return
+	}
+
+	fmt.Println(string(bytes))
+	return
+}
+
+type LogMarkdownCmd struct {
+	Tag string `required:"" help:"filter tag"`
+}
+
+func (cmd *LogMarkdownCmd) Run(g *Globals) (err error) {
+	db, err := openDatabase(g.Config.Database)
+	if err != nil {
+		panic(err)
+	}
+	defer db.Close()
+
+	distros, err := getStats(db, g.WorkDir, cmd.Tag)
+	if err != nil {
+		return
+	}
+
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Distro", "Release", "Kernel", "Reliability"})
+	table.SetBorders(tablewriter.Border{
+		Left: true, Top: false, Right: true, Bottom: false})
+	table.SetCenterSeparator("|")
+
+	for distro, releases := range distros {
+		for release, kernels := range releases {
+			for kernel, stats := range kernels {
+				all := float64(stats.All)
+				ok := float64(stats.TestOK)
+				r := fmt.Sprintf("%6.02f%%", (ok/all)*100)
+				table.Append([]string{distro, release, kernel, r})
+			}
+		}
+	}
+
+	table.Render()
+	return
+}
+
+func logLogEntry(l logEntry) {
+	distroInfo := fmt.Sprintf("%s-%s {%s}", l.DistroType,
+		l.DistroRelease, l.KernelRelease)
+
+	artifactInfo := fmt.Sprintf("{[%s] %s}", l.Type, l.Name)
+
+	colored := ""
+	if l.Type == config.KernelExploit {
+		colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s",
+			l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo,
+			genOkFail("BUILD", l.Build.Ok),
+			genOkFail("LPE", l.Test.Ok))
+	} else {
+		colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s %s",
+			l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo,
+			genOkFail("BUILD", l.Build.Ok),
+			genOkFail("INSMOD", l.Run.Ok),
+			genOkFail("TEST", l.Test.Ok))
+	}
+
+	additional := ""
+	if l.KernelPanic {
+		additional = "(panic)"
+	} else if l.KilledByTimeout {
+		additional = "(timeout)"
+	}
+
+	if additional != "" {
+		fmt.Println(colored, additional)
+	} else {
+		fmt.Println(colored)
+	}
+}
+
 type runstat struct {
 	All, BuildOK, RunOK, TestOK, Timeout, Panic int
 }
@@ -201,45 +297,3 @@ func getStats(db *sql.DB, path, tag string) (
 
 	return
 }
-
-func logJSONHandler(db *sql.DB, path, tag string) (err error) {
-	distros, err := getStats(db, path, tag)
-	if err != nil {
-		return
-	}
-
-	bytes, err := json.Marshal(&distros)
-	if err != nil {
-		return
-	}
-
-	fmt.Println(string(bytes))
-	return
-}
-
-func logMarkdownHandler(db *sql.DB, path, tag string) (err error) {
-	distros, err := getStats(db, path, tag)
-	if err != nil {
-		return
-	}
-
-	table := tablewriter.NewWriter(os.Stdout)
-	table.SetHeader([]string{"Distro", "Release", "Kernel", "Reliability"})
-	table.SetBorders(tablewriter.Border{
-		Left: true, Top: false, Right: true, Bottom: false})
-	table.SetCenterSeparator("|")
-
-	for distro, releases := range distros {
-		for release, kernels := range releases {
-			for kernel, stats := range kernels {
-				all := float64(stats.All)
-				ok := float64(stats.TestOK)
-				r := fmt.Sprintf("%6.02f%%", (ok/all)*100)
-				table.Append([]string{distro, release, kernel, r})
-			}
-		}
-	}
-
-	table.Render()
-	return
-}

main.go

@@ -1,4 +1,4 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Copyright 2023 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
@@ -6,325 +6,172 @@ package main
 
 import (
 	"fmt"
-	"log"
+	"io"
 	"math/rand"
 	"os"
 	"os/exec"
 	"os/user"
-	"runtime"
-	"sort"
+	"runtime/debug"
 	"strconv"
 	"time"
 
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"github.com/natefinch/lumberjack"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+
+	"github.com/alecthomas/kong"
 
 	"code.dumpstack.io/tools/out-of-tree/config"
 )
 
-func findFallback(kcfg config.KernelConfig, ki config.KernelInfo) (rootfs string) {
-	for _, k := range kcfg.Kernels {
-		if !exists(k.RootFS) || k.DistroType != ki.DistroType {
-			continue
-		}
-		if k.RootFS < ki.RootFS {
-			rootfs = k.RootFS
-			return
-		}
-	}
-	return
+type Globals struct {
+	Config config.OutOfTree `help:"path to out-of-tree configuration" default:"~/.out-of-tree/out-of-tree.toml"`
+
+	WorkDir string `help:"path to work directory" default:"./" type:"path"`
 }
 
-func handleFallbacks(kcfg config.KernelConfig) {
-	sort.Sort(sort.Reverse(config.ByRootFS(kcfg.Kernels)))
+type CLI struct {
+	Globals
 
-	for i, k := range kcfg.Kernels {
-		if !exists(k.RootFS) {
-			newRootFS := findFallback(kcfg, k)
+	Pew       PewCmd       `cmd:"" help:"build, run, and test module/exploit"`
+	Kernel    KernelCmd    `cmd:"" help:"manipulate kernels"`
+	Debug     DebugCmd     `cmd:"" help:"debug environment"`
+	Log       LogCmd       `cmd:"" help:"query logs"`
+	Pack      PackCmd      `cmd:"" help:"exploit pack test"`
+	Gen       GenCmd       `cmd:"" help:"generate .out-of-tree.toml skeleton"`
+	Image     ImageCmd     `cmd:"" help:"manage images"`
+	Container ContainerCmd `cmd:"" help:"manage containers"`
 
-			s := k.RootFS + " does not exists "
-			if newRootFS != "" {
-				s += "(fallback to " + newRootFS + ")"
-			} else {
-				s += "(no fallback found)"
+	Version VersionFlag `name:"version" help:"print version information and quit"`
+
+	LogLevel LogLevelFlag `enum:"trace,debug,info,warn,error" default:"info"`
+
+	ContainerRuntime string `enum:"podman,docker" default:"podman"`
+}
+
+type LogLevelFlag string
+
+func (loglevel LogLevelFlag) AfterApply() error {
+	switch loglevel {
+	case "debug", "trace":
+		zerolog.CallerMarshalFunc = func(pc uintptr, file string, line int) string {
+			short := file
+			for i := len(file) - 1; i > 0; i-- {
+				if file[i] == '/' {
+					short = file[i+1:]
+					break
+				}
 			}
-
-			kcfg.Kernels[i].RootFS = newRootFS
-			log.Println(s)
+			file = short
+			return file + ":" + strconv.Itoa(line)
 		}
+		log.Logger = log.With().Caller().Logger()
 	}
+	return nil
 }
 
-func checkRequiredUtils() (err error) {
-	// Check for required commands
-	for _, cmd := range []string{"docker", "qemu-system-x86_64"} {
-		_, err := exec.Command("which", cmd).CombinedOutput()
-		if err != nil {
-			return fmt.Errorf("Command not found: %s", cmd)
-		}
-	}
-	return
+type VersionFlag string
+
+func (v VersionFlag) Decode(ctx *kong.DecodeContext) error { return nil }
+func (v VersionFlag) IsBool() bool                         { return true }
+func (v VersionFlag) BeforeApply(app *kong.Kong, vars kong.Vars) error {
+	fmt.Println(vars["version"])
+	app.Exit(0)
+	return nil
 }
 
-func checkDockerPermissions() (err error) {
-	output, err := exec.Command("docker", "ps").CombinedOutput()
-	if err != nil {
-		err = fmt.Errorf("%s", output)
-	}
-	return
+type LevelWriter struct {
+	io.Writer
+	Level zerolog.Level
 }
 
+func (lw *LevelWriter) WriteLevel(l zerolog.Level, p []byte) (n int, err error) {
+	if l >= lw.Level {
+		return lw.Writer.Write(p)
+	}
+	return len(p), nil
+}
+
+var tempDirBase string
+
 func main() {
-	log.SetFlags(log.Lshortfile)
-
 	rand.Seed(time.Now().UnixNano())
 
-	app := kingpin.New(
-		"out-of-tree",
-		"kernel {module, exploit} development tool",
+	cli := CLI{}
+	ctx := kong.Parse(&cli,
+		kong.Name("out-of-tree"),
+		kong.Description("kernel {module, exploit} development tool"),
+		kong.UsageOnError(),
+		kong.ConfigureHelp(kong.HelpOptions{
+			Compact: true,
+		}),
+		kong.Vars{
+			"version": "2.1.2",
+		},
 	)
 
-	app.Author("Mikhail Klementev <root@dumpstack.io>")
-	app.Version("0.2.0")
-
-	pathFlag := app.Flag("path", "Path to work directory")
-	path := pathFlag.Default(".").ExistingDir()
+	var loglevel zerolog.Level
+	switch cli.LogLevel {
+	case "trace":
+		loglevel = zerolog.TraceLevel
+	case "debug":
+		loglevel = zerolog.DebugLevel
+	case "info":
+		loglevel = zerolog.InfoLevel
+	case "warn":
+		loglevel = zerolog.WarnLevel
+	case "error":
+		loglevel = zerolog.ErrorLevel
+	}
 
 	usr, err := user.Current()
 	if err != nil {
 		return
 	}
-	os.MkdirAll(usr.HomeDir+"/.out-of-tree", os.ModePerm)
 
-	defaultKcfgPath := usr.HomeDir + "/.out-of-tree/kernels.toml"
+	tempDirBase = usr.HomeDir + "/.out-of-tree/tmp/"
+	os.MkdirAll(tempDirBase, os.ModePerm)
 
-	kcfgPathFlag := app.Flag("kernels", "Path to main kernels config")
-	kcfgPath := kcfgPathFlag.Default(defaultKcfgPath).String()
+	log.Logger = log.Output(zerolog.MultiLevelWriter(
+		&LevelWriter{Writer: zerolog.NewConsoleWriter(
+			func(w *zerolog.ConsoleWriter) {
+				w.Out = os.Stderr
+			},
+		),
+			Level: loglevel,
+		},
+		&LevelWriter{Writer: &lumberjack.Logger{
+			Filename: usr.HomeDir + "/.out-of-tree/logs/out-of-tree.log",
+		},
+			Level: zerolog.TraceLevel,
+		},
+	))
 
-	defaultDbPath := usr.HomeDir + "/.out-of-tree/db.sqlite"
-	dbPathFlag := app.Flag("db", "Path to database")
-	dbPath := dbPathFlag.Default(defaultDbPath).String()
+	log.Trace().Msg("start out-of-tree")
+	log.Debug().Msgf("%v", os.Args)
+	log.Debug().Msgf("%v", cli)
 
-	defaultUserKcfgPath := usr.HomeDir + "/.out-of-tree/kernels.user.toml"
-	userKcfgPathFlag := app.Flag("user-kernels", "User kernels config")
-	userKcfgPathEnv := userKcfgPathFlag.Envar("OUT_OF_TREE_KCFG")
-	userKcfgPath := userKcfgPathEnv.Default(defaultUserKcfgPath).String()
+	if buildInfo, ok := debug.ReadBuildInfo(); ok {
+		log.Debug().Msgf("%v", buildInfo.GoVersion)
+		log.Debug().Msgf("%v", buildInfo.Settings)
+	}
 
-	qemuTimeoutFlag := app.Flag("qemu-timeout", "Timeout for qemu")
-	qemuTimeout := qemuTimeoutFlag.Default("1m").Duration()
-
-	dockerTimeoutFlag := app.Flag("docker-timeout", "Timeout for docker")
-	dockerTimeout := dockerTimeoutFlag.Default("1m").Duration()
-
-	pewCommand := app.Command("pew", "Build, run and test module/exploit")
-
-	pewMax := pewCommand.Flag("max", "Test no more than X kernels").
-		PlaceHolder("X").Default(fmt.Sprint(kernelsAll)).Int64()
-
-	pewRuns := pewCommand.Flag("runs", "Runs per each kernel").
-		Default("1").Int64()
-
-	pewKernelFlag := pewCommand.Flag("kernel", "Override kernel regex")
-	pewKernel := pewKernelFlag.String()
-
-	pewGuessFlag := pewCommand.Flag("guess", "Try all defined kernels")
-	pewGuess := pewGuessFlag.Bool()
-
-	pewBinaryFlag := pewCommand.Flag("binary", "Use binary, do not build")
-	pewBinary := pewBinaryFlag.String()
-
-	pewTestFlag := pewCommand.Flag("test", "Override path test")
-	pewTest := pewTestFlag.String()
-
-	pewDistFlag := pewCommand.Flag("dist", "Build result path")
-	pewDist := pewDistFlag.Default(pathDevNull).String()
-
-	pewThreadsFlag := pewCommand.Flag("threads", "Build result path")
-	pewThreads := pewThreadsFlag.Default(strconv.Itoa(runtime.NumCPU())).Int()
-
-	pewTagFlag := pewCommand.Flag("tag", "Log tagging")
-	pewTag := pewTagFlag.String()
-
-	kernelCommand := app.Command("kernel", "Manipulate kernels")
-	kernelNoDownload := kernelCommand.Flag("no-download",
-		"Do not download qemu image while kernel generation").Bool()
-	kernelUseHost := kernelCommand.Flag("host", "Use also host kernels").Bool()
-	kernelListCommand := kernelCommand.Command("list", "List kernels")
-	kernelAutogenCommand := kernelCommand.Command("autogen",
-		"Generate kernels based on a current config")
-	kernelAutogenMax := kernelAutogenCommand.Flag("max",
-		"Download random kernels from set defined by regex in "+
-			"release_mask, but no more than X for each of "+
-			"release_mask").PlaceHolder("X").Default(
-		fmt.Sprint(kernelsAll)).Int64()
-	kernelDockerRegenCommand := kernelCommand.Command("docker-regen",
-		"Regenerate kernels config from out_of_tree_* docker images")
-	kernelGenallCommand := kernelCommand.Command("genall",
-		"Generate all kernels for distro")
-
-	genallDistroFlag := kernelGenallCommand.Flag("distro", "Distributive")
-	distro := genallDistroFlag.Required().String()
-
-	genallVerFlag := kernelGenallCommand.Flag("ver", "Distro version")
-	version := genallVerFlag.Required().String()
-
-	genCommand := app.Command("gen", "Generate .out-of-tree.toml skeleton")
-	genModuleCommand := genCommand.Command("module",
-		"Generate .out-of-tree.toml skeleton for kernel module")
-	genExploitCommand := genCommand.Command("exploit",
-		"Generate .out-of-tree.toml skeleton for kernel exploit")
-
-	debugCommand := app.Command("debug", "Kernel debug environment")
-	debugCommandFlag := debugCommand.Flag("kernel", "Regex (first match)")
-	debugKernel := debugCommandFlag.Required().String()
-	debugFlagGDB := debugCommand.Flag("gdb", "Set gdb listen address")
-	debugGDB := debugFlagGDB.Default("tcp::1234").String()
-
-	yekaslr := debugCommand.Flag("enable-kaslr", "Enable KASLR").Bool()
-	yesmep := debugCommand.Flag("enable-smep", "Enable SMEP").Bool()
-	yesmap := debugCommand.Flag("enable-smap", "Enable SMAP").Bool()
-	yekpti := debugCommand.Flag("enable-kpti", "Enable KPTI").Bool()
-
-	nokaslr := debugCommand.Flag("disable-kaslr", "Disable KASLR").Bool()
-	nosmep := debugCommand.Flag("disable-smep", "Disable SMEP").Bool()
-	nosmap := debugCommand.Flag("disable-smap", "Disable SMAP").Bool()
-	nokpti := debugCommand.Flag("disable-kpti", "Disable KPTI").Bool()
-
-	bootstrapCommand := app.Command("bootstrap", "Apparently nothing")
-
-	logCommand := app.Command("log", "Logs")
-
-	logQueryCommand := logCommand.Command("query", "Query logs")
-	logNum := logQueryCommand.Flag("num", "How much lines").Default("50").Int()
-	logRate := logQueryCommand.Flag("rate", "Show artifact success rate").Bool()
-	logTag := logQueryCommand.Flag("tag", "Filter tag").String()
-
-	logDumpCommand := logCommand.Command("dump",
-		"Show all info for log entry with ID")
-	logDumpID := logDumpCommand.Arg("ID", "").Required().Int()
-
-	logJSONCommand := logCommand.Command("json", "Generate json statistics")
-	logJSONTag := logJSONCommand.Flag("tag", "Filter tag").Required().String()
-
-	logMarkdownCommand := logCommand.Command("markdown", "Generate markdown statistics")
-	logMarkdownTag := logMarkdownCommand.Flag("tag", "Filter tag").Required().String()
-
-	packCommand := app.Command("pack", "Exploit pack test")
-	packAutogen := packCommand.Flag("autogen", "Kernel autogeneration").Bool()
-	packNoDownload := packCommand.Flag("no-download",
-		"Do not download qemu image while kernel generation").Bool()
-	packExploitRuns := packCommand.Flag("exploit-runs",
-		"Amount of runs of each exploit").Default("4").Int64()
-	packKernelRuns := packCommand.Flag("kernel-runs",
-		"Amount of runs of each kernel").Default("1").Int64()
-
-	err = checkRequiredUtils()
+	_, err = exec.LookPath(cli.ContainerRuntime)
 	if err != nil {
-		log.Fatalln(err)
-	}
+		if cli.ContainerRuntime == "podman" { // default value
+			log.Debug().Msgf("podman is not found in $PATH, " +
+				"fall back to docker")
+			cli.ContainerRuntime = "docker"
+		}
 
-	err = checkDockerPermissions()
-	if err != nil {
-		log.Println(err)
-		log.Println("You have two options:")
-		log.Println("\t1. Add user to group docker;")
-		log.Println("\t2. Run out-of-tree with sudo.")
-		os.Exit(1)
-	}
-
-	if !exists(usr.HomeDir + "/.out-of-tree/kernels.toml") {
-		log.Println("No ~/.out-of-tree/kernels.toml: Probably you " +
-			"need to run `out-of-tree kernel autogen` in " +
-			"directory that contains .out-of-tree.toml " +
-			"with defined kernel masks " +
-			"(see docs at https://out-of-tree.io)")
-	}
-
-	kingpin.MustParse(app.Parse(os.Args[1:]))
-
-	if *yekaslr && *nokaslr {
-		log.Fatalln("Only one of disable/enable can be used at once")
-	}
-
-	if *yesmep && *nosmep {
-		log.Fatalln("Only one of disable/enable can be used at once")
-	}
-
-	if *yesmap && *nosmap {
-		log.Fatalln("Only one of disable/enable can be used at once")
-	}
-
-	kcfg, err := config.ReadKernelConfig(*kcfgPath)
-	if err != nil {
-		log.Println(err)
-	}
-
-	if exists(*userKcfgPath) {
-		userKcfg, err := config.ReadKernelConfig(*userKcfgPath)
+		_, err = exec.LookPath(cli.ContainerRuntime)
 		if err != nil {
-			log.Fatalln(err)
-		}
-
-		for _, nk := range userKcfg.Kernels {
-			if !hasKernel(nk, kcfg) {
-				kcfg.Kernels = append(kcfg.Kernels, nk)
-			}
+			log.Fatal().Msgf("%v is not found in $PATH",
+				cli.ContainerRuntime)
 		}
 	}
+	containerRuntime = cli.ContainerRuntime
 
-	handleFallbacks(kcfg)
-
-	db, err := openDatabase(*dbPath)
-	if err != nil {
-		log.Fatalln(err)
-	}
-	defer db.Close()
-
-	switch kingpin.MustParse(app.Parse(os.Args[1:])) {
-	case pewCommand.FullCommand():
-		err = pewHandler(kcfg, *path, *pewKernel, *pewBinary,
-			*pewTest, *pewGuess, *qemuTimeout, *dockerTimeout,
-			*pewMax, *pewRuns, *pewDist, *pewTag, *pewThreads, db)
-	case kernelListCommand.FullCommand():
-		err = kernelListHandler(kcfg)
-	case kernelAutogenCommand.FullCommand():
-		err = kernelAutogenHandler(*path, *kernelAutogenMax,
-			*kernelUseHost, !*kernelNoDownload)
-	case kernelDockerRegenCommand.FullCommand():
-		err = kernelDockerRegenHandler(*kernelUseHost, !*kernelNoDownload)
-	case kernelGenallCommand.FullCommand():
-		err = kernelGenallHandler(*distro, *version,
-			*kernelUseHost, !*kernelNoDownload)
-	case genModuleCommand.FullCommand():
-		err = genConfig(config.KernelModule)
-	case genExploitCommand.FullCommand():
-		err = genConfig(config.KernelExploit)
-	case debugCommand.FullCommand():
-		err = debugHandler(kcfg, *path, *debugKernel, *debugGDB,
-			*dockerTimeout, *yekaslr, *yesmep, *yesmap, *yekpti,
-			*nokaslr, *nosmep, *nosmap, *nokpti)
-	case bootstrapCommand.FullCommand():
-		fmt.Println("bootstrap is no more required, " +
-			"now images downloading on-demand")
-		fmt.Println("please, remove it from any automation scripts, " +
-			"because it'll be removed in the next release")
-	case logQueryCommand.FullCommand():
-		err = logHandler(db, *path, *logTag, *logNum, *logRate)
-	case logDumpCommand.FullCommand():
-		err = logDumpHandler(db, *logDumpID)
-	case logJSONCommand.FullCommand():
-		err = logJSONHandler(db, *path, *logJSONTag)
-	case logMarkdownCommand.FullCommand():
-		err = logMarkdownHandler(db, *path, *logMarkdownTag)
-	case packCommand.FullCommand():
-		err = packHandler(db, *path, kcfg, *packAutogen,
-			!*packNoDownload, *packExploitRuns, *packKernelRuns)
-	}
-
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	if somethingFailed {
-		os.Exit(1)
-	}
+	err = ctx.Run(&cli.Globals)
+	ctx.FatalIfErrorf(err)
 }

pack.go

@@ -1,55 +1,83 @@
-// Copyright 2019 Mikhail Klementev. All rights reserved.
+// Copyright 2023 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
 package main
 
 import (
-	"database/sql"
 	"fmt"
 	"io/ioutil"
-	"log"
-	"runtime"
 	"time"
 
-	"code.dumpstack.io/tools/out-of-tree/config"
+	"github.com/rs/zerolog/log"
 )
 
-func packHandler(db *sql.DB, path string, kcfg config.KernelConfig,
-	autogen, download bool, exploitRuns, kernelRuns int64) (err error) {
+type PackCmd struct {
+	Autogen     bool  `help:"kernel autogeneration"`
+	UseHost     bool  `help:"also use host kernels"`
+	NoDownload  bool  `help:"do not download qemu image while kernel generation"`
+	ExploitRuns int64 `default:"4" help:"amount of runs of each exploit"`
+	KernelRuns  int64 `default:"1" help:"amount of runs of each kernel"`
+	Max         int64 `help:"download random kernels from set defined by regex in release_mask, but no more than X for each of release_mask" default:"1"`
 
-	dockerTimeout := time.Minute
-	qemuTimeout := time.Minute
-	threads := runtime.NumCPU()
+	Threads int `help:"threads" default:"4"`
 
+	Tag string `help:"filter tag"`
+
+	Timeout       time.Duration `help:"timeout after tool will not spawn new tests"`
+	QemuTimeout   time.Duration `help:"timeout for qemu"`
+	DockerTimeout time.Duration `help:"timeout for docker"`
+}
+
+func (cmd *PackCmd) Run(g *Globals) (err error) {
 	tag := fmt.Sprintf("pack_run_%d", time.Now().Unix())
-	log.Println("Tag:", tag)
+	log.Print("Tag:", tag)
 
-	files, err := ioutil.ReadDir(path)
+	files, err := ioutil.ReadDir(g.WorkDir)
 	if err != nil {
 		return
 	}
 
 	for _, f := range files {
-		workPath := path + "/" + f.Name()
+		workPath := g.WorkDir + "/" + f.Name()
 
 		if !exists(workPath + "/.out-of-tree.toml") {
 			continue
 		}
 
-		if autogen {
-			var perRegex int64 = 1
-			err = kernelAutogenHandler(workPath, perRegex, false, download)
+		if cmd.Autogen {
+			err = KernelAutogenCmd{Max: cmd.Max}.Run(
+				&KernelCmd{
+					NoDownload: cmd.NoDownload,
+					UseHost:    cmd.UseHost,
+				},
+				&Globals{
+					Config:  g.Config,
+					WorkDir: workPath,
+				},
+			)
 			if err != nil {
 				return
 			}
 		}
 
-		log.Println(f.Name())
+		log.Print(f.Name())
 
-		pewHandler(kcfg, workPath, "", "", "", false,
-			dockerTimeout, qemuTimeout,
-			kernelRuns, exploitRuns, pathDevNull, tag, threads, db)
+		pew := PewCmd{
+			Max:           cmd.KernelRuns,
+			Runs:          cmd.ExploitRuns,
+			Threads:       cmd.Threads,
+			Tag:           tag,
+			Timeout:       cmd.Timeout,
+			QemuTimeout:   cmd.QemuTimeout,
+			DockerTimeout: cmd.DockerTimeout,
+			Dist:          pathDevNull,
+		}
+
+		pew.Run(&Globals{
+			Config:  g.Config,
+			WorkDir: workPath,
+		})
 	}
 
 	return

pew.go

@@ -1,73 +1,252 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Copyright 2023 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
 package main
 
 import (
+	"bufio"
 	"database/sql"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"math/rand"
 	"os"
 	"os/exec"
-	"os/user"
 	"strings"
 	"time"
 
 	"github.com/otiai10/copy"
 	"github.com/remeh/sizedwaitgroup"
-	"gopkg.in/logrusorgru/aurora.v1"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"gopkg.in/logrusorgru/aurora.v2"
 
 	"code.dumpstack.io/tools/out-of-tree/config"
 	"code.dumpstack.io/tools/out-of-tree/qemu"
 )
 
-var somethingFailed = false
+type PewCmd struct {
+	Max     int64         `help:"test no more than X kernels" default:"100500"`
+	Runs    int64         `help:"runs per each kernel" default:"1"`
+	Kernel  string        `help:"override kernel regex"`
+	Guess   bool          `help:"try all defined kernels"`
+	Shuffle bool          `help:"randomize kernels test order"`
+	Binary  string        `help:"use binary, do not build"`
+	Test    string        `help:"override path for test"`
+	Dist    string        `help:"build result path" default:"/dev/null"`
+	Threads int           `help:"threads" default:"1"`
+	Tag     string        `help:"log tagging"`
+	Timeout time.Duration `help:"timeout after tool will not spawn new tests"`
 
-const pathDevNull = "/dev/null"
+	ArtifactConfig string `help:"path to artifact config" type:"path"`
 
-func dockerRun(timeout time.Duration, container, workdir, command string) (
-	output string, err error) {
+	QemuTimeout   time.Duration `help:"timeout for qemu"`
+	DockerTimeout time.Duration `help:"timeout for docker"`
 
-	cmd := exec.Command("docker", "run", "-v", workdir+":/work",
-		container, "bash", "-c", "cd /work && "+command)
+	Threshold float64 `help:"reliablity threshold for exit code" default:"1.00"`
 
-	timer := time.AfterFunc(timeout, func() {
-		cmd.Process.Kill()
-	})
-	defer timer.Stop()
+	db              *sql.DB
+	kcfg            config.KernelConfig
+	timeoutDeadline time.Time
+}
 
-	raw, err := cmd.CombinedOutput()
+func (cmd *PewCmd) Run(g *Globals) (err error) {
+	cmd.kcfg, err = config.ReadKernelConfig(g.Config.Kernels)
+	if err != nil {
+		log.Fatal().Err(err).Msg("read kernels config")
+	}
+
+	if cmd.Timeout != 0 {
+		log.Info().Msgf("Set global timeout to %s", cmd.Timeout)
+		cmd.timeoutDeadline = time.Now().Add(cmd.Timeout)
+	}
+
+	cmd.db, err = openDatabase(g.Config.Database)
+	if err != nil {
+		log.Fatal().Err(err).
+			Msgf("Cannot open database %s", g.Config.Database)
+	}
+	defer cmd.db.Close()
+
+	var configPath string
+	if cmd.ArtifactConfig == "" {
+		configPath = g.WorkDir + "/.out-of-tree.toml"
+	} else {
+		configPath = cmd.ArtifactConfig
+	}
+	ka, err := config.ReadArtifactConfig(configPath)
 	if err != nil {
-		e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`",
-			err, command, string(raw))
-		err = errors.New(e)
 		return
 	}
 
-	output = string(raw)
+	if ka.SourcePath == "" {
+		ka.SourcePath = g.WorkDir
+	}
+
+	if cmd.Kernel != "" {
+		var km config.KernelMask
+		km, err = kernelMask(cmd.Kernel)
+		if err != nil {
+			return
+		}
+
+		ka.SupportedKernels = []config.KernelMask{km}
+	}
+
+	if cmd.Guess {
+		ka.SupportedKernels, err = genAllKernels()
+		if err != nil {
+			return
+		}
+	}
+
+	if cmd.QemuTimeout != 0 {
+		log.Info().Msgf("Set qemu timeout to %s", cmd.QemuTimeout)
+	} else {
+		cmd.QemuTimeout = g.Config.Qemu.Timeout.Duration
+	}
+
+	if cmd.DockerTimeout != 0 {
+		log.Info().Msgf("Set docker timeout to %s", cmd.DockerTimeout)
+	} else {
+		cmd.DockerTimeout = g.Config.Docker.Timeout.Duration
+	}
+
+	if cmd.Tag == "" {
+		cmd.Tag = fmt.Sprintf("%d", time.Now().Unix())
+	}
+	log.Info().Str("tag", cmd.Tag).Msg("log")
+
+	err = cmd.performCI(ka)
+	if err != nil {
+		return
+	}
+
+	log.Info().Msgf("Success rate: %.02f, Threshold: %.02f",
+		successRate(state), cmd.Threshold)
+	if successRate(state) < cmd.Threshold {
+		err = errors.New("reliability threshold not met")
+	}
+	return
+}
+
+type runstate struct {
+	Overall, Success float64
+}
+
+var (
+	state runstate
+)
+
+func successRate(state runstate) float64 {
+	return state.Success / state.Overall
+}
+
+const pathDevNull = "/dev/null"
+
+func sh(workdir, command string) (output string, err error) {
+	flog := log.With().
+		Str("workdir", workdir).
+		Str("command", command).
+		Logger()
+
+	cmd := exec.Command("sh", "-c", "cd "+workdir+" && "+command)
+
+	flog.Debug().Msgf("%v", cmd)
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	cmd.Stderr = cmd.Stdout
+
+	err = cmd.Start()
+	if err != nil {
+		return
+	}
+
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			m := scanner.Text()
+			output += m + "\n"
+			flog.Trace().Str("stdout", m).Msg("")
+		}
+	}()
+
+	err = cmd.Wait()
+
+	if err != nil {
+		e := fmt.Sprintf("%v %v output: %v", cmd, err, output)
+		err = errors.New(e)
+	}
+	return
+}
+
+func applyPatches(src string, ka config.Artifact) (err error) {
+	for i, patch := range ka.Patches {
+		name := fmt.Sprintf("patch_%02d", i)
+
+		path := src + "/" + name + ".diff"
+		if patch.Source != "" && patch.Path != "" {
+			err = errors.New("path and source are mutually exclusive")
+			return
+		} else if patch.Source != "" {
+			err = os.WriteFile(path, []byte(patch.Source), 0644)
+			if err != nil {
+				return
+			}
+		} else if patch.Path != "" {
+			err = copy.Copy(patch.Path, path)
+			if err != nil {
+				return
+			}
+		}
+
+		if patch.Source != "" || patch.Path != "" {
+			_, err = sh(src, "patch < "+path)
+			if err != nil {
+				return
+			}
+		}
+
+		if patch.Script != "" {
+			script := src + "/" + name + ".sh"
+			err = os.WriteFile(script, []byte(patch.Script), 0755)
+			if err != nil {
+				return
+			}
+			_, err = sh(src, script)
+			if err != nil {
+				return
+			}
+		}
+	}
 	return
 }
 
 func build(tmp string, ka config.Artifact, ki config.KernelInfo,
-	dockerTimeout time.Duration) (outPath, output string, err error) {
+	dockerTimeout time.Duration) (outdir, outpath, output string, err error) {
 
 	target := fmt.Sprintf("%d_%s", rand.Int(), ki.KernelRelease)
 
-	tmpSourcePath := tmp + "/source"
+	outdir = tmp + "/source"
 
-	err = copy.Copy(ka.SourcePath, tmpSourcePath)
+	err = copy.Copy(ka.SourcePath, outdir)
 	if err != nil {
 		return
 	}
 
-	outPath = tmpSourcePath + "/" + target
+	err = applyPatches(outdir, ka)
+	if err != nil {
+		return
+	}
+
+	outpath = outdir + "/" + target
 	if ka.Type == config.KernelModule {
-		outPath += ".ko"
+		outpath += ".ko"
 	}
 
 	kernel := "/lib/modules/" + ki.KernelRelease + "/build"
@@ -75,13 +254,25 @@ func build(tmp string, ka config.Artifact, ki config.KernelInfo,
 		kernel = ki.KernelSource
 	}
 
+	buildCommand := "make KERNEL=" + kernel + " TARGET=" + target
+	if ka.Make.Target != "" {
+		buildCommand += " " + ka.Make.Target
+	}
+
 	if ki.ContainerName != "" {
-		output, err = dockerRun(dockerTimeout, ki.ContainerName,
-			tmpSourcePath, "make KERNEL="+kernel+" TARGET="+target+
-				" && chmod -R 777 /work")
+		var c container
+		c, err = NewContainer(ki.ContainerName, dockerTimeout)
+		if err != nil {
+			log.Fatal().Err(err).Msg("container creation failure")
+		}
+
+		output, err = c.Run(outdir, buildCommand+" && chmod -R 777 /work")
 	} else {
-		command := "make KERNEL=" + kernel + " TARGET=" + target
-		cmd := exec.Command("bash", "-c", "cd "+tmpSourcePath+" && "+command)
+		cmd := exec.Command("bash", "-c", "cd "+outdir+" && "+
+			buildCommand)
+
+		log.Debug().Msgf("%v", cmd)
+
 		timer := time.AfterFunc(dockerTimeout, func() {
 			cmd.Process.Kill()
 		})
@@ -91,7 +282,7 @@ func build(tmp string, ka config.Artifact, ki config.KernelInfo,
 		raw, err = cmd.CombinedOutput()
 		if err != nil {
 			e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`",
-				err, command, string(raw))
+				err, buildCommand, string(raw))
 			err = errors.New(e)
 			return
 		}
@@ -101,6 +292,10 @@ func build(tmp string, ka config.Artifact, ki config.KernelInfo,
 	return
 }
 
+func runScript(q *qemu.System, script string) (output string, err error) {
+	return q.Command("root", script)
+}
+
 func testKernelModule(q *qemu.System, ka config.Artifact,
 	test string) (output string, err error) {
 
@@ -134,18 +329,24 @@ func testKernelExploit(q *qemu.System, ka config.Artifact,
 }
 
 func genOkFail(name string, ok bool) (aurv aurora.Value) {
+	state.Overall += 1
+	s := " " + name
+	if name == "" {
+		s = ""
+	}
 	if ok {
-		s := " " + name + " SUCCESS "
+		state.Success += 1
+		s += " SUCCESS "
 		aurv = aurora.BgGreen(aurora.Black(s))
 	} else {
-		somethingFailed = true
-		s := " " + name + " FAILURE "
-		aurv = aurora.BgRed(aurora.Gray(aurora.Bold(s)))
+		s += " FAILURE "
+		aurv = aurora.BgRed(aurora.White(aurora.Bold(s)))
 	}
 	return
 }
 
 type phasesResult struct {
+	BuildDir         string
 	BuildArtifact    string
 	Build, Run, Test struct {
 		Output string
@@ -180,15 +381,19 @@ func dumpResult(q *qemu.System, ka config.Artifact, ki config.KernelInfo,
 		ki.DistroRelease, ki.KernelRelease)
 
 	colored := ""
-	if ka.Type == config.KernelExploit {
+	switch ka.Type {
+	case config.KernelExploit:
 		colored = aurora.Sprintf("[*] %40s: %s %s", distroInfo,
 			genOkFail("BUILD", res.Build.Ok),
 			genOkFail("LPE", res.Test.Ok))
-	} else {
+	case config.KernelModule:
 		colored = aurora.Sprintf("[*] %40s: %s %s %s", distroInfo,
 			genOkFail("BUILD", res.Build.Ok),
 			genOkFail("INSMOD", res.Run.Ok),
 			genOkFail("TEST", res.Test.Ok))
+	case config.Script:
+		colored = aurora.Sprintf("[*] %40s: %s", distroInfo,
+			genOkFail("", res.Test.Ok))
 	}
 
 	additional := ""
@@ -206,13 +411,13 @@ func dumpResult(q *qemu.System, ka config.Artifact, ki config.KernelInfo,
 
 	err := addToLog(db, q, ka, ki, res, tag)
 	if err != nil {
-		log.Println("[db] addToLog (", ka, ") error:", err)
+		log.Warn().Err(err).Msgf("[db] addToLog (%v)", ka)
 	}
 
 	if binary == "" && dist != pathDevNull {
 		err = os.MkdirAll(dist, os.ModePerm)
 		if err != nil {
-			log.Println("os.MkdirAll (", ka, ") error:", err)
+			log.Warn().Err(err).Msgf("os.MkdirAll (%v)", ka)
 		}
 
 		path := fmt.Sprintf("%s/%s-%s-%s", dist, ki.DistroType,
@@ -223,26 +428,40 @@ func dumpResult(q *qemu.System, ka config.Artifact, ki config.KernelInfo,
 
 		err = copyFile(res.BuildArtifact, path)
 		if err != nil {
-			log.Println("copyFile (", ka, ") error:", err)
+			log.Warn().Err(err).Msgf("copy file (%v)", ka)
 		}
 	}
 }
 
-func copyArtifactAndTest(q *qemu.System, ka config.Artifact,
+func copyArtifactAndTest(slog zerolog.Logger, q *qemu.System, ka config.Artifact,
 	res *phasesResult, remoteTest string) (err error) {
 
+	// Copy all test files to the remote machine
+	for _, f := range ka.TestFiles {
+		if f.Local[0] != '/' {
+			if res.BuildDir != "" {
+				f.Local = res.BuildDir + "/" + f.Local
+			}
+		}
+		err = q.CopyFile(f.User, f.Local, f.Remote)
+		if err != nil {
+			slog.Error().Err(err).Msg("copy test file")
+			return
+		}
+	}
+
 	switch ka.Type {
 	case config.KernelModule:
 		res.Run.Output, err = q.CopyAndInsmod(res.BuildArtifact)
 		if err != nil {
-			log.Println(res.Run.Output, err)
+			slog.Error().Err(err).Msg(res.Run.Output)
 			return
 		}
 		res.Run.Ok = true
 
 		res.Test.Output, err = testKernelModule(q, ka, remoteTest)
 		if err != nil {
-			log.Println(res.Test.Output, err)
+			slog.Error().Err(err).Msg(res.Test.Output)
 			return
 		}
 		res.Test.Ok = true
@@ -256,13 +475,22 @@ func copyArtifactAndTest(q *qemu.System, ka config.Artifact,
 		res.Test.Output, err = testKernelExploit(q, ka, remoteTest,
 			remoteExploit)
 		if err != nil {
-			log.Println(res.Test.Output)
+			slog.Error().Err(err).Msg(res.Test.Output)
 			return
 		}
 		res.Run.Ok = true // does not really used
 		res.Test.Ok = true
+	case config.Script:
+		res.Test.Output, err = runScript(q, remoteTest)
+		if err != nil {
+			slog.Error().Err(err).Msg(res.Test.Output)
+			return
+		}
+		slog.Info().Msg(res.Test.Output)
+		res.Run.Ok = true
+		res.Test.Ok = true
 	default:
-		log.Println("Unsupported artifact type")
+		slog.Fatal().Msg("Unsupported artifact type")
 	}
 
 	return
@@ -289,20 +517,53 @@ func copyTest(q *qemu.System, testPath string, ka config.Artifact) (
 	return
 }
 
-func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
-	ki config.KernelInfo, binaryPath, testPath string,
-	qemuTimeout, dockerTimeout time.Duration, dist, tag string,
-	db *sql.DB) {
+func copyStandardModules(q *qemu.System, ki config.KernelInfo) (err error) {
+	_, err = q.Command("root", "mkdir -p /lib/modules")
+	if err != nil {
+		return
+	}
+
+	files, err := ioutil.ReadDir(ki.ModulesPath)
+	if err != nil {
+		return
+	}
+
+	// FIXME scp cannot ignore symlinks
+	for _, f := range files {
+		if f.Mode()&os.ModeSymlink == os.ModeSymlink {
+			continue
+		}
+
+		path := ki.ModulesPath + "/" + f.Name()
+		err = q.CopyDirectory("root", path, "/lib/modules/"+ki.KernelRelease+"/")
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (cmd PewCmd) testArtifact(swg *sizedwaitgroup.SizedWaitGroup,
+	ka config.Artifact, ki config.KernelInfo) {
 
 	defer swg.Done()
 
+	slog := log.With().
+		Str("distro_type", ki.DistroType.String()).
+		Str("distro_release", ki.DistroRelease).
+		Str("kernel", ki.KernelRelease).
+		Logger()
+
+	slog.Info().Msg("start")
+
 	kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
 	q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
 	if err != nil {
-		log.Println("Qemu creation error:", err)
+		slog.Error().Err(err).Msg("qemu init")
 		return
 	}
-	q.Timeout = qemuTimeout
+	q.Timeout = cmd.QemuTimeout
 
 	if ka.Qemu.Timeout.Duration != 0 {
 		q.Timeout = ka.Qemu.Timeout.Duration
@@ -314,60 +575,98 @@ func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
 		q.Memory = ka.Qemu.Memory
 	}
 
+	if ka.Docker.Timeout.Duration != 0 {
+		cmd.DockerTimeout = ka.Docker.Timeout.Duration
+	}
+
 	q.SetKASLR(!ka.Mitigations.DisableKaslr)
 	q.SetSMEP(!ka.Mitigations.DisableSmep)
 	q.SetSMAP(!ka.Mitigations.DisableSmap)
+	q.SetKPTI(!ka.Mitigations.DisableKpti)
 
 	err = q.Start()
 	if err != nil {
-		log.Println("Qemu start error:", err)
+		slog.Error().Err(err).Msg("qemu start")
 		return
 	}
 	defer q.Stop()
 
-	usr, err := user.Current()
-	if err != nil {
-		return
-	}
-	tmpdir := usr.HomeDir + "/.out-of-tree/tmp"
-	os.MkdirAll(tmpdir, os.ModePerm)
+	go func() {
+		for !q.Died {
+			time.Sleep(time.Minute)
+			slog.Debug().Msg("still alive")
+		}
+	}()
 
-	tmp, err := ioutil.TempDir(tmpdir, "out-of-tree_")
+	tmp, err := ioutil.TempDir(tempDirBase, "out-of-tree_")
 	if err != nil {
-		log.Println("Temporary directory creation error:", err)
+		slog.Error().Err(err).Msg("making tmp directory")
 		return
 	}
 	defer os.RemoveAll(tmp)
 
 	result := phasesResult{}
-	defer dumpResult(q, ka, ki, &result, dist, tag, binaryPath, db)
+	defer dumpResult(q, ka, ki, &result, cmd.Dist, cmd.Tag, cmd.Binary, cmd.db)
 
-	if binaryPath == "" {
-		result.BuildArtifact, result.Build.Output, err = build(tmp, ka,
-			ki, dockerTimeout)
+	if ka.Type == config.Script {
+		result.Build.Ok = true
+		cmd.Test = ka.Script
+	} else if cmd.Binary == "" {
+		// TODO: build should return structure
+		start := time.Now()
+		result.BuildDir, result.BuildArtifact, result.Build.Output, err =
+			build(tmp, ka, ki, cmd.DockerTimeout)
+		slog.Debug().Str("duration", time.Now().Sub(start).String()).
+			Msg("build done")
 		if err != nil {
-			log.Println(err)
+			log.Error().Err(err).Msg("build")
 			return
 		}
 		result.Build.Ok = true
 	} else {
-		result.BuildArtifact = binaryPath
+		result.BuildArtifact = cmd.Binary
 		result.Build.Ok = true
 	}
 
-	if testPath == "" {
-		testPath = result.BuildArtifact + "_test"
-		if !exists(testPath) {
-			testPath = tmp + "/" + "test.sh"
+	if cmd.Test == "" {
+		cmd.Test = result.BuildArtifact + "_test"
+		if !exists(cmd.Test) {
+			cmd.Test = tmp + "/source/" + "test.sh"
 		}
 	}
 
-	remoteTest, err := copyTest(q, testPath, ka)
+	err = q.WaitForSSH(cmd.QemuTimeout)
 	if err != nil {
 		return
 	}
 
-	copyArtifactAndTest(q, ka, &result, remoteTest)
+	remoteTest, err := copyTest(q, cmd.Test, ka)
+	if err != nil {
+		return
+	}
+
+	if ka.StandardModules {
+		// Module depends on one of the standard modules
+		start := time.Now()
+		err = copyStandardModules(q, ki)
+		if err != nil {
+			slog.Fatal().Err(err).Msg("copy standard modules")
+			return
+		}
+		slog.Debug().Str("duration", time.Now().Sub(start).String()).
+			Msg("copy standard modules")
+	}
+
+	err = preloadModules(q, ka, ki, cmd.DockerTimeout)
+	if err != nil {
+		slog.Error().Err(err).Msg("preload modules")
+		return
+	}
+
+	start := time.Now()
+	copyArtifactAndTest(slog, q, ka, &result, remoteTest)
+	slog.Debug().Str("duration", time.Now().Sub(start).String()).
+		Msg("test completed")
 }
 
 func shuffleKernels(a []config.KernelInfo) []config.KernelInfo {
@@ -379,15 +678,15 @@ func shuffleKernels(a []config.KernelInfo) []config.KernelInfo {
 	return a
 }
 
-func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath,
-	testPath string, qemuTimeout, dockerTimeout time.Duration,
-	max, runs int64, dist, tag string, threads int,
-	db *sql.DB) (err error) {
-
+func (cmd PewCmd) performCI(ka config.Artifact) (err error) {
 	found := false
+	max := cmd.Max
 
-	swg := sizedwaitgroup.New(threads)
-	for _, kernel := range shuffleKernels(kcfg.Kernels) {
+	swg := sizedwaitgroup.New(cmd.Threads)
+	if cmd.Shuffle {
+		cmd.kcfg.Kernels = shuffleKernels(cmd.kcfg.Kernels)
+	}
+	for _, kernel := range cmd.kcfg.Kernels {
 		if max <= 0 {
 			break
 		}
@@ -401,11 +700,14 @@ func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath,
 		if supported {
 			found = true
 			max--
-			for i := int64(0); i < runs; i++ {
+			for i := int64(0); i < cmd.Runs; i++ {
+				if !cmd.timeoutDeadline.IsZero() &&
+					time.Now().After(cmd.timeoutDeadline) {
+
+					break
+				}
 				swg.Add()
-				go whatever(&swg, ka, kernel, binaryPath,
-					testPath, qemuTimeout, dockerTimeout,
-					dist, tag, db)
+				go cmd.testArtifact(&swg, ka, kernel)
 			}
 		}
 	}
@@ -420,8 +722,10 @@ func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath,
 
 func exists(path string) bool {
 	if _, err := os.Stat(path); err != nil {
+		log.Debug().Msgf("%s does not exist", path)
 		return false
 	}
+	log.Debug().Msgf("%s exist", path)
 	return true
 }
 
@@ -456,44 +760,3 @@ func genAllKernels() (sk []config.KernelMask, err error) {
 	}
 	return
 }
-
-func pewHandler(kcfg config.KernelConfig,
-	workPath, ovrrdKrnl, binary, test string, guess bool,
-	qemuTimeout, dockerTimeout time.Duration,
-	max, runs int64, dist, tag string, threads int,
-	db *sql.DB) (err error) {
-
-	ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
-	if err != nil {
-		return
-	}
-
-	if ka.SourcePath == "" {
-		ka.SourcePath = workPath
-	}
-
-	if ovrrdKrnl != "" {
-		var km config.KernelMask
-		km, err = kernelMask(ovrrdKrnl)
-		if err != nil {
-			return
-		}
-
-		ka.SupportedKernels = []config.KernelMask{km}
-	}
-
-	if guess {
-		ka.SupportedKernels, err = genAllKernels()
-		if err != nil {
-			return
-		}
-	}
-
-	err = performCI(ka, kcfg, binary, test, qemuTimeout, dockerTimeout,
-		max, runs, dist, tag, threads, db)
-	if err != nil {
-		return
-	}
-
-	return
-}

pew_test.go

@@ -1,44 +0,0 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
-// Use of this source code is governed by a AGPLv3 license
-// (or later) that can be found in the LICENSE file.
-
-package main
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"strings"
-	"testing"
-	"time"
-)
-
-func TestDockerRun(t *testing.T) {
-	tmp, err := ioutil.TempDir("/tmp/", "out-of-tree_test_")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmp)
-
-	start := time.Now()
-
-	timeout := time.Second
-
-	_, err = dockerRun(timeout, "ubuntu", tmp, "sleep 5s")
-	if err == nil {
-		t.Fatal("docker is not killed by timeout")
-	}
-	if time.Since(start) > 3*time.Second {
-		t.Fatal(fmt.Sprintf("timeout failed (%v instead of %v)",
-			time.Since(start), time.Second))
-	}
-
-	output, err := dockerRun(time.Minute, "ubuntu", tmp, "echo hello")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !strings.Contains(output, "hello") {
-		t.Fatal("wrong output (" + output + ")")
-	}
-}

preload.go

@@ -0,0 +1,166 @@
+// Copyright 2020 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package main
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"errors"
+	"io/ioutil"
+	"os"
+	"os/user"
+	"path/filepath"
+	"time"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/rs/zerolog/log"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+	"code.dumpstack.io/tools/out-of-tree/qemu"
+)
+
+func preloadModules(q *qemu.System, ka config.Artifact, ki config.KernelInfo,
+	dockerTimeout time.Duration) (err error) {
+
+	for _, pm := range ka.Preload {
+		err = preload(q, ki, pm, dockerTimeout)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func preload(q *qemu.System, ki config.KernelInfo, pm config.PreloadModule,
+	dockerTimeout time.Duration) (err error) {
+
+	var workPath, cache string
+	if pm.Path != "" {
+		log.Print("Use non-git path for preload module (no cache)")
+		workPath = pm.Path
+	} else if pm.Repo != "" {
+		workPath, cache, err = cloneOrPull(pm.Repo, ki)
+		if err != nil {
+			return
+		}
+	} else {
+		errors.New("No repo/path in preload entry")
+	}
+
+	err = buildAndInsmod(workPath, q, ki, dockerTimeout, cache)
+	if err != nil {
+		return
+	}
+
+	time.Sleep(pm.TimeoutAfterLoad.Duration)
+	return
+}
+
+func buildAndInsmod(workPath string, q *qemu.System, ki config.KernelInfo,
+	dockerTimeout time.Duration, cache string) (err error) {
+
+	tmp, err := ioutil.TempDir(tempDirBase, "out-of-tree_")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(tmp)
+
+	var artifact string
+	if exists(cache) {
+		artifact = cache
+	} else {
+		artifact, err = buildPreload(workPath, tmp, ki, dockerTimeout)
+		if err != nil {
+			return
+		}
+		if cache != "" {
+			err = copyFile(artifact, cache)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	output, err := q.CopyAndInsmod(artifact)
+	if err != nil {
+		log.Print(output)
+		return
+	}
+	return
+}
+
+func buildPreload(workPath, tmp string, ki config.KernelInfo,
+	dockerTimeout time.Duration) (artifact string, err error) {
+
+	ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
+	if err != nil {
+		return
+	}
+
+	ka.SourcePath = workPath
+
+	km := config.KernelMask{DistroType: ki.DistroType,
+		DistroRelease: ki.DistroRelease,
+		ReleaseMask:   ki.KernelRelease,
+	}
+	ka.SupportedKernels = []config.KernelMask{km}
+
+	if ka.Docker.Timeout.Duration != 0 {
+		dockerTimeout = ka.Docker.Timeout.Duration
+	}
+
+	_, artifact, _, err = build(tmp, ka, ki, dockerTimeout)
+	return
+}
+
+func cloneOrPull(repo string, ki config.KernelInfo) (workPath, cache string, err error) {
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+	base := filepath.Join(usr.HomeDir, "/.out-of-tree/preload/")
+	workPath = filepath.Join(base, "/repos/", sha1sum(repo))
+
+	var r *git.Repository
+	if exists(workPath) {
+		r, err = git.PlainOpen(workPath)
+		if err != nil {
+			return
+		}
+
+		var w *git.Worktree
+		w, err = r.Worktree()
+		if err != nil {
+			return
+		}
+
+		err = w.Pull(&git.PullOptions{})
+		if err != nil && err != git.NoErrAlreadyUpToDate {
+			log.Print(repo, "pull error:", err)
+		}
+	} else {
+		r, err = git.PlainClone(workPath, false, &git.CloneOptions{URL: repo})
+		if err != nil {
+			return
+		}
+	}
+
+	ref, err := r.Head()
+	if err != nil {
+		return
+	}
+
+	cachedir := filepath.Join(base, "/cache/")
+	os.MkdirAll(cachedir, 0700)
+
+	filename := sha1sum(repo + ki.KernelPath + ref.Hash().String())
+	cache = filepath.Join(cachedir, filename)
+	return
+}
+
+func sha1sum(data string) string {
+	h := sha1.Sum([]byte(data))
+	return hex.EncodeToString(h[:])
+}

qemu/README.md

@@ -30,7 +30,7 @@ Note: qemu on macOS since v2.12 (24 April 2018) supports Hypervisor.framework.
 
 #### Generate image
 
-    $ cd $GOPATH/src/code.dumpstack.io/tools/out-of-tree/tools/qemu-debian-img
+    $ cd $GOPATH/src/code.dumpstack.io/tools/out-of-tree/tools/qemu-ubuntu-img
     $ ./bootstrap.sh
 
 ### Fill configuration file

qemu/qemu-kernel.go

@@ -5,7 +5,7 @@
 package qemu
 
 import (
-	"bytes"
+	"bufio"
 	"errors"
 	"fmt"
 	"io"
@@ -18,29 +18,11 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 	"golang.org/x/crypto/ssh"
 )
 
-func readUntilEOF(pipe io.ReadCloser, buf *[]byte) (err error) {
-	bufSize := 1024
-	for err != io.EOF {
-		stdout := make([]byte, bufSize)
-		var n int
-
-		n, err = pipe.Read(stdout)
-		if err != nil && err != io.EOF {
-			return
-		}
-
-		*buf = append(*buf, stdout[:n]...)
-	}
-
-	if err == io.EOF {
-		err = nil
-	}
-	return
-}
-
 type arch string
 
 const (
@@ -60,12 +42,14 @@ type Kernel struct {
 	InitrdPath string
 }
 
-// System describe qemu parameters and runned process
+// System describe qemu parameters and executed process
 type System struct {
 	arch      arch
 	kernel    Kernel
 	drivePath string
 
+	Mutable bool
+
 	Cpus   int
 	Memory int
 
@@ -86,7 +70,7 @@ type System struct {
 	Died        bool
 	sshAddrPort string
 
-	// accessible while qemu is runned
+	// accessible while qemu is running
 	cmd  *exec.Cmd
 	pipe struct {
 		stdin  io.WriteCloser
@@ -94,18 +78,25 @@ type System struct {
 		stdout io.ReadCloser
 	}
 
-	Stdout, Stderr []byte
+	Stdout, Stderr string
 
 	// accessible after qemu is closed
 	exitErr error
+
+	log zerolog.Logger
 }
 
 // NewSystem constructor
 func NewSystem(arch arch, kernel Kernel, drivePath string) (q *System, err error) {
+	q = &System{}
+	q.log = log.With().
+		Str("kernel", kernel.KernelPath).
+		Logger()
+
 	if _, err = exec.LookPath("qemu-system-" + string(arch)); err != nil {
 		return
 	}
-	q = &System{}
+
 	q.arch = arch
 
 	if _, err = os.Stat(kernel.KernelPath); err != nil {
@@ -125,6 +116,12 @@ func NewSystem(arch arch, kernel Kernel, drivePath string) (q *System, err error
 	return
 }
 
+func (q *System) SetSSHAddrPort(addr string, port int) (err error) {
+	// TODO validate
+	q.sshAddrPort = fmt.Sprintf("%s:%d", addr, port)
+	return
+}
+
 func getRandomAddrPort() (addr string) {
 	// 127.1-255.0-255.0-255:10000-50000
 	ip := fmt.Sprintf("127.%d.%d.%d",
@@ -178,11 +175,12 @@ func kvmExists() bool {
 func (q *System) panicWatcher() {
 	for {
 		time.Sleep(time.Second)
-		if bytes.Contains(q.Stdout, []byte("Kernel panic")) {
+		if strings.Contains(q.Stdout, "Kernel panic") {
+			q.KernelPanic = true
+			q.log.Debug().Msg("kernel panic")
 			time.Sleep(time.Second)
 			// There is no reason to stay alive after kernel panic
 			q.Stop()
-			q.KernelPanic = true
 			return
 		}
 	}
@@ -210,12 +208,16 @@ func (q System) cmdline() (s string) {
 	return
 }
 
-// Start qemu process
-func (q *System) Start() (err error) {
-	rand.Seed(time.Now().UnixNano()) // Are you sure?
-	q.sshAddrPort = getFreeAddrPort()
+func (q System) Executable() string {
+	return "qemu-system-" + string(q.arch)
+}
+
+func (q *System) Args() (qemuArgs []string) {
+	if q.sshAddrPort == "" {
+		q.sshAddrPort = getFreeAddrPort()
+	}
 	hostfwd := fmt.Sprintf("hostfwd=tcp:%s-:22", q.sshAddrPort)
-	qemuArgs := []string{"-snapshot", "-nographic",
+	qemuArgs = []string{"-nographic",
 		"-hda", q.drivePath,
 		"-kernel", q.kernel.KernelPath,
 		"-smp", fmt.Sprintf("%d", q.Cpus),
@@ -224,6 +226,10 @@ func (q *System) Start() (err error) {
 		"-netdev", "user,id=n1," + hostfwd,
 	}
 
+	if !q.Mutable {
+		qemuArgs = append(qemuArgs, "-snapshot")
+	}
+
 	if q.debug {
 		qemuArgs = append(qemuArgs, "-gdb", q.gdb)
 	}
@@ -241,8 +247,15 @@ func (q *System) Start() (err error) {
 	}
 
 	qemuArgs = append(qemuArgs, "-append", q.cmdline())
+	return
+}
 
-	q.cmd = exec.Command("qemu-system-"+string(q.arch), qemuArgs...)
+// Start qemu process
+func (q *System) Start() (err error) {
+	rand.Seed(time.Now().UnixNano()) // Are you sure?
+
+	q.cmd = exec.Command(q.Executable(), q.Args()...)
+	q.log.Debug().Msgf("%v", q.cmd)
 
 	if q.pipe.stdin, err = q.cmd.StdinPipe(); err != nil {
 		return
@@ -261,8 +274,23 @@ func (q *System) Start() (err error) {
 		return
 	}
 
-	go readUntilEOF(q.pipe.stdout, &q.Stdout)
-	go readUntilEOF(q.pipe.stderr, &q.Stderr)
+	go func() {
+		scanner := bufio.NewScanner(q.pipe.stdout)
+		for scanner.Scan() {
+			m := scanner.Text()
+			q.Stdout += m + "\n"
+			q.log.Trace().Str("stdout", m).Msg("")
+		}
+	}()
+
+	go func() {
+		scanner := bufio.NewScanner(q.pipe.stderr)
+		for scanner.Scan() {
+			m := scanner.Text()
+			q.Stderr += m + "\n"
+			q.log.Trace().Str("stderr", m).Msg("")
+		}
+	}()
 
 	go func() {
 		q.exitErr = q.cmd.Wait()
@@ -301,6 +329,20 @@ func (q *System) Stop() {
 	}
 }
 
+func (q System) WaitForSSH(timeout time.Duration) error {
+	for start := time.Now(); time.Since(start) < timeout; {
+		client, err := q.ssh("root")
+		if err != nil {
+			time.Sleep(time.Second / 10)
+			continue
+		}
+		client.Close()
+		return nil
+	}
+
+	return errors.New("no ssh (timeout)")
+}
+
 func (q System) ssh(user string) (client *ssh.Client, err error) {
 	cfg := &ssh.ClientConfig{
 		User:            user,
@@ -313,6 +355,14 @@ func (q System) ssh(user string) (client *ssh.Client, err error) {
 
 // Command executes shell commands on qemu system
 func (q System) Command(user, cmd string) (output string, err error) {
+	flog := log.With().
+		Str("kernel", q.kernel.KernelPath).
+		Str("user", user).
+		Str("cmd", cmd).
+		Logger()
+
+	flog.Debug().Msg("qemu command")
+
 	client, err := q.ssh(user)
 	if err != nil {
 		return
@@ -324,8 +374,43 @@ func (q System) Command(user, cmd string) (output string, err error) {
 		return
 	}
 
-	bytesOutput, err := session.CombinedOutput(cmd)
-	output = string(bytesOutput)
+	stdout, err := session.StdoutPipe()
+	if err != nil {
+		return
+	}
+
+	stderr, err := session.StderrPipe()
+	if err != nil {
+		return
+	}
+
+	err = session.Start(cmd)
+	if err != nil {
+		return
+	}
+
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			m := scanner.Text()
+			output += m + "\n"
+			flog.Trace().Str("stdout", m).Msg("")
+		}
+		output = strings.TrimSuffix(output, "\n")
+	}()
+
+	go func() {
+		scanner := bufio.NewScanner(stderr)
+		for scanner.Scan() {
+			m := scanner.Text()
+			output += m + "\n"
+			// Note: it prints stderr as stdout
+			flog.Trace().Str("stdout", m).Msg("")
+		}
+		output = strings.TrimSuffix(output, "\n")
+	}()
+
+	err = session.Wait()
 	return
 }
 
@@ -346,24 +431,70 @@ func (q System) AsyncCommand(user, cmd string) (err error) {
 		"nohup sh -c '%s' > /dev/null 2> /dev/null < /dev/null &", cmd))
 }
 
-// CopyFile is copy file from local machine to remote through ssh/scp
-func (q *System) CopyFile(user, localPath, remotePath string) (err error) {
+func (q System) scp(user, localPath, remotePath string, recursive bool) (err error) {
 	addrPort := strings.Split(q.sshAddrPort, ":")
 	addr := addrPort[0]
 	port := addrPort[1]
 
-	cmd := exec.Command("scp", "-P", port,
+	args := []string{
+		"-P", port,
 		"-o", "StrictHostKeyChecking=no",
+		"-o", "UserKnownHostsFile=/dev/null",
 		"-o", "LogLevel=error",
-		localPath, user+"@"+addr+":"+remotePath)
+	}
+
+	if recursive {
+		cmd := exec.Command("ssh", "-V")
+
+		log.Debug().Msgf("%v", cmd)
+
+		var output []byte
+		output, err = cmd.CombinedOutput()
+		if err != nil {
+			return
+		}
+		sshVersion := string(output)
+
+		log.Debug().Str("ssh version", sshVersion).Msg("")
+
+		if strings.Contains(sshVersion, "OpenSSH_9") {
+			// This release switches scp from using the
+			// legacy scp/rcp protocol to using the SFTP
+			// protocol by default.
+			//
+			// To keep compatibility with old distros,
+			// using -O flag to use the legacy scp/rcp.
+			//
+			// Note: old ssh doesn't support -O flag
+			args = append(args, "-O")
+		}
+
+		args = append(args, "-r")
+	}
+
+	args = append(args, localPath, user+"@"+addr+":"+remotePath)
+
+	cmd := exec.Command("scp", args...)
+	log.Debug().Msgf("%v", cmd)
+
 	output, err := cmd.CombinedOutput()
-	if err != nil {
+	if err != nil || string(output) != "" {
 		return errors.New(string(output))
 	}
 
 	return
 }
 
+// CopyFile from local machine to remote via scp
+func (q System) CopyFile(user, localPath, remotePath string) (err error) {
+	return q.scp(user, localPath, remotePath, false)
+}
+
+// CopyDirectory from local machine to remote via scp
+func (q System) CopyDirectory(user, localPath, remotePath string) (err error) {
+	return q.scp(user, localPath, remotePath, true)
+}
+
 // CopyAndInsmod copy kernel module to temporary file on qemu then insmod it
 func (q *System) CopyAndInsmod(localKoPath string) (output string, err error) {
 	remoteKoPath := fmt.Sprintf("/tmp/module_%d.ko", rand.Int())

qemu/qemu-kernel_test.go

@@ -116,6 +116,7 @@ func startTestQemu(t *testing.T, timeout time.Duration) (q *System, err error) {
 		return
 	}
 
+	time.Sleep(time.Second)
 	return
 }
 
@@ -324,6 +325,8 @@ func TestSystemDebug(t *testing.T) {
 		return
 	}
 
+	time.Sleep(time.Second)
+
 	port := 45256
 
 	q.Debug(fmt.Sprintf("tcp::%d", port))

qemu/test.config.go

@@ -4,7 +4,7 @@
 
 package qemu
 
-const testConfigVmlinuz = "../tools/qemu-debian-img/ubuntu1804.vmlinuz"
-const testConfigInitrd = "../tools/qemu-debian-img/ubuntu1804.initrd"
-const testConfigRootfs = "../tools/qemu-debian-img/ubuntu1804.img"
-const testConfigSampleKo = "../tools/qemu-debian-img/ubuntu1804.ko"
+const testConfigVmlinuz = "../tools/qemu-ubuntu-img/ubuntu2204.vmlinuz"
+const testConfigInitrd = "../tools/qemu-ubuntu-img/ubuntu2204.initrd"
+const testConfigRootfs = "../tools/qemu-ubuntu-img/ubuntu2204.img"
+const testConfigSampleKo = "../tools/qemu-ubuntu-img/ubuntu2204.ko"

shell.nix

@@ -0,0 +1,5 @@
+{ pkgs ? import <nixpkgs> {} }:
+
+with pkgs; mkShell {
+  packages = [ go gcc qemu ];
+}

tools/qemu-centos-img/7/Dockerfile

@@ -47,7 +47,7 @@ ENV IMAGE=/shared/out_of_tree_centos_7.img
 
 RUN mkdir $IMAGEDIR
 
-# Must be runned with --privileged because of /dev/loop
+# Must be executed with --privileged because of /dev/loop
 CMD qemu-img create $IMAGE 2G && \
 	mkfs.ext4 -F $IMAGE && \
 	mount -o loop $IMAGE $IMAGEDIR && \

tools/qemu-centos-img/8/Dockerfile

@@ -0,0 +1,56 @@
+# Copyright 2020 Mikhail Klementev. All rights reserved.
+# Use of this source code is governed by a AGPLv3 license
+# (or later) that can be found in the LICENSE file.
+#
+# Usage:
+#
+#     $ sudo docker build -t gen-centos8-image .
+#     $ sudo docker run --privileged -v $(pwd):/shared -t gen-centos8-image
+#     $ tar -Szcf out_of_tree_centos_8.img.tar.gz out_of_tree_centos_8.img
+#
+# out_of_tree_centos_8.img will be created in current directory.
+# You can change $(pwd) to different directory to use different destination
+# for image.
+#
+FROM centos:8
+
+RUN yum -y update
+RUN yum -y groupinstall "Development Tools"
+RUN yum -y install qemu-img e2fsprogs
+
+ENV TMPDIR=/tmp/centos
+
+RUN yum --installroot=$TMPDIR \
+        --releasever=8 \
+        --disablerepo='*' \
+        --enablerepo=BaseOS \
+        -y groupinstall Base
+RUN yum --installroot=$TMPDIR \
+        --releasever=8 \
+        --disablerepo='*' \
+        --enablerepo=BaseOS \
+        -y install openssh-server openssh-clients
+
+RUN chroot $TMPDIR /bin/sh -c 'useradd -m user'
+RUN sed -i 's/root:\*:/root::/' $TMPDIR/etc/shadow
+RUN sed -i 's/user:!!:/user::/' $TMPDIR/etc/shadow
+RUN sed -i '/PermitEmptyPasswords/d' $TMPDIR/etc/ssh/sshd_config
+RUN echo PermitEmptyPasswords yes >> $TMPDIR/etc/ssh/sshd_config
+RUN sed -i '/PermitRootLogin/d' $TMPDIR/etc/ssh/sshd_config
+RUN echo PermitRootLogin yes >> $TMPDIR/etc/ssh/sshd_config
+
+# network workaround
+RUN chmod +x $TMPDIR/etc/rc.local
+RUN echo 'dhclient' >> $TMPDIR/etc/rc.local
+
+ENV IMAGEDIR=/tmp/image
+ENV IMAGE=/shared/out_of_tree_centos_8.img
+
+RUN mkdir $IMAGEDIR
+
+# Must be executed with --privileged because of /dev/loop
+CMD qemu-img create $IMAGE 2G && \
+	mkfs.ext4 -F $IMAGE && \
+	mount -o loop $IMAGE $IMAGEDIR && \
+	cp -a $TMPDIR/* $IMAGEDIR/ && \
+	umount $IMAGEDIR

tools/qemu-centos-img/8/generate.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+cd "$(dirname "$0")"
+
+sudo docker build -t gen-centos8-image .
+sudo docker run --privileged -v $(pwd):/shared -t gen-centos8-image
+tar -Szcf out_of_tree_centos_8.img.tar.gz out_of_tree_centos_8.img

tools/qemu-debian-img/bootstrap.sh

@@ -1,9 +0,0 @@
-#!/bin/sh -eux
-cd $(dirname $(realpath $0))
-
-docker build -t gen-ubuntu1804-image .
-docker run --privileged -v $(pwd):/shared -t gen-ubuntu1804-image
-RUN="docker run -v $(pwd):/shared -t gen-ubuntu1804-image"
-$RUN sh -c 'chmod 644 /vmlinuz && cp /vmlinuz /shared/ubuntu1804.vmlinuz'
-$RUN sh -c 'cp /initrd.img /shared/ubuntu1804.initrd'
-$RUN sh -c 'cp $(find /lib/modules -name test_static_key_base.ko) /shared/ubuntu1804.ko'

tools/qemu-ubuntu-img/14.04/Dockerfile

@@ -4,29 +4,29 @@
 #
 # Usage:
 #
-#     $ docker build -t gen-ubuntu1804-image .
-#     $ docker run --privileged -v $(pwd):/shared -t gen-ubuntu1804-image
+#     $ docker build -t gen-ubuntu1404-image .
+#     $ docker run --privileged -v $(pwd):/shared -t gen-ubuntu1404-image
 #
-# ubuntu1804.img will be created in current directory. You can change $(pwd) to
+# ubuntu1404.img will be created in current directory. You can change $(pwd) to
 # different directory to use different destination for image.
 #
-FROM ubuntu:18.04
+FROM ubuntu:14.04
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt update
-RUN apt install -y debootstrap qemu
-RUN apt install -y linux-image-generic
+RUN apt-get update
+RUN apt-get install -y debootstrap qemu
 
 ENV TMPDIR=/tmp/ubuntu
 ENV IMAGEDIR=/tmp/image
-ENV IMAGE=/shared/ubuntu1804.img
+ENV IMAGE=/shared/out_of_tree_ubuntu_14__04.img
 ENV REPOSITORY=http://archive.ubuntu.com/ubuntu
-ENV RELEASE=bionic
+ENV RELEASE=trusty
 
 RUN mkdir $IMAGEDIR
 
-# Must be runned with --privileged because of /dev/loop
-CMD debootstrap --include=openssh-server $RELEASE $TMPDIR $REPOSITORY && \
+# Must be executed with --privileged because of /dev/loop
+CMD debootstrap --include=openssh-server,policykit-1 \
+	$RELEASE $TMPDIR $REPOSITORY && \
 	/shared/setup.sh $TMPDIR && \
 	qemu-img create $IMAGE 2G && \
 	mkfs.ext4 -F $IMAGE && \

tools/qemu-ubuntu-img/14.04/setup.sh

@@ -0,0 +1,17 @@
+#!/bin/sh -eux
+# Copyright 2018 Mikhail Klementev. All rights reserved.
+# Use of this source code is governed by a AGPLv3 license
+# (or later) that can be found in the LICENSE file.
+TMPDIR=$1
+chroot $TMPDIR /bin/sh -c 'useradd -m user'
+sed -i 's/root:\*:/root::/' $TMPDIR/etc/shadow
+sed -i 's/user:!!:/user::/' $TMPDIR/etc/shadow
+echo auth sufficient pam_permit.so > $TMPDIR/etc/pam.d/sshd
+sed -i '/PermitEmptyPasswords/d' $TMPDIR/etc/ssh/sshd_config
+echo PermitEmptyPasswords yes >> $TMPDIR/etc/ssh/sshd_config
+sed -i '/PermitRootLogin/d' $TMPDIR/etc/ssh/sshd_config
+echo PermitRootLogin yes >> $TMPDIR/etc/ssh/sshd_config
+
+echo '#!/bin/sh' > $TMPDIR/etc/rc.local
+echo 'dhclient eth0' >> $TMPDIR/etc/rc.local
+chmod +x $TMPDIR/etc/rc.local

tools/qemu-ubuntu-img/Dockerfile

@@ -0,0 +1,36 @@
+# Copyright 2018 Mikhail Klementev. All rights reserved.
+# Use of this source code is governed by a AGPLv3 license
+# (or later) that can be found in the LICENSE file.
+#
+# Usage:
+#
+#     $ docker build -t gen-ubuntu2204-image .
+#     $ docker run --privileged -v $(pwd):/shared -t gen-ubuntu2204-image
+#
+# ubuntu2204.img will be created in current directory. You can change $(pwd) to
+# different directory to use different destination for image.
+#
+FROM ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt update
+RUN apt install -y debootstrap qemu-utils
+RUN apt install -y linux-image-generic
+
+ENV TMPDIR=/tmp/ubuntu
+ENV IMAGEDIR=/tmp/image
+ENV IMAGE=/shared/ubuntu2204.img
+ENV REPOSITORY=http://archive.ubuntu.com/ubuntu
+ENV RELEASE=jammy
+
+RUN mkdir $IMAGEDIR
+
+# Must be executed with --privileged because of /dev/loop
+CMD debootstrap --include=openssh-server,policykit-1 \
+	$RELEASE $TMPDIR $REPOSITORY && \
+	/shared/setup.sh $TMPDIR && \
+	qemu-img create $IMAGE 2G && \
+	mkfs.ext4 -F $IMAGE && \
+	mount -o loop $IMAGE $IMAGEDIR && \
+	cp -a $TMPDIR/* $IMAGEDIR/ && \
+	umount $IMAGEDIR

tools/qemu-ubuntu-img/bootstrap.sh

@@ -0,0 +1,9 @@
+#!/bin/sh -eux
+cd $(dirname $(realpath $0))
+
+docker build -t gen-ubuntu2204-image .
+docker run --privileged -v $(pwd):/shared -t gen-ubuntu2204-image
+RUN="docker run -v $(pwd):/shared -t gen-ubuntu2204-image"
+$RUN sh -c 'chmod 644 /boot/vmlinuz && cp /boot/vmlinuz /shared/ubuntu2204.vmlinuz'
+$RUN sh -c 'cp /boot/initrd.img /shared/ubuntu2204.initrd'
+$RUN sh -c 'cp $(find /lib/modules -name test_bpf.ko) /shared/ubuntu2204.ko'