Update changelog

This reverts commit 5931c08de1.

.github/workflows/macos.yml

@@ -0,0 +1,13 @@
+name: macOS
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: Build on macOS
+    runs-on: macOS-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build

.github/workflows/ubuntu.yml

@@ -0,0 +1,130 @@
+name: Ubuntu
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: Build on Ubuntu
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build
+
+  test-unit:
+    name: Unit Testing
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Install dependencies for tests
+      run: |
+        sudo apt-get update
+        sudo apt-get install qemu-system-x86
+
+    - name: Bootstrap
+      run: ./tools/qemu-debian-img/bootstrap.sh
+
+    - name: Unit Testing
+      run: go test -parallel 1 -v ./...
+
+  test-end-to-end:
+    name: End-to-End Testing
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build
+
+    - name: Install dependencies for tests
+      run: |
+        sudo apt-get update
+        sudo apt-get install qemu-system-x86
+
+    - name: End-to-End Testing [Kernel Module]
+      run: |
+        cd examples/kernel-module
+        ../../out-of-tree --log-level=debug kernel autogen --max=1
+        ../../out-of-tree --log-level=debug pew --qemu-timeout=10m
+
+    - name: End-to-End Testing [Kernel Exploit]
+      run: |
+        cd examples/kernel-exploit
+        ../../out-of-tree --log-level=debug kernel autogen --max=1
+        ../../out-of-tree --log-level=debug pew --threshold=0 --qemu-timeout=10m
+
+    - name: Archive logs
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-end-to-end-logs
+        path: /home/runner/.out-of-tree/logs/out-of-tree.log
+
+  test-end-to-end-kernels:
+    name: End-to-End Testing (kernels)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build
+
+    - name: Install dependencies for tests
+      run: |
+        sudo apt-get update
+        sudo apt-get install qemu-system-x86
+
+    - name: End-to-End Testing [Install one Ubuntu 18.04 kernel]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=Ubuntu --ver=18.04 --kernel=4.15.0-70-generic
+
+    - name: End-to-End Testing [Reinstall one Ubuntu 18.04 kernel]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=Ubuntu --ver=18.04 --kernel=4.15.0-70-generic --force
+
+    - name: End-to-End Testing [Install one Ubuntu 22.04 kernel w/o headers]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=Ubuntu --ver=22.04 --kernel=5.19.0-28-generic --no-headers
+
+    - name: End-to-End Testing [Install one CentOS 7 kernel]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=CentOS --ver=7 --kernel=3.10.0-862.6.3
+
+    - name: End-to-End Testing [Install one CentOS 7 kernel w/o headers]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=CentOS --ver=7 --kernel=3.10.0-1160.71.1 --no-headers
+
+    - name: End-to-End Testing [Install one CentOS 8 kernel]
+      run: |
+        ./out-of-tree --log-level=debug kernel install --distro=CentOS --ver=8 --kernel=4.18.0-348.7.1
+
+    - name: Archive logs
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-end-to-end-kernels-log
+        path: /home/runner/.out-of-tree/logs/out-of-tree.log
+
+  test-end-to-end-genall:
+    name: End-to-End Testing (genall)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build
+
+    - name: Install dependencies for tests
+      run: |
+        sudo apt-get update
+        sudo apt-get install qemu-system-x86
+
+    - name: End-to-End Testing [Install all Ubuntu 22.04 kernels]
+      run: |
+        ./out-of-tree --log-level=debug kernel genall --distro=Ubuntu --ver=22.04
+
+    - name: Archive logs
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-end-to-end-genall-logs
+        path: /home/runner/.out-of-tree/logs/out-of-tree.log

.gitignore

@@ -10,3 +10,5 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+
+out-of-tree

CHANGELOG.md

@@ -0,0 +1,286 @@
+# Changelog
+
+[ISO 8601](https://xkcd.com/1179/).
+
+[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [2.1.0]
+
+### Added
+
+- Graceful shutdown on ^C while kernels generation.
+
+- Flag to set the container runtime command.
+
+- out-of-tree image --dry-run for printing full qemu command.
+
+### Changed
+
+- No exit at the end of the retries, will continue with the other
+  kernels.
+
+- All temporary files moved to ~/.out-of-tree/tmp/.
+
+### Fixed
+
+- Discrepancies between /lib/modules and /boot should no longer lead
+  to fatal errors.
+
+- Podman support on macOS.
+
+## [2.0.0]
+
+### Breaking
+
+- Layers with kernels in containers have been abandoned in favor of
+  installation to mounted volumes.
+
+- Command line interface has been changed to alecthomas/kong.
+
+### Added
+
+- Command `kernel install` to install specific kernel.
+
+- Command `containers` to manage containers.
+
+- Command `image edit` to edit qemu image.
+
+- Flag `--force` to force reinstallation of the kernel.
+
+- Flag `--artifact-config` to specify the path to .out-of-tree.toml.
+
+- Flag `--no-headers` flag to install kernel and initrd only.
+
+- Flag `--shuffle` to randomize the order of kernels for
+  installation/testing.
+
+- Support make targets in artifact config.
+
+- Support patches in artifact config.
+
+- Support for copying standard modules to qemu.
+
+- Script artifact type for various automation and information gathering.
+
+- Add TestFiles to artifact config, transfers additional test files to VM.
+
+- Improved logging, with logfile at ~/.out-of-tree/logs/out-of-tree.log
+
+- Kernel installation will retry (10 times by default) in case of
+  network problems.
+
+- Stdout trace (with --log-level=trace, and always to logfile) for
+  qemu and container execution.
+
+- Compatibility with Podman.
+
+- Support for Ubuntu 22.04.
+
+## [1.4.0]
+
+### Added
+
+- Parameter `--docker-timeout` may also be set in the artifact
+  configuration file.
+
+- Preload modules before inserting module or run exploit. Modules can
+  be specified by git repository path in the `repo` parameter of
+  section `[[preload]]`. Also, there is a `path` parameter for local
+  projects. Note that `repo` is using a cache that uses last commit
+  hash to check is project needs to be rebuilt, so it's not suitable
+  for local development (except if you will commit each time before
+  run out-of-tree).
+
+- Flag `--disable-preload` to ignore `[[preload]]` section of
+  configuration file.
+
+- Now `out-of-tree log dump` will show the last log if no ID
+  specified.
+
+## [1.3.0] 2020-05-30
+
+### Added
+
+- Support for Ubuntu 20.04 and CentOS 8.
+
+## [1.2.1] 2019-12-25
+
+### Fixed
+
+- macOS support.
+
+## [1.2.0] 2019-11-15
+
+### Added
+
+- Flag for Verbose output. Right now only qemu status messages is
+  implemented.
+
+### Fixed
+
+- Kpti settings was not affected for regular runs.
+
+## [1.1.2] 2019-09-05
+
+### Added
+
+- Added policykit-1 to rootfs for Ubuntu.
+
+### Fixed
+
+- Avoided slow mirrors with use of mirror://mirrors.ubuntu.com for
+  Ubuntu 16.04 and newer.
+
+## [1.1.1] 2019-08-31
+
+### Fixed
+
+- macOS support.
+
+## [1.1.0] 2019-08-30
+
+### Added
+
+- Global configuration file (~/.out-of-tree/out-of-tree.toml) allow to
+  set up default values for settings.
+
+- rootfs generator for Ubuntu 14.04.
+
+- Parameter for setting up docker registry server.
+
+- Support for (distro-specific) custom docker commands that will be
+  executed before the base template.
+
+- Parameter for setting up a reliability threshold for exit code.
+
+- Parameter for setting up global timeout, after which no new tasks
+  will be started.
+
+### Fixed
+
+- Spelling in output.
+
+- Now kernel generation will not fail if there are no directory
+  /lib/modules inside the container.
+
+## [1.0.0] 2019-08-20
+
+### Added
+
+- New parameter `--max=X` is added for `autogen` (generate kernels
+  base on `.out-of-tree.toml` definitions) and `pew` (automated
+  runs) and allows to specify a maximum number of runs per each
+  supported kernel in module/exploit definition.
+
+- New command `genall` -- generate all kernels for specified
+  distro/version.
+
+- All logs stores in sqlite3 database. Implemented specific commands
+  for making simple queries and export data to markdown and json.
+
+- Implemented success rate calculation for previous runs.
+
+- Save of build results supported by parameter `--dist` for `pew`.
+
+- Support for generating kernels info from host system.
+
+- Support for build on host.
+
+- Support for custom kernels.
+
+- Now debugging environment is automatically looking for debug
+  kernel on the host system.
+
+- Added ability to enable/disable kaslr/smep/smap/kpti for debugging
+  by command line flags.
+
+- New parameter `--threads=N` is added for `pew` and allows to
+  specify maximum number of threads that will be used for parallel
+  build/run/test.
+
+- Tagging for runs. Tags write to log and can be used for
+  statistics.
+
+- Added non-regex way to set kernel version in .out-of-tree.toml (see
+  examples).
+
+- New command `pack` that perform tests in subdirectories.
+
+- Added ability to disable kaslr/smep/smap/kpti for in artifact
+  definition.
+
+- Added ability to change amount of memory/CPUs and set qemu timeout
+  in artifact definition (`.out-of-tree.toml`).
+
+- Now images downloading while `kernel autogen`, bootstrap is not
+  required anymore.
+
+- Support CentOS kernels.
+
+### Changed
+
+- Now if there's no base image found — out-of-tree will try to use
+  an image from closest previous version, e.g. image from Ubuntu
+  18.04 for Ubuntu 18.10.
+
+- Kernel modules tests will not be failed if there are no tests
+  exists.
+
+- Now *out-of-tree* will return negative error code if at least one
+  of the stage was failed.
+
+- Project is switch to use Go modules.
+
+- Now test.sh is used by default if copying is not implemented in
+  Makefile.
+
+- dmesg is not cleaned before the start of module/exploit anymore.
+
+- qemu/kvm will use all host cpu features.
+
+### Removed
+
+- *Kernel factory* is removed completely in favor of incremental
+  Dockerfiles.
+
+- `bootstrap` is not doing anything anymore. It'll be removed in next
+  release.
+
+### Fixed
+
+- Command `timeout` is not required anymore.
+
+- Errors is more meaningful.
+
+- Temporary files is moved to `~/.out-of-tree/tmp/` to avoid docker
+  mounting issues on some systems.
+
+## [0.2.0] - 2018-12-01
+
+The main purpose of the release is to simplify installation.
+
+### Changes
+
+- All configuration moved to `~/.out-of-tree`.
+
+- Now prebuilt images can be downloaded with bootstrap.
+
+- Ability to generate kernels specific to .out-of-tree.toml in
+  current directory. So now there's no need to wait for several
+  hours for start work on specific kernel with module/exploit.
+
+- Now there's no need to keep source tree and _out-of-tree_ can be
+  distributed in binary form.
+
+- New command: **debug**. Creates interactive environment for kernel
+  module/exploit development. Still work-in-progress.
+
+- No warning anymore if test.sh is not exists.
+
+## [0.1.0] - 2018-11-20
+
+Initial release that was never tagged.
+
+Refer to state after first public release on ZeroNights 2018
+([video](https://youtu.be/2tL7bbCdIio),
+[slides](https://2018.zeronights.ru/wp-content/uploads/materials/07-Ways-to-automate-testing-Linux-kernel-exploits.pdf)).

README.md

@@ -1,64 +1,54 @@
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/aba4aad2046b4d1a9a99cf98e22c018b)](https://app.codacy.com/app/jollheef/out-of-tree?utm_source=github.com&utm_medium=referral&utm_content=jollheef/out-of-tree&utm_campaign=Badge_Grade_Dashboard)
+[![Go Report Card](https://goreportcard.com/badge/code.dumpstack.io/tools/out-of-tree)](https://goreportcard.com/report/code.dumpstack.io/tools/out-of-tree)
+[![Documentation Status](https://readthedocs.org/projects/out-of-tree/badge/?version=latest)](https://out-of-tree.readthedocs.io/en/latest/?badge=latest)
+
 # [out-of-tree](https://out-of-tree.io)
 
 out-of-tree kernel {module, exploit} development tool
 
+out-of-tree is for automating some routine actions for creating development environments for debugging kernel modules and exploits, generating reliability statistics for exploits, and also provides the ability to easily integrate into CI (Continuous Integration).
+
 ![Screenshot](https://cloudflare-ipfs.com/ipfs/Qmb88fgdDjbWkxz91sWsgmoZZNfVThnCtj37u3mF2s3T3T)
 
 ## Installation
 
-    $ go get github.com/jollheef/out-of-tree
-    $ out-of-tree bootstrap
+### GNU/Linux (with [Nix](https://nixos.org/nix/))
 
-Then you can check it on kernel module example:
+    $ curl -fsSL https://get.docker.com | sh
+	$ sudo usermod -aG docker user && newgrp docker
+    $ curl -L https://nixos.org/nix/install | sh
+    $ nix-env -iA nixpkgs.out-of-tree # Note: may not be up to date immediately, in this case consider installing from source
 
-    $ cd $GOPATH/github.com/jollheef/out-of-tree/examples/kernel-module
-    $ out-of-tree kernel autogen # generate kernels based on .out-of-tree.toml
-    $ out-of-tree pew
+Note that adding a user to group *docker* has serious security implications. Check Docker documentation for more information.
+
+### macOS
+
+Note: case-sensitive FS is required for the ~/.out-of-tree directory.
+
+    $ brew install podman
+    $ podman machine stop || true
+    $ podman machine rm || true
+    $ podman machine init --cpus=4 --memory=4096 -v $HOME:$HOME
+    $ podman machine start
+    $ brew tap out-of-tree/repo
+    $ brew install out-of-tree
+
+Read [documentation](https://out-of-tree.readthedocs.io) for further info.
 
 ## Examples
 
-Run by absolute path
+Generate all Ubuntu 22.04 kernels:
 
-    $ out-of-tree --path /path/to/exploit/directory pew
+    $ out-of-tree kernel genall --distro=Ubuntu --ver=22.04
 
-Test only with one kernel:
+Run tests based on .out-of-tree.toml definitions:
 
-    $ out-of-tree pew --kernel='Ubuntu:4.10.0-30-generic'
+	$ out-of-tree pew
+
+Test with a specific kernel:
+
+    $ out-of-tree pew --kernel='Ubuntu:5.4.0-29-generic'
 
 Run debug environment:
 
-    $ out-of-tree debug --kernel='Ubuntu:4.10.0-30-generic'
-
-Test binary module/exploit with implicit defined test ($BINARY_test)
-
-    $ out-of-tree pew --binary /path/to/exploit
-
-Test binary module/exploit with explicit defined test
-
-    $ out-of-tree pew --binary /path/to/exploit --test /path/to/exploit_test
-
-Guess work kernels:
-
-    $ out-of-tree pew --guess
-
-Use custom kernels config
-
-    $ out-of-tree --kernels /path/to/kernels.toml pew
-
-## Generate all kernels
-
-Does not required if you dont need to use `--guess`.
-
-    $ cd $GOPATH/src/github.com/jollheef/out-of-tree/tools/kernel-factory
-    $ ./bootstrap.sh # more than 6-8 hours for all kernels
-    $ export OUT_OF_TREE_KCFG=$GOPATH/src/github.com/jollheef/out-of-tree/tools/kernel-factory/output/kernels.toml
-
-## Development
-
-Read [Qemu API](qemu/README.md).
-
-### Generate images
-
-    $ cd $GOPATH/src/github.com/jollheef/out-of-tree/tools/qemu-debian-img/
-    $ docker run --privileged -v $(pwd):/shared -e IMAGE=/shared/ubuntu1404.img -e RELEASE=trusty -t gen-ubuntu1804-image
-    $ docker run --privileged -v $(pwd):/shared -e IMAGE=/shared/ubuntu1604.img -e RELEASE=xenial -t gen-ubuntu1804-image
+    $ out-of-tree debug --kernel='Ubuntu:5.4.0-29-generic'

bootstrap.config.go

@@ -1,7 +0,0 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
-// Use of this source code is governed by a AGPLv3 license
-// (or later) that can be found in the LICENSE file.
-
-package main
-
-const imagesURL = "https://github.com/jollheef/out-of-tree/releases/download/v0.2/images.tar.gz"

bootstrap.go

@@ -1,76 +0,0 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
-// Use of this source code is governed by a AGPLv3 license
-// (or later) that can be found in the LICENSE file.
-
-package main
-
-import (
-	"io"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"os/exec"
-	"os/user"
-)
-
-// inspired by Edd Turtle code
-func downloadFile(filepath string, url string) (err error) {
-	out, err := os.Create(filepath)
-	if err != nil {
-		return
-	}
-	defer out.Close()
-
-	resp, err := http.Get(url)
-	if err != nil {
-		return
-	}
-	defer resp.Body.Close()
-
-	_, err = io.Copy(out, resp.Body)
-	return
-}
-
-func unpackTar(archive, destination string) (err error) {
-	cmd := exec.Command("tar", "xf", archive)
-	cmd.Dir = destination + "/"
-
-	rawOutput, err := cmd.CombinedOutput()
-	if err != nil {
-		// I don't like when some errors printed inside
-		// So if you know way to do it better - FIXME please
-		log.Println("Unpack images error:", string(rawOutput), err)
-		return
-	}
-
-	return
-}
-
-func bootstrapHandler() (err error) {
-	usr, err := user.Current()
-	if err != nil {
-		return
-	}
-
-	imagesPath := usr.HomeDir + "/.out-of-tree/images/"
-	os.MkdirAll(imagesPath, os.ModePerm)
-
-	tmp, err := ioutil.TempDir("/tmp/", "out-of-tree_")
-	if err != nil {
-		log.Println("Temporary directory creation error:", err)
-		return
-	}
-	defer os.RemoveAll(tmp)
-
-	imagesArchive := tmp + "/images.tar.gz"
-
-	err = downloadFile(imagesArchive, imagesURL)
-	if err != nil {
-		log.Println("Download file error:", err)
-		return
-	}
-
-	err = unpackTar(imagesArchive, imagesPath)
-	return
-}

config/config.go

@@ -10,34 +10,54 @@ import (
 	"io/ioutil"
 	"os"
 	"regexp"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/naoina/toml"
 )
 
+type kernel struct {
+	Version []int
+	Major   []int
+	Minor   []int
+	Patch   []int
+}
+
+// KernelMask defines the kernel
 type KernelMask struct {
 	DistroType    DistroType
 	DistroRelease string // 18.04/7.4.1708/9.1
 	ReleaseMask   string
+
+	// Overrides ReleaseMask
+	Kernel kernel
 }
 
+// DockerName is returns stable name for docker container
 func (km KernelMask) DockerName() string {
 	distro := strings.ToLower(km.DistroType.String())
 	release := strings.Replace(km.DistroRelease, ".", "__", -1)
 	return fmt.Sprintf("out_of_tree_%s_%s", distro, release)
 }
 
+// ArtifactType is the kernel module or exploit
 type ArtifactType int
 
 const (
+	// KernelModule is any kind of kernel module
 	KernelModule ArtifactType = iota
+	// KernelExploit is the privilege escalation exploit
 	KernelExploit
+	// Script for information gathering or automation
+	Script
 )
 
 func (at ArtifactType) String() string {
-	return [...]string{"module", "exploit"}[at]
+	return [...]string{"module", "exploit", "script"}[at]
 }
 
+// UnmarshalTOML is for support github.com/naoina/toml
 func (at *ArtifactType) UnmarshalTOML(data []byte) (err error) {
 	stype := strings.Trim(string(data), `"`)
 	stypelower := strings.ToLower(stype)
@@ -45,12 +65,15 @@ func (at *ArtifactType) UnmarshalTOML(data []byte) (err error) {
 		*at = KernelModule
 	} else if strings.Contains(stypelower, "exploit") {
 		*at = KernelExploit
+	} else if strings.Contains(stypelower, "script") {
+		*at = Script
 	} else {
-		err = errors.New(fmt.Sprintf("Type %s is unsupported", stype))
+		err = fmt.Errorf("Type %s is unsupported", stype)
 	}
 	return
 }
 
+// MarshalTOML is for support github.com/naoina/toml
 func (at ArtifactType) MarshalTOML() (data []byte, err error) {
 	s := ""
 	switch at {
@@ -58,18 +81,88 @@ func (at ArtifactType) MarshalTOML() (data []byte, err error) {
 		s = "module"
 	case KernelExploit:
 		s = "exploit"
+	case Script:
+		s = "script"
 	default:
-		err = errors.New(fmt.Sprintf("Cannot marshal %d", at))
+		err = fmt.Errorf("Cannot marshal %d", at)
 	}
 	data = []byte(`"` + s + `"`)
 	return
 }
 
+// Duration type with toml unmarshalling support
+type Duration struct {
+	time.Duration
+}
+
+// UnmarshalTOML for Duration
+func (d *Duration) UnmarshalTOML(data []byte) (err error) {
+	duration := strings.Replace(string(data), "\"", "", -1)
+	d.Duration, err = time.ParseDuration(duration)
+	return
+}
+
+// MarshalTOML for Duration
+func (d Duration) MarshalTOML() (data []byte, err error) {
+	data = []byte(`"` + d.Duration.String() + `"`)
+	return
+}
+
+type PreloadModule struct {
+	Repo             string
+	Path             string
+	TimeoutAfterLoad Duration
+}
+
+// Extra test files to copy over
+type FileTransfer struct {
+	User   string
+	Local  string
+	Remote string
+}
+
+type Patch struct {
+	Path   string
+	Source string
+	Script string
+}
+
+// Artifact is for .out-of-tree.toml
 type Artifact struct {
 	Name             string
 	Type             ArtifactType
+	TestFiles        []FileTransfer
 	SourcePath       string
 	SupportedKernels []KernelMask
+
+	Script string
+
+	Qemu struct {
+		Cpus    int
+		Memory  int
+		Timeout Duration
+	}
+
+	Docker struct {
+		Timeout Duration
+	}
+
+	Mitigations struct {
+		DisableSmep  bool
+		DisableSmap  bool
+		DisableKaslr bool
+		DisableKpti  bool
+	}
+
+	Patches []Patch
+
+	Make struct {
+		Target string
+	}
+
+	StandardModules bool
+
+	Preload []PreloadModule
 }
 
 func (ka Artifact) checkSupport(ki KernelInfo, km KernelMask) (
@@ -90,6 +183,7 @@ func (ka Artifact) checkSupport(ki KernelInfo, km KernelMask) (
 	return
 }
 
+// Supported returns true if given kernel is supported by artifact
 func (ka Artifact) Supported(ki KernelInfo) (supported bool, err error) {
 	for _, km := range ka.SupportedKernels {
 		supported, err = ka.checkSupport(ki, km)
@@ -101,16 +195,22 @@ func (ka Artifact) Supported(ki KernelInfo) (supported bool, err error) {
 	return
 }
 
+// DistroType is enum with all supported distros
 type DistroType int
 
 const (
+	// Ubuntu https://ubuntu.com/
 	Ubuntu DistroType = iota
+	// CentOS https://www.centos.org/
 	CentOS
+	// Debian https://www.debian.org/
 	Debian
 )
 
+// DistroTypeStrings is the string version of enum DistroType
 var DistroTypeStrings = [...]string{"Ubuntu", "CentOS", "Debian"}
 
+// NewDistroType is create new Distro object
 func NewDistroType(dType string) (dt DistroType, err error) {
 	err = dt.UnmarshalTOML([]byte(dType))
 	return
@@ -120,6 +220,7 @@ func (dt DistroType) String() string {
 	return DistroTypeStrings[dt]
 }
 
+// UnmarshalTOML is for support github.com/naoina/toml
 func (dt *DistroType) UnmarshalTOML(data []byte) (err error) {
 	sDistro := strings.Trim(string(data), `"`)
 	if strings.EqualFold(sDistro, "Ubuntu") {
@@ -129,11 +230,12 @@ func (dt *DistroType) UnmarshalTOML(data []byte) (err error) {
 	} else if strings.EqualFold(sDistro, "Debian") {
 		*dt = Debian
 	} else {
-		err = errors.New(fmt.Sprintf("Distro %s is unsupported", sDistro))
+		err = fmt.Errorf("Distro %s is unsupported", sDistro)
 	}
 	return
 }
 
+// MarshalTOML is for support github.com/naoina/toml
 func (dt DistroType) MarshalTOML() (data []byte, err error) {
 	s := ""
 	switch dt {
@@ -144,12 +246,20 @@ func (dt DistroType) MarshalTOML() (data []byte, err error) {
 	case Debian:
 		s = "Debian"
 	default:
-		err = errors.New(fmt.Sprintf("Cannot marshal %d", dt))
+		err = fmt.Errorf("Cannot marshal %d", dt)
 	}
 	data = []byte(`"` + s + `"`)
 	return
 }
 
+// ByRootFS is sorting by .RootFS lexicographically
+type ByRootFS []KernelInfo
+
+func (a ByRootFS) Len() int           { return len(a) }
+func (a ByRootFS) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a ByRootFS) Less(i, j int) bool { return a[i].RootFS < a[j].RootFS }
+
+// KernelInfo defines kernels.toml entries
 type KernelInfo struct {
 	DistroType    DistroType
 	DistroRelease string // 18.04/7.4.1708/9.1
@@ -158,14 +268,21 @@ type KernelInfo struct {
 	KernelRelease string
 
 	// Build-time information
+	KernelSource  string // module/exploit will be build on host
 	ContainerName string
 
 	// Runtime information
-	KernelPath string
-	InitrdPath string
-	RootFS     string
+	KernelPath  string
+	InitrdPath  string
+	ModulesPath string
+
+	RootFS string
+
+	// Debug symbols
+	VmlinuxPath string
 }
 
+// KernelConfig is the ~/.out-of-tree/kernels.toml configuration description
 type KernelConfig struct {
 	Kernels []KernelInfo
 }
@@ -181,6 +298,7 @@ func readFileAll(path string) (buf []byte, err error) {
 	return
 }
 
+// ReadKernelConfig is for read kernels.toml
 func ReadKernelConfig(path string) (kernelCfg KernelConfig, err error) {
 	buf, err := readFileAll(path)
 	if err != nil {
@@ -195,16 +313,93 @@ func ReadKernelConfig(path string) (kernelCfg KernelConfig, err error) {
 	return
 }
 
-func ReadArtifactConfig(path string) (artifactCfg Artifact, err error) {
+func rangeRegexp(start, end int) (s string) {
+	s += "("
+	for i := start; i <= end; i++ {
+		s += strconv.Itoa(i)
+		if i != end {
+			s += "|"
+		}
+	}
+	s += ")"
+	return
+}
+
+func versionRegexp(l []int) (s string, err error) {
+	switch len(l) {
+	case 1:
+		s += strconv.Itoa(l[0])
+	case 2:
+		s += rangeRegexp(l[0], l[1])
+	default:
+		err = errors.New("version must contain one value or range")
+		return
+	}
+	return
+}
+
+func genReleaseMask(km kernel) (mask string, err error) {
+	s, err := versionRegexp(km.Version)
+	if err != nil {
+		return
+	}
+	mask += s + "[.]"
+
+	s, err = versionRegexp(km.Major)
+	if err != nil {
+		return
+	}
+	mask += s + "[.]"
+
+	s, err = versionRegexp(km.Minor)
+	if err != nil {
+		return
+	}
+	mask += s
+
+	switch len(km.Patch) {
+	case 0:
+		// ok
+	case 1:
+		mask += "-" + strconv.Itoa(km.Patch[0]) + "-"
+	case 2:
+		mask += "-" + rangeRegexp(km.Patch[0], km.Patch[1]) + "-"
+	default:
+		err = errors.New("version must contain one value or range")
+		return
+	}
+
+	mask += ".*"
+	return
+}
+
+// ReadArtifactConfig is for read .out-of-tree.toml
+func ReadArtifactConfig(path string) (ka Artifact, err error) {
 	buf, err := readFileAll(path)
 	if err != nil {
 		return
 	}
 
-	err = toml.Unmarshal(buf, &artifactCfg)
+	err = toml.Unmarshal(buf, &ka)
 	if err != nil {
 		return
 	}
 
+	for i, _ := range ka.SupportedKernels {
+		km := &ka.SupportedKernels[i]
+		if len(km.Kernel.Version) != 0 && km.ReleaseMask != "" {
+			s := "Only one way to define kernel version is allowed"
+			err = errors.New(s)
+			return
+		}
+
+		if km.ReleaseMask == "" {
+			km.ReleaseMask, err = genReleaseMask(km.Kernel)
+			if err != nil {
+				return
+			}
+		}
+	}
+
 	return
 }

config/config_test.go

@@ -16,7 +16,7 @@ func TestMarshalUnmarshal(t *testing.T) {
 		Type: KernelModule,
 	}
 	artifactCfg.SupportedKernels = append(artifactCfg.SupportedKernels,
-		KernelMask{Ubuntu, "18.04", ".*"})
+		KernelMask{Ubuntu, "18.04", ".*", kernel{}})
 	buf, err := toml.Marshal(&artifactCfg)
 	if err != nil {
 		t.Fatal(err)
@@ -28,3 +28,38 @@ func TestMarshalUnmarshal(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestKernelRegex(t *testing.T) {
+	mask := "4[.]4[.]0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64|65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96|97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116)-.*"
+	k := kernel{
+		Version: []int{4},
+		Major:   []int{4},
+		Minor:   []int{0},
+		Patch:   []int{1, 116},
+	}
+
+	gmask, err := genReleaseMask(k)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if mask != gmask {
+		t.Fatal("Got", gmask, "instead of", mask)
+	}
+
+	mask = "4[.]4[.]0.*"
+	k = kernel{
+		Version: []int{4},
+		Major:   []int{4},
+		Minor:   []int{0},
+	}
+
+	gmask, err = genReleaseMask(k)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if mask != gmask {
+		t.Fatal("Got", gmask, "instead of", mask)
+	}
+}

config/out-of-tree.go

@@ -0,0 +1,106 @@
+// Copyright 2019 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package config
+
+import (
+	"errors"
+	"os"
+	"os/user"
+	"time"
+
+	"github.com/alecthomas/kong"
+	"github.com/mitchellh/go-homedir"
+	"github.com/naoina/toml"
+)
+
+type DockerCommand struct {
+	DistroType DistroType
+	Command    string
+}
+
+type OutOfTree struct {
+	Kernels     string
+	UserKernels string
+
+	Database string
+
+	Qemu struct {
+		Timeout Duration
+	}
+
+	Docker struct {
+		Timeout  Duration
+		Registry string
+
+		// Commands that will be executed before
+		// the base layer of Dockerfile
+		Commands []DockerCommand
+	}
+}
+
+func (c *OutOfTree) Decode(ctx *kong.DecodeContext) (err error) {
+	if ctx.Value.Set {
+		return
+	}
+
+	s, err := homedir.Expand(ctx.Scan.Pop().String())
+	if err != nil {
+		return
+	}
+
+	defaultValue, err := homedir.Expand(ctx.Value.Default)
+	if err != nil {
+		return
+	}
+
+	_, err = os.Stat(s)
+	if s != defaultValue && errors.Is(err, os.ErrNotExist) {
+		return errors.New("'" + s + "' does not exist")
+	}
+
+	*c, err = ReadOutOfTreeConf(s)
+	return
+}
+
+func ReadOutOfTreeConf(path string) (c OutOfTree, err error) {
+	buf, err := readFileAll(path)
+	if err == nil {
+		err = toml.Unmarshal(buf, &c)
+		if err != nil {
+			return
+		}
+	} else {
+		// It's ok if there's no configuration
+		// then we'll just set default values
+		err = nil
+	}
+
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+
+	if c.Kernels == "" {
+		c.Kernels = usr.HomeDir + "/.out-of-tree/kernels.toml"
+	}
+
+	if c.UserKernels == "" {
+		c.UserKernels = usr.HomeDir + "/.out-of-tree/kernels.user.toml"
+	}
+
+	if c.Database == "" {
+		c.Database = usr.HomeDir + "/.out-of-tree/db.sqlite"
+	}
+
+	if c.Qemu.Timeout.Duration == 0 {
+		c.Qemu.Timeout.Duration = time.Minute
+	}
+
+	if c.Docker.Timeout.Duration == 0 {
+		c.Docker.Timeout.Duration = time.Minute
+	}
+
+	return
+}

container.go

@@ -0,0 +1,252 @@
+// Copyright 2023 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"os/user"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/rs/zerolog/log"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+)
+
+var containerRuntime = "docker"
+
+type ContainerCmd struct {
+	Filter string `help:"filter by name"`
+
+	List    ContainerListCmd    `cmd:"" help:"list containers"`
+	Cleanup ContainerCleanupCmd `cmd:"" help:"cleanup containers"`
+}
+
+func (cmd ContainerCmd) Containers() (names []string) {
+	images, err := listContainerImages()
+	if err != nil {
+		log.Fatal().Err(err).Msg("")
+	}
+
+	for _, img := range images {
+		if cmd.Filter != "" && !strings.Contains(img.Name, cmd.Filter) {
+			continue
+		}
+		names = append(names, img.Name)
+	}
+	return
+}
+
+type ContainerListCmd struct{}
+
+func (cmd ContainerListCmd) Run(containerCmd *ContainerCmd) (err error) {
+	for _, name := range containerCmd.Containers() {
+		fmt.Println(name)
+	}
+	return
+}
+
+type ContainerCleanupCmd struct{}
+
+func (cmd ContainerCleanupCmd) Run(containerCmd *ContainerCmd) (err error) {
+	var output []byte
+	for _, name := range containerCmd.Containers() {
+		output, err = exec.Command(containerRuntime, "image", "rm", name).
+			CombinedOutput()
+		if err != nil {
+			log.Error().Err(err).Str("output", string(output)).Msg("")
+			return
+		}
+	}
+	return
+}
+
+type containerImageInfo struct {
+	Name          string
+	DistroType    config.DistroType
+	DistroRelease string // 18.04/7.4.1708/9.1
+}
+
+func listContainerImages() (diis []containerImageInfo, err error) {
+	cmd := exec.Command(containerRuntime, "images")
+	log.Debug().Msgf("%v", cmd)
+
+	rawOutput, err := cmd.CombinedOutput()
+	if err != nil {
+		return
+	}
+
+	r, err := regexp.Compile("out_of_tree_.*")
+	if err != nil {
+		return
+	}
+
+	containers := r.FindAll(rawOutput, -1)
+	for _, c := range containers {
+		container := strings.Fields(string(c))[0]
+
+		s := strings.Replace(container, "__", ".", -1)
+		values := strings.Split(s, "_")
+		distro, ver := values[3], values[4]
+
+		dii := containerImageInfo{
+			Name:          container,
+			DistroRelease: ver,
+		}
+
+		dii.DistroType, err = config.NewDistroType(distro)
+		if err != nil {
+			return
+		}
+
+		diis = append(diis, dii)
+	}
+	return
+}
+
+type container struct {
+	name    string
+	timeout time.Duration
+	Volumes struct {
+		LibModules string
+		UsrSrc     string
+		Boot       string
+	}
+	// Additional arguments
+	Args []string
+}
+
+func NewContainer(name string, timeout time.Duration) (c container, err error) {
+	c.name = name
+	c.timeout = timeout
+
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+
+	c.Volumes.LibModules = fmt.Sprintf(
+		"%s/.out-of-tree/volumes/%s/lib/modules", usr.HomeDir, name)
+	os.MkdirAll(c.Volumes.LibModules, 0777)
+
+	c.Volumes.UsrSrc = fmt.Sprintf(
+		"%s/.out-of-tree/volumes/%s/usr/src", usr.HomeDir, name)
+	os.MkdirAll(c.Volumes.UsrSrc, 0777)
+
+	c.Volumes.Boot = fmt.Sprintf(
+		"%s/.out-of-tree/volumes/%s/boot", usr.HomeDir, name)
+	os.MkdirAll(c.Volumes.Boot, 0777)
+
+	return
+}
+
+func (c container) Build(imagePath string) (output string, err error) {
+	args := []string{"build"}
+	args = append(args, "-t", c.name, imagePath)
+
+	cmd := exec.Command(containerRuntime, args...)
+
+	flog := log.With().
+		Str("command", fmt.Sprintf("%v", cmd)).
+		Logger()
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	cmd.Stderr = cmd.Stdout
+
+	err = cmd.Start()
+	if err != nil {
+		return
+	}
+
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			m := scanner.Text()
+			output += m + "\n"
+			flog.Trace().Str("stdout", m).Msg("")
+		}
+	}()
+
+	err = cmd.Wait()
+	return
+}
+
+func (c container) Run(workdir string, command string) (output string, err error) {
+	flog := log.With().
+		Str("container", c.name).
+		Str("workdir", workdir).
+		Str("command", command).
+		Logger()
+
+	var args []string
+	args = append(args, "run", "--rm")
+	args = append(args, c.Args...)
+	if workdir != "" {
+		args = append(args, "-v", workdir+":/work")
+	}
+	if c.Volumes.LibModules != "" {
+		args = append(args, "-v", c.Volumes.LibModules+":/lib/modules")
+	}
+	if c.Volumes.UsrSrc != "" {
+		args = append(args, "-v", c.Volumes.UsrSrc+":/usr/src")
+	}
+	if c.Volumes.Boot != "" {
+		args = append(args, "-v", c.Volumes.Boot+":/boot")
+	}
+	args = append(args, c.name, "bash", "-c")
+	if workdir != "" {
+		args = append(args, "cd /work && "+command)
+	} else {
+		args = append(args, command)
+	}
+
+	cmd := exec.Command(containerRuntime, args...)
+
+	log.Debug().Msgf("%v", cmd)
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	cmd.Stderr = cmd.Stdout
+
+	timer := time.AfterFunc(c.timeout, func() {
+		flog.Info().Msg("killing container by timeout")
+		cmd.Process.Kill()
+	})
+	defer timer.Stop()
+
+	err = cmd.Start()
+	if err != nil {
+		return
+	}
+
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			m := scanner.Text()
+			output += m + "\n"
+			flog.Trace().Str("stdout", m).Msg("")
+		}
+	}()
+
+	err = cmd.Wait()
+	if err != nil {
+		e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`",
+			err, command, output)
+		err = errors.New(e)
+		return
+	}
+
+	return
+}

db.go

@@ -0,0 +1,335 @@
+// Copyright 2019 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"strconv"
+	"time"
+
+	_ "github.com/mattn/go-sqlite3"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+	"code.dumpstack.io/tools/out-of-tree/qemu"
+)
+
+// Change on ANY database update
+const currentDatabaseVersion = 2
+
+const versionField = "db_version"
+
+type logEntry struct {
+	ID        int
+	Tag       string
+	Timestamp time.Time
+
+	qemu.System
+	config.Artifact
+	config.KernelInfo
+	phasesResult
+}
+
+func createLogTable(db *sql.DB) (err error) {
+	_, err = db.Exec(`
+	CREATE TABLE IF NOT EXISTS log (
+		id		INTEGER PRIMARY KEY,
+		time		DATETIME DEFAULT CURRENT_TIMESTAMP,
+		tag		TEXT,
+
+		name		TEXT,
+		type		TEXT,
+
+		distro_type	TEXT,
+		distro_release	TEXT,
+		kernel_release	TEXT,
+
+		build_output	TEXT,
+		build_ok	BOOLEAN,
+
+		run_output	TEXT,
+		run_ok		BOOLEAN,
+
+		test_output	TEXT,
+		test_ok		BOOLEAN,
+
+		qemu_stdout	TEXT,
+		qemu_stderr	TEXT,
+
+		kernel_panic	BOOLEAN,
+		timeout_kill	BOOLEAN
+	)`)
+	return
+}
+
+func createMetadataTable(db *sql.DB) (err error) {
+	_, err = db.Exec(`
+	CREATE TABLE IF NOT EXISTS metadata (
+		id	INTEGER PRIMARY KEY,
+		key	TEXT UNIQUE,
+		value	TEXT
+	)`)
+	return
+}
+
+func metaChkValue(db *sql.DB, key string) (exist bool, err error) {
+	sql := "SELECT EXISTS(SELECT id FROM metadata WHERE key = $1)"
+	stmt, err := db.Prepare(sql)
+	if err != nil {
+		return
+	}
+	defer stmt.Close()
+
+	err = stmt.QueryRow(key).Scan(&exist)
+	return
+}
+
+func metaGetValue(db *sql.DB, key string) (value string, err error) {
+	stmt, err := db.Prepare("SELECT value FROM metadata " +
+		"WHERE key = $1")
+	if err != nil {
+		return
+	}
+	defer stmt.Close()
+
+	err = stmt.QueryRow(key).Scan(&value)
+	return
+}
+
+func metaSetValue(db *sql.DB, key, value string) (err error) {
+	stmt, err := db.Prepare("INSERT OR REPLACE INTO metadata " +
+		"(key, value) VALUES ($1, $2)")
+	if err != nil {
+		return
+	}
+	defer stmt.Close()
+
+	_, err = stmt.Exec(key, value)
+	return
+}
+
+func getVersion(db *sql.DB) (version int, err error) {
+	s, err := metaGetValue(db, versionField)
+	if err != nil {
+		return
+	}
+
+	version, err = strconv.Atoi(s)
+	return
+}
+
+func addToLog(db *sql.DB, q *qemu.System, ka config.Artifact,
+	ki config.KernelInfo, res *phasesResult, tag string) (err error) {
+
+	stmt, err := db.Prepare("INSERT INTO log (name, type, tag, " +
+		"distro_type, distro_release, kernel_release, " +
+		"build_output, build_ok, " +
+		"run_output, run_ok, " +
+		"test_output, test_ok, " +
+		"qemu_stdout, qemu_stderr, " +
+		"kernel_panic, timeout_kill) " +
+		"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, " +
+		"$10, $11, $12, $13, $14, $15, $16);")
+	if err != nil {
+		return
+	}
+
+	defer stmt.Close()
+
+	_, err = stmt.Exec(
+		ka.Name, ka.Type, tag,
+		ki.DistroType, ki.DistroRelease, ki.KernelRelease,
+		res.Build.Output, res.Build.Ok,
+		res.Run.Output, res.Run.Ok,
+		res.Test.Output, res.Test.Ok,
+		q.Stdout, q.Stderr,
+		q.KernelPanic, q.KilledByTimeout,
+	)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func getAllLogs(db *sql.DB, tag string, num int) (les []logEntry, err error) {
+	stmt, err := db.Prepare("SELECT id, time, name, type, tag, " +
+		"distro_type, distro_release, kernel_release, " +
+		"build_ok, run_ok, test_ok, kernel_panic, " +
+		"timeout_kill FROM log ORDER BY datetime(time) DESC " +
+		"LIMIT $1")
+	if err != nil {
+		return
+	}
+	defer stmt.Close()
+
+	rows, err := stmt.Query(num)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		le := logEntry{}
+		err = rows.Scan(&le.ID, &le.Timestamp,
+			&le.Name, &le.Type, &le.Tag,
+			&le.DistroType, &le.DistroRelease, &le.KernelRelease,
+			&le.Build.Ok, &le.Run.Ok, &le.Test.Ok,
+			&le.KernelPanic, &le.KilledByTimeout,
+		)
+		if err != nil {
+			return
+		}
+
+		if tag == "" || tag == le.Tag {
+			les = append(les, le)
+		}
+	}
+
+	return
+}
+
+func getAllArtifactLogs(db *sql.DB, tag string, num int, ka config.Artifact) (
+	les []logEntry, err error) {
+
+	stmt, err := db.Prepare("SELECT id, time, name, type, tag, " +
+		"distro_type, distro_release, kernel_release, " +
+		"build_ok, run_ok, test_ok, kernel_panic, " +
+		"timeout_kill FROM log WHERE name=$1 AND type=$2 " +
+		"ORDER BY datetime(time) DESC LIMIT $3")
+	if err != nil {
+		return
+	}
+	defer stmt.Close()
+
+	rows, err := stmt.Query(ka.Name, ka.Type, num)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		le := logEntry{}
+		err = rows.Scan(&le.ID, &le.Timestamp,
+			&le.Name, &le.Type, &le.Tag,
+			&le.DistroType, &le.DistroRelease, &le.KernelRelease,
+			&le.Build.Ok, &le.Run.Ok, &le.Test.Ok,
+			&le.KernelPanic, &le.KilledByTimeout,
+		)
+		if err != nil {
+			return
+		}
+
+		if tag == "" || tag == le.Tag {
+			les = append(les, le)
+		}
+	}
+
+	return
+}
+
+func getLogByID(db *sql.DB, id int) (le logEntry, err error) {
+	stmt, err := db.Prepare("SELECT id, time, name, type, tag, " +
+		"distro_type, distro_release, kernel_release, " +
+		"build_ok, run_ok, test_ok, " +
+		"build_output, run_output, test_output, " +
+		"qemu_stdout, qemu_stderr, " +
+		"kernel_panic, timeout_kill " +
+		"FROM log WHERE id=$1")
+	if err != nil {
+		return
+	}
+	defer stmt.Close()
+
+	err = stmt.QueryRow(id).Scan(&le.ID, &le.Timestamp,
+		&le.Name, &le.Type, &le.Tag,
+		&le.DistroType, &le.DistroRelease, &le.KernelRelease,
+		&le.Build.Ok, &le.Run.Ok, &le.Test.Ok,
+		&le.Build.Output, &le.Run.Output, &le.Test.Output,
+		&le.Stdout, &le.Stderr,
+		&le.KernelPanic, &le.KilledByTimeout,
+	)
+	return
+}
+
+func getLastLog(db *sql.DB) (le logEntry, err error) {
+	err = db.QueryRow("SELECT MAX(id), time, name, type, tag, "+
+		"distro_type, distro_release, kernel_release, "+
+		"build_ok, run_ok, test_ok, "+
+		"build_output, run_output, test_output, "+
+		"qemu_stdout, qemu_stderr, "+
+		"kernel_panic, timeout_kill "+
+		"FROM log").Scan(&le.ID, &le.Timestamp,
+		&le.Name, &le.Type, &le.Tag,
+		&le.DistroType, &le.DistroRelease, &le.KernelRelease,
+		&le.Build.Ok, &le.Run.Ok, &le.Test.Ok,
+		&le.Build.Output, &le.Run.Output, &le.Test.Output,
+		&le.Stdout, &le.Stderr,
+		&le.KernelPanic, &le.KilledByTimeout,
+	)
+	return
+}
+
+func createSchema(db *sql.DB) (err error) {
+	err = createMetadataTable(db)
+	if err != nil {
+		return
+	}
+
+	err = createLogTable(db)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func openDatabase(path string) (db *sql.DB, err error) {
+	db, err = sql.Open("sqlite3", path)
+	if err != nil {
+		return
+	}
+
+	db.SetMaxOpenConns(1)
+
+	exists, _ := metaChkValue(db, versionField)
+	if !exists {
+		err = createSchema(db)
+		if err != nil {
+			return
+		}
+
+		err = metaSetValue(db, versionField,
+			strconv.Itoa(currentDatabaseVersion))
+		return
+	}
+
+	version, err := getVersion(db)
+	if err != nil {
+		return
+	}
+
+	if version == 1 {
+		_, err = db.Exec(`ALTER TABLE log ADD tag TEXT`)
+		if err != nil {
+			return
+		}
+
+		err = metaSetValue(db, versionField, "2")
+		if err != nil {
+			return
+		}
+
+		version = 2
+	}
+
+	if version != currentDatabaseVersion {
+		err = fmt.Errorf("Database is not supported (%d instead of %d)",
+			version, currentDatabaseVersion)
+		return
+	}
+
+	return
+}

debug.go

@@ -8,16 +8,217 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"os"
 	"strings"
 	"time"
 
-	"github.com/jollheef/out-of-tree/config"
-	qemu "github.com/jollheef/out-of-tree/qemu"
-	"github.com/logrusorgru/aurora"
+	"github.com/rs/zerolog/log"
+	"gopkg.in/logrusorgru/aurora.v2"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+	"code.dumpstack.io/tools/out-of-tree/qemu"
 )
 
+type DebugCmd struct {
+	Kernel string `help:"regexp (first match)" required:""`
+	Gdb    string `help:"gdb listen address" default:"tcp::1234"`
+
+	SshAddr string `help:"ssh address to listen" default:"127.0.0.1"`
+	SshPort int    `help:"ssh port to listen" default:"50022"`
+
+	ArtifactConfig string `help:"path to artifact config" type:"path"`
+
+	Kaslr bool `help:"Enable KASLR"`
+	Smep  bool `help:"Enable SMEP"`
+	Smap  bool `help:"Enable SMAP"`
+	Kpti  bool `help:"Enable KPTI"`
+
+	NoKaslr bool `help:"Disable KASLR"`
+	NoSmep  bool `help:"Disable SMEP"`
+	NoSmap  bool `help:"Disable SMAP"`
+	NoKpti  bool `help:"Disable KPTI"`
+}
+
+// TODO: merge with pew.go
+func (cmd *DebugCmd) Run(g *Globals) (err error) {
+	kcfg, err := config.ReadKernelConfig(g.Config.Kernels)
+	if err != nil {
+		log.Print(err)
+	}
+
+	var configPath string
+	if cmd.ArtifactConfig == "" {
+		configPath = g.WorkDir + "/.out-of-tree.toml"
+	} else {
+		configPath = cmd.ArtifactConfig
+	}
+	ka, err := config.ReadArtifactConfig(configPath)
+	if err != nil {
+		return
+	}
+
+	if ka.SourcePath == "" {
+		ka.SourcePath = g.WorkDir
+	}
+
+	ki, err := firstSupported(kcfg, ka, cmd.Kernel)
+	if err != nil {
+		return
+	}
+
+	kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
+	q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
+	if err != nil {
+		return
+	}
+
+	err = q.SetSSHAddrPort(cmd.SshAddr, cmd.SshPort)
+	if err != nil {
+		return
+	}
+
+	if ka.Qemu.Cpus != 0 {
+		q.Cpus = ka.Qemu.Cpus
+	}
+	if ka.Qemu.Memory != 0 {
+		q.Memory = ka.Qemu.Memory
+	}
+
+	if ka.Docker.Timeout.Duration != 0 {
+		g.Config.Docker.Timeout.Duration = ka.Docker.Timeout.Duration
+	}
+
+	q.SetKASLR(false) // set KASLR to false by default because of gdb
+	q.SetSMEP(!ka.Mitigations.DisableSmep)
+	q.SetSMAP(!ka.Mitigations.DisableSmap)
+	q.SetKPTI(!ka.Mitigations.DisableKpti)
+
+	if cmd.Kaslr {
+		q.SetKASLR(true)
+	} else if cmd.NoKaslr {
+		q.SetKASLR(false)
+	}
+
+	if cmd.Smep {
+		q.SetSMEP(true)
+	} else if cmd.NoSmep {
+		q.SetSMEP(false)
+	}
+
+	if cmd.Smap {
+		q.SetSMAP(true)
+	} else if cmd.NoSmap {
+		q.SetSMAP(false)
+	}
+
+	if cmd.Kpti {
+		q.SetKPTI(true)
+	} else if cmd.NoKpti {
+		q.SetKPTI(false)
+	}
+
+	redgreen := func(name string, enabled bool) aurora.Value {
+		if enabled {
+			return aurora.BgGreen(aurora.Black(name))
+		}
+
+		return aurora.BgRed(aurora.White(name))
+	}
+
+	fmt.Printf("[*] %s %s %s %s\n",
+		redgreen("KASLR", q.GetKASLR()),
+		redgreen("SMEP", q.GetSMEP()),
+		redgreen("SMAP", q.GetSMAP()),
+		redgreen("KPTI", q.GetKPTI()))
+
+	fmt.Printf("[*] SMP: %d CPUs\n", q.Cpus)
+	fmt.Printf("[*] Memory: %d MB\n", q.Memory)
+
+	q.Debug(cmd.Gdb)
+	coloredGdbAddress := aurora.BgGreen(aurora.Black(cmd.Gdb))
+	fmt.Printf("[*] gdb is listening on %s\n", coloredGdbAddress)
+
+	err = q.Start()
+	if err != nil {
+		return
+	}
+	defer q.Stop()
+
+	tmp, err := ioutil.TempDir(tempDirBase, "out-of-tree_")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(tmp)
+
+	err = q.WaitForSSH(time.Minute)
+	if err != nil {
+		return
+	}
+
+	if ka.StandardModules {
+		// Module depends on one of the standard modules
+		err = copyStandardModules(q, ki)
+		if err != nil {
+			log.Print(err)
+			return
+		}
+	}
+
+	err = preloadModules(q, ka, ki, g.Config.Docker.Timeout.Duration)
+	if err != nil {
+		log.Print(err)
+		return
+	}
+
+	var buildDir, outFile, output, remoteFile string
+
+	if ka.Type == config.Script {
+		err = q.CopyFile("root", ka.Script, ka.Script)
+		if err != nil {
+			return
+		}
+	} else {
+		buildDir, outFile, output, err = build(tmp, ka, ki, g.Config.Docker.Timeout.Duration)
+		if err != nil {
+			log.Print(err, output)
+			return
+		}
+
+		remoteFile = "/tmp/exploit"
+		if ka.Type == config.KernelModule {
+			remoteFile = "/tmp/module.ko"
+		}
+
+		err = q.CopyFile("user", outFile, remoteFile)
+		if err != nil {
+			return
+		}
+	}
+
+	// Copy all test files to the remote machine
+	for _, f := range ka.TestFiles {
+		if f.Local[0] != '/' {
+			f.Local = buildDir + "/" + f.Local
+		}
+		err = q.CopyFile(f.User, f.Local, f.Remote)
+		if err != nil {
+			log.Print("error copy err:", err, f.Local, f.Remote)
+			return
+		}
+	}
+
+	coloredRemoteFile := aurora.BgGreen(aurora.Black(remoteFile))
+	fmt.Printf("[*] build result copied to %s\n", coloredRemoteFile)
+
+	fmt.Printf("\n%s\n", q.GetSSHCommand())
+	fmt.Printf("gdb %s -ex 'target remote %s'\n\n", ki.VmlinuxPath, cmd.Gdb)
+
+	// TODO set substitute-path /build/.../linux-... /path/to/linux-source
+
+	err = interactive(q)
+	return
+}
+
 func firstSupported(kcfg config.KernelConfig, ka config.Artifact,
 	kernel string) (ki config.KernelInfo, err error) {
 
@@ -40,7 +241,7 @@ func firstSupported(kcfg config.KernelConfig, ka config.Artifact,
 	return
 }
 
-func handleLine(q *qemu.QemuSystem) (err error) {
+func handleLine(q *qemu.System) (err error) {
 	fmt.Print("out-of-tree> ")
 	rawLine := "help"
 	fmt.Scanf("%s", &rawLine)
@@ -56,14 +257,14 @@ func handleLine(q *qemu.QemuSystem) (err error) {
 		fmt.Printf("ssh\t: print arguments to ssh command\n")
 		fmt.Printf("quit\t: quit\n")
 	case "l", "log":
-		fmt.Println(string(q.Stdout))
+		fmt.Println(q.Stdout)
 	case "cl", "clog":
-		fmt.Println(string(q.Stdout))
-		q.Stdout = []byte{}
+		fmt.Println(q.Stdout)
+		q.Stdout = ""
 	case "c", "cleanup":
-		q.Stdout = []byte{}
+		q.Stdout = ""
 	case "s", "ssh":
-		fmt.Println(q.GetSshCommand())
+		fmt.Println(q.GetSSHCommand())
 	case "q", "quit":
 		return errors.New("end of session")
 	default:
@@ -72,7 +273,7 @@ func handleLine(q *qemu.QemuSystem) (err error) {
 	return
 }
 
-func interactive(q *qemu.QemuSystem) (err error) {
+func interactive(q *qemu.System) (err error) {
 	for {
 		err = handleLine(q)
 		if err != nil {
@@ -80,64 +281,3 @@ func interactive(q *qemu.QemuSystem) (err error) {
 		}
 	}
 }
-
-func debugHandler(kcfg config.KernelConfig, workPath, kernRegex, gdb string,
-	dockerTimeout time.Duration) (err error) {
-
-	ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
-	if err != nil {
-		return
-	}
-
-	if ka.SourcePath == "" {
-		ka.SourcePath = workPath
-	}
-
-	ki, err := firstSupported(kcfg, ka, kernRegex)
-	if err != nil {
-		return
-	}
-
-	kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
-	q, err := qemu.NewQemuSystem(qemu.X86_64, kernel, ki.RootFS)
-	if err != nil {
-		return
-	}
-	q.Debug(gdb)
-	coloredGdbAddress := aurora.BgGreen(aurora.Black(gdb))
-	fmt.Printf("[*] gdb runned on %s\n", coloredGdbAddress)
-
-	err = q.Start()
-	if err != nil {
-		return
-	}
-	defer q.Stop()
-
-	tmp, err := ioutil.TempDir("/tmp/", "out-of-tree_")
-	if err != nil {
-		return
-	}
-	defer os.RemoveAll(tmp)
-
-	outFile, output, err := build(tmp, ka, ki, dockerTimeout)
-	if err != nil {
-		log.Println(err, output)
-		return
-	}
-
-	remoteFile := "/tmp/artifact"
-	if ka.Type == config.KernelModule {
-		remoteFile += ".ko"
-	}
-
-	err = q.CopyFile("user", outFile, remoteFile)
-	if err != nil {
-		return
-	}
-
-	coloredRemoteFile := aurora.BgGreen(aurora.Black(remoteFile))
-	fmt.Printf("[*] build result copied to %s\n", coloredRemoteFile)
-
-	err = interactive(q)
-	return
-}

docs/index.rst

@@ -0,0 +1,30 @@
+out-of-tree
+===========
+
+*out-of-tree* is the kernel {module, exploit} development tool.
+
+*out-of-tree* was created on the purpose of decreasing complexity of
+environment for developing, testing and debugging Linux kernel
+exploits and out-of-tree kernel modules (that's why tool got a name
+"out-of-tree").
+
+While I'm trying to keep that documentation up-to-date, there may be
+some missing information. Use ``out-of-tree --help-long`` for checking
+all features.
+
+If you found anything missed here, please make a pull request or send
+patches to patch@dumpstack.io.
+
+If you need personal support, your company is interested in the
+project or you just want to share some thoughts -- feel free to write
+to root@dumpstack.io.
+
+Contents
+========
+
+:ref:`Keyword Index <genindex>`
+
+.. toctree::
+
+   introduction.rst
+   installation.rst

docs/installation.rst

@@ -0,0 +1,79 @@
+Installation (from source)
+============
+
+OS/Distro-specific
+==================
+
+Ubuntu
+------
+
+Install dependencies::
+
+  $ sudo snap install go --classic
+  $ # Install docker: https://docs.docker.com/engine/install/ubuntu/
+  $ sudo apt install qemu-system-x86 build-essential gdb
+
+macOS
+-----
+
+Install dependencies::
+
+  $ brew install go qemu
+  $ brew cask install docker
+
+NixOS
+-----
+
+There's a minimal configuration that you need to apply::
+
+  #!nix
+  { config, pkgs, ... }:
+  {
+    virtualisation.docker.enable = true;
+    virtualisation.libvirtd.enable = true;
+    environment.systemPackages = with pkgs; [
+      go git
+    ];
+  }
+
+Gentoo
+------
+
+Install dependencies::
+
+  $ sudo emerge app-emulation/qemu app-emulation/docker dev-lang/go
+
+Fedora
+------
+
+Install dependencies::
+
+  $ sudo dnf install go qemu moby-engine
+
+Common
+======
+
+Setup environment::
+
+  $ echo 'export PATH=$PATH:$HOME/bin' >> ~/.bashrc
+  $ source ~/.bashrc
+
+Build *out-of-tree*::
+
+  $ git clone https://code.dumpstack.io/tools/out-of-tree
+  $ cd out-of-tree
+  $ CGO_ENABLED=1 go build -o ~/bin/out-of-tree
+
+.. note::
+  On a GNU/Linux you need to add your user to docker group if you want
+  to use *out-of-tree* without sudo. Note that this has a **serious**
+  security implications. Check *Docker* documentation for more
+  information.
+
+Test that everything works::
+
+  $ cd out-of-tree/examples/kernel-exploit
+  $ out-of-tree kernel autogen --max=1
+  $ out-of-tree pew --max=1
+
+Enjoy!

docs/introduction.rst

@@ -0,0 +1,109 @@
+Introduction
+============
+
+*out-of-tree* is written in *Go*, it uses *Docker* for generating
+kernel/filesystem images and *Qemu* for virtualization.
+
+Also it possible to generate kernels from the host system and use the
+custom one.
+
+*out-of-tree* supports *GNU/Linux* (usually it's tested on NixOS and
+latest Ubuntu LTS) and *macOS*. Technically all systems that supported
+by Go, Docker, and Qemu must work well. Create the issue if you'll
+notice any issue in integration for your operating system.
+
+All *Qemu* interaction is stateless.
+
+*out-of-tree* is allow and require metadata (``.out-of-tree.toml``)
+for work. TOML (Tom's Obvious, Minimal Language) is used for kernel
+module/exploit description.
+
+``.out-of-tree.toml`` is mandatory, you need to have in the current
+directory (usually, it's a project of kernel module/exploit) or use
+the ``--path`` flag.
+
+Files
+-----
+
+All data is stored in ``~/.out-of-tree/``.
+
+- *db.sqlite* contains logs related to run with ``out-of-tree pew``,
+  debug mode (``out-of-tree debug``) is not store any data.
+
+- *images* used for filesystem images (rootfs images that used for
+  ``qemu -hda ...``) that can be generated with the
+  ``tools/qemu-*-img/...``.
+
+- *kernels* stores all kernel ``vmlinuz/initrd/config/...`` files that
+  generated previously with a some *Docker magic*.
+
+- *kernels.toml* contains metadata for generated kernels. It's not
+  supposed to be edited by hands.
+
+- *kernels.user.toml* is default path for custom kernels definition.
+
+- *Ubuntu* (or *Centos*/*Debian*/...) is the Dockerfiles tree
+  (DistroName/DistroVersion/Dockerfile). Each Dockerfile contains a
+  base layer and incrementally updated list of kernels that must be
+  installed.
+
+Overview
+---------
+
+*out-of-tree* creating debugging environment based on **defined** kernels::
+
+    $ out-of-tree debug --kernel 'Ubuntu:4.15.0-58-generic'
+    [*] KASLR SMEP SMAP
+    [*] gdb is listening on tcp::1234
+    [*] build result copied to /tmp/exploit
+
+    ssh -o StrictHostKeyChecking=no -p 29308 root@127.133.45.236
+    gdb /usr/lib/debug/boot/vmlinux-4.15.0-58-generic -ex 'target remote tcp::1234'
+
+    out-of-tree> help
+    help    : print this help message
+    log     : print qemu log
+    clog    : print qemu log and cleanup buffer
+    cleanup : cleanup qemu log buffer
+    ssh     : print arguments to ssh command
+    quit    : quit
+    out-of-tree>
+
+*out-of-tree* uses three stages for automated runs:
+
+- Build
+
+  - Inside the docker container (default).
+  - Binary version (de facto skip stage).
+  - On host.
+
+- Run
+
+  - Insmod for the kernel module.
+  - This step is skipped for exploits.
+
+- Test
+
+  - Run the test.sh script on the target machine.
+  - Test script is run from *root* for the kernel module.
+  - Test script is run from *user* for the kernel exploit.
+  - Test script for the kernel module is fully custom (only return
+    value is checked).
+  - Test script for the kernel exploit receives two parameters:
+
+    - Path to exploit
+    - Path to file that must be created with root privileges.
+
+  - If there's no test.sh script then default
+    (``echo touch FILE | exploit``) one is used.
+
+Security
+--------
+
+*out-of-tree* is not supposed to be used on multi-user systems or with
+an untrusted input.
+
+Meanwhile, all modern hypervisors are supporting nested
+virtualization, which means you can use it for isolating *out-of-tree*
+if you want to work with an untrusted input (e.g. with a mass-scale
+testing public proofs-of-concept).

examples/kernel-exploit/.out-of-tree.toml

@@ -6,12 +6,12 @@ type = "exploit"
 [[supported_kernels]]
 distro_type = "Ubuntu"
 distro_release = "16.04"
-release_mask = "4.4.0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64|65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96|97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116)-.*"
+release_mask = "4[.]4[.]0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64|65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96|97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116)-.*"
 
 [[supported_kernels]]
 distro_type = "Ubuntu"
 distro_release = "16.04"
-release_mask = "4.8.0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58)-.*"
+release_mask = "4[.]8[.]0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58)-.*"
 
 [[supported_kernels]]
 # Can be Ubuntu/CentOS/Debian/etc.
@@ -20,14 +20,19 @@ distro_release = "16.04"
 # regex for `uname -r`
 # See also: regex-golang.appspot.com
 # stupid way to generate: $ echo '4.4.0-('$(seq 44 | xargs echo | sed 's/ /|/g')')-.*'
-release_mask = "4.10.0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42)-.*"
+release_mask = "4[.]10[.]0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42)-.*"
 
 [[supported_kernels]]
 distro_type = "Ubuntu"
 distro_release = "16.04"
-release_mask = "4.11.0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14)-.*"
+release_mask = "4[.]11[.]0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14)-.*"
 
 [[supported_kernels]]
 distro_type = "Ubuntu"
 distro_release = "16.04"
-release_mask = "4.13.0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21)-.*"
+# equivalent for "4[.]13[.]0-(1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21)-.*"
+[supported_kernels.kernel]
+version = [ 4 ]
+major = [ 13 ]
+minor = [ 0 ]
+patch = [ 1, 21 ]

examples/kernel-exploit/CVE-2017-16995.c

@@ -317,6 +317,7 @@ void redact(const char *fmt, ...) {
 	va_start(args, fmt);
 	if(doredact) {
 		fprintf(stdout, "[!] ( ( R E D A C T E D ) )\n");
+		va_end(args);
 		return;
 	}
 	fprintf(stdout, "[*] ");

examples/kernel-exploit/Makefile

@@ -2,7 +2,7 @@
 # - KERNEL: kernel headers path
 # - TARGET: name of exploit binary that MUST be produced by makefile.
 # - $(TARGET)_test: name of test binary that MUST be produced by makefile
-# and it's will be runned on a LPE stage. TARGET_TEST MUST accept two argument:
+# and it's will be executed on a LPE stage. TARGET_TEST MUST accept two argument:
 #   - Path to exploit binary
 #   - File that MUST be created with exploit. It uses for test that exploit works
 #     correctly.

examples/kernel-module/.out-of-tree.toml

@@ -9,17 +9,22 @@ distro_type = "Ubuntu"
 distro_release = "16.04"
 # regex for `uname -r`
 # See also: regex-golang.appspot.com
-release_mask = "4.4.0-70-.*"
+release_mask = "4[.]4[.]0-70-.*"
 
 # [[supported_kernels]] may be defined unlimited number of times
 [[supported_kernels]]
 distro_type = "Ubuntu"
 distro_release = "18.04"
 # Also you can use only one kernel
-release_mask = "4.15.0-(24|29)-generic"
+release_mask = "4[.]15[.]0-(24|29)-generic"
 
 [[supported_kernels]]
 distro_type = "Ubuntu"
 distro_release = "18.04"
 # Also you can use only one kernel
-release_mask = "4.15.0-23-generic"
+release_mask = "4[.]15[.]0-23-generic"
+
+[[supported_kernels]]
+distro_type = "CentOS"
+distro_release = "7"
+release_mask = "3[.]10[.]0-862.el7.x86_64"

examples/preload/.out-of-tree.toml

@@ -0,0 +1,12 @@
+name = "out-of-tree preload"
+type = "module"
+
+[[supported_kernels]]
+distro_type = "Ubuntu"
+distro_release = "18.04"
+release_mask = ".*"
+
+[[preload]]
+repo = "https://github.com/openwall/lkrg"
+#path = "/local/path/to/lkrg"
+timeout_after_load = "1s"

examples/preload/Makefile

@@ -0,0 +1,11 @@
+KERNEL := /lib/modules/$(shell uname -r)/build
+TARGET := module
+
+obj-m += $(TARGET).o
+$(TARGET)-objs = module.o
+
+all:
+	make -C $(KERNEL) M=$(PWD) modules
+
+clean:
+	make -C $(KERNEL) M=$(PWD) clean

examples/preload/README.md

@@ -0,0 +1,5 @@
+# out-of-tree kernel module preload example
+
+See .out-of-tree.toml
+
+Note that it should fail to insert module if lkrg is enabled in the preload list.

examples/preload/module.c

@@ -0,0 +1,17 @@
+#include <linux/module.h>
+#include <linux/slab.h>
+
+int init_module(void)
+{
+	char *argv[] = { "/bin/sh", "--help", NULL };
+	char *envp[] = { NULL };
+
+	/* trigger lkrg */
+	return call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC);
+}
+
+void cleanup_module(void)
+{
+}
+
+MODULE_LICENSE("GPL");

examples/script/.out-of-tree.toml

@@ -0,0 +1,11 @@
+# out-of-tree configuration file
+# docs at https://out-of-tree.io
+name = "out-of-tree script example"
+type = "script"
+
+script = "script.sh"
+
+[[supported_kernels]]
+distro_type = "Ubuntu"
+distro_release = "22.04"
+release_mask = ".*"

examples/script/README.md

@@ -0,0 +1,3 @@
+# out-of-tree script example
+
+See .out-of-tree.toml

examples/script/script.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+uname -a
+
+ls /proc | grep config

gen.go

@@ -7,17 +7,40 @@ package main
 import (
 	"fmt"
 
-	"github.com/jollheef/out-of-tree/config"
 	"github.com/naoina/toml"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
 )
 
+type GenCmd struct {
+	Type string `enum:"module,exploit" required:"" help:"module/exploit"`
+}
+
+func (cmd *GenCmd) Run(g *Globals) (err error) {
+	switch cmd.Type {
+	case "module":
+		err = genConfig(config.KernelModule)
+	case "exploit":
+		err = genConfig(config.KernelExploit)
+	}
+	return
+}
+
 func genConfig(at config.ArtifactType) (err error) {
 	a := config.Artifact{
 		Name: "Put name here",
 		Type: at,
 	}
 	a.SupportedKernels = append(a.SupportedKernels, config.KernelMask{
-		config.Ubuntu, "18.04", ".*",
+		DistroType:    config.Ubuntu,
+		DistroRelease: "18.04",
+		ReleaseMask:   ".*",
+	})
+	a.Preload = append(a.Preload, config.PreloadModule{
+		Repo: "Repo name (e.g. https://github.com/openwall/lkrg)",
+	})
+	a.Patches = append(a.Patches, config.Patch{
+		Path: "/path/to/profiling.patch",
 	})
 
 	buf, err := toml.Marshal(&a)

go.mod

@@ -0,0 +1,50 @@
+module code.dumpstack.io/tools/out-of-tree
+
+go 1.17
+
+replace code.dumpstack.io/tools/out-of-tree/qemu => ./qemu
+
+replace code.dumpstack.io/tools/out-of-tree/config => ./config
+
+require (
+	github.com/alecthomas/kong v0.7.1
+	github.com/go-git/go-git/v5 v5.6.1
+	github.com/mattn/go-sqlite3 v1.14.16
+	github.com/mitchellh/go-homedir v1.1.0
+	github.com/naoina/toml v0.1.1
+	github.com/natefinch/lumberjack v2.0.0+incompatible
+	github.com/olekukonko/tablewriter v0.0.5
+	github.com/otiai10/copy v1.10.0
+	github.com/remeh/sizedwaitgroup v1.0.0
+	github.com/rs/zerolog v1.29.0
+	github.com/zcalusic/sysinfo v0.9.5
+	golang.org/x/crypto v0.7.0
+	gopkg.in/logrusorgru/aurora.v2 v2.0.3
+)
+
+require (
+	github.com/BurntSushi/toml v1.2.1 // indirect
+	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
+	github.com/acomagu/bufpipe v1.0.4 // indirect
+	github.com/cloudflare/circl v1.1.0 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/go-git/gcfg v1.5.0 // indirect
+	github.com/go-git/go-billy/v5 v5.4.1 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/naoina/go-stringutil v0.1.0 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
+	github.com/sergi/go-diff v1.1.0 // indirect
+	github.com/skeema/knownhosts v1.1.0 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
+)

go.sum

@@ -0,0 +1,198 @@
+github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
+github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
+github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
+github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA=
+github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
+github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
+github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
+github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0=
+github.com/alecthomas/assert/v2 v2.1.0/go.mod h1:b/+1DI2Q6NckYi+3mXyH3wFb8qG37K/DuK80n7WefXA=
+github.com/alecthomas/kong v0.7.1 h1:azoTh0IOfwlAX3qN9sHWTxACE2oV8Bg2gAwBsMwDQY4=
+github.com/alecthomas/kong v0.7.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U=
+github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
+github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
+github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY=
+github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
+github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
+github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
+github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
+github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
+github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
+github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
+github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
+github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
+github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ=
+github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
+github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk=
+github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
+github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
+github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
+github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
+github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
+github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
+github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
+github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM=
+github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
+github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
+github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
+github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
+github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
+github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/otiai10/copy v1.10.0 h1:znyI7l134wNg/wDktoVQPxPkgvhDfGCYUasey+h0rDQ=
+github.com/otiai10/copy v1.10.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww=
+github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
+github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
+github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
+github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
+github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w=
+github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0=
+github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXijpFO6roag=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
+github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/zcalusic/sysinfo v0.9.5 h1:ivoHyj9aIAYkwzo1+8QgJ5s4oeE6Etx9FmZtqa4wJjQ=
+github.com/zcalusic/sysinfo v0.9.5/go.mod h1:Z/gPVufBrFc8X5sef3m6kkw3r3nlNFp+I6bvASfvBZQ=
+golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
+golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
+golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
+golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
+golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
+golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
+golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/logrusorgru/aurora.v2 v2.0.3 h1:5Hr76hqgwx9PednedYf5Q1dBfiPMZ2IgExR7u3tNXIE=
+gopkg.in/logrusorgru/aurora.v2 v2.0.3/go.mod h1:Wm+IEn1fgFp8E2paL93oFVrHZW4toMKARNE85fDY5w8=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
+gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
+gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

images.config.go

@@ -0,0 +1,7 @@
+// Copyright 2019 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package main
+
+const imagesBaseURL = "https://out-of-tree.fra1.digitaloceanspaces.com/1.0.0/"

images.go

@@ -0,0 +1,196 @@
+// Copyright 2019 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"os/user"
+	"strings"
+	"time"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+	"code.dumpstack.io/tools/out-of-tree/qemu"
+	"github.com/rs/zerolog/log"
+)
+
+type ImageCmd struct {
+	List ImageListCmd `cmd:"" help:"list images"`
+	Edit ImageEditCmd `cmd:"" help:"edit image"`
+}
+
+type ImageListCmd struct{}
+
+func (cmd *ImageListCmd) Run(g *Globals) (err error) {
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+
+	entries, err := os.ReadDir(usr.HomeDir + "/.out-of-tree/images/")
+	if err != nil {
+		return
+	}
+
+	for _, e := range entries {
+		fmt.Println(e.Name())
+	}
+
+	return
+}
+
+type ImageEditCmd struct {
+	Name   string `help:"image name" required:""`
+	DryRun bool   `help:"do nothing, just print commands"`
+}
+
+func (cmd *ImageEditCmd) Run(g *Globals) (err error) {
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+
+	image := usr.HomeDir + "/.out-of-tree/images/" + cmd.Name
+	if !exists(image) {
+		fmt.Println("image does not exist")
+	}
+
+	kcfg, err := config.ReadKernelConfig(g.Config.Kernels)
+	if err != nil {
+		return
+	}
+
+	if len(kcfg.Kernels) == 0 {
+		return errors.New("No kernels found")
+	}
+
+	ki := config.KernelInfo{}
+	for _, k := range kcfg.Kernels {
+		if k.RootFS == image {
+			ki = k
+			break
+		}
+	}
+
+	kernel := qemu.Kernel{
+		KernelPath: ki.KernelPath,
+		InitrdPath: ki.InitrdPath,
+	}
+
+	q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
+
+	q.Mutable = true
+
+	if cmd.DryRun {
+		s := q.Executable()
+		for _, arg := range q.Args() {
+			if strings.Contains(arg, " ") ||
+				strings.Contains(arg, ",") {
+
+				s += fmt.Sprintf(` "%s"`, arg)
+			} else {
+				s += fmt.Sprintf(" %s", arg)
+			}
+		}
+		fmt.Println(s)
+		fmt.Println(q.GetSSHCommand())
+		return
+	}
+
+	err = q.Start()
+	if err != nil {
+		fmt.Println("Qemu start error:", err)
+		return
+	}
+	defer q.Stop()
+
+	fmt.Print("ssh command:\n\n\t")
+	fmt.Println(q.GetSSHCommand())
+
+	fmt.Print("\npress enter to stop")
+	fmt.Scanln()
+
+	q.Command("root", "poweroff")
+
+	for !q.Died {
+		time.Sleep(time.Second)
+	}
+	return
+}
+
+// inspired by Edd Turtle code
+func downloadFile(filepath string, url string) (err error) {
+	out, err := os.Create(filepath)
+	if err != nil {
+		return
+	}
+	defer out.Close()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		break
+	case http.StatusForbidden, http.StatusNotFound:
+		err = fmt.Errorf("Cannot download %s. It looks like you need "+
+			"to generate it manually and place it "+
+			"to ~/.out-of-tree/images/. "+
+			"Check documentation for additional information.", url)
+		return
+	default:
+		err = fmt.Errorf("Something weird happens while "+
+			"download file: %d", resp.StatusCode)
+		return
+	}
+
+	_, err = io.Copy(out, resp.Body)
+	return
+}
+
+func unpackTar(archive, destination string) (err error) {
+	// NOTE: If you're change anything in tar command please check also
+	// BSD tar (or if you're using macOS, do not forget to check GNU Tar)
+	// Also make sure that sparse files are extracting correctly
+	cmd := exec.Command("tar", "-Sxf", archive)
+	cmd.Dir = destination + "/"
+
+	log.Debug().Msgf("%v", cmd)
+
+	rawOutput, err := cmd.CombinedOutput()
+	if err != nil {
+		err = fmt.Errorf("%v: %s", err, rawOutput)
+		return
+	}
+
+	return
+}
+
+func downloadImage(path, file string) (err error) {
+	tmp, err := ioutil.TempDir(tempDirBase, "out-of-tree_")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(tmp)
+
+	archive := tmp + "/" + file + ".tar.gz"
+	url := imagesBaseURL + file + ".tar.gz"
+
+	err = downloadFile(archive, url)
+	if err != nil {
+		return
+	}
+
+	err = unpackTar(archive, path)
+	return
+}

kernel.go

@@ -1,4 +1,4 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Copyright 2023 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
@@ -8,33 +8,187 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"log"
+	"math"
+	"math/rand"
 	"os"
 	"os/exec"
+	"os/signal"
 	"os/user"
 	"regexp"
+	"runtime"
 	"strings"
+	"time"
 
-	"github.com/jollheef/out-of-tree/config"
 	"github.com/naoina/toml"
+	"github.com/rs/zerolog/log"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
 )
 
-func kernelListHandler(kcfg config.KernelConfig) (err error) {
+type KernelCmd struct {
+	NoDownload bool  `help:"do not download qemu image while kernel generation"`
+	UseHost    bool  `help:"also use host kernels"`
+	Force      bool  `help:"force reinstall kernel"`
+	NoHeaders  bool  `help:"do not install kernel headers"`
+	Shuffle    bool  `help:"randomize kernels installation order"`
+	Retries    int64 `help:"amount of tries for each kernel" default:"10"`
+
+	List        KernelListCmd        `cmd:"" help:"list kernels"`
+	Autogen     KernelAutogenCmd     `cmd:"" help:"generate kernels based on the current config"`
+	Genall      KernelGenallCmd      `cmd:"" help:"generate all kernels for distro"`
+	Install     KernelInstallCmd     `cmd:"" help:"install specific kernel"`
+	ConfigRegen KernelConfigRegenCmd `cmd:"" help:"regenerate config"`
+}
+
+type KernelListCmd struct{}
+
+func (cmd *KernelListCmd) Run(g *Globals) (err error) {
+	kcfg, err := config.ReadKernelConfig(g.Config.Kernels)
+	if err != nil {
+		log.Debug().Err(err).Msg("read kernel config")
+	}
+
 	if len(kcfg.Kernels) == 0 {
 		return errors.New("No kernels found")
 	}
+
 	for _, k := range kcfg.Kernels {
 		fmt.Println(k.DistroType, k.DistroRelease, k.KernelRelease)
 	}
+
 	return
 }
 
-func matchDebianKernelPkg(container, mask string, generic bool) (pkgs []string,
-	err error) {
+type KernelAutogenCmd struct {
+	Max int64 `help:"download kernels from set defined by regex in release_mask, but no more than X for each of release_mask" default:"100500"`
+}
 
-	cmd := "apt-cache search linux-image | cut -d ' ' -f 1"
-	c := dockerCommand(container, "/tmp", "1m", cmd)
-	rawOutput, err := c.CombinedOutput()
+func (cmd KernelAutogenCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
+	ka, err := config.ReadArtifactConfig(g.WorkDir + "/.out-of-tree.toml")
+	if err != nil {
+		return
+	}
+
+	shutdown := false
+	setSigintHandler(&shutdown)
+
+	for _, sk := range ka.SupportedKernels {
+		if sk.DistroRelease == "" {
+			err = errors.New("Please set distro_release")
+			return
+		}
+
+		err = generateKernels(sk,
+			g.Config.Docker.Registry,
+			g.Config.Docker.Commands,
+			cmd.Max, kernelCmd.Retries,
+			!kernelCmd.NoDownload,
+			kernelCmd.Force,
+			!kernelCmd.NoHeaders,
+			kernelCmd.Shuffle,
+			&shutdown,
+		)
+		if err != nil {
+			return
+		}
+		if shutdown {
+			break
+		}
+	}
+
+	return updateKernelsCfg(kernelCmd.UseHost, !kernelCmd.NoDownload)
+}
+
+type KernelGenallCmd struct {
+	Distro string `required:"" help:"distribution"`
+	Ver    string `required:"" help:"distro version"`
+}
+
+func (cmd *KernelGenallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
+	distroType, err := config.NewDistroType(cmd.Distro)
+	if err != nil {
+		return
+	}
+
+	shutdown := false
+	setSigintHandler(&shutdown)
+
+	km := config.KernelMask{
+		DistroType:    distroType,
+		DistroRelease: cmd.Ver,
+		ReleaseMask:   ".*",
+	}
+	err = generateKernels(km,
+		g.Config.Docker.Registry,
+		g.Config.Docker.Commands,
+		math.MaxUint32, kernelCmd.Retries,
+		!kernelCmd.NoDownload,
+		kernelCmd.Force,
+		!kernelCmd.NoHeaders,
+		kernelCmd.Shuffle,
+		&shutdown,
+	)
+	if err != nil {
+		return
+	}
+
+	return updateKernelsCfg(kernelCmd.UseHost, !kernelCmd.NoDownload)
+}
+
+type KernelInstallCmd struct {
+	Distro string `required:"" help:"distribution"`
+	Ver    string `required:"" help:"distro version"`
+	Kernel string `required:"" help:"kernel release mask"`
+}
+
+func (cmd *KernelInstallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
+	distroType, err := config.NewDistroType(cmd.Distro)
+	if err != nil {
+		return
+	}
+
+	shutdown := false
+	setSigintHandler(&shutdown)
+
+	km := config.KernelMask{
+		DistroType:    distroType,
+		DistroRelease: cmd.Ver,
+		ReleaseMask:   cmd.Kernel,
+	}
+	err = generateKernels(km,
+		g.Config.Docker.Registry,
+		g.Config.Docker.Commands,
+		math.MaxUint32, kernelCmd.Retries,
+		!kernelCmd.NoDownload,
+		kernelCmd.Force,
+		!kernelCmd.NoHeaders,
+		kernelCmd.Shuffle,
+		&shutdown,
+	)
+	if err != nil {
+		return
+	}
+
+	return updateKernelsCfg(kernelCmd.UseHost, !kernelCmd.NoDownload)
+}
+
+type KernelConfigRegenCmd struct{}
+
+func (cmd *KernelConfigRegenCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
+	return updateKernelsCfg(kernelCmd.UseHost, !kernelCmd.NoDownload)
+}
+
+func matchDebImagePkg(container, mask string) (pkgs []string, err error) {
+
+	cmd := "apt-cache search --names-only '^linux-image-[0-9\\.\\-]*-generic' | awk '{ print $1 }'"
+
+	// FIXME timeout should be in global out-of-tree config
+	c, err := NewContainer(container, time.Hour)
+	if err != nil {
+		return
+	}
+
+	output, err := c.Run(tempDirBase, cmd)
 	if err != nil {
 		return
 	}
@@ -44,14 +198,41 @@ func matchDebianKernelPkg(container, mask string, generic bool) (pkgs []string,
 		return
 	}
 
-	kernels := r.FindAll(rawOutput, -1)
-
-	for _, k := range kernels {
-		pkg := string(k)
-		if generic && !strings.HasSuffix(pkg, "generic") {
-			continue
+	for _, pkg := range strings.Fields(output) {
+		if r.MatchString(pkg) || strings.Contains(pkg, mask) {
+			pkgs = append(pkgs, pkg)
+		}
+	}
+
+	return
+}
+
+func matchCentOSDevelPkg(container, mask string, generic bool) (
+	pkgs []string, err error) {
+
+	cmd := "yum search kernel-devel --showduplicates | " +
+		"grep '^kernel-devel' | cut -d ' ' -f 1"
+
+	// FIXME timeout should be in global out-of-tree config
+	c, err := NewContainer(container, time.Hour)
+	if err != nil {
+		return
+	}
+
+	output, err := c.Run(tempDirBase, cmd)
+	if err != nil {
+		return
+	}
+
+	r, err := regexp.Compile("kernel-devel-" + mask)
+	if err != nil {
+		return
+	}
+
+	for _, pkg := range strings.Fields(output) {
+		if r.MatchString(pkg) || strings.Contains(pkg, mask) {
+			pkgs = append(pkgs, pkg)
 		}
-		pkgs = append(pkgs, pkg)
 	}
 
 	return
@@ -63,12 +244,32 @@ func dockerImagePath(sk config.KernelMask) (path string, err error) {
 		return
 	}
 
-	path = usr.HomeDir + "/.out-of-tree/"
+	path = usr.HomeDir + "/.out-of-tree/containers/"
 	path += sk.DistroType.String() + "/" + sk.DistroRelease
 	return
 }
 
-func generateBaseDockerImage(sk config.KernelMask) (err error) {
+func vsyscallAvailable() (available bool, err error) {
+	if runtime.GOOS != "linux" {
+		// Docker for non-Linux systems is not using the host
+		// kernel but uses kernel inside a virtual machine, so
+		// it builds by the Docker team with vsyscall support.
+		available = true
+		return
+	}
+
+	buf, err := ioutil.ReadFile("/proc/self/maps")
+	if err != nil {
+		return
+	}
+
+	available = strings.Contains(string(buf), "[vsyscall]")
+	return
+}
+
+func generateBaseDockerImage(registry string, commands []config.DockerCommand,
+	sk config.KernelMask) (err error) {
+
 	imagePath, err := dockerImagePath(sk)
 	if err != nil {
 		return
@@ -77,30 +278,104 @@ func generateBaseDockerImage(sk config.KernelMask) (err error) {
 
 	d := "# BASE\n"
 
-	if exists(dockerPath) {
-		log.Printf("Base image for %s:%s found",
-			sk.DistroType.String(), sk.DistroRelease)
+	// TODO move as function to container.go
+	cmd := exec.Command(containerRuntime, "images", "-q", sk.DockerName())
+	log.Debug().Msgf("run %v", cmd)
+
+	rawOutput, err := cmd.CombinedOutput()
+	if err != nil {
 		return
-	} else {
-		log.Printf("Base image for %s:%s not found, start generating",
-			sk.DistroType.String(), sk.DistroRelease)
-		os.MkdirAll(imagePath, os.ModePerm)
 	}
 
-	d += fmt.Sprintf("FROM %s:%s\n",
+	if exists(dockerPath) && string(rawOutput) != "" {
+		log.Info().Msgf("Base image for %s:%s found",
+			sk.DistroType.String(), sk.DistroRelease)
+		return
+	}
+
+	log.Info().Msgf("Base image for %s:%s not found, start generating",
+		sk.DistroType.String(), sk.DistroRelease)
+	os.MkdirAll(imagePath, os.ModePerm)
+
+	d += "FROM "
+	if registry != "" {
+		d += registry + "/"
+	}
+
+	d += fmt.Sprintf("%s:%s\n",
 		strings.ToLower(sk.DistroType.String()),
 		sk.DistroRelease,
 	)
 
+	vsyscall, err := vsyscallAvailable()
+	if err != nil {
+		return
+	}
+
+	for _, c := range commands {
+		d += "RUN " + c.Command + "\n"
+	}
+
 	switch sk.DistroType {
 	case config.Ubuntu:
 		d += "ENV DEBIAN_FRONTEND=noninteractive\n"
 		d += "RUN apt-get update\n"
 		d += "RUN apt-get install -y build-essential libelf-dev\n"
 		d += "RUN apt-get install -y wget git\n"
+		// Install a single kernel and headers to ensure all dependencies are cached
+		d += "RUN export PKGNAME=$(apt-cache search --names-only '^linux-headers-[0-9\\.\\-]*-generic' | awk '{ print $1 }' | head -n 1); " +
+			"apt-get install -y $PKGNAME $(echo $PKGNAME | sed 's/headers/image/'); " +
+			"apt-get remove -y $PKGNAME $(echo $PKGNAME | sed 's/headers/image/')\n"
+		if sk.DistroRelease >= "14.04" {
+			d += "RUN apt-get install -y libseccomp-dev\n"
+		}
+		d += "RUN mkdir -p /lib/modules\n"
+	case config.CentOS:
+		if sk.DistroRelease < "7" && !vsyscall {
+			log.Print("Old CentOS requires `vsyscall=emulate` " +
+				"on the latest kernels")
+			log.Print("Check out `A note about vsyscall` " +
+				"at https://hub.docker.com/_/centos")
+			log.Print("See also https://lwn.net/Articles/446528/")
+			err = fmt.Errorf("vsyscall is not available")
+			return
+		} else if sk.DistroRelease == "8" {
+			// CentOS 8 doesn't have Vault repos by default
+			for _, repover := range []string{
+				"8.0.1905", "8.1.1911", "8.2.2004", "8.3.2011", "8.4.2105", "8.5.2111",
+			} {
+				repo := fmt.Sprintf("[%s]\\nbaseurl=http://vault.centos.org/%s/BaseOS/$basearch/os/\\ngpgcheck=0", repover, repover)
+				d += fmt.Sprintf("RUN echo -e '%s' >> /etc/yum.repos.d/CentOS-Vault.repo\n", repo)
+			}
+			d += "RUN sed -i 's/enabled=1/enabled=0/' /etc/yum.repos.d/*\n"
+		}
+
+		// enable rpms from old minor releases
+		d += "RUN sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/CentOS-Vault.repo\n"
+		// do not remove old kernels
+		d += "RUN sed -i 's;installonly_limit=;installonly_limit=100500;' /etc/yum.conf\n"
+		d += "RUN yum -y update\n"
+
+		d += "RUN yum -y groupinstall 'Development Tools'\n"
+
+		if sk.DistroRelease < "8" {
+			d += "RUN yum -y install deltarpm\n"
+		} else {
+			d += "RUN yum -y install grub2-tools-minimal " +
+				"elfutils-libelf-devel\n"
+		}
+
+		var flags string
+		if sk.DistroRelease >= "8" {
+			flags = "--noautoremove"
+		}
+
+		// Cache kernel package dependencies
+		d += "RUN export PKGNAME=$(yum search kernel-devel --showduplicates | grep '^kernel-devel' | cut -d ' ' -f 1 | head -n 1); " +
+			"yum -y install $PKGNAME $(echo $PKGNAME | sed 's/-devel//'); " +
+			fmt.Sprintf("yum -y remove $PKGNAME $(echo $PKGNAME | sed 's/-devel//') %s\n", flags)
 	default:
-		s := fmt.Sprintf("%s not yet supported", sk.DistroType.String())
-		err = errors.New(s)
+		err = fmt.Errorf("%s not yet supported", sk.DistroType.String())
 		return
 	}
 
@@ -111,203 +386,177 @@ func generateBaseDockerImage(sk config.KernelMask) (err error) {
 		return
 	}
 
-	cmd := exec.Command("docker", "build", "-t", sk.DockerName(), imagePath)
-	rawOutput, err := cmd.CombinedOutput()
+	c, err := NewContainer(sk.DockerName(), time.Hour)
 	if err != nil {
-		log.Printf("Base image for %s:%s generating error, see log",
-			sk.DistroType.String(), sk.DistroRelease)
-		log.Println(string(rawOutput))
 		return
 	}
 
-	log.Printf("Base image for %s:%s generating success",
+	output, err := c.Build(imagePath)
+	if err != nil {
+		log.Error().Err(err).Msgf("Base image for %s:%s generating error",
+			sk.DistroType.String(), sk.DistroRelease)
+		log.Fatal().Msg(output)
+		return
+	}
+
+	log.Info().Msgf("Base image for %s:%s generating success",
 		sk.DistroType.String(), sk.DistroRelease)
 
 	return
 }
 
-func dockerImageAppend(sk config.KernelMask, pkgname string) (err error) {
-	imagePath, err := dockerImagePath(sk)
+func installKernel(sk config.KernelMask, pkgname string, force, headers bool) (err error) {
+	slog := log.With().
+		Str("distro_type", sk.DistroType.String()).
+		Str("distro_release", sk.DistroRelease).
+		Str("pkg", pkgname).
+		Logger()
+
+	c, err := NewContainer(sk.DockerName(), time.Hour) // TODO conf
 	if err != nil {
 		return
 	}
 
-	raw, err := ioutil.ReadFile(imagePath + "/Dockerfile")
+	moddirs, err := ioutil.ReadDir(c.Volumes.LibModules)
 	if err != nil {
 		return
 	}
 
-	if strings.Contains(string(raw), pkgname) {
-		// already installed kernel
-		log.Printf("kernel %s for %s:%s is already exists",
-			pkgname, sk.DistroType.String(), sk.DistroRelease)
-		return
+	for _, krel := range moddirs {
+		if strings.Contains(pkgname, krel.Name()) {
+			if force {
+				slog.Info().Msg("Reinstall")
+			} else {
+				slog.Info().Msg("Already installed")
+				return
+			}
+		}
 	}
 
-	log.Printf("Start adding kernel %s for %s:%s",
-		pkgname, sk.DistroType.String(), sk.DistroRelease)
+	volumes := c.Volumes
 
-	s := fmt.Sprintf("RUN apt-get install -y %s %s\n", pkgname,
-		strings.Replace(pkgname, "image", "headers", -1))
+	c.Volumes.LibModules = ""
+	c.Volumes.UsrSrc = ""
+	c.Volumes.Boot = ""
 
-	err = ioutil.WriteFile(imagePath+"/Dockerfile",
-		append(raw, []byte(s)...), 0644)
-	if err != nil {
-		return
-	}
+	slog.Debug().Msgf("Installing kernel")
 
-	cmd := exec.Command("docker", "build", "-t", sk.DockerName(), imagePath)
-	rawOutput, err := cmd.CombinedOutput()
-	if err != nil {
-		// Fallback to previous state
-		werr := ioutil.WriteFile(imagePath+"/Dockerfile", raw, 0644)
-		if werr != nil {
-			return
+	cmd := "true"
+
+	switch sk.DistroType {
+	case config.Ubuntu:
+		var headerspkg string
+		if headers {
+			headerspkg = strings.Replace(pkgname, "image", "headers", -1)
 		}
 
-		log.Printf("Add kernel %s for %s:%s error, see log",
-			pkgname, sk.DistroType.String(), sk.DistroRelease)
-		log.Println(string(rawOutput))
+		cmd += fmt.Sprintf(" && apt-get install -y %s %s", pkgname, headerspkg)
+	case config.CentOS:
+		imagepkg := strings.Replace(pkgname, "-devel", "", -1)
+
+		version := strings.Replace(pkgname, "kernel-devel-", "", -1)
+
+		if !headers {
+			pkgname = ""
+		}
+		cmd += fmt.Sprintf(" && yum -y install %s %s", imagepkg,
+			pkgname)
+
+		cmd += fmt.Sprintf(" && dracut --add-drivers 'e1000 ext4' -f "+
+			"/boot/initramfs-%s.img %s", version, version)
+	default:
+		err = fmt.Errorf("%s not yet supported", sk.DistroType.String())
 		return
 	}
 
-	log.Printf("Add kernel %s for %s:%s success",
-		pkgname, sk.DistroType.String(), sk.DistroRelease)
+	c.Args = append(c.Args, "-v", volumes.LibModules+":/target/lib/modules")
+	c.Args = append(c.Args, "-v", volumes.UsrSrc+":/target/usr/src")
+	c.Args = append(c.Args, "-v", volumes.Boot+":/target/boot")
 
-	return
-}
+	cmd += " && cp -r /boot /target/"
+	cmd += " && cp -r /lib/modules /target/lib/"
+	cmd += " && cp -r /usr/src /target/usr/"
 
-func kickImage(name string) (err error) {
-	cmd := exec.Command("docker", "run", name, "bash", "-c", "ls")
-	_, err = cmd.CombinedOutput()
-	return
-}
-
-func copyKernels(name string) (err error) {
-	cmd := exec.Command("docker", "ps", "-a")
-	rawOutput, err := cmd.CombinedOutput()
-	if err != nil {
-		log.Println(string(rawOutput))
-		return
-	}
-
-	r, err := regexp.Compile(".*" + name)
+	_, err = c.Run("", cmd)
 	if err != nil {
 		return
 	}
 
-	var containerID string
+	slog.Debug().Msgf("Success")
+	return
+}
 
-	what := r.FindAll(rawOutput, -1)
-	for _, w := range what {
-		containerID = strings.Fields(string(w))[0]
-		break
+func findKernelFile(files []os.FileInfo, kname string) (name string, err error) {
+	for _, file := range files {
+		if strings.HasPrefix(file.Name(), "vmlinuz") {
+			if strings.Contains(file.Name(), kname) {
+				name = file.Name()
+				return
+			}
+		}
 	}
 
+	err = errors.New("cannot find kernel")
+	return
+}
+
+func findInitrdFile(files []os.FileInfo, kname string) (name string, err error) {
+	for _, file := range files {
+		if strings.HasPrefix(file.Name(), "initrd") ||
+			strings.HasPrefix(file.Name(), "initramfs") {
+
+			if strings.Contains(file.Name(), kname) {
+				name = file.Name()
+				return
+			}
+		}
+	}
+
+	err = errors.New("cannot find kernel")
+	return
+}
+
+func genRootfsImage(d containerImageInfo, download bool) (rootfs string, err error) {
 	usr, err := user.Current()
 	if err != nil {
 		return
 	}
+	imageFile := d.Name + ".img"
 
-	target := usr.HomeDir + "/.out-of-tree/kernels/"
-	if !exists(target) {
-		os.MkdirAll(target, os.ModePerm)
-	}
+	imagesPath := usr.HomeDir + "/.out-of-tree/images/"
+	os.MkdirAll(imagesPath, os.ModePerm)
 
-	cmd = exec.Command("docker", "cp", containerID+":/boot/.", target)
-	rawOutput, err = cmd.CombinedOutput()
-	if err != nil {
-		log.Println(string(rawOutput))
-		return
-	}
-
-	return
-}
-
-func genKernelPath(files []os.FileInfo, kname string) string {
-	for _, file := range files {
-		if strings.Contains(file.Name(), "vmlinuz") {
-			if strings.Contains(file.Name(), kname) {
-				return file.Name()
-			}
+	rootfs = imagesPath + imageFile
+	if !exists(rootfs) {
+		if download {
+			log.Info().Msgf("%v not available, start download", imageFile)
+			err = downloadImage(imagesPath, imageFile)
 		}
 	}
-	return "unknown"
-}
-
-func genInitrdPath(files []os.FileInfo, kname string) string {
-	for _, file := range files {
-		if strings.Contains(file.Name(), "initrd") {
-			if strings.Contains(file.Name(), kname) {
-				return file.Name()
-			}
-		}
-	}
-	return "unknown"
-}
-
-func genRootfsImage(d dockerImageInfo) string {
-	usr, err := user.Current()
-	if err != nil {
-		return fmt.Sprintln(err)
-	}
-	imageFile := d.ContainerName + ".img"
-	return usr.HomeDir + "/.out-of-tree/images/" + imageFile
-}
-
-type dockerImageInfo struct {
-	ContainerName string
-	DistroType    config.DistroType
-	DistroRelease string // 18.04/7.4.1708/9.1
-}
-
-func listDockerImages() (diis []dockerImageInfo, err error) {
-	cmd := exec.Command("docker", "images")
-	rawOutput, err := cmd.CombinedOutput()
-	if err != nil {
-		return
-	}
-
-	r, err := regexp.Compile("out_of_tree_.*")
-	if err != nil {
-		return
-	}
-
-	containers := r.FindAll(rawOutput, -1)
-	for _, c := range containers {
-		container := strings.Fields(string(c))[0]
-
-		s := strings.Replace(container, "__", ".", -1)
-		values := strings.Split(s, "_")
-		distro, ver := values[3], values[4]
-
-		dii := dockerImageInfo{
-			ContainerName: container,
-			DistroRelease: ver,
-		}
-
-		dii.DistroType, err = config.NewDistroType(distro)
-		if err != nil {
-			return
-		}
-
-		diis = append(diis, dii)
-	}
 	return
 }
 
-func updateKernelsCfg() (err error) {
-	dockerImages, err := listDockerImages()
-	if err != nil {
-		return
-	}
-
+func updateKernelsCfg(host, download bool) (err error) {
 	newkcfg := config.KernelConfig{}
 
-	for _, d := range dockerImages {
-		err = genKernels(d, &newkcfg)
+	if host {
+		// Get host kernels
+		newkcfg, err = genHostKernels(download)
 		if err != nil {
-			log.Println("gen kernels", d.ContainerName, ":", err)
+			return
+		}
+	}
+
+	// Get docker kernels
+	dockerImages, err := listContainerImages()
+	if err != nil {
+		return
+	}
+
+	for _, d := range dockerImages {
+		err = listContainersKernels(d, &newkcfg, download)
+		if err != nil {
+			log.Print("gen kernels", d.Name, ":", err)
 			continue
 		}
 	}
@@ -338,45 +587,73 @@ func updateKernelsCfg() (err error) {
 		return
 	}
 
-	log.Println(kernelsCfgPath, "is successfully updated")
+	log.Info().Msgf("%s is successfully updated", kernelsCfgPath)
 	return
 }
 
-func genKernels(dii dockerImageInfo, newkcfg *config.KernelConfig) (
-	err error) {
+func listContainersKernels(dii containerImageInfo, newkcfg *config.KernelConfig,
+	download bool) (err error) {
 
-	name := dii.ContainerName
-	cmd := exec.Command("docker", "run", name, "ls", "/lib/modules")
-	rawOutput, err := cmd.CombinedOutput()
-	if err != nil {
-		log.Println(string(rawOutput), err)
-		return
-	}
-
-	usr, err := user.Current()
-	if err != nil {
-		return
-	}
-	kernelsBase := usr.HomeDir + "/.out-of-tree/kernels/"
-	files, err := ioutil.ReadDir(kernelsBase)
+	rootfs, err := genRootfsImage(dii, download)
 	if err != nil {
 		return
 	}
 
-	for _, k := range strings.Fields(string(rawOutput)) {
+	c, err := NewContainer(dii.Name, time.Hour)
+	if err != nil {
+		return
+	}
+
+	moddirs, err := ioutil.ReadDir(c.Volumes.LibModules)
+	if err != nil {
+		return
+	}
+
+	bootfiles, err := ioutil.ReadDir(c.Volumes.Boot)
+	if err != nil {
+		return
+	}
+
+	for _, krel := range moddirs {
+		log.Debug().Msgf("generate config entry for %s", krel.Name())
+
+		var kernelFile, initrdFile string
+		kernelFile, err = findKernelFile(bootfiles, krel.Name())
+		if err != nil {
+			log.Warn().Msgf("cannot find kernel %s", krel.Name())
+			continue
+		}
+
+		initrdFile, err = findInitrdFile(bootfiles, krel.Name())
+		if err != nil {
+			log.Warn().Msgf("cannot find initrd %s", krel.Name())
+			continue
+		}
+
 		ki := config.KernelInfo{
 			DistroType:    dii.DistroType,
 			DistroRelease: dii.DistroRelease,
-			KernelRelease: k,
-			ContainerName: name,
+			KernelRelease: krel.Name(),
+			ContainerName: dii.Name,
 
-			KernelPath: kernelsBase + genKernelPath(files, k),
-			InitrdPath: kernelsBase + genInitrdPath(files, k),
-			RootFS:     genRootfsImage(dii),
+			KernelPath:  c.Volumes.Boot + "/" + kernelFile,
+			InitrdPath:  c.Volumes.Boot + "/" + initrdFile,
+			ModulesPath: c.Volumes.LibModules + "/" + krel.Name(),
+
+			RootFS: rootfs,
 		}
 		newkcfg.Kernels = append(newkcfg.Kernels, ki)
 	}
 
+	for _, cmd := range []string{
+		"find /boot -type f -exec chmod a+r {} \\;",
+	} {
+		_, err = c.Run(tempDirBase, cmd)
+		if err != nil {
+			return
+		}
+	}
+
 	return
 }
 
@@ -389,88 +666,105 @@ func hasKernel(ki config.KernelInfo, kcfg config.KernelConfig) bool {
 	return false
 }
 
-func kernelAutogenHandler(workPath string) (err error) {
-	ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
-	if err != nil {
+func shuffleStrings(a []string) []string {
+	// Fisher–Yates shuffle
+	for i := len(a) - 1; i > 0; i-- {
+		j := rand.Intn(i + 1)
+		a[i], a[j] = a[j], a[i]
+	}
+	return a
+}
+
+func setSigintHandler(variable *bool) {
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+	go func() {
+		counter := 0
+		for _ = range c {
+			if counter == 0 {
+				*variable = true
+				log.Warn().Msg("shutdown requested, finishing work")
+				log.Info().Msg("^C a couple of times more for an unsafe exit")
+			} else if counter >= 3 {
+				log.Fatal().Msg("unsafe exit")
+			}
+
+			counter += 1
+		}
+	}()
+
+}
+
+func generateKernels(km config.KernelMask, registry string,
+	commands []config.DockerCommand, max, retries int64,
+	download, force, headers, shuffle bool, shutdown *bool) (err error) {
+
+	log.Info().Msgf("Generating for kernel mask %v", km)
+
+	_, err = genRootfsImage(containerImageInfo{Name: km.DockerName()},
+		download)
+	if err != nil || *shutdown {
 		return
 	}
 
-	for _, sk := range ka.SupportedKernels {
-		if sk.DistroRelease == "" {
-			err = errors.New("Please set distro_release")
+	err = generateBaseDockerImage(registry, commands, km)
+	if err != nil || *shutdown {
+		return
+	}
+
+	var pkgs []string
+	switch km.DistroType {
+	case config.Ubuntu:
+		pkgs, err = matchDebImagePkg(km.DockerName(), km.ReleaseMask)
+	case config.CentOS:
+		pkgs, err = matchCentOSDevelPkg(km.DockerName(),
+			km.ReleaseMask, true)
+	default:
+		err = fmt.Errorf("%s not yet supported", km.DistroType.String())
+	}
+	if err != nil || *shutdown {
+		return
+	}
+
+	if shuffle {
+		pkgs = shuffleStrings(pkgs)
+	}
+	for i, pkg := range pkgs {
+		if max <= 0 {
+			log.Print("Max is reached")
+			break
+		}
+
+		if *shutdown {
+			err = nil
 			return
 		}
+		log.Info().Msgf("%d/%d %s", i+1, len(pkgs), pkg)
 
-		err = generateBaseDockerImage(sk)
-		if err != nil {
-			return
-		}
+		var attempt int64
+		for {
+			attempt++
 
-		var pkgs []string
-		pkgs, err = matchDebianKernelPkg(sk.DockerName(),
-			sk.ReleaseMask, true)
-		if err != nil {
-			return
-		}
+			if *shutdown {
+				err = nil
+				return
+			}
 
-		for _, pkg := range pkgs {
-			dockerImageAppend(sk, pkg)
-		}
-
-		err = kickImage(sk.DockerName())
-		if err != nil {
-			log.Println("kick image", sk.DockerName(), ":", err)
-			continue
-		}
-
-		err = copyKernels(sk.DockerName())
-		if err != nil {
-			log.Println("copy kernels", sk.DockerName(), ":", err)
-			continue
+			err = installKernel(km, pkg, force, headers)
+			if err == nil {
+				max--
+				break
+			} else if attempt >= retries {
+				log.Error().Err(err).Msg("install kernel")
+				log.Debug().Msg("skip")
+				break
+			} else {
+				log.Warn().Err(err).Msg("install kernel")
+				time.Sleep(time.Second)
+				log.Info().Msg("retry")
+			}
 		}
 	}
 
-	err = updateKernelsCfg()
 	return
 }
-
-func kernelDockerRegenHandler() (err error) {
-	dockerImages, err := listDockerImages()
-	if err != nil {
-		return
-	}
-
-	for _, d := range dockerImages {
-		var imagePath string
-		imagePath, err = dockerImagePath(config.KernelMask{
-			DistroType:    d.DistroType,
-			DistroRelease: d.DistroRelease,
-		})
-		if err != nil {
-			return
-		}
-
-		cmd := exec.Command("docker", "build", "-t",
-			d.ContainerName, imagePath)
-		var rawOutput []byte
-		rawOutput, err = cmd.CombinedOutput()
-		if err != nil {
-			log.Println("docker build:", string(rawOutput))
-			return
-		}
-
-		err = kickImage(d.ContainerName)
-		if err != nil {
-			log.Println("kick image", d.ContainerName, ":", err)
-			continue
-		}
-
-		err = copyKernels(d.ContainerName)
-		if err != nil {
-			log.Println("copy kernels", d.ContainerName, ":", err)
-			continue
-		}
-	}
-
-	return updateKernelsCfg()
-}

kernel_linux.go

@@ -0,0 +1,96 @@
+// Copyright 2023 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+//go:build linux
+// +build linux
+
+package main
+
+import (
+	"io/ioutil"
+	"os/exec"
+	"strings"
+
+	"github.com/rs/zerolog/log"
+	"github.com/zcalusic/sysinfo"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+)
+
+func genHostKernels(download bool) (kcfg config.KernelConfig, err error) {
+	si := sysinfo.SysInfo{}
+	si.GetSysInfo()
+
+	distroType, err := config.NewDistroType(si.OS.Vendor)
+	if err != nil {
+		return
+	}
+
+	cmd := exec.Command("ls", "/lib/modules")
+	log.Debug().Msgf("%v", cmd)
+
+	rawOutput, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Print(string(rawOutput), err)
+		return
+	}
+
+	kernelsBase := "/boot/"
+	bootfiles, err := ioutil.ReadDir(kernelsBase)
+	if err != nil {
+		return
+	}
+
+	// only for compatibility, docker is not really used
+	dii := containerImageInfo{
+		Name: config.KernelMask{
+			DistroType:    distroType,
+			DistroRelease: si.OS.Version,
+		}.DockerName(),
+	}
+
+	rootfs, err := genRootfsImage(dii, download)
+	if err != nil {
+		return
+	}
+
+	for _, krel := range strings.Fields(string(rawOutput)) {
+		log.Debug().Msgf("generate config entry for %s", krel)
+
+		var kernelFile, initrdFile string
+		kernelFile, err = findKernelFile(bootfiles, krel)
+		if err != nil {
+			log.Warn().Msgf("cannot find kernel %s", krel)
+			continue
+		}
+
+		initrdFile, err = findInitrdFile(bootfiles, krel)
+		if err != nil {
+			log.Warn().Msgf("cannot find initrd %s", krel)
+			continue
+		}
+
+		ki := config.KernelInfo{
+			DistroType:    distroType,
+			DistroRelease: si.OS.Version,
+			KernelRelease: krel,
+
+			KernelSource: "/lib/modules/" + krel + "/build",
+
+			KernelPath: kernelsBase + kernelFile,
+			InitrdPath: kernelsBase + initrdFile,
+			RootFS:     rootfs,
+		}
+
+		vmlinux := "/usr/lib/debug/boot/vmlinux-" + krel
+		log.Print("vmlinux", vmlinux)
+		if exists(vmlinux) {
+			ki.VmlinuxPath = vmlinux
+		}
+
+		kcfg.Kernels = append(kcfg.Kernels, ki)
+	}
+
+	return
+}

kernel_macos.go

@@ -0,0 +1,18 @@
+// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+// +build darwin
+
+package main
+
+import (
+	"errors"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+)
+
+func genHostKernels(download bool) (kcfg config.KernelConfig, err error) {
+	err = errors.New("generate host kernels for macOS is not supported")
+	return
+}

log.go

@@ -0,0 +1,299 @@
+// Copyright 2023 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package main
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"math"
+	"os"
+
+	"github.com/olekukonko/tablewriter"
+	"github.com/rs/zerolog/log"
+	"gopkg.in/logrusorgru/aurora.v2"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+)
+
+type LogCmd struct {
+	Query    LogQueryCmd    `cmd:"" help:"query logs"`
+	Dump     LogDumpCmd     `cmd:"" help:"show all info for log entry with ID"`
+	Json     LogJsonCmd     `cmd:"" help:"generate json statistics"`
+	Markdown LogMarkdownCmd `cmd:"" help:"generate markdown statistics"`
+}
+
+type LogQueryCmd struct {
+	Num  int    `help:"how much lines" default:"50"`
+	Rate bool   `help:"show artifact success rate"`
+	Tag  string `help:"filter tag"`
+}
+
+func (cmd *LogQueryCmd) Run(g *Globals) (err error) {
+	db, err := openDatabase(g.Config.Database)
+	if err != nil {
+		panic(err)
+	}
+	defer db.Close()
+
+	var les []logEntry
+
+	ka, kaErr := config.ReadArtifactConfig(g.WorkDir + "/.out-of-tree.toml")
+	if kaErr == nil {
+		log.Print(".out-of-tree.toml found, filter by artifact name")
+		les, err = getAllArtifactLogs(db, cmd.Tag, cmd.Num, ka)
+	} else {
+		les, err = getAllLogs(db, cmd.Tag, cmd.Num)
+	}
+	if err != nil {
+		return
+	}
+
+	s := "\nS"
+	if cmd.Rate {
+		if kaErr != nil {
+			err = kaErr
+			return
+		}
+
+		s = fmt.Sprintf("{[%s] %s} Overall s", ka.Type, ka.Name)
+
+		les, err = getAllArtifactLogs(db, cmd.Tag, math.MaxInt64, ka)
+		if err != nil {
+			return
+		}
+	} else {
+		for _, l := range les {
+			logLogEntry(l)
+		}
+	}
+
+	success := 0
+	for _, l := range les {
+		if l.Test.Ok {
+			success++
+		}
+	}
+
+	overall := float64(success) / float64(len(les))
+	fmt.Printf("%success rate: %.04f (~%.0f%%)\n",
+		s, overall, overall*100)
+
+	return
+}
+
+type LogDumpCmd struct {
+	ID int `help:"id" default:"-1"`
+}
+
+func (cmd *LogDumpCmd) Run(g *Globals) (err error) {
+	db, err := openDatabase(g.Config.Database)
+	if err != nil {
+		panic(err)
+	}
+	defer db.Close()
+
+	var l logEntry
+	if cmd.ID > 0 {
+		l, err = getLogByID(db, cmd.ID)
+	} else {
+		l, err = getLastLog(db)
+	}
+	if err != nil {
+		return
+	}
+
+	fmt.Println("ID:", l.ID)
+	fmt.Println("Date:", l.Timestamp)
+	fmt.Println("Tag:", l.Tag)
+	fmt.Println()
+
+	fmt.Println("Type:", l.Type.String())
+	fmt.Println("Name:", l.Name)
+	fmt.Println()
+
+	fmt.Println("Distro:", l.DistroType.String(), l.DistroRelease)
+	fmt.Println("Kernel:", l.KernelRelease)
+	fmt.Println()
+
+	fmt.Println("Build ok:", l.Build.Ok)
+	if l.Type == config.KernelModule {
+		fmt.Println("Insmod ok:", l.Run.Ok)
+	}
+	fmt.Println("Test ok:", l.Test.Ok)
+	fmt.Println()
+
+	fmt.Printf("Build output:\n%s\n", l.Build.Output)
+	fmt.Println()
+
+	if l.Type == config.KernelModule {
+		fmt.Printf("Insmod output:\n%s\n", l.Run.Output)
+		fmt.Println()
+	}
+
+	fmt.Printf("Test output:\n%s\n", l.Test.Output)
+	fmt.Println()
+
+	fmt.Printf("Qemu stdout:\n%s\n", l.Stdout)
+	fmt.Println()
+
+	fmt.Printf("Qemu stderr:\n%s\n", l.Stderr)
+	fmt.Println()
+
+	return
+}
+
+type LogJsonCmd struct {
+	Tag string `required:"" help:"filter tag"`
+}
+
+func (cmd *LogJsonCmd) Run(g *Globals) (err error) {
+	db, err := openDatabase(g.Config.Database)
+	if err != nil {
+		panic(err)
+	}
+	defer db.Close()
+
+	distros, err := getStats(db, g.WorkDir, cmd.Tag)
+	if err != nil {
+		return
+	}
+
+	bytes, err := json.Marshal(&distros)
+	if err != nil {
+		return
+	}
+
+	fmt.Println(string(bytes))
+	return
+}
+
+type LogMarkdownCmd struct {
+	Tag string `required:"" help:"filter tag"`
+}
+
+func (cmd *LogMarkdownCmd) Run(g *Globals) (err error) {
+	db, err := openDatabase(g.Config.Database)
+	if err != nil {
+		panic(err)
+	}
+	defer db.Close()
+
+	distros, err := getStats(db, g.WorkDir, cmd.Tag)
+	if err != nil {
+		return
+	}
+
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Distro", "Release", "Kernel", "Reliability"})
+	table.SetBorders(tablewriter.Border{
+		Left: true, Top: false, Right: true, Bottom: false})
+	table.SetCenterSeparator("|")
+
+	for distro, releases := range distros {
+		for release, kernels := range releases {
+			for kernel, stats := range kernels {
+				all := float64(stats.All)
+				ok := float64(stats.TestOK)
+				r := fmt.Sprintf("%6.02f%%", (ok/all)*100)
+				table.Append([]string{distro, release, kernel, r})
+			}
+		}
+	}
+
+	table.Render()
+	return
+}
+
+func logLogEntry(l logEntry) {
+	distroInfo := fmt.Sprintf("%s-%s {%s}", l.DistroType,
+		l.DistroRelease, l.KernelRelease)
+
+	artifactInfo := fmt.Sprintf("{[%s] %s}", l.Type, l.Name)
+
+	colored := ""
+	if l.Type == config.KernelExploit {
+		colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s",
+			l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo,
+			genOkFail("BUILD", l.Build.Ok),
+			genOkFail("LPE", l.Test.Ok))
+	} else {
+		colored = aurora.Sprintf("[%4d %4s] [%s] %40s %40s: %s %s %s",
+			l.ID, l.Tag, l.Timestamp, artifactInfo, distroInfo,
+			genOkFail("BUILD", l.Build.Ok),
+			genOkFail("INSMOD", l.Run.Ok),
+			genOkFail("TEST", l.Test.Ok))
+	}
+
+	additional := ""
+	if l.KernelPanic {
+		additional = "(panic)"
+	} else if l.KilledByTimeout {
+		additional = "(timeout)"
+	}
+
+	if additional != "" {
+		fmt.Println(colored, additional)
+	} else {
+		fmt.Println(colored)
+	}
+}
+
+type runstat struct {
+	All, BuildOK, RunOK, TestOK, Timeout, Panic int
+}
+
+func getStats(db *sql.DB, path, tag string) (
+	distros map[string]map[string]map[string]runstat, err error) {
+
+	var les []logEntry
+
+	ka, kaErr := config.ReadArtifactConfig(path + "/.out-of-tree.toml")
+	if kaErr == nil {
+		les, err = getAllArtifactLogs(db, tag, -1, ka)
+	} else {
+		les, err = getAllLogs(db, tag, -1)
+	}
+	if err != nil {
+		return
+	}
+
+	distros = make(map[string]map[string]map[string]runstat)
+
+	for _, l := range les {
+		_, ok := distros[l.DistroType.String()]
+		if !ok {
+			distros[l.DistroType.String()] = make(map[string]map[string]runstat)
+		}
+
+		_, ok = distros[l.DistroType.String()][l.DistroRelease]
+		if !ok {
+			distros[l.DistroType.String()][l.DistroRelease] = make(map[string]runstat)
+		}
+
+		rs := distros[l.DistroType.String()][l.DistroRelease][l.KernelRelease]
+
+		rs.All++
+		if l.Build.Ok {
+			rs.BuildOK++
+		}
+		if l.Run.Ok {
+			rs.RunOK++
+		}
+		if l.Test.Ok {
+			rs.TestOK++
+		}
+		if l.KernelPanic {
+			rs.Panic++
+		}
+		if l.KilledByTimeout {
+			rs.Timeout++
+		}
+
+		distros[l.DistroType.String()][l.DistroRelease][l.KernelRelease] = rs
+	}
+
+	return
+}

main.go

@@ -1,136 +1,177 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Copyright 2023 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
 package main
 
 import (
-	"log"
+	"fmt"
+	"io"
+	"math/rand"
 	"os"
 	"os/exec"
 	"os/user"
+	"runtime/debug"
+	"strconv"
+	"time"
 
-	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"github.com/natefinch/lumberjack"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 
-	"github.com/jollheef/out-of-tree/config"
+	"github.com/alecthomas/kong"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
 )
 
+type Globals struct {
+	Config config.OutOfTree `help:"path to out-of-tree configuration" default:"~/.out-of-tree/out-of-tree.toml"`
+
+	WorkDir string `help:"path to work directory" default:"./" type:"path"`
+}
+
+type CLI struct {
+	Globals
+
+	Pew       PewCmd       `cmd:"" help:"build, run, and test module/exploit"`
+	Kernel    KernelCmd    `cmd:"" help:"manipulate kernels"`
+	Debug     DebugCmd     `cmd:"" help:"debug environment"`
+	Log       LogCmd       `cmd:"" help:"query logs"`
+	Pack      PackCmd      `cmd:"" help:"exploit pack test"`
+	Gen       GenCmd       `cmd:"" help:"generate .out-of-tree.toml skeleton"`
+	Image     ImageCmd     `cmd:"" help:"manage images"`
+	Container ContainerCmd `cmd:"" help:"manage containers"`
+
+	Version VersionFlag `name:"version" help:"print version information and quit"`
+
+	LogLevel LogLevelFlag `enum:"trace,debug,info,warn,error" default:"info"`
+
+	ContainerRuntime string `enum:"podman,docker" default:"podman"`
+}
+
+type LogLevelFlag string
+
+func (loglevel LogLevelFlag) AfterApply() error {
+	switch loglevel {
+	case "debug", "trace":
+		zerolog.CallerMarshalFunc = func(pc uintptr, file string, line int) string {
+			short := file
+			for i := len(file) - 1; i > 0; i-- {
+				if file[i] == '/' {
+					short = file[i+1:]
+					break
+				}
+			}
+			file = short
+			return file + ":" + strconv.Itoa(line)
+		}
+		log.Logger = log.With().Caller().Logger()
+	}
+	return nil
+}
+
+type VersionFlag string
+
+func (v VersionFlag) Decode(ctx *kong.DecodeContext) error { return nil }
+func (v VersionFlag) IsBool() bool                         { return true }
+func (v VersionFlag) BeforeApply(app *kong.Kong, vars kong.Vars) error {
+	fmt.Println(vars["version"])
+	app.Exit(0)
+	return nil
+}
+
+type LevelWriter struct {
+	io.Writer
+	Level zerolog.Level
+}
+
+func (lw *LevelWriter) WriteLevel(l zerolog.Level, p []byte) (n int, err error) {
+	if l >= lw.Level {
+		return lw.Writer.Write(p)
+	}
+	return len(p), nil
+}
+
+var tempDirBase string
+
 func main() {
-	app := kingpin.New(
-		"out-of-tree",
-		"kernel {module, exploit} development tool",
+	rand.Seed(time.Now().UnixNano())
+
+	cli := CLI{}
+	ctx := kong.Parse(&cli,
+		kong.Name("out-of-tree"),
+		kong.Description("kernel {module, exploit} development tool"),
+		kong.UsageOnError(),
+		kong.ConfigureHelp(kong.HelpOptions{
+			Compact: true,
+		}),
+		kong.Vars{
+			"version": "2.1.0",
+		},
 	)
 
-	app.Author("Mikhail Klementev <jollheef@riseup.net>")
-	app.Version("0.1.0")
-
-	pathFlag := app.Flag("path", "Path to work directory")
-	path := pathFlag.Default(".").ExistingDir()
+	var loglevel zerolog.Level
+	switch cli.LogLevel {
+	case "trace":
+		loglevel = zerolog.TraceLevel
+	case "debug":
+		loglevel = zerolog.DebugLevel
+	case "info":
+		loglevel = zerolog.InfoLevel
+	case "warn":
+		loglevel = zerolog.WarnLevel
+	case "error":
+		loglevel = zerolog.ErrorLevel
+	}
 
 	usr, err := user.Current()
 	if err != nil {
 		return
 	}
-	defaultKcfgPath := usr.HomeDir + "/.out-of-tree/kernels.toml"
 
-	kcfgPathFlag := app.Flag("kernels", "Path to main kernels config")
-	kcfgPath := kcfgPathFlag.Default(defaultKcfgPath).ExistingFile()
+	tempDirBase = usr.HomeDir + "/.out-of-tree/tmp/"
+	os.MkdirAll(tempDirBase, os.ModePerm)
 
-	defaultUserKcfgPath := usr.HomeDir + "/.out-of-tree/kernels.user.toml"
-	userKcfgPathFlag := app.Flag("user-kernels", "User kernels config")
-	userKcfgPathEnv := userKcfgPathFlag.Envar("OUT_OF_TREE_KCFG")
-	userKcfgPath := userKcfgPathEnv.Default(defaultUserKcfgPath).String()
+	log.Logger = log.Output(zerolog.MultiLevelWriter(
+		&LevelWriter{Writer: zerolog.NewConsoleWriter(
+			func(w *zerolog.ConsoleWriter) {
+				w.Out = os.Stderr
+			},
+		),
+			Level: loglevel,
+		},
+		&LevelWriter{Writer: &lumberjack.Logger{
+			Filename: usr.HomeDir + "/.out-of-tree/logs/out-of-tree.log",
+		},
+			Level: zerolog.TraceLevel,
+		},
+	))
 
-	qemuTimeoutFlag := app.Flag("qemu-timeout", "Timeout for qemu")
-	qemuTimeout := qemuTimeoutFlag.Default("1m").Duration()
+	log.Trace().Msg("start out-of-tree")
+	log.Debug().Msgf("%v", os.Args)
+	log.Debug().Msgf("%v", cli)
 
-	dockerTimeoutFlag := app.Flag("docker-timeout", "Timeout for docker")
-	dockerTimeout := dockerTimeoutFlag.Default("1m").Duration()
-	pewCommand := app.Command("pew", "Build, run and test module/exploit")
-	pewKernelFlag := pewCommand.Flag("kernel", "Override kernel regex")
-	pewKernel := pewKernelFlag.String()
-
-	pewGuessFlag := pewCommand.Flag("guess", "Try all defined kernels")
-	pewGuess := pewGuessFlag.Bool()
-
-	pewBinaryFlag := pewCommand.Flag("binary", "Use binary, do not build")
-	pewBinary := pewBinaryFlag.String()
-
-	pewTestFlag := pewCommand.Flag("test", "Override path test")
-	pewTest := pewTestFlag.String()
-
-	kernelCommand := app.Command("kernel", "Manipulate kernels")
-	kernelListCommand := kernelCommand.Command("list", "List kernels")
-	kernelAutogenCommand := kernelCommand.Command("autogen",
-		"Generate kernels based on a current config")
-	kernelDockerRegenCommand := kernelCommand.Command("docker-regen",
-		"Regenerate kernels config from out_of_tree_* docker images")
-
-	genCommand := app.Command("gen", "Generate .out-of-tree.toml skeleton")
-	genModuleCommand := genCommand.Command("module",
-		"Generate .out-of-tree.toml skeleton for kernel module")
-	genExploitCommand := genCommand.Command("exploit",
-		"Generate .out-of-tree.toml skeleton for kernel exploit")
-
-	debugCommand := app.Command("debug", "Kernel debug environment")
-	debugCommandFlag := debugCommand.Flag("kernel", "Regex (first match)")
-	debugKernel := debugCommandFlag.Required().String()
-	debugFlagGDB := debugCommand.Flag("gdb", "Set gdb listen address")
-	debugGDB := debugFlagGDB.Default("tcp::1234").String()
-
-	bootstrapCommand := app.Command("bootstrap",
-		"Create directories && download images")
-
-	// Check for required commands
-	for _, cmd := range []string{"timeout", "docker", "qemu"} {
-		_, err := exec.Command("which", cmd).CombinedOutput()
-		if err != nil {
-			log.Fatalln("Command not found:", cmd)
-		}
+	if buildInfo, ok := debug.ReadBuildInfo(); ok {
+		log.Debug().Msgf("%v", buildInfo.GoVersion)
+		log.Debug().Msgf("%v", buildInfo.Settings)
 	}
 
-	kingpin.MustParse(app.Parse(os.Args[1:]))
-
-	kcfg, err := config.ReadKernelConfig(*kcfgPath)
+	_, err = exec.LookPath(cli.ContainerRuntime)
 	if err != nil {
-		log.Fatalln(err)
-	}
+		if cli.ContainerRuntime == "podman" { // default value
+			log.Debug().Msgf("podman is not found in $PATH, " +
+				"fall back to docker")
+			cli.ContainerRuntime = "docker"
+		}
 
-	if exists(*userKcfgPath) {
-		userKcfg, err := config.ReadKernelConfig(*userKcfgPath)
+		_, err = exec.LookPath(cli.ContainerRuntime)
 		if err != nil {
-			log.Fatalln(err)
-		}
-
-		for _, nk := range userKcfg.Kernels {
-			if !hasKernel(nk, kcfg) {
-				kcfg.Kernels = append(kcfg.Kernels, nk)
-			}
+			log.Fatal().Msgf("%v is not found in $PATH",
+				cli.ContainerRuntime)
 		}
 	}
+	containerRuntime = cli.ContainerRuntime
 
-	switch kingpin.MustParse(app.Parse(os.Args[1:])) {
-	case pewCommand.FullCommand():
-		err = pewHandler(kcfg, *path, *pewKernel, *pewBinary,
-			*pewTest, *pewGuess, *qemuTimeout, *dockerTimeout)
-	case kernelListCommand.FullCommand():
-		err = kernelListHandler(kcfg)
-	case kernelAutogenCommand.FullCommand():
-		err = kernelAutogenHandler(*path)
-	case kernelDockerRegenCommand.FullCommand():
-		err = kernelDockerRegenHandler()
-	case genModuleCommand.FullCommand():
-		err = genConfig(config.KernelModule)
-	case genExploitCommand.FullCommand():
-		err = genConfig(config.KernelExploit)
-	case debugCommand.FullCommand():
-		err = debugHandler(kcfg, *path, *debugKernel, *debugGDB,
-			*dockerTimeout)
-	case bootstrapCommand.FullCommand():
-		err = bootstrapHandler()
-	}
-
-	if err != nil {
-		log.Fatalln(err)
-	}
+	err = ctx.Run(&cli.Globals)
+	ctx.FatalIfErrorf(err)
 }

pack.go

@@ -0,0 +1,84 @@
+// Copyright 2023 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"time"
+
+	"github.com/rs/zerolog/log"
+)
+
+type PackCmd struct {
+	Autogen     bool  `help:"kernel autogeneration"`
+	UseHost     bool  `help:"also use host kernels"`
+	NoDownload  bool  `help:"do not download qemu image while kernel generation"`
+	ExploitRuns int64 `default:"4" help:"amount of runs of each exploit"`
+	KernelRuns  int64 `default:"1" help:"amount of runs of each kernel"`
+	Max         int64 `help:"download random kernels from set defined by regex in release_mask, but no more than X for each of release_mask" default:"1"`
+
+	Threads int `help:"threads" default:"4"`
+
+	Tag string `help:"filter tag"`
+
+	Timeout       time.Duration `help:"timeout after tool will not spawn new tests"`
+	QemuTimeout   time.Duration `help:"timeout for qemu"`
+	DockerTimeout time.Duration `help:"timeout for docker"`
+}
+
+func (cmd *PackCmd) Run(g *Globals) (err error) {
+	tag := fmt.Sprintf("pack_run_%d", time.Now().Unix())
+	log.Print("Tag:", tag)
+
+	files, err := ioutil.ReadDir(g.WorkDir)
+	if err != nil {
+		return
+	}
+
+	for _, f := range files {
+		workPath := g.WorkDir + "/" + f.Name()
+
+		if !exists(workPath + "/.out-of-tree.toml") {
+			continue
+		}
+
+		if cmd.Autogen {
+			err = KernelAutogenCmd{Max: cmd.Max}.Run(
+				&KernelCmd{
+					NoDownload: cmd.NoDownload,
+					UseHost:    cmd.UseHost,
+				},
+				&Globals{
+					Config:  g.Config,
+					WorkDir: workPath,
+				},
+			)
+			if err != nil {
+				return
+			}
+		}
+
+		log.Print(f.Name())
+
+		pew := PewCmd{
+			Max:           cmd.KernelRuns,
+			Runs:          cmd.ExploitRuns,
+			Threads:       cmd.Threads,
+			Tag:           tag,
+			Timeout:       cmd.Timeout,
+			QemuTimeout:   cmd.QemuTimeout,
+			DockerTimeout: cmd.DockerTimeout,
+			Dist:          pathDevNull,
+		}
+
+		pew.Run(&Globals{
+			Config:  g.Config,
+			WorkDir: workPath,
+		})
+	}
+
+	return
+}

pew.go

@@ -1,85 +1,302 @@
-// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Copyright 2023 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
 package main
 
 import (
+	"bufio"
+	"database/sql"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
-	"log"
 	"math/rand"
 	"os"
 	"os/exec"
-	"runtime"
 	"strings"
 	"time"
 
-	"github.com/logrusorgru/aurora"
 	"github.com/otiai10/copy"
 	"github.com/remeh/sizedwaitgroup"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"gopkg.in/logrusorgru/aurora.v2"
 
-	"github.com/jollheef/out-of-tree/config"
-	qemu "github.com/jollheef/out-of-tree/qemu"
+	"code.dumpstack.io/tools/out-of-tree/config"
+	"code.dumpstack.io/tools/out-of-tree/qemu"
 )
 
-func dockerCommand(container, workdir, timeout, command string) *exec.Cmd {
-	return exec.Command("timeout", "-k", timeout, timeout, "docker", "run",
-		"-v", workdir+":/work", container,
-		"bash", "-c", "cd /work && "+command)
+type PewCmd struct {
+	Max     int64         `help:"test no more than X kernels" default:"100500"`
+	Runs    int64         `help:"runs per each kernel" default:"1"`
+	Kernel  string        `help:"override kernel regex"`
+	Guess   bool          `help:"try all defined kernels"`
+	Shuffle bool          `help:"randomize kernels test order"`
+	Binary  string        `help:"use binary, do not build"`
+	Test    string        `help:"override path for test"`
+	Dist    string        `help:"build result path" default:"/dev/null"`
+	Threads int           `help:"threads" default:"1"`
+	Tag     string        `help:"log tagging"`
+	Timeout time.Duration `help:"timeout after tool will not spawn new tests"`
+
+	ArtifactConfig string `help:"path to artifact config" type:"path"`
+
+	QemuTimeout   time.Duration `help:"timeout for qemu"`
+	DockerTimeout time.Duration `help:"timeout for docker"`
+
+	Threshold float64 `help:"reliablity threshold for exit code" default:"1.00"`
+
+	db              *sql.DB
+	kcfg            config.KernelConfig
+	timeoutDeadline time.Time
+}
+
+func (cmd *PewCmd) Run(g *Globals) (err error) {
+	cmd.kcfg, err = config.ReadKernelConfig(g.Config.Kernels)
+	if err != nil {
+		log.Fatal().Err(err).Msg("read kernels config")
+	}
+
+	if cmd.Timeout != 0 {
+		log.Info().Msgf("Set global timeout to %s", cmd.Timeout)
+		cmd.timeoutDeadline = time.Now().Add(cmd.Timeout)
+	}
+
+	cmd.db, err = openDatabase(g.Config.Database)
+	if err != nil {
+		log.Fatal().Err(err).
+			Msgf("Cannot open database %s", g.Config.Database)
+	}
+	defer cmd.db.Close()
+
+	var configPath string
+	if cmd.ArtifactConfig == "" {
+		configPath = g.WorkDir + "/.out-of-tree.toml"
+	} else {
+		configPath = cmd.ArtifactConfig
+	}
+	ka, err := config.ReadArtifactConfig(configPath)
+	if err != nil {
+		return
+	}
+
+	if ka.SourcePath == "" {
+		ka.SourcePath = g.WorkDir
+	}
+
+	if cmd.Kernel != "" {
+		var km config.KernelMask
+		km, err = kernelMask(cmd.Kernel)
+		if err != nil {
+			return
+		}
+
+		ka.SupportedKernels = []config.KernelMask{km}
+	}
+
+	if cmd.Guess {
+		ka.SupportedKernels, err = genAllKernels()
+		if err != nil {
+			return
+		}
+	}
+
+	if cmd.QemuTimeout != 0 {
+		log.Info().Msgf("Set qemu timeout to %s", cmd.QemuTimeout)
+	} else {
+		cmd.QemuTimeout = g.Config.Qemu.Timeout.Duration
+	}
+
+	if cmd.DockerTimeout != 0 {
+		log.Info().Msgf("Set docker timeout to %s", cmd.DockerTimeout)
+	} else {
+		cmd.DockerTimeout = g.Config.Docker.Timeout.Duration
+	}
+
+	if cmd.Tag == "" {
+		cmd.Tag = fmt.Sprintf("%d", time.Now().Unix())
+	}
+	log.Info().Str("tag", cmd.Tag).Msg("log")
+
+	err = cmd.performCI(ka)
+	if err != nil {
+		return
+	}
+
+	log.Info().Msgf("Success rate: %.02f, Threshold: %.02f",
+		successRate(state), cmd.Threshold)
+	if successRate(state) < cmd.Threshold {
+		err = errors.New("reliability threshold not met")
+	}
+	return
+}
+
+type runstate struct {
+	Overall, Success float64
+}
+
+var (
+	state runstate
+)
+
+func successRate(state runstate) float64 {
+	return state.Success / state.Overall
+}
+
+const pathDevNull = "/dev/null"
+
+func sh(workdir, command string) (output string, err error) {
+	flog := log.With().
+		Str("workdir", workdir).
+		Str("command", command).
+		Logger()
+
+	cmd := exec.Command("sh", "-c", "cd "+workdir+" && "+command)
+
+	flog.Debug().Msgf("%v", cmd)
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+	cmd.Stderr = cmd.Stdout
+
+	err = cmd.Start()
+	if err != nil {
+		return
+	}
+
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			m := scanner.Text()
+			output += m + "\n"
+			flog.Trace().Str("stdout", m).Msg("")
+		}
+	}()
+
+	err = cmd.Wait()
+
+	if err != nil {
+		e := fmt.Sprintf("%v %v output: %v", cmd, err, output)
+		err = errors.New(e)
+	}
+	return
+}
+
+func applyPatches(src string, ka config.Artifact) (err error) {
+	for i, patch := range ka.Patches {
+		name := fmt.Sprintf("patch_%02d", i)
+
+		path := src + "/" + name + ".diff"
+		if patch.Source != "" && patch.Path != "" {
+			err = errors.New("path and source are mutually exclusive")
+			return
+		} else if patch.Source != "" {
+			err = os.WriteFile(path, []byte(patch.Source), 0644)
+			if err != nil {
+				return
+			}
+		} else if patch.Path != "" {
+			err = copy.Copy(patch.Path, path)
+			if err != nil {
+				return
+			}
+		}
+
+		if patch.Source != "" || patch.Path != "" {
+			_, err = sh(src, "patch < "+path)
+			if err != nil {
+				return
+			}
+		}
+
+		if patch.Script != "" {
+			script := src + "/" + name + ".sh"
+			err = os.WriteFile(script, []byte(patch.Script), 0755)
+			if err != nil {
+				return
+			}
+			_, err = sh(src, script)
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
 }
 
 func build(tmp string, ka config.Artifact, ki config.KernelInfo,
-	dockerTimeout time.Duration) (outPath, output string, err error) {
+	dockerTimeout time.Duration) (outdir, outpath, output string, err error) {
 
 	target := fmt.Sprintf("%d_%s", rand.Int(), ki.KernelRelease)
 
-	tmpSourcePath := tmp + "/source"
+	outdir = tmp + "/source"
 
-	err = copy.Copy(ka.SourcePath, tmpSourcePath)
+	err = copy.Copy(ka.SourcePath, outdir)
 	if err != nil {
 		return
 	}
 
-	outPath = tmpSourcePath + "/" + target
+	err = applyPatches(outdir, ka)
+	if err != nil {
+		return
+	}
+
+	outpath = outdir + "/" + target
 	if ka.Type == config.KernelModule {
-		outPath += ".ko"
+		outpath += ".ko"
 	}
 
 	kernel := "/lib/modules/" + ki.KernelRelease + "/build"
-
-	seconds := fmt.Sprintf("%ds", dockerTimeout/time.Second)
-	cmd := dockerCommand(ki.ContainerName, tmpSourcePath, seconds,
-		"make KERNEL="+kernel+" TARGET="+target)
-	rawOutput, err := cmd.CombinedOutput()
-	output = string(rawOutput)
-	if err != nil {
-		err = errors.New("make execution error")
-		return
+	if ki.KernelSource != "" {
+		kernel = ki.KernelSource
 	}
 
-	return
-}
+	buildCommand := "make KERNEL=" + kernel + " TARGET=" + target
+	if ka.Make.Target != "" {
+		buildCommand += " " + ka.Make.Target
+	}
 
-func cleanDmesg(q *qemu.QemuSystem) (err error) {
-	start := time.Now()
-	for {
-		_, err = q.Command("root", "dmesg -c")
-		if err == nil {
-			break
+	if ki.ContainerName != "" {
+		var c container
+		c, err = NewContainer(ki.ContainerName, dockerTimeout)
+		if err != nil {
+			log.Fatal().Err(err).Msg("container creation failure")
 		}
-		time.Sleep(time.Second)
 
-		if time.Now().After(start.Add(time.Minute)) {
-			err = errors.New("Can't connect to qemu")
-			break
+		output, err = c.Run(outdir, buildCommand+" && chmod -R 777 /work")
+	} else {
+		cmd := exec.Command("bash", "-c", "cd "+outdir+" && "+
+			buildCommand)
+
+		log.Debug().Msgf("%v", cmd)
+
+		timer := time.AfterFunc(dockerTimeout, func() {
+			cmd.Process.Kill()
+		})
+		defer timer.Stop()
+
+		var raw []byte
+		raw, err = cmd.CombinedOutput()
+		if err != nil {
+			e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`",
+				err, buildCommand, string(raw))
+			err = errors.New(e)
+			return
 		}
+
+		output = string(raw)
 	}
 	return
 }
 
-func testKernelModule(q *qemu.QemuSystem, ka config.Artifact,
+func runScript(q *qemu.System, script string) (output string, err error) {
+	return q.Command("root", script)
+}
+
+func testKernelModule(q *qemu.System, ka config.Artifact,
 	test string) (output string, err error) {
 
 	output, err = q.Command("root", test)
@@ -87,7 +304,7 @@ func testKernelModule(q *qemu.QemuSystem, ka config.Artifact,
 	return
 }
 
-func testKernelExploit(q *qemu.QemuSystem, ka config.Artifact,
+func testKernelExploit(q *qemu.System, ka config.Artifact,
 	test, exploit string) (output string, err error) {
 
 	output, err = q.Command("user", "chmod +x "+exploit)
@@ -111,32 +328,72 @@ func testKernelExploit(q *qemu.QemuSystem, ka config.Artifact,
 	return
 }
 
-func genOkFail(name string, ok bool) aurora.Value {
+func genOkFail(name string, ok bool) (aurv aurora.Value) {
+	state.Overall += 1
+	s := " " + name
+	if name == "" {
+		s = ""
+	}
 	if ok {
-		s := " " + name + " SUCCESS "
-		return aurora.BgGreen(aurora.Black(s))
+		state.Success += 1
+		s += " SUCCESS "
+		aurv = aurora.BgGreen(aurora.Black(s))
 	} else {
-		s := " " + name + " FAILURE "
-		return aurora.BgRed(aurora.Gray(aurora.Bold(s)))
+		s += " FAILURE "
+		aurv = aurora.BgRed(aurora.White(aurora.Bold(s)))
+	}
+	return
+}
+
+type phasesResult struct {
+	BuildDir         string
+	BuildArtifact    string
+	Build, Run, Test struct {
+		Output string
+		Ok     bool
 	}
 }
 
-func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo,
-	build_ok, run_ok, test_ok *bool) {
+func copyFile(sourcePath, destinationPath string) (err error) {
+	sourceFile, err := os.Open(sourcePath)
+	if err != nil {
+		return
+	}
+	defer sourceFile.Close()
+
+	destinationFile, err := os.Create(destinationPath)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(destinationFile, sourceFile); err != nil {
+		destinationFile.Close()
+		return err
+	}
+	return destinationFile.Close()
+}
+
+func dumpResult(q *qemu.System, ka config.Artifact, ki config.KernelInfo,
+	res *phasesResult, dist, tag, binary string, db *sql.DB) {
+
+	// TODO merge (problem is it's not 100% same) with log.go:logLogEntry
 
 	distroInfo := fmt.Sprintf("%s-%s {%s}", ki.DistroType,
 		ki.DistroRelease, ki.KernelRelease)
 
 	colored := ""
-	if ka.Type == config.KernelExploit {
+	switch ka.Type {
+	case config.KernelExploit:
 		colored = aurora.Sprintf("[*] %40s: %s %s", distroInfo,
-			genOkFail("BUILD", *build_ok),
-			genOkFail("LPE", *test_ok))
-	} else {
+			genOkFail("BUILD", res.Build.Ok),
+			genOkFail("LPE", res.Test.Ok))
+	case config.KernelModule:
 		colored = aurora.Sprintf("[*] %40s: %s %s %s", distroInfo,
-			genOkFail("BUILD", *build_ok),
-			genOkFail("INSMOD", *run_ok),
-			genOkFail("TEST", *test_ok))
+			genOkFail("BUILD", res.Build.Ok),
+			genOkFail("INSMOD", res.Run.Ok),
+			genOkFail("TEST", res.Test.Ok))
+	case config.Script:
+		colored = aurora.Sprintf("[*] %40s: %s", distroInfo,
+			genOkFail("", res.Test.Ok))
 	}
 
 	additional := ""
@@ -151,129 +408,289 @@ func dumpResult(q *qemu.QemuSystem, ka config.Artifact, ki config.KernelInfo,
 	} else {
 		fmt.Println(colored)
 	}
+
+	err := addToLog(db, q, ka, ki, res, tag)
+	if err != nil {
+		log.Warn().Err(err).Msgf("[db] addToLog (%v)", ka)
+	}
+
+	if binary == "" && dist != pathDevNull {
+		err = os.MkdirAll(dist, os.ModePerm)
+		if err != nil {
+			log.Warn().Err(err).Msgf("os.MkdirAll (%v)", ka)
+		}
+
+		path := fmt.Sprintf("%s/%s-%s-%s", dist, ki.DistroType,
+			ki.DistroRelease, ki.KernelRelease)
+		if ka.Type != config.KernelExploit {
+			path += ".ko"
+		}
+
+		err = copyFile(res.BuildArtifact, path)
+		if err != nil {
+			log.Warn().Err(err).Msgf("copy file (%v)", ka)
+		}
+	}
 }
 
-func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
-	ki config.KernelInfo, binaryPath, testPath string,
-	qemuTimeout, dockerTimeout time.Duration) {
+func copyArtifactAndTest(slog zerolog.Logger, q *qemu.System, ka config.Artifact,
+	res *phasesResult, remoteTest string) (err error) {
 
-	defer swg.Done()
-
-	kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
-	q, err := qemu.NewQemuSystem(qemu.X86_64, kernel, ki.RootFS)
-	if err != nil {
-		log.Println("Qemu creation error:", err)
-		return
-	}
-	q.Timeout = qemuTimeout
-
-	err = q.Start()
-	if err != nil {
-		log.Println("Qemu start error:", err)
-		return
-	}
-	defer q.Stop()
-
-	tmp, err := ioutil.TempDir("/tmp/", "out-of-tree_")
-	if err != nil {
-		log.Println("Temporary directory creation error:", err)
-		return
-	}
-	defer os.RemoveAll(tmp)
-
-	build_ok := false
-	run_ok := false
-	test_ok := false
-	defer dumpResult(q, ka, ki, &build_ok, &run_ok, &test_ok)
-
-	var outFile, output string
-	if binaryPath == "" {
-		// TODO Write build log to file or database
-		outFile, output, err = build(tmp, ka, ki, dockerTimeout)
+	// Copy all test files to the remote machine
+	for _, f := range ka.TestFiles {
+		if f.Local[0] != '/' {
+			if res.BuildDir != "" {
+				f.Local = res.BuildDir + "/" + f.Local
+			}
+		}
+		err = q.CopyFile(f.User, f.Local, f.Remote)
 		if err != nil {
-			log.Println(output)
+			slog.Error().Err(err).Msg("copy test file")
 			return
 		}
-		build_ok = true
-	} else {
-		outFile = binaryPath
-		build_ok = true
 	}
 
-	err = cleanDmesg(q)
-	if err != nil {
-		return
+	switch ka.Type {
+	case config.KernelModule:
+		res.Run.Output, err = q.CopyAndInsmod(res.BuildArtifact)
+		if err != nil {
+			slog.Error().Err(err).Msg(res.Run.Output)
+			return
+		}
+		res.Run.Ok = true
+
+		res.Test.Output, err = testKernelModule(q, ka, remoteTest)
+		if err != nil {
+			slog.Error().Err(err).Msg(res.Test.Output)
+			return
+		}
+		res.Test.Ok = true
+	case config.KernelExploit:
+		remoteExploit := fmt.Sprintf("/tmp/exploit_%d", rand.Int())
+		err = q.CopyFile("user", res.BuildArtifact, remoteExploit)
+		if err != nil {
+			return
+		}
+
+		res.Test.Output, err = testKernelExploit(q, ka, remoteTest,
+			remoteExploit)
+		if err != nil {
+			slog.Error().Err(err).Msg(res.Test.Output)
+			return
+		}
+		res.Run.Ok = true // does not really used
+		res.Test.Ok = true
+	case config.Script:
+		res.Test.Output, err = runScript(q, remoteTest)
+		if err != nil {
+			slog.Error().Err(err).Msg(res.Test.Output)
+			return
+		}
+		slog.Info().Msg(res.Test.Output)
+		res.Run.Ok = true
+		res.Test.Ok = true
+	default:
+		slog.Fatal().Msg("Unsupported artifact type")
 	}
 
-	if testPath == "" {
-		testPath = outFile + "_test"
-	}
+	return
+}
 
-	remoteTest := fmt.Sprintf("/tmp/test_%d", rand.Int())
+func copyTest(q *qemu.System, testPath string, ka config.Artifact) (
+	remoteTest string, err error) {
+
+	remoteTest = fmt.Sprintf("/tmp/test_%d", rand.Int())
 	err = q.CopyFile("user", testPath, remoteTest)
 	if err != nil {
 		if ka.Type == config.KernelExploit {
-			log.Println("Use `echo touch FILE | exploit` for test")
 			q.Command("user",
 				"echo -e '#!/bin/sh\necho touch $2 | $1' "+
 					"> "+remoteTest+
 					" && chmod +x "+remoteTest)
 		} else {
-			log.Println("copy file err", err)
-			// we should not exit because of testing 'insmod' part
-			// for kernel module
+			q.Command("user", "echo '#!/bin/sh' "+
+				"> "+remoteTest+" && chmod +x "+remoteTest)
 		}
 	}
 
 	_, err = q.Command("root", "chmod +x "+remoteTest)
+	return
+}
+
+func copyStandardModules(q *qemu.System, ki config.KernelInfo) (err error) {
+	_, err = q.Command("root", "mkdir -p /lib/modules")
 	if err != nil {
 		return
 	}
 
-	if ka.Type == config.KernelModule {
-		// TODO Write insmod log to file or database
-		output, err := q.CopyAndInsmod(outFile)
-		if err != nil {
-			log.Println(output, err)
-			return
-		}
-		run_ok = true
-
-		// TODO Write test results to file or database
-		output, err = testKernelModule(q, ka, remoteTest)
-		if err != nil {
-			log.Println(output, err)
-			return
-		}
-		test_ok = true
-	} else if ka.Type == config.KernelExploit {
-		remoteExploit := fmt.Sprintf("/tmp/exploit_%d", rand.Int())
-		err = q.CopyFile("user", outFile, remoteExploit)
-		if err != nil {
-			return
-		}
-
-		// TODO Write test results to file or database
-		output, err = testKernelExploit(q, ka, remoteTest, remoteExploit)
-		if err != nil {
-			log.Println(output)
-			return
-		}
-		run_ok = true // does not really used
-		test_ok = true
-	} else {
-		err = errors.New("Unsupported artifact type")
+	files, err := ioutil.ReadDir(ki.ModulesPath)
+	if err != nil {
+		return
 	}
+
+	// FIXME scp cannot ignore symlinks
+	for _, f := range files {
+		if f.Mode()&os.ModeSymlink == os.ModeSymlink {
+			continue
+		}
+
+		path := ki.ModulesPath + "/" + f.Name()
+		err = q.CopyDirectory("root", path, "/lib/modules/"+ki.KernelRelease+"/")
+		if err != nil {
+			return
+		}
+	}
+
 	return
 }
 
-func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath,
-	testPath string, qemuTimeout, dockerTimeout time.Duration) (err error) {
+func (cmd PewCmd) testArtifact(swg *sizedwaitgroup.SizedWaitGroup,
+	ka config.Artifact, ki config.KernelInfo) {
 
+	defer swg.Done()
+
+	slog := log.With().
+		Str("distro_type", ki.DistroType.String()).
+		Str("distro_release", ki.DistroRelease).
+		Str("kernel", ki.KernelRelease).
+		Logger()
+
+	slog.Info().Msg("start")
+
+	kernel := qemu.Kernel{KernelPath: ki.KernelPath, InitrdPath: ki.InitrdPath}
+	q, err := qemu.NewSystem(qemu.X86x64, kernel, ki.RootFS)
+	if err != nil {
+		slog.Error().Err(err).Msg("qemu init")
+		return
+	}
+	q.Timeout = cmd.QemuTimeout
+
+	if ka.Qemu.Timeout.Duration != 0 {
+		q.Timeout = ka.Qemu.Timeout.Duration
+	}
+	if ka.Qemu.Cpus != 0 {
+		q.Cpus = ka.Qemu.Cpus
+	}
+	if ka.Qemu.Memory != 0 {
+		q.Memory = ka.Qemu.Memory
+	}
+
+	if ka.Docker.Timeout.Duration != 0 {
+		cmd.DockerTimeout = ka.Docker.Timeout.Duration
+	}
+
+	q.SetKASLR(!ka.Mitigations.DisableKaslr)
+	q.SetSMEP(!ka.Mitigations.DisableSmep)
+	q.SetSMAP(!ka.Mitigations.DisableSmap)
+	q.SetKPTI(!ka.Mitigations.DisableKpti)
+
+	err = q.Start()
+	if err != nil {
+		slog.Error().Err(err).Msg("qemu start")
+		return
+	}
+	defer q.Stop()
+
+	go func() {
+		for !q.Died {
+			time.Sleep(time.Minute)
+			slog.Debug().Msg("still alive")
+		}
+	}()
+
+	tmp, err := ioutil.TempDir(tempDirBase, "out-of-tree_")
+	if err != nil {
+		slog.Error().Err(err).Msg("making tmp directory")
+		return
+	}
+	defer os.RemoveAll(tmp)
+
+	result := phasesResult{}
+	defer dumpResult(q, ka, ki, &result, cmd.Dist, cmd.Tag, cmd.Binary, cmd.db)
+
+	if ka.Type == config.Script {
+		result.Build.Ok = true
+		cmd.Test = ka.Script
+	} else if cmd.Binary == "" {
+		// TODO: build should return structure
+		start := time.Now()
+		result.BuildDir, result.BuildArtifact, result.Build.Output, err =
+			build(tmp, ka, ki, cmd.DockerTimeout)
+		slog.Debug().Str("duration", time.Now().Sub(start).String()).
+			Msg("build done")
+		if err != nil {
+			log.Error().Err(err).Msg("build")
+			return
+		}
+		result.Build.Ok = true
+	} else {
+		result.BuildArtifact = cmd.Binary
+		result.Build.Ok = true
+	}
+
+	if cmd.Test == "" {
+		cmd.Test = result.BuildArtifact + "_test"
+		if !exists(cmd.Test) {
+			cmd.Test = tmp + "/source/" + "test.sh"
+		}
+	}
+
+	err = q.WaitForSSH(cmd.QemuTimeout)
+	if err != nil {
+		return
+	}
+
+	remoteTest, err := copyTest(q, cmd.Test, ka)
+	if err != nil {
+		return
+	}
+
+	if ka.StandardModules {
+		// Module depends on one of the standard modules
+		start := time.Now()
+		err = copyStandardModules(q, ki)
+		if err != nil {
+			slog.Fatal().Err(err).Msg("copy standard modules")
+			return
+		}
+		slog.Debug().Str("duration", time.Now().Sub(start).String()).
+			Msg("copy standard modules")
+	}
+
+	err = preloadModules(q, ka, ki, cmd.DockerTimeout)
+	if err != nil {
+		slog.Error().Err(err).Msg("preload modules")
+		return
+	}
+
+	start := time.Now()
+	copyArtifactAndTest(slog, q, ka, &result, remoteTest)
+	slog.Debug().Str("duration", time.Now().Sub(start).String()).
+		Msg("test completed")
+}
+
+func shuffleKernels(a []config.KernelInfo) []config.KernelInfo {
+	// Fisher–Yates shuffle
+	for i := len(a) - 1; i > 0; i-- {
+		j := rand.Intn(i + 1)
+		a[i], a[j] = a[j], a[i]
+	}
+	return a
+}
+
+func (cmd PewCmd) performCI(ka config.Artifact) (err error) {
 	found := false
+	max := cmd.Max
+
+	swg := sizedwaitgroup.New(cmd.Threads)
+	if cmd.Shuffle {
+		cmd.kcfg.Kernels = shuffleKernels(cmd.kcfg.Kernels)
+	}
+	for _, kernel := range cmd.kcfg.Kernels {
+		if max <= 0 {
+			break
+		}
 
-	swg := sizedwaitgroup.New(runtime.NumCPU())
-	for _, kernel := range kcfg.Kernels {
 		var supported bool
 		supported, err = ka.Supported(kernel)
 		if err != nil {
@@ -282,9 +699,16 @@ func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath,
 
 		if supported {
 			found = true
-			swg.Add()
-			go whatever(&swg, ka, kernel, binaryPath, testPath,
-				qemuTimeout, dockerTimeout)
+			max--
+			for i := int64(0); i < cmd.Runs; i++ {
+				if !cmd.timeoutDeadline.IsZero() &&
+					time.Now().After(cmd.timeoutDeadline) {
+
+					break
+				}
+				swg.Add()
+				go cmd.testArtifact(&swg, ka, kernel)
+			}
 		}
 	}
 	swg.Wait()
@@ -298,8 +722,10 @@ func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath,
 
 func exists(path string) bool {
 	if _, err := os.Stat(path); err != nil {
+		log.Debug().Msgf("%s does not exist", path)
 		return false
 	}
+	log.Debug().Msgf("%s exist", path)
 	return true
 }
 
@@ -319,47 +745,18 @@ func kernelMask(kernel string) (km config.KernelMask, err error) {
 	return
 }
 
-func pewHandler(kcfg config.KernelConfig,
-	workPath, ovrrdKrnl, binary, test string, guess bool,
-	qemuTimeout, dockerTimeout time.Duration) (err error) {
-
-	ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
-	if err != nil {
-		return
-	}
-
-	if ka.SourcePath == "" {
-		ka.SourcePath = workPath
-	}
-
-	if ovrrdKrnl != "" {
-		var km config.KernelMask
-		km, err = kernelMask(ovrrdKrnl)
+func genAllKernels() (sk []config.KernelMask, err error) {
+	for _, dType := range config.DistroTypeStrings {
+		var dt config.DistroType
+		dt, err = config.NewDistroType(dType)
 		if err != nil {
 			return
 		}
 
-		ka.SupportedKernels = []config.KernelMask{km}
+		sk = append(sk, config.KernelMask{
+			DistroType:  dt,
+			ReleaseMask: ".*",
+		})
 	}
-
-	if guess {
-		ka.SupportedKernels = []config.KernelMask{}
-		for _, dType := range config.DistroTypeStrings {
-			var dt config.DistroType
-			dt, err = config.NewDistroType(dType)
-			if err != nil {
-				return
-			}
-
-			km := config.KernelMask{DistroType: dt, ReleaseMask: ".*"}
-			ka.SupportedKernels = append(ka.SupportedKernels, km)
-		}
-	}
-
-	err = performCI(ka, kcfg, binary, test, qemuTimeout, dockerTimeout)
-	if err != nil {
-		return
-	}
-
 	return
 }

preload.go

@@ -0,0 +1,166 @@
+// Copyright 2020 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package main
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"errors"
+	"io/ioutil"
+	"os"
+	"os/user"
+	"path/filepath"
+	"time"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/rs/zerolog/log"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+	"code.dumpstack.io/tools/out-of-tree/qemu"
+)
+
+func preloadModules(q *qemu.System, ka config.Artifact, ki config.KernelInfo,
+	dockerTimeout time.Duration) (err error) {
+
+	for _, pm := range ka.Preload {
+		err = preload(q, ki, pm, dockerTimeout)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func preload(q *qemu.System, ki config.KernelInfo, pm config.PreloadModule,
+	dockerTimeout time.Duration) (err error) {
+
+	var workPath, cache string
+	if pm.Path != "" {
+		log.Print("Use non-git path for preload module (no cache)")
+		workPath = pm.Path
+	} else if pm.Repo != "" {
+		workPath, cache, err = cloneOrPull(pm.Repo, ki)
+		if err != nil {
+			return
+		}
+	} else {
+		errors.New("No repo/path in preload entry")
+	}
+
+	err = buildAndInsmod(workPath, q, ki, dockerTimeout, cache)
+	if err != nil {
+		return
+	}
+
+	time.Sleep(pm.TimeoutAfterLoad.Duration)
+	return
+}
+
+func buildAndInsmod(workPath string, q *qemu.System, ki config.KernelInfo,
+	dockerTimeout time.Duration, cache string) (err error) {
+
+	tmp, err := ioutil.TempDir(tempDirBase, "out-of-tree_")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(tmp)
+
+	var artifact string
+	if exists(cache) {
+		artifact = cache
+	} else {
+		artifact, err = buildPreload(workPath, tmp, ki, dockerTimeout)
+		if err != nil {
+			return
+		}
+		if cache != "" {
+			err = copyFile(artifact, cache)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	output, err := q.CopyAndInsmod(artifact)
+	if err != nil {
+		log.Print(output)
+		return
+	}
+	return
+}
+
+func buildPreload(workPath, tmp string, ki config.KernelInfo,
+	dockerTimeout time.Duration) (artifact string, err error) {
+
+	ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
+	if err != nil {
+		return
+	}
+
+	ka.SourcePath = workPath
+
+	km := config.KernelMask{DistroType: ki.DistroType,
+		DistroRelease: ki.DistroRelease,
+		ReleaseMask:   ki.KernelRelease,
+	}
+	ka.SupportedKernels = []config.KernelMask{km}
+
+	if ka.Docker.Timeout.Duration != 0 {
+		dockerTimeout = ka.Docker.Timeout.Duration
+	}
+
+	_, artifact, _, err = build(tmp, ka, ki, dockerTimeout)
+	return
+}
+
+func cloneOrPull(repo string, ki config.KernelInfo) (workPath, cache string, err error) {
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+	base := filepath.Join(usr.HomeDir, "/.out-of-tree/preload/")
+	workPath = filepath.Join(base, "/repos/", sha1sum(repo))
+
+	var r *git.Repository
+	if exists(workPath) {
+		r, err = git.PlainOpen(workPath)
+		if err != nil {
+			return
+		}
+
+		var w *git.Worktree
+		w, err = r.Worktree()
+		if err != nil {
+			return
+		}
+
+		err = w.Pull(&git.PullOptions{})
+		if err != nil && err != git.NoErrAlreadyUpToDate {
+			log.Print(repo, "pull error:", err)
+		}
+	} else {
+		r, err = git.PlainClone(workPath, false, &git.CloneOptions{URL: repo})
+		if err != nil {
+			return
+		}
+	}
+
+	ref, err := r.Head()
+	if err != nil {
+		return
+	}
+
+	cachedir := filepath.Join(base, "/cache/")
+	os.MkdirAll(cachedir, 0700)
+
+	filename := sha1sum(repo + ki.KernelPath + ref.Hash().String())
+	cache = filepath.Join(cachedir, filename)
+	return
+}
+
+func sha1sum(data string) string {
+	h := sha1.Sum([]byte(data))
+	return hex.EncodeToString(h[:])
+}

qemu/README.md

@@ -9,7 +9,7 @@ Features:
 
 ## Installation
 
-    $ go get github.com/jollheef/out-of-tree/qemu
+    $ go get code.dumpstack.io/tools/out-of-tree/qemu
 
 ### Generate root image
 
@@ -30,12 +30,12 @@ Note: qemu on macOS since v2.12 (24 April 2018) supports Hypervisor.framework.
 
 #### Generate image
 
-    $ cd $GOPATH/src/github.com/jollheef/out-of-tree/tools/qemu-debian-img
+    $ cd $GOPATH/src/code.dumpstack.io/tools/out-of-tree/tools/qemu-debian-img
     $ ./bootstrap.sh
 
 ### Fill configuration file
 
-    $ $EDITOR $GOPATH/src/github.com/jollheef/out-of-tree/qemu/test.config.go
+    $ $EDITOR $GOPATH/src/code.dumpstack.io/tools/out-of-tree/qemu/test.config.go
 
 ### Run tests
 
@@ -43,7 +43,7 @@ Note: qemu on macOS since v2.12 (24 April 2018) supports Hypervisor.framework.
 
 ## Usage
 
-    $ go get github.com/jollheef/out-of-tree/qemu
+    $ go get code.dumpstack.io/tools/out-of-tree/qemu
 
 Minimal example:
 
@@ -52,7 +52,7 @@ Minimal example:
 		KernelPath: "/path/to/vmlinuz",
 		InitrdPath: "/path/to/initrd", // if required
 	}
-	q, err := qemu.NewQemuSystem(qemu.X86_64, kernel, "/path/to/qcow2")
+	q, err := qemu.NewSystem(qemu.X86_64, kernel, "/path/to/qcow2")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -71,4 +71,4 @@ Minimal example:
 
 More information and list of all functions see at go documentation project, or just run locally:
 
-    $ godoc github.com/jollheef/out-of-tree/qemu
+    $ godoc code.dumpstack.io/tools/out-of-tree/qemu

qemu/qemu-kernel.go

@@ -2,10 +2,10 @@
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
-package qemukernel
+package qemu
 
 import (
-	"bytes"
+	"bufio"
 	"errors"
 	"fmt"
 	"io"
@@ -18,34 +18,18 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 	"golang.org/x/crypto/ssh"
 )
 
-func readUntilEOF(pipe io.ReadCloser, buf *[]byte) (err error) {
-	bufSize := 1024
-	for err != io.EOF {
-		stdout := make([]byte, bufSize)
-		var n int
-
-		n, err = pipe.Read(stdout)
-		if err != nil && err != io.EOF {
-			return
-		}
-
-		*buf = append(*buf, stdout[:n]...)
-	}
-
-	if err == io.EOF {
-		err = nil
-	}
-	return
-}
-
 type arch string
 
 const (
-	X86_64 arch = "x86_64"
-	I386        = "i386"
+	// X86x64 is the qemu-system-x86_64
+	X86x64 arch = "x86_64"
+	// X86x32 is the qemu-system-i386
+	X86x32 = "i386"
 	// TODO add other
 
 	unsupported = "unsupported" // for test purposes
@@ -58,18 +42,25 @@ type Kernel struct {
 	InitrdPath string
 }
 
-// QemuSystem describe qemu parameters and runned process
-type QemuSystem struct {
+// System describe qemu parameters and executed process
+type System struct {
 	arch      arch
 	kernel    Kernel
 	drivePath string
 
+	Mutable bool
+
 	Cpus   int
 	Memory int
 
 	debug bool
 	gdb   string // tcp::1234
 
+	noKASLR bool
+	noSMEP  bool
+	noSMAP  bool
+	noKPTI  bool
+
 	// Timeout works after Start invocation
 	Timeout         time.Duration
 	KilledByTimeout bool
@@ -79,7 +70,7 @@ type QemuSystem struct {
 	Died        bool
 	sshAddrPort string
 
-	// accessible while qemu is runned
+	// accessible while qemu is running
 	cmd  *exec.Cmd
 	pipe struct {
 		stdin  io.WriteCloser
@@ -87,18 +78,25 @@ type QemuSystem struct {
 		stdout io.ReadCloser
 	}
 
-	Stdout, Stderr []byte
+	Stdout, Stderr string
 
 	// accessible after qemu is closed
 	exitErr error
+
+	log zerolog.Logger
 }
 
-// NewQemuSystem constructor
-func NewQemuSystem(arch arch, kernel Kernel, drivePath string) (q *QemuSystem, err error) {
+// NewSystem constructor
+func NewSystem(arch arch, kernel Kernel, drivePath string) (q *System, err error) {
+	q = &System{}
+	q.log = log.With().
+		Str("kernel", kernel.KernelPath).
+		Logger()
+
 	if _, err = exec.LookPath("qemu-system-" + string(arch)); err != nil {
 		return
 	}
-	q = &QemuSystem{}
+
 	q.arch = arch
 
 	if _, err = os.Stat(kernel.KernelPath); err != nil {
@@ -118,6 +116,12 @@ func NewQemuSystem(arch arch, kernel Kernel, drivePath string) (q *QemuSystem, e
 	return
 }
 
+func (q *System) SetSSHAddrPort(addr string, port int) (err error) {
+	// TODO validate
+	q.sshAddrPort = fmt.Sprintf("%s:%d", addr, port)
+	return
+}
+
 func getRandomAddrPort() (addr string) {
 	// 127.1-255.0-255.0-255:10000-50000
 	ip := fmt.Sprintf("127.%d.%d.%d",
@@ -156,37 +160,76 @@ func kvmExists() bool {
 	if _, err := os.Stat("/dev/kvm"); err != nil {
 		return false
 	}
+
+	file, err := os.OpenFile("/dev/kvm", os.O_WRONLY, 0666)
+	if err != nil {
+		if os.IsPermission(err) {
+			return false
+		}
+	}
+	file.Close()
+
 	return true
 }
 
-func (q *QemuSystem) panicWatcher() {
+func (q *System) panicWatcher() {
 	for {
 		time.Sleep(time.Second)
-		if bytes.Contains(q.Stdout, []byte("Kernel panic")) {
+		if strings.Contains(q.Stdout, "Kernel panic") {
+			q.KernelPanic = true
+			q.log.Debug().Msg("kernel panic")
 			time.Sleep(time.Second)
 			// There is no reason to stay alive after kernel panic
 			q.Stop()
-			q.KernelPanic = true
 			return
 		}
 	}
 }
 
-// Start qemu process
-func (q *QemuSystem) Start() (err error) {
-	rand.Seed(time.Now().UnixNano()) // Are you sure?
-	q.sshAddrPort = getFreeAddrPort()
+func (q System) cmdline() (s string) {
+	s = "root=/dev/sda ignore_loglevel console=ttyS0 rw"
+
+	if q.noKASLR {
+		s += " nokaslr"
+	}
+
+	if q.noSMEP {
+		s += " nosmep"
+	}
+
+	if q.noSMAP {
+		s += " nosmap"
+	}
+
+	if q.noKPTI {
+		s += " nokpti"
+	}
+
+	return
+}
+
+func (q System) Executable() string {
+	return "qemu-system-" + string(q.arch)
+}
+
+func (q *System) Args() (qemuArgs []string) {
+	if q.sshAddrPort == "" {
+		q.sshAddrPort = getFreeAddrPort()
+	}
 	hostfwd := fmt.Sprintf("hostfwd=tcp:%s-:22", q.sshAddrPort)
-	qemuArgs := []string{"-snapshot", "-nographic",
+	qemuArgs = []string{"-nographic",
 		"-hda", q.drivePath,
 		"-kernel", q.kernel.KernelPath,
-		"-append", "root=/dev/sda ignore_loglevel console=ttyS0 rw",
 		"-smp", fmt.Sprintf("%d", q.Cpus),
 		"-m", fmt.Sprintf("%d", q.Memory),
 		"-device", "e1000,netdev=n1",
 		"-netdev", "user,id=n1," + hostfwd,
 	}
 
+	if !q.Mutable {
+		qemuArgs = append(qemuArgs, "-snapshot")
+	}
+
 	if q.debug {
 		qemuArgs = append(qemuArgs, "-gdb", q.gdb)
 	}
@@ -195,15 +238,24 @@ func (q *QemuSystem) Start() (err error) {
 		qemuArgs = append(qemuArgs, "-initrd", q.kernel.InitrdPath)
 	}
 
-	if (q.arch == X86_64 || q.arch == I386) && kvmExists() {
-		qemuArgs = append(qemuArgs, "-enable-kvm")
+	if (q.arch == X86x64 || q.arch == X86x32) && kvmExists() {
+		qemuArgs = append(qemuArgs, "-enable-kvm", "-cpu", "host")
 	}
 
-	if q.arch == X86_64 && runtime.GOOS == "darwin" {
+	if q.arch == X86x64 && runtime.GOOS == "darwin" {
 		qemuArgs = append(qemuArgs, "-accel", "hvf", "-cpu", "host")
 	}
 
-	q.cmd = exec.Command("qemu-system-"+string(q.arch), qemuArgs...)
+	qemuArgs = append(qemuArgs, "-append", q.cmdline())
+	return
+}
+
+// Start qemu process
+func (q *System) Start() (err error) {
+	rand.Seed(time.Now().UnixNano()) // Are you sure?
+
+	q.cmd = exec.Command(q.Executable(), q.Args()...)
+	q.log.Debug().Msgf("%v", q.cmd)
 
 	if q.pipe.stdin, err = q.cmd.StdinPipe(); err != nil {
 		return
@@ -222,8 +274,23 @@ func (q *QemuSystem) Start() (err error) {
 		return
 	}
 
-	go readUntilEOF(q.pipe.stdout, &q.Stdout)
-	go readUntilEOF(q.pipe.stderr, &q.Stderr)
+	go func() {
+		scanner := bufio.NewScanner(q.pipe.stdout)
+		for scanner.Scan() {
+			m := scanner.Text()
+			q.Stdout += m + "\n"
+			q.log.Trace().Str("stdout", m).Msg("")
+		}
+	}()
+
+	go func() {
+		scanner := bufio.NewScanner(q.pipe.stderr)
+		for scanner.Scan() {
+			m := scanner.Text()
+			q.Stderr += m + "\n"
+			q.log.Trace().Str("stderr", m).Msg("")
+		}
+	}()
 
 	go func() {
 		q.exitErr = q.cmd.Wait()
@@ -250,7 +317,7 @@ func (q *QemuSystem) Start() (err error) {
 }
 
 // Stop qemu process
-func (q *QemuSystem) Stop() {
+func (q *System) Stop() {
 	// 1  00/01   01  01  SOH  (Ctrl-A)  START OF HEADING
 	fmt.Fprintf(q.pipe.stdin, "%cx", 1)
 	// wait for die
@@ -262,7 +329,21 @@ func (q *QemuSystem) Stop() {
 	}
 }
 
-func (q QemuSystem) ssh(user string) (client *ssh.Client, err error) {
+func (q System) WaitForSSH(timeout time.Duration) error {
+	for start := time.Now(); time.Since(start) < timeout; {
+		client, err := q.ssh("root")
+		if err != nil {
+			time.Sleep(time.Second / 10)
+			continue
+		}
+		client.Close()
+		return nil
+	}
+
+	return errors.New("no ssh (timeout)")
+}
+
+func (q System) ssh(user string) (client *ssh.Client, err error) {
 	cfg := &ssh.ClientConfig{
 		User:            user,
 		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
@@ -273,7 +354,15 @@ func (q QemuSystem) ssh(user string) (client *ssh.Client, err error) {
 }
 
 // Command executes shell commands on qemu system
-func (q QemuSystem) Command(user, cmd string) (output string, err error) {
+func (q System) Command(user, cmd string) (output string, err error) {
+	flog := log.With().
+		Str("kernel", q.kernel.KernelPath).
+		Str("user", user).
+		Str("cmd", cmd).
+		Logger()
+
+	flog.Debug().Msg("qemu command")
+
 	client, err := q.ssh(user)
 	if err != nil {
 		return
@@ -285,13 +374,33 @@ func (q QemuSystem) Command(user, cmd string) (output string, err error) {
 		return
 	}
 
-	bytesOutput, err := session.CombinedOutput(cmd)
-	output = string(bytesOutput)
+	stdout, err := session.StdoutPipe()
+	if err != nil {
+		return
+	}
+	session.Stderr = session.Stdout
+
+	err = session.Start(cmd)
+	if err != nil {
+		return
+	}
+
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			m := scanner.Text()
+			output += m + "\n"
+			flog.Trace().Str("stdout", m).Msg("")
+		}
+		output = strings.TrimSuffix(output, "\n")
+	}()
+
+	err = session.Wait()
 	return
 }
 
 // AsyncCommand executes command on qemu system but does not wait for exit
-func (q QemuSystem) AsyncCommand(user, cmd string) (err error) {
+func (q System) AsyncCommand(user, cmd string) (err error) {
 	client, err := q.ssh(user)
 	if err != nil {
 		return
@@ -307,25 +416,72 @@ func (q QemuSystem) AsyncCommand(user, cmd string) (err error) {
 		"nohup sh -c '%s' > /dev/null 2> /dev/null < /dev/null &", cmd))
 }
 
-// CopyFile is copy file from local machine to remote through ssh/scp
-func (q *QemuSystem) CopyFile(user, localPath, remotePath string) (err error) {
+func (q System) scp(user, localPath, remotePath string, recursive bool) (err error) {
 	addrPort := strings.Split(q.sshAddrPort, ":")
 	addr := addrPort[0]
 	port := addrPort[1]
 
-	cmd := exec.Command("scp", "-P", port,
+	args := []string{
+		"-P", port,
 		"-o", "StrictHostKeyChecking=no",
+		"-o", "UserKnownHostsFile=/dev/null",
 		"-o", "LogLevel=error",
-		localPath, user+"@"+addr+":"+remotePath)
+	}
+
+	if recursive {
+		cmd := exec.Command("ssh", "-V")
+
+		log.Debug().Msgf("%v", cmd)
+
+		var output []byte
+		output, err = cmd.CombinedOutput()
+		if err != nil {
+			return
+		}
+		sshVersion := string(output)
+
+		log.Debug().Str("ssh version", sshVersion).Msg("")
+
+		if strings.Contains(sshVersion, "OpenSSH_9") {
+			// This release switches scp from using the
+			// legacy scp/rcp protocol to using the SFTP
+			// protocol by default.
+			//
+			// To keep compatibility with old distros,
+			// using -O flag to use the legacy scp/rcp.
+			//
+			// Note: old ssh doesn't support -O flag
+			args = append(args, "-O")
+		}
+
+		args = append(args, "-r")
+	}
+
+	args = append(args, localPath, user+"@"+addr+":"+remotePath)
+
+	cmd := exec.Command("scp", args...)
+	log.Debug().Msgf("%v", cmd)
+
 	output, err := cmd.CombinedOutput()
-	if err != nil {
+	if err != nil || string(output) != "" {
 		return errors.New(string(output))
 	}
 
 	return
 }
 
-func (q *QemuSystem) CopyAndInsmod(localKoPath string) (output string, err error) {
+// CopyFile from local machine to remote via scp
+func (q System) CopyFile(user, localPath, remotePath string) (err error) {
+	return q.scp(user, localPath, remotePath, false)
+}
+
+// CopyDirectory from local machine to remote via scp
+func (q System) CopyDirectory(user, localPath, remotePath string) (err error) {
+	return q.scp(user, localPath, remotePath, true)
+}
+
+// CopyAndInsmod copy kernel module to temporary file on qemu then insmod it
+func (q *System) CopyAndInsmod(localKoPath string) (output string, err error) {
 	remoteKoPath := fmt.Sprintf("/tmp/module_%d.ko", rand.Int())
 	err = q.CopyFile("root", localKoPath, remoteKoPath)
 	if err != nil {
@@ -336,7 +492,7 @@ func (q *QemuSystem) CopyAndInsmod(localKoPath string) (output string, err error
 }
 
 // CopyAndRun is copy local file to qemu vm then run it
-func (q *QemuSystem) CopyAndRun(user, path string) (output string, err error) {
+func (q *System) CopyAndRun(user, path string) (output string, err error) {
 	remotePath := fmt.Sprintf("/tmp/executable_%d", rand.Int())
 	err = q.CopyFile(user, path, remotePath)
 	if err != nil {
@@ -346,12 +502,54 @@ func (q *QemuSystem) CopyAndRun(user, path string) (output string, err error) {
 	return q.Command(user, "chmod +x "+remotePath+" && "+remotePath)
 }
 
-func (q *QemuSystem) Debug(conn string) {
+// Debug is for enable qemu debug and set hostname and port for listen
+func (q *System) Debug(conn string) {
 	q.debug = true
 	q.gdb = conn
 }
 
-func (q QemuSystem) GetSshCommand() (cmd string) {
+// SetKASLR is changing KASLR state through kernel boot args
+func (q *System) SetKASLR(state bool) {
+	q.noKASLR = !state
+}
+
+// SetSMEP is changing SMEP state through kernel boot args
+func (q *System) SetSMEP(state bool) {
+	q.noSMEP = !state
+}
+
+// SetSMAP is changing SMAP state through kernel boot args
+func (q *System) SetSMAP(state bool) {
+	q.noSMAP = !state
+}
+
+// SetKPTI is changing KPTI state through kernel boot args
+func (q *System) SetKPTI(state bool) {
+	q.noKPTI = !state
+}
+
+// GetKASLR is retrieve KASLR settings
+func (q *System) GetKASLR() bool {
+	return !q.noKASLR
+}
+
+// GetSMEP is retrieve SMEP settings
+func (q *System) GetSMEP() bool {
+	return !q.noSMEP
+}
+
+// GetSMAP is retrieve SMAP settings
+func (q *System) GetSMAP() bool {
+	return !q.noSMAP
+}
+
+// GetKPTI is retrieve KPTI settings
+func (q *System) GetKPTI() bool {
+	return !q.noKPTI
+}
+
+// GetSSHCommand returns command for connect to qemu machine over ssh
+func (q System) GetSSHCommand() (cmd string) {
 	addrPort := strings.Split(q.sshAddrPort, ":")
 	addr := addrPort[0]
 	port := addrPort[1]

qemu/qemu-kernel_test.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
-package qemukernel
+package qemu
 
 import (
 	"crypto/sha512"
@@ -20,46 +20,46 @@ func init() {
 	rand.Seed(time.Now().UnixNano())
 }
 
-func TestQemuSystemNew_InvalidKernelPath(t *testing.T) {
+func TestSystemNew_InvalidKernelPath(t *testing.T) {
 	kernel := Kernel{Name: "Invalid", KernelPath: "/invalid/path"}
-	if _, err := NewQemuSystem(X86_64, kernel, "/bin/sh"); err == nil {
+	if _, err := NewSystem(X86x64, kernel, "/bin/sh"); err == nil {
 		t.Fatal(err)
 	}
 }
 
-func TestQemuSystemNew_InvalidQemuArch(t *testing.T) {
+func TestSystemNew_InvalidQemuArch(t *testing.T) {
 	kernel := Kernel{Name: "Valid path", KernelPath: testConfigVmlinuz}
-	if _, err := NewQemuSystem(unsupported, kernel, "/bin/sh"); err == nil {
+	if _, err := NewSystem(unsupported, kernel, "/bin/sh"); err == nil {
 		t.Fatal(err)
 	}
 }
 
-func TestQemuSystemNew_InvalidQemuDrivePath(t *testing.T) {
+func TestSystemNew_InvalidQemuDrivePath(t *testing.T) {
 	kernel := Kernel{Name: "Valid path", KernelPath: testConfigVmlinuz}
-	if _, err := NewQemuSystem(X86_64, kernel, "/invalid/path"); err == nil {
+	if _, err := NewSystem(X86x64, kernel, "/invalid/path"); err == nil {
 		t.Fatal(err)
 	}
 }
 
-func TestQemuSystemNew(t *testing.T) {
+func TestSystemNew(t *testing.T) {
 	kernel := Kernel{Name: "Valid path", KernelPath: testConfigVmlinuz}
-	if _, err := NewQemuSystem(X86_64, kernel, "/bin/sh"); err != nil {
+	if _, err := NewSystem(X86x64, kernel, "/bin/sh"); err != nil {
 		t.Fatal(err)
 	}
 }
 
-func TestQemuSystemStart(t *testing.T) {
+func TestSystemStart(t *testing.T) {
 	kernel := Kernel{Name: "Test kernel", KernelPath: testConfigVmlinuz}
-	qemu, err := NewQemuSystem(X86_64, kernel, "/bin/sh")
+	q, err := NewSystem(X86x64, kernel, "/bin/sh")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if err = qemu.Start(); err != nil {
+	if err = q.Start(); err != nil {
 		t.Fatal(err)
 	}
 
-	qemu.Stop()
+	q.Stop()
 }
 
 func TestGetFreeAddrPort(t *testing.T) {
@@ -71,39 +71,39 @@ func TestGetFreeAddrPort(t *testing.T) {
 	ln.Close()
 }
 
-func TestQemuSystemStart_Timeout(t *testing.T) {
+func TestSystemStart_Timeout(t *testing.T) {
 	t.Parallel()
 	kernel := Kernel{Name: "Test kernel", KernelPath: testConfigVmlinuz}
-	qemu, err := NewQemuSystem(X86_64, kernel, "/bin/sh")
+	q, err := NewSystem(X86x64, kernel, "/bin/sh")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	qemu.Timeout = time.Second
+	q.Timeout = time.Second
 
-	if err = qemu.Start(); err != nil {
+	if err = q.Start(); err != nil {
 		t.Fatal(err)
 	}
 
 	time.Sleep(2 * time.Second)
 
-	if !qemu.Died {
+	if !q.Died {
 		t.Fatal("qemu does not died :c")
 	}
 
-	if !qemu.KilledByTimeout {
+	if !q.KilledByTimeout {
 		t.Fatal("qemu died not because of timeout O_o")
 	}
 }
 
-func startTestQemu(t *testing.T, timeout time.Duration) (q *QemuSystem, err error) {
+func startTestQemu(t *testing.T, timeout time.Duration) (q *System, err error) {
 	t.Parallel()
 	kernel := Kernel{
 		Name:       "Test kernel",
 		KernelPath: testConfigVmlinuz,
 		InitrdPath: testConfigInitrd,
 	}
-	q, err = NewQemuSystem(X86_64, kernel, testConfigRootfs)
+	q, err = NewSystem(X86x64, kernel, testConfigRootfs)
 	if err != nil {
 		return
 	}
@@ -116,17 +116,18 @@ func startTestQemu(t *testing.T, timeout time.Duration) (q *QemuSystem, err erro
 		return
 	}
 
+	time.Sleep(time.Second)
 	return
 }
 
-func TestQemuSystemCommand(t *testing.T) {
-	qemu, err := startTestQemu(t, 0)
+func TestSystemCommand(t *testing.T) {
+	q, err := startTestQemu(t, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer qemu.Stop()
+	defer q.Stop()
 
-	output, err := qemu.Command("root", "cat /etc/shadow")
+	output, err := q.Command("root", "cat /etc/shadow")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -134,7 +135,7 @@ func TestQemuSystemCommand(t *testing.T) {
 		t.Fatal("Wrong output from `cat /etc/shadow` by root")
 	}
 
-	output, err = qemu.Command("user", "cat /etc/passwd")
+	output, err = q.Command("user", "cat /etc/passwd")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -142,18 +143,19 @@ func TestQemuSystemCommand(t *testing.T) {
 		t.Fatal("Wrong output from `cat /etc/passwd` by user")
 	}
 
-	output, err = qemu.Command("user", "cat /etc/shadow")
-	if err == nil { // unsucessful is good because user must not read /etc/shadow
+	_, err = q.Command("user", "cat /etc/shadow")
+	// unsuccessful is good because user must not read /etc/shadow
+	if err == nil {
 		t.Fatal("User have rights for /etc/shadow. WAT?!")
 	}
 }
 
-func TestQemuSystemCopyFile(t *testing.T) {
-	qemu, err := startTestQemu(t, 0)
+func TestSystemCopyFile(t *testing.T) {
+	q, err := startTestQemu(t, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer qemu.Stop()
+	defer q.Stop()
 
 	localPath := "/bin/sh"
 
@@ -162,30 +164,31 @@ func TestQemuSystemCopyFile(t *testing.T) {
 		return
 	}
 
-	sha_local := fmt.Sprintf("%x", sha512.Sum512(content))
+	shaLocal := fmt.Sprintf("%x", sha512.Sum512(content))
 
-	err = qemu.CopyFile("user", localPath, "/tmp/test")
+	err = q.CopyFile("user", localPath, "/tmp/test")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	sha_remote, err := qemu.Command("user", "sha512sum /tmp/test")
+	shaRemote, err := q.Command("user", "sha512sum /tmp/test")
 	if err != nil {
 		t.Fatal(err)
 	}
-	sha_remote = strings.Split(sha_remote, " ")[0]
+	shaRemote = strings.Split(shaRemote, " ")[0]
 
-	if sha_local != sha_remote {
-		t.Fatal(fmt.Sprintf("Broken file (%s instead of %s)", sha_remote, sha_local))
+	if shaLocal != shaRemote {
+		t.Fatal(fmt.Sprintf("Broken file (%s instead of %s)",
+			shaRemote, shaLocal))
 	}
 }
 
-func TestQemuSystemCopyAndRun(t *testing.T) {
-	qemu, err := startTestQemu(t, 0)
+func TestSystemCopyAndRun(t *testing.T) {
+	q, err := startTestQemu(t, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer qemu.Stop()
+	defer q.Stop()
 
 	randStr := fmt.Sprintf("%d", rand.Int())
 	content := []byte("#!/bin/sh\n echo -n " + randStr + "\n")
@@ -203,34 +206,35 @@ func TestQemuSystemCopyAndRun(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	output, err := qemu.CopyAndRun("user", tmpfile.Name())
+	output, err := q.CopyAndRun("user", tmpfile.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	if output != randStr {
-		t.Fatal("Wrong output from copyied executable (" + output + "," + randStr + ")")
+		t.Fatal("Wrong output from copyied executable (" +
+			output + "," + randStr + ")")
 	}
 }
 
-func TestQemuSystemCopyAndInsmod(t *testing.T) {
-	qemu, err := startTestQemu(t, 0)
+func TestSystemCopyAndInsmod(t *testing.T) {
+	q, err := startTestQemu(t, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer qemu.Stop()
+	defer q.Stop()
 
-	lsmodBefore, err := qemu.Command("root", "lsmod | wc -l")
+	lsmodBefore, err := q.Command("root", "lsmod | wc -l")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	_, err = qemu.CopyAndInsmod(testConfigSampleKo)
+	_, err = q.CopyAndInsmod(testConfigSampleKo)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	lsmodAfter, err := qemu.Command("root", "lsmod | wc -l")
+	lsmodAfter, err := q.Command("root", "lsmod | wc -l")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -240,21 +244,21 @@ func TestQemuSystemCopyAndInsmod(t *testing.T) {
 	}
 }
 
-func TestQemuSystemKernelPanic(t *testing.T) {
-	qemu, err := startTestQemu(t, time.Minute)
+func TestSystemKernelPanic(t *testing.T) {
+	q, err := startTestQemu(t, 5*time.Minute)
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer qemu.Stop()
+	defer q.Stop()
 
 	// Enable sysrq
-	_, err = qemu.Command("root", "echo 1 > /proc/sys/kernel/sysrq")
+	_, err = q.Command("root", "echo 1 > /proc/sys/kernel/sysrq")
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Trigger kernel panic
-	err = qemu.AsyncCommand("root", "sleep 1s && echo c > /proc/sysrq-trigger")
+	err = q.AsyncCommand("root", "sleep 1s && echo c > /proc/sysrq-trigger")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -262,40 +266,41 @@ func TestQemuSystemKernelPanic(t *testing.T) {
 	// Wait for panic watcher timeout
 	time.Sleep(5 * time.Second)
 
-	if qemu.KilledByTimeout {
+	if q.KilledByTimeout {
 		t.Fatal("qemu is killed by timeout, not because of panic")
 	}
 
-	if !qemu.Died {
+	if !q.Died {
 		t.Fatal("qemu is not killed after kernel panic")
 	}
 
-	if !qemu.KernelPanic {
+	if !q.KernelPanic {
 		t.Fatal("qemu is died but there's no information about panic")
 	}
 }
 
-func TestQemuSystemRun(t *testing.T) {
-	qemu, err := startTestQemu(t, 0)
+func TestSystemRun(t *testing.T) {
+	q, err := startTestQemu(t, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer qemu.Stop()
+	defer q.Stop()
 
 	for {
-		_, err := qemu.Command("root", "echo")
+		_, err := q.Command("root", "echo")
 		if err == nil {
 			break
 		}
 	}
 
 	start := time.Now()
-	err = qemu.AsyncCommand("root", "sleep 10s")
+	err = q.AsyncCommand("root", "sleep 1m")
 	if err != nil {
 		t.Fatal(err)
 	}
-	if time.Since(start) > time.Second {
-		t.Fatalf("qemu.Run does not async (waited %s)", +time.Since(start))
+	if time.Since(start) > 10*time.Second {
+		t.Fatalf("q.AsyncCommand does not async (waited %s)",
+			time.Since(start))
 	}
 
 }
@@ -309,17 +314,19 @@ func openedPort(port int) bool {
 	return true
 }
 
-func TestQemuSystemDebug(t *testing.T) {
+func TestSystemDebug(t *testing.T) {
 	t.Parallel()
 	kernel := Kernel{
 		KernelPath: testConfigVmlinuz,
 		InitrdPath: testConfigInitrd,
 	}
-	q, err := NewQemuSystem(X86_64, kernel, testConfigRootfs)
+	q, err := NewSystem(X86x64, kernel, testConfigRootfs)
 	if err != nil {
 		return
 	}
 
+	time.Sleep(time.Second)
+
 	port := 45256
 
 	q.Debug(fmt.Sprintf("tcp::%d", port))

qemu/test.config.go

@@ -2,9 +2,9 @@
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
-package qemukernel
+package qemu
 
-const testConfigVmlinuz = "../tools/qemu-debian-img/ubuntu1804.vmlinuz"
-const testConfigInitrd = "../tools/qemu-debian-img/ubuntu1804.initrd"
-const testConfigRootfs = "../tools/qemu-debian-img/ubuntu1804.img"
-const testConfigSampleKo = "../tools/qemu-debian-img/ubuntu1804.ko"
+const testConfigVmlinuz = "../tools/qemu-debian-img/ubuntu2204.vmlinuz"
+const testConfigInitrd = "../tools/qemu-debian-img/ubuntu2204.initrd"
+const testConfigRootfs = "../tools/qemu-debian-img/ubuntu2204.img"
+const testConfigSampleKo = "../tools/qemu-debian-img/ubuntu2204.ko"

shell.nix

@@ -0,0 +1,5 @@
+{ pkgs ? import <nixpkgs> {} }:
+
+with pkgs; mkShell {
+  packages = [ go gcc qemu ];
+}

tools/kernel-factory/.gitignore

@@ -1 +0,0 @@
-output

tools/kernel-factory/Ubuntu/14.04/Dockerfile

@@ -1,9 +0,0 @@
-FROM ubuntu:14.04
-
-RUN apt-get update
-
-# for linux-image-3*-generic, so... CRUTCHES!
-# E: Unable to locate package linux-image-3*-generic
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt search linux-image | grep -e linux-image-3 -e linux-image-4 | grep generic | cut -d '/' -f 1 | xargs apt install -y
-RUN apt search linux-headers | grep -e linux-headers-3 -e linux-headers-4 | grep generic | cut -d '/' -f 1 | xargs apt install -y

tools/kernel-factory/Ubuntu/14.04/Image

@@ -1 +0,0 @@
-../../../qemu-debian-img/ubuntu1404.img

tools/kernel-factory/Ubuntu/16.04/Dockerfile

@@ -1,4 +0,0 @@
-FROM ubuntu:16.04
-
-RUN apt-get update
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -y linux-image-4*-generic linux-headers-*-generic build-essential wget git

tools/kernel-factory/Ubuntu/16.04/Image

@@ -1 +0,0 @@
-../../../qemu-debian-img/ubuntu1604.img

tools/kernel-factory/Ubuntu/18.04/Dockerfile

@@ -1,4 +0,0 @@
-FROM ubuntu:18.04
-
-RUN apt-get update
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -y linux-image-4*-generic linux-headers-*-generic build-essential wget git libelf-dev

tools/kernel-factory/Ubuntu/18.04/Image

@@ -1 +0,0 @@
-../../../qemu-debian-img/ubuntu1804.img

tools/kernel-factory/bootstrap.sh

@@ -1,29 +0,0 @@
-#!/bin/sh -eux
-mkdir -p output
-echo > output/kernels.toml
-find | grep Docker | sed 's/Dockerfile//' | while read DOCKER; do
-    CONTAINER_NAME=$(echo $DOCKER | sed -e 's;/;;g' -e 's;\.;;g' -e 's;\(.*\);\L\1;')
-    docker build -t ${CONTAINER_NAME} ${DOCKER}
-    docker run ${CONTAINER_NAME} bash -c 'ls /boot'
-    CONTAINER_ID=$(docker ps -a | grep ${CONTAINER_NAME} | awk '{print $1}' | head -n 1)
-    docker cp ${CONTAINER_ID}:/boot/. output/
-    DISTRO_NAME=$(echo $DOCKER | cut -d '/' -f 2)
-    DISTRO_VER=$(echo $DOCKER | cut -d '/' -f 3)
-
-    BOOT_FILES="$(docker run $CONTAINER_NAME ls /boot)"
-    for KERNEL_RELEASE in $(docker run $CONTAINER_NAME ls /lib/modules); do
-	echo '[[Kernels]]' >> output/kernels.toml
-	echo 'distro_type =' \"$DISTRO_NAME\" >> output/kernels.toml
-	echo 'distro_release =' \"$DISTRO_VER\" >> output/kernels.toml
-	echo 'kernel_release =' \"$KERNEL_RELEASE\" >> output/kernels.toml
-	echo 'container_name =' \"$CONTAINER_NAME\" >> output/kernels.toml
-	KERNEL_PATH=$(echo $BOOT_FILES | sed  's/ /\n/g' | grep $KERNEL_RELEASE | grep vmlinuz)
-	echo 'kernel_path =' \"$(realpath output/$KERNEL_PATH)\" >> output/kernels.toml
-	INITRD_PATH=$(echo $BOOT_FILES | sed  's/ /\n/g' | grep $KERNEL_RELEASE | grep init)
-	echo 'initrd_path =' \"$(realpath output/$INITRD_PATH)\" >> output/kernels.toml
-	ROOTFS_PATH=$(realpath $DOCKER/Image)
-	echo 'root_f_s =' \"$ROOTFS_PATH\" >> output/kernels.toml
-	echo >> output/kernels.toml
-    done
-done
-rm -rf output/grub

tools/qemu-centos-img/7/Dockerfile

@@ -1,4 +1,4 @@
-# Copyright 2018 Mikhail Klementev. All rights reserved.
+# Copyright 2019 Mikhail Klementev. All rights reserved.
 # Use of this source code is governed by a AGPLv3 license
 # (or later) that can be found in the LICENSE file.
 #
@@ -7,11 +7,13 @@
 #     $ docker build -t gen-centos7-image .
 #     $ docker run --privileged -v $(pwd):/shared -t gen-centos7-image
 #
-# centos7.img will be created in current directory. You can change $(pwd) to
-# different directory to use different destination for image.
+# out_of_tree_centos_7.img will be created in current directory.
+# You can change $(pwd) to different directory to use different destination
+# for image.
 #
 FROM centos:7
 
+RUN yum -y update
 RUN yum -y groupinstall "Development Tools"
 RUN yum -y install qemu-img e2fsprogs
 
@@ -26,7 +28,7 @@ RUN yum --installroot=$TMPDIR \
         --releasever=7 \
         --disablerepo='*' \
         --enablerepo=base \
-        -y install openssh-server
+        -y install openssh-server openssh-clients
 
 RUN chroot $TMPDIR /bin/sh -c 'useradd -m user'
 RUN sed -i 's/root:\*:/root::/' $TMPDIR/etc/shadow
@@ -37,17 +39,15 @@ RUN sed -i '/PermitRootLogin/d' $TMPDIR/etc/ssh/sshd_config
 RUN echo PermitRootLogin yes >> $TMPDIR/etc/ssh/sshd_config
 
 # network workaround
-# FIXME kernel module compatibility issues
 RUN chmod +x $TMPDIR/etc/rc.local
-RUN echo 'find /lib/modules | grep e1000.ko | xargs insmod -f' >> $TMPDIR/etc/rc.local
 RUN echo 'dhclient' >> $TMPDIR/etc/rc.local
 
 ENV IMAGEDIR=/tmp/image
-ENV IMAGE=/shared/centos7.img
+ENV IMAGE=/shared/out_of_tree_centos_7.img
 
 RUN mkdir $IMAGEDIR
 
-# Must be runned with --privileged because of /dev/loop
+# Must be executed with --privileged because of /dev/loop
 CMD qemu-img create $IMAGE 2G && \
 	mkfs.ext4 -F $IMAGE && \
 	mount -o loop $IMAGE $IMAGEDIR && \

tools/qemu-centos-img/8/Dockerfile

@@ -0,0 +1,56 @@
+# Copyright 2020 Mikhail Klementev. All rights reserved.
+# Use of this source code is governed by a AGPLv3 license
+# (or later) that can be found in the LICENSE file.
+#
+# Usage:
+#
+#     $ sudo docker build -t gen-centos8-image .
+#     $ sudo docker run --privileged -v $(pwd):/shared -t gen-centos8-image
+#     $ tar -Szcf out_of_tree_centos_8.img.tar.gz out_of_tree_centos_8.img
+#
+# out_of_tree_centos_8.img will be created in current directory.
+# You can change $(pwd) to different directory to use different destination
+# for image.
+#
+FROM centos:8
+
+RUN yum -y update
+RUN yum -y groupinstall "Development Tools"
+RUN yum -y install qemu-img e2fsprogs
+
+ENV TMPDIR=/tmp/centos
+
+RUN yum --installroot=$TMPDIR \
+        --releasever=8 \
+        --disablerepo='*' \
+        --enablerepo=BaseOS \
+        -y groupinstall Base
+RUN yum --installroot=$TMPDIR \
+        --releasever=8 \
+        --disablerepo='*' \
+        --enablerepo=BaseOS \
+        -y install openssh-server openssh-clients
+
+RUN chroot $TMPDIR /bin/sh -c 'useradd -m user'
+RUN sed -i 's/root:\*:/root::/' $TMPDIR/etc/shadow
+RUN sed -i 's/user:!!:/user::/' $TMPDIR/etc/shadow
+RUN sed -i '/PermitEmptyPasswords/d' $TMPDIR/etc/ssh/sshd_config
+RUN echo PermitEmptyPasswords yes >> $TMPDIR/etc/ssh/sshd_config
+RUN sed -i '/PermitRootLogin/d' $TMPDIR/etc/ssh/sshd_config
+RUN echo PermitRootLogin yes >> $TMPDIR/etc/ssh/sshd_config
+
+# network workaround
+RUN chmod +x $TMPDIR/etc/rc.local
+RUN echo 'dhclient' >> $TMPDIR/etc/rc.local
+
+ENV IMAGEDIR=/tmp/image
+ENV IMAGE=/shared/out_of_tree_centos_8.img
+
+RUN mkdir $IMAGEDIR
+
+# Must be executed with --privileged because of /dev/loop
+CMD qemu-img create $IMAGE 2G && \
+	mkfs.ext4 -F $IMAGE && \
+	mount -o loop $IMAGE $IMAGEDIR && \
+	cp -a $TMPDIR/* $IMAGEDIR/ && \
+	umount $IMAGEDIR

tools/qemu-centos-img/8/generate.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+cd "$(dirname "$0")"
+
+sudo docker build -t gen-centos8-image .
+sudo docker run --privileged -v $(pwd):/shared -t gen-centos8-image
+tar -Szcf out_of_tree_centos_8.img.tar.gz out_of_tree_centos_8.img

tools/qemu-debian-img/14.04/Dockerfile

@@ -0,0 +1,35 @@
+# Copyright 2018 Mikhail Klementev. All rights reserved.
+# Use of this source code is governed by a AGPLv3 license
+# (or later) that can be found in the LICENSE file.
+#
+# Usage:
+#
+#     $ docker build -t gen-ubuntu1404-image .
+#     $ docker run --privileged -v $(pwd):/shared -t gen-ubuntu1404-image
+#
+# ubuntu1404.img will be created in current directory. You can change $(pwd) to
+# different directory to use different destination for image.
+#
+FROM ubuntu:14.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update
+RUN apt-get install -y debootstrap qemu
+
+ENV TMPDIR=/tmp/ubuntu
+ENV IMAGEDIR=/tmp/image
+ENV IMAGE=/shared/out_of_tree_ubuntu_14__04.img
+ENV REPOSITORY=http://archive.ubuntu.com/ubuntu
+ENV RELEASE=trusty
+
+RUN mkdir $IMAGEDIR
+
+# Must be executed with --privileged because of /dev/loop
+CMD debootstrap --include=openssh-server,policykit-1 \
+	$RELEASE $TMPDIR $REPOSITORY && \
+	/shared/setup.sh $TMPDIR && \
+	qemu-img create $IMAGE 2G && \
+	mkfs.ext4 -F $IMAGE && \
+	mount -o loop $IMAGE $IMAGEDIR && \
+	cp -a $TMPDIR/* $IMAGEDIR/ && \
+	umount $IMAGEDIR

tools/qemu-debian-img/14.04/setup.sh

@@ -0,0 +1,17 @@
+#!/bin/sh -eux
+# Copyright 2018 Mikhail Klementev. All rights reserved.
+# Use of this source code is governed by a AGPLv3 license
+# (or later) that can be found in the LICENSE file.
+TMPDIR=$1
+chroot $TMPDIR /bin/sh -c 'useradd -m user'
+sed -i 's/root:\*:/root::/' $TMPDIR/etc/shadow
+sed -i 's/user:!!:/user::/' $TMPDIR/etc/shadow
+echo auth sufficient pam_permit.so > $TMPDIR/etc/pam.d/sshd
+sed -i '/PermitEmptyPasswords/d' $TMPDIR/etc/ssh/sshd_config
+echo PermitEmptyPasswords yes >> $TMPDIR/etc/ssh/sshd_config
+sed -i '/PermitRootLogin/d' $TMPDIR/etc/ssh/sshd_config
+echo PermitRootLogin yes >> $TMPDIR/etc/ssh/sshd_config
+
+echo '#!/bin/sh' > $TMPDIR/etc/rc.local
+echo 'dhclient eth0' >> $TMPDIR/etc/rc.local
+chmod +x $TMPDIR/etc/rc.local

tools/qemu-debian-img/Dockerfile

@@ -4,29 +4,30 @@
 #
 # Usage:
 #
-#     $ docker build -t gen-ubuntu1804-image .
-#     $ docker run --privileged -v $(pwd):/shared -t gen-ubuntu1804-image
+#     $ docker build -t gen-ubuntu2204-image .
+#     $ docker run --privileged -v $(pwd):/shared -t gen-ubuntu2204-image
 #
-# centos7.img will be created in current directory. You can change $(pwd) to
+# ubuntu2204.img will be created in current directory. You can change $(pwd) to
 # different directory to use different destination for image.
 #
-FROM ubuntu:18.04
+FROM ubuntu:22.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update
-RUN apt install -y debootstrap qemu
+RUN apt install -y debootstrap qemu-utils
 RUN apt install -y linux-image-generic
 
 ENV TMPDIR=/tmp/ubuntu
 ENV IMAGEDIR=/tmp/image
-ENV IMAGE=/shared/ubuntu1804.img
+ENV IMAGE=/shared/ubuntu2204.img
 ENV REPOSITORY=http://archive.ubuntu.com/ubuntu
-ENV RELEASE=bionic
+ENV RELEASE=jammy
 
 RUN mkdir $IMAGEDIR
 
-# Must be runned with --privileged because of /dev/loop
-CMD debootstrap --include=openssh-server $RELEASE $TMPDIR $REPOSITORY && \
+# Must be executed with --privileged because of /dev/loop
+CMD debootstrap --include=openssh-server,policykit-1 \
+	$RELEASE $TMPDIR $REPOSITORY && \
 	/shared/setup.sh $TMPDIR && \
 	qemu-img create $IMAGE 2G && \
 	mkfs.ext4 -F $IMAGE && \

tools/qemu-debian-img/bootstrap.sh

@@ -1,7 +1,9 @@
 #!/bin/sh -eux
-docker build -t gen-ubuntu1804-image .
-docker run --privileged -v $(pwd):/shared -t gen-ubuntu1804-image
-RUN="docker run -v $(pwd):/shared -t gen-ubuntu1804-image"
-$RUN sh -c 'chmod 644 /vmlinuz && cp /vmlinuz /shared/ubuntu1804.vmlinuz'
-$RUN sh -c 'cp /initrd.img /shared/ubuntu1804.initrd'
-$RUN sh -c 'cp $(find /lib/modules -name test_static_key_base.ko) /shared/ubuntu1804.ko'
+cd $(dirname $(realpath $0))
+
+docker build -t gen-ubuntu2204-image .
+docker run --privileged -v $(pwd):/shared -t gen-ubuntu2204-image
+RUN="docker run -v $(pwd):/shared -t gen-ubuntu2204-image"
+$RUN sh -c 'chmod 644 /boot/vmlinuz && cp /boot/vmlinuz /shared/ubuntu2204.vmlinuz'
+$RUN sh -c 'cp /boot/initrd.img /shared/ubuntu2204.initrd'
+$RUN sh -c 'cp $(find /lib/modules -name test_bpf.ko) /shared/ubuntu2204.ko'