Bump version

Fixes #15

.github/workflows/donate.yml

@@ -0,0 +1,25 @@
+name: donate
+
+on:
+  issues:
+    types: [opened, closed]
+  schedule:
+    - cron: '15 * * * *' # for updating balance
+
+jobs:
+  comment:
+    runs-on: ubuntu-latest
+    steps:
+      - env:
+          # https://github.com/jollheef/donate/blob/master/dashboard/whitelist.go
+          DASHBOARD_ACCESS_TOKEN: ${{ secrets.DONATE_DASHBOARD_ACCESS_TOKEN }}
+          # the scope is current repository only
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TOOL: "https://github.com/jollheef/donate/archive/master.tar.gz"
+        run: |
+          curl https://nixos.org/nix/install | sh
+          . ~/.nix-profile/etc/profile.d/nix.sh
+          # Use latest stable nixpkgs channel
+          nix-channel --add https://nixos.org/channels/nixos-19.09 nixpkgs
+          nix-channel --update
+          nix run -f $TOOL -c donate-ci

.github/workflows/macos.yml

@@ -0,0 +1,13 @@
+name: macOS
+
+on: [push]
+
+jobs:
+  build:
+    name: Build on macOS
+    runs-on: macOS-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build

.github/workflows/ubuntu.yml

@@ -0,0 +1,56 @@
+name: Ubuntu
+
+on: [push]
+
+jobs:
+  build:
+    name: Build on Ubuntu
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build
+
+  test-unit:
+    name: Unit Testing
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Install dependencies for tests
+      run: |
+        sudo apt-get update
+        sudo apt-get install qemu
+
+    - name: Bootstrap
+      run: ./tools/qemu-debian-img/bootstrap.sh
+
+    - name: Unit Testing
+      run: go test -parallel 1 -v ./...
+
+  test-end-to-end:
+    name: End-to-End Testing
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build
+
+    - name: Install dependencies for tests
+      run: |
+        sudo apt-get update
+        sudo apt-get install qemu
+
+    - name: End-to-End Testing [Kernel Module]
+      run: |
+        cd examples/kernel-module
+        ../../out-of-tree kernel autogen --max=1
+        ../../out-of-tree pew --qemu-timeout=10m
+
+    - name: End-to-End Testing [Kernel Exploit]
+      run: |
+        cd examples/kernel-exploit
+        ../../out-of-tree kernel autogen --max=1
+        ../../out-of-tree pew --threshold=0 --qemu-timeout=10m

.travis.yml

@@ -1,30 +0,0 @@
-language: go
-
-go:
-  - 1.x
-  - master
-
-os:
-  - linux
-
-dist:
-  - bionic
-
-addons:
-  apt:
-    packages:
-    - qemu
-
-services:
-  - docker
-
-env:
-  - GO111MODULE=on
-
-install: true
-
-before_script:
-  - ./tools/qemu-debian-img/bootstrap.sh
-
-script:
-  - go test -parallel 1 -v ./...

CHANGELOG.md

@@ -4,6 +4,93 @@
 
 [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.0]
+
+### Added
+
+- Parameter `--docker-timeout` may also be set in the artifact
+  configuration file.
+
+- Preload modules before inserting module or run exploit. Modules can
+  be specified by git repository path in the `repo` parameter of
+  section `[[preload]]`. Also, there is a `path` parameter for local
+  projects. Note that `repo` is using a cache that uses last commit
+  hash to check is project needs to be rebuilt, so it's not suitable
+  for local development (except if you will commit each time before
+  run out-of-tree).
+
+- Flag `--disable-preload` to ignore `[[preload]]` section of
+  configuration file.
+
+- Now `out-of-tree log dump` will show the last log if no ID
+  specified.
+
+## [1.3.0] 2020-05-30
+
+### Added
+
+- Support for Ubuntu 20.04 and CentOS 8.
+
+## [1.2.1] 2019-12-25
+
+### Fixed
+
+- macOS support.
+
+## [1.2.0] 2019-11-15
+
+### Added
+
+- Flag for Verbose output. Right now only qemu status messages is
+  implemented.
+
+### Fixed
+
+- Kpti settings was not affected for regular runs.
+
+## [1.1.2] 2019-09-05
+
+### Added
+
+- Added policykit-1 to rootfs for Ubuntu.
+
+### Fixed
+
+- Avoided slow mirrors with use of mirror://mirrors.ubuntu.com for
+  Ubuntu 16.04 and newer.
+
+## [1.1.1] 2019-08-31
+
+### Fixed
+
+- macOS support.
+
+## [1.1.0] 2019-08-30
+
+### Added
+
+- Global configuration file (~/.out-of-tree/out-of-tree.toml) allow to
+  set up default values for settings.
+
+- rootfs generator for Ubuntu 14.04.
+
+- Parameter for setting up docker registry server.
+
+- Support for (distro-specific) custom docker commands that will be
+  executed before the base template.
+
+- Parameter for setting up a reliability threshold for exit code.
+
+- Parameter for setting up global timeout, after which no new tasks
+  will be started.
+
+### Fixed
+
+- Spelling in output.
+
+- Now kernel generation will not fail if there are no directory
+  /lib/modules inside the container.
+
 ## [1.0.0] 2019-08-20
 
 ### Added
@@ -96,7 +183,7 @@
 - Temporary files is moved to `~/.out-of-tree/tmp/` to avoid docker
   mounting issues on some systems.
 
-## [0.2.0] - 2019-12-01
+## [0.2.0] - 2018-12-01
 
 The main purpose of the release is to simplify installation.
 
@@ -118,7 +205,7 @@ The main purpose of the release is to simplify installation.
 
 - No warning anymore if test.sh is not exists.
 
-## [0.1.0] - 2019-11-20
+## [0.1.0] - 2018-11-20
 
 Initial release that was never tagged.
 

README.md

@@ -1,5 +1,4 @@
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/aba4aad2046b4d1a9a99cf98e22c018b)](https://app.codacy.com/app/jollheef/out-of-tree?utm_source=github.com&utm_medium=referral&utm_content=jollheef/out-of-tree&utm_campaign=Badge_Grade_Dashboard)
-[![Build Status](https://travis-ci.org/jollheef/out-of-tree.svg?branch=master)](https://travis-ci.org/jollheef/out-of-tree)
 [![Go Report Card](https://goreportcard.com/badge/code.dumpstack.io/tools/out-of-tree)](https://goreportcard.com/report/code.dumpstack.io/tools/out-of-tree)
 [![Documentation Status](https://readthedocs.org/projects/out-of-tree/badge/?version=latest)](https://out-of-tree.readthedocs.io/en/latest/?badge=latest)
 [![Donate](https://img.shields.io/badge/donate-paypal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R8W2UQPZ5X5JE&source=url)
@@ -9,42 +8,29 @@
 
 out-of-tree kernel {module, exploit} development tool
 
+out-of-tree is for automating some routine actions for creating development environments for debugging kernel modules and exploits, generating reliability statistics for exploits, and also provides the ability to easily integrate into CI (Continuous Integration).
+
 ![Screenshot](https://cloudflare-ipfs.com/ipfs/Qmb88fgdDjbWkxz91sWsgmoZZNfVThnCtj37u3mF2s3T3T)
 
-## Requirements
+## Installation
 
-[Qemu](https://www.qemu.org), [docker](https://docker.com) and [golang](https://golang.org) is required.
+### GNU/Linux (with [Nix](https://nixos.org/nix/))
 
-Also do not forget to set GOPATH and PATH e.g.:
+    $ curl -fsSL https://get.docker.com | sh
+	$ sudo usermod -aG docker user && newgrp docker
+    $ curl https://nixos.org/nix/install | sh
+    $ nix-env -iA nixpkgs.out-of-tree
 
-    $ echo 'export GOPATH=$HOME' >> ~/.bashrc
-    $ echo 'export PATH=$PATH:$HOME/bin' >> ~/.bashrc
-    $ source ~/.bashrc
-
-### Gentoo
-
-    # emerge app-emulation/qemu app-emulation/docker dev-lang/go
+Note that adding a user to group *docker* has serious security implications. Check Docker documentation for more information.
 
 ### macOS
 
-    $ brew install go qemu
     $ brew cask install docker
+    $ open --background -a Docker && sleep 1m
+    $ brew tap jollheef/repo
+    $ brew install out-of-tree
 
-### Fedora
-
-    $ sudo dnf install go qemu moby-engine
-
-Also check out [docker post-installation steps](https://docs.docker.com/install/linux/linux-postinstall/).
-
-## Build from source
-
-    $ go get -u code.dumpstack.io/tools/out-of-tree
-
-Then you can check it on kernel module example:
-
-    $ cd $GOPATH/src/code.dumpstack.io/tools/out-of-tree/examples/kernel-module
-    $ out-of-tree kernel autogen # generate kernels based on .out-of-tree.toml
-    $ out-of-tree pew
+Read [documentation](https://out-of-tree.readthedocs.io) for further info.
 
 ## Examples
 
@@ -54,11 +40,11 @@ Run by absolute path
 
 Test only with one kernel:
 
-    $ out-of-tree pew --kernel='Ubuntu:4.10.0-30-generic'
+    $ out-of-tree pew --kernel='Ubuntu:5.4.0-29-generic
 
 Run debug environment:
 
-    $ out-of-tree debug --kernel='Ubuntu:4.10.0-30-generic'
+    $ out-of-tree debug --kernel='Ubuntu:5.4.0-29-generic
 
 Test binary module/exploit with implicit defined test ($BINARY_test)
 
@@ -78,8 +64,7 @@ Use custom kernels config
 
 Generate all kernels
 
-    $ out-of-tree kernel genall --distro Ubuntu --ver 16.04
-
+    $ out-of-tree kernel genall --distro Ubuntu --ver 20.04
 
 ## Troubleshooting
 
@@ -90,9 +75,3 @@ But it'll be better if you'll write the bug report.
 ## Development
 
 Read [Qemu API](qemu/README.md).
-
-### Generate images
-
-    $ cd $GOPATH/src/code.dumpstack.io/tools/out-of-tree/tools/qemu-debian-img/
-    $ docker run --privileged -v $(pwd):/shared -e IMAGE=/shared/ubuntu1404.img -e RELEASE=trusty -t gen-ubuntu1804-image
-    $ docker run --privileged -v $(pwd):/shared -e IMAGE=/shared/ubuntu1604.img -e RELEASE=xenial -t gen-ubuntu1804-image

config/config.go

@@ -102,6 +102,12 @@ func (d Duration) MarshalTOML() (data []byte, err error) {
 	return
 }
 
+type PreloadModule struct {
+	Repo             string
+	Path             string
+	TimeoutAfterLoad Duration
+}
+
 // Artifact is for .out-of-tree.toml
 type Artifact struct {
 	Name             string
@@ -115,12 +121,18 @@ type Artifact struct {
 		Timeout Duration
 	}
 
+	Docker struct {
+		Timeout Duration
+	}
+
 	Mitigations struct {
 		DisableSmep  bool
 		DisableSmap  bool
 		DisableKaslr bool
 		DisableKpti  bool
 	}
+
+	Preload []PreloadModule
 }
 
 func (ka Artifact) checkSupport(ki KernelInfo, km KernelMask) (

config/out-of-tree.go

@@ -0,0 +1,77 @@
+// Copyright 2019 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package config
+
+import (
+	"os/user"
+
+	"github.com/naoina/toml"
+)
+
+type DockerCommand struct {
+	DistroType DistroType
+	Command    string
+}
+
+type OutOfTree struct {
+	Kernels     string
+	UserKernels string
+
+	Database string
+
+	Qemu struct {
+		Timeout string
+	}
+
+	Docker struct {
+		Timeout  string
+		Registry string
+
+		// Commands that will be executed before
+		// the base layer of Dockerfile
+		Commands []DockerCommand
+	}
+}
+
+func ReadOutOfTreeConf(path string) (c OutOfTree, err error) {
+	buf, err := readFileAll(path)
+	if err == nil {
+		err = toml.Unmarshal(buf, &c)
+		if err != nil {
+			return
+		}
+	} else {
+		// It's ok if there's no configuration
+		// then we'll just set default values
+		err = nil
+	}
+
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+
+	if c.Kernels == "" {
+		c.Kernels = usr.HomeDir + "/.out-of-tree/kernels.toml"
+	}
+
+	if c.UserKernels == "" {
+		c.UserKernels = usr.HomeDir + "/.out-of-tree/kernels.user.toml"
+	}
+
+	if c.Database == "" {
+		c.Database = usr.HomeDir + "/.out-of-tree/db.sqlite"
+	}
+
+	if c.Qemu.Timeout == "" {
+		c.Qemu.Timeout = "1m"
+	}
+
+	if c.Docker.Timeout == "" {
+		c.Docker.Timeout = "1m"
+	}
+
+	return
+}

db.go

@@ -254,6 +254,24 @@ func getLogByID(db *sql.DB, id int) (le logEntry, err error) {
 	return
 }
 
+func getLastLog(db *sql.DB) (le logEntry, err error) {
+	err = db.QueryRow("SELECT MAX(id), time, name, type, tag, "+
+		"distro_type, distro_release, kernel_release, "+
+		"build_ok, run_ok, test_ok, "+
+		"build_output, run_output, test_output, "+
+		"qemu_stdout, qemu_stderr, "+
+		"kernel_panic, timeout_kill "+
+		"FROM log").Scan(&le.ID, &le.Timestamp,
+		&le.Name, &le.Type, &le.Tag,
+		&le.DistroType, &le.DistroRelease, &le.KernelRelease,
+		&le.Build.Ok, &le.Run.Ok, &le.Test.Ok,
+		&le.Build.Output, &le.Run.Output, &le.Test.Output,
+		&le.Stdout, &le.Stderr,
+		&le.KernelPanic, &le.KilledByTimeout,
+	)
+	return
+}
+
 func createSchema(db *sql.DB) (err error) {
 	err = createMetadataTable(db)
 	if err != nil {

debug.go

@@ -13,7 +13,7 @@ import (
 	"strings"
 	"time"
 
-	"gopkg.in/logrusorgru/aurora.v1"
+	"gopkg.in/logrusorgru/aurora.v2"
 
 	"code.dumpstack.io/tools/out-of-tree/config"
 	"code.dumpstack.io/tools/out-of-tree/qemu"
@@ -113,6 +113,10 @@ func debugHandler(kcfg config.KernelConfig, workPath, kernRegex, gdb string,
 		q.Memory = ka.Qemu.Memory
 	}
 
+	if ka.Docker.Timeout.Duration != 0 {
+		dockerTimeout = ka.Docker.Timeout.Duration
+	}
+
 	q.SetKASLR(false) // set KASLR to false by default because of gdb
 	q.SetSMEP(!ka.Mitigations.DisableSmep)
 	q.SetSMAP(!ka.Mitigations.DisableSmap)
@@ -147,7 +151,7 @@ func debugHandler(kcfg config.KernelConfig, workPath, kernRegex, gdb string,
 			return aurora.BgGreen(aurora.Black(name))
 		}
 
-		return aurora.BgRed(aurora.Gray(name))
+		return aurora.BgRed(aurora.White(name))
 	}
 
 	fmt.Printf("[*] %s %s %s %s\n",
@@ -161,7 +165,7 @@ func debugHandler(kcfg config.KernelConfig, workPath, kernRegex, gdb string,
 
 	q.Debug(gdb)
 	coloredGdbAddress := aurora.BgGreen(aurora.Black(gdb))
-	fmt.Printf("[*] gdb runned on %s\n", coloredGdbAddress)
+	fmt.Printf("[*] gdb is listening on %s\n", coloredGdbAddress)
 
 	err = q.Start()
 	if err != nil {
@@ -175,6 +179,12 @@ func debugHandler(kcfg config.KernelConfig, workPath, kernRegex, gdb string,
 	}
 	defer os.RemoveAll(tmp)
 
+	err = preloadModules(q, ka, ki, dockerTimeout)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
 	outFile, output, err := build(tmp, ka, ki, dockerTimeout)
 	if err != nil {
 		log.Println(err, output)

docs/installation.rst

@@ -1,4 +1,4 @@
-Installation
+Installation (from source)
 ============
 
 OS/Distro-specific
@@ -36,6 +36,20 @@ There's a minimal configuration that you need to apply::
     ];
   }
 
+Gentoo
+------
+
+Install dependencies::
+
+  $ sudo emerge app-emulation/qemu app-emulation/docker dev-lang/go
+
+Fedora
+------
+
+Install dependencies::
+
+  $ sudo dnf install go qemu moby-engine
+
 Common
 ======
 

docs/introduction.rst

@@ -54,7 +54,7 @@ Overview
 
     $ out-of-tree debug --kernel 'Ubuntu:4.15.0-58-generic'
     [*] KASLR SMEP SMAP
-    [*] gdb runned on tcp::1234
+    [*] gdb is listening on tcp::1234
     [*] build result copied to /tmp/exploit
 
     ssh -o StrictHostKeyChecking=no -p 29308 root@127.133.45.236

examples/kernel-exploit/Makefile

@@ -2,7 +2,7 @@
 # - KERNEL: kernel headers path
 # - TARGET: name of exploit binary that MUST be produced by makefile.
 # - $(TARGET)_test: name of test binary that MUST be produced by makefile
-# and it's will be runned on a LPE stage. TARGET_TEST MUST accept two argument:
+# and it's will be executed on a LPE stage. TARGET_TEST MUST accept two argument:
 #   - Path to exploit binary
 #   - File that MUST be created with exploit. It uses for test that exploit works
 #     correctly.

examples/preload/.out-of-tree.toml

@@ -0,0 +1,12 @@
+name = "out-of-tree preload"
+type = "module"
+
+[[supported_kernels]]
+distro_type = "Ubuntu"
+distro_release = "18.04"
+release_mask = ".*"
+
+[[preload]]
+repo = "https://github.com/jollheef/lkrg"
+#path = "/local/path/to/lkrg"
+timeout_after_load = "1s"

examples/preload/Makefile

@@ -0,0 +1,11 @@
+KERNEL := /lib/modules/$(shell uname -r)/build
+TARGET := module
+
+obj-m += $(TARGET).o
+$(TARGET)-objs = module.o
+
+all:
+	make -C $(KERNEL) M=$(PWD) modules
+
+clean:
+	make -C $(KERNEL) M=$(PWD) clean

examples/preload/README.md

@@ -0,0 +1,5 @@
+# out-of-tree kernel module preload example
+
+See .out-of-tree.toml
+
+Note that it should fail to insert module if lkrg is enabled in the preload list.

examples/preload/module.c

@@ -0,0 +1,17 @@
+#include <linux/module.h>
+#include <linux/slab.h>
+
+int init_module(void)
+{
+	char *argv[] = { "/bin/sh", "--help", NULL };
+	char *envp[] = { NULL };
+
+	/* trigger lkrg */
+	return call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC);
+}
+
+void cleanup_module(void)
+{
+}
+
+MODULE_LICENSE("GPL");

gen.go

@@ -22,6 +22,9 @@ func genConfig(at config.ArtifactType) (err error) {
 		DistroRelease: "18.04",
 		ReleaseMask:   ".*",
 	})
+	a.Preload = append(a.Preload, config.PreloadModule{
+		Repo: "Repo name (e.g. https://github.com/jollheef/lkrg)",
+	})
 
 	buf, err := toml.Marshal(&a)
 	if err != nil {

go.mod

@@ -1,5 +1,7 @@
 module code.dumpstack.io/tools/out-of-tree
 
+go 1.14
+
 replace code.dumpstack.io/tools/out-of-tree/qemu => ./qemu
 
 replace code.dumpstack.io/tools/out-of-tree/config => ./config
@@ -7,15 +9,19 @@ replace code.dumpstack.io/tools/out-of-tree/config => ./config
 require (
 	github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect
 	github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect
+	github.com/go-git/go-git/v5 v5.1.0
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mattn/go-runewidth v0.0.4 // indirect
 	github.com/mattn/go-sqlite3 v1.11.0
 	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/naoina/toml v0.1.1
 	github.com/olekukonko/tablewriter v0.0.1
 	github.com/otiai10/copy v1.0.1
+	github.com/otiai10/curr v1.0.0 // indirect
 	github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce
+	github.com/stretchr/testify v1.5.1 // indirect
 	github.com/zcalusic/sysinfo v0.0.0-20190429151633-fbadb57345c2
-	golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5
+	golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
-	gopkg.in/logrusorgru/aurora.v1 v1.0.0-20181002194514-a7b3b318ed4e
+	gopkg.in/logrusorgru/aurora.v2 v2.0.0-20190417123914-21d75270181e
 )

go.sum

@@ -1,33 +1,108 @@
+bou.ke/monkey v1.0.1 h1:zEMLInw9xvNakzUUPjfS4Ds6jYPqCFx3m7bRmG5NH2U=
 bou.ke/monkey v1.0.1/go.mod h1:FgHuK96Rv2Nlf+0u1OOVDpCMdsWyOFmeeketDHE7LIg=
+github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
+github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
+github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
+github.com/go-git/go-billy/v5 v5.0.0 h1:7NQHvd9FVid8VL4qVUMm8XifBK+2xCoZ2lSk0agRrHM=
+github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
+github.com/go-git/go-git v1.0.0 h1:YcN9iDGDoXuIw0vHls6rINwV416HYa0EB2X+RBsyYp4=
+github.com/go-git/go-git v4.7.0+incompatible h1:+W9rgGY4DOKKdX2x6HxSR7HNeTxqiKrOvKnuittYVdA=
+github.com/go-git/go-git-fixtures/v4 v4.0.1/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw=
+github.com/go-git/go-git/v5 v5.1.0 h1:HxJn9g/E7eYvKW3Fm7Jt4ee8LXfPOm/H1cdDu8vEssk=
+github.com/go-git/go-git/v5 v5.1.0/go.mod h1:ZKfuPUoY1ZqIG4QG9BDBh3G4gLM5zvPuSJAozQrZuyM=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=
+github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
+github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
 github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
 github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
 github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
 github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
 github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/otiai10/copy v1.0.1 h1:gtBjD8aq4nychvRZ2CyJvFWAw0aja+VHazDdruZKGZA=
 github.com/otiai10/copy v1.0.1/go.mod h1:8bMCJrAqOtN/d9oyh5HR7HhLQMvcGMpGdwRDYsfOCHc=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI=
+github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
+github.com/otiai10/mint v1.2.3 h1:PsrRBmrxR68kyNu6YlqYHbNlItc5vOkuS6LBEsNttVA=
 github.com/otiai10/mint v1.2.3/go.mod h1:YnfyPNhBvnY8bW4SGQHCs/aAFhkgySlMZbrF5U0bOVw=
+github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc=
+github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce h1:aP+C+YbHZfOQlutA4p4soHi7rVUqHQdWEVMSkHfDTqY=
 github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
+github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
 github.com/zcalusic/sysinfo v0.0.0-20190429151633-fbadb57345c2 h1:uMiaKNX5zFLOa6nNtun+d/lpV5bOBh7BvE4q9jfZacQ=
 github.com/zcalusic/sysinfo v0.0.0-20190429151633-fbadb57345c2/go.mod h1:zAn3FAIbgZPYnutDND49Ivf8sb/mXYk8UjZdqMswgHg=
+golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
-gopkg.in/logrusorgru/aurora.v1 v1.0.0-20181002194514-a7b3b318ed4e h1:uKdf1KQDFZDYqNzSDhxB5hFxj5Fq4e3/C/ejtRJxlY0=
-gopkg.in/logrusorgru/aurora.v1 v1.0.0-20181002194514-a7b3b318ed4e/go.mod h1:DGR33jeYG1jxERD2W4hGjuW94Pxf3mkUf/Ddhf5BskA=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/logrusorgru/aurora.v2 v2.0.0-20190417123914-21d75270181e h1:Wc0601/F/0TByNewL9UAKk18FfwumyYyT8pJMIHcolA=
+gopkg.in/logrusorgru/aurora.v2 v2.0.0-20190417123914-21d75270181e/go.mod h1:Wm+IEn1fgFp8E2paL93oFVrHZW4toMKARNE85fDY5w8=
+gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
+gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

kernel.go

@@ -15,11 +15,11 @@ import (
 	"os/exec"
 	"os/user"
 	"regexp"
+	"runtime"
 	"strings"
 	"time"
 
 	"github.com/naoina/toml"
-	"github.com/zcalusic/sysinfo"
 
 	"code.dumpstack.io/tools/out-of-tree/config"
 )
@@ -69,7 +69,7 @@ func matchDebianHeadersPkg(container, mask string, generic bool) (
 func matchCentOSDevelPkg(container, mask string, generic bool) (
 	pkgs []string, err error) {
 
-	cmd := "yum search kernel-devel --show-duplicates | " +
+	cmd := "yum search kernel-devel --showduplicates | " +
 		"grep '^kernel-devel' | cut -d ' ' -f 1"
 	output, err := dockerRun(time.Minute, container, "/tmp", cmd)
 	if err != nil {
@@ -100,6 +100,14 @@ func dockerImagePath(sk config.KernelMask) (path string, err error) {
 }
 
 func vsyscallAvailable() (available bool, err error) {
+	if runtime.GOOS != "linux" {
+		// Docker for non-Linux systems is not using the host
+		// kernel but uses kernel inside a virtual machine, so
+		// it builds by the Docker team with vsyscall support.
+		available = true
+		return
+	}
+
 	buf, err := ioutil.ReadFile("/proc/self/maps")
 	if err != nil {
 		return
@@ -109,7 +117,9 @@ func vsyscallAvailable() (available bool, err error) {
 	return
 }
 
-func generateBaseDockerImage(sk config.KernelMask) (err error) {
+func generateBaseDockerImage(registry string, commands []config.DockerCommand,
+	sk config.KernelMask) (err error) {
+
 	imagePath, err := dockerImagePath(sk)
 	if err != nil {
 		return
@@ -128,7 +138,12 @@ func generateBaseDockerImage(sk config.KernelMask) (err error) {
 		sk.DistroType.String(), sk.DistroRelease)
 	os.MkdirAll(imagePath, os.ModePerm)
 
-	d += fmt.Sprintf("FROM %s:%s\n",
+	d += "FROM "
+	if registry != "" {
+		d += registry + "/"
+	}
+
+	d += fmt.Sprintf("%s:%s\n",
 		strings.ToLower(sk.DistroType.String()),
 		sk.DistroRelease,
 	)
@@ -138,16 +153,38 @@ func generateBaseDockerImage(sk config.KernelMask) (err error) {
 		return
 	}
 
+	for _, c := range commands {
+		switch c.DistroType {
+		case config.Ubuntu:
+			d += "RUN " + c.Command + "\n"
+		case config.CentOS:
+			d += "RUN " + c.Command + "\n"
+		case config.Debian:
+			d += "RUN " + c.Command + "\n"
+		default:
+			err = fmt.Errorf("%s not yet supported",
+				sk.DistroType.String())
+			return
+		}
+	}
+
 	switch sk.DistroType {
 	case config.Ubuntu:
 		d += "ENV DEBIAN_FRONTEND=noninteractive\n"
+		if sk.DistroRelease >= "16.04" {
+			from := "http://.*ubuntu/"
+			to := "mirror://mirrors.ubuntu.com/mirrors.txt"
+			file := "/etc/apt/sources.list"
+			s := fmt.Sprintf("sed -i 's;%s;%s;' %s", from, to, file)
+			d += "RUN " + s + "\n"
+		}
 		d += "RUN apt-get update\n"
 		d += "RUN apt-get install -y build-essential libelf-dev\n"
 		d += "RUN apt-get install -y wget git\n"
 		if sk.DistroRelease >= "14.04" {
 			d += "RUN apt-get install -y libseccomp-dev\n"
 		}
-		d += "RUN mkdir /lib/modules\n"
+		d += "RUN mkdir -p /lib/modules\n"
 	case config.CentOS:
 		if sk.DistroRelease < "7" && !vsyscall {
 			log.Println("Old CentOS requires `vsyscall=emulate` " +
@@ -164,8 +201,24 @@ func generateBaseDockerImage(sk config.KernelMask) (err error) {
 		// do not remove old kernels
 		d += "RUN sed -i 's;installonly_limit=;installonly_limit=100500;' /etc/yum.conf\n"
 		d += "RUN yum -y update\n"
+
+		if sk.DistroRelease == "8" {
+			// FIXME CentOS Vault repository list for 8 is empty
+			// at the time of this fix; check for it and use a
+			// workaround if it's still empty
+			d += `RUN grep enabled /etc/yum.repos.d/CentOS-Vault.repo` +
+				` || echo -e '[8.0.1905]\nbaseurl=http://vault.centos.org/8.0.1905/BaseOS/$basearch/os/'` +
+				` >> /etc/yum.repos.d/CentOS-Vault.repo` + "\n"
+		}
+
 		d += "RUN yum -y groupinstall 'Development Tools'\n"
+
+		if sk.DistroRelease < "8" {
 			d += "RUN yum -y install deltarpm\n"
+		} else {
+			d += "RUN yum -y install drpm grub2-tools-minimal " +
+				"elfutils-libelf-devel\n"
+		}
 	default:
 		err = fmt.Errorf("%s not yet supported", sk.DistroType.String())
 		return
@@ -236,6 +289,7 @@ func dockerImageAppend(sk config.KernelMask, pkgname string) (err error) {
 			"/boot/initramfs-%s.img %s\n", version, version)
 	default:
 		err = fmt.Errorf("%s not yet supported", sk.DistroType.String())
+		return
 	}
 
 	err = ioutil.WriteFile(imagePath+"/Dockerfile",
@@ -397,66 +451,6 @@ func listDockerImages() (diis []dockerImageInfo, err error) {
 	return
 }
 
-func genHostKernels(download bool) (kcfg config.KernelConfig, err error) {
-	si := sysinfo.SysInfo{}
-	si.GetSysInfo()
-
-	distroType, err := config.NewDistroType(si.OS.Vendor)
-	if err != nil {
-		return
-	}
-
-	cmd := exec.Command("ls", "/lib/modules")
-	rawOutput, err := cmd.CombinedOutput()
-	if err != nil {
-		log.Println(string(rawOutput), err)
-		return
-	}
-
-	kernelsBase := "/boot/"
-	files, err := ioutil.ReadDir(kernelsBase)
-	if err != nil {
-		return
-	}
-
-	// only for compatibility, docker is not really used
-	dii := dockerImageInfo{
-		ContainerName: config.KernelMask{
-			DistroType:    distroType,
-			DistroRelease: si.OS.Version,
-		}.DockerName(),
-	}
-
-	rootfs, err := genRootfsImage(dii, download)
-	if err != nil {
-		return
-	}
-
-	for _, k := range strings.Fields(string(rawOutput)) {
-		ki := config.KernelInfo{
-			DistroType:    distroType,
-			DistroRelease: si.OS.Version,
-			KernelRelease: k,
-
-			KernelSource: "/lib/modules/" + k + "/build",
-
-			KernelPath: kernelsBase + genKernelPath(files, k),
-			InitrdPath: kernelsBase + genInitrdPath(files, k),
-			RootFS:     rootfs,
-		}
-
-		vmlinux := "/usr/lib/debug/boot/vmlinux-" + k
-		log.Println("vmlinux", vmlinux)
-		if exists(vmlinux) {
-			ki.VmlinuxPath = vmlinux
-		}
-
-		kcfg.Kernels = append(kcfg.Kernels, ki)
-	}
-
-	return
-}
-
 func updateKernelsCfg(host, download bool) (err error) {
 	newkcfg := config.KernelConfig{}
 
@@ -573,7 +567,10 @@ func shuffle(a []string) []string {
 	return a
 }
 
-func generateKernels(km config.KernelMask, max int64, download bool) (err error) {
+func generateKernels(km config.KernelMask, registry string,
+	commands []config.DockerCommand, max int64,
+	download bool) (err error) {
+
 	log.Println("Generating for kernel mask", km)
 
 	_, err = genRootfsImage(dockerImageInfo{ContainerName: km.DockerName()},
@@ -582,7 +579,7 @@ func generateKernels(km config.KernelMask, max int64, download bool) (err error)
 		return
 	}
 
-	err = generateBaseDockerImage(km)
+	err = generateBaseDockerImage(registry, commands, km)
 	if err != nil {
 		return
 	}
@@ -632,7 +629,10 @@ func generateKernels(km config.KernelMask, max int64, download bool) (err error)
 	return
 }
 
-func kernelAutogenHandler(workPath string, max int64, host, download bool) (err error) {
+func kernelAutogenHandler(workPath, registry string,
+	commands []config.DockerCommand,
+	max int64, host, download bool) (err error) {
+
 	ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
 	if err != nil {
 		return
@@ -644,7 +644,7 @@ func kernelAutogenHandler(workPath string, max int64, host, download bool) (err
 			return
 		}
 
-		err = generateKernels(sk, max, download)
+		err = generateKernels(sk, registry, commands, max, download)
 		if err != nil {
 			return
 		}
@@ -695,7 +695,9 @@ func kernelDockerRegenHandler(host, download bool) (err error) {
 	return updateKernelsCfg(host, download)
 }
 
-func kernelGenallHandler(distro, version string, host, download bool) (err error) {
+func kernelGenallHandler(distro, version, registry string,
+	commands []config.DockerCommand, host, download bool) (err error) {
+
 	distroType, err := config.NewDistroType(distro)
 	if err != nil {
 		return
@@ -706,7 +708,7 @@ func kernelGenallHandler(distro, version string, host, download bool) (err error
 		DistroRelease: version,
 		ReleaseMask:   ".*",
 	}
-	err = generateKernels(km, kernelsAll, download)
+	err = generateKernels(km, registry, commands, kernelsAll, download)
 	if err != nil {
 		return
 	}

kernel_linux.go

@@ -0,0 +1,77 @@
+// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+// +build linux
+
+package main
+
+import (
+	"io/ioutil"
+	"log"
+	"os/exec"
+	"strings"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+	"github.com/zcalusic/sysinfo"
+)
+
+func genHostKernels(download bool) (kcfg config.KernelConfig, err error) {
+	si := sysinfo.SysInfo{}
+	si.GetSysInfo()
+
+	distroType, err := config.NewDistroType(si.OS.Vendor)
+	if err != nil {
+		return
+	}
+
+	cmd := exec.Command("ls", "/lib/modules")
+	rawOutput, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Println(string(rawOutput), err)
+		return
+	}
+
+	kernelsBase := "/boot/"
+	files, err := ioutil.ReadDir(kernelsBase)
+	if err != nil {
+		return
+	}
+
+	// only for compatibility, docker is not really used
+	dii := dockerImageInfo{
+		ContainerName: config.KernelMask{
+			DistroType:    distroType,
+			DistroRelease: si.OS.Version,
+		}.DockerName(),
+	}
+
+	rootfs, err := genRootfsImage(dii, download)
+	if err != nil {
+		return
+	}
+
+	for _, k := range strings.Fields(string(rawOutput)) {
+		ki := config.KernelInfo{
+			DistroType:    distroType,
+			DistroRelease: si.OS.Version,
+			KernelRelease: k,
+
+			KernelSource: "/lib/modules/" + k + "/build",
+
+			KernelPath: kernelsBase + genKernelPath(files, k),
+			InitrdPath: kernelsBase + genInitrdPath(files, k),
+			RootFS:     rootfs,
+		}
+
+		vmlinux := "/usr/lib/debug/boot/vmlinux-" + k
+		log.Println("vmlinux", vmlinux)
+		if exists(vmlinux) {
+			ki.VmlinuxPath = vmlinux
+		}
+
+		kcfg.Kernels = append(kcfg.Kernels, ki)
+	}
+
+	return
+}

kernel_macos.go

@@ -0,0 +1,18 @@
+// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+// +build darwin
+
+package main
+
+import (
+	"errors"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+)
+
+func genHostKernels(download bool) (kcfg config.KernelConfig, err error) {
+	err = errors.New("generate host kernels for macOS is not supported")
+	return
+}

log.go

@@ -13,7 +13,7 @@ import (
 	"os"
 
 	"github.com/olekukonko/tablewriter"
-	"gopkg.in/logrusorgru/aurora.v1"
+	"gopkg.in/logrusorgru/aurora.v2"
 
 	"code.dumpstack.io/tools/out-of-tree/config"
 )
@@ -100,7 +100,12 @@ func logHandler(db *sql.DB, path, tag string, num int, rate bool) (err error) {
 }
 
 func logDumpHandler(db *sql.DB, id int) (err error) {
-	l, err := getLogByID(db, id)
+	var l logEntry
+	if id > 0 {
+		l, err = getLogByID(db, id)
+	} else {
+		l, err = getLastLog(db)
+	}
 	if err != nil {
 		return
 	}

main.go

@@ -84,7 +84,7 @@ func main() {
 	)
 
 	app.Author("Mikhail Klementev <root@dumpstack.io>")
-	app.Version("1.0.1")
+	app.Version("1.4.0")
 
 	pathFlag := app.Flag("path", "Path to work directory")
 	path := pathFlag.Default(".").ExistingDir()
@@ -95,25 +95,39 @@ func main() {
 	}
 	os.MkdirAll(usr.HomeDir+"/.out-of-tree", os.ModePerm)
 
-	defaultKcfgPath := usr.HomeDir + "/.out-of-tree/kernels.toml"
+	confPath := usr.HomeDir + "/.out-of-tree/out-of-tree.toml"
+	conf, err := config.ReadOutOfTreeConf(confPath)
+	if err != nil {
+		return
+	}
 
 	kcfgPathFlag := app.Flag("kernels", "Path to main kernels config")
-	kcfgPath := kcfgPathFlag.Default(defaultKcfgPath).String()
+	kcfgPath := kcfgPathFlag.Default(conf.Kernels).String()
 
-	defaultDbPath := usr.HomeDir + "/.out-of-tree/db.sqlite"
 	dbPathFlag := app.Flag("db", "Path to database")
-	dbPath := dbPathFlag.Default(defaultDbPath).String()
+	dbPath := dbPathFlag.Default(conf.Database).String()
 
-	defaultUserKcfgPath := usr.HomeDir + "/.out-of-tree/kernels.user.toml"
 	userKcfgPathFlag := app.Flag("user-kernels", "User kernels config")
 	userKcfgPathEnv := userKcfgPathFlag.Envar("OUT_OF_TREE_KCFG")
-	userKcfgPath := userKcfgPathEnv.Default(defaultUserKcfgPath).String()
+	userKcfgPath := userKcfgPathEnv.Default(conf.UserKernels).String()
+
+	timeoutFlag := app.Flag("timeout", "Timeout after tool will not spawn new tests")
+	timeout := timeoutFlag.Duration()
 
 	qemuTimeoutFlag := app.Flag("qemu-timeout", "Timeout for qemu")
-	qemuTimeout := qemuTimeoutFlag.Default("1m").Duration()
+	qemuTimeout := qemuTimeoutFlag.Default(conf.Qemu.Timeout).Duration()
 
 	dockerTimeoutFlag := app.Flag("docker-timeout", "Timeout for docker")
-	dockerTimeout := dockerTimeoutFlag.Default("1m").Duration()
+	dockerTimeout := dockerTimeoutFlag.Default(conf.Docker.Timeout).Duration()
+
+	dockerRegistryFlag := app.Flag("docker-registry", "Registry for docker")
+	dockerRegistry := dockerRegistryFlag.Default(conf.Docker.Registry).String()
+
+	thresholdFlag := app.Flag("threshold", "Reliablity threshold for exit code")
+	threshold := thresholdFlag.Default("1.00").Float64()
+
+	disablePreloadFlag := app.Flag("disable-preload", "Disable module preload")
+	disablePreload = disablePreloadFlag.Bool()
 
 	pewCommand := app.Command("pew", "Build, run and test module/exploit")
 
@@ -144,6 +158,9 @@ func main() {
 	pewTagFlag := pewCommand.Flag("tag", "Log tagging")
 	pewTag := pewTagFlag.String()
 
+	pewVerboseFlag := pewCommand.Flag("verbose", "Show more information")
+	pewVerbose := pewVerboseFlag.Bool()
+
 	kernelCommand := app.Command("kernel", "Manipulate kernels")
 	kernelNoDownload := kernelCommand.Flag("no-download",
 		"Do not download qemu image while kernel generation").Bool()
@@ -200,7 +217,7 @@ func main() {
 
 	logDumpCommand := logCommand.Command("dump",
 		"Show all info for log entry with ID")
-	logDumpID := logDumpCommand.Arg("ID", "").Required().Int()
+	logDumpID := logDumpCommand.Arg("ID", "").Default("-1").Int()
 
 	logJSONCommand := logCommand.Command("json", "Generate json statistics")
 	logJSONTag := logJSONCommand.Flag("tag", "Filter tag").Required().String()
@@ -253,6 +270,10 @@ func main() {
 		log.Fatalln("Only one of disable/enable can be used at once")
 	}
 
+	if *yekpti && *nokpti {
+		log.Fatalln("Only one of disable/enable can be used at once")
+	}
+
 	kcfg, err := config.ReadKernelConfig(*kcfgPath)
 	if err != nil {
 		log.Println(err)
@@ -279,20 +300,28 @@ func main() {
 	}
 	defer db.Close()
 
+	stop := time.Time{} // never stop
+	if *timeout != 0 {
+		stop = time.Now().Add(*timeout)
+	}
+
 	switch kingpin.MustParse(app.Parse(os.Args[1:])) {
 	case pewCommand.FullCommand():
 		err = pewHandler(kcfg, *path, *pewKernel, *pewBinary,
-			*pewTest, *pewGuess, *qemuTimeout, *dockerTimeout,
-			*pewMax, *pewRuns, *pewDist, *pewTag, *pewThreads, db)
+			*pewTest, *pewGuess, stop, *qemuTimeout, *dockerTimeout,
+			*pewMax, *pewRuns, *pewDist, *pewTag, *pewThreads,
+			db, *pewVerbose)
 	case kernelListCommand.FullCommand():
 		err = kernelListHandler(kcfg)
 	case kernelAutogenCommand.FullCommand():
-		err = kernelAutogenHandler(*path, *kernelAutogenMax,
+		err = kernelAutogenHandler(*path, *dockerRegistry,
+			conf.Docker.Commands, *kernelAutogenMax,
 			*kernelUseHost, !*kernelNoDownload)
 	case kernelDockerRegenCommand.FullCommand():
 		err = kernelDockerRegenHandler(*kernelUseHost, !*kernelNoDownload)
 	case kernelGenallCommand.FullCommand():
 		err = kernelGenallHandler(*distro, *version,
+			*dockerRegistry, conf.Docker.Commands,
 			*kernelUseHost, !*kernelNoDownload)
 	case genModuleCommand.FullCommand():
 		err = genConfig(config.KernelModule)
@@ -316,7 +345,8 @@ func main() {
 	case logMarkdownCommand.FullCommand():
 		err = logMarkdownHandler(db, *path, *logMarkdownTag)
 	case packCommand.FullCommand():
-		err = packHandler(db, *path, kcfg, *packAutogen,
+		err = packHandler(db, *path, *dockerRegistry, stop,
+			conf.Docker.Commands, kcfg, *packAutogen,
 			!*packNoDownload, *packExploitRuns, *packKernelRuns)
 	}
 
@@ -324,7 +354,7 @@ func main() {
 		log.Fatalln(err)
 	}
 
-	if somethingFailed {
+	if successRate(state) < *threshold {
 		os.Exit(1)
 	}
 }

pack.go

@@ -15,7 +15,8 @@ import (
 	"code.dumpstack.io/tools/out-of-tree/config"
 )
 
-func packHandler(db *sql.DB, path string, kcfg config.KernelConfig,
+func packHandler(db *sql.DB, path, registry string, stop time.Time,
+	commands []config.DockerCommand, kcfg config.KernelConfig,
 	autogen, download bool, exploitRuns, kernelRuns int64) (err error) {
 
 	dockerTimeout := time.Minute
@@ -39,7 +40,8 @@ func packHandler(db *sql.DB, path string, kcfg config.KernelConfig,
 
 		if autogen {
 			var perRegex int64 = 1
-			err = kernelAutogenHandler(workPath, perRegex, false, download)
+			err = kernelAutogenHandler(workPath, registry,
+				commands, perRegex, false, download)
 			if err != nil {
 				return
 			}
@@ -48,8 +50,9 @@ func packHandler(db *sql.DB, path string, kcfg config.KernelConfig,
 		log.Println(f.Name())
 
 		pewHandler(kcfg, workPath, "", "", "", false,
-			dockerTimeout, qemuTimeout,
-			kernelRuns, exploitRuns, pathDevNull, tag, threads, db)
+			stop, dockerTimeout, qemuTimeout,
+			kernelRuns, exploitRuns, pathDevNull,
+			tag, threads, db, false)
 	}
 
 	return

pew.go

@@ -20,13 +20,23 @@ import (
 
 	"github.com/otiai10/copy"
 	"github.com/remeh/sizedwaitgroup"
-	"gopkg.in/logrusorgru/aurora.v1"
+	"gopkg.in/logrusorgru/aurora.v2"
 
 	"code.dumpstack.io/tools/out-of-tree/config"
 	"code.dumpstack.io/tools/out-of-tree/qemu"
 )
 
-var somethingFailed = false
+type runstate struct {
+	Overall, Success float64
+}
+
+var (
+	state runstate
+)
+
+func successRate(state runstate) float64 {
+	return state.Success / state.Overall
+}
 
 const pathDevNull = "/dev/null"
 
@@ -134,13 +144,14 @@ func testKernelExploit(q *qemu.System, ka config.Artifact,
 }
 
 func genOkFail(name string, ok bool) (aurv aurora.Value) {
+	state.Overall += 1
 	if ok {
+		state.Success += 1
 		s := " " + name + " SUCCESS "
 		aurv = aurora.BgGreen(aurora.Black(s))
 	} else {
-		somethingFailed = true
 		s := " " + name + " FAILURE "
-		aurv = aurora.BgRed(aurora.Gray(aurora.Bold(s)))
+		aurv = aurora.BgRed(aurora.White(aurora.Bold(s)))
 	}
 	return
 }
@@ -292,7 +303,7 @@ func copyTest(q *qemu.System, testPath string, ka config.Artifact) (
 func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
 	ki config.KernelInfo, binaryPath, testPath string,
 	qemuTimeout, dockerTimeout time.Duration, dist, tag string,
-	db *sql.DB) {
+	db *sql.DB, verbose bool) {
 
 	defer swg.Done()
 
@@ -314,9 +325,14 @@ func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
 		q.Memory = ka.Qemu.Memory
 	}
 
+	if ka.Docker.Timeout.Duration != 0 {
+		dockerTimeout = ka.Docker.Timeout.Duration
+	}
+
 	q.SetKASLR(!ka.Mitigations.DisableKaslr)
 	q.SetSMEP(!ka.Mitigations.DisableSmep)
 	q.SetSMAP(!ka.Mitigations.DisableSmap)
+	q.SetKPTI(!ka.Mitigations.DisableKpti)
 
 	err = q.Start()
 	if err != nil {
@@ -325,6 +341,18 @@ func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
 	}
 	defer q.Stop()
 
+	if verbose {
+		go func() {
+			for !q.Died {
+				time.Sleep(time.Minute)
+				log.Println(ka.Name, ki.DistroType,
+					ki.DistroRelease, ki.KernelRelease,
+					"still alive")
+
+			}
+		}()
+	}
+
 	usr, err := user.Current()
 	if err != nil {
 		return
@@ -367,6 +395,12 @@ func whatever(swg *sizedwaitgroup.SizedWaitGroup, ka config.Artifact,
 		return
 	}
 
+	err = preloadModules(q, ka, ki, dockerTimeout)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
 	copyArtifactAndTest(q, ka, &result, remoteTest)
 }
 
@@ -380,9 +414,10 @@ func shuffleKernels(a []config.KernelInfo) []config.KernelInfo {
 }
 
 func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath,
-	testPath string, qemuTimeout, dockerTimeout time.Duration,
+	testPath string, stop time.Time,
+	qemuTimeout, dockerTimeout time.Duration,
 	max, runs int64, dist, tag string, threads int,
-	db *sql.DB) (err error) {
+	db *sql.DB, verbose bool) (err error) {
 
 	found := false
 
@@ -402,10 +437,13 @@ func performCI(ka config.Artifact, kcfg config.KernelConfig, binaryPath,
 			found = true
 			max--
 			for i := int64(0); i < runs; i++ {
+				if !stop.IsZero() && time.Now().After(stop) {
+					break
+				}
 				swg.Add()
 				go whatever(&swg, ka, kernel, binaryPath,
 					testPath, qemuTimeout, dockerTimeout,
-					dist, tag, db)
+					dist, tag, db, verbose)
 			}
 		}
 	}
@@ -457,11 +495,12 @@ func genAllKernels() (sk []config.KernelMask, err error) {
 	return
 }
 
+// TODO: Now too many parameters, move all of them to some structure
 func pewHandler(kcfg config.KernelConfig,
 	workPath, ovrrdKrnl, binary, test string, guess bool,
-	qemuTimeout, dockerTimeout time.Duration,
+	stop time.Time, qemuTimeout, dockerTimeout time.Duration,
 	max, runs int64, dist, tag string, threads int,
-	db *sql.DB) (err error) {
+	db *sql.DB, verbose bool) (err error) {
 
 	ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
 	if err != nil {
@@ -489,8 +528,9 @@ func pewHandler(kcfg config.KernelConfig,
 		}
 	}
 
-	err = performCI(ka, kcfg, binary, test, qemuTimeout, dockerTimeout,
-		max, runs, dist, tag, threads, db)
+	err = performCI(ka, kcfg, binary, test,
+		stop, qemuTimeout, dockerTimeout,
+		max, runs, dist, tag, threads, db, verbose)
 	if err != nil {
 		return
 	}

preload.go

@@ -0,0 +1,172 @@
+// Copyright 2020 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a AGPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package main
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"errors"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/user"
+	"path/filepath"
+	"time"
+
+	"github.com/go-git/go-git/v5"
+
+	"code.dumpstack.io/tools/out-of-tree/config"
+	"code.dumpstack.io/tools/out-of-tree/qemu"
+)
+
+var disablePreload *bool
+
+func preloadModules(q *qemu.System, ka config.Artifact, ki config.KernelInfo,
+	dockerTimeout time.Duration) (err error) {
+
+	if *disablePreload {
+		return
+	}
+
+	for _, pm := range ka.Preload {
+		err = preload(q, ki, pm, dockerTimeout)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func preload(q *qemu.System, ki config.KernelInfo, pm config.PreloadModule,
+	dockerTimeout time.Duration) (err error) {
+
+	var workPath, cache string
+	if pm.Path != "" {
+		log.Println("Use non-git path for preload module (no cache)")
+		workPath = pm.Path
+	} else if pm.Repo != "" {
+		workPath, cache, err = cloneOrPull(pm.Repo, ki)
+		if err != nil {
+			return
+		}
+	} else {
+		errors.New("No repo/path in preload entry")
+	}
+
+	err = buildAndInsmod(workPath, q, ki, dockerTimeout, cache)
+	if err != nil {
+		return
+	}
+
+	time.Sleep(pm.TimeoutAfterLoad.Duration)
+	return
+}
+
+func buildAndInsmod(workPath string, q *qemu.System, ki config.KernelInfo,
+	dockerTimeout time.Duration, cache string) (err error) {
+
+	tmp, err := ioutil.TempDir("", "out-of-tree_")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(tmp)
+
+	var artifact string
+	if exists(cache) {
+		artifact = cache
+	} else {
+		artifact, err = buildPreload(workPath, tmp, ki, dockerTimeout)
+		if err != nil {
+			return
+		}
+		if cache != "" {
+			err = copyFile(artifact, cache)
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	output, err := q.CopyAndInsmod(artifact)
+	if err != nil {
+		log.Println(output)
+		return
+	}
+	return
+}
+
+func buildPreload(workPath, tmp string, ki config.KernelInfo,
+	dockerTimeout time.Duration) (artifact string, err error) {
+
+	ka, err := config.ReadArtifactConfig(workPath + "/.out-of-tree.toml")
+	if err != nil {
+		return
+	}
+
+	ka.SourcePath = workPath
+
+	km := config.KernelMask{DistroType: ki.DistroType,
+		DistroRelease: ki.DistroRelease,
+		ReleaseMask:   ki.KernelRelease,
+	}
+	ka.SupportedKernels = []config.KernelMask{km}
+
+	if ka.Docker.Timeout.Duration != 0 {
+		dockerTimeout = ka.Docker.Timeout.Duration
+	}
+
+	artifact, _, err = build(tmp, ka, ki, dockerTimeout)
+	return
+}
+
+func cloneOrPull(repo string, ki config.KernelInfo) (workPath, cache string, err error) {
+	usr, err := user.Current()
+	if err != nil {
+		return
+	}
+	base := filepath.Join(usr.HomeDir, "/.out-of-tree/preload/")
+	workPath = filepath.Join(base, "/repos/", sha1sum(repo))
+
+	var r *git.Repository
+	if exists(workPath) {
+		r, err = git.PlainOpen(workPath)
+		if err != nil {
+			return
+		}
+
+		var w *git.Worktree
+		w, err = r.Worktree()
+		if err != nil {
+			return
+		}
+
+		err = w.Pull(&git.PullOptions{})
+		if err != nil && err != git.NoErrAlreadyUpToDate {
+			log.Println(repo, "pull error:", err)
+		}
+	} else {
+		r, err = git.PlainClone(workPath, false, &git.CloneOptions{URL: repo})
+		if err != nil {
+			return
+		}
+	}
+
+	ref, err := r.Head()
+	if err != nil {
+		return
+	}
+
+	cachedir := filepath.Join(base, "/cache/")
+	os.MkdirAll(cachedir, 0700)
+
+	filename := sha1sum(repo + ki.KernelPath + ref.Hash().String())
+	cache = filepath.Join(cachedir, filename)
+	return
+}
+
+func sha1sum(data string) string {
+	h := sha1.Sum([]byte(data))
+	return hex.EncodeToString(h[:])
+}

qemu/qemu-kernel.go

@@ -60,7 +60,7 @@ type Kernel struct {
 	InitrdPath string
 }
 
-// System describe qemu parameters and runned process
+// System describe qemu parameters and executed process
 type System struct {
 	arch      arch
 	kernel    Kernel
@@ -86,7 +86,7 @@ type System struct {
 	Died        bool
 	sshAddrPort string
 
-	// accessible while qemu is runned
+	// accessible while qemu is running
 	cmd  *exec.Cmd
 	pipe struct {
 		stdin  io.WriteCloser

qemu/qemu-kernel_test.go

@@ -116,6 +116,7 @@ func startTestQemu(t *testing.T, timeout time.Duration) (q *System, err error) {
 		return
 	}
 
+	time.Sleep(time.Second)
 	return
 }
 
@@ -324,6 +325,8 @@ func TestSystemDebug(t *testing.T) {
 		return
 	}
 
+	time.Sleep(time.Second)
+
 	port := 45256
 
 	q.Debug(fmt.Sprintf("tcp::%d", port))

qemu/test.config.go

@@ -4,7 +4,7 @@
 
 package qemu
 
-const testConfigVmlinuz = "../tools/qemu-debian-img/ubuntu1804.vmlinuz"
-const testConfigInitrd = "../tools/qemu-debian-img/ubuntu1804.initrd"
-const testConfigRootfs = "../tools/qemu-debian-img/ubuntu1804.img"
-const testConfigSampleKo = "../tools/qemu-debian-img/ubuntu1804.ko"
+const testConfigVmlinuz = "../tools/qemu-debian-img/ubuntu2004.vmlinuz"
+const testConfigInitrd = "../tools/qemu-debian-img/ubuntu2004.initrd"
+const testConfigRootfs = "../tools/qemu-debian-img/ubuntu2004.img"
+const testConfigSampleKo = "../tools/qemu-debian-img/ubuntu2004.ko"

tools/qemu-centos-img/7/Dockerfile

@@ -47,7 +47,7 @@ ENV IMAGE=/shared/out_of_tree_centos_7.img
 
 RUN mkdir $IMAGEDIR
 
-# Must be runned with --privileged because of /dev/loop
+# Must be executed with --privileged because of /dev/loop
 CMD qemu-img create $IMAGE 2G && \
 	mkfs.ext4 -F $IMAGE && \
 	mount -o loop $IMAGE $IMAGEDIR && \

tools/qemu-centos-img/8/Dockerfile

@@ -0,0 +1,56 @@
+# Copyright 2020 Mikhail Klementev. All rights reserved.
+# Use of this source code is governed by a AGPLv3 license
+# (or later) that can be found in the LICENSE file.
+#
+# Usage:
+#
+#     $ sudo docker build -t gen-centos8-image .
+#     $ sudo docker run --privileged -v $(pwd):/shared -t gen-centos8-image
+#     $ tar -Szcf out_of_tree_centos_8.img.tar.gz out_of_tree_centos_8.img
+#
+# out_of_tree_centos_8.img will be created in current directory.
+# You can change $(pwd) to different directory to use different destination
+# for image.
+#
+FROM centos:8
+
+RUN yum -y update
+RUN yum -y groupinstall "Development Tools"
+RUN yum -y install qemu-img e2fsprogs
+
+ENV TMPDIR=/tmp/centos
+
+RUN yum --installroot=$TMPDIR \
+        --releasever=8 \
+        --disablerepo='*' \
+        --enablerepo=BaseOS \
+        -y groupinstall Base
+RUN yum --installroot=$TMPDIR \
+        --releasever=8 \
+        --disablerepo='*' \
+        --enablerepo=BaseOS \
+        -y install openssh-server openssh-clients
+
+RUN chroot $TMPDIR /bin/sh -c 'useradd -m user'
+RUN sed -i 's/root:\*:/root::/' $TMPDIR/etc/shadow
+RUN sed -i 's/user:!!:/user::/' $TMPDIR/etc/shadow
+RUN sed -i '/PermitEmptyPasswords/d' $TMPDIR/etc/ssh/sshd_config
+RUN echo PermitEmptyPasswords yes >> $TMPDIR/etc/ssh/sshd_config
+RUN sed -i '/PermitRootLogin/d' $TMPDIR/etc/ssh/sshd_config
+RUN echo PermitRootLogin yes >> $TMPDIR/etc/ssh/sshd_config
+
+# network workaround
+RUN chmod +x $TMPDIR/etc/rc.local
+RUN echo 'dhclient' >> $TMPDIR/etc/rc.local
+
+ENV IMAGEDIR=/tmp/image
+ENV IMAGE=/shared/out_of_tree_centos_8.img
+
+RUN mkdir $IMAGEDIR
+
+# Must be executed with --privileged because of /dev/loop
+CMD qemu-img create $IMAGE 2G && \
+	mkfs.ext4 -F $IMAGE && \
+	mount -o loop $IMAGE $IMAGEDIR && \
+	cp -a $TMPDIR/* $IMAGEDIR/ && \
+	umount $IMAGEDIR

tools/qemu-centos-img/8/generate.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+cd "$(dirname "$0")"
+
+sudo docker build -t gen-centos8-image .
+sudo docker run --privileged -v $(pwd):/shared -t gen-centos8-image
+tar -Szcf out_of_tree_centos_8.img.tar.gz out_of_tree_centos_8.img

tools/qemu-debian-img/14.04/Dockerfile

@@ -0,0 +1,35 @@
+# Copyright 2018 Mikhail Klementev. All rights reserved.
+# Use of this source code is governed by a AGPLv3 license
+# (or later) that can be found in the LICENSE file.
+#
+# Usage:
+#
+#     $ docker build -t gen-ubuntu1404-image .
+#     $ docker run --privileged -v $(pwd):/shared -t gen-ubuntu1404-image
+#
+# ubuntu1404.img will be created in current directory. You can change $(pwd) to
+# different directory to use different destination for image.
+#
+FROM ubuntu:14.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update
+RUN apt-get install -y debootstrap qemu
+
+ENV TMPDIR=/tmp/ubuntu
+ENV IMAGEDIR=/tmp/image
+ENV IMAGE=/shared/out_of_tree_ubuntu_14__04.img
+ENV REPOSITORY=http://archive.ubuntu.com/ubuntu
+ENV RELEASE=trusty
+
+RUN mkdir $IMAGEDIR
+
+# Must be executed with --privileged because of /dev/loop
+CMD debootstrap --include=openssh-server,policykit-1 \
+	$RELEASE $TMPDIR $REPOSITORY && \
+	/shared/setup.sh $TMPDIR && \
+	qemu-img create $IMAGE 2G && \
+	mkfs.ext4 -F $IMAGE && \
+	mount -o loop $IMAGE $IMAGEDIR && \
+	cp -a $TMPDIR/* $IMAGEDIR/ && \
+	umount $IMAGEDIR

tools/qemu-debian-img/14.04/setup.sh

@@ -0,0 +1,17 @@
+#!/bin/sh -eux
+# Copyright 2018 Mikhail Klementev. All rights reserved.
+# Use of this source code is governed by a AGPLv3 license
+# (or later) that can be found in the LICENSE file.
+TMPDIR=$1
+chroot $TMPDIR /bin/sh -c 'useradd -m user'
+sed -i 's/root:\*:/root::/' $TMPDIR/etc/shadow
+sed -i 's/user:!!:/user::/' $TMPDIR/etc/shadow
+echo auth sufficient pam_permit.so > $TMPDIR/etc/pam.d/sshd
+sed -i '/PermitEmptyPasswords/d' $TMPDIR/etc/ssh/sshd_config
+echo PermitEmptyPasswords yes >> $TMPDIR/etc/ssh/sshd_config
+sed -i '/PermitRootLogin/d' $TMPDIR/etc/ssh/sshd_config
+echo PermitRootLogin yes >> $TMPDIR/etc/ssh/sshd_config
+
+echo '#!/bin/sh' > $TMPDIR/etc/rc.local
+echo 'dhclient eth0' >> $TMPDIR/etc/rc.local
+chmod +x $TMPDIR/etc/rc.local

tools/qemu-debian-img/Dockerfile

@@ -4,29 +4,30 @@
 #
 # Usage:
 #
-#     $ docker build -t gen-ubuntu1804-image .
-#     $ docker run --privileged -v $(pwd):/shared -t gen-ubuntu1804-image
+#     $ docker build -t gen-ubuntu2004-image .
+#     $ docker run --privileged -v $(pwd):/shared -t gen-ubuntu2004-image
 #
-# ubuntu1804.img will be created in current directory. You can change $(pwd) to
+# ubuntu2004.img will be created in current directory. You can change $(pwd) to
 # different directory to use different destination for image.
 #
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update
-RUN apt install -y debootstrap qemu
+RUN apt install -y debootstrap qemu-utils
 RUN apt install -y linux-image-generic
 
 ENV TMPDIR=/tmp/ubuntu
 ENV IMAGEDIR=/tmp/image
-ENV IMAGE=/shared/ubuntu1804.img
+ENV IMAGE=/shared/ubuntu2004.img
 ENV REPOSITORY=http://archive.ubuntu.com/ubuntu
-ENV RELEASE=bionic
+ENV RELEASE=focal
 
 RUN mkdir $IMAGEDIR
 
-# Must be runned with --privileged because of /dev/loop
-CMD debootstrap --include=openssh-server $RELEASE $TMPDIR $REPOSITORY && \
+# Must be executed with --privileged because of /dev/loop
+CMD debootstrap --include=openssh-server,policykit-1 \
+	$RELEASE $TMPDIR $REPOSITORY && \
 	/shared/setup.sh $TMPDIR && \
 	qemu-img create $IMAGE 2G && \
 	mkfs.ext4 -F $IMAGE && \

tools/qemu-debian-img/bootstrap.sh

@@ -1,9 +1,9 @@
 #!/bin/sh -eux
 cd $(dirname $(realpath $0))
 
-docker build -t gen-ubuntu1804-image .
-docker run --privileged -v $(pwd):/shared -t gen-ubuntu1804-image
-RUN="docker run -v $(pwd):/shared -t gen-ubuntu1804-image"
-$RUN sh -c 'chmod 644 /vmlinuz && cp /vmlinuz /shared/ubuntu1804.vmlinuz'
-$RUN sh -c 'cp /initrd.img /shared/ubuntu1804.initrd'
-$RUN sh -c 'cp $(find /lib/modules -name test_static_key_base.ko) /shared/ubuntu1804.ko'
+docker build -t gen-ubuntu2004-image .
+docker run --privileged -v $(pwd):/shared -t gen-ubuntu2004-image
+RUN="docker run -v $(pwd):/shared -t gen-ubuntu2004-image"
+$RUN sh -c 'chmod 644 /boot/vmlinuz && cp /boot/vmlinuz /shared/ubuntu2004.vmlinuz'
+$RUN sh -c 'cp /boot/initrd.img /shared/ubuntu2004.initrd'
+$RUN sh -c 'cp $(find /lib/modules -name test_bpf.ko) /shared/ubuntu2004.ko'