fix: skip retracted kernels

BREAKING CHANGE: Parameters for the kernel command are changed

from

--distro= --ver= --kernel=

to

--distro-id= --distro-release= --kernel-regex=

.github/workflows/e2e.yml

@@ -14,6 +14,7 @@ on:
       - ".github/workflows/macos.yml"
       - ".github/workflows/debian-cache.yml"
       - "docs/**"
+      - 'tools/**'
       - ".readthedocs.yaml"
       - "README.md"
 
@@ -125,7 +126,7 @@ jobs:
         echo 'distro = { id = "${{ matrix.os.distro }}", release = "${{ matrix.os.release }}" }' >> examples/kernel-module/.out-of-tree.toml
         echo 'kernel = { regex = ".*" }' >> examples/kernel-module/.out-of-tree.toml
         echo '[qemu]' >> examples/kernel-module/.out-of-tree.toml
-        echo 'timeout = "10m"' >> examples/kernel-module/.out-of-tree.toml
+        echo 'timeout = "5m"' >> examples/kernel-module/.out-of-tree.toml
         echo 'after_start_timeout = "10s"' >> examples/kernel-module/.out-of-tree.toml
 
         echo 'modprobe uio || modprobe 9p || modprobe xfs' >> examples/kernel-module/test.sh
@@ -142,7 +143,7 @@ jobs:
         echo 'WorkingDirectory=/root/test' >> test.service
         echo 'TimeoutStopSec=1' >> test.service
         echo 'ExecStart=/usr/local/bin/out-of-tree kernel --no-prebuilt-containers autogen --threads=8 --max=64 --shuffle' >> test.service
-        echo 'ExecStart=/usr/local/bin/out-of-tree pew --qemu-timeout=10m --threads=4 --include-internal-errors' >> test.service
+        echo 'ExecStart=/usr/local/bin/out-of-tree pew --threads=4 --include-internal-errors' >> test.service
 
         scp test.service root@$IP:/etc/systemd/system/test.service
 

.github/workflows/images-centos.yml

@@ -0,0 +1,87 @@
+name: CentOS images
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - 'tools/qemu-centos-img/**'
+      - '.github/workflows/images-centos.yml'
+
+concurrency:
+  group: ${{ github.workflow_ref }}
+  cancel-in-progress: true
+
+jobs:
+  images-centos:
+    name: Qemu Images
+    runs-on: ubuntu-latest
+    steps:
+    - uses: digitalocean/action-doctl@v2
+      with:
+        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+
+    - uses: webfactory/ssh-agent@v0.8.0
+      with:
+        ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+
+    - name: create droplet
+      run: doctl compute droplet create --ssh-keys='b4:4c:66:7d:be:19:25:43:1c:e0:02:61:9f:49:12:94,37:46:77:a8:4a:96:3b:20:16:46:35:04:95:ca:0c:5c' --tag-name=github-actions ga-out-of-tree-images-centos-$GITHUB_SHA --size s-1vcpu-1gb --image ubuntu-22-04-x64 --wait
+
+      # TODO Move to common script
+    - name: generate images
+      shell: bash
+      run: |
+        sleep 1m
+
+        IP=$(doctl compute droplet list --tag-name=github-actions --format "Name,Public IPv4" | grep -v ID | grep ga-out-of-tree-images-centos-$GITHUB_SHA | awk '{print $2}')
+
+        while ! ssh -o StrictHostKeyChecking=accept-new root@$IP echo
+        do
+          sleep 1s
+        done
+
+        ssh root@$IP "cloud-init status --wait | grep done"
+
+        ssh root@$IP apt-get update
+        ssh root@$IP apt-get install -y git podman s3cmd
+        ssh root@$IP git clone https://github.com/out-of-tree/out-of-tree
+        ssh root@$IP "cd out-of-tree && git checkout $GITHUB_SHA"
+
+        ssh root@$IP "echo -e '[Unit]\nDescription=CentOS image generator and uploader\n[Service]\nRemainAfterExit=yes\nStandardError=append:/var/log/images-centos.log\nStandardOutput=append:/var/log/images-centos.log\nType=oneshot' >> /etc/systemd/system/images-centos.service"
+
+        ssh root@$IP "echo Environment=HOST=fra1.digitaloceanspaces.com >> /etc/systemd/system/images-centos.service"
+        ssh root@$IP "echo Environment=HOST_BUCKET=out-of-tree.fra1.digitaloceanspaces.com >> /etc/systemd/system/images-centos.service"
+        ssh root@$IP "echo Environment=ACCESS_KEY=${{ secrets.DIGITALOCEAN_SPACES_ACCESS_KEY }} >> /etc/systemd/system/images-centos.service"
+        ssh root@$IP "echo Environment=SECRET_KEY=${{ secrets.DIGITALOCEAN_SPACES_SECRET_KEY }} >> /etc/systemd/system/images-centos.service"
+
+        ssh root@$IP "echo 'ExecStart=/root/out-of-tree/tools/qemu-centos-img/6/generate.sh' >> /etc/systemd/system/images-centos.service"
+        ssh root@$IP "echo 'ExecStart=/root/out-of-tree/tools/qemu-centos-img/7/generate.sh' >> /etc/systemd/system/images-centos.service"
+        ssh root@$IP "echo 'ExecStart=/root/out-of-tree/tools/qemu-centos-img/8/generate.sh' >> /etc/systemd/system/images-centos.service"
+
+        ssh root@$IP 'echo ExecStart=/bin/sh -c \"s3cmd put --acl-public /root/out-of-tree/tools/qemu-centos-img/*/*.tar.gz s3://out-of-tree/3.0.0/ --host=\$HOST --host-bucket=\$HOST_BUCKET --access_key=\$ACCESS_KEY --secret_key=\$SECRET_KEY\" >> /etc/systemd/system/images-centos.service'
+
+        ssh root@$IP "echo TimeoutStopSec=1 >> /etc/systemd/system/images-centos.service"
+
+        ssh root@$IP systemctl daemon-reload
+
+        ssh root@$IP systemctl start images-centos --no-block
+
+        while ! ssh root@$IP systemctl show images-centos -p SubState --value | grep -E '(failed|exited)'
+        do
+          sleep 3m
+        done
+
+        scp root@$IP:/var/log/images-centos.log .
+
+        ssh root@$IP systemctl is-active images-centos
+
+    - name: Archive logs
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: images-centos-log
+        path: images-centos.log
+
+    - name: delete droplet
+      if: always()
+      run: doctl compute droplet delete -f ga-out-of-tree-images-centos-$GITHUB_SHA

.github/workflows/images-debian.yml

@@ -1,4 +1,4 @@
-name: Debian
+name: Debian images
 
 on:
   workflow_dispatch:
@@ -25,7 +25,7 @@ jobs:
         ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
 
     - name: create droplet
-      run: doctl compute droplet create --ssh-keys='b4:4c:66:7d:be:19:25:43:1c:e0:02:61:9f:49:12:94' --tag-name=github-actions ga-out-of-tree-images-debian-$GITHUB_SHA --size s-1vcpu-1gb --image ubuntu-22-04-x64 --wait
+      run: doctl compute droplet create --ssh-keys='b4:4c:66:7d:be:19:25:43:1c:e0:02:61:9f:49:12:94,37:46:77:a8:4a:96:3b:20:16:46:35:04:95:ca:0c:5c' --tag-name=github-actions ga-out-of-tree-images-debian-$GITHUB_SHA --size s-1vcpu-1gb --image ubuntu-22-04-x64 --wait
 
       # TODO Move to common script
     - name: generate images
@@ -40,8 +40,7 @@ jobs:
           sleep 1s
         done
 
-        sleep 5m
-        ssh root@$IP pkill apt-get || true
+        ssh root@$IP "cloud-init status --wait | grep done"
 
         ssh root@$IP apt-get update
         ssh root@$IP apt-get install -y git podman s3cmd
@@ -57,7 +56,7 @@ jobs:
 
         ssh root@$IP "echo 'ExecStart=/root/out-of-tree/tools/qemu-debian-img/generate-images.sh' >> /etc/systemd/system/images-debian.service"
 
-        ssh root@$IP 'echo ExecStart=/bin/sh -c \"s3cmd put --acl-public /root/out-of-tree/tools/qemu-debian-img/*.tar.gz s3://out-of-tree/1.0.0/ --host=\$HOST --host-bucket=\$HOST_BUCKET --access_key=\$ACCESS_KEY --secret_key=\$SECRET_KEY\" >> /etc/systemd/system/images-debian.service'
+        ssh root@$IP 'echo ExecStart=/bin/sh -c \"s3cmd put --acl-public /root/out-of-tree/tools/qemu-debian-img/*.tar.gz s3://out-of-tree/3.0.0/ --host=\$HOST --host-bucket=\$HOST_BUCKET --access_key=\$ACCESS_KEY --secret_key=\$SECRET_KEY\" >> /etc/systemd/system/images-debian.service'
 
         ssh root@$IP "echo TimeoutStopSec=1 >> /etc/systemd/system/images-debian.service"
 

.github/workflows/images-oraclelinux.yml

@@ -1,4 +1,4 @@
-name: Oracle Linux
+name: Oracle Linux images
 
 on:
   workflow_dispatch:
@@ -25,7 +25,7 @@ jobs:
         ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
 
     - name: create droplet
-      run: doctl compute droplet create --ssh-keys='b4:4c:66:7d:be:19:25:43:1c:e0:02:61:9f:49:12:94' --tag-name=github-actions ga-out-of-tree-images-oraclelinux-$GITHUB_SHA --size s-1vcpu-1gb --image ubuntu-22-04-x64 --wait
+      run: doctl compute droplet create --ssh-keys='b4:4c:66:7d:be:19:25:43:1c:e0:02:61:9f:49:12:94,37:46:77:a8:4a:96:3b:20:16:46:35:04:95:ca:0c:5c' --tag-name=github-actions ga-out-of-tree-images-oraclelinux-$GITHUB_SHA --size s-1vcpu-2gb --image ubuntu-22-04-x64 --wait
 
       # TODO Move to common script
     - name: generate images
@@ -40,8 +40,7 @@ jobs:
           sleep 1s
         done
 
-        sleep 5m
-        ssh root@$IP pkill apt-get || true
+        ssh root@$IP "cloud-init status --wait | grep done"
 
         ssh root@$IP apt-get update
         ssh root@$IP apt-get install -y git podman s3cmd
@@ -57,7 +56,7 @@ jobs:
 
         ssh root@$IP "echo 'ExecStart=/root/out-of-tree/tools/qemu-oraclelinux-img/generate-images.sh' >> /etc/systemd/system/images-oraclelinux.service"
 
-        ssh root@$IP 'echo ExecStart=/bin/sh -c \"s3cmd put --acl-public /root/out-of-tree/tools/qemu-oraclelinux-img/*.tar.gz s3://out-of-tree/1.0.0/ --host=\$HOST --host-bucket=\$HOST_BUCKET --access_key=\$ACCESS_KEY --secret_key=\$SECRET_KEY\" >> /etc/systemd/system/images-oraclelinux.service'
+        ssh root@$IP 'echo ExecStart=/bin/sh -c \"s3cmd put --acl-public /root/out-of-tree/tools/qemu-oraclelinux-img/*.tar.gz s3://out-of-tree/3.0.0/ --host=\$HOST --host-bucket=\$HOST_BUCKET --access_key=\$ACCESS_KEY --secret_key=\$SECRET_KEY\" >> /etc/systemd/system/images-oraclelinux.service'
 
         ssh root@$IP "echo TimeoutStopSec=1 >> /etc/systemd/system/images-oraclelinux.service"
 
@@ -74,6 +73,13 @@ jobs:
 
         ssh root@$IP systemctl is-active images-oraclelinux
 
+    - name: Archive logs
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: images-oraclelinux-log
+        path: images-oraclelinux.log
+
     - name: delete droplet
       if: always()
       run: doctl compute droplet delete -f ga-out-of-tree-images-oraclelinux-$GITHUB_SHA

.github/workflows/images-ubuntu.yml

@@ -58,7 +58,7 @@ jobs:
 
         ssh root@$IP "echo 'ExecStart=/root/out-of-tree/tools/qemu-ubuntu-img/generate-images.py' >> /etc/systemd/system/images-ubuntu.service"
 
-        ssh root@$IP 'echo ExecStart=/bin/sh -c \"s3cmd put --acl-public /root/out-of-tree/tools/qemu-ubuntu-img/*.tar.gz s3://out-of-tree/1.0.0/ --host=\$HOST --host-bucket=\$HOST_BUCKET --access_key=\$ACCESS_KEY --secret_key=\$SECRET_KEY\" >> /etc/systemd/system/images-ubuntu.service'
+        ssh root@$IP 'echo ExecStart=/bin/sh -c \"s3cmd put --acl-public /root/out-of-tree/tools/qemu-ubuntu-img/*.tar.gz s3://out-of-tree/3.0.0/ --host=\$HOST --host-bucket=\$HOST_BUCKET --access_key=\$ACCESS_KEY --secret_key=\$SECRET_KEY\" >> /etc/systemd/system/images-ubuntu.service'
 
         ssh root@$IP "echo TimeoutStopSec=1 >> /etc/systemd/system/images-ubuntu.service"
 

.github/workflows/ubuntu.yml

@@ -203,7 +203,7 @@ jobs:
 
           cp ../examples/kernel-module/{module.c,Makefile,test.sh} .
 
-          ../out-of-tree --log-level=debug kernel list-remote --distro=${{ matrix.os.distro }} --ver=${{ matrix.os.release }}
+          ../out-of-tree --log-level=debug kernel list-remote --distro-id=${{ matrix.os.distro }} --distro-release=${{ matrix.os.release }}
           ../out-of-tree --log-level=debug kernel autogen --max=1 --shuffle
           ../out-of-tree --log-level=debug pew --qemu-timeout=20m --include-internal-errors
 

README.md

@@ -8,8 +8,6 @@
 
 *out-of-tree* was created to reduce the complexity of the environment for developing, testing and debugging Linux kernel exploits and out-of-tree kernel modules (hence the name "out-of-tree").
 
-![Screenshot](https://cloudflare-ipfs.com/ipfs/Qmb88fgdDjbWkxz91sWsgmoZZNfVThnCtj37u3mF2s3T3T)
-
 ## Installation
 
 ### GNU/Linux (with [Nix](https://nixos.org/nix/))
@@ -42,9 +40,9 @@ Read [documentation](https://out-of-tree.readthedocs.io) for further info.
 
 ## Examples
 
-Generate all Ubuntu 22.04 kernels:
+Download all Ubuntu 24.04 kernels:
 
-    $ out-of-tree kernel genall --distro=Ubuntu --ver=22.04
+    $ out-of-tree kernel genall --distro-id=Ubuntu --distro-release=24.04
 
 Run tests based on .out-of-tree.toml definitions:
 
@@ -52,8 +50,8 @@ Run tests based on .out-of-tree.toml definitions:
 
 Test with a specific kernel:
 
-    $ out-of-tree pew --kernel='Ubuntu:5.4.0-29-generic'
+    $ out-of-tree pew --realtime-output  --distro-id=ubuntu --kernel-regex=6.8.0-41-generic
 
 Run debug environment:
 
-    $ out-of-tree debug --kernel='Ubuntu:5.4.0-29-generic'
+    $ out-of-tree debug --distro-id=ubuntu --distro-release=24.04 --kernel-regex=6.8.0-41-generic

artifact/artifact.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/naoina/toml"
 	"github.com/rs/zerolog"
-	"github.com/rs/zerolog/log"
 
 	"code.dumpstack.io/tools/out-of-tree/config/dotfiles"
 	"code.dumpstack.io/tools/out-of-tree/distro"
@@ -241,8 +240,9 @@ func (ka Artifact) Supported(ki distro.KernelInfo) (supported bool, err error) {
 	return
 }
 
+// TODO too many parameters
 func (ka Artifact) Process(slog zerolog.Logger, ki distro.KernelInfo,
-	endless bool, cBinary,
+	outputOnSuccess, realtimeOutput, endless bool, cBinary,
 	cEndlessStress string, cEndlessTimeout time.Duration,
 	dump func(q *qemu.System, ka Artifact, ki distro.KernelInfo,
 		result *Result)) {
@@ -334,12 +334,22 @@ func (ka Artifact) Process(slog zerolog.Logger, ki distro.KernelInfo,
 		// TODO: build should return structure
 		start := time.Now()
 		result.BuildDir, result.BuildArtifact, result.Build.Output, err =
-			Build(slog, tmp, ka, ki, ka.Docker.Timeout.Duration)
+			Build(slog, tmp, ka, ki, ka.Docker.Timeout.Duration, realtimeOutput)
 		slog.Debug().Str("duration", time.Since(start).String()).
 			Msg("build done")
 		if err != nil {
-			log.Error().Err(err).Msg("build")
+			if !realtimeOutput {
+				slog.Error().Err(err).Msgf("build failure\n%v\n", result.Build.Output)
+			} else {
+				slog.Error().Err(err).Msg("build failure")
+			}
 			return
+		} else {
+			if outputOnSuccess && !realtimeOutput {
+				slog.Info().Msgf("build success\n%v\n", result.Build.Output)
+			} else {
+				slog.Info().Msg("build success")
+			}
 		}
 		result.Build.Ok = true
 	} else {
@@ -361,6 +371,8 @@ func (ka Artifact) Process(slog zerolog.Logger, ki distro.KernelInfo,
 		ka.Qemu.Timeout.Duration = time.Minute
 	}
 
+	slog.Info().Msg("wait for vm initialisation")
+
 	err = q.WaitForSSH(ka.Qemu.Timeout.Duration)
 	if err != nil {
 		result.InternalError = err
@@ -397,11 +409,40 @@ func (ka Artifact) Process(slog zerolog.Logger, ki distro.KernelInfo,
 		return
 	}
 
+	var qemuTestOutput string
+	q.SetQemuOutputHandler(func(s string) {
+		if realtimeOutput {
+			fmt.Printf("kmsg: %s\n", s)
+		} else {
+			qemuTestOutput += s + "\n"
+		}
+	})
+
+	if realtimeOutput {
+		q.SetCommandsOutputHandler(func(s string) {
+			fmt.Printf("test: %s\n", s)
+		})
+	}
+
 	start := time.Now()
-	copyArtifactAndTest(slog, q, ka, &result, remoteTest)
+	slog.Info().Msg("copy artifact and run test")
+	copyArtifactAndTest(slog, q, ka, &result, remoteTest, outputOnSuccess, realtimeOutput)
 	slog.Debug().Str("duration", time.Since(start).String()).
 		Msgf("test completed (success: %v)", result.Test.Ok)
 
+	if result.Build.Ok && !realtimeOutput {
+		if !result.Run.Ok || !result.Test.Ok {
+			slog.Error().Msgf("qemu output\n%v\n", qemuTestOutput)
+		} else if outputOnSuccess {
+			slog.Info().Msgf("qemu output\n%v\n", qemuTestOutput)
+		}
+	}
+
+	if realtimeOutput {
+		q.CloseCommandsOutputHandler()
+	}
+	q.CloseQemuOutputHandler()
+
 	if !endless {
 		return
 	}

artifact/preload.go

@@ -111,7 +111,7 @@ func buildPreload(workPath, tmp string, ki distro.KernelInfo,
 		dockerTimeout = ka.Docker.Timeout.Duration
 	}
 
-	_, af, _, err = Build(log.Logger, tmp, ka, ki, dockerTimeout)
+	_, af, _, err = Build(log.Logger, tmp, ka, ki, dockerTimeout, false)
 	return
 }
 

artifact/process.go

@@ -103,7 +103,7 @@ func applyPatches(src string, ka Artifact) (err error) {
 }
 
 func Build(flog zerolog.Logger, tmp string, ka Artifact,
-	ki distro.KernelInfo, dockerTimeout time.Duration) (
+	ki distro.KernelInfo, dockerTimeout time.Duration, realtimeOutput bool) (
 	outdir, outpath, output string, err error) {
 
 	target := strings.Replace(ka.Name, " ", "_", -1)
@@ -157,9 +157,19 @@ func Build(flog zerolog.Logger, tmp string, ka Artifact,
 
 		c.Args = append(c.Args, "--network", "none")
 
+		if realtimeOutput {
+			c.SetCommandsOutputHandler(func(s string) {
+				fmt.Printf("%s\n", s)
+			})
+		}
+
 		output, err = c.Run(outdir, []string{
 			buildCommand + " && chmod -R 777 /work",
 		})
+
+		if realtimeOutput {
+			c.CloseCommandsOutputHandler()
+		}
 	} else {
 		cmd := exec.Command("bash", "-c", "cd "+outdir+" && "+
 			buildCommand)
@@ -281,7 +291,7 @@ func CopyFile(sourcePath, destinationPath string) (err error) {
 }
 
 func copyArtifactAndTest(slog zerolog.Logger, q *qemu.System, ka Artifact,
-	res *Result, remoteTest string) (err error) {
+	res *Result, remoteTest string, outputOnSuccess, realtimeOutput bool) (err error) {
 
 	// Copy all test files to the remote machine
 	for _, f := range ka.TestFiles {
@@ -313,8 +323,7 @@ func copyArtifactAndTest(slog zerolog.Logger, q *qemu.System, ka Artifact,
 
 		res.Test.Output, err = testKernelModule(q, ka, remoteTest)
 		if err != nil {
-			slog.Error().Err(err).Msg(res.Test.Output)
-			return
+			break
 		}
 		res.Test.Ok = true
 	case KernelExploit:
@@ -327,16 +336,14 @@ func copyArtifactAndTest(slog zerolog.Logger, q *qemu.System, ka Artifact,
 		res.Test.Output, err = testKernelExploit(q, ka, remoteTest,
 			remoteExploit)
 		if err != nil {
-			slog.Error().Err(err).Msg(res.Test.Output)
-			return
+			break
 		}
 		res.Run.Ok = true // does not really used
 		res.Test.Ok = true
 	case Script:
 		res.Test.Output, err = runScript(q, remoteTest)
 		if err != nil {
-			slog.Error().Err(err).Msg(res.Test.Output)
-			return
+			break
 		}
 		res.Run.Ok = true
 		res.Test.Ok = true
@@ -344,7 +351,20 @@ func copyArtifactAndTest(slog zerolog.Logger, q *qemu.System, ka Artifact,
 		slog.Fatal().Msg("Unsupported artifact type")
 	}
 
-	slog.Info().Msgf("\n%v\n", res.Test.Output)
+	if err != nil || !res.Test.Ok {
+		if !realtimeOutput {
+			slog.Error().Err(err).Msgf("test failure\n%v\n", res.Test.Output)
+		} else {
+			slog.Error().Err(err).Msg("test failure")
+		}
+		return
+	}
+
+	if outputOnSuccess && !realtimeOutput {
+		slog.Info().Msgf("test success\n%v\n", res.Test.Output)
+	} else {
+		slog.Info().Msg("test success")
+	}
 
 	_, err = q.Command("root", "echo")
 	if err != nil {

client/client.go

@@ -198,9 +198,11 @@ func (c Client) PushRepo(repo api.Repo) (err error) {
 	remote := fmt.Sprintf("git://%s/%s", addr, repo.Name)
 	log.Debug().Msgf("git proxy remote: %v", remote)
 
-	raw, err := exec.Command("git", "--work-tree", repo.Path, "push", "--force", remote).
+	raw, err := exec.Command("git", "-c", "push.default=current",
+		"--work-tree", repo.Path, "push", "--force", remote).
 		CombinedOutput()
 	if err != nil {
+		log.Error().Msgf("push repo %v\n%v", repo, string(raw))
 		return
 	}
 

cmd/container.go

@@ -1,40 +1,74 @@
-// Copyright 2023 Mikhail Klementev. All rights reserved.
+// Copyright 2024 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
 package cmd
 
 import (
+	"errors"
 	"fmt"
 	"os/exec"
 	"path/filepath"
-	"strings"
 
 	"github.com/rs/zerolog/log"
 
 	"code.dumpstack.io/tools/out-of-tree/container"
+	"code.dumpstack.io/tools/out-of-tree/distro"
 )
 
 type ContainerCmd struct {
-	Filter string `help:"filter by name"`
+	DistroID      string `help:"filter by distribution"`
+	DistroRelease string `help:"filter by distribution release"`
 
 	List    ContainerListCmd    `cmd:"" help:"list containers"`
 	Update  ContainerUpdateCmd  `cmd:"" help:"update containers"`
 	Save    ContainerSaveCmd    `cmd:"" help:"save containers"`
 	Cleanup ContainerCleanupCmd `cmd:"" help:"cleanup containers"`
+
+	RealtimeOutput RealtimeContainerOutputFlag `help:"show realtime output"`
 }
 
-func (cmd ContainerCmd) Containers() (names []string) {
+type RealtimeContainerOutputFlag bool
+
+func (f RealtimeContainerOutputFlag) AfterApply() (err error) {
+	container.Stdout = bool(f)
+	return
+}
+
+func (cmd ContainerCmd) Containers() (diis []container.Image, err error) {
 	images, err := container.Images()
 	if err != nil {
-		log.Fatal().Err(err).Msg("")
+		return
+	}
+
+	var dt distro.Distro
+	if cmd.DistroID != "" {
+		dt.ID, err = distro.NewID(cmd.DistroID)
+		if err != nil {
+			return
+		}
+
+		if cmd.DistroRelease != "" {
+			dt.Release = cmd.DistroRelease
+		}
+	} else if cmd.DistroRelease != "" {
+		err = errors.New("--distro-release has no use on its own")
+		return
 	}
 
 	for _, img := range images {
-		if cmd.Filter != "" && !strings.Contains(img.Name, cmd.Filter) {
+		if dt.ID != distro.None && dt.ID != img.Distro.ID {
+			log.Debug().Msgf("skip %s", img.Name)
 			continue
 		}
-		names = append(names, img.Name)
+
+		if dt.Release != "" && dt.Release != img.Distro.Release {
+			log.Debug().Msgf("skip %s", img.Name)
+			continue
+		}
+
+		log.Debug().Msgf("append %s", img.Name)
+		diis = append(diis, img)
 	}
 	return
 }
@@ -42,8 +76,13 @@ func (cmd ContainerCmd) Containers() (names []string) {
 type ContainerListCmd struct{}
 
 func (cmd ContainerListCmd) Run(containerCmd *ContainerCmd) (err error) {
-	for _, name := range containerCmd.Containers() {
-		fmt.Println(name)
+	images, err := containerCmd.Containers()
+	if err != nil {
+		return
+	}
+
+	for _, img := range images {
+		fmt.Printf("%s\n", img.Distro.String())
 	}
 	return
 }
@@ -51,7 +90,7 @@ func (cmd ContainerListCmd) Run(containerCmd *ContainerCmd) (err error) {
 type ContainerUpdateCmd struct{}
 
 func (cmd ContainerUpdateCmd) Run(g *Globals, containerCmd *ContainerCmd) (err error) {
-	images, err := container.Images()
+	images, err := containerCmd.Containers()
 	if err != nil {
 		return
 	}
@@ -65,13 +104,6 @@ func (cmd ContainerUpdateCmd) Run(g *Globals, containerCmd *ContainerCmd) (err e
 	container.Timeout = g.Config.Docker.Timeout.Duration
 
 	for _, img := range images {
-		if containerCmd.Filter != "" {
-			if !strings.Contains(img.Name, containerCmd.Filter) {
-				log.Debug().Msgf("skip %s", img.Name)
-				continue
-			}
-		}
-
 		_, err = img.Distro.Packages()
 		if err != nil {
 			return
@@ -86,13 +118,18 @@ type ContainerSaveCmd struct {
 }
 
 func (cmd ContainerSaveCmd) Run(containerCmd *ContainerCmd) (err error) {
-	for _, name := range containerCmd.Containers() {
-		nlog := log.With().Str("name", name).Logger()
+	images, err := containerCmd.Containers()
+	if err != nil {
+		return
+	}
 
-		output := filepath.Join(cmd.OutDir, name+".tar")
+	for _, img := range images {
+		nlog := log.With().Str("name", img.Name).Logger()
+
+		output := filepath.Join(cmd.OutDir, img.Name+".tar")
 		nlog.Info().Msgf("saving to %v", output)
 
-		err = container.Save(name, output)
+		err = container.Save(img.Name, output)
 		if err != nil {
 			return
 		}
@@ -115,9 +152,14 @@ func (cmd ContainerSaveCmd) Run(containerCmd *ContainerCmd) (err error) {
 type ContainerCleanupCmd struct{}
 
 func (cmd ContainerCleanupCmd) Run(containerCmd *ContainerCmd) (err error) {
+	images, err := containerCmd.Containers()
+	if err != nil {
+		return
+	}
+
 	var output []byte
-	for _, name := range containerCmd.Containers() {
-		output, err = exec.Command(container.Runtime, "image", "rm", name).
+	for _, img := range images {
+		output, err = exec.Command(container.Runtime, "image", "rm", img.Name).
 			CombinedOutput()
 		if err != nil {
 			log.Error().Err(err).Str("output", string(output)).Msg("")

cmd/debug.go

@@ -193,7 +193,8 @@ func (cmd *DebugCmd) Run(g *Globals) (err error) {
 			return
 		}
 	} else {
-		buildDir, outFile, output, err = artifact.Build(log.Logger, tmp, ka, ki, g.Config.Docker.Timeout.Duration)
+		buildDir, outFile, output, err = artifact.Build(
+			log.Logger, tmp, ka, ki, g.Config.Docker.Timeout.Duration, false)
 		if err != nil {
 			log.Error().Err(err).Msg(output)
 			return

cmd/kernel.go

@@ -39,6 +39,8 @@ type KernelCmd struct {
 
 	ContainerTimeout time.Duration `help:"container timeout"`
 
+	RealtimeOutput RealtimeContainerOutputFlag `help:"show realtime output"`
+
 	List        KernelListCmd        `cmd:"" help:"list kernels"`
 	ListRemote  KernelListRemoteCmd  `cmd:"" help:"list remote kernels"`
 	Autogen     KernelAutogenCmd     `cmd:"" help:"generate kernels based on the current config"`
@@ -265,8 +267,8 @@ func (cmd *KernelListCmd) Run(g *Globals) (err error) {
 }
 
 type KernelListRemoteCmd struct {
-	Distro string `required:"" help:"distribution"`
-	Ver    string `help:"distro version"`
+	DistroID      string `required:"" help:"distribution"`
+	DistroRelease string `help:"distro version"`
 }
 
 func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
@@ -279,13 +281,13 @@ func (cmd *KernelListRemoteCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error
 
 	container.UsePrebuilt = kernelCmd.PrebuiltContainers
 
-	distroType, err := distro.NewID(cmd.Distro)
+	distroType, err := distro.NewID(cmd.DistroID)
 	if err != nil {
 		return
 	}
 
 	km := artifact.Target{
-		Distro: distro.Distro{ID: distroType, Release: cmd.Ver},
+		Distro: distro.Distro{ID: distroType, Release: cmd.DistroRelease},
 		Kernel: artifact.Kernel{Regex: ".*"},
 	}
 
@@ -336,12 +338,12 @@ func (cmd *KernelAutogenCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
 }
 
 type KernelGenallCmd struct {
-	Distro string `help:"distribution"`
-	Ver    string `help:"distro version"`
+	DistroID      string `help:"distribution"`
+	DistroRelease string `help:"distro version"`
 }
 
 func (cmd *KernelGenallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
-	distroType, err := distro.NewID(cmd.Distro)
+	distroType, err := distro.NewID(cmd.DistroID)
 	if err != nil {
 		return
 	}
@@ -357,7 +359,7 @@ func (cmd *KernelGenallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
 			continue
 		}
 
-		if cmd.Ver != "" && dist.Release != cmd.Ver {
+		if cmd.DistroRelease != "" && dist.Release != cmd.DistroRelease {
 			continue
 		}
 
@@ -376,13 +378,13 @@ func (cmd *KernelGenallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
 }
 
 type KernelInstallCmd struct {
-	Distro string `required:"" help:"distribution"`
-	Ver    string `required:"" help:"distro version"`
-	Kernel string `required:"" help:"kernel release mask"`
+	DistroID      string `required:"" help:"distribution"`
+	DistroRelease string `required:"" help:"distro version"`
+	KernelRegex   string `required:"" help:"kernel release mask"`
 }
 
 func (cmd *KernelInstallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
-	distroType, err := distro.NewID(cmd.Distro)
+	distroType, err := distro.NewID(cmd.DistroID)
 	if err != nil {
 		return
 	}
@@ -390,8 +392,8 @@ func (cmd *KernelInstallCmd) Run(kernelCmd *KernelCmd, g *Globals) (err error) {
 	kernel.SetSigintHandler(&kernelCmd.shutdown)
 
 	km := artifact.Target{
-		Distro: distro.Distro{ID: distroType, Release: cmd.Ver},
-		Kernel: artifact.Kernel{Regex: cmd.Kernel},
+		Distro: distro.Distro{ID: distroType, Release: cmd.DistroRelease},
+		Kernel: artifact.Kernel{Regex: cmd.KernelRegex},
 	}
 	err = kernelCmd.Generate(g, km)
 	if err != nil {

cmd/log.go

@@ -1,4 +1,4 @@
-// Copyright 2023 Mikhail Klementev. All rights reserved.
+// Copyright 2024 Mikhail Klementev. All rights reserved.
 // Use of this source code is governed by a AGPLv3 license
 // (or later) that can be found in the LICENSE file.
 
@@ -212,7 +212,12 @@ func center(s string, w int) string {
 }
 
 func genOkFailCentered(name string, ok bool) (aurv aurora.Value) {
-	name = center(name, 10)
+	if ok {
+		name += " OK"
+	} else {
+		name += " FAIL"
+	}
+	name = center(name, 14)
 	if ok {
 		aurv = aurora.BgGreen(aurora.Black(name))
 	} else {
@@ -225,7 +230,7 @@ func logLogEntry(l logEntry) {
 	distroInfo := fmt.Sprintf("%s-%s {%s}", l.Distro.ID,
 		l.Distro.Release, l.KernelRelease)
 
-	artifactInfo := fmt.Sprintf("{[%s] %s}", l.Type, l.Name)
+	artifactInfo := fmt.Sprintf("%s", l.Name)
 
 	timestamp := l.Timestamp.Format("2006-01-02 15:04")
 
@@ -257,7 +262,10 @@ func logLogEntry(l logEntry) {
 		additional = "(timeout)"
 	}
 
-	colored := aurora.Sprintf("[%4d %4s] [%s] %s    %-70s: %s %s",
+	if len(distroInfo) > 40 {
+		distroInfo = distroInfo[:40]
+	}
+	colored := aurora.Sprintf("[%4d %4s] [%s] %s    %-40s: %s %s",
 		l.ID, l.Tag, timestamp, artifactInfo, distroInfo, status,
 		additional)
 

cmd/pew.go

@@ -12,6 +12,7 @@ import (
 	"math/rand"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -86,6 +87,12 @@ type PewCmd struct {
 
 	Threshold             float64 `help:"reliablity threshold for exit code" default:"1.00"`
 	IncludeInternalErrors bool    `help:"count internal errors as part of the success rate"`
+	InternalErrorsRetries int     `help:"amount of retries on internal errors" default:"3"`
+
+	OutputOnSuccess bool `help:"show output on success"`
+	RealtimeOutput  bool `help:"show realtime output"`
+
+	LogDir string `help:"write logs to directory"`
 
 	Endless        bool          `help:"endless tests"`
 	EndlessTimeout time.Duration `help:"timeout between tests" default:"1m"`
@@ -161,6 +168,11 @@ func (cmd *PewCmd) Run(g *Globals) (err error) {
 	cmd.useRemote = g.Remote
 	cmd.remoteAddr = g.RemoteAddr
 
+	if cmd.RealtimeOutput && cmd.Threads != 1 {
+		log.Warn().Msg("realtime output disables multithreading")
+		cmd.Threads = 1
+	}
+
 	if cmd.useRemote {
 		c := client.Client{RemoteAddr: cmd.remoteAddr}
 		cmd.Kcfg.Kernels, err = c.Kernels()
@@ -400,39 +412,54 @@ func (cmd PewCmd) testArtifact(swg *sizedwaitgroup.SizedWaitGroup,
 
 	defer swg.Done()
 
-	logdir := "logs/" + cmd.Tag
-	err := os.MkdirAll(logdir, os.ModePerm)
-	if err != nil {
-		log.Error().Err(err).Msgf("mkdir %s", logdir)
-		return
-	}
+	var logDirWriter *zerolog.ConsoleWriter
+	if cmd.LogDir != "" {
+		logdir := filepath.Join(cmd.LogDir, cmd.Tag)
+		err := os.MkdirAll(logdir, os.ModePerm)
+		if err != nil {
+			log.Error().Err(err).Msgf("mkdir %s", logdir)
+			return
+		}
 
-	logfile := fmt.Sprintf("logs/%s/%s-%s-%s.log",
-		cmd.Tag,
-		ki.Distro.ID.String(),
-		ki.Distro.Release,
-		ki.KernelRelease,
-	)
-	f, err := os.Create(logfile)
-	if err != nil {
-		log.Error().Err(err).Msgf("create %s", logfile)
-		return
-	}
-	defer f.Close()
+		logfile := fmt.Sprintf("logs/%s/%s-%s-%s.log",
+			cmd.Tag,
+			ki.Distro.ID.String(),
+			ki.Distro.Release,
+			ki.KernelRelease,
+		)
+		f, err := os.Create(logfile)
+		if err != nil {
+			log.Error().Err(err).Msgf("create %s", logfile)
+			return
+		}
+		defer f.Close()
 
-	slog := zerolog.New(zerolog.MultiLevelWriter(
-		&ConsoleWriter,
-		&FileWriter,
-		&zerolog.ConsoleWriter{
+		logDirWriter = &zerolog.ConsoleWriter{
 			Out: f,
 			FieldsExclude: []string{
 				"distro_release",
 				"distro_type",
 				"kernel",
+				"command",
+				"workdir",
 			},
 			NoColor: true,
-		},
-	))
+		}
+	}
+
+	var slog zerolog.Logger
+	if logDirWriter != nil {
+		slog = zerolog.New(zerolog.MultiLevelWriter(
+			&ConsoleWriter,
+			&FileWriter,
+			logDirWriter,
+		))
+	} else {
+		slog = zerolog.New(zerolog.MultiLevelWriter(
+			&ConsoleWriter,
+			&FileWriter,
+		))
+	}
 
 	switch LogLevel {
 	case zerolog.TraceLevel, zerolog.DebugLevel:
@@ -445,12 +472,33 @@ func (cmd PewCmd) testArtifact(swg *sizedwaitgroup.SizedWaitGroup,
 		Str("kernel", ki.KernelRelease).
 		Logger()
 
-	ka.Process(slog, ki,
-		cmd.Endless, cmd.Binary, cmd.EndlessStress, cmd.EndlessTimeout,
-		func(q *qemu.System, ka artifact.Artifact, ki distro.KernelInfo, result *artifact.Result) {
-			dumpResult(q, ka, ki, result, cmd.Dist, cmd.Tag, cmd.Binary, cmd.DB)
-		},
-	)
+	retriesLeft := cmd.InternalErrorsRetries
+	var stop bool
+	for !stop {
+		ka.Process(slog, ki, cmd.OutputOnSuccess, cmd.RealtimeOutput,
+			cmd.Endless, cmd.Binary, cmd.EndlessStress, cmd.EndlessTimeout,
+			func(q *qemu.System, ka artifact.Artifact, ki distro.KernelInfo, res *artifact.Result) {
+				if res.InternalError == nil {
+					cmd.dumpResult(q, ka, ki, res)
+					stop = true
+					return
+				}
+
+				q.Log.Warn().Err(res.InternalError).
+					Str("panic", fmt.Sprintf("%v", q.KernelPanic)).
+					Str("timeout", fmt.Sprintf("%v", q.KilledByTimeout)).
+					Int("retries_left", retriesLeft).
+					Msg("internal")
+
+				if retriesLeft == 0 {
+					state.InternalErrors += 1
+					stop = true
+					return
+				}
+				retriesLeft -= 1
+			},
+		)
+	}
 }
 
 func shuffleKernels(a []distro.KernelInfo) []distro.KernelInfo {
@@ -546,76 +594,70 @@ func genOkFail(name string, ok bool) (aurv aurora.Value) {
 	return
 }
 
-func dumpResult(q *qemu.System, ka artifact.Artifact, ki distro.KernelInfo,
-	res *artifact.Result, dist, tag, binary string, db *sql.DB) {
+func (cmd PewCmd) dumpResult(q *qemu.System, ka artifact.Artifact, ki distro.KernelInfo, res *artifact.Result) {
+	state.Overall += 1
 
-	// TODO refactor
+	if res.Test.Ok {
+		state.Success += 1
+	}
 
-	if res.InternalError != nil {
-		q.Log.Warn().Err(res.InternalError).
-			Str("panic", fmt.Sprintf("%v", q.KernelPanic)).
-			Str("timeout", fmt.Sprintf("%v", q.KilledByTimeout)).
-			Msg("internal")
-		res.InternalErrorString = res.InternalError.Error()
-		state.InternalErrors += 1
+	colored := ""
+	switch ka.Type {
+	case artifact.KernelExploit:
+		colored = aurora.Sprintf("%s %s",
+			genOkFail("BUILD", res.Build.Ok),
+			genOkFail("LPE", res.Test.Ok))
+	case artifact.KernelModule:
+		colored = aurora.Sprintf("%s %s %s",
+			genOkFail("BUILD", res.Build.Ok),
+			genOkFail("INSMOD", res.Run.Ok),
+			genOkFail("TEST", res.Test.Ok))
+	case artifact.Script:
+		colored = aurora.Sprintf("%s",
+			genOkFail("", res.Test.Ok))
+	}
+
+	additional := ""
+	if q.KernelPanic {
+		additional = "(panic)"
+	} else if q.KilledByTimeout {
+		additional = "(timeout)"
+	}
+
+	if additional != "" {
+		q.Log.Info().Msgf("%v %v", colored, additional)
 	} else {
-		colored := ""
-
-		state.Overall += 1
-
-		if res.Test.Ok {
-			state.Success += 1
-		}
-
-		switch ka.Type {
-		case artifact.KernelExploit:
-			colored = aurora.Sprintf("%s %s",
-				genOkFail("BUILD", res.Build.Ok),
-				genOkFail("LPE", res.Test.Ok))
-		case artifact.KernelModule:
-			colored = aurora.Sprintf("%s %s %s",
-				genOkFail("BUILD", res.Build.Ok),
-				genOkFail("INSMOD", res.Run.Ok),
-				genOkFail("TEST", res.Test.Ok))
-		case artifact.Script:
-			colored = aurora.Sprintf("%s",
-				genOkFail("", res.Test.Ok))
-		}
-
-		additional := ""
-		if q.KernelPanic {
-			additional = "(panic)"
-		} else if q.KilledByTimeout {
-			additional = "(timeout)"
-		}
-
-		if additional != "" {
-			q.Log.Info().Msgf("%v %v", colored, additional)
-		} else {
-			q.Log.Info().Msgf("%v", colored)
-		}
+		q.Log.Info().Msgf("%v", colored)
 	}
 
-	err := addToLog(db, q, ka, ki, res, tag)
+	err := addToLog(cmd.DB, q, ka, ki, res, cmd.Tag)
 	if err != nil {
-		q.Log.Warn().Err(err).Msgf("[db] addToLog (%v)", ka)
+		q.Log.Error().Err(err).Msgf("[db] addToLog (%v)", ka)
 	}
 
-	if binary == "" && dist != pathDevNull {
-		err = os.MkdirAll(dist, os.ModePerm)
-		if err != nil {
-			log.Warn().Err(err).Msgf("os.MkdirAll (%v)", ka)
-		}
+	if cmd.Binary != "" {
+		return
+	}
 
-		path := fmt.Sprintf("%s/%s-%s-%s", dist, ki.Distro.ID,
-			ki.Distro.Release, ki.KernelRelease)
-		if ka.Type != artifact.KernelExploit {
-			path += ".ko"
-		}
+	if cmd.Dist == pathDevNull { // why?
+		return
+	}
 
-		err = artifact.CopyFile(res.BuildArtifact, path)
-		if err != nil {
-			log.Warn().Err(err).Msgf("copy file (%v)", ka)
-		}
+	err = os.MkdirAll(cmd.Dist, os.ModePerm)
+	if err != nil {
+		log.Error().Err(err).Msgf("os.MkdirAll (%v)", ka)
+		return
+	}
+
+	path := fmt.Sprintf("%s/%s-%s-%s", cmd.Dist, ki.Distro.ID,
+		ki.Distro.Release, ki.KernelRelease)
+	if ka.Type != artifact.KernelExploit {
+		path += ".ko"
+	}
+
+	err = artifact.CopyFile(res.BuildArtifact, path)
+	if err != nil {
+		log.Error().Err(err).Msgf("copy file (%v)", ka)
+		return
 	}
 }

container/container.go

@@ -14,6 +14,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/cavaliergopher/grab/v3"
@@ -45,6 +46,8 @@ var UsePrebuilt = true
 
 var Prune = true
 
+var Stdout = false
+
 type Image struct {
 	Name   string
 	Distro distro.Distro
@@ -175,7 +178,15 @@ type Container struct {
 	// Additional arguments
 	Args []string
 
+	// Base of container is local-only
+	LocalBase bool
+
 	Log zerolog.Logger
+
+	commandsOutput struct {
+		listener chan string
+		mu       sync.Mutex
+	}
 }
 
 func New(dist distro.Distro) (c Container, err error) {
@@ -234,6 +245,43 @@ func NewFromKernelInfo(ki distro.KernelInfo) (
 	return
 }
 
+// c.SetCommandsOutputHandler(func(s string) { fmt.Println(s) })
+// defer c.CloseCommandsOutputHandler()
+func (c *Container) SetCommandsOutputHandler(handler func(s string)) {
+	c.commandsOutput.mu.Lock()
+	defer c.commandsOutput.mu.Unlock()
+
+	c.commandsOutput.listener = make(chan string)
+
+	go func(l chan string) {
+		for m := range l {
+			if m != "" {
+				handler(m)
+			}
+		}
+	}(c.commandsOutput.listener)
+}
+
+func (c *Container) CloseCommandsOutputHandler() {
+	c.commandsOutput.mu.Lock()
+	defer c.commandsOutput.mu.Unlock()
+
+	close(c.commandsOutput.listener)
+	c.commandsOutput.listener = nil
+}
+
+func (c *Container) handleCommandsOutput(m string) {
+	if c.commandsOutput.listener == nil {
+		return
+	}
+	c.commandsOutput.mu.Lock()
+	defer c.commandsOutput.mu.Unlock()
+
+	if c.commandsOutput.listener != nil {
+		c.commandsOutput.listener <- m
+	}
+}
+
 func (c Container) Name() string {
 	return c.name
 }
@@ -383,7 +431,10 @@ func (c Container) build(imagePath string) (output string, err error) {
 
 	args := []string{"build"}
 	if !UseCache {
-		args = append(args, "--pull", "--no-cache")
+		if !c.LocalBase {
+			args = append(args, "--pull")
+		}
+		args = append(args, "--no-cache")
 	}
 	args = append(args, "-t", c.name, imagePath)
 
@@ -408,6 +459,10 @@ func (c Container) build(imagePath string) (output string, err error) {
 		scanner := bufio.NewScanner(stdout)
 		for scanner.Scan() {
 			m := scanner.Text()
+			if Stdout {
+				fmt.Println(m)
+			}
+			c.handleCommandsOutput(m)
 			output += m + "\n"
 			flog.Trace().Str("stdout", m).Msg("")
 		}
@@ -417,7 +472,7 @@ func (c Container) build(imagePath string) (output string, err error) {
 	return
 }
 
-func (c Container) Run(workdir string, cmds []string) (out string, err error) {
+func (c *Container) Run(workdir string, cmds []string) (out string, err error) {
 	flog := c.Log.With().
 		Str("workdir", workdir).
 		Str("command", fmt.Sprintf("%v", cmds)).
@@ -481,16 +536,17 @@ func (c Container) Run(workdir string, cmds []string) (out string, err error) {
 		scanner := bufio.NewScanner(stdout)
 		for scanner.Scan() {
 			m := scanner.Text()
+			if Stdout {
+				fmt.Println(m)
+			}
+			c.handleCommandsOutput(m)
 			out += m + "\n"
-			flog.Trace().Str("stdout", m).Msg("")
+			flog.Trace().Str("container stdout", m).Msg("")
 		}
 	}()
 
 	err = cmd.Wait()
 	if err != nil {
-		e := fmt.Sprintf("error `%v` for cmd `%v` with output `%v`",
-			err, cmds, out)
-		err = errors.New(e)
 		return
 	}
 

daemon/process.go

@@ -131,7 +131,7 @@ func (pj *jobProcessor) Process(res *Resources) (err error) {
 	var result *artifact.Result
 	var dq *qemu.System
 
-	pj.job.Artifact.Process(pj.log, pj.job.Target, false, "", "", 0,
+	pj.job.Artifact.Process(pj.log, pj.job.Target, false, false, false, "", "", 0,
 		func(q *qemu.System, ka artifact.Artifact, ki distro.KernelInfo,
 			res *artifact.Result) {
 

distro/opensuse/opensuse.go

@@ -57,6 +57,8 @@ func (suse OpenSUSE) Packages() (pkgs []string, err error) {
 		if err != nil {
 			return
 		}
+		c.LocalBase = true
+
 	} else if strings.HasPrefix(suse.release, "13") {
 		name = "opensuse:13"
 		cnturl := cache.ContainerURL("openSUSE-13.2")
@@ -64,6 +66,8 @@ func (suse OpenSUSE) Packages() (pkgs []string, err error) {
 		if err != nil {
 			return
 		}
+		c.LocalBase = true
+
 	} else if strings.HasPrefix(suse.release, "42") {
 		name = "opensuse/leap:42"
 	} else if strings.HasPrefix(suse.release, "15") {
@@ -83,7 +87,32 @@ func (suse OpenSUSE) Packages() (pkgs []string, err error) {
 		return
 	}
 
-	pkgs = append(pkgs, strings.Fields(output)...)
+	// TODO Find a way for non-interactive installation of
+	// retracted kernels
+	retracted := []string{
+		"5.14.21-150400.24.49.3",
+		"5.14.21-150400.24.84.1",
+		"5.14.21-150500.55.22.1",
+		"5.3.18-150300.59.81.1",
+		"5.3.18-59.30.1",
+		"5.3.18-lp152.98.1",
+	}
+
+	for _, k := range strings.Fields(output) {
+		skip := false
+		for _, rk := range retracted {
+			if rk == k {
+				skip = true
+				break
+			}
+		}
+		if skip {
+			continue
+		}
+
+		pkgs = append(pkgs, k)
+	}
+
 	return
 }
 

qemu/qemu-kernel.go

@@ -101,6 +101,16 @@ type System struct {
 
 	Stdout, Stderr string
 
+	qemuOutput struct {
+		listener chan string
+		mu       sync.Mutex
+	}
+
+	commandsOutput struct {
+		listener chan string
+		mu       sync.Mutex
+	}
+
 	// accessible after qemu is closed
 	exitErr error
 
@@ -138,6 +148,80 @@ func NewSystem(arch arch, kernel Kernel, drivePath string) (q *System, err error
 	return
 }
 
+// q.SetQemuOutputHandler(func(s string) { fmt.Println(s) })
+// defer q.CloseQemuOutputHandler()
+func (q *System) SetQemuOutputHandler(handler func(s string)) {
+	q.qemuOutput.mu.Lock()
+	defer q.qemuOutput.mu.Unlock()
+
+	q.qemuOutput.listener = make(chan string)
+
+	go func(l chan string) {
+		for m := range l {
+			if m != "" {
+				handler(m)
+			}
+		}
+	}(q.qemuOutput.listener)
+}
+
+func (q *System) CloseQemuOutputHandler() {
+	q.qemuOutput.mu.Lock()
+	defer q.qemuOutput.mu.Unlock()
+
+	close(q.qemuOutput.listener)
+	q.qemuOutput.listener = nil
+}
+
+func (q *System) handleQemuOutput(m string) {
+	if q.qemuOutput.listener == nil {
+		return
+	}
+	q.qemuOutput.mu.Lock()
+	defer q.qemuOutput.mu.Unlock()
+
+	if q.qemuOutput.listener != nil {
+		q.qemuOutput.listener <- m
+	}
+}
+
+// q.SetCommandsOutputHandler(func(s string) { fmt.Println(s) })
+// defer q.CloseCommandsOutputHandler()
+func (q *System) SetCommandsOutputHandler(handler func(s string)) {
+	q.commandsOutput.mu.Lock()
+	defer q.commandsOutput.mu.Unlock()
+
+	q.commandsOutput.listener = make(chan string)
+
+	go func(l chan string) {
+		for m := range l {
+			if m != "" {
+				handler(m)
+			}
+		}
+	}(q.commandsOutput.listener)
+}
+
+func (q *System) CloseCommandsOutputHandler() {
+	q.commandsOutput.mu.Lock()
+	defer q.commandsOutput.mu.Unlock()
+
+	close(q.commandsOutput.listener)
+	q.commandsOutput.listener = nil
+}
+
+func (q *System) handleCommandsOutput(m string) {
+	if q.commandsOutput.listener == nil {
+		return
+	}
+	q.commandsOutput.mu.Lock()
+	defer q.commandsOutput.mu.Unlock()
+
+	if q.commandsOutput.listener != nil {
+		q.commandsOutput.listener <- m
+	}
+}
+
 func (q *System) SetSSHAddrPort(addr string, port int) (err error) {
 	// TODO validate
 	q.SSH.AddrPort = fmt.Sprintf("%s:%d", addr, port)
@@ -312,7 +396,8 @@ func (q *System) Start() (err error) {
 	go func() {
 		scanner := bufio.NewScanner(q.pipe.stdout)
 		for scanner.Scan() {
-			m := scanner.Text()
+			m := strings.TrimSpace(scanner.Text())
+			q.handleQemuOutput(m)
 			q.Stdout += m + "\n"
 			q.Log.Trace().Str("stdout", m).Msg("qemu")
 			go q.checkOopsPanic(m)
@@ -322,7 +407,8 @@ func (q *System) Start() (err error) {
 	go func() {
 		scanner := bufio.NewScanner(q.pipe.stderr)
 		for scanner.Scan() {
-			m := scanner.Text()
+			m := strings.TrimSpace(scanner.Text())
+			q.handleQemuOutput(m)
 			q.Stderr += m + "\n"
 			q.Log.Trace().Str("stderr", m).Msg("qemu")
 		}
@@ -474,7 +560,8 @@ func (q System) Command(user, cmd string) (output string, err error) {
 
 		scanner := bufio.NewScanner(stdout)
 		for scanner.Scan() {
-			m := scanner.Text()
+			m := strings.TrimSpace(scanner.Text())
+			q.handleCommandsOutput(m)
 			output += m + "\n"
 			flog.Trace().Str("stdout", m).Msg("qemu command")
 		}
@@ -487,7 +574,8 @@ func (q System) Command(user, cmd string) (output string, err error) {
 
 		scanner := bufio.NewScanner(stderr)
 		for scanner.Scan() {
-			m := scanner.Text()
+			m := strings.TrimSpace(scanner.Text())
+			q.handleCommandsOutput(m)
 			output += m + "\n"
 			// Note: it prints stderr as stdout
 			flog.Trace().Str("stdout", m).Msg("qemu command")

tools/qemu-centos-img/6/Dockerfile

@@ -28,6 +28,8 @@ RUN sed -i '/PermitEmptyPasswords/d' $TMPDIR/etc/ssh/sshd_config
 RUN echo PermitEmptyPasswords yes >> $TMPDIR/etc/ssh/sshd_config
 RUN sed -i '/PermitRootLogin/d' $TMPDIR/etc/ssh/sshd_config
 RUN echo PermitRootLogin yes >> $TMPDIR/etc/ssh/sshd_config
+RUN sed -i '/UseDNS/d' $TMPDIR/etc/ssh/sshd_config
+RUN echo UseDNS no >> $TMPDIR/etc/ssh/sshd_config
 
 # network workaround
 RUN chmod +x $TMPDIR/etc/rc.local

tools/qemu-centos-img/6/generate.sh

@@ -4,6 +4,6 @@ set -eux
 
 cd "$(dirname "$0")"
 
-sudo docker build -t gen-centos6-image .
-sudo docker run --privileged -v $(pwd):/shared -t gen-centos6-image
+sudo podman build -t gen-centos6-image .
+sudo podman run --privileged -v $(pwd):/shared -t gen-centos6-image
 tar -Szcf out_of_tree_centos_6.img.tar.gz out_of_tree_centos_6.img

tools/qemu-centos-img/7/Dockerfile

@@ -13,6 +13,11 @@
 #
 FROM centos:7
 
+RUN sed -i 's/enabled=1/enabled=0/' /etc/yum.repos.d/* || true
+RUN sed -i 's/name/enabled=0\nname/' /etc/yum.repos.d/* || true
+RUN echo -e '[7.9.2009-os]\nbaseurl=https://vault.centos.org/7.9.2009/os/$basearch/\ngpgcheck=0' >> /etc/yum.repos.d/oot.repo
+RUN echo -e '[7.9.2009-updates]\nbaseurl=https://vault.centos.org/7.9.2009/updates/$basearch/\ngpgcheck=0' >> /etc/yum.repos.d/oot.repo
+
 RUN yum -y update
 RUN yum -y groupinstall "Development Tools"
 RUN yum -y install qemu-img e2fsprogs
@@ -21,13 +26,13 @@ ENV TMPDIR=/tmp/centos
 
 RUN yum --installroot=$TMPDIR \
         --releasever=7 \
-        --disablerepo='*' \
-        --enablerepo=base \
         -y groupinstall Base
+
+RUN rm $TMPDIR/etc/yum.repos.d/*
+RUN cp /etc/yum.repos.d/* $TMPDIR/etc/yum.repos.d/
+
 RUN yum --installroot=$TMPDIR \
         --releasever=7 \
-        --disablerepo='*' \
-        --enablerepo=base \
         -y install openssh-server openssh-clients
 
 RUN chroot $TMPDIR /bin/sh -c 'useradd -m user'
@@ -37,6 +42,8 @@ RUN sed -i '/PermitEmptyPasswords/d' $TMPDIR/etc/ssh/sshd_config
 RUN echo PermitEmptyPasswords yes >> $TMPDIR/etc/ssh/sshd_config
 RUN sed -i '/PermitRootLogin/d' $TMPDIR/etc/ssh/sshd_config
 RUN echo PermitRootLogin yes >> $TMPDIR/etc/ssh/sshd_config
+RUN sed -i '/UseDNS/d' $TMPDIR/etc/ssh/sshd_config
+RUN echo UseDNS no >> $TMPDIR/etc/ssh/sshd_config
 
 # network workaround
 RUN chmod +x $TMPDIR/etc/rc.local

tools/qemu-centos-img/7/generate.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -eux
+
+cd "$(dirname "$0")"
+
+sudo podman build -t gen-centos7-image .
+sudo podman run --privileged -v $(pwd):/shared -t gen-centos7-image
+tar -Szcf out_of_tree_centos_7.img.tar.gz out_of_tree_centos_7.img

tools/qemu-centos-img/8/Dockerfile

@@ -28,6 +28,8 @@ RUN sed -i '/PermitEmptyPasswords/d' $TMPDIR/etc/ssh/sshd_config
 RUN echo PermitEmptyPasswords yes >> $TMPDIR/etc/ssh/sshd_config
 RUN sed -i '/PermitRootLogin/d' $TMPDIR/etc/ssh/sshd_config
 RUN echo PermitRootLogin yes >> $TMPDIR/etc/ssh/sshd_config
+RUN sed -i '/UseDNS/d' $TMPDIR/etc/ssh/sshd_config
+RUN echo UseDNS no >> $TMPDIR/etc/ssh/sshd_config
 
 # network workaround
 RUN chmod +x $TMPDIR/etc/rc.local

tools/qemu-centos-img/8/generate.sh

@@ -4,6 +4,6 @@ set -eux
 
 cd "$(dirname "$0")"
 
-sudo docker build -t gen-centos8-image .
-sudo docker run --privileged -v $(pwd):/shared -t gen-centos8-image
+sudo podman build -t gen-centos8-image .
+sudo podman run --privileged -v $(pwd):/shared -t gen-centos8-image
 tar -Szcf out_of_tree_centos_8.img.tar.gz out_of_tree_centos_8.img

tools/qemu-debian-img/setup.sh

@@ -11,6 +11,8 @@ sed -i '/PermitEmptyPasswords/d' $TMPDIR/etc/ssh/sshd_config
 echo PermitEmptyPasswords yes >> $TMPDIR/etc/ssh/sshd_config
 sed -i '/PermitRootLogin/d' $TMPDIR/etc/ssh/sshd_config
 echo PermitRootLogin yes >> $TMPDIR/etc/ssh/sshd_config
+sed -i '/UseDNS/d' $TMPDIR/etc/ssh/sshd_config
+echo UseDNS no >> $TMPDIR/etc/ssh/sshd_config
 
 echo '#!/bin/sh' > $TMPDIR/etc/rc.local
 echo 'dhclient' >> $TMPDIR/etc/rc.local

tools/qemu-oraclelinux-img/Dockerfile.template

@@ -13,6 +13,7 @@ RUN yum --installroot=$TMPDIR \
         --releasever=_VERSION_ \
         --disablerepo='*' \
         --enablerepo=ol_VERSION__baseos_latest \
+        --enablerepo=ol_VERSION__appstream \
         -y groupinstall Base
 
 RUN cp /etc/yum.repos.d/* $TMPDIR/etc/yum.repos.d/
@@ -21,6 +22,7 @@ RUN yum --installroot=$TMPDIR \
         --releasever=_VERSION_ \
         --disablerepo='*' \
         --enablerepo=ol_VERSION__baseos_latest \
+        --enablerepo=ol_VERSION__appstream \
         -y install openssh-server openssh-clients dhclient yum
 
 RUN chroot $TMPDIR /bin/sh -c 'useradd -m user'
@@ -30,6 +32,8 @@ RUN sed -i '/PermitEmptyPasswords/d' $TMPDIR/etc/ssh/sshd_config
 RUN echo PermitEmptyPasswords yes >> $TMPDIR/etc/ssh/sshd_config
 RUN sed -i '/PermitRootLogin/d' $TMPDIR/etc/ssh/sshd_config
 RUN echo PermitRootLogin yes >> $TMPDIR/etc/ssh/sshd_config
+RUN sed -i '/UseDNS/d' $TMPDIR/etc/ssh/sshd_config
+RUN echo UseDNS no >> $TMPDIR/etc/ssh/sshd_config
 
 # network workaround
 RUN chmod +x $TMPDIR/etc/rc.local

tools/qemu-oraclelinux-img/generate-images.sh

@@ -11,9 +11,11 @@ for version in 6 7 8 9; do
 
     if [[ $version -eq 6 ]]; then
 	sed -i 's/baseos_latest/u10_base/' $version/Dockerfile
+	sed -i '/appstream/d' $version/Dockerfile
     fi
     if [[ $version -eq 7 ]]; then
 	sed -i 's/baseos_latest/u9_base/' $version/Dockerfile
+	sed -i '/appstream/d' $version/Dockerfile
     fi
 
     podman build -t gen-oraclelinux${version}-image $version

tools/qemu-ubuntu-img/14.04/Dockerfile

@@ -1,35 +0,0 @@
-# Copyright 2018 Mikhail Klementev. All rights reserved.
-# Use of this source code is governed by a AGPLv3 license
-# (or later) that can be found in the LICENSE file.
-#
-# Usage:
-#
-#     $ docker build -t gen-ubuntu1404-image .
-#     $ docker run --privileged -v $(pwd):/shared -t gen-ubuntu1404-image
-#
-# ubuntu1404.img will be created in current directory. You can change $(pwd) to
-# different directory to use different destination for image.
-#
-FROM ubuntu:14.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update
-RUN apt-get install -y debootstrap qemu
-
-ENV TMPDIR=/tmp/ubuntu
-ENV IMAGEDIR=/tmp/image
-ENV IMAGE=/shared/out_of_tree_ubuntu_14__04.img
-ENV REPOSITORY=http://archive.ubuntu.com/ubuntu
-ENV RELEASE=trusty
-
-RUN mkdir $IMAGEDIR
-
-# Must be executed with --privileged because of /dev/loop
-CMD debootstrap --include=openssh-server,policykit-1 \
-	$RELEASE $TMPDIR $REPOSITORY && \
-	/shared/setup.sh $TMPDIR && \
-	qemu-img create $IMAGE 2G && \
-	mkfs.ext4 -F $IMAGE && \
-	mount -o loop $IMAGE $IMAGEDIR && \
-	cp -a $TMPDIR/* $IMAGEDIR/ && \
-	umount $IMAGEDIR

tools/qemu-ubuntu-img/14.04/setup.sh

@@ -1,17 +0,0 @@
-#!/bin/sh -eux
-# Copyright 2018 Mikhail Klementev. All rights reserved.
-# Use of this source code is governed by a AGPLv3 license
-# (or later) that can be found in the LICENSE file.
-TMPDIR=$1
-chroot $TMPDIR /bin/sh -c 'useradd -m user'
-sed -i 's/root:\*:/root::/' $TMPDIR/etc/shadow
-sed -i 's/user:!!:/user::/' $TMPDIR/etc/shadow
-echo auth sufficient pam_permit.so > $TMPDIR/etc/pam.d/sshd
-sed -i '/PermitEmptyPasswords/d' $TMPDIR/etc/ssh/sshd_config
-echo PermitEmptyPasswords yes >> $TMPDIR/etc/ssh/sshd_config
-sed -i '/PermitRootLogin/d' $TMPDIR/etc/ssh/sshd_config
-echo PermitRootLogin yes >> $TMPDIR/etc/ssh/sshd_config
-
-echo '#!/bin/sh' > $TMPDIR/etc/rc.local
-echo 'dhclient eth0' >> $TMPDIR/etc/rc.local
-chmod +x $TMPDIR/etc/rc.local

tools/qemu-ubuntu-img/generate-images.py

@@ -7,27 +7,29 @@ script_dir = os.path.dirname(os.path.realpath(__file__))
 os.chdir(script_dir)
 
 releases = [
-    # ('12.04', 'precies'),
-    # ('14.04', 'trusty'),
-    # ('16.04', 'xenial'),
-    # ('18.04', 'bionic'),
-    # ('20.04', 'focal'),
-    # ('22.04', 'jammy'),
-    ('24.04', 'noble'),
+    ('12.04', 'precise', 'http://old-releases.ubuntu.com/ubuntu'),
+    ('14.04', 'trusty', 'http://archive.ubuntu.com/ubuntu'),
+    ('16.04', 'xenial', 'http://archive.ubuntu.com/ubuntu'),
+    ('18.04', 'bionic', 'http://archive.ubuntu.com/ubuntu'),
+    ('20.04', 'focal', 'http://archive.ubuntu.com/ubuntu'),
+    ('22.04', 'jammy', 'http://archive.ubuntu.com/ubuntu'),
+    ('24.04', 'noble', 'http://archive.ubuntu.com/ubuntu')
 ]
 
 template = '''
 FROM ubuntu:{version}
 
+RUN sed -i 's;http://archive.ubuntu.com/ubuntu;{repository};' /etc/apt/sources.list
+
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt update
-RUN apt install -y debootstrap qemu-utils
-RUN apt install -y linux-image-generic
+RUN apt-get update
+RUN apt-get install -y debootstrap qemu-utils
+RUN apt-get install -y linux-image-generic
 
 ENV TMPDIR=/tmp/ubuntu
 ENV IMAGEDIR=/tmp/image
 ENV IMAGE=/shared/out_of_tree_ubuntu_{img_version}.img
-ENV REPOSITORY=http://archive.ubuntu.com/ubuntu
+ENV REPOSITORY={repository}
 ENV RELEASE={codename}
 
 RUN mkdir $IMAGEDIR
@@ -47,7 +49,7 @@ def run_cmd(cmd):
     print(f"+ {cmd}")
     subprocess.run(cmd, shell=True, check=True, executable='/bin/bash')
 
-for version, codename in releases:
+for version, codename, repository in releases:
     numeric_version = version.replace('.', '')
     img_version=version.replace(".","__")
 
@@ -55,6 +57,7 @@ for version, codename in releases:
         version=version,
         img_version=img_version,
         codename=codename,
+        repository=repository,
         numeric_version=numeric_version)
 
     os.makedirs(str(version), exist_ok=True)

tools/qemu-ubuntu-img/setup.sh

@@ -11,6 +11,8 @@ sed -i '/PermitEmptyPasswords/d' $TMPDIR/etc/ssh/sshd_config
 echo PermitEmptyPasswords yes >> $TMPDIR/etc/ssh/sshd_config
 sed -i '/PermitRootLogin/d' $TMPDIR/etc/ssh/sshd_config
 echo PermitRootLogin yes >> $TMPDIR/etc/ssh/sshd_config
+sed -i '/UseDNS/d' $TMPDIR/etc/ssh/sshd_config
+echo UseDNS no >> $TMPDIR/etc/ssh/sshd_config
 
 echo '#!/bin/sh' > $TMPDIR/etc/rc.local
 echo 'dhclient || dhcpcd' >> $TMPDIR/etc/rc.local