github.com/jollheef/go-qemu-kernel -> github.com/jollheef/out-of-tree/qemu

qemu/README.md

@@ -0,0 +1,76 @@
+# out-of-tree/qemu
+
+Qemu wrapper for kernel-related CI tasks. Supports *GNU/Linux* and *macOS*.
+
+Features:
+* Uses upstream virtualization -- KVM in GNU/Linux and Hypervisor.framework in macOS.
+* Run files and kernel modules directly from local filesystem. No need to copy byself!
+* Run commands inside qemu virtual machine at the same way as you run in it locally.
+
+## Installation
+
+    $ go get github.com/jollheef/out-of-tree/qemu
+
+### Generate root image
+
+First of all we need to generate rootfs for run qemu.
+
+#### GNU/Linux
+
+    $ sudo apt install -y debootstrap qemu
+    $ sudo qemu-debian-img generate sid.img
+
+#### macOS
+
+Note: qemu on macOS since v2.12 (24 April 2018) supports Hypervisor.framework.
+
+    $ brew install qemu
+
+Because it's a very complicated to debootstrap qemu images from macOS,
+preferred way is to use Vagrant with any hypervisor.
+
+    $ brew cask install vagrant
+    $ cd $GOPATH/src/github.com/jollheef/out-of-tree/qemu/tools/qemu-debian-image
+    $ vagrant up && vagrant destroy -f
+
+bionic.img and bionic-vmlinuz will be created in current directory.
+
+### Fill configuration file
+
+    $ $EDITOR $GOPATH/src/github.com/jollheef/out-of-tree/qemu/test.config.go
+
+### Run tests
+
+    $ go test -v
+
+## Usage
+
+    $ go get github.com/jollheef/out-of-tree/qemu
+
+Minimal example:
+
+	kernel := qemu.Kernel{
+		Name:       "Some kernel name",
+		KernelPath: "/path/to/vmlinuz",
+		InitrdPath: "/path/to/initrd", // if required
+	}
+	q, err := qemu.NewQemuSystem(qemu.X86_64, kernel, "/path/to/qcow2")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if err = q.Start(); err != nil {
+		log.Fatal(err)
+	}
+	defer q.Stop()
+
+	output, err = q.Command("root", "echo Hello, World!")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// output == "Hello, World!\n"
+
+More information and list of all functions see at go documentation project, or just run locally:
+
+    $ godoc github.com/jollheef/out-of-tree/qemu

qemu/qemu-kernel.go

@@ -0,0 +1,302 @@
+// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a GPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package qemukernel
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"net"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+	"syscall"
+	"time"
+
+	"golang.org/x/crypto/ssh"
+)
+
+func readBytesUntilEOF(pipe io.ReadCloser) (buf []byte, err error) {
+	bufSize := 1024
+	for err != io.EOF {
+		stdout := make([]byte, bufSize)
+		var n int
+
+		n, err = pipe.Read(stdout)
+		if err != nil && err != io.EOF {
+			return
+		}
+
+		buf = append(buf, stdout[:n]...)
+	}
+
+	if err == io.EOF {
+		err = nil
+	}
+	return
+}
+
+func readUntilEOF(pipe io.ReadCloser) (str string, err error) {
+	buf, err := readBytesUntilEOF(pipe)
+	str = string(buf)
+	return
+}
+
+type arch string
+
+const (
+	X86_64 arch = "x86_64"
+	I386        = "i386"
+	// TODO add other
+
+	unsupported = "unsupported" // for test purposes
+)
+
+// Kernel describe kernel parameters for qemu
+type Kernel struct {
+	Name       string
+	KernelPath string
+	InitrdPath string
+}
+
+// QemuSystem describe qemu parameters and runned process
+type QemuSystem struct {
+	arch      arch
+	kernel    Kernel
+	drivePath string
+
+	Cpus   int
+	Memory int
+
+	// Timeout works after Start invocation
+	Timeout         time.Duration
+	KilledByTimeout bool
+
+	Died        bool
+	sshAddrPort string
+
+	// accessible while qemu is runned
+	cmd  *exec.Cmd
+	pipe struct {
+		stdin  io.WriteCloser
+		stderr io.ReadCloser
+		stdout io.ReadCloser
+	}
+
+	// accessible after qemu is closed
+	Stdout, Stderr string
+	exitErr        error
+}
+
+// NewQemuSystem constructor
+func NewQemuSystem(arch arch, kernel Kernel, drivePath string) (q *QemuSystem, err error) {
+	if _, err = exec.LookPath("qemu-system-" + string(arch)); err != nil {
+		return
+	}
+	q = &QemuSystem{}
+	q.arch = arch
+
+	if _, err = os.Stat(kernel.KernelPath); err != nil {
+		return
+	}
+	q.kernel = kernel
+
+	if _, err = os.Stat(drivePath); err != nil {
+		return
+	}
+	q.drivePath = drivePath
+
+	// Default values
+	q.Cpus = 1
+	q.Memory = 512 // megabytes
+
+	return
+}
+
+func getRandomAddrPort() (addr string) {
+	// 127.1-255.0-255.0-255:10000-50000
+	ip := fmt.Sprintf("127.%d.%d.%d",
+		rand.Int()%254+1, rand.Int()%255, rand.Int()%254)
+	port := rand.Int()%40000 + 10000
+	return fmt.Sprintf("%s:%d", ip, port)
+}
+
+func getRandomPort(ip string) (addr string) {
+	// ip:1024-65535
+	port := rand.Int()%(65535-1024) + 1024
+	return fmt.Sprintf("%s:%d", ip, port)
+}
+
+func getFreeAddrPort() (addrPort string) {
+	timeout := time.Now().Add(time.Second)
+	for {
+		if runtime.GOOS == "linux" {
+			addrPort = getRandomAddrPort()
+		} else {
+			addrPort = getRandomPort("127.0.0.1")
+		}
+		ln, err := net.Listen("tcp", addrPort)
+		if err == nil {
+			ln.Close()
+			return
+		}
+
+		if time.Now().After(timeout) {
+			panic("Can't found free address:port on localhost")
+		}
+	}
+}
+
+func kvmExists() bool {
+	if _, err := os.Stat("/dev/kvm"); err != nil {
+		return false
+	}
+	return true
+}
+
+// Start qemu process
+func (q *QemuSystem) Start() (err error) {
+	rand.Seed(time.Now().UnixNano()) // Are you sure?
+	q.sshAddrPort = getFreeAddrPort()
+	hostfwd := fmt.Sprintf("hostfwd=tcp:%s-:22", q.sshAddrPort)
+	qemuArgs := []string{"-snapshot", "-nographic",
+		"-hda", q.drivePath,
+		"-kernel", q.kernel.KernelPath,
+		"-append", "root=/dev/sda ignore_loglevel console=ttyS0 rw",
+		"-smp", fmt.Sprintf("%d", q.Cpus),
+		"-m", fmt.Sprintf("%d", q.Memory),
+		"-device", "e1000,netdev=n1",
+		"-netdev", "user,id=n1," + hostfwd,
+	}
+
+	if q.kernel.InitrdPath != "" {
+		qemuArgs = append(qemuArgs, "-initrd", q.kernel.InitrdPath)
+	}
+
+	if (q.arch == X86_64 || q.arch == I386) && kvmExists() {
+		qemuArgs = append(qemuArgs, "-enable-kvm")
+	}
+
+	if q.arch == X86_64 && runtime.GOOS == "darwin" {
+		qemuArgs = append(qemuArgs, "-accel", "hvf", "-cpu", "host")
+	}
+
+	q.cmd = exec.Command("qemu-system-"+string(q.arch), qemuArgs...)
+
+	if q.pipe.stdin, err = q.cmd.StdinPipe(); err != nil {
+		return
+	}
+
+	if q.pipe.stdout, err = q.cmd.StdoutPipe(); err != nil {
+		return
+	}
+
+	if q.pipe.stderr, err = q.cmd.StderrPipe(); err != nil {
+		return
+	}
+
+	err = q.cmd.Start()
+	if err != nil {
+		return
+	}
+
+	go func() {
+		q.Stdout, _ = readUntilEOF(q.pipe.stdout)
+		q.Stderr, _ = readUntilEOF(q.pipe.stderr)
+		q.exitErr = q.cmd.Wait()
+		q.Died = true
+	}()
+
+	time.Sleep(time.Second / 10) // wait for immediately die
+
+	if q.Died {
+		err = errors.New("qemu died immediately: " + q.Stderr)
+	}
+
+	if q.Timeout != 0 {
+		go func() {
+			time.Sleep(q.Timeout)
+			q.KilledByTimeout = true
+			q.Stop()
+		}()
+	}
+
+	return
+}
+
+// Stop qemu process
+func (q *QemuSystem) Stop() {
+	// 1  00/01   01  01  SOH  (Ctrl-A)  START OF HEADING
+	fmt.Fprintf(q.pipe.stdin, "%cx", 1)
+	// wait for die
+	time.Sleep(time.Second / 10)
+	if !q.Died {
+		q.cmd.Process.Signal(syscall.SIGTERM)
+		time.Sleep(time.Second / 10)
+		q.cmd.Process.Signal(syscall.SIGKILL)
+	}
+}
+
+// Command executes shell commands on qemu system
+func (q *QemuSystem) Command(user, cmd string) (output string, err error) {
+	cfg := &ssh.ClientConfig{
+		User: user,
+	}
+	cfg.HostKeyCallback = ssh.InsecureIgnoreHostKey()
+
+	client, err := ssh.Dial("tcp", q.sshAddrPort, cfg)
+	if err != nil {
+		return
+	}
+	defer client.Close()
+
+	session, err := client.NewSession()
+	if err != nil {
+		return
+	}
+
+	bytesOutput, err := session.CombinedOutput(cmd)
+	output = string(bytesOutput)
+	return
+}
+
+// CopyFile is copy file from local machine to remote through ssh/scp
+func (q *QemuSystem) CopyFile(user, localPath, remotePath string) (err error) {
+	addrPort := strings.Split(q.sshAddrPort, ":")
+	addr := addrPort[0]
+	port := addrPort[1]
+
+	cmd := exec.Command("scp", "-P", port, "-o", "StrictHostKeyChecking=no",
+		localPath, user+"@"+addr+":"+remotePath)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return errors.New(string(output))
+	}
+
+	return
+}
+
+func (q *QemuSystem) CopyAndInsmod(localKoPath string) (output string, err error) {
+	remoteKoPath := fmt.Sprintf("/tmp/module_%d.ko", rand.Int())
+	err = q.CopyFile("root", localKoPath, remoteKoPath)
+	if err != nil {
+		return
+	}
+
+	return q.Command("root", "insmod "+remoteKoPath)
+}
+
+// CopyAndRun is copy local file to qemu vm then run it
+func (q *QemuSystem) CopyAndRun(user, path string) (output string, err error) {
+	remotePath := fmt.Sprintf("/tmp/executable_%d", rand.Int())
+	err = q.CopyFile(user, path, remotePath)
+	if err != nil {
+		return
+	}
+
+	return q.Command(user, "chmod +x "+remotePath+" && "+remotePath)
+}

qemu/qemu-kernel_test.go

@@ -0,0 +1,237 @@
+// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a GPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package qemukernel
+
+import (
+	"crypto/sha512"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net"
+	"os"
+	"strings"
+	"testing"
+	"time"
+)
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+func TestQemuSystemNew_InvalidKernelPath(t *testing.T) {
+	kernel := Kernel{Name: "Invalid", KernelPath: "/invalid/path"}
+	if _, err := NewQemuSystem(X86_64, kernel, "/bin/sh"); err == nil {
+		t.Fatal(err)
+	}
+}
+
+func TestQemuSystemNew_InvalidQemuArch(t *testing.T) {
+	kernel := Kernel{Name: "Valid path", KernelPath: testConfigVmlinuz}
+	if _, err := NewQemuSystem(unsupported, kernel, "/bin/sh"); err == nil {
+		t.Fatal(err)
+	}
+}
+
+func TestQemuSystemNew_InvalidQemuDrivePath(t *testing.T) {
+	kernel := Kernel{Name: "Valid path", KernelPath: testConfigVmlinuz}
+	if _, err := NewQemuSystem(X86_64, kernel, "/invalid/path"); err == nil {
+		t.Fatal(err)
+	}
+}
+
+func TestQemuSystemNew(t *testing.T) {
+	kernel := Kernel{Name: "Valid path", KernelPath: testConfigVmlinuz}
+	if _, err := NewQemuSystem(X86_64, kernel, "/bin/sh"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestQemuSystemStart(t *testing.T) {
+	kernel := Kernel{Name: "Test kernel", KernelPath: testConfigVmlinuz}
+	qemu, err := NewQemuSystem(X86_64, kernel, "/bin/sh")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = qemu.Start(); err != nil {
+		t.Fatal(err)
+	}
+
+	qemu.Stop()
+}
+
+func TestGetFreeAddrPort(t *testing.T) {
+	addrPort := getFreeAddrPort()
+	ln, err := net.Listen("tcp", addrPort)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ln.Close()
+}
+
+func TestQemuSystemStart_Timeout(t *testing.T) {
+	t.Parallel()
+	kernel := Kernel{Name: "Test kernel", KernelPath: testConfigVmlinuz}
+	qemu, err := NewQemuSystem(X86_64, kernel, "/bin/sh")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	qemu.Timeout = time.Second
+
+	if err = qemu.Start(); err != nil {
+		t.Fatal(err)
+	}
+
+	time.Sleep(2 * time.Second)
+
+	if !qemu.Died {
+		t.Fatal("qemu does not died :c")
+	}
+
+	if !qemu.KilledByTimeout {
+		t.Fatal("qemu died not because of timeout O_o")
+	}
+}
+
+func startTestQemu(t *testing.T) (q *QemuSystem, err error) {
+	t.Parallel()
+	kernel := Kernel{
+		Name:       "Test kernel",
+		KernelPath: testConfigVmlinuz,
+		InitrdPath: testConfigInitrd,
+	}
+	q, err = NewQemuSystem(X86_64, kernel, testConfigRootfs)
+	if err != nil {
+		return
+	}
+
+	if err = q.Start(); err != nil {
+		return
+	}
+
+	return
+}
+
+func TestQemuSystemCommand(t *testing.T) {
+	qemu, err := startTestQemu(t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer qemu.Stop()
+
+	output, err := qemu.Command("root", "cat /etc/shadow")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(output, "root::") {
+		t.Fatal("Wrong output from `cat /etc/shadow` by root")
+	}
+
+	output, err = qemu.Command("user", "cat /etc/passwd")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(output, "root:x:0:0:root:/root:/bin/bash") {
+		t.Fatal("Wrong output from `cat /etc/passwd` by user")
+	}
+
+	output, err = qemu.Command("user", "cat /etc/shadow")
+	if err == nil { // unsucessful is good because user must not read /etc/shadow
+		t.Fatal("User have rights for /etc/shadow. WAT?!")
+	}
+}
+
+func TestQemuSystemCopyFile(t *testing.T) {
+	qemu, err := startTestQemu(t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer qemu.Stop()
+
+	localPath := "/bin/sh"
+
+	content, err := ioutil.ReadFile(localPath)
+	if err != nil {
+		return
+	}
+
+	sha_local := fmt.Sprintf("%x", sha512.Sum512(content))
+
+	err = qemu.CopyFile("user", localPath, "/tmp/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sha_remote, err := qemu.Command("user", "sha512sum /tmp/test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	sha_remote = strings.Split(sha_remote, " ")[0]
+
+	if sha_local != sha_remote {
+		t.Fatal(fmt.Sprintf("Broken file (%s instead of %s)", sha_remote, sha_local))
+	}
+}
+
+func TestQemuSystemCopyAndRun(t *testing.T) {
+	qemu, err := startTestQemu(t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer qemu.Stop()
+
+	randStr := fmt.Sprintf("%d", rand.Int())
+	content := []byte("#!/bin/sh\n echo -n " + randStr + "\n")
+
+	tmpfile, err := ioutil.TempFile("", "executable")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name())
+
+	if _, err := tmpfile.Write(content); err != nil {
+		t.Fatal(err)
+	}
+	if err := tmpfile.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	output, err := qemu.CopyAndRun("user", tmpfile.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if output != randStr {
+		t.Fatal("Wrong output from copyied executable (" + output + "," + randStr + ")")
+	}
+}
+
+func TestQemuSystemCopyAndInsmod(t *testing.T) {
+	qemu, err := startTestQemu(t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer qemu.Stop()
+
+	lsmodBefore, err := qemu.Command("root", "lsmod | wc -l")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = qemu.CopyAndInsmod(testConfigSampleKo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	lsmodAfter, err := qemu.Command("root", "lsmod | wc -l")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if lsmodBefore == lsmodAfter {
+		t.Fatal("insmod returns ok but there is no new kernel modules")
+	}
+}

qemu/test.config.go

@@ -0,0 +1,10 @@
+// Copyright 2018 Mikhail Klementev. All rights reserved.
+// Use of this source code is governed by a GPLv3 license
+// (or later) that can be found in the LICENSE file.
+
+package qemukernel
+
+const testConfigVmlinuz = "../tools/qemu-debian-img/vmlinuz-bionic"
+const testConfigInitrd = "../tools/qemu-debian-img/initrd-bionic"
+const testConfigRootfs = "../tools/qemu-debian-img/bionic.img"
+const testConfigSampleKo = "../tools/qemu-debian-img/sample.ko"