1
0
out-of-tree/qemu/qemu-kernel.go

671 lines
13 KiB
Go
Raw Normal View History

2018-09-18 21:45:21 +00:00
// Copyright 2018 Mikhail Klementev. All rights reserved.
2018-10-08 20:51:32 +00:00
// Use of this source code is governed by a AGPLv3 license
2018-09-18 21:45:21 +00:00
// (or later) that can be found in the LICENSE file.
2019-02-02 21:33:27 +00:00
package qemu
2018-09-18 21:45:21 +00:00
import (
2023-04-06 18:20:55 +00:00
"bufio"
2018-09-18 21:45:21 +00:00
"errors"
"fmt"
"io"
2018-09-19 06:13:28 +00:00
"math/rand"
"net"
2018-09-18 21:45:21 +00:00
"os"
"os/exec"
2018-09-20 23:47:36 +00:00
"runtime"
"strings"
2023-05-08 21:35:47 +00:00
"sync"
2018-09-18 21:45:21 +00:00
"syscall"
"time"
2023-05-07 18:14:59 +00:00
"github.com/povsister/scp"
2023-04-06 19:50:57 +00:00
"github.com/rs/zerolog"
2023-03-19 13:14:14 +00:00
"github.com/rs/zerolog/log"
"golang.org/x/crypto/ssh"
2018-09-18 21:45:21 +00:00
)
type arch string
const (
2019-08-17 09:05:06 +00:00
// X86x64 is the qemu-system-x86_64
X86x64 arch = "x86_64"
// X86x32 is the qemu-system-i386
X86x32 = "i386"
2018-09-18 21:45:21 +00:00
// TODO add other
unsupported = "unsupported" // for test purposes
)
const (
DefaultCPUs = 1
DefaultMemory = 512 // megabytes
DefaultSSHRetries = 4
DefaultSSHRetryTimeout = time.Second / 4
)
2018-09-18 21:45:21 +00:00
// Kernel describe kernel parameters for qemu
type Kernel struct {
2018-09-22 10:34:43 +00:00
Name string
KernelPath string
InitrdPath string
2018-09-18 21:45:21 +00:00
}
type CPU struct {
Model string
Flags []string
}
2019-08-21 06:16:25 +00:00
// System describe qemu parameters and executed process
2019-08-17 09:05:06 +00:00
type System struct {
2018-09-19 06:13:28 +00:00
arch arch
kernel Kernel
drivePath string
2023-02-15 09:48:13 +00:00
Mutable bool
2018-09-19 06:13:28 +00:00
Cpus int
Memory int
2018-09-18 21:45:21 +00:00
CPU CPU
2018-11-25 13:12:06 +00:00
debug bool
gdb string // tcp::1234
noKASLR bool
noSMEP bool
noSMAP bool
2019-08-20 00:05:19 +00:00
noKPTI bool
2018-09-22 07:28:08 +00:00
// Timeout works after Start invocation
Timeout time.Duration
KilledByTimeout bool
2018-10-06 14:00:48 +00:00
KernelPanic bool
2023-05-08 14:54:28 +00:00
Died bool
SSH struct {
AddrPort string
2023-05-01 15:10:39 +00:00
Retries int
RetryTimeout time.Duration
}
2018-09-22 07:28:08 +00:00
2019-08-21 06:16:25 +00:00
// accessible while qemu is running
2018-09-18 21:45:21 +00:00
cmd *exec.Cmd
pipe struct {
stdin io.WriteCloser
stderr io.ReadCloser
stdout io.ReadCloser
}
2023-04-06 19:50:57 +00:00
Stdout, Stderr string
2018-09-18 21:45:21 +00:00
// accessible after qemu is closed
exitErr error
2023-04-06 19:50:57 +00:00
Log zerolog.Logger
2018-09-18 21:45:21 +00:00
}
2019-08-17 09:05:06 +00:00
// NewSystem constructor
func NewSystem(arch arch, kernel Kernel, drivePath string) (q *System, err error) {
2023-04-06 19:50:57 +00:00
q = &System{}
q.Log = log.With().
2023-04-06 19:50:57 +00:00
Str("kernel", kernel.KernelPath).
Logger()
2018-09-18 21:45:21 +00:00
if _, err = exec.LookPath("qemu-system-" + string(arch)); err != nil {
return
}
2023-04-06 19:50:57 +00:00
2018-09-18 21:45:21 +00:00
q.arch = arch
2018-09-22 10:34:43 +00:00
if _, err = os.Stat(kernel.KernelPath); err != nil {
2018-09-18 21:45:21 +00:00
return
}
q.kernel = kernel
2018-09-19 06:13:28 +00:00
if _, err = os.Stat(drivePath); err != nil {
return
}
q.drivePath = drivePath
q.Cpus = DefaultCPUs
q.Memory = DefaultMemory
q.SSH.Retries = DefaultSSHRetries
q.SSH.RetryTimeout = DefaultSSHRetryTimeout
2018-09-19 06:13:28 +00:00
2018-09-18 21:45:21 +00:00
return
}
func (q *System) SetSSHAddrPort(addr string, port int) (err error) {
// TODO validate
2023-05-08 14:54:28 +00:00
q.SSH.AddrPort = fmt.Sprintf("%s:%d", addr, port)
return
}
2018-09-19 06:13:28 +00:00
func getRandomAddrPort() (addr string) {
// 127.1-255.0-255.0-255:10000-50000
ip := fmt.Sprintf("127.%d.%d.%d",
rand.Int()%254+1, rand.Int()%255, rand.Int()%254)
port := rand.Int()%40000 + 10000
return fmt.Sprintf("%s:%d", ip, port)
}
2018-09-20 23:47:36 +00:00
func getRandomPort(ip string) (addr string) {
// ip:1024-65535
2018-10-08 18:59:05 +00:00
port := rand.Int()%(65536-1024) + 1024
2018-09-20 23:47:36 +00:00
return fmt.Sprintf("%s:%d", ip, port)
}
2024-02-20 11:58:41 +00:00
func GetFreeAddrPort() (addrPort string) {
2018-09-20 23:47:36 +00:00
timeout := time.Now().Add(time.Second)
2018-09-19 06:13:28 +00:00
for {
2018-09-20 23:47:36 +00:00
if runtime.GOOS == "linux" {
addrPort = getRandomAddrPort()
} else {
addrPort = getRandomPort("127.0.0.1")
}
2018-09-19 06:13:28 +00:00
ln, err := net.Listen("tcp", addrPort)
if err == nil {
ln.Close()
return
}
2018-09-20 23:47:36 +00:00
if time.Now().After(timeout) {
panic("Can't found free address:port on localhost")
}
2018-09-19 06:13:28 +00:00
}
}
func kvmExists() bool {
if _, err := os.Stat("/dev/kvm"); err != nil {
return false
}
2019-08-18 12:53:13 +00:00
file, err := os.OpenFile("/dev/kvm", os.O_WRONLY, 0666)
if err != nil {
if os.IsPermission(err) {
return false
}
}
file.Close()
2018-09-19 06:13:28 +00:00
return true
}
2023-12-25 15:03:07 +00:00
func (q *System) checkOopsPanic(s string) {
if strings.Contains(s, "Kernel panic") {
q.KernelPanic = true
q.Log.Warn().Msg("kernel panic")
2018-10-06 14:00:48 +00:00
time.Sleep(time.Second)
2023-12-25 15:03:07 +00:00
// There is no reason to stay alive after kernel panic
q.Stop()
} else if strings.Contains(s, "BUG") {
q.Log.Warn().Msg(s)
time.Sleep(time.Second)
// consider BUG() as non-recoverable state
q.Stop()
} else if strings.Contains(s, "WARNING") {
q.Log.Warn().Msg(s)
2018-10-06 14:00:48 +00:00
}
}
2019-08-17 10:04:45 +00:00
func (q System) cmdline() (s string) {
s = "root=/dev/sda ignore_loglevel console=ttyS0 rw"
if q.noKASLR {
s += " nokaslr"
}
if q.noSMEP {
s += " nosmep"
}
if q.noSMAP {
s += " nosmap"
}
2019-08-20 00:05:19 +00:00
if q.noKPTI {
s += " nokpti"
}
2019-08-17 10:04:45 +00:00
return
}
2023-04-07 21:17:34 +00:00
func (q System) Executable() string {
return "qemu-system-" + string(q.arch)
}
func (q *System) Args() (qemuArgs []string) {
2023-05-08 14:54:28 +00:00
if q.SSH.AddrPort == "" {
2024-02-20 11:58:41 +00:00
q.SSH.AddrPort = GetFreeAddrPort()
}
2023-05-08 14:54:28 +00:00
hostfwd := fmt.Sprintf("hostfwd=tcp:%s-:22", q.SSH.AddrPort)
2023-04-07 21:17:34 +00:00
qemuArgs = []string{"-nographic",
2018-09-19 06:13:28 +00:00
"-hda", q.drivePath,
2018-09-22 10:34:43 +00:00
"-kernel", q.kernel.KernelPath,
2018-09-19 06:13:28 +00:00
"-smp", fmt.Sprintf("%d", q.Cpus),
"-m", fmt.Sprintf("%d", q.Memory),
"-device", "e1000,netdev=n1",
"-netdev", "user,id=n1," + hostfwd,
2018-09-19 06:13:28 +00:00
}
2023-02-15 09:48:13 +00:00
if !q.Mutable {
qemuArgs = append(qemuArgs, "-snapshot")
}
2018-11-25 13:12:06 +00:00
if q.debug {
qemuArgs = append(qemuArgs, "-gdb", q.gdb)
}
2018-09-22 10:34:43 +00:00
if q.kernel.InitrdPath != "" {
qemuArgs = append(qemuArgs, "-initrd", q.kernel.InitrdPath)
}
cpu := "max"
if q.CPU.Model != "" {
cpu = q.CPU.Model
}
for _, flag := range q.CPU.Flags {
cpu += "," + flag
}
qemuArgs = append(qemuArgs, "-cpu", cpu)
2023-05-17 17:45:34 +00:00
if q.arch == X86x64 || q.arch == X86x32 {
if kvmExists() {
qemuArgs = append(qemuArgs, "-enable-kvm")
}
2018-09-19 06:13:28 +00:00
}
2019-08-17 09:05:06 +00:00
if q.arch == X86x64 && runtime.GOOS == "darwin" {
qemuArgs = append(qemuArgs, "-accel", "hvf")
2018-09-21 00:13:29 +00:00
}
2019-08-17 10:04:45 +00:00
qemuArgs = append(qemuArgs, "-append", q.cmdline())
2023-04-07 21:17:34 +00:00
return
}
// Start qemu process
func (q *System) Start() (err error) {
q.cmd = exec.Command(q.Executable(), q.Args()...)
q.Log.Debug().Msgf("%v", q.cmd)
2018-09-18 21:45:21 +00:00
if q.pipe.stdin, err = q.cmd.StdinPipe(); err != nil {
return
}
if q.pipe.stdout, err = q.cmd.StdoutPipe(); err != nil {
return
}
if q.pipe.stderr, err = q.cmd.StderrPipe(); err != nil {
return
}
2024-02-20 12:03:35 +00:00
q.Log.Debug().Msg("start qemu")
2018-09-18 21:45:21 +00:00
err = q.cmd.Start()
if err != nil {
return
}
2023-04-06 19:50:57 +00:00
go func() {
scanner := bufio.NewScanner(q.pipe.stdout)
for scanner.Scan() {
m := scanner.Text()
q.Stdout += m + "\n"
q.Log.Trace().Str("stdout", m).Msg("qemu")
2023-12-25 15:03:07 +00:00
go q.checkOopsPanic(m)
2023-04-06 19:50:57 +00:00
}
}()
go func() {
scanner := bufio.NewScanner(q.pipe.stderr)
for scanner.Scan() {
m := scanner.Text()
q.Stderr += m + "\n"
q.Log.Trace().Str("stderr", m).Msg("qemu")
2023-04-06 19:50:57 +00:00
}
}()
2018-09-18 21:45:21 +00:00
go func() {
q.exitErr = q.cmd.Wait()
2018-09-22 07:28:08 +00:00
q.Died = true
2024-02-20 12:03:35 +00:00
q.Log.Debug().Msg("qemu died")
2018-09-18 21:45:21 +00:00
}()
time.Sleep(time.Second / 10) // wait for immediately die
2018-09-22 07:28:08 +00:00
if q.Died {
err = errors.New("qemu died immediately: " + string(q.Stderr))
2018-09-18 21:45:21 +00:00
}
2018-09-22 07:28:08 +00:00
if q.Timeout != 0 {
go func() {
time.Sleep(q.Timeout)
q.KilledByTimeout = true
q.Stop()
}()
}
2018-09-18 21:45:21 +00:00
return
}
// Stop qemu process
2019-08-17 09:05:06 +00:00
func (q *System) Stop() {
2024-02-20 12:03:35 +00:00
q.Log.Debug().Msg("stop qemu process")
2018-09-18 21:45:21 +00:00
// 1 00/01 01 01 SOH (Ctrl-A) START OF HEADING
fmt.Fprintf(q.pipe.stdin, "%cx", 1)
// wait for die
time.Sleep(time.Second / 10)
2018-09-22 07:28:08 +00:00
if !q.Died {
2018-09-18 21:45:21 +00:00
q.cmd.Process.Signal(syscall.SIGTERM)
time.Sleep(time.Second / 10)
q.cmd.Process.Signal(syscall.SIGKILL)
}
}
func (q *System) WaitForSSH(timeout time.Duration) error {
2024-02-20 15:07:02 +00:00
q.Log.Debug().Msgf("wait for ssh for %v", timeout)
2024-02-20 12:03:35 +00:00
2023-02-16 06:27:17 +00:00
for start := time.Now(); time.Since(start) < timeout; {
time.Sleep(time.Second / 4)
2023-05-10 12:30:01 +00:00
if q.Died || q.KernelPanic {
return errors.New("no ssh (qemu is dead)")
}
2023-02-16 06:27:17 +00:00
client, err := q.ssh("root")
if err != nil {
2024-02-20 12:03:35 +00:00
q.Log.Debug().Err(err).Msg("")
2023-02-16 06:27:17 +00:00
continue
}
session, err := client.NewSession()
if err != nil {
client.Close()
2024-02-20 12:03:35 +00:00
q.Log.Debug().Err(err).Msg("")
continue
}
_, err = session.CombinedOutput("echo")
if err != nil {
client.Close()
2024-02-20 12:03:35 +00:00
q.Log.Debug().Err(err).Msg("")
continue
}
2023-02-16 06:27:17 +00:00
client.Close()
2023-05-10 12:30:01 +00:00
q.Log.Debug().Msg("ssh is available")
2023-02-16 06:27:17 +00:00
return nil
}
return errors.New("no ssh (timeout)")
}
func (q *System) ssh(user string) (client *ssh.Client, err error) {
cfg := &ssh.ClientConfig{
2018-10-06 14:00:31 +00:00
User: user,
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}
2023-05-08 14:54:28 +00:00
for retries := q.SSH.Retries; retries > 0; retries-- {
if q.Died {
2023-05-24 19:46:11 +00:00
err = errors.New("qemu is dead")
return
}
2023-05-08 14:54:28 +00:00
client, err = ssh.Dial("tcp", q.SSH.AddrPort, cfg)
if err == nil {
break
}
time.Sleep(q.SSH.RetryTimeout)
}
2018-10-06 14:00:31 +00:00
return
}
// Command executes shell commands on qemu system
2019-08-17 09:05:06 +00:00
func (q System) Command(user, cmd string) (output string, err error) {
2023-05-08 14:30:09 +00:00
flog := q.Log.With().Str("kernel", q.kernel.KernelPath).
2023-04-06 18:20:55 +00:00
Str("user", user).
2023-05-08 14:30:09 +00:00
Str("cmd", cmd).
Logger()
2023-04-06 18:20:55 +00:00
2023-05-08 14:30:09 +00:00
flog.Debug().Msg("qemu command")
2023-03-19 13:30:10 +00:00
2018-10-06 14:00:31 +00:00
client, err := q.ssh(user)
if err != nil {
2023-05-08 14:30:09 +00:00
flog.Debug().Err(err).Msg("ssh connection")
return
}
defer client.Close()
session, err := client.NewSession()
if err != nil {
2023-05-08 14:30:09 +00:00
flog.Debug().Err(err).Msg("new session")
return
}
2023-04-06 18:20:55 +00:00
stdout, err := session.StdoutPipe()
if err != nil {
2023-05-08 14:30:09 +00:00
flog.Debug().Err(err).Msg("get stdout pipe")
2023-04-06 18:20:55 +00:00
return
}
2023-04-26 14:35:20 +00:00
stderr, err := session.StderrPipe()
if err != nil {
2023-05-08 14:30:09 +00:00
flog.Debug().Err(err).Msg("get stderr pipe")
2023-04-26 14:35:20 +00:00
return
}
2023-04-06 18:20:55 +00:00
err = session.Start(cmd)
if err != nil {
2023-05-08 14:30:09 +00:00
flog.Debug().Err(err).Msg("start session")
2023-04-06 18:20:55 +00:00
return
}
2023-05-08 21:35:47 +00:00
var wg sync.WaitGroup
wg.Add(1)
2023-04-06 18:20:55 +00:00
go func() {
2023-05-08 21:35:47 +00:00
defer wg.Done()
2023-04-06 18:20:55 +00:00
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
m := scanner.Text()
output += m + "\n"
2023-05-08 14:30:09 +00:00
flog.Trace().Str("stdout", m).Msg("qemu command")
2023-04-06 18:20:55 +00:00
}
2023-04-06 20:45:20 +00:00
output = strings.TrimSuffix(output, "\n")
2023-04-06 18:20:55 +00:00
}()
2023-05-08 21:35:47 +00:00
wg.Add(1)
2023-04-26 14:35:20 +00:00
go func() {
2023-05-08 21:35:47 +00:00
defer wg.Done()
2023-04-26 14:35:20 +00:00
scanner := bufio.NewScanner(stderr)
for scanner.Scan() {
m := scanner.Text()
output += m + "\n"
// Note: it prints stderr as stdout
2023-05-08 14:30:09 +00:00
flog.Trace().Str("stdout", m).Msg("qemu command")
2023-04-26 14:35:20 +00:00
}
output = strings.TrimSuffix(output, "\n")
}()
2023-04-06 18:20:55 +00:00
err = session.Wait()
2023-05-08 21:35:47 +00:00
wg.Wait()
return
}
2018-10-06 14:00:31 +00:00
// AsyncCommand executes command on qemu system but does not wait for exit
2019-08-17 09:05:06 +00:00
func (q System) AsyncCommand(user, cmd string) (err error) {
2018-10-06 14:00:31 +00:00
client, err := q.ssh(user)
if err != nil {
return
}
defer client.Close()
session, err := client.NewSession()
if err != nil {
return
}
return session.Run(fmt.Sprintf(
"nohup sh -c '%s' > /dev/null 2> /dev/null < /dev/null &", cmd))
}
2023-02-15 11:20:30 +00:00
func (q System) scp(user, localPath, remotePath string, recursive bool) (err error) {
2023-05-07 18:14:59 +00:00
q.Log.Debug().Msgf("scp[%s] %s -> %s", user, localPath, remotePath)
2023-05-08 14:54:28 +00:00
sshClient, err := q.ssh(user)
if err != nil {
return
2023-02-15 11:20:30 +00:00
}
2023-05-08 14:54:28 +00:00
defer sshClient.Close()
2023-02-15 11:20:30 +00:00
2023-05-08 14:54:28 +00:00
client, err := scp.NewClientFromExistingSSH(sshClient, &scp.ClientOption{})
2023-05-07 18:14:59 +00:00
if err != nil {
return
2023-02-15 11:20:30 +00:00
}
2023-05-07 18:14:59 +00:00
if recursive {
err = client.CopyDirToRemote(
localPath,
remotePath,
&scp.DirTransferOption{},
)
} else {
err = client.CopyFileToRemote(
localPath,
remotePath,
&scp.FileTransferOption{},
)
}
return
}
func (q *System) scpWithRetry(user, localPath, remotePath string, recursive bool) (err error) {
2023-05-08 14:54:28 +00:00
for retries := q.SSH.Retries; retries > 0; retries-- {
if q.Died {
2023-05-24 19:46:11 +00:00
err = errors.New("qemu is dead")
return
}
2023-05-01 15:10:39 +00:00
err = q.scp(user, localPath, remotePath, recursive)
if err == nil {
break
}
2024-02-21 13:30:49 +00:00
q.Log.Debug().Err(err).Msgf(
"scp[%v] (r: %v) %v -> %v: failed",
user, recursive, localPath, remotePath)
2023-05-08 14:54:28 +00:00
time.Sleep(q.SSH.RetryTimeout)
q.Log.Warn().Msgf("scp: %d retries left", retries)
2023-05-01 15:10:39 +00:00
}
return
}
2023-02-15 11:20:30 +00:00
// CopyFile from local machine to remote via scp
func (q System) CopyFile(user, localPath, remotePath string) (err error) {
2023-05-01 15:10:39 +00:00
return q.scpWithRetry(user, localPath, remotePath, false)
2023-02-15 11:20:30 +00:00
}
// CopyDirectory from local machine to remote via scp
func (q System) CopyDirectory(user, localPath, remotePath string) (err error) {
2023-05-01 15:10:39 +00:00
return q.scpWithRetry(user, localPath, remotePath, true)
2023-02-15 11:20:30 +00:00
}
2018-12-10 03:06:26 +00:00
// CopyAndInsmod copy kernel module to temporary file on qemu then insmod it
2019-08-17 09:05:06 +00:00
func (q *System) CopyAndInsmod(localKoPath string) (output string, err error) {
remoteKoPath := fmt.Sprintf("/tmp/module_%d.ko", rand.Int())
err = q.CopyFile("root", localKoPath, remoteKoPath)
if err != nil {
return
}
return q.Command("root", "insmod "+remoteKoPath)
}
// CopyAndRun is copy local file to qemu vm then run it
2019-08-17 09:05:06 +00:00
func (q *System) CopyAndRun(user, path string) (output string, err error) {
remotePath := fmt.Sprintf("/tmp/executable_%d", rand.Int())
err = q.CopyFile(user, path, remotePath)
if err != nil {
return
}
return q.Command(user, "chmod +x "+remotePath+" && "+remotePath)
}
2018-11-25 13:12:06 +00:00
// CopyAndRunAsync is copy local file to qemu vm then run it w/o wait for exit
func (q *System) CopyAndRunAsync(user, path string) (err error) {
remotePath := fmt.Sprintf("/tmp/executable_%d", rand.Int())
err = q.CopyFile(user, path, remotePath)
if err != nil {
return
}
return q.AsyncCommand(user, "chmod +x "+remotePath+" && "+remotePath)
}
2018-12-10 03:06:26 +00:00
// Debug is for enable qemu debug and set hostname and port for listen
2019-08-17 09:05:06 +00:00
func (q *System) Debug(conn string) {
2018-11-25 13:12:06 +00:00
q.debug = true
q.gdb = conn
}
2018-11-25 14:44:14 +00:00
// SetKASLR is changing KASLR state through kernel boot args
2019-08-17 09:05:06 +00:00
func (q *System) SetKASLR(state bool) {
q.noKASLR = !state
}
// SetSMEP is changing SMEP state through kernel boot args
2019-08-17 09:05:06 +00:00
func (q *System) SetSMEP(state bool) {
q.noSMEP = !state
}
// SetSMAP is changing SMAP state through kernel boot args
2019-08-17 09:05:06 +00:00
func (q *System) SetSMAP(state bool) {
q.noSMAP = !state
}
2019-08-20 00:05:19 +00:00
// SetKPTI is changing KPTI state through kernel boot args
func (q *System) SetKPTI(state bool) {
q.noKPTI = !state
}
// GetKASLR is retrieve KASLR settings
func (q *System) GetKASLR() bool {
return !q.noKASLR
}
// GetSMEP is retrieve SMEP settings
func (q *System) GetSMEP() bool {
return !q.noSMEP
}
// GetSMAP is retrieve SMAP settings
func (q *System) GetSMAP() bool {
return !q.noSMAP
}
2019-08-20 00:05:19 +00:00
// GetKPTI is retrieve KPTI settings
func (q *System) GetKPTI() bool {
return !q.noKPTI
}
2019-08-17 09:05:06 +00:00
// GetSSHCommand returns command for connect to qemu machine over ssh
func (q System) GetSSHCommand() (cmd string) {
2023-05-08 14:54:28 +00:00
addrPort := strings.Split(q.SSH.AddrPort, ":")
2018-11-25 14:44:14 +00:00
addr := addrPort[0]
port := addrPort[1]
cmd = "ssh -o StrictHostKeyChecking=no"
cmd += " -p " + port + " root@" + addr
return
}