ci: add full e2e tests on vm

.github/workflows/e2e.yml

@@ -0,0 +1,146 @@
+name: E2E
+
+on:
+  workflow_dispatch:
+  push:
+    paths-ignore:
+      - ".github/workflows/images-*"
+      - ".github/workflows/ubuntu.yml"
+      - ".github/workflows/macos.yml"
+      - ".github/workflows/debian-cache.yml"
+
+concurrency:
+  group: ${{ github.workflow_ref }}
+  cancel-in-progress: true
+
+jobs:
+  test-end-to-end:
+    name: Full
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [
+          { distro: Ubuntu, release: 12.04 },
+          { distro: Ubuntu, release: 14.04 },
+          { distro: Ubuntu, release: 16.04 },
+          { distro: Ubuntu, release: 18.04 },
+          { distro: Ubuntu, release: 20.04 },
+          { distro: Ubuntu, release: 22.04 },
+          { distro: CentOS, release: 6 },
+          { distro: CentOS, release: 7 },
+          { distro: CentOS, release: 8 },
+          { distro: OracleLinux, release: 6 },
+          { distro: OracleLinux, release: 7 },
+          { distro: OracleLinux, release: 8 },
+          { distro: OracleLinux, release: 9 },
+          { distro: Debian, release: 7 },
+          { distro: Debian, release: 8 },
+          { distro: Debian, release: 9 },
+          { distro: Debian, release: 10 },
+          { distro: Debian, release: 11 },
+          { distro: Debian, release: 12 }
+          ]
+
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Build
+      run: go build
+
+    - uses: digitalocean/action-doctl@v2
+      with:
+        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+
+    - uses: webfactory/ssh-agent@v0.8.0
+      with:
+        ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+
+    - name: Create droplet
+      run: >-
+        doctl compute droplet create
+        --size s-4vcpu-8gb
+        --tag-name=github-actions
+        --image almalinux-9-x64
+        --ssh-keys='b4:4c:66:7d:be:19:25:43:1c:e0:02:61:9f:49:12:94'
+        --wait
+        --region fra1
+        ga-out-of-tree-e2e-${{ matrix.os.distro }}-${{ matrix.os.release }}-$GITHUB_SHA
+
+    - name: End-to-End Testing [${{ matrix.os.distro }} ${{ matrix.os.release }}]
+      shell: bash
+      run: |
+        sleep 1m
+
+        IP=$(doctl compute droplet list \
+          --tag-name=github-actions \
+          --format "Name,Public IPv4" \
+          | grep -v ID \
+          | grep ga-out-of-tree-e2e-${{ matrix.os.distro }}-${{ matrix.os.release }}-$GITHUB_SHA \
+          | awk '{print $2}')
+
+        while ! ssh -o StrictHostKeyChecking=accept-new root@$IP echo
+        do
+          sleep 1s
+        done
+
+        ssh root@$IP "cloud-init status --wait"
+
+        ssh root@$IP "dnf install -y podman qemu-kvm-core"
+
+        ssh root@$IP "ln -s /usr/libexec/qemu-kvm /usr/bin/qemu-system-x86_64"
+
+        scp ./out-of-tree root@$IP:/usr/local/bin/
+
+        echo 'name = "test"' > examples/kernel-module/.out-of-tree.toml
+        echo 'type = "module"' >> examples/kernel-module/.out-of-tree.toml
+        echo 'standard_modules = true' >> examples/kernel-module/.out-of-tree.toml
+
+        echo '[[targets]]' >> examples/kernel-module/.out-of-tree.toml
+        echo 'distro = { id = "${{ matrix.os.distro }}", release = "${{ matrix.os.release }}" }' >> examples/kernel-module/.out-of-tree.toml
+        echo 'kernel = { regex = ".*" }' >> examples/kernel-module/.out-of-tree.toml
+
+        echo 'modprobe 9p' >> examples/kernel-module/test.sh
+
+        scp -r examples/kernel-module root@$IP:test
+
+        echo '[Unit]' >> test.service
+        echo 'Description=e2e' >> test.service
+        echo '[Service]' >> test.service
+        echo 'RemainAfterExit=yes' >> test.service
+        echo 'StandardError=append:/var/log/test.log' >> test.service
+        echo 'StandardOutput=append:/var/log/test.log' >> test.service
+        echo 'Type=oneshot' >> test.service
+        echo 'WorkingDirectory=/root/test' >> test.service
+        echo 'TimeoutStopSec=1' >> test.service
+        echo 'ExecStart=/usr/local/bin/out-of-tree kernel autogen --threads=4' >> test.service
+        echo 'ExecStart=/usr/local/bin/out-of-tree pew --threads=4' >> test.service
+
+        scp test.service root@$IP:/etc/systemd/system/test.service
+
+        ssh root@$IP systemctl daemon-reload
+
+        ssh root@$IP systemctl start test --no-block
+
+        while ! ssh root@$IP systemctl show test -p SubState --value | grep -E '(failed|exited)'
+        do
+          sleep 30s
+        done
+
+        ssh root@$IP "cat /var/log/test.log"
+
+        scp -r root@$IP:.out-of-tree/logs .
+
+        ssh root@$IP systemctl is-active test
+
+    - name: Archive logs
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-end-to-end-${{ matrix.os.distro }}-${{ matrix.os.release }}-logs
+        path: logs
+
+    - name: Delete droplet
+      if: always()
+      run: doctl compute droplet delete -f ga-out-of-tree-e2e-${{ matrix.os.distro }}-${{ matrix.os.release }}-$GITHUB_SHA