name: E2E

on:
  workflow_dispatch:
  schedule:
    - cron: '0 0 * * 0'
  push:
    branches:
      - master
      - dev
    paths-ignore:
      - ".github/workflows/images-*"
      - ".github/workflows/ubuntu.yml"
      - ".github/workflows/macos.yml"
      - ".github/workflows/debian-cache.yml"
      - "docs/**"
      - ".readthedocs.yaml"
      - "README.md"

concurrency:
  group: ${{ github.workflow_ref }}
  cancel-in-progress: true

jobs:
  test-end-to-end:
    name: Module
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        os: [
          { distro: Ubuntu, release: 12.04 },
          { distro: Ubuntu, release: 14.04 },
          { distro: Ubuntu, release: 16.04 },
          { distro: Ubuntu, release: 18.04 },
          { distro: Ubuntu, release: 20.04 },
          { distro: Ubuntu, release: 22.04 },
          { distro: Ubuntu, release: 24.04 },
          { distro: CentOS, release: 6 },
          { distro: CentOS, release: 7 },
          { distro: CentOS, release: 8 },
          { distro: OracleLinux, release: 6 },
          { distro: OracleLinux, release: 7 },
          { distro: OracleLinux, release: 8 },
          { distro: OracleLinux, release: 9 },
          { distro: Debian, release: 7 },
          { distro: Debian, release: 8 },
          { distro: Debian, release: 9 },
          { distro: Debian, release: 10 },
          { distro: Debian, release: 11 },
          { distro: Debian, release: 12 },
          { distro: OpenSUSE, release: "12.1" },
          { distro: OpenSUSE, release: "12.2" },
          { distro: OpenSUSE, release: "12.3" },
          { distro: OpenSUSE, release: "13.1" },
          { distro: OpenSUSE, release: "13.2" },
          { distro: OpenSUSE, release: "42.1" },
          { distro: OpenSUSE, release: "42.2" },
          { distro: OpenSUSE, release: "42.3" },
          { distro: OpenSUSE, release: "15.0" },
          { distro: OpenSUSE, release: "15.1" },
          { distro: OpenSUSE, release: "15.2" },
          { distro: OpenSUSE, release: "15.3" },
          { distro: OpenSUSE, release: "15.4" },
          { distro: OpenSUSE, release: "15.5" }
          ]

    steps:
    - uses: actions/checkout@v1

    - name: Build
      run: go build

    - uses: digitalocean/action-doctl@v2
      with:
        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}

    - uses: webfactory/ssh-agent@v0.9.0
      with:
        ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}

    - name: Create droplet
      run: >-
        doctl compute droplet create
        --size s-4vcpu-8gb-intel
        --tag-name=github-actions
        --image almalinux-9-x64
        --ssh-keys='b4:4c:66:7d:be:19:25:43:1c:e0:02:61:9f:49:12:94,37:46:77:a8:4a:96:3b:20:16:46:35:04:95:ca:0c:5c'
        --wait
        --region fra1
        ga-out-of-tree-e2e-${{ matrix.os.distro }}-${{ matrix.os.release }}-$GITHUB_SHA

    - name: End-to-End Testing [${{ matrix.os.distro }} ${{ matrix.os.release }}]
      shell: bash
      run: |
        sleep 1m

        IP=$(doctl compute droplet list \
          --tag-name=github-actions \
          --format "Name,Public IPv4" \
          | grep -v ID \
          | grep ga-out-of-tree-e2e-${{ matrix.os.distro }}-${{ matrix.os.release }}-$GITHUB_SHA \
          | awk '{print $2}')

        while ! ssh -o StrictHostKeyChecking=accept-new root@$IP echo
        do
          sleep 1s
        done

        ssh root@$IP "cloud-init status --wait"

        ssh root@$IP "dnf install -y podman qemu-kvm-core epel-release"
        ssh root@$IP "dnf install -y s3cmd"

        ssh root@$IP "ln -s /usr/libexec/qemu-kvm /usr/bin/qemu-system-x86_64"

        scp ./out-of-tree root@$IP:/usr/local/bin/

        echo 'name = "test"' > examples/kernel-module/.out-of-tree.toml
        echo 'type = "module"' >> examples/kernel-module/.out-of-tree.toml
        echo 'standard_modules = true' >> examples/kernel-module/.out-of-tree.toml

        echo '[[targets]]' >> examples/kernel-module/.out-of-tree.toml
        echo 'distro = { id = "${{ matrix.os.distro }}", release = "${{ matrix.os.release }}" }' >> examples/kernel-module/.out-of-tree.toml
        echo 'kernel = { regex = ".*" }' >> examples/kernel-module/.out-of-tree.toml
        echo '[qemu]' >> examples/kernel-module/.out-of-tree.toml
        echo 'timeout = "5m"' >> examples/kernel-module/.out-of-tree.toml
        echo 'after_start_timeout = "10s"' >> examples/kernel-module/.out-of-tree.toml

        echo 'modprobe uio || modprobe 9p || modprobe xfs' >> examples/kernel-module/test.sh

        scp -r examples/kernel-module root@$IP:test

        echo '[Unit]' >> test.service
        echo 'Description=e2e' >> test.service
        echo '[Service]' >> test.service
        echo 'RemainAfterExit=yes' >> test.service
        echo 'StandardError=append:/var/log/test.log' >> test.service
        echo 'StandardOutput=append:/var/log/test.log' >> test.service
        echo 'Type=oneshot' >> test.service
        echo 'WorkingDirectory=/root/test' >> test.service
        echo 'TimeoutStopSec=1' >> test.service
        echo 'ExecStart=/usr/local/bin/out-of-tree kernel --no-prebuilt-containers autogen --threads=8 --max=64 --shuffle' >> test.service
        echo 'ExecStart=/usr/local/bin/out-of-tree pew --threads=4 --include-internal-errors' >> test.service

        scp test.service root@$IP:/etc/systemd/system/test.service

        ssh root@$IP systemctl daemon-reload

        ssh root@$IP setenforce 0

        ssh root@$IP systemctl start test --no-block

        while ! ssh root@$IP systemctl show test -p SubState --value | grep -E '(failed|exited)'
        do
          sleep 30s
        done

        ssh root@$IP "cat /var/log/test.log"

        scp -r root@$IP:.out-of-tree/logs .

        ssh root@$IP systemctl is-active test || exit 1

        ssh root@$IP "/usr/local/bin/out-of-tree container save"
        ssh root@$IP "s3cmd put --acl-public *.tar.gz s3://out-of-tree/1.0.0/containers/ --host=fra1.digitaloceanspaces.com --host-bucket='%(bucket)s.fra1.digitaloceanspaces.com' --access_key=${{ secrets.DIGITALOCEAN_SPACES_ACCESS_KEY }} --secret_key=${{ secrets.DIGITALOCEAN_SPACES_SECRET_KEY }}"

    - name: Archive logs
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: test-end-to-end-${{ matrix.os.distro }}-${{ matrix.os.release }}-logs
        path: logs

    - name: Delete droplet
      if: always()
      run: doctl compute droplet delete -f ga-out-of-tree-e2e-${{ matrix.os.distro }}-${{ matrix.os.release }}-$GITHUB_SHA