ci: retry all setup commands that need network access, till they succeed (so the CI jobs have less false positives) (#21103)

.github/workflows/build_vinix_locally.sh

@@ -21,11 +21,11 @@ mkdir -p $BUILD
 
 cd $BUILD
 echo "Clone current Vinix"
-git clone https://github.com/vlang/vinix.git --depth=1
+.github/workflows/retry.sh git clone https://github.com/vlang/vinix.git --depth=1
 
 cd $BUILD
 echo "Clone current mlibc"
-git clone https://github.com/managarm/mlibc.git --depth=1
+.github/workflows/retry.sh git clone https://github.com/managarm/mlibc.git --depth=1
 
 cd $BUILD
 echo "Patch mlibc for Vinix"

.github/workflows/c2v_ci.yml

@@ -37,19 +37,19 @@ jobs:
         run: make && ./v symlink -githubci
       - name: Install C2V
         run: |
-          v install --git https://github.com/vlang/c2v
+          .github/workflows/retry.sh v install --git https://github.com/vlang/c2v
           v -g ~/.vmodules/c2v/ || true
       - name: Install dependencies
         run: |
           if [ "${{ runner.os }}" == "Linux" ]; then
-            sudo apt update -y -qq
-            sudo apt install libsdl2-dev libsdl2-mixer-dev libsdl2-net-dev libpng-dev libsamplerate0-dev
+            .github/workflows/retry.sh sudo apt update -y -qq
+            .github/workflows/retry.sh sudo apt install libsdl2-dev libsdl2-mixer-dev libsdl2-net-dev libpng-dev libsamplerate0-dev
           else
-            brew install sdl2 sdl2_mixer sdl2_net libpng libsamplerate
+            .github/workflows/retry.sh brew install sdl2 sdl2_mixer sdl2_net libpng libsamplerate
           fi
       - name: Build original Chocolate Doom
         run: |
-          git clone --quiet --depth 1 https://github.com/vlang/doom ~/code/doom
+          .github/workflows/retry.sh git clone --quiet --depth 1 https://github.com/vlang/doom ~/code/doom
           cd ~/code/doom/chocolate-doom
           cmake -DCMAKE_BUILD_TYPE=Debug .
           make chocolate-doom
@@ -73,12 +73,12 @@ jobs:
         run: make && ./v symlink -githubci
       - name: Install C2V
         run: |
-          v install --git https://github.com/vlang/c2v
+          .github/workflows/retry.sh v install --git https://github.com/vlang/c2v
           v -g ~/.vmodules/c2v/ || true
       - name: Install dependencies
         run: |
-          sudo apt update -y -qq
-          sudo apt install libsdl2-dev libsdl2-mixer-dev libsdl2-net-dev libpng-dev libsamplerate0-dev
+          .github/workflows/retry.sh sudo apt update -y -qq
+          .github/workflows/retry.sh sudo apt install libsdl2-dev libsdl2-mixer-dev libsdl2-net-dev libpng-dev libsamplerate0-dev
           # c2v / DOOM dependencies
           # vgret dependencies
           # imagemagick              : convert, mogrify, import
@@ -86,19 +86,19 @@ jobs:
           # openimageio-tools        : idiff
           # libgl1-mesa-dri          : For headless rendering / software DRI driver (LIBGL_ALWAYS_SOFTWARE=true)
           # freeglut3-dev            : Fixes graphic apps compilation with tcc
-          sudo apt install imagemagick openimageio-tools freeglut3-dev libgl1-mesa-dri xvfb xsel xclip
+          .github/workflows/retry.sh sudo apt install imagemagick openimageio-tools freeglut3-dev libgl1-mesa-dri xvfb xsel xclip
       - name: Setup test tools
         run: |
           # Fetch the free ~4MB DOOM1.WAD from the link at https://doomwiki.org/wiki/DOOM1.WAD
-          wget https://distro.ibiblio.org/slitaz/sources/packages/d/doom1.wad -O ~/doom1.wad
+          .github/workflows/retry.sh wget https://distro.ibiblio.org/slitaz/sources/packages/d/doom1.wad -O ~/doom1.wad
           # Get imgur upload script
-          wget https://raw.githubusercontent.com/tremby/imgur.sh/c98345d/imgur.sh
+          .github/workflows/retry.sh wget https://raw.githubusercontent.com/tremby/imgur.sh/c98345d/imgur.sh
           chmod +x ./imgur.sh
           # Get regression images to test against
-          git clone https://github.com/Larpon/doom-regression-images
+          .github/workflows/retry.sh git clone https://github.com/Larpon/doom-regression-images
       - name: Build original Chocolate Doom
         run: |
-          git clone --quiet --depth 1 https://github.com/vlang/doom ~/code/doom
+          .github/workflows/retry.sh git clone --quiet --depth 1 https://github.com/vlang/doom ~/code/doom
           cd ~/code/doom/chocolate-doom
           cmake -DCMAKE_BUILD_TYPE=Debug .
           make chocolate-doom

.github/workflows/compile_v_with_vtcc.sh

@@ -10,7 +10,7 @@ show "Prepare"
 rm -rf vtcc/
 
 show "Clone vtcc"
-git clone https://github.com/felipensp/vtcc --branch stable --quiet vtcc/
+.github/workflows/retry.sh git clone https://github.com/felipensp/vtcc --branch stable --quiet vtcc/
 du -s vtcc/
 ## TODO: just `./v vtcc`, later will cause V, to detect the compiler as tcc (which it is), and add `-fwrapv`, which causes the vtcc compiler to panic currently
 show "Compile vtcc"

.github/workflows/cross_ci.yml

@@ -23,7 +23,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          brew install mingw-w64
+          .github/workflows/retry.sh brew install mingw-w64
           export LIBRARY_PATH="$LIBRARY_PATH:/usr/local/opt/openssl/lib/"
 
       - name: Build V
@@ -55,10 +55,10 @@ jobs:
       - name: Install dependencies
         run: |
           ## sudo dpkg --add-architecture i386
-          sudo apt update
-          sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev
-          sudo apt-get install --quiet -y mingw-w64 wine-stable winetricks
-          ## sudo apt-get install --quiet -y wine32
+          .github/workflows/retry.sh sudo apt update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev
+          .github/workflows/retry.sh sudo apt-get install --quiet -y mingw-w64 wine-stable winetricks
+          ## .github/workflows/retry.sh sudo apt-get install --quiet -y wine32
 
       - name: Turn off the wine crash dialog
         run: winetricks nocrashdialog

.github/workflows/freebsd_build_tcc.sh

@@ -7,7 +7,7 @@ rm -rf thirdparty/tcc/
 
 pushd .
 
-git clone git://repo.or.cz/tinycc.git
+.github/workflows/retry.sh git clone git://repo.or.cz/tinycc.git
 cd tinycc
 
 export CC=clang

.github/workflows/gen_vc_ci.yml

@@ -26,7 +26,7 @@ jobs:
           COMMIT_MSG=$(git log -1 --oneline --pretty='%s' HEAD)
 
           rm -rf vc
-          git clone --depth=1 \
+          .github/workflows/retry.sh git clone --depth=1 \
             https://vlang-bot:${{ secrets.VLANG_BOT_SECRET }}@$VREPO
 
           rm -rf vc/v.c vc/v_win.c
@@ -45,7 +45,7 @@ jobs:
           git -C vc commit -m "[v:master] $COMMIT_HASH - $COMMIT_MSG"
 
           # in case there are recent commits:
-          git -C vc pull --rebase origin master
+          .github/workflows/retry.sh git -C vc pull --rebase origin master
           # Note that failure below may happen, due to vlang/vc rejecting the push
           # from forked repos; that is not usually a problem.
           git -C vc push || true 

.github/workflows/gg_regressions_ci.yml

@@ -33,10 +33,10 @@ jobs:
           # libxcursor-dev libxi-dev : V gfx deps
           # libgl1-mesa-dri          : For headless rendering / software DRI driver (LIBGL_ALWAYS_SOFTWARE=true)
           # freeglut3-dev            : Fixes graphic apps compilation with tcc
-          sudo apt-get update
-          sudo apt-get install imagemagick openimageio-tools libgl1-mesa-dri xvfb libxcursor-dev libxi-dev freeglut3-dev xsel xclip
-          wget https://raw.githubusercontent.com/tremby/imgur.sh/c98345d/imgur.sh
-          git clone https://github.com/Larpon/gg-regression-images gg-regression-images
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install imagemagick openimageio-tools libgl1-mesa-dri xvfb libxcursor-dev libxi-dev freeglut3-dev xsel xclip
+          .github/workflows/retry.sh wget https://raw.githubusercontent.com/tremby/imgur.sh/c98345d/imgur.sh
+          .github/workflows/retry.sh git clone https://github.com/Larpon/gg-regression-images gg-regression-images
           chmod +x ./imgur.sh
 
       - name: Sample and compare

.github/workflows/linux_ci.yml

@@ -14,14 +14,14 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
           # The following is needed for examples/wkhtmltopdf.v
-          wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
-          sudo apt-get install --quiet -y xfonts-75dpi xfonts-base
-          sudo apt-get install --quiet -y expect
-          sudo dpkg -i wkhtmltox_0.12.6-1.focal_amd64.deb
+          .github/workflows/retry.sh wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
+          .github/workflows/retry.sh sudo apt-get install --quiet -y xfonts-75dpi xfonts-base
+          .github/workflows/retry.sh sudo apt-get install --quiet -y expect
+          .github/workflows/retry.sh sudo dpkg -i wkhtmltox_0.12.6-1.focal_amd64.deb
       - name: Build v
         run: |
           echo $VFLAGS
@@ -94,14 +94,14 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
-          sudo apt-get install --quiet -y libgc-dev
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libgc-dev
           # The following is needed for examples/wkhtmltopdf.v
-          wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
-          sudo apt-get install --quiet -y xfonts-75dpi xfonts-base
-          sudo dpkg -i wkhtmltox_0.12.6-1.focal_amd64.deb
+          .github/workflows/retry.sh wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
+          .github/workflows/retry.sh sudo apt-get install --quiet -y xfonts-75dpi xfonts-base
+          .github/workflows/retry.sh sudo dpkg -i wkhtmltox_0.12.6-1.focal_amd64.deb
       - name: Build v
         run: |
           echo $VFLAGS
@@ -149,9 +149,9 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
       - name: Build V
         run: make -j4 && ./v -cc gcc -cg -cstrict -o v cmd/v
       - name: Valgrind v.c
@@ -169,9 +169,9 @@ jobs:
       # - name: Test v->js
       #   run: ./v -o hi.js examples/hello_v_js.v && node hi.js
       # - name: Build Vorum
-      #   run: git clone --depth 1 https://github.com/vlang/vorum && cd vorum && ../v . && cd ..
+      #   run: .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/vorum && cd vorum && ../v . && cd ..
       - name: Build vpm
-        run: ./v install markdown && git clone --depth 1 https://github.com/vlang/vpm && cd vpm && ../v . || ../v cmd/vpm && cd ..
+        run: .github/workflows/retry.sh ./v install markdown && .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/vpm && cd vpm && ../v . || ../v cmd/vpm && cd ..
       - name: Freestanding
         run: ./v -freestanding run vlib/os/bare/bare_example_linux.v
       - name: V self compilation
@@ -259,10 +259,10 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
-          sudo apt-get install --quiet -y clang
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get install --quiet -y clang
       - name: Build V
         run: make -j4 && ./v -cc clang -cg -cstrict -o v cmd/v
       - name: Valgrind
@@ -363,7 +363,7 @@ jobs:
   #   - uses: actions/checkout@v4
   #   - name: Install dependencies
   #     run: |
-  #        sudo apt-get install --quiet -y musl musl-tools libssl-dev sqlite3 libsqlite3-dev valgrind
+  #        .github/workflows/retry.sh sudo apt-get install --quiet -y musl musl-tools libssl-dev sqlite3 libsqlite3-dev valgrind
   #   - name: Build v
   #     run: echo $VFLAGS && make -j4 && ./v -cg -o v cmd/v
   #   # - name: Test v binaries

.github/workflows/macos_ci.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Install dependencies
         run: |
           echo "PKG_CONFIG_PATH is '$PKG_CONFIG_PATH'"
-          brew install libpq openssl mercurial
+          .github/workflows/retry.sh brew install libpq openssl mercurial
           export LIBRARY_PATH="$LIBRARY_PATH:/usr/local/opt/openssl/lib/"
           echo "LIBRARY_PATH is '$LIBRARY_PATH'"
       - name: Build V
@@ -30,7 +30,7 @@ jobs:
       - name: Test symlink
         run: ./v symlink
       - name: Build vpm
-        run: ./v install markdown && git clone --depth 1 https://github.com/vlang/vpm && cd vpm && ../v . || ../v cmd/vpm && cd ..
+        run: .github/workflows/retry.sh ./v install markdown && .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/vpm && cd vpm && ../v . || ../v cmd/vpm && cd ..
       # - name: Set up pg database
       #   run: |
       #     pg_ctl -D /usr/local/var/postgres -l /usr/local/var/postgres/server.log start
@@ -68,14 +68,14 @@ jobs:
           ./v doctor
       - name: Test ved
         run: |
-          git clone --depth 1 https://github.com/vlang/ved
+          .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/ved
           cd ved && ../v -o ved .
           ../v -autofree .
           ../v -prod .
           cd ..
       - name: Build V UI examples
         run: |
-          git clone --depth 1 https://github.com/vlang/ui
+          .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/ui
           cd ui
           mkdir -p ~/.vmodules
           ln -s $(pwd) ~/.vmodules/ui

.github/workflows/native_backend_tests_ci.yml

@@ -63,7 +63,7 @@ jobs:
       - name: Install linker
         if: ${{ startsWith(matrix.os, 'ubuntu')}}
         run: |
-          sudo apt-get install --quiet -y binutils
+          .github/workflows/retry.sh sudo apt-get install --quiet -y binutils
 
       - name: Build V with make.bat
         if: ${{ startsWith(matrix.os, 'windows') }}

.github/workflows/other_ci.yml

@@ -54,7 +54,7 @@ jobs:
         run: make -j4
       - name: Clone & Build previous vmaster/v
         run: |
-          git clone --depth=1 https://github.com/vlang/v vmaster/
+          .github/workflows/retry.sh git clone --depth=1 https://github.com/vlang/v vmaster/
           (cd vmaster; make -j4)
       - name: V versions
         run: ./v version && ./vmaster/v version
@@ -78,10 +78,10 @@ jobs:
 
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y libsodium-dev libssl-dev sqlite3 libsqlite3-dev postgresql libpq-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev libxi-dev libxcursor-dev libgl-dev xfonts-75dpi xfonts-base
-          sudo apt-get install --quiet -y g++-9 g++-11
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libsodium-dev libssl-dev sqlite3 libsqlite3-dev postgresql libpq-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev libxi-dev libxcursor-dev libgl-dev xfonts-75dpi xfonts-base
+          .github/workflows/retry.sh sudo apt-get install --quiet -y g++-9 g++-11
 
       - name: Build v
         run: make
@@ -112,7 +112,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install zzuf
-        run: sudo apt install -qq zzuf
+        run: .github/workflows/retry.sh sudo apt install -qq zzuf
       - name: Build local v
         run: |
           make -j4

.github/workflows/periodic_ci.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install dependencies 1
-        run: sudo apt-get install --quiet -y sqlite3 libsqlite3-dev
+        run: .github/workflows/retry.sh sudo apt-get install --quiet -y sqlite3 libsqlite3-dev
       - name: Build v
         run: make
       - name: Symlink V

.github/workflows/retry.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+while ! $@; do "command failed, retrying ..."; sleep 1; done
+

.github/workflows/sanitized_ci.yml

@@ -86,10 +86,10 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
-          sudo apt-get install clang
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get install clang
       - name: Build V
         run: make && ./v -cg -cstrict -o v cmd/v
       - name: Ensure code is well formatted
@@ -111,9 +111,9 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
       - name: Build V
         run: make && ./v -cg -cstrict -o v cmd/v
       - name: Ensure code is well formatted
@@ -135,10 +135,10 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
-          sudo apt-get install clang
+          .github/workflows/retry.sh sudo apt update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get install clang
       - name: Build V
         run: make && ./v -cg -cstrict -o v cmd/v
       - name: Ensure code is well formatted
@@ -193,10 +193,10 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
-          sudo apt-get install clang
+          .github/workflows/retry.sh sudo apt update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get install clang
       - name: Build V
         run: make && ./v -cg -cstrict -o v cmd/v
       - name: Ensure code is well formatted
@@ -222,10 +222,10 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
-          sudo apt-get install clang
+          .github/workflows/retry.sh sudo apt update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get install clang
       - name: Build V
         run: make && ./v -cc clang -cg -cstrict -o v cmd/v
       - name: Ensure code is well formatted

.github/workflows/sdl_ci.yml

@@ -28,16 +28,16 @@ jobs:
 
       - name: Clone sdl into .vmodules
         run: |
-          git clone --depth 1 https://github.com/vlang/sdl
+          .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/sdl
           cd sdl
           mkdir -p ~/.vmodules
           ln -s $(pwd) ~/.vmodules/sdl
 
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y libsdl2-dev libsdl2-ttf-dev
-          sudo apt-get install --quiet -y libsdl2-mixer-dev libsdl2-image-dev
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libsdl2-dev libsdl2-ttf-dev
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libsdl2-mixer-dev libsdl2-image-dev
 
       - name: Run tests
         run: ./v test sdl

.github/workflows/toml_ci.yml

@@ -27,8 +27,8 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y jq libgc-dev
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y jq libgc-dev
       - name: Build V
         run: make -j2 && ./v -cc gcc -o v cmd/v
 
@@ -41,15 +41,15 @@ jobs:
         # Tests found at https://github.com/BurntSushi/toml-test
       - name: Clone BurntSushi/toml-test
         run: |
-          git clone -n https://github.com/BurntSushi/toml-test.git $TOML_BS_TESTS_PATH
-          git -C $TOML_BS_TESTS_PATH checkout $TOML_BS_TESTS_PINNED_COMMIT
+          .github/workflows/retry.sh git clone -n https://github.com/BurntSushi/toml-test.git $TOML_BS_TESTS_PATH
+          .github/workflows/retry.sh git -C $TOML_BS_TESTS_PATH checkout $TOML_BS_TESTS_PINNED_COMMIT
 
       - name: Run BurntSushi TOML tests
         run: ./v vlib/toml/tests/burntsushi_toml_test.v
 
         # Tests found at gist
       - name: Get large_toml_file_test.toml
-        run: wget https://gist.githubusercontent.com/Larpon/89b0e3d94c6903851ff15559e5df7a05/raw/62a1f87a4e37bf157f2e0bfb32d85d840c98e422/large_toml_file_test.toml -O vlib/toml/tests/testdata/large_toml_file_test.toml
+        run: .github/workflows/retry.sh wget https://gist.githubusercontent.com/Larpon/89b0e3d94c6903851ff15559e5df7a05/raw/62a1f87a4e37bf157f2e0bfb32d85d840c98e422/large_toml_file_test.toml -O vlib/toml/tests/testdata/large_toml_file_test.toml
 
       - name: Run large TOML file tests
         run: ./v vlib/toml/tests/large_toml_file_test.v
@@ -57,7 +57,7 @@ jobs:
         # Tests found at https://github.com/iarna/toml-spec-tests
       - name: Clone iarna/toml-spec-tests
         run: |
-          git clone -n https://github.com/iarna/toml-spec-tests.git $TOML_IARNA_TESTS_PATH
+          .github/workflows/retry.sh git clone -n https://github.com/iarna/toml-spec-tests.git $TOML_IARNA_TESTS_PATH
           git -C $TOML_IARNA_TESTS_PATH checkout $TOML_IARNA_TESTS_PINNED_COMMIT
 
       - name: Run iarna TOML tests
@@ -66,7 +66,7 @@ jobs:
         # Tests found at https://github.com/alexcrichton/toml-rs
       - name: Clone alexcrichton/toml-rs
         run: |
-          git clone -n https://github.com/alexcrichton/toml-rs.git $TOML_AC_TESTS_PATH
+          .github/workflows/retry.sh git clone -n https://github.com/alexcrichton/toml-rs.git $TOML_AC_TESTS_PATH
           git -C $TOML_AC_TESTS_PATH checkout $TOML_AC_TESTS_PINNED_COMMIT
 
       - name: Run alexcrichton TOML tests

.github/workflows/v_apps_and_modules_compile_ci.yml

@@ -23,9 +23,9 @@ jobs:
 
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y libgc-dev libsodium-dev libssl-dev sqlite3 libsqlite3-dev libfreetype6-dev libxi-dev libxcursor-dev libgl-dev xfonts-75dpi xfonts-base
-          sudo apt-get install --quiet -y --no-install-recommends sassc libgit2-dev ## needed by gitly
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libgc-dev libsodium-dev libssl-dev sqlite3 libsqlite3-dev libfreetype6-dev libxi-dev libxcursor-dev libgl-dev xfonts-75dpi xfonts-base
+          .github/workflows/retry.sh sudo apt-get install --quiet -y --no-install-recommends sassc libgit2-dev ## needed by gitly
 
       - name: Test vtcc
         run: .github/workflows/compile_v_with_vtcc.sh
@@ -33,7 +33,7 @@ jobs:
       - name: Test vsql compilation and examples
         run: |
           echo "Install vsql"
-          v install elliotchance.vsql ; cd ~/.vmodules/elliotchance/vsql
+          .github/workflows/retry.sh v install elliotchance.vsql ; cd ~/.vmodules/elliotchance/vsql
           echo "Generate vsql/grammar.v"
           make vsql/grammar.v
           echo "Compile vsql"
@@ -46,7 +46,7 @@ jobs:
       - name: Test discord.v
         run: |
           echo "Clone https://github.com/DarpHome/discord.v/"
-          git clone https://github.com/DarpHome/discord.v/ discord && cd discord
+          .github/workflows/retry.sh git clone https://github.com/DarpHome/discord.v/ discord && cd discord
           echo "Checkout last known good commit"
           git checkout 65448b687ae759e385c127c8739b97b6ac62d3e3
           echo "Execute Tests"
@@ -55,7 +55,7 @@ jobs:
       - name: Build vlang/vab
         run: |
           echo "Install VAB"
-          v install vab
+          .github/workflows/retry.sh v install vab
           echo "Build vab"
           v ~/.vmodules/vab
           echo "Build vab with -gc boehm -skip-unused"
@@ -63,7 +63,7 @@ jobs:
 
       - name: Build vlang/ved
         run: |
-          git clone --depth 1 https://github.com/vlang/ved
+          .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/ved
           cd ved && ../v -o ved .
           ../v -autofree .
           ../v -prod .
@@ -71,32 +71,32 @@ jobs:
 
       - name: Build vlang/pdf
         run: |
-          v install pdf
+          .github/workflows/retry.sh v install pdf
           echo "PDF examples should compile"
           v should-compile-all ~/.vmodules/pdf/examples
 
       - name: Build vlang/libsodium
         run: |
           echo "Install the libsodium wrapper"
-          v install libsodium
+          .github/workflows/retry.sh v install libsodium
           echo "Test libsodium"
           VJOBS=1 v test ~/.vmodules/libsodium
 
       - name: Build vlang/coreutils
         run: |
           echo "Clone Coreutils"
-          git clone --depth 1 https://github.com/vlang/coreutils /tmp/coreutils
+          .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/coreutils /tmp/coreutils
           echo "Build Coreutils"
           cd /tmp/coreutils; make
 
       - name: Build vlang/gitly
         run: |
           echo "Install markdown"
-          v install markdown
+          .github/workflows/retry.sh v install markdown
           echo "Install pcre"
-          v install pcre
+          .github/workflows/retry.sh v install pcre
           echo "Clone Gitly"
-          git clone https://github.com/vlang/gitly /tmp/gitly
+          .github/workflows/retry.sh git clone https://github.com/vlang/gitly /tmp/gitly
           echo "Build Gitly"
           v -cc gcc /tmp/gitly
           ## echo "Build Gitly with -autofree"
@@ -110,7 +110,7 @@ jobs:
       - name: Build V Language Server (v-analyzer) vlang/v-analyzer
         run: |
           echo "Clone v-analyzer"
-          git clone --depth=1 --filter=blob:none --recursive --shallow-submodules https://github.com/vlang/v-analyzer /tmp/v-analyzer
+          .github/workflows/retry.sh git clone --depth=1 --filter=blob:none --recursive --shallow-submodules https://github.com/vlang/v-analyzer /tmp/v-analyzer
           cd /tmp/v-analyzer
           echo "Installing dependencies"
           v install
@@ -122,7 +122,7 @@ jobs:
       - name: Build vlang/go2v
         run: |
           echo "Clone Go2V"
-          git clone --depth=1 https://github.com/vlang/go2v /tmp/go2v/
+          .github/workflows/retry.sh git clone --depth=1 https://github.com/vlang/go2v /tmp/go2v/
           echo "Build Go2V"
           v /tmp/go2v/
           echo "Run Go2V tests"
@@ -131,23 +131,23 @@ jobs:
       - name: Install UI through VPM and make sure its examples compile
         run: |
           echo "Official VPM modules should be installable"
-          v install ui
+          .github/workflows/retry.sh v install ui
           echo "Examples of UI should compile"
           v ~/.vmodules/ui/examples/build_examples.vsh
 
       - name: Build vlang/adventofcode
         run: |
           echo "Clone the AdventOfCode repo"
-          git clone --depth 1 https://github.com/vlang/adventofcode  /tmp/adventofcode
+          .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/adventofcode  /tmp/adventofcode
           echo "Install dependencies"
-          v install pcre
+          .github/workflows/retry.sh v install pcre
           echo "Execute Tests"
           cd /tmp/adventofcode && v run verify.v
 
       - name: Build vlang/msgpack
         run: |
           echo "Install msgpack"
-          v install msgpack
+          .github/workflows/retry.sh v install msgpack
           echo "Build msgpack"
           v -shared ~/.vmodules/msgpack/
           echo "Run msgpack tests"
@@ -159,7 +159,7 @@ jobs:
 #      - name: Build VEX
 #        run: |
 #          echo "Install Vex"
-#          v install nedpals.vex
+#          .github/workflows/retry.sh v install nedpals.vex
 #          echo "Compile all of the Vex examples"
 #          v should-compile-all ~/.vmodules/nedpals/vex/examples
 #          echo "Compile the simple Vex example with -skip-unused"
@@ -184,15 +184,15 @@ jobs:
 
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y libgc-dev   libsodium-dev libssl-dev sqlite3 libsqlite3-dev libfreetype6-dev libxi-dev libxcursor-dev libgl-dev xfonts-75dpi xfonts-base
-          sudo apt-get install --quiet -y --no-install-recommends gfortran liblapacke-dev libopenblas-dev ## for vsl/vtl
-          sudo apt-get install --quiet -y --no-install-recommends libhdf5-cpp-103 libhdf5-dev libhdf5-mpi-dev hdf5-tools libopenmpi-dev opencl-headers liblapacke-dev libopenblas-dev ## needed by VSL
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libgc-dev   libsodium-dev libssl-dev sqlite3 libsqlite3-dev libfreetype6-dev libxi-dev libxcursor-dev libgl-dev xfonts-75dpi xfonts-base
+          .github/workflows/retry.sh sudo apt-get install --quiet -y --no-install-recommends gfortran liblapacke-dev libopenblas-dev ## for vsl/vtl
+          .github/workflows/retry.sh sudo apt-get install --quiet -y --no-install-recommends libhdf5-cpp-103 libhdf5-dev libhdf5-mpi-dev hdf5-tools libopenmpi-dev opencl-headers liblapacke-dev libopenblas-dev ## needed by VSL
 
       - name: Build vlang/vsl
         run: |
           echo "Installing dependencies"
-          sudo apt-get install --quiet -y --no-install-recommends \
+          .github/workflows/retry.sh sudo apt-get install --quiet -y --no-install-recommends \
             gfortran \
             libxi-dev \
             libxcursor-dev \
@@ -204,7 +204,7 @@ jobs:
             libopenmpi-dev \
             opencl-headers
           echo "Install VSL"
-          v install vsl
+          .github/workflows/retry.sh v install vsl
           echo "Execute Tests using Pure V Backend"
           ~/.vmodules/vsl/bin/test
           echo "Execute Tests using Pure V Backend with Pure V Math"
@@ -217,7 +217,7 @@ jobs:
       - name: Build vlang/vtl
         run: |
           echo "Install VTL"
-          v install vtl
+          .github/workflows/retry.sh v install vtl
           echo "Install dependencies"
           echo "Execute Tests using Pure V Backend"
           ~/.vmodules/vtl/bin/test

.github/workflows/vab_ci.yml

@@ -28,7 +28,7 @@ jobs:
 
       - name: Install vab
         run: |
-          v install vab
+          .github/workflows/retry.sh v install vab
           v -g ~/.vmodules/vab
           sudo ln -s ~/.vmodules/vab/vab /usr/local/bin/vab
 
@@ -64,7 +64,7 @@ jobs:
 
       - name: Install vab
         run: |
-          v install vab
+          .github/workflows/retry.sh v install vab
           v -g ~/.vmodules/vab
           sudo ln -s ~/.vmodules/vab/vab /usr/local/bin/vab
 

.github/workflows/vinix_ci.yml

@@ -25,14 +25,14 @@ jobs:
 
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install build-essential meson -y
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install build-essential meson -y
 
       - name: Build V
         run: make
 
       - name: Clone current Vinix
-        run: git clone https://github.com/vlang/vinix.git
+        run: .github/workflows/retry.sh git clone https://github.com/vlang/vinix.git
 
       - name: Download Vinix kernel dependencies
         run: cd vinix/kernel && ./get-deps

.github/workflows/websockets_ci.yml

@@ -18,7 +18,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install dependencies
-        run: sudo apt-get install --quiet -y libssl-dev
+        run: .github/workflows/retry.sh sudo apt-get install --quiet -y libssl-dev
       - name: Build v
         run: |
           echo $VFLAGS