Auto-generate tool list for CI

.github/.cspell/organization-dictionary.txt

@@ -151,6 +151,7 @@ endianness
 esac
 euxo
 gsub
+libc
 moreutils
 msys
 noninteractive

.github/.cspell/project-dictionary.txt

@@ -6,7 +6,7 @@ dprint
 enablerepo
 epel
 grcov
-libc
+incompat
 linkcheck
 mdbook
 microdnf

.github/workflows/ci.yml

@@ -38,25 +38,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os:
-          - ubuntu-20.04
-          - ubuntu-22.04
-        tool:
-          # cargo-watch/watchexec-cli is supported by cargo-binstall (through quickinstall)
-          # TODO: valgrind installation sometime hangs.
-          - cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,zola,syft
         include:
-          # Note: Specifying the version of valgrind and cargo-binstall is not supported.
           - os: ubuntu-20.04
-            tool: [email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected]
+          - os: ubuntu-22.04
           - os: ubuntu-20.04
-            tool: [email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected],[email protected]
+            tool: major.minor.patch
           - os: ubuntu-20.04
-            tool: cargo-valgrind@2, just@1,protoc@3 , shfmt@3 ,wasmtime@7,cargo-watch@8,watchexec-cli@1
+            tool: major.minor
+          - os: ubuntu-20.04
+            tool: major
           - os: macos-11
-            tool: cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,zola,syft
+          - os: macos-12
           - os: windows-2019
-            tool: cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,zola,syft
+          - os: windows-2022
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     steps:
@@ -65,9 +59,12 @@ jobs:
           persist-credentials: false
       # cross attempts to install rust-src when Cargo.toml is available even if `cross --version`
       - run: rm Cargo.toml
+      - name: Generate tool list
+        id: tool-list
+        run: tools/ci/tool-list.sh "${{ matrix.tool }}" >>"${GITHUB_OUTPUT}"
       - uses: ./
         with:
-          tool: ${{ matrix.tool }}
+          tool: ${{ steps.tool-list.outputs.tool }}
       # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell
       - name: Test bash
         run: just --version; shfmt --version; protoc --version
@@ -97,33 +94,17 @@ jobs:
       fail-fast: false
       matrix:
         container:
-          # glibc >= 2.31
+          - ubuntu:18.04 # glibc 2.27
           - ubuntu:20.04 # glibc 2.31
           - ubuntu:22.04 # glibc 2.35
+          - debian:10-slim # glibc 2.28
           - debian:11-slim # glibc 2.31
           - debian:12-slim # glibc 2.36
           - fedora:latest # glibc 2.37 (as of fedora 38)
-        tool:
-          # valgrind: installing snap to container is difficult...
-          - cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,zola,syft
-        include:
-          # glibc < 2.31
-          # zola don't provide prebuilt binaries for musl or old glibc host.
-          - container: ubuntu:18.04 # glibc 2.27
-            tool: cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,syft
-          - container: debian:10-slim # glibc 2.28
-            tool: cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,syft
-          - container: rockylinux:8 # glibc 2.28
-            tool: cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,syft
-          - container: rockylinux:8-minimal # glibc 2.28
-            tool: cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,syft
-          # glibc < 2.27 or musl
-          - container: centos:7 # glibc 2.17
-            # protoc,valgrind,wasmtime,mdbook-linkcheck,cargo-watch,zola don't provide prebuilt binaries for musl or old glibc host.
-            tool: cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,shellcheck,shfmt,wasm-pack,mdbook,cargo-binstall,grcov,watchexec-cli,cargo-tarpaulin,syft
-          - container: alpine:latest # musl 1.2.4 (as of alpine 3.18)
-            # protoc,valgrind,wasmtime,mdbook-linkcheck,cargo-watch,zola don't provide prebuilt binaries for musl host.
-            tool: cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,shellcheck,shfmt,wasm-pack,mdbook,cargo-binstall,grcov,watchexec-cli,cargo-tarpaulin,syft
+          - rockylinux:8 # glibc 2.28
+          - rockylinux:8-minimal # glibc 2.28
+          - centos:7 # glibc 2.17
+          - alpine:latest # musl 1.2.4 (as of alpine 3.18)
     runs-on: ubuntu-latest
     timeout-minutes: 60
     container: ${{ matrix.container }}
@@ -161,9 +142,12 @@ jobs:
           persist-credentials: false
       # cross attempts to install rust-src when Cargo.toml is available even if `cross --version`
       - run: rm Cargo.toml
+      - name: Generate tool list
+        id: tool-list
+        run: tools/ci/tool-list.sh >>"${GITHUB_OUTPUT}"
       - uses: ./
         with:
-          tool: ${{ matrix.tool }}
+          tool: ${{ steps.tool-list.outputs.tool }}
 
   manifest:
     runs-on: ubuntu-latest

DEVELOPMENT.md

@@ -21,13 +21,14 @@ See JSON files in `tools/codegen/base` directory for examples of the manifest.
 > GITHUB_TOKEN=$(gh auth status --show-token 2>&1 | sed -n 's/^.*Token: \(.*\)$/\1/p') ./tools/manifest.sh <tool>
 > ```
 
-3\. Add tool name to test matrix in `.github/workflows/ci.yml`.
-
-4\. Add tool name to the table in ["Supported tools" section in `README.md`](https://github.com/taiki-e/install-action#supported-tools).
+3\. Add tool name to the table in ["Supported tools" section in `README.md`](https://github.com/taiki-e/install-action#supported-tools).
 
 ## Troubleshooting
 
-If one of the CI builds fails due to an bin path or release asset_name, fix the problem in the base
+If one of the CI builds fails due to a bin path or release asset_name, fix the problem in the base
 manifest, and re-run the manifest tool `tools/manifest.sh` to regenerate the manifest json file. The
 base manifest supports overriding the bin path per platform by adding the `"bin"` / `"asset_name"`
 to the platform object.
+
+If CI fails only for containers using older versions of glibc or musl, you may need to add the tool
+name to one of the `*_incompat` arrays in `tools/ci/tool-list.sh`.

tools/ci/manifest.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: Apache-2.0 OR MIT
-set -euxo pipefail
+set -euo pipefail
 IFS=$'\n\t'
 cd "$(dirname "$0")"/../..
 
@@ -16,6 +16,8 @@ fi
 git config user.name "Taiki Endo"
 git config user.email "[email protected]"
 
+set -x
+
 for manifest in manifests/*.json; do
     git add -N "${manifest}"
     if ! git diff --exit-code -- "${manifest}"; then

tools/ci/tool-list.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+set -euo pipefail
+IFS=$'\n\t'
+cd "$(dirname "$0")"/../..
+
+# They don't provide prebuilt binaries for musl or old glibc host.
+glibc_pre_2_31_incompat=(
+    zola
+)
+glibc_pre_2_27_incompat=(
+    "${glibc_pre_2_31_incompat[@]}"
+    cargo-watch
+    mdbook-linkcheck
+    protoc
+    valgrind
+    wasmtime
+)
+musl_incompat=(
+    "${glibc_pre_2_27_incompat[@]}"
+)
+
+incompat_tools=()
+case "${1:-}" in
+    '') version=latest ;;
+    major.minor.patch | major.minor | major)
+        version="$1"
+        # Specifying the version of valgrind and cargo-binstall is not supported.
+        incompat_tools+=(valgrind cargo-binstall)
+        ;;
+    *)
+        echo "tool=$1"
+        exit 0
+        ;;
+esac
+case "$(uname -s)" in
+    Linux)
+        host_os=linux
+        ldd_version=$(ldd --version 2>&1 || true)
+        if grep <<<"${ldd_version}" -q 'musl'; then
+            incompat_tools+=("${musl_incompat[@]}")
+        else
+            host_glibc_version=$(grep <<<"${ldd_version}" -E "GLIBC|GNU libc" | sed "s/.* //g")
+            higher_glibc_version=$(sort <<<"2.31"$'\n'"${host_glibc_version}" -Vu | tail -1)
+            if [[ "${higher_glibc_version}" != "${host_glibc_version}" ]]; then
+                higher_glibc_version=$(sort <<<"2.27"$'\n'"${host_glibc_version}" -Vu | tail -1)
+                if [[ "${higher_glibc_version}" == "${host_glibc_version}" ]]; then
+                    incompat_tools+=("${glibc_pre_2_31_incompat[@]}")
+                else
+                    incompat_tools+=("${glibc_pre_2_27_incompat[@]}")
+                fi
+            fi
+        fi
+        if ! type -P snap &>/dev/null; then
+            incompat_tools+=(valgrind)
+        fi
+        ;;
+    Darwin) host_os=macos ;;
+    MINGW* | MSYS* | CYGWIN* | Windows_NT) host_os=windows ;;
+    *) bail "unrecognized OS type '$(uname -s)'" ;;
+esac
+
+tools=()
+for manifest in tools/codegen/base/*.json; do
+    tool_name=$(basename "${manifest%.*}")
+    case "${host_os}" in
+        linux*)
+            for incompat in ${incompat_tools[@]+"${incompat_tools[@]}"}; do
+                if [[ "${incompat}" == "${tool_name}" ]]; then
+                    tool_name=''
+                    break
+                fi
+            done
+            ;;
+        *)
+            if [[ "$(jq -r ".platform.x86_64_${host_os}" "${manifest}")" == "null" ]]; then
+                continue
+            fi
+            ;;
+    esac
+    if [[ -n "${tool_name}" ]]; then
+        if [[ "${version}" != "latest" ]]; then
+            latest_version="$(jq -r ".latest.version" "manifests/${tool_name}.json")"
+            case "${version}" in
+                major.minor.patch) tool_name+="@${latest_version}" ;;
+                major.minor) tool_name+="@${latest_version%.*}" ;;
+                major) tool_name+="@${latest_version%%.*}" ;;
+                *) exit 1 ;;
+            esac
+        fi
+        if [[ "${tool_name}" != *"@0" ]] && [[ "${tool_name}" != *"@0.0" ]]; then
+            tools+=("${tool_name}")
+        fi
+    fi
+done
+if [[ "${version}" != "latest" ]]; then
+    tools_tmp=()
+    for tool in "${tools[@]}"; do
+        tools_tmp+=("${tool}")
+    done
+    tools=("${tools_tmp[@]}")
+fi
+
+# Not manifest-based
+case "${host_os}" in
+    linux*)
+        # Installing snap to container is difficult...
+        # Specifying the version of valgrind is not supported.
+        if type -P snap &>/dev/null && [[ "${version}" == "latest" ]]; then
+            tools+=(valgrind)
+        fi
+        ;;
+esac
+# cargo-watch/watchexec-cli is supported by cargo-binstall (through quickinstall)
+case "${version}" in
+    latest) tools+=(cargo-watch watchexec-cli) ;;
+    major.minor.patch) tools+=([email protected] [email protected]) ;;
+    major.minor) tools+=([email protected] [email protected]) ;;
+    major) tools+=(cargo-watch@8 watchexec-cli@1) ;;
+    *) exit 1 ;;
+esac
+
+# sort and dedup
+IFS=$'\n'
+# shellcheck disable=SC2207
+tools=($(LC_ALL=C sort -u <<<"${tools[*]}"))
+IFS=$'\n\t'
+
+# TODO: inject random space before/after of tool name for testing https://github.com/taiki-e/install-action/issues/115.
+IFS=','
+echo "tool=${tools[*]}"
+IFS=$'\n\t'