From 0d1173ff5e1b6da152edde2ace3db54c6365ef5b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 6 Nov 2024 01:55:28 -0500 Subject: [PATCH 1/7] Add Windows CI job with `GIX_TEST_IGNORE_ARCHIVES=1` Some of these fail. We report the step as failing if any fail, which currently at least 14 and usually 15 are expected to (one of them is a performance test). If more than 15 fail, we'll fail this job, which will fail the workflow overall, but the job is still deliberately not treated as a required check for PR auto-merge. Right now, the expected number of failures is delibreately set too low, to 13, which is unlikely to be satisfied. This is just to test the new job. After verifying that it can trigger job failure when an excessive number of test case failures occurs, and fixing any readily apparent bugs in the job definition, this value will be raised, probably to 15. See #1358 for information on the known-failing tests on Windows with `GIX_TEST_IGNORE_ARCHIVES=1`. --- .github/workflows/ci.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5d04efc9a9..1eca14bc9a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,6 +80,37 @@ jobs: - name: Check that tracked archives are up to date run: git diff --exit-code # If this fails, the fix is usually to commit a regenerated archive. + test-fixtures-windows: + runs-on: windows-latest + env: + # FIXME: Change to 15 after verifying that the last step can fail. + # See https://github.com/GitoxideLabs/gitoxide/issues/1358. + EXPECTED_FAILURE_COUNT: 13 + defaults: + run: + shell: bash # Includes `-o pipefail`. + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/install-action@v2 + with: + tool: nextest + - name: "Test (nextest)" + env: + GIX_TEST_IGNORE_ARCHIVES: 1 + run: | + cargo nextest --color=always run --workspace --no-fail-fast |& tee nextest.log + continue-on-error: true + - name: Allow up to ${{ env.EXPECTED_FAILURE_COUNT }} failures + if: failure() + run: | + set -x + sed -Ei 's/\x1B\[[[:digit:];]*m//g' nextest.log # Remove ANSI color codes. + pattern='\n-{10,}\r?\n[ \t]+Summary\b[^\n]+[ \t]\K\d+(?= failed\b)' + count="$(grep -zoP "$pattern" nextest.log)" + ((count <= EXPECTED_FAILURE_COUNT)) + test-32bit: runs-on: ubuntu-latest strategy: From 8bf4517bf8c487977b3ea63093dc71cd8083c11a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 6 Nov 2024 02:28:23 -0500 Subject: [PATCH 2/7] Fix various problems in the new job - Check the specific step outcome (and specifically the outcome, not the conclusion which is turned into `success` by the value of `continue-on-error`). Without this, the last step never runs, so the test does not fail when it is supposed to. - Give the last step a fixed (literal) name, since the `env` key does not interpolate into its name like a `matrix` value would. Accordingly, move the `env` key into the step, since it is now only used inside its logic. - Line-buffer standard output for the `cargo nextest` command for more immedate (less staggered) output. This may not happen automatically since its stdout is a pipe. (I think its stderr is already either unbuffered or line-buffered even when not a terminal.) --- .github/workflows/ci.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1eca14bc9a6..f79d743ada1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,10 +82,6 @@ jobs: test-fixtures-windows: runs-on: windows-latest - env: - # FIXME: Change to 15 after verifying that the last step can fail. - # See https://github.com/GitoxideLabs/gitoxide/issues/1358. - EXPECTED_FAILURE_COUNT: 13 defaults: run: shell: bash # Includes `-o pipefail`. @@ -97,13 +93,19 @@ jobs: with: tool: nextest - name: "Test (nextest)" + id: nextest env: GIX_TEST_IGNORE_ARCHIVES: 1 run: | - cargo nextest --color=always run --workspace --no-fail-fast |& tee nextest.log + stdbuf -oL -- cargo nextest --color=always run --workspace --no-fail-fast |& + tee nextest.log continue-on-error: true - - name: Allow up to ${{ env.EXPECTED_FAILURE_COUNT }} failures - if: failure() + - name: Check how many tests failed + if: steps.nextest.outcome == 'failure' + env: + # FIXME: Change to 15 after verifying that the last step can fail. + # See https://github.com/GitoxideLabs/gitoxide/issues/1358. + EXPECTED_FAILURE_COUNT: 13 run: | set -x sed -Ei 's/\x1B\[[[:digit:];]*m//g' nextest.log # Remove ANSI color codes. From 5df05764490a525ff7b20926ff2c5c2c3ebbaac4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 6 Nov 2024 04:44:54 -0500 Subject: [PATCH 3/7] Set LC_ALL=C to simplify parsing and let grep -P work --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f79d743ada1..0a19e371b60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,6 +103,7 @@ jobs: - name: Check how many tests failed if: steps.nextest.outcome == 'failure' env: + LC_ALL: C # FIXME: Change to 15 after verifying that the last step can fail. # See https://github.com/GitoxideLabs/gitoxide/issues/1358. EXPECTED_FAILURE_COUNT: 13 From f0f8534a5887a597e63418d42fdd95c4ee3e3d83 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 6 Nov 2024 13:45:44 -0500 Subject: [PATCH 4/7] Set LC_ALL=C.utf8, since LC_ALL=C doesn't make grep -P work --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a19e371b60..3d441a166cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,7 +103,7 @@ jobs: - name: Check how many tests failed if: steps.nextest.outcome == 'failure' env: - LC_ALL: C + LC_ALL: C.utf8 # FIXME: Change to 15 after verifying that the last step can fail. # See https://github.com/GitoxideLabs/gitoxide/issues/1358. EXPECTED_FAILURE_COUNT: 13 From 6b6436bea45ce26abc432e0c598a21593bfa9d55 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 6 Nov 2024 15:10:25 -0500 Subject: [PATCH 5/7] Stop explicitly line-buffering standard output This removes `stdbuf -oL`. I had added the `stdbuf -oL` along with some other changes, but it seems the most likely of any of them to be producing the numerous new failures reporting "error while loading shared libraries: C: cannot open shared object file" (treating `C:` as though it is itself the name of a DLL to load) such as: FAIL [ 0.028s] gix-filter::filter pipeline::convert_to_git::only_driver_means_streaming_is_possible --- STDOUT: gix-filter::filter pipeline::convert_to_git::only_driver_means_streaming_is_possible --- running 1 test test pipeline::convert_to_git::only_driver_means_streaming_is_possible ... FAILED failures: failures: pipeline::convert_to_git::only_driver_means_streaming_is_possible test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 47 filtered out; finished in 0.02s --- STDERR: gix-filter::filter pipeline::convert_to_git::only_driver_means_streaming_is_possible --- failed to extract 'tests\fixtures\generated-archives\pipeline_repos.tar': Ignoring archive at 'tests\fixtures\generated-archives\pipeline_repos.tar' as GIX_TEST_IGNORE_ARCHIVES is set. thread 'pipeline::convert_to_git::only_driver_means_streaming_is_possible' panicked at tests\tools\src\lib.rs:567:17: fixture script of "bash" "D:\\a\\gitoxide\\gitoxide\\gix-filter\\tests\\fixtures\\pipeline_repos.sh" failed: stdout: stderr: 0 [main] bash 547 C:\Program Files\Git\usr\bin\bash.exe: *** fatal error - error while loading shared libraries: C: cannot open shared object file: No such file or directory note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace There were about 700 such failures, with a total of 719 failing tests. Let's see if no longer running the `cargo nextest` command through `stdbuf` makes them go away. --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d441a166cd..3c70d2e0cb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,8 +97,7 @@ jobs: env: GIX_TEST_IGNORE_ARCHIVES: 1 run: | - stdbuf -oL -- cargo nextest --color=always run --workspace --no-fail-fast |& - tee nextest.log + cargo nextest --color=always run --workspace --no-fail-fast |& tee nextest.log continue-on-error: true - name: Check how many tests failed if: steps.nextest.outcome == 'failure' From 145ae49e3380cbcc9edcb541664faa2673528340 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Nov 2024 00:28:33 -0500 Subject: [PATCH 6/7] Use XML nextest output instead of ad-hoc parsing This eliminates the piping, tricky parsing, and special-casing to preserve colorization, and runs the tests and the XML parsing in the default `pwsh` shell (since this is a Windows job), using PowerShell facilities to parse the XML. This also checks that there are no *errors*, in addition to (still) checking that there are no more *failures* than expected. In the preceding commit, five additional tests, not currently noted in #1358, failed: FAIL [ 0.010s] gix-credentials::credentials program::from_custom_definition::empty FAIL [ 0.008s] gix-credentials::credentials program::from_custom_definition::name FAIL [ 0.010s] gix-credentials::credentials program::from_custom_definition::name_with_args FAIL [ 0.009s] gix-credentials::credentials program::from_custom_definition::name_with_special_args FAIL [ 0.014s] gix-discover::discover upwards::from_dir_with_dot_dot In addition, one test noted in #1358 does not always fail on CI, because it is a performance test and the CI runner is fast enough so that it usually passes: FAIL [ 181.270s] gix-ref-tests::refs packed::iter::performance Let's see if running the tests more similarly to the way they are run on Windows without `GIX_TEST_IGNORE_ARCHIVES`, i.e. without piping and with the Windows default of `pwsh` as the shell, affects any of those new/CI-specific failures. --- .config/nextest.toml | 2 ++ .github/workflows/ci.yml | 15 ++++----------- 2 files changed, 6 insertions(+), 11 deletions(-) create mode 100644 .config/nextest.toml diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 00000000000..4dd517b63c4 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,2 @@ +[profile.with-xml.junit] +path = "junit.xml" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c70d2e0cb0..359c42bcb55 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,9 +82,6 @@ jobs: test-fixtures-windows: runs-on: windows-latest - defaults: - run: - shell: bash # Includes `-o pipefail`. steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable @@ -96,22 +93,18 @@ jobs: id: nextest env: GIX_TEST_IGNORE_ARCHIVES: 1 - run: | - cargo nextest --color=always run --workspace --no-fail-fast |& tee nextest.log + run: cargo nextest --profile=with-xml run --workspace --no-fail-fast continue-on-error: true - name: Check how many tests failed if: steps.nextest.outcome == 'failure' env: - LC_ALL: C.utf8 # FIXME: Change to 15 after verifying that the last step can fail. # See https://github.com/GitoxideLabs/gitoxide/issues/1358. EXPECTED_FAILURE_COUNT: 13 run: | - set -x - sed -Ei 's/\x1B\[[[:digit:];]*m//g' nextest.log # Remove ANSI color codes. - pattern='\n-{10,}\r?\n[ \t]+Summary\b[^\n]+[ \t]\K\d+(?= failed\b)' - count="$(grep -zoP "$pattern" nextest.log)" - ((count <= EXPECTED_FAILURE_COUNT)) + [xml]$junit = Get-Content -Path 'target/nextest/with-xml/junit.xml' + if ($junit.testsuites.errors -ne 0) { exit 1 } + if ($junit.testsuites.failures -gt $env:EXPECTED_FAILURE_COUNT) { exit 1 } test-32bit: runs-on: ubuntu-latest From 319e9d871c75299a22aa8dd41b86acef99802e82 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Nov 2024 02:12:14 -0500 Subject: [PATCH 7/7] Raise the expected failure count This should let the `test-fixtures-windows` job succeed. As suspected (but not, as of now, explained), the five additional failures when running the tests in `bash` and piping the output went away with `pwsh` and no pipe. This brings the number of failures down to 14, with the possibility of 15 failing tests if the performance test fails (see discussion in #1358). However, while it fails when I run it locally, that performance test seems rarely if ever to fail on CI (anymore?). So let's try setting the maximum number of failing tests, above which the job will report failure, to 14 rather than 15. So that they can be investigated later, the tests that failed with `bash` and `|&` piping of stdout and stderr, when run on Windows on CI with `GIX_TEST_IGNORE_ARCHIVES=1` -- which are listed in the previous commit where the failures were corrected -- have as the most significant reported details as follows: --- STDERR: gix-credentials::credentials program::from_custom_definition::empty --- thread 'program::from_custom_definition::empty' panicked at gix-credentials\tests\program\from_custom_definition.rs:16:5: assertion `left == right` failed: not useful, but allowed, would have to be caught elsewhere left: "\"sh\" \"-c\" \"C:\\\\Program Files\\\\Git\\\\mingw64\\\\bin\\\\git.exe credential- \\\"$@\\\"\" \"--\" \"store\"" right: "\"C:\\Program Files\\Git\\mingw64\\bin\\git.exe\" \"credential-\" \"store\"" note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace --- STDERR: gix-credentials::credentials program::from_custom_definition::name --- thread 'program::from_custom_definition::name' panicked at gix-credentials\tests\program\from_custom_definition.rs:64:5: assertion `left == right` failed: we detect that this can run without shell, which is also more portable on windows left: "\"sh\" \"-c\" \"C:\\\\Program Files\\\\Git\\\\mingw64\\\\bin\\\\git.exe credential-name \\\"$@\\\"\" \"--\" \"store\"" right: "\"C:\\Program Files\\Git\\mingw64\\bin\\git.exe\" \"credential-name\" \"store\"" note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace --- STDERR: gix-credentials::credentials program::from_custom_definition::name_with_args --- thread 'program::from_custom_definition::name_with_args' panicked at gix-credentials\tests\program\from_custom_definition.rs:40:5: assertion `left == right` failed left: "\"sh\" \"-c\" \"C:\\\\Program Files\\\\Git\\\\mingw64\\\\bin\\\\git.exe credential-name --arg --bar=\\\"a b\\\" \\\"$@\\\"\" \"--\" \"store\"" right: "\"C:\\Program Files\\Git\\mingw64\\bin\\git.exe\" \"credential-name\" \"--arg\" \"--bar=a b\" \"store\"" note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace --- STDERR: gix-credentials::credentials program::from_custom_definition::name_with_special_args --- thread 'program::from_custom_definition::name_with_special_args' panicked at gix-credentials\tests\program\from_custom_definition.rs:52:5: assertion `left == right` failed left: "\"sh\" \"-c\" \"C:\\\\Program Files\\\\Git\\\\mingw64\\\\bin\\\\git.exe credential-name --arg --bar=~/folder/in/home \\\"$@\\\"\" \"--\" \"store\"" right: "\"sh\" \"-c\" \"C:\\Program Files\\Git\\mingw64\\bin\\git.exe credential-name --arg --bar=~/folder/in/home \\\"$@\\\"\" \"--\" \"store\"" note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace --- STDERR: gix-discover::discover upwards::from_dir_with_dot_dot --- thread 'upwards::from_dir_with_dot_dot' panicked at gix-discover\tests\discover\upwards\mod.rs:155:5: assertion `left == right` failed left: Reduced right: Full note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace To be clear, those failures do *not* occur in the preivous commit and are not expected to occur in this one. Those prior failures consist of four `gix-credentials` tests and one `gix-discover` test. (The `gix-discover` failure resembles a problem observed locally on a Windows Server 2022 system as an administrator with UAC disabled, reported in #1429, and suggests that the conditions of that failure may differ from, or be more complex than, what I had described there.) --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 359c42bcb55..523e909c8e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,9 +98,8 @@ jobs: - name: Check how many tests failed if: steps.nextest.outcome == 'failure' env: - # FIXME: Change to 15 after verifying that the last step can fail. # See https://github.com/GitoxideLabs/gitoxide/issues/1358. - EXPECTED_FAILURE_COUNT: 13 + EXPECTED_FAILURE_COUNT: 14 run: | [xml]$junit = Get-Content -Path 'target/nextest/with-xml/junit.xml' if ($junit.testsuites.errors -ne 0) { exit 1 }