From 2365befbfe9cd3c322e8ec562a16483082ca8734 Mon Sep 17 00:00:00 2001 From: Qiu Yuzhou Date: Sat, 24 Oct 2026 02:51:51 +0800 Subject: [PATCH] ci: run smoke tests and retain failure evidence --- .github/workflows/shadowsocksx-ng2-ci.yml | 21 +++++++++++++++++++-- ShadowsocksX-NG2/Taskfile.dist.yml | 1 + docs/development.md | 4 ++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/workflows/shadowsocksx-ng2-ci.yml b/.github/workflows/shadowsocksx-ng2-ci.yml index 285a97af..b10f4f90 100644 --- a/.github/workflows/shadowsocksx-ng2-ci.yml +++ b/.github/workflows/shadowsocksx-ng2-ci.yml @@ -16,7 +16,11 @@ concurrency: jobs: checks: - name: Lint, test, build (Xcode 27 arm64) + name: ${{ matrix.gate == 'test' && 'Lint, test, build' || 'Smoke tests' }} (Xcode 27 arm64) + strategy: + fail-fast: false + matrix: + gate: [test, 'test:smoke'] runs-on: xcode-27 timeout-minutes: 60 @@ -92,17 +96,30 @@ jobs: key: ng2-external-binaries-macos-arm64-${{ hashFiles('ShadowsocksX-NG2/Vendor/sslocal/manifest.json', 'ShadowsocksX-NG2/Vendor/v2ray-plugin/manifest.json', 'ShadowsocksX-NG2/Vendor/ck-client/manifest.json') }} - name: Check Swift formatting + if: matrix.gate == 'test' working-directory: ShadowsocksX-NG2 run: task format:check - name: Lint + if: matrix.gate == 'test' working-directory: ShadowsocksX-NG2 run: task --exit-code lint - name: Test working-directory: ShadowsocksX-NG2 - run: task --exit-code test STRICT=true + run: task --exit-code validate STRICT=true -- ${{ matrix.gate }} - name: Build + if: matrix.gate == 'test' working-directory: ShadowsocksX-NG2 run: task --exit-code build STRICT=true + + - name: Upload failed test evidence + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: validation-${{ strategy.job-index }} + path: ShadowsocksX-NG2/.build/validation/ + include-hidden-files: true + retention-days: 7 + if-no-files-found: warn diff --git a/ShadowsocksX-NG2/Taskfile.dist.yml b/ShadowsocksX-NG2/Taskfile.dist.yml index 6364101d..77926b16 100644 --- a/ShadowsocksX-NG2/Taskfile.dist.yml +++ b/ShadowsocksX-NG2/Taskfile.dist.yml @@ -224,6 +224,7 @@ tasks: -derivedDataPath {{.DERIVED_DATA_DIR | shellQuote}} {{if .RESULT_BUNDLE_PATH}}-resultBundlePath {{.RESULT_BUNDLE_PATH | shellQuote}}{{end}} {{.TEST_TIMEOUT_ARGS}} + {{.STRICT_BUILD_ARGS}} test clean: diff --git a/docs/development.md b/docs/development.md index b3ab1270..b47391f0 100644 --- a/docs/development.md +++ b/docs/development.md @@ -22,7 +22,7 @@ By default, stages run in this order: `lint → test → build`. Real-process sm Each local `validate` run creates a separate `.build/validation/-/` directory and prints its absolute path at the start and end. `summary.txt` records each stage's status, exit code, and evidence status. For every executed stage, the corresponding subdirectory contains a combined stdout/stderr `raw.log` and a `status.txt` (`test:smoke` uses `test-smoke/`); Xcode stages also save `result.xcresult`. The console shows progress and a summary. Read the corresponding logs for complete errors. All artifacts are inside existing ignored directories. `task clean` deletes them, so copy anything you need to keep. -CI runs `task --exit-code lint`, `task --exit-code test STRICT=true`, and `task --exit-code build STRICT=true` in separate steps after the formatting check. Tests and builds treat compiler warnings as errors. Each step streams its output directly to the GitHub Actions log. CI does not invoke `validate` or upload its evidence files. +CI runs two independent matrix jobs: `test` and `test:smoke`, with fail-fast disabled. Both run through `task --exit-code validate STRICT=true -- ` and treat compiler warnings as errors. The `test` job also checks formatting, runs lint, and builds. Failed jobs upload retained validation logs, summaries, and result bundles for seven days. Read the uploaded `summary.txt` and `test-summary.json` first; test output is retained in `raw.log`. For test stages, the workflow uses the local `xcrun xcresulttool get test-results summary` and system `plutil` to read test counts on a best-effort basis; it adds no scripts or tool dependencies. `test-summary.json` stores the raw summary, `test-summary.stderr.log` stores xcresulttool errors, and `count-reader.log` stores count-reading errors. The summary shows passed, failed, skipped, expected-failure, and total counts using Xcode's summary semantics; parameterized tests may have a different actual execution count. Stage and overall exit codes come from the original command, and a failed command always remains a failure. A missing result bundle or unreadable summary is marked `evidence incomplete`, with counts shown as unknown. This does not change the command result, and missing counts are never reported as zero. Build stages do not read test counts; a stage that did not run is not the same as a skipped test. @@ -64,7 +64,7 @@ Swift Package Manager resolves the pinned release of AgeSwift from `project.yml` You can build and run unit tests without a certificate: the Debug baseline uses ad-hoc signing and does not require `Configs/Signing.local.xcconfig`. Create that file only when signing with Developer ID, either for release or to make a local Debug build pass the packaging gate. See [Signing and runtime baseline](distribution.md#signing-and-runtime-baseline) for its contents. -`Tests/Smoke/` contains real-process smoke tests. A wrapper launches real `sslocal` to verify startup, handshake, ACL routing, and plugin supervision. These tests are not part of the default `task test`; run them on demand with their separate scheme. +`Tests/Smoke/` contains real-process smoke tests. A wrapper launches real `sslocal` to verify startup, handshake, ACL routing, and plugin supervision. These tests are not part of the default `task test`; run them locally with their separate scheme. CI runs them as an independent job. Pending reload-signal accumulation is tested without child processes in `RuntimeReloadStateTests`; real signal delivery and combined server/ACL routing remain covered by `RuntimePublicationSmokeTests`. ## Code style tools