From c77bc30b2e084ba5d4ce9a68df3275def8092221 Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Thu, 29 Oct 2026 13:17:47 +0200 Subject: [PATCH] ci: Report rejected changes to done nodes kernelci-api rejects state/result changes to nodes that are already done and publishes each attempt (and each admin override) on the node-integrity channel. Such nodes may hold a wrong result in Maestro or on the dashboard and need a manual look. Add tests/check_node_integrity.py to list those events for the validation window, run it in the forecast job, and include them in the Discord summary with a link to each affected node. Signed-off-by: Denys Fedoryshchenko --- .github/workflows/validate-dashboard.yml | 17 ++++- tests/check_node_integrity.py | 97 ++++++++++++++++++++++++ tests/notify_discord.py | 28 +++++++ 3 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 tests/check_node_integrity.py diff --git a/.github/workflows/validate-dashboard.yml b/.github/workflows/validate-dashboard.yml index ddad16e74..a60bfd81f 100644 --- a/.github/workflows/validate-dashboard.yml +++ b/.github/workflows/validate-dashboard.yml @@ -117,12 +117,25 @@ jobs: --days "${KCI_VALIDATE_DAYS}" \ --json-out "${RUNNER_TEMP}/forecast.json" - - name: Upload forecast report + - name: Check for changes to done nodes + # kernelci-api rejects state/result changes to done nodes and logs + # each attempt on the node-integrity channel. + if: always() + env: + KCI_VALIDATE_DAYS: ${{ github.event.inputs.days || '1' }} + run: | + python tests/check_node_integrity.py \ + --days "${KCI_VALIDATE_DAYS}" \ + --json-out "${RUNNER_TEMP}/integrity.json" + + - name: Upload forecast and integrity reports if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: report-forecast - path: ${{ runner.temp }}/forecast.json + path: | + ${{ runner.temp }}/forecast.json + ${{ runner.temp }}/integrity.json if-no-files-found: ignore retention-days: 7 diff --git a/tests/check_node_integrity.py b/tests/check_node_integrity.py new file mode 100644 index 000000000..8b5d112d6 --- /dev/null +++ b/tests/check_node_integrity.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +""" +Report attempts to change nodes that were already done. + +kernelci-api rejects (HTTP 409) any update that changes the state or result of +a done node, e.g. a lab reporting pass and then fail for the same job, and +publishes each attempt on the `node-integrity` channel. Admin overrides +(force=true) are accepted but published there as well. Such nodes may hold the +wrong result in Maestro and on the dashboard, so they need a manual look. + +This script lists those events for the last N days. Exit status is 1 if any +attempt was found. +""" + +import argparse +import collections +import datetime +import json +import sys + +import requests + +CHANNEL = "node-integrity" + + +def fetch_events(api_url, since, limit, timeout=60): + url = f"{api_url.rstrip('/')}/latest/events" + params = { + "channel": CHANNEL, + "from": since.isoformat(timespec="seconds"), + "limit": limit, + } + res = requests.get(url, params=params, timeout=timeout) + res.raise_for_status() + return [event["data"] for event in res.json()] + + +def main(): + parser = argparse.ArgumentParser( + description="Report rejected/forced changes to done nodes" + ) + parser.add_argument( + "--api-url", + default="https://api.kernelci.org", + help="Maestro API base URL", + ) + parser.add_argument( + "--days", + type=float, + default=1, + help="Report events from the last N days", + ) + parser.add_argument( + "--limit", type=int, default=1000, help="Maximum events to fetch" + ) + parser.add_argument("--json-out", help="Write the report to this file") + args = parser.parse_args() + + now = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) + since = now - datetime.timedelta(days=args.days) + events = fetch_events(args.api_url, since, args.limit) + + for event in events: + print( + f"::warning::[{event['op']}] {event['id']} {event['name']} " + f"({event.get('runtime')}/{event.get('platform')}): " + f"{event['state']}/{event['result']} -> " + f"{event['attempted_state']}/{event['attempted_result']} " + f"by {event['user']} " + f"(https://api.kernelci.org/viewer?node_id={event['id']})" + ) + + ops = collections.Counter(event["op"] for event in events) + report = { + "window": {"since": since.isoformat(timespec="seconds")}, + "summary": { + "events": len(events), + "rejected": ops.get("done_update_rejected", 0), + "forced": ops.get("done_update_forced", 0), + "ok": not events, + }, + "events": events, + } + + print("\n=== Done node integrity ===") + for key, value in report["summary"].items(): + print(f"{key}: {value}") + + if args.json_out: + with open(args.json_out, "w") as fp: + json.dump(report, fp, indent=2) + + return 0 if report["summary"]["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/notify_discord.py b/tests/notify_discord.py index 96b902f4b..8f7508ff1 100644 --- a/tests/notify_discord.py +++ b/tests/notify_discord.py @@ -6,6 +6,7 @@ * .json - `kci-dev maestro validate ... --json` (dashboard checks) * forecast.json - tests/check_forecast_coverage.py --json-out + * integrity.json - tests/check_node_integrity.py --json-out and posts one embed with a line per check. The webhook URL is taken from DISCORD_WEBHOOK_URL; without it the message is only printed. @@ -100,6 +101,29 @@ def forecast_lines(report, top): return lines, summary["ok"] +def integrity_lines(report, top): + if report is None: + return ["❌ **done nodes**: no report (integrity step crashed?)"], False + summary = report["summary"] + lines = [ + f"{flag(summary['ok'])} **done nodes**: " + f"{summary['rejected']} rejected changes, " + f"{summary['forced']} forced by admin" + ] + # These are rare and each may hide a wrong result, so link every one. + for event in report["events"][:top]: + viewer = f"https://api.kernelci.org/viewer?node_id={event['id']}" + lines.append( + f" • [{event['name']}]({viewer}) " + f"{event.get('platform') or ''} ({event.get('runtime')}): " + f"{event['result']} → {event['attempted_result']}" + ) + more = len(report["events"]) - top + if more > 0: + lines.append(f" • +{more} more") + return lines, summary["ok"] + + def main(): parser = argparse.ArgumentParser( description="Post dashboard validation summary to Discord" @@ -130,6 +154,10 @@ def main(): flines, ok = forecast_lines(report, args.top) lines.extend(flines) all_ok = all_ok and ok + report = load_report(os.path.join(args.reports_dir, "integrity.json")) + ilines, ok = integrity_lines(report, args.top) + lines.extend(ilines) + all_ok = all_ok and ok description = "\n".join(lines) if len(description) > MAX_DESCRIPTION: