ci: Report rejected changes to done nodes

.github/workflows/validate-dashboard.yml

@@ -117,12 +117,25 @@ jobs:
             --days "${KCI_VALIDATE_DAYS}" \
             --json-out "${RUNNER_TEMP}/forecast.json"
 
-      - name: Upload forecast report
+      - name: Check for changes to done nodes
+        # kernelci-api rejects state/result changes to done nodes and logs
+        # each attempt on the node-integrity channel.
+        if: always()
+        env:
+          KCI_VALIDATE_DAYS: ${{ github.event.inputs.days || '1' }}
+        run: |
+          python tests/check_node_integrity.py \
+            --days "${KCI_VALIDATE_DAYS}" \
+            --json-out "${RUNNER_TEMP}/integrity.json"
+
+      - name: Upload forecast and integrity reports
         if: always()
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
           name: report-forecast
-          path: ${{ runner.temp }}/forecast.json
+          path: |
+            ${{ runner.temp }}/forecast.json
+            ${{ runner.temp }}/integrity.json
           if-no-files-found: ignore
           retention-days: 7
 

tests/check_node_integrity.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""
+Report attempts to change nodes that were already done.
+
+kernelci-api rejects (HTTP 409) any update that changes the state or result of
+a done node, e.g. a lab reporting pass and then fail for the same job, and
+publishes each attempt on the `node-integrity` channel. Admin overrides
+(force=true) are accepted but published there as well. Such nodes may hold the
+wrong result in Maestro and on the dashboard, so they need a manual look.
+
+This script lists those events for the last N days. Exit status is 1 if any
+attempt was found.
+"""
+
+import argparse
+import collections
+import datetime
+import json
+import sys
+
+import requests
+
+CHANNEL = "node-integrity"
+
+
+def fetch_events(api_url, since, limit, timeout=60):
+    url = f"{api_url.rstrip('/')}/latest/events"
+    params = {
+        "channel": CHANNEL,
+        "from": since.isoformat(timespec="seconds"),
+        "limit": limit,
+    }
+    res = requests.get(url, params=params, timeout=timeout)
+    res.raise_for_status()
+    return [event["data"] for event in res.json()]
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Report rejected/forced changes to done nodes"
+    )
+    parser.add_argument(
+        "--api-url",
+        default="https://api.kernelci.org",
+        help="Maestro API base URL",
+    )
+    parser.add_argument(
+        "--days",
+        type=float,
+        default=1,
+        help="Report events from the last N days",
+    )
+    parser.add_argument(
+        "--limit", type=int, default=1000, help="Maximum events to fetch"
+    )
+    parser.add_argument("--json-out", help="Write the report to this file")
+    args = parser.parse_args()
+
+    now = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
+    since = now - datetime.timedelta(days=args.days)
+    events = fetch_events(args.api_url, since, args.limit)
+
+    for event in events:
+        print(
+            f"::warning::[{event['op']}] {event['id']} {event['name']} "
+            f"({event.get('runtime')}/{event.get('platform')}): "
+            f"{event['state']}/{event['result']} -> "
+            f"{event['attempted_state']}/{event['attempted_result']} "
+            f"by {event['user']} "
+            f"(https://api.kernelci.org/viewer?node_id={event['id']})"
+        )
+
+    ops = collections.Counter(event["op"] for event in events)
+    report = {
+        "window": {"since": since.isoformat(timespec="seconds")},
+        "summary": {
+            "events": len(events),
+            "rejected": ops.get("done_update_rejected", 0),
+            "forced": ops.get("done_update_forced", 0),
+            "ok": not events,
+        },
+        "events": events,
+    }
+
+    print("\n=== Done node integrity ===")
+    for key, value in report["summary"].items():
+        print(f"{key}: {value}")
+
+    if args.json_out:
+        with open(args.json_out, "w") as fp:
+            json.dump(report, fp, indent=2)
+
+    return 0 if report["summary"]["ok"] else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/notify_discord.py

@@ -6,6 +6,7 @@
 
   * <name>.json   - `kci-dev maestro validate ... --json` (dashboard checks)
   * forecast.json - tests/check_forecast_coverage.py --json-out
+  * integrity.json - tests/check_node_integrity.py --json-out
 
 and posts one embed with a line per check. The webhook URL is taken from
 DISCORD_WEBHOOK_URL; without it the message is only printed.
@@ -100,6 +101,29 @@ def forecast_lines(report, top):
     return lines, summary["ok"]
 
 
+def integrity_lines(report, top):
+    if report is None:
+        return ["❌ **done nodes**: no report (integrity step crashed?)"], False
+    summary = report["summary"]
+    lines = [
+        f"{flag(summary['ok'])} **done nodes**: "
+        f"{summary['rejected']} rejected changes, "
+        f"{summary['forced']} forced by admin"
+    ]
+    # These are rare and each may hide a wrong result, so link every one.
+    for event in report["events"][:top]:
+        viewer = f"https://api.kernelci.org/viewer?node_id={event['id']}"
+        lines.append(
+            f"  • [{event['name']}]({viewer}) "
+            f"{event.get('platform') or ''} ({event.get('runtime')}): "
+            f"{event['result']} → {event['attempted_result']}"
+        )
+    more = len(report["events"]) - top
+    if more > 0:
+        lines.append(f"  • +{more} more")
+    return lines, summary["ok"]
+
+
 def main():
     parser = argparse.ArgumentParser(
         description="Post dashboard validation summary to Discord"
@@ -130,6 +154,10 @@ def main():
     flines, ok = forecast_lines(report, args.top)
     lines.extend(flines)
     all_ok = all_ok and ok
+    report = load_report(os.path.join(args.reports_dir, "integrity.json"))
+    ilines, ok = integrity_lines(report, args.top)
+    lines.extend(ilines)
+    all_ok = all_ok and ok
 
     description = "\n".join(lines)
     if len(description) > MAX_DESCRIPTION: