From 7c255d7362ca22b131b6d311ca519102d2612ac4 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Mon, 16 Nov 2026 22:34:15 +0100 Subject: [PATCH 01/14] Add endpoint to get a presigned upload url for desync reports GET /game/{gameId}/desync/upload?size= checks that the player took part in the game (optionally also that the lobby server recorded a desync for them) and returns a presigned PUT url for desync//.zip in a dedicated bucket. The size and content type are signed into the url, so the object storage rejects any other upload. Co-Authored-By: Claude Sonnet 5.5 --- compose.yaml | 2 +- src/inttest/resources/config/application.yml | 6 + .../api/config/FafApiProperties.java | 21 +++ .../com/faforever/api/config/S3Config.java | 20 +++ .../api/desync/DesyncReportController.java | 39 ++++++ .../api/desync/DesyncReportService.java | 86 ++++++++++++ .../desync/DesyncReportUploadResponse.java | 11 ++ .../com/faforever/api/error/ErrorCode.java | 1 + .../api/game/GamePlayerStatsRepository.java | 10 ++ .../faforever/api/security/OAuthScope.java | 2 + src/main/resources/config/application.yml | 8 ++ .../api/desync/DesyncReportServiceTest.java | 130 ++++++++++++++++++ 12 files changed, 335 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/faforever/api/desync/DesyncReportController.java create mode 100644 src/main/java/com/faforever/api/desync/DesyncReportService.java create mode 100644 src/main/java/com/faforever/api/desync/DesyncReportUploadResponse.java create mode 100644 src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java diff --git a/compose.yaml b/compose.yaml index a2fc41955..cc14adc7d 100644 --- a/compose.yaml +++ b/compose.yaml @@ -42,7 +42,7 @@ services: environment: MINIO_ROOT_USER: admin MINIO_ROOT_PASSWORD: banana123 - MINIO_DEFAULT_BUCKETS: user-uploads + MINIO_DEFAULT_BUCKETS: user-uploads,desync-reports MINIO_SCHEME: http healthcheck: test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/ready" ] diff --git a/src/inttest/resources/config/application.yml b/src/inttest/resources/config/application.yml index 7645f3291..de98bc473 100644 --- a/src/inttest/resources/config/application.yml +++ b/src/inttest/resources/config/application.yml @@ -102,6 +102,12 @@ faf-api: endpoint: http://minio region: auto user-upload-bucket: user-uploads + desync-report: + access-key: admin + secret-key: banana123 + endpoint: http://minio + region: auto + bucket: desync-reports logging: level: diff --git a/src/main/java/com/faforever/api/config/FafApiProperties.java b/src/main/java/com/faforever/api/config/FafApiProperties.java index 22ad0ab44..90572dddb 100644 --- a/src/main/java/com/faforever/api/config/FafApiProperties.java +++ b/src/main/java/com/faforever/api/config/FafApiProperties.java @@ -3,6 +3,7 @@ import lombok.Data; import org.checkerframework.checker.index.qual.Positive; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.util.unit.DataSize; import java.nio.file.Path; import java.nio.file.Paths; @@ -44,6 +45,7 @@ public class FafApiProperties { private Monitoring monitoring = new Monitoring(); private Coturn coturn = new Coturn(); private S3 s3 = new S3(); + private DesyncReport desyncReport = new DesyncReport(); @Data public static class OAuth2 { @@ -304,4 +306,23 @@ public static class S3 { private String accessKey; private String secretKey; } + + @Data + public static class DesyncReport { + private String endpoint; + private String region; + private String bucket; + private String accessKey; + private String secretKey; + /** + * Upper limit for the size of a single desync report. The size is signed into the upload url, so the object storage + * rejects any upload of a different size. + */ + private DataSize maxUploadSize = DataSize.ofMegabytes(10); + /** + * If enabled, a player can only upload a report for games in which the lobby server has recorded a desync report (via + * GPGNet) for them. + */ + private boolean requireGpgnetReport = false; + } } diff --git a/src/main/java/com/faforever/api/config/S3Config.java b/src/main/java/com/faforever/api/config/S3Config.java index e562ebe81..c8256138b 100644 --- a/src/main/java/com/faforever/api/config/S3Config.java +++ b/src/main/java/com/faforever/api/config/S3Config.java @@ -3,6 +3,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; @@ -33,6 +34,7 @@ public S3Client s3Client() { } @Bean + @Primary public S3Presigner s3Presigner() { return S3Presigner.builder() .endpointOverride(URI.create(properties.getS3().getEndpoint())) @@ -46,4 +48,22 @@ public S3Presigner s3Presigner() { .build()) .build(); } + + /** + * Presigner for the dedicated desync report bucket, which has its own endpoint and credentials. + */ + @Bean + public S3Presigner desyncReportS3Presigner() { + FafApiProperties.DesyncReport desyncReport = properties.getDesyncReport(); + return S3Presigner.builder() + .endpointOverride(URI.create(desyncReport.getEndpoint())) + .region(Region.of(desyncReport.getRegion())) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create(desyncReport.getAccessKey(), desyncReport.getSecretKey()) + )) + .serviceConfiguration(S3Configuration.builder() + .pathStyleAccessEnabled(true) + .build()) + .build(); + } } diff --git a/src/main/java/com/faforever/api/desync/DesyncReportController.java b/src/main/java/com/faforever/api/desync/DesyncReportController.java new file mode 100644 index 000000000..53402cd7e --- /dev/null +++ b/src/main/java/com/faforever/api/desync/DesyncReportController.java @@ -0,0 +1,39 @@ +package com.faforever.api.desync; + +import com.faforever.api.player.PlayerService; +import com.faforever.api.security.OAuthScope; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import lombok.RequiredArgsConstructor; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.security.core.Authentication; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; + +@RestController +@RequestMapping(path = "/game/{gameId}/desync") +@RequiredArgsConstructor +public class DesyncReportController { + + private final PlayerService playerService; + private final DesyncReportService desyncReportService; + + @Operation(summary = "Get a presigned url to upload the desync report of the authenticated player for a game", + description = "The report has to be uploaded with a PUT request to the returned url, with exactly 'size' bytes as body and " + + "the returned headers. A repeated request for the same game replaces the previous report.") + @ApiResponse(responseCode = "200", description = "Upload url created") + @ApiResponse(responseCode = "403", description = "The player did not participate in this game") + @ApiResponse(responseCode = "422", description = "The size is not within the allowed range") + @GetMapping(path = "/upload", produces = APPLICATION_JSON_VALUE) + @PreAuthorize("hasScope('" + OAuthScope._UPLOAD_DESYNC_REPORT + "')") + public DesyncReportUploadResponse getUploadUrl(Authentication authentication, + @PathVariable("gameId") int gameId, + @RequestParam("size") long size) { + return desyncReportService.createUploadUrl(playerService.getPlayer(authentication).getId(), gameId, size); + } +} diff --git a/src/main/java/com/faforever/api/desync/DesyncReportService.java b/src/main/java/com/faforever/api/desync/DesyncReportService.java new file mode 100644 index 000000000..de449996d --- /dev/null +++ b/src/main/java/com/faforever/api/desync/DesyncReportService.java @@ -0,0 +1,86 @@ +package com.faforever.api.desync; + +import com.faforever.api.config.FafApiProperties; +import com.faforever.api.config.FafApiProperties.DesyncReport; +import com.faforever.api.error.ApiException; +import com.faforever.api.error.Error; +import com.faforever.api.error.ErrorCode; +import com.faforever.api.game.GamePlayerStatsRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.stereotype.Service; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest; +import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; + +@Slf4j +@Service +public class DesyncReportService { + static final String CONTENT_TYPE = "application/zip"; + private static final Duration URL_LIFETIME = Duration.ofMinutes(15); + + private final DesyncReport properties; + private final GamePlayerStatsRepository gamePlayerStatsRepository; + private final S3Presigner s3Presigner; + + public DesyncReportService(FafApiProperties properties, + GamePlayerStatsRepository gamePlayerStatsRepository, + @Qualifier("desyncReportS3Presigner") S3Presigner s3Presigner) { + this.properties = properties.getDesyncReport(); + this.gamePlayerStatsRepository = gamePlayerStatsRepository; + this.s3Presigner = s3Presigner; + } + + static String getObjectKey(int gameId, int playerId) { + return "desync/%d/%d.zip".formatted(gameId, playerId); + } + + /** + * Creates a presigned url that allows the player to upload exactly one desync report of exactly {@code size} bytes. + * The size is part of the signature, so the object storage rejects uploads with any other size. + */ + public DesyncReportUploadResponse createUploadUrl(int playerId, int gameId, long size) { + long maxSize = properties.getMaxUploadSize().toBytes(); + if (size < 1 || size > maxSize) { + throw new ApiException(new Error(ErrorCode.DESYNC_REPORT_INVALID_SIZE, maxSize)); + } + + if (!gamePlayerStatsRepository.existsByGameIdAndPlayerId(gameId, playerId)) { + throw new AccessDeniedException("Player %d did not participate in game %d".formatted(playerId, gameId)); + } + + if (properties.isRequireGpgnetReport() && gamePlayerStatsRepository.countGpgnetDesyncReports(gameId, playerId) == 0) { + throw new AccessDeniedException("No desync was reported for player %d in game %d".formatted(playerId, gameId)); + } + + String key = getObjectKey(gameId, playerId); + log.info("Player {} requested a desync report upload url for game {} ({} bytes)", playerId, gameId, size); + + PutObjectRequest putObjectRequest = PutObjectRequest.builder() + .bucket(properties.getBucket()) + .key(key) + .contentLength(size) + .contentType(CONTENT_TYPE) + .build(); + + PresignedPutObjectRequest presigned = s3Presigner.presignPutObject(PutObjectPresignRequest.builder() + .signatureDuration(URL_LIFETIME) + .putObjectRequest(putObjectRequest) + .build()); + + Map headers = new LinkedHashMap<>(); + presigned.signedHeaders().forEach((name, values) -> { + if (!"host".equalsIgnoreCase(name) && !values.isEmpty()) { + headers.put(name, values.getFirst()); + } + }); + + return new DesyncReportUploadResponse(presigned.url().toString(), key, headers); + } +} diff --git a/src/main/java/com/faforever/api/desync/DesyncReportUploadResponse.java b/src/main/java/com/faforever/api/desync/DesyncReportUploadResponse.java new file mode 100644 index 000000000..54ad5ac3c --- /dev/null +++ b/src/main/java/com/faforever/api/desync/DesyncReportUploadResponse.java @@ -0,0 +1,11 @@ +package com.faforever.api.desync; + +import java.util.Map; + +/** + * @param url the presigned url to PUT the report to + * @param key the object key the report will be stored under + * @param headers headers that have to be sent with the PUT request exactly as given (they are part of the signature) + */ +public record DesyncReportUploadResponse(String url, String key, Map headers) { +} diff --git a/src/main/java/com/faforever/api/error/ErrorCode.java b/src/main/java/com/faforever/api/error/ErrorCode.java index 34e1c8c32..df2eeb422 100644 --- a/src/main/java/com/faforever/api/error/ErrorCode.java +++ b/src/main/java/com/faforever/api/error/ErrorCode.java @@ -121,6 +121,7 @@ public enum ErrorCode { MAP_VERSION_INVALID_RANGE(211, "Invalid map version", "The map version must be a whole number in range {0, number} to {1, number}."), MOD_VERSION_INVALID_RANGE(212, "Invalid mod version", "The mod version must be a whole number in range {0, number} to {1, number}."), AVATAR_NOT_ASSIGNED(213, "Avatar not assigned", "Avatar ''{0}'' is not assigned to player ''{1}''."), + DESYNC_REPORT_INVALID_SIZE(214, "Invalid desync report size", "The size of a desync report must be between 1 and {0, number} bytes."), ; diff --git a/src/main/java/com/faforever/api/game/GamePlayerStatsRepository.java b/src/main/java/com/faforever/api/game/GamePlayerStatsRepository.java index 1c139dcb3..378b61155 100644 --- a/src/main/java/com/faforever/api/game/GamePlayerStatsRepository.java +++ b/src/main/java/com/faforever/api/game/GamePlayerStatsRepository.java @@ -4,7 +4,17 @@ import com.faforever.api.data.domain.Player; import com.faforever.api.data.domain.Validity; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; public interface GamePlayerStatsRepository extends JpaRepository { int countByPlayerAndGameValidity(Player player, Validity validity); + + boolean existsByGameIdAndPlayerId(int gameId, int playerId); + + /** + * Counts the desync reports that the lobby server has recorded for a player in a game (table {@code game_desync}). + */ + @Query(value = "SELECT COUNT(*) FROM game_desync WHERE game_id = :gameId AND player_id = :playerId", nativeQuery = true) + long countGpgnetDesyncReports(@Param("gameId") int gameId, @Param("playerId") int playerId); } diff --git a/src/main/java/com/faforever/api/security/OAuthScope.java b/src/main/java/com/faforever/api/security/OAuthScope.java index a1afbb032..a8b6fa09c 100644 --- a/src/main/java/com/faforever/api/security/OAuthScope.java +++ b/src/main/java/com/faforever/api/security/OAuthScope.java @@ -11,6 +11,7 @@ public enum OAuthScope { WRITE_EVENTS(OAuthScope._WRITE_EVENTS, "Write events"), UPLOAD_MAP(OAuthScope._UPLOAD_MAP, "Upload maps"), UPLOAD_MOD(OAuthScope._UPLOAD_MOD, "Upload mods"), + UPLOAD_DESYNC_REPORT(OAuthScope._UPLOAD_DESYNC_REPORT, "Upload desync reports"), WRITE_ACCOUNT_DATA(OAuthScope._WRITE_ACCOUNT_DATA, "Edit account data"), EDIT_CLAN_DATA(OAuthScope._EDIT_CLAN_DATA, "Edit clan data"), VOTE(OAuthScope._VOTE, "Vote"), @@ -24,6 +25,7 @@ public enum OAuthScope { public static final String _WRITE_EVENTS = "write_events"; public static final String _UPLOAD_MAP = "upload_map"; public static final String _UPLOAD_MOD = "upload_mod"; + public static final String _UPLOAD_DESYNC_REPORT = "upload_desync_report"; public static final String _WRITE_ACCOUNT_DATA = "write_account_data"; public static final String _EDIT_CLAN_DATA = "edit_clan_data"; public static final String _VOTE = "vote"; diff --git a/src/main/resources/config/application.yml b/src/main/resources/config/application.yml index 23f715ae6..7735743a8 100644 --- a/src/main/resources/config/application.yml +++ b/src/main/resources/config/application.yml @@ -94,6 +94,14 @@ faf-api: endpoint: ${S3_ENDPOINT:http://minio} region: ${S3_REGION:auto} # auto is a Cloudflare value user-upload-bucket: ${S3_USER_UPLOAD_BUCKET:user-uploads} + desync-report: + access-key: ${DESYNC_REPORT_S3_ACCESS_KEY:admin} + secret-key: ${DESYNC_REPORT_S3_SECRET_KEY:banana123} + endpoint: ${DESYNC_REPORT_S3_ENDPOINT:http://minio} + region: ${DESYNC_REPORT_S3_REGION:auto} + bucket: ${DESYNC_REPORT_S3_BUCKET:desync-reports} + max-upload-size: ${DESYNC_REPORT_MAX_UPLOAD_SIZE:10MB} + require-gpgnet-report: ${DESYNC_REPORT_REQUIRE_GPGNET_REPORT:false} spring: application: diff --git a/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java b/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java new file mode 100644 index 000000000..b43eadcd7 --- /dev/null +++ b/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java @@ -0,0 +1,130 @@ +package com.faforever.api.desync; + +import com.faforever.api.config.FafApiProperties; +import com.faforever.api.error.ApiException; +import com.faforever.api.error.ErrorCode; +import com.faforever.api.game.GamePlayerStatsRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.util.unit.DataSize; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; + +import java.net.URI; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.startsWith; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class DesyncReportServiceTest { + private static final int GAME_ID = 1234; + private static final int PLAYER_ID = 42; + + @Mock + private GamePlayerStatsRepository gamePlayerStatsRepository; + + private FafApiProperties properties; + private DesyncReportService instance; + + @BeforeEach + void setUp() { + properties = new FafApiProperties(); + properties.getDesyncReport().setBucket("desync-reports"); + properties.getDesyncReport().setMaxUploadSize(DataSize.ofMegabytes(10)); + + // signing is done locally, so a real presigner with dummy credentials does not need any network access + S3Presigner presigner = S3Presigner.builder() + .endpointOverride(URI.create("http://localhost:9000")) + .region(Region.of("auto")) + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("admin", "banana123"))) + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) + .build(); + + instance = new DesyncReportService(properties, gamePlayerStatsRepository, presigner); + } + + @Test + void createUploadUrlSignsKeyAndSize() { + when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); + + DesyncReportUploadResponse response = instance.createUploadUrl(PLAYER_ID, GAME_ID, 1_200_000); + + assertThat(response.key(), is("desync/1234/42.zip")); + assertThat(response.url(), startsWith("http://localhost:9000/desync-reports/desync/1234/42.zip?")); + // the size and content type are part of the signature, so the storage rejects anything else + assertThat(response.url(), containsString("content-length")); + assertThat(response.headers(), hasEntry("content-length", "1200000")); + assertThat(response.headers(), hasEntry("content-type", "application/zip")); + assertThat(response.headers().containsKey("host"), is(false)); + } + + @Test + void createUploadUrlSizeOutOfRange() { + ApiException tooLarge = assertThrows(ApiException.class, + () -> instance.createUploadUrl(PLAYER_ID, GAME_ID, DataSize.ofMegabytes(10).toBytes() + 1)); + assertThat(tooLarge.getErrors()[0].getErrorCode(), is(ErrorCode.DESYNC_REPORT_INVALID_SIZE)); + + ApiException empty = assertThrows(ApiException.class, () -> instance.createUploadUrl(PLAYER_ID, GAME_ID, 0)); + assertThat(empty.getErrors()[0].getErrorCode(), is(ErrorCode.DESYNC_REPORT_INVALID_SIZE)); + + verifyNoInteractions(gamePlayerStatsRepository); + } + + @Test + void createUploadUrlAtMaxSize() { + when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); + + instance.createUploadUrl(PLAYER_ID, GAME_ID, DataSize.ofMegabytes(10).toBytes()); + } + + @Test + void createUploadUrlPlayerNotInGame() { + when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(false); + + assertThrows(AccessDeniedException.class, () -> instance.createUploadUrl(PLAYER_ID, GAME_ID, 100)); + } + + @Test + void gpgnetReportNotCheckedByDefault() { + when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); + + instance.createUploadUrl(PLAYER_ID, GAME_ID, 100); + + verify(gamePlayerStatsRepository, never()).countGpgnetDesyncReports(GAME_ID, PLAYER_ID); + } + + @Test + void gpgnetReportRequiredButMissing() { + properties.getDesyncReport().setRequireGpgnetReport(true); + when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); + when(gamePlayerStatsRepository.countGpgnetDesyncReports(GAME_ID, PLAYER_ID)).thenReturn(0L); + + assertThrows(AccessDeniedException.class, () -> instance.createUploadUrl(PLAYER_ID, GAME_ID, 100)); + } + + @Test + void gpgnetReportRequiredAndPresent() { + properties.getDesyncReport().setRequireGpgnetReport(true); + when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); + when(gamePlayerStatsRepository.countGpgnetDesyncReports(GAME_ID, PLAYER_ID)).thenReturn(1L); + + DesyncReportUploadResponse response = instance.createUploadUrl(PLAYER_ID, GAME_ID, 100); + + assertThat(response.key(), is("desync/1234/42.zip")); + } +} From 3786eba1b97fe0435400b889da7d18bddac2acdc Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Mon, 16 Nov 2026 22:35:21 +0100 Subject: [PATCH 02/14] Bump faf-db-migrations used by the integration tests to v146 Adds the game_desync table. Co-Authored-By: Claude Sonnet 5.5 --- .../java/com/faforever/api/config/MainDbTestContainers.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inttest/java/com/faforever/api/config/MainDbTestContainers.java b/src/inttest/java/com/faforever/api/config/MainDbTestContainers.java index 9188d15aa..ec3a76d2c 100644 --- a/src/inttest/java/com/faforever/api/config/MainDbTestContainers.java +++ b/src/inttest/java/com/faforever/api/config/MainDbTestContainers.java @@ -23,7 +23,7 @@ @Configuration public class MainDbTestContainers { private static final MariaDBContainer fafDBContainer = new MariaDBContainer<>("mariadb:11.7"); - private static final GenericContainer flywayMigrationsContainer = new GenericContainer<>("faforever/faf-db-migrations:v144"); + private static final GenericContainer flywayMigrationsContainer = new GenericContainer<>("faforever/faf-db-migrations:v146"); private static final Network sharedNetwork = Network.newNetwork(); @Bean From cf4a3d5150fc8a0997c4b0daf741945a718ad731 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Mon, 16 Nov 2026 23:20:43 +0100 Subject: [PATCH 03/14] Expose game_desync read-only via Elide and bump migrations to v147 Adds the GameDesync entity (/data/gameDesync) so the time of a desync can be queried, and uses it for the optional gpgnet check instead of a native query. Needs the surrogate id added in faf-db-migrations v147. Co-Authored-By: Claude Sonnet 5.5 --- compose.yaml | 2 +- .../api/config/MainDbTestContainers.java | 2 +- .../faforever/api/data/domain/GameDesync.java | 77 +++++++++++++++++++ .../api/desync/DesyncReportService.java | 6 +- .../api/game/GameDesyncRepository.java | 8 ++ .../api/game/GamePlayerStatsRepository.java | 8 -- .../api/desync/DesyncReportServiceTest.java | 14 ++-- 7 files changed, 100 insertions(+), 17 deletions(-) create mode 100644 src/main/java/com/faforever/api/data/domain/GameDesync.java create mode 100644 src/main/java/com/faforever/api/game/GameDesyncRepository.java diff --git a/compose.yaml b/compose.yaml index cc14adc7d..e234ddddb 100644 --- a/compose.yaml +++ b/compose.yaml @@ -16,7 +16,7 @@ services: - "3306:3306" faf-db-migrations: - image: faforever/faf-db-migrations:v146 + image: faforever/faf-db-migrations:v147 command: migrate environment: FLYWAY_URL: jdbc:mysql://faf-db/faf?useSSL=false diff --git a/src/inttest/java/com/faforever/api/config/MainDbTestContainers.java b/src/inttest/java/com/faforever/api/config/MainDbTestContainers.java index ec3a76d2c..1c7f15b72 100644 --- a/src/inttest/java/com/faforever/api/config/MainDbTestContainers.java +++ b/src/inttest/java/com/faforever/api/config/MainDbTestContainers.java @@ -23,7 +23,7 @@ @Configuration public class MainDbTestContainers { private static final MariaDBContainer fafDBContainer = new MariaDBContainer<>("mariadb:11.7"); - private static final GenericContainer flywayMigrationsContainer = new GenericContainer<>("faforever/faf-db-migrations:v146"); + private static final GenericContainer flywayMigrationsContainer = new GenericContainer<>("faforever/faf-db-migrations:v147"); private static final Network sharedNetwork = Network.newNetwork(); @Bean diff --git a/src/main/java/com/faforever/api/data/domain/GameDesync.java b/src/main/java/com/faforever/api/data/domain/GameDesync.java new file mode 100644 index 000000000..1a67173ca --- /dev/null +++ b/src/main/java/com/faforever/api/data/domain/GameDesync.java @@ -0,0 +1,77 @@ +package com.faforever.api.data.domain; + +import com.faforever.api.data.checks.Prefab; +import com.yahoo.elide.annotation.CreatePermission; +import com.yahoo.elide.annotation.DeletePermission; +import com.yahoo.elide.annotation.Include; +import com.yahoo.elide.annotation.UpdatePermission; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; +import lombok.Setter; +import org.hibernate.annotations.Immutable; + +import java.time.OffsetDateTime; + +/** + * A player whose client reported a desync in a game via GPGNet. Written by the lobby server only; one row per game and + * player, recording the first report. + */ +@Entity +@Table(name = "game_desync") +@Include(name = "gameDesync") +@Immutable +@Setter +@CreatePermission(expression = Prefab.NONE) +@UpdatePermission(expression = Prefab.NONE) +@DeletePermission(expression = Prefab.NONE) +public class GameDesync { + + private int id; + private Game game; + private Player player; + private OffsetDateTime reportedAt; + private Integer gameTime; + + @Id + @Column(name = "id") + @GeneratedValue(strategy = GenerationType.IDENTITY) + public int getId() { + return id; + } + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "game_id") + public Game getGame() { + return game; + } + + /** + * The player whose client reported the desync. Every client that detects the mismatch reports it, so this is not the + * culprit. + */ + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "player_id") + public Player getPlayer() { + return player; + } + + @Column(name = "reported_at") + public OffsetDateTime getReportedAt() { + return reportedAt; + } + + /** + * Seconds since the game was launched, derived by the lobby server. Not a simulation tick. + */ + @Column(name = "game_time") + public Integer getGameTime() { + return gameTime; + } +} diff --git a/src/main/java/com/faforever/api/desync/DesyncReportService.java b/src/main/java/com/faforever/api/desync/DesyncReportService.java index de449996d..a878472c6 100644 --- a/src/main/java/com/faforever/api/desync/DesyncReportService.java +++ b/src/main/java/com/faforever/api/desync/DesyncReportService.java @@ -5,6 +5,7 @@ import com.faforever.api.error.ApiException; import com.faforever.api.error.Error; import com.faforever.api.error.ErrorCode; +import com.faforever.api.game.GameDesyncRepository; import com.faforever.api.game.GamePlayerStatsRepository; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; @@ -27,13 +28,16 @@ public class DesyncReportService { private final DesyncReport properties; private final GamePlayerStatsRepository gamePlayerStatsRepository; + private final GameDesyncRepository gameDesyncRepository; private final S3Presigner s3Presigner; public DesyncReportService(FafApiProperties properties, GamePlayerStatsRepository gamePlayerStatsRepository, + GameDesyncRepository gameDesyncRepository, @Qualifier("desyncReportS3Presigner") S3Presigner s3Presigner) { this.properties = properties.getDesyncReport(); this.gamePlayerStatsRepository = gamePlayerStatsRepository; + this.gameDesyncRepository = gameDesyncRepository; this.s3Presigner = s3Presigner; } @@ -55,7 +59,7 @@ public DesyncReportUploadResponse createUploadUrl(int playerId, int gameId, long throw new AccessDeniedException("Player %d did not participate in game %d".formatted(playerId, gameId)); } - if (properties.isRequireGpgnetReport() && gamePlayerStatsRepository.countGpgnetDesyncReports(gameId, playerId) == 0) { + if (properties.isRequireGpgnetReport() && !gameDesyncRepository.existsByGameIdAndPlayerId(gameId, playerId)) { throw new AccessDeniedException("No desync was reported for player %d in game %d".formatted(playerId, gameId)); } diff --git a/src/main/java/com/faforever/api/game/GameDesyncRepository.java b/src/main/java/com/faforever/api/game/GameDesyncRepository.java new file mode 100644 index 000000000..029f0c6d3 --- /dev/null +++ b/src/main/java/com/faforever/api/game/GameDesyncRepository.java @@ -0,0 +1,8 @@ +package com.faforever.api.game; + +import com.faforever.api.data.domain.GameDesync; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface GameDesyncRepository extends JpaRepository { + boolean existsByGameIdAndPlayerId(int gameId, int playerId); +} diff --git a/src/main/java/com/faforever/api/game/GamePlayerStatsRepository.java b/src/main/java/com/faforever/api/game/GamePlayerStatsRepository.java index 378b61155..b9ec9dbfc 100644 --- a/src/main/java/com/faforever/api/game/GamePlayerStatsRepository.java +++ b/src/main/java/com/faforever/api/game/GamePlayerStatsRepository.java @@ -4,17 +4,9 @@ import com.faforever.api.data.domain.Player; import com.faforever.api.data.domain.Validity; import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Query; -import org.springframework.data.repository.query.Param; public interface GamePlayerStatsRepository extends JpaRepository { int countByPlayerAndGameValidity(Player player, Validity validity); boolean existsByGameIdAndPlayerId(int gameId, int playerId); - - /** - * Counts the desync reports that the lobby server has recorded for a player in a game (table {@code game_desync}). - */ - @Query(value = "SELECT COUNT(*) FROM game_desync WHERE game_id = :gameId AND player_id = :playerId", nativeQuery = true) - long countGpgnetDesyncReports(@Param("gameId") int gameId, @Param("playerId") int playerId); } diff --git a/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java b/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java index b43eadcd7..f3df7c3f7 100644 --- a/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java +++ b/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java @@ -3,6 +3,7 @@ import com.faforever.api.config.FafApiProperties; import com.faforever.api.error.ApiException; import com.faforever.api.error.ErrorCode; +import com.faforever.api.game.GameDesyncRepository; import com.faforever.api.game.GamePlayerStatsRepository; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -25,8 +26,6 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.startsWith; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @@ -38,6 +37,9 @@ class DesyncReportServiceTest { @Mock private GamePlayerStatsRepository gamePlayerStatsRepository; + @Mock + private GameDesyncRepository gameDesyncRepository; + private FafApiProperties properties; private DesyncReportService instance; @@ -55,7 +57,7 @@ void setUp() { .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) .build(); - instance = new DesyncReportService(properties, gamePlayerStatsRepository, presigner); + instance = new DesyncReportService(properties, gamePlayerStatsRepository, gameDesyncRepository, presigner); } @Test @@ -105,14 +107,14 @@ void gpgnetReportNotCheckedByDefault() { instance.createUploadUrl(PLAYER_ID, GAME_ID, 100); - verify(gamePlayerStatsRepository, never()).countGpgnetDesyncReports(GAME_ID, PLAYER_ID); + verifyNoInteractions(gameDesyncRepository); } @Test void gpgnetReportRequiredButMissing() { properties.getDesyncReport().setRequireGpgnetReport(true); when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); - when(gamePlayerStatsRepository.countGpgnetDesyncReports(GAME_ID, PLAYER_ID)).thenReturn(0L); + when(gameDesyncRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(false); assertThrows(AccessDeniedException.class, () -> instance.createUploadUrl(PLAYER_ID, GAME_ID, 100)); } @@ -121,7 +123,7 @@ void gpgnetReportRequiredButMissing() { void gpgnetReportRequiredAndPresent() { properties.getDesyncReport().setRequireGpgnetReport(true); when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); - when(gamePlayerStatsRepository.countGpgnetDesyncReports(GAME_ID, PLAYER_ID)).thenReturn(1L); + when(gameDesyncRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); DesyncReportUploadResponse response = instance.createUploadUrl(PLAYER_ID, GAME_ID, 100); From 65ae522b62826fdeb3307fcb9bfc18b16248f531 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Mon, 16 Nov 2026 23:41:43 +0100 Subject: [PATCH 04/14] Accept desync reports as upload and forward them to the object storage Replace the presigned url flow: the API now receives the report as multipart upload, enforces the size limit itself, stores it in the desync bucket and only then marks it as uploaded in game_desync (report_uploaded_at), inserting the row if the lobby server did not record a GPGNet desync for the player. Needs the report_uploaded_at column from faf-db-migrations V148. Co-Authored-By: Claude Sonnet 5.5 --- .../api/config/FafApiProperties.java | 8 +- .../com/faforever/api/config/S3Config.java | 8 +- .../faforever/api/data/domain/GameDesync.java | 9 ++ .../api/desync/DesyncReportController.java | 27 +++-- .../api/desync/DesyncReportService.java | 60 ++++------ .../desync/DesyncReportUploadResponse.java | 11 -- .../api/game/GameDesyncRepository.java | 19 ++- src/main/resources/config/application.yml | 1 - .../api/desync/DesyncReportServiceTest.java | 108 ++++++++---------- 9 files changed, 117 insertions(+), 134 deletions(-) delete mode 100644 src/main/java/com/faforever/api/desync/DesyncReportUploadResponse.java diff --git a/src/main/java/com/faforever/api/config/FafApiProperties.java b/src/main/java/com/faforever/api/config/FafApiProperties.java index 90572dddb..47fb5d7dd 100644 --- a/src/main/java/com/faforever/api/config/FafApiProperties.java +++ b/src/main/java/com/faforever/api/config/FafApiProperties.java @@ -315,14 +315,8 @@ public static class DesyncReport { private String accessKey; private String secretKey; /** - * Upper limit for the size of a single desync report. The size is signed into the upload url, so the object storage - * rejects any upload of a different size. + * Upper limit for the size of a single desync report. */ private DataSize maxUploadSize = DataSize.ofMegabytes(10); - /** - * If enabled, a player can only upload a report for games in which the lobby server has recorded a desync report (via - * GPGNet) for them. - */ - private boolean requireGpgnetReport = false; } } diff --git a/src/main/java/com/faforever/api/config/S3Config.java b/src/main/java/com/faforever/api/config/S3Config.java index c8256138b..a52ccfb0d 100644 --- a/src/main/java/com/faforever/api/config/S3Config.java +++ b/src/main/java/com/faforever/api/config/S3Config.java @@ -20,6 +20,7 @@ public class S3Config { private final FafApiProperties properties; @Bean + @Primary public S3Client s3Client() { return S3Client.builder() .endpointOverride(URI.create(properties.getS3().getEndpoint())) @@ -34,7 +35,6 @@ public S3Client s3Client() { } @Bean - @Primary public S3Presigner s3Presigner() { return S3Presigner.builder() .endpointOverride(URI.create(properties.getS3().getEndpoint())) @@ -50,12 +50,12 @@ public S3Presigner s3Presigner() { } /** - * Presigner for the dedicated desync report bucket, which has its own endpoint and credentials. + * Client for the dedicated desync report bucket, which has its own endpoint and credentials. */ @Bean - public S3Presigner desyncReportS3Presigner() { + public S3Client desyncReportS3Client() { FafApiProperties.DesyncReport desyncReport = properties.getDesyncReport(); - return S3Presigner.builder() + return S3Client.builder() .endpointOverride(URI.create(desyncReport.getEndpoint())) .region(Region.of(desyncReport.getRegion())) .credentialsProvider(StaticCredentialsProvider.create( diff --git a/src/main/java/com/faforever/api/data/domain/GameDesync.java b/src/main/java/com/faforever/api/data/domain/GameDesync.java index 1a67173ca..f97888d2e 100644 --- a/src/main/java/com/faforever/api/data/domain/GameDesync.java +++ b/src/main/java/com/faforever/api/data/domain/GameDesync.java @@ -38,6 +38,7 @@ public class GameDesync { private Player player; private OffsetDateTime reportedAt; private Integer gameTime; + private OffsetDateTime reportUploadedAt; @Id @Column(name = "id") @@ -74,4 +75,12 @@ public OffsetDateTime getReportedAt() { public Integer getGameTime() { return gameTime; } + + /** + * Time the player uploaded the desync report file through the API, null if there is none. + */ + @Column(name = "report_uploaded_at") + public OffsetDateTime getReportUploadedAt() { + return reportUploadedAt; + } } diff --git a/src/main/java/com/faforever/api/desync/DesyncReportController.java b/src/main/java/com/faforever/api/desync/DesyncReportController.java index 53402cd7e..4568c66f6 100644 --- a/src/main/java/com/faforever/api/desync/DesyncReportController.java +++ b/src/main/java/com/faforever/api/desync/DesyncReportController.java @@ -5,15 +5,18 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.responses.ApiResponse; import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.core.Authentication; -import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; -import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; +import java.io.IOException; @RestController @RequestMapping(path = "/game/{gameId}/desync") @@ -23,17 +26,17 @@ public class DesyncReportController { private final PlayerService playerService; private final DesyncReportService desyncReportService; - @Operation(summary = "Get a presigned url to upload the desync report of the authenticated player for a game", - description = "The report has to be uploaded with a PUT request to the returned url, with exactly 'size' bytes as body and " - + "the returned headers. A repeated request for the same game replaces the previous report.") - @ApiResponse(responseCode = "200", description = "Upload url created") + @Operation(summary = "Upload the desync report (a zip file) of the authenticated player for a game", + description = "A repeated upload for the same game replaces the previous report.") + @ApiResponse(responseCode = "204", description = "Report stored") @ApiResponse(responseCode = "403", description = "The player did not participate in this game") - @ApiResponse(responseCode = "422", description = "The size is not within the allowed range") - @GetMapping(path = "/upload", produces = APPLICATION_JSON_VALUE) + @ApiResponse(responseCode = "422", description = "The file is empty or larger than the allowed size") + @PostMapping(path = "/report", consumes = "multipart/form-data") + @ResponseStatus(HttpStatus.NO_CONTENT) @PreAuthorize("hasScope('" + OAuthScope._UPLOAD_DESYNC_REPORT + "')") - public DesyncReportUploadResponse getUploadUrl(Authentication authentication, - @PathVariable("gameId") int gameId, - @RequestParam("size") long size) { - return desyncReportService.createUploadUrl(playerService.getPlayer(authentication).getId(), gameId, size); + public void uploadReport(Authentication authentication, + @PathVariable("gameId") int gameId, + @RequestParam("file") MultipartFile file) throws IOException { + desyncReportService.storeReport(playerService.getPlayer(authentication).getId(), gameId, file); } } diff --git a/src/main/java/com/faforever/api/desync/DesyncReportService.java b/src/main/java/com/faforever/api/desync/DesyncReportService.java index a878472c6..bb0234e52 100644 --- a/src/main/java/com/faforever/api/desync/DesyncReportService.java +++ b/src/main/java/com/faforever/api/desync/DesyncReportService.java @@ -11,34 +11,32 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.security.access.AccessDeniedException; import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.presigner.S3Presigner; -import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest; -import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest; -import java.time.Duration; -import java.util.LinkedHashMap; -import java.util.Map; +import java.io.IOException; +import java.io.InputStream; @Slf4j @Service public class DesyncReportService { static final String CONTENT_TYPE = "application/zip"; - private static final Duration URL_LIFETIME = Duration.ofMinutes(15); private final DesyncReport properties; private final GamePlayerStatsRepository gamePlayerStatsRepository; private final GameDesyncRepository gameDesyncRepository; - private final S3Presigner s3Presigner; + private final S3Client s3Client; public DesyncReportService(FafApiProperties properties, GamePlayerStatsRepository gamePlayerStatsRepository, GameDesyncRepository gameDesyncRepository, - @Qualifier("desyncReportS3Presigner") S3Presigner s3Presigner) { + @Qualifier("desyncReportS3Client") S3Client s3Client) { this.properties = properties.getDesyncReport(); this.gamePlayerStatsRepository = gamePlayerStatsRepository; this.gameDesyncRepository = gameDesyncRepository; - this.s3Presigner = s3Presigner; + this.s3Client = s3Client; } static String getObjectKey(int gameId, int playerId) { @@ -46,11 +44,12 @@ static String getObjectKey(int gameId, int playerId) { } /** - * Creates a presigned url that allows the player to upload exactly one desync report of exactly {@code size} bytes. - * The size is part of the signature, so the object storage rejects uploads with any other size. + * Stores the desync report of a player and marks it as uploaded. A repeated upload for the same game replaces the + * previous report. */ - public DesyncReportUploadResponse createUploadUrl(int playerId, int gameId, long size) { + public void storeReport(int playerId, int gameId, MultipartFile report) throws IOException { long maxSize = properties.getMaxUploadSize().toBytes(); + long size = report.getSize(); if (size < 1 || size > maxSize) { throw new ApiException(new Error(ErrorCode.DESYNC_REPORT_INVALID_SIZE, maxSize)); } @@ -59,32 +58,19 @@ public DesyncReportUploadResponse createUploadUrl(int playerId, int gameId, long throw new AccessDeniedException("Player %d did not participate in game %d".formatted(playerId, gameId)); } - if (properties.isRequireGpgnetReport() && !gameDesyncRepository.existsByGameIdAndPlayerId(gameId, playerId)) { - throw new AccessDeniedException("No desync was reported for player %d in game %d".formatted(playerId, gameId)); - } - String key = getObjectKey(gameId, playerId); - log.info("Player {} requested a desync report upload url for game {} ({} bytes)", playerId, gameId, size); - - PutObjectRequest putObjectRequest = PutObjectRequest.builder() - .bucket(properties.getBucket()) - .key(key) - .contentLength(size) - .contentType(CONTENT_TYPE) - .build(); + log.info("Player {} uploads a desync report for game {} ({} bytes)", playerId, gameId, size); - PresignedPutObjectRequest presigned = s3Presigner.presignPutObject(PutObjectPresignRequest.builder() - .signatureDuration(URL_LIFETIME) - .putObjectRequest(putObjectRequest) - .build()); - - Map headers = new LinkedHashMap<>(); - presigned.signedHeaders().forEach((name, values) -> { - if (!"host".equalsIgnoreCase(name) && !values.isEmpty()) { - headers.put(name, values.getFirst()); - } - }); + try (InputStream inputStream = report.getInputStream()) { + s3Client.putObject(PutObjectRequest.builder() + .bucket(properties.getBucket()) + .key(key) + .contentType(CONTENT_TYPE) + .contentLength(size) + .build(), RequestBody.fromInputStream(inputStream, size)); + } - return new DesyncReportUploadResponse(presigned.url().toString(), key, headers); + // only after the file is stored, so the column can be relied on to filter for existing reports + gameDesyncRepository.markReportUploaded(gameId, playerId); } } diff --git a/src/main/java/com/faforever/api/desync/DesyncReportUploadResponse.java b/src/main/java/com/faforever/api/desync/DesyncReportUploadResponse.java deleted file mode 100644 index 54ad5ac3c..000000000 --- a/src/main/java/com/faforever/api/desync/DesyncReportUploadResponse.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.faforever.api.desync; - -import java.util.Map; - -/** - * @param url the presigned url to PUT the report to - * @param key the object key the report will be stored under - * @param headers headers that have to be sent with the PUT request exactly as given (they are part of the signature) - */ -public record DesyncReportUploadResponse(String url, String key, Map headers) { -} diff --git a/src/main/java/com/faforever/api/game/GameDesyncRepository.java b/src/main/java/com/faforever/api/game/GameDesyncRepository.java index 029f0c6d3..bbc63df45 100644 --- a/src/main/java/com/faforever/api/game/GameDesyncRepository.java +++ b/src/main/java/com/faforever/api/game/GameDesyncRepository.java @@ -2,7 +2,24 @@ import com.faforever.api.data.domain.GameDesync; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; public interface GameDesyncRepository extends JpaRepository { - boolean existsByGameIdAndPlayerId(int gameId, int playerId); + + /** + * Marks the desync report of a player as uploaded. The lobby server records desyncs reported via GPGNet in the same + * table, so the row is created if it does not exist yet. This is a single atomic statement, so it is safe against the + * lobby server inserting the row at the same time. + */ + @Transactional + @Modifying + @Query(value = """ + INSERT INTO game_desync (game_id, player_id, reported_at, report_uploaded_at) + VALUES (:gameId, :playerId, NOW(3), NOW(3)) + ON DUPLICATE KEY UPDATE report_uploaded_at = NOW(3) + """, nativeQuery = true) + void markReportUploaded(@Param("gameId") int gameId, @Param("playerId") int playerId); } diff --git a/src/main/resources/config/application.yml b/src/main/resources/config/application.yml index 7735743a8..47d609f04 100644 --- a/src/main/resources/config/application.yml +++ b/src/main/resources/config/application.yml @@ -101,7 +101,6 @@ faf-api: region: ${DESYNC_REPORT_S3_REGION:auto} bucket: ${DESYNC_REPORT_S3_BUCKET:desync-reports} max-upload-size: ${DESYNC_REPORT_MAX_UPLOAD_SIZE:10MB} - require-gpgnet-report: ${DESYNC_REPORT_REQUIRE_GPGNET_REPORT:false} spring: application: diff --git a/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java b/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java index f3df7c3f7..9ff853420 100644 --- a/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java +++ b/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java @@ -8,24 +8,25 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.mock.web.MockMultipartFile; import org.springframework.security.access.AccessDeniedException; import org.springframework.util.unit.DataSize; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Configuration; -import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; -import java.net.URI; +import java.io.IOException; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.startsWith; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @@ -36,97 +37,82 @@ class DesyncReportServiceTest { @Mock private GamePlayerStatsRepository gamePlayerStatsRepository; - @Mock private GameDesyncRepository gameDesyncRepository; + @Mock + private S3Client s3Client; - private FafApiProperties properties; private DesyncReportService instance; @BeforeEach void setUp() { - properties = new FafApiProperties(); + FafApiProperties properties = new FafApiProperties(); properties.getDesyncReport().setBucket("desync-reports"); - properties.getDesyncReport().setMaxUploadSize(DataSize.ofMegabytes(10)); + properties.getDesyncReport().setMaxUploadSize(DataSize.ofBytes(1000)); - // signing is done locally, so a real presigner with dummy credentials does not need any network access - S3Presigner presigner = S3Presigner.builder() - .endpointOverride(URI.create("http://localhost:9000")) - .region(Region.of("auto")) - .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("admin", "banana123"))) - .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) - .build(); + instance = new DesyncReportService(properties, gamePlayerStatsRepository, gameDesyncRepository, s3Client); + } - instance = new DesyncReportService(properties, gamePlayerStatsRepository, gameDesyncRepository, presigner); + private static MockMultipartFile report(int size) { + return new MockMultipartFile("file", "dump.zip", "application/zip", new byte[size]); } @Test - void createUploadUrlSignsKeyAndSize() { + void storeReportStoresFileThenMarksUploaded() throws IOException { when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); - DesyncReportUploadResponse response = instance.createUploadUrl(PLAYER_ID, GAME_ID, 1_200_000); - - assertThat(response.key(), is("desync/1234/42.zip")); - assertThat(response.url(), startsWith("http://localhost:9000/desync-reports/desync/1234/42.zip?")); - // the size and content type are part of the signature, so the storage rejects anything else - assertThat(response.url(), containsString("content-length")); - assertThat(response.headers(), hasEntry("content-length", "1200000")); - assertThat(response.headers(), hasEntry("content-type", "application/zip")); - assertThat(response.headers().containsKey("host"), is(false)); - } - - @Test - void createUploadUrlSizeOutOfRange() { - ApiException tooLarge = assertThrows(ApiException.class, - () -> instance.createUploadUrl(PLAYER_ID, GAME_ID, DataSize.ofMegabytes(10).toBytes() + 1)); - assertThat(tooLarge.getErrors()[0].getErrorCode(), is(ErrorCode.DESYNC_REPORT_INVALID_SIZE)); + instance.storeReport(PLAYER_ID, GAME_ID, report(500)); - ApiException empty = assertThrows(ApiException.class, () -> instance.createUploadUrl(PLAYER_ID, GAME_ID, 0)); - assertThat(empty.getErrors()[0].getErrorCode(), is(ErrorCode.DESYNC_REPORT_INVALID_SIZE)); + ArgumentCaptor request = ArgumentCaptor.forClass(PutObjectRequest.class); + InOrder inOrder = inOrder(s3Client, gameDesyncRepository); + inOrder.verify(s3Client).putObject(request.capture(), any(RequestBody.class)); + inOrder.verify(gameDesyncRepository).markReportUploaded(GAME_ID, PLAYER_ID); - verifyNoInteractions(gamePlayerStatsRepository); + assertThat(request.getValue().bucket(), is("desync-reports")); + assertThat(request.getValue().key(), is("desync/1234/42.zip")); + assertThat(request.getValue().contentLength(), is(500L)); + assertThat(request.getValue().contentType(), is("application/zip")); } @Test - void createUploadUrlAtMaxSize() { + void storeReportAtMaxSize() throws IOException { when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); - instance.createUploadUrl(PLAYER_ID, GAME_ID, DataSize.ofMegabytes(10).toBytes()); + instance.storeReport(PLAYER_ID, GAME_ID, report(1000)); } @Test - void createUploadUrlPlayerNotInGame() { - when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(false); + void storeReportTooLarge() { + ApiException exception = assertThrows(ApiException.class, () -> instance.storeReport(PLAYER_ID, GAME_ID, report(1001))); - assertThrows(AccessDeniedException.class, () -> instance.createUploadUrl(PLAYER_ID, GAME_ID, 100)); + assertThat(exception.getErrors()[0].getErrorCode(), is(ErrorCode.DESYNC_REPORT_INVALID_SIZE)); + verifyNoInteractions(gamePlayerStatsRepository, s3Client, gameDesyncRepository); } @Test - void gpgnetReportNotCheckedByDefault() { - when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); - - instance.createUploadUrl(PLAYER_ID, GAME_ID, 100); + void storeReportEmpty() { + ApiException exception = assertThrows(ApiException.class, () -> instance.storeReport(PLAYER_ID, GAME_ID, report(0))); - verifyNoInteractions(gameDesyncRepository); + assertThat(exception.getErrors()[0].getErrorCode(), is(ErrorCode.DESYNC_REPORT_INVALID_SIZE)); + verifyNoInteractions(gamePlayerStatsRepository, s3Client, gameDesyncRepository); } @Test - void gpgnetReportRequiredButMissing() { - properties.getDesyncReport().setRequireGpgnetReport(true); - when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); - when(gameDesyncRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(false); + void storeReportPlayerNotInGame() { + when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(false); + + assertThrows(AccessDeniedException.class, () -> instance.storeReport(PLAYER_ID, GAME_ID, report(500))); - assertThrows(AccessDeniedException.class, () -> instance.createUploadUrl(PLAYER_ID, GAME_ID, 100)); + verifyNoInteractions(s3Client, gameDesyncRepository); } @Test - void gpgnetReportRequiredAndPresent() { - properties.getDesyncReport().setRequireGpgnetReport(true); + void storeReportNotMarkedWhenStorageFails() { when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); - when(gameDesyncRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); + when(s3Client.putObject(any(PutObjectRequest.class), any(RequestBody.class))).thenThrow(S3Exception.builder().build()); - DesyncReportUploadResponse response = instance.createUploadUrl(PLAYER_ID, GAME_ID, 100); + assertThrows(S3Exception.class, () -> instance.storeReport(PLAYER_ID, GAME_ID, report(500))); - assertThat(response.key(), is("desync/1234/42.zip")); + verifyNoInteractions(gameDesyncRepository); } } From 41594ee313d96357b5e5aab5c582812093e27bf7 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Tue, 17 Nov 2026 00:10:49 +0100 Subject: [PATCH 05/14] Bump faf-db-migrations to v148 Adds game_desync.report_uploaded_at. Co-Authored-By: Claude Sonnet 5.5 --- compose.yaml | 2 +- .../java/com/faforever/api/config/MainDbTestContainers.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose.yaml b/compose.yaml index e234ddddb..47b4fe6c6 100644 --- a/compose.yaml +++ b/compose.yaml @@ -16,7 +16,7 @@ services: - "3306:3306" faf-db-migrations: - image: faforever/faf-db-migrations:v147 + image: faforever/faf-db-migrations:v148 command: migrate environment: FLYWAY_URL: jdbc:mysql://faf-db/faf?useSSL=false diff --git a/src/inttest/java/com/faforever/api/config/MainDbTestContainers.java b/src/inttest/java/com/faforever/api/config/MainDbTestContainers.java index 1c7f15b72..23e18f0d9 100644 --- a/src/inttest/java/com/faforever/api/config/MainDbTestContainers.java +++ b/src/inttest/java/com/faforever/api/config/MainDbTestContainers.java @@ -23,7 +23,7 @@ @Configuration public class MainDbTestContainers { private static final MariaDBContainer fafDBContainer = new MariaDBContainer<>("mariadb:11.7"); - private static final GenericContainer flywayMigrationsContainer = new GenericContainer<>("faforever/faf-db-migrations:v147"); + private static final GenericContainer flywayMigrationsContainer = new GenericContainer<>("faforever/faf-db-migrations:v148"); private static final Network sharedNetwork = Network.newNetwork(); @Bean From 9b547139161d016d272ab6b5169d44aaac86cc8f Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Tue, 17 Nov 2026 00:14:49 +0100 Subject: [PATCH 06/14] Expose the report file name of a desync as computed attribute Co-Authored-By: Claude Sonnet 5.5 --- .../faforever/api/data/domain/GameDesync.java | 16 ++++++++++++++ .../api/desync/DesyncReportService.java | 7 ++---- .../api/data/domain/GameDesyncTest.java | 22 +++++++++++++++++++ 3 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 src/test/java/com/faforever/api/data/domain/GameDesyncTest.java diff --git a/src/main/java/com/faforever/api/data/domain/GameDesync.java b/src/main/java/com/faforever/api/data/domain/GameDesync.java index f97888d2e..a68c92ba5 100644 --- a/src/main/java/com/faforever/api/data/domain/GameDesync.java +++ b/src/main/java/com/faforever/api/data/domain/GameDesync.java @@ -1,6 +1,7 @@ package com.faforever.api.data.domain; import com.faforever.api.data.checks.Prefab; +import com.yahoo.elide.annotation.ComputedAttribute; import com.yahoo.elide.annotation.CreatePermission; import com.yahoo.elide.annotation.DeletePermission; import com.yahoo.elide.annotation.Include; @@ -14,6 +15,7 @@ import jakarta.persistence.JoinColumn; import jakarta.persistence.ManyToOne; import jakarta.persistence.Table; +import jakarta.persistence.Transient; import lombok.Setter; import org.hibernate.annotations.Immutable; @@ -83,4 +85,18 @@ public Integer getGameTime() { public OffsetDateTime getReportUploadedAt() { return reportUploadedAt; } + + /** + * Object key under which the report of this player is stored in the object storage. Only points to an existing file if + * {@link #getReportUploadedAt()} is set. + */ + @ComputedAttribute + @Transient + public String getReportFileName() { + return reportFileName(game.getId(), player.getId()); + } + + public static String reportFileName(int gameId, int playerId) { + return "desync/%d/%d.zip".formatted(gameId, playerId); + } } diff --git a/src/main/java/com/faforever/api/desync/DesyncReportService.java b/src/main/java/com/faforever/api/desync/DesyncReportService.java index bb0234e52..7ef1fb351 100644 --- a/src/main/java/com/faforever/api/desync/DesyncReportService.java +++ b/src/main/java/com/faforever/api/desync/DesyncReportService.java @@ -2,6 +2,7 @@ import com.faforever.api.config.FafApiProperties; import com.faforever.api.config.FafApiProperties.DesyncReport; +import com.faforever.api.data.domain.GameDesync; import com.faforever.api.error.ApiException; import com.faforever.api.error.Error; import com.faforever.api.error.ErrorCode; @@ -39,10 +40,6 @@ public DesyncReportService(FafApiProperties properties, this.s3Client = s3Client; } - static String getObjectKey(int gameId, int playerId) { - return "desync/%d/%d.zip".formatted(gameId, playerId); - } - /** * Stores the desync report of a player and marks it as uploaded. A repeated upload for the same game replaces the * previous report. @@ -58,7 +55,7 @@ public void storeReport(int playerId, int gameId, MultipartFile report) throws I throw new AccessDeniedException("Player %d did not participate in game %d".formatted(playerId, gameId)); } - String key = getObjectKey(gameId, playerId); + String key = GameDesync.reportFileName(gameId, playerId); log.info("Player {} uploads a desync report for game {} ({} bytes)", playerId, gameId, size); try (InputStream inputStream = report.getInputStream()) { diff --git a/src/test/java/com/faforever/api/data/domain/GameDesyncTest.java b/src/test/java/com/faforever/api/data/domain/GameDesyncTest.java new file mode 100644 index 000000000..a6f62e17a --- /dev/null +++ b/src/test/java/com/faforever/api/data/domain/GameDesyncTest.java @@ -0,0 +1,22 @@ +package com.faforever.api.data.domain; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +class GameDesyncTest { + + @Test + void reportFileName() { + Game game = new Game(); + game.setId(1234); + Player player = new Player(); + player.setId(42); + GameDesync gameDesync = new GameDesync(); + gameDesync.setGame(game); + gameDesync.setPlayer(player); + + assertThat(gameDesync.getReportFileName(), is("desync/1234/42.zip")); + } +} From af7d8a5c7e67f164f3c2ed604a5978b05b0cd953 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Tue, 17 Nov 2026 00:17:14 +0100 Subject: [PATCH 07/14] Fill the desync report file name in an entity listener Follows the enricher pattern of the other entities instead of computing it in the entity. Co-Authored-By: Claude Sonnet 5.5 --- .../faforever/api/data/domain/GameDesync.java | 16 ++++++++-------- .../api/data/listeners/GameDesyncEnricher.java | 15 +++++++++++++++ .../api/desync/DesyncReportService.java | 7 +++++-- .../GameDesyncEnricherTest.java} | 11 ++++++++--- 4 files changed, 36 insertions(+), 13 deletions(-) create mode 100644 src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java rename src/test/java/com/faforever/api/data/{domain/GameDesyncTest.java => listeners/GameDesyncEnricherTest.java} (59%) diff --git a/src/main/java/com/faforever/api/data/domain/GameDesync.java b/src/main/java/com/faforever/api/data/domain/GameDesync.java index a68c92ba5..c35278f9f 100644 --- a/src/main/java/com/faforever/api/data/domain/GameDesync.java +++ b/src/main/java/com/faforever/api/data/domain/GameDesync.java @@ -1,6 +1,7 @@ package com.faforever.api.data.domain; import com.faforever.api.data.checks.Prefab; +import com.faforever.api.data.listeners.GameDesyncEnricher; import com.yahoo.elide.annotation.ComputedAttribute; import com.yahoo.elide.annotation.CreatePermission; import com.yahoo.elide.annotation.DeletePermission; @@ -8,6 +9,7 @@ import com.yahoo.elide.annotation.UpdatePermission; import jakarta.persistence.Column; import jakarta.persistence.Entity; +import jakarta.persistence.EntityListeners; import jakarta.persistence.FetchType; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; @@ -30,6 +32,7 @@ @Include(name = "gameDesync") @Immutable @Setter +@EntityListeners(GameDesyncEnricher.class) @CreatePermission(expression = Prefab.NONE) @UpdatePermission(expression = Prefab.NONE) @DeletePermission(expression = Prefab.NONE) @@ -41,6 +44,7 @@ public class GameDesync { private OffsetDateTime reportedAt; private Integer gameTime; private OffsetDateTime reportUploadedAt; + private String reportFileName; @Id @Column(name = "id") @@ -87,16 +91,12 @@ public OffsetDateTime getReportUploadedAt() { } /** - * Object key under which the report of this player is stored in the object storage. Only points to an existing file if - * {@link #getReportUploadedAt()} is set. + * Object key under which the report of this player is stored in the object storage, filled by the + * {@link GameDesyncEnricher}. Only points to an existing file if {@link #getReportUploadedAt()} is set. */ - @ComputedAttribute @Transient + @ComputedAttribute public String getReportFileName() { - return reportFileName(game.getId(), player.getId()); - } - - public static String reportFileName(int gameId, int playerId) { - return "desync/%d/%d.zip".formatted(gameId, playerId); + return reportFileName; } } diff --git a/src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java b/src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java new file mode 100644 index 000000000..dd6011d8b --- /dev/null +++ b/src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java @@ -0,0 +1,15 @@ +package com.faforever.api.data.listeners; + +import com.faforever.api.data.domain.GameDesync; +import com.faforever.api.desync.DesyncReportService; +import jakarta.persistence.PostLoad; +import org.springframework.stereotype.Component; + +@Component +public class GameDesyncEnricher { + + @PostLoad + public void enrich(GameDesync gameDesync) { + gameDesync.setReportFileName(DesyncReportService.getObjectKey(gameDesync.getGame().getId(), gameDesync.getPlayer().getId())); + } +} diff --git a/src/main/java/com/faforever/api/desync/DesyncReportService.java b/src/main/java/com/faforever/api/desync/DesyncReportService.java index 7ef1fb351..ea54859bd 100644 --- a/src/main/java/com/faforever/api/desync/DesyncReportService.java +++ b/src/main/java/com/faforever/api/desync/DesyncReportService.java @@ -2,7 +2,6 @@ import com.faforever.api.config.FafApiProperties; import com.faforever.api.config.FafApiProperties.DesyncReport; -import com.faforever.api.data.domain.GameDesync; import com.faforever.api.error.ApiException; import com.faforever.api.error.Error; import com.faforever.api.error.ErrorCode; @@ -40,6 +39,10 @@ public DesyncReportService(FafApiProperties properties, this.s3Client = s3Client; } + public static String getObjectKey(int gameId, int playerId) { + return "desync/%d/%d.zip".formatted(gameId, playerId); + } + /** * Stores the desync report of a player and marks it as uploaded. A repeated upload for the same game replaces the * previous report. @@ -55,7 +58,7 @@ public void storeReport(int playerId, int gameId, MultipartFile report) throws I throw new AccessDeniedException("Player %d did not participate in game %d".formatted(playerId, gameId)); } - String key = GameDesync.reportFileName(gameId, playerId); + String key = getObjectKey(gameId, playerId); log.info("Player {} uploads a desync report for game {} ({} bytes)", playerId, gameId, size); try (InputStream inputStream = report.getInputStream()) { diff --git a/src/test/java/com/faforever/api/data/domain/GameDesyncTest.java b/src/test/java/com/faforever/api/data/listeners/GameDesyncEnricherTest.java similarity index 59% rename from src/test/java/com/faforever/api/data/domain/GameDesyncTest.java rename to src/test/java/com/faforever/api/data/listeners/GameDesyncEnricherTest.java index a6f62e17a..1c36e980b 100644 --- a/src/test/java/com/faforever/api/data/domain/GameDesyncTest.java +++ b/src/test/java/com/faforever/api/data/listeners/GameDesyncEnricherTest.java @@ -1,14 +1,17 @@ -package com.faforever.api.data.domain; +package com.faforever.api.data.listeners; +import com.faforever.api.data.domain.Game; +import com.faforever.api.data.domain.GameDesync; +import com.faforever.api.data.domain.Player; import org.junit.jupiter.api.Test; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; -class GameDesyncTest { +class GameDesyncEnricherTest { @Test - void reportFileName() { + void enrichSetsReportFileName() { Game game = new Game(); game.setId(1234); Player player = new Player(); @@ -17,6 +20,8 @@ void reportFileName() { gameDesync.setGame(game); gameDesync.setPlayer(player); + new GameDesyncEnricher().enrich(gameDesync); + assertThat(gameDesync.getReportFileName(), is("desync/1234/42.zip")); } } From acafa2a1719fd57f0f81fa2db64115f5d1321314 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Tue, 17 Nov 2026 00:19:19 +0100 Subject: [PATCH 08/14] Make the desync report file name configurable and build it in the service The key format was hardcoded and the entity listener called a static method. Add faf-api.desync-report.report-file-name-format and let the listener use the service like the other enrichers do. Co-Authored-By: Claude Sonnet 5.5 --- .../faforever/api/config/FafApiProperties.java | 4 ++++ .../api/data/listeners/GameDesyncEnricher.java | 10 +++++++++- .../faforever/api/desync/DesyncReportService.java | 6 +++--- src/main/resources/config/application.yml | 1 + .../data/listeners/GameDesyncEnricherTest.java | 15 ++++++++++++++- .../api/desync/DesyncReportServiceTest.java | 10 ++++++++++ 6 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/faforever/api/config/FafApiProperties.java b/src/main/java/com/faforever/api/config/FafApiProperties.java index 47fb5d7dd..e2713d4df 100644 --- a/src/main/java/com/faforever/api/config/FafApiProperties.java +++ b/src/main/java/com/faforever/api/config/FafApiProperties.java @@ -318,5 +318,9 @@ public static class DesyncReport { * Upper limit for the size of a single desync report. */ private DataSize maxUploadSize = DataSize.ofMegabytes(10); + /** + * Object key of a report in the bucket, with the game id and the player id as arguments. + */ + private String reportFileNameFormat = "desync/%d/%d.zip"; } } diff --git a/src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java b/src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java index dd6011d8b..783a28d5d 100644 --- a/src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java +++ b/src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java @@ -2,14 +2,22 @@ import com.faforever.api.data.domain.GameDesync; import com.faforever.api.desync.DesyncReportService; +import jakarta.inject.Inject; import jakarta.persistence.PostLoad; import org.springframework.stereotype.Component; @Component public class GameDesyncEnricher { + private static DesyncReportService desyncReportService; + + @Inject + public void init(DesyncReportService desyncReportService) { + GameDesyncEnricher.desyncReportService = desyncReportService; + } + @PostLoad public void enrich(GameDesync gameDesync) { - gameDesync.setReportFileName(DesyncReportService.getObjectKey(gameDesync.getGame().getId(), gameDesync.getPlayer().getId())); + gameDesync.setReportFileName(desyncReportService.getReportFileName(gameDesync.getGame().getId(), gameDesync.getPlayer().getId())); } } diff --git a/src/main/java/com/faforever/api/desync/DesyncReportService.java b/src/main/java/com/faforever/api/desync/DesyncReportService.java index ea54859bd..ae466daf1 100644 --- a/src/main/java/com/faforever/api/desync/DesyncReportService.java +++ b/src/main/java/com/faforever/api/desync/DesyncReportService.java @@ -39,8 +39,8 @@ public DesyncReportService(FafApiProperties properties, this.s3Client = s3Client; } - public static String getObjectKey(int gameId, int playerId) { - return "desync/%d/%d.zip".formatted(gameId, playerId); + public String getReportFileName(int gameId, int playerId) { + return properties.getReportFileNameFormat().formatted(gameId, playerId); } /** @@ -58,7 +58,7 @@ public void storeReport(int playerId, int gameId, MultipartFile report) throws I throw new AccessDeniedException("Player %d did not participate in game %d".formatted(playerId, gameId)); } - String key = getObjectKey(gameId, playerId); + String key = getReportFileName(gameId, playerId); log.info("Player {} uploads a desync report for game {} ({} bytes)", playerId, gameId, size); try (InputStream inputStream = report.getInputStream()) { diff --git a/src/main/resources/config/application.yml b/src/main/resources/config/application.yml index 47d609f04..d7209d7f4 100644 --- a/src/main/resources/config/application.yml +++ b/src/main/resources/config/application.yml @@ -101,6 +101,7 @@ faf-api: region: ${DESYNC_REPORT_S3_REGION:auto} bucket: ${DESYNC_REPORT_S3_BUCKET:desync-reports} max-upload-size: ${DESYNC_REPORT_MAX_UPLOAD_SIZE:10MB} + report-file-name-format: ${DESYNC_REPORT_FILE_NAME_FORMAT:desync/%d/%d.zip} spring: application: diff --git a/src/test/java/com/faforever/api/data/listeners/GameDesyncEnricherTest.java b/src/test/java/com/faforever/api/data/listeners/GameDesyncEnricherTest.java index 1c36e980b..8b22e7c2f 100644 --- a/src/test/java/com/faforever/api/data/listeners/GameDesyncEnricherTest.java +++ b/src/test/java/com/faforever/api/data/listeners/GameDesyncEnricherTest.java @@ -3,13 +3,22 @@ import com.faforever.api.data.domain.Game; import com.faforever.api.data.domain.GameDesync; import com.faforever.api.data.domain.Player; +import com.faforever.api.desync.DesyncReportService; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.when; +@ExtendWith(MockitoExtension.class) class GameDesyncEnricherTest { + @Mock + private DesyncReportService desyncReportService; + @Test void enrichSetsReportFileName() { Game game = new Game(); @@ -20,7 +29,11 @@ void enrichSetsReportFileName() { gameDesync.setGame(game); gameDesync.setPlayer(player); - new GameDesyncEnricher().enrich(gameDesync); + when(desyncReportService.getReportFileName(1234, 42)).thenReturn("desync/1234/42.zip"); + GameDesyncEnricher enricher = new GameDesyncEnricher(); + enricher.init(desyncReportService); + + enricher.enrich(gameDesync); assertThat(gameDesync.getReportFileName(), is("desync/1234/42.zip")); } diff --git a/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java b/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java index 9ff853420..009102112 100644 --- a/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java +++ b/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java @@ -74,6 +74,16 @@ void storeReportStoresFileThenMarksUploaded() throws IOException { assertThat(request.getValue().contentType(), is("application/zip")); } + @Test + void reportFileNameUsesConfiguredFormat() { + FafApiProperties properties = new FafApiProperties(); + properties.getDesyncReport().setReportFileNameFormat("dumps/%d-%d.zip"); + DesyncReportService custom = new DesyncReportService(properties, gamePlayerStatsRepository, gameDesyncRepository, s3Client); + + assertThat(custom.getReportFileName(1234, 42), is("dumps/1234-42.zip")); + assertThat(instance.getReportFileName(1234, 42), is("desync/1234/42.zip")); + } + @Test void storeReportAtMaxSize() throws IOException { when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); From cbe39c7d5f479694893f34808cffac95731658b1 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Tue, 17 Nov 2026 00:23:00 +0100 Subject: [PATCH 09/14] Use constructor injection in GameDesyncEnricher The faf entity manager factory uses a SpringBeanContainer, so entity listeners are created by Spring and do not need static field injection. Co-Authored-By: Claude Sonnet 5.5 --- .../api/data/listeners/GameDesyncEnricher.java | 10 +++------- .../api/data/listeners/GameDesyncEnricherTest.java | 3 +-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java b/src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java index 783a28d5d..c7efa418d 100644 --- a/src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java +++ b/src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java @@ -2,19 +2,15 @@ import com.faforever.api.data.domain.GameDesync; import com.faforever.api.desync.DesyncReportService; -import jakarta.inject.Inject; import jakarta.persistence.PostLoad; +import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; @Component +@RequiredArgsConstructor public class GameDesyncEnricher { - private static DesyncReportService desyncReportService; - - @Inject - public void init(DesyncReportService desyncReportService) { - GameDesyncEnricher.desyncReportService = desyncReportService; - } + private final DesyncReportService desyncReportService; @PostLoad public void enrich(GameDesync gameDesync) { diff --git a/src/test/java/com/faforever/api/data/listeners/GameDesyncEnricherTest.java b/src/test/java/com/faforever/api/data/listeners/GameDesyncEnricherTest.java index 8b22e7c2f..be929dedd 100644 --- a/src/test/java/com/faforever/api/data/listeners/GameDesyncEnricherTest.java +++ b/src/test/java/com/faforever/api/data/listeners/GameDesyncEnricherTest.java @@ -30,8 +30,7 @@ void enrichSetsReportFileName() { gameDesync.setPlayer(player); when(desyncReportService.getReportFileName(1234, 42)).thenReturn("desync/1234/42.zip"); - GameDesyncEnricher enricher = new GameDesyncEnricher(); - enricher.init(desyncReportService); + GameDesyncEnricher enricher = new GameDesyncEnricher(desyncReportService); enricher.enrich(gameDesync); From 2d0403a5438239621a120fd8c96d2806cca6f702 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Tue, 17 Nov 2026 00:39:28 +0100 Subject: [PATCH 10/14] Revert "Use constructor injection in GameDesyncEnricher" The listener is created while the entity manager factory is built, so it cannot depend on beans that need the entity manager factory (circular dependency). That is why the other entity listeners use static injection. Co-Authored-By: Claude Sonnet 5.5 --- .../api/data/listeners/GameDesyncEnricher.java | 10 +++++++--- .../api/data/listeners/GameDesyncEnricherTest.java | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java b/src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java index c7efa418d..783a28d5d 100644 --- a/src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java +++ b/src/main/java/com/faforever/api/data/listeners/GameDesyncEnricher.java @@ -2,15 +2,19 @@ import com.faforever.api.data.domain.GameDesync; import com.faforever.api.desync.DesyncReportService; +import jakarta.inject.Inject; import jakarta.persistence.PostLoad; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; @Component -@RequiredArgsConstructor public class GameDesyncEnricher { - private final DesyncReportService desyncReportService; + private static DesyncReportService desyncReportService; + + @Inject + public void init(DesyncReportService desyncReportService) { + GameDesyncEnricher.desyncReportService = desyncReportService; + } @PostLoad public void enrich(GameDesync gameDesync) { diff --git a/src/test/java/com/faforever/api/data/listeners/GameDesyncEnricherTest.java b/src/test/java/com/faforever/api/data/listeners/GameDesyncEnricherTest.java index be929dedd..8b22e7c2f 100644 --- a/src/test/java/com/faforever/api/data/listeners/GameDesyncEnricherTest.java +++ b/src/test/java/com/faforever/api/data/listeners/GameDesyncEnricherTest.java @@ -30,7 +30,8 @@ void enrichSetsReportFileName() { gameDesync.setPlayer(player); when(desyncReportService.getReportFileName(1234, 42)).thenReturn("desync/1234/42.zip"); - GameDesyncEnricher enricher = new GameDesyncEnricher(desyncReportService); + GameDesyncEnricher enricher = new GameDesyncEnricher(); + enricher.init(desyncReportService); enricher.enrich(gameDesync); From 7f764483777d21ff6d99888281b7e05569bb08ec Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Tue, 17 Nov 2026 10:57:31 +0100 Subject: [PATCH 11/14] Assert the outcome in the max size desync report test Co-Authored-By: Claude Sonnet 5.5 --- .../com/faforever/api/desync/DesyncReportServiceTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java b/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java index 009102112..b699fb531 100644 --- a/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java +++ b/src/test/java/com/faforever/api/desync/DesyncReportServiceTest.java @@ -27,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @@ -89,6 +90,9 @@ void storeReportAtMaxSize() throws IOException { when(gamePlayerStatsRepository.existsByGameIdAndPlayerId(GAME_ID, PLAYER_ID)).thenReturn(true); instance.storeReport(PLAYER_ID, GAME_ID, report(1000)); + + verify(s3Client).putObject(any(PutObjectRequest.class), any(RequestBody.class)); + verify(gameDesyncRepository).markReportUploaded(GAME_ID, PLAYER_ID); } @Test From a8a4b580cfa3e88b9901e0396b7053a4280ab8b2 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Tue, 17 Nov 2026 10:59:30 +0100 Subject: [PATCH 12/14] Require db schema version 148 Co-Authored-By: Claude Sonnet 5.5 --- src/main/resources/config/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/config/application.yml b/src/main/resources/config/application.yml index d7209d7f4..bc16f22c0 100644 --- a/src/main/resources/config/application.yml +++ b/src/main/resources/config/application.yml @@ -16,7 +16,7 @@ faf-api: hmac-secret: ${CLOUDFLARE_HMAC_SECRET} hmac-param: ${CLOUDFLARE_HMAC_PARAM:verify} database: - schema-version: ${DATABASE_SCHEMA_VERSION:140} + schema-version: ${DATABASE_SCHEMA_VERSION:148} deployment: forged-alliance-exe-path: ${FORGED_ALLIANCE_EXE_PATH:/content/legacy-featured-mod-files/updates_faf_files/ForgedAlliance.exe} repositories-directory: ${REPOSITORIES_DIRECTORY:/repositories} From 1cbd764d8e30be331f521f6593f2c61605b1a3d3 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Tue, 17 Nov 2026 21:31:29 +0100 Subject: [PATCH 13/14] Disable chunked encoding on the desync report client Co-Authored-By: Claude Sonnet 5.5 --- src/main/java/com/faforever/api/config/S3Config.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/faforever/api/config/S3Config.java b/src/main/java/com/faforever/api/config/S3Config.java index a52ccfb0d..d22591c04 100644 --- a/src/main/java/com/faforever/api/config/S3Config.java +++ b/src/main/java/com/faforever/api/config/S3Config.java @@ -63,6 +63,7 @@ public S3Client desyncReportS3Client() { )) .serviceConfiguration(S3Configuration.builder() .pathStyleAccessEnabled(true) + .chunkedEncodingEnabled(false) // R2 rejects the chunked signature of putObject with 403 .build()) .build(); } From 72f1c2684c8cb5c0937c6848ce52a922ab7bd577 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Tue, 17 Nov 2026 21:48:05 +0100 Subject: [PATCH 14/14] Upload desync reports from a repeatable stream Co-Authored-By: Claude Sonnet 5.5 --- .../api/desync/DesyncReportService.java | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/faforever/api/desync/DesyncReportService.java b/src/main/java/com/faforever/api/desync/DesyncReportService.java index ae466daf1..8d4f2d4ef 100644 --- a/src/main/java/com/faforever/api/desync/DesyncReportService.java +++ b/src/main/java/com/faforever/api/desync/DesyncReportService.java @@ -13,11 +13,12 @@ import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.ContentStreamProvider; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import java.io.IOException; -import java.io.InputStream; +import java.io.UncheckedIOException; @Slf4j @Service @@ -61,14 +62,21 @@ public void storeReport(int playerId, int gameId, MultipartFile report) throws I String key = getReportFileName(gameId, playerId); log.info("Player {} uploads a desync report for game {} ({} bytes)", playerId, gameId, size); - try (InputStream inputStream = report.getInputStream()) { - s3Client.putObject(PutObjectRequest.builder() - .bucket(properties.getBucket()) - .key(key) - .contentType(CONTENT_TYPE) - .contentLength(size) - .build(), RequestBody.fromInputStream(inputStream, size)); - } + // the SDK reads the body more than once (checksum, then upload), so every read needs a fresh stream + ContentStreamProvider streamProvider = () -> { + try { + return report.getInputStream(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + + s3Client.putObject(PutObjectRequest.builder() + .bucket(properties.getBucket()) + .key(key) + .contentType(CONTENT_TYPE) + .contentLength(size) + .build(), RequestBody.fromContentProvider(streamProvider, size, CONTENT_TYPE)); // only after the file is stored, so the column can be relied on to filter for existing reports gameDesyncRepository.markReportUploaded(gameId, playerId);