From a6d041fc93cb035e195d147f34c13da2dee6eab4 Mon Sep 17 00:00:00 2001 From: Tracy Barlow Date: Fri, 20 Nov 2026 16:57:07 +0000 Subject: [PATCH 1/7] Read only the _51DEGREES_ secrets in the workflows, and pass them to common-ci unnamed --- .github/workflows/cloud-51did.yml | 7 ++++--- .github/workflows/example-assets-update.yml | 3 +-- .github/workflows/monthly-copyright-update.yml | 3 +-- .github/workflows/nightly-pipeline.yml | 18 +++--------------- .github/workflows/publish.yml | 8 +------- .github/workflows/pull-requests.yml | 7 +------ .github/workflows/release-pull-request.yml | 7 +------ .github/workflows/submodule-update.yml | 3 +-- 8 files changed, 13 insertions(+), 43 deletions(-) diff --git a/.github/workflows/cloud-51did.yml b/.github/workflows/cloud-51did.yml index 2a2e6ee2..eeb05540 100644 --- a/.github/workflows/cloud-51did.yml +++ b/.github/workflows/cloud-51did.yml @@ -64,8 +64,9 @@ jobs: - name: Cloud 51Did test for each resource key secret shell: pwsh env: - _51DEGREES_RESOURCE_KEY_BESPOKE: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - _51DEGREES_RESOURCE_KEY_51DID: ${{ vars._51DEGREES_RESOURCE_KEY_51DID }} + # The 51Did live tests use the Bespoke resource key, which carries + # FODid at CloudV5Bespoke. + _51DEGREES_RESOURCE_KEY_51DID: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} # The redemption tests check a sealed result reads back only with a # license key, and the service only opens a sealed result for a # license key of the account whose resource key sealed it. So each @@ -96,5 +97,5 @@ jobs: - name: Cloud 51Did tier test with the Basic key if: ${{ !cancelled() }} env: - _51DEGREES_RESOURCE_KEY_BASIC: ${{ vars._51DEGREES_RESOURCE_KEY_BASIC }} + _51DEGREES_RESOURCE_KEY_BASIC: ${{ secrets._51DEGREES_RESOURCE_KEY_FREE }} run: tox -- tests/test_did_tier_live.py -v -rs diff --git a/.github/workflows/example-assets-update.yml b/.github/workflows/example-assets-update.yml index 7c7b1851..1fcf37e1 100644 --- a/.github/workflows/example-assets-update.yml +++ b/.github/workflows/example-assets-update.yml @@ -17,5 +17,4 @@ jobs: uses: 51Degrees/common-ci/.github/workflows/nightly-example-assets-update.yml@main with: repo-name: ${{ github.event.repository.name }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} + secrets: inherit diff --git a/.github/workflows/monthly-copyright-update.yml b/.github/workflows/monthly-copyright-update.yml index 7a344b62..68fe9fd6 100644 --- a/.github/workflows/monthly-copyright-update.yml +++ b/.github/workflows/monthly-copyright-update.yml @@ -11,5 +11,4 @@ jobs: uses: 51Degrees/common-ci/.github/workflows/monthly-copyright-update.yml@main with: repo-name: ${{ github.event.repository.name }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} + secrets: inherit diff --git a/.github/workflows/nightly-pipeline.yml b/.github/workflows/nightly-pipeline.yml index f0455882..3c05d8e2 100644 --- a/.github/workflows/nightly-pipeline.yml +++ b/.github/workflows/nightly-pipeline.yml @@ -32,13 +32,7 @@ jobs: dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - PypiToken: ${{ secrets.PYPI_TOKEN }} + secrets: inherit SubmoduleUpdate: name: Nightly Submodule Update @@ -48,8 +42,7 @@ jobs: org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} + secrets: inherit PullRequests: if: ${{ !cancelled() }} @@ -65,9 +58,4 @@ jobs: cache-assets: true # One configuration that fails does not cancel the others. fail-fast: false - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} + secrets: inherit diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b75b3038..81d37252 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -27,10 +27,4 @@ jobs: common-ci-ref: ${{ inputs.common-ci-ref || '' }} allow-early-release: ${{ inputs.allow-early-release || false }} cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - PypiToken: ${{ secrets.PYPI_TOKEN }} + secrets: inherit diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index e154435f..e072cfba 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -35,9 +35,4 @@ jobs: full-matrix: ${{ inputs.full-matrix || false }} fail-fast: false build-without-pull-requests: false - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} + secrets: inherit diff --git a/.github/workflows/release-pull-request.yml b/.github/workflows/release-pull-request.yml index 6faa5356..8e4af949 100644 --- a/.github/workflows/release-pull-request.yml +++ b/.github/workflows/release-pull-request.yml @@ -22,9 +22,4 @@ jobs: repo-name: ${{ github.event.repository.name }} org-name: ${{ github.event.repository.owner.login }} cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} + secrets: inherit diff --git a/.github/workflows/submodule-update.yml b/.github/workflows/submodule-update.yml index 7bf3291c..d3b64cce 100644 --- a/.github/workflows/submodule-update.yml +++ b/.github/workflows/submodule-update.yml @@ -19,5 +19,4 @@ jobs: org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} + secrets: inherit From 4005c82c79894b75f393ef493caa42e7df0cbcc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eddie=20Vereb=C3=A9lyi?= Date: Fri, 20 Nov 2026 20:54:37 +0000 Subject: [PATCH 2/7] Test that every key is read under its _51DEGREES_ name Adds ci/key-names.Tests.ps1. It runs each product's run-integration-tests.ps1 and fetch-assets.ps1 against a stand-in common-ci, and checks that: - the resource key comes from _51DEGREES_RESOURCE_KEY_BESPOKE, with no TestResourceKey option - the data license common-ci exports as _51DEGREES_LICENSE_KEY is left as it is - the TAC data file, the CSV and the Enterprise IP Intelligence file are asked for with no key or address option - no CI script reads TestResourceKey, DeviceDetection or IpIntelligence - every tox.ini passes each _51DEGREES_ variable its package reads - the cloud license-key tests read _51DEGREES_LICENSE_KEY_BESPOKE, and the 51Did live test reads _51DEGREES_RESOURCE_KEY_51DID and _51DEGREES_LICENSE_KEY_51DID The CI script tests workflow now also runs when those files change. --- .github/workflows/ci-script-tests.yml | 13 +- ci/key-names.Tests.ps1 | 323 ++++++++++++++++++++++++++ 2 files changed, 335 insertions(+), 1 deletion(-) create mode 100644 ci/key-names.Tests.ps1 diff --git a/.github/workflows/ci-script-tests.yml b/.github/workflows/ci-script-tests.yml index 8f68b5e0..ad98f21e 100644 --- a/.github/workflows/ci-script-tests.yml +++ b/.github/workflows/ci-script-tests.yml @@ -1,12 +1,18 @@ name: CI script tests # Runs the Pester tests of the scripts in ci/, which choose what a pull request -# builds and what a publish releases. common-ci's main is checked out beside +# builds and what a publish releases, and check that every key is read under +# its _51DEGREES_ name (ci/key-names.Tests.ps1). common-ci's main is checked out beside # this repository, where ci/dispatch.Tests.ps1 looks for it. on: pull_request: paths: - 'ci/**' + - '*/ci/**' + - '**/tox.ini' + - 'pipeline/fiftyone_pipeline_cloudrequestengine/**/tests/**' + - 'pipeline/fiftyone_pipeline_did/tests/**' + - '.github/workflows/cloud-51did.yml' - '.github/workflows/ci-script-tests.yml' push: branches: @@ -14,6 +20,11 @@ on: - rc paths: - 'ci/**' + - '*/ci/**' + - '**/tox.ini' + - 'pipeline/fiftyone_pipeline_cloudrequestengine/**/tests/**' + - 'pipeline/fiftyone_pipeline_did/tests/**' + - '.github/workflows/cloud-51did.yml' - '.github/workflows/ci-script-tests.yml' workflow_dispatch: diff --git a/ci/key-names.Tests.ps1 b/ci/key-names.Tests.ps1 new file mode 100644 index 00000000..1dd8dbbd --- /dev/null +++ b/ci/key-names.Tests.ps1 @@ -0,0 +1,323 @@ +# Tests that the CI scripts, the tox configurations and the cloud tests read +# each key under its _51DEGREES_ name, and never under an old name. +# +# common-ci exports every _51DEGREES_ secret under its own name before any of +# this repository's scripts run, and fills _51DEGREES_RESOURCE_KEY and +# _51DEGREES_RESOURCE_KEY_51DID from _51DEGREES_RESOURCE_KEY_BESPOKE. It +# downloads every data file with _51DEGREES_LICENSE_KEY, which is the data +# license and not a cloud license. No key reaches a script as an option. + +BeforeDiscovery { + $script:DiscoveryRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +} + +BeforeAll { + $script:Root = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path + + # The variables the scripts read or write, saved before each test and put + # back after it. + $script:Saved = @( + '_51DEGREES_RESOURCE_KEY', '_51DEGREES_RESOURCE_KEY_BESPOKE', + '_51DEGREES_LICENSE_KEY', 'PAID_RESOURCE_KEY', 'TEMP', 'GITHUB_ENV', + 'GITHUB_JOB', 'PYTEST_ADDOPTS') + + function New-TempFolder { + $path = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) + return [IO.Directory]::CreateDirectory($path).FullName + } + + # A folder laid out like a common-ci checkout, which the product scripts + # run in. Its python/run-integration-tests.ps1 writes the keys it was run + # with to seen.json, and its steps/fetch-assets.ps1 writes each call to + # fetch.json and leaves an empty file for each asset. + function New-CommonCi { + $path = New-TempFolder + $null = [IO.Directory]::CreateDirectory((Join-Path $path 'python')) + $null = [IO.Directory]::CreateDirectory((Join-Path $path 'steps')) + Set-Content (Join-Path $path 'python' 'run-integration-tests.ps1') @' +param ([string]$RepoName, [string[]]$Packages, [hashtable]$Keys) +@{ + Resource = $env:_51DEGREES_RESOURCE_KEY + License = $env:_51DEGREES_LICENSE_KEY +} | ConvertTo-Json | Set-Content (Join-Path $PSScriptRoot '..' 'seen.json') +$global:LASTEXITCODE = 0 +'@ + Set-Content (Join-Path $path 'steps' 'fetch-assets.ps1') @' +param ( + [Parameter(Mandatory)][string[]]$Assets, + [string]$LicenseKey, + [string]$DeviceDetection, + [string]$DeviceDetectionUrl, + [string]$IpIntelligence, + [string]$IpIntelligenceUrl, + [string]$CsvUrl +) +$log = Join-Path $PSScriptRoot '..' 'fetch.json' +$calls = @() +if (Test-Path $log) { $calls = @(Get-Content -Raw $log | ConvertFrom-Json) } +$calls += [pscustomobject]@{ + Assets = @($Assets) + Old = @($PSBoundParameters.Keys | Where-Object { $_ -notin 'Assets', 'LicenseKey' }) +} +ConvertTo-Json -InputObject @($calls) -Depth 4 | Set-Content $log +$null = [IO.Directory]::CreateDirectory((Join-Path $PWD 'assets')) +foreach ($asset in $Assets) { Set-Content -LiteralPath "assets/$asset" '' } +'@ + return $path + } + + # A copy of one product's CI script in a repository of its own, with the + # scripts it calls replaced by ones that do nothing, and the folders it + # writes to. + function New-ProductRepo([string]$Product, [string]$Script) { + $repo = New-TempFolder + $ci = [IO.Directory]::CreateDirectory((Join-Path $repo $Product 'ci')).FullName + Copy-Item (Join-Path $script:Root $Product 'ci' $Script) $ci + Set-Content (Join-Path $ci 'report-key-products.ps1') 'param ([string]$ResourceKey)' + Set-Content (Join-Path $ci 'copy-owid-source.ps1') 'param ([string]$RepoRoot)' + $null = [IO.Directory]::CreateDirectory((Join-Path $repo 'cxx' 'data' 'device-detection')) + $null = [IO.Directory]::CreateDirectory((Join-Path $repo 'cxx' 'data' 'ip-intelligence')) + return Join-Path $repo $Product + } +} + +Describe 'Keys under their _51DEGREES_ names' { + + BeforeEach { + $script:Before = @{} + foreach ($name in $script:Saved) { + $script:Before[$name] = [Environment]::GetEnvironmentVariable($name) + [Environment]::SetEnvironmentVariable($name, $null) + } + } + + AfterEach { + foreach ($name in $script:Saved) { + [Environment]::SetEnvironmentVariable($name, $script:Before[$name]) + } + } + + Describe 'Each product''s run-integration-tests.ps1' { + + BeforeEach { + $script:CommonCi = New-CommonCi + } + + AfterEach { + Remove-Item -Recurse -Force $script:CommonCi + } + + It 'takes the resource key from _51DEGREES_RESOURCE_KEY_BESPOKE in , with no key option' -ForEach @( + @{ Product = 'device-detection'; Extra = @{ SeleniumContract = $false } } + @{ Product = 'ip-intelligence'; Extra = @{} } + @{ Product = 'location'; Extra = @{} } + @{ Product = 'pipeline'; Extra = @{} } + ) { + $repoName = New-ProductRepo $Product 'run-integration-tests.ps1' + $env:_51DEGREES_RESOURCE_KEY_BESPOKE = 'bespoke-resource-key' + $env:_51DEGREES_LICENSE_KEY = 'data-license' + Push-Location $script:CommonCi + try { + & (Join-Path $repoName 'ci' 'run-integration-tests.ps1') ` + -RepoName $repoName -Keys @{} @Extra + } finally { + Pop-Location + } + + $seen = Get-Content -Raw (Join-Path $script:CommonCi 'seen.json') | ConvertFrom-Json + $seen.Resource | Should -Be 'bespoke-resource-key' + } + + It 'leaves the data license common-ci exported as it was in ' -ForEach @( + @{ Product = 'device-detection'; Extra = @{ SeleniumContract = $false } } + @{ Product = 'ip-intelligence'; Extra = @{} } + @{ Product = 'location'; Extra = @{} } + @{ Product = 'pipeline'; Extra = @{} } + ) { + $repoName = New-ProductRepo $Product 'run-integration-tests.ps1' + $env:_51DEGREES_RESOURCE_KEY_BESPOKE = 'bespoke-resource-key' + $env:_51DEGREES_LICENSE_KEY = 'data-license' + Push-Location $script:CommonCi + try { + # An old key option, as a caller passing every secret might, must + # change nothing. + & (Join-Path $repoName 'ci' 'run-integration-tests.ps1') ` + -RepoName $repoName -Keys @{ DeviceDetection = 'old-license'; TestResourceKey = 'old-key' } @Extra + } finally { + Pop-Location + } + + $seen = Get-Content -Raw (Join-Path $script:CommonCi 'seen.json') | ConvertFrom-Json + $seen.License | Should -Be 'data-license' + $seen.Resource | Should -Be 'bespoke-resource-key' + } + + It 'fails naming _51DEGREES_RESOURCE_KEY_BESPOKE when it is not set in ' -ForEach @( + @{ Product = 'device-detection'; Extra = @{ SeleniumContract = $false } } + @{ Product = 'ip-intelligence'; Extra = @{} } + @{ Product = 'location'; Extra = @{} } + @{ Product = 'pipeline'; Extra = @{} } + ) { + $repoName = New-ProductRepo $Product 'run-integration-tests.ps1' + Push-Location $script:CommonCi + try { + { & (Join-Path $repoName 'ci' 'run-integration-tests.ps1') ` + -RepoName $repoName -Keys @{} @Extra } | + Should -Throw '*_51DEGREES_RESOURCE_KEY_BESPOKE*' + } finally { + Pop-Location + } + } + } + + Describe 'Each product''s fetch-assets.ps1' { + + BeforeEach { + $script:CommonCi = New-CommonCi + # The scripts make symbolic links, which need a privilege a test + # run may not have, and folders for them. Where they point is not + # what these tests check, so New-Item does nothing here, and the + # helpers above make their folders without it. + Mock New-Item { } + } + + AfterEach { + Remove-Item -Recurse -Force $script:CommonCi + } + + It 'takes no key or address option in