diff --git a/ci/run-integration-tests.Tests.ps1 b/ci/run-integration-tests.Tests.ps1 new file mode 100644 index 00000000..22296cdf --- /dev/null +++ b/ci/run-integration-tests.Tests.ps1 @@ -0,0 +1,183 @@ +# Tests for the integration test step when no resource key is passed. +# +# Each product's ci/run-integration-tests.ps1 is run as common-ci runs it, +# through ci/dispatch.ps1 and common-ci's steps/run-script.ps1, from a folder +# laid out like a common-ci checkout. Its python/run-integration-tests.ps1 is +# a stand-in that runs no tests and records, for each call, the packages it +# was given and the PYTEST_ADDOPTS the tests would see. + +BeforeAll { + . (Join-Path $PSScriptRoot 'dispatch.ps1') + + # common-ci checked out beside this repository, or wherever COMMON_CI says. + $commonCi = $env:COMMON_CI + if ([string]::IsNullOrEmpty($commonCi) -eq $true) { + $commonCi = Join-Path $PSScriptRoot '..' '..' 'common-ci' + } + $script:Steps = Join-Path $commonCi 'steps' + if ((Test-Path (Join-Path $script:Steps 'run-script.ps1')) -eq $false) { + throw "common-ci not found at '$commonCi'; set COMMON_CI to a checkout of it" + } + + $script:Repo = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path + + # The products whose integration tests call the cloud service. + $script:Products = 'device-detection', 'ip-intelligence', 'location', + 'pipeline' + + # The environment variables the scripts set, kept to be put back. + $script:Saved = '_51DEGREES_RESOURCE_KEY', '_51DEGREES_LICENSE_KEY', + 'PYTEST_ADDOPTS', 'TEMP' + + # The stand-in for common-ci's python/run-integration-tests.ps1. It + # writes one line of JSON for each call to calls.jsonl in the folder it + # runs from. + $script:StandIn = @' +param ( + [Parameter(Mandatory=$true)][string]$RepoName, + [Parameter(Mandatory=$true)][string[]]$Packages, + [Parameter(Mandatory=$true)][hashtable]$Keys +) +@{ + Product = Split-Path $RepoName -Leaf + Packages = @($Packages) + AddOpts = "$env:PYTEST_ADDOPTS" +} | ConvertTo-Json -Compress | Add-Content (Join-Path $PWD 'calls.jsonl') +$global:LASTEXITCODE = 0 +'@ + + # A folder laid out like a common-ci checkout, which the step runs in. + function New-CommonCi { + $path = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) + $null = New-Item -ItemType Directory -Force (Join-Path $path 'python') + Copy-Item -Recurse $script:Steps (Join-Path $path 'steps') + Set-Content (Join-Path $path 'python' 'run-integration-tests.ps1') ` + $script:StandIn + return $path + } + + # The calls the stand-in recorded for a product. + function Get-Calls([string]$Product) { + $file = Join-Path $script:CommonCi 'calls.jsonl' + if ((Test-Path $file) -eq $false) { + return @() + } + return @(Get-Content $file | ForEach-Object { $_ | ConvertFrom-Json } | + Where-Object Product -eq $Product) + } +} + +Describe 'run-integration-tests.ps1' { + +BeforeEach { + $script:Environment = @{} + foreach ($name in $script:Saved) { + $script:Environment[$name] = [Environment]::GetEnvironmentVariable($name) + } + $script:CommonCi = New-CommonCi + Push-Location $script:CommonCi +} + +AfterEach { + Pop-Location + Remove-Item -Recurse -Force $script:CommonCi + foreach ($name in $script:Saved) { + [Environment]::SetEnvironmentVariable($name, $script:Environment[$name]) + } +} + +Context 'with no TestResourceKey' { + BeforeEach { + $env:PYTEST_ADDOPTS = $null + $script:Output = Invoke-ProductStep 'run-integration-tests' $script:Repo ` + @{ RepoName = $script:Repo; Keys = @{} } $script:Products *>&1 | + Out-String + } + + It 'runs the tests of every product' { + foreach ($product in $script:Products) { + (Get-Calls $product).Count | Should -BeGreaterThan 0 -Because $product + } + } + + It 'leaves out only the tests marked live, which need a resource key' { + foreach ($product in $script:Products) { + foreach ($call in (Get-Calls $product)) { + $call.AddOpts | Should -Be '-m "not live"' -Because $product + } + } + } + + It 'says for every product that the tests needing a key are skipped' { + foreach ($product in $script:Products) { + # What the step printed for this product, between its group + # markers. + $group = ($script:Output -split '::group::') | + Where-Object { $_ -like "run-integration-tests for $product*" } + $group | Should -Match ("No TestResourceKey was passed, so the " + + "tests that need a resource key are skipped") -Because $product + } + } + + It 'runs the tests of the device detection and IP intelligence examples' { + (Get-Calls 'device-detection').Packages | + Should -Contain 'fiftyone_devicedetection_examples' + (Get-Calls 'device-detection').Packages | + Should -Contain 'fiftyone_devicedetection_cloud' + (Get-Calls 'ip-intelligence').Packages | + Should -Contain 'fiftyone_ipintelligence_examples' + } + + It 'runs the cloud request engine and its examples, and skips the 51Did live tests' { + $packages = (Get-Calls 'pipeline').Packages + $packages | Should -Contain 'fiftyone_pipeline_cloudrequestengine' + $packages | Should -Contain 'fiftyone_pipeline_cloudrequestengine/examples' + $packages | Should -Not -Contain 'fiftyone_pipeline_did' + $script:Output | Should -Match 'fiftyone_pipeline_did.*skipped' + } + + It 'skips the Selenium contract against the cloud example and saying so' { + $script:Output | Should -Match 'Selenium contract tests will not run against the cloud example' + Test-Path (Join-Path $script:CommonCi 'selenium-api-tests') | + Should -BeFalse + } +} + +Context 'with no TestResourceKey and options a caller set' { + It 'adds the selection to the options, and puts them back after' { + $env:PYTEST_ADDOPTS = '-p no:cacheprovider' + Invoke-ProductStep 'run-integration-tests' $script:Repo ` + @{ RepoName = $script:Repo; Keys = @{} } $script:Products *>&1 | + Out-Null + + foreach ($product in $script:Products) { + $calls = Get-Calls $product + $calls.Count | Should -BeGreaterThan 0 -Because $product + foreach ($call in $calls) { + $call.AddOpts | Should -Be '-p no:cacheprovider -m "not live"' ` + -Because $product + } + } + $env:PYTEST_ADDOPTS | Should -Be '-p no:cacheprovider' + } +} + +Context 'with a TestResourceKey' { + It 'runs every test, the ones marked live included' { + Invoke-ProductStep 'run-integration-tests' $script:Repo @{ + RepoName = $script:Repo + Keys = @{ TestResourceKey = 'key' } + SeleniumContract = $false + } 'device-detection', 'location' *>&1 | Out-Null + + foreach ($product in 'device-detection', 'location') { + $calls = Get-Calls $product + $calls.Count | Should -BeGreaterThan 0 -Because $product + foreach ($call in $calls) { + $call.AddOpts | Should -Not -Match 'live' -Because $product + } + } + } +} + +} diff --git a/device-detection/ci/run-integration-tests.ps1 b/device-detection/ci/run-integration-tests.ps1 index ed7a17d2..32bff92c 100644 --- a/device-detection/ci/run-integration-tests.ps1 +++ b/device-detection/ci/run-integration-tests.ps1 @@ -16,10 +16,14 @@ if ($IsWindows) { $packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_examples" +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. A live test still fails when it finds +# no key, so a key that is passed but does not reach it is not missed. +# The selection is added to any options a caller set in PYTEST_ADDOPTS. +$addopts = $env:PYTEST_ADDOPTS if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = "$addopts -m `"not live`"".Trim() } # The cloud package runs without the CSV file too, because only its property # coverage tests read the file, and they say so when it is missing. @@ -44,7 +48,11 @@ $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey # files from _51DEGREES_LICENSE_KEY, which common-ci sets as license_key. $env:_51DEGREES_LICENSE_KEY = $Keys.DeviceDetection -./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +try { + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +} finally { + $env:PYTEST_ADDOPTS = $addopts +} $failures = @() if ($LASTEXITCODE -ne 0) { $failures += "the Python integration tests exited with code $LASTEXITCODE" @@ -58,6 +66,25 @@ if (-not $SeleniumContract) { return } +# The contract tests run against the cloud example when there is a resource +# key, and against the on-premise example when there is the TAC data file, +# which is only fetched when a licence is given. +$tac = "$PWD/assets/TAC-HashV41.hash" +$runCloud = [bool]$Keys.TestResourceKey +$runOnPremise = Test-Path $tac +if (-not $runCloud) { + Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No resource key::No TestResourceKey was passed, so the Selenium contract tests will not run against the cloud example." +} +if (-not $runOnPremise) { + Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No TAC Data File::The TAC data file wasn't found, so the Selenium tests will not run against the on-premise example." +} +if (-not ($runCloud -or $runOnPremise)) { + if ($failures.Count -gt 0) { + throw "Integration tests failed: $($failures -join '; ')" + } + return +} + # The shared Selenium contract tests are run against the cloud example and the # on-premise example in turn. Each result is also written as a trx file next # to the other integration results, so it is published with them. @@ -141,22 +168,21 @@ function Test-Example([string]$Name, [string]$Module, [int]$Port) { } # The cloud example, pointed at the live cloud. -$env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey -$env:_51DEGREES_CLOUD_ENDPOINT = "https://cloud.51degrees.com/api/v4/" -$failures += Test-Example -Name 'cloud' ` - -Module 'fiftyone_devicedetection_examples.cloud.gettingstarted_web' -Port 8097 +if ($runCloud) { + $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey + $env:_51DEGREES_CLOUD_ENDPOINT = "https://cloud.51degrees.com/api/v4/" + $failures += Test-Example -Name 'cloud' ` + -Module 'fiftyone_devicedetection_examples.cloud.gettingstarted_web' -Port 8097 +} # The on-premise example. The contract tests need the device type and the # JavaScript properties, which the Lite data file does not have, so it runs -# against the TAC data file, which is only fetched when a licence is given. -$tac = "$PWD/assets/TAC-HashV41.hash" -if (Test-Path $tac) { +# against the TAC data file. +if ($runOnPremise) { $env:_51DEGREES_DD_PATH = (Resolve-Path $tac).Path $failures += Test-Example -Name 'onpremise' ` -Module 'fiftyone_devicedetection_examples.onpremise.gettingstarted_web' -Port 8098 Remove-Item Env:_51DEGREES_DD_PATH -} else { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No TAC Data File::The TAC data file wasn't found, so the Selenium tests will not run against the on-premise example." } # A failure anywhere above must fail the job. This script is dot-sourced by diff --git a/device-detection/fiftyone_devicedetection_examples/conftest.py b/device-detection/fiftyone_devicedetection_examples/conftest.py new file mode 100644 index 00000000..4096653b --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/conftest.py @@ -0,0 +1,76 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The tests that call the live cloud service are marked live here, so the +# released test files stay as they are. ci/run-integration-tests.ps1 leaves +# the live tests out when no resource key is passed, and the other tests run. + +from pathlib import Path + +import pytest +from _pytest.mark.expression import Expression + +TESTS = Path(__file__).resolve().parent / "tests" + +# Every test in these modules is live. +LIVE_MODULES = { + TESTS / "test_cloudexamples.py", + TESTS / "test_cloudgettingstartedweb.py", +} + +BUILT_AT_IMPORT = {TESTS / "test_cloudgettingstartedweb.py"} + + +def _is_live(item): + return Path(item.path).resolve() in LIVE_MODULES + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(config, items): + # Runs before -m deselects, so the selection sees the marker. + for item in items: + if _is_live(item): + item.add_marker(pytest.mark.live) + + +def _leaves_out_live(config): + """Whether the -m expression leaves out a test marked live alone.""" + expression = config.getoption("markexpr") + return bool(expression) and not Expression.compile(expression).evaluate( + lambda name, **kwargs: name == "live") + + +# A module that builds a cloud Pipeline when imported raises with no resource +# key, so it is not collected when the live tests are left out. Every test in +# it is live, so the selection is the same. +def pytest_ignore_collect(collection_path, config): + if (Path(collection_path).resolve() in BUILT_AT_IMPORT and + _leaves_out_live(config)): + return True + return None diff --git a/device-detection/fiftyone_devicedetection_examples/readme.md b/device-detection/fiftyone_devicedetection_examples/readme.md index 59971991..e6214dcb 100644 --- a/device-detection/fiftyone_devicedetection_examples/readme.md +++ b/device-detection/fiftyone_devicedetection_examples/readme.md @@ -53,7 +53,9 @@ the Enterprise data file. The cloud tests need a valid 51Degrees resource key (see above), and fail without one. They read it from the environment variable -`_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`. +`_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`. They are +marked `live`, so `python -m pytest -m "not live"` runs every other test, +which is what CI does when no resource key is passed. For example, use following command to set the resource key on Linux: `export _51DEGREES_RESOURCE_KEY=MY-RESOURCE-KEY` diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py index 1ac2b995..4852ed0c 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py @@ -34,10 +34,14 @@ from contextlib import redirect_stdout from unittest import mock +import pytest + from fiftyone_devicedetection_examples.example_utils import ExampleUtils from .live_cloud import answered_without, find_resource_key, no_resource_key +pytestmark = pytest.mark.live + MODULE = "fiftyone_devicedetection_examples.cloud.failuretomatch" NO_KEY = no_resource_key("the device product", "the IsMobile property") diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py b/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py index afb5e23c..e00f9d97 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py @@ -29,6 +29,8 @@ import re import unittest +import pytest + from fiftyone_pipeline_core.logger import Logger from fiftyone_devicedetection_examples.cloud.useragentclienthints_web.app \ @@ -36,6 +38,8 @@ from .live_cloud import find_resource_key, no_resource_key +pytestmark = pytest.mark.live + NO_KEY = no_resource_key( "the device product", "device detection properties such as BrowserName") diff --git a/device-detection/fiftyone_devicedetection_examples/tox.ini b/device-detection/fiftyone_devicedetection_examples/tox.ini index 48032ef6..1a7c3f74 100644 --- a/device-detection/fiftyone_devicedetection_examples/tox.ini +++ b/device-detection/fiftyone_devicedetection_examples/tox.ini @@ -42,6 +42,9 @@ pass_env = # running the examples. GITHUB_ACTIONS GITHUB_REPOSITORY + # The marker selection of ../ci/run-integration-tests.ps1, which leaves + # the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] deps = diff --git a/ip-intelligence/ci/run-integration-tests.ps1 b/ip-intelligence/ci/run-integration-tests.ps1 index a64f611e..0d73e949 100644 --- a/ip-intelligence/ci/run-integration-tests.ps1 +++ b/ip-intelligence/ci/run-integration-tests.ps1 @@ -13,10 +13,14 @@ if ($IsWindows) { $packages = , "fiftyone_ipintelligence_examples" +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. A live test still fails when it finds +# no key, so a key that is passed but does not reach it is not missed. +# The selection is added to any options a caller set in PYTEST_ADDOPTS. +$addopts = $env:PYTEST_ADDOPTS if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = "$addopts -m `"not live`"".Trim() } # The tests read the resource key from _51DEGREES_RESOURCE_KEY, and common-ci @@ -28,7 +32,9 @@ $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey # only, never the key. Found through this script's own folder, because # common-ci runs it from the common-ci checkout, where # './ci/report-key-products.ps1' does not exist. -& (Join-Path $PSScriptRoot 'report-key-products.ps1') -ResourceKey $Keys.TestResourceKey +if ($Keys.TestResourceKey) { + & (Join-Path $PSScriptRoot 'report-key-products.ps1') -ResourceKey $Keys.TestResourceKey +} # nightly-publish-main workflow doesn't create the examples package, so # install-package.ps1 won't install it and its test dependencies won't get @@ -38,6 +44,10 @@ if ($env:GITHUB_JOB -eq "Test") { pip install $RepoName/fiftyone_ipintelligence_examples || $(throw "pip install failed") } -./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +try { + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/conftest.py b/ip-intelligence/fiftyone_ipintelligence_examples/conftest.py new file mode 100644 index 00000000..c235aa3b --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/conftest.py @@ -0,0 +1,61 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The tests that call the live cloud service are marked live here, so the +# released test files stay as they are. ci/run-integration-tests.ps1 leaves +# the live tests out when no resource key is passed, and the other tests run. + +from pathlib import Path + +import pytest + +TEST_EXAMPLES = ( + Path(__file__).resolve().parent / "tests" / "test_examples.py") + +# The tests in test_examples.py that need a resource key. +LIVE_TESTS = { + "test_cloud_gettingstarted_console", + "test_cloud_metadata_console", + "test_cloud_getallproperties_console", + "test_cloud_mixed_console", + "test_cloud_countries_console", +} + + +def _is_live(item): + return (Path(item.path).resolve() == TEST_EXAMPLES and + item.name in LIVE_TESTS) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(config, items): + # Runs before -m deselects, so the selection sees the marker. + for item in items: + if _is_live(item): + item.add_marker(pytest.mark.live) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py index 40a52678..d8a8f4e4 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py @@ -27,11 +27,15 @@ import unittest +import pytest + from fiftyone_ipintelligence_examples.cloud.gettingstarted_web \ import app as example from .live_cloud import find_resource_key, no_resource_key +pytestmark = pytest.mark.live + NO_KEY = no_resource_key( "the ip product", "IP Intelligence properties such as RegisteredName") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini index ed00ed06..f873d855 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini @@ -57,6 +57,9 @@ pass_env = # running the examples. GITHUB_ACTIONS GITHUB_REPOSITORY + # The marker selection of ../ci/run-integration-tests.ps1, which leaves + # the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] # The examples package itself must be installed, because its tests import diff --git a/location/ci/run-integration-tests.ps1 b/location/ci/run-integration-tests.ps1 index 01ae9b5c..b2025094 100644 --- a/location/ci/run-integration-tests.ps1 +++ b/location/ci/run-integration-tests.ps1 @@ -5,16 +5,24 @@ param ( [Hashtable]$Keys ) +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. A live test still fails when it finds +# no key, so a key that is passed but does not reach it is not missed. +# The selection is added to any options a caller set in PYTEST_ADDOPTS. +$addopts = $env:PYTEST_ADDOPTS if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = "$addopts -m `"not live`"".Trim() } # The examples and tests read the resource key from _51DEGREES_RESOURCE_KEY, # and common-ci sets only the older resource_key, so the key is set here. $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey -./python/run-integration-tests.ps1 -RepoName $RepoName -Packages . -Keys $Keys +try { + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages . -Keys $Keys +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/location/tests/conftest.py b/location/tests/conftest.py new file mode 100644 index 00000000..df687fc0 --- /dev/null +++ b/location/tests/conftest.py @@ -0,0 +1,76 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The tests that call the live cloud service are marked live here, so the +# released test files stay as they are. ci/run-integration-tests.ps1 leaves +# the live tests out when no resource key is passed, and the other tests run. + +from pathlib import Path + +import pytest +from _pytest.mark.expression import Expression + +HERE = Path(__file__).resolve().parent + +# Every test in these modules is live. test_web_example.py marks its own. +LIVE_MODULES = { + HERE / "test_examples.py", + HERE / "test_properties51D.py", +} + +BUILT_AT_IMPORT = {HERE / "test_properties51D.py"} + + +def _is_live(item): + return Path(item.path).resolve() in LIVE_MODULES + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(config, items): + # Runs before -m deselects, so the selection sees the marker. + for item in items: + if _is_live(item): + item.add_marker(pytest.mark.live) + + +def _leaves_out_live(config): + """Whether the -m expression leaves out a test marked live alone.""" + expression = config.getoption("markexpr") + return bool(expression) and not Expression.compile(expression).evaluate( + lambda name, **kwargs: name == "live") + + +# A module that builds a cloud Pipeline when imported raises with no resource +# key, so it is not collected when the live tests are left out. Every test in +# it is live, so the selection is the same. +def pytest_ignore_collect(collection_path, config): + if (Path(collection_path).resolve() in BUILT_AT_IMPORT and + _leaves_out_live(config)): + return True + return None diff --git a/location/tests/test_web_example.py b/location/tests/test_web_example.py index 1fafe50b..de01960f 100644 --- a/location/tests/test_web_example.py +++ b/location/tests/test_web_example.py @@ -37,6 +37,8 @@ from pathlib import Path from unittest import mock +import pytest + from .live_cloud import answered_without, find_resource_key, no_resource_key EXAMPLE = Path(__file__).resolve().parents[1] / "examples" / "cloud" / "web.py" @@ -108,6 +110,7 @@ def load_example(resource_key): return module +@pytest.mark.live class LocationWebExampleTests(unittest.TestCase): resource_key = find_resource_key() diff --git a/location/tox.ini b/location/tox.ini index dff16572..9b7bf58d 100644 --- a/location/tox.ini +++ b/location/tox.ini @@ -26,6 +26,9 @@ commands = pass_env = _51DEGREES_RESOURCE_KEY _51DEGREES_LICENSE_KEY + # The marker selection of ci/run-integration-tests.ps1, which leaves the + # live tests out when no resource key is passed. + PYTEST_ADDOPTS # Tests the packages just built, so it takes the test tools from [common] # and every package from the requirements file, never the editable copies diff --git a/pipeline/ci/run-integration-tests.ps1 b/pipeline/ci/run-integration-tests.ps1 index a4845edd..c34ea46a 100644 --- a/pipeline/ci/run-integration-tests.ps1 +++ b/pipeline/ci/run-integration-tests.ps1 @@ -5,12 +5,6 @@ param ( [Hashtable]$Keys ) -if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." -} - # The examples call the cloud service, so they are integration tests rather # than unit tests. Listing them here runs them on every Python version and # operating system in options.json, and in the nightly, which a standalone @@ -26,23 +20,48 @@ if (!$Keys.TestResourceKey) { # The 51Did package runs only its tests marked live here, as its others ran # in the unit step. Its tier tests need a key for another tier, which only # the Cloud 51Did workflow has. Only a package whose tox.ini passes -# PYTEST_ADDOPTS sees the selection. Its build needs the OWID source, which -# build-project.ps1 copies, and the publish workflow's test job does not run -# that script, so it is copied here too. -& "$PSScriptRoot/copy-owid-source.ps1" -RepoRoot $RepoName -$packages = "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_cloudrequestengine/examples", "fiftyone_pipeline_did" +# PYTEST_ADDOPTS sees the selection, so the 51Did package is run on its own, +# after the others. Its build needs the OWID source, which build-project.ps1 +# copies, and the publish workflow's test job does not run that script, so +# it is copied here too. +# +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. That leaves nothing of the 51Did +# package, so it is not run. A live test still fails when it finds no key, so +# a key that is passed but does not reach it is not missed. The selection is +# added to any options a caller set in PYTEST_ADDOPTS. +$packages = "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_cloudrequestengine/examples" $pipelineKeys = $Keys.Clone() $pipelineKeys.Remove('DeviceDetection') # The tests read the resource key from _51DEGREES_RESOURCE_KEY, and common-ci # sets only the older resource_key, so the key is set here. $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey +$failed = $false $addopts = $env:PYTEST_ADDOPTS -$env:PYTEST_ADDOPTS = '-m "live and not tier"' try { + if (!$Keys.TestResourceKey) { + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. fiftyone_pipeline_did runs only live tests here, so it is skipped. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = "$addopts -m `"not live`"".Trim() + } ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $pipelineKeys + if ($LASTEXITCODE -ne 0) { $failed = $true } + + if ($Keys.TestResourceKey) { + # common-ci's script combines the coverage of the packages it ran + # into .coverage, and a second run replaces that file, so the first + # run's is kept under a name the second run's combine reads. + $coverage = Join-Path $RepoName '.coverage' + if (Test-Path $coverage) { + Move-Item -Force $coverage (Join-Path $RepoName '.coverage.cloudrequestengine') + } + & "$PSScriptRoot/copy-owid-source.ps1" -RepoRoot $RepoName + $env:PYTEST_ADDOPTS = '-m "live and not tier"' + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages "fiftyone_pipeline_did" -Keys $pipelineKeys + if ($LASTEXITCODE -ne 0) { $failed = $true } + } } finally { $env:PYTEST_ADDOPTS = $addopts } -exit $LASTEXITCODE +exit ($failed ? 1 : 0) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md index 77caac69..8099464d 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md @@ -99,14 +99,16 @@ then open . Set `PORT` to listen on another port. The tests that call the cloud service with a resource key fail when none is set, naming the variables they read, which are `_51DEGREES_RESOURCE_KEY` -and `_51DEGREES_RESOURCE_KEY_BESPOKE`, in that order. The +and `_51DEGREES_RESOURCE_KEY_BESPOKE`, in that order. They are marked +`live`, so `python -m pytest -m "not live"` runs every other test. The tests that use a licence key skip themselves when none is set. In CI the examples are run by [ci/run-integration-tests.ps1](../../ci/run-integration-tests.ps1) through common-ci, on every Python version and operating system in [ci/options.json](../../ci/options.json), with the resource key the pipeline -holds. No licence key is passed to the pipeline, so the licence-key tests -skip themselves there. That uses the [tox.ini](tox.ini) in this folder, +holds. When no resource key is passed, the script leaves out the tests +marked `live`, says so, and runs the rest. No licence key is passed to the +pipeline, so the licence-key tests skip themselves there. That uses the [tox.ini](tox.ini) in this folder, which can also be run by hand with `python -m tox -e py` from here. ## Cloud engine diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/conftest.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/conftest.py new file mode 100644 index 00000000..16bcd169 --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/conftest.py @@ -0,0 +1,49 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The tests that call the live cloud service are marked live here, so the +# released test files stay as they are. ci/run-integration-tests.ps1 leaves +# the live tests out when no resource key is passed, and the other tests run. + +import pytest + + +# The tests given the "resource key" case of helpers.KEYS. The "licence key" +# case skips itself when no licence key is set. +def _is_live(item): + callspec = getattr(item, "callspec", None) + return callspec is not None and "resource key" in callspec.id.split("-") + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(config, items): + # Runs before -m deselects, so the selection sees the marker. + for item in items: + if _is_live(item): + item.add_marker(pytest.mark.live) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini index fbd5e095..3ab7052a 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini @@ -41,6 +41,9 @@ pass_env = _51DEGREES_LICENSE_KEY _51DEGREES_CLOUD_ENDPOINT _51DEGREES_CLOUD_REQUEST_ORIGIN + # The marker selection of ../../ci/run-integration-tests.ps1, which + # leaves the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] deps = diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/conftest.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/conftest.py new file mode 100644 index 00000000..0ecb4c70 --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/conftest.py @@ -0,0 +1,53 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The tests that call the live cloud service are marked live here, so the +# released test files stay as they are. ci/run-integration-tests.ps1 leaves +# the live tests out when no resource key is passed, and the other tests run. + +from pathlib import Path + +import pytest + +TEST_CLOUDENGINE = Path(__file__).resolve().parent / "test_cloudengine.py" + + +# CloudEngineTests needs a resource key for each of its tests, in setUp. +def _is_live(item): + return (Path(item.path).resolve() == TEST_CLOUDENGINE and + getattr(item, "cls", None) is not None and + item.cls.__name__ == "CloudEngineTests") + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(config, items): + # Runs before -m deselects, so the selection sees the marker. + for item in items: + if _is_live(item): + item.add_marker(pytest.mark.live) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini index 646020a0..35bf9b29 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini @@ -21,6 +21,9 @@ commands = pass_env = _51DEGREES_RESOURCE_KEY _51DEGREES_LICENSE_KEY + # The marker selection of ../ci/run-integration-tests.ps1, which leaves + # the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] package = skip