Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
183 changes: 183 additions & 0 deletions ci/run-integration-tests.Tests.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
# Tests for the integration test step when no resource key is passed.
#
# Each product's ci/run-integration-tests.ps1 is run as common-ci runs it,
# through ci/dispatch.ps1 and common-ci's steps/run-script.ps1, from a folder
# laid out like a common-ci checkout. Its python/run-integration-tests.ps1 is
# a stand-in that runs no tests and records, for each call, the packages it
# was given and the PYTEST_ADDOPTS the tests would see.

BeforeAll {
. (Join-Path $PSScriptRoot 'dispatch.ps1')

# common-ci checked out beside this repository, or wherever COMMON_CI says.
$commonCi = $env:COMMON_CI
if ([string]::IsNullOrEmpty($commonCi) -eq $true) {
$commonCi = Join-Path $PSScriptRoot '..' '..' 'common-ci'
}
$script:Steps = Join-Path $commonCi 'steps'
if ((Test-Path (Join-Path $script:Steps 'run-script.ps1')) -eq $false) {
throw "common-ci not found at '$commonCi'; set COMMON_CI to a checkout of it"
}

$script:Repo = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path

# The products whose integration tests call the cloud service.
$script:Products = 'device-detection', 'ip-intelligence', 'location',
'pipeline'

# The environment variables the scripts set, kept to be put back.
$script:Saved = '_51DEGREES_RESOURCE_KEY', '_51DEGREES_LICENSE_KEY',
'PYTEST_ADDOPTS', 'TEMP'

# The stand-in for common-ci's python/run-integration-tests.ps1. It
# writes one line of JSON for each call to calls.jsonl in the folder it
# runs from.
$script:StandIn = @'
param (
[Parameter(Mandatory=$true)][string]$RepoName,
[Parameter(Mandatory=$true)][string[]]$Packages,
[Parameter(Mandatory=$true)][hashtable]$Keys
)
@{
Product = Split-Path $RepoName -Leaf
Packages = @($Packages)
AddOpts = "$env:PYTEST_ADDOPTS"
} | ConvertTo-Json -Compress | Add-Content (Join-Path $PWD 'calls.jsonl')
$global:LASTEXITCODE = 0
'@

# A folder laid out like a common-ci checkout, which the step runs in.
function New-CommonCi {
$path = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString())
$null = New-Item -ItemType Directory -Force (Join-Path $path 'python')
Copy-Item -Recurse $script:Steps (Join-Path $path 'steps')
Set-Content (Join-Path $path 'python' 'run-integration-tests.ps1') `
$script:StandIn
return $path
}

# The calls the stand-in recorded for a product.
function Get-Calls([string]$Product) {
$file = Join-Path $script:CommonCi 'calls.jsonl'
if ((Test-Path $file) -eq $false) {
return @()
}
return @(Get-Content $file | ForEach-Object { $_ | ConvertFrom-Json } |
Where-Object Product -eq $Product)
}
}

Describe 'run-integration-tests.ps1' {

BeforeEach {
$script:Environment = @{}
foreach ($name in $script:Saved) {
$script:Environment[$name] = [Environment]::GetEnvironmentVariable($name)
}
$script:CommonCi = New-CommonCi
Push-Location $script:CommonCi
}

AfterEach {
Pop-Location
Remove-Item -Recurse -Force $script:CommonCi
foreach ($name in $script:Saved) {
[Environment]::SetEnvironmentVariable($name, $script:Environment[$name])
}
}

Context 'with no TestResourceKey' {
BeforeEach {
$env:PYTEST_ADDOPTS = $null
$script:Output = Invoke-ProductStep 'run-integration-tests' $script:Repo `
@{ RepoName = $script:Repo; Keys = @{} } $script:Products *>&1 |
Out-String
}

It 'runs the tests of every product' {
foreach ($product in $script:Products) {
(Get-Calls $product).Count | Should -BeGreaterThan 0 -Because $product
}
}

It 'leaves out only the tests marked live, which need a resource key' {
foreach ($product in $script:Products) {
foreach ($call in (Get-Calls $product)) {
$call.AddOpts | Should -Be '-m "not live"' -Because $product
}
}
}

It 'says for every product that the tests needing a key are skipped' {
foreach ($product in $script:Products) {
# What the step printed for this product, between its group
# markers.
$group = ($script:Output -split '::group::') |
Where-Object { $_ -like "run-integration-tests for $product*" }
$group | Should -Match ("No TestResourceKey was passed, so the " +
"tests that need a resource key are skipped") -Because $product
}
}

It 'runs the tests of the device detection and IP intelligence examples' {
(Get-Calls 'device-detection').Packages |
Should -Contain 'fiftyone_devicedetection_examples'
(Get-Calls 'device-detection').Packages |
Should -Contain 'fiftyone_devicedetection_cloud'
(Get-Calls 'ip-intelligence').Packages |
Should -Contain 'fiftyone_ipintelligence_examples'
}

It 'runs the cloud request engine and its examples, and skips the 51Did live tests' {
$packages = (Get-Calls 'pipeline').Packages
$packages | Should -Contain 'fiftyone_pipeline_cloudrequestengine'
$packages | Should -Contain 'fiftyone_pipeline_cloudrequestengine/examples'
$packages | Should -Not -Contain 'fiftyone_pipeline_did'
$script:Output | Should -Match 'fiftyone_pipeline_did.*skipped'
}

It 'skips the Selenium contract against the cloud example and saying so' {
$script:Output | Should -Match 'Selenium contract tests will not run against the cloud example'
Test-Path (Join-Path $script:CommonCi 'selenium-api-tests') |
Should -BeFalse
}
}

Context 'with no TestResourceKey and options a caller set' {
It 'adds the selection to the options, and puts them back after' {
$env:PYTEST_ADDOPTS = '-p no:cacheprovider'
Invoke-ProductStep 'run-integration-tests' $script:Repo `
@{ RepoName = $script:Repo; Keys = @{} } $script:Products *>&1 |
Out-Null

foreach ($product in $script:Products) {
$calls = Get-Calls $product
$calls.Count | Should -BeGreaterThan 0 -Because $product
foreach ($call in $calls) {
$call.AddOpts | Should -Be '-p no:cacheprovider -m "not live"' `
-Because $product
}
}
$env:PYTEST_ADDOPTS | Should -Be '-p no:cacheprovider'
}
}

Context 'with a TestResourceKey' {
It 'runs every test, the ones marked live included' {
Invoke-ProductStep 'run-integration-tests' $script:Repo @{
RepoName = $script:Repo
Keys = @{ TestResourceKey = 'key' }
SeleniumContract = $false
} 'device-detection', 'location' *>&1 | Out-Null

foreach ($product in 'device-detection', 'location') {
$calls = Get-Calls $product
$calls.Count | Should -BeGreaterThan 0 -Because $product
foreach ($call in $calls) {
$call.AddOpts | Should -Not -Match 'live' -Because $product
}
}
}
}

}
52 changes: 39 additions & 13 deletions device-detection/ci/run-integration-tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,14 @@ if ($IsWindows) {

$packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_examples"

# Without a resource key the tests that need one, which are marked live, are
# left out and every other test runs. A live test still fails when it finds
# no key, so a key that is passed but does not reach it is not missed.
# The selection is added to any options a caller set in PYTEST_ADDOPTS.
$addopts = $env:PYTEST_ADDOPTS
if (!$Keys.TestResourceKey) {
throw "No TestResourceKey was passed, so the cloud tests cannot call " +
"the cloud service. The workflows set it from the " +
"_51DEGREES_RESOURCE_KEY_BESPOKE secret."
Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret."
$env:PYTEST_ADDOPTS = "$addopts -m `"not live`"".Trim()
}
# The cloud package runs without the CSV file too, because only its property
# coverage tests read the file, and they say so when it is missing.
Expand All @@ -44,7 +48,11 @@ $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey
# files from _51DEGREES_LICENSE_KEY, which common-ci sets as license_key.
$env:_51DEGREES_LICENSE_KEY = $Keys.DeviceDetection

./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys
try {
./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys
} finally {
$env:PYTEST_ADDOPTS = $addopts
}
$failures = @()
if ($LASTEXITCODE -ne 0) {
$failures += "the Python integration tests exited with code $LASTEXITCODE"
Expand All @@ -58,6 +66,25 @@ if (-not $SeleniumContract) {
return
}

# The contract tests run against the cloud example when there is a resource
# key, and against the on-premise example when there is the TAC data file,
# which is only fetched when a licence is given.
$tac = "$PWD/assets/TAC-HashV41.hash"
$runCloud = [bool]$Keys.TestResourceKey
$runOnPremise = Test-Path $tac
if (-not $runCloud) {
Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No resource key::No TestResourceKey was passed, so the Selenium contract tests will not run against the cloud example."
}
if (-not $runOnPremise) {
Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No TAC Data File::The TAC data file wasn't found, so the Selenium tests will not run against the on-premise example."
}
if (-not ($runCloud -or $runOnPremise)) {
if ($failures.Count -gt 0) {
throw "Integration tests failed: $($failures -join '; ')"
}
return
}

# The shared Selenium contract tests are run against the cloud example and the
# on-premise example in turn. Each result is also written as a trx file next
# to the other integration results, so it is published with them.
Expand Down Expand Up @@ -141,22 +168,21 @@ function Test-Example([string]$Name, [string]$Module, [int]$Port) {
}

# The cloud example, pointed at the live cloud.
$env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey
$env:_51DEGREES_CLOUD_ENDPOINT = "https://cloud.51degrees.com/api/v4/"
$failures += Test-Example -Name 'cloud' `
-Module 'fiftyone_devicedetection_examples.cloud.gettingstarted_web' -Port 8097
if ($runCloud) {
$env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey
$env:_51DEGREES_CLOUD_ENDPOINT = "https://cloud.51degrees.com/api/v4/"
$failures += Test-Example -Name 'cloud' `
-Module 'fiftyone_devicedetection_examples.cloud.gettingstarted_web' -Port 8097
}

# The on-premise example. The contract tests need the device type and the
# JavaScript properties, which the Lite data file does not have, so it runs
# against the TAC data file, which is only fetched when a licence is given.
$tac = "$PWD/assets/TAC-HashV41.hash"
if (Test-Path $tac) {
# against the TAC data file.
if ($runOnPremise) {
$env:_51DEGREES_DD_PATH = (Resolve-Path $tac).Path
$failures += Test-Example -Name 'onpremise' `
-Module 'fiftyone_devicedetection_examples.onpremise.gettingstarted_web' -Port 8098
Remove-Item Env:_51DEGREES_DD_PATH
} else {
Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No TAC Data File::The TAC data file wasn't found, so the Selenium tests will not run against the on-premise example."
}

# A failure anywhere above must fail the job. This script is dot-sourced by
Expand Down
76 changes: 76 additions & 0 deletions device-detection/fiftyone_devicedetection_examples/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
# *********************************************************************
# This Original Work is copyright of 51 Degrees Mobile Experts Limited.
# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House,
# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU.
#
# This Original Work is licensed under the European Union Public Licence
# (EUPL) v.1.2 and is subject to its terms as set out below.
#
# If a copy of the EUPL was not distributed with this file, You can obtain
# one at https://opensource.org/licenses/EUPL-1.2.
#
# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
# amended by the European Commission) shall be deemed incompatible for
# the purposes of the Work and the provisions of the compatibility
# clause in Article 5 of the EUPL shall not apply.
#
# If using the Work as, or as part of, a network application, by
# including the attribution notice(s) required under Article 5 of the EUPL
# in the end user terms of the application under an appropriate heading,
# such notice(s) shall fulfill the requirements of that article.
# *********************************************************************

# The tests that call the live cloud service are marked live here, so the
# released test files stay as they are. ci/run-integration-tests.ps1 leaves
# the live tests out when no resource key is passed, and the other tests run.

from pathlib import Path

import pytest
from _pytest.mark.expression import Expression

TESTS = Path(__file__).resolve().parent / "tests"

# Every test in these modules is live.
LIVE_MODULES = {
TESTS / "test_cloudexamples.py",
TESTS / "test_cloudgettingstartedweb.py",
}

BUILT_AT_IMPORT = {TESTS / "test_cloudgettingstartedweb.py"}


def _is_live(item):
return Path(item.path).resolve() in LIVE_MODULES


def pytest_configure(config):
config.addinivalue_line(
"markers",
"live: calls the live 51Degrees cloud service and needs a resource "
"key")


@pytest.hookimpl(tryfirst=True)
def pytest_collection_modifyitems(config, items):
# Runs before -m deselects, so the selection sees the marker.
for item in items:
if _is_live(item):
item.add_marker(pytest.mark.live)


def _leaves_out_live(config):
"""Whether the -m expression leaves out a test marked live alone."""
expression = config.getoption("markexpr")
return bool(expression) and not Expression.compile(expression).evaluate(
lambda name, **kwargs: name == "live")


# A module that builds a cloud Pipeline when imported raises with no resource
# key, so it is not collected when the live tests are left out. Every test in
# it is live, so the selection is the same.
def pytest_ignore_collect(collection_path, config):
if (Path(collection_path).resolve() in BUILT_AT_IMPORT and
_leaves_out_live(config)):
return True
return None
4 changes: 3 additions & 1 deletion device-detection/fiftyone_devicedetection_examples/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,9 @@ the Enterprise data file.

The cloud tests need a valid 51Degrees resource key (see above), and fail
without one. They read it from the environment variable
`_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`.
`_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`. They are
marked `live`, so `python -m pytest -m "not live"` runs every other test,
which is what CI does when no resource key is passed.

For example, use following command to set the resource key on Linux:
`export _51DEGREES_RESOURCE_KEY=MY-RESOURCE-KEY`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,14 @@
from contextlib import redirect_stdout
from unittest import mock

import pytest

from fiftyone_devicedetection_examples.example_utils import ExampleUtils

from .live_cloud import answered_without, find_resource_key, no_resource_key

pytestmark = pytest.mark.live

MODULE = "fiftyone_devicedetection_examples.cloud.failuretomatch"

NO_KEY = no_resource_key("the device product", "the IsMobile property")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,17 @@
import re
import unittest

import pytest

from fiftyone_pipeline_core.logger import Logger

from fiftyone_devicedetection_examples.cloud.useragentclienthints_web.app \
import UserAgentClientHintsWeb

from .live_cloud import find_resource_key, no_resource_key

pytestmark = pytest.mark.live

NO_KEY = no_resource_key(
"the device product", "device detection properties such as BrowserName")

Expand Down
Loading
Loading