From df0949d7313b11214c1758fbe7ee8c3334c53476 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 08:17:47 +0000 Subject: [PATCH 001/136] Say in AGENTS.md that a test failing on timing is fixed, not run again (#115) --- AGENTS.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 1b697ff7..39b710d7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -334,3 +334,11 @@ into `main` from any other branch. The four repositories were brought in with `git subtree add`, so commit SHAs match the originals. Use `git log --follow` to trace a file back through the import. + +## Tests that fail on timing + +A test that fails on a time limit, a performance threshold or a timeout is +fixed, not run again until it passes. Either the limit is set to fit the +slowest runner the test runs on, with each runner's measurement behind it, or +the reason it takes too long is found and fixed. A failure that comes and goes +costs everyone who meets it the time to look into it. From d7480eee6b0cf39ac154c1bdd2e2b8bfc602d321 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 19:10:51 +0000 Subject: [PATCH 002/136] Label a one-package release with its version, not its last character (#54) * Label a one-package release with its version, not its last character When the release plan holds one package, $plan.Version is a single string, Sort-Object hands it back as a string, and [-1] took its last character, so a plan releasing 4.5.3 alone gave the run the label 3, which the publish's version check refuses. The sort is now collected into an array first. * Test the publish label for a plan of one package, several packages and none --- ci/get-next-package-version.Tests.ps1 | 46 +++++++++++++++++++++++++++ ci/get-next-package-version.ps1 | 2 +- 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 ci/get-next-package-version.Tests.ps1 diff --git a/ci/get-next-package-version.Tests.ps1 b/ci/get-next-package-version.Tests.ps1 new file mode 100644 index 00000000..706e6f0c --- /dev/null +++ b/ci/get-next-package-version.Tests.ps1 @@ -0,0 +1,46 @@ +# Tests for ci/get-next-package-version.ps1, which labels a publish with the +# highest version in the release plan. +# +# Each test copies the script to a throwaway folder beside a packages.ps1 +# whose release plan is the versions the test gives, so the test states the +# plan it labels. + +Describe 'get-next-package-version.ps1' { + BeforeAll { + $script:Label = Join-Path $PSScriptRoot 'get-next-package-version.ps1' + + function Get-Label([string[]]$Versions) { + $folder = Join-Path ([IO.Path]::GetTempPath()) ` + ([Guid]::NewGuid().ToString()) + $null = New-Item -ItemType Directory $folder + try { + Copy-Item $script:Label $folder + $list = ($Versions | ForEach-Object { "'$_'" }) -join ', ' + Set-Content (Join-Path $folder 'packages.ps1') @" +function Get-ReleasePlan(`$RepoPath) { + @(@($list) | ForEach-Object { + [pscustomobject]@{ Name = "package-`$_"; Version = `$_ } + }) +} +"@ + & (Join-Path $folder 'get-next-package-version.ps1') ` + -RepoName 'repository' -VariableName 'TestLabel' + return $global:TestLabel + } finally { + Remove-Item -Recurse -Force $folder + } + } + } + + It 'labels a plan of one package with its whole version' { + Get-Label @('4.5.3') | Should -Be '4.5.3' + } + + It 'labels a plan of several packages with the highest version' { + Get-Label @('4.5.3', '4.5.10', '4.5.9') | Should -Be '4.5.10' + } + + It 'gives a valid version when nothing is released' { + Get-Label @() | Should -Be '4.5.0' + } +} diff --git a/ci/get-next-package-version.ps1 b/ci/get-next-package-version.ps1 index d94aa4d7..0456e988 100644 --- a/ci/get-next-package-version.ps1 +++ b/ci/get-next-package-version.ps1 @@ -13,7 +13,7 @@ $ErrorActionPreference = "Stop" # build-package.ps1 and publish-package.ps1, all from the same release plan. $plan = @(Get-ReleasePlan $RepoName) if ($plan.Count -gt 0) { - $label = ($plan.Version | Sort-Object { [version]$_ })[-1] + $label = @($plan.Version | Sort-Object { [version]$_ })[-1] } else { # Nothing to release. Any valid version will do, because # package-update-required.ps1 stops the publish before it is used. From 94073ec8422a85850dd84039627fe55daef6827c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 19:11:21 +0000 Subject: [PATCH 003/136] Point AGENTS.md at the one list of environment variable names (#109) * DOC: Point at the one list of environment variable names AGENTS.md gains a section linking common-ci's list of environment variable names (environment-variables.md, made from environment-variables.json) at commit 18f9a570, and saying the list decides. Documentation only. * DOC: Link the list of environment variable names at d8a9c596 The list at that commit holds the names review found missing. * Keep one Releases section in AGENTS.md --- AGENTS.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 39b710d7..f09f4b1a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -250,6 +250,19 @@ example logs an error that names the variable and the configurator. assets, and the IP Intelligence example tests use the Asn file and throw their output away. +## Environment variable names + +[common-ci's list of environment variable names](https://github.com/51Degrees/common-ci/blob/d8a9c596eabf69dd5581a08e9360cdd1c9b3d9e8/environment-variables.md) +holds every environment variable name that the packages, tests, examples and +CI scripts of every language read, with what each name holds and the +deprecated names each replaced. The list decides. Use the names it gives, and +when a change needs a new name, or reads an old name again, change +[`environment-variables.json`](https://github.com/51Degrees/common-ci/blob/d8a9c596eabf69dd5581a08e9360cdd1c9b3d9e8/environment-variables.json) +in common-ci in the same piece of work. This repository's own name +check, [`ci/check-environment-variable-names.ps1`](ci/check-environment-variable-names.ps1), still keeps a +list of its own. Where the two differ, common-ci's list is the one to +follow. + ## Products keep their own CI scripts Each product folder still has the `ci/` scripts it had as its own repository, @@ -321,14 +334,6 @@ draft and has no review requested. When a release is wanted, one pull request merges `rc` into `main`, and the "Only rc into main" check fails a pull request into `main` from any other branch. -## Releases - -Pull requests go to `rc`, not `main`, and stay drafts until they have been -reviewed, because the nightly merges any pull request into `rc` that is not a -draft and has no review requested. When a release is wanted, one pull request -merges `rc` into `main`, and the "Only rc into main" check fails a pull request -into `main` from any other branch. - ## History The four repositories were brought in with `git subtree add`, so commit SHAs From 5b5a833239d435676a1ab0450be22bce0c0807a2 Mon Sep 17 00:00:00 2001 From: Eugene Dorfman Date: Fri, 13 Nov 2026 20:12:07 +0100 Subject: [PATCH 004/136] Run the Cloud 51Did tests on pull requests into rc (#88) Pull requests go to rc, so a 51Did change was first checked against the live cloud only when the release pull request into main opened. --- .github/workflows/cloud-51did.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cloud-51did.yml b/.github/workflows/cloud-51did.yml index 5f258dc1..2a2e6ee2 100644 --- a/.github/workflows/cloud-51did.yml +++ b/.github/workflows/cloud-51did.yml @@ -29,7 +29,7 @@ on: schedule: - cron: '30 5 * * *' pull_request: - branches: [main] + branches: [main, rc] paths: - 'pipeline/fiftyone_pipeline_did/**' - 'pipeline/owid-python/**' From 5eb295ce76d4666d71234dcbf0ff2eab5d0c1e07 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 22:29:52 +0000 Subject: [PATCH 005/136] Release a package only for a change that counts, leaving out documentation, tests, examples and manifest metadata (#60) Reviewed and approved at head ec8b2fdd94f4f31fbda1dfad1ad31227fbdca734. The pull request body says what changed and how it was tested. --- AGENTS.md | 17 ++- ci/packages.Tests.ps1 | 295 +++++++++++++++++++++++++++++++++++++ ci/packages.json | 19 +++ ci/packages.ps1 | 110 ++++++++++++-- ci/release-plan-filter.ps1 | 39 +++++ ci/release-plan-filter.py | 194 ++++++++++++++++++++++++ 6 files changed, 657 insertions(+), 17 deletions(-) create mode 100644 ci/release-plan-filter.ps1 create mode 100644 ci/release-plan-filter.py diff --git a/AGENTS.md b/AGENTS.md index f09f4b1a..3fe39ce4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -280,11 +280,18 @@ touches all of them ([ci/packages.ps1](ci/packages.ps1), ## Versions are per package Every package is versioned and tagged on its own, `/`. A -package is released when its folders changed since its last tag, or when a -package it depends on is released. The examples packages are tested but never -published. common-ci's publish carries one version; here it is only a label, -and the release plan in `ci/packages.ps1` decides what is built, published -and tagged. Never push a plain `4.5.x` tag: nothing here reads one. +package is released when a file that counts in its folders changed since its +last tag, or when a package it depends on is released. Documentation +(`*.md`, `docs/`), tests (`tests/`, `test/`), examples (`examples/`) and +`LICENSE` files do not count, nor do files a package's `Ignore` globs in +`ci/packages.json` match, unless it sets `IgnoreDefaults` to false. A +`setup.py`, `setup.cfg` or `pyproject.toml` whose change is only to the +description, keywords, urls, classifiers, readme, authors or maintainers does +not count either, which `ci/release-plan-filter.ps1` works out with Python. +The examples packages are tested but never published. common-ci's publish +carries one version, which here is only a label, and the release plan in +`ci/packages.ps1` decides what is built, published and tagged. Never push a +plain `4.5.x` tag, because nothing here reads one. `ci/packages.json` is the list of packages and what each depends on. Add a package there, with a `FirstVersion` if it has never been released, or it is diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index 766d1380..243d87a6 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -247,6 +247,267 @@ Describe 'Get-ReleasePlan' { $script:Repo = New-Repo @('p/new/a.py') @() { Get-ReleasePlan $script:Repo $graph } | Should -Throw '*FirstVersion*' } + + It 'says how many files counted' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/a.py', 'device/dd/b.py') + (@(Get-ReleasePlan $script:Repo $script:Graph) | + Where-Object Name -eq 'dd').Reason | + Should -Be 'changed since 4.5.96 (2 files)' + } + + It 'releases nothing for a change only to documentation, tests or examples' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @( + 'device/dd/readme.md', 'device/dd/docs/guide.txt', + 'device/dd/tests/test_a.py', 'device/dd/test/a.py', + 'device/dd/examples/a.py', 'device/dd/LICENSE', + 'pipeline/core/sub/notes.md') + @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 + } + + It 'ignores documentation, tests and examples whatever their case' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @( + 'device/dd/README.MD', 'device/dd/Tests/a.py', + 'device/dd/Examples/a.py') + @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 + } + + It 'releases for the code changed beside the documentation' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/readme.md', 'device/dd/a.py') + $plan = @(Get-ReleasePlan $script:Repo $script:Graph) + $plan.Name | Should -Be @('dd') + $plan[0].Reason | Should -Be 'changed since 4.5.96 (1 file)' + } + + It 'leaves out the files a package''s Ignore globs match' { + $graph = @($script:Graph | ForEach-Object { $_.PSObject.Copy() }) + ($graph | Where-Object Name -eq 'dd') | + Add-Member Ignore @('device/dd/tox.ini') + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/tox.ini') + @(Get-ReleasePlan $script:Repo $graph).Count | Should -Be 0 + } + + It 'counts documentation for a package that turns the defaults off' { + $graph = @($script:Graph | ForEach-Object { $_.PSObject.Copy() }) + ($graph | Where-Object Name -eq 'dd') | Add-Member IgnoreDefaults $false + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/readme.md') + @(Get-ReleasePlan $script:Repo $graph).Name | Should -Be @('dd') + } + + It 'refuses IgnoreDefaults given as text' { + $graph = @($script:Graph | ForEach-Object { $_.PSObject.Copy() }) + ($graph | Where-Object Name -eq 'dd') | Add-Member IgnoreDefaults 'false' + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/readme.md') + { Get-ReleasePlan $script:Repo $graph } | + Should -Throw '*IgnoreDefaults*true or false*' + } +} + +Describe 'release-plan-filter.ps1' { + BeforeAll { + # A setup.py, setup.cfg and pyproject.toml like the ones here, with + # the metadata the filter leaves out and settings it does not. Line + # endings are made LF, whatever the checkout gave this file. + $script:SetupPy = @' +import setuptools + +setuptools.setup( + name="dd", + version=read("version.txt"), + author="51Degrees Engineering", + author_email="engineering@51degrees.com", + project_urls={"Source": "https://github.com/51Degrees/51degrees-python"}, + description=("Device detection."), + keywords=["51degrees", "device-detection"], + long_description=read("readme.md"), + long_description_content_type='text/markdown', + python_requires=">=3.10", + install_requires=["fiftyone_pipeline_core"], + classifiers=["Programming Language :: Python"], +) +'@.Replace("`r`n", "`n") + $script:SetupCfg = @' +[metadata] +name = dd +description = Device detection. +url = https://github.com/51Degrees/51degrees-python + +[options] +install_requires = + fiftyone_pipeline_core +'@.Replace("`r`n", "`n") + "`n" + $script:Pyproject = @' +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "dd" +description = "Device detection." +keywords = [ + "51degrees", + "device-detection", +] +dependencies = ["fiftyone_pipeline_core"] + +[project.urls] +Source = "https://github.com/51Degrees/51degrees-python" +'@.Replace("`r`n", "`n") + "`n" + + # A repository with the given files, holding the filter as this + # repository does, with the tags of the small graph and the tag of + # dd on the commit that adds the files. + function New-FilterRepo([hashtable]$Files) { + $repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + $null = New-Item -ItemType Directory -Force (Join-Path $repo 'ci') + foreach ($name in 'release-plan-filter.ps1', 'release-plan-filter.py') { + Copy-Item (Join-Path $PSScriptRoot $name) (Join-Path $repo 'ci') + } + Set-Files $repo $Files + git -C $repo tag -f 'dd/4.5.96' | Out-Null + return $repo + } + + # Commits the files with exactly the text given. + function Set-Files([string]$Repo, [hashtable]$Files) { + foreach ($file in $Files.Keys) { + $full = Join-Path $Repo $file + $null = New-Item -ItemType Directory -Force (Split-Path $full) + [IO.File]::WriteAllText($full, $Files[$file]) + } + git -C $Repo add -A + git -C $Repo -c user.name=test -c user.email=test@example.com ` + commit --quiet --allow-empty -m 'Change' + } + + function Get-DdPlan([string]$Repo) { + return @(Get-ReleasePlan $Repo $script:Graph | + Where-Object Name -eq 'dd') + } + } + + It 'does not release for a setup.py change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + $changed = $script:SetupPy + $changed = $changed.Replace('Device detection.', 'Device detection for Python.') + $changed = $changed.Replace('"device-detection"]', '"device-detection", "user-agent"]') + $changed = $changed.Replace('51degrees-python"}', '51degrees-python", "Docs": "x"}') + $changed = $changed.Replace('51Degrees Engineering', '51Degrees') + $changed = $changed.Replace('"Programming Language :: Python"]', + '"Programming Language :: Python", "Typing :: Typed"]') + $changed = $changed.Replace("'text/markdown'", "'text/x-rst'") + Set-Files $script:Repo @{ 'device/dd/setup.py' = $changed + "# A note.`n" } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + + It 'releases for a setup.py change to anything else' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + $changed = $script:SetupPy.Replace('["fiftyone_pipeline_core"]', + '["fiftyone_pipeline_core>=4.5.40"]') + Set-Files $script:Repo @{ 'device/dd/setup.py' = $changed } + $plan = Get-DdPlan $script:Repo + $plan.Count | Should -Be 1 + $plan[0].Reason | Should -Be 'changed since 4.5.96 (1 file)' + } + + It 'counts the code changed beside a setup.py change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + Set-Files $script:Repo @{ + 'device/dd/setup.py' = $script:SetupPy.Replace( + 'Device detection.', 'Device detection for Python.') + 'device/dd/src/a.py' = 'print(1)' + } + (Get-DdPlan $script:Repo)[0].Reason | + Should -Be 'changed since 4.5.96 (1 file)' + } + + It 'does not release for a setup.cfg change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.cfg' = $script:SetupCfg } + $changed = $script:SetupCfg.Replace('Device detection.', 'Device detection for Python.') + $changed = $changed.Replace('51degrees-python', '51degrees-python/tree/main') + Set-Files $script:Repo @{ 'device/dd/setup.cfg' = $changed } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + + It 'releases for a setup.cfg change to its options' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.cfg' = $script:SetupCfg } + Set-Files $script:Repo @{ 'device/dd/setup.cfg' = + $script:SetupCfg + " fiftyone_pipeline_engines`n" } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } + + It 'does not release for a pyproject.toml change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/pyproject.toml' = $script:Pyproject } + $changed = $script:Pyproject.Replace('"Device detection."', '"Device detection for Python."') + $changed = $changed.Replace(" `"device-detection`",`n", + " `"device-detection`",`n `"user-agent`",`n") + $changed = $changed.Replace('Source = ', "Docs = `"x`"`nSource = ") + $changed += "`n[[project.authors]]`nname = `"51Degrees`"`n" + Set-Files $script:Repo @{ 'device/dd/pyproject.toml' = $changed } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + + It 'does not release for a change only to the maintainers in ' -ForEach @( + @{ Case = 'setup.py'; File = 'setup.py'; Start = 'SetupPy' + Was = 'author_email="engineering@51degrees.com",' + Changed = "author_email=`"engineering@51degrees.com`",`n maintainer=`"51Degrees`",`n maintainer_email=`"support@51degrees.com`"," } + @{ Case = 'setup.cfg'; File = 'setup.cfg'; Start = 'SetupCfg' + Was = "name = dd`n" + Changed = "name = dd`nmaintainer = 51Degrees`nmaintainer_email = support@51degrees.com`n" } + @{ Case = 'a pyproject.toml key'; File = 'pyproject.toml'; Start = 'Pyproject' + Was = 'dependencies = ' + Changed = "maintainers = [{ name = `"51Degrees`" }]`ndependencies = " } + @{ Case = 'a pyproject.toml table'; File = 'pyproject.toml'; Start = 'Pyproject' + Was = '[project.urls]' + Changed = "[[project.maintainers]]`nname = `"51Degrees`"`n`n[project.urls]" } + ) { + $text = @{ + SetupPy = $script:SetupPy + SetupCfg = $script:SetupCfg + Pyproject = $script:Pyproject + }[$Start] + $changed = $text.Replace($Was, $Changed) + $changed | Should -Not -Be $text + $script:Repo = New-FilterRepo @{ "device/dd/$File" = $text } + Set-Files $script:Repo @{ "device/dd/$File" = $changed } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + + It 'releases for a pyproject.toml change to ' -ForEach @( + @{ Was = '["fiftyone_pipeline_core"]'; Changed = '["fiftyone_pipeline_core>=4.5"]' } + @{ Was = '"setuptools>=42"'; Changed = '"setuptools>=61"' } + ) { + $script:Repo = New-FilterRepo @{ 'device/dd/pyproject.toml' = $script:Pyproject } + Set-Files $script:Repo @{ 'device/dd/pyproject.toml' = + $script:Pyproject.Replace($Was, $Changed) } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } + + It 'releases for a manifest that is new since the last tag' { + $script:Repo = New-FilterRepo @{} + Set-Files $script:Repo @{ 'device/dd/setup.py' = $script:SetupPy } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } + + It 'releases for a manifest it cannot read' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + Set-Files $script:Repo @{ 'device/dd/setup.py' = 'setup(' } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } + + It 'releases for a manifest that is not UTF-8' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + [IO.File]::WriteAllBytes((Join-Path $script:Repo 'device/dd/setup.py'), + [byte[]](0x23, 0xff, 0xfe, 0x0a)) + Set-Files $script:Repo @{} + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } } Describe 'Get-BuildPlan' { @@ -320,6 +581,40 @@ Describe 'the real packages.json' { Get-ToxGaps (Split-Path $PSScriptRoot) $script:Real | Should -BeNullOrEmpty } + + Context 'a change to a package''s tox.ini' { + BeforeAll { + $published = @($script:Real | Where-Object Publish -eq $true) + $script:RealTags = @($published | ForEach-Object { "$($_.Name)/4.5.1" }) + $script:RealFiles = @($script:Real | ForEach-Object { + "$($_.Path)/src/a.py" + "$($_.Path)/tox.ini" + }) + } + + It 'releases nothing, whichever package it sits beside' { + $script:Repo = New-Repo $script:RealFiles $script:RealTags + Add-Commit $script:Repo ($script:Real | ForEach-Object { "$($_.Path)/tox.ini" }) + @(Get-ReleasePlan $script:Repo $script:Real).Count | Should -Be 0 + } + + It 'still releases a package for a change to its source' { + $script:Repo = New-Repo $script:RealFiles $script:RealTags + Add-Commit $script:Repo @( + 'pipeline/fiftyone_pipeline_core/tox.ini', + 'pipeline/fiftyone_pipeline_core/src/a.py') + $plan = @(Get-ReleasePlan $script:Repo $script:Real) + $plan.Name | Should -Contain 'fiftyone_pipeline_core' + ($plan | Where-Object Name -eq 'fiftyone_pipeline_core').Reason | + Should -Be 'changed since 4.5.1 (1 file)' + } + + It 'is ignored by every package, so one added later counts for nothing' { + foreach ($package in $script:Real) { + $package.Ignore | Should -Contain '**/tox.ini' -Because $package.Name + } + } + } } } diff --git a/ci/packages.json b/ci/packages.json index deb06ce9..d9dcd87d 100644 --- a/ci/packages.json +++ b/ci/packages.json @@ -3,6 +3,7 @@ "Name": "fiftyone_pipeline_core", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_core", + "Ignore": ["**/tox.ini"], "DependsOn": [], "Publish": true }, @@ -10,6 +11,7 @@ "Name": "fiftyone_pipeline_engines", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_engines", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_core"], "Publish": true }, @@ -17,6 +19,7 @@ "Name": "fiftyone_pipeline_engines_fiftyone", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_engines_fiftyone", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_engines"], "Publish": true }, @@ -24,6 +27,7 @@ "Name": "fiftyone_pipeline_cloudrequestengine", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_cloudrequestengine", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_engines"], "Publish": true }, @@ -31,6 +35,7 @@ "Name": "fiftyone_pipeline_translation", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_translation", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_core"], "Publish": true }, @@ -38,6 +43,7 @@ "Name": "fiftyone_pipeline_did", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_did", + "Ignore": ["**/tox.ini"], "ExtraPaths": ["pipeline/owid-python"], "DependsOn": [], "Publish": true @@ -46,6 +52,7 @@ "Name": "fiftyone_pipeline_examples", "Product": "pipeline", "Path": "pipeline/examples", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -57,6 +64,7 @@ "Name": "fiftyone_devicedetection_shared", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_shared", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -69,6 +77,7 @@ "Name": "fiftyone_devicedetection_cloud", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_cloud", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_devicedetection_shared"], "Publish": true }, @@ -76,6 +85,7 @@ "Name": "fiftyone_devicedetection_onpremise", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_onpremise", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_devicedetection_shared", "fiftyone_pipeline_engines_fiftyone" @@ -87,6 +97,7 @@ "Name": "fiftyone_devicedetection", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_devicedetection_shared", "fiftyone_devicedetection_cloud", @@ -98,6 +109,7 @@ "Name": "fiftyone_devicedetection_examples", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_examples", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_devicedetection_cloud", "fiftyone_devicedetection" @@ -108,6 +120,7 @@ "Name": "fiftyone_location", "Product": "location", "Path": "location", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -119,6 +132,7 @@ "Name": "fiftyone_ipintelligence_shared", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_shared", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -132,6 +146,7 @@ "Name": "fiftyone_ipintelligence_cloud", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_cloud", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_ipintelligence_shared"], "Publish": true, "FirstVersion": "4.5.0" @@ -140,6 +155,7 @@ "Name": "fiftyone_ipintelligence_onpremise", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_onpremise", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_ipintelligence_shared", "fiftyone_pipeline_engines_fiftyone" @@ -152,6 +168,7 @@ "Name": "fiftyone_ipintelligence_translation", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_translation", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_translation" @@ -163,6 +180,7 @@ "Name": "fiftyone_ipintelligence", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_ipintelligence_shared", "fiftyone_ipintelligence_cloud", @@ -175,6 +193,7 @@ "Name": "fiftyone_ipintelligence_examples", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_examples", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_ipintelligence", "fiftyone_ipintelligence_translation" diff --git a/ci/packages.ps1 b/ci/packages.ps1 index 429c43ff..7529a333 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -4,10 +4,20 @@ # which version. # # Each package has its own version, tagged '/'. A package is -# released when a file under its folder changed since its last tag, or when a -# package it depends on is being released, because its requirement on that -# package moves. This is the scheme in 51Degrees/rust#205, which the per -# language repositories follow. +# released when a file that counts under its folders changed since its last +# tag, or when a package it depends on is being released, because its +# requirement on that package moves. This is the scheme in 51Degrees/rust#205, +# which the per language repositories follow. +# +# Every file counts except those matched by the package's Ignore globs in +# packages.json and, unless its IgnoreDefaults is false, by +# $ReleasePlanDefaultIgnore, so a change only to documentation, tests or +# examples releases nothing. release-plan-filter.ps1 then leaves out a +# manifest whose change is only to metadata such as links. These are the +# fields and the filter of common-ci's steps/release-plan.ps1. +# +# A submodule in Path or ExtraPaths changes as one path, its own, so any move +# of it counts. Neither the globs nor the filter see inside it. # Reads packages.json. Kept as a function so the tests can give their own list. function Get-Packages([string]$RepoPath) { @@ -30,6 +40,79 @@ function Test-PathUnder([string]$Path, [string]$Folder) { return $Path -eq $Folder -or $Path.StartsWith("$Folder/") } +# The files whose changes do not release a package, being documentation, +# tests, examples and LICENSE files wherever they sit and whatever their case, +# unless the package sets IgnoreDefaults to false. These are git glob +# pathspecs from the repository root, where '**/' matches any number of +# folders, none included. +$ReleasePlanDefaultIgnore = @( + '**/*.md' + '**/docs/**' + '**/tests/**' + '**/test/**' + '**/examples/**' + '**/LICENSE*' +) + +# The pathspecs that leave out of a package's changes the files its Ignore +# globs match and, unless IgnoreDefaults is false, the default set. +# PowerShell on Linux and macOS replaces a native command's argument that +# holds a '*' with the files it matches, if any. Each of these starts with +# ':(', which no file in a checkout starts with, so git receives it as written. +function Get-PackageExcludes($Package) { + $defaults = $Package.IgnoreDefaults + if ($null -ne $defaults -and $defaults -isnot [bool]) { + throw "$($Package.Name) gives IgnoreDefaults as '$defaults', where " + + "packages.json must give true or false." + } + $excludes = @($Package.Ignore | Where-Object { $_ } | + ForEach-Object { ":(exclude,glob)$_" }) + if ($defaults -ne $false) { + # The defaults ignore case, as one language names a folder Tests or + # Examples where another names it tests or examples. + $excludes = @($ReleasePlanDefaultIgnore | + ForEach-Object { ":(exclude,glob,icase)$_" }) + $excludes + } + return $excludes +} + +# The files under the package's folders that changed since the given tag, as +# paths from the repository root, apart from those it ignores. +function Get-ChangedFiles([string]$RepoPath, $Package, [string]$From) { + # A failure is reported below, naming the package, whatever the caller's + # preference. Set for this function only. + $PSNativeCommandUseErrorActionPreference = $false + $pathspecs = @(Get-PackagePaths $Package) + @(Get-PackageExcludes $Package) + $files = @(& git -C $RepoPath diff --name-only $From HEAD -- $pathspecs) + if ($LASTEXITCODE -ne 0) { + throw "git could not list the changes to $($Package.Name) since $From." + } + return $files +} + +# The files that count among a package's changed files. The repository's +# ci/release-plan-filter.ps1 decides when there is one, and otherwise every +# file counts. A path the filter returns that it was not given stops the +# plan, because a filter that rewrote a path would otherwise release a +# package on a file that never changed. +function Select-CountedFiles([string]$RepoPath, $Package, [string]$From, + [string[]]$Files) { + $filter = Join-Path $RepoPath 'ci/release-plan-filter.ps1' + if ($Files.Count -eq 0 -or (Test-Path $filter -PathType Leaf) -eq $false) { + return $Files + } + $filter = (Resolve-Path $filter).ProviderPath + $kept = @(& $filter -RepoPath $RepoPath -Package $Package -From $From ` + -Files $Files | Where-Object { $_ }) + foreach ($file in $kept) { + if ($Files -cnotcontains $file) { + throw "ci/release-plan-filter.ps1 returned '$file' for " + + "$($Package.Name), which is not one of the files it was given." + } + } + return $kept +} + # Each tox.ini that does not install every package its package depends on, # directly or through another, from this checkout, as a line naming what is # missing. Taken from a package index instead, a dependency is tested at its @@ -224,11 +307,6 @@ function Get-ReleasedVersions([string]$RepoPath, [string]$Name, # from packages.json. Packages that are not published, the examples, are never # in the plan; a change to them is tested but releases nothing. function Get-ReleasePlan([string]$RepoPath, $Packages = $null) { - # 'git diff --quiet' exits 1 to say a package changed. That is an answer, - # not a failure, but a caller that stops on a failing native command, as - # publish-package.ps1 does, stopped there and failed the publish. Set for - # this function only. - $PSNativeCommandUseErrorActionPreference = $false if ($null -eq $Packages) { $Packages = Get-Packages $RepoPath } @@ -254,10 +332,18 @@ function Get-ReleasePlan([string]$RepoPath, $Packages = $null) { $last = $released[-1] $previous[$package.Name] = $last $tag = "$($package.Name)/$last" - & git -C $RepoPath diff --quiet $tag HEAD -- (Get-PackagePaths $package) - if ($LASTEXITCODE -eq 1) { - $reasons[$package.Name] = "changed since $last" + $files = @(Get-ChangedFiles $RepoPath $package $tag) + $counted = @(Select-CountedFiles $RepoPath $package $tag $files) + if ($counted.Count -gt 0) { + $noun = 'files' + if ($counted.Count -eq 1) { + $noun = 'file' + } + $reasons[$package.Name] = + "changed since $last ($($counted.Count) $noun)" } + # A filter may leave an exit code that was an answer, which must not + # read as a failure to the script that called this. $global:LASTEXITCODE = 0 } diff --git a/ci/release-plan-filter.ps1 b/ci/release-plan-filter.ps1 new file mode 100644 index 00000000..dcd316aa --- /dev/null +++ b/ci/release-plan-filter.ps1 @@ -0,0 +1,39 @@ +param ( + [Parameter(Mandatory)][string]$RepoPath, + [Parameter(Mandatory)]$Package, + [Parameter(Mandatory)][string]$From, + [Parameter(Mandatory)][string[]]$Files +) +$ErrorActionPreference = "Stop" + +# Called by the release plan for each package with changed files left once +# the Ignore globs have left theirs out, and returns the files that count. +# A pyproject.toml, setup.py or setup.cfg whose change since $From is only to +# the description, keywords, urls, classifiers, readme, authors or maintainers +# does not count, which release-plan-filter.py works out. Every other file +# counts. +# +# Python is required rather than optional, because a job that counted every +# file would plan different releases from a job that could compare them. + +$manifests = 'pyproject.toml', 'setup.py', 'setup.cfg' +$compare = @($Files | Where-Object { (Split-Path -Leaf $_) -in $manifests }) +$counted = @($Files | Where-Object { (Split-Path -Leaf $_) -notin $manifests }) + +if ($compare.Count -gt 0) { + $python = Get-Command python, python3 -CommandType Application ` + -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($null -eq $python) { + throw "Python is needed to tell whether a change to " + + "$($compare -join ', ') is only to metadata, and none was found." + } + $PSNativeCommandUseErrorActionPreference = $false + $script = Join-Path $PSScriptRoot 'release-plan-filter.py' + $changed = @(& $python.Source -I $script $RepoPath $From @compare) + if ($LASTEXITCODE -ne 0) { + throw "release-plan-filter.py failed for $($Package.Name) since $From." + } + $counted += $changed | Where-Object { $_ } +} + +$counted diff --git a/ci/release-plan-filter.py b/ci/release-plan-filter.py new file mode 100644 index 00000000..94191c90 --- /dev/null +++ b/ci/release-plan-filter.py @@ -0,0 +1,194 @@ +"""Prints the manifest files whose change since a tag is more than metadata. + +Usage: release-plan-filter.py [ ...] + +Each file is a pyproject.toml, setup.py or setup.cfg, as a path from the +repository root. Its text at the tag and at HEAD is compared with the +description, keywords, urls, classifiers, readme, authors and maintainers +left out, so a file is printed when anything else changed, and left out when +only those did. A file that is missing on either side, or that cannot be read as what +its name says, is printed, so a release is never skipped by mistake. + +Only the standard library is used, and the same way on every Python this +repository supports, so every job of a publish works out the same plan. +""" + +import ast +import configparser +import os +import re +import subprocess +import sys + +# The setup() arguments and setup.cfg [metadata] options that are metadata. +SETUP_METADATA = frozenset(( + "description", "long_description", "long_description_content_type", + "keywords", "url", "project_urls", "download_url", "classifiers", + "author", "author_email", "maintainer", "maintainer_email")) + +# The pyproject.toml [project] keys that are metadata. +PROJECT_METADATA = frozenset(( + "description", "keywords", "urls", "classifiers", "readme", "authors", + "maintainers")) + +# A table header, such as [project.urls] or [[project.maintainers]]. +HEADER = re.compile(r"^\[\[?\s*([^\]]+?)\s*\]\]?\s*(#.*)?$") + +# The start of a key and value. The first part of a dotted key is the one +# that says what it is, as urls in urls.Homepage. +ASSIGNMENT = re.compile( + r"^(\"[^\"]*\"|'[^']*'|[A-Za-z0-9_-]+)" + r"(?:\s*\.\s*(?:\"[^\"]*\"|'[^']*'|[A-Za-z0-9_-]+))*\s*=") + + +def bytes_at(repository, revision, path): + """The file's content at the revision, or None when it is not there.""" + found = subprocess.run( + ["git", "-C", repository, "show", revision + ":" + path], + capture_output=True) + if found.returncode != 0: + return None + return found.stdout + + +class StripSetupMetadata(ast.NodeTransformer): + """Removes the metadata arguments from every call of setup().""" + + def visit_Call(self, node): + self.generic_visit(node) + name = node.func.attr if isinstance(node.func, ast.Attribute) \ + else getattr(node.func, "id", None) + if name == "setup": + node.keywords = [ + keyword for keyword in node.keywords + if keyword.arg not in SETUP_METADATA] + return node + + +def setup_py(text): + # The syntax tree leaves out comments and layout as well. + return ast.dump(StripSetupMetadata().visit(ast.parse(text))) + + +def setup_cfg(text): + parser = configparser.RawConfigParser() + parser.optionxform = str + parser.read_string(text) + return { + section: { + key: value for key, value in parser.items(section) + if section != "metadata" or key not in SETUP_METADATA} + for section in parser.sections()} + + +def toml_lines(text): + """The lines of a TOML file that are not metadata, each with the table + and key it belongs to, leaving out blank lines and comments. + + A small reader rather than tomllib, which Python 3.10 does not have. It + follows tables, keys, values that run over several lines in brackets or + in triple quotes, and comments, which is enough to place every line. + """ + kept = [] + table = "" + key = None + depth = 0 + quote = None + for line in text.splitlines(): + stripped = line.strip() + if quote is None and depth == 0: + if stripped == "" or stripped.startswith("#"): + continue + header = HEADER.match(stripped) + if header: + table = header.group(1).replace(" ", "") + key = None + if not is_metadata(table, None): + kept.append((table, None, stripped)) + continue + assignment = ASSIGNMENT.match(stripped) + if assignment is None: + # Not TOML this reader knows, so it counts unless the whole + # table is metadata. + if not is_metadata(table, "?"): + kept.append((table, "?", stripped)) + continue + key = assignment.group(1).strip("\"'") + value = stripped[assignment.end():] + else: + value = stripped + depth, quote = scan(value, depth, quote) + if not is_metadata(table, key): + kept.append((table, key, stripped)) + return kept + + +def scan(value, depth, quote): + """The bracket depth and the open triple quote after the value.""" + position = 0 + while position < len(value): + if quote is not None: + end = value.find(quote, position) + if end < 0: + return depth, quote + position = end + 3 + quote = None + continue + character = value[position] + if value.startswith('"""', position) or \ + value.startswith("'''", position): + quote = value[position:position + 3] + position += 3 + continue + if character in "\"'": + end = value.find(character, position + 1) + position = len(value) if end < 0 else end + 1 + continue + if character == "#": + break + if character in "[{": + depth += 1 + elif character in "]}": + depth -= 1 + position += 1 + return depth, quote + + +def is_metadata(table, key): + if table == "project": + return key in PROJECT_METADATA + return table in ( + "project.urls", "project.authors", "project.maintainers", + "project.readme") + + +READERS = { + "setup.py": setup_py, + "setup.cfg": setup_cfg, + "pyproject.toml": toml_lines, +} + + +def counts(repository, tag, path): + """Whether the change to the file since the tag is more than metadata.""" + reader = READERS.get(os.path.basename(path)) + before = bytes_at(repository, tag, path) + after = bytes_at(repository, "HEAD", path) + if reader is None or before is None or after is None: + return True + try: + return reader(before.decode("utf-8")) != reader(after.decode("utf-8")) + except (SyntaxError, ValueError, configparser.Error, + UnicodeDecodeError): + return True + + +def main(arguments): + repository, tag = arguments[0], arguments[1] + for path in arguments[2:]: + if counts(repository, tag, path): + print(path) + + +if __name__ == "__main__": + main(sys.argv[1:]) From e1bcf98abe395311d01ab8c9033662f13403a981 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 22:30:39 +0000 Subject: [PATCH 006/136] Measure changes from rc and build pull requests on fewer entries (#33) Reviewed and approved at head 2e8d370b289685606789a3c8ee66ae61f81003b7. The pull request body says what changed and how it was tested. --- .github/workflows/nightly-pipeline.yml | 52 +++++++------ .github/workflows/pull-requests.yml | 10 +++ .github/workflows/release-pull-request.yml | 32 ++++++++ AGENTS.md | 11 ++- ci/build-project.ps1 | 5 +- ci/dispatch.ps1 | 7 +- ci/fetch-assets.ps1 | 5 +- ci/packages.ps1 | 11 ++- ci/run-integration-tests.ps1 | 8 +- ci/run-performance-tests.ps1 | 5 +- ci/run-unit-tests.ps1 | 5 +- ci/select-options.Tests.ps1 | 69 ++++++++++++++++++ ci/select-options.ps1 | 73 +++++++++++++++++++ ci/setup-environment.ps1 | 5 +- device-detection/ci/run-integration-tests.ps1 | 13 +++- 15 files changed, 273 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/release-pull-request.yml create mode 100644 ci/select-options.Tests.ps1 create mode 100644 ci/select-options.ps1 diff --git a/.github/workflows/nightly-pipeline.yml b/.github/workflows/nightly-pipeline.yml index e44f9d29..7f434943 100644 --- a/.github/workflows/nightly-pipeline.yml +++ b/.github/workflows/nightly-pipeline.yml @@ -15,53 +15,59 @@ on: default: '' schedule: # The 51degrees-cxx nightly starts at 00:35 and ends by about 02:35, so - # its release reaches this submodule update the same night. - - cron: '0 3 * * *' + # its release reaches this submodule update the same night. The Java and + # Node nightlies start at 03:00 and want macOS runners too, so this one + # starts after them. + - cron: '0 5 * * *' jobs: - SubmoduleUpdate: - name: Nightly Submodule Update - uses: 51Degrees/common-ci/.github/workflows/nightly-submodule-update.yml@main + # The publish builds main, which the pull requests into rc do not change, so + # it runs first and has the runners before the pull request builds do. + Publish: + name: Nightly Publish + uses: 51Degrees/common-ci/.github/workflows/nightly-publish.yml@main with: repo-name: ${{ github.event.repository.name }} org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} + cache-assets: true secrets: token: ${{ secrets.ACCESS_TOKEN }} + DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} + DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} + CsvUrl: ${{ secrets.CSV_URL }} + IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} + IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} + TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} + PypiToken: ${{ secrets.PYPI_TOKEN }} + TestPypiToken: ${{ secrets.TEST_PYPI_TOKEN }} - PullRequests: - if: ${{ !cancelled() }} - needs: SubmoduleUpdate - name: Nightly Pull Requests - uses: 51Degrees/common-ci/.github/workflows/nightly-pull-requests.yml@main + SubmoduleUpdate: + name: Nightly Submodule Update + uses: 51Degrees/common-ci/.github/workflows/nightly-submodule-update.yml@main with: repo-name: ${{ github.event.repository.name }} org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} - skip-lite-tests: ${{ inputs.skip-lite-tests || false }} - cache-assets: true secrets: token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - CsvUrl: ${{ secrets.CSV_URL }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - Publish: + PullRequests: if: ${{ !cancelled() }} - needs: PullRequests - name: Nightly Publish - uses: 51Degrees/common-ci/.github/workflows/nightly-publish.yml@main + needs: [Publish, SubmoduleUpdate] + name: Nightly Pull Requests + uses: 51Degrees/common-ci/.github/workflows/nightly-pull-requests.yml@main with: repo-name: ${{ github.event.repository.name }} org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} + skip-lite-tests: ${{ inputs.skip-lite-tests || false }} cache-assets: true + # One configuration that fails does not cancel the others. + fail-fast: false secrets: token: ${{ secrets.ACCESS_TOKEN }} DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} @@ -70,5 +76,3 @@ jobs: IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - PypiToken: ${{ secrets.PYPI_TOKEN }} - TestPypiToken: ${{ secrets.TEST_PYPI_TOKEN }} diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index 0d600a54..10b95bd4 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -13,6 +13,10 @@ on: common-ci-ref: type: string default: '' + full-matrix: + description: 'Build every configuration, not only those the changes need' + type: boolean + default: false jobs: PullRequests: @@ -25,6 +29,12 @@ jobs: common-ci-ref: ${{ inputs.common-ci-ref || '' }} skip-lite-tests: ${{ inputs.skip-lite-tests || false }} cache-assets: true + # Each pull request is built on the configurations its changes need, + # as ci/select-options.ps1 chooses, and one that fails does not cancel + # the others. With nothing eligible, the nightly builds rc instead. + full-matrix: ${{ inputs.full-matrix || false }} + fail-fast: false + build-without-pull-requests: false secrets: token: ${{ secrets.ACCESS_TOKEN }} DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} diff --git a/.github/workflows/release-pull-request.yml b/.github/workflows/release-pull-request.yml new file mode 100644 index 00000000..8af10442 --- /dev/null +++ b/.github/workflows/release-pull-request.yml @@ -0,0 +1,32 @@ +name: Release Pull Request + +# Builds and tests the release pull request, rc into main, on every entry in +# ci/options.json, and never merges it. A pull request into rc is built only on +# the entries its changes need, so this is the full build of a release before +# it reaches main. See common-ci .github/workflows/release-pull-request.yml. +on: + pull_request: + branches: [main] + +# A push to rc while the release is open builds it again, in place of the +# build of the older rc. +concurrency: + group: release-pull-request-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + Release: + name: Release Pull Request + uses: 51Degrees/common-ci/.github/workflows/release-pull-request.yml@main + with: + repo-name: ${{ github.event.repository.name }} + org-name: ${{ github.event.repository.owner.login }} + cache-assets: true + secrets: + token: ${{ secrets.ACCESS_TOKEN }} + DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} + DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} + CsvUrl: ${{ secrets.CSV_URL }} + IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} + IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} + TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} diff --git a/AGENTS.md b/AGENTS.md index 3fe39ce4..c47453bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -275,7 +275,16 @@ product is built or tested belongs in that product's `ci/`, not in the root. A change touches a product when it is in that product's folder, or in a package another product depends on; a change outside every product folder touches all of them ([ci/packages.ps1](ci/packages.ps1), -`Resolve-AffectedProducts`). +`Resolve-AffectedProducts`). A pull request's changes are measured from the +branch it merges into, `rc`. + +[ci/select-options.ps1](ci/select-options.ps1) chooses the entries a pull +request runs. The standard set is Python 3.14 and 3.10 on Linux, with the +Selenium contract on 3.14 alone. A native change adds Windows, a packaging +change adds the other Python versions on Linux, and a change outside the +product folders or to a `ci/` folder adds every entry but macOS. macOS and +performance run only when `rc` itself or the release pull request is built, +which `.github/workflows/release-pull-request.yml` does on every entry. ## Versions are per package diff --git a/ci/build-project.ps1 b/ci/build-project.ps1 index 3461d091..639d7435 100644 --- a/ci/build-project.ps1 +++ b/ci/build-project.ps1 @@ -1,5 +1,8 @@ param ( - [Parameter(Mandatory)][string]$RepoName + [Parameter(Mandatory)][string]$RepoName, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main' ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/ci/dispatch.ps1 b/ci/dispatch.ps1 index 4249d40b..06ba77ef 100644 --- a/ci/dispatch.ps1 +++ b/ci/dispatch.ps1 @@ -10,12 +10,15 @@ # Runs $Step for each affected product, or for each of $Products when given. # Every product is run even when an earlier one fails, so one run reports -# every broken product, and the step fails at the end if any did. +# every broken product, and the step fails at the end if any did. A pull +# request's changes are measured from the Branch in $Options, the branch it +# merges into. function Invoke-ProductStep([string]$Step, [string]$RepoName, [Collections.IDictionary]$Options, [string[]]$Products) { $failed = [Collections.Generic.List[string]]::new() if (-not $Products) { - $Products = Get-AffectedProducts $RepoName + $branch = $Options.Branch ? $Options.Branch : 'main' + $Products = Get-AffectedProducts $RepoName $branch } foreach ($product in $Products) { $script = Join-Path $RepoName $product 'ci' "$Step.ps1" diff --git a/ci/fetch-assets.ps1 b/ci/fetch-assets.ps1 index 802476f1..7a37794a 100644 --- a/ci/fetch-assets.ps1 +++ b/ci/fetch-assets.ps1 @@ -8,7 +8,10 @@ param ( [string]$DeviceDetectionUrl, [string]$CsvUrl, [string]$IpIntelligence, - [string]$IpIntelligenceUrl + [string]$IpIntelligenceUrl, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main' ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/ci/packages.ps1 b/ci/packages.ps1 index 7529a333..ec4bb89b 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -251,15 +251,18 @@ function Resolve-AffectedProducts($Packages, [string[]]$Changed) { } # The products a pull request changes, or every product when the checkout is -# not a pull request (a publish, or a run on main), as 51degrees-cxx does. -function Get-AffectedProducts([string]$RepoPath) { - # merge-base fails where there is no origin/main, which is an answer here. +# not a pull request (a publish, or a build of the branch itself), as +# 51degrees-cxx does. A pull request is checked out with the branch it merges +# into merged in, so the merge base with that branch leaves only the pull +# request's own changes. +function Get-AffectedProducts([string]$RepoPath, [string]$Branch = 'main') { + # merge-base fails where there is no such branch, which is an answer here. # Set for this function only, so a caller that stops on a failing native # command, as publish-package.ps1 does, is not stopped by it. $PSNativeCommandUseErrorActionPreference = $false $packages = Get-Packages $RepoPath $head = git -C $RepoPath rev-parse HEAD - $base = git -C $RepoPath merge-base origin/main HEAD 2>$null + $base = git -C $RepoPath merge-base "origin/$Branch" HEAD 2>$null if ($LASTEXITCODE -ne 0 -or $null -eq $base -or $base -eq $head) { $global:LASTEXITCODE = 0 return @($packages.Product | Select-Object -Unique) diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index ae1c20f8..59885cae 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -1,6 +1,12 @@ param ( [Parameter(Mandatory)][string]$RepoName, - [Parameter(Mandatory)][hashtable]$Keys + [Parameter(Mandatory)][hashtable]$Keys, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main', + # False, from select-options.ps1, on every entry of a pull request but + # one, so the browser contract runs once. + [bool]$SeleniumContract = $true ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/ci/run-performance-tests.ps1 b/ci/run-performance-tests.ps1 index 58401f22..4fa56758 100644 --- a/ci/run-performance-tests.ps1 +++ b/ci/run-performance-tests.ps1 @@ -1,7 +1,10 @@ param ( [Parameter(Mandatory)][string]$RepoName, [string]$Name, - [bool]$RunPerformance + [bool]$RunPerformance, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main' ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/ci/run-unit-tests.ps1 b/ci/run-unit-tests.ps1 index e74a60ff..38e3ee6e 100644 --- a/ci/run-unit-tests.ps1 +++ b/ci/run-unit-tests.ps1 @@ -1,5 +1,8 @@ param ( - [Parameter(Mandatory)][string]$RepoName + [Parameter(Mandatory)][string]$RepoName, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main' ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/ci/select-options.Tests.ps1 b/ci/select-options.Tests.ps1 new file mode 100644 index 00000000..9e0042dc --- /dev/null +++ b/ci/select-options.Tests.ps1 @@ -0,0 +1,69 @@ +# Tests for ci/select-options.ps1, with the entries of the real options.json +# read as common-ci's steps/select-options.ps1 reads them, as hashtables. + +BeforeAll { + $script:Select = Join-Path $PSScriptRoot 'select-options.ps1' + + # The names of the entries chosen for the changed files given, and the + # entries themselves in $script:Chosen. + function Get-Chosen([string[]]$ChangedFiles, [string[]]$CommentOnlyFiles = @()) { + $options = @(Get-Content -Raw (Join-Path $PSScriptRoot 'options.json') | + ConvertFrom-Json -AsHashtable) + $script:Chosen = @(& $script:Select -Options $options ` + -ChangedFiles $ChangedFiles -CommentOnlyFiles $CommentOnlyFiles 6>$null) + return @($script:Chosen | ForEach-Object { $_.Name } | Sort-Object) + } + + $script:StandardSet = @('Ubuntu_Python_3.10', 'Ubuntu_Python_3.14') +} + +Describe 'select-options.ps1' { + It 'builds the standard set for a change to a package''s code' { + Get-Chosen @('pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py') | + Should -Be $script:StandardSet + } + + It 'runs the Selenium contract on Python 3.14 alone, and no performance tests' { + $null = Get-Chosen @('location/src/fiftyone_location/location_cloud.py') + foreach ($entry in $script:Chosen) { + $entry.SeleniumContract | Should -Be ($entry.Name -eq 'Ubuntu_Python_3.14') + $entry.RunPerformance | Should -BeFalse + } + } + + It 'adds Windows for a change to an on-premise package' { + Get-Chosen @('device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py') | + Should -Be (@($script:StandardSet + 'Windows_Python_3.14') | Sort-Object) + } + + It 'adds Windows for a change to the cxx submodule' { + Get-Chosen @('cxx') | + Should -Be (@($script:StandardSet + 'Windows_Python_3.14') | Sort-Object) + } + + It 'leaves Windows out for a change only to the comments of an on-premise file' { + $file = 'ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py' + Get-Chosen @($file) @($file) | Should -Be $script:StandardSet + } + + It 'adds Python 3.11 to 3.13 on Linux for a change to a setup.py' { + Get-Chosen @('location/setup.py') | Should -Be (@($script:StandardSet + + 'Ubuntu_Python_3.11' + 'Ubuntu_Python_3.12' + 'Ubuntu_Python_3.13') | + Sort-Object) + } + + It 'builds every entry but macOS for a change to ' -ForEach @( + @{ Case = 'a ci folder'; File = 'device-detection/ci/build-project.ps1' } + @{ Case = 'the root ci folder'; File = 'ci/packages.ps1' } + @{ Case = 'a file outside the product folders'; File = '.github/workflows/pull-requests.yml' } + ) { + $chosen = Get-Chosen @($File) + $chosen.Count | Should -Be 10 + $chosen | Where-Object { $_ -like 'macOS*' } | Should -BeNullOrEmpty + } + + It 'never builds macOS on a pull request' { + Get-Chosen @('cxx', 'ci/packages.ps1', 'location/setup.py') | + Where-Object { $_ -like 'macOS*' } | Should -BeNullOrEmpty + } +} diff --git a/ci/select-options.ps1 b/ci/select-options.ps1 new file mode 100644 index 00000000..cc22871f --- /dev/null +++ b/ci/select-options.ps1 @@ -0,0 +1,73 @@ +param( + # Every entry of options.json. + [Parameter(Mandatory)][object[]]$Options, + # The files the pull request changes, less documentation. + [string[]]$ChangedFiles, + # Those of the changed files whose change touches only comments. + [string[]]$CommentOnlyFiles +) +# Chooses the options.json entries a pull request is built on, for common-ci's +# steps/select-options.ps1. The branch itself and a release build every entry. +# Within each entry, packages.ps1 still builds only the products the change +# touches. +# +# The standard set is Python 3.14 and 3.10 on Linux, the newest and the oldest, +# with the Selenium contract on 3.14 alone. A change in more than its comments +# to an on-premise package, its data or the cxx submodule adds Windows, where +# the native module is built differently. A change to a setup.py, setup.cfg or +# pyproject.toml adds Python 3.11 to 3.13 on Linux, and a change outside the +# product folders or to a ci folder adds every entry. Only five macOS jobs run +# at once across the organisation, and the releases need them, so macOS is left +# to the branch and release builds, as is performance. + +$Standard = @('Ubuntu_Python_3.14', 'Ubuntu_Python_3.10') +$Contract = 'Ubuntu_Python_3.14' +$Native = @('Windows_Python_3.14') +$Versions = @('Ubuntu_Python_3.13', 'Ubuntu_Python_3.12', 'Ubuntu_Python_3.11') +$ProductFolders = @('pipeline/', 'device-detection/', 'ip-intelligence/', + 'location/') +# The data files come in through the cxx submodule, so a change to them is a +# change to cxx. +$NativePaths = @('device-detection/fiftyone_devicedetection_onpremise/', + 'ip-intelligence/fiftyone_ipintelligence_onpremise/') + +function Test-Under([string]$Path, [string[]]$Prefixes) { + return @($Prefixes | Where-Object { + $Path -eq $_.TrimEnd('/') -or $Path.StartsWith($_.TrimEnd('/') + '/') + }).Count -gt 0 +} + +# PowerShell names are not case sensitive, so these must not share a name with +# the lists above, such as $Native. +$code = @($ChangedFiles | Where-Object { $CommentOnlyFiles -notcontains $_ }) +$nativeChanged = @($code | Where-Object { + $_ -eq 'cxx' -or (Test-Under $_ $NativePaths) +}).Count -gt 0 +$setupChanged = @($code | Where-Object { + $_ -match '(^|/)(setup\.py|setup\.cfg|pyproject\.toml)$' +}).Count -gt 0 +$otherChanged = @($code | Where-Object { + $_ -ne 'cxx' -and (!(Test-Under $_ $ProductFolders) -or $_ -match '(^|/)ci/') +}).Count -gt 0 +Write-Host ("Native: $nativeChanged. Setup: $setupChanged. " + + "Outside the product folders or in a ci folder: $otherChanged.") + +foreach ($entry in $Options) { + $name = $entry.Name + if ($entry.Image -like 'macos*') { + Write-Host "$name is left to the branch and release builds" + continue + } + $why = if ($Standard -contains $name) { 'the standard set' } + elseif ($otherChanged) { 'a change outside the product folders or to CI' } + elseif ($nativeChanged -and $Native -contains $name) { 'a native change' } + elseif ($setupChanged -and $Versions -contains $name) { 'a packaging change' } + if (!$why) { + Write-Host "$name is not needed for this change" + continue + } + $entry.SeleniumContract = $name -eq $Contract + $entry.RunPerformance = $false + Write-Host "$name runs for $why" + $entry +} diff --git a/ci/setup-environment.ps1 b/ci/setup-environment.ps1 index 932f5d81..24c1b8bd 100644 --- a/ci/setup-environment.ps1 +++ b/ci/setup-environment.ps1 @@ -1,6 +1,9 @@ param ( [Parameter(Mandatory)][string]$RepoName, - [Parameter(Mandatory)][string]$LanguageVersion + [Parameter(Mandatory)][string]$LanguageVersion, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main' ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/device-detection/ci/run-integration-tests.ps1 b/device-detection/ci/run-integration-tests.ps1 index 9c45928e..ed7a17d2 100644 --- a/device-detection/ci/run-integration-tests.ps1 +++ b/device-detection/ci/run-integration-tests.ps1 @@ -2,7 +2,10 @@ param ( [Parameter(Mandatory=$true)] [string]$RepoName, [Parameter(Mandatory=$true)] - [Hashtable]$Keys + [Hashtable]$Keys, + # False, from the root select-options.ps1, on every entry of a pull + # request but one, so the browser contract runs once. + [bool]$SeleniumContract = $true ) if ($IsWindows) { @@ -47,6 +50,14 @@ if ($LASTEXITCODE -ne 0) { $failures += "the Python integration tests exited with code $LASTEXITCODE" } +if (-not $SeleniumContract) { + Write-Output "Another entry runs the Selenium contract tests for this pull request." + if ($failures.Count -gt 0) { + throw "Integration tests failed: $($failures -join '; ')" + } + return +} + # The shared Selenium contract tests are run against the cloud example and the # on-premise example in turn. Each result is also written as a trx file next # to the other integration results, so it is published with them. From 32ffdcf1205a5ce593fb55f5be733de5a18dffde Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 22:31:25 +0000 Subject: [PATCH 007/136] Point configuration page links at 51degrees.com/configure (#107) Reviewed and approved at head 60b403678245772f824bca80aa7d05b8d774e3d7. The pull request body says what changed and how it was tested. --- AGENTS.md | 4 ++-- README.md | 2 +- device-detection/fiftyone_devicedetection_cloud/readme.md | 2 +- .../fiftyone_devicedetection_cloud/tests/live_cloud.py | 2 +- .../cloud/configurator_console.py | 4 ++-- .../cloud/failuretomatch.py | 4 ++-- .../cloud/gettingstarted_console.json | 2 +- .../cloud/gettingstarted_console.py | 2 +- .../cloud/gettingstarted_web/app.py | 2 +- .../cloud/metadata_console.py | 2 +- .../cloud/nativemodellookup_console.py | 2 +- .../cloud/taclookup_console.json | 2 +- .../cloud/taclookup_console.py | 2 +- .../cloud/useragentclienthints_web/app.py | 2 +- .../src/fiftyone_devicedetection_examples/example_utils.py | 2 +- .../fiftyone_devicedetection_examples/tests/live_cloud.py | 2 +- device-detection/readme.md | 2 +- ip-intelligence/fiftyone_ipintelligence_cloud/readme.md | 2 +- ip-intelligence/fiftyone_ipintelligence_examples/readme.md | 2 +- .../cloud/getallproperties_console.py | 4 ++-- .../cloud/gettingstarted_console.py | 6 +++--- .../cloud/gettingstarted_web/app.py | 4 ++-- .../cloud/metadata_console.py | 4 ++-- .../fiftyone_ipintelligence_examples/cloud/mixed_console.py | 4 ++-- .../fiftyone_ipintelligence_examples/tests/live_cloud.py | 2 +- .../fiftyone_ipintelligence_onpremise/tests/live_cloud.py | 2 +- location/examples/cloud/gettingstarted.py | 6 +++--- location/examples/cloud/web.py | 6 +++--- location/readme.md | 2 +- location/tests/live_cloud.py | 2 +- .../fiftyone_pipeline_cloudrequestengine/examples/readme.md | 2 +- .../cloud/mixed/gettingstarted_console.py | 4 ++-- .../cloud/mixed/gettingstarted_web/app.py | 4 ++-- .../src/fiftyone_pipeline_examples/cloud/mixed/pipeline.py | 2 +- .../examples/tests/live_cloud.py | 2 +- pipeline/fiftyone_pipeline_cloudrequestengine/readme.md | 2 +- .../src/fiftyone_pipeline_cloudrequestengine/constants.py | 2 +- .../tests/live_cloud.py | 2 +- .../tests/test_cloudengine.py | 2 +- pipeline/fiftyone_pipeline_did/tests/live_cloud.py | 2 +- 40 files changed, 55 insertions(+), 55 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c47453bf..01221404 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -202,10 +202,10 @@ describes. The cloud examples need a resource key, which a person creates in the configurator, so ask for one if none is set. A key with the properties the Device Detection cloud example uses can be made -[here](https://configure.51degrees.com/1QWJwHxl?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=cloud-examples), +[here](https://51degrees.com/configure/1QWJwHxl?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=cloud-examples), which is the configuration the example's own message names, and a key for IP Intelligence in the -[configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=cloud-examples). +[configuration page](https://51degrees.com/configure/?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=cloud-examples). The Device Detection and IP Intelligence cloud examples take the key as their first argument, or from `_51DEGREES_RESOURCE_KEY`. diff --git a/README.md b/README.md index 04fb4d05..0a8ba812 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ python -m pip install fiftyone-ipintelligence The table below lists the other packages. To build from source, see [Getting the code](#getting-the-code). -Try the [User-Agent tester](https://51degrees.com/developers/user-agent-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=install) and [IP tester](https://51degrees.com/developers/ip-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=install). For cloud use, [create a Resource Key](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=install). +Try the [User-Agent tester](https://51degrees.com/developers/user-agent-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=install) and [IP tester](https://51degrees.com/developers/ip-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=install). For cloud use, [create a Resource Key](https://51degrees.com/configure/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=install). Created from `pipeline-python`, `device-detection-python`, and `location-python`, this repository continues their work. diff --git a/device-detection/fiftyone_devicedetection_cloud/readme.md b/device-detection/fiftyone_devicedetection_cloud/readme.md index 07923fdc..c41fda5c 100644 --- a/device-detection/fiftyone_devicedetection_cloud/readme.md +++ b/device-detection/fiftyone_devicedetection_cloud/readme.md @@ -12,7 +12,7 @@ You can confirm this is working with the following micro-example. -* Create a resource key for free with the 51Degrees [configurator](https://configure.51degrees.com/np5M4nlF?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=from-pypi). This defines the properties you want to access. +* Create a resource key for free with the 51Degrees [configuration page](https://51degrees.com/configure/np5M4nlF?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=from-pypi). This defines the properties you want to access. * On the 'implement' page of the configurator, copy the resource key and replace YOUR_RESOURCE_KEY in the example below. Save this as exampledd.py * Run the example with `python exampledd.py` * Feel free to try different user-agents and property values. diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py index 36bc26ad..2109e4a2 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py @@ -51,7 +51,7 @@ ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") -CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" +CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" def _report_old_name(name, old_name): diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py index 9e6d0b5f..6cd8ac4c 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py @@ -25,7 +25,7 @@ from fiftyone_devicedetection_cloud.devicedetection_cloud_pipelinebuilder import DeviceDetectionCloudPipelineBuilder from fiftyone_devicedetection_examples.example_utils import ExampleUtils -# This example is displayed at the end of the [Configurator](https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=top) +# This example is displayed at the end of the [configuration page](https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=top) # process, which is used to create resource keys for use with the 51Degrees cloud service. # # It shows how to call the cloud with the newly created key and how to access the values @@ -99,7 +99,7 @@ def main(argv): "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + - "https://configure.51degrees.com/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=resource-key-required. " + + "https://51degrees.com/configure/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=resource-key-required. " + "Once complete, populate the environment variable " + "mentioned at the start of this message with the key.") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/failuretomatch.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/failuretomatch.py index 8258122c..c10939eb 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/failuretomatch.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/failuretomatch.py @@ -24,7 +24,7 @@ # First create the device detection pipeline with the desired settings. -# You need to create a resource key at https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-failuretomatch.py&utm_term=top +# You need to create a resource key at https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-failuretomatch.py&utm_term=top # and paste it into the code, replacing !!YOUR_RESOURCE_KEY!! below. # Alternatively, set the _51DEGREES_RESOURCE_KEY environment variable. The # key is read through ExampleUtils, as every other example reads it, so @@ -40,7 +40,7 @@ # The address stays on one line, because the campaign lint reads a # line at a time and a split address looks to it like a missing tag. print(" To include the properties used in this example, go to " - "https://configure.51degrees.com/bxXqZhLT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-failuretomatch.py&utm_term=resource-key-required") + "https://51degrees.com/configure/bxXqZhLT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-failuretomatch.py&utm_term=resource-key-required") else: pipeline = DeviceDetectionCloudPipelineBuilder({ diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.json b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.json index 214a588a..2caf8d14 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.json +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.json @@ -7,7 +7,7 @@ "elementParameters": { "settings": { // Obtain a resource key with the properties required to run this - // example for free: https://configure.51degrees.com/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.json&utm_term=settings + // example for free: https://51degrees.com/configure/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.json&utm_term=settings "resource_key": "!!ENTER_YOUR_RESOURCE_KEY_HERE!!", "cloud_request_origin": "51Degrees.example.com" } diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py index 3a92915b..f763cfcf 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py @@ -176,7 +176,7 @@ def main(argv): "see " + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + "A resource key with the properties required by this example can be " + - "created for free at https://configure.51degrees.com/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + + "created for free at https://51degrees.com/configure/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + "Once complete, populate the config file or environment variable " + "mentioned at the start of this message with the key.") else: diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py index 93f448b3..94a2855d 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py @@ -204,7 +204,7 @@ def main(argv): "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + - "https://configure.51degrees.com/g3gMZdPY?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required. " + + "https://51degrees.com/configure/g3gMZdPY?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required. " + "Once complete, populated the environment variable " + "mentioned at the start of this message with the key.") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py index d8b06656..7fc8ba08 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py @@ -123,7 +123,7 @@ def main(argv): "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + - "https://configure.51degrees.com/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=resource-key-required. " + + "https://51degrees.com/configure/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=resource-key-required. " + "Once complete, populate the environment variable " + "mentioned at the start of this message with the key.") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py index 98b7212e..868d0665 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py @@ -133,7 +133,7 @@ def main(argv): "you will first need a license key, which can be purchased from our " + "pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=resource-key-required. Once this is done, a resource " + "key with the properties required by this example can be created at " + - "https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=resource-key-required. You can now populate the " + + "https://51degrees.com/configure/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=resource-key-required. You can now populate the " + "environment variable mentioned at the start of this message with the " + "resource key or pass it as the first argument on the command line.") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.json b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.json index 2bb00c22..9a393c31 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.json +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.json @@ -8,7 +8,7 @@ "settings": { // TAC lookup requires a license key, which can be purchased from our pricing page: // https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.json&utm_term=settings. You can then obtain a resource key with the properties - // required to run this example here: https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.json&utm_term=settings + // required to run this example here: https://51degrees.com/configure/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.json&utm_term=settings "resource_key": "!!ENTER_YOUR_RESOURCE_KEY_HERE!!", // Update this with your domain name. "cloud_request_origin": "51Degrees.example.com" diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py index 63355916..0d07605e 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py @@ -134,7 +134,7 @@ def main(argv): "that you will first need a license key, which can be purchased " + "from our pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=resource-key-required. Once this is " + "done, a resource key with the properties required by this example " + - "can be created at https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=resource-key-required. You " + + "can be created at https://51degrees.com/configure/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=resource-key-required. You " + "can now populate the environment variable mentioned at the start " + "of this message with the resource key or pass it as the first " + "argument on the command line.") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py index 728f0594..b47cce81 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py @@ -139,7 +139,7 @@ def main(argv): "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + - "https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=resource-key-required. " + + "https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=resource-key-required. " + "Once complete, populated the environment variable " + "mentioned at the start of this message with the key.") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py index 98f38b8d..b6a96ff9 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py @@ -213,7 +213,7 @@ def get_missing_resource_key_message(): return (f"No resource key found. Set the environment variable " f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. Create a resource key for free at " - "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-example_utils.py&utm_term=resource-key-required") + "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-example_utils.py&utm_term=resource-key-required") @staticmethod def get_data_file_path(): diff --git a/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py b/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py index 36bc26ad..2109e4a2 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py @@ -51,7 +51,7 @@ ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") -CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" +CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" def _report_old_name(name, old_name): diff --git a/device-detection/readme.md b/device-detection/readme.md index 62f44556..0559335d 100644 --- a/device-detection/readme.md +++ b/device-detection/readme.md @@ -36,7 +36,7 @@ The API can either use our cloud service to get its data or it can use a local ( You will require a [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=cloud) to use the Cloud API. You can create resource keys using our -[configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=cloud), see our +[configuration page](https://51degrees.com/configure/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=cloud), see our [documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=cloud) on how to use this. #### On-Premise diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md b/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md index fc189448..760a2e04 100644 --- a/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md @@ -11,7 +11,7 @@ location, network and registration information using the [51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=introduction). To use the cloud service you will need a resource key. You can obtain one -from the [51Degrees Configurator](https://configure.51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=introduction). +from the [51Degrees configuration page](https://51degrees.com/configure/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=introduction). ## Usage diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md index 127fc3af..f812b3e8 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md @@ -11,7 +11,7 @@ Usage examples for the 51Degrees IP intelligence engines. ## Cloud examples Require a resource key in the `_51DEGREES_RESOURCE_KEY` environment variable. -Obtain one from the [51Degrees Configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=cloud-examples). +Obtain one from the [51Degrees configuration page](https://51degrees.com/configure/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=cloud-examples). - `cloud/gettingstarted_console.py` - resolve IP addresses using the cloud service from the console. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py index fc950959..e082fc68 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py @@ -28,7 +28,7 @@ # how to enumerate the values returned. # # A resource key is required. You can obtain one from -# https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-getallproperties_console.py&utm_term=top - make sure to include all of the +# https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-getallproperties_console.py&utm_term=top - make sure to include all of the # properties that you want to see displayed by this example. # # Required PyPi Dependencies: @@ -107,7 +107,7 @@ def main(argv): "see " "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-getallproperties_console.py&utm_term=resource-key-required. " "A resource key with the properties required by this example " - "can be created for free at https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-getallproperties_console.py&utm_term=resource-key-required.") + "can be created for free at https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-getallproperties_console.py&utm_term=resource-key-required.") if __name__ == "__main__": main(sys.argv[1:]) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py index 48bfe83b..f9faf25f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py @@ -32,7 +32,7 @@ # 3. How to retrieve the results # # A resource key is required. You can obtain one from -# https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=top +# https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=top # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -83,7 +83,7 @@ def run(self, resource_key, logger, output, cloud_endpoint = None): # general see the documentation at # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=run settings = { - # Obtain a resource key from https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=run + # Obtain a resource key from https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=run "resource_key": resource_key } if cloud_endpoint: @@ -174,7 +174,7 @@ def main(argv): "information see " "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " "A resource key with the properties required by this example " - "can be created for free at https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + "can be created for free at https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " "Once complete, populate the environment variable mentioned at " "the start of this message with the key.") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py index 59183683..90270ef4 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py @@ -28,7 +28,7 @@ # the 51Degrees cloud service. # # A resource key must be supplied in the `_51DEGREES_RESOURCE_KEY` environment -# variable. Obtain one from https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_web-app.py&utm_term=top +# variable. Obtain one from https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_web-app.py&utm_term=top # # Run with: # ``` @@ -92,7 +92,7 @@ def main(resource_key = None): raise Exception( "No resource key specified in the environment variable " f"'{ExampleUtils.CLOUD_RESOURCE_KEY_ENV_VAR}'. Obtain one from " - "https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required") + "https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required") build_pipeline(resource_key) app.run(debug=False) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py index 3d6d8f00..cef146be 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py @@ -32,7 +32,7 @@ # resource key, rather than all properties the cloud service can return. # # A resource key is required. You can obtain one from -# https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-metadata_console.py&utm_term=top +# https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-metadata_console.py&utm_term=top # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -106,7 +106,7 @@ def main(argv): "see " "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-metadata_console.py&utm_term=resource-key-required. " "A resource key with the properties required by this example " - "can be created for free at https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-metadata_console.py&utm_term=resource-key-required.") + "can be created for free at https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-metadata_console.py&utm_term=resource-key-required.") if __name__ == "__main__": main(sys.argv[1:]) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/mixed_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/mixed_console.py index 2c228035..b9a0b2c3 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/mixed_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/mixed_console.py @@ -30,7 +30,7 @@ # # A resource key configured with both device detection and IP intelligence # properties is required. You can obtain one from -# https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-mixed_console.py&utm_term=top +# https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-mixed_console.py&utm_term=top # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -172,7 +172,7 @@ def main(argv): f"'{ExampleUtils.CLOUD_RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " "service is accessed using a 'ResourceKey'. A resource key " "configured with both device detection and IP intelligence " - "properties can be created at https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-mixed_console.py&utm_term=resource-key-required.") + "properties can be created at https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-mixed_console.py&utm_term=resource-key-required.") if __name__ == "__main__": main(sys.argv[1:]) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py index 36bc26ad..2109e4a2 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py @@ -51,7 +51,7 @@ ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") -CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" +CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" def _report_old_name(name, old_name): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/live_cloud.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/live_cloud.py index 36bc26ad..2109e4a2 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/live_cloud.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/live_cloud.py @@ -51,7 +51,7 @@ ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") -CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" +CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" def _report_old_name(name, old_name): diff --git a/location/examples/cloud/gettingstarted.py b/location/examples/cloud/gettingstarted.py index 143cc373..adaacef0 100644 --- a/location/examples/cloud/gettingstarted.py +++ b/location/examples/cloud/gettingstarted.py @@ -36,7 +36,7 @@ # First create the device detection pipeline with the desired settings. -# You need to create a resource key at https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-gettingstarted.py&utm_term=top +# You need to create a resource key at https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-gettingstarted.py&utm_term=top # and paste it into the code, replacing !!YOUR_RESOURCE_KEY!! below. # Alternatively, add a _51DEGREES_RESOURCE_KEY environment variable. # resource_key is the old name of that variable. It is deprecated, and is @@ -70,9 +70,9 @@ def read_environment(name, old_name): if resource_key == "!!YOUR_RESOURCE_KEY!!": print(""" You need to create a resource key at - https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-gettingstarted.py&utm_term=resource-key-required and paste it into the code, + https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-gettingstarted.py&utm_term=resource-key-required and paste it into the code, 'replacing !!YOUR_RESOURCE_KEY!! - To get a resourcekey with the properties used in this example go to https://configure.51degrees.com/GCrtGh1L?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-gettingstarted.py&utm_term=resource-key-required + To get a resourcekey with the properties used in this example go to https://51degrees.com/configure/GCrtGh1L?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-gettingstarted.py&utm_term=resource-key-required """) else: diff --git a/location/examples/cloud/web.py b/location/examples/cloud/web.py index f893824f..c2b7d13a 100644 --- a/location/examples/cloud/web.py +++ b/location/examples/cloud/web.py @@ -36,7 +36,7 @@ # First create the location pipeline with the desired settings. -# You need to create a resource key at https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-web.py&utm_term=top +# You need to create a resource key at https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-web.py&utm_term=top # and paste it into the code, replacing !!YOUR_RESOURCE_KEY!! below. # Alternatively, add a _51DEGREES_RESOURCE_KEY environment variable. # resource_key is the old name of that variable. It is deprecated, and is @@ -70,9 +70,9 @@ def read_environment(name, old_name): if resource_key == "!!YOUR_RESOURCE_KEY!!": print(""" You need to create a resource key at - https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-web.py&utm_term=resource-key-required and paste it into the code, + https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-web.py&utm_term=resource-key-required and paste it into the code, 'replacing !!YOUR_RESOURCE_KEY!! - To get a resourcekey with the properties used in this example go to https://configure.51degrees.com/GCrtGh1L?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-web.py&utm_term=resource-key-required + To get a resourcekey with the properties used in this example go to https://51degrees.com/configure/GCrtGh1L?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-web.py&utm_term=resource-key-required """) else: diff --git a/location/readme.md b/location/readme.md index dd21ab30..931eaf16 100644 --- a/location/readme.md +++ b/location/readme.md @@ -23,7 +23,7 @@ The [tested versions](https://51degrees.com/documentation/_product_summaries__te You can confirm this is working with the following micro-example. -* Create a resource key for free with the 51Degrees [configurator](https://configure.51degrees.com/6CTsmbPx?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=from-pypi). This defines the properties you want to access. +* Create a resource key for free with the 51Degrees [configuration page](https://51degrees.com/configure/6CTsmbPx?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=from-pypi). This defines the properties you want to access. * On the 'implement' page of the configurator, copy the resource key and replace YOUR_RESOURCE_KEY in the example below. Save this as examplelocation.py * Run the example with `python examplelocation.py` * Feel free to try different locations and property values. diff --git a/location/tests/live_cloud.py b/location/tests/live_cloud.py index 36bc26ad..2109e4a2 100644 --- a/location/tests/live_cloud.py +++ b/location/tests/live_cloud.py @@ -51,7 +51,7 @@ ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") -CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" +CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" def _report_old_name(name, old_name): diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md index c3457cd2..65d762ef 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md @@ -31,7 +31,7 @@ The examples call the cloud service with a resource key or a licence key. A resource key carries the list of properties it returns. Create one carrying device detection and IP intelligence properties for free with the -[configurator](https://configure.51degrees.com?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-readme.md&utm_term=resource-key), +[configuration page](https://51degrees.com/configure?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-readme.md&utm_term=resource-key), and set the `_51DEGREES_RESOURCE_KEY` environment variable to it. A resource key limited to particular domains only answers a request whose origin is one of them, so set diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py index 42ef1f62..2f4fcf70 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py @@ -39,7 +39,7 @@ # # A resource key carries the list of properties it returns. Create one # carrying device detection and IP intelligence properties for free at -# https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_console.py&utm_term=header +# https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_console.py&utm_term=header # and set the _51DEGREES_RESOURCE_KEY environment variable to it, or pass it # as the first argument. # @@ -296,7 +296,7 @@ def main(argv): "A resource key carrying the device detection and IP " "intelligence properties this example uses can be created for " "free at " - "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_console.py&utm_term=resource-key-required. " + "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_console.py&utm_term=resource-key-required. " "Once complete, set the first environment variable named at " "the start of this message to the key.") return diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py index f86331cf..445429d8 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py @@ -37,7 +37,7 @@ # # A resource key carries the list of properties it returns. Create one # carrying device detection and IP intelligence properties for free at -# https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_web-app.py&utm_term=header +# https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_web-app.py&utm_term=header # and set the _51DEGREES_RESOURCE_KEY environment variable to it, or pass it # as the first argument. # @@ -434,7 +434,7 @@ def main(argv): "A resource key carrying the device detection and IP " "intelligence properties this example uses can be created for " "free at " - "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_web-app.py&utm_term=resource-key-required. " + "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_web-app.py&utm_term=resource-key-required. " "Once complete, set the first environment variable named at " "the start of this message to the key.") sys.exit(1) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/pipeline.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/pipeline.py index 41895b64..7ce672ee 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/pipeline.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/pipeline.py @@ -114,7 +114,7 @@ def can_fill(engine): "The key carries neither the device detection nor the IP " "intelligence properties this example shows. Create a resource " "key that carries them for free at " - "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-pipeline.py&utm_term=key-carries-nothing") + "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-pipeline.py&utm_term=key-carries-nothing") builder = PipelineBuilder(pipeline_settings or {}).add(request_engine) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py index 36bc26ad..2109e4a2 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py @@ -51,7 +51,7 @@ ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") -CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" +CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" def _report_old_name(name, old_name): diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md index e123445a..88b3e3cc 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md @@ -16,7 +16,7 @@ The Pipeline is a generic web request intelligence and data processing solution This package uses the `engines` class created by the `fiftyone-pipeline-engines`. It makes available: -* A `Cloud Request Engine` which calls the 51Degrees cloud service to fetch properties and metadata about them based on a provided resource key. Get a resource key at https://configure.51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=this-package-fiftyone_pipeline_cloudrequestengine +* A `Cloud Request Engine` which calls the 51Degrees cloud service to fetch properties and metadata about them based on a provided resource key. Get a resource key at https://51degrees.com/configure/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=this-package-fiftyone_pipeline_cloudrequestengine * A `Cloud Engine` template which reads data from the Cloud Request Engine. ## Credentials diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py index fa7c879c..1ff994b8 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py @@ -122,7 +122,7 @@ class Constants: "CloudRequestEngine needs a resource key or a licence key to " "authenticate with, as the cloud service answers 401 to a request " "that carries neither. Create a resource key for free at " - "https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-constants.py&utm_term=no-credential") + "https://51degrees.com/configure?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-constants.py&utm_term=no-credential") MESSAGE_LICENSE_KEY_NEEDS_PROPERTIES = ( "CloudRequestEngine has a licence key and no requested_properties. " "A licence key names no properties of its own, so the cloud service " diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/live_cloud.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/live_cloud.py index 36bc26ad..2109e4a2 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/live_cloud.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/live_cloud.py @@ -51,7 +51,7 @@ ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") -CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" +CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" def _report_old_name(name, old_name): diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py index b88ed79d..2e34a95f 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py @@ -41,7 +41,7 @@ def setUp(self): self.assertIsNotNone( self.resource_key, "You need to create a resource key at" - " https://configure.51degrees.com and paste it" + " https://51degrees.com/configure and paste it" " into the code, replacing !!YOUR_RESOURCE_KEY!!." " Please make sure to include IsMobile property." ) diff --git a/pipeline/fiftyone_pipeline_did/tests/live_cloud.py b/pipeline/fiftyone_pipeline_did/tests/live_cloud.py index 36bc26ad..2109e4a2 100644 --- a/pipeline/fiftyone_pipeline_did/tests/live_cloud.py +++ b/pipeline/fiftyone_pipeline_did/tests/live_cloud.py @@ -51,7 +51,7 @@ ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") -CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" +CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" def _report_old_name(name, old_name): From 37fca5db5d760f500c8554585e36e6051b981a4c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 23:05:47 +0000 Subject: [PATCH 008/136] Let Publish be run with common-ci's early release option (#118) --- .github/workflows/publish.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 195969ae..c02871b1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,6 +9,12 @@ on: common-ci-ref: type: string default: '' + # common-ci refuses a release less than a week after the last unless + # this is set, which is only for a release agreed to go out early. + allow-early-release: + description: 'Publish even though a package was released less than a week ago' + type: boolean + default: false jobs: Publish: @@ -19,6 +25,7 @@ jobs: org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} + allow-early-release: ${{ inputs.allow-early-release || false }} cache-assets: true secrets: token: ${{ secrets.ACCESS_TOKEN }} From bc34f817984615d6b7b396ab20a4488c2d18db1c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 23:27:13 +0000 Subject: [PATCH 009/136] Use common-ci's release plan in place of this repository's copy (#47) Step 1 of [common-ci#306](https://github.com/51Degrees/common-ci/issues/306). The release plan in `ci/packages.ps1`, which works out which packages a publish releases and at which version, moved to common-ci as `steps/release-plan.ps1` in [common-ci#307](https://github.com/51Degrees/common-ci/pull/307), and the rule that documentation, tests and examples do not count was added there by [common-ci#310](https://github.com/51Degrees/common-ci/pull/310). Both reached common-ci's `main`, which the workflows here use, in common-ci's release of 12 November. This repository then uses the shared step in place of its own copy, so that the rule is written once for every package repository. --- AGENTS.md | 7 +- ci/dispatch.Tests.ps1 | 12 +- ci/packages.Tests.ps1 | 206 +---------------------------- ci/packages.ps1 | 299 ++++-------------------------------------- 4 files changed, 40 insertions(+), 484 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 01221404..505c110a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -298,9 +298,10 @@ last tag, or when a package it depends on is released. Documentation description, keywords, urls, classifiers, readme, authors or maintainers does not count either, which `ci/release-plan-filter.ps1` works out with Python. The examples packages are tested but never published. common-ci's publish -carries one version, which here is only a label, and the release plan in -`ci/packages.ps1` decides what is built, published and tagged. Never push a -plain `4.5.x` tag, because nothing here reads one. +carries one version, which here is only a label, and the release plan +decides what is built, published and tagged. The release plan is common-ci's +`steps/release-plan.ps1`, which `ci/packages.ps1` loads, and common-ci tests +it. Never push a plain `4.5.x` tag, because nothing here reads one. `ci/packages.json` is the list of packages and what each depends on. Add a package there, with a `FirstVersion` if it has never been released, or it is diff --git a/ci/dispatch.Tests.ps1 b/ci/dispatch.Tests.ps1 index 3d75e0a2..352e667b 100644 --- a/ci/dispatch.Tests.ps1 +++ b/ci/dispatch.Tests.ps1 @@ -2,8 +2,8 @@ # # Each test builds a throwaway repository with two products, 'alpha' and # 'beta', whose ci/ scripts are written by the test, and runs the dispatcher -# from a folder laid out like a common-ci checkout, with common-ci's real -# steps/run-script.ps1, because that is how common-ci calls it. +# from a folder laid out like a common-ci checkout, with a copy of common-ci's +# real steps folder, because that is how common-ci calls it. BeforeAll { . (Join-Path $PSScriptRoot 'dispatch.ps1') @@ -17,6 +17,10 @@ BeforeAll { if ((Test-Path $script:RunScript) -eq $false) { throw "common-ci not found at '$commonCi'; set COMMON_CI to a checkout of it" } + # The whole steps folder is copied, because ci/packages.ps1 loads the + # release plan from the common-ci checkout that holds this repository's + # folder, and the release plan may load other steps beside it. + $script:Steps = Join-Path $commonCi 'steps' # A repository with packages.json naming one package in each product. function New-ProductRepo { @@ -45,8 +49,8 @@ BeforeAll { # A folder laid out like a common-ci checkout, which the dispatcher runs in. function New-CommonCi { $path = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) - $steps = New-Item -ItemType Directory -Force (Join-Path $path 'steps') - Copy-Item $script:RunScript $steps + $null = New-Item -ItemType Directory -Force $path + Copy-Item -Recurse $script:Steps (Join-Path $path 'steps') return $path } } diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index 243d87a6..63066049 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -1,9 +1,11 @@ -# Tests for ci/packages.ps1. +# Tests for ci/packages.ps1 and ci/release-plan-filter.ps1. # # The selection rules are tested against a small package list written here, so -# a test states exactly which graph it relies on. The release plan reads tags -# and diffs, so those tests build a throwaway git repository. One test reads the -# real packages.json, to catch a typo in a dependency name or a cycle. +# a test states exactly which graph it relies on. The release plan, which +# packages.ps1 loads from common-ci, is tested there. The tests of this +# repository's filter and of the build plan read tags and diffs, so they build +# a throwaway git repository. Some tests read the real packages.json, to catch +# a typo in a dependency name or a cycle, and to check its Ignore globs. BeforeAll { . (Join-Path $PSScriptRoot 'packages.ps1') @@ -75,40 +77,6 @@ AfterEach { $script:Repo = $null } -Describe 'Get-DependentClosure' { - It 'adds every package that depends on a changed one, through others' { - $closure = Get-DependentClosure $script:Graph @('core') - $closure | Sort-Object | Should -Be @( - 'core', 'dd', 'engines', 'examples', 'ipi', 'location') - } - - It 'adds nothing for a package nothing depends on' { - Get-DependentClosure $script:Graph @('did') | Should -Be @('did') - } - - It 'is empty when given nothing' { - @(Get-DependentClosure $script:Graph @()).Count | Should -Be 0 - } -} - -Describe 'Sort-ByDependency' { - It 'puts every package after the packages it depends on' { - $order = Sort-ByDependency $script:Graph @('dd', 'engines', 'core') - $order | Should -Be @('core', 'engines', 'dd') - } - - It 'ignores dependencies that are not being sorted' { - Sort-ByDependency $script:Graph @('dd') | Should -Be @('dd') - } - - It 'refuses a cycle rather than looping' { - $cycle = @( - New-Package 'a' 'p' 'p/a' @('b') - New-Package 'b' 'p' 'p/b' @('a')) - { Sort-ByDependency $cycle @('a', 'b') } | Should -Throw '*cycle*' - } -} - Describe 'Resolve-AffectedProducts' { It 'tests only the product of a package nothing depends on' { Resolve-AffectedProducts $script:Graph @('device/dd/a.py') | @@ -147,168 +115,6 @@ Describe 'Resolve-AffectedProducts' { } } -Describe 'Get-ReleasePlan' { - It 'releases nothing but first releases when nothing changed since the tags' { - $script:Repo = New-Repo $script:AllFiles $script:AllTags - Add-Commit $script:Repo @() - $plan = @(Get-ReleasePlan $script:Repo $script:Graph) - $plan.Name | Should -Be @('ipi') - $plan[0].Version | Should -Be '4.5.0' - $plan[0].Reason | Should -Be 'first release' - } - - It 'releases a changed package at the next patch, and its dependents' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('pipeline/engines/a.py') - $plan = @(Get-ReleasePlan $script:Repo $script:Graph) - $plan.Name | Should -Be @('engines', 'dd', 'ipi') - ($plan | Where-Object Name -eq 'engines').Version | Should -Be '4.5.39' - ($plan | Where-Object Name -eq 'dd').Version | Should -Be '4.5.97' - ($plan | Where-Object Name -eq 'dd').Reason | - Should -Be 'a package it depends on is released' - ($plan | Where-Object Name -eq 'ipi').Version | Should -Be '4.5.1' - } - - It 'releases a vendoring package when the vendored folder changes' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('pipeline/owid/a.py') - @(Get-ReleasePlan $script:Repo $script:Graph).Name | Should -Be @('did') - } - - It 'releases nothing for a change to an unpublished package' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('device/examples/a.py') - @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 - } - - It 'releases nothing for a change outside every package' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('ci/build.ps1', 'device/docs/a.md') - @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 - } - - It 'releases nothing a second time from a commit already released' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('device/dd/a.py') - git -C $script:Repo tag 'dd/4.5.97' - @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 - } - - It 'continues from the highest tag, not the latest one made' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + @( - 'ipi/4.5.0', 'dd/4.5.9')) - Add-Commit $script:Repo @('device/dd/a.py') - (@(Get-ReleasePlan $script:Repo $script:Graph) | - Where-Object Name -eq 'dd').Version | Should -Be '4.5.97' - } - - It 'does not read another package''s tags as its own' { - # 'ipi' must not see tags of a package whose name starts with 'ipi'. - $graph = $script:Graph + (New-Package 'ipi_cloud' 'ipi' 'ipi/cloud' -FirstVersion '4.5.0') - $script:Repo = New-Repo ($script:AllFiles + 'ipi/cloud/a.py') ( - $script:AllTags + 'ipi_cloud/4.5.7') - $plan = @(Get-ReleasePlan $script:Repo $graph) - ($plan | Where-Object Name -eq 'ipi').Version | Should -Be '4.5.0' - } - - It 'plans a changed package for a caller that stops on native errors' { - # publish-package.ps1 stops on a failing native command; 'git diff - # --quiet' exiting 1 for a changed package once failed the publish. - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('device/dd/a.py') - $plan = & { - $ErrorActionPreference = 'Stop' - $PSNativeCommandUseErrorActionPreference = $true - @(Get-ReleasePlan $script:Repo $script:Graph) - } - $plan.Name | Should -Be @('dd') - } - - It 'reads the tags whatever folder it runs from' { - # PowerShell expands a wildcard in a native command's argument into - # matching files, so a tag pattern run from a folder that holds a - # folder of the same name, as the repository root holds 'location', - # once found no tags. - $script:Repo = New-Repo $script:AllFiles $script:AllTags - $here = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) - $null = New-Item -ItemType Directory -Force (Join-Path $here 'location') - $null = New-Item -ItemType File (Join-Path $here 'location' 'setup.py') - Push-Location $here - try { - (Get-ReleasedVersions $script:Repo 'location')[-1] | Should -Be ([version]'4.5.4') - } finally { - Pop-Location - Remove-Item -Recurse -Force $here - } - } - - It 'refuses a never-released package with no FirstVersion' { - $graph = @(New-Package 'new' 'p' 'p/new') - $script:Repo = New-Repo @('p/new/a.py') @() - { Get-ReleasePlan $script:Repo $graph } | Should -Throw '*FirstVersion*' - } - - It 'says how many files counted' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('device/dd/a.py', 'device/dd/b.py') - (@(Get-ReleasePlan $script:Repo $script:Graph) | - Where-Object Name -eq 'dd').Reason | - Should -Be 'changed since 4.5.96 (2 files)' - } - - It 'releases nothing for a change only to documentation, tests or examples' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @( - 'device/dd/readme.md', 'device/dd/docs/guide.txt', - 'device/dd/tests/test_a.py', 'device/dd/test/a.py', - 'device/dd/examples/a.py', 'device/dd/LICENSE', - 'pipeline/core/sub/notes.md') - @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 - } - - It 'ignores documentation, tests and examples whatever their case' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @( - 'device/dd/README.MD', 'device/dd/Tests/a.py', - 'device/dd/Examples/a.py') - @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 - } - - It 'releases for the code changed beside the documentation' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('device/dd/readme.md', 'device/dd/a.py') - $plan = @(Get-ReleasePlan $script:Repo $script:Graph) - $plan.Name | Should -Be @('dd') - $plan[0].Reason | Should -Be 'changed since 4.5.96 (1 file)' - } - - It 'leaves out the files a package''s Ignore globs match' { - $graph = @($script:Graph | ForEach-Object { $_.PSObject.Copy() }) - ($graph | Where-Object Name -eq 'dd') | - Add-Member Ignore @('device/dd/tox.ini') - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('device/dd/tox.ini') - @(Get-ReleasePlan $script:Repo $graph).Count | Should -Be 0 - } - - It 'counts documentation for a package that turns the defaults off' { - $graph = @($script:Graph | ForEach-Object { $_.PSObject.Copy() }) - ($graph | Where-Object Name -eq 'dd') | Add-Member IgnoreDefaults $false - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('device/dd/readme.md') - @(Get-ReleasePlan $script:Repo $graph).Name | Should -Be @('dd') - } - - It 'refuses IgnoreDefaults given as text' { - $graph = @($script:Graph | ForEach-Object { $_.PSObject.Copy() }) - ($graph | Where-Object Name -eq 'dd') | Add-Member IgnoreDefaults 'false' - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('device/dd/readme.md') - { Get-ReleasePlan $script:Repo $graph } | - Should -Throw '*IgnoreDefaults*true or false*' - } -} - Describe 'release-plan-filter.ps1' { BeforeAll { # A setup.py, setup.cfg and pyproject.toml like the ones here, with diff --git a/ci/packages.ps1 b/ci/packages.ps1 index ec4bb89b..17f73ca2 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -1,117 +1,26 @@ -# Shared by the scripts in this folder, which dot-source it. Reads the -# packages this repository publishes from packages.json, works out which -# products a pull request touches, and which packages a publish releases at -# which version. +# Shared by the scripts in this folder, which dot-source it. Works out which +# products a pull request touches, what a build of HEAD makes, and checks each +# package's tox.ini. # -# Each package has its own version, tagged '/'. A package is -# released when a file that counts under its folders changed since its last -# tag, or when a package it depends on is being released, because its -# requirement on that package moves. This is the scheme in 51Degrees/rust#205, -# which the per language repositories follow. -# -# Every file counts except those matched by the package's Ignore globs in -# packages.json and, unless its IgnoreDefaults is false, by -# $ReleasePlanDefaultIgnore, so a change only to documentation, tests or -# examples releases nothing. release-plan-filter.ps1 then leaves out a -# manifest whose change is only to metadata such as links. These are the -# fields and the filter of common-ci's steps/release-plan.ps1. -# -# A submodule in Path or ExtraPaths changes as one path, its own, so any move -# of it counts. Neither the globs nor the filter see inside it. - -# Reads packages.json. Kept as a function so the tests can give their own list. -function Get-Packages([string]$RepoPath) { - return Get-Content -Raw (Join-Path $RepoPath 'ci/packages.json') | - ConvertFrom-Json -} - -# The folders whose changes count as a change to the package: its own folder, -# and any folder it takes files from at build time, such as a submodule it -# vendors. -function Get-PackagePaths($Package) { - $paths = @($Package.Path) - if ($Package.ExtraPaths) { - $paths += $Package.ExtraPaths - } - return $paths -} - -function Test-PathUnder([string]$Path, [string]$Folder) { - return $Path -eq $Folder -or $Path.StartsWith("$Folder/") -} - -# The files whose changes do not release a package, being documentation, -# tests, examples and LICENSE files wherever they sit and whatever their case, -# unless the package sets IgnoreDefaults to false. These are git glob -# pathspecs from the repository root, where '**/' matches any number of -# folders, none included. -$ReleasePlanDefaultIgnore = @( - '**/*.md' - '**/docs/**' - '**/tests/**' - '**/test/**' - '**/examples/**' - '**/LICENSE*' -) - -# The pathspecs that leave out of a package's changes the files its Ignore -# globs match and, unless IgnoreDefaults is false, the default set. -# PowerShell on Linux and macOS replaces a native command's argument that -# holds a '*' with the files it matches, if any. Each of these starts with -# ':(', which no file in a checkout starts with, so git receives it as written. -function Get-PackageExcludes($Package) { - $defaults = $Package.IgnoreDefaults - if ($null -ne $defaults -and $defaults -isnot [bool]) { - throw "$($Package.Name) gives IgnoreDefaults as '$defaults', where " + - "packages.json must give true or false." - } - $excludes = @($Package.Ignore | Where-Object { $_ } | - ForEach-Object { ":(exclude,glob)$_" }) - if ($defaults -ne $false) { - # The defaults ignore case, as one language names a folder Tests or - # Examples where another names it tests or examples. - $excludes = @($ReleasePlanDefaultIgnore | - ForEach-Object { ":(exclude,glob,icase)$_" }) + $excludes - } - return $excludes -} - -# The files under the package's folders that changed since the given tag, as -# paths from the repository root, apart from those it ignores. -function Get-ChangedFiles([string]$RepoPath, $Package, [string]$From) { - # A failure is reported below, naming the package, whatever the caller's - # preference. Set for this function only. - $PSNativeCommandUseErrorActionPreference = $false - $pathspecs = @(Get-PackagePaths $Package) + @(Get-PackageExcludes $Package) - $files = @(& git -C $RepoPath diff --name-only $From HEAD -- $pathspecs) - if ($LASTEXITCODE -ne 0) { - throw "git could not list the changes to $($Package.Name) since $From." - } - return $files -} - -# The files that count among a package's changed files. The repository's -# ci/release-plan-filter.ps1 decides when there is one, and otherwise every -# file counts. A path the filter returns that it was not given stops the -# plan, because a filter that rewrote a path would otherwise release a -# package on a file that never changed. -function Select-CountedFiles([string]$RepoPath, $Package, [string]$From, - [string[]]$Files) { - $filter = Join-Path $RepoPath 'ci/release-plan-filter.ps1' - if ($Files.Count -eq 0 -or (Test-Path $filter -PathType Leaf) -eq $false) { - return $Files - } - $filter = (Resolve-Path $filter).ProviderPath - $kept = @(& $filter -RepoPath $RepoPath -Package $Package -From $From ` - -Files $Files | Where-Object { $_ }) - foreach ($file in $kept) { - if ($Files -cnotcontains $file) { - throw "ci/release-plan-filter.ps1 returned '$file' for " + - "$($Package.Name), which is not one of the files it was given." - } - } - return $kept +# Which packages a publish releases, and at which version, is worked out by +# common-ci's steps/release-plan.ps1 from ci/packages.json. common-ci runs +# these scripts from its own root, with this repository checked out inside +# it. A checkout of common-ci beside this repository is found too, or one +# elsewhere can be named with COMMON_CI. +$releasePlan = @( + $env:COMMON_CI, + (Join-Path $PSScriptRoot '..' '..'), + (Join-Path $PSScriptRoot '..' '..' 'common-ci')) | + Where-Object { $_ } | + ForEach-Object { Join-Path $_ 'steps' 'release-plan.ps1' } | + Where-Object { Test-Path $_ } | + Select-Object -First 1 +if ($null -eq $releasePlan) { + throw "common-ci's steps/release-plan.ps1 was not found. Set COMMON_CI " + + "to a checkout of common-ci." } +. $releasePlan +Remove-Variable releasePlan # Each tox.ini that does not install every package its package depends on, # directly or through another, from this checkout, as a line naming what is @@ -158,56 +67,6 @@ function Get-ToxGaps([string]$RepoPath, $Packages = $null) { return $gaps } -# The names of the given packages and of every package that depends on one of -# them, directly or through another. -function Get-DependentClosure($Packages, [string[]]$Names) { - $closure = [Collections.Generic.HashSet[string]]::new() - foreach ($name in @($Names | Where-Object { $_ })) { - $null = $closure.Add($name) - } - $grew = $true - while ($grew -eq $true) { - $grew = $false - foreach ($package in $Packages) { - if ($closure.Contains($package.Name) -eq $true) { - continue - } - foreach ($dependency in $package.DependsOn) { - if ($closure.Contains($dependency) -eq $true) { - $null = $closure.Add($package.Name) - $grew = $true - break - } - } - } - } - return @($closure) -} - -# The named packages, each after every package it depends on, so a release -# never publishes a package before the version it requires exists. -function Sort-ByDependency($Packages, [string[]]$Names) { - $remaining = [Collections.Generic.List[string]]::new([string[]]@($Names)) - $sorted = [Collections.Generic.List[string]]::new() - while ($remaining.Count -gt 0) { - $ready = @($remaining | Where-Object { - $name = $_ - $package = $Packages | Where-Object { $_.Name -eq $name } - @($package.DependsOn | Where-Object { - $remaining.Contains($_) -eq $true - }).Count -eq 0 - } | Sort-Object) - if ($ready.Count -eq 0) { - throw "packages.json has a dependency cycle among: $($remaining -join ', ')" - } - foreach ($name in $ready) { - $sorted.Add($name) - $null = $remaining.Remove($name) - } - } - return @($sorted) -} - # The products a set of changed files touches. A file in a package's folders # touches that package's product and the product of every package depending on # it, because a change to pipeline can break device-detection. @@ -251,10 +110,7 @@ function Resolve-AffectedProducts($Packages, [string[]]$Changed) { } # The products a pull request changes, or every product when the checkout is -# not a pull request (a publish, or a build of the branch itself), as -# 51degrees-cxx does. A pull request is checked out with the branch it merges -# into merged in, so the merge base with that branch leaves only the pull -# request's own changes. +# not a pull request (a publish, or a run on main), as 51degrees-cxx does. function Get-AffectedProducts([string]$RepoPath, [string]$Branch = 'main') { # merge-base fails where there is no such branch, which is an answer here. # Set for this function only, so a caller that stops on a failing native @@ -271,117 +127,6 @@ function Get-AffectedProducts([string]$RepoPath, [string]$Branch = 'main') { return Resolve-AffectedProducts $packages $changed } -# The parts of a version such as 4.5.38, or $null for anything else. -function ConvertTo-VersionParts([string]$Text) { - $match = [regex]::Match($Text, '^(\d+)\.(\d+)\.(\d+)$') - if ($match.Success -eq $false) { - return $null - } - return [version]::new([int]$match.Groups[1].Value, - [int]$match.Groups[2].Value, [int]$match.Groups[3].Value) -} - -# The package's released versions, read from its '/' tags, -# highest last. The '/' after the name keeps fiftyone_ipintelligence from -# reading the tags of fiftyone_ipintelligence_cloud. -function Get-ReleasedVersions([string]$RepoPath, [string]$Name, - [switch]$AtHead) { - # Tags are found by prefix, not with a 'name/*' pattern: PowerShell on - # Linux and macOS expands a '*' in an argument to a native command into - # matching files, so from a folder holding a 'location' folder git would - # be handed file names and find no tags. - $arguments = @('-C', $RepoPath, 'for-each-ref', - '--format=%(refname:strip=2)', "refs/tags/$Name/") - if ($AtHead -eq $true) { - $arguments += @('--points-at', 'HEAD') - } - $versions = foreach ($tag in (& git @arguments)) { - ConvertTo-VersionParts $tag.Substring($Name.Length + 1) - } - return @($versions | Where-Object { $null -ne $_ } | Sort-Object) -} - -# What a publish of HEAD releases: one entry per package, in the order to -# publish them, each with the version it is released at and why. -# -# A package already tagged at HEAD was released from this commit, so it is -# left out, which makes a second publish of the same commit release nothing. -# A package with no tag has never been released and starts at its FirstVersion -# from packages.json. Packages that are not published, the examples, are never -# in the plan; a change to them is tested but releases nothing. -function Get-ReleasePlan([string]$RepoPath, $Packages = $null) { - if ($null -eq $Packages) { - $Packages = Get-Packages $RepoPath - } - $published = @($Packages | Where-Object { $_.Publish -eq $true }) - - $previous = @{} - $reasons = @{} - foreach ($package in $published) { - $atHead = Get-ReleasedVersions $RepoPath $package.Name -AtHead - if ($atHead.Count -gt 0) { - continue - } - $released = Get-ReleasedVersions $RepoPath $package.Name - if ($released.Count -eq 0) { - if ([string]::IsNullOrEmpty($package.FirstVersion) -eq $true) { - throw "$($package.Name) has never been released and " + - "packages.json gives it no FirstVersion." - } - $previous[$package.Name] = $null - $reasons[$package.Name] = 'first release' - continue - } - $last = $released[-1] - $previous[$package.Name] = $last - $tag = "$($package.Name)/$last" - $files = @(Get-ChangedFiles $RepoPath $package $tag) - $counted = @(Select-CountedFiles $RepoPath $package $tag $files) - if ($counted.Count -gt 0) { - $noun = 'files' - if ($counted.Count -eq 1) { - $noun = 'file' - } - $reasons[$package.Name] = - "changed since $last ($($counted.Count) $noun)" - } - # A filter may leave an exit code that was an answer, which must not - # read as a failure to the script that called this. - $global:LASTEXITCODE = 0 - } - - # A dependent is released with its dependency, as long as it was not - # already released from this commit. - $candidates = @($published.Name | Where-Object { - $previous.ContainsKey($_) -eq $true - }) - $changed = @($reasons.Keys) - $toRelease = @(Get-DependentClosure $published $changed | Where-Object { - $candidates -contains $_ - }) - - foreach ($name in (Sort-ByDependency $published $toRelease)) { - $package = $published | Where-Object { $_.Name -eq $name } - $last = $previous[$name] - if ($null -eq $last) { - $version = $package.FirstVersion - } else { - $version = "$($last.Major).$($last.Minor).$($last.Build + 1)" - } - $reason = $reasons[$name] - if ($null -eq $reason) { - $reason = 'a package it depends on is released' - } - [pscustomobject]@{ - Name = $name - Path = $package.Path - Version = $version - Previous = $last - Reason = $reason - } - } -} - # What a build of HEAD makes: the release plan, or, when nothing changed since # the last release, every published package at its released version. Nothing # is published then, because package-update-required.ps1 finds the release From f084e8534ecbff33e68b69f04b7c48a3a20f4b8a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 23:27:24 +0000 Subject: [PATCH 010/136] Keep certificate checks on in the usage sharing example (#116) Closes #51. --- .../onpremise/shareusage.py | 7 -- .../test_shareusage_certificate_checks.py | 66 +++++++++++++++++++ 2 files changed, 66 insertions(+), 7 deletions(-) create mode 100644 device-detection/fiftyone_devicedetection_examples/tests/test_shareusage_certificate_checks.py diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py index 51bcff0c..fec5b948 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py @@ -65,13 +65,6 @@ from fiftyone_devicedetection_shared.example_constants import EVIDENCE_FILE_NAME from ruamel.yaml import YAML -# Disable SSL verification for testing -import urllib3 -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) -import requests -_original_post = requests.post -requests.post = lambda *args, **kwargs: _original_post(*args, **{**kwargs, 'verify': False}) - # Configurable settings REQUESTED_PACKAGE_SIZE = 10 # Send after 10 entries (default is 10) SHARE_PERCENTAGE = 1 # Share 100% of data (1 = 100%) diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_shareusage_certificate_checks.py b/device-detection/fiftyone_devicedetection_examples/tests/test_shareusage_certificate_checks.py new file mode 100644 index 00000000..cf23df01 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_shareusage_certificate_checks.py @@ -0,0 +1,66 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +The usage sharing example sends with certificate checks on, because it is the +code people copy into their own applications. The checks read the example's +source, so they need no data file and no network. +""" + +import ast +import os +import unittest + +EXAMPLE = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "src", + "fiftyone_devicedetection_examples", "onpremise", "shareusage.py") + + +class ShareUsageCertificateCheckTests(unittest.TestCase): + + def setUp(self): + with open(EXAMPLE, encoding="utf-8") as source: + self.tree = ast.parse(source.read()) + + def test_does_not_replace_requests_post(self): + for node in ast.walk(self.tree): + if isinstance(node, ast.Assign): + for target in node.targets: + self.assertNotEqual("requests.post", ast.unparse(target)) + + def test_does_not_turn_off_certificate_checks(self): + for node in ast.walk(self.tree): + if isinstance(node, ast.keyword): + self.assertNotEqual("verify", node.arg) + if isinstance(node, ast.Dict): + for key in node.keys: + self.assertFalse( + isinstance(key, ast.Constant) and key.value == "verify") + + def test_does_not_silence_certificate_warnings(self): + for node in ast.walk(self.tree): + if isinstance(node, ast.Call): + self.assertNotIn("disable_warnings", ast.unparse(node.func)) + + +if __name__ == "__main__": + unittest.main() From 175ca4adab7d765f22e2ba61ed37f3c410bbbac3 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 23:46:34 +0000 Subject: [PATCH 011/136] Hand the web request to the pipeline in the IP Intelligence web examples (#119) The Usage Sharing Element accepts no Evidence, and shares what it finds in the web request that the web integration adds as `server.http-request`, together with the Evidence given to the Pipeline (`found_evidence` in [`share_usage.py` lines 432 to 458 at 5b5a833](https://github.com/51Degrees/51degrees-python/blob/5b5a833239d435676a1ab0450be22bce0c0807a2/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py#L432-L458)). `webevidence` in [`pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py` line 139](https://github.com/51Degrees/51degrees-python/blob/5b5a833239d435676a1ab0450be22bce0c0807a2/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py#L139) adds it for every request it reads. --- .../cloud/gettingstarted_web/app.py | 5 ++ .../onpremise/gettingstarted_web/app.py | 5 ++ .../test_web_examples_hand_over_request.py | 71 +++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 ip-intelligence/fiftyone_ipintelligence_examples/tests/test_web_examples_hand_over_request.py diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py index 90270ef4..c7100800 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py @@ -37,6 +37,7 @@ from flask import Flask, request +from fiftyone_pipeline_core.web import webevidence from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_ipintelligence_shared.constants import Constants @@ -63,6 +64,10 @@ def index(): # Create the flowdata object for this request. flowdata = pipeline.create_flowdata() + # Hand the whole request to the pipeline. Each engine takes the + # Evidence it reads from it, and usage sharing takes the request. + flowdata.evidence.add_from_dict(webevidence(request)) + # Use the IP address supplied as a query parameter, or fall back # to the address of the requester. client_ip = request.args.get("client-ip", request.remote_addr) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py index e6f25806..c65b5a4b 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py @@ -38,6 +38,7 @@ from flask import Flask, request +from fiftyone_pipeline_core.web import webevidence from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_ipintelligence_examples.onpremise.countries_pipeline import ( @@ -103,6 +104,10 @@ def index(): # the results of the lookup straight away. with pipeline.create_flowdata() as flowdata: + # Hand the whole request to the pipeline. Each engine takes the + # Evidence it reads from it, and usage sharing takes the request. + flowdata.evidence.add_from_dict(webevidence(request)) + # Use the IP address supplied as a query parameter, or fall back # to the address of the requester. client_ip = request.args.get("client-ip", request.remote_addr) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_web_examples_hand_over_request.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_web_examples_hand_over_request.py new file mode 100644 index 00000000..c77f83af --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_web_examples_hand_over_request.py @@ -0,0 +1,71 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Each web example hands the whole web request to the pipeline with +webevidence, which adds it as server.http-request. The usage sharing element +accepts no Evidence and shares what it finds in that request, so an example +that adds only the Evidence its engine reads shares nothing else. The checks +read the examples' source, so they need no data file and no network. +""" + +import ast +import os +import unittest + +EXAMPLES = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "src", + "fiftyone_ipintelligence_examples") +WEB_EXAMPLES = [ + os.path.join(EXAMPLES, "cloud", "gettingstarted_web", "app.py"), + os.path.join(EXAMPLES, "onpremise", "gettingstarted_web", "app.py"), +] + + +def called_names(function): + # The names of every function the given function calls. + names = set() + for node in ast.walk(function): + if isinstance(node, ast.Call): + target = node.func + names.add(target.attr if isinstance(target, ast.Attribute) + else getattr(target, "id", None)) + return names + + +class WebExamplesHandOverRequestTests(unittest.TestCase): + + def test_every_route_that_makes_flow_data_adds_the_web_request(self): + for path in WEB_EXAMPLES: + with open(path, encoding="utf-8") as source: + tree = ast.parse(source.read()) + routes = [node for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef) + and "create_flowdata" in called_names(node)] + self.assertTrue(routes, path) + for route in routes: + with self.subTest(example=path, route=route.name): + self.assertIn("webevidence", called_names(route)) + + +if __name__ == "__main__": + unittest.main() From b1fc0903d7fdead3b743f59d1cd59361f37537a5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 23:57:47 +0000 Subject: [PATCH 012/136] Test each cloud example with no resource key set Each must end without an error, naming the environment variable to set and giving a tagged link to the page where a key is created. --- .../tests/test_cloud_examples_without_key.py | 132 ++++++++++++++++++ .../tests/test_cloud_examples_without_key.py | 122 ++++++++++++++++ location/tests/test_examples_without_key.py | 70 ++++++++++ .../tests/test_cloud_mixed_without_key.py | 82 +++++++++++ .../tests/test_creator_context_without_key.py | 53 +++++++ 5 files changed, 459 insertions(+) create mode 100644 device-detection/fiftyone_devicedetection_examples/tests/test_cloud_examples_without_key.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloud_examples_without_key.py create mode 100644 location/tests/test_examples_without_key.py create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_without_key.py create mode 100644 pipeline/fiftyone_pipeline_did/tests/test_creator_context_without_key.py diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_cloud_examples_without_key.py b/device-detection/fiftyone_devicedetection_examples/tests/test_cloud_examples_without_key.py new file mode 100644 index 00000000..cf6802fe --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_cloud_examples_without_key.py @@ -0,0 +1,132 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Runs each cloud example as it runs from a fresh clone, with no resource key +set. Each must end without an error, naming the environment variable to set +and giving a tagged link to the page where a key is created. +""" + +import io +import logging +import os +import runpy +import unittest +from contextlib import redirect_stdout +from unittest import mock + +from fiftyone_devicedetection_examples.example_utils import ExampleUtils +from fiftyone_devicedetection_examples.cloud import configurator_console +from fiftyone_devicedetection_examples.cloud import gettingstarted_console +from fiftyone_devicedetection_examples.cloud import metadata_console +from fiftyone_devicedetection_examples.cloud import nativemodellookup_console +from fiftyone_devicedetection_examples.cloud import taclookup_console +from fiftyone_devicedetection_examples.cloud.gettingstarted_web import ( + app as gettingstarted_web) +from fiftyone_devicedetection_examples.cloud.useragentclienthints_web import ( + app as useragentclienthints_web) + +# Every name a key can be read from, which a fresh clone does not set. +KEY_SETTINGS = [ + "_51DEGREES_RESOURCE_KEY", "resource_key", "RESOURCE_KEY", + "_51DEGREES_LICENSE_KEY", "license_key"] + +# A link to the page where a resource key is created, with the campaign +# tags every example's links carry. +CONFIGURE_LINK = ( + r"https://51degrees\.com/configure[^\s?]*\?utm_source=code" + r"&utm_medium=example&utm_campaign=51degrees-python&utm_content=") + + +class Recorder(logging.Handler): + """Keeps every message logged while it is attached.""" + + def __init__(self): + super().__init__(logging.DEBUG) + self.messages = [] + + def emit(self, record): + self.messages.append(record.getMessage()) + + +def run_without_key(run): + """Runs an example with no key in the environment, and returns what it + printed and logged.""" + printed = io.StringIO() + recorder = Recorder() + root = logging.getLogger() + root.addHandler(recorder) + try: + with mock.patch.dict(os.environ), redirect_stdout(printed): + for name in KEY_SETTINGS: + os.environ.pop(name, None) + run() + finally: + root.removeHandler(recorder) + return printed.getvalue() + "\n".join(recorder.messages) + + +class CloudExamplesWithoutKeyTests(unittest.TestCase): + + def assert_says_how_to_get_a_key(self, said, names_variable=True): + self.assertRegex(said, CONFIGURE_LINK) + if names_variable: + self.assertIn(ExampleUtils.RESOURCE_KEY_ENV_VAR, said) + + def test_configurator_console(self): + self.assert_says_how_to_get_a_key( + run_without_key(lambda: configurator_console.main([]))) + + def test_gettingstarted_console(self): + self.assert_says_how_to_get_a_key( + run_without_key(lambda: gettingstarted_console.main([]))) + + def test_metadata_console(self): + self.assert_says_how_to_get_a_key( + run_without_key(lambda: metadata_console.main([]))) + + def test_nativemodellookup_console(self): + self.assert_says_how_to_get_a_key( + run_without_key(lambda: nativemodellookup_console.main([]))) + + def test_taclookup_console(self): + self.assert_says_how_to_get_a_key( + run_without_key(lambda: taclookup_console.main([]))) + + def test_gettingstarted_web(self): + self.assert_says_how_to_get_a_key( + run_without_key(lambda: gettingstarted_web.main([]))) + + def test_useragentclienthints_web(self): + self.assert_says_how_to_get_a_key( + run_without_key(lambda: useragentclienthints_web.main([]))) + + def test_failuretomatch(self): + # This example has no main and runs when it is run as a script. + self.assert_says_how_to_get_a_key(run_without_key( + lambda: runpy.run_module( + "fiftyone_devicedetection_examples.cloud.failuretomatch", + run_name="__main__"))) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloud_examples_without_key.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloud_examples_without_key.py new file mode 100644 index 00000000..3d5f02c3 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloud_examples_without_key.py @@ -0,0 +1,122 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Runs each cloud example as it runs from a fresh clone, with no resource key +set. Each must end without an error, naming the environment variable to set +and giving a tagged link to the page where a key is created. +""" + +import io +import logging +import os +import unittest +from contextlib import redirect_stdout +from unittest import mock + +from fiftyone_ipintelligence_examples.example_utils import ExampleUtils +from fiftyone_ipintelligence_examples.cloud import getallproperties_console +from fiftyone_ipintelligence_examples.cloud import gettingstarted_console +from fiftyone_ipintelligence_examples.cloud import metadata_console +from fiftyone_ipintelligence_examples.cloud.gettingstarted_web import ( + app as gettingstarted_web) + +# The mixed example combines IP intelligence with device detection, which +# is an optional dependency. +try: + from fiftyone_ipintelligence_examples.cloud import mixed_console +except ImportError: + mixed_console = None + +# Every name a key can be read from, which a fresh clone does not set. +KEY_SETTINGS = [ + "_51DEGREES_RESOURCE_KEY", "resource_key", "RESOURCE_KEY", + "_51DEGREES_LICENSE_KEY", "license_key"] + +# A link to the page where a resource key is created, with the campaign +# tags every example's links carry. +CONFIGURE_LINK = ( + r"https://51degrees\.com/configure[^\s?]*\?utm_source=code" + r"&utm_medium=example&utm_campaign=51degrees-python&utm_content=") + + +class Recorder(logging.Handler): + """Keeps every message logged while it is attached.""" + + def __init__(self): + super().__init__(logging.DEBUG) + self.messages = [] + + def emit(self, record): + self.messages.append(record.getMessage()) + + +def run_without_key(run): + """Runs an example with no key in the environment, and returns what it + printed and logged.""" + printed = io.StringIO() + recorder = Recorder() + root = logging.getLogger() + root.addHandler(recorder) + try: + with mock.patch.dict(os.environ), redirect_stdout(printed): + for name in KEY_SETTINGS: + os.environ.pop(name, None) + run() + finally: + root.removeHandler(recorder) + return printed.getvalue() + "\n".join(recorder.messages) + + +class CloudExamplesWithoutKeyTests(unittest.TestCase): + + def assert_says_how_to_get_a_key(self, said): + self.assertIn(ExampleUtils.CLOUD_RESOURCE_KEY_ENV_VAR, said) + self.assertRegex(said, CONFIGURE_LINK) + + def test_getallproperties_console(self): + self.assert_says_how_to_get_a_key( + run_without_key(lambda: getallproperties_console.main([]))) + + def test_gettingstarted_console(self): + self.assert_says_how_to_get_a_key( + run_without_key(lambda: gettingstarted_console.main([]))) + + def test_metadata_console(self): + self.assert_says_how_to_get_a_key( + run_without_key(lambda: metadata_console.main([]))) + + @unittest.skipIf( + mixed_console is None, "device detection package not available") + def test_mixed_console(self): + self.assert_says_how_to_get_a_key( + run_without_key(lambda: mixed_console.main([]))) + + def test_gettingstarted_web(self): + # The web example takes the key as its one argument, and starts no + # server when there is none. + self.assert_says_how_to_get_a_key( + run_without_key(lambda: gettingstarted_web.main())) + + +if __name__ == "__main__": + unittest.main() diff --git a/location/tests/test_examples_without_key.py b/location/tests/test_examples_without_key.py new file mode 100644 index 00000000..33f4158b --- /dev/null +++ b/location/tests/test_examples_without_key.py @@ -0,0 +1,70 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Runs each cloud example as it runs from a fresh clone, with no resource key +set. Each must end without an error and give a tagged link to the page +where a key is created. +""" + +import io +import os +import runpy +import unittest +from contextlib import redirect_stdout +from pathlib import Path +from unittest import mock + +EXAMPLES = Path(__file__).resolve().parent.parent / "examples" / "cloud" + +# Every name a key can be read from, which a fresh clone does not set. +KEY_SETTINGS = ["_51DEGREES_RESOURCE_KEY", "resource_key", "RESOURCE_KEY"] + +# A link to the page where a resource key is created, with the campaign +# tags every example's links carry. +CONFIGURE_LINK = ( + r"https://51degrees\.com/configure[^\s?]*\?utm_source=code" + r"&utm_medium=example&utm_campaign=51degrees-python&utm_content=") + + +def run_without_key(example): + """Runs an example file as a script with no key in the environment, and + returns what it printed.""" + printed = io.StringIO() + with mock.patch.dict(os.environ), redirect_stdout(printed): + for name in KEY_SETTINGS: + os.environ.pop(name, None) + runpy.run_path(str(EXAMPLES / example), run_name="__main__") + return printed.getvalue() + + +class LocationExamplesWithoutKeyTests(unittest.TestCase): + + def test_cloud_getting_started(self): + self.assertRegex(run_without_key("gettingstarted.py"), CONFIGURE_LINK) + + def test_cloud_web(self): + self.assertRegex(run_without_key("web.py"), CONFIGURE_LINK) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_without_key.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_without_key.py new file mode 100644 index 00000000..50c4ff0c --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_without_key.py @@ -0,0 +1,82 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Runs the mixed cloud console as it runs from a fresh clone, with no key +set. It must end without an error, naming the environment variables to set +and giving a tagged link to the page where a key is created. +""" + +import io +import logging +import os +import unittest +from contextlib import redirect_stdout +from unittest import mock + +from fiftyone_pipeline_examples.cloud.mixed import gettingstarted_console +from fiftyone_pipeline_examples.example_utils import ExampleUtils + +# Every name a key can be read from, which a fresh clone does not set. +KEY_SETTINGS = [ + "_51DEGREES_RESOURCE_KEY", "resource_key", "RESOURCE_KEY", + "_51DEGREES_LICENSE_KEY", "license_key", "LICENSE_KEY"] + +# A link to the page where a resource key is created, with the campaign +# tags every example's links carry. +CONFIGURE_LINK = ( + r"https://51degrees\.com/configure[^\s?]*\?utm_source=code" + r"&utm_medium=example&utm_campaign=51degrees-python&utm_content=") + + +class Recorder(logging.Handler): + """Keeps every message logged while it is attached.""" + + def __init__(self): + super().__init__(logging.DEBUG) + self.messages = [] + + def emit(self, record): + self.messages.append(record.getMessage()) + + +class MixedConsoleWithoutKeyTests(unittest.TestCase): + + def test_says_how_to_get_a_key(self): + printed = io.StringIO() + recorder = Recorder() + root = logging.getLogger() + root.addHandler(recorder) + try: + with mock.patch.dict(os.environ), redirect_stdout(printed): + for name in KEY_SETTINGS: + os.environ.pop(name, None) + gettingstarted_console.main([]) + finally: + root.removeHandler(recorder) + said = printed.getvalue() + "\n".join(recorder.messages) + self.assertIn(ExampleUtils.RESOURCE_KEY_ENV_VAR, said) + self.assertRegex(said, CONFIGURE_LINK) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_did/tests/test_creator_context_without_key.py b/pipeline/fiftyone_pipeline_did/tests/test_creator_context_without_key.py new file mode 100644 index 00000000..c1a3beae --- /dev/null +++ b/pipeline/fiftyone_pipeline_did/tests/test_creator_context_without_key.py @@ -0,0 +1,53 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +The creator context demo, started with no resource key, stops and says +which environment variable to set and where to create a key. +""" + +import unittest +from unittest import mock + +from .test_creator_context_server import load_server + +# A link to the page where a resource key is created, with the campaign +# tags every example's links carry. +CONFIGURE_LINK = ( + r"https://51degrees\.com/configure[^\s?]*\?utm_source=code" + r"&utm_medium=example&utm_campaign=51degrees-python&utm_content=") + + +class CreatorContextWithoutKeyTests(unittest.TestCase): + + def test_says_where_to_create_a_resource_key(self): + server = load_server() + with mock.patch.object(server, "RESOURCE", None): + with self.assertRaises(SystemExit) as stopped: + server.run() + said = str(stopped.exception.code) + self.assertIn("_51DEGREES_RESOURCE_KEY", said) + self.assertRegex(said, CONFIGURE_LINK) + + +if __name__ == "__main__": + unittest.main() From b1364f8b45e611d1537a4352f4fa5cd2b999c853 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 00:09:34 +0000 Subject: [PATCH 013/136] Test that every example ends with a Find out more section Each examples package, location and the pipeline packages that hold examples gain test_find_out_more.py. It reads every example file and fails, naming the file, when the file has no Find out more section, meaning a list of links whose 51degrees.com addresses carry a utm_term starting with find-out-more and a utm_content naming that file, and a link to the product's folder in this repository. Where an example can stop early without a key, a data file or the network, the test also runs it to that end and checks the section is printed there. The existing tests that capture what an example prints, or the page a web example serves, now also check for the section. The device lines check of the TAC and native model lookups leaves out the section's links, which are indented as device lines are. These fail until the examples print the section, which the next commits add. --- .../tests/test_cloudexamples.py | 11 +- .../tests/test_cloudgettingstartedweb.py | 1 + .../tests/test_examples_without_settings.py | 6 + .../tests/test_find_out_more.py | 191 ++++++++++++++++++ .../tests/test_onpremisegettingstartedweb.py | 1 + .../tests/test_examples_without_settings.py | 8 + .../tests/test_find_out_more.py | 173 ++++++++++++++++ location/tests/test_find_out_more.py | 99 +++++++++ .../examples/tests/test_find_out_more.py | 63 ++++++ .../tests/test_find_out_more.py | 71 +++++++ .../tests/test_find_out_more.py | 77 +++++++ .../tests/test_find_out_more.py | 71 +++++++ 12 files changed, 771 insertions(+), 1 deletion(-) create mode 100644 device-detection/fiftyone_devicedetection_examples/tests/test_find_out_more.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_examples/tests/test_find_out_more.py create mode 100644 location/tests/test_find_out_more.py create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_find_out_more.py create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_find_out_more.py create mode 100644 pipeline/fiftyone_pipeline_did/tests/test_find_out_more.py create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_find_out_more.py diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py index eebe2361..97b50e11 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py @@ -92,7 +92,10 @@ def assert_device_lines_are_meaningful(self, output): has no value and gives the reason the cloud service supplied. """ - lines = [line for line in output.split("\n") + # The links of the section the example ends with are left out, + # because they are indented as device lines are. + devices = output.split("Find out more:")[0] + lines = [line for line in devices.split("\n") if line.startswith("\t")] self.assertGreater(len(lines), 0, @@ -112,6 +115,7 @@ def test_cloud_getting_started_console(self): self.assertIn("Input values:", output) self.assertIn("Mobile Device:", output) + self.assertIn("Find out more:", output) def test_cloud_nativemodellookup_console(self): example = NativeModelLookupConsole() @@ -123,6 +127,7 @@ def test_cloud_nativemodellookup_console(self): "Which devices are associated with the native model name " "'SC-03L'?", output) self.assert_device_lines_are_meaningful(output) + self.assertIn("Find out more:", output) def test_cloud_taclookup_console(self): example = TacLookupConsole() @@ -136,6 +141,7 @@ def test_cloud_taclookup_console(self): self.assertIn( "Which devices are associated with the TAC '35925406'?", output) self.assert_device_lines_are_meaningful(output) + self.assertIn("Find out more:", output) def test_cloud_metadata_console(self): example = MetaDataConsole() @@ -145,6 +151,7 @@ def test_cloud_metadata_console(self): self.assertIn("Accepted evidence keys:", output) self.assertIn("Property - ", output) + self.assertIn("Find out more:", output) def test_cloud_configurator_console(self): example = ConfiguratorConsole() @@ -154,6 +161,7 @@ def test_cloud_configurator_console(self): self.assertIn("device.ismobile: ", output) self.assertNotEqual("device.ismobile:", output.strip()) + self.assertIn("Find out more:", output) def test_cloud_failuretomatch(self): """The failure to match example is a script rather than a class, @@ -174,5 +182,6 @@ def test_cloud_failuretomatch(self): self.assertEqual(0, finished.returncode, output) self.assertNotIn("No resource key found", output) self.assertIn("a mobile device?", output) + self.assertIn("Find out more:", output) for marker in FAULT_MARKERS: self.assertNotIn(marker, output) diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudgettingstartedweb.py b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudgettingstartedweb.py index d3239b6a..440f3a81 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudgettingstartedweb.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudgettingstartedweb.py @@ -41,6 +41,7 @@ class CloudGettingStartedWebTests(flask_unittest.ClientTestCase): def test_cloud_getting_started_web(self, client): response = client.get('/') self.assertEqual(200, response.status_code) + self.assertIn(b"Find out more", response.data) # The device id used to appear on the page only by accident, inside the JSON of the # inlined client-side script. Now that the script is served separately, check that the diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py b/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py index 6e7fbef4..eee099e2 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py @@ -85,18 +85,21 @@ def test_gettingstarted_console(self): self.assertIn( ExampleUtils.get_enterprise_note(gettingstarted_console.CONTACT_US), printed) + self.assertIn("Find out more:", printed) def test_match_metrics(self): printed = run_without_settings(match_metrics.main) self.assertIn( ExampleUtils.get_enterprise_offer(match_metrics.CONTACT_US), printed) + self.assertIn("Find out more:", printed) def test_metadata_console(self): printed = run_without_settings(metadata_console.main) self.assertIn( ExampleUtils.get_enterprise_offer(metadata_console.CONTACT_US), printed) + self.assertIn("Find out more:", printed) def test_offline_processing(self): output = io.StringIO() @@ -119,6 +122,7 @@ def test_datafileupdate_console(self): printed = run_without_settings(datafileupdate_console.main) self.assertIn("The data file in use is the 'Lite' file", printed) self.assertIn(datafileupdate_console.CONTACT_US, printed) + self.assertIn("Find out more:", printed) # The two examples below need files that are not free, so in a fresh # clone the search for them finds nothing, which the tests stand in for. @@ -128,12 +132,14 @@ def test_csv_profile_metadata(self): printed = run_without_settings(csv_profile_metadata.main) self.assertIn(csv_profile_metadata.CSV_FILE_NAME, printed) self.assertIn(csv_profile_metadata.CONTACT_US, printed) + self.assertIn("Find out more:", printed) def test_profile_metadata_console(self): with mock.patch.object(ExampleUtils, "find_file", return_value=None): printed = run_without_settings(profile_metadata_console.main) self.assertIn("only the Enterprise data file has", printed) self.assertIn(profile_metadata_console.CONTACT_US, printed) + self.assertIn("Find out more:", printed) if __name__ == "__main__": diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_find_out_more.py b/device-detection/fiftyone_devicedetection_examples/tests/test_find_out_more.py new file mode 100644 index 00000000..6f381ace --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_find_out_more.py @@ -0,0 +1,191 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every example ends with a "Find out more" section, which gives the person +running it links to follow up, wherever the example ends. The first checks +read the examples' source. The others run each example to an early end, with +no resource key or no data file, so none of them needs a key, a data file or +the network. +""" + +import io +import os +import re +import runpy +import unittest +from contextlib import redirect_stdout +from unittest import mock + +from fiftyone_devicedetection_examples.example_utils import ExampleUtils +from fiftyone_devicedetection_examples.cloud import configurator_console +from fiftyone_devicedetection_examples.cloud import ( + gettingstarted_console as cloud_gettingstarted_console) +from fiftyone_devicedetection_examples.cloud import ( + metadata_console as cloud_metadata_console) +from fiftyone_devicedetection_examples.cloud import nativemodellookup_console +from fiftyone_devicedetection_examples.cloud import taclookup_console +from fiftyone_devicedetection_examples.cloud.gettingstarted_web import ( + app as cloud_gettingstarted_web) +from fiftyone_devicedetection_examples.cloud.useragentclienthints_web import ( + app as cloud_useragentclienthints_web) +from fiftyone_devicedetection_examples.onpremise import gettingstarted_console +from fiftyone_devicedetection_examples.onpremise import match_metrics +from fiftyone_devicedetection_examples.onpremise import metadata_console +from fiftyone_devicedetection_examples.onpremise import shareusage + +# The product folder. Each link's utm_content is the path of its file below +# this folder. +PRODUCT = os.path.abspath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "..")) +EXAMPLES = os.path.join( + PRODUCT, "fiftyone_devicedetection_examples", "src", + "fiftyone_devicedetection_examples") +CLOUD = os.path.join(EXAMPLES, "cloud") +ONPREMISE = os.path.join(EXAMPLES, "onpremise") + +# Every file that holds a section. A web example has one in its page, and +# one in its app for the console. +SOURCES = [ + os.path.join(CLOUD, "configurator_console.py"), + os.path.join(CLOUD, "failuretomatch.py"), + os.path.join(CLOUD, "gettingstarted_console.py"), + os.path.join(CLOUD, "gettingstarted_web", "app.py"), + os.path.join(CLOUD, "gettingstarted_web", "templates", "index.html"), + os.path.join(CLOUD, "metadata_console.py"), + os.path.join(CLOUD, "nativemodellookup_console.py"), + os.path.join(CLOUD, "taclookup_console.py"), + os.path.join(CLOUD, "useragentclienthints_web", "app.py"), + os.path.join( + CLOUD, "useragentclienthints_web", "templates", "index.html"), + os.path.join(ONPREMISE, "csv_profile_metadata.py"), + os.path.join(ONPREMISE, "datafileupdate_console.py"), + os.path.join(ONPREMISE, "failuretomatch.py"), + os.path.join(ONPREMISE, "gettingstarted_console.py"), + os.path.join(ONPREMISE, "gettingstarted_web", "app.py"), + os.path.join(ONPREMISE, "gettingstarted_web", "templates", "index.html"), + os.path.join(ONPREMISE, "match_metrics.py"), + os.path.join(ONPREMISE, "metadata_console.py"), + os.path.join(ONPREMISE, "offlineprocessing.py"), + os.path.join(ONPREMISE, "performance.py"), + os.path.join(ONPREMISE, "profile_metadata_console.py"), + os.path.join(ONPREMISE, "shareusage.py"), + os.path.join(ONPREMISE, "useragentclienthints_web", "app.py"), + os.path.join( + ONPREMISE, "useragentclienthints_web", "templates", "index.html"), + os.path.join( + PRODUCT, "fiftyone_devicedetection", "examples", "gettingstarted.py"), +] + +# The name of the section, or of the list that holds its links. +SECTION_NAME = re.compile(r"(?i)find[ _]out[ _]more") + +# A 51degrees.com link in the section, which carries a utm_term that starts +# with "find-out-more". +SECTION_LINK = re.compile( + r"https://51degrees\.com/[^\s\"'<>)]*utm_term=find-out-more[^\s\"'<>)]*") + +# This product's folder in the repository, which every section links to. +REPOSITORY = ( + "https://github.com/51Degrees/51degrees-python/tree/main/" + "device-detection") + +# The settings that let an example get past its early end. +SETTINGS = [ + "_51DEGREES_RESOURCE_KEY", "resource_key", "RESOURCE_KEY", + "_51DEGREES_LICENSE_KEY", "license_key", "LICENSE_KEY"] + + +def slug(path): + """The utm_content of a link in the file, which is the path below the + product folder with each separator a hyphen.""" + return os.path.relpath(path, PRODUCT).replace(os.sep, "-") + + +def printed_by(run): + """Runs an example with the settings taken out of the environment, and + returns what it printed.""" + printed = io.StringIO() + with mock.patch.dict(os.environ), redirect_stdout(printed): + for name in SETTINGS: + os.environ.pop(name, None) + run() + return printed.getvalue() + + +class FindOutMoreSourceTests(unittest.TestCase): + + def test_every_example_has_a_find_out_more_section(self): + for path in SOURCES: + with self.subTest(example=slug(path)): + with open(path, encoding="utf-8") as source: + # A page separates the parameters with "&". + text = source.read().replace("&", "&") + self.assertRegex(text, SECTION_NAME) + links = SECTION_LINK.findall(text) + self.assertTrue( + links, "The section has no tagged 51degrees.com link") + for link in links: + self.assertIn(f"&utm_content={slug(path)}&", link) + self.assertIn(REPOSITORY, text) + + +class FindOutMoreAtEarlyEndsTests(unittest.TestCase): + + def test_cloud_examples_without_a_resource_key(self): + examples = { + "configurator_console": configurator_console.main, + "gettingstarted_console": cloud_gettingstarted_console.main, + "metadata_console": cloud_metadata_console.main, + "nativemodellookup_console": nativemodellookup_console.main, + "taclookup_console": taclookup_console.main, + "gettingstarted_web": cloud_gettingstarted_web.main, + "useragentclienthints_web": cloud_useragentclienthints_web.main, + } + for name, main in examples.items(): + with self.subTest(example=name): + self.assertIn("Find out more:", printed_by(lambda: main([]))) + + def test_cloud_failuretomatch_without_a_resource_key(self): + # This example has no main, and runs when it is run as a script. + printed = printed_by(lambda: runpy.run_module( + "fiftyone_devicedetection_examples.cloud.failuretomatch", + run_name="__main__")) + self.assertIn("Find out more:", printed) + + def test_onpremise_examples_without_a_data_file(self): + examples = { + "gettingstarted_console": gettingstarted_console.main, + "match_metrics": match_metrics.main, + "metadata_console": metadata_console.main, + "shareusage": shareusage.main, + } + for name, main in examples.items(): + with self.subTest(example=name): + with mock.patch.object( + ExampleUtils, "find_file", return_value=None): + printed = printed_by(lambda: main([])) + self.assertIn("Find out more:", printed) + + +if __name__ == "__main__": + unittest.main() diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py b/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py index 5c8978ee..9787fe3c 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py @@ -49,6 +49,7 @@ def tearDownClass(cls): def test_onpremise_getting_started_web(self, client): response = client.get('/') self.assertEqual(200, response.status_code) + self.assertIn(b"Find out more", response.data) # The page references the client-side script by the '/51Degrees.core.js' name used # by the web integrations in the other Pipeline APIs, so check that the route diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py index e3cdf702..557740d9 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py @@ -78,6 +78,7 @@ def test_gettingstarted_console(self): self.assertIn( ExampleUtils.get_enterprise_note(gettingstarted_console.CONTACT_US), printed) + self.assertIn("Find out more:", printed) def test_countries_console(self): printed = run_without_settings(countries_console.main) @@ -86,12 +87,14 @@ def test_countries_console(self): "CountryCodesGeographical and CountryCodesPopulation properties", printed) self.assertIn(countries_console.CONTACT_US, printed) + self.assertIn("Find out more:", printed) def test_metadata_console(self): printed = run_without_settings(metadata_console.main) self.assertIn( ExampleUtils.get_enterprise_offer(metadata_console.CONTACT_US), printed) + self.assertIn("Find out more:", printed) def test_offlineprocessing(self): printed = run_without_settings(offlineprocessing.main) @@ -100,6 +103,7 @@ def test_offlineprocessing(self): "# " + ExampleUtils.get_enterprise_note( offlineprocessing.CONTACT_US), printed) + self.assertIn("Find out more:", printed) def test_compare_console(self): printed = run_without_settings(compare_console.main) @@ -108,6 +112,7 @@ def test_compare_console(self): "Latitude, Longitude, LocationConfidence and Country properties", printed) self.assertIn(compare_console.CONTACT_US, printed) + self.assertIn("Find out more:", printed) def test_performance_console(self): printed = run_without_settings( @@ -115,11 +120,13 @@ def test_performance_console(self): self.assertIn( ExampleUtils.get_enterprise_offer(performance_console.CONTACT_US), printed) + self.assertIn("Find out more:", printed) def test_datafileupdate_console(self): printed = run_without_settings(datafileupdate_console.main) self.assertIn("The data file in use is the", printed) self.assertIn(datafileupdate_console.CONTACT_US, printed) + self.assertIn("Find out more:", printed) @unittest.skipUnless( device_detection_available, "device detection is not installed") @@ -129,6 +136,7 @@ def test_mixed_console(self): self.assertIn( ExampleUtils.get_enterprise_note(mixed_console.CONTACT_US), printed) + self.assertIn("Find out more:", printed) if __name__ == "__main__": diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_find_out_more.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_find_out_more.py new file mode 100644 index 00000000..219fb7cb --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_find_out_more.py @@ -0,0 +1,173 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every example ends with a "Find out more" section, which gives the person +running it links to follow up, wherever the example ends. The first checks +read the examples' source. The others run each example to an early end, with +no resource key or no data file, so none of them needs a key, a data file or +the network. +""" + +import io +import os +import re +import unittest +from contextlib import redirect_stdout +from unittest import mock + +from fiftyone_ipintelligence_examples.example_utils import ExampleUtils +from fiftyone_ipintelligence_examples.cloud import getallproperties_console +from fiftyone_ipintelligence_examples.cloud import ( + gettingstarted_console as cloud_gettingstarted_console) +from fiftyone_ipintelligence_examples.cloud import ( + metadata_console as cloud_metadata_console) +from fiftyone_ipintelligence_examples.cloud import ( + mixed_console as cloud_mixed_console) +from fiftyone_ipintelligence_examples.onpremise import compare_console +from fiftyone_ipintelligence_examples.onpremise import countries_console +from fiftyone_ipintelligence_examples.onpremise import datafileupdate_console +from fiftyone_ipintelligence_examples.onpremise import gettingstarted_console +from fiftyone_ipintelligence_examples.onpremise import metadata_console +from fiftyone_ipintelligence_examples.onpremise import mixed_console +from fiftyone_ipintelligence_examples.onpremise import offlineprocessing +from fiftyone_ipintelligence_examples.onpremise import performance_console + +# The product folder. Each link's utm_content is the path of its file below +# this folder. +PRODUCT = os.path.abspath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "..")) +EXAMPLES = os.path.join( + PRODUCT, "fiftyone_ipintelligence_examples", "src", + "fiftyone_ipintelligence_examples") +CLOUD = os.path.join(EXAMPLES, "cloud") +ONPREMISE = os.path.join(EXAMPLES, "onpremise") + +# Every file that holds a section. A web example builds its page in its app, +# so the app holds the section for the page. +SOURCES = [ + os.path.join(CLOUD, "getallproperties_console.py"), + os.path.join(CLOUD, "gettingstarted_console.py"), + os.path.join(CLOUD, "gettingstarted_web", "app.py"), + os.path.join(CLOUD, "metadata_console.py"), + os.path.join(CLOUD, "mixed_console.py"), + os.path.join(ONPREMISE, "compare_console.py"), + os.path.join(ONPREMISE, "countries_console.py"), + os.path.join(ONPREMISE, "datafileupdate_console.py"), + os.path.join(ONPREMISE, "gettingstarted_console.py"), + os.path.join(ONPREMISE, "gettingstarted_web", "app.py"), + os.path.join(ONPREMISE, "metadata_console.py"), + os.path.join(ONPREMISE, "mixed_console.py"), + os.path.join(ONPREMISE, "offlineprocessing.py"), + os.path.join(ONPREMISE, "performance_console.py"), + os.path.join( + PRODUCT, "fiftyone_ipintelligence", "examples", "gettingstarted.py"), +] + +# The name of the section, or of the list that holds its links. +SECTION_NAME = re.compile(r"(?i)find[ _]out[ _]more") + +# A 51degrees.com link in the section, which carries a utm_term that starts +# with "find-out-more". +SECTION_LINK = re.compile( + r"https://51degrees\.com/[^\s\"'<>)]*utm_term=find-out-more[^\s\"'<>)]*") + +# This product's folder in the repository, which every section links to. +REPOSITORY = ( + "https://github.com/51Degrees/51degrees-python/tree/main/" + "ip-intelligence") + +# The settings that let an example get past its early end. +SETTINGS = [ + "_51DEGREES_RESOURCE_KEY", "resource_key", "RESOURCE_KEY", + "_51DEGREES_LICENSE_KEY", "license_key", + ExampleUtils.IPI_PATH_ENV_VAR, ExampleUtils.DATA_FILE_ENV_VAR] + + +def slug(path): + """The utm_content of a link in the file, which is the path below the + product folder with each separator a hyphen.""" + return os.path.relpath(path, PRODUCT).replace(os.sep, "-") + + +def printed_by(run): + """Runs an example with the settings taken out of the environment, and + returns what it printed.""" + printed = io.StringIO() + with mock.patch.dict(os.environ), redirect_stdout(printed): + for name in SETTINGS: + os.environ.pop(name, None) + run() + return printed.getvalue() + + +class FindOutMoreSourceTests(unittest.TestCase): + + def test_every_example_has_a_find_out_more_section(self): + for path in SOURCES: + with self.subTest(example=slug(path)): + with open(path, encoding="utf-8") as source: + # A page separates the parameters with "&". + text = source.read().replace("&", "&") + self.assertRegex(text, SECTION_NAME) + links = SECTION_LINK.findall(text) + self.assertTrue( + links, "The section has no tagged 51degrees.com link") + for link in links: + self.assertIn(f"&utm_content={slug(path)}&", link) + self.assertIn(REPOSITORY, text) + + +class FindOutMoreAtEarlyEndsTests(unittest.TestCase): + + def test_cloud_examples_without_a_resource_key(self): + examples = { + "getallproperties_console": getallproperties_console.main, + "gettingstarted_console": cloud_gettingstarted_console.main, + "metadata_console": cloud_metadata_console.main, + "mixed_console": cloud_mixed_console.main, + } + for name, main in examples.items(): + with self.subTest(example=name): + self.assertIn("Find out more:", printed_by(lambda: main([]))) + + def test_onpremise_examples_without_a_data_file(self): + examples = { + "compare_console": compare_console.main, + "countries_console": countries_console.main, + "datafileupdate_console": datafileupdate_console.main, + "gettingstarted_console": gettingstarted_console.main, + "metadata_console": metadata_console.main, + "mixed_console": mixed_console.main, + "offlineprocessing": offlineprocessing.main, + "performance_console": performance_console.main, + } + for name, main in examples.items(): + with self.subTest(example=name): + with mock.patch.object( + ExampleUtils, "find_file", return_value=None): + printed = printed_by(lambda: main([])) + self.assertIn("Find out more:", printed) + + +if __name__ == "__main__": + unittest.main() diff --git a/location/tests/test_find_out_more.py b/location/tests/test_find_out_more.py new file mode 100644 index 00000000..a8b1ef44 --- /dev/null +++ b/location/tests/test_find_out_more.py @@ -0,0 +1,99 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every example ends with a "Find out more" section, which gives the person +running it links to follow up, wherever the example ends. The first check +reads the examples' source. The other runs each example with no resource +key, so none of them needs a key or the network. +""" + +import io +import os +import re +import runpy +import unittest +from contextlib import redirect_stdout +from unittest import mock + +# The product folder. Each link's utm_content is the path of its file below +# this folder. +PRODUCT = os.path.abspath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..")) +EXAMPLES = os.path.join(PRODUCT, "examples", "cloud") + +# Every example. The web example has one section for the console and the +# same links at the end of its page. +SOURCES = [ + os.path.join(EXAMPLES, "gettingstarted.py"), + os.path.join(EXAMPLES, "web.py"), +] + +# The name of the section, or of the list that holds its links. +SECTION_NAME = re.compile(r"(?i)find[ _]out[ _]more") + +# A 51degrees.com link in the section, which carries a utm_term that starts +# with "find-out-more". +SECTION_LINK = re.compile( + r"https://51degrees\.com/[^\s\"'<>)]*utm_term=find-out-more[^\s\"'<>)]*") + +# This product's folder in the repository, which every section links to. +REPOSITORY = "https://github.com/51Degrees/51degrees-python/tree/main/location" + +# The settings that let an example get past its early end. +SETTINGS = ["_51DEGREES_RESOURCE_KEY", "resource_key", "RESOURCE_KEY"] + + +def slug(path): + """The utm_content of a link in the file, which is the path below the + product folder with each separator a hyphen.""" + return os.path.relpath(path, PRODUCT).replace(os.sep, "-") + + +class FindOutMoreTests(unittest.TestCase): + + def test_every_example_has_a_find_out_more_section(self): + for path in SOURCES: + with self.subTest(example=slug(path)): + with open(path, encoding="utf-8") as source: + text = source.read() + self.assertRegex(text, SECTION_NAME) + links = SECTION_LINK.findall(text) + self.assertTrue( + links, "The section has no tagged 51degrees.com link") + for link in links: + self.assertIn(f"&utm_content={slug(path)}&", link) + self.assertIn(REPOSITORY, text) + + def test_every_example_without_a_resource_key(self): + for path in SOURCES: + with self.subTest(example=slug(path)): + printed = io.StringIO() + with mock.patch.dict(os.environ), redirect_stdout(printed): + for name in SETTINGS: + os.environ.pop(name, None) + runpy.run_path(path, run_name="__main__") + self.assertIn("Find out more:", printed.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_find_out_more.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_find_out_more.py new file mode 100644 index 00000000..34195956 --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_find_out_more.py @@ -0,0 +1,63 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +The mixed console ends with its "Find out more" section when it stops early, +as it does after a run. These run it to each early end, so they need no key +and no network. +""" + +from fiftyone_pipeline_examples.cloud.mixed import gettingstarted_console +from fiftyone_pipeline_examples.example_utils import ExampleUtils + +# Every name a key is read from. +KEY_SETTINGS = [ + ExampleUtils.RESOURCE_KEY_ENV_VAR, + ExampleUtils.LEGACY_RESOURCE_KEY_ENV_VAR, + ExampleUtils.LICENSE_KEY_ENV_VAR, + ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR, +] + + +def test_without_a_key(monkeypatch, capsys): + for name in KEY_SETTINGS: + monkeypatch.delenv(name, raising=False) + + gettingstarted_console.main([]) + + printed = capsys.readouterr().out + assert "Find out more:" in printed + for _, url in gettingstarted_console.GettingStartedConsole.FIND_OUT_MORE: + assert url in printed + + +def test_when_the_pipeline_cannot_be_built(monkeypatch, capsys): + # Stands in for a key the cloud service refuses, or one that carries + # none of the properties the example shows. + def refuse(*args, **kwargs): + raise ValueError("The key carries none of the properties.") + + monkeypatch.setattr(gettingstarted_console, "create_pipeline", refuse) + + gettingstarted_console.main(["a-resource-key"]) + + assert "Find out more:" in capsys.readouterr().out diff --git a/pipeline/fiftyone_pipeline_core/tests/test_find_out_more.py b/pipeline/fiftyone_pipeline_core/tests/test_find_out_more.py new file mode 100644 index 00000000..919888fe --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_find_out_more.py @@ -0,0 +1,71 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every example ends with a "Find out more" section, which gives the person +running it links to follow up. The check reads the example's source, so it +needs no network. +""" + +import os +import re +import unittest + +# The product folder. Each link's utm_content is the path of its file below +# this folder. +PRODUCT = os.path.abspath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "..")) + +# The example of a custom flow element, a web page that ends with the +# section. +EXAMPLE = os.path.join( + PRODUCT, "fiftyone_pipeline_core", "examples", + "client_side_evidence_custom_flow_element.py") + +# The name of the section, or of the list that holds its links. +SECTION_NAME = re.compile(r"(?i)find[ _]out[ _]more") + +# A 51degrees.com link in the section, which carries a utm_term that starts +# with "find-out-more". +SECTION_LINK = re.compile( + r"https://51degrees\.com/[^\s\"'<>)]*utm_term=find-out-more[^\s\"'<>)]*") + +# This product's folder in the repository, which every section links to. +REPOSITORY = "https://github.com/51Degrees/51degrees-python/tree/main/pipeline" + + +class FindOutMoreTests(unittest.TestCase): + + def test_the_example_has_a_find_out_more_section(self): + with open(EXAMPLE, encoding="utf-8") as source: + text = source.read() + self.assertRegex(text, SECTION_NAME) + links = SECTION_LINK.findall(text) + self.assertTrue(links, "The section has no tagged 51degrees.com link") + content = os.path.relpath(EXAMPLE, PRODUCT).replace(os.sep, "-") + for link in links: + self.assertIn(f"&utm_content={content}&", link) + self.assertIn(REPOSITORY, text) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_did/tests/test_find_out_more.py b/pipeline/fiftyone_pipeline_did/tests/test_find_out_more.py new file mode 100644 index 00000000..8574d07a --- /dev/null +++ b/pipeline/fiftyone_pipeline_did/tests/test_find_out_more.py @@ -0,0 +1,77 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +The 51Did example ends with a "Find out more" section, which gives the +person running it links to follow up. The example works offline, so it is +run as well as read. +""" + +import io +import os +import re +import runpy +import unittest +from contextlib import redirect_stdout + +# The product folder. Each link's utm_content is the path of its file below +# this folder. +PRODUCT = os.path.abspath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "..")) + +EXAMPLE = os.path.join( + PRODUCT, "fiftyone_pipeline_did", "examples", "fodid_example.py") + +# The name of the section, or of the list that holds its links. +SECTION_NAME = re.compile(r"(?i)find[ _]out[ _]more") + +# A 51degrees.com link in the section, which carries a utm_term that starts +# with "find-out-more". +SECTION_LINK = re.compile( + r"https://51degrees\.com/[^\s\"'<>)]*utm_term=find-out-more[^\s\"'<>)]*") + +# This product's folder in the repository, which every section links to. +REPOSITORY = "https://github.com/51Degrees/51degrees-python/tree/main/pipeline" + + +class FindOutMoreTests(unittest.TestCase): + + def test_the_example_has_a_find_out_more_section(self): + with open(EXAMPLE, encoding="utf-8") as source: + text = source.read() + self.assertRegex(text, SECTION_NAME) + links = SECTION_LINK.findall(text) + self.assertTrue(links, "The section has no tagged 51degrees.com link") + content = os.path.relpath(EXAMPLE, PRODUCT).replace(os.sep, "-") + for link in links: + self.assertIn(f"&utm_content={content}&", link) + self.assertIn(REPOSITORY, text) + + def test_the_example_ends_with_the_section(self): + printed = io.StringIO() + with redirect_stdout(printed): + runpy.run_path(EXAMPLE, run_name="__main__") + self.assertIn("Find out more:", printed.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_find_out_more.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_find_out_more.py new file mode 100644 index 00000000..e2489ad1 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_find_out_more.py @@ -0,0 +1,71 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every example ends with a "Find out more" section, which gives the person +running it links to follow up. The check reads the example's source rather +than running it, because the example builds a pipeline that shares usage +with 51Degrees. +""" + +import os +import re +import unittest + +# The product folder. Each link's utm_content is the path of its file below +# this folder. +PRODUCT = os.path.abspath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "..")) + +# The usage sharing example, a script that ends with the section. +EXAMPLE = os.path.join( + PRODUCT, "fiftyone_pipeline_engines_fiftyone", "examples", + "usagesharing", "usagesharing.py") + +# The name of the section, or of the list that holds its links. +SECTION_NAME = re.compile(r"(?i)find[ _]out[ _]more") + +# A 51degrees.com link in the section, which carries a utm_term that starts +# with "find-out-more". +SECTION_LINK = re.compile( + r"https://51degrees\.com/[^\s\"'<>)]*utm_term=find-out-more[^\s\"'<>)]*") + +# This product's folder in the repository, which every section links to. +REPOSITORY = "https://github.com/51Degrees/51degrees-python/tree/main/pipeline" + + +class FindOutMoreTests(unittest.TestCase): + + def test_the_example_has_a_find_out_more_section(self): + with open(EXAMPLE, encoding="utf-8") as source: + text = source.read() + self.assertRegex(text, SECTION_NAME) + links = SECTION_LINK.findall(text) + self.assertTrue(links, "The section has no tagged 51degrees.com link") + content = os.path.relpath(EXAMPLE, PRODUCT).replace(os.sep, "-") + for link in links: + self.assertIn(f"&utm_content={content}&", link) + self.assertIn(REPOSITORY, text) + + +if __name__ == "__main__": + unittest.main() From 63566d06c9b72f613ed2a23c3139f7da5e94a8bb Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 00:12:45 +0000 Subject: [PATCH 014/136] Share a key found in the request and the Evidence once, whatever its letters (#120) `ShareUsage.found_evidence` merges the values of the web request, which the web integration adds as `server.http-request`, with every Evidence value in the Flow Data, so that a key found in both places is shared once with the Evidence value ([`share_usage.py` lines 432 to 458 at 32ffdcf1](https://github.com/51Degrees/51degrees-python/blob/32ffdcf1205a5ce593fb55f5be733de5a18dffde/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py#L432-L458)). It merged on the exact key. `webevidence` writes each header's key in lower case ([`web.py` line 91](https://github.com/51Degrees/51degrees-python/blob/32ffdcf1205a5ce593fb55f5be733de5a18dffde/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py#L91)), so a header an application added by hand in other letters, such as `header.User-Agent`, was found twice and both values were shared. --- .../share_usage.py | 16 ++++-- .../test_share_usage_accepts_no_evidence.py | 55 +++++++++++++++++++ 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 943674b8..3a91a980 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -435,9 +435,10 @@ def found_evidence(self, flow_data): The evidence the element finds to share. These are the values of the web request, when the web integration added it as server.http-request, and every evidence value in the flow data, - whether an element accepted it or not. A key found in both places - takes the evidence value. The web request itself is never one of - them. + whether an element accepted it or not. Keys are compared without + regard to case, so a key found in more than one place is found + once, with the evidence value and the evidence's spelling. The web + request itself is never one of them. @type flow_data: FlowData @param flow_data: The flow data being processed @@ -450,12 +451,15 @@ def found_evidence(self, flow_data): unaccepted = getattr(evidence, "get_unaccepted", dict)() web_request = unaccepted.get(HTTP_REQUEST_KEY) or \ evidence.get(HTTP_REQUEST_KEY) or {} + # The key of each value found, by its lower case form. A later + # source replaces both the value and the spelling of an earlier one. found = {} for source in (web_request, unaccepted, evidence.get_all()): for key, value in source.items(): - if str(key).lower() != HTTP_REQUEST_KEY: - found[key] = value - return found + folded = str(key).lower() + if folded != HTTP_REQUEST_KEY: + found[folded] = (key, value) + return dict(found.values()) def share_send_usage(self): """! diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py index e683f856..503abff0 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py @@ -25,8 +25,13 @@ depend on it. It shares what it finds in the flow data and in the web request instead, by the same rules as before.""" +import gzip +import re import unittest +from types import MappingProxyType +from unittest import mock +import requests from flask import Flask, request from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ @@ -82,6 +87,22 @@ def request_evidence(headers, path="/"): return webevidence(request) +def sent_on_close(usage): + """Closes the usage sharing element, which sends what it has queued, + and returns the text of each message it posted. The post is answered + with a 200 without leaving the process.""" + + sent = [] + + def post(url, headers=None, data=None, **kwargs): + sent.append(gzip.decompress(data).decode("utf8")) + return mock.Mock(status_code=200, text="") + + with mock.patch.object(requests, "post", post): + usage.close() + return sent + + class ShareUsageAcceptsNoEvidenceTests(unittest.TestCase): def test_the_pipeline_accepts_no_header_for_usage_sharing(self): @@ -171,6 +192,40 @@ def test_a_key_in_both_places_is_shared_once_with_the_evidence_value(self): self.assertIn( '
from the evidence
', shared) + def test_a_header_no_element_reads_is_in_the_message_sent(self): + usage = usage_element() + flowdata = pipeline_with(usage).create_flowdata() + flowdata.evidence.add_from_dict(request_evidence({ + "User-Agent": CHROME, + "X-Test": "only usage sharing reads this", + })) + flowdata.process() + + sent = sent_on_close(usage) + self.assertEqual(1, len(sent)) + self.assertIn( + '
only usage sharing reads this
', + sent[0]) + self.assertNotIn("http-request", sent[0]) + + def test_a_key_in_both_places_whose_letters_differ_is_shared_once(self): + usage = usage_element() + flowdata = pipeline_with(usage).create_flowdata() + # Added before the request, so the evidence keeps this spelling and + # the request carries the other one. + flowdata.evidence.add("HEADER.User-Agent", "evidence agent") + flowdata.evidence.add("server.http-request", MappingProxyType({ + "header.user-agent": "request agent"})) + flowdata.process() + + sent = sent_on_close(usage) + self.assertEqual(1, len(sent)) + names = re.findall( + r'
Date: Sat, 14 Nov 2026 00:26:25 +0000 Subject: [PATCH 015/136] Say in AGENTS.md that the IP Intelligence getting started example uses the Lite file (#124) [AGENTS.md](https://github.com/51Degrees/51degrees-python/blob/175ca4a/AGENTS.md) said the IP Intelligence on-premise `gettingstarted_console` "with no argument looks for the Asn file", shows registration values only once given the Lite file, and prints "This property is not present in the data file." for a property the file lacks. None of that is what the example does on `rc` at 175ca4a. --- AGENTS.md | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 505c110a..7530c8a4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,10 +10,10 @@ layout and how to get the code are in the [README](README.md). Device Detection's on-premise `gettingstarted_console` is the quickest example to show, because its data file is about 35 MB. IP Intelligence's -on-premise `gettingstarted_console` runs on the free Asn data file, which is -about 6 MB, and shows registration values once it is given the Lite file, -which is a download of about 230 MB. Run each command from the repository root -unless the step says otherwise. +on-premise `gettingstarted_console` runs on the free Lite data file, which is a +download of about 230 MB and has the registration values. The free Asn data +file, about 6 MB, has only the network properties. Run each command from the +repository root unless the step says otherwise. The on-premise engines are C and C++ code that is compiled when the package is installed. PyPI has only source distributions of @@ -150,16 +150,19 @@ macOS (`find_file` in python -m fiftyone_devicedetection_examples.onpremise.gettingstarted_console ``` -The IP Intelligence example with no argument looks for the Asn file in the -folder it runs in and in its `assets` and `cxx/data/ip-intelligence` folders, -then does the same in each of the seven folders above it, and downloads the -file into `cxx/data/ip-intelligence` when it is missing. To see registration -values, download the Lite file first, as "Data files" says, and give its path -as the first argument. +The IP Intelligence example with no argument looks for the Lite file, +`51Degrees-LiteV41.ipi`, in the folder it runs in and in its `assets`, +`cxx/data/ip-intelligence` and `cxx/data/device-detection` folders, then does +the same in each of the seven folders above it. When none of them has the +file, it downloads the file into the `cxx/data/ip-intelligence` folder that +holds the file's `.link` (`find_file` in +`ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py`). +To run it on the smaller Asn file, download that file first, as "Data files" +says, and give its path as the first argument. ``` python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_console -python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_console cxx/data/ip-intelligence/51Degrees-LiteV41.ipi +python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_console cxx/data/ip-intelligence/51Degrees-IPIV4AsnIpiV41.ipi ``` Either example uses a path given as the first argument as it is, so a file @@ -187,11 +190,12 @@ The IP Intelligence example logs `Using a '' data file created ''`, then for each of four IP addresses, starting with `116.154.188.222`, prints `Input values:` with `query.client-ip`, then `Results:` with a line for each of 16 properties, starting with `RegisteredName`, `RegisteredOwner` and -`RegisteredCountry`. A property the data file lacks reads -`This property is not present in the data file.` With the Asn file that is -every registration and location property, as the examples' README says. The -Lite file has RegisteredName, RegisteredOwner and RegisteredCountry, so with it -those three lines are read from the file. +`RegisteredCountry`. The Lite file has RegisteredName, RegisteredOwner and +RegisteredCountry, so those three lines are read from the file. Each property +the data file lacks reads `needs the Enterprise data file`, and after the last +address the example prints `Some of these properties need the Enterprise data +file. Contact us for a copy at` followed by a link. With the Asn file the +registration properties need the Enterprise data file as well. The paid data files have many more properties, as the [pricing page](https://51degrees.com/pricing?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=correct-output) From 15f6f1bdb3defa7283b2bdfd6d266753b8aa9a53 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 03:51:16 +0000 Subject: [PATCH 016/136] Link to where resource keys are created when the 51Did demo has none The creator context demo's exit message with no resource key named the environment variable only. It now gives a tagged link to the configuration page, where a resource key is created. --- .../examples/creator_context_web/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py index a8b26698..9ead9d2c 100644 --- a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py +++ b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py @@ -296,7 +296,8 @@ def _is_json(text): def run(): if not RESOURCE: sys.exit("Set _51DEGREES_RESOURCE_KEY to the resource key of the " - "page.") + "page. Resource keys are created at " + "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did-examples-creator_context_web-server.py&utm_term=resource-key-required") if not LICENCE: # Only an account that holds licence keys needs one to redeem, # because the licence key is what keeps redemption to the acting From 7277528c62b3fd799e56997cda52e8dc02b48223 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 03:51:17 +0000 Subject: [PATCH 017/136] Say how to get a resource key, and start nothing, when the IP Intelligence cloud web example has none The IP Intelligence cloud web example logged nothing and raised an exception with no key. It now logs the message the cloud console examples log, with tagged links to the resource keys page and the configuration page, and returns. --- .../cloud/gettingstarted_web/app.py | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py index c7100800..248d6dc0 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py @@ -37,6 +37,7 @@ from flask import Flask, request +from fiftyone_pipeline_core.logger import Logger from fiftyone_pipeline_core.web import webevidence from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder from fiftyone_ipintelligence_examples.example_utils import ExampleUtils @@ -93,11 +94,26 @@ def index(): def main(resource_key = None): if not resource_key: resource_key = ExampleUtils.get_resource_key() + + # Configure a logger to output to the console. + logger = Logger(min_level="info") + + # With no key there is nothing to serve, so say how to get one and + # start no server. if not resource_key: - raise Exception( + logger.log("error", "No resource key specified in the environment variable " - f"'{ExampleUtils.CLOUD_RESOURCE_KEY_ENV_VAR}'. Obtain one from " - "https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required") + f"'{ExampleUtils.CLOUD_RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " + "service is accessed using a 'ResourceKey'. For more " + "information see " + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required. " + "A resource key with the properties required by this example " + "can be created for free at " + "https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required. " + "Once complete, populate the environment variable mentioned at " + "the start of this message with the key.") + return + build_pipeline(resource_key) app.run(debug=False) From d57fa8ddc0010f9ad4586549d563a72b06fec512 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 03:51:59 +0000 Subject: [PATCH 018/136] End the custom flow element, usage sharing and 51Did examples with a Find out more section The custom flow element example ends its page with the Build your own page, the Pipeline API documentation and this repository's pipeline folder. The usage sharing example prints the usage sharing documentation, the Pipeline API documentation and the pipeline folder last, and its expected output comment says the section follows. The 51Did example prints the 51Did page, its documentation, the 51Did inspector, the pipeline folder and the specifications repository after a successful run. --- ...lient_side_evidence_custom_flow_element.py | 29 +++++++++++++++++++ .../examples/fodid_example.py | 19 ++++++++++++ .../examples/usagesharing/usagesharing.py | 17 ++++++++++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py b/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py index e25c5723..d284ab95 100644 --- a/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py +++ b/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py @@ -30,12 +30,38 @@ # This latitude is also then passed in to the flowdata to calculate if # a person is in the northern or southern hemispheres. +import html + from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_pipeline_core.web import json_response, webevidence from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter from fiftyone_pipeline_core.flowelement import FlowElement from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +# Where to go next, shown at the end of the page. +FIND_OUT_MORE = [ + ("Build your own", + "https://51degrees.com/build-your-own?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core-examples-client_side_evidence_custom_flow_element.py&utm_term=find-out-more-build-your-own"), + ("Pipeline API documentation", + "https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core-examples-client_side_evidence_custom_flow_element.py&utm_term=find-out-more-documentation"), + ("Pipeline API for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/pipeline"), +] + + +def find_out_more_html(): + """The Find out more section for the end of the page. A repository is + shown by its address, and any other link by its label.""" + items = "" + for label, url in FIND_OUT_MORE: + href = html.escape(url) + label = html.escape(label) + if url.startswith("https://github.com/"): + items += f"
  • {label}: {href}
  • " + else: + items += f"
  • {label}
  • " + return f"

    Find out more

      {items}
    " + # Function to get star sign from month and day def getStarSign(month, day): @@ -247,6 +273,9 @@ def server(): output += "" + # Where to go next. + output += find_out_more_html() + # Return the full output to the page return output diff --git a/pipeline/fiftyone_pipeline_did/examples/fodid_example.py b/pipeline/fiftyone_pipeline_did/examples/fodid_example.py index 18e16f37..a52b3c93 100644 --- a/pipeline/fiftyone_pipeline_did/examples/fodid_example.py +++ b/pipeline/fiftyone_pipeline_did/examples/fodid_example.py @@ -52,6 +52,20 @@ SAMPLE_TERMS = 1 # index 1, the Model Terms for Marketing # version 2 +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("51Did", + "https://51degrees.com/developers/51did?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did-examples-fodid_example.py&utm_term=find-out-more-51did"), + ("51Did documentation", + "https://51degrees.com/documentation/_identifiers_51_did.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did-examples-fodid_example.py&utm_term=find-out-more-documentation"), + ("51Did inspector", + "https://51degrees.com/developers/51did-inspector?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did-examples-fodid_example.py&utm_term=find-out-more-51did-inspector"), + ("Pipeline API for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/pipeline"), + ("Specifications", + "https://github.com/51Degrees/specifications"), +] + def sample_payload(): """A canonical 38-byte Probabilistic payload: the flags byte, License @@ -109,6 +123,11 @@ def run(): "Expected a different envelope but the same match key across " "reissues.") + print("=" * 79) + print("Find out more:") + for label, url in FIND_OUT_MORE: + print(f"\t{label}: {url}") + if __name__ == "__main__": run() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py index ade418a8..375f3352 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py @@ -44,7 +44,7 @@ # # This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py). # -# Expected output: +# Expected output, before the Find out more section: # ``` # Constructing pipeline from configuration file. # @@ -58,6 +58,16 @@ from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Usage sharing", + "https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone-examples-usagesharing-usagesharing.py&utm_term=find-out-more-usage-sharing"), + ("Pipeline API documentation", + "https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone-examples-usagesharing-usagesharing.py&utm_term=find-out-more-documentation"), + ("Pipeline API for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/pipeline"), +] + print("Constructing pipeline from configuration file.") print() @@ -78,3 +88,8 @@ print("""Pipeline created with share usage element. Evidence processed with this pipeline will now be periodically shared with 51Degrees using the specified configuration.""") + +print("=" * 79) +print("Find out more:") +for label, url in FIND_OUT_MORE: + print(f"\t{label}: {url}") From 0559392a5c82adf76fae585d1da36df0ff98de74 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 03:52:00 +0000 Subject: [PATCH 019/136] End the mixed cloud console with a Find out more section at every end The mixed cloud console printed its section only after a run. Writing the section moves into GettingStartedConsole.print_find_out_more, which run still calls, and main now calls it too when there is no key and when the pipeline cannot be built, so each of the example's ends finishes with it. --- .../cloud/mixed/gettingstarted_console.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py index 2f4fcf70..91b9c644 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py @@ -159,7 +159,7 @@ class GettingStartedConsole(): ("Time Zone Offset", "timezoneoffset"), ] - # Where to go next, printed after the results. + # Where to go next, printed when the example ends. FIND_OUT_MORE = [ ("Device detection", "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_console.py&utm_term=find-out-more-device-detection"), @@ -199,8 +199,19 @@ def run(self, pipeline, output): for evidence in self.EVIDENCE: self.analyse_evidence(evidence, pipeline, output) + self.print_find_out_more(output) + + @classmethod + def print_find_out_more(cls, output): + + """! + Write the "Find out more" section, a line of "=" and then each label + with its link. The example ends with it, whether it ran or stopped + early. + """ + message = ["=" * 79, "Find out more:"] - for label, url in self.FIND_OUT_MORE: + for label, url in cls.FIND_OUT_MORE: message.append(f"\t{label}: {url}") output("\n".join(message) + "\n") @@ -299,6 +310,7 @@ def main(argv): "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_console.py&utm_term=resource-key-required. " "Once complete, set the first environment variable named at " "the start of this message to the key.") + GettingStartedConsole.print_find_out_more(print) return try: @@ -311,6 +323,7 @@ def main(argv): # refused it, for example because the key is limited to domains # that do not include the origin sent. logger.log("error", str(error)) + GettingStartedConsole.print_find_out_more(print) return GettingStartedConsole().run(pipeline, print) From 00621e5ab204a8bada713cb221734fa2f1ff0b4c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 14 Nov 2026 00:14:43 +0000 Subject: [PATCH 020/136] End every Device Detection example with a Find out more section Each example now lists its own links, the Device Detection page, the resource keys documentation and pricing for the cloud examples, the Device Detection page and where to ask about the Enterprise data file for the on-premise ones, then this repository's device-detection folder, and 51degrees-cxx where the example uses the native engine. ExampleUtils gains print_find_out_more, which writes the section in the form the mixed cloud console uses, a line of "=" then "Find out more:" then each label and link on a tab-indented line. The section is printed at every clean end. A cloud console prints it after its results, and from main when there is no resource key. An on-premise console prints it from main, so it follows the results or the message that no data file was found. The data file update example prints it whether it updated the file, had no licence key or was stopped before downloading. The CSV and profile metadata examples write it where their progress messages go, so JSON on standard output stays valid. The offline processing example prints it to the console, because its results go to a file. The web examples end their pages with the section and print it to the console when the server stops, and the cloud ones also when there is no resource key. The scripts, failuretomatch, performance and the getting started example in fiftyone_devicedetection, print it last. --- .../examples/gettingstarted.py | 19 +++++++++++++++++++ .../cloud/configurator_console.py | 15 +++++++++++++++ .../cloud/failuretomatch.py | 14 ++++++++++++++ .../cloud/gettingstarted_console.py | 15 +++++++++++++++ .../cloud/gettingstarted_web/app.py | 16 ++++++++++++++++ .../gettingstarted_web/templates/index.html | 8 ++++++++ .../cloud/metadata_console.py | 15 +++++++++++++++ .../cloud/nativemodellookup_console.py | 15 +++++++++++++++ .../cloud/taclookup_console.py | 15 +++++++++++++++ .../cloud/useragentclienthints_web/app.py | 16 ++++++++++++++++ .../templates/index.html | 8 ++++++++ .../example_utils.py | 12 ++++++++++++ .../onpremise/csv_profile_metadata.py | 16 ++++++++++++++++ .../onpremise/datafileupdate_console.py | 16 ++++++++++++++++ .../onpremise/failuretomatch.py | 14 ++++++++++++++ .../onpremise/gettingstarted_console.py | 14 ++++++++++++++ .../onpremise/gettingstarted_web/app.py | 15 +++++++++++++++ .../gettingstarted_web/templates/index.html | 8 ++++++++ .../onpremise/match_metrics.py | 14 ++++++++++++++ .../onpremise/metadata_console.py | 14 ++++++++++++++ .../onpremise/offlineprocessing.py | 16 ++++++++++++++++ .../onpremise/performance.py | 14 ++++++++++++++ .../onpremise/profile_metadata_console.py | 16 ++++++++++++++++ .../onpremise/shareusage.py | 14 ++++++++++++++ .../onpremise/useragentclienthints_web/app.py | 15 +++++++++++++++ .../templates/index.html | 8 ++++++++ 26 files changed, 362 insertions(+) diff --git a/device-detection/fiftyone_devicedetection/examples/gettingstarted.py b/device-detection/fiftyone_devicedetection/examples/gettingstarted.py index b439c3d3..5cdf9325 100644 --- a/device-detection/fiftyone_devicedetection/examples/gettingstarted.py +++ b/device-detection/fiftyone_devicedetection/examples/gettingstarted.py @@ -22,6 +22,20 @@ from fiftyone_devicedetection.devicedetection_pipelinebuilder import DeviceDetectionPipelineBuilder +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection-examples-gettingstarted.py&utm_term=find-out-more-device-detection"), + ("Enterprise data file", + "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection-examples-gettingstarted.py&utm_term=find-out-more-enterprise-data-file"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection-examples-gettingstarted.py&utm_term=find-out-more-resource-keys"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), + ("Device detection engine", + "https://github.com/51Degrees/51degrees-cxx"), +] + # First create the device detection pipeline with the desired settings. data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" @@ -81,3 +95,8 @@ else: # Output why the value isn't meaningful print(flowdata2.device.ismobile.no_value_message()) + +print("=" * 79) +print("Find out more:") +for label, url in FIND_OUT_MORE: + print(f"\t{label}: {url}") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py index 6cd8ac4c..0ef8d9b8 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py @@ -25,6 +25,18 @@ from fiftyone_devicedetection_cloud.devicedetection_cloud_pipelinebuilder import DeviceDetectionCloudPipelineBuilder from fiftyone_devicedetection_examples.example_utils import ExampleUtils +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + # This example is displayed at the end of the [configuration page](https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=top) # process, which is used to create resource keys for use with the 51Degrees cloud service. # @@ -79,6 +91,8 @@ def run(self, resource_key, logger, output): output("device.ismobile: " + f"{ExampleUtils.get_human_readable(device, 'ismobile')}") + ExampleUtils.print_find_out_more(FIND_OUT_MORE, output) + def main(argv): # Use the command line args to get the resource key if present. # Otherwise, get it from the environment variable. @@ -102,6 +116,7 @@ def main(argv): "https://51degrees.com/configure/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=resource-key-required. " + "Once complete, populate the environment variable " + "mentioned at the start of this message with the key.") + ExampleUtils.print_find_out_more(FIND_OUT_MORE) if __name__ == "__main__": main(sys.argv[1:]) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/failuretomatch.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/failuretomatch.py index c10939eb..ae926b41 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/failuretomatch.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/failuretomatch.py @@ -33,6 +33,18 @@ # it has no key. from fiftyone_devicedetection_examples.example_utils import ExampleUtils +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-failuretomatch.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-failuretomatch.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-failuretomatch.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + resource_key = ExampleUtils.get_resource_key() or "!!YOUR_RESOURCE_KEY!!" if resource_key == "!!YOUR_RESOURCE_KEY!!": @@ -90,3 +102,5 @@ else: # Output why the value isn't meaningful print(flowdata2.device.ismobile.no_value_message()) + +ExampleUtils.print_find_out_more(FIND_OUT_MORE) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py index f763cfcf..7d525358 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py @@ -45,6 +45,18 @@ from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_devicedetection_examples.example_utils import ExampleUtils +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + class GettingStartedConsole(): def run(self, config, logger, output): @@ -59,6 +71,8 @@ def run(self, config, logger, output): for values in self.EvidenceValues: self.analyseEvidence(values, pipeline, output) + ExampleUtils.print_find_out_more(FIND_OUT_MORE, output) + def analyseEvidence(self, evidence, pipeline, output): # FlowData is a data structure that is used to convey @@ -179,6 +193,7 @@ def main(argv): "created for free at https://51degrees.com/configure/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + "Once complete, populate the config file or environment variable " + "mentioned at the start of this message with the key.") + ExampleUtils.print_find_out_more(FIND_OUT_MORE) else: GettingStartedConsole().run(config, logger, print) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py index 94a2855d..7d9b1fd8 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py @@ -84,6 +84,19 @@ import os import sys +# Where to go next, printed when the example ends. The page ends with the +# same links, in templates/index.html. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + class GettingStartedWeb(): app = Flask(__name__) @@ -208,5 +221,8 @@ def main(argv): "Once complete, populated the environment variable " + "mentioned at the start of this message with the key.") + # Printed once the server stops, or straight away when there is no key. + ExampleUtils.print_find_out_more(FIND_OUT_MORE) + if __name__ == "__main__": main(sys.argv[1:]) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/templates/index.html b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/templates/index.html index f9ff4bd8..0d055c53 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/templates/index.html +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/templates/index.html @@ -153,6 +153,14 @@

    Client-side evidence and Apple models

    Want to try on-premise? Contact us to discuss requirements.

    Contact us + +

    Find out more

    + {# diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py index 7fc8ba08..b628ac2d 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py @@ -53,6 +53,18 @@ from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter from fiftyone_devicedetection_examples.example_utils import ExampleUtils +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + bgRed = "\u001b[41m" fgWhite = "\u001b[37;1m" colReset = "\u001b[0m" @@ -73,6 +85,8 @@ def run(self, resource_key, logger, output): # them to the cloud, so that's the engine we want the keys from. self.outputEvidenceKeyDetails(pipeline.get_element("cloud"), output) + ExampleUtils.print_find_out_more(FIND_OUT_MORE, output) + def outputEvidenceKeyDetails(self, engine, output): if (issubclass(type(engine.get_evidence_key_filter()), BasicListEvidenceKeyFilter)): @@ -126,6 +140,7 @@ def main(argv): "https://51degrees.com/configure/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=resource-key-required. " + "Once complete, populate the environment variable " + "mentioned at the start of this message with the key.") + ExampleUtils.print_find_out_more(FIND_OUT_MORE) if __name__ == "__main__": main(sys.argv[1:]) \ No newline at end of file diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py index 868d0665..4b0b28b5 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py @@ -45,6 +45,18 @@ from fiftyone_devicedetection_shared.constants import Constants from fiftyone_devicedetection_cloud.hardwareprofile_cloud import HardwareProfileCloud +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + class NativeModelLookupConsole(): def run(self, resource_key, logger, output, cloudEndPoint = ""): @@ -82,6 +94,8 @@ def run(self, resource_key, logger, output, cloudEndPoint = ""): # Repeat for an alternative native model name. self.analyseModel(self._nativeModel2, pipeline, output) + ExampleUtils.print_find_out_more(FIND_OUT_MORE, output) + def analyseModel(self, nativemodel, pipeline, output): # Create the FlowData instance. data = pipeline.create_flowdata() @@ -136,6 +150,7 @@ def main(argv): "https://51degrees.com/configure/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=resource-key-required. You can now populate the " + "environment variable mentioned at the start of this message with the " + "resource key or pass it as the first argument on the command line.") + ExampleUtils.print_find_out_more(FIND_OUT_MORE) if __name__ == "__main__": main(sys.argv[1:]) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py index 0d07605e..dca57cc6 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py @@ -48,6 +48,18 @@ from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_devicedetection_shared.constants import Constants +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + class TacLookupConsole(): def run(self, config, logger, output): @@ -72,6 +84,8 @@ def run(self, config, logger, output): # Repeat for an alternative TAC. self.analyseTac(self._tac2, pipeline, output) + ExampleUtils.print_find_out_more(FIND_OUT_MORE, output) + def analyseTac(self, tac, pipeline, output): # Create the FlowData instance. data = pipeline.create_flowdata() @@ -138,6 +152,7 @@ def main(argv): "can now populate the environment variable mentioned at the start " + "of this message with the resource key or pass it as the first " + "argument on the command line.") + ExampleUtils.print_find_out_more(FIND_OUT_MORE) else: TacLookupConsole().run(config, logger, print) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py index b47cce81..c534ffbd 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py @@ -52,6 +52,19 @@ from fiftyone_pipeline_core.logger import Logger from fiftyone_pipeline_core.web import webevidence, set_response_header +# Where to go next, printed when the example ends. The page ends with the +# same links, in templates/index.html. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + class UserAgentClientHintsWeb(): app = Flask(__name__) @@ -143,6 +156,9 @@ def main(argv): "Once complete, populated the environment variable " + "mentioned at the start of this message with the key.") + # Printed once the server stops, or straight away when there is no key. + ExampleUtils.print_find_out_more(FIND_OUT_MORE) + if __name__ == "__main__": main(sys.argv[1:]) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/templates/index.html b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/templates/index.html index b206b40c..1b84bc94 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/templates/index.html +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/templates/index.html @@ -101,6 +101,14 @@

    Detection results

    Want to try on-premise? Contact us to discuss requirements.

    Contact us
    + +

    Find out more

    + " diff --git a/location/readme.md b/location/readme.md index 931eaf16..36b90325 100644 --- a/location/readme.md +++ b/location/readme.md @@ -60,6 +60,12 @@ Execute `export FLASK_APP=web` where `web` is the example file, and start your a Execute `$env:FLASK_APP = "web"` where `web` is the example file, and start your application via `flask run`. +A Flow Data cannot be used by two threads at once, so the web example +creates a Flow Data for each request, uses it only in the code that handles +that request, and closes it before the request ends. To hand work to +another thread or task, read the values it needs from the Flow Data first +and pass those values, not the Flow Data. + ## Find out more - [Geolocation](https://51degrees.com/geolocation?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=find-out-more-geolocation) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md index 5dc64d97..77caac69 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md @@ -26,6 +26,12 @@ answer. Those engines are in `src/fiftyone_pipeline_examples/cloud/engines.py`, and `src/fiftyone_pipeline_examples/cloud/mixed/pipeline.py` puts them together. +A Flow Data cannot be used by two threads at once, so the web example +creates a Flow Data for each request, uses it only in the code that handles +that request, and closes it when the request ends. To hand work to another +thread or task, read the values it needs from the Flow Data first and pass +those values, not the Flow Data. + ### Keys The examples call the cloud service with a resource key or a licence key. diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py index ce10a5af..b2f1480e 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py @@ -73,6 +73,14 @@ # Required PyPI dependencies: # - [fiftyone_pipeline_cloudrequestengine](https://pypi.org/project/fiftyone-pipeline-cloudrequestengine/) # - [flask](https://pypi.org/project/flask/) +# +# ## Flow Data and threads +# +# A Flow Data cannot be used by two threads at once. This example creates one +# for each request, uses it only in the code that handles that request, and +# closes it when the request ends. To hand work to another thread or task, +# read the values it needs from the Flow Data first and pass those values, +# not the Flow Data. import os import sys @@ -232,35 +240,32 @@ def create_app(pipeline): # else can set the header to any address. app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1) - def process(): - - flowdata = pipeline.create_flowdata() - - # Add the evidence from the request, being its headers, cookies, - # query parameters and client address, along with any form fields - # the client-side script posts. - flowdata.evidence.add_from_dict(webevidence(request)) - - flowdata.process() - - return flowdata - @app.route("/") def index(): - flowdata = process() + # A Flow Data cannot be used by two threads at once, so this one is + # made for this request, used only here, and closed when the route + # returns. + with pipeline.create_flowdata() as flowdata: + + # Add the evidence from the request, being its headers, cookies, + # query parameters and client address, along with any form + # fields the client-side script posts. + flowdata.evidence.add_from_dict(webevidence(request)) + + flowdata.process() - response = make_response() + response = make_response() - # Some browsers only send User-Agent Client Hints that are asked for, - # so set the Accept-CH header the results call for. More about this - # at https://51degrees.com/blog/user-agent-client-hints?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_web-app.py&utm_term=index - set_response_header(flowdata, response) + # Some browsers only send User-Agent Client Hints that are asked + # for, so set the Accept-CH header the results call for. More + # about this at https://51degrees.com/blog/user-agent-client-hints?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_web-app.py&utm_term=index + set_response_header(flowdata, response) - response.set_data(render_template( - "index.html", **page_model(flowdata))) + response.set_data(render_template( + "index.html", **page_model(flowdata))) - return response + return response # The other Pipeline APIs ship web integrations that serve the # client-side script at '/51Degrees.core.js', so the page refers to it by diff --git a/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py b/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py index d284ab95..58e0987c 100644 --- a/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py +++ b/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py @@ -29,6 +29,14 @@ # which gets a user's geolocation and saves the latitude as a cookie. # This latitude is also then passed in to the flowdata to calculate if # a person is in the northern or southern hemispheres. +# +# ## Flow Data and threads +# +# A Flow Data cannot be used by two threads at once. This example creates one +# for each request, uses it only in the code that handles that request, and +# closes it before the request ends. To hand work to another thread or task, +# read the values it needs from the Flow Data first and pass those values, +# not the Flow Data. import html @@ -205,17 +213,25 @@ def jsonroute(): @app.route('/') def server(): - - flowdata = myPipeline.create_flowdata() - # Add any information from the request (headers, cookies and additional - # client side provided information) + # A Flow Data cannot be used by two threads at once, so this one is made + # for this request, used only here, and closed once the page's values + # have been read from it. + with myPipeline.create_flowdata() as flowdata: + + # Add any information from the request (headers, cookies and + # additional client side provided information) + + flowdata.evidence.add_from_dict(webevidence(request)) - flowdata.evidence.add_from_dict(webevidence(request)) + # Process the flowdata - # Process the flowdata + flowdata.process() - flowdata.process() + # The values the page shows + star_sign = flowdata.astrology.starSign + hemisphere = flowdata.astrology.hemisphere + javascript = flowdata.javascriptbuilder.javascript # Generate the HTML for the form that gets a user's starsign @@ -227,13 +243,13 @@ def server(): ## Add the results if they're available - if (flowdata.astrology.starSign): - output += "

    Your starsign is " + flowdata.astrology.starSign + "

    " + if (star_sign): + output += "

    Your starsign is " + star_sign + "

    " output += "
    " - if (flowdata.astrology.hemisphere): - output += "

    Look at the " + flowdata.astrology.hemisphere + " hemisphere stars tonight!

    " + if (hemisphere): + output += "

    Look at the " + hemisphere + " hemisphere stars tonight!

    " output += "
    " @@ -247,7 +263,7 @@ def server(): ## 3. The web server responds with new JSON data that contains the hemisphere based on the location. ## 4. The JavaScript integrates the new JSON data and fires the onChange callback below. - output += flowdata.javascriptbuilder.javascript + output += javascript output += """ const loadHemisphere = function() { From c6842b62dc78738b0d43bdcf3c5d67afba4a828f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 20:05:51 +0000 Subject: [PATCH 117/136] Keep usage sharing last in the pause tests The pause tests put an element after usage sharing to show that processing went on while sending was paused. Usage sharing must be the last Flow Element, so the element goes, and the tests check instead that each request processed during the pause returns with no errors, which still shows that processing is neither held up nor stopped by the back-off. These tests were added since the last release. --- .../tests/test_share_usage_pause.py | 45 +++++-------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pause.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pause.py index 53a742a9..26daee1e 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pause.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pause.py @@ -32,9 +32,6 @@ import time import unittest -from fiftyone_pipeline_core.elementdata_dictionary import \ - ElementDataDictionary -from fiftyone_pipeline_core.flowelement import FlowElement from fiftyone_pipeline_core.logger import Logger from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage @@ -111,22 +108,6 @@ def __call__(self, endpoint, data): return self.succeed.is_set() -class After(FlowElement): - """! - An element after usage sharing, which records that processing reached - it. - """ - - def __init__(self): - super(After, self).__init__() - self.datakey = "after" - self.properties = {"reached": {"type": "bool"}} - - def process_internal(self, flow_data): - flow_data.set_element_data( - ElementDataDictionary(self, {"reached": True})) - - class BlockedSend: """! Stands in for sending a message. The first send fails, which starts a @@ -160,19 +141,19 @@ def wait_until(self, condition, message): def process(self, pipeline, user_agent): """! - Process a request, returning whether processing reached the element - after usage sharing. + Process a request, returning the errors processing recorded. """ flow_data = pipeline.create_flowdata() flow_data.evidence.add("header.user-agent", user_agent) flow_data.process() - return flow_data.get("after").get("reached") + return dict(flow_data.errors) def test_processing_does_not_wait_for_the_pause(self): """! - Requests processed while sending is paused return, reach the - elements after usage sharing and are queued, and none of them waits - for a send. + Requests processed while sending is paused return with no errors + and are queued, and none of them waits for a send. Usage sharing is + the last element, so no element follows it to show that processing + went on. """ send = BlockedSend() @@ -186,8 +167,7 @@ def test_processing_does_not_wait_for_the_pause(self): self.addCleanup(usage.close) # Runs before closing, so closing never waits on a held send. self.addCleanup(send.release.set) - pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS) \ - .add(usage).add(After()).build() + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS).add(usage).build() # One failed send starts the pause. self.process(pipeline, "before the pause") @@ -195,11 +175,11 @@ def test_processing_does_not_wait_for_the_pause(self): lambda: usage.backoff.remaining(time.monotonic()) > 0, "The failed send did not pause sending.") - reached = [] + errors = [] def process_during_pause(): for i in range(REQUESTS_DURING_PAUSE): - reached.append(self.process(pipeline, f"during pause {i}")) + errors.append(self.process(pipeline, f"during pause {i}")) worker = threading.Thread(target=process_during_pause, daemon=True) worker.start() @@ -209,9 +189,8 @@ def process_during_pause(): worker.is_alive(), "Processing waited for the pause after failed sends.") self.assertEqual( - [True] * REQUESTS_DURING_PAUSE, reached, - "Processing during the pause did not reach the element after " - "usage sharing.") + [{}] * REQUESTS_DURING_PAUSE, errors, + "Processing during the pause recorded errors.") self.assertEqual( REQUESTS_DURING_PAUSE, len(usage.share_data), "The requests processed during the pause were not queued.") @@ -241,7 +220,7 @@ def test_pause_longer_pause_and_resume_are_logged(self): usage.send_thread = send self.addCleanup(usage.close) pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS) \ - .add_logger(recorder).add(usage).add(After()).build() + .add_logger(recorder).add(usage).build() def index_of(logged, matches): return next((index for index, (level, message) From f419d621258b1e42bb1cb3f246db3f844b212cd7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 20:05:54 +0000 Subject: [PATCH 118/136] Run usage sharing last and refuse a Pipeline that puts it elsewhere The usage sharing element shares the Evidence that every other Flow Element adds, so the specification requires it to be the last Flow Element and requires a builder to refuse a Pipeline that puts it anywhere else. The Device Detection and IP Intelligence builders added it before the engine. They now hold it and put it after every other element when they build, those the caller adds after making the builder included. The core builder keeps a usage sharing element that ends the caller's list after the JSON Bundler, JavaScript Builder and Set Headers elements it appends. The Pipeline constructor refuses a usage sharing element that is not last, before it registers any element, so a Pipeline built in code, from a configuration or directly is refused with the same error. The element marks itself with a class attribute, because pipeline core cannot import it. --- .../devicedetection_pipelinebuilder.py | 29 +++++++++++-- ...vicedetection_onpremise_pipelinebuilder.py | 26 ++++++++--- .../ipintelligence_pipelinebuilder.py | 29 +++++++++++-- ...pintelligence_onpremise_pipelinebuilder.py | 26 ++++++++--- .../src/fiftyone_pipeline_core/messages.py | 10 ++++- .../src/fiftyone_pipeline_core/pipeline.py | 43 ++++++++++++++++++- .../fiftyone_pipeline_core/pipelinebuilder.py | 19 ++++++-- .../share_usage.py | 5 +++ 8 files changed, 164 insertions(+), 23 deletions(-) diff --git a/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py b/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py index e227fdd2..e695c9c3 100644 --- a/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py +++ b/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py @@ -72,7 +72,8 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): On Premise Engine Settings @type usage_sharing : bool - @param usage_sharing : Whether to enable usage sharing if using the on premise engine + @param usage_sharing : Whether to enable usage sharing if using the + on premise engine. The usage sharing element runs last. @type data_file_path: string @param data_file_path: path to the data file @type auto_update: bool @@ -143,14 +144,17 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): # Add specific engines + # Usage sharing, when on with the on premise engine, runs after + # every other element, so build() moves it after any element the + # caller adds later. + self._usage_sharing = None + if "data_file_path" in settings: del settings["settings"] device = DeviceDetectionOnPremise(**settings) - - # If usage sharing enabled, add usage sharing engine if usage_sharing: - self.add(ShareUsage()) + self._usage_sharing = ShareUsage() else : self.add(CloudRequestEngine(settings)) @@ -161,3 +165,20 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): self.add(device) + if self._usage_sharing is not None: + self.add(self._usage_sharing) + + def build(self): + + """! + Build the Pipeline, with usage sharing after every other element. + + @rtype: Pipeline + @return: Returns a Pipeline + """ + + if self._usage_sharing in self.flow_elements: + self.flow_elements.remove(self._usage_sharing) + self.flow_elements.append(self._usage_sharing) + return super(DeviceDetectionPipelineBuilder, self).build() + diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py index e3589f2f..9e1df02a 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py @@ -69,7 +69,8 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): On Premise Engine Settings @type usage_sharing : bool - @param usage_sharing : Whether to enable usage sharing if using the on premise engine + @param usage_sharing : Whether to enable usage sharing if using the + on premise engine. The usage sharing element runs last. @type data_file_path: string @param data_file_path: path to the data file @type auto_update: bool @@ -155,12 +156,27 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): del settings["settings"] device = DeviceDetectionOnPremise(**settings) - # If usage sharing enabled, add usage sharing engine + self.add(device) - if usage_sharing: - self.add(ShareUsage()) + # Usage sharing, when on, runs after every other element, so build() + # moves it after any element the caller adds later. + self._usage_sharing = ShareUsage() if usage_sharing else None + if self._usage_sharing is not None: + self.add(self._usage_sharing) - self.add(device) + def build(self): + + """! + Build the Pipeline, with usage sharing after every other element. + + @rtype: Pipeline + @return: Returns a Pipeline + """ + + if self._usage_sharing in self.flow_elements: + self.flow_elements.remove(self._usage_sharing) + self.flow_elements.append(self._usage_sharing) + return super(DeviceDetectionOnPremisePipelineBuilder, self).build() def _map_properties_names(self, mappings, arguments): if arguments is None: diff --git a/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py b/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py index d5ac3c0e..a3eb676f 100644 --- a/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py +++ b/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py @@ -70,7 +70,8 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): On Premise Engine Settings: @type usage_sharing : bool - @param usage_sharing : Whether to enable usage sharing if using the on premise engine + @param usage_sharing : Whether to enable usage sharing if using the + on premise engine. The usage sharing element runs last. @type data_file_path: string @param data_file_path: path to the data file (.ipi) @type auto_update: bool @@ -120,14 +121,17 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): # Add specific engines + # Usage sharing, when on with the on premise engine, runs after + # every other element, so build() moves it after any element the + # caller adds later. + self._usage_sharing = None + if "data_file_path" in settings: del settings["settings"] ip_engine = IpIntelligenceOnPremise(**settings) - # If usage sharing enabled, add usage sharing engine - if usage_sharing: - self.add(ShareUsage()) + self._usage_sharing = ShareUsage() else : self.add(CloudRequestEngine(settings)) @@ -137,3 +141,20 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): ip_engine.set_cache(settings["cache"]) self.add(ip_engine) + + if self._usage_sharing is not None: + self.add(self._usage_sharing) + + def build(self): + + """! + Build the Pipeline, with usage sharing after every other element. + + @rtype: Pipeline + @return: Returns a Pipeline + """ + + if self._usage_sharing in self.flow_elements: + self.flow_elements.remove(self._usage_sharing) + self.flow_elements.append(self._usage_sharing) + return super(IpIntelligencePipelineBuilder, self).build() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py index 1ff2ace4..6d5873dd 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py @@ -50,7 +50,8 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): On Premise Engine Settings: @type usage_sharing : bool - @param usage_sharing : Whether to enable usage sharing + @param usage_sharing : Whether to enable usage sharing. The usage + sharing element runs last. @type data_file_path: string @param data_file_path: path to the data file (.ipi) @type auto_update: bool @@ -115,12 +116,27 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): del settings["settings"] ip_engine = IpIntelligenceOnPremise(**settings) - # If usage sharing enabled, add usage sharing engine + self.add(ip_engine) - if usage_sharing: - self.add(ShareUsage()) + # Usage sharing, when on, runs after every other element, so build() + # moves it after any element the caller adds later. + self._usage_sharing = ShareUsage() if usage_sharing else None + if self._usage_sharing is not None: + self.add(self._usage_sharing) - self.add(ip_engine) + def build(self): + + """! + Build the Pipeline, with usage sharing after every other element. + + @rtype: Pipeline + @return: Returns a Pipeline + """ + + if self._usage_sharing in self.flow_elements: + self.flow_elements.remove(self._usage_sharing) + self.flow_elements.append(self._usage_sharing) + return super(IpIntelligenceOnPremisePipelineBuilder, self).build() def _map_properties_names(self, mappings, arguments): if arguments is None: diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py index 05f2996e..5698b031 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py @@ -46,4 +46,12 @@ class Messages(): "Property '{}' is not present in the FlowData against '{}' ElementData. " # FlowData already processed. - FLOW_DATA_PROCESSED = "FlowData already processed" \ No newline at end of file + FLOW_DATA_PROCESSED = "FlowData already processed" + + # The usage sharing element is not the last Flow Element. This takes the + # element's class name, its position counting from 1 and the number of + # Flow Elements as format arguments. + USAGE_SHARING_NOT_LAST = ( + "The usage sharing element must be the last Flow Element and " + "outside any parallel group. {} is Flow Element {} of {} in this " + "Pipeline.") \ No newline at end of file diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py index c6d93557..e21f0b54 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py @@ -25,6 +25,43 @@ from .evidence_keyfilter import EvidenceKeyFilter from .flowdata import FlowData from .logger import Logger +from .messages import Messages + + +def is_usage_sharing_element(flow_element): + + """! + Whether a FlowElement is the usage sharing element. That element sets + the class attribute is_usage_sharing_element to True, because this + package cannot import it. + + @type flow_element: FlowElement + @param flow_element: The FlowElement to check + @rtype: bool + @return: True for the usage sharing element + """ + + return getattr(flow_element, "is_usage_sharing_element", False) is True + + +def check_usage_sharing_last(flow_elements): + + """! + Refuse FlowElements that put the usage sharing element anywhere but + last. It shares the Evidence the other FlowElements add, so it runs after + all of them. A Pipeline here has no parallel groups, so the element is + always outside one. + + @type flow_elements: sequence of FlowElement + @param flow_elements: The FlowElements, in the order they run + @raise ValueError: if a usage sharing element is not the last one + """ + + count = len(flow_elements) + for index, flow_element in enumerate(flow_elements): + if index < count - 1 and is_usage_sharing_element(flow_element): + raise ValueError(Messages.USAGE_SHARING_NOT_LAST.format( + type(flow_element).__name__, index + 1, count)) class PipelineEvidenceKeyFilter(EvidenceKeyFilter): @@ -70,7 +107,7 @@ def __init__(self, flow_elements, logger=Logger(), suppress_process_exceptions = @type flow_elements: list[FlowElement] @param flow_elements: The flowElements, in the order they run. The pipeline keeps a copy, so changing this list afterwards changes - nothing. + nothing. A usage sharing element must be the last one. @type logger: Logger @param logger: A logger to attach to the pipeline @@ -88,11 +125,15 @@ def __init__(self, flow_elements, logger=Logger(), suppress_process_exceptions = @rtype: Pipeline @return: Returns a Pipeline + @raise ValueError: if a usage sharing element is not the last + flowElement, before any element is registered """ self._flow_elements = tuple(flow_elements) + check_usage_sharing_last(self._flow_elements) + self.logger = logger # Filled below as each element is registered, so an element finds diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py index 2f0746d1..cee4e482 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py @@ -20,7 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -from .pipeline import Pipeline +from .pipeline import Pipeline, is_usage_sharing_element from .logger import Logger from .javascriptbuilder import JavascriptBuilderElement from .jsonbundler import JSONBundlerElement @@ -196,9 +196,13 @@ def build(self): """ Construct an immutable Pipeline using the list of flowElements, (Engines) and (Logger) currently in this Pipeline Builder. Call build after all items to be included in the pipeline have been added. + A usage sharing element added last stays last, after the web + integration elements. @rtype: Pipeline @return: Returns a Pipeline + @raise ValueError: if a usage sharing element was added anywhere + but last """ @@ -214,7 +218,8 @@ def _add_web_elements(self, flow_elements): SequenceElement goes first and the others after every other element, except that a JSONBundler goes immediately before a JavaScriptBuilder the caller added, because the JavaScriptBuilder reads the - JSONBundler's results. + JSONBundler's results, and a usage sharing element that ends the + caller's list stays last, because it runs after every other element. @type flow_elements: list @param flow_elements: The FlowElements the caller added @@ -227,6 +232,11 @@ def _add_web_elements(self, flow_elements): def absent(element): return _data_key(element) not in present + last = [] + if flow_elements and is_usage_sharing_element(flow_elements[-1]): + last = [flow_elements[-1]] + flow_elements = flow_elements[:-1] + elements = [element for element in self.get_sequence_elements() if absent(element)] elements.extend(flow_elements) @@ -246,6 +256,8 @@ def absent(element): elements.extend(element for element in self.get_setheader_elements() if absent(element)) + elements.extend(last) + return elements def add_logger(self, logger): @@ -275,7 +287,8 @@ def build_from_configuration(self, config): @return: Returns built pipeline If an element fails to build, the elements already built are closed - and the error is raised. + and the error is raised. The same happens, with a ValueError, when + the configuration lists a usage sharing element anywhere but last. """ diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 88cb2f61..ee9b6c24 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -98,6 +98,11 @@ def devices_message(entries): class ShareUsage(Engine): + + # Marks the usage sharing element, which a Pipeline refuses anywhere but + # last. Pipeline core reads this, as it cannot import this class. + is_usage_sharing_element = True + def __init__( self, interval = 1200, From 5d6ac344b3654aee133fb3465b47abd5206ab1a9 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 19 Nov 2026 12:32:05 +0000 Subject: [PATCH 119/136] Describe the repository to Context7 (#249) Adds context7.json at the root. It gives the title and description Context7 lists the repository under, the folders it skips, the fields that let the 51Degrees Context7 account claim the library, and rules that state the EUPL 1.2 and the notices it asks for. No package changes. The file was checked against Context7's schema. --- context7.json | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 context7.json diff --git a/context7.json b/context7.json new file mode 100644 index 00000000..3d7f343d --- /dev/null +++ b/context7.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://context7.com/schema/context7.json", + "url": "https://context7.com/51degrees/51degrees-python", + "public_key": "pk_NN7JecmrkO5zrSbbVRsy9", + "projectTitle": "51Degrees Device Detection, IP Intelligence, Human Confidence, and Identifiers for Python", + "description": "PyPI packages for Python 3.10+. DeviceDetectionPipelineBuilder(data_file_path=path, auto_update=False) for on-premise or DeviceDetectionPipelineBuilder(resource_key=key) for the cloud, then .build(). Use with pipeline.create_flowdata() as data, data.evidence.add('header.user-agent', ua), data.process(), then data.device.ismobile.value(). IpIntelligencePipelineBuilder covers IP Intelligence. fiftyone_pipeline_did reads 51Did. Install with pip install fiftyone-devicedetection", + "rules": [ + "This library is copyright of 51 Degrees Mobile Experts Limited and licensed under the European Union Public Licence (EUPL) v1.2, https://opensource.org/licenses/EUPL-1.2.", + "Code copied or adapted from this library must keep its copyright notices and the notices that refer to the EUPL, and must come with a copy of the EUPL, as Article 5 of the EUPL requires.", + "Code from this library that has been changed must carry a prominent notice saying that it was modified and the date it was modified, as Article 5 of the EUPL requires.", + "In a network application, the attribution notices that Article 5 of the EUPL requires may be given in the application's end user terms under an appropriate heading." + ], + "excludeFolders": [ + ".github", + "ci", + "cxx", + "tests", + "performance-tests", + "pipeline/owid-python", + "pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/js_templates", + "pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived-properties", + "archive", + "archived", + "old", + "deprecated", + "legacy", + "previous", + "outdated", + "superseded", + "benchmark", + "benchmarks", + "benches" + ] +} From 07ddcfddc8a880ed50809a510b3146bbca601530 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 19 Nov 2026 16:05:45 +0000 Subject: [PATCH 120/136] Stop reading the deprecated environment variable names The packages, examples, tests and tox settings read and pass only the current _51DEGREES_ names. The fallbacks to the old names, their deprecation warnings, and the tests and allow-list entries that existed only for the old names are removed. Setting and option names such as resource_key and cloud_endpoint are unchanged. The deprecated constants that hold an old name stay, so code that uses them still works, but the variables they name are no longer read. --- ci/check-environment-variable-names.Tests.ps1 | 19 +-- ci/check-environment-variable-names.ps1 | 61 ++------- .../tests/live_cloud.py | 56 ++------ .../fiftyone_devicedetection_cloud/tox.ini | 2 - .../example_utils.py | 51 ++----- .../onpremise/datafileupdate_console.py | 2 +- .../tests/live_cloud.py | 56 ++------ .../tests/test_cloud_examples_without_key.py | 4 +- .../tests/test_environment_names.py | 89 +------------ .../tests/test_find_out_more.py | 4 +- .../tests/test_onpremiseexamples.py | 9 +- .../test_package_gettingstarted_from_root.py | 3 +- .../fiftyone_devicedetection_examples/tox.ini | 3 - .../tox.ini | 2 - .../fiftyone_ipintelligence_cloud/tox.ini | 2 - .../example_utils.py | 64 +++------ .../onpremise/datafileupdate_console.py | 2 +- .../tests/live_cloud.py | 56 ++------ .../tests/test_cloud_examples_without_key.py | 4 +- .../tests/test_countries_cloud_without_key.py | 4 +- .../tests/test_datafiledownload.py | 1 - .../tests/test_environment_names.py | 126 +----------------- .../tests/test_examples_without_settings.py | 3 +- .../tests/test_find_out_more.py | 5 +- .../test_package_gettingstarted_from_root.py | 3 +- .../fiftyone_ipintelligence_examples/tox.ini | 3 - .../tests/constants.py | 5 +- .../tests/live_cloud.py | 56 ++------ .../tests/test_environment_names.py | 66 --------- .../fiftyone_ipintelligence_onpremise/tox.ini | 3 - .../fiftyone_ipintelligence_shared/tox.ini | 2 - .../tox.ini | 2 - location/examples/cloud/gettingstarted.py | 23 +--- location/examples/cloud/web.py | 23 +--- location/tests/live_cloud.py | 56 ++------ location/tests/test_environment_names.py | 114 ---------------- location/tests/test_examples.py | 2 +- location/tests/test_properties51D.py | 2 +- location/tox.ini | 2 - .../example_utils.py | 43 ++---- .../examples/tests/live_cloud.py | 56 ++------ .../tests/test_cloud_mixed_without_key.py | 4 +- .../examples/tests/test_example_utils.py | 48 ++----- .../examples/tests/test_find_out_more.py | 2 - .../examples/tox.ini | 5 - .../readme.md | 6 +- .../cloudrequestengine.py | 41 +----- .../constants.py | 12 +- .../tests/live_cloud.py | 56 ++------ .../tests/test_cloudengine.py | 2 +- .../tests/test_cloudrequestengine.py | 122 +---------------- .../tests/test_licensekey_live.py | 2 +- .../tox.ini | 2 - pipeline/fiftyone_pipeline_core/tox.ini | 2 - .../examples/creator_context_web/server.py | 35 ++--- pipeline/fiftyone_pipeline_did/readme.md | 4 +- .../src/fiftyone_pipeline_did/did_client.py | 47 +------ .../fiftyone_pipeline_did/tests/live_cloud.py | 56 ++------ .../tests/test_creator_context_server.py | 69 ++-------- .../tests/test_did_client.py | 86 +----------- .../tests/test_did_client_live.py | 2 +- .../tests/test_live_cloud.py | 121 +++-------------- pipeline/fiftyone_pipeline_did/tox.ini | 3 - pipeline/fiftyone_pipeline_engines/tox.ini | 2 - .../tox.ini | 2 - 65 files changed, 240 insertions(+), 1580 deletions(-) delete mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_environment_names.py delete mode 100644 location/tests/test_environment_names.py diff --git a/ci/check-environment-variable-names.Tests.ps1 b/ci/check-environment-variable-names.Tests.ps1 index f70f1f79..86ea7af8 100644 --- a/ci/check-environment-variable-names.Tests.ps1 +++ b/ci/check-environment-variable-names.Tests.ps1 @@ -82,18 +82,15 @@ Describe 'Find-EnvironmentNameProblem' { Get-Kind $Line | Should -BeNullOrEmpty } - # A variable whose name changed is still read under its old name when - # the new one is not set. That read gives the old name as the argument - # straight after the new one, and it is left alone. - It 'leaves the deprecated read alone' -ForEach @( - @{ Line = 'RESOURCE = env("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY")' } - @{ Line = 'LICENCE = env("_51DEGREES_LICENSE_KEY", "license_key")' } + # No variable is read under its old name any longer, so a read that + # gives the old name straight after the new one is found like any other. + It 'finds the old name in the read ' -ForEach @( @{ Line = 'API = env("_51DEGREES_CLOUD_ENDPOINT", "FOD_CLOUD_API_URL")' } @{ Line = 'read_environment("_51DEGREES_IPI_PATH", "IPINTELLIGENCEDATAFILE")' } @{ Line = ' "_51DEGREES_RUN_PERFORMANCE_TESTS", "run_performance_tests")' } @{ Line = "read_environment('_51DEGREES_DD_PATH', '51DEGREES_DD_PATH')" } ) { - Get-Kind $Line | Should -BeNullOrEmpty + Get-Kind $Line | Should -Not -BeNullOrEmpty } } @@ -126,14 +123,14 @@ Describe 'Find-EnvironmentNameProblem with the old names a file may hold' { Describe 'Get-FallbackName' { It 'lets hold ' -ForEach @( - @{ Path = 'pipeline/fiftyone_pipeline_did/tox.ini'; Name = 'FOD_CLOUD_API_URL' } + @{ Path = 'pipeline/fiftyone_pipeline_did/tests/test_did_client.py'; Name = 'FOD_CLOUD_API_URL' } @{ Path = 'device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py'; Name = '51DEGREES_DD_PATH' } @{ Path = 'device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py'; Name = 'cloud_endpoint' } @{ Path = 'ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py'; Name = 'IPINTELLIGENCEDATAFILE' } @{ Path = 'pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py'; Name = 'FOD_CLOUD_API_URL' } @{ Path = 'pipeline/fiftyone_pipeline_cloudrequestengine/readme.md'; Name = 'FOD_CLOUD_API_URL' } @{ Path = 'pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py'; Name = 'FOD_CLOUD_API_URL' } - @{ Path = 'pipeline/fiftyone_pipeline_did/tests/test_live_cloud.py'; Name = 'resource_key' } + @{ Path = 'pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/example_utils.py'; Name = 'resource_key' } ) { @(Get-FallbackName $Path) | Should -Contain $Name } @@ -150,6 +147,10 @@ Describe 'Get-FallbackName' { It 'lets hold no old name' -ForEach @( @{ Path = 'ci/run-unit-tests.ps1' } + # These read old names before and no longer may. + @{ Path = 'pipeline/fiftyone_pipeline_did/tox.ini' } + @{ Path = 'pipeline/fiftyone_pipeline_did/tests/test_live_cloud.py' } + @{ Path = 'pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py' } @{ Path = 'readme.md' } @{ Path = 'pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py' } @{ Path = 'pipeline/fiftyone_pipeline_core/readme.md' } diff --git a/ci/check-environment-variable-names.ps1 b/ci/check-environment-variable-names.ps1 index 5283cd81..ee2cff3e 100644 --- a/ci/check-environment-variable-names.ps1 +++ b/ci/check-environment-variable-names.ps1 @@ -4,12 +4,11 @@ # which leaves out build output and the submodules, and fails on any name # that starts with the digit and on any old name. # -# A variable whose name changed is still read under its old name when the new -# one is not set, which is deprecated. So an old name is left alone in the two -# places that read needs it. One is a line where the old name is the argument -# straight after its _51DEGREES_ name, which is how each deprecated read is -# written. The other is the files $FallbackNames lists, each of which may hold -# the old names it is listed with and no others. +# No variable is read under its old name. An old name is left alone only in +# the files $FallbackNames lists, which hold the deprecated constants that keep +# an old name, the tests of those constants and the documentation that says the +# name is no longer read. Each may hold the old names it is listed with and no +# others. # # Names are matched in any case, because Windows reads an environment # variable name in any case, so a lower case old name is still the old name @@ -84,27 +83,19 @@ $script:FallbackPattern = [regex]::new( "[A-Za-z0-9_]*\s*=\s*['`"]$sharedOld['`"]", [Text.RegularExpressions.RegexOptions]::IgnoreCase) -# A name given as the argument straight after a _51DEGREES_ name, which is how -# a deprecated read of an old name is written. The old name is taken out of -# the line before the line is checked. -$script:DeprecatedReadPattern = [regex]::new( - "(['`"]_51DEGREES_[A-Za-z0-9_]*['`"]\s*,\s*)['`"][A-Za-z0-9_]+['`"]") - # The old names each file may hold. The files are the ones that hold the -# deprecated constants, the documentation that says an old name is still -# read, the tests that set an old name to check it is read, and the lists of -# names tox passes to the tests. A file is given by its path from the root of +# deprecated constants, the tests of those constants, and the documentation +# that says an old name is no longer read. A file is given by its path from the root of # the repository, and may hold only the names it is listed with, so any other # old name still fails there, and so does any other name that starts with a # digit. The names that are matched wherever they appear are built from # pieces, so that this file does not name one itself. $endpointOld = 'FOD_CLOUD' + '_API_URL' $ipiFileOld = 'IPINTELLIGENCE' + 'DATAFILE' -$performanceOld = 'run_performance' + '_tests' $ddPathOld = '5' + '1DEGREES_DD_PATH' $script:FallbackNames = @( @{ - Names = @($ddPathOld, 'LICENSE_KEY') + Names = @($ddPathOld) Paths = @( 'device-detection/fiftyone_devicedetection_examples/tests/test_environment_names.py' ) @@ -118,34 +109,15 @@ $script:FallbackNames = @( @{ Names = @($endpointOld) Paths = @( - 'pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini' 'pipeline/fiftyone_pipeline_cloudrequestengine/readme.md' - 'pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py' 'pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py' 'pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py' - 'pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py' - 'pipeline/fiftyone_pipeline_did/readme.md' 'pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py' 'pipeline/fiftyone_pipeline_did/tests/test_did_client.py' - 'pipeline/fiftyone_pipeline_did/tox.ini' - ) - } - @{ - Names = @($endpointOld, 'LICENSE_KEY', 'RESOURCE_KEY') - Paths = @( - 'pipeline/fiftyone_pipeline_did/tests/test_creator_context_server.py' ) } @{ Names = @($ipiFileOld) - Paths = @( - 'ip-intelligence/fiftyone_ipintelligence_examples/tox.ini' - 'ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_environment_names.py' - 'ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini' - ) - } - @{ - Names = @($ipiFileOld, 'LICENSE_KEY') Paths = @( 'ip-intelligence/fiftyone_ipintelligence_examples/tests/test_environment_names.py' ) @@ -162,18 +134,6 @@ $script:FallbackNames = @( 'pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/example_utils.py' ) } - @{ - Names = @('license_key', 'resource_key') - Paths = @( - 'pipeline/fiftyone_pipeline_did/tests/test_live_cloud.py' - ) - } - @{ - Names = @($performanceOld) - Paths = @( - 'device-detection/fiftyone_devicedetection_examples/tox.ini' - ) - } ) # The old names a file may hold, given its path from the root of the @@ -191,7 +151,6 @@ function Get-FallbackName([string]$Relative) { # are taken out of the line, in the case they are written in, before the # line is checked. function Find-EnvironmentNameProblem([string]$Line, [string[]]$Allowed = @()) { - $Line = $script:DeprecatedReadPattern.Replace($Line, '$1""') if ($Allowed.Count -gt 0) { $names = ($Allowed | ForEach-Object { [regex]::Escape($_) }) -join '|' $Line = [regex]::Replace( @@ -260,8 +219,8 @@ if ($found.Count -gt 0 -or $fallbacks.Count -gt 0) { throw ("$($found.Count) names start with a digit, which a POSIX shell " + 'cannot set, so each must start with an underscore, and ' + "$($fallbacks.Count) old names are read, set or kept outside the " + - 'deprecated reads and the files listed with them.') + 'files listed with them.') } Write-Output ('No tracked file names a variable starting with a digit, or ' + - 'an old name outside the deprecated reads.') + 'an old name outside the files that may hold one.') diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py index 2109e4a2..6cdbab0f 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py @@ -22,8 +22,8 @@ """Where the tests that call the live 51Degrees cloud service find their resource key, and what they fail with when there is none or when the -answer lacks what they asked for. Also how a test reads any setting from -an environment variable whose name has changed. +answer lacks what they asked for. Also how a test reads a setting from an +environment variable. The same file is in each test folder whose tests read a setting from the environment. There is no fallback to a skip, because a cloud test that @@ -31,7 +31,6 @@ """ import os -import sys # Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud # 51Did workflow sets it to each key in turn, and each product's @@ -41,68 +40,31 @@ "_51DEGREES_RESOURCE_KEY_BESPOKE", ) -# Deprecated. The name the resource key variable had before, read when -# none of RESOURCE_KEY_NAMES is set. -OLD_RESOURCE_KEY_NAME = "resource_key" - -# The old names already reported as deprecated, so each is reported once. -_reported_old_names = set() - ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" -def _report_old_name(name, old_name): - """Prints, once for each old name, that it is deprecated and which - name replaces it.""" - if old_name in _reported_old_names: - return - _reported_old_names.add(old_name) - print( - f"The {old_name} environment variable is deprecated and {name} " - f"replaces it. The value was read from {old_name} because {name} " - f"is not set, so set {name} to the same value.", - file=sys.stderr) - - -def read_environment(name, old_name): +def read_environment(name): """The value of the environment variable name, or None when it has - none. When name is unset or empty the name the variable had before is - read, and using it prints once that the old name is deprecated.""" + none.""" value = os.environ.get(name, "").strip() - if value: - return value - value = os.environ.get(old_name, "").strip() - if not value: - return None - _report_old_name(name, old_name) - return value + return value or None -def is_set(name, old_name): +def is_set(name): """Whether the environment variable name is set, whatever it holds, - which is how a switch is read. When it is not set, the name the - variable had before counts, and using it prints once that the old name - is deprecated.""" - if name in os.environ: - return True - if old_name not in os.environ: - return False - _report_old_name(name, old_name) - return True + which is how a switch is read.""" + return name in os.environ def find_resource_key(names=RESOURCE_KEY_NAMES): - """The first resource key set under any of the names, or None. When - none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" + """The first resource key set under any of the names, or None.""" for name in names: value = os.environ.get(name, "").strip() if value: return value - if names is RESOURCE_KEY_NAMES: - return read_environment(names[0], OLD_RESOURCE_KEY_NAME) return None diff --git a/device-detection/fiftyone_devicedetection_cloud/tox.ini b/device-detection/fiftyone_devicedetection_cloud/tox.ini index 28643c83..27672a13 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tox.ini +++ b/device-detection/fiftyone_devicedetection_cloud/tox.ini @@ -25,10 +25,8 @@ commands = # which leaves the live tests out of the unit step. pass_env = _51DEGREES_RESOURCE_KEY - resource_key _51DEGREES_RESOURCE_KEY_BESPOKE _51DEGREES_LICENSE_KEY - license_key PYTEST_ADDOPTS [testenv:pre-publish] diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py index 63bb4d78..328ebefc 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py @@ -44,7 +44,7 @@ class ExampleUtils: RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY" # Deprecated. Use RESOURCE_KEY_ENV_VAR. The name the resource key - # variable had before, read when RESOURCE_KEY_ENV_VAR is not set. + # variable had before, which is no longer read. LEGACY_RESOURCE_KEY_ENV_VAR = "resource_key" # The environment variable the data file update example reads a licence @@ -52,7 +52,7 @@ class ExampleUtils: LICENSE_KEY_ENV_VAR = "_51DEGREES_LICENSE_KEY" # Deprecated. Use LICENSE_KEY_ENV_VAR. The name the licence key - # variable had before, read when LICENSE_KEY_ENV_VAR is not set. + # variable had before, which is no longer read. LEGACY_LICENSE_KEY_ENV_VAR = "license_key" # The environment variable naming a cloud service other than the public @@ -60,7 +60,7 @@ class ExampleUtils: CLOUD_ENDPOINT_ENV_VAR = "_51DEGREES_CLOUD_ENDPOINT" # Deprecated. Use CLOUD_ENDPOINT_ENV_VAR. The name that variable had - # before, read when CLOUD_ENDPOINT_ENV_VAR is not set. + # before, which is no longer read. ENDPOINT_ENV_VAR = "cloud_endpoint" # The environment variable an on-premise example reads the path of its @@ -69,7 +69,7 @@ class ExampleUtils: DD_PATH_ENV_VAR = "_51DEGREES_DD_PATH" # Deprecated. Use DD_PATH_ENV_VAR. The name that variable had before, - # read when DD_PATH_ENV_VAR is not set. A POSIX shell cannot set it, + # which is no longer read. A POSIX shell cannot set it, # because it starts with a digit. DATA_FILE_ENV_VAR = "51DEGREES_DD_PATH" @@ -186,23 +186,16 @@ def get_no_profiles_message(): @staticmethod def get_resource_key(): return ExampleUtils.__get_env_variable( - ExampleUtils.RESOURCE_KEY_ENV_VAR, - ExampleUtils.LEGACY_RESOURCE_KEY_ENV_VAR) + ExampleUtils.RESOURCE_KEY_ENV_VAR) @staticmethod def get_license_key(): """! - The licence key from the environment, or an empty string. The old - name is also read in upper case, as it was before. + The licence key from the environment, or an empty string. """ - return ( - ExampleUtils.__get_env_variable( - ExampleUtils.LICENSE_KEY_ENV_VAR, - ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR) - or ExampleUtils.__get_env_variable( - ExampleUtils.LICENSE_KEY_ENV_VAR, - ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR.upper())) + return ExampleUtils.__get_env_variable( + ExampleUtils.LICENSE_KEY_ENV_VAR) @staticmethod def get_missing_resource_key_message(): @@ -218,40 +211,20 @@ def get_missing_resource_key_message(): @staticmethod def get_data_file_path(): return ExampleUtils.__get_env_variable( - ExampleUtils.DD_PATH_ENV_VAR, ExampleUtils.DATA_FILE_ENV_VAR) + ExampleUtils.DD_PATH_ENV_VAR) @staticmethod def get_cloud_endpoint(): return ExampleUtils.__get_env_variable( - ExampleUtils.CLOUD_ENDPOINT_ENV_VAR, - ExampleUtils.ENDPOINT_ENV_VAR) - - # The old names already reported as deprecated, so each is reported - # once. - __reported_old_names = set() + ExampleUtils.CLOUD_ENDPOINT_ENV_VAR) @staticmethod - def __get_env_variable(name, old_name): + def __get_env_variable(name): """! The value of the environment variable name, or an empty string. - When name is unset or empty the name the variable had before is - read, and using it prints once that the old name is deprecated. """ - value = os.environ.get(name) - if value: - return value - value = os.environ.get(old_name) - if not value: - return "" - if old_name not in ExampleUtils.__reported_old_names: - ExampleUtils.__reported_old_names.add(old_name) - print( - f"The {old_name} environment variable is deprecated and {name} " - f"replaces it. The value was read from {old_name} because {name} " - f"is not set, so set {name} to the same value.", - file=sys.stderr) - return value + return os.environ.get(name) or "" # Find the specified file. A path to a file that exists is returned as # it is. Next, the data repository's folder is checked in the working diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py index 2ad8ee26..39a834e9 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py @@ -204,7 +204,7 @@ # on the command line. UPDATE_EXAMPLE_LICENSE_KEY_VARIABLE = ExampleUtils.LICENSE_KEY_ENV_VAR # Deprecated. Use UPDATE_EXAMPLE_LICENSE_KEY_VARIABLE. The name that -# variable had before, read when the new one is not set. +# variable had before, which is no longer read. UPDATE_EXAMPLE_LICENSE_KEY_NAME = ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR # Where to ask for a licence key and a copy of the Enterprise data file, tagged with this # example. diff --git a/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py b/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py index 2109e4a2..6cdbab0f 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py @@ -22,8 +22,8 @@ """Where the tests that call the live 51Degrees cloud service find their resource key, and what they fail with when there is none or when the -answer lacks what they asked for. Also how a test reads any setting from -an environment variable whose name has changed. +answer lacks what they asked for. Also how a test reads a setting from an +environment variable. The same file is in each test folder whose tests read a setting from the environment. There is no fallback to a skip, because a cloud test that @@ -31,7 +31,6 @@ """ import os -import sys # Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud # 51Did workflow sets it to each key in turn, and each product's @@ -41,68 +40,31 @@ "_51DEGREES_RESOURCE_KEY_BESPOKE", ) -# Deprecated. The name the resource key variable had before, read when -# none of RESOURCE_KEY_NAMES is set. -OLD_RESOURCE_KEY_NAME = "resource_key" - -# The old names already reported as deprecated, so each is reported once. -_reported_old_names = set() - ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" -def _report_old_name(name, old_name): - """Prints, once for each old name, that it is deprecated and which - name replaces it.""" - if old_name in _reported_old_names: - return - _reported_old_names.add(old_name) - print( - f"The {old_name} environment variable is deprecated and {name} " - f"replaces it. The value was read from {old_name} because {name} " - f"is not set, so set {name} to the same value.", - file=sys.stderr) - - -def read_environment(name, old_name): +def read_environment(name): """The value of the environment variable name, or None when it has - none. When name is unset or empty the name the variable had before is - read, and using it prints once that the old name is deprecated.""" + none.""" value = os.environ.get(name, "").strip() - if value: - return value - value = os.environ.get(old_name, "").strip() - if not value: - return None - _report_old_name(name, old_name) - return value + return value or None -def is_set(name, old_name): +def is_set(name): """Whether the environment variable name is set, whatever it holds, - which is how a switch is read. When it is not set, the name the - variable had before counts, and using it prints once that the old name - is deprecated.""" - if name in os.environ: - return True - if old_name not in os.environ: - return False - _report_old_name(name, old_name) - return True + which is how a switch is read.""" + return name in os.environ def find_resource_key(names=RESOURCE_KEY_NAMES): - """The first resource key set under any of the names, or None. When - none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" + """The first resource key set under any of the names, or None.""" for name in names: value = os.environ.get(name, "").strip() if value: return value - if names is RESOURCE_KEY_NAMES: - return read_environment(names[0], OLD_RESOURCE_KEY_NAME) return None diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_cloud_examples_without_key.py b/device-detection/fiftyone_devicedetection_examples/tests/test_cloud_examples_without_key.py index cf6802fe..23820cbd 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_cloud_examples_without_key.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_cloud_examples_without_key.py @@ -47,8 +47,8 @@ # Every name a key can be read from, which a fresh clone does not set. KEY_SETTINGS = [ - "_51DEGREES_RESOURCE_KEY", "resource_key", "RESOURCE_KEY", - "_51DEGREES_LICENSE_KEY", "license_key"] + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_LICENSE_KEY"] # A link to the page where a resource key is created, with the campaign # tags every example's links carry. diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_environment_names.py b/device-detection/fiftyone_devicedetection_examples/tests/test_environment_names.py index be74700c..bc1ec6d5 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_environment_names.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_environment_names.py @@ -20,13 +20,10 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -"""Checks that the example helper still reads each setting under the name -its environment variable had before, when the new name is not set, and -prints once that the old name is deprecated. These need no resource key. +"""Checks that the constants of the example helper that name a variable keep +their values, including those that hold a name that is no longer read. """ -import contextlib -import io import os import unittest from unittest import mock @@ -34,16 +31,8 @@ from fiftyone_devicedetection_examples.example_utils import ExampleUtils from fiftyone_devicedetection_shared.key_utils import KeyUtils -# Each reader, with the new name of its variable and the old one. -PAIRS = { - "get_resource_key": ("_51DEGREES_RESOURCE_KEY", "resource_key"), - "get_license_key": ("_51DEGREES_LICENSE_KEY", "license_key"), - "get_cloud_endpoint": ("_51DEGREES_CLOUD_ENDPOINT", "cloud_endpoint"), - "get_data_file_path": ("_51DEGREES_DD_PATH", "51DEGREES_DD_PATH"), -} - # Each constant that names a variable, with the value it has to hold. The -# ones that held an old name hold it still. +# ones that held an old name hold it still, although it is no longer read. CONSTANTS = { "RESOURCE_KEY_ENV_VAR": "_51DEGREES_RESOURCE_KEY", "LEGACY_RESOURCE_KEY_ENV_VAR": "resource_key", @@ -56,83 +45,13 @@ } -class OldNameTests(unittest.TestCase): - - def setUp(self): - # Each test starts with none of the names set and no old name - # reported yet, and what it sets is put back afterwards. - environment = mock.patch.dict(os.environ) - environment.start() - self.addCleanup(environment.stop) - for pair in PAIRS.values(): - for name in pair: - os.environ.pop(name, None) - os.environ.pop(name.upper(), None) - reported = mock.patch.object( - ExampleUtils, "_ExampleUtils__reported_old_names", set()) - reported.start() - self.addCleanup(reported.stop) - - def printed(self, call): - """What a call returned and what it printed to the error stream.""" - errors = io.StringIO() - with contextlib.redirect_stderr(errors): - value = call() - return value, errors.getvalue() - - def test_each_setting_is_read_from_its_old_name(self): - for getter, (name, old_name) in PAIRS.items(): - with self.subTest(getter=getter): - os.environ[old_name] = "old-value" - value, errors = self.printed(getattr(ExampleUtils, getter)) - self.assertEqual("old-value", value) - self.assertIn( - old_name + " environment variable is deprecated", errors) - self.assertIn(name, errors) - del os.environ[old_name] - - def test_new_name_wins_and_nothing_is_printed(self): - for getter, (name, old_name) in PAIRS.items(): - with self.subTest(getter=getter): - os.environ[name] = "new-value" - os.environ[old_name] = "old-value" - value, errors = self.printed(getattr(ExampleUtils, getter)) - self.assertEqual("new-value", value) - self.assertEqual("", errors) - del os.environ[name] - del os.environ[old_name] - - def test_neither_name_gives_nothing(self): - for getter in PAIRS: - with self.subTest(getter=getter): - value, errors = self.printed(getattr(ExampleUtils, getter)) - self.assertFalse(value) - self.assertEqual("", errors) - - def test_old_name_is_reported_once(self): - getter, (name, old_name) = next(iter(PAIRS.items())) - os.environ[old_name] = "old-value" - - def twice(): - return (getattr(ExampleUtils, getter)(), - getattr(ExampleUtils, getter)()) - - value, errors = self.printed(twice) - self.assertEqual(("old-value", "old-value"), value) - self.assertEqual( - 1, errors.count(old_name + " environment variable is deprecated")) +class ConstantTests(unittest.TestCase): def test_constants_hold_the_old_and_the_new_names(self): for constant, value in CONSTANTS.items(): with self.subTest(constant=constant): self.assertEqual(value, getattr(ExampleUtils, constant)) - def test_licence_key_is_read_from_the_old_name_in_upper_case(self): - os.environ.pop("LICENSE_KEY", None) - os.environ["LICENSE_KEY"] = "old-value" - value, _ = self.printed(ExampleUtils.get_license_key) - self.assertEqual("old-value", value) - class KeyUtilsTests(unittest.TestCase): diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_find_out_more.py b/device-detection/fiftyone_devicedetection_examples/tests/test_find_out_more.py index 6f381ace..2b789927 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_find_out_more.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_find_out_more.py @@ -111,8 +111,8 @@ # The settings that let an example get past its early end. SETTINGS = [ - "_51DEGREES_RESOURCE_KEY", "resource_key", "RESOURCE_KEY", - "_51DEGREES_LICENSE_KEY", "license_key", "LICENSE_KEY"] + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_LICENSE_KEY"] def slug(path): diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_onpremiseexamples.py b/device-detection/fiftyone_devicedetection_examples/tests/test_onpremiseexamples.py index b8160f23..7be3e128 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_onpremiseexamples.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_onpremiseexamples.py @@ -32,9 +32,8 @@ from .live_cloud import is_set -# Deprecated old name, read when the new one is not set. -RUN_PERFORMANCE_TESTS = ( - "_51DEGREES_RUN_PERFORMANCE_TESTS", "run_performance_tests") +# Set to run the performance tests. +RUN_PERFORMANCE_TESTS = "_51DEGREES_RUN_PERFORMANCE_TESTS" class DeviceDetectionExampleTests(unittest.TestCase): @@ -80,7 +79,7 @@ def test_onpremise_datafileupdate_programmatic(self): def test_onpremise_offline_processing(self): # Only run if environment variable set - if is_set(*RUN_PERFORMANCE_TESTS): + if is_set(RUN_PERFORMANCE_TESTS): example = OfflineProcessing() with open(self.evidence_file, "r") as input: with open("./offlineprocessing-output.yml", "w") as output: @@ -104,5 +103,5 @@ def test_onpremise_offline_processing_with_tabs(self): def test_onpremise_performance(self): # Only run if environment variable set - if is_set(*RUN_PERFORMANCE_TESTS): + if is_set(RUN_PERFORMANCE_TESTS): import fiftyone_devicedetection_examples.onpremise.performance diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_package_gettingstarted_from_root.py b/device-detection/fiftyone_devicedetection_examples/tests/test_package_gettingstarted_from_root.py index 1449018c..d15ff027 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_package_gettingstarted_from_root.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_package_gettingstarted_from_root.py @@ -60,8 +60,7 @@ def test_runs_from_the_repository_root(self): # aside. The builder shares usage, so nothing is sent from a # test. with mock.patch.dict(os.environ): - for name in (ExampleUtils.DD_PATH_ENV_VAR, - ExampleUtils.DATA_FILE_ENV_VAR): + for name in (ExampleUtils.DD_PATH_ENV_VAR,): os.environ.pop(name, None) with mock.patch("requests.post"), redirect_stdout(printed): runpy.run_path(str(EXAMPLE), run_name="__main__") diff --git a/device-detection/fiftyone_devicedetection_examples/tox.ini b/device-detection/fiftyone_devicedetection_examples/tox.ini index 4b9fc088..48032ef6 100644 --- a/device-detection/fiftyone_devicedetection_examples/tox.ini +++ b/device-detection/fiftyone_devicedetection_examples/tox.ini @@ -33,12 +33,9 @@ commands = {[common]commands} pass_env = _51DEGREES_RESOURCE_KEY - resource_key _51DEGREES_RESOURCE_KEY_BESPOKE _51DEGREES_LICENSE_KEY - license_key _51DEGREES_RUN_PERFORMANCE_TESTS - run_performance_tests # A download the examples make in GitHub Actions leaves out their own # source name, and the data repository's script names the build from # these instead, so a test's download is not counted as a person diff --git a/device-detection/fiftyone_devicedetection_onpremise/tox.ini b/device-detection/fiftyone_devicedetection_onpremise/tox.ini index 4e3d7d7c..261fd422 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tox.ini +++ b/device-detection/fiftyone_devicedetection_onpremise/tox.ini @@ -33,9 +33,7 @@ commands = {[common]commands} pass_env = _51DEGREES_RESOURCE_KEY - resource_key _51DEGREES_LICENSE_KEY - license_key TMPDIR TEMP TMP diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini b/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini index f4a02df3..996a428f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini @@ -23,9 +23,7 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_cloud {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY - resource_key _51DEGREES_LICENSE_KEY - license_key [testenv:pre-publish] package = skip diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py index a5c7c29d..fe1f2f59 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py @@ -43,7 +43,7 @@ class ExampleUtils: CLOUD_RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY" # Deprecated. Use CLOUD_RESOURCE_KEY_ENV_VAR. The name the resource key - # variable had before, read when CLOUD_RESOURCE_KEY_ENV_VAR is not set. + # variable had before, which is no longer read. RESOURCE_KEY_ENV_VAR = "resource_key" # The environment variable the data file update example reads a licence @@ -51,7 +51,7 @@ class ExampleUtils: LICENSE_KEY_ENV_VAR = "_51DEGREES_LICENSE_KEY" # Deprecated. Use LICENSE_KEY_ENV_VAR. The name the licence key - # variable had before, read when LICENSE_KEY_ENV_VAR is not set. + # variable had before, which is no longer read. LEGACY_LICENSE_KEY_ENV_VAR = "license_key" # The environment variable naming a cloud service other than the public @@ -59,7 +59,7 @@ class ExampleUtils: CLOUD_ENDPOINT_ENV_VAR = "_51DEGREES_CLOUD_ENDPOINT" # Deprecated. Use CLOUD_ENDPOINT_ENV_VAR. The name that variable had - # before, read when CLOUD_ENDPOINT_ENV_VAR is not set. + # before, which is no longer read. ENDPOINT_ENV_VAR = "cloud_endpoint" # The default environment variable used to get the data file path @@ -67,7 +67,7 @@ class ExampleUtils: IPI_PATH_ENV_VAR = "_51DEGREES_IPI_PATH" # Deprecated. Use IPI_PATH_ENV_VAR. The name that variable had before, - # read when IPI_PATH_ENV_VAR is not set. + # which is no longer read. DATA_FILE_ENV_VAR = "IPINTELLIGENCEDATAFILE" # The data repository's IP intelligence folder, from the root of a @@ -152,43 +152,30 @@ def get_data_file_path(): """ return ExampleUtils.__get_env_variable( - ExampleUtils.IPI_PATH_ENV_VAR, ExampleUtils.DATA_FILE_ENV_VAR) + ExampleUtils.IPI_PATH_ENV_VAR) @staticmethod def get_resource_key(): """! - The resource key from the environment, or None. The old name is - also read in upper case, as it was before. + The resource key from the environment, or None. """ - return ( - ExampleUtils.__get_env_variable( - ExampleUtils.CLOUD_RESOURCE_KEY_ENV_VAR, - ExampleUtils.RESOURCE_KEY_ENV_VAR) - or ExampleUtils.__get_env_variable( - ExampleUtils.CLOUD_RESOURCE_KEY_ENV_VAR, - ExampleUtils.RESOURCE_KEY_ENV_VAR.upper())) + return ExampleUtils.__get_env_variable( + ExampleUtils.CLOUD_RESOURCE_KEY_ENV_VAR) @staticmethod def get_license_key(): return ExampleUtils.__get_env_variable( - ExampleUtils.LICENSE_KEY_ENV_VAR, - ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR) + ExampleUtils.LICENSE_KEY_ENV_VAR) @staticmethod def get_cloud_endpoint(): """! - The cloud endpoint from the environment, or None. The old name is - also read in upper case, as it was before. + The cloud endpoint from the environment, or None. """ - return ( - ExampleUtils.__get_env_variable( - ExampleUtils.CLOUD_ENDPOINT_ENV_VAR, - ExampleUtils.ENDPOINT_ENV_VAR) - or ExampleUtils.__get_env_variable( - ExampleUtils.CLOUD_ENDPOINT_ENV_VAR, - ExampleUtils.ENDPOINT_ENV_VAR.upper())) + return ExampleUtils.__get_env_variable( + ExampleUtils.CLOUD_ENDPOINT_ENV_VAR) @staticmethod def find_file(file_name): @@ -202,7 +189,7 @@ def find_file(file_name): """ env = ExampleUtils.__get_env_variable( - ExampleUtils.IPI_PATH_ENV_VAR, ExampleUtils.DATA_FILE_ENV_VAR) + ExampleUtils.IPI_PATH_ENV_VAR) if env and os.path.basename(env) == file_name and os.path.isfile(env): return env @@ -475,29 +462,10 @@ def check_data_file(pipeline, logger): f"or cxx/data/get-data-files.ps1 ip-intelligence -Update " f"on Windows.") - # The old names already reported as deprecated, so each is reported - # once. - __reported_old_names = set() - @staticmethod - def __get_env_variable(name, old_name): + def __get_env_variable(name): """! - The value of the environment variable name, or None. When name is - unset or empty the name the variable had before is read, and using - it prints once that the old name is deprecated. + The value of the environment variable name, or None. """ - value = os.environ.get(name) - if value: - return value - value = os.environ.get(old_name) - if not value: - return None - if old_name not in ExampleUtils.__reported_old_names: - ExampleUtils.__reported_old_names.add(old_name) - print( - f"The {old_name} environment variable is deprecated and {name} " - f"replaces it. The value was read from {old_name} because {name} " - f"is not set, so set {name} to the same value.", - file=sys.stderr) - return value + return os.environ.get(name) or None diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py index b4da1f26..a4f1eba7 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py @@ -90,7 +90,7 @@ UPDATE_EXAMPLE_LICENSE_KEY_VARIABLE = ExampleUtils.LICENSE_KEY_ENV_VAR # Deprecated. Use UPDATE_EXAMPLE_LICENSE_KEY_VARIABLE. The name that -# variable had before, read when the new one is not set. +# variable had before, which is no longer read. UPDATE_EXAMPLE_LICENSE_KEY_NAME = ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR # Where to ask for a licence key and a copy of the Enterprise data file, tagged with this diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py index 2109e4a2..6cdbab0f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py @@ -22,8 +22,8 @@ """Where the tests that call the live 51Degrees cloud service find their resource key, and what they fail with when there is none or when the -answer lacks what they asked for. Also how a test reads any setting from -an environment variable whose name has changed. +answer lacks what they asked for. Also how a test reads a setting from an +environment variable. The same file is in each test folder whose tests read a setting from the environment. There is no fallback to a skip, because a cloud test that @@ -31,7 +31,6 @@ """ import os -import sys # Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud # 51Did workflow sets it to each key in turn, and each product's @@ -41,68 +40,31 @@ "_51DEGREES_RESOURCE_KEY_BESPOKE", ) -# Deprecated. The name the resource key variable had before, read when -# none of RESOURCE_KEY_NAMES is set. -OLD_RESOURCE_KEY_NAME = "resource_key" - -# The old names already reported as deprecated, so each is reported once. -_reported_old_names = set() - ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" -def _report_old_name(name, old_name): - """Prints, once for each old name, that it is deprecated and which - name replaces it.""" - if old_name in _reported_old_names: - return - _reported_old_names.add(old_name) - print( - f"The {old_name} environment variable is deprecated and {name} " - f"replaces it. The value was read from {old_name} because {name} " - f"is not set, so set {name} to the same value.", - file=sys.stderr) - - -def read_environment(name, old_name): +def read_environment(name): """The value of the environment variable name, or None when it has - none. When name is unset or empty the name the variable had before is - read, and using it prints once that the old name is deprecated.""" + none.""" value = os.environ.get(name, "").strip() - if value: - return value - value = os.environ.get(old_name, "").strip() - if not value: - return None - _report_old_name(name, old_name) - return value + return value or None -def is_set(name, old_name): +def is_set(name): """Whether the environment variable name is set, whatever it holds, - which is how a switch is read. When it is not set, the name the - variable had before counts, and using it prints once that the old name - is deprecated.""" - if name in os.environ: - return True - if old_name not in os.environ: - return False - _report_old_name(name, old_name) - return True + which is how a switch is read.""" + return name in os.environ def find_resource_key(names=RESOURCE_KEY_NAMES): - """The first resource key set under any of the names, or None. When - none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" + """The first resource key set under any of the names, or None.""" for name in names: value = os.environ.get(name, "").strip() if value: return value - if names is RESOURCE_KEY_NAMES: - return read_environment(names[0], OLD_RESOURCE_KEY_NAME) return None diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloud_examples_without_key.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloud_examples_without_key.py index 3d5f02c3..95c3057b 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloud_examples_without_key.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloud_examples_without_key.py @@ -49,8 +49,8 @@ # Every name a key can be read from, which a fresh clone does not set. KEY_SETTINGS = [ - "_51DEGREES_RESOURCE_KEY", "resource_key", "RESOURCE_KEY", - "_51DEGREES_LICENSE_KEY", "license_key"] + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_LICENSE_KEY"] # A link to the page where a resource key is created, with the campaign # tags every example's links carry. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_countries_cloud_without_key.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_countries_cloud_without_key.py index 1b150b1b..a7bbcbf1 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_countries_cloud_without_key.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_countries_cloud_without_key.py @@ -36,8 +36,8 @@ from fiftyone_ipintelligence_examples.example_utils import ExampleUtils KEY_SETTINGS = [ - "_51DEGREES_RESOURCE_KEY", "resource_key", "RESOURCE_KEY", - "_51DEGREES_LICENSE_KEY", "license_key"] + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_LICENSE_KEY"] CONFIGURE_LINK = ( r"https://51degrees\.com/configure[^\s?]*\?utm_source=code" diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_datafiledownload.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_datafiledownload.py index 633a1250..7fdbcf96 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_datafiledownload.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_datafiledownload.py @@ -99,7 +99,6 @@ def test_find_file_downloads_a_missing_asn_file(tmp_path, monkeypatch, capfd): # A data file named in the environment would be used in place of the # search. monkeypatch.delenv(ExampleUtils.IPI_PATH_ENV_VAR, raising=False) - monkeypatch.delenv(ExampleUtils.DATA_FILE_ENV_VAR, raising=False) monkeypatch.chdir(tmp_path) with serving(source) as address: diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_environment_names.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_environment_names.py index 27d3844d..5b22afcd 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_environment_names.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_environment_names.py @@ -20,33 +20,18 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -"""Checks that the example helper still reads each setting under the name -its environment variable had before, when the new name is not set, and -prints once that the old name is deprecated. These need no resource key. +"""Checks that the constants of the example helper that name a variable keep +their values, including those that hold a name that is no longer read. """ -import contextlib -import io -import os import unittest -from unittest import mock -import tempfile from fiftyone_ipintelligence_examples.example_utils import ExampleUtils -# Each reader, with the new name of its variable and the old one. -PAIRS = { - "get_resource_key": ("_51DEGREES_RESOURCE_KEY", "resource_key"), - "get_license_key": ("_51DEGREES_LICENSE_KEY", "license_key"), - "get_cloud_endpoint": ("_51DEGREES_CLOUD_ENDPOINT", "cloud_endpoint"), -} - -# The data file variable, which find_file reads, under both names. -DATA_FILE = ("_51DEGREES_IPI_PATH", "IPINTELLIGENCEDATAFILE") # Each constant that names a variable, with the value it has to hold. The -# ones that held an old name hold it still. +# ones that held an old name hold it still, although it is no longer read. CONSTANTS = { "CLOUD_RESOURCE_KEY_ENV_VAR": "_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY_ENV_VAR": "resource_key", @@ -59,116 +44,13 @@ } -class OldNameTests(unittest.TestCase): - - def setUp(self): - # Each test starts with none of the names set and no old name - # reported yet, and what it sets is put back afterwards. - environment = mock.patch.dict(os.environ) - environment.start() - self.addCleanup(environment.stop) - for pair in PAIRS.values(): - for name in pair: - os.environ.pop(name, None) - os.environ.pop(name.upper(), None) - reported = mock.patch.object( - ExampleUtils, "_ExampleUtils__reported_old_names", set()) - reported.start() - self.addCleanup(reported.stop) - - def printed(self, call): - """What a call returned and what it printed to the error stream.""" - errors = io.StringIO() - with contextlib.redirect_stderr(errors): - value = call() - return value, errors.getvalue() - - def test_each_setting_is_read_from_its_old_name(self): - for getter, (name, old_name) in PAIRS.items(): - with self.subTest(getter=getter): - os.environ[old_name] = "old-value" - value, errors = self.printed(getattr(ExampleUtils, getter)) - self.assertEqual("old-value", value) - self.assertIn( - old_name + " environment variable is deprecated", errors) - self.assertIn(name, errors) - del os.environ[old_name] - - def test_new_name_wins_and_nothing_is_printed(self): - for getter, (name, old_name) in PAIRS.items(): - with self.subTest(getter=getter): - os.environ[name] = "new-value" - os.environ[old_name] = "old-value" - value, errors = self.printed(getattr(ExampleUtils, getter)) - self.assertEqual("new-value", value) - self.assertEqual("", errors) - del os.environ[name] - del os.environ[old_name] - - def test_neither_name_gives_nothing(self): - for getter in PAIRS: - with self.subTest(getter=getter): - value, errors = self.printed(getattr(ExampleUtils, getter)) - self.assertFalse(value) - self.assertEqual("", errors) - - def test_old_name_is_reported_once(self): - getter, (name, old_name) = next(iter(PAIRS.items())) - os.environ[old_name] = "old-value" - - def twice(): - return (getattr(ExampleUtils, getter)(), - getattr(ExampleUtils, getter)()) - - value, errors = self.printed(twice) - self.assertEqual(("old-value", "old-value"), value) - self.assertEqual( - 1, errors.count(old_name + " environment variable is deprecated")) +class ConstantTests(unittest.TestCase): def test_constants_hold_the_old_and_the_new_names(self): for constant, value in CONSTANTS.items(): with self.subTest(constant=constant): self.assertEqual(value, getattr(ExampleUtils, constant)) - def test_key_and_endpoint_are_read_from_the_old_names_in_upper_case(self): - # Windows reads a variable name in any case, so the name the - # message gives is compared without regard to case. - for getter, upper_name in ( - ("get_resource_key", "RESOURCE_KEY"), - ("get_cloud_endpoint", "CLOUD_ENDPOINT")): - with self.subTest(getter=getter): - os.environ[upper_name] = "old-value" - value, errors = self.printed(getattr(ExampleUtils, getter)) - self.assertEqual("old-value", value) - self.assertIn( - upper_name.lower() - + " environment variable is deprecated", errors.lower()) - del os.environ[upper_name] - - def test_licence_key_is_not_read_in_upper_case(self): - # The licence key was only ever read under the lower case name. - # Windows reads a name in any case, so this holds elsewhere. - if os.name == "nt": - self.skipTest("Windows reads a variable name in any case") - os.environ["LICENSE_KEY"] = "old-value" - value, errors = self.printed(ExampleUtils.get_license_key) - self.assertFalse(value) - self.assertEqual("", errors) - - def test_data_file_is_found_under_the_old_name(self): - for name in DATA_FILE: - os.environ.pop(name, None) - with tempfile.TemporaryDirectory() as folder: - path = os.path.join(folder, "only-for-this-test.ipi") - with open(path, "wb") as data_file: - data_file.write(b"not a data file") - os.environ[DATA_FILE[1]] = path - value, errors = self.printed( - lambda: ExampleUtils.find_file("only-for-this-test.ipi")) - self.assertEqual(path, value) - self.assertIn( - DATA_FILE[1] + " environment variable is deprecated", errors) - if __name__ == "__main__": unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py index d6cca06c..6cf3b37b 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py @@ -43,8 +43,7 @@ # The settings the examples read, which a fresh clone does not have. SETTINGS = [ - "_51DEGREES_LICENSE_KEY", "license_key", - ExampleUtils.IPI_PATH_ENV_VAR, ExampleUtils.DATA_FILE_ENV_VAR] + "_51DEGREES_LICENSE_KEY", ExampleUtils.IPI_PATH_ENV_VAR] # Options that keep the performance example short. They leave the data file # to the example. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_find_out_more.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_find_out_more.py index fde30e0c..d9131276 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_find_out_more.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_find_out_more.py @@ -103,9 +103,8 @@ # The settings that let an example get past its early end. SETTINGS = [ - "_51DEGREES_RESOURCE_KEY", "resource_key", "RESOURCE_KEY", - "_51DEGREES_LICENSE_KEY", "license_key", - ExampleUtils.IPI_PATH_ENV_VAR, ExampleUtils.DATA_FILE_ENV_VAR] + "_51DEGREES_RESOURCE_KEY", "_51DEGREES_LICENSE_KEY", + ExampleUtils.IPI_PATH_ENV_VAR] def slug(path): diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_package_gettingstarted_from_root.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_package_gettingstarted_from_root.py index 454bfd06..bb364299 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_package_gettingstarted_from_root.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_package_gettingstarted_from_root.py @@ -59,8 +59,7 @@ def test_runs_from_the_repository_root(self): # the environment, as CI names the Enterprise file, is set # aside. with mock.patch.dict(os.environ): - for name in (ExampleUtils.IPI_PATH_ENV_VAR, - ExampleUtils.DATA_FILE_ENV_VAR): + for name in (ExampleUtils.IPI_PATH_ENV_VAR,): os.environ.pop(name, None) with redirect_stdout(printed): runpy.run_path(str(EXAMPLE), run_name="__main__") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini index dbce9c11..ed00ed06 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini @@ -45,12 +45,9 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_examples {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY - resource_key _51DEGREES_RESOURCE_KEY_BESPOKE _51DEGREES_LICENSE_KEY - license_key _51DEGREES_IPI_PATH - IPINTELLIGENCEDATAFILE TMPDIR TEMP TMP diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py index 3e5cdf7b..c03256a2 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py @@ -33,13 +33,12 @@ class TestConstants: IPV6_ADDRESS = "2001:4860:4860::8888" # The data file can be supplied via the _51DEGREES_IPI_PATH - # environment variable, or under its deprecated old name. By default + # environment variable. By default # the Lite file downloaded to the # repository's assets folder by ci/fetch-assets.ps1 is used. @staticmethod def get_data_file_path(): - env = read_environment( - "_51DEGREES_IPI_PATH", "IPINTELLIGENCEDATAFILE") + env = read_environment("_51DEGREES_IPI_PATH") if env: return env return os.path.join( diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/live_cloud.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/live_cloud.py index 2109e4a2..6cdbab0f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/live_cloud.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/live_cloud.py @@ -22,8 +22,8 @@ """Where the tests that call the live 51Degrees cloud service find their resource key, and what they fail with when there is none or when the -answer lacks what they asked for. Also how a test reads any setting from -an environment variable whose name has changed. +answer lacks what they asked for. Also how a test reads a setting from an +environment variable. The same file is in each test folder whose tests read a setting from the environment. There is no fallback to a skip, because a cloud test that @@ -31,7 +31,6 @@ """ import os -import sys # Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud # 51Did workflow sets it to each key in turn, and each product's @@ -41,68 +40,31 @@ "_51DEGREES_RESOURCE_KEY_BESPOKE", ) -# Deprecated. The name the resource key variable had before, read when -# none of RESOURCE_KEY_NAMES is set. -OLD_RESOURCE_KEY_NAME = "resource_key" - -# The old names already reported as deprecated, so each is reported once. -_reported_old_names = set() - ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" -def _report_old_name(name, old_name): - """Prints, once for each old name, that it is deprecated and which - name replaces it.""" - if old_name in _reported_old_names: - return - _reported_old_names.add(old_name) - print( - f"The {old_name} environment variable is deprecated and {name} " - f"replaces it. The value was read from {old_name} because {name} " - f"is not set, so set {name} to the same value.", - file=sys.stderr) - - -def read_environment(name, old_name): +def read_environment(name): """The value of the environment variable name, or None when it has - none. When name is unset or empty the name the variable had before is - read, and using it prints once that the old name is deprecated.""" + none.""" value = os.environ.get(name, "").strip() - if value: - return value - value = os.environ.get(old_name, "").strip() - if not value: - return None - _report_old_name(name, old_name) - return value + return value or None -def is_set(name, old_name): +def is_set(name): """Whether the environment variable name is set, whatever it holds, - which is how a switch is read. When it is not set, the name the - variable had before counts, and using it prints once that the old name - is deprecated.""" - if name in os.environ: - return True - if old_name not in os.environ: - return False - _report_old_name(name, old_name) - return True + which is how a switch is read.""" + return name in os.environ def find_resource_key(names=RESOURCE_KEY_NAMES): - """The first resource key set under any of the names, or None. When - none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" + """The first resource key set under any of the names, or None.""" for name in names: value = os.environ.get(name, "").strip() if value: return value - if names is RESOURCE_KEY_NAMES: - return read_environment(names[0], OLD_RESOURCE_KEY_NAME) return None diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_environment_names.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_environment_names.py deleted file mode 100644 index 5fe9f2e5..00000000 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_environment_names.py +++ /dev/null @@ -1,66 +0,0 @@ -# ********************************************************************* -# This Original Work is copyright of 51 Degrees Mobile Experts Limited. -# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, -# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. -# -# This Original Work is licensed under the European Union Public Licence -# (EUPL) v.1.2 and is subject to its terms as set out below. -# -# If a copy of the EUPL was not distributed with this file, You can obtain -# one at https://opensource.org/licenses/EUPL-1.2. -# -# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be -# amended by the European Commission) shall be deemed incompatible for -# the purposes of the Work and the provisions of the compatibility -# clause in Article 5 of the EUPL shall not apply. -# -# If using the Work as, or as part of, a network application, by -# including the attribution notice(s) required under Article 5 of the EUPL -# in the end user terms of the application under an appropriate heading, -# such notice(s) shall fulfill the requirements of that article. -# ********************************************************************* - -"""Checks that the tests still read the data file path under the name its -environment variable had before, when the new name is not set. -""" - -import contextlib -import io -import os -import unittest -from unittest import mock - -from .constants import TestConstants - -NEW_NAME = "_51DEGREES_IPI_PATH" -OLD_NAME = "IPINTELLIGENCEDATAFILE" - - -class OldNameTests(unittest.TestCase): - - def setUp(self): - environment = mock.patch.dict(os.environ) - environment.start() - self.addCleanup(environment.stop) - os.environ.pop(NEW_NAME, None) - os.environ.pop(OLD_NAME, None) - - def path(self): - with contextlib.redirect_stderr(io.StringIO()): - return TestConstants.get_data_file_path() - - def test_data_file_path_is_read_from_the_old_name(self): - os.environ[OLD_NAME] = "old.ipi" - self.assertEqual("old.ipi", self.path()) - - def test_new_name_wins(self): - os.environ[NEW_NAME] = "new.ipi" - os.environ[OLD_NAME] = "old.ipi" - self.assertEqual("new.ipi", self.path()) - - def test_neither_name_gives_the_lite_file(self): - self.assertTrue(self.path().endswith("51Degrees-LiteIpiV41.ipi")) - - -if __name__ == "__main__": - unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini b/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini index 8a13f223..b14580c8 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini @@ -35,11 +35,8 @@ commands = {[common]commands} pass_env = _51DEGREES_RESOURCE_KEY - resource_key _51DEGREES_LICENSE_KEY - license_key _51DEGREES_IPI_PATH - IPINTELLIGENCEDATAFILE TMPDIR TEMP TMP diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini b/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini index 43f49189..c304f959 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini @@ -22,9 +22,7 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_shared {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY - resource_key _51DEGREES_LICENSE_KEY - license_key [testenv:pre-publish] package = skip diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini b/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini index e033217e..fabb28ba 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini @@ -20,9 +20,7 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_translation {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY - resource_key _51DEGREES_LICENSE_KEY - license_key [testenv:pre-publish] package = skip diff --git a/location/examples/cloud/gettingstarted.py b/location/examples/cloud/gettingstarted.py index 6392d7ea..54366221 100644 --- a/location/examples/cloud/gettingstarted.py +++ b/location/examples/cloud/gettingstarted.py @@ -39,10 +39,7 @@ # You need to create a resource key at https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-gettingstarted.py&utm_term=top # and paste it into the code, replacing !!YOUR_RESOURCE_KEY!! below. # Alternatively, add a _51DEGREES_RESOURCE_KEY environment variable. -# resource_key is the old name of that variable. It is deprecated, and is -# read only when _51DEGREES_RESOURCE_KEY is not set. import os -import sys # Where to go next, printed when the example ends. FIND_OUT_MORE = [ @@ -57,25 +54,7 @@ ] -def read_environment(name, old_name): - """The value of the environment variable name, or None when it has - none. When name is unset or empty the name the variable had before is - read, and using it prints that the old name is deprecated.""" - value = os.environ.get(name) - if value: - return value - value = os.environ.get(old_name) - if not value: - return None - print( - f"The {old_name} environment variable is deprecated and {name} " - f"replaces it. The value was read from {old_name} because {name} " - f"is not set, so set {name} to the same value.", - file=sys.stderr) - return value - - -resource_key = read_environment("_51DEGREES_RESOURCE_KEY", "resource_key") +resource_key = os.environ.get("_51DEGREES_RESOURCE_KEY") if not resource_key: resource_key = "!!YOUR_RESOURCE_KEY!!" diff --git a/location/examples/cloud/web.py b/location/examples/cloud/web.py index 9fe962ec..6529e518 100644 --- a/location/examples/cloud/web.py +++ b/location/examples/cloud/web.py @@ -47,11 +47,8 @@ # You need to create a resource key at https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-web.py&utm_term=top # and paste it into the code, replacing !!YOUR_RESOURCE_KEY!! below. # Alternatively, add a _51DEGREES_RESOURCE_KEY environment variable. -# resource_key is the old name of that variable. It is deprecated, and is -# read only when _51DEGREES_RESOURCE_KEY is not set. import html import os -import sys # Where to go next, shown at the end of the page, and printed when there is # no resource key. @@ -90,25 +87,7 @@ def find_out_more_html(): return f"

    Find out more

      {items}
    " -def read_environment(name, old_name): - """The value of the environment variable name, or None when it has - none. When name is unset or empty the name the variable had before is - read, and using it prints that the old name is deprecated.""" - value = os.environ.get(name) - if value: - return value - value = os.environ.get(old_name) - if not value: - return None - print( - f"The {old_name} environment variable is deprecated and {name} " - f"replaces it. The value was read from {old_name} because {name} " - f"is not set, so set {name} to the same value.", - file=sys.stderr) - return value - - -resource_key = read_environment("_51DEGREES_RESOURCE_KEY", "resource_key") +resource_key = os.environ.get("_51DEGREES_RESOURCE_KEY") if not resource_key: resource_key = "!!YOUR_RESOURCE_KEY!!" diff --git a/location/tests/live_cloud.py b/location/tests/live_cloud.py index 2109e4a2..6cdbab0f 100644 --- a/location/tests/live_cloud.py +++ b/location/tests/live_cloud.py @@ -22,8 +22,8 @@ """Where the tests that call the live 51Degrees cloud service find their resource key, and what they fail with when there is none or when the -answer lacks what they asked for. Also how a test reads any setting from -an environment variable whose name has changed. +answer lacks what they asked for. Also how a test reads a setting from an +environment variable. The same file is in each test folder whose tests read a setting from the environment. There is no fallback to a skip, because a cloud test that @@ -31,7 +31,6 @@ """ import os -import sys # Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud # 51Did workflow sets it to each key in turn, and each product's @@ -41,68 +40,31 @@ "_51DEGREES_RESOURCE_KEY_BESPOKE", ) -# Deprecated. The name the resource key variable had before, read when -# none of RESOURCE_KEY_NAMES is set. -OLD_RESOURCE_KEY_NAME = "resource_key" - -# The old names already reported as deprecated, so each is reported once. -_reported_old_names = set() - ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" -def _report_old_name(name, old_name): - """Prints, once for each old name, that it is deprecated and which - name replaces it.""" - if old_name in _reported_old_names: - return - _reported_old_names.add(old_name) - print( - f"The {old_name} environment variable is deprecated and {name} " - f"replaces it. The value was read from {old_name} because {name} " - f"is not set, so set {name} to the same value.", - file=sys.stderr) - - -def read_environment(name, old_name): +def read_environment(name): """The value of the environment variable name, or None when it has - none. When name is unset or empty the name the variable had before is - read, and using it prints once that the old name is deprecated.""" + none.""" value = os.environ.get(name, "").strip() - if value: - return value - value = os.environ.get(old_name, "").strip() - if not value: - return None - _report_old_name(name, old_name) - return value + return value or None -def is_set(name, old_name): +def is_set(name): """Whether the environment variable name is set, whatever it holds, - which is how a switch is read. When it is not set, the name the - variable had before counts, and using it prints once that the old name - is deprecated.""" - if name in os.environ: - return True - if old_name not in os.environ: - return False - _report_old_name(name, old_name) - return True + which is how a switch is read.""" + return name in os.environ def find_resource_key(names=RESOURCE_KEY_NAMES): - """The first resource key set under any of the names, or None. When - none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" + """The first resource key set under any of the names, or None.""" for name in names: value = os.environ.get(name, "").strip() if value: return value - if names is RESOURCE_KEY_NAMES: - return read_environment(names[0], OLD_RESOURCE_KEY_NAME) return None diff --git a/location/tests/test_environment_names.py b/location/tests/test_environment_names.py deleted file mode 100644 index f3ad6f0a..00000000 --- a/location/tests/test_environment_names.py +++ /dev/null @@ -1,114 +0,0 @@ -# ********************************************************************* -# This Original Work is copyright of 51 Degrees Mobile Experts Limited. -# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, -# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. -# -# This Original Work is licensed under the European Union Public Licence -# (EUPL) v.1.2 and is subject to its terms as set out below. -# -# If a copy of the EUPL was not distributed with this file, You can obtain -# one at https://opensource.org/licenses/EUPL-1.2. -# -# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be -# amended by the European Commission) shall be deemed incompatible for -# the purposes of the Work and the provisions of the compatibility -# clause in Article 5 of the EUPL shall not apply. -# -# If using the Work as, or as part of, a network application, by -# including the attribution notice(s) required under Article 5 of the EUPL -# in the end user terms of the application under an appropriate heading, -# such notice(s) shall fulfill the requirements of that article. -# ********************************************************************* - -"""Checks that the two cloud examples still read the resource key under -the name its environment variable had before, when the new name is not set. - -Importing an example runs it, which calls the cloud service, so the -function each example reads its key with is taken out of the example's -source and run on its own. -""" - -import ast -import contextlib -import io -import os -import sys -import unittest -from pathlib import Path -from unittest import mock - -EXAMPLES = Path(__file__).resolve().parent.parent / "examples" / "cloud" - -NEW_NAME = "_51DEGREES_RESOURCE_KEY" -OLD_NAME = "resource_key" - - -def reader(example): - """The read_environment function of an example, or None when the - example has none.""" - source = (EXAMPLES / example).read_text(encoding="utf-8") - for node in ast.parse(source).body: - if isinstance(node, ast.FunctionDef) \ - and node.name == "read_environment": - module = ast.Module(body=[node], type_ignores=[]) - names = {"os": os, "sys": sys} - exec(compile(module, example, "exec"), names) - return names["read_environment"] - return None - - -class OldNameTests(unittest.TestCase): - - def setUp(self): - environment = mock.patch.dict(os.environ) - environment.start() - self.addCleanup(environment.stop) - os.environ.pop(NEW_NAME, None) - os.environ.pop(OLD_NAME, None) - - def read(self, example): - """What the example reads for its resource key, and what it prints - to the error stream.""" - read_environment = reader(example) - self.assertIsNotNone( - read_environment, - example + " has no read_environment function, so it does not " - "read the old name") - # The example has to read the key through the function. - self.assertIn( - 'read_environment("' + NEW_NAME + '", "' + OLD_NAME + '")', - (EXAMPLES / example).read_text(encoding="utf-8")) - errors = io.StringIO() - with contextlib.redirect_stderr(errors): - value = read_environment(NEW_NAME, OLD_NAME) - return value, errors.getvalue() - - def test_resource_key_is_read_from_the_old_name(self): - for example in ("gettingstarted.py", "web.py"): - with self.subTest(example=example): - os.environ[OLD_NAME] = "old-key" - value, errors = self.read(example) - self.assertEqual("old-key", value) - self.assertIn( - OLD_NAME + " environment variable is deprecated", errors) - self.assertIn(NEW_NAME, errors) - - def test_new_name_wins_and_nothing_is_printed(self): - for example in ("gettingstarted.py", "web.py"): - with self.subTest(example=example): - os.environ[NEW_NAME] = "new-key" - os.environ[OLD_NAME] = "old-key" - value, errors = self.read(example) - self.assertEqual("new-key", value) - self.assertEqual("", errors) - - def test_neither_name_gives_none(self): - for example in ("gettingstarted.py", "web.py"): - with self.subTest(example=example): - value, errors = self.read(example) - self.assertIsNone(value) - self.assertEqual("", errors) - - -if __name__ == "__main__": - unittest.main() diff --git a/location/tests/test_examples.py b/location/tests/test_examples.py index d01a5bee..df9b0447 100644 --- a/location/tests/test_examples.py +++ b/location/tests/test_examples.py @@ -30,7 +30,7 @@ class LocationExampleTests(unittest.TestCase): def test_cloud_getting_started(self): - if read_environment("_51DEGREES_RESOURCE_KEY", "resource_key"): + if read_environment("_51DEGREES_RESOURCE_KEY"): # this is required to import the example code, # since pytest sets sys.path to the test directory sys.path.append(str(Path(__file__).parent.parent.absolute() / "examples")) diff --git a/location/tests/test_properties51D.py b/location/tests/test_properties51D.py index a31468c2..2ba50631 100644 --- a/location/tests/test_properties51D.py +++ b/location/tests/test_properties51D.py @@ -30,7 +30,7 @@ location_LAT = '51.4578261'; location_LON = '-0.975922996290084'; -resource_key = read_environment("_51DEGREES_RESOURCE_KEY", "resource_key") +resource_key = read_environment("_51DEGREES_RESOURCE_KEY") if not resource_key: raise Exception("To run the cloud tests, please set a valid 51Degrees " "cloud resource key as the _51DEGREES_RESOURCE_KEY environment variable.") diff --git a/location/tox.ini b/location/tox.ini index f8d1cda1..dff16572 100644 --- a/location/tox.ini +++ b/location/tox.ini @@ -25,9 +25,7 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_location {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY - resource_key _51DEGREES_LICENSE_KEY - license_key # Tests the packages just built, so it takes the test tools from [common] # and every package from the requirements file, never the editable copies diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/example_utils.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/example_utils.py index e1534c4d..6d5e6d2d 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/example_utils.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/example_utils.py @@ -37,7 +37,7 @@ class ExampleUtils: RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY" # Deprecated. Use RESOURCE_KEY_ENV_VAR. The name the resource key - # variable had before, read when RESOURCE_KEY_ENV_VAR is not set. + # variable had before, which is no longer read. LEGACY_RESOURCE_KEY_ENV_VAR = "resource_key" # The environment variable the examples read a licence key from, to use @@ -45,7 +45,7 @@ class ExampleUtils: LICENSE_KEY_ENV_VAR = "_51DEGREES_LICENSE_KEY" # Deprecated. Use LICENSE_KEY_ENV_VAR. The name the licence key - # variable had before, read when LICENSE_KEY_ENV_VAR is not set. + # variable had before, which is no longer read. LEGACY_LICENSE_KEY_ENV_VAR = "license_key" # The environment variable naming a cloud service other than the public @@ -54,7 +54,7 @@ class ExampleUtils: CLOUD_ENDPOINT_ENV_VAR = "_51DEGREES_CLOUD_ENDPOINT" # Deprecated. Use CLOUD_ENDPOINT_ENV_VAR. The name that variable had - # before, read when CLOUD_ENDPOINT_ENV_VAR is not set. + # before, which is no longer read. ENDPOINT_ENV_VAR = "cloud_endpoint" # The environment variable holding the origin to send with each request @@ -63,7 +63,7 @@ class ExampleUtils: CLOUD_REQUEST_ORIGIN_ENV_VAR = "_51DEGREES_CLOUD_REQUEST_ORIGIN" # Deprecated. Use CLOUD_REQUEST_ORIGIN_ENV_VAR. The name that variable - # had before, read when CLOUD_REQUEST_ORIGIN_ENV_VAR is not set. + # had before, which is no longer read. ORIGIN_ENV_VAR = "cloud_request_origin" # The text shown for a value the results do not hold. @@ -72,53 +72,30 @@ class ExampleUtils: @staticmethod def get_resource_key(): return ExampleUtils.__get_env_variable( - ExampleUtils.RESOURCE_KEY_ENV_VAR, - ExampleUtils.LEGACY_RESOURCE_KEY_ENV_VAR) + ExampleUtils.RESOURCE_KEY_ENV_VAR) @staticmethod def get_license_key(): return ExampleUtils.__get_env_variable( - ExampleUtils.LICENSE_KEY_ENV_VAR, - ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR) + ExampleUtils.LICENSE_KEY_ENV_VAR) @staticmethod def get_cloud_endpoint(): return ExampleUtils.__get_env_variable( - ExampleUtils.CLOUD_ENDPOINT_ENV_VAR, - ExampleUtils.ENDPOINT_ENV_VAR) + ExampleUtils.CLOUD_ENDPOINT_ENV_VAR) @staticmethod def get_cloud_request_origin(): return ExampleUtils.__get_env_variable( - ExampleUtils.CLOUD_REQUEST_ORIGIN_ENV_VAR, - ExampleUtils.ORIGIN_ENV_VAR) - - # The old names already reported as deprecated, so each is reported - # once. - __reported_old_names = set() + ExampleUtils.CLOUD_REQUEST_ORIGIN_ENV_VAR) @staticmethod - def __get_env_variable(name, old_name): + def __get_env_variable(name): """! The value of the environment variable name, or an empty string. - When name is unset or empty the name the variable had before is - read, and using it prints once that the old name is deprecated. """ - value = os.environ.get(name) - if value: - return value - value = os.environ.get(old_name) - if not value: - return "" - if old_name not in ExampleUtils.__reported_old_names: - ExampleUtils.__reported_old_names.add(old_name) - print( - f"The {old_name} environment variable is deprecated and {name} " - f"replaces it. The value was read from {old_name} because {name} " - f"is not set, so set {name} to the same value.", - file=sys.stderr) - return value + return os.environ.get(name) or "" @staticmethod def name_values(section, property_names): diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py index 2109e4a2..6cdbab0f 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py @@ -22,8 +22,8 @@ """Where the tests that call the live 51Degrees cloud service find their resource key, and what they fail with when there is none or when the -answer lacks what they asked for. Also how a test reads any setting from -an environment variable whose name has changed. +answer lacks what they asked for. Also how a test reads a setting from an +environment variable. The same file is in each test folder whose tests read a setting from the environment. There is no fallback to a skip, because a cloud test that @@ -31,7 +31,6 @@ """ import os -import sys # Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud # 51Did workflow sets it to each key in turn, and each product's @@ -41,68 +40,31 @@ "_51DEGREES_RESOURCE_KEY_BESPOKE", ) -# Deprecated. The name the resource key variable had before, read when -# none of RESOURCE_KEY_NAMES is set. -OLD_RESOURCE_KEY_NAME = "resource_key" - -# The old names already reported as deprecated, so each is reported once. -_reported_old_names = set() - ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" -def _report_old_name(name, old_name): - """Prints, once for each old name, that it is deprecated and which - name replaces it.""" - if old_name in _reported_old_names: - return - _reported_old_names.add(old_name) - print( - f"The {old_name} environment variable is deprecated and {name} " - f"replaces it. The value was read from {old_name} because {name} " - f"is not set, so set {name} to the same value.", - file=sys.stderr) - - -def read_environment(name, old_name): +def read_environment(name): """The value of the environment variable name, or None when it has - none. When name is unset or empty the name the variable had before is - read, and using it prints once that the old name is deprecated.""" + none.""" value = os.environ.get(name, "").strip() - if value: - return value - value = os.environ.get(old_name, "").strip() - if not value: - return None - _report_old_name(name, old_name) - return value + return value or None -def is_set(name, old_name): +def is_set(name): """Whether the environment variable name is set, whatever it holds, - which is how a switch is read. When it is not set, the name the - variable had before counts, and using it prints once that the old name - is deprecated.""" - if name in os.environ: - return True - if old_name not in os.environ: - return False - _report_old_name(name, old_name) - return True + which is how a switch is read.""" + return name in os.environ def find_resource_key(names=RESOURCE_KEY_NAMES): - """The first resource key set under any of the names, or None. When - none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" + """The first resource key set under any of the names, or None.""" for name in names: value = os.environ.get(name, "").strip() if value: return value - if names is RESOURCE_KEY_NAMES: - return read_environment(names[0], OLD_RESOURCE_KEY_NAME) return None diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_without_key.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_without_key.py index 255b2571..31507975 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_without_key.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_without_key.py @@ -40,8 +40,8 @@ # Every name a key can be read from, which a fresh clone does not set. KEY_SETTINGS = [ - "_51DEGREES_RESOURCE_KEY", "resource_key", "RESOURCE_KEY", - "_51DEGREES_LICENSE_KEY", "license_key", "LICENSE_KEY"] + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_LICENSE_KEY"] # A link to the page where a resource key is created, with the campaign # tags every example's links carry. diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_example_utils.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_example_utils.py index 1b6cc187..44fbacd9 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_example_utils.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_example_utils.py @@ -32,56 +32,32 @@ import pytest -# Each reader of a setting, with the new name of its variable and the old -# one, which is still read when the new name is not set. -OLD_NAMES = [ - ("get_resource_key", "_51DEGREES_RESOURCE_KEY", "resource_key"), - ("get_license_key", "_51DEGREES_LICENSE_KEY", "license_key"), - ("get_cloud_endpoint", "_51DEGREES_CLOUD_ENDPOINT", "cloud_endpoint"), - ("get_cloud_request_origin", "_51DEGREES_CLOUD_REQUEST_ORIGIN", - "cloud_request_origin"), +# Each reader of a setting, with the name of its variable. +NAMES = [ + ("get_resource_key", "_51DEGREES_RESOURCE_KEY"), + ("get_license_key", "_51DEGREES_LICENSE_KEY"), + ("get_cloud_endpoint", "_51DEGREES_CLOUD_ENDPOINT"), + ("get_cloud_request_origin", "_51DEGREES_CLOUD_REQUEST_ORIGIN"), ] @pytest.fixture def no_settings(monkeypatch): - """None of the settings in the environment, and no old name reported - as deprecated yet.""" - for _, name, old_name in OLD_NAMES: + """None of the settings in the environment.""" + for _, name in NAMES: monkeypatch.delenv(name, raising=False) - monkeypatch.delenv(old_name, raising=False) - monkeypatch.setattr( - ExampleUtils, "_ExampleUtils__reported_old_names", set()) return monkeypatch -@pytest.mark.parametrize("getter, name, old_name", OLD_NAMES) -def test_setting_is_read_from_its_old_name_and_reported_once( - getter, name, old_name, no_settings, capsys): - no_settings.setenv(old_name, "old-value") - assert getattr(ExampleUtils, getter)() == "old-value" - assert getattr(ExampleUtils, getter)() == "old-value" - errors = capsys.readouterr().err - assert errors.count( - old_name + " environment variable is deprecated") == 1 - assert name in errors - - -@pytest.mark.parametrize("getter, name, old_name", OLD_NAMES) -def test_new_name_wins_and_nothing_is_printed( - getter, name, old_name, no_settings, capsys): +@pytest.mark.parametrize("getter, name", NAMES) +def test_setting_is_read(getter, name, no_settings): no_settings.setenv(name, "new-value") - no_settings.setenv(old_name, "old-value") assert getattr(ExampleUtils, getter)() == "new-value" - assert capsys.readouterr().err == "" -@pytest.mark.parametrize("getter, name, old_name", OLD_NAMES) -def test_neither_name_gives_an_empty_string( - getter, name, old_name, no_settings, capsys): +@pytest.mark.parametrize("getter, name", NAMES) +def test_no_setting_gives_an_empty_string(getter, name, no_settings): assert getattr(ExampleUtils, getter)() == "" - assert capsys.readouterr().err == "" - def test_constants_hold_the_old_and_the_new_names(): assert ExampleUtils.RESOURCE_KEY_ENV_VAR == "_51DEGREES_RESOURCE_KEY" diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_find_out_more.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_find_out_more.py index 34195956..20a186c2 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_find_out_more.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_find_out_more.py @@ -32,9 +32,7 @@ # Every name a key is read from. KEY_SETTINGS = [ ExampleUtils.RESOURCE_KEY_ENV_VAR, - ExampleUtils.LEGACY_RESOURCE_KEY_ENV_VAR, ExampleUtils.LICENSE_KEY_ENV_VAR, - ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR, ] diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini index cd1bd54a..fbd5e095 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini @@ -37,15 +37,10 @@ commands = # examples for a resource key limited to particular domains. pass_env = _51DEGREES_RESOURCE_KEY - resource_key _51DEGREES_RESOURCE_KEY_BESPOKE _51DEGREES_LICENSE_KEY - license_key _51DEGREES_CLOUD_ENDPOINT - FOD_CLOUD_API_URL - cloud_endpoint _51DEGREES_CLOUD_REQUEST_ORIGIN - cloud_request_origin [testenv:pre-publish] deps = diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md index 1ff74cdd..8b8dc507 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md @@ -123,10 +123,8 @@ The engine calls `https://cloud.51degrees.com/api/v4/` unless told otherwise. Set the `_51DEGREES_CLOUD_ENDPOINT` environment variable, or pass `cloud_endpoint` in the engine settings, to the API base of another host including the `/api/v4/` segment. `FOD_CLOUD_API_URL`, the old name -of that variable, is deprecated. The engine reads it only when -`_51DEGREES_CLOUD_ENDPOINT` is not set, and warns once that the old name -was used. `Constants.FOD_CLOUD_API_URL` is deprecated as well, and -`Constants.CLOUD_ENDPOINT` replaces it. A host other than +of that variable, is no longer read. `Constants.FOD_CLOUD_API_URL` is +deprecated, and `Constants.CLOUD_ENDPOINT` replaces it. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons. This is the private hosting option of the cloud diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py index 9e17100b..ac18349d 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py @@ -26,7 +26,6 @@ import json import math import os -import threading import warnings from functools import cached_property from json import JSONDecodeError @@ -51,48 +50,17 @@ from urllib.parse import urlencode -# Whether the old end point variable has been reported as deprecated, so -# that it is reported once however many engines are built. -_old_endpoint_reported = False -_old_endpoint_lock = threading.Lock() - - def _endpoint_from_environment(): """! - The cloud end point the environment gives, or None when it gives none. - - Constants.CLOUD_ENDPOINT is read first. When it is unset or empty the - variable Constants.FOD_CLOUD_API_URL names is read, which is the old - name, and the first use of it is reported as deprecated. The report is - a FutureWarning, which Python shows by default, because the people who - set the variable are the ones running the application. + The cloud end point that Constants.CLOUD_ENDPOINT names, or None when + it is unset or empty. @rtype: string @return: the end point, or None """ - global _old_endpoint_reported - - endpoint = os.environ.get(Constants.CLOUD_ENDPOINT) - if endpoint: - return endpoint - - old_name = Constants.FOD_CLOUD_API_URL - endpoint = os.environ.get(old_name) - if not endpoint: - return None - - with _old_endpoint_lock: - report = not _old_endpoint_reported - _old_endpoint_reported = True - if report: - # The third frame is the code that built the engine. - warnings.warn( - Constants.MESSAGE_OLD_ENDPOINT_VARIABLE.format( - old_name, Constants.CLOUD_ENDPOINT), - FutureWarning, stacklevel=3) - return endpoint + return os.environ.get(Constants.CLOUD_ENDPOINT) or None def _takes_timeout(client): @@ -196,8 +164,7 @@ def __init__(self, settings = {}): self.reported_not_covered = set() # The endpoint is the cloud_endpoint setting, then the - # _51DEGREES_CLOUD_ENDPOINT environment variable, then that - # variable's deprecated old name FOD_CLOUD_API_URL, then + # _51DEGREES_CLOUD_ENDPOINT environment variable, then # cloud.51degrees.com. A host other than cloud.51degrees.com # would be used to (a) use an on premise web server, or (b) use # a privately hosted version of the 51Degrees cloud for diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py index 0783fe76..38d31ba5 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py @@ -25,19 +25,9 @@ class Constants: CLOUD_ENDPOINT = "_51DEGREES_CLOUD_ENDPOINT" # @deprecated Use Constants.CLOUD_ENDPOINT. This holds the old name of - # the cloud end point environment variable, which the engine reads - # through this constant when CLOUD_ENDPOINT is not set, so code that - # passes this constant to the environment reads and sets the old - # variable. + # the cloud end point environment variable, which is no longer read. FOD_CLOUD_API_URL = "FOD_CLOUD_API_URL" - # Message when the cloud end point came from the old environment - # variable. The first value is the old name and the second the new one. - MESSAGE_OLD_ENDPOINT_VARIABLE = ( - "The {0} environment variable is deprecated and {1} replaces it. " - "The cloud endpoint was read from {0} because {1} is not set, so " - "set {1} to the same value.") - # Default cloud end point BASE_URL_DEFAULT = "https://cloud.51degrees.com/api/v4/" diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/live_cloud.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/live_cloud.py index 2109e4a2..6cdbab0f 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/live_cloud.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/live_cloud.py @@ -22,8 +22,8 @@ """Where the tests that call the live 51Degrees cloud service find their resource key, and what they fail with when there is none or when the -answer lacks what they asked for. Also how a test reads any setting from -an environment variable whose name has changed. +answer lacks what they asked for. Also how a test reads a setting from an +environment variable. The same file is in each test folder whose tests read a setting from the environment. There is no fallback to a skip, because a cloud test that @@ -31,7 +31,6 @@ """ import os -import sys # Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud # 51Did workflow sets it to each key in turn, and each product's @@ -41,68 +40,31 @@ "_51DEGREES_RESOURCE_KEY_BESPOKE", ) -# Deprecated. The name the resource key variable had before, read when -# none of RESOURCE_KEY_NAMES is set. -OLD_RESOURCE_KEY_NAME = "resource_key" - -# The old names already reported as deprecated, so each is reported once. -_reported_old_names = set() - ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" -def _report_old_name(name, old_name): - """Prints, once for each old name, that it is deprecated and which - name replaces it.""" - if old_name in _reported_old_names: - return - _reported_old_names.add(old_name) - print( - f"The {old_name} environment variable is deprecated and {name} " - f"replaces it. The value was read from {old_name} because {name} " - f"is not set, so set {name} to the same value.", - file=sys.stderr) - - -def read_environment(name, old_name): +def read_environment(name): """The value of the environment variable name, or None when it has - none. When name is unset or empty the name the variable had before is - read, and using it prints once that the old name is deprecated.""" + none.""" value = os.environ.get(name, "").strip() - if value: - return value - value = os.environ.get(old_name, "").strip() - if not value: - return None - _report_old_name(name, old_name) - return value + return value or None -def is_set(name, old_name): +def is_set(name): """Whether the environment variable name is set, whatever it holds, - which is how a switch is read. When it is not set, the name the - variable had before counts, and using it prints once that the old name - is deprecated.""" - if name in os.environ: - return True - if old_name not in os.environ: - return False - _report_old_name(name, old_name) - return True + which is how a switch is read.""" + return name in os.environ def find_resource_key(names=RESOURCE_KEY_NAMES): - """The first resource key set under any of the names, or None. When - none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" + """The first resource key set under any of the names, or None.""" for name in names: value = os.environ.get(name, "").strip() if value: return value - if names is RESOURCE_KEY_NAMES: - return read_environment(names[0], OLD_RESOURCE_KEY_NAME) return None diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py index 831734c7..fe08226c 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py @@ -36,7 +36,7 @@ class CloudEngineTests(unittest.TestCase): def setUp(self): self.resource_key = read_environment( - "_51DEGREES_RESOURCE_KEY", "resource_key") + "_51DEGREES_RESOURCE_KEY") self.assertIsNotNone( self.resource_key, diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py index 1078da4f..aa7cfcea 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py @@ -47,26 +47,19 @@ class TestCloudRequestEngine(CloudRequestEngineTestsBase): testResourceKey = "resource_key" testEndPoint = "http://testEndPoint/" testEnvVarEndPoint = "http://testEnvVarEndPoint/" - testOldEnvVarEndPoint = "http://testOldEnvVarEndPoint/" # The name the endpoint environment variable had before - # Constants.CLOUD_ENDPOINT, written out so that these tests do not - # depend on the deprecated constant that holds it. + # Constants.CLOUD_ENDPOINT, which is no longer read, written out so that + # the test of the deprecated constant that holds it does not depend on it. oldEndPointVariable = "FOD_CLOUD_API_URL" def setUp(self): - # Each test starts with neither name of the endpoint variable set - # and with the old name not yet reported as deprecated, and what - # it sets is put back afterwards. + # Each test starts with the endpoint variable not set, and what it + # sets is put back afterwards. environment = mock.patch.dict(os.environ) environment.start() self.addCleanup(environment.stop) os.environ.pop(Constants.CLOUD_ENDPOINT, None) - os.environ.pop(self.oldEndPointVariable, None) - reported = mock.patch.object( - cloudrequestengine, "_old_endpoint_reported", False) - reported.start() - self.addCleanup(reported.stop) def new_engine(self, **settings): """! @@ -80,18 +73,6 @@ def new_engine(self, **settings): **settings }) - def old_name_reports(self, caught): - """! - The warnings in a recorded list that say the old endpoint variable - is deprecated and name the one that replaces it. - """ - - return [ - warning for warning in caught - if issubclass(warning.category, FutureWarning) - and self.oldEndPointVariable in str(warning.message) - and Constants.CLOUD_ENDPOINT in str(warning.message)] - def testConfigEndPoint_ExplicitSetting(self): """! Check that the explicitly setting the cloud endpoint via @@ -130,86 +111,6 @@ def testConfigEndPoint_DefaultSetting(self): }) self.assertEqual(Constants.BASE_URL_DEFAULT, cloud_request_engine.baseURL) - def testConfigEndPoint_OldEnvironmentVariable(self): - """! - Check that the old name of the endpoint environment variable is - still read when the new name is not set - """ - - os.environ[self.oldEndPointVariable] = self.testOldEnvVarEndPoint - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - engine = self.new_engine() - self.assertEqual(self.testOldEnvVarEndPoint, engine.baseURL) - - def testConfigEndPoint_OldEnvironmentVariable_NewNameEmpty(self): - """! - Check that the old name is read when the new name is set to an - empty value, which counts as not set - """ - - os.environ[Constants.CLOUD_ENDPOINT] = "" - os.environ[self.oldEndPointVariable] = self.testOldEnvVarEndPoint - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - engine = self.new_engine() - self.assertEqual(self.testOldEnvVarEndPoint, engine.baseURL) - - def testConfigEndPoint_OldEnvironmentVariable_Empty(self): - """! - Check that the default is used when the old name is set to an - empty value and the new name is not set, with nothing reported - """ - - os.environ[self.oldEndPointVariable] = "" - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - engine = self.new_engine() - self.assertEqual(Constants.BASE_URL_DEFAULT, engine.baseURL) - self.assertEqual([], self.old_name_reports(caught)) - - def testConfigEndPoint_NewEnvironmentVariableWins(self): - """! - Check that the new name wins when both names are set, and that the - old name is not reported because it was not used - """ - - os.environ[Constants.CLOUD_ENDPOINT] = self.testEnvVarEndPoint - os.environ[self.oldEndPointVariable] = self.testOldEnvVarEndPoint - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - engine = self.new_engine() - self.assertEqual(self.testEnvVarEndPoint, engine.baseURL) - self.assertEqual([], self.old_name_reports(caught)) - - def testConfigEndPoint_ExplicitSettingOverOldEnvironmentVariable(self): - """! - Check that the cloud_endpoint setting wins over the old name, and - that the old name is not reported because it was not used - """ - - os.environ[self.oldEndPointVariable] = self.testOldEnvVarEndPoint - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - engine = self.new_engine(cloud_endpoint=self.testEndPoint) - self.assertEqual(self.testEndPoint, engine.baseURL) - self.assertEqual([], self.old_name_reports(caught)) - - def testConfigEndPoint_OldEnvironmentVariableReportedOnce(self): - """! - Check that using the old name is reported as deprecated once, - naming the new name, however many engines are built - """ - - os.environ[self.oldEndPointVariable] = self.testOldEnvVarEndPoint - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - first = self.new_engine() - second = self.new_engine() - self.assertEqual(self.testOldEnvVarEndPoint, first.baseURL) - self.assertEqual(self.testOldEnvVarEndPoint, second.baseURL) - self.assertEqual(1, len(self.old_name_reports(caught))) - def testOldEndPointConstant(self): """! Check that the constant the old name was held in is still there @@ -224,21 +125,6 @@ def testOldEndPointConstant(self): vars(Constants).get("FOD_CLOUD_API_URL")) self.assertIn("FOD_CLOUD_API_URL", dir(Constants)) - def testConfigEndPoint_VariableTheOldConstantNames(self): - """! - Check that the engine reads the old name through the constant, as - it did before, so that code which points the constant at another - variable still decides what is read - """ - - os.environ["ANOTHER_VARIABLE"] = self.testOldEnvVarEndPoint - with mock.patch.object( - Constants, "FOD_CLOUD_API_URL", "ANOTHER_VARIABLE"): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - engine = self.new_engine() - self.assertEqual(self.testOldEnvVarEndPoint, engine.baseURL) - def testEndPointConstant(self): """! Check that the constant for the new name holds the new name diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licensekey_live.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licensekey_live.py index aff8943b..2458881b 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licensekey_live.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licensekey_live.py @@ -34,7 +34,7 @@ # The licence key the live tests authenticate with, read from the variable # the examples in this repository read it from. LICENSE_KEY = read_environment( - "_51DEGREES_LICENSE_KEY", "license_key") or "" + "_51DEGREES_LICENSE_KEY") or "" USER_AGENT = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) " diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini index 6c759a83..646020a0 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini @@ -20,9 +20,7 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_cloudrequestengine {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY - resource_key _51DEGREES_LICENSE_KEY - license_key [testenv:pre-publish] package = skip diff --git a/pipeline/fiftyone_pipeline_core/tox.ini b/pipeline/fiftyone_pipeline_core/tox.ini index 1f52d5d0..9a03f3a3 100644 --- a/pipeline/fiftyone_pipeline_core/tox.ini +++ b/pipeline/fiftyone_pipeline_core/tox.ini @@ -22,9 +22,7 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_core {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY - resource_key _51DEGREES_LICENSE_KEY - license_key [testenv:pre-publish] package = skip diff --git a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py index 343d87d4..a4a5e034 100644 --- a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py +++ b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py @@ -63,13 +63,10 @@ Environment variables. ``_51DEGREES_RESOURCE_KEY`` is required. ``_51DEGREES_LICENSE_KEY`` is optional, and the comment in ``run`` says -why. ``RESOURCE_KEY`` and ``LICENSE_KEY`` are the old names of those two. -They are deprecated, and each is read only when its new name is not set. +why. ``_51DEGREES_CLOUD_ENDPOINT`` is the cloud API base including the ``/api/v4/`` segment, defaulting to ``https://cloud.51degrees.com/api/v4/``, and is the same variable the cloud request engine and the ``DidClient`` honour. -``FOD_CLOUD_API_URL`` is that variable's old name. It is deprecated, and -is read only when ``_51DEGREES_CLOUD_ENDPOINT`` is not set. ``PORT`` is the port to listen on, defaulting to 5100. Standard library plus this package. Run ``python server.py`` then open @@ -82,7 +79,6 @@ import secrets import sys import urllib.parse -import warnings from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path @@ -106,27 +102,14 @@ DEFAULT_API = "https://cloud.51degrees.com/api/v4/" -def env(name, old_name): +def env(name): """The named environment variable when it is set and not empty, - otherwise None. When it is unset or empty the name the variable had - before is read, with a warning that the old name is deprecated, as - the ``DidClient`` gives for the cloud endpoint.""" - value = os.environ.get(name) - if value: - return value - value = os.environ.get(old_name) - if not value: - return None - warnings.warn( - "The {0} environment variable is deprecated and {1} replaces it. " - "The value was read from {0} because {1} is not set, so set {1} " - "to the same value.".format(old_name, name), - FutureWarning, stacklevel=2) - return value - - -RESOURCE = env("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY") -LICENCE = env("_51DEGREES_LICENSE_KEY", "LICENSE_KEY") or "" + otherwise None.""" + return os.environ.get(name) or None + + +RESOURCE = env("_51DEGREES_RESOURCE_KEY") +LICENCE = env("_51DEGREES_LICENSE_KEY") or "" # The cloud API base, normalised to end in exactly one slash so that # every URL is the base followed by its path. The page receives the same # value through its __API__ placeholder and builds its two cloud calls @@ -136,7 +119,7 @@ def env(name, old_name): # 51Degrees cloud for performance reasons. That is the private hosting # option of the cloud service, and both run the same service, so this # demo works unchanged against either. -API = (env("_51DEGREES_CLOUD_ENDPOINT", "FOD_CLOUD_API_URL") +API = (env("_51DEGREES_CLOUD_ENDPOINT") or DEFAULT_API).rstrip("/") + "/" PORT = int(os.environ.get("PORT", "5100")) diff --git a/pipeline/fiftyone_pipeline_did/readme.md b/pipeline/fiftyone_pipeline_did/readme.md index 55164459..8c2b2730 100644 --- a/pipeline/fiftyone_pipeline_did/readme.md +++ b/pipeline/fiftyone_pipeline_did/readme.md @@ -438,7 +438,7 @@ client = DidClient(resource_key, licence_key, transport=httpx_transport) | --- | --- | | `resource_key` | Required. The page's resource key, public by nature. It travels in the route of the key and verify requests and in the form body of the redeem request | | `licence_key` | Optional. A licence key of the same account, server side only. Needed to redeem where the account holds licence keys. Sent only in the body of the redeem request, never in a URL | -| `endpoint` | Optional. The API base including the `/api/v4/` segment. Defaults to the `_51DEGREES_CLOUD_ENDPOINT` environment variable, the same variable the cloud request engine honours, then to `FOD_CLOUD_API_URL`, the deprecated old name of that variable, then to `https://cloud.51degrees.com/api/v4/`. The client warns once when the old name is the one used. A value without a trailing slash gains one | +| `endpoint` | Optional. The API base including the `/api/v4/` segment. Defaults to the `_51DEGREES_CLOUD_ENDPOINT` environment variable, the same variable the cloud request engine honours, then to `https://cloud.51degrees.com/api/v4/`. A value without a trailing slash gains one | | `transport` | Optional. The HTTP transport, either an async callable taking the prepared `urllib.request.Request` and answering `(status, body_bytes)`, or an `urllib.request.OpenerDirector`, whose blocking `open` runs on a worker thread. Defaults to `urllib.request.urlopen` on a worker thread. Tests inject one | | `timeout` | Optional. Seconds the default transport waits for the cloud, 30 by default | | `now` | Optional. The clock, returning an aware UTC `datetime`. Tests inject one | @@ -674,7 +674,7 @@ unreachable cloud answers 502 with `{ "error": ... }`. | --- | --- | | `_51DEGREES_RESOURCE_KEY` | Required. The page's resource key, public by nature | | `_51DEGREES_LICENSE_KEY` | Optional. A licence key of the same account, held server side only. Only an account that holds licence keys needs one to redeem, so an account holding none runs without it | -| `_51DEGREES_CLOUD_ENDPOINT` | Optional. The cloud API base including the `/api/v4/` segment, defaulting to `https://cloud.51degrees.com/api/v4/`. This is the same variable the cloud request engine honours. `FOD_CLOUD_API_URL` is its old name, which is deprecated and read only when `_51DEGREES_CLOUD_ENDPOINT` is not set. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons, which is the private hosting option of the cloud service. Both run the same service, so the demo works unchanged against either | +| `_51DEGREES_CLOUD_ENDPOINT` | Optional. The cloud API base including the `/api/v4/` segment, defaulting to `https://cloud.51degrees.com/api/v4/`. This is the same variable the cloud request engine honours. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons, which is the private hosting option of the cloud service. Both run the same service, so the demo works unchanged against either | | `PORT` | The port to listen on, defaulting to `5100` | ### Running diff --git a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py index c91cdf09..84797ab7 100644 --- a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py +++ b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py @@ -53,11 +53,9 @@ import json import os import re -import threading import urllib.error import urllib.parse import urllib.request -import warnings from dataclasses import dataclass from datetime import datetime, timedelta, timezone from enum import Enum @@ -93,8 +91,7 @@ def __getattr__(name: str) -> int: #: The public cloud API base, used when neither the ``endpoint`` argument -#: nor the ``_51DEGREES_CLOUD_ENDPOINT`` environment variable, nor that -#: variable's deprecated old name ``FOD_CLOUD_API_URL``, is set. +#: nor the ``_51DEGREES_CLOUD_ENDPOINT`` environment variable is set. DEFAULT_ENDPOINT = "https://cloud.51degrees.com/api/v4/" #: The environment variable the cloud request engine reads for the API @@ -102,47 +99,14 @@ def __getattr__(name: str) -> int: CLOUD_ENDPOINT_VARIABLE = "_51DEGREES_CLOUD_ENDPOINT" #: Deprecated. Use :data:`CLOUD_ENDPOINT_VARIABLE`. This holds the name that -#: variable had before, which is read when :data:`CLOUD_ENDPOINT_VARIABLE` -#: is not set, so code that passes this constant to the environment reads -#: and sets the old variable. +#: variable had before, which is no longer read. ENDPOINT_VARIABLE = "FOD_CLOUD_API_URL" -# Whether the old variable has been reported as deprecated, so that it is -# reported once however many clients are made. -_old_endpoint_reported = False -_old_endpoint_lock = threading.Lock() - - def _endpoint_from_environment() -> Optional[str]: - """The API base the environment gives, or None when it gives none. - - :data:`CLOUD_ENDPOINT_VARIABLE` is read first. When it is unset or - empty the variable :data:`ENDPOINT_VARIABLE` names is read, which is - the old name, and the first use of it is reported as deprecated. The - report is a :class:`FutureWarning`, which Python shows by default, - because the people who set the variable are the ones running the - application. + """The API base :data:`CLOUD_ENDPOINT_VARIABLE` gives, or None when it is + unset or empty. """ - global _old_endpoint_reported - endpoint = os.environ.get(CLOUD_ENDPOINT_VARIABLE) - if endpoint: - return endpoint - old_name = ENDPOINT_VARIABLE - endpoint = os.environ.get(old_name) - if not endpoint: - return None - with _old_endpoint_lock: - report = not _old_endpoint_reported - _old_endpoint_reported = True - if report: - # The third frame is the code that made the client. - warnings.warn( - "The {0} environment variable is deprecated and {1} replaces " - "it. The cloud endpoint was read from {0} because {1} is not " - "set, so set {1} to the same value.".format( - old_name, CLOUD_ENDPOINT_VARIABLE), - FutureWarning, stacklevel=3) - return endpoint + return os.environ.get(CLOUD_ENDPOINT_VARIABLE) or None _BOUNDARY_TOLERANCE = timedelta(minutes=15) @@ -522,7 +486,6 @@ class DidClient: sent only in the body of the redeem request, never in a URL. :param endpoint: the API base including the ``/api/v4/`` segment. Defaults to the ``_51DEGREES_CLOUD_ENDPOINT`` environment variable, - then to that variable's deprecated old name ``FOD_CLOUD_API_URL``, then to the public cloud. A value without a trailing slash gains one. :param transport: the HTTP transport, either an async callable taking diff --git a/pipeline/fiftyone_pipeline_did/tests/live_cloud.py b/pipeline/fiftyone_pipeline_did/tests/live_cloud.py index 2109e4a2..6cdbab0f 100644 --- a/pipeline/fiftyone_pipeline_did/tests/live_cloud.py +++ b/pipeline/fiftyone_pipeline_did/tests/live_cloud.py @@ -22,8 +22,8 @@ """Where the tests that call the live 51Degrees cloud service find their resource key, and what they fail with when there is none or when the -answer lacks what they asked for. Also how a test reads any setting from -an environment variable whose name has changed. +answer lacks what they asked for. Also how a test reads a setting from an +environment variable. The same file is in each test folder whose tests read a setting from the environment. There is no fallback to a skip, because a cloud test that @@ -31,7 +31,6 @@ """ import os -import sys # Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud # 51Did workflow sets it to each key in turn, and each product's @@ -41,68 +40,31 @@ "_51DEGREES_RESOURCE_KEY_BESPOKE", ) -# Deprecated. The name the resource key variable had before, read when -# none of RESOURCE_KEY_NAMES is set. -OLD_RESOURCE_KEY_NAME = "resource_key" - -# The old names already reported as deprecated, so each is reported once. -_reported_old_names = set() - ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" -def _report_old_name(name, old_name): - """Prints, once for each old name, that it is deprecated and which - name replaces it.""" - if old_name in _reported_old_names: - return - _reported_old_names.add(old_name) - print( - f"The {old_name} environment variable is deprecated and {name} " - f"replaces it. The value was read from {old_name} because {name} " - f"is not set, so set {name} to the same value.", - file=sys.stderr) - - -def read_environment(name, old_name): +def read_environment(name): """The value of the environment variable name, or None when it has - none. When name is unset or empty the name the variable had before is - read, and using it prints once that the old name is deprecated.""" + none.""" value = os.environ.get(name, "").strip() - if value: - return value - value = os.environ.get(old_name, "").strip() - if not value: - return None - _report_old_name(name, old_name) - return value + return value or None -def is_set(name, old_name): +def is_set(name): """Whether the environment variable name is set, whatever it holds, - which is how a switch is read. When it is not set, the name the - variable had before counts, and using it prints once that the old name - is deprecated.""" - if name in os.environ: - return True - if old_name not in os.environ: - return False - _report_old_name(name, old_name) - return True + which is how a switch is read.""" + return name in os.environ def find_resource_key(names=RESOURCE_KEY_NAMES): - """The first resource key set under any of the names, or None. When - none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" + """The first resource key set under any of the names, or None.""" for name in names: value = os.environ.get(name, "").strip() if value: return value - if names is RESOURCE_KEY_NAMES: - return read_environment(names[0], OLD_RESOURCE_KEY_NAME) return None diff --git a/pipeline/fiftyone_pipeline_did/tests/test_creator_context_server.py b/pipeline/fiftyone_pipeline_did/tests/test_creator_context_server.py index f0784a29..e4348d2a 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_creator_context_server.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_creator_context_server.py @@ -181,67 +181,32 @@ def test_unreachable_cloud_answers_502_with_an_error(self): class ServerEndpointVariableTests(unittest.TestCase): """The demo reads the cloud API base from the environment itself, because its page needs the same value as its client, so it follows the - same two names as the DidClient. The old name is still read when the - new one is not set.""" + same variable as the DidClient.""" NEW_VARIABLE = "_51DEGREES_CLOUD_ENDPOINT" - OLD_VARIABLE = "FOD_CLOUD_API_URL" def setUp(self): - # Each test starts with neither name set, and what it sets is put - # back afterwards. + # Each test starts with the variable not set, and what it sets is + # put back afterwards. environment = mock.patch.dict(os.environ) environment.start() self.addCleanup(environment.stop) os.environ.pop(self.NEW_VARIABLE, None) - os.environ.pop(self.OLD_VARIABLE, None) - - def load_recording_warnings(self): - """The server module, loaded afresh so that it reads the - environment again, with the warnings its import gave that say the - old endpoint variable is deprecated and name the new one.""" - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - module = load_server() - reports = [ - warning for warning in caught - if issubclass(warning.category, FutureWarning) - and self.OLD_VARIABLE in str(warning.message) - and self.NEW_VARIABLE in str(warning.message)] - return module, reports def test_api_defaults_to_the_public_cloud(self): - module, reports = self.load_recording_warnings() + module = load_server() self.assertEqual("https://cloud.51degrees.com/api/v4/", module.API) - self.assertEqual([], reports) def test_api_read_from_the_new_variable(self): os.environ[self.NEW_VARIABLE] = "https://private.example/api/v4" - module, reports = self.load_recording_warnings() + module = load_server() self.assertEqual("https://private.example/api/v4/", module.API) - self.assertEqual([], reports) - - def test_api_read_from_the_old_variable_and_reported(self): - os.environ[self.OLD_VARIABLE] = "https://old.example/api/v4" - module, reports = self.load_recording_warnings() - self.assertEqual("https://old.example/api/v4/", module.API) - self.assertEqual(1, len(reports)) - - def test_new_variable_wins_over_the_old_one(self): - os.environ[self.NEW_VARIABLE] = "https://private.example/api/v4" - os.environ[self.OLD_VARIABLE] = "https://old.example/api/v4" - module, reports = self.load_recording_warnings() - self.assertEqual("https://private.example/api/v4/", module.API) - self.assertEqual([], reports) class ServerKeyVariableTests(unittest.TestCase): - """The demo reads its two keys from the environment, each under its - new name first and then under the name it had before.""" + """The demo reads its two keys from the environment.""" - NAMES = ( - "_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY", - "_51DEGREES_LICENSE_KEY", "LICENSE_KEY") + NAMES = ("_51DEGREES_RESOURCE_KEY", "_51DEGREES_LICENSE_KEY") def setUp(self): # Each test starts with none of the names set, and what it sets is @@ -254,7 +219,7 @@ def setUp(self): def load(self): """The server module, loaded afresh with the environment as it is, - and the warnings that say an old name is deprecated.""" + and any warnings that say a name is deprecated.""" spec = importlib.util.spec_from_file_location( "creator_context_server_keys", SERVER_PY) module = importlib.util.module_from_spec(spec) @@ -267,25 +232,9 @@ def load(self): and "is deprecated" in str(warning.message)] return module, reports - def test_keys_are_read_from_their_old_names_and_reported(self): - os.environ["RESOURCE_KEY"] = "old-resource-key" - os.environ["LICENSE_KEY"] = "old-licence-key" - module, reports = self.load() - self.assertEqual("old-resource-key", module.RESOURCE) - self.assertEqual("old-licence-key", module.LICENCE) - self.assertEqual(2, len(reports)) - self.assertTrue(any( - "RESOURCE_KEY" in report and "_51DEGREES_RESOURCE_KEY" in report - for report in reports)) - self.assertTrue(any( - "LICENSE_KEY" in report and "_51DEGREES_LICENSE_KEY" in report - for report in reports)) - - def test_new_names_win_and_nothing_is_reported(self): + def test_keys_are_read_and_nothing_is_reported(self): os.environ["_51DEGREES_RESOURCE_KEY"] = "new-resource-key" os.environ["_51DEGREES_LICENSE_KEY"] = "new-licence-key" - os.environ["RESOURCE_KEY"] = "old-resource-key" - os.environ["LICENSE_KEY"] = "old-licence-key" module, reports = self.load() self.assertEqual("new-resource-key", module.RESOURCE) self.assertEqual("new-licence-key", module.LICENCE) diff --git a/pipeline/fiftyone_pipeline_did/tests/test_did_client.py b/pipeline/fiftyone_pipeline_did/tests/test_did_client.py index 562c954f..9e24bfc4 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_did_client.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_did_client.py @@ -175,33 +175,18 @@ def test_resource_key_is_required(self): DidClient(None) # The two names of the endpoint environment variable, written out so - # that these tests pin the names themselves. The old one is still read - # when the new one is not set. + # that these tests pin the names themselves. The old one is no longer + # read. NEW_VARIABLE = "_51DEGREES_CLOUD_ENDPOINT" OLD_VARIABLE = "FOD_CLOUD_API_URL" def setUp(self): - # Each test starts with neither name set and with the old name not - # yet reported as deprecated, and what it sets is put back - # afterwards. + # Each test starts with the variable not set, and what it sets is + # put back afterwards. environment = mock.patch.dict(os.environ) environment.start() self.addCleanup(environment.stop) os.environ.pop(self.NEW_VARIABLE, None) - os.environ.pop(self.OLD_VARIABLE, None) - reported = mock.patch.object( - did_client, "_old_endpoint_reported", False) - reported.start() - self.addCleanup(reported.stop) - - def old_name_reports(self, caught): - """The warnings in a recorded list that say the old endpoint - variable is deprecated and name the one that replaces it.""" - return [ - warning for warning in caught - if issubclass(warning.category, FutureWarning) - and self.OLD_VARIABLE in str(warning.message) - and self.NEW_VARIABLE in str(warning.message)] def test_endpoint_defaults_to_the_public_cloud(self): self.assertEqual(DEFAULT_ENDPOINT, DidClient(RESOURCE).endpoint) @@ -211,63 +196,12 @@ def test_endpoint_read_from_the_environment(self): self.assertEqual("https://private.example/api/v4/", DidClient(RESOURCE).endpoint) - def test_endpoint_read_from_the_old_variable(self): - os.environ[self.OLD_VARIABLE] = "https://old.example/api/v4" - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - client = DidClient(RESOURCE) - self.assertEqual("https://old.example/api/v4/", client.endpoint) - - def test_old_variable_read_when_the_new_one_is_empty(self): - os.environ[self.NEW_VARIABLE] = "" - os.environ[self.OLD_VARIABLE] = "https://old.example/api/v4" - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - client = DidClient(RESOURCE) - self.assertEqual("https://old.example/api/v4/", client.endpoint) - - def test_empty_old_variable_gives_the_public_cloud(self): - os.environ[self.OLD_VARIABLE] = "" - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - client = DidClient(RESOURCE) - self.assertEqual(DEFAULT_ENDPOINT, client.endpoint) - self.assertEqual([], self.old_name_reports(caught)) - - def test_new_variable_wins_over_the_old_one(self): - os.environ[self.NEW_VARIABLE] = "https://private.example/api/v4" - os.environ[self.OLD_VARIABLE] = "https://old.example/api/v4" - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - client = DidClient(RESOURCE) - self.assertEqual("https://private.example/api/v4/", client.endpoint) - # The old name was not used, so it is not reported. - self.assertEqual([], self.old_name_reports(caught)) - - def test_old_variable_is_reported_as_deprecated_once(self): - os.environ[self.OLD_VARIABLE] = "https://old.example/api/v4" - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - first = DidClient(RESOURCE) - second = DidClient(RESOURCE) - self.assertEqual("https://old.example/api/v4/", first.endpoint) - self.assertEqual("https://old.example/api/v4/", second.endpoint) - self.assertEqual(1, len(self.old_name_reports(caught))) - def test_endpoint_argument_wins_and_is_normalised(self): self.assertEqual(ENDPOINT, DidClient( RESOURCE, endpoint="https://cloud.example/api/v4").endpoint) self.assertEqual(ENDPOINT, DidClient( RESOURCE, endpoint="https://cloud.example/api/v4//").endpoint) - def test_endpoint_argument_wins_over_the_old_variable(self): - os.environ[self.OLD_VARIABLE] = "https://old.example/api/v4" - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - client = DidClient(RESOURCE, endpoint=ENDPOINT) - self.assertEqual(ENDPOINT, client.endpoint) - self.assertEqual([], self.old_name_reports(caught)) - def test_old_constant_keeps_its_old_value(self): # Code that passes ENDPOINT_VARIABLE to the environment has to keep # reading the variable it read before. @@ -285,18 +219,6 @@ def test_old_constant_is_given_by_dir_and_a_star_import(self): self.assertEqual( self.NEW_VARIABLE, names.get("CLOUD_ENDPOINT_VARIABLE")) - def test_client_reads_the_variable_the_old_constant_names(self): - # The client reads the old name through the constant, as it did - # before, so code that points the constant at another variable - # still decides what is read. - os.environ["ANOTHER_VARIABLE"] = "https://other.example/api/v4" - with mock.patch.object( - did_client, "ENDPOINT_VARIABLE", "ANOTHER_VARIABLE"): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - client = DidClient(RESOURCE) - self.assertEqual("https://other.example/api/v4/", client.endpoint) - def test_new_constant_holds_the_new_name(self): self.assertEqual( self.NEW_VARIABLE, did_client.CLOUD_ENDPOINT_VARIABLE) diff --git a/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py b/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py index 8ef0fc8a..4ea84a84 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py @@ -57,7 +57,7 @@ pytestmark = pytest.mark.live RESOURCE_KEY = find_resource_key() -LICENCE_KEY = read_environment("_51DEGREES_LICENSE_KEY", "license_key") +LICENCE_KEY = read_environment("_51DEGREES_LICENSE_KEY") FODID = "the FODid product" MARKETING = "the CloudV5FODiD product" diff --git a/pipeline/fiftyone_pipeline_did/tests/test_live_cloud.py b/pipeline/fiftyone_pipeline_did/tests/test_live_cloud.py index 415e24ec..4d6bd2b0 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_live_cloud.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_live_cloud.py @@ -21,13 +21,9 @@ # ********************************************************************* """Checks live_cloud.py, which every test folder that reads a setting from -the environment holds a copy of. A variable whose name changed is still -read under its old name when the new one is not set, and using the old name -prints once that it is deprecated. These need no resource key. +the environment holds a copy of. These need no resource key. """ -import contextlib -import io import os import unittest from pathlib import Path @@ -50,120 +46,43 @@ NAMES = ( "_51DEGREES_RESOURCE_KEY", "_51DEGREES_RESOURCE_KEY_BESPOKE", - "_51DEGREES_RESOURCE_KEY_BASIC", "resource_key", - "_51DEGREES_LICENSE_KEY", "license_key") + "_51DEGREES_RESOURCE_KEY_BASIC", "_51DEGREES_LICENSE_KEY") -class OldNameTests(unittest.TestCase): +class ReadTests(unittest.TestCase): def setUp(self): - # Each test starts with none of the names set and no old name - # reported yet, and what it sets is put back afterwards. + # Each test starts with none of the names set, and what it sets is + # put back afterwards. environment = mock.patch.dict(os.environ) environment.start() self.addCleanup(environment.stop) for name in NAMES: os.environ.pop(name, None) - reported = mock.patch.object( - live_cloud, "_reported_old_names", set()) - reported.start() - self.addCleanup(reported.stop) - - def printed(self, call): - """What a call returned and what it printed to the error stream.""" - errors = io.StringIO() - with contextlib.redirect_stderr(errors): - value = call() - return value, errors.getvalue() - - def test_resource_key_is_read_from_the_old_name(self): - os.environ["resource_key"] = "old-key" - value, errors = self.printed(live_cloud.find_resource_key) - self.assertEqual("old-key", value) - self.assertIn("resource_key environment variable is deprecated", - errors) - self.assertIn("_51DEGREES_RESOURCE_KEY", errors) - - def test_new_resource_key_name_wins_and_nothing_is_printed(self): + + def test_resource_key_is_found(self): os.environ["_51DEGREES_RESOURCE_KEY"] = "new-key" - os.environ["resource_key"] = "old-key" - value, errors = self.printed(live_cloud.find_resource_key) - self.assertEqual("new-key", value) - self.assertEqual("", errors) + self.assertEqual("new-key", live_cloud.find_resource_key()) def test_no_resource_key_gives_none(self): - value, errors = self.printed(live_cloud.find_resource_key) - self.assertIsNone(value) - self.assertEqual("", errors) - - def test_other_names_do_not_fall_back_to_the_old_name(self): - # The old name was one key, so a search for a key of another tier - # does not take it. - os.environ["resource_key"] = "old-key" - value, _ = self.printed(lambda: live_cloud.find_resource_key( - ("_51DEGREES_RESOURCE_KEY_BASIC",))) - self.assertIsNone(value) - - def test_old_name_is_read_and_reported_once(self): - os.environ["license_key"] = "old-licence" - - def twice(): - return ( - live_cloud.read_environment( - "_51DEGREES_LICENSE_KEY", "license_key"), - live_cloud.read_environment( - "_51DEGREES_LICENSE_KEY", "license_key")) - - value, errors = self.printed(twice) - self.assertEqual(("old-licence", "old-licence"), value) - self.assertEqual( - 1, errors.count("license_key environment variable is deprecated")) - self.assertIn("_51DEGREES_LICENSE_KEY", errors) - - def test_old_name_is_read_when_the_new_one_is_empty(self): - os.environ["_51DEGREES_LICENSE_KEY"] = "" - os.environ["license_key"] = "old-licence" - value, _ = self.printed(lambda: live_cloud.read_environment( - "_51DEGREES_LICENSE_KEY", "license_key")) - self.assertEqual("old-licence", value) + self.assertIsNone(live_cloud.find_resource_key()) - def test_new_name_wins_and_nothing_is_printed(self): + def test_a_variable_is_read(self): os.environ["_51DEGREES_LICENSE_KEY"] = "new-licence" - os.environ["license_key"] = "old-licence" - value, errors = self.printed(lambda: live_cloud.read_environment( - "_51DEGREES_LICENSE_KEY", "license_key")) - self.assertEqual("new-licence", value) - self.assertEqual("", errors) - - def test_neither_name_gives_none(self): - value, errors = self.printed(lambda: live_cloud.read_environment( - "_51DEGREES_LICENSE_KEY", "license_key")) - self.assertIsNone(value) - self.assertEqual("", errors) - - def test_old_name_set_to_nothing_still_counts_as_set(self): - # A switch is on when its variable is set, whatever it holds. - os.environ["license_key"] = "" - value, errors = self.printed(lambda: live_cloud.is_set( - "_51DEGREES_LICENSE_KEY", "license_key")) - self.assertTrue(value) self.assertEqual( - 1, errors.count("license_key environment variable is deprecated")) - self.assertIn("_51DEGREES_LICENSE_KEY", errors) + "new-licence", + live_cloud.read_environment("_51DEGREES_LICENSE_KEY")) - def test_new_name_set_to_nothing_counts_as_set(self): - os.environ["_51DEGREES_LICENSE_KEY"] = "" - value, errors = self.printed(lambda: live_cloud.is_set( - "_51DEGREES_LICENSE_KEY", "license_key")) - self.assertTrue(value) - self.assertEqual("", errors) + def test_an_unset_variable_gives_none(self): + self.assertIsNone( + live_cloud.read_environment("_51DEGREES_LICENSE_KEY")) - def test_neither_name_is_not_set(self): - value, errors = self.printed(lambda: live_cloud.is_set( - "_51DEGREES_LICENSE_KEY", "license_key")) - self.assertFalse(value) - self.assertEqual("", errors) + def test_a_variable_set_to_nothing_counts_as_set(self): + os.environ["_51DEGREES_LICENSE_KEY"] = "" + self.assertTrue(live_cloud.is_set("_51DEGREES_LICENSE_KEY")) + def test_an_unset_variable_is_not_set(self): + self.assertFalse(live_cloud.is_set("_51DEGREES_LICENSE_KEY")) class CopiesTests(unittest.TestCase): diff --git a/pipeline/fiftyone_pipeline_did/tox.ini b/pipeline/fiftyone_pipeline_did/tox.ini index f42e48fb..7317c461 100644 --- a/pipeline/fiftyone_pipeline_did/tox.ini +++ b/pipeline/fiftyone_pipeline_did/tox.ini @@ -16,13 +16,10 @@ wheel_build_env = .pkg # scripts select with. passenv = _51DEGREES_RESOURCE_KEY - resource_key _51DEGREES_RESOURCE_KEY_BESPOKE _51DEGREES_RESOURCE_KEY_BASIC _51DEGREES_LICENSE_KEY - license_key _51DEGREES_CLOUD_ENDPOINT - FOD_CLOUD_API_URL PYTEST_ADDOPTS deps = {[common]deps} diff --git a/pipeline/fiftyone_pipeline_engines/tox.ini b/pipeline/fiftyone_pipeline_engines/tox.ini index 7ca15783..1e8e9619 100644 --- a/pipeline/fiftyone_pipeline_engines/tox.ini +++ b/pipeline/fiftyone_pipeline_engines/tox.ini @@ -19,9 +19,7 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_engines {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY - resource_key _51DEGREES_LICENSE_KEY - license_key [testenv:pre-publish] package = skip diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tox.ini b/pipeline/fiftyone_pipeline_engines_fiftyone/tox.ini index 5e9b48a8..cddb3cd4 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tox.ini +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tox.ini @@ -20,9 +20,7 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_engines_fiftyone {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY - resource_key _51DEGREES_LICENSE_KEY - license_key [testenv:pre-publish] package = skip From f9b2d26dd1e151c96ab7f3d063e0c4fa38004440 Mon Sep 17 00:00:00 2001 From: Tracy Barlow Date: Thu, 19 Nov 2026 16:37:34 +0000 Subject: [PATCH 121/136] Keep the threading and warnings imports a star import gives --- .../cloudrequestengine.py | 2 ++ .../src/fiftyone_pipeline_did/did_client.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py index ac18349d..f62223cf 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py @@ -26,6 +26,8 @@ import json import math import os +# Kept so that ``from ... import *`` of this module still gives it. +import threading import warnings from functools import cached_property from json import JSONDecodeError diff --git a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py index 84797ab7..d2f7e2db 100644 --- a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py +++ b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py @@ -53,9 +53,13 @@ import json import os import re +# Kept so that ``from ... import *`` of this module still gives it. +import threading import urllib.error import urllib.parse import urllib.request +# Kept so that ``from ... import *`` of this module still gives it. +import warnings from dataclasses import dataclass from datetime import datetime, timedelta, timezone from enum import Enum From 74123a9068eecb2857fdb2d3e69dac81097bba11 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 19 Nov 2026 20:15:56 +0000 Subject: [PATCH 122/136] Give the released missing property messages again, and share a parameter only when it starts with 51D_ or the Pipeline uses it (#246) Usage sharing now shares a query string or form parameter, from a GET or a POST, only when its name starts with 51D_ or a Flow Element in the Pipeline accepts it as evidence, whatever the configuration and even with share_all set. The query_whitelist and share_all_query_string_parameters options are deprecated and do nothing, so this changes released behavior for anyone who used them, or a list of None, to share other parameters. Missing property errors give the messages of the last release again, but on-premise Device Detection and IP Intelligence data now raise for a listed property with no value, where the release returned None. Run 44465779200 built this head with only a temporary workflow file added and passed the unit and integration tests on Ubuntu and Windows with Python 3.10 to 3.14, without macOS. --- .../hardwareprofile_cloud.py | 4 +- .../tests/test_cloud.py | 16 +- .../swig_data.py | 19 +- .../tests/test_devicedetection.py | 31 +-- .../tests/test_missing_property_raises.py | 56 +++++ .../swig_data.py | 21 +- .../tests/test_missing_property_raises.py | 56 +++++ .../clouddata.py | 65 ++---- .../cloudengine.py | 26 +-- .../test_cloudengine_missing_property.py | 133 ----------- .../tests/test_credentials.py | 20 +- .../fiftyone_pipeline_engines/aspectdata.py | 11 +- .../missingproperty_service.py | 215 +----------------- .../tests/test_engines.py | 14 +- .../tests/test_missing_property_reasons.py | 121 ---------- .../share_usage.py | 26 +-- .../share_usage_evidencekeyfilter.py | 85 +++++-- .../test_share_usage_evidencekeyfilter.py | 124 ++++++---- .../tests/test_share_usage_lists.py | 26 ++- .../tests/test_share_usage_rules.py | 129 +++++++---- .../tests/test_usage.py | 38 +++- 21 files changed, 507 insertions(+), 729 deletions(-) create mode 100644 device-detection/fiftyone_devicedetection_onpremise/tests/test_missing_property_raises.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_missing_property_raises.py delete mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_property.py delete mode 100644 pipeline/fiftyone_pipeline_engines/tests/test_missing_property_reasons.py diff --git a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py index a19b7da2..b6e6fe5e 100644 --- a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py @@ -23,7 +23,7 @@ from __future__ import absolute_import from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine from fiftyone_pipeline_cloudrequestengine.clouddata import \ - CloudMissingPropertyService + OnPremiseMissingPropertyService from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary @@ -91,7 +91,7 @@ def __init__(self, flow_element, contents): # The device data's service, so a name that is not a property is # reported in the same way for both. - self.missing_property_service = CloudMissingPropertyService() + self.missing_property_service = OnPremiseMissingPropertyService() def _matching_key(self, key): """! diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py index 702e5c04..7d769b84 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py @@ -30,8 +30,6 @@ from fiftyone_devicedetection_cloud.devicedetection_cloud import DeviceDetectionCloud -from fiftyone_pipeline_engines.missingproperty_service import CONFIGURATOR_URL - from .live_cloud import find_resource_key, no_resource_key pytestmark = pytest.mark.live @@ -158,16 +156,10 @@ def test_missing_property_service_not_found_anywhere(self): fd.device.get("notpresent") result = str(raised.exception) - properties = pipeline.get_element("device").get_properties() - self.assertIn( - "Property 'notpresent' not found in data for element 'device'. " - "This is because your resource key does not include access to " - "this property. The properties included for your key under " - "'device' are ", result) - self.assertIn( - ",".join(meta.get("name", name) - for name, meta in properties.items()), result) - self.assertIn(CONFIGURATOR_URL, result) + self.assertIn("notpresent", result) + self.assertIn("device", result) + self.assertIn(', '.join(pipeline.get_element("device").get_properties().keys()), result) + self.assertIn("https://51degrees.com/documentation/_services__cloud__resource_keys.html", result) def test_cloud_request_origin(self): """! diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index 05f4cc0f..7d7891fa 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -25,8 +25,7 @@ from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue -from fiftyone_pipeline_engines.missingproperty_service import \ - OnPremiseMissingPropertyService +from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService from .constants import RESULTS_CLOSED @@ -34,6 +33,22 @@ # fiftyoneDegreesHashMatchMethod values, which getMethod returns. MATCH_METHODS = ("NONE", "PERFORMANCE", "COMBINED", "PREDICTIVE") +class OnPremiseMissingPropertyService(MissingPropertyService): + + def check(self, key, flow_element): + + if key in flow_element.properties: + datafiles = flow_element.properties[key]["datafiles"] + if(not flow_element.engine.getProduct() in datafiles): + raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. This is because your datafile does not contain the property. The property is available in' + str(datafiles)) + # The data file holds the property, so it is missing because + # it is not available. A missing property always raises. + raise Exception( + "Property '" + key + "' not found in data for element '" + + flow_element.datakey + "'. This is because the property " + "has been excluded when configuring the engine.") + else: + raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. Please check that the element and property names are correct.') class SwigData(AspectData): diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py index aa6222ca..f8151eaf 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py @@ -261,9 +261,7 @@ def test_missing_property_service_not_found_anywhere(self): result = str(e) self.assertEqual( - "Property 'notpresent' not found in data for element 'device'. " - "The reason for this is unknown. Please check that the aspect and " - "property name are correct.", result) + result, "Property notpresent not found in data for element device. Please check that the element and property names are correct.") def test_missing_property_service_not_found_in_current(self): """! @@ -288,20 +286,15 @@ def test_missing_property_service_not_found_in_current(self): fd.process() - # The name is found however it is spelled, and the message gives it - # as the caller did. - for name in ("mock", "Mock"): - with self.assertRaises(Exception) as raised: - fd.device.get(name) - self.assertEqual( - "Property '" + name + "' not found in data for element " - "'device'. This is because your license or data file does not " - "include this property. It is available with the Enterprise " - "license/data for the DeviceDetectionOnPremise. To upgrade, " - "see " - "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-pricing" - ".", - str(raised.exception)) + result = "" + + try: + fd.device.get("mock") + except Exception as e: + result = str(e) + + self.assertEqual( + result, "Property mock not found in data for element device. This is because your datafile does not contain the property. The property is available in['Enterprise']") def test_excluded_property(self): """! @@ -329,9 +322,7 @@ def test_excluded_property(self): result = str(e) self.assertEqual( - "Property 'screenpixelsheight' not found in data for element " - "'device'. This is because the property has been excluded when " - "configuring the engine.", result) + result, "Property screenpixelsheight was excluded from device") def test_evidencekey_filter_contains_user_agent(self): """! diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_missing_property_raises.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_missing_property_raises.py new file mode 100644 index 00000000..2fe00eaa --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_missing_property_raises.py @@ -0,0 +1,56 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""A property the engine lists, in a data file tier that holds it, raises +when the data has no value for it, as every missing property does.""" + +import unittest +from types import SimpleNamespace + +from fiftyone_devicedetection_onpremise.swig_data import \ + OnPremiseMissingPropertyService + + +class StandInEngine: + """An engine whose data file is Lite, listing one property that the + Lite file holds.""" + + datakey = "device" + properties = {"ismobile": {"datafiles": ["Lite", "Enterprise"]}} + engine = SimpleNamespace(getProduct=lambda: "Lite") + + +class MissingPropertyRaisesTests(unittest.TestCase): + + def test_a_listed_property_of_this_tier_raises(self): + with self.assertRaises(Exception) as raised: + OnPremiseMissingPropertyService().check( + "ismobile", StandInEngine()) + + self.assertEqual( + "Property 'ismobile' not found in data for element 'device'. " + "This is because the property has been excluded when " + "configuring the engine.", str(raised.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 2070a22b..85fa7094 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -27,8 +27,7 @@ from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue -from fiftyone_pipeline_engines.missingproperty_service import \ - OnPremiseMissingPropertyService +from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService from .constants import RESULTS_CLOSED @@ -43,6 +42,24 @@ COORDINATE_PROPERTIES = ("latitude", "longitude") +class OnPremiseMissingPropertyService(MissingPropertyService): + + def check(self, key, flow_element): + + if key in flow_element.properties: + datafiles = flow_element.properties[key]["datafiles"] + if(not flow_element.engine.getProduct() in datafiles): + raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. This is because your datafile does not contain the property. The property is available in' + str(datafiles)) + # The data file holds the property, so it is missing because + # it is not available. A missing property always raises. + raise Exception( + "Property '" + key + "' not found in data for element '" + + flow_element.datakey + "'. This is because the property " + "has been excluded when configuring the engine.") + else: + raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. Please check that the element and property names are correct.') + + class SwigData(AspectData): """! diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_missing_property_raises.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_missing_property_raises.py new file mode 100644 index 00000000..f8e282e7 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_missing_property_raises.py @@ -0,0 +1,56 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""A property the engine lists, in a data file tier that holds it, raises +when the data has no value for it, as every missing property does.""" + +import unittest +from types import SimpleNamespace + +from fiftyone_ipintelligence_onpremise.swig_data import \ + OnPremiseMissingPropertyService + + +class StandInEngine: + """An engine whose data file is Lite, listing one property that the + Lite file holds.""" + + datakey = "ip" + properties = {"registeredname": {"datafiles": ["Lite", "Enterprise"]}} + engine = SimpleNamespace(getProduct=lambda: "Lite") + + +class MissingPropertyRaisesTests(unittest.TestCase): + + def test_a_listed_property_of_this_tier_raises(self): + with self.assertRaises(Exception) as raised: + OnPremiseMissingPropertyService().check( + "registeredname", StandInEngine()) + + self.assertEqual( + "Property 'registeredname' not found in data for element 'ip'. " + "This is because the property has been excluded when " + "configuring the engine.", str(raised.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py index e45f67fd..4f4f8c88 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py @@ -22,48 +22,27 @@ from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary -from fiftyone_pipeline_engines.missingproperty_service import \ - MissingPropertyReason, MissingPropertyService - - -class CloudMissingPropertyService(MissingPropertyService): - - """! - The missing property service of a Cloud Aspect Engine. A property its - metadata does not list is one the key in use does not include, or, - while the engine's property list is not known, one whose request for - that list failed. - """ - - def engine_reason(self, flow_element): - - """! - The reason for a property the metadata does not list. - - @type flow_element: CloudEngine - @param flow_element: the engine - @rtype: tuple - @return: the reason, and the property names the key includes - """ - - properties_known = getattr(flow_element, "properties_known", None) - if properties_known is not None and not properties_known(): - # The Cloud Request Engine asks for the property list when it is - # built, so a list that is still not known is one whose request - # failed. - return MissingPropertyReason.CLOUD_REQUEST_FAILED, () - properties = self.properties(flow_element) - if not properties: - return ( - MissingPropertyReason.PRODUCT_NOT_ACCESSIBLE_WITH_RESOURCE_KEY, - ()) - return ( - MissingPropertyReason.PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY, - [meta.get("name", name) for name, meta in properties.items()]) - - -# The name this service had before, kept for code that imports it. -OnPremiseMissingPropertyService = CloudMissingPropertyService +from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService + +class OnPremiseMissingPropertyService(MissingPropertyService): + + def check(self, key, flow_element): + + # The properties listed here are the ones the key carries, which is + # not the same as the ones one answer holds: a request made with a + # licence key carries only the properties it asked for, so a + # property can be listed and still be absent. The message names + # both causes, so the reader checks the right one. + raise Exception( + "Property " + key + " not found in data for element " + + flow_element.datakey + ". Either the key in use does not " + "include access to this property, or the request was made " + "with a licence key and the property was not among the " + "requested_properties. Properties the key carries under " + + flow_element.datakey + " are " + + ", ".join(list(flow_element.get_properties().keys())) + + ". For more details on resource keys, see our explainer: " + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-clouddata.py&utm_term=property-not-found") class CloudData(AspectDataDictionary): @@ -83,4 +62,4 @@ def __init__(self, flow_element, contents): super(CloudData, self).__init__(flow_element, contents) - self.missing_property_service = CloudMissingPropertyService() + self.missing_property_service = OnPremiseMissingPropertyService() diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py index 1352adde..248ec92e 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py @@ -26,12 +26,14 @@ from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue from fiftyone_pipeline_core.jsonbundler import WKT_TYPES -from .clouddata import CloudData, CloudMissingPropertyService +from .clouddata import CloudData from .constants import Constants import json import threading +from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService + # Why a pipeline with a second CloudRequestEngine is refused. _ANOTHER_REQUEST_ENGINE = ( "This cloud engine is already in a pipeline with another Cloud Request " @@ -39,9 +41,11 @@ "pipeline with a different one is refused. Build another cloud engine " "for this pipeline, or use the same Cloud Request Engine in both.") -# The name the cloud missing property service had here, kept for code that -# imports it. -OnPremiseMissingPropertyService = CloudMissingPropertyService +class OnPremiseMissingPropertyService(MissingPropertyService): + + def check(self, key, flow_element): + + raise Exception('Property ' + key + ' not found in data for element ' + flow_element.datakey + '. Please check that the element and property names are correct.') class CloudEngine(Engine): @@ -128,20 +132,6 @@ def get_properties(self): self._load_properties(self._log) return super(CloudEngine, self).get_properties() - def properties_known(self): - - """! - Whether this engine has its properties from the CloudRequestEngine, - which it has once that engine's discovery has completed. The - properties are looked for first, as get_properties does. - - @rtype: bool - @return: True once the properties are known - """ - - self._load_properties(self._log) - return self._properties_final - def _load_properties(self, log): """! diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_property.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_property.py deleted file mode 100644 index 92117abc..00000000 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_property.py +++ /dev/null @@ -1,133 +0,0 @@ -# ********************************************************************* -# This Original Work is copyright of 51 Degrees Mobile Experts Limited. -# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, -# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. -# -# This Original Work is licensed under the European Union Public Licence -# (EUPL) v.1.2 and is subject to its terms as set out below. -# -# If a copy of the EUPL was not distributed with this file, You can obtain -# one at https://opensource.org/licenses/EUPL-1.2. -# -# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be -# amended by the European Commission) shall be deemed incompatible for -# the purposes of the Work and the provisions of the compatibility -# clause in Article 5 of the EUPL shall not apply. -# -# If using the Work as, or as part of, a network application, by -# including the attribution notice(s) required under Article 5 of the EUPL -# in the end user terms of the application under an appropriate heading, -# such notice(s) shall fulfill the requirements of that article. -# ********************************************************************* - -""" -The reason a Cloud Aspect Engine gives for a missing property, decided in -the order the properties specification sets out, with the message of the -missing properties table for that reason. -""" - -import json -import unittest - -from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine -from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary -from fiftyone_pipeline_core.flowelement import FlowElement -from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder - -# The configurator the Resource Key rows link to, with this repository's -# campaign parameters. -CONFIGURATOR = "https://configure.51degrees.com/?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-configurator" - - -class FakeCloudRequestEngine(FlowElement): - """Stands in for the Cloud Request Engine of a key that carries only the - device product, with two of its properties.""" - - def __init__(self): - super(FakeCloudRequestEngine, self).__init__() - self.datakey = "cloud" - self.flow_element_properties = {"device": { - "ismobile": {"name": "IsMobile", "type": "Boolean"}, - "hardwarevendor": {"name": "HardwareVendor", "type": "String"}}} - - def process_internal(self, flowdata): - flowdata.set_element_data(ElementDataDictionary( - self, {"cloud": json.dumps({"device": {"ismobile": True}})})) - - -class UndiscoveredCloudRequestEngine(FakeCloudRequestEngine): - """Stands in for a Cloud Request Engine whose request for the property - list failed when it was built, so its properties are not known.""" - - def known_flow_element_properties(self): - return None - - -def build(request_engine, datakey, restricted=None): - engine = CloudEngine() - engine.datakey = datakey - if restricted is not None: - engine.set_restricted_properties(restricted) - pipeline = PipelineBuilder({ - "add_javascript_builder": False, - "use_setheader_properties": False}).add( - request_engine).add(engine).build() - flowdata = pipeline.create_flowdata() - flowdata.process() - return pipeline, flowdata - - -class CloudReasonTests(unittest.TestCase): - - def message(self, request_engine, datakey, name, restricted=None): - pipeline, flowdata = build(request_engine, datakey, restricted) - self.addCleanup(pipeline.close) - with self.assertRaises(Exception) as raised: - flowdata.get(datakey).get(name) - return str(raised.exception) - - def test_a_product_the_key_does_not_include(self): - self.assertEqual( - "Property 'country' not found in data for element 'location'. " - "This is because your resource key does not include access to " - "any properties under 'location'. Check or create a resource key " - "with the properties you need in the configurator at " - + CONFIGURATOR + ".", - self.message(FakeCloudRequestEngine(), "location", "country")) - - def test_a_property_the_key_does_not_include(self): - self.assertEqual( - "Property 'browsername' not found in data for element 'device'. " - "This is because your resource key does not include access to " - "this property. The properties included for your key under " - "'device' are IsMobile,HardwareVendor. Check or change your key " - "in the configurator at " + CONFIGURATOR + ".", - self.message(FakeCloudRequestEngine(), "device", "browsername")) - - def test_a_listed_property_not_in_the_answer_was_excluded(self): - self.assertEqual( - "Property 'HardwareVendor' not found in data for element " - "'device'. This is because the property has been excluded when " - "configuring the engine.", - self.message( - FakeCloudRequestEngine(), "device", "HardwareVendor")) - - def test_a_property_left_out_of_the_restricted_list_was_excluded(self): - self.assertIn( - "This is because the property has been excluded when " - "configuring the engine.", - self.message( - FakeCloudRequestEngine(), "device", "hardwarevendor", - restricted=["IsMobile"])) - - def test_a_failed_request_for_the_property_list(self): - self.assertEqual( - "Property 'browsername' not found in data for element 'device'. " - "This is because an upstream cloud request failed, so no data " - "was available to populate the property.", - self.message( - UndiscoveredCloudRequestEngine(), "device", "browsername")) - - -if __name__ == "__main__": - unittest.main() diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py index 1f42abc6..72b5a11c 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py @@ -595,12 +595,13 @@ def test_no_report_when_every_property_came_back(self): str(warning.message) for warning in caught if "not covered" in str(warning.message)]) - def test_missing_property_message_names_the_properties_carried(self): + def test_missing_property_message_names_both_causes(self): """ - With a licence key, a property the key does not carry gives the - missing properties table's message for it, which lists the - properties the key carries under the element it looked in rather - than assuming device. + With a licence key the accessible properties list everything the + licence entitles, so a property can be listed as carried and still + be absent from an answer because it was not requested. The message + for a property that is not in the data says so, and names the + element it looked in rather than assuming device. """ engine = CloudRequestEngine({ @@ -620,8 +621,7 @@ def test_missing_property_message_names_the_properties_carried(self): data.device.get("other") message = str(context.exception) - self.assertIn( - "Property 'other' not found in data for element 'device'. This " - "is because your resource key does not include access to this " - "property. The properties included for your key under 'device' " - "are value.", message) + self.assertIn("Property other not found", message) + self.assertIn("requested_properties", message) + self.assertIn("under device are value", message) + self.assertNotIn("your resource key", message) diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py index 28faf87e..fa02ff16 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py @@ -61,17 +61,16 @@ def get(self, key): @rtype mixed @return The value stored under the chosen key @raise Exception: where the property was left out of the engine's - restricted properties, or the data has no value for it, with the - message the missing property service gives for the reason + restricted properties, saying so, or where the data has no value + for it, with the message the missing property service gives """ if hasattr(self.flow_element, "restricted_property_keys"): if key.lower() not in self.flow_element.restricted_property_keys: - # The service gives the reason, as the property was - # excluded or is in a data tier the data file lacks. - return self.missing_property_service.check( - key, self.flow_element) + raise Exception( + "Property " + key + " was excluded from " + + self.flow_element.datakey) try: if self.get_internal(key) == None: diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py index 35c4a7c6..c99faa00 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/missingproperty_service.py @@ -20,104 +20,13 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -from enum import Enum - - -class MissingPropertyReason(Enum): - """! - Why a property an engine defines is missing from its data, named as the - .NET packages name each reason. - """ - - DATA_FILE_UPGRADE_REQUIRED = "DataFileUpgradeRequired" - PROPERTY_EXCLUDED_FROM_ENGINE_CONFIGURATION = \ - "PropertyExcludedFromEngineConfiguration" - PRODUCT_NOT_ACCESSIBLE_WITH_RESOURCE_KEY = \ - "ProductNotAccessibleWithResourceKey" - PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY = \ - "PropertyNotAccessibleWithResourceKey" - CLOUD_REQUEST_FAILED = "CloudRequestFailed" - UNKNOWN = "Unknown" - - -# The pages the messages link to, with this repository's campaign -# parameters. -PRICING_URL = "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-pricing" -CONFIGURATOR_URL = "https://configure.51degrees.com/?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-configurator" - -# Lists in a message are joined with a comma and no space. -LIST_SEPARATOR = "," - - -def missing_property_message( - reason, name, element, tiers=(), engine_type="", properties=()): - """! - The message of the missing properties table for a reason. - - @type reason: MissingPropertyReason - @param reason: why the property is missing - @type name: str - @param name: the property, as the caller named it - @type element: str - @param element: the data key of the element asked - @type tiers: list - @param tiers: the data tiers that hold the property, for - DATA_FILE_UPGRADE_REQUIRED, where an empty list gives the row for - tiers that are not known - @type engine_type: str - @param engine_type: the engine's type name, for - DATA_FILE_UPGRADE_REQUIRED - @type properties: list - @param properties: the property names the key includes, for - PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY - @rtype: str - @return: the message - """ - - text = ("Property '" + name + "' not found in data for element '" + - element + "'. ") - if reason is MissingPropertyReason.DATA_FILE_UPGRADE_REQUIRED: - text += ("This is because your license or data file does not " - "include this property. ") - if tiers: - return text + ( - "It is available with the " + LIST_SEPARATOR.join(tiers) + - " license/data for the " + engine_type + - ". To upgrade, see " + PRICING_URL + ".") - return text + ( - "To see which licenses include it, see " + PRICING_URL + ".") - if reason is MissingPropertyReason.PROPERTY_EXCLUDED_FROM_ENGINE_CONFIGURATION: - return text + ("This is because the property has been excluded " - "when configuring the engine.") - if reason is MissingPropertyReason.PRODUCT_NOT_ACCESSIBLE_WITH_RESOURCE_KEY: - return text + ( - "This is because your resource key does not include access to " - "any properties under '" + element + "'. Check or create a " - "resource key with the properties you need in the configurator " - "at " + CONFIGURATOR_URL + ".") - if reason is MissingPropertyReason.PROPERTY_NOT_ACCESSIBLE_WITH_RESOURCE_KEY: - return text + ( - "This is because your resource key does not include access to " - "this property. The properties included for your key under '" + - element + "' are " + LIST_SEPARATOR.join(properties) + - ". Check or change your key in the configurator at " + - CONFIGURATOR_URL + ".") - if reason is MissingPropertyReason.CLOUD_REQUEST_FAILED: - return text + ( - "This is because an upstream cloud request failed, so no data " - "was available to populate the property.") - return text + ("The reason for this is unknown. Please check that the " - "aspect and property name are correct.") - class MissingPropertyService(): """! - Raises the error for a property an engine's data does not hold, giving - the reason in the order the properties specification sets out, with - the missing properties table's message for it. Property names are - compared ignoring case. Engines of a kind override needs_upgrade and - engine_reason to add their own steps. + A missing property service runs when a property is not available in the + aspectData. It can be extended to provide a specific message for why the property + is not available """ def check(self, key, flow_element): @@ -129,121 +38,7 @@ def check(self, key, flow_element): @param key: the property asked for @type flow_element: FlowElement @param flow_element: the element whose data was asked - @raise Exception: always, with the message for the reason the - property is missing - """ - - raise Exception(self.message(key, flow_element)) - - def message(self, key, flow_element): - - """! - The message for the reason a property is missing. - - @type key: string - @param key: the property asked for - @type flow_element: FlowElement - @param flow_element: the element whose data was asked - @rtype: str - @return: the message - """ - - name = key.lower() - listed = None - for property_name, meta in self.properties(flow_element).items(): - if property_name.lower() == name: - listed = meta - break - restricted = getattr(flow_element, "restricted_property_keys", None) - - # A property the metadata lists, or one the restricted list leaves - # out, is in a data tier the data file does not hold, or was - # excluded. - if listed is not None or ( - restricted is not None and name not in restricted): - tiers = list((listed or {}).get("datafiles") or ()) - if listed is not None and self.needs_upgrade(tiers, flow_element): - return missing_property_message( - MissingPropertyReason.DATA_FILE_UPGRADE_REQUIRED, key, - flow_element.datakey, tiers=tiers, - engine_type=type(flow_element).__name__) - return missing_property_message( - MissingPropertyReason.PROPERTY_EXCLUDED_FROM_ENGINE_CONFIGURATION, - key, flow_element.datakey) - - reason, properties = self.engine_reason(flow_element) - return missing_property_message( - reason, key, flow_element.datakey, properties=properties) - - def properties(self, flow_element): - - """! - The engine's property metadata, by name. - - @type flow_element: FlowElement - @param flow_element: the engine - @rtype: dict - @return: the metadata of each property, by name - """ - - get_properties = getattr(flow_element, "get_properties", None) - if get_properties is not None: - return get_properties() or {} - return getattr(flow_element, "properties", None) or {} - - def needs_upgrade(self, tiers, flow_element): - - """! - Whether a property the metadata lists is missing because the - engine's data file is not of one of the property's tiers. An engine - with no data file never needs an upgrade. - - @type tiers: list - @param tiers: the data tiers that hold the property - @type flow_element: FlowElement - @param flow_element: the engine - @rtype: bool - @return: True where an upgrade is needed - """ - - return False - - def engine_reason(self, flow_element): - - """! - The reason for a property the metadata does not list, which a kind - of engine with a step of its own overrides. - - @type flow_element: FlowElement - @param flow_element: the engine - @rtype: tuple - @return: the reason, and the property names for a message that - lists them - """ - - return MissingPropertyReason.UNKNOWN, () - - -class OnPremiseMissingPropertyService(MissingPropertyService): - - """! - The missing property service of an on-premise engine, whose data file - tier decides whether a property the metadata lists needs an upgrade. - The engine's native engine gives the tier. - """ - - def needs_upgrade(self, tiers, flow_element): - - """! - Whether the tier of the data file the engine uses is not among a - property's tiers. - - @type tiers: list - @param tiers: the data tiers that hold the property - @type flow_element: FlowElement - @param flow_element: the engine - @rtype: bool - @return: True where an upgrade is needed + @raise Exception: always, saying why the property is missing """ - return flow_element.engine.getProduct() not in tiers + raise Exception("Property " + key + " not found in " + flow_element.datakey) diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_engines.py b/pipeline/fiftyone_pipeline_engines/tests/test_engines.py index 86553b9c..3b30e850 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_engines.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_engines.py @@ -140,10 +140,7 @@ def testRestrictedProperty(self): except Exception as e: result = str(e) - self.assertEqual( - "Property 'integer' not found in data for element 'example'. " - "This is because the property has been excluded when configuring " - "the engine.", result) + self.assertTrue(result == "Property integer was excluded from example") # Restricted property names are compared ignoring case def testRestrictedPropertyIgnoresCase(self): @@ -167,9 +164,7 @@ def testRestrictedPropertyIgnoresCase(self): with self.assertRaises(Exception) as refused: flowdata.example.get(key) self.assertEqual( - "Property '" + key + "' not found in data for element " - "'example'. This is because the property has been excluded " - "when configuring the engine.", + "Property " + key + " was excluded from example", str(refused.exception)) # Test restricting properties @@ -188,10 +183,7 @@ def testMissingPropertyService(self): except Exception as e: result = str(e) - self.assertEqual( - "Property 'missing' not found in data for element 'example'. " - "The reason for this is unknown. Please check that the aspect and " - "property name are correct.", result) + self.assertTrue(result == "Property missing not found in example") # Test cache diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_missing_property_reasons.py b/pipeline/fiftyone_pipeline_engines/tests/test_missing_property_reasons.py deleted file mode 100644 index c9944ed4..00000000 --- a/pipeline/fiftyone_pipeline_engines/tests/test_missing_property_reasons.py +++ /dev/null @@ -1,121 +0,0 @@ -# ********************************************************************* -# This Original Work is copyright of 51 Degrees Mobile Experts Limited. -# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, -# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. -# -# This Original Work is licensed under the European Union Public Licence -# (EUPL) v.1.2 and is subject to its terms as set out below. -# -# If a copy of the EUPL was not distributed with this file, You can obtain -# one at https://opensource.org/licenses/EUPL-1.2. -# -# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be -# amended by the European Commission) shall be deemed incompatible for -# the purposes of the Work and the provisions of the compatibility -# clause in Article 5 of the EUPL shall not apply. -# -# If using the Work as, or as part of, a network application, by -# including the attribution notice(s) required under Article 5 of the EUPL -# in the end user terms of the application under an appropriate heading, -# such notice(s) shall fulfill the requirements of that article. -# ********************************************************************* - -""" -The reason an on-premise engine gives for a missing property, decided in -the order the properties specification sets out, with the message of the -missing properties table for that reason. -""" - -import unittest - -from fiftyone_pipeline_engines.missingproperty_service import \ - OnPremiseMissingPropertyService - -# The pricing page the data file rows link to, with this repository's -# campaign parameters. -PRICING = "https://51degrees.com/pricing?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-src-fiftyone_pipeline_engines-missingproperty_service.py&utm_term=missing-property-pricing" - - -class StandInNativeEngine: - """Stands in for the native engine, giving the data file's tier.""" - - def __init__(self, tier): - self.tier = tier - - def getProduct(self): - return self.tier - - -class StandInEngine: - """Stands in for an on-premise engine, with its property metadata keyed - by lower-case name as the engines keep it.""" - - def __init__(self, tier, properties): - self.datakey = "device" - self.engine = StandInNativeEngine(tier) - self.properties = properties - - def get_properties(self): - return self.properties - - -def message(name, engine): - with unittest.TestCase().assertRaises(Exception) as raised: - OnPremiseMissingPropertyService().check(name, engine) - return str(raised.exception) - - -class OnPremiseReasonTests(unittest.TestCase): - - def test_a_property_of_other_tiers_needs_an_upgrade(self): - engine = StandInEngine("Lite", { - "screenpixelsheight": {"datafiles": ["Premium", "Enterprise"]}}) - self.assertEqual( - "Property 'ScreenPixelsHeight' not found in data for element " - "'device'. This is because your license or data file does not " - "include this property. It is available with the " - "Premium,Enterprise license/data for the StandInEngine. To " - "upgrade, see " + PRICING + ".", - message("ScreenPixelsHeight", engine)) - - def test_a_property_with_no_tiers_known_needs_an_upgrade(self): - engine = StandInEngine("Lite", {"screenpixelsheight": { - "datafiles": []}}) - self.assertEqual( - "Property 'screenpixelsheight' not found in data for element " - "'device'. This is because your license or data file does not " - "include this property. To see which licenses include it, see " - + PRICING + ".", - message("screenpixelsheight", engine)) - - def test_a_property_of_this_tier_was_excluded(self): - engine = StandInEngine("Enterprise", { - "screenpixelsheight": {"datafiles": ["Premium", "Enterprise"]}}) - self.assertEqual( - "Property 'screenpixelsheight' not found in data for element " - "'device'. This is because the property has been excluded when " - "configuring the engine.", - message("screenpixelsheight", engine)) - - def test_a_property_the_metadata_does_not_list_is_unknown(self): - engine = StandInEngine("Enterprise", { - "screenpixelsheight": {"datafiles": ["Enterprise"]}}) - self.assertEqual( - "Property 'notthere' not found in data for element 'device'. " - "The reason for this is unknown. Please check that the aspect " - "and property name are correct.", - message("notthere", engine)) - - def test_names_are_compared_ignoring_case(self): - engine = StandInEngine("Lite", { - "screenpixelsheight": {"datafiles": ["Enterprise"]}}) - for name in ("screenpixelsheight", "SCREENPIXELSHEIGHT", - "ScreenPixelsHeight"): - with self.subTest(name=name): - self.assertIn( - "It is available with the Enterprise license/data", - message(name, engine)) - - -if __name__ == "__main__": - unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index ee9b6c24..dc95ca16 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -149,11 +149,10 @@ def __init__( needs cookie to be set, and works the same whether or not share_all is True. @type query_whitelist: list or str - @param query_whitelist: The query string and HTTP form parameters - to share, as well as those prefixed with '51D_', when - share_all_query_string_parameters is False. A string of names - separated by commas, which is how a configuration file gives a - list, is read as those names. None holds no names. + @param query_whitelist: Deprecated, and has no effect. A query + string or HTTP form parameter is shared only when its name starts + with '51D_' or the Pipeline uses it as evidence, whatever the + settings. @type header_blacklist: list or str @param header_blacklist: By default, all HTTP headers except Cookie are shared. @@ -165,13 +164,13 @@ def __init__( @type share_all: bool @param share_all: If True, all evidence is shared apart from the email address, the salt used for hashed email identifiers, the - request's query string and path, and every cookie whose name does - not start with '51D_'. False by default. + request's query string and path, every cookie whose name does not + start with '51D_', and every query string or HTTP form parameter + whose name does not start with '51D_' and that the Pipeline does + not use as evidence. False by default. @type share_all_query_string_parameters: bool - @param share_all_query_string_parameters: If True, every query - string and HTTP form parameter is shared. If False, only those - prefixed with '51D_' and those in query_whitelist are. True by - default. + @param share_all_query_string_parameters: Deprecated, and has no + effect, as query_whitelist. @type max_queue_size: int @param max_queue_size: The most entries held while waiting to be sent. When the queue is full, new entries are dropped. The default @@ -746,7 +745,7 @@ def process_internal(self, flow_data): # A hash of the values that would be shared, as the element # accepts no evidence of its own. cachekey = self.tracker_evidence_key_filter.filter_evidence( - self.found_evidence(flow_data)) + self.found_evidence(flow_data), flow_data.pipeline) # Evidence keys ignore case, so a repeat is found whatever the # letters of its keys. cachekey = json.dumps( @@ -789,7 +788,8 @@ def get_data_from_evidence(self, flow_data): f"The value '{value}' could not be parsed to an integer.") else: # Check if we can send this piece of evidence - if self.data_evidence_key_filter.filter_evidence_key(key): + if self.data_evidence_key_filter.filter_evidence_key( + key, flow_data.pipeline): data.try_add(key, value) return data diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py index 9438ff5a..e37fc724 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py @@ -21,6 +21,7 @@ # ********************************************************************* from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.javascriptbuilder import JavascriptBuilderElement # Cookies, session values and query string parameters whose names start with # this are the ones 51Degrees sets, and are shared. @@ -37,6 +38,30 @@ "id.email", "id.salt", "request-query", "request-path", "header.cookie") +# The query string parameters the JavaScript Builder's specification lists as +# its accepted Evidence. Its filter also passes every other parameter, only so +# that it can send them back to the callback. +JAVASCRIPT_BUILDER_PARAMETERS = ( + "query.fod-js-object-name", "query.fod-js-enable-cookies") + + +def _used_by(pipeline, key): + """! + Whether a FlowElement of the Pipeline accepts the key as evidence. The + JavaScript Builder is asked only about the parameters its specification + says it accepts. + """ + if pipeline is None: + return False + for element in pipeline.flow_elements: + if isinstance(element, JavascriptBuilderElement): + if key.lower() in JAVASCRIPT_BUILDER_PARAMETERS: + return True + elif element.get_evidence_key_filter().filter_evidence_key(key): + return True + return False + + def _split_names(names): """! A configuration file gives a list of names as one comma-separated @@ -53,10 +78,10 @@ class ShareUsageEvidenceKeyFilter(BasicListEvidenceKeyFilter): """ The ShareUsageEvidenceKeyFilter decides which evidence the 51Degrees ShareUsage service is sent. Cookies are shared only when their names - start with 51D_, whatever the other settings. Headers are shared unless - they are blocked, session values only when their names start with 51D_, - and query string and form parameters when all of them are shared or - when their names start with 51D_ or are on the list to share. Any other + start with 51D_, and query string and form parameters only when their + names start with 51D_ or the Pipeline uses them as evidence, whatever + the other settings. Headers are shared unless they are blocked, and + session values only when their names start with 51D_. Any other evidence is shared. """ @@ -73,8 +98,9 @@ def __init__( @type cookie: string @param cookie: the name of the cookie that identifies the session @type query_whitelist: list or str - @param query_whitelist: query string and form parameters to share as - well as those starting 51D_. None holds no names. A string is read as names separated by commas. + @param query_whitelist: deprecated, and has no effect. A query + string or form parameter is shared only when its name starts with + 51D_ or the Pipeline uses it as evidence. @type header_blacklist: list or str @param header_blacklist: headers not to share. A string is read as names separated by commas. @@ -84,14 +110,16 @@ def __init__( @param share_all: whether all evidence is shared, apart from the evidence that is never shared and cookies not starting 51D_ @type share_all_query_string_parameters: bool - @param share_all_query_string_parameters: whether every query - string and form parameter is shared, whatever query_whitelist holds + @param share_all_query_string_parameters: deprecated, and has no + effect, as query_whitelist. """ self.cookie = cookie.lower() if cookie else None query_whitelist = _split_names(query_whitelist) header_blacklist = _split_names(header_blacklist) + # Kept so that code reading them still works. Neither decides what + # is shared. self.query_whitelist = \ [parameter.lower() for parameter in query_whitelist or ()] @@ -105,18 +133,40 @@ def __init__( self.share_all_query_string_parameters = \ share_all_query_string_parameters - def filter_evidence_key(self, key): + def filter_evidence(self, evidence_key_object, pipeline=None): + + """! + The evidence to share, from a dictionary of evidence. + + @type evidence_key_object: dict + @param evidence_key_object: evidence keys and values + @type pipeline: Pipeline + @param pipeline: the Pipeline the evidence came to, whose evidence + decides which query string and form parameters are shared + @rtype: dict + @return: the evidence to share + """ + + return {key: value for key, value in evidence_key_object.items() + if self.filter_evidence_key(key, pipeline)} + + def filter_evidence_key(self, key, pipeline=None): """! Check if a specific key should be filtered. @type key: string @param key: to check in the filter + @type pipeline: Pipeline + @param pipeline: the Pipeline the evidence came to. Without one, a + query string or form parameter is shared only when its name starts + with 51D_. @rtype: bool @return: Is this key in the filter's keys list? """ + original = key key = key.lower() if any(key == never_shared or key.endswith("." + never_shared) @@ -133,6 +183,13 @@ def filter_evidence_key(self, key): return field.startswith(FIFTYONE_PREFIX) or \ (self.include_session and field == self.cookie) + # A query string or form parameter is shared only when 51Degrees set + # it or the Pipeline uses it as evidence, even when all evidence is, + # so that a value such as a password is never sent. + if separator and prefix == "query": + return field.startswith(FIFTYONE_PREFIX) or \ + _used_by(pipeline, original) + if self.share_all: return True @@ -146,12 +203,6 @@ def filter_evidence_key(self, key): # Only session values that start with the 51Degrees # cookie prefix return field.startswith(FIFTYONE_PREFIX) - elif prefix == "query": - # Share every parameter only when asked to. Otherwise only - # those starting 51d_ or in the list. - return self.share_all_query_string_parameters or \ - field.startswith(FIFTYONE_PREFIX) or \ - field in self.query_whitelist else: # Add anything that is not a cookie, header, session # variable or query parameter @@ -164,10 +215,10 @@ class ShareUsageTrackerEvidenceKeyFilter(ShareUsageEvidenceKeyFilter): out, because they make every request different. """ - def filter_evidence_key(self, key): + def filter_evidence_key(self, key, pipeline=None): if key.lower() in ("query.session-id", "query.sequence"): return False return super(ShareUsageTrackerEvidenceKeyFilter, self)\ - .filter_evidence_key(key) + .filter_evidence_key(key, pipeline) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py index 0256ca15..eb3a0fdc 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py @@ -22,10 +22,31 @@ import unittest +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.flowelement import FlowElement from fiftyone_pipeline_engines_fiftyone.share_usage_evidencekeyfilter import \ ShareUsageEvidenceKeyFilter +class UsingElement(FlowElement): + """Accepts the evidence keys given.""" + + def __init__(self, keys): + super(UsingElement, self).__init__() + self.keys = keys + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(self.keys) + + +class UsingPipeline: + """Stands in for a Pipeline whose FlowElement uses the keys given.""" + + def __init__(self, *keys): + self.flow_elements = [UsingElement(list(keys))] + + class ShareUsageEvidenceKeyFilterTests(unittest.TestCase): def test_default_rules(self): @@ -53,20 +74,22 @@ def test_default_rules(self): def test_field_is_everything_after_the_first_dot(self): """! - Check that a name containing a dot is matched as a whole against - the parameters to share. + Check that a parameter whose name contains a dot is matched as a + whole against the evidence the Pipeline uses. """ - keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + keys = ShareUsageEvidenceKeyFilter() + pipeline = UsingPipeline("query.valid.email") - self.assertTrue(keys.filter_evidence_key("query.valid.email")) - self.assertFalse(keys.filter_evidence_key("query.valid")) - self.assertFalse(keys.filter_evidence_key("query.other.name")) + self.assertTrue(keys.filter_evidence_key("query.valid.email", pipeline)) + self.assertFalse(keys.filter_evidence_key("query.valid", pipeline)) + self.assertFalse(keys.filter_evidence_key("query.other.name", pipeline)) def test_lists_ignore_case(self): """! - Check that the headers not to share and the parameters to share - are matched without regard to case. + Check that the headers not to share are matched without regard to + case, and that a parameter on the deprecated list to share is not + shared. """ keys = ShareUsageEvidenceKeyFilter( @@ -74,45 +97,55 @@ def test_lists_ignore_case(self): header_blacklist=["X-Secret"]) self.assertFalse(keys.filter_evidence_key("header.x-secret")) - self.assertTrue(keys.filter_evidence_key("query.campaign")) + self.assertFalse(keys.filter_evidence_key("query.campaign")) self.assertTrue(keys.filter_evidence_key("header.user-agent")) def test_lists_from_comma_separated_strings(self): """! Check that a string given for either list, the form a configuration file gives a list in, is read as names separated by - commas, each trimmed. + commas, each trimmed, and that the parameters named still are not + shared. """ keys = ShareUsageEvidenceKeyFilter( query_whitelist="Campaign, utm_source", header_blacklist="X-Secret,X-Other") - self.assertTrue(keys.filter_evidence_key("query.campaign")) - self.assertTrue(keys.filter_evidence_key("query.utm_source")) - self.assertFalse(keys.filter_evidence_key("query.c")) + self.assertEqual(["campaign", "utm_source"], keys.query_whitelist) + self.assertFalse(keys.filter_evidence_key("query.campaign")) + self.assertFalse(keys.filter_evidence_key("query.utm_source")) self.assertFalse(keys.filter_evidence_key("header.x-secret")) self.assertFalse(keys.filter_evidence_key("header.x-other")) self.assertTrue(keys.filter_evidence_key("header.x")) def test_no_parameter_list_shares_only_51d_parameters(self): """! - Check that None in place of the parameters to share holds no - names, as an empty list does, so only parameters starting 51D_ are - shared unless every parameter is. + Check that None in place of the deprecated parameters to share, and + an empty list, both share only parameters starting 51D_ when the + Pipeline uses no others. """ for query_whitelist in (None, []): - prefixed = ShareUsageEvidenceKeyFilter( - query_whitelist=query_whitelist) - every = ShareUsageEvidenceKeyFilter( - query_whitelist=query_whitelist, - share_all_query_string_parameters=True) + keys = ShareUsageEvidenceKeyFilter(query_whitelist=query_whitelist) - self.assertFalse(prefixed.filter_evidence_key("query.campaign")) - self.assertTrue( - prefixed.filter_evidence_key("query.51d_campaign")) - self.assertTrue(every.filter_evidence_key("query.campaign")) + self.assertFalse(keys.filter_evidence_key("query.campaign")) + self.assertTrue(keys.filter_evidence_key("query.51d_campaign")) + + def test_the_deprecated_option_changes_nothing(self): + """! + Check that the deprecated share_all_query_string_parameters option + shares nothing more, on or off, whatever the list. + """ + + for query_whitelist in (None, [], ["campaign"]): + for share_every in (False, True): + keys = ShareUsageEvidenceKeyFilter( + query_whitelist=query_whitelist, + share_all_query_string_parameters=share_every) + + self.assertFalse(keys.filter_evidence_key("query.campaign")) + self.assertTrue(keys.filter_evidence_key("query.51d_campaign")) def test_session_cookie(self): """! @@ -131,8 +164,9 @@ def test_session_cookie(self): def test_share_all(self): """! Check that sharing everything shares the evidence the other rules - leave out, apart from the evidence that is never shared and the - cookies that do not start 51D_. + leave out, apart from the evidence that is never shared, the + cookies that do not start 51D_, and the parameters that neither start + 51D_ nor are used by the Pipeline. """ keys = ShareUsageEvidenceKeyFilter(share_all=True) @@ -141,8 +175,9 @@ def test_share_all(self): self.assertFalse(keys.filter_evidence_key("cookie.sessionid")) self.assertTrue( keys.filter_evidence_key("cookie.51D_screenpixelswidth")) - self.assertTrue(keys.filter_evidence_key("query.password")) - self.assertTrue(keys.filter_evidence_key("query.id.usage")) + self.assertFalse(keys.filter_evidence_key("query.password")) + self.assertFalse(keys.filter_evidence_key("query.id.usage")) + self.assertTrue(keys.filter_evidence_key("query.51d_usage")) self.assertTrue(keys.filter_evidence_key("server.request-method")) self.assertFalse(keys.filter_evidence_key("query.id.email")) self.assertFalse(keys.filter_evidence_key("header.id.email")) @@ -153,18 +188,18 @@ def test_share_all(self): def test_every_query_parameter_when_sharing_them_all(self): """! - Check that sharing every query string parameter shares one not on - the list, and that a filter made without it keeps to the list. + Check that a parameter is shared when the Pipeline uses it, and + another is not, whatever the deprecated options say. """ every = ShareUsageEvidenceKeyFilter( - query_whitelist=["campaign"], - share_all_query_string_parameters=True) - listed = ShareUsageEvidenceKeyFilter(query_whitelist=["campaign"]) + query_whitelist=["password"], + share_all_query_string_parameters=True, + share_all=True) + pipeline = UsingPipeline("query.campaign") - self.assertTrue(every.filter_evidence_key("query.password")) - self.assertFalse(listed.filter_evidence_key("query.password")) - self.assertTrue(listed.filter_evidence_key("query.campaign")) + self.assertFalse(every.filter_evidence_key("query.password", pipeline)) + self.assertTrue(every.filter_evidence_key("query.campaign", pipeline)) def test_the_session_cookie_is_kept_for_the_tracker_alone(self): """! @@ -207,12 +242,13 @@ def test_never_shared_keys_match_whole_parts(self): never shared is treated as an ordinary key. """ - keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + keys = ShareUsageEvidenceKeyFilter() - self.assertTrue(keys.filter_evidence_key("query.valid.email")) + self.assertTrue(keys.filter_evidence_key( + "query.valid.email", UsingPipeline("query.valid.email"))) self.assertTrue( ShareUsageEvidenceKeyFilter(share_all=True) - .filter_evidence_key("query.valid.email")) + .filter_evidence_key("header.valid.email")) def test_naming_a_parameter_does_not_share_the_query_string(self): """! @@ -220,7 +256,9 @@ def test_naming_a_parameter_does_not_share_the_query_string(self): string, which carries every parameter. """ - keys = ShareUsageEvidenceKeyFilter(query_whitelist=["utm_source"]) + keys = ShareUsageEvidenceKeyFilter() + pipeline = UsingPipeline("query.utm_source") - self.assertTrue(keys.filter_evidence_key("query.utm_source")) - self.assertFalse(keys.filter_evidence_key("server.request-query")) + self.assertTrue(keys.filter_evidence_key("query.utm_source", pipeline)) + self.assertFalse( + keys.filter_evidence_key("server.request-query", pipeline)) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py index 9209ae53..b31b9f1f 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py @@ -21,8 +21,9 @@ # ********************************************************************* """A configuration file gives a list of names as one comma-separated string, -so the usage sharing element takes its query string parameters to share and -its headers not to share in that form, as well as in a list.""" +so the usage sharing element takes its headers not to share in that form, as +well as in a list. Its deprecated query string parameters to share are read +the same way and shared no more than any other parameter.""" import json import unittest @@ -44,8 +45,7 @@ "endpoint": "http://127.0.0.1:9", } -# Shares only the query string parameters listed and those starting 51D_, -# where every one is shared by default. +# The deprecated option, which no longer changes what is shared. LISTED_ONLY = {"share_all_query_string_parameters": False} # A configuration file that gives both lists as comma-separated strings. @@ -114,8 +114,9 @@ def test_query_parameters_to_share_from_a_string(self): ShareUsage(query_whitelist="first,second", **LISTED_ONLY, **QUEUE_ONLY)) - self.assert_shared(shared, ("first value", "second value")) - self.assert_not_shared(shared, ("third value", "letter value")) + self.assert_not_shared( + shared, + ("first value", "second value", "third value", "letter value")) def test_headers_not_to_share_from_a_string(self): shared = shared_alone_by( @@ -130,10 +131,11 @@ def test_lists_from_a_configuration_file(self): usage = pipeline.get_element("shareusage") shared = shared_by(pipeline, usage) - self.assert_shared( - shared, ("first value", "second value", "agent value")) + self.assertEqual(["first", "second"], usage.query_whitelist) + self.assert_shared(shared, ("agent value",)) self.assert_not_shared( - shared, ("third value", "letter value", "one value", "two value")) + shared, ("first value", "second value", "third value", + "letter value", "one value", "two value")) def test_names_are_trimmed_and_empty_entries_left_out(self): usage = ShareUsage( @@ -165,10 +167,10 @@ def test_a_list_is_used_as_it_is(self): self.assertIs(names, usage.query_whitelist) self.assertIs(headers, usage.header_blacklist) shared = shared_alone_by(usage) - self.assert_shared( - shared, ("first value", "second value", "agent value")) + self.assert_shared(shared, ("agent value",)) self.assert_not_shared( - shared, ("third value", "letter value", "one value", "two value")) + shared, ("first value", "second value", "third value", + "letter value", "one value", "two value")) if __name__ == "__main__": diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py index 5df7e9be..40a4bd51 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py @@ -21,10 +21,11 @@ # ********************************************************************* """ -What usage sharing shares by default and when all Evidence is shared. -Every query string and form parameter is shared unless that is turned off, -no cookie but a 51D_ one is ever shared, and the session cookie is still -compared when sessions are tracked. +What usage sharing shares by default and when all Evidence is shared. A query +string or form parameter is shared only when its name starts with 51D_ or the +Pipeline uses it as evidence, whatever the settings, no cookie but a 51D_ one +is ever shared, and the session cookie is still compared when sessions are +tracked. """ import gzip @@ -36,6 +37,8 @@ import requests from flask import Flask, request +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter from fiftyone_pipeline_core.flowelement import FlowElement from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_pipeline_core.web import webevidence @@ -53,8 +56,8 @@ "endpoint": "http://127.0.0.1:9", } -# Turns off sharing every query string parameter, from a configuration -# file. +# The deprecated option to share every query string parameter, from a +# configuration file. CONFIGURATION = """ { "PipelineOptions": { @@ -104,50 +107,91 @@ def records_of(usage, requests_evidence, *elements): return sent_on_close(usage) -class QueryAndFormValuesTests(unittest.TestCase): +class UsingElement(FlowElement): + """Accepts the query string parameter given as evidence, and adds no + data.""" + + def __init__(self, key): + super(UsingElement, self).__init__() + self.datakey = "using" + self.key = key + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter([self.key]) + + +# Every way the deprecated options could once share more. +DEPRECATED_SETTINGS = ( + {}, + {"share_all": True}, + {"query_whitelist": ["password", "campaign"]}, + {"share_all_query_string_parameters": True}, + {"share_all_query_string_parameters": False, + "query_whitelist": "password,campaign"}, +) + + +def shared_from_web_request(settings, method): + """The record usage sharing sends for one web request with the + parameters below, in its query string for GET or its form body for + POST, through a Pipeline whose other element uses query.used.""" + app = Flask(__name__) + usage = ShareUsage(**dict(QUEUE_ONLY, **settings)) + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS) \ + .add(UsingElement("query.used")).add(usage).build() + parameters = {"51D_screen": "fiftyone value", "used": "used value", + "password": "password value", "campaign": "campaign value"} + if method == "GET": + context = app.test_request_context("/", query_string=parameters) + else: + context = app.test_request_context( + "/", method="POST", data=parameters) + with context: + flowdata = pipeline.create_flowdata() + flowdata.evidence.add_from_dict(webevidence(request)) + flowdata.process() + records = sent_on_close(usage) + return records[0] if records else "" - def test_every_query_value_is_shared_by_default(self): - records = records_of(ShareUsage(**QUEUE_ONLY), [{ - "header.user-agent": "agent value", - "query.campaign": "campaign value", - "query.password": "password value", - }]) - self.assertEqual(1, len(records)) - self.assertIn('campaign value', - records[0]) - self.assertIn('password value', - records[0]) +class QueryAndFormValuesTests(unittest.TestCase): - def test_every_form_value_of_the_web_request_is_shared_by_default(self): - app = Flask(__name__) + def test_a_51d_parameter_is_shared_from_a_query_string_and_a_form(self): + for method in ("GET", "POST"): + with self.subTest(method=method): + record = shared_from_web_request({}, method) + self.assertIn("fiftyone value", record) + + def test_a_parameter_the_pipeline_uses_is_shared_from_both(self): + for method in ("GET", "POST"): + with self.subTest(method=method): + record = shared_from_web_request({}, method) + self.assertIn('used value', record) + + def test_no_other_parameter_is_ever_shared_from_either(self): + for settings in DEPRECATED_SETTINGS: + for method in ("GET", "POST"): + with self.subTest(settings=settings, method=method): + record = shared_from_web_request(settings, method) + self.assertIn("fiftyone value", record) + self.assertNotIn("password value", record) + self.assertNotIn("campaign value", record) + + def test_the_javascript_builder_uses_only_the_parameters_it_lists(self): usage = ShareUsage(**QUEUE_ONLY) - pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS).add(usage).build() - with app.test_request_context( - "/", method="POST", data={"field": "form value"}): - flowdata = pipeline.create_flowdata() - flowdata.evidence.add_from_dict(webevidence(request)) - flowdata.process() + pipeline = PipelineBuilder().add(usage).build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "agent value") + flowdata.evidence.add("query.password", "password value") + flowdata.evidence.add("query.fod-js-object-name", "object value") + flowdata.process() records = sent_on_close(usage) - self.assertEqual(1, len(records)) - self.assertIn('form value', records[0]) - - def test_turned_off_only_51d_and_listed_values_are_shared(self): - records = records_of(ShareUsage( - share_all_query_string_parameters=False, - query_whitelist=["campaign"], **QUEUE_ONLY), [{ - "header.user-agent": "agent value", - "query.campaign": "campaign value", - "query.51d_gethighentropyvalues": "hev", - "query.password": "password value", - }]) - - self.assertIn("campaign value", records[0]) - self.assertIn("hev", records[0]) + self.assertIn("agent value", records[0]) + self.assertIn("object value", records[0]) self.assertNotIn("password value", records[0]) - def test_turned_off_from_a_configuration_file(self): + def test_the_deprecated_option_from_a_configuration_file(self): pipeline = PipelineBuilder().build_from_configuration( json.loads(CONFIGURATION)) usage = pipeline.get_element("shareusage") @@ -161,7 +205,6 @@ def test_turned_off_from_a_configuration_file(self): self.assertIn("agent value", records[0]) self.assertNotIn("password value", records[0]) - class CookieTests(unittest.TestCase): def test_only_51d_cookies_are_shared_when_all_evidence_is_shared(self): diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index c415387e..c3749389 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -38,6 +38,22 @@ import requests from unittest import mock +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.flowelement import FlowElement + + +class UsingElement(FlowElement): + """Accepts the evidence keys given, and adds no data.""" + + def __init__(self, keys): + super(UsingElement, self).__init__() + self.datakey = "using" + self.keys = keys + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(self.keys) + testHost = "127.0.0.1" testPort = 5000 testEndpoint = f"http://{testHost}:{testPort}" @@ -919,9 +935,9 @@ def test_escapes_xml_characters(self): def test_default_shares_only_accepted_evidence(self): """! - Check that with no options the element shares headers, the - 51Degrees cookies and every query string and form parameter, and - leaves out other cookies, the Cookie header and the email address. + Check that with no options the element shares headers and the + 51Degrees cookies and parameters, and leaves out other cookies, the + Cookie header, and other query string and form parameters. """ usage = ShareUsage( @@ -948,7 +964,7 @@ def test_default_shares_only_accepted_evidence(self): self.assertIn('hev', app.received[0]) self.assertNotIn("cookie value", app.received[0]) self.assertNotIn("header value", app.received[0]) - self.assertIn('password value', app.received[0]) + self.assertNotIn("password value", app.received[0]) self.assertNotIn("email value", app.received[0]) def test_never_shared_evidence(self): @@ -984,8 +1000,9 @@ def test_never_shared_evidence(self): def test_share_all(self): """! Check that sharing all evidence shares what the other rules leave - out, but not the evidence that is never shared or a cookie that - does not start 51D_. + out, but not the evidence that is never shared, a cookie that + does not start 51D_, or a parameter that neither starts 51D_ nor is + used by the Pipeline. """ usage = ShareUsage( @@ -1007,22 +1024,21 @@ def test_share_all(self): self.assertEqual(len(app.received), 1) self.assertNotIn("cookie value", app.received[0]) self.assertIn('1080', app.received[0]) - self.assertIn('campaign value', app.received[0]) + self.assertNotIn("campaign value", app.received[0]) self.assertNotIn("email value", app.received[0]) def test_parameter_names_with_dots(self): """! Check that a query string parameter whose name contains a dot is - matched by its whole name against the parameters to share. + matched by its whole name against the evidence the Pipeline uses. """ usage = ShareUsage( requested_package_size=1, - endpoint=testEndpoint, - share_all_query_string_parameters=False, - query_whitelist=["valid.email"]) + endpoint=testEndpoint) pipeline = (PipelineBuilder())\ + .add(UsingElement(["query.valid.email"]))\ .add(usage)\ .build() flowdata = pipeline.create_flowdata() From 34faff8289f07c53d3a8938c5f141b1c2457aa4e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 19 Nov 2026 20:16:05 +0000 Subject: [PATCH 123/136] Count only a whole number or digits as the sequence in usage sharing (#234) Usage sharing now records query.sequence only when it is an integer or a string of the digits 0 to 9, from 0 up to 2147483647, the largest value the Sequence Element writes. In a Pipeline without a Sequence Element, a floating point 2.0, True, a signed or spaced number, or a number with an underscore used to be turned into a number by int(), and now leaves the Sequence empty, which changes released behavior. A value that cannot be read no longer logs an error, and a Pipeline with a Sequence Element records the same sequences as before. Run 44479401972 built this head with only a temporary workflow file added and passed on Ubuntu and Windows with Python 3.10 to 3.14, without macOS. --- .../share_usage.py | 46 +++++++++-- .../tests/test_share_usage_record_fields.py | 80 +++++++++++++++++++ 2 files changed, 119 insertions(+), 7 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index dc95ca16..ef6ade7b 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -51,6 +51,41 @@ # fiftyone_pipeline_core. HTTP_REQUEST_KEY = "server.http-request" +# A sequence counts when it is an integer, or a string of the digits 0 to 9 +# with no sign or spaces, from 0 to _MAX_SEQUENCE. That is the largest 32 +# bit signed integer, which is the largest sequence the Sequence Element +# writes, because it adds one to at most 2147483646. Written out for the +# same reason as HTTP_REQUEST_KEY. +_SEQUENCE_DIGITS_PATTERN = re.compile(r"[0-9]+") +_MAX_SEQUENCE = 2147483647 + + +def _read_sequence(value): + """! + The sequence a page sent, or None when it is not present. + + @type value: object + @param value: The query.sequence Evidence, which may be anything + @rtype: int + @return: The sequence, or None. Nothing raises. + """ + + if isinstance(value, bool): + return None + if isinstance(value, int): + return value if 0 <= value <= _MAX_SEQUENCE else None + if not isinstance(value, str) or \ + not _SEQUENCE_DIGITS_PATTERN.fullmatch(value): + return None + # The length is checked first, because int() refuses a very long run + # of digits. + digits = value.lstrip("0") or "0" + if len(digits) > len(str(_MAX_SEQUENCE)): + return None + sequence = int(digits) + return sequence if sequence <= _MAX_SEQUENCE else None + + # The maximum length of a piece of evidence's value which can be # added to the usage data being sent. SHARE_USAGE_MAX_EVIDENCE_LENGTH = 10000 @@ -778,14 +813,11 @@ def get_data_from_evidence(self, flow_data): # The SessionID is dealt with separately. data.session_id = value elif folded == 'query.sequence': - # The Sequence is dealt with separately. - try: - sequence = int(value) + # The Sequence is dealt with separately. A value that is + # not present leaves it out, with no error. + sequence = _read_sequence(value) + if sequence is not None: data.sequence = sequence - except: - self._log( - "error", - f"The value '{value}' could not be parsed to an integer.") else: # Check if we can send this piece of evidence if self.data_evidence_key_filter.filter_evidence_key( diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record_fields.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record_fields.py index 2586e325..2ae24f76 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record_fields.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record_fields.py @@ -63,6 +63,23 @@ H2 = "1:65536;4:6291456|15663105|0|m,a,s,p" RESOURCE_KEY = "AQS5HKcyAAAA" +# The largest sequence the Sequence Element takes, from sequence-element.md. +MAX_INPUT_SEQUENCE = 2147483646 + +# The largest sequence a record carries, the largest 32 bit signed integer, +# which the Sequence Element writes when it is given MAX_INPUT_SEQUENCE. +MAX_SEQUENCE = MAX_INPUT_SEQUENCE + 1 + +# Sequence values that are not present. A floating point 2.0, True, a +# signed or spaced number, a number with an underscore in it, digits other +# than 0 to 9, a trailing newline, bytes, and a number above MAX_SEQUENCE, +# held as an integer, as digits or as a 5,000 digit string that int() +# cannot read by default. +ABSENT_SEQUENCES = ( + 2.0, True, False, " 7 ", "-3", -3, "+3", "1_0", "0x10", "2.0", "", + "two", "Ù£", "ï¼’", "7\n", b"2", + MAX_SEQUENCE + 1, str(MAX_SEQUENCE + 1), "9" * 5000) + def usage_element(**settings): """A usage sharing element that keeps what it would share, because its @@ -122,6 +139,69 @@ def test_session_id_and_sequence_are_recorded_whatever_their_case(self): self.assertEqual(2, data.sequence, sequence_key) self.assertEqual("203.0.113.7", data.client_ip, ip_key) + def test_a_sequence_that_is_not_a_whole_number_is_absent(self): + # A Pipeline with no Sequence Element hands the page's own value + # straight to usage sharing, which counts only a whole number or a + # string made of the digits 0 to 9. A floating point 2.0, True, + # a signed or spaced number and a number with an underscore in it + # are each absent, so the record carries no sequence rather than a + # number the page never sent. + usage = usage_element() + self.addCleanup(usage.close) + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS).add(usage).build() + for value in ABSENT_SEQUENCES: + flow_data = pipeline.create_flowdata() + flow_data.evidence.add("header.user-agent", "agent") + flow_data.evidence.add("query.sequence", value) + data = usage.get_data_from_evidence(flow_data) + self.assertEqual("", data.sequence, repr(value)[:20]) + + def test_an_absent_sequence_logs_no_error(self): + # sequence-element.md says of a value that is not present "No + # error is recorded". + usage = usage_element() + self.addCleanup(usage.close) + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS).add(usage).build() + with mock.patch.object(usage, "_log") as log: + for value in ABSENT_SEQUENCES: + flow_data = pipeline.create_flowdata() + flow_data.evidence.add("header.user-agent", "agent") + flow_data.evidence.add("query.sequence", value) + usage.get_data_from_evidence(flow_data) + self.assertEqual( + [], [c for c in log.call_args_list if c.args[0] == "error"]) + + def test_a_sequence_of_digits_is_recorded(self): + usage = usage_element() + self.addCleanup(usage.close) + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS).add(usage).build() + for value, expected in ((2, 2), ("2", 2), ("0002", 2), + (0, 0), ("0", 0), ("000", 0), + (MAX_SEQUENCE, MAX_SEQUENCE), + (str(MAX_SEQUENCE), MAX_SEQUENCE)): + flow_data = pipeline.create_flowdata() + flow_data.evidence.add("header.user-agent", "agent") + flow_data.evidence.add("query.sequence", value) + data = usage.get_data_from_evidence(flow_data) + self.assertEqual(expected, data.sequence, repr(value)) + + def test_the_largest_sequence_the_sequence_element_writes_is_recorded( + self): + # Usage sharing runs after the Sequence Element and reads what it + # wrote, which is one more than the largest sequence it takes. + usage = usage_element() + pipeline = PipelineBuilder().add(usage).build() + flow_data = pipeline.create_flowdata() + flow_data.evidence.add("header.user-agent", "agent") + flow_data.evidence.add("query.sequence", str(MAX_INPUT_SEQUENCE)) + flow_data.process() + + sent = sent_on_close(usage) + + self.assertEqual(1, len(sent)) + self.assertIn( + "" + str(MAX_SEQUENCE) + "", sent[0]) + def test_the_record_names_the_server_ip(self): with mock.patch.object(share_usage, "host_address", lambda: SERVER_IP): From 3abcb1c137ddb9a7024665fbfd1918dbae9411c8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 19 Nov 2026 20:16:15 +0000 Subject: [PATCH 124/136] Write a process identifier on every usage sharing record, the invariant values in the first record and once an hour, and a Package element for every 51Degrees package (#193) Every usage sharing record now opens with a ProcessId, a version 4 UUID created when the ShareUsage element is built. Values that never change, such as Version, Product, the flow elements, ServerIP, and Platform, go only in the element's first record, in the first record once an hour has passed, and in the first record after a server address that was still being looked up becomes known. Those records also name every 51Degrees package in use, with its installed version, as Package elements. This changes released behavior for any reader that expects the unchanging values in every record, which now has to match records to them by ProcessId. Run 44465747922 built this head with only a temporary workflow file added and passed on Ubuntu and Windows with Python 3.10 to 3.14, without macOS. --- .../share_usage.py | 203 ++++++++- .../test_share_usage_invariant_values.py | 407 ++++++++++++++++++ .../tests/test_share_usage_packages.py | 169 ++++++++ .../tests/test_share_usage_pipelines.py | 126 +++++- 4 files changed, 870 insertions(+), 35 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_invariant_values.py create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_packages.py diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index ef6ade7b..3e4076bf 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -30,7 +30,9 @@ import datetime import threading import time +import uuid from concurrent.futures import ThreadPoolExecutor +from importlib import metadata from importlib.metadata import version import requests @@ -122,6 +124,11 @@ def _read_sequence(value): # parts, as a timer cannot wait for any length of time on every platform. LONGEST_WAIT_SECONDS = 86400 +# The values that do not change while the element lives are written to its +# first record, and again to the first record written once this many +# seconds have passed since they were last written. +INVARIANT_VALUES_INTERVAL_SECONDS = 3600 + def devices_message(entries): """! @@ -318,6 +325,25 @@ def __init__( # is in, as each names its own flow elements. self._constant_xml_by_pipeline = {} + # Written to every record as its ProcessId. Created here and never + # stored, so an element built again, in this process or another, + # has a new one. + self.process_id = str(uuid.uuid4()) + + # The clock, in seconds, that the hour between writes of the values + # that do not change is measured by. Not a user setting, and here so + # that a test can control it. + self._clock = time.monotonic + + # When the element last wrote the values that do not change to a + # record, by the clock, or None before it has. One time for the + # element, whichever Pipeline each record is from. + self._invariants_written = None + # Whether the last record to carry them was written before the + # lookup of the server's address ended, so with an empty ServerIP. + self._invariants_without_server_ip = False + self._invariants_lock = threading.Lock() + # The address of this machine for the record's ServerIP, looked up # on its own thread so that processing never waits for a slow name # lookup. It is None until the lookup ends, and records made before @@ -447,32 +473,86 @@ def _send_remaining(self, deadline, done): def get_constant_xml(self, pipeline=None): """! The part of a record that is the same for every request of a - Pipeline. Given the Pipeline a request came from, it names that + Pipeline, being the values that do not change while the element + lives. Given the Pipeline a request came from, it names that Pipeline's flow elements, so an element in several Pipelines writes each one's own. With no Pipeline it names those of the one Pipeline - the element is in. + the element is in. A record carries it only when + _write_invariants says so. + """ + return self._constant_xml(pipeline, self._server_ip) + + def _constant_xml(self, pipeline, server_ip): + """! + get_constant_xml with the server's address read once, as server_ip, + which is None while the lookup is running. A part made then carries + an empty ServerIP and is not kept, so that ServerIP carries the + address once the lookup has ended. """ if pipeline is not None: xml = self._constant_xml_by_pipeline.get(pipeline) if xml is None: xml = self._make_constant_xml( - self.get_flow_elements(pipeline)) - if self._server_ip is not None: + self.get_flow_elements(pipeline), + list(pipeline.flow_elements), server_ip) + if server_ip is not None: self._constant_xml_by_pipeline[pipeline] = xml return xml if self.constant_xml is None: - xml = self._make_constant_xml(self.get_flow_elements()) - # Kept once the lookup of the address has ended, so that - # ServerIP carries the address from then on. - if self._server_ip is None: + xml = self._make_constant_xml( + self.get_flow_elements(), self._pipeline_flow_elements(), + server_ip) + if server_ip is None: return xml self.constant_xml = xml return self.constant_xml - def _make_constant_xml(self, flow_element_names): + def _invariant_xml(self, pipeline): + """! + The values that do not change, for the record being written for a + request from the Pipeline, or None when that record leaves them + out. The decision and the values are made under one lock from one + reading of the server's address, so that the address a record + carries is the one the decision was made on. + """ + with self._invariants_lock: + server_ip = self._server_ip + if not self._write_invariants(server_ip): + return None + return self._constant_xml(pipeline, server_ip) + + def _write_invariants(self, server_ip): + """! + Whether the record being written carries the values that do not + change, given the server's address, None while the lookup is + running. They go in the first record the element writes, and again + in the first one written once INVARIANT_VALUES_INTERVAL_SECONDS + have passed since they were last written, whether or not that + record is ever sent. When the last record to carry them was written + before the lookup ended, so with an empty ServerIP, they go again + in the first record written once the lookup has found the address. + No other record carries them. The element keeps one time for all + the Pipelines it is in, so the record that carries them can be from + any of them, and names that Pipeline's flow elements. Called with + _invariants_lock held. + """ + now = self._clock() + written = self._invariants_written + found_since = bool(server_ip) and self._invariants_without_server_ip + if written is not None and not found_since and \ + now - written < INVARIANT_VALUES_INTERVAL_SECONDS: + return False + self._invariants_written = now + self._invariants_without_server_ip = server_ip is None + return True + + def _make_constant_xml(self, flow_element_names, flow_elements=(), + server_ip=None): """! The constant part of a record for the flow elements of one - Pipeline. + Pipeline, given their names, the elements themselves, whose + packages the record names, and the server's address, written empty + when it is None. """ coreVersion = version("fiftyone_pipeline_core") osVersion = f"{ReplacedString(platform.system()).result} {ReplacedString(platform.release()).result}" @@ -488,11 +568,14 @@ def _make_constant_xml(self, flow_element_names): # The flow elements in the Pipeline for flow_element in flow_element_names: xml += f"{flow_element}" + # The release of each 51Degrees package in use + for name, package_version in fiftyone_packages(flow_elements): + xml += record_element("Package", package_version, name) xml += "Python" # The software language version xml += f"{pyVersion}" # The address of this machine - xml += plain_element("ServerIP", self._server_ip or "") + xml += plain_element("ServerIP", server_ip or "") # The OS name and version xml += f"{osVersion}" return xml @@ -504,6 +587,16 @@ def _look_up_server_ip(self): self._server_ip = "" self._server_ip_found.set() + def _pipeline_flow_elements(self): + """! + The flow elements of the one Pipeline this element is in, or none + when it is in more than one, as get_flow_elements gives no names + then either. + """ + if len(self.pipelines) == 1: + return list(self.pipelines[0].flow_elements) + return [] + def get_flow_elements(self, pipeline=None): """! The class names of the flow elements of the given Pipeline, or of @@ -707,8 +800,16 @@ def add_to_share_usage(self, data): xml += "" - # --- write invariant data - xml += self.get_constant_xml(data.pipeline) + # The process identifier, in every record, which ties a record that + # leaves out the values that do not change to those values. + xml += plain_element("ProcessId", self.process_id) + + # --- write invariant data, to the first record, once an hour, and + # to the first record once the server's address is found when the + # last record to carry them went without it + invariants = self._invariant_xml(data.pipeline) + if invariants is not None: + xml += invariants # --- write variable data # The SessionID used to track a series of requests @@ -882,6 +983,82 @@ def host_address(): return str(addresses[0]) if addresses else "" +# The text the author or author email of every package 51Degrees +# publishes holds. +FIFTYONE_AUTHOR = "51degrees" + +# The package that holds the Pipeline. +PIPELINE_PACKAGE = "fiftyone_pipeline_core" + + +def canonical_package_name(name): + """! + A package name as the package index compares names, so that each + package is named once however its name is spelled. + """ + return re.sub(r"[-_.]+", "_", name).lower() + + +def _fiftyone_distribution(name): + """! + The installed package of that name if 51Degrees publishes it, or None. + """ + try: + distribution = metadata.distribution(name) + except (metadata.PackageNotFoundError, ValueError): + return None + author = " ".join( + distribution.metadata.get(field) or "" + for field in ("Author", "Author-email")).lower() + return distribution if FIFTYONE_AUTHOR in author else None + + +def _required_names(distribution): + """! + The names of the packages a package requires, leaving out those it + requires only for an extra. + """ + names = [] + for requirement in distribution.requires or []: + if re.search(r"\bextra\s*==", requirement): + continue + match = re.match(r"\s*([A-Za-z0-9][A-Za-z0-9._-]*)", requirement) + if match: + names.append(match.group(1)) + return names + + +def fiftyone_packages(flow_elements): + """! + The name and version of every 51Degrees package in use, sorted by name + and each given once. These are the package that holds the Pipeline, + the package that holds each flow element and every 51Degrees package + those require. A flow element from no installed package, such as one + an application defines, adds none. + + @type flow_elements: list + @param flow_elements: the flow elements of the Pipeline + @rtype: list + @return: (name, version) pairs, with the name the package is published + under + """ + names = [PIPELINE_PACKAGE] + [ + type(element).__module__.partition(".")[0] + for element in flow_elements] + found = {} + while names: + distribution = _fiftyone_distribution(names.pop()) + if distribution is None: + continue + name = distribution.metadata["Name"] + key = canonical_package_name(name) + if key in found: + continue + found[key] = (name, distribution.version) + names.extend(_required_names(distribution)) + return [found[key] for key in sorted(found)] + + def plain_element(tag, value): """! An element of a usage sharing record that is not an Evidence entry, with diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_invariant_values.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_invariant_values.py new file mode 100644 index 00000000..0147066d --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_invariant_values.py @@ -0,0 +1,407 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +The values that do not change while a usage sharing element lives, written +to its first record and again once an hour, and the process identifier +written to every record. + +The rules are those of the specifications repository at commit d79bbe3b, +pipeline-specification/pipeline-elements/usage-sharing-element.md, which +holds specifications#189 as merged in 1d9836cf. The tests control the +element's clock rather than waiting an hour. +""" + +import gzip +import os +import re +import subprocess +import sys +import threading +import unittest +from unittest import mock + +import requests + +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines_fiftyone import share_usage +from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage + +NO_DEFAULT_ELEMENTS = { + "add_javascript_builder": False, + "use_setheader_properties": False, +} + +# The package size is never reached, so the element keeps what it would +# share and sends nothing until it is closed. +QUEUE_ONLY = { + "requested_package_size": 1000, + "endpoint": "http://127.0.0.1:9", +} + +# usage-sharing-element.md lines 181 to 188 at d79bbe3b list the invariant +# values, and lines 265 to 267 say "A record that leaves them out has no +# Version, Product, FlowElement, Package, Language, LanguageVersion, +# ServerIP or Platform element". +INVARIANT_ELEMENTS = ( + "Version", "Product", "FlowElement", "Package", "Language", + "LanguageVersion", "ServerIP", "Platform", +) + +HOUR = 3600 + +# An address from the range RFC 5737 sets aside for documentation. +SERVER_IP = "192.0.2.10" + +# How long a test waits for the lookup of the server's address it holds up. +LOOKUP_WAIT_SECONDS = 10 + +# Lines 167 and 168: "32 hexadecimal digits in groups of 8, 4, 4, 4 and 12 +# separated by hyphens", for a version 4 UUID (line 162). +UUID4 = re.compile( + r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}" + r"-[0-9a-fA-F]{12}\Z") + + +class Clock: + """A clock the test moves by hand, in seconds.""" + + def __init__(self): + self.now = 1000.0 + + def __call__(self): + return self.now + + +def sent_on_close(usage): + """Closes the element, which sends what it has queued, and returns each + record it posted, in the order written. The post is answered without + leaving the process.""" + sent = [] + + def post(url, headers=None, data=None, **kwargs): + sent.append(gzip.decompress(data).decode("utf8")) + return mock.Mock(status_code=200, text="") + + with mock.patch.object(requests, "post", post): + usage.close() + return [record for message in sent + for record in re.findall(r".*?", message)] + + +def process(pipeline, user_agent): + """Processes one request. Each test gives each request its own + User-Agent, so that no request repeats one already shared.""" + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("header.user-agent", user_agent) + flowdata.process() + + +def invariants_in(record): + """The names of the invariant elements a record carries.""" + return [name for name in INVARIANT_ELEMENTS + if re.search("<" + name + r"[ >]", record)] + + +def process_ids_in(record): + return re.findall(r"([^<]*)", record) + + +def element_with_clock(wait_for_server_ip=True): + """A usage sharing element in a Pipeline of its own, with a clock the + test moves. Unless asked not to, it waits until the lookup of the + server's address has ended, as a record written before then does not + count as carrying the invariant values.""" + usage = ShareUsage(**QUEUE_ONLY) + if wait_for_server_ip: + usage._server_ip_found.wait(LOOKUP_WAIT_SECONDS) + clock = Clock() + usage._clock = clock + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS).add(usage).build() + return usage, clock, pipeline + + +class InvariantValuesTests(unittest.TestCase): + + def test_the_first_record_carries_the_invariant_values(self): + # Line 203: "The invariant values MUST be written to the first + # record the element writes." + usage, clock, pipeline = element_with_clock() + process(pipeline, "first") + + records = sent_on_close(usage) + + self.assertEqual(1, len(records)) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[0])) + + def test_records_within_the_hour_leave_the_invariant_values_out(self): + # Lines 204 and 205: "They MUST be written again to the first + # record the element writes once an hour or more has passed since it + # last wrote them." Lines 208 and 209: "They MUST NOT be written to + # any other record." + usage, clock, pipeline = element_with_clock() + process(pipeline, "first") + clock.now += 1 + process(pipeline, "second") + clock.now += HOUR - 1.5 + process(pipeline, "third") + + records = sent_on_close(usage) + + self.assertEqual(3, len(records)) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[0])) + self.assertEqual([], invariants_in(records[1])) + self.assertEqual([], invariants_in(records[2])) + + def test_the_invariant_values_are_written_again_once_the_hour_is_up(self): + # Lines 204 and 205, "once an hour or more has passed since it last + # wrote them", so the hour runs from the record that last carried + # them, here the one written exactly an hour after the first. + usage, clock, pipeline = element_with_clock() + process(pipeline, "first") + clock.now += HOUR + process(pipeline, "an hour later") + clock.now += 1 + process(pipeline, "a second after that") + clock.now += HOUR - 1 + process(pipeline, "an hour after the second write") + clock.now += 2 * HOUR + 5 + process(pipeline, "long after") + + records = sent_on_close(usage) + + self.assertEqual( + [True, True, False, True, True], + [invariants_in(record) == list(INVARIANT_ELEMENTS) + for record in records]) + self.assertEqual([], invariants_in(records[2])) + + def test_a_record_that_leaves_them_out_is_otherwise_the_same(self): + # Lines 264 to 267: a record that leaves the invariant values out + # "is otherwise the same". + usage, clock, pipeline = element_with_clock() + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("header.user-agent", "first") + flowdata.evidence.add("query.51D_value", "one") + flowdata.process() + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("header.user-agent", "second") + flowdata.evidence.add("query.51D_value", "two") + flowdata.process() + + later = sent_on_close(usage)[1] + + for name in ("ProcessId", "SessionId", "Sequence", "ClientIP", + "DateSent"): + self.assertIn("<" + name + ">", later) + self.assertIn('
    second
    ', later) + self.assertIn('two', later) + + def test_a_record_written_before_the_server_ip_is_known_is_not_counted( + self): + # Processing does not wait for the lookup of the server's address, + # so a record written before it ends has an empty ServerIP. That + # record does not hold every invariant value, so the next record + # written once the lookup has ended carries them all, and the hour + # runs from that record. + release = threading.Event() + self.addCleanup(release.set) + + def slow(): + release.wait(LOOKUP_WAIT_SECONDS) + return SERVER_IP + + with mock.patch.object(share_usage, "host_address", slow): + usage, clock, pipeline = element_with_clock( + wait_for_server_ip=False) + process(pipeline, "before the lookup ends") + release.set() + self.assertTrue(usage._server_ip_found.wait(LOOKUP_WAIT_SECONDS)) + clock.now += 1 + process(pipeline, "after the lookup ends") + clock.now += HOUR - 2 + process(pipeline, "within the hour of that") + + records = sent_on_close(usage) + + self.assertEqual(3, len(records)) + self.assertIn("", records[0]) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[1])) + self.assertIn("" + SERVER_IP + "", records[1]) + self.assertEqual([], invariants_in(records[2])) + + def test_the_values_go_again_only_in_the_first_record_once_known(self): + # Lines 205 to 209: "When one or more of them is not yet known as + # the first record is written, for example a server address still + # being looked up, the element MUST write all of them again to the + # first record it writes after each such value becomes known. They + # MUST NOT be written to any other record." + # So a second record written while the lookup is still running + # leaves them out, and the hour runs from the record that carried + # the address. + release = threading.Event() + self.addCleanup(release.set) + + def slow(): + release.wait(LOOKUP_WAIT_SECONDS) + return SERVER_IP + + with mock.patch.object(share_usage, "host_address", slow): + usage, clock, pipeline = element_with_clock( + wait_for_server_ip=False) + process(pipeline, "first, while the lookup runs") + clock.now += 1 + process(pipeline, "second, while the lookup runs") + release.set() + self.assertTrue(usage._server_ip_found.wait(LOOKUP_WAIT_SECONDS)) + clock.now += 1 + process(pipeline, "first once the address is known") + clock.now += 1 + process(pipeline, "next after that") + # An hour less a second after the record that carried the address, + # and more than an hour after the first record. + clock.now += HOUR - 2 + process(pipeline, "within the hour of the address") + clock.now += 1 + process(pipeline, "an hour after the address") + + records = sent_on_close(usage) + + self.assertEqual(6, len(records)) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[0])) + self.assertIn("", records[0]) + self.assertEqual([], invariants_in(records[1])) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[2])) + self.assertIn("" + SERVER_IP + "", records[2]) + self.assertEqual([], invariants_in(records[3])) + self.assertEqual([], invariants_in(records[4])) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[5])) + self.assertIn("" + SERVER_IP + "", records[5]) + + def test_a_lookup_that_finds_no_address_does_not_send_them_again(self): + # A lookup that ends without an address leaves ServerIP empty for + # good, so no value becomes known and only the hour sends the + # values again. + with mock.patch.object(share_usage, "host_address", lambda: ""): + usage, clock, pipeline = element_with_clock() + process(pipeline, "first") + clock.now += 1 + process(pipeline, "second") + clock.now += HOUR - 1 + process(pipeline, "an hour after the first") + + records = sent_on_close(usage) + + self.assertEqual(3, len(records)) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[0])) + self.assertIn("", records[0]) + self.assertEqual([], invariants_in(records[1])) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[2])) + + +# Writes one record from a new element and prints its ProcessId, so that a +# test can compare the identifiers of two processes. +ONE_RECORD = """ +import gzip, re +from unittest import mock +import requests +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage +sent = [] +def post(url, headers=None, data=None, **kwargs): + sent.append(gzip.decompress(data).decode("utf8")) + return mock.Mock(status_code=200, text="") +usage = ShareUsage(requested_package_size=1000, endpoint="http://127.0.0.1:9") +pipeline = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}).add(usage).build() +flowdata = pipeline.create_flowdata() +flowdata.evidence.add("header.user-agent", "agent") +flowdata.process() +with mock.patch.object(requests, "post", post): + usage.close() +print(",".join(re.findall("([^<]*)", "".join(sent)))) +""" + + +class ProcessIdentifierTests(unittest.TestCase): + + def test_every_record_carries_the_same_identifier(self): + # Lines 158 and 159: "The element MUST create a process identifier + # when it is built, and MUST write it to every record as the + # ProcessId element." + usage, clock, pipeline = element_with_clock() + for index in range(5): + process(pipeline, f"agent {index}") + clock.now += HOUR / 2 + + records = sent_on_close(usage) + + self.assertEqual(5, len(records)) + identifiers = [process_ids_in(record) for record in records] + for found in identifiers: + self.assertEqual(1, len(found)) + self.assertEqual(1, len(set(found[0] for found in identifiers))) + self.assertRegex(identifiers[0][0], UUID4) + + def test_the_identifier_is_written_first_in_the_record(self): + # The snippet at lines 270 and 271 opens each Device with its + # ProcessId. + usage, clock, pipeline = element_with_clock() + process(pipeline, "first") + + record = sent_on_close(usage)[0] + + self.assertTrue(record.startswith(""), record) + + def test_a_fresh_element_has_a_new_identifier(self): + # Lines 171 to 174: a process that starts again creates a new + # identifier, and "A process that builds more than one Usage + # Sharing Element ... creates an identifier for each". + identifiers = [] + for _ in range(2): + usage, clock, pipeline = element_with_clock() + process(pipeline, "agent") + identifiers.extend(process_ids_in(sent_on_close(usage)[0])) + + self.assertEqual(2, len(identifiers)) + self.assertNotEqual(identifiers[0].lower(), identifiers[1].lower()) + + def test_a_new_process_has_a_new_identifier(self): + # Line 171: "The identifier is never stored, so a process that + # starts again creates a new one." + environment = dict(os.environ) + environment["PYTHONPATH"] = os.pathsep.join( + path for path in sys.path if path) + identifiers = [] + for _ in range(2): + output = subprocess.run( + [sys.executable, "-c", ONE_RECORD], env=environment, + capture_output=True, text=True, timeout=120, check=True) + identifiers.append(output.stdout.strip()) + + for identifier in identifiers: + self.assertRegex(identifier, UUID4) + self.assertNotEqual(identifiers[0].lower(), identifiers[1].lower()) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_packages.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_packages.py new file mode 100644 index 00000000..33245b35 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_packages.py @@ -0,0 +1,169 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +The Package elements of a usage sharing record. A record that carries the +invariant values names every 51Degrees package in use with its version. +""" + +import gzip +import re +import unittest +from importlib import metadata +from unittest import mock + +import requests + +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage + +NO_DEFAULT_ELEMENTS = { + "add_javascript_builder": False, + "use_setheader_properties": False, +} + +# The package size is never reached, so the element keeps what it would +# share and sends nothing until it is closed. +QUEUE_ONLY = { + "requested_package_size": 1000, + "endpoint": "http://127.0.0.1:9", +} + +# The packages a Pipeline holding only usage sharing uses, being the one +# holding the Pipeline, the one holding the element and the one that +# requires. +USAGE_SHARING_PACKAGES = ( + "fiftyone_pipeline_core", + "fiftyone_pipeline_engines", + "fiftyone_pipeline_engines_fiftyone", +) + + +def canonical(name): + """A package name as the package index compares names.""" + return re.sub(r"[-_.]+", "_", name).lower() + + +def sent_on_close(usage): + """Closes the element, which sends what it has queued, and returns each + record it posted. The post is answered without leaving the process.""" + sent = [] + + def post(url, headers=None, data=None, **kwargs): + sent.append(gzip.decompress(data).decode("utf8")) + return mock.Mock(status_code=200, text="") + + with mock.patch.object(requests, "post", post): + usage.close() + return [record for message in sent + for record in re.findall(r".*?", message)] + + +def records_of(usage, requests_evidence, *elements): + """Processes each request's Evidence through a Pipeline holding the + elements and the usage sharing element, and returns the records sent.""" + builder = PipelineBuilder(NO_DEFAULT_ELEMENTS) + for element in elements: + builder.add(element) + pipeline = builder.add(usage).build() + for evidence in requests_evidence: + flowdata = pipeline.create_flowdata() + for key, value in evidence.items(): + flowdata.evidence.add(key, value) + flowdata.process() + return sent_on_close(usage) + + +def packages_in(record): + """The Package elements of a record, as name and version pairs.""" + return re.findall(r'([^<]*)', record) + + +class StarSign(FlowElement): + """An element from no installed package.""" + + def __init__(self): + super(StarSign, self).__init__() + self.datakey = "starsign" + + def process_internal(self, flow_data): + pass + + +class PackageTests(unittest.TestCase): + """A record that carries the invariant values names every 51Degrees + package in use with its version (usage-sharing-element.md lines 313 to + 321 at d79bbe3b).""" + + def test_the_first_record_names_every_51degrees_package_in_use(self): + # The packages are invariant values, so the record after the first, + # written within the hour, names none (usage-sharing-element.md + # lines 181 to 209 at d79bbe3b). + usage = ShareUsage(**QUEUE_ONLY) + # A record written before the lookup of the server's address ends + # does not count as carrying them, so wait for it. + usage._server_ip_found.wait(10) + records = records_of(usage, [ + {"header.user-agent": "first"}, + {"header.user-agent": "second"}, + ]) + + self.assertEqual(2, len(records)) + packages = packages_in(records[0]) + self.assertEqual( + sorted(USAGE_SHARING_PACKAGES), + sorted(canonical(name) for name, _ in packages)) + for name, version in packages: + self.assertEqual(metadata.version(name), version, name) + self.assertEqual([], packages_in(records[1])) + + def test_packages_follow_the_flow_elements(self): + record = records_of(ShareUsage(**QUEUE_ONLY), [ + {"header.user-agent": "agent value"}])[0] + + self.assertRegex( + record, + r"(]*>[^<]*)+") + + def test_an_element_from_no_package_names_no_package(self): + record = records_of(ShareUsage(**QUEUE_ONLY), [ + {"header.user-agent": "agent value"}], StarSign())[0] + + self.assertIn("StarSign", record) + self.assertEqual( + sorted(USAGE_SHARING_PACKAGES), + sorted(canonical(name) for name, _ in packages_in(record))) + + def test_packages_that_are_not_51degrees_are_not_named(self): + record = records_of(ShareUsage(**QUEUE_ONLY), [ + {"header.user-agent": "agent value"}])[0] + + names = [canonical(name) for name, _ in packages_in(record)] + # A record that named no package would pass the checks below. + self.assertIn("fiftyone_pipeline_core", names) + for other in ("requests", "cachetools", "flask"): + self.assertNotIn(other, names) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pipelines.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pipelines.py index 57722ecd..d5a8ad32 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pipelines.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pipelines.py @@ -21,10 +21,11 @@ # ********************************************************************* """ -One usage sharing element in two Pipelines. Each record names the flow -elements of the Pipeline its request came from, as -multi-pipeline-elements.md asks of an element that keeps state for a -Pipeline. +One usage sharing element in two Pipelines. A record that carries the +invariant values names the flow elements of the Pipeline its request came +from, as multi-pipeline-elements.md asks of an element that keeps state for +a Pipeline. The element writes those values to the first record it writes, +whichever Pipeline that is from, and keeps one hour for every Pipeline. """ import gzip @@ -43,6 +44,23 @@ "use_setheader_properties": False, } +HOUR = 3600 + +INVARIANT_ELEMENTS = ( + "Version", "Product", "FlowElement", "Package", "Language", + "LanguageVersion", "ServerIP", "Platform", +) + + +class Clock: + """A clock the test moves by hand, in seconds.""" + + def __init__(self): + self.now = 1000.0 + + def __call__(self): + return self.now + class FirstElement(FlowElement): @@ -85,30 +103,94 @@ def process(pipeline, user_agent): flowdata.process() +def flow_elements_in(record): + return re.findall(r"([^<]*)", record) + + +def invariants_in(record): + """The names of the invariant elements a record carries.""" + return [name for name in INVARIANT_ELEMENTS + if re.search("<" + name + r"[ >]", record)] + + +def element_in_two_pipelines(): + """One usage sharing element added to two Pipelines, with a clock the + test moves. It waits until the lookup of the server's address has + ended, as a record written before then does not count as carrying the + invariant values.""" + usage = ShareUsage(requested_package_size=1000, + endpoint="http://127.0.0.1:9") + usage._server_ip_found.wait(10) + clock = Clock() + usage._clock = clock + first = PipelineBuilder(NO_DEFAULT_ELEMENTS) \ + .add(FirstElement()).add(usage).build() + second = PipelineBuilder(NO_DEFAULT_ELEMENTS) \ + .add(SecondElement()).add(usage).build() + return usage, clock, first, second + + class ShareUsagePipelinesTests(unittest.TestCase): def test_each_record_names_the_elements_of_its_own_pipeline(self): - usage = ShareUsage(requested_package_size=1000, - endpoint="http://127.0.0.1:9") - first = PipelineBuilder(NO_DEFAULT_ELEMENTS) \ - .add(FirstElement()).add(usage).build() - second = PipelineBuilder(NO_DEFAULT_ELEMENTS) \ - .add(SecondElement()).add(usage).build() - - for _ in range(2): - process(first, "first agent") - process(second, "second agent") + # The flow elements are invariant values, so only a record that + # carries them names them. The second request comes an hour after + # the first, so that its record carries them too. + usage, clock, first, second = element_in_two_pipelines() + + process(first, "first agent") + clock.now += HOUR + process(second, "second agent") + records = sent_on_close(usage) + + self.assertEqual( + [["FirstElement", "ShareUsage"], ["SecondElement", "ShareUsage"]], + [flow_elements_in(record) for record in records]) + + def test_only_the_first_record_the_element_writes_carries_them(self): + # usage-sharing-element.md lines 203 to 209 at d79bbe3b: "The + # invariant values MUST be written to the first record the element + # writes" and "They MUST NOT be written to any other record". The + # first record from the second Pipeline is not the first record + # the element writes. + usage, clock, first, second = element_in_two_pipelines() + + for index in range(2): + process(first, f"first agent {index}") + clock.now += 1 + process(second, f"second agent {index}") + clock.now += 1 + records = sent_on_close(usage) + + self.assertEqual(4, len(records)) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[0])) + self.assertEqual(["FirstElement", "ShareUsage"], + flow_elements_in(records[0])) + for record in records[1:]: + self.assertEqual([], invariants_in(record)) + + def test_the_element_keeps_one_hour_for_every_pipeline(self): + # Lines 204 and 205: "They MUST be written again to the first + # record the element writes once an hour or more has passed since + # it last wrote them", from whichever Pipeline that record is. + usage, clock, first, second = element_in_two_pipelines() + + process(first, "first") + clock.now += HOUR / 2 + process(second, "half an hour later") + clock.now += HOUR / 2 + process(second, "an hour after the first") + clock.now += HOUR / 2 + process(first, "half an hour after that") records = sent_on_close(usage) - by_agent = {} - for record in records: - agent = re.search(r'Name="user-agent">([^<]*)<', record).group(1) - by_agent.setdefault(agent, []).append( - re.findall(r"([^<]*)", record)) self.assertEqual( - {"first agent": [["FirstElement", "ShareUsage"]], - "second agent": [["SecondElement", "ShareUsage"]]}, - by_agent) + [["FirstElement", "ShareUsage"], [], + ["SecondElement", "ShareUsage"], []], + [flow_elements_in(record) for record in records]) + self.assertEqual([], invariants_in(records[1])) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[2])) + self.assertEqual([], invariants_in(records[3])) if __name__ == "__main__": From f9013a0b9189c874a942db5861d1a25e7b082e42 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 20 Nov 2026 06:37:23 +0000 Subject: [PATCH 125/136] Report how the update on start-up ended, and schedule the next check (#233) The update on start-up reports how it ended and schedules the next check. An error in it is reported and followed, the data file is unregistered when the update raises, and closing the engine stops its checks after a failed build. The runner build of this head, run 44590980688, passed. --- .../datafile_update_service.py | 15 ++++- .../src/fiftyone_pipeline_engines/engine.py | 14 ++++- .../tests/test_datafile_update_http.py | 61 +++++++++++++++++++ .../tests/test_engine_in_pipelines.py | 40 ++++++++++++ 4 files changed, 125 insertions(+), 5 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py index c962eac9..aa46c997 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py @@ -786,9 +786,18 @@ def check_next_update(self, data_file, run_after_error = False): if data_file.update_on_start and not data_file.updated_on_start and not run_after_error: self._log("info", "Updating on start-up") - # The update on start-up logs and reports a failure as update - # polling does. - self._check(data_file, to_caller=False) + # An error from a row of the errors table is logged and + # reported here as update polling does, and is not raised. Any + # other error is reported, so every check is reported once it + # ends, then raised to the caller that registered the data file + # without being logged. Reporting schedules the next check, so + # one error never stops later checks. + try: + self._check(data_file, to_caller=False) + except Exception: + self._end_check( + data_file, UpdateStatus.AUTO_UPDATE_UNKNOWN_ERROR, True) + raise elif data_file.auto_update: diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py index 64bde2ed..9999fcd3 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py @@ -191,16 +191,26 @@ def on_registration(self, pipeline): The data file is registered with the update service of the first pipeline only, so an engine in several pipelines has its data file checked, downloaded and refreshed once. + An error from the update on start-up is passed on with the data file + unregistered, so no check is left running for a Pipeline that was + never made. @type pipeline: Pipeline @param pipeline: The pipeline the engine has been attached to - + """ if(self.data_file): if not hasattr(pipeline, "data_file_update_service"): pipeline.data_file_update_service = DataFileUpdateService(pipeline) if not getattr(self.data_file, "registered", False): - pipeline.data_file_update_service.register_data_file(self.data_file) + service = pipeline.data_file_update_service + try: + service.register_data_file(self.data_file) + except Exception: + # The Pipeline is not made, so close() could not reach + # this service. Cancel the check already scheduled. + service.unregister_data_file(self.data_file) + raise def close(self): diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py index bd43899d..62bfb516 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py @@ -1290,5 +1290,66 @@ def testTheNextPolledCheckStillAppliesAGoodFile(self): UpdateStatus.AUTO_UPDATE_SUCCESS], self.statuses) +class StartUpCheckErrorTests(UpdateServiceTestCase): + """! + An error the errors table does not name, raised in the update on + start-up. data-updates.md line 354 says every check MUST be reported + to onUpdateCallbacks once it ends, "including the checks made by + update polling, by the file watcher, by the update on start-up and by + a programmatic call". Lines 477 to 478 say "A programmatic check, the + update on start-up and an update from memory pass the error to their + caller instead of logging it", and lines 475 to 476 say a failure + "MUST NOT stop later checks for updates". + + So the check is reported as AUTO_UPDATE_UNKNOWN_ERROR, the error + reaches the caller that registered the data file, nothing is logged, + and the next check is still scheduled. + """ + + def setUp(self): + super().setUp() + self.data_file.update_on_start = True + self.data_file.updated_on_start = False + self.data_file.auto_update = True + self.hold_timers() + self.failure = ValueError("the update URL could not be made") + + def update_url(): + raise self.failure + + self.data_file.get_update_url = update_url + + def register(self): + """Registers the data file, which makes the update on start-up, + and returns the error it passed to this caller.""" + with self.assertRaises(ValueError) as caught: + self.service.register_data_file(self.data_file) + return caught.exception + + def testTheErrorIsReportedAndPassedToTheCaller(self): + self.assertIs(self.failure, self.register()) + self.assertEqual( + [UpdateStatus.AUTO_UPDATE_UNKNOWN_ERROR], self.statuses) + self.assertFalse(self.data_file.updating) + + def testTheErrorIsNotLogged(self): + self.register() + self.assertEqual( + [], [message for level, message in self.pipeline.logs + if level == "error"]) + + def testTheNextCheckIsScheduledAndStillAppliesAGoodFile(self): + self.register() + self.assertEqual(1, len(self.pending)) + self.get(response(200)) + self.data_file.get_update_url = lambda: URL + scheduled = self.pending.pop() + scheduled.function(*scheduled.args).join() + self.assertEqual([b"new"], self.data_file.refreshed) + self.assertEqual( + [UpdateStatus.AUTO_UPDATE_UNKNOWN_ERROR, + UpdateStatus.AUTO_UPDATE_SUCCESS], self.statuses) + + if __name__ == "__main__": unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_engine_in_pipelines.py b/pipeline/fiftyone_pipeline_engines/tests/test_engine_in_pipelines.py index a985fb9c..e537ef21 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_engine_in_pipelines.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_engine_in_pipelines.py @@ -91,5 +91,45 @@ def build(): self.assertTrue(engine.data_file.registered) +class StartUpErrorTests(unittest.TestCase): + """! + An error the errors table does not name, raised by the update on + start-up while a Pipeline is built. The error reaches the caller, so + the Pipeline is never made. resource-cleanup.md says "There MUST be + some mechanism to clean up system resources", so closing the engine + still stops its data file's checks. + """ + + def failing_engine(self): + engine = DataFileEngine() + self.addCleanup(engine.close) + data_file = engine.data_file + data_file.update_on_start = True + data_file.updated_on_start = False + data_file.auto_update = True + + def update_url(): + raise ValueError("the update URL could not be made") + + data_file.get_update_url = update_url + return engine + + def test_closing_the_engine_stops_the_checks_after_a_failed_build(self): + engine = self.failing_engine() + with self.assertRaises(ValueError): + Pipeline([engine]) + engine.close() + timer = getattr(engine.data_file, "update_timer", None) + if timer is not None: + timer.join(5) + self.assertFalse(timer.is_alive()) + + def test_a_failed_build_leaves_the_data_file_unregistered(self): + engine = self.failing_engine() + with self.assertRaises(ValueError): + Pipeline([engine]) + self.assertFalse(engine.data_file.registered) + + if __name__ == "__main__": unittest.main() From c692df98cf37772f1d2b711332c63bf9573a7006 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 20 Nov 2026 07:09:10 +0000 Subject: [PATCH 126/136] End the two web examples with their Find out more section The mixed cloud web example stopped with no key, or when the Pipeline could not be built, and the 51Did creator context demo stopped with no resource key. Neither printed the Find out more links its page ends with. Each now prints them on those early ends, with every 51degrees.com link tagged, and tests check the section and the tags of its links. --- .../cloud/mixed/gettingstarted_web/app.py | 35 ++++++++++ .../examples/tests/test_find_out_more.py | 65 ++++++++++++++++++- .../examples/creator_context_web/server.py | 23 +++++++ .../tests/test_creator_context_without_key.py | 24 ++++++- 4 files changed, 143 insertions(+), 4 deletions(-) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py index b2f1480e..7d0d3048 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py @@ -154,6 +154,39 @@ # Evidence values longer than this are shortened on the page. EVIDENCE_DISPLAY_LENGTH = 100 +# Where to go next. The page ends with these links, and the example prints +# them when it stops before the page is served. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_web-app.py&utm_term=find-out-more-device-detection"), + ("IP intelligence", + "https://51degrees.com/ip-intelligence?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_web-app.py&utm_term=find-out-more-ip-intelligence"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_web-app.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_web-app.py&utm_term=find-out-more-pricing"), + ("Pipeline API for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/pipeline"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), + ("IP intelligence engine", + "https://github.com/51Degrees/51degrees-cxx"), +] + + +def print_find_out_more(): + + """! + Print the "Find out more" section, a line of "=" and then each label + with its link, to standard output. The example prints it when it stops + before the page is served, which ends with the same links. + """ + + message = ["=" * 79, "Find out more:"] + for label, url in FIND_OUT_MORE: + message.append(f"\t{label}: {url}") + print("\n".join(message) + "\n") + def values(): @@ -442,6 +475,7 @@ def main(argv): "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_web-app.py&utm_term=resource-key-required. " "Once complete, set the first environment variable named at " "the start of this message to the key.") + print_find_out_more() return try: @@ -454,6 +488,7 @@ def main(argv): # refused it, for example because the key is limited to domains # that do not include the origin sent. logger.log("error", str(error)) + print_find_out_more() sys.exit(1) create_app(pipeline).run(port=int(os.environ.get("PORT", 5000))) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_find_out_more.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_find_out_more.py index 20a186c2..b9617572 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_find_out_more.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_find_out_more.py @@ -21,12 +21,17 @@ # ********************************************************************* """ -The mixed console ends with its "Find out more" section when it stops early, -as it does after a run. These run it to each early end, so they need no key -and no network. +The mixed console and the mixed web example each end with their "Find out +more" section when they stop early, as they do after a run. These run each +of them to each early end, so they need no key and no network. """ +import re + +import pytest + from fiftyone_pipeline_examples.cloud.mixed import gettingstarted_console +from fiftyone_pipeline_examples.cloud.mixed.gettingstarted_web import app from fiftyone_pipeline_examples.example_utils import ExampleUtils # Every name a key is read from. @@ -35,6 +40,25 @@ ExampleUtils.LICENSE_KEY_ENV_VAR, ] +# A 51degrees.com link. +LINK = re.compile(r"https://51degrees\.com/\S+") + +# The utm_content the web example's links carry, naming its file. +WEB_CONTENT = ( + "&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-" + "fiftyone_pipeline_examples-cloud-mixed-gettingstarted_web-app.py&") + + +def assert_tagged(printed, content): + """Each 51degrees.com link printed in the section is tagged as being + in the section and in the file named.""" + links = LINK.findall(printed.split("Find out more:", 1)[1]) + assert links + for link in links: + assert "utm_medium=example" in link + assert "utm_term=find-out-more" in link + assert content in link + def test_without_a_key(monkeypatch, capsys): for name in KEY_SETTINGS: @@ -59,3 +83,38 @@ def refuse(*args, **kwargs): gettingstarted_console.main(["a-resource-key"]) assert "Find out more:" in capsys.readouterr().out + + +def test_the_web_example_without_a_key(monkeypatch, capsys): + for name in KEY_SETTINGS: + monkeypatch.delenv(name, raising=False) + + def started(*args, **kwargs): + raise AssertionError("The web server was started") + + monkeypatch.setattr(app, "create_app", started) + + app.main([]) + + printed = capsys.readouterr().out + assert "Find out more:" in printed + for _, url in app.FIND_OUT_MORE: + assert url in printed + assert_tagged(printed, WEB_CONTENT) + + +def test_the_web_example_when_the_pipeline_cannot_be_built(monkeypatch, + capsys): + # Stands in for a key the cloud service refuses, or one that carries + # none of the properties the example shows. + def refuse(*args, **kwargs): + raise ValueError("The key carries none of the properties.") + + monkeypatch.setattr(app, "create_pipeline", refuse) + + with pytest.raises(SystemExit): + app.main(["a-resource-key"]) + + printed = capsys.readouterr().out + assert "Find out more:" in printed + assert_tagged(printed, WEB_CONTENT) diff --git a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py index a4a5e034..810a04a3 100644 --- a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py +++ b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py @@ -123,6 +123,28 @@ def env(name): or DEFAULT_API).rstrip("/") + "/" PORT = int(os.environ.get("PORT", "5100")) +# Where to go next. The page ends with these links, and the demo prints +# them when it stops before the page is served. +FIND_OUT_MORE = [ + ("What a 51Did is, and what the creator context binds it to", + "https://51degrees.com/documentation/_identifiers__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did-examples-creator_context_web-server.py&utm_term=find-out-more-identifiers"), + ("Talk to 51Degrees about using 51Did", + "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did-examples-creator_context_web-server.py&utm_term=find-out-more-contact-us"), + ("Pipeline API for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/pipeline"), + ("Specifications", + "https://github.com/51Degrees/specifications"), +] + + +def print_find_out_more(): + """Print the "Find out more" section, a line of "=" and then each + label with its link.""" + message = ["=" * 79, "Find out more:"] + for label, url in FIND_OUT_MORE: + message.append(f"\t{label}: {url}") + print("\n".join(message) + "\n") + def page_html(): # Both files are read PER REQUEST, not once at start-up. A demo left @@ -281,6 +303,7 @@ def run(): print("Set _51DEGREES_RESOURCE_KEY to the resource key of the " "page. Resource keys are created at " "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did-examples-creator_context_web-server.py&utm_term=resource-key-required") + print_find_out_more() return if not LICENCE: # Only an account that holds licence keys needs one to redeem, diff --git a/pipeline/fiftyone_pipeline_did/tests/test_creator_context_without_key.py b/pipeline/fiftyone_pipeline_did/tests/test_creator_context_without_key.py index 52261b7a..99239db0 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_creator_context_without_key.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_creator_context_without_key.py @@ -22,10 +22,13 @@ """ The creator context demo, started with no resource key, ends without an -error and says which environment variable to set and where to create a key. +error, says which environment variable to set and where to create a key, +and ends with its "Find out more" section, which the page carries and a +person who never reaches the page would otherwise never see. """ import io +import re import unittest from contextlib import redirect_stdout from unittest import mock @@ -38,6 +41,14 @@ r"https://51degrees\.com/configure[^\s?]*\?utm_source=code" r"&utm_medium=example&utm_campaign=51degrees-python&utm_content=") +# A 51degrees.com link. +LINK = re.compile(r"https://51degrees\.com/\S+") + +# The utm_content the demo's links carry, naming its file. +CONTENT = ( + "&utm_content=fiftyone_pipeline_did-examples-creator_context_web-" + "server.py&") + class CreatorContextWithoutKeyTests(unittest.TestCase): @@ -55,6 +66,17 @@ def test_says_where_to_create_a_resource_key(self): said = printed.getvalue() self.assertIn("_51DEGREES_RESOURCE_KEY", said) self.assertRegex(said, CONFIGURE_LINK) + self.assertIn("Find out more:", said) + for _, url in server.FIND_OUT_MORE: + self.assertIn(url, said) + # Each 51degrees.com link in the section is tagged as being in the + # section and in this file. + links = LINK.findall(said.split("Find out more:", 1)[1]) + self.assertTrue(links) + for link in links: + self.assertIn("utm_medium=example", link) + self.assertIn("utm_term=find-out-more", link) + self.assertIn(CONTENT, link) if __name__ == "__main__": From 70ae25f73c29a902b24c5ea109aa4c06460600fa Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 20 Nov 2026 08:07:50 +0000 Subject: [PATCH 127/136] Keep a Flow Data's Element Data in a linked list, added with no lock (#250) A Flow Data now keeps its Element Data in a linked list that a reader follows with no lock, so a read part way through is never upset by an add. FlowData.data returns a read-and-add view of that list rather than a dict, and Element Data can no longer be removed through it. Green run 44624783486. --- .../src/fiftyone_pipeline_core/flowdata.py | 151 ++++++++++- .../tests/test_flowdata_element_data_list.py | 253 ++++++++++++++++++ 2 files changed, 400 insertions(+), 4 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_flowdata_element_data_list.py diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py index 35b0c5ad..5d433bfe 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py @@ -25,6 +25,104 @@ from .messages import Messages import traceback import sys +from collections.abc import MutableMapping + + +def _find(node, key): + + # Follows the links from the node given to the first, so newest, node + # for the key. + while node is not None: + if node.key == key: + return node + node = node.next + return None + + +class _ElementDataNode(object): + + """! + One Element Data in a FlowData's list. A node never changes once it is + in the list, so a reader follows the links with no lock. + """ + + __slots__ = ("key", "element_data", "next") + + def __init__(self, key, element_data, next): + self.key = key + self.element_data = element_data + self.next = next + + +class ElementDataView(MutableMapping): + + """! + The Element Data of a FlowData, keyed by data key, as the list was when + the view was taken. Adds made elsewhere after that are not seen, so a + reader part way through always sees a whole list. + + Setting a key adds to the FlowData, and this view then shows the list + as it is after the add. Element Data is not removed, so del, pop, + popitem and clear raise TypeError. copy gives a dict. + + Iteration gives each key once, in the order the keys were first added. + While a Pipeline processes, each Flow Element adds its own in turn, so + that is Pipeline order. Element Data added another way, such as before + processing, is listed by when it was added. + """ + + def __init__(self, flowdata, head): + self._flowdata = flowdata + self._head = head + + def _latest(self): + # The list holds the newest first, so it is sorted into the order + # the keys were added, keeping the newest Element Data for each. + nodes = [] + node = self._head + while node is not None: + nodes.append(node) + node = node.next + latest = {} + for node in reversed(nodes): + latest[node.key] = node.element_data + return latest + + def __getitem__(self, key): + node = _find(self._head, key) + if node is None: + raise KeyError(key) + return node.element_data + + def __setitem__(self, key, element_data): + self._flowdata._add(key, element_data) + # This view now shows the list with the add, as a mapping should. + self._head = self._flowdata._head + + def __delitem__(self, key): + raise TypeError("Element Data is not removed from a FlowData") + + def __iter__(self): + return iter(self._latest()) + + def __len__(self): + return len(self._latest()) + + def __repr__(self): + return repr(self._latest()) + + def copy(self): + return self._latest() + + def keys(self): + return self._latest().keys() + + def values(self): + return self._latest().values() + + def items(self): + return self._latest().items() + class FlowData: """! @@ -33,6 +131,11 @@ class FlowData: It passes evidence to FlowElements in the Pipeline These elements can return ElementData or populate an errors object + The Element Data is kept in a linked list, newest first. An add puts a + new node at the head with a plain write and takes no lock, because no + parallel group adds at the same time. Readers follow the links with no + lock. + A FlowData is not safe to use from several threads at once. It is used by one thread at a time, and where one is shared across threads, the caller is responsible for any locking, for example by wrapping each use @@ -52,7 +155,8 @@ def __init__(self, pipeline): """ - self.data = {} + # The newest node of the Element Data list. See data. + self._head = None self.errors = {} self.pipeline = pipeline self.processed = False @@ -170,12 +274,15 @@ def get(self, flow_element_key): """ try: - return self.data[flow_element_key.lower()] - + node = _find(self._head, flow_element_key.lower()) except Exception: + node = None + if node is None: return self.pipeline.element_not_found(flow_element_key, self) + return node.element_data + def __getattr__(self, flow_element_key): """! @@ -221,7 +328,43 @@ def set_element_data(self, element_data): """ # Kept under the data key in lower case, which is how get reads it. - self.data[element_data.flow_element.datakey.lower()] = element_data + self._add(element_data.flow_element.datakey.lower(), element_data) + + def _add(self, key, element_data): + + # A new node at the head. A reader that has already read the head + # keeps the list it saw. + self._write_head(_ElementDataNode(key, element_data, self._head)) + + def _write_head(self, head): + + # The only write of the list, with a plain write, as no parallel + # group adds at the same time. + self._head = head + + @property + def data(self): + + """! + The Element Data in this FlowData, keyed by data key in lower case, + as the list is now. Iteration gives Pipeline order. + + @rtype: ElementDataView + """ + + return ElementDataView(self, self._head) + + @data.setter + def data(self, contents): + + # Replaces the list with the contents given, in their order. The + # new list is built first and put in place with one write, so a + # reader sees the old list or the new one. The mapping given is + # copied, so later changes to it are not seen. + head = None + for key, element_data in contents.items(): + head = _ElementDataNode(key, element_data, head) + self._write_head(head) def set_error(self, flow_error): diff --git a/pipeline/fiftyone_pipeline_core/tests/test_flowdata_element_data_list.py b/pipeline/fiftyone_pipeline_core/tests/test_flowdata_element_data_list.py new file mode 100644 index 00000000..694a658a --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_flowdata_element_data_list.py @@ -0,0 +1,253 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +A Flow Data keeps its Element Data in a linked list, as the thread safety +page of the specification says. An add puts a new node at the head with a +plain write and takes no lock. A node never changes once added, so a reader +that has started sees a whole list, as it was when it read the head. +""" + +import threading +import unittest + +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + + +class AddsItsData(FlowElement): + """Adds Element Data holding its own name.""" + + def __init__(self, datakey): + super(AddsItsData, self).__init__() + self.datakey = datakey + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter([]) + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"name": self.datakey})) + + +class ReadsWhileItAdds(AddsItsData): + """Starts to list the Element Data already added, adds its own part way + through, then finishes the list, as a reader on another thread would + meet an add.""" + + def process_internal(self, flowdata): + values = flowdata.data.values() + reading = iter(values) + self.first = next(reading) + super(ReadsWhileItAdds, self).process_internal(flowdata) + self.rest = list(reading) + self.taken_before = list(values) + + +def pipeline(*elements): + builder = PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False}) + for element in elements: + builder.add(element) + return builder.build() + + +def lock_types(): + return (type(threading.Lock()), type(threading.RLock())) + + +class ElementDataListTests(unittest.TestCase): + + def test_a_reader_part_way_through_is_not_upset_by_an_add(self): + reader = ReadsWhileItAdds("reader") + flowdata = pipeline( + AddsItsData("one"), AddsItsData("two"), reader).create_flowdata() + + flowdata.process() + + names = [reader.first.name] + [data.name for data in reader.rest] + self.assertEqual(["one", "two"], names) + + def test_a_view_taken_before_an_add_keeps_the_list_it_saw(self): + reader = ReadsWhileItAdds("reader") + flowdata = pipeline( + AddsItsData("one"), AddsItsData("two"), reader).create_flowdata() + + flowdata.process() + + self.assertEqual( + ["one", "two"], [data.name for data in reader.taken_before]) + self.assertEqual( + ["one", "two", "reader"], + [data.name for data in flowdata.data.values()]) + + def test_element_data_is_listed_in_pipeline_order(self): + flowdata = pipeline( + AddsItsData("one"), AddsItsData("two"), + AddsItsData("three")).create_flowdata() + + flowdata.process() + + self.assertEqual(["one", "two", "three"], list(flowdata.data)) + + def test_the_newest_add_for_a_key_is_the_one_read(self): + element = AddsItsData("one") + flowdata = pipeline(element).create_flowdata() + first = ElementDataDictionary(element, {"name": "first"}) + second = ElementDataDictionary(element, {"name": "second"}) + + flowdata.set_element_data(first) + flowdata.set_element_data(second) + + self.assertIs(second, flowdata.get("one")) + self.assertEqual(["one"], list(flowdata.data)) + + def test_an_add_takes_no_lock(self): + """A smoke test. It looks only at the FlowData's own attributes, so + a lock kept anywhere else would not be seen.""" + flowdata = pipeline(AddsItsData("one")).create_flowdata() + + flowdata.process() + + held = [name for name, value in vars(flowdata).items() + if isinstance(value, lock_types())] + self.assertEqual([], held) + + def test_a_write_through_a_view_shows_in_that_view(self): + element = AddsItsData("one") + flowdata = pipeline(element).create_flowdata() + view = flowdata.data + first = ElementDataDictionary(element, {"name": "first"}) + second = ElementDataDictionary(element, {"name": "second"}) + third = ElementDataDictionary(element, {"name": "third"}) + + view["first"] = first + view.update({"second": second}) + view.setdefault("third", third) + + self.assertIs(first, view["first"]) + self.assertIs(second, view["second"]) + self.assertIs(third, view["third"]) + self.assertEqual(["first", "second", "third"], list(view)) + self.assertIs(first, flowdata.get("first")) + + def test_a_copy_is_a_dict_of_the_element_data(self): + flowdata = pipeline( + AddsItsData("one"), AddsItsData("two")).create_flowdata() + flowdata.process() + + copy = flowdata.data.copy() + + self.assertIs(dict, type(copy)) + self.assertEqual(["one", "two"], list(copy)) + self.assertIs(flowdata.get("two"), copy["two"]) + + def test_element_data_is_not_removed(self): + flowdata = pipeline( + AddsItsData("one"), AddsItsData("two")).create_flowdata() + flowdata.process() + + def remove(view): + del view["one"] + + for removal in ( + remove, + lambda view: view.pop("one"), + lambda view: view.popitem(), + lambda view: view.clear()): + with self.assertRaises(TypeError): + removal(flowdata.data) + self.assertEqual(["one", "two"], list(flowdata.data)) + + def test_setting_data_copies_the_mapping_given(self): + element = AddsItsData("one") + flowdata = pipeline(element).create_flowdata() + given = {"one": ElementDataDictionary(element, {"name": "one"})} + + flowdata.data = given + given["two"] = ElementDataDictionary(element, {"name": "two"}) + + self.assertEqual(["one"], list(flowdata.data)) + + def test_setting_data_replaces_the_whole_list_at_once(self): + element = AddsItsData("one") + flowdata = pipeline(element).create_flowdata() + flowdata.data = { + "old": ElementDataDictionary(element, {"name": "old"})} + seen = [] + + class Contents(dict): + # Lists the Element Data part way through the replacement, as + # a reader on another thread could. + def items(self): + for key, value in dict.items(self): + yield key, value + seen.append(list(flowdata.data)) + + flowdata.data = Contents({ + "a": ElementDataDictionary(element, {"name": "a"}), + "b": ElementDataDictionary(element, {"name": "b"})}) + + self.assertEqual([["old"], ["old"]], seen) + self.assertEqual(["a", "b"], list(flowdata.data)) + + def test_a_reader_on_another_thread_always_sees_a_whole_list(self): + element = AddsItsData("one") + flowdata = pipeline(element).create_flowdata() + keys = ["key%d" % index for index in range(2000)] + seen = [] + failures = [] + done = threading.Event() + + def read(): + try: + while not done.is_set(): + listed = list(flowdata.data) + # A whole list is every key added so far, in order. + if listed != keys[:len(listed)]: + failures.append(listed) + seen.append(len(listed)) + except Exception as error: + failures.append(error) + + reader = threading.Thread(target=read) + reader.start() + try: + for key in keys: + flowdata.data[key] = ElementDataDictionary( + element, {"name": key}) + finally: + done.set() + reader.join() + + self.assertEqual([], failures) + self.assertTrue(seen) + self.assertEqual(keys, list(flowdata.data)) + + +if __name__ == "__main__": + unittest.main() From b64b480dc0fce574151383fe180fa01f49a45f10 Mon Sep 17 00:00:00 2001 From: 51DTechResponse <143603565+51DTechResponse@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:40:23 +0000 Subject: [PATCH 128/136] Listen on port 5151 by default in the web examples (#253) Listen on port 5151 by default in the web examples. Reviewed READY at c2eaa51d, with all 4 checks on that head passing. --- .../cloud/gettingstarted_web/app.py | 2 +- .../cloud/useragentclienthints_web/app.py | 2 +- .../onpremise/gettingstarted_web/app.py | 2 +- .../onpremise/useragentclienthints_web/app.py | 2 +- .../test_devicedetection_web_example_ports.py | 89 +++++++++++++++++++ .../cloud/gettingstarted_web/app.py | 2 +- .../onpremise/gettingstarted_web/app.py | 2 +- .../test_ipintelligence_web_example_ports.py | 78 ++++++++++++++++ location/readme.md | 4 +- .../examples/readme.md | 2 +- .../cloud/mixed/gettingstarted_web/app.py | 4 +- .../tests/test_cloud_mixed_web_port.py | 65 ++++++++++++++ .../examples/creator_context_web/server.py | 6 +- pipeline/fiftyone_pipeline_did/readme.md | 4 +- .../tests/creator_context_page_harness.js | 2 +- .../tests/test_creator_context_port.py | 59 ++++++++++++ pipeline/readme.md | 4 +- 17 files changed, 310 insertions(+), 19 deletions(-) create mode 100644 device-detection/fiftyone_devicedetection_examples/tests/test_devicedetection_web_example_ports.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_examples/tests/test_ipintelligence_web_example_ports.py create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_port.py create mode 100644 pipeline/fiftyone_pipeline_did/tests/test_creator_context_port.py diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py index 26a58f07..f2087c98 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py @@ -141,7 +141,7 @@ def build(self, resource_key, logger): def run(self): - GettingStartedWeb.app.run(port=int(os.environ.get("PORT", 5000))) + GettingStartedWeb.app.run(port=int(os.environ.get("PORT", 5151))) # First we make a JSON route that will be called from the client side and will return # a JSON encoded property database using any additional evidence provided by the client. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py index bee57eaa..3e7fedc6 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py @@ -101,7 +101,7 @@ def build(self, resource_key, logger): return self def run(self): - UserAgentClientHintsWeb.app.run(port=int(os.environ.get("PORT", 5000))) + UserAgentClientHintsWeb.app.run(port=int(os.environ.get("PORT", 5151))) # The main route performs device detection and renders the results. On the # second request, any additional client hints headers requested via the diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py index a1fdf296..833004cf 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py @@ -127,7 +127,7 @@ def run(self): # The pipeline lives as long as the app, and is closed when the server # stops, which releases the native data it holds. with GettingStartedWeb.pipeline: - GettingStartedWeb.app.run(port=int(os.environ.get("PORT", 5001))) + GettingStartedWeb.app.run(port=int(os.environ.get("PORT", 5151))) # First we make a JSON route that will be called from the client side and will return # a JSON encoded property database using any additional evidence provided by the client. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py index 9db4aa1b..cfd83889 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py @@ -97,7 +97,7 @@ def run(self): # The pipeline lives as long as the app, and is closed when the server # stops, which releases the native data it holds. with UserAgentClientHintsWeb.pipeline: - UserAgentClientHintsWeb.app.run(port=5001) + UserAgentClientHintsWeb.app.run(port=5151) # The main route performs device detection and renders the results. On the # second request, any additional client hints headers requested via the diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_devicedetection_web_example_ports.py b/device-detection/fiftyone_devicedetection_examples/tests/test_devicedetection_web_example_ports.py new file mode 100644 index 00000000..e78b48fa --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_devicedetection_web_example_ports.py @@ -0,0 +1,89 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Each web example listens on port 5151 by default. The examples that read +the PORT environment variable still do, because that is how the CI starts +them on ports of its own. The checks read the examples' source, so they +start no server and need no data file and no network. +""" + +import ast +import os +import unittest + +EXAMPLES = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "src", + "fiftyone_devicedetection_examples") +WEB_EXAMPLES = [ + os.path.join(EXAMPLES, "cloud", "gettingstarted_web", "app.py"), + os.path.join(EXAMPLES, "cloud", "useragentclienthints_web", "app.py"), + os.path.join(EXAMPLES, "onpremise", "gettingstarted_web", "app.py"), + os.path.join(EXAMPLES, "onpremise", "useragentclienthints_web", "app.py"), +] +# The examples whose port the PORT environment variable sets. +PORT_FROM_ENVIRONMENT = WEB_EXAMPLES[:3] + + +def read_port(node): + # The default port and the environment variable that overrides it, from + # a number, int() of one, or os.environ.get(name, number). + if isinstance(node, ast.Constant): + return int(node.value), None + if isinstance(node, ast.Call): + if getattr(node.func, "id", None) == "int" and node.args: + return read_port(node.args[0]) + if getattr(node.func, "attr", None) == "get" and len(node.args) == 2: + port, _ = read_port(node.args[1]) + return port, getattr(node.args[0], "value", None) + return None, None + + +def port_setting(path): + # The port the example passes to run, as read_port gives it. + with open(path, encoding="utf-8") as source: + tree = ast.parse(source.read()) + for node in ast.walk(tree): + if (isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "run"): + for keyword in node.keywords: + if keyword.arg == "port": + return read_port(keyword.value) + return None, None + + +class WebExamplePortTests(unittest.TestCase): + + def test_every_web_example_listens_on_5151_by_default(self): + for path in WEB_EXAMPLES: + with self.subTest(example=path): + self.assertEqual(5151, port_setting(path)[0]) + + def test_the_port_environment_variable_still_sets_the_port(self): + for path in PORT_FROM_ENVIRONMENT: + with self.subTest(example=path): + self.assertEqual("PORT", port_setting(path)[1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py index 82949a87..117c1fbf 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py @@ -136,7 +136,7 @@ def main(resource_key = None): "the start of this message with the key.") else: build_pipeline(resource_key) - app.run(debug=False) + app.run(port=5151, debug=False) # Every end, with no key or once the server stops, finishes here. ExampleUtils.print_find_out_more(FIND_OUT_MORE) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py index 5a567308..9a33b91e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py @@ -182,7 +182,7 @@ def main(data_file = None): # The pipeline lives as long as the app, and is closed when the server # stops, which releases the native data it holds. with build_pipeline(data_file): - app.run(debug=False) + app.run(port=5151, debug=False) ExampleUtils.print_find_out_more(FIND_OUT_MORE) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_ipintelligence_web_example_ports.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_ipintelligence_web_example_ports.py new file mode 100644 index 00000000..add45b31 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_ipintelligence_web_example_ports.py @@ -0,0 +1,78 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Each web example listens on port 5151 by default, rather than on the port +Flask picks when none is given. The checks read the examples' source, so +they start no server and need no data file and no network. +""" + +import ast +import os +import unittest + +EXAMPLES = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "src", + "fiftyone_ipintelligence_examples") +WEB_EXAMPLES = [ + os.path.join(EXAMPLES, "cloud", "gettingstarted_web", "app.py"), + os.path.join(EXAMPLES, "onpremise", "gettingstarted_web", "app.py"), +] + + +def read_port(node): + # The default port, from a number, int() of one, or + # os.environ.get(name, number). + if isinstance(node, ast.Constant): + return int(node.value) + if isinstance(node, ast.Call): + if getattr(node.func, "id", None) == "int" and node.args: + return read_port(node.args[0]) + if getattr(node.func, "attr", None) == "get" and len(node.args) == 2: + return read_port(node.args[1]) + return None + + +def default_port(path): + # The port the example passes to app.run, or None when it passes none. + with open(path, encoding="utf-8") as source: + tree = ast.parse(source.read()) + for node in ast.walk(tree): + if (isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "run"): + for keyword in node.keywords: + if keyword.arg == "port": + return read_port(keyword.value) + return None + + +class WebExamplePortTests(unittest.TestCase): + + def test_every_web_example_listens_on_5151_by_default(self): + for path in WEB_EXAMPLES: + with self.subTest(example=path): + self.assertEqual(5151, default_port(path)) + + +if __name__ == "__main__": + unittest.main() diff --git a/location/readme.md b/location/readme.md index 36b90325..bbd04a36 100644 --- a/location/readme.md +++ b/location/readme.md @@ -54,11 +54,11 @@ To run the web example navigate into Examples folder: #### Linux -Execute `export FLASK_APP=web` where `web` is the example file, and start your application via `flask run`. +Execute `export FLASK_APP=web` where `web` is the example file, and start your application via `flask run --port 5151`. #### Windows -Execute `$env:FLASK_APP = "web"` where `web` is the example file, and start your application via `flask run`. +Execute `$env:FLASK_APP = "web"` where `web` is the example file, and start your application via `flask run --port 5151`. A Flow Data cannot be used by two threads at once, so the web example creates a Flow Data for each request, uses it only in the code that handles diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md index 77caac69..824391d0 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md @@ -89,7 +89,7 @@ and the web example with .venv/bin/python -m fiftyone_pipeline_examples.cloud.mixed.gettingstarted_web ``` -then open . Set `PORT` to listen on another port. +then open . Set `PORT` to listen on another port. ### Tests diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py index 7d0d3048..8a4b46b8 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py @@ -58,7 +58,7 @@ # ``` # python -m fiftyone_pipeline_examples.cloud.mixed.gettingstarted_web # ``` -# and open http://localhost:5000, or the port named by the PORT environment +# and open http://localhost:5151, or the port named by the PORT environment # variable. Use the form on the page, or add ?client-ip=8.8.8.8 to the # address, to look up an IP address other than your own. # @@ -491,7 +491,7 @@ def main(argv): print_find_out_more() sys.exit(1) - create_app(pipeline).run(port=int(os.environ.get("PORT", 5000))) + create_app(pipeline).run(port=int(os.environ.get("PORT", 5151))) if __name__ == "__main__": diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_port.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_port.py new file mode 100644 index 00000000..6bc04ffa --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_port.py @@ -0,0 +1,65 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +The mixed web example listens on port 5151 by default, and on the port the +PORT environment variable names when it is set. The check reads the +example's source, so it starts no server and needs no network. +""" + +import ast +import os +import unittest + +APP = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "src", + "fiftyone_pipeline_examples", "cloud", "mixed", "gettingstarted_web", + "app.py") + + +def run_port(): + # The expression the example passes to run as its port. + with open(APP, encoding="utf-8") as source: + tree = ast.parse(source.read()) + for node in ast.walk(tree): + if (isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "run"): + for keyword in node.keywords: + if keyword.arg == "port": + return keyword.value + return None + + +class CloudMixedWebPortTests(unittest.TestCase): + + def test_listens_on_5151_unless_port_names_another(self): + port = run_port() + # int(os.environ.get("PORT", 5151)) + self.assertIsInstance(port, ast.Call) + get = port.args[0] + self.assertEqual("get", get.func.attr) + self.assertEqual(["PORT", 5151], [arg.value for arg in get.args]) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py index 810a04a3..c810585a 100644 --- a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py +++ b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py @@ -67,10 +67,10 @@ ``_51DEGREES_CLOUD_ENDPOINT`` is the cloud API base including the ``/api/v4/`` segment, defaulting to ``https://cloud.51degrees.com/api/v4/``, and is the same variable the cloud request engine and the ``DidClient`` honour. -``PORT`` is the port to listen on, defaulting to 5100. +``PORT`` is the port to listen on, defaulting to 5151. Standard library plus this package. Run ``python server.py`` then open -``http://localhost:5100/``. +``http://localhost:5151/``. """ import asyncio @@ -121,7 +121,7 @@ def env(name): # demo works unchanged against either. API = (env("_51DEGREES_CLOUD_ENDPOINT") or DEFAULT_API).rstrip("/") + "/" -PORT = int(os.environ.get("PORT", "5100")) +PORT = int(os.environ.get("PORT", "5151")) # Where to go next. The page ends with these links, and the demo prints # them when it stops before the page is served. diff --git a/pipeline/fiftyone_pipeline_did/readme.md b/pipeline/fiftyone_pipeline_did/readme.md index 8c2b2730..d7261af5 100644 --- a/pipeline/fiftyone_pipeline_did/readme.md +++ b/pipeline/fiftyone_pipeline_did/readme.md @@ -675,7 +675,7 @@ unreachable cloud answers 502 with `{ "error": ... }`. | `_51DEGREES_RESOURCE_KEY` | Required. The page's resource key, public by nature | | `_51DEGREES_LICENSE_KEY` | Optional. A licence key of the same account, held server side only. Only an account that holds licence keys needs one to redeem, so an account holding none runs without it | | `_51DEGREES_CLOUD_ENDPOINT` | Optional. The cloud API base including the `/api/v4/` segment, defaulting to `https://cloud.51degrees.com/api/v4/`. This is the same variable the cloud request engine honours. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons, which is the private hosting option of the cloud service. Both run the same service, so the demo works unchanged against either | -| `PORT` | The port to listen on, defaulting to `5100` | +| `PORT` | The port to listen on, defaulting to `5151` | ### Running @@ -686,7 +686,7 @@ cd fiftyone_pipeline_did/examples/creator_context_web python server.py ``` -then open `http://localhost:5100/`. To demonstrate across two devices, +then open `http://localhost:5151/`. To demonstrate across two devices, serve on an address both can reach and open the copied link on the second device. diff --git a/pipeline/fiftyone_pipeline_did/tests/creator_context_page_harness.js b/pipeline/fiftyone_pipeline_did/tests/creator_context_page_harness.js index 44c4bcaf..fa51d4ac 100644 --- a/pipeline/fiftyone_pipeline_did/tests/creator_context_page_harness.js +++ b/pipeline/fiftyone_pipeline_did/tests/creator_context_page_harness.js @@ -151,7 +151,7 @@ const sandbox = { navigator: { clipboard: { writeText: function () {} } }, location: { search: given ? '?51did=' + encodeURIComponent(given) : '', - origin: 'http://localhost:5100', + origin: 'http://localhost:5151', pathname: '/' } }; diff --git a/pipeline/fiftyone_pipeline_did/tests/test_creator_context_port.py b/pipeline/fiftyone_pipeline_did/tests/test_creator_context_port.py new file mode 100644 index 00000000..ba9a4ceb --- /dev/null +++ b/pipeline/fiftyone_pipeline_did/tests/test_creator_context_port.py @@ -0,0 +1,59 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The creator context example listens on port 5151 by default, and on the +port the PORT environment variable names when it is set. The check reads +the example's source, so it starts no server and needs no resource key.""" + +import ast +import unittest +from pathlib import Path + +SERVER_PY = Path(__file__).resolve().parents[1] / "examples" \ + / "creator_context_web" / "server.py" + + +def port_assignment(): + """The expression server.py assigns to PORT.""" + tree = ast.parse(SERVER_PY.read_text(encoding="utf-8")) + for node in tree.body: + if (isinstance(node, ast.Assign) + and any(getattr(target, "id", None) == "PORT" + for target in node.targets)): + return node.value + return None + + +class CreatorContextPortTests(unittest.TestCase): + + def test_listens_on_5151_unless_port_names_another(self): + port = port_assignment() + # int(os.environ.get("PORT", "5151")) + self.assertIsInstance(port, ast.Call) + get = port.args[0] + self.assertEqual("get", get.func.attr) + self.assertEqual( + ["PORT", "5151"], [str(arg.value) for arg in get.args]) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/readme.md b/pipeline/readme.md index c9bed17b..a6eb86b8 100644 --- a/pipeline/readme.md +++ b/pipeline/readme.md @@ -95,7 +95,7 @@ export FLASK_APP=fiftyone_pipeline_core/examples/client_side_evidence_custom_flo ``` then start your application with ```sh -flask run +flask run --port 5151 ``` ### Windows @@ -110,5 +110,5 @@ $env:FLASK_APP = "fiftyone_pipeline_core/examples/client_side_evidence_custom_fl ``` then start your application with ```pwsh -flask run +flask run --port 5151 ``` From 92e6c7f7659518efb1ae39b3492ee8c49c0808d7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 21 Nov 2026 16:20:16 +0000 Subject: [PATCH 129/136] Require the version of each package that holds the names imported (#231) Require the version of each package that holds the names imported. Reviewed at c123a0cc with no high finding open, and all 5 checks on that head passing. --- ci/write-package-versions.Tests.ps1 | 84 ++++++++ ci/write-package-versions.ps1 | 53 +++++ .../fiftyone_devicedetection_cloud/setup.py | 7 +- .../setup.py | 5 +- .../setup.py | 5 +- .../setup.py | 4 +- .../tests/test_package_requirements.py | 198 ++++++++++++++++++ pipeline/fiftyone_pipeline_engines/tox.ini | 4 + 8 files changed, 356 insertions(+), 4 deletions(-) create mode 100644 ci/write-package-versions.Tests.ps1 create mode 100644 pipeline/fiftyone_pipeline_engines/tests/test_package_requirements.py diff --git a/ci/write-package-versions.Tests.ps1 b/ci/write-package-versions.Tests.ps1 new file mode 100644 index 00000000..f97dae8d --- /dev/null +++ b/ci/write-package-versions.Tests.ps1 @@ -0,0 +1,84 @@ +# Tests for ci/write-package-versions.ps1, which writes the version.txt each +# published package's setup.py reads. +# +# Each test copies the script to a throwaway folder beside a packages.ps1 +# whose release plan is the one the test gives, so the test states the plan +# it writes from. Two packages stand in for the real ones. engines was last +# released at 4.5.43 and cre at 4.5.44, and cre requires engines>=4.5.44. + +Describe 'write-package-versions.ps1' { + BeforeAll { + $script:Script = Join-Path $PSScriptRoot 'write-package-versions.ps1' + + # Runs the script on a repository holding engines and cre, with the + # release plan given as package name to version. + function Invoke-Write([hashtable]$Plan) { + $ci = Join-Path $script:Folder 'ci' + $entries = @($Plan.GetEnumerator() | ForEach-Object { + "[pscustomobject]@{ Name = '$($_.Key)'; Version = '$($_.Value)' }" + }) -join ', ' + Set-Content (Join-Path $ci 'packages.ps1') @" +function Get-ReleasePlan(`$RepoPath) { @($entries) } +function Get-Packages(`$RepoPath) { + @([pscustomobject]@{ Name = 'engines'; Path = 'engines'; Publish = `$true }, + [pscustomobject]@{ Name = 'cre'; Path = 'cre'; Publish = `$true }) +} +function Get-ReleasedVersions(`$RepoPath, `$Name) { + @{ engines = @('4.5.42', '4.5.43'); cre = @('4.5.43', '4.5.44') }[`$Name] +} +"@ + & (Join-Path $ci 'write-package-versions.ps1') -RepoName $script:Repo + } + + function Get-Written([string]$Name) { + $file = Join-Path $script:Repo $Name 'version.txt' + if (Test-Path $file) { + return (Get-Content $file -Raw).Trim() + } + return $null + } + } + + BeforeEach { + $script:Folder = Join-Path ([IO.Path]::GetTempPath()) ` + ([Guid]::NewGuid().ToString()) + $script:Repo = Join-Path $script:Folder 'repo' + $null = New-Item -ItemType Directory (Join-Path $script:Folder 'ci'), + (Join-Path $script:Repo 'engines'), (Join-Path $script:Repo 'cre') + git init --quiet $script:Repo + Set-Content (Join-Path $script:Repo 'engines' 'setup.py') ` + 'install_requires=["requests"]' + Set-Content (Join-Path $script:Repo 'cre' 'setup.py') ` + 'install_requires=["engines>=4.5.44", "requests"]' + Copy-Item $script:Script (Join-Path $script:Folder 'ci') + } + + AfterEach { + Remove-Item -Recurse -Force $script:Folder + } + + It 'writes the planned versions when the plan releases a dependency at the version required' { + $null = Invoke-Write @{ engines = '4.5.44'; cre = '4.5.45' } + Get-Written 'engines' | Should -Be '4.5.44' + Get-Written 'cre' | Should -Be '4.5.45' + } + + It 'fails, naming both versions, when the plan leaves out a dependency required above its last release' { + { Invoke-Write @{ cre = '4.5.45' } } | + Should -Throw '*engines is required at 4.5.44 or later by cre*last release*4.5.43*' + Get-Written 'engines' | Should -BeNullOrEmpty + } + + It 'fails when the plan releases nothing and a requirement is above a last release' { + { Invoke-Write @{} } | + Should -Throw '*engines is required at 4.5.44 or later by cre*' + Get-Written 'engines' | Should -BeNullOrEmpty + } + + It 'fails when the plan gives a dependency a version below the one required' { + Set-Content (Join-Path $script:Repo 'cre' 'setup.py') ` + 'install_requires=["engines>=4.5.45"]' + { Invoke-Write @{ engines = '4.5.44'; cre = '4.5.45' } } | + Should -Throw '*engines is required at 4.5.45 or later by cre*release plan gives it 4.5.44*' + } +} diff --git a/ci/write-package-versions.ps1 b/ci/write-package-versions.ps1 index 44b23f25..378dd186 100644 --- a/ci/write-package-versions.ps1 +++ b/ci/write-package-versions.ps1 @@ -4,14 +4,49 @@ param ( $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'packages.ps1') +# The highest minimum version each package here is required at by another +# package here, read from the install_requires of every setup.py, with the +# package that names it. +function Get-RequiredMinimums($RepoName, $Packages) { + $minimums = @{} + foreach ($package in $Packages) { + $setup = Join-Path $RepoName $package.Path 'setup.py' + if (-not (Test-Path $setup)) { + continue + } + $text = Get-Content $setup -Raw + $pattern = '["''](?[A-Za-z0-9_.-]+)\s*>=\s*(?\d+(\.\d+)+)["'']' + foreach ($match in [regex]::Matches($text, $pattern)) { + $name = $match.Groups['name'].Value + $version = [version]$match.Groups['version'].Value + if (-not $minimums.ContainsKey($name) -or + $minimums[$name].Version -lt $version) { + $minimums[$name] = [pscustomobject]@{ + Version = $version + By = $package.Name + } + } + } + } + return $minimums +} + # Writes the version.txt that each published package's setup.py reads. Without # one a package installed from this checkout is 0.0.0, which is below the # versions the packages here require of each other, so pip refuses it, or # replaces it with a release from PyPI. Each package gets the version this # commit would release it at, as the publish gives it when it builds the # package, or its last release when this commit would not release it. +# +# When that is below the version another package here requires of it, no +# release from this commit would meet the requirement, and the package that +# names it could not be installed once published. So the script stops, before +# writing anything, and names both versions. $plan = @(Get-ReleasePlan $RepoName) $published = @(Get-Packages $RepoName | Where-Object { $_.Publish -eq $true }) +$minimums = Get-RequiredMinimums $RepoName $published +$versions = [ordered]@{} +$problems = @() foreach ($package in $published) { # A version.txt committed with the package is left as it is. if (git -C $RepoName ls-files -- "$($package.Path)/version.txt") { @@ -20,9 +55,27 @@ foreach ($package in $published) { $release = $plan | Where-Object { $_.Name -eq $package.Name } if ($null -ne $release) { $version = $release.Version + $source = "the release plan gives it $version" } else { $version = "$((Get-ReleasedVersions $RepoName $package.Name)[-1])" + $source = ("the release plan leaves it out, so it stays at its " + + "last release, $version") } + $required = $minimums[$package.Name] + if ($null -ne $required -and [version]$version -lt $required.Version) { + $problems += ("$($package.Name) is required at " + + "$($required.Version) or later by $($required.By), but $source.") + } + $versions[$package] = $version +} +if ($problems.Count -gt 0) { + throw (($problems + ("No release from this commit would meet these " + + "requirements. Find why the release plan does not release each " + + "package at the version required, or lower the requirement.")) -join + [Environment]::NewLine) +} +foreach ($package in $versions.Keys) { + $version = $versions[$package] $version | Out-File (Join-Path $RepoName $package.Path 'version.txt') Write-Output "$($package.Name) $version" } diff --git a/device-detection/fiftyone_devicedetection_cloud/setup.py b/device-detection/fiftyone_devicedetection_cloud/setup.py index 7318548d..cbf3f7dc 100644 --- a/device-detection/fiftyone_devicedetection_cloud/setup.py +++ b/device-detection/fiftyone_devicedetection_cloud/setup.py @@ -65,7 +65,12 @@ def read(file_name): python_requires=">=3.10", packages=["fiftyone_devicedetection_cloud"], package_dir={"": "src"}, - install_requires=["fiftyone_devicedetection_shared"], + install_requires=[ + "fiftyone_devicedetection_shared", + # The first version with CloudMissingPropertyService, which the + # hardware profile results of this package use. + "fiftyone_pipeline_cloudrequestengine>=4.5.45", + ], license="EUPL-1.2", classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/device-detection/fiftyone_devicedetection_onpremise/setup.py b/device-detection/fiftyone_devicedetection_onpremise/setup.py index 83b58226..b482bddc 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/setup.py +++ b/device-detection/fiftyone_devicedetection_onpremise/setup.py @@ -267,6 +267,9 @@ def stage_native_source(): # The first versions with close(), which releases the native memory # this package holds. "fiftyone_pipeline_core>=4.5.42", - "fiftyone_pipeline_engines>=4.5.42", + # The first version with OnPremiseMissingPropertyService, which the + # results of this package give a caller that reads a Property the + # data file in use does not hold. + "fiftyone_pipeline_engines>=4.5.44", ], ) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py index 2d850cf7..265e714f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py @@ -283,6 +283,9 @@ def stage_native_source(): # The first versions with close(), which releases the native memory # this package holds. "fiftyone_pipeline_core>=4.5.42", - "fiftyone_pipeline_engines>=4.5.42", + # The first version with OnPremiseMissingPropertyService, which the + # results of this package give a caller that reads a Property the + # data file in use does not hold. + "fiftyone_pipeline_engines>=4.5.44", ], ) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py b/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py index 223d23d8..5c51df1a 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py @@ -54,7 +54,9 @@ def read(file_name): packages=["fiftyone_pipeline_cloudrequestengine"], package_dir={"": "src"}, install_requires=[ - 'fiftyone_pipeline_engines', + # The first version with MissingPropertyReason, which this + # package gives as the reason a cloud Property has no value. + 'fiftyone_pipeline_engines>=4.5.44', 'requests', ], license="EUPL-1.2", diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_package_requirements.py b/pipeline/fiftyone_pipeline_engines/tests/test_package_requirements.py new file mode 100644 index 00000000..a0eeabe4 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines/tests/test_package_requirements.py @@ -0,0 +1,198 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import ast +import os +import re +import unittest + +# Each row is a name one package here imports from another package here, +# the package that holds it and the version of that package it came in at, +# so the importing package has to require at least that version. Without +# the minimum, pip leaves an older copy of the dependency in place when the +# importing package alone is upgraded, and the import fails. The test checks +# each import, the module holding each name and each requirement against the +# checkout. The version in each row is written here by hand, as the next +# release of the holding package after the last one without the name, and is +# not checked. +IMPORTED_NAMES = ( + ("pipeline/fiftyone_pipeline_cloudrequestengine", + "fiftyone_pipeline_engines", + "4.5.44", + "fiftyone_pipeline_engines.missingproperty_service", + "MissingPropertyReason"), + ("device-detection/fiftyone_devicedetection_cloud", + "fiftyone_pipeline_cloudrequestengine", + "4.5.45", + "fiftyone_pipeline_cloudrequestengine.clouddata", + "CloudMissingPropertyService"), + ("device-detection/fiftyone_devicedetection_onpremise", + "fiftyone_pipeline_engines", + "4.5.44", + "fiftyone_pipeline_engines.missingproperty_service", + "OnPremiseMissingPropertyService"), + ("ip-intelligence/fiftyone_ipintelligence_onpremise", + "fiftyone_pipeline_engines", + "4.5.44", + "fiftyone_pipeline_engines.missingproperty_service", + "OnPremiseMissingPropertyService"), +) + +# Where each package that holds one of the names above keeps its source. +PACKAGE_FOLDERS = { + "fiftyone_pipeline_engines": "pipeline/fiftyone_pipeline_engines", + "fiftyone_pipeline_cloudrequestengine": + "pipeline/fiftyone_pipeline_cloudrequestengine", +} + +# A requirement that gives a minimum version, as setup.py writes one. +MINIMUM = re.compile(r"^>=\s*([0-9]+(?:\.[0-9]+)*)$") + +# The name and the rest of one entry of an install_requires list. +REQUIREMENT = re.compile(r"^([A-Za-z0-9_.-]+)\s*(.*)$") + +# The entries of an install_requires list, each in its own quotes. +QUOTED = re.compile("[\"']([^\"']+)[\"']") + + +def repository_root(): + """The folder holding ci/packages.json, found by walking up from this + file, so the test reads the checkout it runs from.""" + folder = os.path.dirname(os.path.abspath(__file__)) + while True: + if os.path.isfile(os.path.join(folder, "ci", "packages.json")): + return folder + parent = os.path.dirname(folder) + if parent == folder: + raise AssertionError( + "no ci/packages.json above " + os.path.abspath(__file__)) + folder = parent + + +def requirements(package_folder): + """The install_requires of a package's setup.py, as the name required + mapped to the rest of the requirement.""" + path = os.path.join(repository_root(), package_folder, "setup.py") + with open(path, "rb") as file: + text = file.read().decode("utf-8") + found = re.search(r"install_requires\s*=\s*\[(.*?)\]", text, re.S) + assert found is not None, "no install_requires in " + path + out = {} + for item in QUOTED.findall(found.group(1)): + parts = REQUIREMENT.match(item.strip()) + if parts: + out[parts.group(1)] = parts.group(2).strip() + return out + + +def version_numbers(text): + """A version as a tuple of its numbers, for comparing two of them.""" + return tuple(int(part) for part in text.split(".")) + + +def imports_in(package_folder, module, name): + """Whether any source file of the package imports the name from the + module.""" + source = os.path.join(repository_root(), package_folder, "src") + for folder, _, files in os.walk(source): + if "cxx" in folder.split(os.sep): + continue + for file_name in files: + if not file_name.endswith(".py"): + continue + with open(os.path.join(folder, file_name), "rb") as file: + text = file.read().decode("utf-8", "replace") + try: + tree = ast.parse(text) + except SyntaxError: + continue + for node in ast.walk(tree): + if (isinstance(node, ast.ImportFrom) + and node.module == module + and any(alias.name == name + for alias in node.names)): + return True + return False + + +def module_names(module): + """The names a module of this checkout holds at its top level.""" + parts = module.split(".") + path = os.path.join(repository_root(), PACKAGE_FOLDERS[parts[0]], + "src", *parts) + ".py" + with open(path, "rb") as file: + text = file.read().decode("utf-8") + out = set() + for node in ast.parse(text).body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, + ast.ClassDef)): + out.add(node.name) + elif isinstance(node, ast.Assign): + out.update(target.id for target in node.targets + if isinstance(target, ast.Name)) + return out + + +class PackageRequirementTests(unittest.TestCase): + + """A package that imports a name from another package here requires + that package at the version the name came in at. Upgrading one package + on its own then brings the dependency with it, where a requirement + with no version leaves an older copy in place and the import fails.""" + + def test_each_name_is_imported_where_the_row_says(self): + for folder, _, _, module, name in IMPORTED_NAMES: + with self.subTest(package=folder, name=name): + self.assertTrue( + imports_in(folder, module, name), + "{0} does not import {1} from {2}".format( + folder, name, module)) + + def test_each_name_is_held_by_the_package_the_row_names(self): + for _, package, _, module, name in IMPORTED_NAMES: + with self.subTest(package=package, name=name): + self.assertEqual(package, module.split(".")[0]) + self.assertIn(name, module_names(module)) + + def test_each_package_is_required_at_the_version_of_the_name(self): + for folder, package, version, module, name in IMPORTED_NAMES: + with self.subTest(package=folder, requires=package): + required = requirements(folder) + self.assertIn( + package, required, + "{0} imports {1} from {2} and does not require " + "{3}".format(folder, name, module, package)) + found = MINIMUM.match(required[package]) + self.assertIsNotNone( + found, + "{0} requires {1} as {2!r}, with no minimum " + "version".format(folder, package, required[package])) + self.assertGreaterEqual( + version_numbers(found.group(1)), + version_numbers(version), + "{0} requires {1} {2}, below the {3} that holds " + "{4}".format(folder, package, required[package], + version, name)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines/tox.ini b/pipeline/fiftyone_pipeline_engines/tox.ini index 1e8e9619..80b97706 100644 --- a/pipeline/fiftyone_pipeline_engines/tox.ini +++ b/pipeline/fiftyone_pipeline_engines/tox.ini @@ -10,10 +10,14 @@ deps = description = run the tests with pytest package = wheel wheel_build_env = .pkg +# The Cloud Request Engine requires this package at the version this checkout +# releases, which a package index may not have yet, so this package is taken +# from the checkout too. The built wheel replaces it before the tests run. deps = {[common]deps} -e../fiftyone_pipeline_cloudrequestengine/ -e../fiftyone_pipeline_core/ + -e../fiftyone_pipeline_engines/ -e../fiftyone_pipeline_engines_fiftyone/ commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_engines {tty:--color=yes} {posargs} From 1952ff59655fdc14707cf1a83e2fa18630985488 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 21 Nov 2026 16:32:10 +0000 Subject: [PATCH 130/136] Specification conformance, integration tests that skip only what needs a key (#239) Specification conformance, integration tests that skip only what needs a key. Reviewed at 2ec7915f with no high finding open, and all 5 checks on that head passing. --- ci/run-integration-tests.Tests.ps1 | 183 ++++++++++++++++++ device-detection/ci/run-integration-tests.ps1 | 52 +++-- .../conftest.py | 76 ++++++++ .../readme.md | 4 +- .../tests/test_cloudfailuretomatch.py | 4 + .../test_clouduseragentclienthintsweb.py | 4 + .../fiftyone_devicedetection_examples/tox.ini | 3 + ip-intelligence/ci/run-integration-tests.ps1 | 20 +- .../conftest.py | 61 ++++++ .../tests/test_cloudgettingstartedweb.py | 4 + .../fiftyone_ipintelligence_examples/tox.ini | 3 + location/ci/run-integration-tests.ps1 | 16 +- location/tests/conftest.py | 76 ++++++++ location/tests/test_web_example.py | 3 + location/tox.ini | 3 + pipeline/ci/run-integration-tests.ps1 | 45 +++-- .../examples/readme.md | 8 +- .../examples/tests/conftest.py | 49 +++++ .../examples/tox.ini | 3 + .../tests/conftest.py | 53 +++++ .../tox.ini | 3 + 21 files changed, 634 insertions(+), 39 deletions(-) create mode 100644 ci/run-integration-tests.Tests.ps1 create mode 100644 device-detection/fiftyone_devicedetection_examples/conftest.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_examples/conftest.py create mode 100644 location/tests/conftest.py create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/conftest.py create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/tests/conftest.py diff --git a/ci/run-integration-tests.Tests.ps1 b/ci/run-integration-tests.Tests.ps1 new file mode 100644 index 00000000..22296cdf --- /dev/null +++ b/ci/run-integration-tests.Tests.ps1 @@ -0,0 +1,183 @@ +# Tests for the integration test step when no resource key is passed. +# +# Each product's ci/run-integration-tests.ps1 is run as common-ci runs it, +# through ci/dispatch.ps1 and common-ci's steps/run-script.ps1, from a folder +# laid out like a common-ci checkout. Its python/run-integration-tests.ps1 is +# a stand-in that runs no tests and records, for each call, the packages it +# was given and the PYTEST_ADDOPTS the tests would see. + +BeforeAll { + . (Join-Path $PSScriptRoot 'dispatch.ps1') + + # common-ci checked out beside this repository, or wherever COMMON_CI says. + $commonCi = $env:COMMON_CI + if ([string]::IsNullOrEmpty($commonCi) -eq $true) { + $commonCi = Join-Path $PSScriptRoot '..' '..' 'common-ci' + } + $script:Steps = Join-Path $commonCi 'steps' + if ((Test-Path (Join-Path $script:Steps 'run-script.ps1')) -eq $false) { + throw "common-ci not found at '$commonCi'; set COMMON_CI to a checkout of it" + } + + $script:Repo = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path + + # The products whose integration tests call the cloud service. + $script:Products = 'device-detection', 'ip-intelligence', 'location', + 'pipeline' + + # The environment variables the scripts set, kept to be put back. + $script:Saved = '_51DEGREES_RESOURCE_KEY', '_51DEGREES_LICENSE_KEY', + 'PYTEST_ADDOPTS', 'TEMP' + + # The stand-in for common-ci's python/run-integration-tests.ps1. It + # writes one line of JSON for each call to calls.jsonl in the folder it + # runs from. + $script:StandIn = @' +param ( + [Parameter(Mandatory=$true)][string]$RepoName, + [Parameter(Mandatory=$true)][string[]]$Packages, + [Parameter(Mandatory=$true)][hashtable]$Keys +) +@{ + Product = Split-Path $RepoName -Leaf + Packages = @($Packages) + AddOpts = "$env:PYTEST_ADDOPTS" +} | ConvertTo-Json -Compress | Add-Content (Join-Path $PWD 'calls.jsonl') +$global:LASTEXITCODE = 0 +'@ + + # A folder laid out like a common-ci checkout, which the step runs in. + function New-CommonCi { + $path = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) + $null = New-Item -ItemType Directory -Force (Join-Path $path 'python') + Copy-Item -Recurse $script:Steps (Join-Path $path 'steps') + Set-Content (Join-Path $path 'python' 'run-integration-tests.ps1') ` + $script:StandIn + return $path + } + + # The calls the stand-in recorded for a product. + function Get-Calls([string]$Product) { + $file = Join-Path $script:CommonCi 'calls.jsonl' + if ((Test-Path $file) -eq $false) { + return @() + } + return @(Get-Content $file | ForEach-Object { $_ | ConvertFrom-Json } | + Where-Object Product -eq $Product) + } +} + +Describe 'run-integration-tests.ps1' { + +BeforeEach { + $script:Environment = @{} + foreach ($name in $script:Saved) { + $script:Environment[$name] = [Environment]::GetEnvironmentVariable($name) + } + $script:CommonCi = New-CommonCi + Push-Location $script:CommonCi +} + +AfterEach { + Pop-Location + Remove-Item -Recurse -Force $script:CommonCi + foreach ($name in $script:Saved) { + [Environment]::SetEnvironmentVariable($name, $script:Environment[$name]) + } +} + +Context 'with no TestResourceKey' { + BeforeEach { + $env:PYTEST_ADDOPTS = $null + $script:Output = Invoke-ProductStep 'run-integration-tests' $script:Repo ` + @{ RepoName = $script:Repo; Keys = @{} } $script:Products *>&1 | + Out-String + } + + It 'runs the tests of every product' { + foreach ($product in $script:Products) { + (Get-Calls $product).Count | Should -BeGreaterThan 0 -Because $product + } + } + + It 'leaves out only the tests marked live, which need a resource key' { + foreach ($product in $script:Products) { + foreach ($call in (Get-Calls $product)) { + $call.AddOpts | Should -Be '-m "not live"' -Because $product + } + } + } + + It 'says for every product that the tests needing a key are skipped' { + foreach ($product in $script:Products) { + # What the step printed for this product, between its group + # markers. + $group = ($script:Output -split '::group::') | + Where-Object { $_ -like "run-integration-tests for $product*" } + $group | Should -Match ("No TestResourceKey was passed, so the " + + "tests that need a resource key are skipped") -Because $product + } + } + + It 'runs the tests of the device detection and IP intelligence examples' { + (Get-Calls 'device-detection').Packages | + Should -Contain 'fiftyone_devicedetection_examples' + (Get-Calls 'device-detection').Packages | + Should -Contain 'fiftyone_devicedetection_cloud' + (Get-Calls 'ip-intelligence').Packages | + Should -Contain 'fiftyone_ipintelligence_examples' + } + + It 'runs the cloud request engine and its examples, and skips the 51Did live tests' { + $packages = (Get-Calls 'pipeline').Packages + $packages | Should -Contain 'fiftyone_pipeline_cloudrequestengine' + $packages | Should -Contain 'fiftyone_pipeline_cloudrequestengine/examples' + $packages | Should -Not -Contain 'fiftyone_pipeline_did' + $script:Output | Should -Match 'fiftyone_pipeline_did.*skipped' + } + + It 'skips the Selenium contract against the cloud example and saying so' { + $script:Output | Should -Match 'Selenium contract tests will not run against the cloud example' + Test-Path (Join-Path $script:CommonCi 'selenium-api-tests') | + Should -BeFalse + } +} + +Context 'with no TestResourceKey and options a caller set' { + It 'adds the selection to the options, and puts them back after' { + $env:PYTEST_ADDOPTS = '-p no:cacheprovider' + Invoke-ProductStep 'run-integration-tests' $script:Repo ` + @{ RepoName = $script:Repo; Keys = @{} } $script:Products *>&1 | + Out-Null + + foreach ($product in $script:Products) { + $calls = Get-Calls $product + $calls.Count | Should -BeGreaterThan 0 -Because $product + foreach ($call in $calls) { + $call.AddOpts | Should -Be '-p no:cacheprovider -m "not live"' ` + -Because $product + } + } + $env:PYTEST_ADDOPTS | Should -Be '-p no:cacheprovider' + } +} + +Context 'with a TestResourceKey' { + It 'runs every test, the ones marked live included' { + Invoke-ProductStep 'run-integration-tests' $script:Repo @{ + RepoName = $script:Repo + Keys = @{ TestResourceKey = 'key' } + SeleniumContract = $false + } 'device-detection', 'location' *>&1 | Out-Null + + foreach ($product in 'device-detection', 'location') { + $calls = Get-Calls $product + $calls.Count | Should -BeGreaterThan 0 -Because $product + foreach ($call in $calls) { + $call.AddOpts | Should -Not -Match 'live' -Because $product + } + } + } +} + +} diff --git a/device-detection/ci/run-integration-tests.ps1 b/device-detection/ci/run-integration-tests.ps1 index ed7a17d2..32bff92c 100644 --- a/device-detection/ci/run-integration-tests.ps1 +++ b/device-detection/ci/run-integration-tests.ps1 @@ -16,10 +16,14 @@ if ($IsWindows) { $packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_examples" +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. A live test still fails when it finds +# no key, so a key that is passed but does not reach it is not missed. +# The selection is added to any options a caller set in PYTEST_ADDOPTS. +$addopts = $env:PYTEST_ADDOPTS if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = "$addopts -m `"not live`"".Trim() } # The cloud package runs without the CSV file too, because only its property # coverage tests read the file, and they say so when it is missing. @@ -44,7 +48,11 @@ $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey # files from _51DEGREES_LICENSE_KEY, which common-ci sets as license_key. $env:_51DEGREES_LICENSE_KEY = $Keys.DeviceDetection -./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +try { + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +} finally { + $env:PYTEST_ADDOPTS = $addopts +} $failures = @() if ($LASTEXITCODE -ne 0) { $failures += "the Python integration tests exited with code $LASTEXITCODE" @@ -58,6 +66,25 @@ if (-not $SeleniumContract) { return } +# The contract tests run against the cloud example when there is a resource +# key, and against the on-premise example when there is the TAC data file, +# which is only fetched when a licence is given. +$tac = "$PWD/assets/TAC-HashV41.hash" +$runCloud = [bool]$Keys.TestResourceKey +$runOnPremise = Test-Path $tac +if (-not $runCloud) { + Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No resource key::No TestResourceKey was passed, so the Selenium contract tests will not run against the cloud example." +} +if (-not $runOnPremise) { + Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No TAC Data File::The TAC data file wasn't found, so the Selenium tests will not run against the on-premise example." +} +if (-not ($runCloud -or $runOnPremise)) { + if ($failures.Count -gt 0) { + throw "Integration tests failed: $($failures -join '; ')" + } + return +} + # The shared Selenium contract tests are run against the cloud example and the # on-premise example in turn. Each result is also written as a trx file next # to the other integration results, so it is published with them. @@ -141,22 +168,21 @@ function Test-Example([string]$Name, [string]$Module, [int]$Port) { } # The cloud example, pointed at the live cloud. -$env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey -$env:_51DEGREES_CLOUD_ENDPOINT = "https://cloud.51degrees.com/api/v4/" -$failures += Test-Example -Name 'cloud' ` - -Module 'fiftyone_devicedetection_examples.cloud.gettingstarted_web' -Port 8097 +if ($runCloud) { + $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey + $env:_51DEGREES_CLOUD_ENDPOINT = "https://cloud.51degrees.com/api/v4/" + $failures += Test-Example -Name 'cloud' ` + -Module 'fiftyone_devicedetection_examples.cloud.gettingstarted_web' -Port 8097 +} # The on-premise example. The contract tests need the device type and the # JavaScript properties, which the Lite data file does not have, so it runs -# against the TAC data file, which is only fetched when a licence is given. -$tac = "$PWD/assets/TAC-HashV41.hash" -if (Test-Path $tac) { +# against the TAC data file. +if ($runOnPremise) { $env:_51DEGREES_DD_PATH = (Resolve-Path $tac).Path $failures += Test-Example -Name 'onpremise' ` -Module 'fiftyone_devicedetection_examples.onpremise.gettingstarted_web' -Port 8098 Remove-Item Env:_51DEGREES_DD_PATH -} else { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No TAC Data File::The TAC data file wasn't found, so the Selenium tests will not run against the on-premise example." } # A failure anywhere above must fail the job. This script is dot-sourced by diff --git a/device-detection/fiftyone_devicedetection_examples/conftest.py b/device-detection/fiftyone_devicedetection_examples/conftest.py new file mode 100644 index 00000000..4096653b --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/conftest.py @@ -0,0 +1,76 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The tests that call the live cloud service are marked live here, so the +# released test files stay as they are. ci/run-integration-tests.ps1 leaves +# the live tests out when no resource key is passed, and the other tests run. + +from pathlib import Path + +import pytest +from _pytest.mark.expression import Expression + +TESTS = Path(__file__).resolve().parent / "tests" + +# Every test in these modules is live. +LIVE_MODULES = { + TESTS / "test_cloudexamples.py", + TESTS / "test_cloudgettingstartedweb.py", +} + +BUILT_AT_IMPORT = {TESTS / "test_cloudgettingstartedweb.py"} + + +def _is_live(item): + return Path(item.path).resolve() in LIVE_MODULES + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(config, items): + # Runs before -m deselects, so the selection sees the marker. + for item in items: + if _is_live(item): + item.add_marker(pytest.mark.live) + + +def _leaves_out_live(config): + """Whether the -m expression leaves out a test marked live alone.""" + expression = config.getoption("markexpr") + return bool(expression) and not Expression.compile(expression).evaluate( + lambda name, **kwargs: name == "live") + + +# A module that builds a cloud Pipeline when imported raises with no resource +# key, so it is not collected when the live tests are left out. Every test in +# it is live, so the selection is the same. +def pytest_ignore_collect(collection_path, config): + if (Path(collection_path).resolve() in BUILT_AT_IMPORT and + _leaves_out_live(config)): + return True + return None diff --git a/device-detection/fiftyone_devicedetection_examples/readme.md b/device-detection/fiftyone_devicedetection_examples/readme.md index 59971991..e6214dcb 100644 --- a/device-detection/fiftyone_devicedetection_examples/readme.md +++ b/device-detection/fiftyone_devicedetection_examples/readme.md @@ -53,7 +53,9 @@ the Enterprise data file. The cloud tests need a valid 51Degrees resource key (see above), and fail without one. They read it from the environment variable -`_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`. +`_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`. They are +marked `live`, so `python -m pytest -m "not live"` runs every other test, +which is what CI does when no resource key is passed. For example, use following command to set the resource key on Linux: `export _51DEGREES_RESOURCE_KEY=MY-RESOURCE-KEY` diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py index 1ac2b995..4852ed0c 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py @@ -34,10 +34,14 @@ from contextlib import redirect_stdout from unittest import mock +import pytest + from fiftyone_devicedetection_examples.example_utils import ExampleUtils from .live_cloud import answered_without, find_resource_key, no_resource_key +pytestmark = pytest.mark.live + MODULE = "fiftyone_devicedetection_examples.cloud.failuretomatch" NO_KEY = no_resource_key("the device product", "the IsMobile property") diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py b/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py index afb5e23c..e00f9d97 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py @@ -29,6 +29,8 @@ import re import unittest +import pytest + from fiftyone_pipeline_core.logger import Logger from fiftyone_devicedetection_examples.cloud.useragentclienthints_web.app \ @@ -36,6 +38,8 @@ from .live_cloud import find_resource_key, no_resource_key +pytestmark = pytest.mark.live + NO_KEY = no_resource_key( "the device product", "device detection properties such as BrowserName") diff --git a/device-detection/fiftyone_devicedetection_examples/tox.ini b/device-detection/fiftyone_devicedetection_examples/tox.ini index 48032ef6..1a7c3f74 100644 --- a/device-detection/fiftyone_devicedetection_examples/tox.ini +++ b/device-detection/fiftyone_devicedetection_examples/tox.ini @@ -42,6 +42,9 @@ pass_env = # running the examples. GITHUB_ACTIONS GITHUB_REPOSITORY + # The marker selection of ../ci/run-integration-tests.ps1, which leaves + # the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] deps = diff --git a/ip-intelligence/ci/run-integration-tests.ps1 b/ip-intelligence/ci/run-integration-tests.ps1 index a64f611e..0d73e949 100644 --- a/ip-intelligence/ci/run-integration-tests.ps1 +++ b/ip-intelligence/ci/run-integration-tests.ps1 @@ -13,10 +13,14 @@ if ($IsWindows) { $packages = , "fiftyone_ipintelligence_examples" +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. A live test still fails when it finds +# no key, so a key that is passed but does not reach it is not missed. +# The selection is added to any options a caller set in PYTEST_ADDOPTS. +$addopts = $env:PYTEST_ADDOPTS if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = "$addopts -m `"not live`"".Trim() } # The tests read the resource key from _51DEGREES_RESOURCE_KEY, and common-ci @@ -28,7 +32,9 @@ $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey # only, never the key. Found through this script's own folder, because # common-ci runs it from the common-ci checkout, where # './ci/report-key-products.ps1' does not exist. -& (Join-Path $PSScriptRoot 'report-key-products.ps1') -ResourceKey $Keys.TestResourceKey +if ($Keys.TestResourceKey) { + & (Join-Path $PSScriptRoot 'report-key-products.ps1') -ResourceKey $Keys.TestResourceKey +} # nightly-publish-main workflow doesn't create the examples package, so # install-package.ps1 won't install it and its test dependencies won't get @@ -38,6 +44,10 @@ if ($env:GITHUB_JOB -eq "Test") { pip install $RepoName/fiftyone_ipintelligence_examples || $(throw "pip install failed") } -./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +try { + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/conftest.py b/ip-intelligence/fiftyone_ipintelligence_examples/conftest.py new file mode 100644 index 00000000..c235aa3b --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/conftest.py @@ -0,0 +1,61 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The tests that call the live cloud service are marked live here, so the +# released test files stay as they are. ci/run-integration-tests.ps1 leaves +# the live tests out when no resource key is passed, and the other tests run. + +from pathlib import Path + +import pytest + +TEST_EXAMPLES = ( + Path(__file__).resolve().parent / "tests" / "test_examples.py") + +# The tests in test_examples.py that need a resource key. +LIVE_TESTS = { + "test_cloud_gettingstarted_console", + "test_cloud_metadata_console", + "test_cloud_getallproperties_console", + "test_cloud_mixed_console", + "test_cloud_countries_console", +} + + +def _is_live(item): + return (Path(item.path).resolve() == TEST_EXAMPLES and + item.name in LIVE_TESTS) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(config, items): + # Runs before -m deselects, so the selection sees the marker. + for item in items: + if _is_live(item): + item.add_marker(pytest.mark.live) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py index 40a52678..d8a8f4e4 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py @@ -27,11 +27,15 @@ import unittest +import pytest + from fiftyone_ipintelligence_examples.cloud.gettingstarted_web \ import app as example from .live_cloud import find_resource_key, no_resource_key +pytestmark = pytest.mark.live + NO_KEY = no_resource_key( "the ip product", "IP Intelligence properties such as RegisteredName") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini index ed00ed06..f873d855 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini @@ -57,6 +57,9 @@ pass_env = # running the examples. GITHUB_ACTIONS GITHUB_REPOSITORY + # The marker selection of ../ci/run-integration-tests.ps1, which leaves + # the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] # The examples package itself must be installed, because its tests import diff --git a/location/ci/run-integration-tests.ps1 b/location/ci/run-integration-tests.ps1 index 01ae9b5c..b2025094 100644 --- a/location/ci/run-integration-tests.ps1 +++ b/location/ci/run-integration-tests.ps1 @@ -5,16 +5,24 @@ param ( [Hashtable]$Keys ) +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. A live test still fails when it finds +# no key, so a key that is passed but does not reach it is not missed. +# The selection is added to any options a caller set in PYTEST_ADDOPTS. +$addopts = $env:PYTEST_ADDOPTS if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = "$addopts -m `"not live`"".Trim() } # The examples and tests read the resource key from _51DEGREES_RESOURCE_KEY, # and common-ci sets only the older resource_key, so the key is set here. $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey -./python/run-integration-tests.ps1 -RepoName $RepoName -Packages . -Keys $Keys +try { + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages . -Keys $Keys +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/location/tests/conftest.py b/location/tests/conftest.py new file mode 100644 index 00000000..df687fc0 --- /dev/null +++ b/location/tests/conftest.py @@ -0,0 +1,76 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The tests that call the live cloud service are marked live here, so the +# released test files stay as they are. ci/run-integration-tests.ps1 leaves +# the live tests out when no resource key is passed, and the other tests run. + +from pathlib import Path + +import pytest +from _pytest.mark.expression import Expression + +HERE = Path(__file__).resolve().parent + +# Every test in these modules is live. test_web_example.py marks its own. +LIVE_MODULES = { + HERE / "test_examples.py", + HERE / "test_properties51D.py", +} + +BUILT_AT_IMPORT = {HERE / "test_properties51D.py"} + + +def _is_live(item): + return Path(item.path).resolve() in LIVE_MODULES + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(config, items): + # Runs before -m deselects, so the selection sees the marker. + for item in items: + if _is_live(item): + item.add_marker(pytest.mark.live) + + +def _leaves_out_live(config): + """Whether the -m expression leaves out a test marked live alone.""" + expression = config.getoption("markexpr") + return bool(expression) and not Expression.compile(expression).evaluate( + lambda name, **kwargs: name == "live") + + +# A module that builds a cloud Pipeline when imported raises with no resource +# key, so it is not collected when the live tests are left out. Every test in +# it is live, so the selection is the same. +def pytest_ignore_collect(collection_path, config): + if (Path(collection_path).resolve() in BUILT_AT_IMPORT and + _leaves_out_live(config)): + return True + return None diff --git a/location/tests/test_web_example.py b/location/tests/test_web_example.py index 1fafe50b..de01960f 100644 --- a/location/tests/test_web_example.py +++ b/location/tests/test_web_example.py @@ -37,6 +37,8 @@ from pathlib import Path from unittest import mock +import pytest + from .live_cloud import answered_without, find_resource_key, no_resource_key EXAMPLE = Path(__file__).resolve().parents[1] / "examples" / "cloud" / "web.py" @@ -108,6 +110,7 @@ def load_example(resource_key): return module +@pytest.mark.live class LocationWebExampleTests(unittest.TestCase): resource_key = find_resource_key() diff --git a/location/tox.ini b/location/tox.ini index dff16572..9b7bf58d 100644 --- a/location/tox.ini +++ b/location/tox.ini @@ -26,6 +26,9 @@ commands = pass_env = _51DEGREES_RESOURCE_KEY _51DEGREES_LICENSE_KEY + # The marker selection of ci/run-integration-tests.ps1, which leaves the + # live tests out when no resource key is passed. + PYTEST_ADDOPTS # Tests the packages just built, so it takes the test tools from [common] # and every package from the requirements file, never the editable copies diff --git a/pipeline/ci/run-integration-tests.ps1 b/pipeline/ci/run-integration-tests.ps1 index a4845edd..c34ea46a 100644 --- a/pipeline/ci/run-integration-tests.ps1 +++ b/pipeline/ci/run-integration-tests.ps1 @@ -5,12 +5,6 @@ param ( [Hashtable]$Keys ) -if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." -} - # The examples call the cloud service, so they are integration tests rather # than unit tests. Listing them here runs them on every Python version and # operating system in options.json, and in the nightly, which a standalone @@ -26,23 +20,48 @@ if (!$Keys.TestResourceKey) { # The 51Did package runs only its tests marked live here, as its others ran # in the unit step. Its tier tests need a key for another tier, which only # the Cloud 51Did workflow has. Only a package whose tox.ini passes -# PYTEST_ADDOPTS sees the selection. Its build needs the OWID source, which -# build-project.ps1 copies, and the publish workflow's test job does not run -# that script, so it is copied here too. -& "$PSScriptRoot/copy-owid-source.ps1" -RepoRoot $RepoName -$packages = "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_cloudrequestengine/examples", "fiftyone_pipeline_did" +# PYTEST_ADDOPTS sees the selection, so the 51Did package is run on its own, +# after the others. Its build needs the OWID source, which build-project.ps1 +# copies, and the publish workflow's test job does not run that script, so +# it is copied here too. +# +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. That leaves nothing of the 51Did +# package, so it is not run. A live test still fails when it finds no key, so +# a key that is passed but does not reach it is not missed. The selection is +# added to any options a caller set in PYTEST_ADDOPTS. +$packages = "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_cloudrequestengine/examples" $pipelineKeys = $Keys.Clone() $pipelineKeys.Remove('DeviceDetection') # The tests read the resource key from _51DEGREES_RESOURCE_KEY, and common-ci # sets only the older resource_key, so the key is set here. $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey +$failed = $false $addopts = $env:PYTEST_ADDOPTS -$env:PYTEST_ADDOPTS = '-m "live and not tier"' try { + if (!$Keys.TestResourceKey) { + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. fiftyone_pipeline_did runs only live tests here, so it is skipped. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = "$addopts -m `"not live`"".Trim() + } ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $pipelineKeys + if ($LASTEXITCODE -ne 0) { $failed = $true } + + if ($Keys.TestResourceKey) { + # common-ci's script combines the coverage of the packages it ran + # into .coverage, and a second run replaces that file, so the first + # run's is kept under a name the second run's combine reads. + $coverage = Join-Path $RepoName '.coverage' + if (Test-Path $coverage) { + Move-Item -Force $coverage (Join-Path $RepoName '.coverage.cloudrequestengine') + } + & "$PSScriptRoot/copy-owid-source.ps1" -RepoRoot $RepoName + $env:PYTEST_ADDOPTS = '-m "live and not tier"' + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages "fiftyone_pipeline_did" -Keys $pipelineKeys + if ($LASTEXITCODE -ne 0) { $failed = $true } + } } finally { $env:PYTEST_ADDOPTS = $addopts } -exit $LASTEXITCODE +exit ($failed ? 1 : 0) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md index 824391d0..c4b0df5a 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md @@ -99,14 +99,16 @@ then open . Set `PORT` to listen on another port. The tests that call the cloud service with a resource key fail when none is set, naming the variables they read, which are `_51DEGREES_RESOURCE_KEY` -and `_51DEGREES_RESOURCE_KEY_BESPOKE`, in that order. The +and `_51DEGREES_RESOURCE_KEY_BESPOKE`, in that order. They are marked +`live`, so `python -m pytest -m "not live"` runs every other test. The tests that use a licence key skip themselves when none is set. In CI the examples are run by [ci/run-integration-tests.ps1](../../ci/run-integration-tests.ps1) through common-ci, on every Python version and operating system in [ci/options.json](../../ci/options.json), with the resource key the pipeline -holds. No licence key is passed to the pipeline, so the licence-key tests -skip themselves there. That uses the [tox.ini](tox.ini) in this folder, +holds. When no resource key is passed, the script leaves out the tests +marked `live`, says so, and runs the rest. No licence key is passed to the +pipeline, so the licence-key tests skip themselves there. That uses the [tox.ini](tox.ini) in this folder, which can also be run by hand with `python -m tox -e py` from here. ## Cloud engine diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/conftest.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/conftest.py new file mode 100644 index 00000000..16bcd169 --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/conftest.py @@ -0,0 +1,49 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The tests that call the live cloud service are marked live here, so the +# released test files stay as they are. ci/run-integration-tests.ps1 leaves +# the live tests out when no resource key is passed, and the other tests run. + +import pytest + + +# The tests given the "resource key" case of helpers.KEYS. The "licence key" +# case skips itself when no licence key is set. +def _is_live(item): + callspec = getattr(item, "callspec", None) + return callspec is not None and "resource key" in callspec.id.split("-") + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(config, items): + # Runs before -m deselects, so the selection sees the marker. + for item in items: + if _is_live(item): + item.add_marker(pytest.mark.live) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini index fbd5e095..3ab7052a 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini @@ -41,6 +41,9 @@ pass_env = _51DEGREES_LICENSE_KEY _51DEGREES_CLOUD_ENDPOINT _51DEGREES_CLOUD_REQUEST_ORIGIN + # The marker selection of ../../ci/run-integration-tests.ps1, which + # leaves the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] deps = diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/conftest.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/conftest.py new file mode 100644 index 00000000..0ecb4c70 --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/conftest.py @@ -0,0 +1,53 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The tests that call the live cloud service are marked live here, so the +# released test files stay as they are. ci/run-integration-tests.ps1 leaves +# the live tests out when no resource key is passed, and the other tests run. + +from pathlib import Path + +import pytest + +TEST_CLOUDENGINE = Path(__file__).resolve().parent / "test_cloudengine.py" + + +# CloudEngineTests needs a resource key for each of its tests, in setUp. +def _is_live(item): + return (Path(item.path).resolve() == TEST_CLOUDENGINE and + getattr(item, "cls", None) is not None and + item.cls.__name__ == "CloudEngineTests") + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(config, items): + # Runs before -m deselects, so the selection sees the marker. + for item in items: + if _is_live(item): + item.add_marker(pytest.mark.live) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini index 646020a0..35bf9b29 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini @@ -21,6 +21,9 @@ commands = pass_env = _51DEGREES_RESOURCE_KEY _51DEGREES_LICENSE_KEY + # The marker selection of ../ci/run-integration-tests.ps1, which leaves + # the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] package = skip From 0dd02ca408d2201ed83b2ecf773aedf4eccd9778 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 21 Nov 2026 16:43:33 +0000 Subject: [PATCH 131/136] Results instances are not thread-safe, so reads take no lock (#241) Results instances are not thread-safe, so reads take no lock. Reviewed at def9267a with no high finding open, and all 3 checks on that head passing. --- .../swig_data.py | 32 ++--- .../tests/test_swig_data_threads.py | 114 ++++------------ .../swig_data.py | 31 +++-- .../tests/test_swig_data_threads.py | 125 ++++-------------- .../src/fiftyone_pipeline_core/elementdata.py | 20 ++- .../tests/test_elementdata_documentation.py | 30 +++-- 6 files changed, 120 insertions(+), 232 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index 7d7891fa..9982124b 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -20,8 +20,6 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -import threading - from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue @@ -55,6 +53,17 @@ class SwigData(AspectData): """! Extention of AspectData made for retrieving results created by the SWIG wrapper of the On Premise Device Detection engine + + An instance reads its values from a results instance, the object the + native code fills for one request, which references unmanaged memory. + Multi-threaded access to a results instance is not supported, for + performance reasons. Reading a value from it writes into it, so it is not + safe to use from several threads at once, and neither is this instance. + The engine, its results instances and this instance take no lock to make + those reads safe for several threads. The caller is responsible for + thread-safe access, and either gives each thread its own Flow Data, and + so its own results instance, or coordinates its own access, for example + with a lock of its own around every read and around close(). """ def __init__(self, flow_element, swig_results): @@ -73,11 +82,6 @@ def __init__(self, flow_element, swig_results): self.missing_property_service = OnPremiseMissingPropertyService() - # Reading a value writes into the native results, and the first read - # of a component can walk its graph, so one thread at a time reads - # this instance. - self._read_lock = threading.RLock() - def close(self): """! @@ -87,8 +91,7 @@ def close(self): exception. Closing the FlowData closes its results. """ - with self._read_lock: - self.swig_results = None + self.swig_results = None def get(self, key): @@ -99,13 +102,12 @@ def get(self, key): @returns mixed """ - with self._read_lock: - # Checked here, because AspectData.get turns an exception from - # get_internal into a missing property message. - if self.swig_results is None: - raise Exception(RESULTS_CLOSED) + # Checked here, because AspectData.get turns an exception from + # get_internal into a missing property message. + if self.swig_results is None: + raise Exception(RESULTS_CLOSED) - return super(SwigData, self).get(key) + return super(SwigData, self).get(key) def get_internal(self, key): diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py index e07f169d..538cc0b7 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_threads.py @@ -21,59 +21,21 @@ # ********************************************************************* """ -One thread at a time reads a native results instance, because reading a value -writes into the results. +A results instance is not safe to use from several threads at once and the +caller is responsible for thread-safe access, so SwigData takes no lock and +its documentation says so. """ import threading -import time import unittest from fiftyone_devicedetection_onpremise.swig_data import SwigData -# How long a stand-in read takes, long enough that overlapping reads are seen. -READ_SECONDS = 0.02 -# The longest a test waits for another thread. -WAIT_SECONDS = 10 -# How long closing is given while a read holds the results. It must still be -# waiting at the end. -BLOCKED_SECONDS = 0.2 - -KEY = "deviceid" -VALUE = "1-2-3" - - -class OverlapCountingResults: - """Stands in for the native results and records how many reads overlap.""" - - def __init__(self): - self.guard = threading.Lock() - self.active = 0 - self.most_active = 0 - - def getDeviceId(self): - with self.guard: - self.active += 1 - self.most_active = max(self.most_active, self.active) - time.sleep(READ_SECONDS) - with self.guard: - self.active -= 1 - return "1-2-3" - - -class BlockingResults: - """Stands in for the native results with a read that waits to be let go, - so a test can act while the read is in progress.""" - - def __init__(self): - self.reading = threading.Event() - self.release = threading.Event() - - def getDeviceId(self): - self.reading.set() - self.release.wait(WAIT_SECONDS) - return "1-2-3" +# The types of the locks the threading module makes. +LOCK_TYPES = ( + type(threading.Lock()), type(threading.RLock()), + threading.Condition, threading.Semaphore) class FakeElement: @@ -83,50 +45,24 @@ class FakeElement: class SwigDataThreadTests(unittest.TestCase): - def test_reads_from_several_threads_do_not_overlap(self): - results = OverlapCountingResults() - data = SwigData(FakeElement(), results) - errors = [] - - def read(): - try: - for _ in range(3): - data.get("deviceid") - except Exception as e: - errors.append(e) - - threads = [threading.Thread(target=read) for _ in range(6)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - - self.assertEqual([], errors) - self.assertEqual(1, results.most_active) - - def test_closing_waits_for_a_read_in_progress(self): - results = BlockingResults() - data = SwigData(FakeElement(), results) - values = [] - reader = threading.Thread( - target=lambda: values.append(data.get(KEY))) - reader.start() - self.assertTrue(results.reading.wait(WAIT_SECONDS)) - - closer = threading.Thread(target=data.close) - closer.start() - closer.join(BLOCKED_SECONDS) - # The read holds the results, so closing waits for it. - self.assertTrue(closer.is_alive()) - - results.release.set() - reader.join(WAIT_SECONDS) - closer.join(WAIT_SECONDS) - self.assertFalse(closer.is_alive()) - self.assertEqual([VALUE], [value.value() for value in values]) - self.assertIsNone(data.swig_results) - with self.assertRaises(Exception): - data.get(KEY) + def test_the_documentation_says_the_caller_coordinates_threads(self): + text = " ".join(SwigData.__doc__.split()) + self.assertIn("unmanaged memory", text) + self.assertIn( + "Multi-threaded access to a results instance is not supported, " + "for performance reasons", text) + self.assertIn("not safe to use from several threads at once", text) + self.assertIn("take no lock", text) + self.assertIn("caller is responsible for thread-safe access", text) + + def test_an_instance_holds_no_lock(self): + """A smoke test. It looks only at the instance's own attributes, + so a lock at module level, on the engine or in AspectData would + still pass.""" + data = SwigData(FakeElement(), object()) + locks = [name for name, value in vars(data).items() + if isinstance(value, LOCK_TYPES)] + self.assertEqual([], locks) if __name__ == "__main__": diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 85fa7094..f26cd27f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -21,7 +21,6 @@ # ********************************************************************* import math -import threading from decimal import Decimal, ROUND_HALF_UP from fiftyone_pipeline_engines.aspectdata import AspectData @@ -65,6 +64,17 @@ class SwigData(AspectData): """! Extension of AspectData made for retrieving results created by the SWIG wrapper of the On Premise IP Intelligence engine + + An instance reads its values from a results instance, the object the + native code fills for one request, which references unmanaged memory. + Multi-threaded access to a results instance is not supported, for + performance reasons. Reading a value from it writes into it, so it is not + safe to use from several threads at once, and neither is this instance. + The engine, its results instances and this instance take no lock to make + those reads safe for several threads. The caller is responsible for + thread-safe access, and either gives each thread its own Flow Data, and + so its own results instance, or coordinates its own access, for example + with a lock of its own around every read and around close(). """ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): @@ -89,11 +99,6 @@ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): self.missing_property_service = OnPremiseMissingPropertyService() - # Reading a value writes into the native results, and the first read - # of a component can walk its graph, so one thread at a time reads - # this instance. - self._read_lock = threading.RLock() - def close(self): """! @@ -103,8 +108,7 @@ def close(self): exception. Closing the FlowData closes its results. """ - with self._read_lock: - self.swig_results = None + self.swig_results = None def get(self, key): @@ -115,13 +119,12 @@ def get(self, key): @returns mixed """ - with self._read_lock: - # Checked here, because AspectData.get turns an exception from - # get_internal into a missing property message. - if self.swig_results is None: - raise Exception(RESULTS_CLOSED) + # Checked here, because AspectData.get turns an exception from + # get_internal into a missing property message. + if self.swig_results is None: + raise Exception(RESULTS_CLOSED) - return super(SwigData, self).get(key) + return super(SwigData, self).get(key) @staticmethod def utf8_string_to_str(utf8_string): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py index c1476c43..41313401 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_threads.py @@ -21,121 +21,48 @@ # ********************************************************************* """ -One thread at a time reads a native results instance, because reading a value -writes into the results. +A results instance is not safe to use from several threads at once and the +caller is responsible for thread-safe access, so SwigData takes no lock and +its documentation says so. """ import threading -import time import unittest from fiftyone_ipintelligence_onpremise.swig_data import SwigData -class Value: - - def hasValue(self): - return True - - def getValue(self): - return 5 - - -# How long a stand-in read takes, long enough that overlapping reads are seen. -READ_SECONDS = 0.02 -# The longest a test waits for another thread. -WAIT_SECONDS = 10 -# How long closing is given while a read holds the results. It must still be -# waiting at the end. -BLOCKED_SECONDS = 0.2 - -KEY = "count" -VALUE = 5 - - -class OverlapCountingResults: - """Stands in for the native results and records how many reads overlap.""" - - def __init__(self): - self.guard = threading.Lock() - self.active = 0 - self.most_active = 0 - - def getValueAsInteger(self, name): - with self.guard: - self.active += 1 - self.most_active = max(self.most_active, self.active) - time.sleep(READ_SECONDS) - with self.guard: - self.active -= 1 - return Value() - - -class BlockingResults: - """Stands in for the native results with a read that waits to be let go, - so a test can act while the read is in progress.""" - - def __init__(self): - self.reading = threading.Event() - self.release = threading.Event() - - def getValueAsInteger(self, name): - self.reading.set() - self.release.wait(WAIT_SECONDS) - return Value() +# The types of the locks the threading module makes. +LOCK_TYPES = ( + type(threading.Lock()), type(threading.RLock()), + threading.Condition, threading.Semaphore) class FakeElement: datakey = "ip" - properties = {"count": {"name": "Count", "type": "int"}} + properties = {} class SwigDataThreadTests(unittest.TestCase): - def test_reads_from_several_threads_do_not_overlap(self): - results = OverlapCountingResults() - data = SwigData(FakeElement(), results) - errors = [] - - def read(): - try: - for _ in range(3): - data.get("count") - except Exception as e: - errors.append(e) - - threads = [threading.Thread(target=read) for _ in range(6)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - - self.assertEqual([], errors) - self.assertEqual(1, results.most_active) - - def test_closing_waits_for_a_read_in_progress(self): - results = BlockingResults() - data = SwigData(FakeElement(), results) - values = [] - reader = threading.Thread( - target=lambda: values.append(data.get(KEY))) - reader.start() - self.assertTrue(results.reading.wait(WAIT_SECONDS)) - - closer = threading.Thread(target=data.close) - closer.start() - closer.join(BLOCKED_SECONDS) - # The read holds the results, so closing waits for it. - self.assertTrue(closer.is_alive()) - - results.release.set() - reader.join(WAIT_SECONDS) - closer.join(WAIT_SECONDS) - self.assertFalse(closer.is_alive()) - self.assertEqual([VALUE], [value.value() for value in values]) - self.assertIsNone(data.swig_results) - with self.assertRaises(Exception): - data.get(KEY) + def test_the_documentation_says_the_caller_coordinates_threads(self): + text = " ".join(SwigData.__doc__.split()) + self.assertIn("unmanaged memory", text) + self.assertIn( + "Multi-threaded access to a results instance is not supported, " + "for performance reasons", text) + self.assertIn("not safe to use from several threads at once", text) + self.assertIn("take no lock", text) + self.assertIn("caller is responsible for thread-safe access", text) + + def test_an_instance_holds_no_lock(self): + """A smoke test. It looks only at the instance's own attributes, + so a lock at module level, on the engine or in AspectData would + still pass.""" + data = SwigData(FakeElement(), object()) + locks = [name for name, value in vars(data).items() + if isinstance(value, LOCK_TYPES)] + self.assertEqual([], locks) if __name__ == "__main__": diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py index 5ee546d3..008bc22b 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py @@ -28,12 +28,20 @@ class ElementData(object): Stores information created by a FlowElement based on FlowData. Stored in FlowData - Only the Flow Element that creates an instance writes it, so an instance - does not need to be safe to write from several threads. A finished - instance, once the Flow Element that writes it has finished, is safe to - read from several threads at once, such as the caller's own threads. An - instance that is written from several threads, or read while another - thread writes it, is not catered for. + An instance is written only by the Flow Element that provides it, or by + the Flow Elements that share it, and only while they process. Once they + have finished, it is not written again, apart from lazy properties, so it + is safe to read from several threads at once, as its values are Python's + own managed data. The Flow Data that holds it is still used by one thread + at a time. + + The exception is Element Data that reads its values from a results + instance, such as an on-premise engine's, which references unmanaged + memory. Multi-threaded access to a results instance is not supported, for + performance reasons, so such Element Data is not safe to use from several + threads at once. It takes no lock to make those reads safe. The caller is + responsible for thread-safe access, and either gives each thread its own + Flow Data or coordinates its own access. """ diff --git a/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py b/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py index 58d88e28..96ccbfb5 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py @@ -1,9 +1,11 @@ """ -The documentation of Element Data says it is safe to read from several threads -once the Flow Element that writes it has finished, and is not catered for when -written from several threads or read while another thread writes, as the -thread safety page of the specification requires. +The documentation of Element Data says it is written only by the Flow Element +that provides it, or those that share it, and only while they process, and +that it is then safe to read from several threads. It also says that Element +Data reading from a results instance, which references unmanaged memory, is +not safe to use from several threads at once, takes no lock and leaves the +caller responsible, as the thread safety page of the specification requires. """ import unittest @@ -13,13 +15,23 @@ class ElementDataDocumentationTests(unittest.TestCase): - def test_the_documentation_says_what_is_safe_with_threads(self): + def test_the_documentation_says_who_writes_and_when(self): text = " ".join(ElementData.__doc__.split()) + self.assertIn( + "written only by the Flow Element that provides it, or by the " + "Flow Elements that share it, and only while they process", text) self.assertIn("safe to read from several threads", text) - self.assertIn("once the Flow Element that writes it has finished", text) - self.assertIn("not catered for", text) - self.assertIn("written from several threads", text) - self.assertIn("read while another thread writes", text) + self.assertIn("used by one thread at a time", text) + + def test_the_documentation_says_results_instances_are_the_exception(self): + text = " ".join(ElementData.__doc__.split()) + self.assertIn("results instance", text) + self.assertIn("unmanaged memory", text) + self.assertIn( + "Multi-threaded access to a results instance is not supported, " + "for performance reasons", text) + self.assertIn("takes no lock", text) + self.assertIn("caller is responsible for thread-safe access", text) if __name__ == "__main__": From 1af685bac112336ce32c4dbfae34522505536c52 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 21 Nov 2026 16:54:14 +0000 Subject: [PATCH 132/136] Refuse writes to a finished Flow Data and to finished Element Data (#247) Refuse writes to a finished Flow Data and to finished Element Data. Reviewed at 2bc91de9 with no high finding open, and all 3 checks on that head passing. --- .../hardwareprofile_cloud.py | 21 +- .../tests/test_finished_writes.py | 140 ++++++ .../tests/test_finished_writes.py | 101 +++++ .../tests/test_finished_writes.py | 113 +++++ .../tests/test_finished_writes.py | 101 +++++ .../tests/test_finished_writes.py | 99 +++++ location/tests/test_finished_writes.py | 108 +++++ .../tests/test_finished_writes.py | 147 +++++++ .../src/fiftyone_pipeline_core/elementdata.py | 185 ++++++++ .../elementdata_dictionary.py | 5 +- .../src/fiftyone_pipeline_core/evidence.py | 37 +- .../src/fiftyone_pipeline_core/flowdata.py | 117 +++-- .../src/fiftyone_pipeline_core/messages.py | 15 +- .../tests/test_finished_writes.py | 416 ++++++++++++++++++ .../tests/test_setheader.py | 8 +- .../derived_property_data.py | 8 +- .../tests/test_finished_writes.py | 105 +++++ .../aspectdata_dictionary.py | 5 + .../tests/test_finished_writes.py | 97 ++++ .../tests/test_finished_writes.py | 80 ++++ .../tests/test_finished_writes.py | 99 +++++ 21 files changed, 1972 insertions(+), 35 deletions(-) create mode 100644 device-detection/fiftyone_devicedetection_cloud/tests/test_finished_writes.py create mode 100644 device-detection/fiftyone_devicedetection_onpremise/tests/test_finished_writes.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_cloud/tests/test_finished_writes.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_finished_writes.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_translation/tests/test_finished_writes.py create mode 100644 location/tests/test_finished_writes.py create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_finished_writes.py create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py create mode 100644 pipeline/fiftyone_pipeline_derivedproperty/tests/test_finished_writes.py create mode 100644 pipeline/fiftyone_pipeline_engines/tests/test_finished_writes.py create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_finished_writes.py create mode 100644 pipeline/fiftyone_pipeline_translation/tests/test_finished_writes.py diff --git a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py index b6e6fe5e..0344f1d2 100644 --- a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py @@ -28,6 +28,7 @@ from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue +from fiftyone_pipeline_core.elementdata import _CheckedDictWrites import json @@ -60,7 +61,7 @@ def _cloud_response(flowdata): return data.get("cloud") -class HardwareProfileData(dict, AspectData): +class HardwareProfileData(_CheckedDictWrites, dict, AspectData): """! One hardware profile from a profile lookup, read in the way the device data is read. It is Aspect Data, so a property is read as an attribute, @@ -72,6 +73,10 @@ class HardwareProfileData(dict, AspectData): Attribute access, get(), [] and in compare names ignoring letter case. A property whose name is also a dict method, such as keys, is read with get() or []. + + A profile is part of the hardware data, and is written only while the + engine processes. Once it has finished, setting an attribute or writing + through a dict method raises an exception. """ # Attributes set in the constructor, which are never read as properties. @@ -93,6 +98,18 @@ def __init__(self, flow_element, contents): # reported in the same way for both. self.missing_property_service = OnPremiseMissingPropertyService() + @property + def _finished(self): + # Finished when the hardware data that holds the profile is, as + # FlowData records that on the data it holds, not on each profile. + holder = self.__dict__.get("_holder") + return (self.__dict__.get("_own_finished", False) + or (holder is not None and holder._finished)) + + @_finished.setter + def _finished(self, finished): + self.__dict__["_own_finished"] = finished + def _matching_key(self, key): """! The key held for a property name given in another letter case, or @@ -247,6 +264,8 @@ def process_internal(self, flowdata): contents["profiles"] = devices data = AspectDataDictionary(self, contents) + for device in devices: + device._holder = data flowdata.set_element_data(data) \ No newline at end of file diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_finished_writes.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_finished_writes.py new file mode 100644 index 00000000..4d0c9e49 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_finished_writes.py @@ -0,0 +1,140 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. +""" + +import importlib +import inspect +import json +import pkgutil +import unittest + +import fiftyone_devicedetection_cloud +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_devicedetection_cloud.devicedetection_cloud import \ + DeviceDetectionCloud +from fiftyone_devicedetection_cloud.hardwareprofile_cloud import \ + HardwareProfileCloud + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +# FlowElements that only other FlowElements build on. +BASES = set() + + +class StandInRequestEngine(FlowElement): + """Stands in for a Cloud Request Engine whose answer has an empty + section for each of the given data keys.""" + + def __init__(self, keys): + super(StandInRequestEngine, self).__init__() + self.datakey = "cloud" + self.flow_element_properties = {key: {} for key in keys} + self.answer = json.dumps({key: {} for key in keys}) + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"cloud": self.answer})) + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + processed = set() + for element in [DeviceDetectionCloud(), HardwareProfileCloud()]: + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + flowdata = builder \ + .add(StandInRequestEngine([element.datakey])) \ + .add(element).build().create_flowdata() + flowdata.process() + data = flowdata.get(element.datakey) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + processed.add(type(element)) + self.assertEqual( + defined_flow_elements(fiftyone_devicedetection_cloud) - BASES, processed) + + def test_a_hardware_profile_refuses_writes_once_finished(self): + writes = ( + ("attribute", lambda p: setattr(p, "written", True)), + ("[]=", lambda p: p.__setitem__("written", True)), + ("del", lambda p: p.__delitem__("hardwarevendor")), + ("update", lambda p: p.update(written=True)), + ("pop", lambda p: p.pop("hardwarevendor")), + ("popitem", lambda p: p.popitem()), + ("setdefault", lambda p: p.setdefault("written", True)), + ("clear", lambda p: p.clear()), + ("|=", lambda p: p.__ior__({"written": True})), + ) + element = HardwareProfileCloud() + request = StandInRequestEngine([element.datakey]) + request.answer = json.dumps({element.datakey: { + "profiles": [{"hardwarevendor": "Apple"}]}}) + flowdata = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(request).add(element).build().create_flowdata() + flowdata.process() + profile = flowdata.get(element.datakey).get("profiles")[0] + for name, write in writes: + with self.subTest(name): + with self.assertRaises(Exception) as caught: + write(profile) + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual(["hardwarevendor"], list(profile)) + self.assertEqual("Apple", profile.hardwarevendor.value()) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_finished_writes.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_finished_writes.py new file mode 100644 index 00000000..3bb1cb5a --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_finished_writes.py @@ -0,0 +1,101 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. Reading values from the results is not a write. +""" + +import importlib +import inspect +import pkgutil +import unittest + +import fiftyone_devicedetection_onpremise +from fiftyone_devicedetection_onpremise.constants import RESULTS_CLOSED +from fiftyone_devicedetection_onpremise.devicedetection_onpremise import \ + DeviceDetectionOnPremise +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +DATA_FILE = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" +USER_AGENT = ("Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) " + "AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148") +FINISHED = "finished processing" + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + engine = DeviceDetectionOnPremise( + data_file_path=DATA_FILE, licence_keys="", auto_update=False) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("header.user-agent", USER_AGENT) + flowdata.process() + data = flowdata.get("device") + self.assertTrue(data.ismobile.value()) + with self.assertRaises(Exception) as caught: + data.ismobile = False + self.assertIn(FINISHED, str(caught.exception)) + self.assertTrue(data.ismobile.value()) + self.assertEqual( + {DeviceDetectionOnPremise}, + defined_flow_elements(fiftyone_devicedetection_onpremise)) + + def test_the_data_can_be_closed_once_finished(self): + # Closing releases the native results, which is not a write, so it + # works on finished data, as closing the FlowData does. + engine = DeviceDetectionOnPremise( + data_file_path=DATA_FILE, licence_keys="", auto_update=False) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("header.user-agent", USER_AGENT) + flowdata.process() + data = flowdata.get("device") + data.close() + with self.assertRaises(Exception) as caught: + data.get("ismobile") + self.assertEqual(RESULTS_CLOSED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + data.ismobile = False + self.assertIn(FINISHED, str(caught.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/tests/test_finished_writes.py b/ip-intelligence/fiftyone_ipintelligence_cloud/tests/test_finished_writes.py new file mode 100644 index 00000000..4ab707da --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/tests/test_finished_writes.py @@ -0,0 +1,113 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. +""" + +import importlib +import inspect +import json +import pkgutil +import unittest + +import fiftyone_ipintelligence_cloud +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_ipintelligence_cloud.countries_translation_cloud import \ + CountriesTranslationCloud +from fiftyone_ipintelligence_cloud.ipintelligence_cloud import \ + IpIntelligenceCloud +from fiftyone_ipintelligence_cloud.weighted_cloud_engine import \ + WeightedCloudEngine + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +# FlowElements that only other FlowElements build on. +BASES = {WeightedCloudEngine} + + +class StandInRequestEngine(FlowElement): + """Stands in for a Cloud Request Engine whose answer has an empty + section for each of the given data keys.""" + + def __init__(self, keys): + super(StandInRequestEngine, self).__init__() + self.datakey = "cloud" + self.flow_element_properties = {key: {} for key in keys} + self.answer = json.dumps({key: {} for key in keys}) + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"cloud": self.answer})) + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + processed = set() + for element in [IpIntelligenceCloud(), CountriesTranslationCloud()]: + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + flowdata = builder \ + .add(StandInRequestEngine([element.datakey])) \ + .add(element).build().create_flowdata() + flowdata.process() + data = flowdata.get(element.datakey) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + processed.add(type(element)) + self.assertEqual( + defined_flow_elements(fiftyone_ipintelligence_cloud) - BASES, processed) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_finished_writes.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_finished_writes.py new file mode 100644 index 00000000..4474ad83 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_finished_writes.py @@ -0,0 +1,101 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. Reading values from the results is not a write. +""" + +import importlib +import inspect +import pkgutil +import unittest + +import fiftyone_ipintelligence_onpremise +from fiftyone_ipintelligence_onpremise.constants import RESULTS_CLOSED +from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise import \ + IpIntelligenceOnPremise +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +from .constants import TestConstants + +DATA_FILE = TestConstants.get_data_file_path() +FINISHED = "finished processing" + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + engine = IpIntelligenceOnPremise( + data_file_path=DATA_FILE, licence_keys="", auto_update=False) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("query.client-ip", "185.28.167.77") + flowdata.process() + data = flowdata.get("ip") + name = data.get("registeredname") + with self.assertRaises(Exception) as caught: + data.registeredname = "written after processing" + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual(name.value(), data.get("registeredname").value()) + self.assertEqual( + {IpIntelligenceOnPremise}, + defined_flow_elements(fiftyone_ipintelligence_onpremise)) + + def test_the_data_can_be_closed_once_finished(self): + # Closing releases the native results, which is not a write, so it + # works on finished data, as closing the FlowData does. + engine = IpIntelligenceOnPremise( + data_file_path=DATA_FILE, licence_keys="", auto_update=False) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("query.client-ip", "185.28.167.77") + flowdata.process() + data = flowdata.get("ip") + data.close() + with self.assertRaises(Exception) as caught: + data.get("registeredname") + self.assertEqual(RESULTS_CLOSED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + data.registeredname = "written after processing" + self.assertIn(FINISHED, str(caught.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_finished_writes.py b/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_finished_writes.py new file mode 100644 index 00000000..26e75e85 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_finished_writes.py @@ -0,0 +1,99 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. +""" + +import importlib +import inspect +import pkgutil +import unittest + +import fiftyone_ipintelligence_translation +from fiftyone_ipintelligence_translation.countries_translation_engine import \ + CountriesTranslationEngine +from fiftyone_ipintelligence_translation.countries_translation_engine_builder \ + import CountriesTranslationEngineBuilder +from fiftyone_ipintelligence_translation.country_code_translation_engine \ + import CountryCodeTranslationEngine +from fiftyone_ipintelligence_translation \ + .country_code_translation_engine_builder import \ + CountryCodeTranslationEngineBuilder +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +from .mock_ip_engine import mock_ip_element + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + engines = [CountryCodeTranslationEngineBuilder().build(), + CountriesTranslationEngineBuilder().build()] + builder = PipelineBuilder().add(mock_ip_element(["DE"], ["DE"])) + for engine in engines: + builder.add(engine) + flowdata = builder.build().create_flowdata() + flowdata.process() + for engine in engines: + data = flowdata.get(engine.datakey) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual( + {type(engine) for engine in engines}, + defined_flow_elements(fiftyone_ipintelligence_translation)) + self.assertEqual( + {CountriesTranslationEngine, CountryCodeTranslationEngine}, + {type(engine) for engine in engines}) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/location/tests/test_finished_writes.py b/location/tests/test_finished_writes.py new file mode 100644 index 00000000..e4f5111e --- /dev/null +++ b/location/tests/test_finished_writes.py @@ -0,0 +1,108 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. +""" + +import importlib +import inspect +import json +import pkgutil +import unittest + +import fiftyone_location +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_location.location_cloud import LocationCloud + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +# FlowElements that only other FlowElements build on. +BASES = set() + + +class StandInRequestEngine(FlowElement): + """Stands in for a Cloud Request Engine whose answer has an empty + section for each of the given data keys.""" + + def __init__(self, keys): + super(StandInRequestEngine, self).__init__() + self.datakey = "cloud" + self.flow_element_properties = {key: {} for key in keys} + self.answer = json.dumps({key: {} for key in keys}) + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"cloud": self.answer})) + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + processed = set() + for element in [LocationCloud()]: + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + flowdata = builder \ + .add(StandInRequestEngine([element.datakey])) \ + .add(element).build().create_flowdata() + flowdata.process() + data = flowdata.get(element.datakey) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + processed.add(type(element)) + self.assertEqual( + defined_flow_elements(fiftyone_location) - BASES, processed) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_finished_writes.py new file mode 100644 index 00000000..dc85cf5e --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_finished_writes.py @@ -0,0 +1,147 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines, and every one its examples define, is +processed, then a write to its ElementData is expected to fail, as the thread +safety page of the specification requires. +""" + +import importlib +import importlib.util +import inspect +import json +import os +import pkgutil + +import fiftyone_pipeline_cloudrequestengine +from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine +from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import \ + CloudRequestEngine +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +from .classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +EXAMPLES = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "examples", "src", "fiftyone_pipeline_examples", "cloud") + + +class StandInRequestEngine(FlowElement): + """Stands in for a Cloud Request Engine whose answer has an empty + section for each of the given data keys.""" + + def __init__(self, keys): + super(StandInRequestEngine, self).__init__() + self.datakey = "cloud" + self.flow_element_properties = {key: {} for key in keys} + self.answer = json.dumps({key: {} for key in keys}) + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"cloud": self.answer})) + + +def load_example(name): + """An example module, loaded from its file under its own name.""" + spec = importlib.util.spec_from_file_location( + "example_" + name, os.path.join(EXAMPLES, name + ".py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(CloudRequestEngineTestsBase): + + def assert_writes_fail(self, flowdata, key): + data = flowdata.get(key) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + + def test_every_flow_element_refuses_writes_once_finished(self): + cloud = CloudRequestEngine({ + "license_key": "a-licence-key", + "requested_properties": ["device.ismobile"], + "http_client": self.mock_http(), + }) + engine = CloudEngine() + engine.datakey = "device" + flowdata = PipelineBuilder().add(cloud).add(engine).build() \ + .create_flowdata() + flowdata.evidence.add("query.User-Agent", "iPhone") + flowdata.process() + self.assert_writes_fail(flowdata, "cloud") + self.assert_writes_fail(flowdata, "device") + self.assertEqual( + {CloudEngine, CloudRequestEngine}, + defined_flow_elements(fiftyone_pipeline_cloudrequestengine)) + + def test_every_example_flow_element_refuses_writes_once_finished(self): + engines = load_example("engines") + simple = load_example("simple_cloud_engine") + elements = [engines.DeviceDetectionCloud(), + engines.IpIntelligenceCloud(), + engines.CountriesTranslationCloud(), + simple.SimpleCloudEngine()] + for element in elements: + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + flowdata = builder \ + .add(StandInRequestEngine([element.datakey])) \ + .add(element).build().create_flowdata() + flowdata.process() + self.assert_writes_fail(flowdata, element.datakey) + + +if __name__ == "__main__": + import unittest + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py index 008bc22b..256ac1f4 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py @@ -20,6 +20,137 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +import functools +import inspect + +from .messages import Messages + + +class _CheckedDictWrites(object): + + """! + The write methods of a dict, each of which calls _check_writable first, + so a dict that holds Element Data values refuses writes once the + FlowElement that writes them has finished. Reads are the dict's own. + """ + + # The class this is mixed into gives _check_writable and the dict's own + # methods, which pylint cannot see from here. + # pylint: disable=no-member + + __slots__ = () + + def __setitem__(self, key, value): + self._check_writable(key) + super(_CheckedDictWrites, self).__setitem__(key, value) + + def __delitem__(self, key): + self._check_writable(key) + super(_CheckedDictWrites, self).__delitem__(key) + + def __ior__(self, other): + self._check_writable("|=") + return super(_CheckedDictWrites, self).__ior__(other) + + def update(self, *args, **kwargs): + self._check_writable("update") + super(_CheckedDictWrites, self).update(*args, **kwargs) + + def pop(self, *args): + self._check_writable(args[0] if args else "pop") + return super(_CheckedDictWrites, self).pop(*args) + + def popitem(self): + self._check_writable("popitem") + return super(_CheckedDictWrites, self).popitem() + + def setdefault(self, key, default=None): + self._check_writable(key) + return super(_CheckedDictWrites, self).setdefault(key, default) + + def clear(self): + self._check_writable("clear") + super(_CheckedDictWrites, self).clear() + + +class _ContentsDict(_CheckedDictWrites, dict): + + """! + The contents of Element Data that keeps its values in a dict. A write + raises once the FlowElement that writes the data has finished, as a + write to the data itself does. A copy is a plain dict. + """ + + __slots__ = ("_owner",) + + def __init__(self, owner, values): + dict.__init__(self, values) + self._owner = owner + + def _check_writable(self, name): + self._owner._check_writable(name) + + def __reduce_ex__(self, protocol): + return (dict, (dict(self),)) + + +class _FlowDataDict(_CheckedDictWrites, dict): + + """! + A dict that holds what a FlowData has been given, being its ElementData, + its errors or its Evidence. A write raises once the FlowData has + finished processing. A copy is a plain dict. + """ + + __slots__ = ("_flowdata", "_what") + + def __init__(self, flowdata, what, values=()): + dict.__init__(self, values) + self._flowdata = flowdata + self._what = what + + def _check_writable(self, name): + # Read from the FlowData's own fields, because FlowData treats an + # attribute it does not have as the key of ElementData. + if getattr(self._flowdata, "__dict__", {}).get("processed"): + raise Exception(Messages.FLOW_DATA_FINISHED.format(self._what)) + + def __reduce_ex__(self, protocol): + return (dict, (dict(self),)) + + +def _get_contents(data): + return data.__dict__.get("_contents") + + +def _set_contents(data, values): + data._contents = _ContentsDict(data, values) + + +# The contents property of the Element Data classes that keep their values +# in a dict. Setting it keeps a copy, which refuses writes once finished. +_contents_property = property( + _get_contents, _set_contents, + doc="The values, keyed by property name. Written only while the " + "FlowElement that provides the data processes.") + + +def _open_while_closing(close): + + # Releasing what the data holds is not a write to it, so a close method + # runs with the data open for writing, and finished again afterwards. + @functools.wraps(close) + def opened(self, *args, **kwargs): + finished = self._finished + object.__setattr__(self, "_finished", False) + try: + return close(self, *args, **kwargs) + finally: + object.__setattr__(self, "_finished", finished) + + opened._opens_data = True + return opened + class ElementData(object): @@ -43,8 +174,23 @@ class ElementData(object): responsible for thread-safe access, and either gives each thread its own Flow Data or coordinates its own access. + A subclass that holds something to release, such as native results, + gives itself a close method. Closing is not a write, so close may set + the data's attributes even once it is finished, whether it is called + directly or by FlowData.close. + """ + def __init_subclass__(cls, **kwargs): + + # A close method a subclass defines or inherits runs with the data + # open. Read without binding, so a static method is left alone. + super(ElementData, cls).__init_subclass__(**kwargs) + close = inspect.getattr_static(cls, "close", None) + if inspect.isfunction(close) and \ + not getattr(close, "_opens_data", False): + cls.close = _open_while_closing(close) + def __init__(self, flow_element): """! @@ -57,6 +203,45 @@ def __init__(self, flow_element): self.flow_element = flow_element + # Set once the FlowElement that writes the data has finished processing, + # and cleared while one that shares the instance processes. A class + # default, so data whose constructor does not call this one has it too. + _finished = False + + def __setattr__(self, name, value): + + """! + Set an attribute, unless the FlowElement that writes the data has + finished processing, when a public attribute cannot be set. Names + starting with an underscore hold the data's own state, such as + values kept once read, and are not writes. + + @raise Exception: when the FlowElement has finished processing + """ + + if self._finished and not name.startswith("_"): + self._raise_finished(name) + object.__setattr__(self, name, value) + + def _check_writable(self, name): + + """! + Raise unless the FlowElement that writes the data is processing. A + write method calls this first. + + @type name: str + @param name: what is being written, for the message + @raise Exception: when the FlowElement has finished processing + """ + + if self._finished: + self._raise_finished(name) + + def _raise_finished(self, name): + flow_element = self.__dict__.get("flow_element") + key = getattr(flow_element, "datakey", "") + raise Exception(Messages.ELEMENT_DATA_FINISHED.format(name, key)) + def get(self, key): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py index 1becbadb..961a722c 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py @@ -20,7 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -from .elementdata import ElementData +from .elementdata import ElementData, _contents_property class ElementDataDictionary(ElementData): @@ -30,6 +30,9 @@ class ElementDataDictionary(ElementData): """ + # A dict that refuses writes once the FlowElement has finished. + contents = _contents_property + def __init__(self, flow_element, contents): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py index ee6b8f5d..290b1af1 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py @@ -20,11 +20,17 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from .elementdata import _FlowDataDict +from .messages import Messages + # The key the web integration gives the web request itself, which the usage # sharing element reads. No element uses it as evidence, so filter() always # sets it aside, whatever the elements' filters accept. HTTP_REQUEST_KEY = "server.http-request" +# The attributes that hold Evidence values. +_VALUES = ("evidence", "unaccepted") + class Evidence: def __init__(self, flowdata): @@ -34,14 +40,33 @@ def __init__(self, flowdata): @type flowdata: FlowData @param flowdata: Parent FlowData """ + self.flowdata = flowdata self.evidence = {} # Each key in lower case to the key as first added, because keys # ignore letter case. self.keys = {} self.evidence_filtered = False - self.flowdata = flowdata self.unaccepted = {} + def __setattr__(self, name, value): + + """! + Set an attribute. The Evidence values, and the values filter() sets + aside, are kept in dicts that refuse writes once the FlowData has + finished processing, and neither can be replaced then. + + @raise Exception: when one of them is replaced once the FlowData + has finished processing + """ + + if name in _VALUES: + flowdata = self.__dict__.get("flowdata") + if name in self.__dict__ and \ + getattr(flowdata, "__dict__", {}).get("processed"): + raise Exception(Messages.FLOW_DATA_FINISHED.format("Evidence")) + value = _FlowDataDict(flowdata, "Evidence", value) + object.__setattr__(self, name, value) + def add(self, key, value): """! Add a single piece of evidence by its element and value @@ -51,8 +76,12 @@ def add(self, key, value): @type value: mixed @param value: a piece of evidence + @raise Exception: once the FlowData has finished processing """ + if self.flowdata.processed: + raise Exception(Messages.FLOW_DATA_FINISHED.format("Evidence")) + folded = str(key).lower() if folded in self.keys: key = self.keys[folded] @@ -66,9 +95,13 @@ def add_from_dict(self, evidence_dictionary): @type evidence_dictionary: dict @param evidence_dictionary: Dict of evidence + @raise Exception: once the FlowData has finished processing """ - if not type(evidence_dictionary) is dict: + if self.flowdata.processed: + raise Exception(Messages.FLOW_DATA_FINISHED.format("Evidence")) + + if not isinstance(evidence_dictionary, dict): self.flowdata.set_error("core", "Must pass valid dictionary.") for key, value in evidence_dictionary.items(): diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py index 5d433bfe..6eb981dc 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py @@ -20,9 +20,10 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from .elementdata import ElementData, _FlowDataDict from .evidence import Evidence from .flowerror import FlowError -from .messages import Messages +from .messages import Messages import traceback import sys from collections.abc import MutableMapping @@ -124,6 +125,20 @@ def items(self): return self._latest().items() + +def _set_finished(element_data, finished): + + # Records whether the FlowElement that writes the data has finished, which + # ElementData checks before a write. Data of another type is left alone. + if isinstance(element_data, ElementData): + object.__setattr__(element_data, "_finished", finished) + + +# The attributes that hold what a FlowData has been given, each with what +# it holds for the message when a finished FlowData is written. +_GIVEN = {"data": "ElementData", "errors": "an error", "evidence": "Evidence"} + + class FlowData: """! FlowData is created by a specific Pipeline @@ -143,6 +158,11 @@ class FlowData: request, uses it only in the code that handles that request, and closes it when the request ends. + Once processing has finished, a FlowData is read only. Adding Evidence, + ElementData or errors to it raises, whether through its methods or + through the dicts data, errors and evidence.get_all() return. Only + process() sets processed, and setting it back is not supported. + """ def __init__(self, pipeline): @@ -163,6 +183,27 @@ def __init__(self, pipeline): self.stopped = False self.evidence = Evidence(self) + def __setattr__(self, name, value): + + """! + Set an attribute. The ElementData and errors are kept in dicts that + refuse writes once processing has finished, and they and the + Evidence cannot be replaced then. + + @raise Exception: when one of them is set once processing has + finished + """ + + if name in _GIVEN: + # Read from the instance's own fields, because a missing + # attribute is treated as the key of ElementData. + if self.__dict__.get("processed"): + raise Exception(Messages.FLOW_DATA_FINISHED.format( + _GIVEN[name])) + if name != "evidence": + value = _FlowDataDict(self, _GIVEN[name], value) + object.__setattr__(self, name, value) + def process(self): """! @@ -171,37 +212,49 @@ def process(self): @rtype: FlowData @return: Returns flowdata - @raise Exception: the first error a flow element raised, or the - error saying this FlowData has already been processed, unless the - Pipeline suppresses process exceptions. Every error is in errors - either way. + @raise Exception: the first error a flow element raised, unless the + Pipeline suppresses process exceptions, and every error is in errors + either way. A second call raises the error saying this FlowData has + already been processed, as a finished FlowData takes no more errors. """ - if not self.processed: - # filter evidence before processing - try: - self.evidence.filter() - except Exception as e: - flow_error = FlowError("global", e, traceback.format_exc()) - self.set_error(flow_error) - - for flow_element in self.pipeline.flow_elements: - if self.stopped is not True: - # All errors are caught and stored in an errors array keyed by the - # flowElement that set the error - try: - flow_element.process(self) - except Exception: - flow_error = FlowError(flow_element.datakey, sys.exc_info()[1], traceback.format_exc()) - self.set_error(flow_error) - - # Set processed flag to true. flowdata can only be processed once - self.processed = True - else: - flow_error = FlowError("global", Exception(Messages.FLOW_DATA_PROCESSED), Messages.FLOW_DATA_PROCESSED) + # A finished FlowData takes no more errors, so a second call raises. + if self.processed: + raise Exception(Messages.FLOW_DATA_PROCESSED) + + # filter evidence before processing + try: + self.evidence.filter() + except Exception as e: + flow_error = FlowError("global", e, traceback.format_exc()) self.set_error(flow_error) + for flow_element in self.pipeline.flow_elements: + if self.stopped is not True: + key = str(flow_element.datakey).lower() + # A FlowElement that shares data another added may write it + # again while it processes. + _set_finished(self.data.get(key), False) + # All errors are caught and stored in an errors array keyed by the + # flowElement that set the error + try: + flow_element.process(self) + except Exception: + flow_error = FlowError(flow_element.datakey, sys.exc_info()[1], traceback.format_exc()) + self.set_error(flow_error) + finally: + _set_finished(self.data.get(key), True) + + # Data added before processing, or under another FlowElement's key, + # is finished too. + for element_data in list(self.data.values()): + _set_finished(element_data, True) + + # Set processed flag to true. flowdata can only be processed once, + # and is read only from now on. + self.processed = True + if self.errors and self.pipeline.suppress_process_exceptions is False: errored_flow_element_key = next(iter(self.errors)) flowErrorObj = self.errors[errored_flow_element_key] @@ -229,6 +282,8 @@ def close(self): # attribute it does not have as the name of a property. close = getattr(type(element_data), "close", None) if callable(close): + # Closing is not a write, and ElementData opens itself for + # its close method. close(element_data) def __enter__(self): @@ -324,9 +379,13 @@ def set_element_data(self, element_data): @type element_data: ElementData @param element_data: elementData to be added to flowdata + @raise Exception: once this FlowData has finished processing """ + if self.processed: + raise Exception(Messages.FLOW_DATA_FINISHED.format("ElementData")) + # Kept under the data key in lower case, which is how get reads it. self._add(element_data.flow_element.datakey.lower(), element_data) @@ -374,9 +433,13 @@ def set_error(self, flow_error): @type error: FlowError @param error: Flow Error + @raise Exception: once this FlowData has finished processing """ + if self.processed: + raise Exception(Messages.FLOW_DATA_FINISHED.format("an error")) + self.errors[flow_error.flow_element] = flow_error log_message = "Error occurred during processing" diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py index 5698b031..d183e5e3 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py @@ -48,10 +48,23 @@ class Messages(): # FlowData already processed. FLOW_DATA_PROCESSED = "FlowData already processed" + # A write to a FlowData that has finished processing. This takes what was + # being added as a format argument. + FLOW_DATA_FINISHED = ( + "This FlowData has finished processing, so {} cannot be added to it. " + "A FlowData is read only once processing has finished.") + + # A write to ElementData once its FlowElement has finished. This takes + # the name written and the element's data key as format arguments. + ELEMENT_DATA_FINISHED = ( + "'{}' cannot be written to the '{}' ElementData, because the " + "FlowElement that writes it has finished processing. ElementData is " + "written only while the FlowElement that provides it processes.") + # The usage sharing element is not the last Flow Element. This takes the # element's class name, its position counting from 1 and the number of # Flow Elements as format arguments. USAGE_SHARING_NOT_LAST = ( "The usage sharing element must be the last Flow Element and " "outside any parallel group. {} is Flow Element {} of {} in this " - "Pipeline.") \ No newline at end of file + "Pipeline.") diff --git a/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py new file mode 100644 index 00000000..39d7ba75 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_finished_writes.py @@ -0,0 +1,416 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +A FlowData that has finished processing refuses Evidence, ElementData and +errors, and ElementData refuses a write once the FlowElement that writes it +has finished, as the thread safety page of the specification requires. The +last tests process every FlowElement this package defines, and its example +FlowElements, then write to each one's ElementData and expect the error. +""" + +import copy +import importlib +import importlib.util +import inspect +import json +import os +import pkgutil +import unittest + +import fiftyone_pipeline_core +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.evidence import Evidence +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.flowerror import FlowError +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +FINISHED = "finished processing" + +# FlowElements that only other FlowElements build on, which have no +# ElementData of their own to write. +BASES = {FlowElement} + +EXAMPLES = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples") + + +class WritingElement(FlowElement): + """Adds data and then writes it, as a FlowElement may while it + processes.""" + + def __init__(self, datakey="writer", value="first"): + super(WritingElement, self).__init__() + self.datakey = datakey + self.value = value + + def process_internal(self, flowdata): + data = flowdata.data.get(self.datakey) + if data is None: + data = ElementDataDictionary(self, {}) + flowdata.set_element_data(data) + data.written = self.value + + +class RaisingElement(FlowElement): + """Raises while it processes, so the FlowData records an error.""" + + def __init__(self, datakey="raiser"): + super(RaisingElement, self).__init__() + self.datakey = datakey + + def process_internal(self, flowdata): + raise Exception("raised while processing") + + +class AddingElement(FlowElement): + """Adds data under another FlowElement's key, and a value to its + contents while it processes.""" + + def __init__(self): + super(AddingElement, self).__init__() + self.datakey = "adder" + + def process_internal(self, flowdata): + data = ElementDataDictionary(WritingElement("other"), {}) + data.contents["item"] = "while processing" + flowdata.set_element_data(data) + + +class ClosingData(ElementDataDictionary): + """Sets a public attribute when it is closed, as a close method that + releases what the data holds may.""" + + def close(self): + self.closed = True + + +class ClosingElement(FlowElement): + """Adds data that has a close method of its own.""" + + def __init__(self): + super(ClosingElement, self).__init__() + self.datakey = "closing" + + def process_internal(self, flowdata): + flowdata.set_element_data(ClosingData(self, {})) + + +def item_store(data): + """The dictionary the data keeps its values in, or None.""" + if isinstance(data, dict): + return data + if (isinstance(getattr(type(data), "contents", None), property) + or "contents" in vars(data)): + return data.contents + return None + + +# Each way a dictionary can be written. +DICT_WRITES = ( + ("[]=", lambda d: d.__setitem__("b", 2)), + ("del", lambda d: d.__delitem__("a")), + ("update", lambda d: d.update(b=2)), + ("pop", lambda d: d.pop("a")), + ("popitem", lambda d: d.popitem()), + ("setdefault", lambda d: d.setdefault("b", 2)), + ("clear", lambda d: d.clear()), + ("|=", lambda d: d.__ior__({"b": 2})), +) + + +def process(*elements, **settings): + builder = PipelineBuilder(dict({"add_javascript_builder": False, + "use_setheader_properties": False}, + **settings)) + for element in elements: + builder.add(element) + flowdata = builder.build().create_flowdata() + flowdata.process() + return flowdata + + +def load_example(name): + """An example module, loaded from its file under its own name.""" + spec = importlib.util.spec_from_file_location( + "example_" + name, os.path.join(EXAMPLES, name + ".py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedFlowDataTests(unittest.TestCase): + + def test_evidence_cannot_be_added_once_processed(self): + flowdata = process(WritingElement()) + with self.assertRaises(Exception) as caught: + flowdata.evidence.add("query.later", "value") + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + flowdata.evidence.add_from_dict({"query.later": "value"}) + self.assertIn(FINISHED, str(caught.exception)) + self.assertIsNone(flowdata.evidence.get("query.later")) + + def test_element_data_cannot_be_added_once_processed(self): + element = WritingElement("late") + flowdata = process(WritingElement()) + with self.assertRaises(Exception) as caught: + flowdata.set_element_data(ElementDataDictionary(element, {})) + self.assertIn(FINISHED, str(caught.exception)) + self.assertNotIn("late", flowdata.data) + + def test_errors_cannot_be_added_once_processed(self): + flowdata = process(WritingElement()) + with self.assertRaises(Exception) as caught: + flowdata.set_error(FlowError("late", Exception("late"), "")) + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual({}, flowdata.errors) + + def test_a_second_process_raises_even_when_exceptions_are_suppressed(self): + flowdata = process(WritingElement(), + suppress_process_exceptions=True) + with self.assertRaises(Exception) as caught: + flowdata.process() + self.assertIn("already processed", str(caught.exception)) + self.assertEqual({}, flowdata.errors) + + def test_evidence_can_be_added_before_processing(self): + flowdata = PipelineBuilder().add(WritingElement()).build() \ + .create_flowdata() + flowdata.evidence.add("query.early", "value") + self.assertEqual("value", flowdata.evidence.get("query.early")) + + def assert_dict_writes_fail(self, finished, store): + """Each way of writing the dictionary store(finished) raises, and + leaves it as it was.""" + for name, write in DICT_WRITES: + with self.subTest(name): + flowdata = finished() + before = dict(store(flowdata)) + with self.assertRaises(Exception) as caught: + write(store(flowdata)) + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual(before, dict(store(flowdata))) + + def test_the_element_data_dictionary_cannot_be_written_once_processed( + self): + self.assert_dict_writes_fail( + lambda: process(WritingElement("a")), lambda f: f.data) + + def test_the_errors_dictionary_cannot_be_written_once_processed(self): + self.assert_dict_writes_fail( + lambda: process(RaisingElement("a"), + suppress_process_exceptions=True), + lambda f: f.errors) + + def test_the_evidence_dictionaries_cannot_be_written_once_processed( + self): + def finished(): + flowdata = PipelineBuilder().add(WritingElement()).build() \ + .create_flowdata() + flowdata.evidence.add("a", "value") + flowdata.process() + return flowdata + + for store in (lambda f: f.evidence.get_all(), + lambda f: f.evidence.evidence, + lambda f: f.evidence.get_unaccepted()): + self.assert_dict_writes_fail(finished, store) + + def test_the_dictionaries_cannot_be_replaced_once_processed(self): + flowdata = process(WritingElement()) + evidence = flowdata.evidence + for owner, name, value in ( + (flowdata, "data", {}), + (flowdata, "errors", {}), + (flowdata, "evidence", Evidence(flowdata)), + (evidence, "evidence", {}), + (evidence, "unaccepted", {})): + with self.subTest(name): + before = getattr(owner, name) + with self.assertRaises(Exception) as caught: + setattr(owner, name, value) + self.assertIn(FINISHED, str(caught.exception)) + self.assertIs(before, getattr(owner, name)) + + def test_data_given_before_processing_is_read_only_after(self): + flowdata = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(WritingElement()).build().create_flowdata() + flowdata.data = {"given": "before processing"} + flowdata.errors = {} + flowdata.data["added"] = "before processing" + flowdata.process() + self.assertEqual("before processing", flowdata.get("given")) + self.assertEqual("before processing", flowdata.get("added")) + for store in (flowdata.data, flowdata.errors): + with self.assertRaises(Exception) as caught: + store["later"] = "after processing" + self.assertIn(FINISHED, str(caught.exception)) + + +class FinishedElementDataTests(unittest.TestCase): + + def test_data_written_while_processing_cannot_be_written_after(self): + flowdata = process(WritingElement()) + data = flowdata.get("writer") + self.assertEqual("first", data.written) + with self.assertRaises(Exception) as caught: + data.written = "after" + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual("first", data.written) + + def test_an_element_sharing_the_data_writes_it_while_it_processes(self): + flowdata = process(WritingElement("shared", "first"), + WritingElement("shared", "second")) + data = flowdata.get("shared") + self.assertEqual("second", data.written) + with self.assertRaises(Exception): + data.written = "after" + + def test_the_data_keeps_its_own_state_once_finished(self): + data = process(WritingElement()).get("writer") + data._kept = "a value kept once read" + self.assertEqual("a value kept once read", data._kept) + + def test_the_contents_cannot_be_written_once_finished(self): + for name, write in DICT_WRITES: + with self.subTest(name): + data = process(WritingElement()).get("writer") + dict.__setitem__(data.contents, "a", 1) + for contents in (data.contents, data.as_dictionary()): + with self.assertRaises(Exception) as caught: + write(contents) + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual({"a": 1}, dict(data.contents)) + + def test_the_contents_cannot_be_replaced_once_finished(self): + data = process(WritingElement()).get("writer") + with self.assertRaises(Exception) as caught: + data.contents = {"b": 2} + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual({}, dict(data.contents)) + + def test_the_contents_are_a_dict_that_is_written_while_processing(self): + flowdata = process(AddingElement()) + contents = flowdata.get("other").contents + self.assertIsInstance(contents, dict) + self.assertEqual({"item": "while processing"}, contents) + self.assertEqual('{"item": "while processing"}', + json.dumps(contents)) + self.assertIs(dict, type(copy.deepcopy(contents))) + self.assertIs(dict, type(contents.copy())) + + def test_data_under_another_key_is_finished_too(self): + flowdata = process(AddingElement()) + data = flowdata.get("other") + with self.assertRaises(Exception) as caught: + data.written = "after" + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + data.contents["item"] = "after" + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual("while processing", data.contents["item"]) + + def test_data_added_before_processing_is_finished_with_it(self): + flowdata = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(WritingElement()).build().create_flowdata() + data = ElementDataDictionary(WritingElement("early"), {}) + flowdata.set_element_data(data) + data.written = "before processing" + flowdata.process() + with self.assertRaises(Exception) as caught: + data.written = "after" + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual("before processing", data.written) + + def test_closing_the_flow_data_is_not_a_write(self): + flowdata = process(ClosingElement()) + flowdata.close() + self.assertTrue(flowdata.get("closing").closed) + + def test_closing_the_data_itself_is_not_a_write(self): + data = process(ClosingElement()).get("closing") + data.close() + self.assertTrue(data.closed) + with self.assertRaises(Exception) as caught: + data.closed = False + self.assertIn(FINISHED, str(caught.exception)) + self.assertTrue(data.closed) + + +class EveryFlowElementTests(unittest.TestCase): + + def assert_writes_fail(self, flowdata, key): + data = flowdata.get(key) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + contents = item_store(data) + if contents is not None: + with self.assertRaises(Exception) as caught: + contents["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + + def test_the_package_flow_elements_refuse_writes_once_finished(self): + flowdata = PipelineBuilder().build().create_flowdata() + flowdata.evidence.add("header.user-agent", "a user agent") + flowdata.process() + processed = set() + for element in flowdata.pipeline.flow_elements: + self.assert_writes_fail(flowdata, element.datakey) + processed.add(type(element)) + self.assertEqual( + defined_flow_elements(fiftyone_pipeline_core) - BASES, processed) + + def test_the_example_flow_elements_refuse_writes_once_finished(self): + for name, element in ( + ("simple_flow_element", "SimpleFlowElement"), + ("client_side_evidence_custom_flow_element", + "AstrologyFlowElement")): + module = load_example(name) + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + flowdata = builder.add(getattr(module, element)()).build() \ + .create_flowdata() + flowdata.evidence.add("query.dateOfBirth", "1992-12-18") + flowdata.process() + self.assert_writes_fail(flowdata, "astrology") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_setheader.py b/pipeline/fiftyone_pipeline_core/tests/test_setheader.py index 9f96df6a..67311854 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_setheader.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_setheader.py @@ -57,7 +57,9 @@ class SetHeaderUACHTests(unittest.TestCase): def test_get_response_header_value(self, device, expected_value): testPipeline = TestPipeline() setHeaderElement = SetHeaderElement() - flowdata = testPipeline.flowdata + # A FlowData that has not been processed, because a finished one + # takes no more ElementData. + flowdata = testPipeline.pipeline.create_flowdata() flowdata.data = device actual_value = setHeaderElement.get_response_header_value(flowdata, setHeader_properties_dict) self.assertEqual(expected_value, actual_value[Constants.ACCEPTCH_HEADER]) @@ -80,7 +82,9 @@ def test_set_response_header(self, data, expected_value): response = session.get('mock://test.com/3') testPipeline = TestPipeline() - flowdata = testPipeline.flowdata + # A FlowData that has not been processed, because a finished one + # takes no more ElementData. + flowdata = testPipeline.pipeline.create_flowdata() flowdata.data = data response = set_response_header(flowdata, response) diff --git a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py index dcca50e6..96e9c9b7 100644 --- a/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py +++ b/pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived_property_data.py @@ -36,5 +36,11 @@ def __init__(self, flow_element): super().__init__(flow_element, {}) def set_value(self, name, value): - """! Writes the value of one property. """ + """! + Writes the value of one property. + + @raise Exception: once every element that writes the data has + finished processing + """ + self._check_writable(name) self.contents[name.lower()] = value diff --git a/pipeline/fiftyone_pipeline_derivedproperty/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_finished_writes.py new file mode 100644 index 00000000..d02452c1 --- /dev/null +++ b/pipeline/fiftyone_pipeline_derivedproperty/tests/test_finished_writes.py @@ -0,0 +1,105 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines, and the one its example defines, is +processed, then a write to its ElementData is expected to fail, as the +thread safety page of the specification requires. The derived property +elements of a pipeline share one ElementData, which each writes while it +processes and none writes once they have all finished. +""" + +import importlib +import importlib.util +import inspect +import os +import pkgutil + +import pytest + +import fiftyone_pipeline_derivedproperty +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_derivedproperty import DerivedPropertyElementBuilder +from fiftyone_pipeline_derivedproperty.derived_property_element import \ + DerivedPropertyElement + +from .helpers import SourceElement, band_script, pipeline_of, processed + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +EXAMPLES = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples") + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +def derived(name): + return (DerivedPropertyElementBuilder() + .add_script(name, band_script(name, "src.N")).build()) + + +def test_every_derived_element_writes_only_while_it_processes(): + flowdata = processed(pipeline_of( + [SourceElement("src", {"N": 12}), derived("Demo"), derived("Other")])) + data = flowdata.derived + assert data.get("Demo").value() == "High" + assert data.get("Other").value() == "High" + with pytest.raises(Exception, match=FINISHED): + data.set_value("Demo", "written after processing") + with pytest.raises(Exception, match=FINISHED): + data.written_after_processing = True + with pytest.raises(Exception, match=FINISHED): + item_store(data)["written_after_processing"] = True + assert data.get("Demo").value() == "High" + assert defined_flow_elements(fiftyone_pipeline_derivedproperty) == \ + {DerivedPropertyElement} + + +def test_the_example_flow_element_refuses_writes_once_finished(): + spec = importlib.util.spec_from_file_location( + "example_screen_element", os.path.join(EXAMPLES, "screen_element.py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + flowdata = processed(pipeline_of([module.ScreenElement()])) + with pytest.raises(Exception, match=FINISHED): + flowdata.get("screen").written_after_processing = True + with pytest.raises(Exception, match=FINISHED): + item_store(flowdata.get("screen"))["written_after_processing"] = True diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata_dictionary.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata_dictionary.py index b259bdcb..88a6dd14 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata_dictionary.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata_dictionary.py @@ -20,6 +20,8 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from fiftyone_pipeline_core.elementdata import _contents_property + from .aspectdata import AspectData class AspectDataDictionary(AspectData): @@ -30,6 +32,9 @@ class AspectDataDictionary(AspectData): """ + # A dict that refuses writes once the FlowElement has finished. + contents = _contents_property + def __init__(self, flow_element, contents): """! diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_engines/tests/test_finished_writes.py new file mode 100644 index 00000000..fdb45ade --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines/tests/test_finished_writes.py @@ -0,0 +1,97 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +The engines package defines only the Engine that other FlowElements build +on. Its example engine is processed, then a write to its ElementData is +expected to fail, as the thread safety page of the specification requires. +""" + +import importlib +import importlib.util +import inspect +import os +import pkgutil +import unittest + +import fiftyone_pipeline_engines +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines.engine import Engine + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +EXAMPLES = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "examples") + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_the_package_defines_only_the_engine_others_build_on(self): + self.assertEqual( + {Engine}, defined_flow_elements(fiftyone_pipeline_engines)) + + def test_the_example_engine_refuses_writes_once_finished(self): + spec = importlib.util.spec_from_file_location( + "example_simple_onpremise_engine", + os.path.join(EXAMPLES, "simple_onpremise_engine.py")) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + engine = module.SimpleOnPremiseEngine(module.DATA_FILE) + pipeline = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(engine).build() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.evidence.add(module.DATE_OF_BIRTH, "1992-12-18") + flowdata.process() + with self.assertRaises(Exception) as caught: + flowdata.get("astrology").written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(flowdata.get("astrology"))["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_finished_writes.py new file mode 100644 index 00000000..6c4b6400 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_finished_writes.py @@ -0,0 +1,80 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, as the thread safety +page of the specification requires. The usage sharing element adds no +ElementData, so a finished FlowData holds none for it, and the FlowData +refuses one being added. +""" + +import importlib +import inspect +import pkgutil +import unittest + +import fiftyone_pipeline_engines_fiftyone +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage + +FINISHED = "finished processing" + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + # Nothing listens on the discard port, and one record is far short + # of a package, so nothing is sent. + usage = ShareUsage(requested_package_size=1000, + endpoint="http://127.0.0.1:9") + pipeline = PipelineBuilder().add(usage).build() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "a user agent") + flowdata.process() + self.assertNotIn(usage.datakey, flowdata.data) + with self.assertRaises(Exception) as caught: + flowdata.set_element_data(ElementDataDictionary(usage, {})) + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual( + {ShareUsage}, + defined_flow_elements(fiftyone_pipeline_engines_fiftyone)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_translation/tests/test_finished_writes.py b/pipeline/fiftyone_pipeline_translation/tests/test_finished_writes.py new file mode 100644 index 00000000..ac741857 --- /dev/null +++ b/pipeline/fiftyone_pipeline_translation/tests/test_finished_writes.py @@ -0,0 +1,99 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every FlowElement this package defines is processed, then a write to its +ElementData is expected to fail, as the thread safety page of the +specification requires. +""" + +import importlib +import inspect +import pkgutil +import unittest + +import fiftyone_pipeline_translation +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_translation.translation_engine_base import ( + TranslationEngineBase) + +FINISHED = "finished processing" + + +def item_store(data): + """The dictionary the data keeps its values in.""" + if isinstance(data, dict): + return data + return data.contents + + +def defined_flow_elements(package): + """Every FlowElement class defined in the package's own modules.""" + found = set() + for module in pkgutil.walk_packages(package.__path__, + package.__name__ + "."): + loaded = importlib.import_module(module.name) + for _, cls in inspect.getmembers(loaded, inspect.isclass): + if (issubclass(cls, FlowElement) + and cls.__module__.startswith(package.__name__)): + found.add(cls) + return found + + +class SourceElement(FlowElement): + """Gives fixed names to translate.""" + + def __init__(self): + super(SourceElement, self).__init__() + self.datakey = "source" + + def process_internal(self, flowdata): + flowdata.set_element_data( + ElementDataDictionary(self, {"names": ["Germany"]})) + + +class FinishedWritesTests(unittest.TestCase): + + def test_every_flow_element_refuses_writes_once_finished(self): + engine = TranslationEngineBase( + "source", [("Names", "NamesTranslated")], + {"countries.fr_FR.yml": "Germany: Allemagne\n"}, "fr_FR") + flowdata = PipelineBuilder().add(SourceElement()).add(engine) \ + .build().create_flowdata() + flowdata.process() + data = flowdata.get(engine.datakey) + with self.assertRaises(Exception) as caught: + data.written_after_processing = True + self.assertIn(FINISHED, str(caught.exception)) + with self.assertRaises(Exception) as caught: + item_store(data)["written_after_processing"] = True + self.assertIn(FINISHED, str(caught.exception)) + self.assertEqual( + {TranslationEngineBase}, + defined_flow_elements(fiftyone_pipeline_translation)) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From d85e95b7aea104b5971e247a98e6a4ecbce585e6 Mon Sep 17 00:00:00 2001 From: 51DTechResponse <143603565+51DTechResponse@users.noreply.github.com> Date: Sat, 21 Nov 2026 17:04:48 +0000 Subject: [PATCH 133/136] Build the Pipeline's evidence key filter once, when the Pipeline is created (#252) Build the Pipeline's evidence key filter once, when the Pipeline is created. Reviewed at e8ffe2ea with no high finding open, and all 3 checks on that head passing. --- .../src/fiftyone_pipeline_core/evidence.py | 18 +- .../src/fiftyone_pipeline_core/pipeline.py | 27 ++- .../src/fiftyone_pipeline_core/web.py | 11 +- .../test_pipeline_evidence_key_filter.py | 159 ++++++++++++++++++ 4 files changed, 193 insertions(+), 22 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py index 290b1af1..053f470c 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py @@ -152,22 +152,22 @@ def get_unaccepted(self): def filter(self) -> None: """! - Filter evidence using `filter_evidence_key` and - `filter_evidence` methods of each FlowElement in the pipeline. The - keys that do not pass, and the web request under HTTP_REQUEST_KEY, - move to get_unaccepted(). + Filter evidence using the Pipeline's evidence key filter, which is + built once with the Pipeline and asks each FlowElement's + `filter_evidence_key` when a key is checked. The keys that do not + pass, and the web request under HTTP_REQUEST_KEY, move to + get_unaccepted(). """ if self.evidence_filtered: return + accepted = self.flowdata.pipeline.get_evidence_key_filter() + # Loop over a copy of the keys, as keys are removed on the way. for key in list(self.evidence): - if str(key).lower() == HTTP_REQUEST_KEY or not any([ - flow_element.filter_evidence_key(key) - for flow_element - in self.flowdata.pipeline.flow_elements - ]): + if str(key).lower() == HTTP_REQUEST_KEY or \ + not accepted.filter_evidence_key(key): self.unaccepted[key] = self.evidence.pop(key) del self.keys[str(key).lower()] diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py index e21f0b54..39d125c9 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py @@ -68,9 +68,14 @@ class PipelineEvidenceKeyFilter(EvidenceKeyFilter): """! The Evidence a Pipeline accepts, being every key one of its FlowElements - accepts. Each element's filter is read when a key is checked, as an - element's list can change once its data or the cloud service's answer - arrives. + accepts. Each element is asked when a key is checked, in the order the + elements run, and nothing an element answers is kept, as an element's + list can change once its data or the cloud service's answer arrives. + Making the filter asks no element, so a Pipeline is built while an + element cannot yet say what it accepts. + + This is the filter that decides which evidence processing keeps, and + which evidence the web integration counts as used. """ @@ -79,8 +84,8 @@ def __init__(self, pipeline): def filter_evidence_key(self, key): return any( - element.get_evidence_key_filter().filter_evidence_key(key) - for element in self.pipeline.flow_elements_list.values()) + element.filter_evidence_key(key) + for element in self.pipeline.flow_elements) class Pipeline: @@ -134,6 +139,13 @@ def __init__(self, flow_elements, logger=Logger(), suppress_process_exceptions = check_usage_sharing_last(self._flow_elements) + # Built once here, as it is always needed and the Pipeline's + # elements do not change. It is made before the elements are + # registered, so one that filters evidence as it is registered + # finds it. It asks each element when a key is checked, so an + # element whose keys arrive later is still read. + self._evidence_key_filter = PipelineEvidenceKeyFilter(self) + self.logger = logger # Filled below as each element is registered, so an element finds @@ -271,14 +283,15 @@ def get_evidence_key_filter(self): """! The Evidence the Pipeline accepts, being every key one of its - FlowElements accepts. + FlowElements accepts. The filter is built when the Pipeline is + created, and the same one is returned every time. @rtype: EvidenceKeyFilter @return: Returns an EvidenceKeyFilter """ - return PipelineEvidenceKeyFilter(self) + return self._evidence_key_filter def element_not_found(self, element, flowdata): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py index 208f58eb..122f2f2a 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py @@ -298,17 +298,16 @@ def _client_side_response( def _uses(pipeline, key): - # Whether the key is evidence one of the pipeline's elements accepts. The - # web request itself, which the usage sharing element reads, is evidence - # no element uses, whatever the filters accept. + # Whether the key is evidence one of the pipeline's elements accepts, + # which the pipeline's own evidence key filter says. The web request + # itself, which the usage sharing element reads, is evidence no element + # uses, whatever the filters accept. if key.lower() == HTTP_REQUEST_KEY: return False if key.startswith("header.") and \ key[len("header."):] in _REVALIDATION_HEADERS: return False - return any( - element.filter_evidence_key(key) - for element in pipeline.flow_elements) + return pipeline.get_evidence_key_filter().filter_evidence_key(key) def _opaque_tag(pipeline, evidence): diff --git a/pipeline/fiftyone_pipeline_core/tests/test_pipeline_evidence_key_filter.py b/pipeline/fiftyone_pipeline_core/tests/test_pipeline_evidence_key_filter.py index 16cb4612..d994a36d 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_pipeline_evidence_key_filter.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_pipeline_evidence_key_filter.py @@ -21,7 +21,10 @@ # ********************************************************************* import unittest +from unittest import mock +from fiftyone_pipeline_core import pipeline as pipeline_module +from fiftyone_pipeline_core import web from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter from fiftyone_pipeline_core.flowelement import FlowElement from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder @@ -47,6 +50,54 @@ def process_internal(self, flowdata): pass +class NotReady(Exception): + """Raised by an element that cannot yet say what it accepts.""" + + +class Recovering(FlowElement): + """An element that cannot say what it accepts until it has recovered, + as an element that learns its keys from a service does while the + service cannot be reached.""" + + def __init__(self): + super(Recovering, self).__init__() + self.datakey = "recovering" + self.recovered = False + + def get_evidence_key_filter(self): + if not self.recovered: + raise NotReady("The stand-in service cannot be reached.") + return BasicListEvidenceKeyFilter(["header.x-recovered"]) + + def process_internal(self, flowdata): + pass + + +class FiltersWhenRegistered(FlowElement): + """An element that filters a FlowData's evidence as soon as it is + registered, before the Pipeline's constructor has finished.""" + + def __init__(self): + super(FiltersWhenRegistered, self).__init__() + self.datakey = "early" + self.kept = None + self.set_aside = None + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(["header.user-agent"]) + + def on_registration(self, pipeline): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "a browser") + flowdata.evidence.add("cookie.other", "not read") + flowdata.evidence.filter() + self.kept = dict(flowdata.evidence.get_all()) + self.set_aside = dict(flowdata.evidence.get_unaccepted()) + + def process_internal(self, flowdata): + pass + + class PipelineEvidenceKeyFilterTests(unittest.TestCase): """! advertize-accepted-evidence.md line 5: "All Flow Elements and Pipelines @@ -78,6 +129,27 @@ def test_the_pipeline_reads_its_elements_filters_when_asked(self): later.keys = ["header.x-later"] self.assertTrue(accepted.filter_evidence_key("header.x-later")) + def test_the_pipeline_gives_the_same_filter_every_time(self): + pipeline = self.build(Element("one", ["header.user-agent"])) + self.assertIs( + pipeline.get_evidence_key_filter(), + pipeline.get_evidence_key_filter()) + + def test_the_pipeline_builds_its_filter_once_when_it_is_created(self): + # The Pipeline's filter is always needed and does not change, so it + # is built in the constructor and not on each call. + real = pipeline_module.PipelineEvidenceKeyFilter + with mock.patch.object( + pipeline_module, "PipelineEvidenceKeyFilter", + side_effect=real) as made: + pipeline = self.build(Element("one", ["header.user-agent"])) + self.assertEqual(1, made.call_count) + first = pipeline.get_evidence_key_filter() + pipeline.get_evidence_key_filter() + self.assertEqual(1, made.call_count) + self.assertIsInstance(first, real) + self.assertTrue(first.filter_evidence_key("header.user-agent")) + def test_the_pipeline_filters_evidence_to_what_it_accepts(self): pipeline = self.build(Element("one", ["header.user-agent"])) flowdata = pipeline.create_flowdata() @@ -88,6 +160,93 @@ def test_the_pipeline_filters_evidence_to_what_it_accepts(self): pipeline.get_evidence_key_filter().filter_evidence( flowdata.evidence.get_all())) + def test_processing_filters_evidence_with_the_pipelines_filter(self): + # The filter built with the Pipeline is the one processing uses, so + # there is one place that decides what the Pipeline accepts. + pipeline = self.build(Element("one", ["header.user-agent"])) + accepted = pipeline.get_evidence_key_filter() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "a browser") + flowdata.evidence.add("cookie.other", "not read") + with mock.patch.object( + accepted, "filter_evidence_key", + wraps=accepted.filter_evidence_key) as asked: + flowdata.process() + self.assertEqual( + [mock.call("header.user-agent"), mock.call("cookie.other")], + asked.call_args_list) + self.assertEqual( + {"header.user-agent": "a browser"}, flowdata.evidence.get_all()) + self.assertEqual( + {"cookie.other": "not read"}, flowdata.evidence.get_unaccepted()) + + def test_processing_follows_an_element_whose_keys_arrive_later(self): + # Processing reads the Pipeline's filter, and that filter asks each + # element at the time, so keys an element gains after the Pipeline + # is built are kept. + later = Element("later", []) + pipeline = self.build(later) + accepted = pipeline.get_evidence_key_filter() + later.keys = ["header.x-later"] + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.x-later", "kept") + with mock.patch.object( + accepted, "filter_evidence_key", + wraps=accepted.filter_evidence_key) as asked: + flowdata.process() + self.assertEqual( + [mock.call("header.x-later")], asked.call_args_list) + self.assertEqual( + {"header.x-later": "kept"}, flowdata.evidence.get_all()) + + def test_the_web_integration_asks_the_pipelines_filter(self): + # The entity tag and the Vary header of the client-side responses + # are made from the evidence the Pipeline accepts. + pipeline = self.build(Element("one", ["header.user-agent"])) + accepted = pipeline.get_evidence_key_filter() + with mock.patch.object( + accepted, "filter_evidence_key", + wraps=accepted.filter_evidence_key) as asked: + self.assertTrue(web._uses(pipeline, "header.user-agent")) + self.assertFalse(web._uses(pipeline, "cookie.other")) + # Neither of these is evidence an element uses, whatever the + # filters accept, so the filter is not asked about them. + self.assertFalse(web._uses(pipeline, "server.http-request")) + self.assertFalse(web._uses(pipeline, "header.if-none-match")) + self.assertEqual( + [mock.call("header.user-agent"), mock.call("cookie.other")], + asked.call_args_list) + + def test_the_pipeline_is_built_while_an_element_cannot_answer(self): + # Building the Pipeline asks no element for its filter, so an + # element that cannot answer yet does not stop the build. The + # element is asked each time a key is checked, and nothing is kept + # from before it could answer. + recovering = Recovering() + pipeline = self.build( + Element("one", ["header.user-agent"]), recovering) + accepted = pipeline.get_evidence_key_filter() + with self.assertRaises(NotReady): + accepted.filter_evidence_key("header.x-recovered") + recovering.recovered = True + self.assertTrue(accepted.filter_evidence_key("header.x-recovered")) + self.assertTrue(accepted.filter_evidence_key("header.user-agent")) + self.assertFalse(accepted.filter_evidence_key("cookie.other")) + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.x-recovered", "kept") + flowdata.evidence.add("cookie.other", "not read") + flowdata.process() + self.assertEqual( + {"header.x-recovered": "kept"}, flowdata.evidence.get_all()) + + def test_evidence_is_filtered_while_the_pipeline_is_being_built(self): + # An element may use the Pipeline as soon as it is registered, so + # the filter is there before the first element is. + early = FiltersWhenRegistered() + self.build(early) + self.assertEqual({"header.user-agent": "a browser"}, early.kept) + self.assertEqual({"cookie.other": "not read"}, early.set_aside) + if __name__ == "__main__": unittest.main() From 497078ce9478b32f91e1ec7de8344d878e690884 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 21 Nov 2026 19:18:11 +0000 Subject: [PATCH 134/136] Test and document the on-premise packages on free-threaded Python (#243) Test and document the on-premise packages on free-threaded Python. Reviewed at c7a07ee6 with no high finding open, and all 9 checks on that head passing. --- .github/workflows/ci-script-tests.yml | 6 +- .github/workflows/free-threaded-python.yml | 211 ++++++++++++++++++ ci/free-threaded-python.Tests.ps1 | 174 +++++++++++++++ .../readme.md | 12 + .../swig_data.py | 10 + .../tests/test_free_threaded.py | 85 +++++++ .../readme.md | 12 + .../swig_data.py | 10 + .../tests/test_free_threaded.py | 85 +++++++ 9 files changed, 604 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/free-threaded-python.yml create mode 100644 ci/free-threaded-python.Tests.ps1 create mode 100644 device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py diff --git a/.github/workflows/ci-script-tests.yml b/.github/workflows/ci-script-tests.yml index 8f68b5e0..3a74d841 100644 --- a/.github/workflows/ci-script-tests.yml +++ b/.github/workflows/ci-script-tests.yml @@ -1,13 +1,16 @@ name: CI script tests # Runs the Pester tests of the scripts in ci/, which choose what a pull request -# builds and what a publish releases. common-ci's main is checked out beside +# builds and what a publish releases, and the test of how +# free-threaded-python.yml fetches the free data files. common-ci's main is +# checked out beside # this repository, where ci/dispatch.Tests.ps1 looks for it. on: pull_request: paths: - 'ci/**' - '.github/workflows/ci-script-tests.yml' + - '.github/workflows/free-threaded-python.yml' push: branches: - main @@ -15,6 +18,7 @@ on: paths: - 'ci/**' - '.github/workflows/ci-script-tests.yml' + - '.github/workflows/free-threaded-python.yml' workflow_dispatch: permissions: diff --git a/.github/workflows/free-threaded-python.yml b/.github/workflows/free-threaded-python.yml new file mode 100644 index 00000000..d1dc2c87 --- /dev/null +++ b/.github/workflows/free-threaded-python.yml @@ -0,0 +1,211 @@ +name: Free-threaded Python + +# Builds the Device Detection and IP Intelligence on-premise packages on the +# free-threaded builds of Python and runs their tests there, with the free +# Lite data files. The first job downloads those once for each commit of the +# data submodule, which links them, and puts them in the cache. The test jobs +# restore them from there and never download them. +# ci/free-threaded-python.Tests.ps1 checks this. The native modules do not declare that they run without +# the GIL, so Python turns the GIL back on when it loads them. +# tests/test_free_threaded.py in each package checks that it does, and is +# skipped on every other build of Python. +on: + pull_request: + paths: + - 'device-detection/fiftyone_devicedetection_onpremise/**' + - 'ip-intelligence/fiftyone_ipintelligence_onpremise/**' + - 'pipeline/fiftyone_pipeline_core/**' + - 'pipeline/fiftyone_pipeline_engines/**' + - 'pipeline/fiftyone_pipeline_engines_fiftyone/**' + - 'cxx' + - '.github/workflows/free-threaded-python.yml' + push: + branches: + - main + - rc + paths: + - 'device-detection/fiftyone_devicedetection_onpremise/**' + - 'ip-intelligence/fiftyone_ipintelligence_onpremise/**' + - 'pipeline/fiftyone_pipeline_core/**' + - 'pipeline/fiftyone_pipeline_engines/**' + - 'pipeline/fiftyone_pipeline_engines_fiftyone/**' + - 'cxx' + - '.github/workflows/free-threaded-python.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + free-data-files: + name: Fetch the free Lite data files once + runs-on: ubuntu-latest + outputs: + key: ${{ steps.key.outputs.key }} + defaults: + run: + shell: pwsh + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + submodules: recursive + + - name: Name the cache after the data submodule + id: key + run: | + # The data submodule's .link files say which files are fetched and + # from where, so a new commit of it needs the files again. Change + # v1 when the list of files below changes. + $data = git -C cxx rev-parse HEAD:data + if ($LASTEXITCODE) { exit $LASTEXITCODE } + "key=free-lite-data-v1-$data" >> $env:GITHUB_OUTPUT + + - name: Look for the files in the cache + id: cache + uses: actions/cache@v4 + with: + path: | + cxx/data/**/51Degrees-LiteV4.1.hash + cxx/data/**/20000 Evidence Records.yml + cxx/data/**/20000 User Agents.csv + cxx/data/**/51Degrees-LiteV41.ipi + cxx/data/**/51Degrees-IPIV4AsnIpiV41.ipi + key: ${{ steps.key.outputs.key }} + # The Windows job restores what this Linux job saves. + enableCrossOsArchive: true + lookup-only: true + + # Runs only when the cache has no copy, and the cache saves the files + # when the job ends. + - name: Fetch the free Lite data files + if: steps.cache.outputs.cache-hit != 'true' + run: ./cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash '20000 Evidence Records.yml' '20000 User Agents.csv' 51Degrees-LiteV41.ipi 51Degrees-IPIV4AsnIpiV41.ipi + + free-threaded: + name: Python ${{ matrix.python }} on ${{ matrix.os }} + needs: free-data-files + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + python: '3.13t' + - os: ubuntu-latest + python: '3.14t' + - os: windows-latest + python: '3.14t' + runs-on: ${{ matrix.os }} + defaults: + run: + shell: pwsh + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + # The native source is the cxx submodule, and the data files are + # linked from its own data submodule. + submodules: recursive + # The package versions written below come from the release tags. + fetch-depth: 0 + + # ci/write-package-versions.ps1 works out the versions with common-ci's + # release plan. + - name: Checkout common-ci + uses: actions/checkout@v4 + with: + repository: 51Degrees/common-ci + ref: main + path: .common-ci + persist-credentials: false + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Check the build is free-threaded + run: | + python -c "import sys, sysconfig; print(sys.version); assert sysconfig.get_config_var('Py_GIL_DISABLED') == 1, 'not a free-threaded build'; assert not sys._is_gil_enabled(), 'the GIL is on at start'" + + - name: Install the packages the tests use + env: + COMMON_CI: ${{ github.workspace }}/.common-ci + run: | + # A pip install that fails stops the step here, rather than + # leaving the tests to fail on a missing module. + $PSNativeCommandUseErrorActionPreference = $true + python -m pip install --upgrade pip + python -m pip install pytest flask flask-unittest json5 ruamel.yaml Cython setuptools + # A package installed from this checkout is 0.0.0 without the + # version.txt this writes, which is below the versions the packages + # here require of each other, so pip would refuse them. + ./ci/write-package-versions.ps1 + # The pipeline packages come from this checkout, as the tests run + # with the pipeline code they are released with. + python -m pip install ` + -e pipeline/fiftyone_pipeline_core ` + -e pipeline/fiftyone_pipeline_engines ` + -e pipeline/fiftyone_pipeline_engines_fiftyone ` + -e pipeline/fiftyone_pipeline_cloudrequestengine ` + -e device-detection/fiftyone_devicedetection_shared ` + -e device-detection/fiftyone_devicedetection_cloud ` + -e ip-intelligence/fiftyone_ipintelligence_shared + # These two require the on-premise package by name, which the + # tests read from its source folder instead. + python -m pip install --no-deps ` + -e device-detection/fiftyone_devicedetection ` + -e device-detection/fiftyone_devicedetection_examples + + - name: Restore the free Lite data files + uses: actions/cache/restore@v4 + with: + path: | + cxx/data/**/51Degrees-LiteV4.1.hash + cxx/data/**/20000 Evidence Records.yml + cxx/data/**/20000 User Agents.csv + cxx/data/**/51Degrees-LiteV41.ipi + cxx/data/**/51Degrees-IPIV4AsnIpiV41.ipi + key: ${{ needs.free-data-files.outputs.key }} + enableCrossOsArchive: true + # The first job fetched them, so a miss is a fault, never a reason + # to download them here. + fail-on-cache-miss: true + + - name: Put the IP Intelligence file where the tests look for it + run: | + New-Item -ItemType Directory -Force -Path ip-intelligence/assets | Out-Null + New-Item -ItemType HardLink ` + -Path ip-intelligence/assets/51Degrees-LiteIpiV41.ipi ` + -Target cxx/data/ip-intelligence/51Degrees-LiteV41.ipi | Out-Null + + - name: Build the native modules + run: | + if ($IsWindows) { + # A short temporary path, for the MSVC path length limit. + $env:TEMP = New-Item -ItemType Directory -Force -Path 'C:\tmp' + $env:TMP = $env:TEMP + } + foreach ($package in 'device-detection/fiftyone_devicedetection_onpremise', 'ip-intelligence/fiftyone_ipintelligence_onpremise') { + Push-Location $package + python setup.py build_clib build_ext --inplace + if ($LASTEXITCODE) { exit $LASTEXITCODE } + Pop-Location + } + + - name: Run the tests + run: | + $failed = $false + foreach ($package in 'device-detection/fiftyone_devicedetection_onpremise', 'ip-intelligence/fiftyone_ipintelligence_onpremise') { + Write-Host "::group::$package" + Push-Location $package + $env:PYTHONPATH = Join-Path $PWD 'src' + # -rs lists any test that skipped, and none should here. + python -m pytest -rs tests + if ($LASTEXITCODE) { $failed = $true } + Pop-Location + Write-Host '::endgroup::' + } + if ($failed) { exit 1 } diff --git a/ci/free-threaded-python.Tests.ps1 b/ci/free-threaded-python.Tests.ps1 new file mode 100644 index 00000000..d7028ece --- /dev/null +++ b/ci/free-threaded-python.Tests.ps1 @@ -0,0 +1,174 @@ +# Tests that .github/workflows/free-threaded-python.yml fetches the free Lite +# data files once, in one job, and that the jobs that test take them from the +# cache that job fills, rather than each downloading them again. +# The workflow is read as text, as the runners have no YAML module. + +BeforeAll { + $script:Workflow = Join-Path $PSScriptRoot '..' '.github' 'workflows' ` + 'free-threaded-python.yml' + $lines = @(Get-Content -LiteralPath $script:Workflow) + + # Each job's lines, by the job's name, from the two-space keys under jobs. + $script:Jobs = [ordered]@{} + $inJobs = $false + $name = $null + foreach ($line in $lines) { + if ($line -match '^\S') { + $inJobs = $line -match '^jobs:\s*$' + $name = $null + continue + } + if (-not $inJobs) { continue } + if ($line -match '^ ([\w-]+):\s*$') { + $name = $Matches[1] + $script:Jobs[$name] = [System.Collections.Generic.List[string]]::new() + continue + } + if ($name) { $script:Jobs[$name].Add($line) } + } + + # The text of the step that uses the action given, from its "- name:" or + # "- uses:" line to the next step. + function Get-Step([string]$Job, [string]$Uses) { + $steps = @() + $current = $null + foreach ($line in $script:Jobs[$Job]) { + if ($line -match '^ - ') { + if ($null -ne $current) { $steps += , $current } + $current = [System.Collections.Generic.List[string]]::new() + } + if ($null -ne $current) { $current.Add($line) } + } + if ($null -ne $current) { $steps += , $current } + return @($steps | Where-Object { + ($_ -join "`n") -match "uses:\s*$([regex]::Escape($Uses))@" }) + } + + # The lines of a "path: |" block in a step, trimmed. + function Get-Paths($Step) { + $paths = @() + $inPaths = $false + foreach ($line in $Step) { + if ($line -match '^\s+path:\s*\|\s*$') { $inPaths = $true; continue } + if ($inPaths -and $line -match '^\s{12,}\S') { + $paths += $line.Trim() + continue + } + $inPaths = $false + } + return $paths + } + + function Get-Value($Step, [string]$Key) { + foreach ($line in $Step) { + if ($line -match "^\s+$Key\:\s*(.+?)\s*$") { return $Matches[1] } + } + } + + $script:Fetching = @($script:Jobs.Keys | Where-Object { + ($script:Jobs[$_] -join "`n") -match 'get-data-files\.ps1' }) + $script:Testing = @($script:Jobs.Keys | Where-Object { + ($script:Jobs[$_] -join "`n") -match 'pytest' }) +} + +Describe 'free-threaded-python.yml' { + It 'downloads the free data files in one job only' { + $script:Fetching.Count | Should -Be 1 + } + + It 'downloads them in a job that is not a matrix' { + foreach ($job in $script:Fetching) { + ($script:Jobs[$job] -join "`n") | Should -Not -Match 'matrix:' + } + } + + It 'never downloads them in a job that runs the tests' { + $script:Testing.Count | Should -BeGreaterThan 0 + foreach ($job in $script:Testing) { + $script:Fetching | Should -Not -Contain $job + } + } + + It 'saves them in the cache, and downloads them only when it misses' { + $job = $script:Fetching[0] + $cache = @(Get-Step $job 'actions/cache') + $cache.Count | Should -Be 1 + Get-Value $cache[0] 'id' | Should -Not -BeNullOrEmpty + $id = Get-Value $cache[0] 'id' + $fetch = ($script:Jobs[$job] -join "`n") + $fetch | Should -Match ("if:\s*steps\.$id\.outputs\.cache-hit\s*!=\s*" + + "'true'\s*\n\s+run:\s*[|]?\s*\n?\s*\./cxx/data/get-data-files\.ps1") + } + + It 'restores them in each job that runs the tests, from the same key and paths' { + $job = $script:Fetching[0] + $save = @(Get-Step $job 'actions/cache')[0] + $keyStep = Get-Value $save 'key' + $keyStep | Should -Match '^\$\{\{\s*steps\.([\w-]+)\.outputs\.key\s*\}\}$' + ($script:Jobs[$job] -join "`n") | Should -Match ( + '\n outputs:\s*\n\s+key:\s*' + [regex]::Escape($keyStep)) + foreach ($testing in $script:Testing) { + $text = $script:Jobs[$testing] -join "`n" + $text | Should -Match "\n needs:\s*\[?\s*$([regex]::Escape($job))\b" + $restore = @(Get-Step $testing 'actions/cache/restore') + $restore.Count | Should -Be 1 + Get-Value $restore[0] 'key' | + Should -Be ('${{ needs.' + $job + '.outputs.key }}') + Get-Value $restore[0] 'fail-on-cache-miss' | Should -Be 'true' + Get-Paths $restore[0] | Should -Be (Get-Paths $save) + } + } + + It 'saves and restores across operating systems, for the Windows jobs' { + $job = $script:Fetching[0] + $steps = @(Get-Step $job 'actions/cache') + @($script:Testing | + ForEach-Object { Get-Step $_ 'actions/cache/restore' }) + $steps.Count | Should -BeGreaterThan 1 + foreach ($step in $steps) { + Get-Value $step 'enableCrossOsArchive' | Should -Be 'true' + } + } + + It 'writes the package versions before it installs the packages from this checkout' { + foreach ($testing in $script:Testing) { + $text = $script:Jobs[$testing] -join "`n" + $write = $text.IndexOf('./ci/write-package-versions.ps1') + $install = $text.IndexOf('-e pipeline/') + $write | Should -BeGreaterThan -1 + $install | Should -BeGreaterThan $write + } + } + + It 'checks out the release tags and common-ci, which the versions are worked out from' { + foreach ($testing in $script:Testing) { + $checkouts = @(Get-Step $testing 'actions/checkout') + $own = @($checkouts | Where-Object { + -not (Get-Value $_ 'repository') }) + $own.Count | Should -Be 1 + Get-Value $own[0] 'fetch-depth' | Should -Be '0' + $commonCi = @($checkouts | Where-Object { + (Get-Value $_ 'repository') -eq '51Degrees/common-ci' }) + $commonCi.Count | Should -Be 1 + $path = Get-Value $commonCi[0] 'path' + ($script:Jobs[$testing] -join "`n") | + Should -Match "COMMON_CI:.*$([regex]::Escape($path))" + } + } + + It 'fails the install step when pip fails' { + foreach ($testing in $script:Testing) { + $text = $script:Jobs[$testing] -join "`n" + $text | Should -Match ('- name: Install the packages the tests use' + + '\s*\n(\s+\S.*\n)*?\s+\$PSNativeCommandUseErrorActionPreference = \$true') + } + } + + It 'caches each of the free data files the tests use' { + $paths = Get-Paths (@(Get-Step $script:Fetching[0] 'actions/cache')[0]) + foreach ($file in '51Degrees-LiteV4.1.hash', '51Degrees-LiteV41.ipi', + '51Degrees-IPIV4AsnIpiV41.ipi', '20000 Evidence Records.yml', + '20000 User Agents.csv') { + $paths | Should -Contain "cxx/data/**/$file" + } + } +} diff --git a/device-detection/fiftyone_devicedetection_onpremise/readme.md b/device-detection/fiftyone_devicedetection_onpremise/readme.md index eb1b0178..208ce8ff 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/readme.md +++ b/device-detection/fiftyone_devicedetection_onpremise/readme.md @@ -78,6 +78,18 @@ To run the tests use: `python -m unittest discover -s tests -p test*.py -b` +## Free-threaded Python + +The package is tested on the free-threaded builds of Python 3.13 and 3.14 on Linux, and of Python 3.14 on Windows. It is published as source, so installing it on a free-threaded build needs a C and C++ compiler that builds for that Python. Its native module does not declare that it can run without the global interpreter lock (GIL), so Python turns the GIL back on for the whole process when it loads the module, and prints this warning. + +``` +RuntimeWarning: The global interpreter lock (GIL) has been enabled to load module 'fiftyone_devicedetection_onpremise._DeviceDetectionHashEngineModule', which has not declared that it can run safely without the GIL. To override this behavior and keep the GIL disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0. +``` + +From then on every thread in the program runs with the GIL, the package's native code included, as on the standard build of Python, so neither the package nor the rest of the program gains anything from free-threading. To start with the GIL on, and so without the warning, set the environment variable `PYTHON_GIL=1` or run Python with `-X gil=1`. The package has not been tested with the GIL kept off by `PYTHON_GIL=0`. + +Whichever build of Python runs it, the results of one request are not safe to use from several threads at once. Give each thread its own Flow Data, or coordinate access with a lock of your own. + ## Find out more - [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-device-detection) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index 9982124b..955c71a6 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -64,6 +64,16 @@ class SwigData(AspectData): thread-safe access, and either gives each thread its own Flow Data, and so its own results instance, or coordinates its own access, for example with a lock of its own around every read and around close(). + + The package is tested on the free-threaded builds of Python 3.13 and + 3.14 on Linux, and of Python 3.14 on Windows. The native module does not + declare that it can run without the GIL, so Python turns the GIL back on + for the whole process when it loads the module, with a RuntimeWarning. + Every thread in the program then runs with the GIL, the native code + included, as on the standard build of Python, so nothing gains from + free-threading. Set PYTHON_GIL=1, or run Python with -X gil=1, to start + with the GIL on and so without the warning. A results instance is still + not safe to use from several threads at once. """ def __init__(self, flow_element, swig_results): diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py new file mode 100644 index 00000000..baf0fc78 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py @@ -0,0 +1,85 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +On a free-threaded build of Python, loading the native module turns the GIL +back on, with a RuntimeWarning, because the module does not declare that it +runs without the GIL. A change that loaded it with the GIL still off fails +here. Each check runs in a new interpreter, as this one may have loaded the +module already. +""" + +import os +import subprocess +import sys +import sysconfig +import unittest + +# The SWIG module, which loads the native module when it is imported. +MODULE = "fiftyone_devicedetection_onpremise.DeviceDetectionHashEngineModule" +NATIVE_MODULE = "_DeviceDetectionHashEngineModule" + +FREE_THREADED = sysconfig.get_config_var("Py_GIL_DISABLED") == 1 + +# Prints whether the GIL is on before and after the import, then the +# warnings the import gave, one to a line. +CHECK = "\n".join([ + "import sys, warnings", + "warnings.simplefilter('always')", + "before = sys._is_gil_enabled()", + "with warnings.catch_warnings(record=True) as caught:", + " import " + MODULE, + "print(before, sys._is_gil_enabled())", + "for warning in caught:", + " print(warning.category.__name__, str(warning.message))", +]) + + +@unittest.skipUnless(FREE_THREADED, "Python is not a free-threaded build") +class FreeThreadedTests(unittest.TestCase): + + def import_in_a_new_interpreter(self): + env = dict(os.environ) + # PYTHON_GIL would set the GIL whatever the module declares. + env.pop("PYTHON_GIL", None) + result = subprocess.run( + [sys.executable, "-c", CHECK], env=env, capture_output=True, + text=True) + # The new interpreter's own error says why an import failed. + self.assertEqual(0, result.returncode, result.stderr) + return result.stdout.splitlines() + + def test_loading_the_native_module_turns_the_gil_on(self): + lines = self.import_in_a_new_interpreter() + self.assertEqual("False True", lines[0]) + + def test_loading_the_native_module_warns_that_the_gil_is_on(self): + warnings = self.import_in_a_new_interpreter()[1:] + self.assertTrue( + any(line.startswith("RuntimeWarning ") + and "GIL" in line and NATIVE_MODULE in line + for line in warnings), + warnings) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md index 5cd4011b..1422561f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md @@ -56,6 +56,18 @@ flowdata.process() print(flowdata.ip.registeredname.value()) ``` +## Free-threaded Python + +The package is tested on the free-threaded builds of Python 3.13 and 3.14 on Linux, and of Python 3.14 on Windows. It is published as source, so installing it on a free-threaded build needs a C and C++ compiler that builds for that Python. Its native module does not declare that it can run without the global interpreter lock (GIL), so Python turns the GIL back on for the whole process when it loads the module, and prints this warning. + +``` +RuntimeWarning: The global interpreter lock (GIL) has been enabled to load module 'fiftyone_ipintelligence_onpremise._IpIntelligenceEngineModule', which has not declared that it can run safely without the GIL. To override this behavior and keep the GIL disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0. +``` + +From then on every thread in the program runs with the GIL, the package's native code included, as on the standard build of Python, so neither the package nor the rest of the program gains anything from free-threading. To start with the GIL on, and so without the warning, set the environment variable `PYTHON_GIL=1` or run Python with `-X gil=1`. The package has not been tested with the GIL kept off by `PYTHON_GIL=0`. + +Whichever build of Python runs it, the results of one request are not safe to use from several threads at once. Give each thread its own Flow Data, or coordinate access with a lock of your own. + ## Find out more - [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-ip-intelligence) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index f26cd27f..704d9a97 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -75,6 +75,16 @@ class SwigData(AspectData): thread-safe access, and either gives each thread its own Flow Data, and so its own results instance, or coordinates its own access, for example with a lock of its own around every read and around close(). + + The package is tested on the free-threaded builds of Python 3.13 and + 3.14 on Linux, and of Python 3.14 on Windows. The native module does not + declare that it can run without the GIL, so Python turns the GIL back on + for the whole process when it loads the module, with a RuntimeWarning. + Every thread in the program then runs with the GIL, the native code + included, as on the standard build of Python, so nothing gains from + free-threading. Set PYTHON_GIL=1, or run Python with -X gil=1, to start + with the GIL on and so without the warning. A results instance is still + not safe to use from several threads at once. """ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py new file mode 100644 index 00000000..91253600 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py @@ -0,0 +1,85 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +On a free-threaded build of Python, loading the native module turns the GIL +back on, with a RuntimeWarning, because the module does not declare that it +runs without the GIL. A change that loaded it with the GIL still off fails +here. Each check runs in a new interpreter, as this one may have loaded the +module already. +""" + +import os +import subprocess +import sys +import sysconfig +import unittest + +# The SWIG module, which loads the native module when it is imported. +MODULE = "fiftyone_ipintelligence_onpremise.IpIntelligenceEngineModule" +NATIVE_MODULE = "_IpIntelligenceEngineModule" + +FREE_THREADED = sysconfig.get_config_var("Py_GIL_DISABLED") == 1 + +# Prints whether the GIL is on before and after the import, then the +# warnings the import gave, one to a line. +CHECK = "\n".join([ + "import sys, warnings", + "warnings.simplefilter('always')", + "before = sys._is_gil_enabled()", + "with warnings.catch_warnings(record=True) as caught:", + " import " + MODULE, + "print(before, sys._is_gil_enabled())", + "for warning in caught:", + " print(warning.category.__name__, str(warning.message))", +]) + + +@unittest.skipUnless(FREE_THREADED, "Python is not a free-threaded build") +class FreeThreadedTests(unittest.TestCase): + + def import_in_a_new_interpreter(self): + env = dict(os.environ) + # PYTHON_GIL would set the GIL whatever the module declares. + env.pop("PYTHON_GIL", None) + result = subprocess.run( + [sys.executable, "-c", CHECK], env=env, capture_output=True, + text=True) + # The new interpreter's own error says why an import failed. + self.assertEqual(0, result.returncode, result.stderr) + return result.stdout.splitlines() + + def test_loading_the_native_module_turns_the_gil_on(self): + lines = self.import_in_a_new_interpreter() + self.assertEqual("False True", lines[0]) + + def test_loading_the_native_module_warns_that_the_gil_is_on(self): + warnings = self.import_in_a_new_interpreter()[1:] + self.assertTrue( + any(line.startswith("RuntimeWarning ") + and "GIL" in line and NATIVE_MODULE in line + for line in warnings), + warnings) + + +if __name__ == "__main__": + unittest.main() From ad8c2a7e547bd42311b6ed9b85659de858cf350b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 21 Nov 2026 19:28:31 +0000 Subject: [PATCH 135/136] Reuse Element Data through a bounded pool for each Flow Element (#248) Reuse Element Data through a bounded pool for each Flow Element. Reviewed at 04a81a27 with no high finding open, and all 3 checks on that head passing. --- .../devicedetection_onpremise.py | 9 +- .../swig_data.py | 9 + .../tests/test_element_data_pools.py | 91 ++++++ .../ipintelligence_onpremise.py | 11 +- .../swig_data.py | 12 + .../tests/test_element_data_pools.py | 90 ++++++ pipeline/fiftyone_pipeline_core/readme.md | 4 + .../src/fiftyone_pipeline_core/elementdata.py | 16 + .../elementdata_dictionary.py | 10 + .../elementdata_pool.py | 112 +++++++ .../src/fiftyone_pipeline_core/flowdata.py | 35 +++ .../src/fiftyone_pipeline_core/flowelement.py | 17 ++ .../javascriptbuilder.py | 8 +- .../src/fiftyone_pipeline_core/jsonbundler.py | 8 +- .../fiftyone_pipeline_core/sequenceelement.py | 10 +- .../setheaderelement.py | 10 +- .../tests/test_element_data_pools.py | 277 ++++++++++++++++++ .../tests/test_element_data_pools.py | 112 +++++++ 18 files changed, 833 insertions(+), 8 deletions(-) create mode 100644 device-detection/fiftyone_devicedetection_onpremise/tests/test_element_data_pools.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_element_data_pools.py create mode 100644 pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_pool.py create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_element_data_pools.py create mode 100644 pipeline/fiftyone_pipeline_engines/tests/test_element_data_pools.py diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py index 8134f89a..1ddf8983 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py @@ -30,6 +30,7 @@ from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.elementdata_pool import ElementDataPool from .DeviceDetectionHashEngineModule import * @@ -162,6 +163,8 @@ def __init__( self.datakey = "device" + self._element_data_pool = ElementDataPool() + # The evidence key filter, with the key list it was made from. See # get_evidence_key_filter. self._evidence_key_filter = None @@ -515,7 +518,11 @@ def process_internal(self, flow_data): result = self.engine.process(evidence) - data = SwigData(self, result) + data = self._element_data_pool.take() + if data is None: + data = SwigData(self, result) + else: + data.swig_results = result flow_data.set_element_data(data) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index 955c71a6..4e9e792a 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -103,6 +103,15 @@ def close(self): self.swig_results = None + def _clear(self): + + """! + Drop the results, ready for a new request. See ElementData._clear. + """ + + super(SwigData, self)._clear() + self.swig_results = None + def get(self, key): """! diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_element_data_pools.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_element_data_pools.py new file mode 100644 index 00000000..f24519f5 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_element_data_pools.py @@ -0,0 +1,91 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +The engine takes its ElementData from a pool, as the resource cleanup page +of the specification describes, so over many requests, each closing its +FlowData, it makes one instance rather than one for each request, and a +reused instance gives the new request's results. +""" + +import unittest +from unittest import mock + +from fiftyone_devicedetection_onpremise.devicedetection_onpremise import \ + DeviceDetectionOnPremise +from fiftyone_devicedetection_onpremise.swig_data import SwigData +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +DATA_FILE = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" +MOBILE = ("Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) " + "AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148") +DESKTOP = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + +# Far more requests than the pool's bound. +REQUESTS = 200 + + +class ElementDataPoolTests(unittest.TestCase): + + def setUp(self): + engine = DeviceDetectionOnPremise( + data_file_path=DATA_FILE, licence_keys="", auto_update=False) + self.pipeline = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(engine).build() + self.addCleanup(self.pipeline.close) + + def process(self, user_agent): + flowdata = self.pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", user_agent) + flowdata.process() + return flowdata + + def test_one_instance_is_made_over_many_requests(self): + made = [] + original = SwigData.__init__ + + def counting(data, *args, **kwargs): + made.append(data) + original(data, *args, **kwargs) + + with mock.patch.object(SwigData, "__init__", counting): + for index in range(REQUESTS): + user_agent = MOBILE if index % 2 else DESKTOP + with self.process(user_agent) as flowdata: + self.assertEqual(bool(index % 2), + flowdata.device.ismobile.value()) + self.assertEqual(1, len(made)) + + def test_a_reused_instance_gives_the_new_request_results(self): + with self.process(MOBILE) as flowdata: + first = flowdata.device + self.assertTrue(first.ismobile.value()) + with self.process(DESKTOP) as flowdata: + self.assertIs(first, flowdata.device) + self.assertFalse(flowdata.device.ismobile.value()) + self.assertEqual({}, flowdata.errors) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py index 650b15f3..425ed21e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py @@ -29,6 +29,7 @@ from fiftyone_pipeline_engines.engine import Engine from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.elementdata_pool import ElementDataPool from .IpIntelligenceEngineModule import * @@ -132,6 +133,8 @@ def __init__( self.datakey = "ip" + self._element_data_pool = ElementDataPool() + # The evidence key filter, with the key list it was made from. See # get_evidence_key_filter. self._evidence_key_filter = None @@ -468,7 +471,13 @@ def process_internal(self, flow_data): ipv4, ipv6 = reported_addresses(found) - data = SwigData(self, result, echo_ip=ipv4, echo_ipv6=ipv6) + data = self._element_data_pool.take() + if data is None: + data = SwigData(self, result, echo_ip=ipv4, echo_ipv6=ipv6) + else: + data.swig_results = result + data.echo_ip = ipv4 + data.echo_ipv6 = ipv6 flow_data.set_element_data(data) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 704d9a97..0e700208 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -120,6 +120,18 @@ def close(self): self.swig_results = None + def _clear(self): + + """! + Drop the results and the addresses, ready for a new request. See + ElementData._clear. + """ + + super(SwigData, self)._clear() + self.swig_results = None + self.echo_ip = None + self.echo_ipv6 = None + def get(self, key): """! diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_element_data_pools.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_element_data_pools.py new file mode 100644 index 00000000..a92441e0 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_element_data_pools.py @@ -0,0 +1,90 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +The engine takes its ElementData from a pool, as the resource cleanup page +of the specification describes, so over many requests, each closing its +FlowData, it makes one instance rather than one for each request, and a +reused instance gives the new request's results with none of the last. +""" + +import unittest +from unittest import mock + +from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise import \ + IpIntelligenceOnPremise +from fiftyone_ipintelligence_onpremise.swig_data import SwigData +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +from .constants import TestConstants + +DATA_FILE = TestConstants.get_data_file_path() + +# Far more requests than the pool's bound. +REQUESTS = 200 + + +class ElementDataPoolTests(unittest.TestCase): + + def setUp(self): + engine = IpIntelligenceOnPremise( + data_file_path=DATA_FILE, licence_keys="", auto_update=False) + self.pipeline = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(engine).build() + self.addCleanup(self.pipeline.close) + + def process(self, address): + flowdata = self.pipeline.create_flowdata() + flowdata.evidence.add("query.client-ip", address) + flowdata.process() + return flowdata + + def test_one_instance_is_made_over_many_requests(self): + made = [] + original = SwigData.__init__ + + def counting(data, *args, **kwargs): + made.append(data) + original(data, *args, **kwargs) + + with mock.patch.object(SwigData, "__init__", counting): + for _ in range(REQUESTS): + with self.process(TestConstants.IPV4_ADDRESS) as flowdata: + self.assertEqual(TestConstants.IPV4_ADDRESS, + flowdata.ip.get("ip").value()) + self.assertEqual(1, len(made)) + + def test_a_reused_instance_has_no_address_from_the_last_request(self): + with self.process(TestConstants.IPV4_ADDRESS) as flowdata: + first = flowdata.ip + self.assertTrue(first.get("ip").has_value()) + with self.process(TestConstants.IPV6_ADDRESS) as flowdata: + self.assertIs(first, flowdata.ip) + self.assertFalse(flowdata.ip.get("ip").has_value()) + self.assertEqual(TestConstants.IPV6_ADDRESS, + flowdata.ip.get("ipv6").value()) + self.assertEqual({}, flowdata.errors) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/readme.md b/pipeline/fiftyone_pipeline_core/readme.md index e340273e..e02c1a58 100644 --- a/pipeline/fiftyone_pipeline_core/readme.md +++ b/pipeline/fiftyone_pipeline_core/readme.md @@ -54,6 +54,10 @@ def index(): return response ``` +## Element data is reused + +Some flow elements reuse their element data from one request to the next. When a flow data is closed, by calling `close()` or at the end of a `with` block, its element data goes back to a pool, and a later request may change its values. So read the values you need before the flow data is closed, and don't keep a reference to element data, such as `flowdata.device`, after that. Keeping one isn't supported, and reads aren't checked for it. A value you've already read is unchanged. + ## Requirements * Python 3.8+ diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py index 256ac1f4..d5d56481 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py @@ -179,6 +179,12 @@ class ElementData(object): the data's attributes even once it is finished, whether it is called directly or by FlowData.close. + A FlowElement may take its instances from an ElementDataPool, and + closing the FlowData gives them back for a later request to reuse. So + keeping a reference to ElementData after its FlowData is closed is not + supported, because another request may then change its values. Reads + are not checked for this. + """ def __init_subclass__(cls, **kwargs): @@ -242,6 +248,16 @@ def _raise_finished(self, name): key = getattr(flow_element, "datakey", "") raise Exception(Messages.ELEMENT_DATA_FINISHED.format(name, key)) + def _clear(self): + + """! + Ready an instance taken from an ElementDataPool for a new request, + open for writing. A class with values of its own overrides this to + drop them as well, so that none of the last request's remain. + """ + + self._finished = False + def get(self, key): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py index 961a722c..841e02ea 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_dictionary.py @@ -56,6 +56,16 @@ def __init__(self, flow_element, contents): ElementData(flow_element) + def _clear(self): + + """! + Drop the contents, ready for a new request. See ElementData._clear. + A new dictionary, so one returned by as_dictionary() is unchanged. + """ + + super(ElementDataDictionary, self)._clear() + self.contents = {} + def as_dictionary(self): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_pool.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_pool.py new file mode 100644 index 00000000..d5bf1b85 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata_pool.py @@ -0,0 +1,112 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import os +import threading + +# The most instances a pool keeps, twice the number of processors. An +# instance given back to a full pool is left to the garbage collector. +DEFAULT_MAXIMUM_INSTANCES = 2 * (os.cpu_count() or 1) + + +class ElementDataPool(object): + + """! + A bounded pool of the ElementData one FlowElement makes, so that a + request reuses an instance an earlier request has finished with rather + than making a new one. The FlowElement keeps it as _element_data_pool, + which FlowElement.get_element_data_pool() returns, and takes from it + while it processes. FlowData.close() gives each instance back after + closing it. + + An instance from the pool has no values and is open for writing, as + take() calls its _clear method, which a class with values of its own + overrides to drop them. An instance a cache may hold never goes back, + because the cache shares it with other requests, and the instances of a + FlowData that is not closed are left to the garbage collector. + + Requests on several threads share the pool, so it takes a lock of its + own. It never locks a FlowData or ElementData. + + """ + + def __init__(self, maximum_instances=DEFAULT_MAXIMUM_INSTANCES): + + """! + @type maximum_instances: int + @param maximum_instances: the most instances the pool keeps + + """ + + self.maximum_instances = maximum_instances + self._instances = [] + self._lock = threading.Lock() + + def take(self): + + """! + An instance an earlier request has finished with, with no values and + open for writing. + + @rtype: ElementData + @return: the instance, or None when the pool has none, when the + FlowElement makes a new one + + """ + + with self._lock: + if not self._instances: + return None + element_data = self._instances.pop() + element_data._clear() + return element_data + + def give_back(self, element_data): + + """! + Keep an instance for a later request, unless the pool is full or the + FlowElement that made it has a cache, which may hold it. + + @type element_data: ElementData + @param element_data: an instance whose FlowData has been closed + + """ + + # The check is on the element, not on this instance. A cache may + # hold any instance its element makes, so none comes back, even one + # the cache did not keep. The specification asks for no less. + if getattr(element_data.flow_element, "cache", None) is not None: + return + with self._lock: + if len(self._instances) < self.maximum_instances: + self._instances.append(element_data) + + def __len__(self): + + """! + @rtype: int + @return: the number of instances the pool holds + + """ + + with self._lock: + return len(self._instances) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py index 6eb981dc..b4f948c5 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py @@ -139,6 +139,19 @@ def _set_finished(element_data, finished): _GIVEN = {"data": "ElementData", "errors": "an error", "evidence": "Evidence"} +def _give_back(element_data): + + # Gives the data back to the pool of the FlowElement that made it, if + # that element keeps one. An element that is not a FlowElement has no + # get_element_data_pool and keeps no pool. + if isinstance(element_data, ElementData): + flow_element = element_data.__dict__.get("flow_element") + get_pool = getattr(flow_element, "get_element_data_pool", None) + pool = get_pool() if callable(get_pool) else None + if pool is not None: + pool.give_back(element_data) + + class FlowData: """! FlowData is created by a specific Pipeline @@ -182,6 +195,8 @@ def __init__(self, pipeline): self.processed = False self.stopped = False self.evidence = Evidence(self) + # Set by close(), so that a second call gives nothing back again. + self._closed = False def __setattr__(self, name, value): @@ -275,9 +290,25 @@ def close(self): element data that needs no cleanup, such as an entry in a results cache, is left alone. Values already read are unaffected. + Each element data is then given back to the ElementDataPool of the + FlowElement that made it, if that element keeps one, for a later + request to reuse. So keeping a reference to element data after its + FlowData is closed is not supported, because another request may + then change its values. Reads are not checked for this. Data a + cache may hold is never given back. A second call does nothing. + """ + if self._closed: + return + self._closed = True + + given_back = set() for element_data in list(self.data.values()): + # Data held under two keys is closed and given back once. + if id(element_data) in given_back: + continue + given_back.add(id(element_data)) # Looked up on the class, because ElementData treats an # attribute it does not have as the name of a property. close = getattr(type(element_data), "close", None) @@ -285,6 +316,10 @@ def close(self): # Closing is not a write, and ElementData opens itself for # its close method. close(element_data) + # Given back only once its close has returned. Data whose close + # waits for a lazy task still writing it so goes back only after + # that task has finished, which the specification requires. + _give_back(element_data) def __enter__(self): return self diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py index b04553a0..ff43da42 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py @@ -21,6 +21,7 @@ # ********************************************************************* from .basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +from .elementdata_pool import ElementDataPool class FlowElement(object): @@ -88,6 +89,22 @@ def close(self): pass + def get_element_data_pool(self): + + """! + The ElementDataPool this element takes its ElementData from, which + FlowData.close() gives each instance back to. An element that keeps + one holds it as _element_data_pool. Anything else held under that + name, such as an attribute of a user's own element, is not a pool. + + @rtype: ElementDataPool + @return: the pool, or None when the element keeps none + + """ + + pool = getattr(self, "_element_data_pool", None) + return pool if isinstance(pool, ElementDataPool) else None + def get_evidence_key_filter(self): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py index 8a3d8a0c..aaebd986 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py @@ -37,6 +37,7 @@ from .flowelement import FlowElement from .evidence_keyfilter import EvidenceKeyFilter from .elementdata_dictionary import ElementDataDictionary +from .elementdata_pool import ElementDataPool from .constants import Constants from .callback_url import callback_url, is_scheme @@ -327,6 +328,8 @@ def __init__(self, settings = {} ): self.exclude_from_messages = True + self._element_data_pool = ElementDataPool() + # Set once a callback URL that is not valid has been logged, because # a misconfigured element would otherwise log on every request. self._invalid_url_logged = False @@ -517,7 +520,10 @@ def process_internal(self, flowdata): # Minify the output output = jsmin(output) - data = ElementDataDictionary(self, {"javascript": output}) + data = self._element_data_pool.take() + if data is None: + data = ElementDataDictionary(self, {}) + data.contents["javascript"] = output flowdata.set_element_data(data) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py index a7c42d67..c9b84470 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py @@ -23,6 +23,7 @@ from .basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter from .flowelement import FlowElement from .elementdata_dictionary import ElementDataDictionary +from .elementdata_pool import ElementDataPool # The property types that hold a shape in well-known text. The JSON holds such # a value as an object with one member, value, holding the text. @@ -50,6 +51,8 @@ def __init__(self): self.properties = {"json" : { "type": "dict"} } + self._element_data_pool = ElementDataPool() + # The delayed execution information of each flow element, keyed by # the element rather than its data key, as two Pipelines sharing # this element can hold different elements under one data key. @@ -258,7 +261,10 @@ def process_internal(self, flowdata): output["javascriptProperties"].append(flow_element.datakey.lower() + "." + propertykey.lower()) - data = ElementDataDictionary(self, {"json": output}) + data = self._element_data_pool.take() + if data is None: + data = ElementDataDictionary(self, {}) + data.contents["json"] = output flowdata.set_element_data(data) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py index 5092cd84..af258f55 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py @@ -25,6 +25,7 @@ from .evidence_keyfilter import EvidenceKeyFilter from .javascriptbuilder import parse_sequence from .elementdata_dictionary import ElementDataDictionary +from .elementdata_pool import ElementDataPool import re import uuid @@ -70,6 +71,7 @@ def __init__(self): "session-id": {"type": "string"}, "sequence": {"type": "int"}} self.exclude_from_messages = True + self._element_data_pool = ElementDataPool() def get_evidence_key_filter(self): @@ -113,5 +115,9 @@ def process_internal(self, flowdata): flowdata.evidence.add("query.session-id", session_id) flowdata.evidence.add("query.sequence", sequence) - flowdata.set_element_data(ElementDataDictionary( - self, {"session-id": session_id, "sequence": sequence})) + data = self._element_data_pool.take() + if data is None: + data = ElementDataDictionary(self, {}) + data.contents["session-id"] = session_id + data.contents["sequence"] = sequence + flowdata.set_element_data(data) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py index 2845b00b..87783c71 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py @@ -23,6 +23,7 @@ from .basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter from .flowelement import FlowElement from .elementdata_dictionary import ElementDataDictionary +from .elementdata_pool import ElementDataPool from .messages import Messages # The start of the name of every property whose value is for a response @@ -56,9 +57,11 @@ def __init__(self): # returns for that Pipeline. The element already holds each Pipeline # it is added to in pipelines, so the keys keep none alive longer. self._setheader_properties_by_pipeline = {} - + self.exclude_from_messages = True + self._element_data_pool = ElementDataPool() + def get_evidence_key_filter(self): """! @@ -98,7 +101,10 @@ def process_internal(self, flowdata): response_headers = self.get_response_header_value( flowdata, setheader_properties) - data = ElementDataDictionary(self, {"responseheaderdictionary": response_headers}) + data = self._element_data_pool.take() + if data is None: + data = ElementDataDictionary(self, {}) + data.contents["responseheaderdictionary"] = response_headers flowdata.set_element_data(data) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_element_data_pools.py b/pipeline/fiftyone_pipeline_core/tests/test_element_data_pools.py new file mode 100644 index 00000000..8c848388 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_element_data_pools.py @@ -0,0 +1,277 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +ElementData comes from a bounded pool for each FlowElement that keeps one, +and closing a FlowData gives its ElementData back, as the resource cleanup +page of the specification describes. Over many requests the instances made +stay at the pool's bound rather than one for each request. A reused +instance starts with no values and open for writing, and the instances of a +FlowData that is not closed are left to the garbage collector. +""" + +import gc +import threading +import unittest +import weakref +from unittest import mock + +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_core.sequenceelement import SequenceElement + +# Far more requests than any pool's bound. +REQUESTS = 200 + +# A small bound, so that a test can hold more FlowData open than it. +BOUND = 4 + +# The threads that process requests at once in the threaded test. +THREADS = 4 + +FINISHED = "finished processing" + + +def count_constructions(test, cls): + """Counts each instance of cls made until the test ends, keyed by the + data key of the FlowElement that made it.""" + counts = {} + original = cls.__init__ + + def counting(self, flow_element, *args, **kwargs): + key = flow_element.datakey + counts[key] = counts.get(key, 0) + 1 + original(self, flow_element, *args, **kwargs) + + patcher = mock.patch.object(cls, "__init__", counting) + patcher.start() + test.addCleanup(patcher.stop) + return counts + + +class PooledElement(FlowElement): + """Takes its ElementData from a pool and writes the query name and + value into it, as contents and as a public attribute, which only an + instance open for writing takes.""" + + def __init__(self, bound=BOUND): + from fiftyone_pipeline_core.elementdata_pool import ElementDataPool + super(PooledElement, self).__init__() + self.datakey = "pooled" + self._element_data_pool = ElementDataPool(bound) + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(["query.name", "query.value"]) + + def process_internal(self, flowdata): + data = self._element_data_pool.take() + if data is None: + data = ElementDataDictionary(self, {}) + name = flowdata.evidence.get("query.name") + data.contents[name] = flowdata.evidence.get("query.value") + data.last_name = name + flowdata.set_element_data(data) + + +class TwoKeysElement(PooledElement): + """Adds its ElementData under its own data key and under a second key, + so one instance is in the FlowData's list twice.""" + + def process_internal(self, flowdata): + super(TwoKeysElement, self).process_internal(flowdata) + flowdata.data["second"] = flowdata.get("pooled") + + +class OwnAttributeElement(FlowElement): + """Keeps no pool, but has an attribute of its own with the name the + pooled elements use, holding something else.""" + + def __init__(self): + super(OwnAttributeElement, self).__init__() + self.datakey = "own" + self._element_data_pool = "a value of the element's own" + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary(self, {"a": 1})) + + +def build(*elements): + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + for element in elements: + builder.add(element) + return builder.build() + + +def process(pipeline, name="name", value="value"): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "a user agent") + flowdata.evidence.add("query.name", name) + flowdata.evidence.add("query.value", value) + flowdata.process() + return flowdata + + +class AllocationCountTests(unittest.TestCase): + + def test_each_core_element_makes_one_instance_over_many_requests(self): + counts = count_constructions(self, ElementDataDictionary) + pipeline = PipelineBuilder().build() + self.addCleanup(pipeline.close) + for _ in range(REQUESTS): + with process(pipeline): + pass + keys = {element.datakey for element in pipeline.flow_elements} + self.assertEqual( + {"sequence", "jsonbundler", "javascriptbuilder", "set-headers"}, + keys) + self.assertEqual({key: 1 for key in keys}, counts) + + def test_requests_at_once_make_no_more_instances_than_the_bound(self): + element = SequenceElement() + element._element_data_pool.maximum_instances = BOUND + counts = count_constructions(self, ElementDataDictionary) + pipeline = build(element) + for _ in range(REQUESTS // BOUND): + open_flowdata = [process(pipeline) for _ in range(BOUND)] + for flowdata in open_flowdata: + flowdata.close() + self.assertEqual({"sequence": BOUND}, counts) + + def test_the_pool_keeps_no_more_than_its_bound(self): + element = SequenceElement() + pool = element._element_data_pool + pool.maximum_instances = BOUND + pipeline = build(element) + open_flowdata = [process(pipeline) for _ in range(BOUND * 2)] + for flowdata in open_flowdata: + flowdata.close() + self.assertEqual(BOUND, len(pool)) + + def test_threads_make_no_more_instances_than_run_at_once(self): + counts = count_constructions(self, ElementDataDictionary) + pipeline = build(PooledElement(THREADS)) + wrong = [] + + def requests(name): + for index in range(REQUESTS): + with process(pipeline, name, index) as flowdata: + if flowdata.pooled.get(name) != index: + wrong.append((name, index)) + + threads = [threading.Thread(target=requests, args=("t" + str(n),)) + for n in range(THREADS)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + self.assertEqual([], wrong) + self.assertLessEqual(counts["pooled"], THREADS) + + +class ReuseTests(unittest.TestCase): + + def test_a_reused_instance_has_no_values_from_the_last_request(self): + pipeline = build(PooledElement()) + with process(pipeline, "first", 1) as flowdata: + first = flowdata.get("pooled") + with process(pipeline, "second", 2) as flowdata: + second = flowdata.get("pooled") + self.assertIs(first, second) + self.assertIsNone(second.get("first")) + self.assertEqual(2, second.get("second")) + self.assertEqual("second", second.last_name) + + def test_a_reused_instance_is_written_and_then_finished_again(self): + pipeline = build(PooledElement()) + process(pipeline, "first", 1).close() + with process(pipeline, "second", 2) as flowdata: + self.assertEqual({}, flowdata.errors) + with self.assertRaises(Exception) as caught: + flowdata.pooled.last_name = "after" + self.assertIn(FINISHED, str(caught.exception)) + + def test_a_core_element_reuses_its_instance_with_new_values(self): + pipeline = build(SequenceElement()) + with process(pipeline) as flowdata: + first = flowdata.sequence + session = first.get("session-id") + with process(pipeline) as flowdata: + self.assertIs(first, flowdata.sequence) + self.assertNotEqual(session, flowdata.sequence.get("session-id")) + self.assertEqual(1, flowdata.sequence.get("sequence")) + + def test_closing_twice_gives_an_instance_back_once(self): + element = PooledElement() + pipeline = build(element) + flowdata = process(pipeline) + flowdata.close() + flowdata.close() + self.assertEqual(1, len(element._element_data_pool)) + with process(pipeline) as one, process(pipeline) as two: + self.assertIsNot(one.pooled, two.pooled) + + def test_data_under_two_keys_is_given_back_once(self): + element = TwoKeysElement() + pipeline = build(element) + flowdata = process(pipeline) + self.assertIs(flowdata.get("pooled"), flowdata.get("second")) + flowdata.close() + self.assertEqual(1, len(element._element_data_pool)) + with process(pipeline) as one, process(pipeline) as two: + self.assertIsNot(one.pooled, two.pooled) + + def test_a_flow_data_not_closed_leaves_its_instances_to_the_gc(self): + element = PooledElement() + counts = count_constructions(self, ElementDataDictionary) + pipeline = build(element) + made = [weakref.ref(process(pipeline).pooled) + for _ in range(REQUESTS)] + gc.collect() + self.assertEqual({"pooled": REQUESTS}, counts) + self.assertEqual(0, len(element._element_data_pool)) + self.assertEqual([], [ref for ref in made if ref() is not None]) + + +class PoolLookupTests(unittest.TestCase): + + def test_an_element_names_the_pool_it_keeps(self): + element = SequenceElement() + self.assertIs(element._element_data_pool, + element.get_element_data_pool()) + self.assertIsNone(FlowElement().get_element_data_pool()) + + def test_an_attribute_that_is_not_a_pool_is_not_taken_for_one(self): + element = OwnAttributeElement() + pipeline = build(element) + flowdata = process(pipeline) + flowdata.close() + self.assertIsNone(element.get_element_data_pool()) + self.assertEqual(1, flowdata.own.get("a")) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_element_data_pools.py b/pipeline/fiftyone_pipeline_engines/tests/test_element_data_pools.py new file mode 100644 index 00000000..d9e17f7c --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines/tests/test_element_data_pools.py @@ -0,0 +1,112 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +An instance a cache holds never goes back to a pool, because the cache +shares it with other requests, as the resource cleanup page of the +specification requires. +""" + +import unittest + +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines.datakeyed_cache import DataKeyedCache +from fiftyone_pipeline_engines.engine import Engine + + +class DictionaryCache(DataKeyedCache): + + def __init__(self): + self.entries = {} + + def get_cache_value(self, key): + return self.entries.get(key) + + def set_cache_value(self, key, value): + self.entries[key] = value + + +class PooledEngine(Engine): + """Takes its ElementData from a pool and writes the query name into + it.""" + + def __init__(self): + from fiftyone_pipeline_core.elementdata_pool import ElementDataPool + super(PooledEngine, self).__init__() + self.datakey = "pooled" + self.properties = {"name": {"type": "string"}} + self._element_data_pool = ElementDataPool() + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(["query.name"]) + + def process_internal(self, flowdata): + data = self._element_data_pool.take() + if data is None: + data = ElementDataDictionary(self, {}) + data.contents["name"] = flowdata.evidence.get("query.name") + flowdata.set_element_data(data) + + +def process(pipeline, name): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.name", name) + flowdata.process() + return flowdata + + +class CachedElementDataTests(unittest.TestCase): + + def test_an_instance_held_by_a_cache_is_never_reused(self): + engine = PooledEngine() + engine.set_cache(DictionaryCache()) + pipeline = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(engine).build() + with process(pipeline, "first") as flowdata: + cached = flowdata.pooled + with process(pipeline, "first") as flowdata: + self.assertIs(cached, flowdata.pooled) + with process(pipeline, "second") as flowdata: + self.assertIsNot(cached, flowdata.pooled) + self.assertEqual("second", flowdata.pooled.get("name")) + self.assertEqual("first", cached.get("name")) + self.assertEqual(0, len(engine._element_data_pool)) + + def test_an_engine_without_a_cache_reuses_its_instance(self): + engine = PooledEngine() + pipeline = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(engine).build() + with process(pipeline, "first") as flowdata: + first = flowdata.pooled + with process(pipeline, "second") as flowdata: + self.assertIs(first, flowdata.pooled) + self.assertEqual("second", flowdata.pooled.get("name")) + + +if __name__ == "__main__": + unittest.main() From 60e1b5e84ac43d221d778d98fc5f5ef794d4ee9a Mon Sep 17 00:00:00 2001 From: 51DTechResponse <143603565+51DTechResponse@users.noreply.github.com> Date: Sat, 21 Nov 2026 21:01:30 +0000 Subject: [PATCH 136/136] Read every key under its _51DEGREES_ name, and skip a live cloud test without one (#254) Read every key under its _51DEGREES_ name, and skip a live cloud test without one. Reviewed at 3455f448 with no high finding open, and all 10 checks on that head passing. --- .github/workflows/ci-script-tests.yml | 13 +- .github/workflows/cloud-51did.yml | 64 ++- .github/workflows/example-assets-update.yml | 3 +- .../workflows/monthly-copyright-update.yml | 3 +- .github/workflows/nightly-pipeline.yml | 18 +- .github/workflows/publish.yml | 8 +- .github/workflows/pull-requests.yml | 7 +- .github/workflows/release-pull-request.yml | 7 +- .github/workflows/submodule-update.yml | 3 +- ci/fetch-assets.ps1 | 5 +- ci/key-names.Tests.ps1 | 429 ++++++++++++++++++ ci/run-integration-tests.Tests.ps1 | 21 +- device-detection/ci/README.md | 4 +- device-detection/ci/fetch-assets.ps1 | 16 +- device-detection/ci/run-integration-tests.ps1 | 35 +- .../tests/live_cloud.py | 11 +- .../tests/test_cloud.py | 2 +- .../tests/test_properties.py | 2 +- .../fiftyone_devicedetection_cloud/tox.ini | 5 +- .../tests/live_cloud.py | 11 +- .../tests/test_cloudexamples.py | 2 +- .../tests/test_cloudfailuretomatch.py | 2 +- .../tests/test_cloudgettingstartedweb.py | 11 +- .../test_clouduseragentclienthintsweb.py | 2 +- .../fiftyone_devicedetection_examples/tox.ini | 6 +- .../tox.ini | 4 +- ip-intelligence/ci/README.md | 5 +- ip-intelligence/ci/fetch-assets.ps1 | 23 +- ip-intelligence/ci/run-integration-tests.ps1 | 22 +- .../fiftyone_ipintelligence_cloud/tox.ini | 4 +- .../tests/live_cloud.py | 11 +- .../tests/test_cloudgettingstartedweb.py | 2 +- .../tests/test_examples.py | 10 +- .../fiftyone_ipintelligence_examples/tox.ini | 6 +- .../tests/live_cloud.py | 11 +- .../fiftyone_ipintelligence_onpremise/tox.ini | 5 +- .../fiftyone_ipintelligence_shared/tox.ini | 4 +- .../tox.ini | 4 +- location/ci/README.md | 2 +- location/ci/run-integration-tests.ps1 | 18 +- location/tests/live_cloud.py | 11 +- location/tests/test_examples.py | 4 + location/tests/test_properties51D.py | 5 +- location/tests/test_web_example.py | 2 +- location/tox.ini | 4 +- pipeline/ci/README.md | 4 +- pipeline/ci/run-integration-tests.ps1 | 31 +- .../examples/tests/helpers.py | 30 +- .../examples/tests/live_cloud.py | 11 +- .../examples/tox.ini | 7 +- .../tests/live_cloud.py | 11 +- .../tests/test_cloudengine.py | 12 +- .../tests/test_licensekey_live.py | 13 +- .../tox.ini | 4 +- pipeline/fiftyone_pipeline_core/tox.ini | 4 +- .../fiftyone_pipeline_did/tests/live_cloud.py | 11 +- .../tests/test_did_client_live.py | 21 +- .../tests/test_did_tier_live.py | 6 +- pipeline/fiftyone_pipeline_did/tox.ini | 7 +- pipeline/fiftyone_pipeline_engines/tox.ini | 4 +- .../tox.ini | 4 +- 61 files changed, 728 insertions(+), 299 deletions(-) create mode 100644 ci/key-names.Tests.ps1 diff --git a/.github/workflows/ci-script-tests.yml b/.github/workflows/ci-script-tests.yml index 3a74d841..f1aa5f4a 100644 --- a/.github/workflows/ci-script-tests.yml +++ b/.github/workflows/ci-script-tests.yml @@ -1,7 +1,8 @@ name: CI script tests # Runs the Pester tests of the scripts in ci/, which choose what a pull request -# builds and what a publish releases, and the test of how +# builds and what a publish releases, check that every key is read under its +# _51DEGREES_ name (ci/key-names.Tests.ps1), and test how # free-threaded-python.yml fetches the free data files. common-ci's main is # checked out beside # this repository, where ci/dispatch.Tests.ps1 looks for it. @@ -9,6 +10,11 @@ on: pull_request: paths: - 'ci/**' + - '*/ci/**' + - '**/tox.ini' + - 'pipeline/fiftyone_pipeline_cloudrequestengine/**/tests/**' + - 'pipeline/fiftyone_pipeline_did/tests/**' + - '.github/workflows/cloud-51did.yml' - '.github/workflows/ci-script-tests.yml' - '.github/workflows/free-threaded-python.yml' push: @@ -17,6 +23,11 @@ on: - rc paths: - 'ci/**' + - '*/ci/**' + - '**/tox.ini' + - 'pipeline/fiftyone_pipeline_cloudrequestengine/**/tests/**' + - 'pipeline/fiftyone_pipeline_did/tests/**' + - '.github/workflows/cloud-51did.yml' - '.github/workflows/ci-script-tests.yml' - '.github/workflows/free-threaded-python.yml' workflow_dispatch: diff --git a/.github/workflows/cloud-51did.yml b/.github/workflows/cloud-51did.yml index 2a2e6ee2..adb9d522 100644 --- a/.github/workflows/cloud-51did.yml +++ b/.github/workflows/cloud-51did.yml @@ -1,10 +1,10 @@ name: Cloud 51Did Tests # Runs the live cloud 51Did integration test -# (pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py) once for -# every _51DEGREES_RESOURCE_KEY* key passed to its step below. Each run sets -# the single _51DEGREES_RESOURCE_KEY the test reads first. Those keys carry -# the FODid product and the CloudV5FODiD product the marketing usages need. +# (pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py) with the +# keys it reads, _51DEGREES_RESOURCE_KEY_51DID and +# _51DEGREES_LICENSE_KEY_51DID. The resource key carries the FODid product +# and the CloudV5FODiD product the marketing usages need. # The Basic key carries FODid without CloudV5FODiD, so it runs only the # tests of what the Basic tier is entitled to (test_did_tier_live.py). # @@ -14,8 +14,9 @@ name: Cloud 51Did Tests # A new resource key therefore needs a line here as well as the organisation # secret. # -# A run with no key fails, because a cloud test that passes without calling -# the cloud has proved nothing. A pull request from a fork gets no secrets, +# A run with no key fails, naming the variable, because a cloud test that +# passes without calling the cloud has proved nothing. The tests themselves +# skip without a key. A pull request from a fork gets no secrets, # so run this workflow from a branch in this repository. on: @@ -61,40 +62,37 @@ jobs: run: ./pipeline/ci/copy-owid-source.ps1 - name: Install tox run: python -m pip install --upgrade pip tox - - name: Cloud 51Did test for each resource key secret - shell: pwsh + - name: Cloud 51Did test env: - _51DEGREES_RESOURCE_KEY_BESPOKE: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - _51DEGREES_RESOURCE_KEY_51DID: ${{ vars._51DEGREES_RESOURCE_KEY_51DID }} + # The 51Did live tests read their resource key from + # _51DEGREES_RESOURCE_KEY_51DID, the secret of the same name. It has + # to be a key of the same account as _51DEGREES_LICENSE_KEY_51DID. + # With the Bespoke key in its place, the licensed redeem came back + # unreadable. + _51DEGREES_RESOURCE_KEY_51DID: ${{ secrets._51DEGREES_RESOURCE_KEY_51DID }} # The redemption tests check a sealed result reads back only with a # license key, and the service only opens a sealed result for a - # license key of the account whose resource key sealed it. So each - # license key is passed only with the resource key of the same - # suffix, and a resource key without one checks that the result - # redeems as unreadable. + # license key of the account whose resource key sealed it. The tests + # read it from _51DEGREES_LICENSE_KEY_51DID. _51DEGREES_LICENSE_KEY_51DID: ${{ secrets._51DEGREES_LICENSE_KEY_51DID }} + # The tests skip without a key, so the run fails here instead, + # naming the variable. -rs lists any test that skipped, and none + # should. + shell: pwsh run: | - $keys = Get-ChildItem env: | - Where-Object { $_.Name -like '_51DEGREES_RESOURCE_KEY*' -and $_.Value } - if (-not $keys) { - Write-Host '::error::No _51DEGREES_RESOURCE_KEY* secret reached this job. The tests need a key that carries the FODid product, and the CloudV5FODiD product for the marketing usages.' + if (-not $env:_51DEGREES_RESOURCE_KEY_51DID) { + Write-Host '::error::_51DEGREES_RESOURCE_KEY_51DID is not set. It comes from the _51DEGREES_RESOURCE_KEY_51DID secret.' exit 1 } - $failed = $false - foreach ($k in $keys) { - Write-Host "::group::test_did_client_live with $($k.Name)" - $env:_51DEGREES_RESOURCE_KEY = $k.Value - $licenseName = $k.Name -replace '^_51DEGREES_RESOURCE_KEY', '_51DEGREES_LICENSE_KEY' - $env:_51DEGREES_LICENSE_KEY = [Environment]::GetEnvironmentVariable($licenseName) - Write-Host "License key: $(if ($env:_51DEGREES_LICENSE_KEY) { $licenseName } else { 'none' })" - # -rs lists any test that skipped, and none should. - tox -- tests/test_did_client_live.py -v -rs - if ($LASTEXITCODE -ne 0) { $failed = $true } - Write-Host '::endgroup::' - } - if ($failed) { exit 1 } + tox -- tests/test_did_client_live.py -v -rs - name: Cloud 51Did tier test with the Basic key if: ${{ !cancelled() }} env: - _51DEGREES_RESOURCE_KEY_BASIC: ${{ vars._51DEGREES_RESOURCE_KEY_BASIC }} - run: tox -- tests/test_did_tier_live.py -v -rs + _51DEGREES_RESOURCE_KEY_BASIC: ${{ secrets._51DEGREES_RESOURCE_KEY_BASIC }} + shell: pwsh + run: | + if (-not $env:_51DEGREES_RESOURCE_KEY_BASIC) { + Write-Host '::error::_51DEGREES_RESOURCE_KEY_BASIC is not set. It comes from the _51DEGREES_RESOURCE_KEY_BASIC secret.' + exit 1 + } + tox -- tests/test_did_tier_live.py -v -rs diff --git a/.github/workflows/example-assets-update.yml b/.github/workflows/example-assets-update.yml index 7c7b1851..1fcf37e1 100644 --- a/.github/workflows/example-assets-update.yml +++ b/.github/workflows/example-assets-update.yml @@ -17,5 +17,4 @@ jobs: uses: 51Degrees/common-ci/.github/workflows/nightly-example-assets-update.yml@main with: repo-name: ${{ github.event.repository.name }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} + secrets: inherit diff --git a/.github/workflows/monthly-copyright-update.yml b/.github/workflows/monthly-copyright-update.yml index 7a344b62..68fe9fd6 100644 --- a/.github/workflows/monthly-copyright-update.yml +++ b/.github/workflows/monthly-copyright-update.yml @@ -11,5 +11,4 @@ jobs: uses: 51Degrees/common-ci/.github/workflows/monthly-copyright-update.yml@main with: repo-name: ${{ github.event.repository.name }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} + secrets: inherit diff --git a/.github/workflows/nightly-pipeline.yml b/.github/workflows/nightly-pipeline.yml index f0455882..3c05d8e2 100644 --- a/.github/workflows/nightly-pipeline.yml +++ b/.github/workflows/nightly-pipeline.yml @@ -32,13 +32,7 @@ jobs: dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - PypiToken: ${{ secrets.PYPI_TOKEN }} + secrets: inherit SubmoduleUpdate: name: Nightly Submodule Update @@ -48,8 +42,7 @@ jobs: org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} + secrets: inherit PullRequests: if: ${{ !cancelled() }} @@ -65,9 +58,4 @@ jobs: cache-assets: true # One configuration that fails does not cancel the others. fail-fast: false - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} + secrets: inherit diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b75b3038..81d37252 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -27,10 +27,4 @@ jobs: common-ci-ref: ${{ inputs.common-ci-ref || '' }} allow-early-release: ${{ inputs.allow-early-release || false }} cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - PypiToken: ${{ secrets.PYPI_TOKEN }} + secrets: inherit diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index e154435f..e072cfba 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -35,9 +35,4 @@ jobs: full-matrix: ${{ inputs.full-matrix || false }} fail-fast: false build-without-pull-requests: false - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} + secrets: inherit diff --git a/.github/workflows/release-pull-request.yml b/.github/workflows/release-pull-request.yml index 6faa5356..8e4af949 100644 --- a/.github/workflows/release-pull-request.yml +++ b/.github/workflows/release-pull-request.yml @@ -22,9 +22,4 @@ jobs: repo-name: ${{ github.event.repository.name }} org-name: ${{ github.event.repository.owner.login }} cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} + secrets: inherit diff --git a/.github/workflows/submodule-update.yml b/.github/workflows/submodule-update.yml index 7bf3291c..d3b64cce 100644 --- a/.github/workflows/submodule-update.yml +++ b/.github/workflows/submodule-update.yml @@ -19,5 +19,4 @@ jobs: org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} + secrets: inherit diff --git a/ci/fetch-assets.ps1 b/ci/fetch-assets.ps1 index 62209ec2..6ca6495c 100644 --- a/ci/fetch-assets.ps1 +++ b/ci/fetch-assets.ps1 @@ -4,9 +4,8 @@ param ( # this repository's folder is the default. It is relative to the # common-ci checkout, which is where both callers run it from. [string]$RepoName = (Split-Path -Leaf (Split-Path -Parent $PSScriptRoot)), - [string]$DeviceDetection, - [string]$IpIntelligence, - [string]$IpIntelligenceUrl, + # No key is taken: common-ci downloads the data files with the + # _51DEGREES_LICENSE_KEY it exports. # The branch a pull request merges into, which its changes are measured # against. [string]$Branch = 'main' diff --git a/ci/key-names.Tests.ps1 b/ci/key-names.Tests.ps1 new file mode 100644 index 00000000..b683e6b7 --- /dev/null +++ b/ci/key-names.Tests.ps1 @@ -0,0 +1,429 @@ +# Tests that the CI scripts, the tox configurations and the cloud tests read +# each key under its _51DEGREES_ name, and never under an old name. +# +# common-ci exports every _51DEGREES_ secret under its own name before any of +# this repository's scripts run, and fills _51DEGREES_RESOURCE_KEY from +# _51DEGREES_RESOURCE_KEY_BESPOKE. The Cloud 51Did workflow takes +# _51DEGREES_RESOURCE_KEY_51DID from the secret of the same name. It +# downloads every data file with _51DEGREES_LICENSE_KEY, which is the data +# license and not a cloud license. No key reaches a script as an option. + +BeforeDiscovery { + $script:DiscoveryRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +} + +BeforeAll { + $script:Root = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path + + # The variables the scripts read or write, saved before each test and put + # back after it. + $script:Saved = @( + '_51DEGREES_RESOURCE_KEY', '_51DEGREES_RESOURCE_KEY_BESPOKE', + '_51DEGREES_LICENSE_KEY', 'PAID_RESOURCE_KEY', 'TEMP', 'GITHUB_ENV', + 'GITHUB_JOB', 'PYTEST_ADDOPTS') + + function New-TempFolder { + $path = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) + return [IO.Directory]::CreateDirectory($path).FullName + } + + # A folder laid out like a common-ci checkout, which the product scripts + # run in. Its python/run-integration-tests.ps1 writes the keys it was run + # with to seen.json, and its steps/fetch-assets.ps1 writes each call to + # fetch.json and leaves an empty file for each asset. + function New-CommonCi { + $path = New-TempFolder + $null = [IO.Directory]::CreateDirectory((Join-Path $path 'python')) + $null = [IO.Directory]::CreateDirectory((Join-Path $path 'steps')) + Set-Content (Join-Path $path 'python' 'run-integration-tests.ps1') @' +param ([string]$RepoName, [string[]]$Packages, [hashtable]$Keys) +@{ + Resource = $env:_51DEGREES_RESOURCE_KEY + License = $env:_51DEGREES_LICENSE_KEY +} | ConvertTo-Json | Set-Content (Join-Path $PSScriptRoot '..' 'seen.json') +$global:LASTEXITCODE = 0 +'@ + Set-Content (Join-Path $path 'steps' 'fetch-assets.ps1') @' +param ( + [Parameter(Mandatory)][string[]]$Assets, + [string]$LicenseKey, + [string]$DeviceDetection, + [string]$DeviceDetectionUrl, + [string]$IpIntelligence, + [string]$IpIntelligenceUrl, + [string]$CsvUrl +) +$log = Join-Path $PSScriptRoot '..' 'fetch.json' +$calls = @() +if (Test-Path $log) { $calls = @(Get-Content -Raw $log | ConvertFrom-Json) } +$calls += [pscustomobject]@{ + Assets = @($Assets) + Old = @($PSBoundParameters.Keys | Where-Object { $_ -notin 'Assets', 'LicenseKey' }) +} +ConvertTo-Json -InputObject @($calls) -Depth 4 | Set-Content $log +$null = [IO.Directory]::CreateDirectory((Join-Path $PWD 'assets')) +foreach ($asset in $Assets) { Set-Content -LiteralPath "assets/$asset" '' } +'@ + return $path + } + + # A copy of one product's CI script in a repository of its own, with the + # scripts it calls replaced by ones that do nothing, and the folders it + # writes to. + function New-ProductRepo([string]$Product, [string]$Script) { + $repo = New-TempFolder + $ci = [IO.Directory]::CreateDirectory((Join-Path $repo $Product 'ci')).FullName + Copy-Item (Join-Path $script:Root $Product 'ci' $Script) $ci + Set-Content (Join-Path $ci 'report-key-products.ps1') 'param ([string]$ResourceKey)' + Set-Content (Join-Path $ci 'copy-owid-source.ps1') 'param ([string]$RepoRoot)' + $null = [IO.Directory]::CreateDirectory((Join-Path $repo 'cxx' 'data' 'device-detection')) + $null = [IO.Directory]::CreateDirectory((Join-Path $repo 'cxx' 'data' 'ip-intelligence')) + return Join-Path $repo $Product + } +} + +Describe 'Keys under their _51DEGREES_ names' { + + BeforeEach { + $script:Before = @{} + foreach ($name in $script:Saved) { + $script:Before[$name] = [Environment]::GetEnvironmentVariable($name) + [Environment]::SetEnvironmentVariable($name, $null) + } + } + + AfterEach { + foreach ($name in $script:Saved) { + [Environment]::SetEnvironmentVariable($name, $script:Before[$name]) + } + } + + Describe 'Each product''s run-integration-tests.ps1' { + + BeforeEach { + $script:CommonCi = New-CommonCi + } + + AfterEach { + Remove-Item -Recurse -Force $script:CommonCi + } + + It 'takes the resource key from _51DEGREES_RESOURCE_KEY_BESPOKE in , with no key option' -ForEach @( + @{ Product = 'device-detection'; Extra = @{ SeleniumContract = $false } } + @{ Product = 'ip-intelligence'; Extra = @{} } + @{ Product = 'location'; Extra = @{} } + @{ Product = 'pipeline'; Extra = @{} } + ) { + $repoName = New-ProductRepo $Product 'run-integration-tests.ps1' + $env:_51DEGREES_RESOURCE_KEY_BESPOKE = 'bespoke-resource-key' + $env:_51DEGREES_LICENSE_KEY = 'data-license' + Push-Location $script:CommonCi + try { + & (Join-Path $repoName 'ci' 'run-integration-tests.ps1') ` + -RepoName $repoName -Keys @{} @Extra + } finally { + Pop-Location + } + + $seen = Get-Content -Raw (Join-Path $script:CommonCi 'seen.json') | ConvertFrom-Json + $seen.Resource | Should -Be 'bespoke-resource-key' + } + + It 'leaves the data license common-ci exported as it was in ' -ForEach @( + @{ Product = 'device-detection'; Extra = @{ SeleniumContract = $false } } + @{ Product = 'ip-intelligence'; Extra = @{} } + @{ Product = 'location'; Extra = @{} } + @{ Product = 'pipeline'; Extra = @{} } + ) { + $repoName = New-ProductRepo $Product 'run-integration-tests.ps1' + $env:_51DEGREES_RESOURCE_KEY_BESPOKE = 'bespoke-resource-key' + $env:_51DEGREES_LICENSE_KEY = 'data-license' + Push-Location $script:CommonCi + try { + # An old key option, as a caller passing every secret might, must + # change nothing. + & (Join-Path $repoName 'ci' 'run-integration-tests.ps1') ` + -RepoName $repoName -Keys @{ DeviceDetection = 'old-license'; TestResourceKey = 'old-key' } @Extra + } finally { + Pop-Location + } + + $seen = Get-Content -Raw (Join-Path $script:CommonCi 'seen.json') | ConvertFrom-Json + $seen.License | Should -Be 'data-license' + $seen.Resource | Should -Be 'bespoke-resource-key' + } + + It 'runs the tests that need no key, naming _51DEGREES_RESOURCE_KEY_BESPOKE, when it is not set in ' -ForEach @( + @{ Product = 'device-detection'; Extra = @{ SeleniumContract = $false } } + @{ Product = 'ip-intelligence'; Extra = @{} } + @{ Product = 'location'; Extra = @{} } + @{ Product = 'pipeline'; Extra = @{} } + ) { + $repoName = New-ProductRepo $Product 'run-integration-tests.ps1' + Push-Location $script:CommonCi + try { + $output = & (Join-Path $repoName 'ci' 'run-integration-tests.ps1') ` + -RepoName $repoName -Keys @{} @Extra *>&1 | Out-String + } finally { + Pop-Location + } + + $output | Should -Match '_51DEGREES_RESOURCE_KEY_BESPOKE is not set' + Join-Path $script:CommonCi 'seen.json' | Should -Exist + } + } + + Describe 'Each product''s fetch-assets.ps1' { + + BeforeEach { + $script:CommonCi = New-CommonCi + # The scripts make symbolic links, which need a privilege a test + # run may not have, and folders for them. Where they point is not + # what these tests check, so New-Item does nothing here, and the + # helpers above make their folders without it. + Mock New-Item { } + } + + AfterEach { + Remove-Item -Recurse -Force $script:CommonCi + } + + It 'takes no key or address option in